From 0a46532005b5be94c85d264de7cb8a21a2a3dee0 Mon Sep 17 00:00:00 2001
From: liyang830 <liyang830@jd.com>
Date: Mon, 14 Mar 2022 18:25:28 +0800
Subject: [PATCH 001/249] add check copier drop partition success

---
 programs/copier/ClusterCopier.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/programs/copier/ClusterCopier.cpp b/programs/copier/ClusterCopier.cpp
index 4d491a06795..b1cafd4ad6d 100644
--- a/programs/copier/ClusterCopier.cpp
+++ b/programs/copier/ClusterCopier.cpp
@@ -598,6 +598,8 @@ TaskStatus ClusterCopier::tryMoveAllPiecesToDestinationTable(const TaskTable & t
         ss << "ALTER TABLE " << getQuotedTable(original_table) << ((partition_name == "'all'") ? " DROP PARTITION ID " : " DROP PARTITION ") << partition_name;
 
         UInt64 num_shards_drop_partition = executeQueryOnCluster(task_table.cluster_push, ss.str(), task_cluster->settings_push, ClusterExecutionMode::ON_EACH_SHARD);
+        if (num_shards_drop_partition != task_table.cluster_push->getShardCount())
+            return TaskStatus::Error;
 
         LOG_INFO(log, "Drop partition {} in original table {} have been executed successfully on {} shards of {}",
             partition_name, getQuotedTable(original_table), num_shards_drop_partition, task_table.cluster_push->getShardCount());

From 230f79a3229daf11c169c20be909c0ea4f04958e Mon Sep 17 00:00:00 2001
From: Dmitrii Kovalkov <dakovalkov@yandex-team.ru>
Date: Wed, 10 May 2023 11:51:25 +0300
Subject: [PATCH 002/249] Fix bug in StorageBuffer::reschedule()

---
 src/Storages/StorageBuffer.cpp | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp
index a4cb15d5711..757d3161067 100644
--- a/src/Storages/StorageBuffer.cpp
+++ b/src/Storages/StorageBuffer.cpp
@@ -996,8 +996,11 @@ void StorageBuffer::reschedule()
         std::unique_lock lock(buffer.tryLock());
         if (lock.owns_lock())
         {
-            min_first_write_time = buffer.first_write_time;
-            rows += buffer.data.rows();
+            if (buffer.data)
+            {
+                min_first_write_time = std::min(min_first_write_time, buffer.first_write_time);
+                rows += buffer.data.rows();
+            }
         }
     }
 

From 28fb99cd260dcf7d6c36386364645f19c3ef7ab6 Mon Sep 17 00:00:00 2001
From: Suzy Wang <suzy.wang@ibm.com>
Date: Mon, 5 Jun 2023 13:12:19 -0700
Subject: [PATCH 003/249] Maintain same aggregate function mergeY behavior for
 small and big endian machine

---
 src/AggregateFunctions/UniquesHashSet.h | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/AggregateFunctions/UniquesHashSet.h b/src/AggregateFunctions/UniquesHashSet.h
index ca6d31a716d..369fce9374f 100644
--- a/src/AggregateFunctions/UniquesHashSet.h
+++ b/src/AggregateFunctions/UniquesHashSet.h
@@ -108,7 +108,13 @@ private:
     inline size_t buf_size() const           { return 1ULL << size_degree; } /// NOLINT
     inline size_t max_fill() const           { return 1ULL << (size_degree - 1); } /// NOLINT
     inline size_t mask() const               { return buf_size() - 1; }
-    inline size_t place(HashValue x) const { return (x >> UNIQUES_HASH_BITS_FOR_SKIP) & mask(); }
+
+    inline size_t place(HashValue x) const {
+        if constexpr (std::endian::native == std::endian::little)
+            return (x >> UNIQUES_HASH_BITS_FOR_SKIP) & mask();
+        else
+            return (std::byteswap(x) >> UNIQUES_HASH_BITS_FOR_SKIP) & mask();
+    }
 
     /// The value is divided by 2 ^ skip_degree
     inline bool good(HashValue hash) const

From 55c526b6e644c9bab04972711efc2512a64cbfdd Mon Sep 17 00:00:00 2001
From: Suzy Wang <suzy.wang@ibm.com>
Date: Wed, 5 Jul 2023 10:20:37 -0400
Subject: [PATCH 004/249] Update src/AggregateFunctions/UniquesHashSet.h

Co-authored-by: Yakov Olkhovskiy <99031427+yakov-olkhovskiy@users.noreply.github.com>
---
 src/AggregateFunctions/UniquesHashSet.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/AggregateFunctions/UniquesHashSet.h b/src/AggregateFunctions/UniquesHashSet.h
index 369fce9374f..3e501b29414 100644
--- a/src/AggregateFunctions/UniquesHashSet.h
+++ b/src/AggregateFunctions/UniquesHashSet.h
@@ -109,7 +109,8 @@ private:
     inline size_t max_fill() const           { return 1ULL << (size_degree - 1); } /// NOLINT
     inline size_t mask() const               { return buf_size() - 1; }
 
-    inline size_t place(HashValue x) const {
+    inline size_t place(HashValue x) const
+    {
         if constexpr (std::endian::native == std::endian::little)
             return (x >> UNIQUES_HASH_BITS_FOR_SKIP) & mask();
         else

From 77e45e5669eae8ae05c5acdc5b0cbf6647d1f50a Mon Sep 17 00:00:00 2001
From: pufit <pufit@clickhouse.com>
Date: Mon, 10 Jul 2023 20:23:03 -0400
Subject: [PATCH 005/249] Add `DROP ROLE ... FROM` and `CREATE ROLE ... AT`
 syntax

---
 src/Access/AccessControl.cpp                  |  2 +-
 src/Access/DiskAccessStorage.h                |  2 +-
 src/Access/MultipleAccessStorage.cpp          | 35 +++++++++
 src/Access/MultipleAccessStorage.h            |  5 ++
 src/Access/UsersConfigAccessStorage.h         |  2 +-
 .../Access/InterpreterCreateRoleQuery.cpp     | 42 +++++++++--
 .../InterpreterDropAccessEntityQuery.cpp      | 18 +++--
 src/Parsers/Access/ASTCreateRoleQuery.cpp     |  6 ++
 src/Parsers/Access/ASTCreateRoleQuery.h       |  1 +
 .../Access/ASTDropAccessEntityQuery.cpp       |  5 ++
 src/Parsers/Access/ASTDropAccessEntityQuery.h |  1 +
 src/Parsers/Access/ParserCreateRoleQuery.cpp  |  5 ++
 .../Access/ParserDropAccessEntityQuery.cpp    |  5 ++
 src/Parsers/Access/parseUserName.h            |  5 ++
 .../System/StorageSystemRoleGrants.cpp        |  7 +-
 .../__init__.py                               |  0
 .../configs/roles.xml                         | 13 ++++
 .../test.py                                   | 72 +++++++++++++++++++
 18 files changed, 211 insertions(+), 15 deletions(-)
 create mode 100644 tests/integration/test_multi_access_storage_role_management/__init__.py
 create mode 100644 tests/integration/test_multi_access_storage_role_management/configs/roles.xml
 create mode 100644 tests/integration/test_multi_access_storage_role_management/test.py

diff --git a/src/Access/AccessControl.cpp b/src/Access/AccessControl.cpp
index 6179c823b56..0c61a9e3e1b 100644
--- a/src/Access/AccessControl.cpp
+++ b/src/Access/AccessControl.cpp
@@ -415,7 +415,7 @@ void AccessControl::addStoragesFromUserDirectoriesConfig(
         String type = key_in_user_directories;
         if (size_t bracket_pos = type.find('['); bracket_pos != String::npos)
             type.resize(bracket_pos);
-        if ((type == "users_xml") || (type == "users_config"))
+        if ((type == "users.xml") || (type == "users_config"))
             type = UsersConfigAccessStorage::STORAGE_TYPE;
         else if ((type == "local") || (type == "local_directory"))
             type = DiskAccessStorage::STORAGE_TYPE;
diff --git a/src/Access/DiskAccessStorage.h b/src/Access/DiskAccessStorage.h
index 069a966c8e9..e3bd32c0292 100644
--- a/src/Access/DiskAccessStorage.h
+++ b/src/Access/DiskAccessStorage.h
@@ -13,7 +13,7 @@ class AccessChangesNotifier;
 class DiskAccessStorage : public IAccessStorage
 {
 public:
-    static constexpr char STORAGE_TYPE[] = "local directory";
+    static constexpr char STORAGE_TYPE[] = "local_directory";
 
     DiskAccessStorage(const String & storage_name_, const String & directory_path_, AccessChangesNotifier & changes_notifier_, bool readonly_, bool allow_backup_);
     ~DiskAccessStorage() override;
diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp
index f36a670ddbf..f36beb3f80b 100644
--- a/src/Access/MultipleAccessStorage.cpp
+++ b/src/Access/MultipleAccessStorage.cpp
@@ -16,6 +16,7 @@ namespace ErrorCodes
 {
     extern const int ACCESS_ENTITY_ALREADY_EXISTS;
     extern const int ACCESS_STORAGE_FOR_INSERTION_NOT_FOUND;
+    extern const int ACCESS_ENTITY_NOT_FOUND;
 }
 
 using Storage = IAccessStorage;
@@ -178,6 +179,40 @@ ConstStoragePtr MultipleAccessStorage::getStorage(const UUID & id) const
     return const_cast<MultipleAccessStorage *>(this)->getStorage(id);
 }
 
+StoragePtr MultipleAccessStorage::findStorageByName(const DB::String & storage_name)
+{
+    auto storages = getStoragesInternal();
+    for (const auto & storage : *storages)
+    {
+        if (storage->getStorageName() == storage_name)
+            return storage;
+    }
+
+    return nullptr;
+}
+
+
+ConstStoragePtr MultipleAccessStorage::findStorageByName(const DB::String & storage_name) const
+{
+    return const_cast<MultipleAccessStorage *>(this)->findStorageByName(storage_name);
+}
+
+
+StoragePtr MultipleAccessStorage::getStorageByName(const DB::String & storage_name)
+{
+    auto storage = findStorageByName(storage_name);
+    if (storage)
+        return storage;
+
+    throw Exception(ErrorCodes::ACCESS_ENTITY_NOT_FOUND, "Access storage with name {} is not found", storage_name);
+}
+
+
+ConstStoragePtr MultipleAccessStorage::getStorageByName(const DB::String & storage_name) const
+{
+    return const_cast<MultipleAccessStorage *>(this)->getStorageByName(storage_name);
+}
+
 AccessEntityPtr MultipleAccessStorage::readImpl(const UUID & id, bool throw_if_not_exists) const
 {
     if (auto storage = findStorage(id))
diff --git a/src/Access/MultipleAccessStorage.h b/src/Access/MultipleAccessStorage.h
index 6a0c1bdfc02..1898c6ba5d3 100644
--- a/src/Access/MultipleAccessStorage.h
+++ b/src/Access/MultipleAccessStorage.h
@@ -41,6 +41,11 @@ public:
     ConstStoragePtr getStorage(const UUID & id) const;
     StoragePtr getStorage(const UUID & id);
 
+    ConstStoragePtr findStorageByName(const String & storage_name) const;
+    StoragePtr findStorageByName(const String & storage_name);
+    ConstStoragePtr getStorageByName(const String & storage_name) const;
+    StoragePtr getStorageByName(const String & storage_name);
+
     bool exists(const UUID & id) const override;
 
     bool isBackupAllowed() const override;
diff --git a/src/Access/UsersConfigAccessStorage.h b/src/Access/UsersConfigAccessStorage.h
index b533ccbf200..d8ddf07ff40 100644
--- a/src/Access/UsersConfigAccessStorage.h
+++ b/src/Access/UsersConfigAccessStorage.h
@@ -20,7 +20,7 @@ class UsersConfigAccessStorage : public IAccessStorage
 {
 public:
 
-    static constexpr char STORAGE_TYPE[] = "users.xml";
+    static constexpr char STORAGE_TYPE[] = "users_xml";
 
     UsersConfigAccessStorage(const String & storage_name_, AccessControl & access_control_, bool allow_backup_);
     ~UsersConfigAccessStorage() override;
diff --git a/src/Interpreters/Access/InterpreterCreateRoleQuery.cpp b/src/Interpreters/Access/InterpreterCreateRoleQuery.cpp
index 3386dfb8792..87abe002d3d 100644
--- a/src/Interpreters/Access/InterpreterCreateRoleQuery.cpp
+++ b/src/Interpreters/Access/InterpreterCreateRoleQuery.cpp
@@ -8,6 +8,12 @@
 
 namespace DB
 {
+
+namespace ErrorCodes
+{
+    extern const int ACCESS_ENTITY_ALREADY_EXISTS;
+}
+
 namespace
 {
     void updateRoleFromQueryImpl(
@@ -52,6 +58,15 @@ BlockIO InterpreterCreateRoleQuery::execute()
     if (!query.cluster.empty())
         return executeDDLQueryOnCluster(query_ptr, getContext());
 
+    IAccessStorage * storage = &access_control;
+    MultipleAccessStorage::StoragePtr storage_ptr;
+
+    if (!query.storage_name.empty())
+    {
+        storage_ptr = access_control.getStorageByName(query.storage_name);
+        storage = storage_ptr.get();
+    }
+
     if (query.alter)
     {
         auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr
@@ -62,11 +77,11 @@ BlockIO InterpreterCreateRoleQuery::execute()
         };
         if (query.if_exists)
         {
-            auto ids = access_control.find<Role>(query.names);
-            access_control.tryUpdate(ids, update_func);
+            auto ids = storage->find<Role>(query.names);
+            storage->tryUpdate(ids, update_func);
         }
         else
-            access_control.update(access_control.getIDs<Role>(query.names), update_func);
+            storage->update(storage->getIDs<Role>(query.names), update_func);
     }
     else
     {
@@ -78,12 +93,27 @@ BlockIO InterpreterCreateRoleQuery::execute()
             new_roles.emplace_back(std::move(new_role));
         }
 
+        if (!query.storage_name.empty())
+        {
+            for (const auto & name : query.names)
+            {
+                auto id = access_control.find<Role>(name);
+
+                if (!id)
+                    continue;
+
+                auto another_storage_ptr = access_control.findStorage(*id);
+                if (another_storage_ptr != storage_ptr)
+                    throw Exception(ErrorCodes::ACCESS_ENTITY_ALREADY_EXISTS, "Role {} already exists in storage {}", name, another_storage_ptr->getStorageName());
+            }
+        }
+
         if (query.if_not_exists)
-            access_control.tryInsert(new_roles);
+            storage->tryInsert(new_roles);
         else if (query.or_replace)
-            access_control.insertOrReplace(new_roles);
+            storage->insertOrReplace(new_roles);
         else
-            access_control.insert(new_roles);
+            storage->insert(new_roles);
     }
 
     return {};
diff --git a/src/Interpreters/Access/InterpreterDropAccessEntityQuery.cpp b/src/Interpreters/Access/InterpreterDropAccessEntityQuery.cpp
index d4c37064065..54e3b95226c 100644
--- a/src/Interpreters/Access/InterpreterDropAccessEntityQuery.cpp
+++ b/src/Interpreters/Access/InterpreterDropAccessEntityQuery.cpp
@@ -26,18 +26,26 @@ BlockIO InterpreterDropAccessEntityQuery::execute()
 
     query.replaceEmptyDatabase(getContext()->getCurrentDatabase());
 
-    auto do_drop = [&](const Strings & names)
+    auto do_drop = [&](const Strings & names, const String & storage_name)
     {
+        IAccessStorage * storage = &access_control;
+        MultipleAccessStorage::StoragePtr storage_ptr;
+        if (!storage_name.empty())
+        {
+            storage_ptr = access_control.getStorageByName(storage_name);
+            storage = storage_ptr.get();
+        }
+
         if (query.if_exists)
-            access_control.tryRemove(access_control.find(query.type, names));
+            storage->tryRemove(storage->find(query.type, names));
         else
-            access_control.remove(access_control.getIDs(query.type, names));
+            storage->remove(storage->getIDs(query.type, names));
     };
 
     if (query.type == AccessEntityType::ROW_POLICY)
-        do_drop(query.row_policy_names->toStrings());
+        do_drop(query.row_policy_names->toStrings(), query.storage_name);
     else
-        do_drop(query.names);
+        do_drop(query.names, query.storage_name);
 
     return {};
 }
diff --git a/src/Parsers/Access/ASTCreateRoleQuery.cpp b/src/Parsers/Access/ASTCreateRoleQuery.cpp
index d624b9a9157..14946f2d3cd 100644
--- a/src/Parsers/Access/ASTCreateRoleQuery.cpp
+++ b/src/Parsers/Access/ASTCreateRoleQuery.cpp
@@ -71,6 +71,12 @@ void ASTCreateRoleQuery::formatImpl(const FormatSettings & format, FormatState &
         format.ostr << (format.hilite ? hilite_keyword : "") << " OR REPLACE" << (format.hilite ? hilite_none : "");
 
     formatNames(names, format);
+
+    if (!storage_name.empty())
+        format.ostr << (format.hilite ? IAST::hilite_keyword : "")
+                    << " AT " << (format.hilite ? IAST::hilite_none : "")
+                    << backQuoteIfNeed(storage_name);
+
     formatOnCluster(format);
 
     if (!new_name.empty())
diff --git a/src/Parsers/Access/ASTCreateRoleQuery.h b/src/Parsers/Access/ASTCreateRoleQuery.h
index 42d1a4031b6..4e465553164 100644
--- a/src/Parsers/Access/ASTCreateRoleQuery.h
+++ b/src/Parsers/Access/ASTCreateRoleQuery.h
@@ -28,6 +28,7 @@ public:
 
     Strings names;
     String new_name;
+    String storage_name;
 
     std::shared_ptr<ASTSettingsProfileElements> settings;
 
diff --git a/src/Parsers/Access/ASTDropAccessEntityQuery.cpp b/src/Parsers/Access/ASTDropAccessEntityQuery.cpp
index 88f2d7bce63..02b29f3fd2c 100644
--- a/src/Parsers/Access/ASTDropAccessEntityQuery.cpp
+++ b/src/Parsers/Access/ASTDropAccessEntityQuery.cpp
@@ -53,6 +53,11 @@ void ASTDropAccessEntityQuery::formatImpl(const FormatSettings & settings, Forma
     else
         formatNames(names, settings);
 
+    if (!storage_name.empty())
+        settings.ostr << (settings.hilite ? IAST::hilite_keyword : "")
+                      << " FROM " << (settings.hilite ? IAST::hilite_none : "")
+                      << backQuoteIfNeed(storage_name);
+
     formatOnCluster(settings);
 }
 
diff --git a/src/Parsers/Access/ASTDropAccessEntityQuery.h b/src/Parsers/Access/ASTDropAccessEntityQuery.h
index f3a065c50df..32f4a8f8047 100644
--- a/src/Parsers/Access/ASTDropAccessEntityQuery.h
+++ b/src/Parsers/Access/ASTDropAccessEntityQuery.h
@@ -21,6 +21,7 @@ public:
     AccessEntityType type;
     bool if_exists = false;
     Strings names;
+    String storage_name;
     std::shared_ptr<ASTRowPolicyNames> row_policy_names;
 
     String getID(char) const override;
diff --git a/src/Parsers/Access/ParserCreateRoleQuery.cpp b/src/Parsers/Access/ParserCreateRoleQuery.cpp
index da9749958ee..de9e5baed18 100644
--- a/src/Parsers/Access/ParserCreateRoleQuery.cpp
+++ b/src/Parsers/Access/ParserCreateRoleQuery.cpp
@@ -91,6 +91,10 @@ bool ParserCreateRoleQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
     String new_name;
     std::shared_ptr<ASTSettingsProfileElements> settings;
     String cluster;
+    String storage_name;
+
+    if (ParserKeyword{"AT"}.ignore(pos, expected))
+        parseStorageName(pos, expected, storage_name);
 
     while (true)
     {
@@ -125,6 +129,7 @@ bool ParserCreateRoleQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
     query->names = std::move(names);
     query->new_name = std::move(new_name);
     query->settings = std::move(settings);
+    query->storage_name = std::move(storage_name);
 
     return true;
 }
diff --git a/src/Parsers/Access/ParserDropAccessEntityQuery.cpp b/src/Parsers/Access/ParserDropAccessEntityQuery.cpp
index d21164bc1a2..d5ecdf27449 100644
--- a/src/Parsers/Access/ParserDropAccessEntityQuery.cpp
+++ b/src/Parsers/Access/ParserDropAccessEntityQuery.cpp
@@ -53,6 +53,7 @@ bool ParserDropAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expected &
 
     Strings names;
     std::shared_ptr<ASTRowPolicyNames> row_policy_names;
+    String storage_name;
     String cluster;
 
     if ((type == AccessEntityType::USER) || (type == AccessEntityType::ROLE))
@@ -76,6 +77,9 @@ bool ParserDropAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expected &
             return false;
     }
 
+    if (ParserKeyword{"FROM"}.ignore(pos, expected))
+        parseStorageName(pos, expected, storage_name);
+
     if (cluster.empty())
         parseOnCluster(pos, expected, cluster);
 
@@ -87,6 +91,7 @@ bool ParserDropAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expected &
     query->cluster = std::move(cluster);
     query->names = std::move(names);
     query->row_policy_names = std::move(row_policy_names);
+    query->storage_name = std::move(storage_name);
 
     return true;
 }
diff --git a/src/Parsers/Access/parseUserName.h b/src/Parsers/Access/parseUserName.h
index fbad78e7bce..717911bf2f1 100644
--- a/src/Parsers/Access/parseUserName.h
+++ b/src/Parsers/Access/parseUserName.h
@@ -34,4 +34,9 @@ inline bool parseRoleNames(IParser::Pos & pos, Expected & expected, Strings & ro
     return parseUserNames(pos, expected, role_names);
 }
 
+inline bool parseStorageName(IParser::Pos & pos, Expected & expected, String & storage_name)
+{
+    return parseUserName(pos, expected, storage_name);
+}
+
 }
diff --git a/src/Storages/System/StorageSystemRoleGrants.cpp b/src/Storages/System/StorageSystemRoleGrants.cpp
index cf5a24f88cd..8fcd0fd7cf5 100644
--- a/src/Storages/System/StorageSystemRoleGrants.cpp
+++ b/src/Storages/System/StorageSystemRoleGrants.cpp
@@ -3,6 +3,7 @@
 #include <DataTypes/DataTypeNullable.h>
 #include <DataTypes/DataTypeString.h>
 #include <DataTypes/DataTypesNumber.h>
+#include <DataTypes/DataTypeUUID.h>
 #include <Columns/ColumnString.h>
 #include <Columns/ColumnNullable.h>
 #include <Columns/ColumnsNumber.h>
@@ -22,6 +23,7 @@ NamesAndTypesList StorageSystemRoleGrants::getNamesAndTypes()
         {"user_name", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeString>())},
         {"role_name", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeString>())},
         {"granted_role_name", std::make_shared<DataTypeString>()},
+        {"granted_role_id", std::make_shared<DataTypeUUID>()},
         {"granted_role_is_default", std::make_shared<DataTypeUInt8>()},
         {"with_admin_option", std::make_shared<DataTypeUInt8>()},
     };
@@ -45,12 +47,14 @@ void StorageSystemRoleGrants::fillData(MutableColumns & res_columns, ContextPtr
     auto & column_role_name = assert_cast<ColumnString &>(assert_cast<ColumnNullable &>(*res_columns[column_index]).getNestedColumn());
     auto & column_role_name_null_map = assert_cast<ColumnNullable &>(*res_columns[column_index++]).getNullMapData();
     auto & column_granted_role_name = assert_cast<ColumnString &>(*res_columns[column_index++]);
+    auto & column_granted_role_id = assert_cast<ColumnUUID &>(*res_columns[column_index++]).getData();
     auto & column_is_default = assert_cast<ColumnUInt8 &>(*res_columns[column_index++]).getData();
     auto & column_admin_option = assert_cast<ColumnUInt8 &>(*res_columns[column_index++]).getData();
 
     auto add_row = [&](const String & grantee_name,
                        AccessEntityType grantee_type,
                        const String & granted_role_name,
+                       const UUID & granted_role_id,
                        bool is_default,
                        bool with_admin_option)
     {
@@ -72,6 +76,7 @@ void StorageSystemRoleGrants::fillData(MutableColumns & res_columns, ContextPtr
             assert(false);
 
         column_granted_role_name.insertData(granted_role_name.data(), granted_role_name.length());
+        column_granted_role_id.push_back(granted_role_id.toUnderType());
         column_is_default.push_back(is_default);
         column_admin_option.push_back(with_admin_option);
     };
@@ -90,7 +95,7 @@ void StorageSystemRoleGrants::fillData(MutableColumns & res_columns, ContextPtr
                     continue;
 
                 bool is_default = !default_roles || default_roles->match(role_id);
-                add_row(grantee_name, grantee_type, *role_name, is_default, element.admin_option);
+                add_row(grantee_name, grantee_type, *role_name, role_id, is_default, element.admin_option);
             }
         }
     };
diff --git a/tests/integration/test_multi_access_storage_role_management/__init__.py b/tests/integration/test_multi_access_storage_role_management/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/integration/test_multi_access_storage_role_management/configs/roles.xml b/tests/integration/test_multi_access_storage_role_management/configs/roles.xml
new file mode 100644
index 00000000000..672c7e24728
--- /dev/null
+++ b/tests/integration/test_multi_access_storage_role_management/configs/roles.xml
@@ -0,0 +1,13 @@
+<clickhouse>
+    <user_directories>
+        <memory/>
+    </user_directories>
+
+    <roles>
+        <default_role>
+            <grants>
+                <query>GRANT ALL ON *.* WITH GRANT OPTION</query>
+            </grants>
+        </default_role>
+    </roles>
+</clickhouse>
diff --git a/tests/integration/test_multi_access_storage_role_management/test.py b/tests/integration/test_multi_access_storage_role_management/test.py
new file mode 100644
index 00000000000..473a37695e9
--- /dev/null
+++ b/tests/integration/test_multi_access_storage_role_management/test.py
@@ -0,0 +1,72 @@
+import pytest
+import os
+from helpers.cluster import ClickHouseCluster
+from helpers.test_tools import TSV
+from helpers.client import QueryRuntimeException
+
+cluster = ClickHouseCluster(__file__)
+node = cluster.add_instance(
+    "node",
+    stay_alive=True,
+)
+
+SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
+
+
+@pytest.fixture(scope="module", autouse=True)
+def started_cluster():
+    try:
+        cluster.start()
+        node.query("CREATE DATABASE mydb")
+        node.query("CREATE TABLE mydb.table1(x UInt32) ENGINE=Log")
+
+        node.query("CREATE USER test_user")
+        node.query("CREATE USER test_user2")
+        yield cluster
+
+    finally:
+        cluster.shutdown()
+
+
+def test_role_from_different_storages():
+    node.query("CREATE ROLE default_role")
+    node.query("GRANT SELECT ON system.* TO default_role")
+
+    assert node.query("SHOW GRANTS FOR default_role") == TSV(["GRANT SELECT ON system.* TO default_role"])
+    assert node.query("SHOW ROLES") == TSV(["default_role"])
+
+    node.query("GRANT default_role TO test_user")
+
+    node.copy_file_to_container(
+        os.path.join(SCRIPT_DIR, "configs/roles.xml"),
+        "/etc/clickhouse-server/users.d/roles.xml",
+    )
+
+    node.restart_clickhouse()
+
+    assert node.query("SELECT name, storage FROM system.roles") == TSV(
+        [
+            ["default_role", "users_xml"],
+            ["default_role", "local_directory"]
+        ]
+    )
+
+    # Role from users.xml will have priority
+    assert node.query("SHOW GRANTS FOR default_role") == TSV(["GRANT ALL ON *.* TO default_role WITH GRANT OPTION"])
+
+    node.query("GRANT default_role TO test_user")
+    node.query("GRANT default_role TO test_user2")
+    assert node.query("SELECT granted_role_id FROM system.role_grants WHERE user_name = 'test_user2'") == TSV(
+        ["62bedbf3-7fb1-94cb-3a35-e479693223b3"]  # roles from users.xml have deterministic ids
+    )
+
+    node.query("DROP ROLE default_role FROM local_directory")
+    assert node.query("SELECT granted_role_id FROM system.role_grants WHERE user_name = 'test_user'") == TSV(["62bedbf3-7fb1-94cb-3a35-e479693223b3"])
+
+    # Already exists
+    with pytest.raises(QueryRuntimeException):
+        node.query("CREATE ROLE default_role AT memory")
+
+    node.query("CREATE ROLE other_role AT memory")
+
+    assert node.query("SELECT storage FROM system.roles WHERE name = 'other_role'") == TSV(["memory"])

From ee591ef7f52f3f53dd8cb59a6acf33a734561f65 Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Tue, 11 Jul 2023 00:33:34 +0000
Subject: [PATCH 006/249] Automatic style fix

---
 .../test.py                                   | 29 ++++++++++++-------
 1 file changed, 19 insertions(+), 10 deletions(-)

diff --git a/tests/integration/test_multi_access_storage_role_management/test.py b/tests/integration/test_multi_access_storage_role_management/test.py
index 473a37695e9..d9a4a73c56b 100644
--- a/tests/integration/test_multi_access_storage_role_management/test.py
+++ b/tests/integration/test_multi_access_storage_role_management/test.py
@@ -32,7 +32,9 @@ def test_role_from_different_storages():
     node.query("CREATE ROLE default_role")
     node.query("GRANT SELECT ON system.* TO default_role")
 
-    assert node.query("SHOW GRANTS FOR default_role") == TSV(["GRANT SELECT ON system.* TO default_role"])
+    assert node.query("SHOW GRANTS FOR default_role") == TSV(
+        ["GRANT SELECT ON system.* TO default_role"]
+    )
     assert node.query("SHOW ROLES") == TSV(["default_role"])
 
     node.query("GRANT default_role TO test_user")
@@ -45,23 +47,28 @@ def test_role_from_different_storages():
     node.restart_clickhouse()
 
     assert node.query("SELECT name, storage FROM system.roles") == TSV(
-        [
-            ["default_role", "users_xml"],
-            ["default_role", "local_directory"]
-        ]
+        [["default_role", "users_xml"], ["default_role", "local_directory"]]
     )
 
     # Role from users.xml will have priority
-    assert node.query("SHOW GRANTS FOR default_role") == TSV(["GRANT ALL ON *.* TO default_role WITH GRANT OPTION"])
+    assert node.query("SHOW GRANTS FOR default_role") == TSV(
+        ["GRANT ALL ON *.* TO default_role WITH GRANT OPTION"]
+    )
 
     node.query("GRANT default_role TO test_user")
     node.query("GRANT default_role TO test_user2")
-    assert node.query("SELECT granted_role_id FROM system.role_grants WHERE user_name = 'test_user2'") == TSV(
-        ["62bedbf3-7fb1-94cb-3a35-e479693223b3"]  # roles from users.xml have deterministic ids
+    assert node.query(
+        "SELECT granted_role_id FROM system.role_grants WHERE user_name = 'test_user2'"
+    ) == TSV(
+        [
+            "62bedbf3-7fb1-94cb-3a35-e479693223b3"
+        ]  # roles from users.xml have deterministic ids
     )
 
     node.query("DROP ROLE default_role FROM local_directory")
-    assert node.query("SELECT granted_role_id FROM system.role_grants WHERE user_name = 'test_user'") == TSV(["62bedbf3-7fb1-94cb-3a35-e479693223b3"])
+    assert node.query(
+        "SELECT granted_role_id FROM system.role_grants WHERE user_name = 'test_user'"
+    ) == TSV(["62bedbf3-7fb1-94cb-3a35-e479693223b3"])
 
     # Already exists
     with pytest.raises(QueryRuntimeException):
@@ -69,4 +76,6 @@ def test_role_from_different_storages():
 
     node.query("CREATE ROLE other_role AT memory")
 
-    assert node.query("SELECT storage FROM system.roles WHERE name = 'other_role'") == TSV(["memory"])
+    assert node.query(
+        "SELECT storage FROM system.roles WHERE name = 'other_role'"
+    ) == TSV(["memory"])

From 0e9f0e6837cbbb9c7fa784fc9139451ce9779c57 Mon Sep 17 00:00:00 2001
From: HarryLeeIBM <hleeatwork@outlook.com>
Date: Tue, 11 Jul 2023 11:09:01 -0700
Subject: [PATCH 007/249] Fix ForEach Aggregate state for s390x

---
 src/AggregateFunctions/AggregateFunctionForEach.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/AggregateFunctions/AggregateFunctionForEach.h b/src/AggregateFunctions/AggregateFunctionForEach.h
index 480b4cc690e..ec59fd2e616 100644
--- a/src/AggregateFunctions/AggregateFunctionForEach.h
+++ b/src/AggregateFunctions/AggregateFunctionForEach.h
@@ -240,7 +240,7 @@ public:
     void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional<size_t> /* version */) const override
     {
         const AggregateFunctionForEachData & state = data(place);
-        writeBinary(state.dynamic_array_size, buf);
+        writeBinaryLittleEndian(state.dynamic_array_size, buf);
 
         const char * nested_state = state.array_of_aggregate_datas;
         for (size_t i = 0; i < state.dynamic_array_size; ++i)
@@ -255,7 +255,7 @@ public:
         AggregateFunctionForEachData & state = data(place);
 
         size_t new_size = 0;
-        readBinary(new_size, buf);
+        readBinaryLittleEndian(new_size, buf);
 
         ensureAggregateData(place, new_size, *arena);
 

From 1a4d48324e3ffe1f8c0c05a337b590ce8aa9ff33 Mon Sep 17 00:00:00 2001
From: pufit <pufit@clickhouse.com>
Date: Tue, 11 Jul 2023 19:58:28 -0400
Subject: [PATCH 008/249] Fix fast tests

---
 .../queries/0_stateless/01293_create_role.reference  |  2 +-
 .../01294_create_settings_profile.reference          | 12 ++++++------
 .../0_stateless/01295_create_row_policy.reference    |  6 +++---
 .../queries/0_stateless/01297_create_quota.reference |  8 ++++----
 .../02117_show_create_table_system.reference         |  1 +
 5 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/tests/queries/0_stateless/01293_create_role.reference b/tests/queries/0_stateless/01293_create_role.reference
index 9b3c4eabd47..42f091bddd5 100644
--- a/tests/queries/0_stateless/01293_create_role.reference
+++ b/tests/queries/0_stateless/01293_create_role.reference
@@ -28,7 +28,7 @@ CREATE ROLE r2_01293
 CREATE ROLE r1_01293 SETTINGS readonly = 1
 CREATE ROLE r2_01293 SETTINGS readonly = 1
 -- system.roles
-r1_01293	local directory
+r1_01293	local_directory
 -- system.settings_profile_elements
 \N	\N	r1_01293	0	readonly	1	\N	\N	\N	\N
 \N	\N	r2_01293	0	\N	\N	\N	\N	\N	default
diff --git a/tests/queries/0_stateless/01294_create_settings_profile.reference b/tests/queries/0_stateless/01294_create_settings_profile.reference
index dbb73bca851..a10d5758752 100644
--- a/tests/queries/0_stateless/01294_create_settings_profile.reference
+++ b/tests/queries/0_stateless/01294_create_settings_profile.reference
@@ -53,12 +53,12 @@ CREATE SETTINGS PROFILE s4_01294 SETTINGS INHERIT readonly, INHERIT readonly
 CREATE SETTINGS PROFILE s5_01294 SETTINGS INHERIT readonly, readonly = 1
 CREATE SETTINGS PROFILE s6_01294 SETTINGS INHERIT readonly, readonly CONST
 -- system.settings_profiles
-s1_01294	local directory	0	0	[]	[]
-s2_01294	local directory	1	0	['r1_01294']	[]
-s3_01294	local directory	1	0	['r1_01294']	[]
-s4_01294	local directory	1	0	['r1_01294']	[]
-s5_01294	local directory	3	0	['u1_01294']	[]
-s6_01294	local directory	0	1	[]	['r1_01294','u1_01294']
+s1_01294	local_directory	0	0	[]	[]
+s2_01294	local_directory	1	0	['r1_01294']	[]
+s3_01294	local_directory	1	0	['r1_01294']	[]
+s4_01294	local_directory	1	0	['r1_01294']	[]
+s5_01294	local_directory	3	0	['u1_01294']	[]
+s6_01294	local_directory	0	1	[]	['r1_01294','u1_01294']
 -- system.settings_profile_elements
 s2_01294	\N	\N	0	readonly	0	\N	\N	\N	\N
 s3_01294	\N	\N	0	max_memory_usage	5000000	4000000	6000000	CONST	\N
diff --git a/tests/queries/0_stateless/01295_create_row_policy.reference b/tests/queries/0_stateless/01295_create_row_policy.reference
index 6e3169b7fec..d73d9752bc1 100644
--- a/tests/queries/0_stateless/01295_create_row_policy.reference
+++ b/tests/queries/0_stateless/01295_create_row_policy.reference
@@ -30,6 +30,6 @@ CREATE ROW POLICY p5_01295 ON db2.table2 FOR SELECT USING a = b
 CREATE ROW POLICY p1_01295 ON db.table FOR SELECT USING 1 TO ALL
 CREATE ROW POLICY p2_01295 ON db.table FOR SELECT USING 1 TO ALL
 -- system.row_policies
-p1_01295 ON db.table	p1_01295	db	table	local directory	(a < b) AND (c > d)	0	0	[]	[]
-p2_01295 ON db.table	p2_01295	db	table	local directory	id = currentUser()	1	0	['u1_01295']	[]
-p3_01295 ON db.table	p3_01295	db	table	local directory	1	0	1	[]	['r1_01295']
+p1_01295 ON db.table	p1_01295	db	table	local_directory	(a < b) AND (c > d)	0	0	[]	[]
+p2_01295 ON db.table	p2_01295	db	table	local_directory	id = currentUser()	1	0	['u1_01295']	[]
+p3_01295 ON db.table	p3_01295	db	table	local_directory	1	0	1	[]	['r1_01295']
diff --git a/tests/queries/0_stateless/01297_create_quota.reference b/tests/queries/0_stateless/01297_create_quota.reference
index 6b844e0d2f6..308bbf79024 100644
--- a/tests/queries/0_stateless/01297_create_quota.reference
+++ b/tests/queries/0_stateless/01297_create_quota.reference
@@ -52,10 +52,10 @@ CREATE QUOTA q2_01297 FOR INTERVAL 1 day MAX errors = 5
 CREATE QUOTA q1_01297 FOR INTERVAL 1 day TRACKING ONLY TO r1_01297
 CREATE QUOTA q2_01297 FOR INTERVAL 1 day TRACKING ONLY TO r1_01297
 -- system.quotas
-q1_01297	local directory	['user_name']	[]	0	['r1_01297']	[]
-q2_01297	local directory	[]	[5259492]	0	['r1_01297','u1_01297']	[]
-q3_01297	local directory	['client_key','user_name']	[5259492,15778476]	0	[]	[]
-q4_01297	local directory	[]	[604800]	1	[]	['u1_01297']
+q1_01297	local_directory	['user_name']	[]	0	['r1_01297']	[]
+q2_01297	local_directory	[]	[5259492]	0	['r1_01297','u1_01297']	[]
+q3_01297	local_directory	['client_key','user_name']	[5259492,15778476]	0	[]	[]
+q4_01297	local_directory	[]	[604800]	1	[]	['u1_01297']
 -- system.quota_limits
 q2_01297	5259492	0	100	\N	\N	11	1000	10000	1001	10001	2.5	\N
 q3_01297	5259492	0	\N	\N	\N	\N	1002	\N	\N	\N	\N	\N
diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference
index 3834b05601f..aaff6d92d32 100644
--- a/tests/queries/0_stateless/02117_show_create_table_system.reference
+++ b/tests/queries/0_stateless/02117_show_create_table_system.reference
@@ -955,6 +955,7 @@ CREATE TABLE system.role_grants
     `user_name` Nullable(String),
     `role_name` Nullable(String),
     `granted_role_name` String,
+    `granted_role_id` UUID,
     `granted_role_is_default` UInt8,
     `with_admin_option` UInt8
 )

From 8e3bd222f00324667a1d41fb62f50adc91045b6c Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 19 Jul 2023 03:42:32 +0200
Subject: [PATCH 009/249] Add SQLTest

---
 .github/workflows/pull_request.yml |  36 +++++++
 docker/images.json                 |   7 +-
 docker/test/sqltest/Dockerfile     |  29 ++++++
 docker/test/sqltest/run.sh         |  41 ++++++++
 docker/test/sqltest/test.py        | 118 ++++++++++++++++++++++
 tests/ci/ast_fuzzer_check.py       |   2 +-
 tests/ci/sqltest.py                | 157 +++++++++++++++++++++++++++++
 tests/ci/tests/docker_images.json  |   4 +
 8 files changed, 391 insertions(+), 3 deletions(-)
 create mode 100644 docker/test/sqltest/Dockerfile
 create mode 100755 docker/test/sqltest/run.sh
 create mode 100644 docker/test/sqltest/test.py
 create mode 100644 tests/ci/sqltest.py

diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index dd834959578..b3283c3afbe 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -5182,3 +5182,39 @@ jobs:
           docker ps --quiet | xargs --no-run-if-empty docker kill ||:
           docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH"
+##############################################################################################
+##################################### SQL TEST ###############################################
+##############################################################################################
+  SQLTest:
+    needs: [BuilderDebRelease]
+    runs-on: [self-hosted, fuzzer-unit-tester]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/sqltest
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          CHECK_NAME="SQLTest"
+          REPO_COPY=${{runner.temp}}/sqltest/ClickHouse
+          EOF
+      - name: Download json reports
+        uses: actions/download-artifact@v3
+        with:
+          path: ${{ env.REPORTS_PATH }}
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+      - name: SQLTest
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci"
+          python3 sqltest.py "$CHECK_NAME"
+      - name: Cleanup
+        if: always()
+        run: |
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+          sudo rm -fr "$TEMP_PATH"
diff --git a/docker/images.json b/docker/images.json
index e8fc329a640..663fa21a6c9 100644
--- a/docker/images.json
+++ b/docker/images.json
@@ -155,11 +155,14 @@
     },
     "docker/docs/builder": {
         "name": "clickhouse/docs-builder",
-        "dependent": [
-        ]
+        "dependent": []
     },
     "docker/test/sqllogic": {
         "name": "clickhouse/sqllogic-test",
         "dependent": []
+    },
+    "docker/test/sqltest": {
+        "name": "clickhouse/sqltest",
+        "dependent": []
     }
 }
diff --git a/docker/test/sqltest/Dockerfile b/docker/test/sqltest/Dockerfile
new file mode 100644
index 00000000000..c58907a98a8
--- /dev/null
+++ b/docker/test/sqltest/Dockerfile
@@ -0,0 +1,29 @@
+# docker build -t clickhouse/sqltest .
+ARG FROM_TAG=latest
+FROM clickhouse/test-base:$FROM_TAG
+
+RUN apt-get update --yes \
+    && env DEBIAN_FRONTEND=noninteractive \
+        apt-get install --yes --no-install-recommends \
+            wget \
+            git \
+            python3 \
+            python3-dev \
+            python3-pip \
+            sudo \
+    && apt-get clean
+
+RUN pip3 install \
+    pyyaml \
+    clickhouse-driver
+
+ARG sqltest_repo="https://github.com/elliotchance/sqltest/"
+
+RUN git clone ${sqltest_repo}
+
+ENV TZ=UTC
+ENV MAX_RUN_TIME=900
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+COPY run.sh /
+CMD ["/bin/bash", "/run.sh"]
diff --git a/docker/test/sqltest/run.sh b/docker/test/sqltest/run.sh
new file mode 100755
index 00000000000..ef9ecaec33b
--- /dev/null
+++ b/docker/test/sqltest/run.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+set -x
+set -e
+set -u
+set -o pipefail
+
+BINARY_TO_DOWNLOAD=${BINARY_TO_DOWNLOAD:="clang-16_debug_none_unsplitted_disable_False_binary"}
+BINARY_URL_TO_DOWNLOAD=${BINARY_URL_TO_DOWNLOAD:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/$BINARY_TO_DOWNLOAD/clickhouse"}
+
+function wget_with_retry
+{
+    for _ in 1 2 3 4; do
+        if wget -nv -nd -c "$1";then
+            return 0
+        else
+            sleep 0.5
+        fi
+    done
+    return 1
+}
+
+wget_with_retry "$BINARY_URL_TO_DOWNLOAD"
+chmod +x clickhouse
+./clickhouse install --noninteractive
+clickhouse start
+
+# Wait for start
+for _ in {1..100}
+do
+    clickhouse-client --query "SELECT 1" && break ||:
+    sleep 1
+done
+
+# Run the test
+pushd sqltest/standards/2016/
+./test.py
+popd
+
+zstd --threads=0 /var/log/clickhouse-server/clickhouse-server.log
+zstd --threads=0 /var/log/clickhouse-server/clickhouse-server.err.log
diff --git a/docker/test/sqltest/test.py b/docker/test/sqltest/test.py
new file mode 100644
index 00000000000..7b38fdc52ad
--- /dev/null
+++ b/docker/test/sqltest/test.py
@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+
+import os
+import yaml
+import random
+import string
+from clickhouse_driver import Client
+
+
+client = Client(host = 'localhost', port = 9000)
+settings = {'default_table_engine': 'Memory', 'union_default_mode': 'DISTINCT', 'calculate_text_stack_trace': 0}
+
+database_name = 'sqltest_' + ''.join(random.choice(string.ascii_lowercase) for _ in range(10))
+
+client.execute(f"DROP DATABASE IF EXISTS {database_name}", settings = settings)
+client.execute(f"CREATE DATABASE {database_name}", settings = settings)
+
+client = Client(host = 'localhost', port = 9000, database = database_name)
+
+summary = {'success': 0, 'total': 0, 'results': {}}
+
+log_file = open("test.log", "w")
+report_html_file = open("report.html", "w")
+
+with open('features.yml', 'r') as file:
+    yaml_content = yaml.safe_load(file)
+
+    for category in yaml_content:
+        log_file.write(category.capitalize() + " features:")
+        summary['results'][category] = {'success': 0, 'total': 0, 'results': {}}
+
+        for test in yaml_content[category]:
+            log_file.write(test + ": " + yaml_content[category][test])
+            summary['results'][category]['results'][test] = {'success': 0, 'total': 0, 'description': yaml_content[category][test]}
+
+            test_path = test[0] + "/" + test + ".tests.yml"
+            if os.path.exists(test_path):
+                with open(test_path, 'r') as test_file:
+                    test_yaml_content = yaml.load_all(test_file, Loader = yaml.FullLoader)
+
+                    for test_case in test_yaml_content:
+
+                        queries = test_case['sql'];
+                        if not isinstance(queries, list):
+                            queries = [queries]
+
+                        for query in queries:
+                            # Example: E011-01
+                            test_group = ''
+                            if '-' in test:
+                                test_group = test.split("-", 1)[0]
+                                summary['results'][category]['results'][test_group]['total'] += 1
+                            summary['results'][category]['results'][test]['total'] += 1
+                            summary['results'][category]['total'] += 1
+                            summary['total'] += 1
+
+                            log_file.write(query)
+
+                            try:
+                                result = client.execute(query, settings = settings)
+                                log_file.write(result)
+
+                                if test_group:
+                                    summary['results'][category]['results'][test_group]['success'] += 1
+                                summary['results'][category]['results'][test]['success'] += 1
+                                summary['results'][category]['success'] += 1
+                                summary['success'] += 1
+
+                            except Exception as e:
+                                log_file.write(f"Error occurred: {str(e)}")
+
+client.execute(f"DROP DATABASE {database_name}", settings = settings)
+
+def enable_color(ratio):
+    if ratio == 0:
+        return "<b style='color: red;'>"
+    elif ratio < 0.5:
+        return "<b style='color: orange;'>"
+    elif ratio < 1:
+        return "<b style='color: gray;'>"
+    else:
+        return "<b style='color: green;'>"
+
+reset_color = "</b>"
+
+def print_ratio(indent, name, success, total, description):
+    report_html_file.write("{}{}: {}{} / {} ({:.1%}){}{}<br/>".format(
+        ' ' * indent,
+        name,
+        enable_color(success / total),
+        success,
+        total,
+        success / total,
+        reset_color,
+        f" - {description}" if description else ''))
+
+
+report_html_file.write("<html style='font: 16pt Monospace; padding: 1em;'><body><p>")
+
+print_ratio(0, 'Total', summary['success'], summary['total'], '')
+
+for category in summary['results']:
+    cat_summary = summary['results'][category]
+
+    if cat_summary['total'] == 0:
+        continue
+
+    print_ratio(2, category, cat_summary['success'], cat_summary['total'], '')
+
+    for test in summary['results'][category]['results']:
+        test_summary = summary['results'][category]['results'][test]
+
+        if test_summary['total'] == 0:
+            continue
+
+        print_ratio(4, test, test_summary['success'], test_summary['total'], test_summary['description'])
+
+report_html_file.write("</p></body></html>")
diff --git a/tests/ci/ast_fuzzer_check.py b/tests/ci/ast_fuzzer_check.py
index 514aaf7e2ac..74b875c23be 100644
--- a/tests/ci/ast_fuzzer_check.py
+++ b/tests/ci/ast_fuzzer_check.py
@@ -79,7 +79,7 @@ def main():
             build_url = url
             break
     else:
-        raise Exception("Cannot binary clickhouse among build results")
+        raise Exception("Cannot find the clickhouse binary among build results")
 
     logging.info("Got build url %s", build_url)
 
diff --git a/tests/ci/sqltest.py b/tests/ci/sqltest.py
new file mode 100644
index 00000000000..5c20cc4849b
--- /dev/null
+++ b/tests/ci/sqltest.py
@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+
+import logging
+import subprocess
+import os
+import sys
+
+from github import Github
+
+from build_download_helper import get_build_name_for_check, read_build_urls
+from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse
+from commit_status_helper import (
+    RerunHelper,
+    format_description,
+    get_commit,
+    post_commit_status,
+)
+from docker_pull_helper import get_image_with_version
+from env_helper import (
+    GITHUB_RUN_URL,
+    REPORTS_PATH,
+    TEMP_PATH,
+)
+from get_robot_token import get_best_robot_token
+from pr_info import PRInfo
+from report import TestResult
+from s3_helper import S3Helper
+from stopwatch import Stopwatch
+
+IMAGE_NAME = "clickhouse/sqltest"
+
+
+def get_run_command(pr_number, sha, download_url, workspace_path, image):
+    return (
+        f"docker run "
+        # For sysctl
+        "--privileged "
+        "--network=host "
+        f"--volume={workspace_path}:/workspace "
+        "--cap-add syslog --cap-add sys_admin --cap-add=SYS_PTRACE "
+        f'-e PR_TO_TEST={pr_number} -e SHA_TO_TEST={sha} -e BINARY_URL_TO_DOWNLOAD="{download_url}" '
+        f"{image}"
+    )
+
+
+def main():
+    logging.basicConfig(level=logging.INFO)
+
+    stopwatch = Stopwatch()
+
+    temp_path = TEMP_PATH
+    reports_path = REPORTS_PATH
+
+    check_name = sys.argv[1]
+
+    if not os.path.exists(temp_path):
+        os.makedirs(temp_path)
+
+    pr_info = PRInfo()
+
+    gh = Github(get_best_robot_token(), per_page=100)
+    commit = get_commit(gh, pr_info.sha)
+
+    rerun_helper = RerunHelper(commit, check_name)
+    if rerun_helper.is_already_finished_by_status():
+        logging.info("Check is already finished according to github status, exiting")
+        sys.exit(0)
+
+    docker_image = get_image_with_version(reports_path, IMAGE_NAME)
+
+    build_name = get_build_name_for_check(check_name)
+    print(build_name)
+    urls = read_build_urls(build_name, reports_path)
+    if not urls:
+        raise Exception("No build URLs found")
+
+    for url in urls:
+        if url.endswith("/clickhouse"):
+            build_url = url
+            break
+    else:
+        raise Exception("Cannot find the clickhouse binary among build results")
+
+    logging.info("Got build url %s", build_url)
+
+    workspace_path = os.path.join(temp_path, "workspace")
+    if not os.path.exists(workspace_path):
+        os.makedirs(workspace_path)
+
+    run_command = get_run_command(
+        pr_info.number, pr_info.sha, build_url, workspace_path, docker_image
+    )
+    logging.info("Going to run %s", run_command)
+
+    run_log_path = os.path.join(temp_path, "run.log")
+    with open(run_log_path, "w", encoding="utf-8") as log:
+        with subprocess.Popen(
+            run_command, shell=True, stderr=log, stdout=log
+        ) as process:
+            retcode = process.wait()
+            if retcode == 0:
+                logging.info("Run successfully")
+            else:
+                logging.info("Run failed")
+
+    subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True)
+
+    check_name_lower = (
+        check_name.lower().replace("(", "").replace(")", "").replace(" ", "")
+    )
+    s3_prefix = f"{pr_info.number}/{pr_info.sha}/sqltest_{check_name_lower}/"
+    paths = {
+        "run.log": run_log_path,
+        "main.log": os.path.join(workspace_path, "main.log"),
+        "server.log.zst": os.path.join(workspace_path, "server.log.zst"),
+        "server.err.log.zst": os.path.join(workspace_path, "server.err.log.zst"),
+        "report.html": os.path.join(workspace_path, "report.html"),
+        "test.log": os.path.join(workspace_path, "test.log"),
+    }
+
+    s3_helper = S3Helper()
+    for f in paths:
+        try:
+            paths[f] = s3_helper.upload_test_report_to_s3(paths[f], s3_prefix + f)
+        except Exception as ex:
+            logging.info("Exception uploading file %s text %s", f, ex)
+            paths[f] = ""
+
+    report_url = GITHUB_RUN_URL
+    if paths["report.html"]:
+        report_url = paths["report.html"]
+
+    status = "success"
+    description = "See the report"
+    test_result = TestResult(description, "OK")
+
+    ch_helper = ClickHouseHelper()
+
+    prepared_events = prepare_tests_results_for_clickhouse(
+        pr_info,
+        [test_result],
+        status,
+        stopwatch.duration_seconds,
+        stopwatch.start_time_str,
+        report_url,
+        check_name,
+    )
+
+    ch_helper.insert_events_into(db="default", table="checks", events=prepared_events)
+
+    logging.info("Result: '%s', '%s', '%s'", status, description, report_url)
+    print(f"::notice ::Report url: {report_url}")
+    post_commit_status(commit, status, report_url, description, check_name, pr_info)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tests/ci/tests/docker_images.json b/tests/ci/tests/docker_images.json
index 0d40d43c33f..0637058e184 100644
--- a/tests/ci/tests/docker_images.json
+++ b/tests/ci/tests/docker_images.json
@@ -153,5 +153,9 @@
     "docker/test/sqllogic": {
         "name": "clickhouse/sqllogic-test",
         "dependent": []
+    },
+    "docker/test/sqltest": {
+        "name": "clickhouse/sqltest",
+        "dependent": []
     }
 }

From df5abcf685a141d163e4aefc213521fc64b613ca Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 19 Jul 2023 03:45:43 +0200
Subject: [PATCH 010/249] Update Docker

---
 docker/test/sqltest/Dockerfile | 1 +
 docker/test/sqltest/test.py    | 0
 2 files changed, 1 insertion(+)
 mode change 100644 => 100755 docker/test/sqltest/test.py

diff --git a/docker/test/sqltest/Dockerfile b/docker/test/sqltest/Dockerfile
index c58907a98a8..437677f4fd1 100644
--- a/docker/test/sqltest/Dockerfile
+++ b/docker/test/sqltest/Dockerfile
@@ -26,4 +26,5 @@ ENV MAX_RUN_TIME=900
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
 COPY run.sh /
+COPY test.py /
 CMD ["/bin/bash", "/run.sh"]
diff --git a/docker/test/sqltest/test.py b/docker/test/sqltest/test.py
old mode 100644
new mode 100755

From 174166b23382488f1a6fc12cbf121b1a1da7f49b Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 19 Jul 2023 03:47:41 +0200
Subject: [PATCH 011/249] Fix error

---
 docker/test/sqltest/run.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/test/sqltest/run.sh b/docker/test/sqltest/run.sh
index ef9ecaec33b..b26d4d89f8b 100755
--- a/docker/test/sqltest/run.sh
+++ b/docker/test/sqltest/run.sh
@@ -34,7 +34,7 @@ done
 
 # Run the test
 pushd sqltest/standards/2016/
-./test.py
+/test.py
 popd
 
 zstd --threads=0 /var/log/clickhouse-server/clickhouse-server.log

From fcf4a6ea887f957b8d911b1611932ecc59260b7a Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 19 Jul 2023 03:48:48 +0200
Subject: [PATCH 012/249] Fix error

---
 docker/test/sqltest/run.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker/test/sqltest/run.sh b/docker/test/sqltest/run.sh
index b26d4d89f8b..24032874403 100755
--- a/docker/test/sqltest/run.sh
+++ b/docker/test/sqltest/run.sh
@@ -35,6 +35,7 @@ done
 # Run the test
 pushd sqltest/standards/2016/
 /test.py
+mv report.html test.log /
 popd
 
 zstd --threads=0 /var/log/clickhouse-server/clickhouse-server.log

From 2255307a477ab86dc7740537a31e4791f3e769a1 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 19 Jul 2023 03:49:47 +0200
Subject: [PATCH 013/249] Improvement

---
 docker/test/sqltest/test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/test/sqltest/test.py b/docker/test/sqltest/test.py
index 7b38fdc52ad..945446358c3 100755
--- a/docker/test/sqltest/test.py
+++ b/docker/test/sqltest/test.py
@@ -95,7 +95,7 @@ def print_ratio(indent, name, success, total, description):
         f" - {description}" if description else ''))
 
 
-report_html_file.write("<html style='font: 16pt Monospace; padding: 1em;'><body><p>")
+report_html_file.write("<html style='font: 16pt; padding: 1em;'><body><pre>")
 
 print_ratio(0, 'Total', summary['success'], summary['total'], '')
 
@@ -115,4 +115,4 @@ for category in summary['results']:
 
         print_ratio(4, test, test_summary['success'], test_summary['total'], test_summary['description'])
 
-report_html_file.write("</p></body></html>")
+report_html_file.write("</pre></body></html>")

From 50dac21483b9003ad546da6d87340124bf108d69 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 19 Jul 2023 03:54:09 +0200
Subject: [PATCH 014/249] Improvement

---
 docker/test/sqltest/test.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/docker/test/sqltest/test.py b/docker/test/sqltest/test.py
index 945446358c3..ae2d6406c78 100755
--- a/docker/test/sqltest/test.py
+++ b/docker/test/sqltest/test.py
@@ -2,6 +2,7 @@
 
 import os
 import yaml
+import html
 import random
 import string
 from clickhouse_driver import Client
@@ -26,11 +27,11 @@ with open('features.yml', 'r') as file:
     yaml_content = yaml.safe_load(file)
 
     for category in yaml_content:
-        log_file.write(category.capitalize() + " features:")
+        log_file.write(category.capitalize() + " features:\n")
         summary['results'][category] = {'success': 0, 'total': 0, 'results': {}}
 
         for test in yaml_content[category]:
-            log_file.write(test + ": " + yaml_content[category][test])
+            log_file.write(test + ": " + yaml_content[category][test] + "\n")
             summary['results'][category]['results'][test] = {'success': 0, 'total': 0, 'description': yaml_content[category][test]}
 
             test_path = test[0] + "/" + test + ".tests.yml"
@@ -54,11 +55,11 @@ with open('features.yml', 'r') as file:
                             summary['results'][category]['total'] += 1
                             summary['total'] += 1
 
-                            log_file.write(query)
+                            log_file.write(query + "\n")
 
                             try:
                                 result = client.execute(query, settings = settings)
-                                log_file.write(result)
+                                log_file.write(result + "\n")
 
                                 if test_group:
                                     summary['results'][category]['results'][test_group]['success'] += 1
@@ -67,7 +68,7 @@ with open('features.yml', 'r') as file:
                                 summary['success'] += 1
 
                             except Exception as e:
-                                log_file.write(f"Error occurred: {str(e)}")
+                                log_file.write(f"Error occurred: {str(e)}\n")
 
 client.execute(f"DROP DATABASE {database_name}", settings = settings)
 
@@ -84,7 +85,7 @@ def enable_color(ratio):
 reset_color = "</b>"
 
 def print_ratio(indent, name, success, total, description):
-    report_html_file.write("{}{}: {}{} / {} ({:.1%}){}{}<br/>".format(
+    report_html_file.write("{}{}: {}{} / {} ({:.1%}){}{}<br/>\n".format(
         ' ' * indent,
         name,
         enable_color(success / total),
@@ -92,10 +93,10 @@ def print_ratio(indent, name, success, total, description):
         total,
         success / total,
         reset_color,
-        f" - {description}" if description else ''))
+        f" - " + html.escape(description) if description else ''))
 
 
-report_html_file.write("<html style='font: 16pt; padding: 1em;'><body><pre>")
+report_html_file.write("<html style='font: 16pt; padding: 1em;'><body><pre>\n")
 
 print_ratio(0, 'Total', summary['success'], summary['total'], '')
 
@@ -115,4 +116,4 @@ for category in summary['results']:
 
         print_ratio(4, test, test_summary['success'], test_summary['total'], test_summary['description'])
 
-report_html_file.write("</pre></body></html>")
+report_html_file.write("</pre></body></html>\n")

From 76677e009b8cf9a501fde1f7273fa5d546f4ffa3 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 19 Jul 2023 04:00:50 +0200
Subject: [PATCH 015/249] Improvement

---
 docker/test/sqltest/test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/test/sqltest/test.py b/docker/test/sqltest/test.py
index ae2d6406c78..cf2f976e05a 100755
--- a/docker/test/sqltest/test.py
+++ b/docker/test/sqltest/test.py
@@ -59,7 +59,7 @@ with open('features.yml', 'r') as file:
 
                             try:
                                 result = client.execute(query, settings = settings)
-                                log_file.write(result + "\n")
+                                log_file.write(str(result) + "\n")
 
                                 if test_group:
                                     summary['results'][category]['results'][test_group]['success'] += 1

From fe5972953703edb8a7e13c6066e5af369359b82c Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 19 Jul 2023 04:01:00 +0200
Subject: [PATCH 016/249] Improvement

---
 docker/test/sqltest/run.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docker/test/sqltest/run.sh b/docker/test/sqltest/run.sh
index 24032874403..03678971f60 100755
--- a/docker/test/sqltest/run.sh
+++ b/docker/test/sqltest/run.sh
@@ -23,6 +23,12 @@ function wget_with_retry
 wget_with_retry "$BINARY_URL_TO_DOWNLOAD"
 chmod +x clickhouse
 ./clickhouse install --noninteractive
+
+echo "
+users:
+  default:
+    access_management: 1" > /etc/clickhouse-server/users.d/access_management.yaml
+
 clickhouse start
 
 # Wait for start

From 9b10e0a5bc2b7e3f84614d1aa3fa84c60de76929 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 19 Jul 2023 04:09:19 +0200
Subject: [PATCH 017/249] Improvement

---
 docker/test/sqltest/test.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docker/test/sqltest/test.py b/docker/test/sqltest/test.py
index cf2f976e05a..ff321d3e5df 100755
--- a/docker/test/sqltest/test.py
+++ b/docker/test/sqltest/test.py
@@ -85,9 +85,9 @@ def enable_color(ratio):
 reset_color = "</b>"
 
 def print_ratio(indent, name, success, total, description):
-    report_html_file.write("{}{}: {}{} / {} ({:.1%}){}{}<br/>\n".format(
+    report_html_file.write("{}{}: {}{} / {} ({:.1%}){}{}\n".format(
         ' ' * indent,
-        name,
+        name.capitalize(),
         enable_color(success / total),
         success,
         total,
@@ -96,7 +96,7 @@ def print_ratio(indent, name, success, total, description):
         f" - " + html.escape(description) if description else ''))
 
 
-report_html_file.write("<html style='font: 16pt; padding: 1em;'><body><pre>\n")
+report_html_file.write("<html><body><pre style='font-size: 16pt; padding: 1em; line-height: 1.25;'>\n")
 
 print_ratio(0, 'Total', summary['success'], summary['total'], '')
 
@@ -114,6 +114,6 @@ for category in summary['results']:
         if test_summary['total'] == 0:
             continue
 
-        print_ratio(4, test, test_summary['success'], test_summary['total'], test_summary['description'])
+        print_ratio(6 if '-' in test else 4, test, test_summary['success'], test_summary['total'], test_summary['description'])
 
 report_html_file.write("</pre></body></html>\n")

From 891c01e6ab30c2d4377a118cb4c449134a56e17a Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Wed, 19 Jul 2023 02:37:02 +0000
Subject: [PATCH 018/249] Automatic style fix

---
 docker/test/sqltest/test.py | 119 ++++++++++++++++++++++--------------
 1 file changed, 74 insertions(+), 45 deletions(-)

diff --git a/docker/test/sqltest/test.py b/docker/test/sqltest/test.py
index ff321d3e5df..5807ca79b02 100755
--- a/docker/test/sqltest/test.py
+++ b/docker/test/sqltest/test.py
@@ -8,69 +8,85 @@ import string
 from clickhouse_driver import Client
 
 
-client = Client(host = 'localhost', port = 9000)
-settings = {'default_table_engine': 'Memory', 'union_default_mode': 'DISTINCT', 'calculate_text_stack_trace': 0}
+client = Client(host="localhost", port=9000)
+settings = {
+    "default_table_engine": "Memory",
+    "union_default_mode": "DISTINCT",
+    "calculate_text_stack_trace": 0,
+}
 
-database_name = 'sqltest_' + ''.join(random.choice(string.ascii_lowercase) for _ in range(10))
+database_name = "sqltest_" + "".join(
+    random.choice(string.ascii_lowercase) for _ in range(10)
+)
 
-client.execute(f"DROP DATABASE IF EXISTS {database_name}", settings = settings)
-client.execute(f"CREATE DATABASE {database_name}", settings = settings)
+client.execute(f"DROP DATABASE IF EXISTS {database_name}", settings=settings)
+client.execute(f"CREATE DATABASE {database_name}", settings=settings)
 
-client = Client(host = 'localhost', port = 9000, database = database_name)
+client = Client(host="localhost", port=9000, database=database_name)
 
-summary = {'success': 0, 'total': 0, 'results': {}}
+summary = {"success": 0, "total": 0, "results": {}}
 
 log_file = open("test.log", "w")
 report_html_file = open("report.html", "w")
 
-with open('features.yml', 'r') as file:
+with open("features.yml", "r") as file:
     yaml_content = yaml.safe_load(file)
 
     for category in yaml_content:
         log_file.write(category.capitalize() + " features:\n")
-        summary['results'][category] = {'success': 0, 'total': 0, 'results': {}}
+        summary["results"][category] = {"success": 0, "total": 0, "results": {}}
 
         for test in yaml_content[category]:
             log_file.write(test + ": " + yaml_content[category][test] + "\n")
-            summary['results'][category]['results'][test] = {'success': 0, 'total': 0, 'description': yaml_content[category][test]}
+            summary["results"][category]["results"][test] = {
+                "success": 0,
+                "total": 0,
+                "description": yaml_content[category][test],
+            }
 
             test_path = test[0] + "/" + test + ".tests.yml"
             if os.path.exists(test_path):
-                with open(test_path, 'r') as test_file:
-                    test_yaml_content = yaml.load_all(test_file, Loader = yaml.FullLoader)
+                with open(test_path, "r") as test_file:
+                    test_yaml_content = yaml.load_all(test_file, Loader=yaml.FullLoader)
 
                     for test_case in test_yaml_content:
-
-                        queries = test_case['sql'];
+                        queries = test_case["sql"]
                         if not isinstance(queries, list):
                             queries = [queries]
 
                         for query in queries:
                             # Example: E011-01
-                            test_group = ''
-                            if '-' in test:
+                            test_group = ""
+                            if "-" in test:
                                 test_group = test.split("-", 1)[0]
-                                summary['results'][category]['results'][test_group]['total'] += 1
-                            summary['results'][category]['results'][test]['total'] += 1
-                            summary['results'][category]['total'] += 1
-                            summary['total'] += 1
+                                summary["results"][category]["results"][test_group][
+                                    "total"
+                                ] += 1
+                            summary["results"][category]["results"][test]["total"] += 1
+                            summary["results"][category]["total"] += 1
+                            summary["total"] += 1
 
                             log_file.write(query + "\n")
 
                             try:
-                                result = client.execute(query, settings = settings)
+                                result = client.execute(query, settings=settings)
                                 log_file.write(str(result) + "\n")
 
                                 if test_group:
-                                    summary['results'][category]['results'][test_group]['success'] += 1
-                                summary['results'][category]['results'][test]['success'] += 1
-                                summary['results'][category]['success'] += 1
-                                summary['success'] += 1
+                                    summary["results"][category]["results"][test_group][
+                                        "success"
+                                    ] += 1
+                                summary["results"][category]["results"][test][
+                                    "success"
+                                ] += 1
+                                summary["results"][category]["success"] += 1
+                                summary["success"] += 1
 
                             except Exception as e:
                                 log_file.write(f"Error occurred: {str(e)}\n")
 
-client.execute(f"DROP DATABASE {database_name}", settings = settings)
+client.execute(f"DROP DATABASE {database_name}", settings=settings)
+
 
 def enable_color(ratio):
     if ratio == 0:
@@ -82,38 +98,51 @@ def enable_color(ratio):
     else:
         return "<b style='color: green;'>"
 
+
 reset_color = "</b>"
 
+
 def print_ratio(indent, name, success, total, description):
-    report_html_file.write("{}{}: {}{} / {} ({:.1%}){}{}\n".format(
-        ' ' * indent,
-        name.capitalize(),
-        enable_color(success / total),
-        success,
-        total,
-        success / total,
-        reset_color,
-        f" - " + html.escape(description) if description else ''))
+    report_html_file.write(
+        "{}{}: {}{} / {} ({:.1%}){}{}\n".format(
+            " " * indent,
+            name.capitalize(),
+            enable_color(success / total),
+            success,
+            total,
+            success / total,
+            reset_color,
+            f" - " + html.escape(description) if description else "",
+        )
+    )
 
 
-report_html_file.write("<html><body><pre style='font-size: 16pt; padding: 1em; line-height: 1.25;'>\n")
+report_html_file.write(
+    "<html><body><pre style='font-size: 16pt; padding: 1em; line-height: 1.25;'>\n"
+)
 
-print_ratio(0, 'Total', summary['success'], summary['total'], '')
+print_ratio(0, "Total", summary["success"], summary["total"], "")
 
-for category in summary['results']:
-    cat_summary = summary['results'][category]
+for category in summary["results"]:
+    cat_summary = summary["results"][category]
 
-    if cat_summary['total'] == 0:
+    if cat_summary["total"] == 0:
         continue
 
-    print_ratio(2, category, cat_summary['success'], cat_summary['total'], '')
+    print_ratio(2, category, cat_summary["success"], cat_summary["total"], "")
 
-    for test in summary['results'][category]['results']:
-        test_summary = summary['results'][category]['results'][test]
+    for test in summary["results"][category]["results"]:
+        test_summary = summary["results"][category]["results"][test]
 
-        if test_summary['total'] == 0:
+        if test_summary["total"] == 0:
             continue
 
-        print_ratio(6 if '-' in test else 4, test, test_summary['success'], test_summary['total'], test_summary['description'])
+        print_ratio(
+            6 if "-" in test else 4,
+            test,
+            test_summary["success"],
+            test_summary["total"],
+            test_summary["description"],
+        )
 
 report_html_file.write("</pre></body></html>\n")

From 29094a22cf4cc787cc50f5d494b29dcf511cc765 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 19 Jul 2023 20:00:00 +0200
Subject: [PATCH 019/249] Fix Docker

---
 docker/images.json                | 1 +
 tests/ci/tests/docker_images.json | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/docker/images.json b/docker/images.json
index 663fa21a6c9..8e19401ba72 100644
--- a/docker/images.json
+++ b/docker/images.json
@@ -125,6 +125,7 @@
             "docker/test/keeper-jepsen",
             "docker/test/server-jepsen",
             "docker/test/sqllogic",
+            "docker/test/sqltest",
             "docker/test/stateless"
          ]
     },
diff --git a/tests/ci/tests/docker_images.json b/tests/ci/tests/docker_images.json
index 0637058e184..70db8760561 100644
--- a/tests/ci/tests/docker_images.json
+++ b/tests/ci/tests/docker_images.json
@@ -119,7 +119,8 @@
             "docker/test/stateless",
             "docker/test/integration/base",
             "docker/test/fuzzer",
-            "docker/test/keeper-jepsen"
+            "docker/test/keeper-jepsen",
+            "docker/test/sqltest"
          ]
     },
     "docker/test/integration/kerberized_hadoop": {

From f458108c44f2d9f0c92725f05d3959658c97ee6f Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 19 Jul 2023 22:15:23 +0200
Subject: [PATCH 020/249] Fix style

---
 docker/test/sqllogic/run.sh | 2 ++
 tests/ci/sqltest.py         | 1 -
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docker/test/sqllogic/run.sh b/docker/test/sqllogic/run.sh
index 444252837a3..3b900c097e2 100755
--- a/docker/test/sqllogic/run.sh
+++ b/docker/test/sqllogic/run.sh
@@ -1,4 +1,6 @@
 #!/bin/bash
+# shellcheck disable=SC2015
+
 set -exu
 trap "exit" INT TERM
 
diff --git a/tests/ci/sqltest.py b/tests/ci/sqltest.py
index 5c20cc4849b..b752d4e4aee 100644
--- a/tests/ci/sqltest.py
+++ b/tests/ci/sqltest.py
@@ -11,7 +11,6 @@ from build_download_helper import get_build_name_for_check, read_build_urls
 from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse
 from commit_status_helper import (
     RerunHelper,
-    format_description,
     get_commit,
     post_commit_status,
 )

From fa44f84377bc097cbe0c25852603de9be536f75e Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 20 Jul 2023 04:50:12 +0200
Subject: [PATCH 021/249] Fix style

---
 docker/test/sqllogic/run.sh | 1 -
 docker/test/sqltest/run.sh  | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/test/sqllogic/run.sh b/docker/test/sqllogic/run.sh
index 3b900c097e2..5be44fc148c 100755
--- a/docker/test/sqllogic/run.sh
+++ b/docker/test/sqllogic/run.sh
@@ -1,5 +1,4 @@
 #!/bin/bash
-# shellcheck disable=SC2015
 
 set -exu
 trap "exit" INT TERM
diff --git a/docker/test/sqltest/run.sh b/docker/test/sqltest/run.sh
index 03678971f60..42aeef9df15 100755
--- a/docker/test/sqltest/run.sh
+++ b/docker/test/sqltest/run.sh
@@ -1,4 +1,5 @@
 #!/bin/bash
+# shellcheck disable=SC2015
 
 set -x
 set -e

From 2a00e2aa6ae7c417fdd5aafd63225ef44460de6f Mon Sep 17 00:00:00 2001
From: pufit <pufit@clickhouse.com>
Date: Thu, 20 Jul 2023 22:20:36 -0400
Subject: [PATCH 022/249] Implement `MOVE` query for the access entities.

---
 src/Access/AccessControl.cpp                  |  4 +-
 src/Access/AccessControl.h                    |  2 +-
 src/Access/DiskAccessStorage.cpp              |  4 +-
 src/Access/DiskAccessStorage.h                |  2 +-
 src/Access/IAccessStorage.cpp                 | 46 +++++++--
 src/Access/IAccessStorage.h                   | 21 +++-
 src/Access/MemoryAccessStorage.cpp            |  4 +-
 src/Access/MemoryAccessStorage.h              |  2 +-
 src/Access/MultipleAccessStorage.cpp          | 40 +++++++-
 src/Access/MultipleAccessStorage.h            |  9 +-
 src/Access/ReplicatedAccessStorage.cpp        |  4 +-
 src/Access/ReplicatedAccessStorage.h          |  2 +-
 .../Access/InterpreterCreateQuotaQuery.cpp    | 36 +++++--
 .../Access/InterpreterCreateRoleQuery.cpp     |  8 +-
 .../InterpreterCreateRowPolicyQuery.cpp       | 38 ++++++--
 .../InterpreterCreateSettingsProfileQuery.cpp | 37 +++++--
 .../Access/InterpreterCreateUserQuery.cpp     | 33 +++++--
 .../InterpreterMoveAccessEntityQuery.cpp      | 96 +++++++++++++++++++
 .../Access/InterpreterMoveAccessEntityQuery.h | 24 +++++
 src/Interpreters/InterpreterFactory.cpp       |  6 ++
 src/Parsers/Access/ASTCreateQuotaQuery.cpp    |  6 ++
 src/Parsers/Access/ASTCreateQuotaQuery.h      |  1 +
 src/Parsers/Access/ASTCreateRoleQuery.cpp     |  2 +-
 .../Access/ASTCreateRowPolicyQuery.cpp        |  5 +
 src/Parsers/Access/ASTCreateRowPolicyQuery.h  |  1 +
 .../Access/ASTCreateSettingsProfileQuery.cpp  |  6 ++
 .../Access/ASTCreateSettingsProfileQuery.h    |  1 +
 src/Parsers/Access/ASTCreateUserQuery.cpp     |  5 +
 src/Parsers/Access/ASTCreateUserQuery.h       |  1 +
 .../Access/ASTDropAccessEntityQuery.cpp       |  4 +-
 .../Access/ASTMoveAccessEntityQuery.cpp       | 64 +++++++++++++
 src/Parsers/Access/ASTMoveAccessEntityQuery.h | 32 +++++++
 src/Parsers/Access/ParserCreateQuotaQuery.cpp |  6 ++
 src/Parsers/Access/ParserCreateRoleQuery.cpp  |  6 +-
 .../Access/ParserCreateRowPolicyQuery.cpp     |  6 ++
 .../ParserCreateSettingsProfileQuery.cpp      |  6 ++
 src/Parsers/Access/ParserCreateUserQuery.cpp  |  5 +
 .../Access/ParserMoveAccessEntityQuery.cpp    | 93 ++++++++++++++++++
 .../Access/ParserMoveAccessEntityQuery.h      | 17 ++++
 src/Parsers/Access/parseUserName.h            |  3 +-
 src/Parsers/IAST.h                            |  1 +
 src/Parsers/ParserQuery.cpp                   |  3 +
 .../integration/test_grant_and_revoke/test.py |  4 +-
 .../configs/memory.xml                        |  5 +
 .../configs/roles.xml                         |  4 -
 .../test.py                                   | 67 ++++++++++++-
 tests/integration/test_role/test.py           |  2 +-
 .../integration/test_settings_profile/test.py | 14 +--
 .../integration/test_user_directories/test.py | 46 ++++-----
 49 files changed, 731 insertions(+), 103 deletions(-)
 create mode 100644 src/Interpreters/Access/InterpreterMoveAccessEntityQuery.cpp
 create mode 100644 src/Interpreters/Access/InterpreterMoveAccessEntityQuery.h
 create mode 100644 src/Parsers/Access/ASTMoveAccessEntityQuery.cpp
 create mode 100644 src/Parsers/Access/ASTMoveAccessEntityQuery.h
 create mode 100644 src/Parsers/Access/ParserMoveAccessEntityQuery.cpp
 create mode 100644 src/Parsers/Access/ParserMoveAccessEntityQuery.h
 create mode 100644 tests/integration/test_multi_access_storage_role_management/configs/memory.xml

diff --git a/src/Access/AccessControl.cpp b/src/Access/AccessControl.cpp
index 0c61a9e3e1b..22b00d5dde7 100644
--- a/src/Access/AccessControl.cpp
+++ b/src/Access/AccessControl.cpp
@@ -525,9 +525,9 @@ scope_guard AccessControl::subscribeForChanges(const std::vector<UUID> & ids, co
     return changes_notifier->subscribeForChanges(ids, handler);
 }
 
-std::optional<UUID> AccessControl::insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists)
+std::optional<UUID> AccessControl::insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, std::optional<UUID> set_id)
 {
-    auto id = MultipleAccessStorage::insertImpl(entity, replace_if_exists, throw_if_exists);
+    auto id = MultipleAccessStorage::insertImpl(entity, replace_if_exists, throw_if_exists, set_id);
     if (id)
         changes_notifier->sendNotifications();
     return id;
diff --git a/src/Access/AccessControl.h b/src/Access/AccessControl.h
index 2a8293a49e7..85feaa70462 100644
--- a/src/Access/AccessControl.h
+++ b/src/Access/AccessControl.h
@@ -229,7 +229,7 @@ private:
     class CustomSettingsPrefixes;
     class PasswordComplexityRules;
 
-    std::optional<UUID> insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override;
+    std::optional<UUID> insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, std::optional<UUID> set_id) override;
     bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
     bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;
 
diff --git a/src/Access/DiskAccessStorage.cpp b/src/Access/DiskAccessStorage.cpp
index 938881fafff..acbd7b12569 100644
--- a/src/Access/DiskAccessStorage.cpp
+++ b/src/Access/DiskAccessStorage.cpp
@@ -498,9 +498,9 @@ std::optional<std::pair<String, AccessEntityType>> DiskAccessStorage::readNameWi
 }
 
 
-std::optional<UUID> DiskAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists)
+std::optional<UUID> DiskAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, std::optional<UUID> set_id)
 {
-    UUID id = generateRandomID();
+    UUID id = set_id ? *set_id : generateRandomID();
     if (insertWithID(id, new_entity, replace_if_exists, throw_if_exists, /* write_on_disk= */ true))
         return id;
 
diff --git a/src/Access/DiskAccessStorage.h b/src/Access/DiskAccessStorage.h
index e3bd32c0292..3541c3228e3 100644
--- a/src/Access/DiskAccessStorage.h
+++ b/src/Access/DiskAccessStorage.h
@@ -39,7 +39,7 @@ private:
     std::vector<UUID> findAllImpl(AccessEntityType type) const override;
     AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override;
     std::optional<std::pair<String, AccessEntityType>> readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const override;
-    std::optional<UUID> insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override;
+    std::optional<UUID> insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, std::optional<UUID> set_id) override;
     bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
     bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;
 
diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp
index 9468e8d220a..11bc97f6cc3 100644
--- a/src/Access/IAccessStorage.cpp
+++ b/src/Access/IAccessStorage.cpp
@@ -93,6 +93,17 @@ String IAccessStorage::readName(const UUID & id) const
 }
 
 
+bool IAccessStorage::exists(const std::vector<UUID> & ids) const
+{
+    for (const auto & id : ids)
+    {
+        if (!exists(id))
+            return false;
+    }
+
+    return true;
+}
+
 std::optional<String> IAccessStorage::readName(const UUID & id, bool throw_if_not_exists) const
 {
     if (auto name_and_type = readNameWithType(id, throw_if_not_exists))
@@ -167,15 +178,26 @@ UUID IAccessStorage::insert(const AccessEntityPtr & entity)
     return *insert(entity, /* replace_if_exists = */ false, /* throw_if_exists = */ true);
 }
 
-
 std::optional<UUID> IAccessStorage::insert(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists)
 {
-    return insertImpl(entity, replace_if_exists, throw_if_exists);
+    return *insert(entity, replace_if_exists, throw_if_exists, /* set_id = */ std::nullopt);
 }
 
+std::optional<UUID> IAccessStorage::insert(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, std::optional<UUID> set_id)
+{
+    return insertImpl(entity, replace_if_exists, throw_if_exists, set_id);
+}
 
 std::vector<UUID> IAccessStorage::insert(const std::vector<AccessEntityPtr> & multiple_entities, bool replace_if_exists, bool throw_if_exists)
 {
+    return insert(multiple_entities, /* ids = */ {}, replace_if_exists, throw_if_exists);
+}
+
+std::vector<UUID> IAccessStorage::insert(const std::vector<AccessEntityPtr> & multiple_entities, const std::vector<UUID> & ids, bool replace_if_exists, bool throw_if_exists)
+{
+    if (!ids.empty())
+        assert(multiple_entities.size() == ids.size());
+
     if (multiple_entities.empty())
         return {};
 
@@ -189,16 +211,24 @@ std::vector<UUID> IAccessStorage::insert(const std::vector<AccessEntityPtr> & mu
     std::vector<AccessEntityPtr> successfully_inserted;
     try
     {
-        std::vector<UUID> ids;
-        for (const auto & entity : multiple_entities)
+        std::vector<UUID> new_ids;
+        for (size_t i = 0; i < multiple_entities.size(); ++i)
         {
-            if (auto id = insertImpl(entity, replace_if_exists, throw_if_exists))
+            const auto & entity = multiple_entities[i];
+
+            std::optional<UUID> id;
+            if (!ids.empty())
+                id = ids[i];
+
+            auto new_id = insertImpl(entity, replace_if_exists, throw_if_exists, id);
+
+            if (new_id)
             {
                 successfully_inserted.push_back(entity);
-                ids.push_back(*id);
+                new_ids.push_back(*new_id);
             }
         }
-        return ids;
+        return new_ids;
     }
     catch (Exception & e)
     {
@@ -244,7 +274,7 @@ std::vector<UUID> IAccessStorage::insertOrReplace(const std::vector<AccessEntity
 }
 
 
-std::optional<UUID> IAccessStorage::insertImpl(const AccessEntityPtr & entity, bool, bool)
+std::optional<UUID> IAccessStorage::insertImpl(const AccessEntityPtr & entity, bool, bool, std::optional<UUID>)
 {
     if (isReadOnly())
         throwReadonlyCannotInsert(entity->getType(), entity->getName());
diff --git a/src/Access/IAccessStorage.h b/src/Access/IAccessStorage.h
index aa3947201e7..894220e95ad 100644
--- a/src/Access/IAccessStorage.h
+++ b/src/Access/IAccessStorage.h
@@ -92,6 +92,7 @@ public:
 
     /// Returns whether there is an entity with such identifier in the storage.
     virtual bool exists(const UUID & id) const = 0;
+    bool exists(const std::vector<UUID> & ids) const;
 
     /// Reads an entity. Throws an exception if not found.
     template <typename EntityClassT = IAccessEntity>
@@ -100,6 +101,9 @@ public:
     template <typename EntityClassT = IAccessEntity>
     std::shared_ptr<const EntityClassT> read(const String & name, bool throw_if_not_exists = true) const;
 
+    template <typename EntityClassT = IAccessEntity>
+    std::vector<AccessEntityPtr> read(const std::vector<UUID> & ids, bool throw_if_not_exists = true) const;
+
     /// Reads an entity. Returns nullptr if not found.
     template <typename EntityClassT = IAccessEntity>
     std::shared_ptr<const EntityClassT> tryRead(const UUID & id) const;
@@ -128,7 +132,9 @@ public:
     /// Throws an exception if the specified name already exists.
     UUID insert(const AccessEntityPtr & entity);
     std::optional<UUID> insert(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists);
+    std::optional<UUID> insert(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, std::optional<UUID> set_id);
     std::vector<UUID> insert(const std::vector<AccessEntityPtr> & multiple_entities, bool replace_if_exists = false, bool throw_if_exists = true);
+    std::vector<UUID> insert(const std::vector<AccessEntityPtr> & multiple_entities, const std::vector<UUID> & ids, bool replace_if_exists = false, bool throw_if_exists = true);
 
     /// Inserts an entity to the storage. Returns ID of a new entry in the storage.
     std::optional<UUID> tryInsert(const AccessEntityPtr & entity);
@@ -179,7 +185,7 @@ protected:
     virtual std::vector<UUID> findAllImpl(AccessEntityType type) const = 0;
     virtual AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const = 0;
     virtual std::optional<std::pair<String, AccessEntityType>> readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const;
-    virtual std::optional<UUID> insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists);
+    virtual std::optional<UUID> insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, std::optional<UUID> set_id);
     virtual bool removeImpl(const UUID & id, bool throw_if_not_exists);
     virtual bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists);
     virtual std::optional<UUID> authenticateImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool throw_if_user_not_exists, bool allow_no_password, bool allow_plaintext_password) const;
@@ -240,6 +246,19 @@ std::shared_ptr<const EntityClassT> IAccessStorage::read(const String & name, bo
 }
 
 
+template <typename EntityClassT>
+std::vector<AccessEntityPtr> IAccessStorage::read(const std::vector<UUID> & ids, bool throw_if_not_exists) const
+{
+    std::vector<AccessEntityPtr> result;
+    result.reserve(ids.size());
+
+    for (const auto & id : ids)
+        result.push_back(read<EntityClassT>(id, throw_if_not_exists));
+
+    return result;
+}
+
+
 template <typename EntityClassT>
 std::shared_ptr<const EntityClassT> IAccessStorage::tryRead(const UUID & id) const
 {
diff --git a/src/Access/MemoryAccessStorage.cpp b/src/Access/MemoryAccessStorage.cpp
index c4192ee4552..16f3b3793a0 100644
--- a/src/Access/MemoryAccessStorage.cpp
+++ b/src/Access/MemoryAccessStorage.cpp
@@ -63,9 +63,9 @@ AccessEntityPtr MemoryAccessStorage::readImpl(const UUID & id, bool throw_if_not
 }
 
 
-std::optional<UUID> MemoryAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists)
+std::optional<UUID> MemoryAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, std::optional<UUID> set_id)
 {
-    UUID id = generateRandomID();
+    UUID id = set_id ? *set_id : generateRandomID();
     if (insertWithID(id, new_entity, replace_if_exists, throw_if_exists))
         return id;
 
diff --git a/src/Access/MemoryAccessStorage.h b/src/Access/MemoryAccessStorage.h
index b63132147da..7c19ab9c4ab 100644
--- a/src/Access/MemoryAccessStorage.h
+++ b/src/Access/MemoryAccessStorage.h
@@ -44,7 +44,7 @@ private:
     std::optional<UUID> findImpl(AccessEntityType type, const String & name) const override;
     std::vector<UUID> findAllImpl(AccessEntityType type) const override;
     AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override;
-    std::optional<UUID> insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override;
+    std::optional<UUID> insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, std::optional<UUID> set_id) override;
     bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
     bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;
 
diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp
index f36beb3f80b..169dc889509 100644
--- a/src/Access/MultipleAccessStorage.cpp
+++ b/src/Access/MultipleAccessStorage.cpp
@@ -213,6 +213,42 @@ ConstStoragePtr MultipleAccessStorage::getStorageByName(const DB::String & stora
     return const_cast<MultipleAccessStorage *>(this)->getStorageByName(storage_name);
 }
 
+StoragePtr MultipleAccessStorage::findExcludingStorage(AccessEntityType type, const DB::String & name, DB::MultipleAccessStorage::StoragePtr exclude) const
+{
+    auto storages = getStoragesInternal();
+    for (const auto & storage : *storages)
+    {
+        if (storage == exclude)
+            continue;
+
+        if (storage->find(type, name))
+            return storage;
+    }
+
+    return nullptr;
+}
+
+void MultipleAccessStorage::moveAccessEntities(const std::vector<UUID> & ids, const String & source_storage_name, const String & destination_storage_name)
+{
+    auto source_storage = findStorageByName(source_storage_name);
+    auto destination_storage = findStorageByName(destination_storage_name);
+
+    auto to_move = source_storage->read(ids);
+    source_storage->remove(ids);
+
+    try
+    {
+        destination_storage->insert(to_move, ids);
+    }
+    catch (Exception & e)
+    {
+        e.addMessage("while moving access entities");
+
+        source_storage->insert(to_move, ids);
+        throw;
+    }
+}
+
 AccessEntityPtr MultipleAccessStorage::readImpl(const UUID & id, bool throw_if_not_exists) const
 {
     if (auto storage = findStorage(id))
@@ -280,7 +316,7 @@ void MultipleAccessStorage::reload(ReloadMode reload_mode)
 }
 
 
-std::optional<UUID> MultipleAccessStorage::insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists)
+std::optional<UUID> MultipleAccessStorage::insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, std::optional<UUID> set_id)
 {
     std::shared_ptr<IAccessStorage> storage_for_insertion;
 
@@ -303,7 +339,7 @@ std::optional<UUID> MultipleAccessStorage::insertImpl(const AccessEntityPtr & en
             getStorageName());
     }
 
-    auto id = storage_for_insertion->insert(entity, replace_if_exists, throw_if_exists);
+    auto id = storage_for_insertion->insert(entity, replace_if_exists, throw_if_exists, set_id);
     if (id)
     {
         std::lock_guard lock{mutex};
diff --git a/src/Access/MultipleAccessStorage.h b/src/Access/MultipleAccessStorage.h
index 1898c6ba5d3..1d95296b796 100644
--- a/src/Access/MultipleAccessStorage.h
+++ b/src/Access/MultipleAccessStorage.h
@@ -46,6 +46,11 @@ public:
     ConstStoragePtr getStorageByName(const String & storage_name) const;
     StoragePtr getStorageByName(const String & storage_name);
 
+    /// Search for an access entity storage, excluding one. Returns nullptr if not found.
+    StoragePtr findExcludingStorage(AccessEntityType type, const String & name, StoragePtr exclude) const;
+
+    void moveAccessEntities(const std::vector<UUID> & ids, const String & source_storage_name, const String & destination_storage_name);
+
     bool exists(const UUID & id) const override;
 
     bool isBackupAllowed() const override;
@@ -58,7 +63,7 @@ protected:
     std::vector<UUID> findAllImpl(AccessEntityType type) const override;
     AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override;
     std::optional<std::pair<String, AccessEntityType>> readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const override;
-    std::optional<UUID> insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override;
+    std::optional<UUID> insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, std::optional<UUID> set_id) override;
     bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
     bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;
     std::optional<UUID> authenticateImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool throw_if_user_not_exists, bool allow_no_password, bool allow_plaintext_password) const override;
@@ -70,6 +75,8 @@ private:
     std::shared_ptr<const Storages> nested_storages TSA_GUARDED_BY(mutex);
     mutable CacheBase<UUID, Storage> ids_cache TSA_GUARDED_BY(mutex);
     mutable std::mutex mutex;
+
+    mutable std::mutex move_mutex;
 };
 
 }
diff --git a/src/Access/ReplicatedAccessStorage.cpp b/src/Access/ReplicatedAccessStorage.cpp
index 56d68be9268..efa9dbbdfaa 100644
--- a/src/Access/ReplicatedAccessStorage.cpp
+++ b/src/Access/ReplicatedAccessStorage.cpp
@@ -108,9 +108,9 @@ static void retryOnZooKeeperUserError(size_t attempts, Func && function)
     }
 }
 
-std::optional<UUID> ReplicatedAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists)
+std::optional<UUID> ReplicatedAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, std::optional<UUID> set_id)
 {
-    const UUID id = generateRandomID();
+    const UUID id = set_id ? *set_id : generateRandomID();
     if (insertWithID(id, new_entity, replace_if_exists, throw_if_exists))
         return id;
 
diff --git a/src/Access/ReplicatedAccessStorage.h b/src/Access/ReplicatedAccessStorage.h
index 555d58e6b04..0a0c53d3bc4 100644
--- a/src/Access/ReplicatedAccessStorage.h
+++ b/src/Access/ReplicatedAccessStorage.h
@@ -46,7 +46,7 @@ private:
     std::unique_ptr<ThreadFromGlobalPool> watching_thread;
     std::shared_ptr<ConcurrentBoundedQueue<UUID>> watched_queue;
 
-    std::optional<UUID> insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override;
+    std::optional<UUID> insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, std::optional<UUID> set_id) override;
     bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
     bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;
 
diff --git a/src/Interpreters/Access/InterpreterCreateQuotaQuery.cpp b/src/Interpreters/Access/InterpreterCreateQuotaQuery.cpp
index 83a620d11c6..e271497ff5c 100644
--- a/src/Interpreters/Access/InterpreterCreateQuotaQuery.cpp
+++ b/src/Interpreters/Access/InterpreterCreateQuotaQuery.cpp
@@ -14,6 +14,12 @@
 
 namespace DB
 {
+
+namespace ErrorCodes
+{
+    extern const int ACCESS_ENTITY_ALREADY_EXISTS;
+}
+
 namespace
 {
     void updateQuotaFromQueryImpl(
@@ -90,6 +96,15 @@ BlockIO InterpreterCreateQuotaQuery::execute()
     if (query.roles)
         roles_from_query = RolesOrUsersSet{*query.roles, access_control, getContext()->getUserID()};
 
+    IAccessStorage * storage = &access_control;
+    MultipleAccessStorage::StoragePtr storage_ptr;
+
+    if (!query.storage_name.empty())
+    {
+        storage_ptr = access_control.getStorageByName(query.storage_name);
+        storage = storage_ptr.get();
+    }
+
     if (query.alter)
     {
         auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr
@@ -100,11 +115,11 @@ BlockIO InterpreterCreateQuotaQuery::execute()
         };
         if (query.if_exists)
         {
-            auto ids = access_control.find<Quota>(query.names);
-            access_control.tryUpdate(ids, update_func);
+            auto ids = storage->find<Quota>(query.names);
+            storage->tryUpdate(ids, update_func);
         }
         else
-            access_control.update(access_control.getIDs<Quota>(query.names), update_func);
+            storage->update(storage->getIDs<Quota>(query.names), update_func);
     }
     else
     {
@@ -116,12 +131,21 @@ BlockIO InterpreterCreateQuotaQuery::execute()
             new_quotas.emplace_back(std::move(new_quota));
         }
 
+        if (!query.storage_name.empty())
+        {
+            for (const auto & name : query.names)
+            {
+                if (auto another_storage_ptr = access_control.findExcludingStorage(AccessEntityType::QUOTA, name, storage_ptr))
+                    throw Exception(ErrorCodes::ACCESS_ENTITY_ALREADY_EXISTS, "Quota {} already exists in storage {}", name, another_storage_ptr->getStorageName());
+            }
+        }
+
         if (query.if_not_exists)
-            access_control.tryInsert(new_quotas);
+            storage->tryInsert(new_quotas);
         else if (query.or_replace)
-            access_control.insertOrReplace(new_quotas);
+            storage->insertOrReplace(new_quotas);
         else
-            access_control.insert(new_quotas);
+            storage->insert(new_quotas);
     }
 
     return {};
diff --git a/src/Interpreters/Access/InterpreterCreateRoleQuery.cpp b/src/Interpreters/Access/InterpreterCreateRoleQuery.cpp
index 87abe002d3d..f6c3103a7a9 100644
--- a/src/Interpreters/Access/InterpreterCreateRoleQuery.cpp
+++ b/src/Interpreters/Access/InterpreterCreateRoleQuery.cpp
@@ -97,13 +97,7 @@ BlockIO InterpreterCreateRoleQuery::execute()
         {
             for (const auto & name : query.names)
             {
-                auto id = access_control.find<Role>(name);
-
-                if (!id)
-                    continue;
-
-                auto another_storage_ptr = access_control.findStorage(*id);
-                if (another_storage_ptr != storage_ptr)
+                if (auto another_storage_ptr = access_control.findExcludingStorage(AccessEntityType::ROLE, name, storage_ptr))
                     throw Exception(ErrorCodes::ACCESS_ENTITY_ALREADY_EXISTS, "Role {} already exists in storage {}", name, another_storage_ptr->getStorageName());
             }
         }
diff --git a/src/Interpreters/Access/InterpreterCreateRowPolicyQuery.cpp b/src/Interpreters/Access/InterpreterCreateRowPolicyQuery.cpp
index 87dc9039c17..a938d7afc16 100644
--- a/src/Interpreters/Access/InterpreterCreateRowPolicyQuery.cpp
+++ b/src/Interpreters/Access/InterpreterCreateRowPolicyQuery.cpp
@@ -14,6 +14,12 @@
 
 namespace DB
 {
+
+namespace ErrorCodes
+{
+    extern const int ACCESS_ENTITY_ALREADY_EXISTS;
+}
+
 namespace
 {
     void updateRowPolicyFromQueryImpl(
@@ -66,6 +72,16 @@ BlockIO InterpreterCreateRowPolicyQuery::execute()
     if (query.roles)
         roles_from_query = RolesOrUsersSet{*query.roles, access_control, getContext()->getUserID()};
 
+    IAccessStorage * storage = &access_control;
+    MultipleAccessStorage::StoragePtr storage_ptr;
+
+    if (!query.storage_name.empty())
+    {
+        storage_ptr = access_control.getStorageByName(query.storage_name);
+        storage = storage_ptr.get();
+    }
+
+    Strings names = query.names->toStrings();
     if (query.alter)
     {
         auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr
@@ -74,14 +90,13 @@ BlockIO InterpreterCreateRowPolicyQuery::execute()
             updateRowPolicyFromQueryImpl(*updated_policy, query, {}, roles_from_query);
             return updated_policy;
         };
-        Strings names = query.names->toStrings();
         if (query.if_exists)
         {
-            auto ids = access_control.find<RowPolicy>(names);
-            access_control.tryUpdate(ids, update_func);
+            auto ids = storage->find<RowPolicy>(names);
+            storage->tryUpdate(ids, update_func);
         }
         else
-            access_control.update(access_control.getIDs<RowPolicy>(names), update_func);
+            storage->update(storage->getIDs<RowPolicy>(names), update_func);
     }
     else
     {
@@ -93,12 +108,21 @@ BlockIO InterpreterCreateRowPolicyQuery::execute()
             new_policies.emplace_back(std::move(new_policy));
         }
 
+        if (!query.storage_name.empty())
+        {
+            for (const auto & name : names)
+            {
+                if (auto another_storage_ptr = access_control.findExcludingStorage(AccessEntityType::ROW_POLICY, name, storage_ptr))
+                    throw Exception(ErrorCodes::ACCESS_ENTITY_ALREADY_EXISTS, "Row policy {} already exists in storage {}", name, another_storage_ptr->getStorageName());
+            }
+        }
+
         if (query.if_not_exists)
-            access_control.tryInsert(new_policies);
+            storage->tryInsert(new_policies);
         else if (query.or_replace)
-            access_control.insertOrReplace(new_policies);
+            storage->insertOrReplace(new_policies);
         else
-            access_control.insert(new_policies);
+            storage->insert(new_policies);
     }
 
     return {};
diff --git a/src/Interpreters/Access/InterpreterCreateSettingsProfileQuery.cpp b/src/Interpreters/Access/InterpreterCreateSettingsProfileQuery.cpp
index 0727b6f2182..bac0146e821 100644
--- a/src/Interpreters/Access/InterpreterCreateSettingsProfileQuery.cpp
+++ b/src/Interpreters/Access/InterpreterCreateSettingsProfileQuery.cpp
@@ -10,6 +10,12 @@
 
 namespace DB
 {
+
+namespace ErrorCodes
+{
+    extern const int ACCESS_ENTITY_ALREADY_EXISTS;
+}
+
 namespace
 {
     void updateSettingsProfileFromQueryImpl(
@@ -67,6 +73,16 @@ BlockIO InterpreterCreateSettingsProfileQuery::execute()
     if (query.to_roles)
         roles_from_query = RolesOrUsersSet{*query.to_roles, access_control, getContext()->getUserID()};
 
+
+    IAccessStorage * storage = &access_control;
+    MultipleAccessStorage::StoragePtr storage_ptr;
+
+    if (!query.storage_name.empty())
+    {
+        storage_ptr = access_control.getStorageByName(query.storage_name);
+        storage = storage_ptr.get();
+    }
+
     if (query.alter)
     {
         auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr
@@ -77,11 +93,11 @@ BlockIO InterpreterCreateSettingsProfileQuery::execute()
         };
         if (query.if_exists)
         {
-            auto ids = access_control.find<SettingsProfile>(query.names);
-            access_control.tryUpdate(ids, update_func);
+            auto ids = storage->find<SettingsProfile>(query.names);
+            storage->tryUpdate(ids, update_func);
         }
         else
-            access_control.update(access_control.getIDs<SettingsProfile>(query.names), update_func);
+            storage->update(storage->getIDs<SettingsProfile>(query.names), update_func);
     }
     else
     {
@@ -93,12 +109,21 @@ BlockIO InterpreterCreateSettingsProfileQuery::execute()
             new_profiles.emplace_back(std::move(new_profile));
         }
 
+        if (!query.storage_name.empty())
+        {
+            for (const auto & name : query.names)
+            {
+                if (auto another_storage_ptr = access_control.findExcludingStorage(AccessEntityType::SETTINGS_PROFILE, name, storage_ptr))
+                    throw Exception(ErrorCodes::ACCESS_ENTITY_ALREADY_EXISTS, "Settings profile {} already exists in storage {}", name, another_storage_ptr->getStorageName());
+            }
+        }
+
         if (query.if_not_exists)
-            access_control.tryInsert(new_profiles);
+            storage->tryInsert(new_profiles);
         else if (query.or_replace)
-            access_control.insertOrReplace(new_profiles);
+            storage->insertOrReplace(new_profiles);
         else
-            access_control.insert(new_profiles);
+            storage->insert(new_profiles);
     }
 
     return {};
diff --git a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp
index 165937560cc..f1d921a9eb2 100644
--- a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp
+++ b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp
@@ -17,6 +17,7 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int BAD_ARGUMENTS;
+    extern const int ACCESS_ENTITY_ALREADY_EXISTS;
 }
 namespace
 {
@@ -139,6 +140,16 @@ BlockIO InterpreterCreateUserQuery::execute()
     if (!query.cluster.empty())
         return executeDDLQueryOnCluster(query_ptr, getContext());
 
+    IAccessStorage * storage = &access_control;
+    MultipleAccessStorage::StoragePtr storage_ptr;
+
+    if (!query.storage_name.empty())
+    {
+        storage_ptr = access_control.getStorageByName(query.storage_name);
+        storage = storage_ptr.get();
+    }
+
+    Strings names = query.names->toStrings();
     if (query.alter)
     {
         std::optional<RolesOrUsersSet> grantees_from_query;
@@ -152,14 +163,13 @@ BlockIO InterpreterCreateUserQuery::execute()
             return updated_user;
         };
 
-        Strings names = query.names->toStrings();
         if (query.if_exists)
         {
-            auto ids = access_control.find<User>(names);
-            access_control.tryUpdate(ids, update_func);
+            auto ids = storage->find<User>(names);
+            storage->tryUpdate(ids, update_func);
         }
         else
-            access_control.update(access_control.getIDs<User>(names), update_func);
+            storage->update(storage->getIDs<User>(names), update_func);
     }
     else
     {
@@ -171,13 +181,22 @@ BlockIO InterpreterCreateUserQuery::execute()
             new_users.emplace_back(std::move(new_user));
         }
 
+        if (!query.storage_name.empty())
+        {
+            for (const auto & name : names)
+            {
+                if (auto another_storage_ptr = access_control.findExcludingStorage(AccessEntityType::USER, name, storage_ptr))
+                    throw Exception(ErrorCodes::ACCESS_ENTITY_ALREADY_EXISTS, "User {} already exists in storage {}", name, another_storage_ptr->getStorageName());
+            }
+        }
+
         std::vector<UUID> ids;
         if (query.if_not_exists)
-            ids = access_control.tryInsert(new_users);
+            ids = storage->tryInsert(new_users);
         else if (query.or_replace)
-            ids = access_control.insertOrReplace(new_users);
+            ids = storage->insertOrReplace(new_users);
         else
-            ids = access_control.insert(new_users);
+            ids = storage->insert(new_users);
 
         if (query.grantees)
         {
diff --git a/src/Interpreters/Access/InterpreterMoveAccessEntityQuery.cpp b/src/Interpreters/Access/InterpreterMoveAccessEntityQuery.cpp
new file mode 100644
index 00000000000..ac7ff4efcb6
--- /dev/null
+++ b/src/Interpreters/Access/InterpreterMoveAccessEntityQuery.cpp
@@ -0,0 +1,96 @@
+#include <Interpreters/Access/InterpreterMoveAccessEntityQuery.h>
+#include <Parsers/Access/ASTMoveAccessEntityQuery.h>
+#include <Parsers/Access/ASTRowPolicyName.h>
+#include <Access/AccessControl.h>
+#include <Access/Common/AccessRightsElement.h>
+#include <Interpreters/executeDDLQueryOnCluster.h>
+
+
+namespace DB
+{
+namespace ErrorCodes
+{
+    extern const int NOT_IMPLEMENTED;
+    extern const int ACCESS_ENTITY_NOT_FOUND;
+}
+
+
+BlockIO InterpreterMoveAccessEntityQuery::execute()
+{
+    auto & query = query_ptr->as<ASTMoveAccessEntityQuery &>();
+    auto & access_control = getContext()->getAccessControl();
+    getContext()->checkAccess(getRequiredAccess());
+
+    if (!query.cluster.empty())
+        return executeDDLQueryOnCluster(query_ptr, getContext());
+
+    query.replaceEmptyDatabase(getContext()->getCurrentDatabase());
+
+    std::vector<UUID> ids;
+    if (query.type == AccessEntityType::ROW_POLICY)
+        ids = access_control.find(query.type, query.row_policy_names->toStrings());
+    else
+        ids = access_control.find(query.type, query.names);
+
+    if (ids.empty())
+        return {};
+
+    /// Validate that all entities are from the same storage.
+    const auto source_storage = access_control.findStorage(ids.front());
+    if (!source_storage->exists(ids))
+        throw Exception(ErrorCodes::ACCESS_ENTITY_NOT_FOUND, "All access entities must be from the same storage in order to be moved");
+
+    access_control.moveAccessEntities(ids, source_storage->getStorageName(), query.storage_name);
+    return {};
+}
+
+
+AccessRightsElements InterpreterMoveAccessEntityQuery::getRequiredAccess() const
+{
+    const auto & query = query_ptr->as<const ASTMoveAccessEntityQuery &>();
+    AccessRightsElements res;
+    switch (query.type)
+    {
+        case AccessEntityType::USER:
+        {
+            res.emplace_back(AccessType::DROP_USER);
+            res.emplace_back(AccessType::CREATE_USER);
+            return res;
+        }
+        case AccessEntityType::ROLE:
+        {
+            res.emplace_back(AccessType::DROP_ROLE);
+            res.emplace_back(AccessType::CREATE_ROLE);
+            return res;
+        }
+        case AccessEntityType::SETTINGS_PROFILE:
+        {
+            res.emplace_back(AccessType::DROP_SETTINGS_PROFILE);
+            res.emplace_back(AccessType::CREATE_SETTINGS_PROFILE);
+            return res;
+        }
+        case AccessEntityType::ROW_POLICY:
+        {
+            if (query.row_policy_names)
+            {
+                for (const auto & row_policy_name : query.row_policy_names->full_names)
+                {
+                    res.emplace_back(AccessType::DROP_ROW_POLICY, row_policy_name.database, row_policy_name.table_name);
+                    res.emplace_back(AccessType::CREATE_ROW_POLICY, row_policy_name.database, row_policy_name.table_name);
+                }
+            }
+            return res;
+        }
+        case AccessEntityType::QUOTA:
+        {
+            res.emplace_back(AccessType::DROP_QUOTA);
+            res.emplace_back(AccessType::CREATE_QUOTA);
+            return res;
+        }
+        case AccessEntityType::MAX:
+            break;
+    }
+    throw Exception(ErrorCodes::NOT_IMPLEMENTED, "{}: type is not supported by DROP query", toString(query.type));
+}
+
+}
diff --git a/src/Interpreters/Access/InterpreterMoveAccessEntityQuery.h b/src/Interpreters/Access/InterpreterMoveAccessEntityQuery.h
new file mode 100644
index 00000000000..1a70bf7c065
--- /dev/null
+++ b/src/Interpreters/Access/InterpreterMoveAccessEntityQuery.h
@@ -0,0 +1,24 @@
+#pragma once
+
+#include <Interpreters/IInterpreter.h>
+#include <Parsers/IAST_fwd.h>
+
+
+namespace DB
+{
+class AccessRightsElements;
+
+class InterpreterMoveAccessEntityQuery : public IInterpreter, WithMutableContext
+{
+public:
+    InterpreterMoveAccessEntityQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) : WithMutableContext(context_), query_ptr(query_ptr_) {}
+
+    BlockIO execute() override;
+
+private:
+    AccessRightsElements getRequiredAccess() const;
+
+    ASTPtr query_ptr;
+};
+
+}
diff --git a/src/Interpreters/InterpreterFactory.cpp b/src/Interpreters/InterpreterFactory.cpp
index 9cd1f2a251c..27c94119750 100644
--- a/src/Interpreters/InterpreterFactory.cpp
+++ b/src/Interpreters/InterpreterFactory.cpp
@@ -39,6 +39,7 @@
 #include <Parsers/Access/ASTCreateUserQuery.h>
 #include <Parsers/Access/ASTDropAccessEntityQuery.h>
 #include <Parsers/Access/ASTGrantQuery.h>
+#include <Parsers/Access/ASTMoveAccessEntityQuery.h>
 #include <Parsers/Access/ASTSetRoleQuery.h>
 #include <Parsers/Access/ASTShowAccessEntitiesQuery.h>
 #include <Parsers/Access/ASTShowAccessQuery.h>
@@ -96,6 +97,7 @@
 #include <Interpreters/Access/InterpreterCreateUserQuery.h>
 #include <Interpreters/Access/InterpreterDropAccessEntityQuery.h>
 #include <Interpreters/Access/InterpreterGrantQuery.h>
+#include <Interpreters/Access/InterpreterMoveAccessEntityQuery.h>
 #include <Interpreters/Access/InterpreterSetRoleQuery.h>
 #include <Interpreters/Access/InterpreterShowAccessEntitiesQuery.h>
 #include <Interpreters/Access/InterpreterShowAccessQuery.h>
@@ -314,6 +316,10 @@ std::unique_ptr<IInterpreter> InterpreterFactory::get(ASTPtr & query, ContextMut
     {
         return std::make_unique<InterpreterDropAccessEntityQuery>(query, context);
     }
+    else if (query->as<ASTMoveAccessEntityQuery>())
+    {
+        return std::make_unique<InterpreterMoveAccessEntityQuery>(query, context);
+    }
     else if (query->as<ASTDropNamedCollectionQuery>())
     {
         return std::make_unique<InterpreterDropNamedCollectionQuery>(query, context);
diff --git a/src/Parsers/Access/ASTCreateQuotaQuery.cpp b/src/Parsers/Access/ASTCreateQuotaQuery.cpp
index 56abedf5235..091b62b0a9f 100644
--- a/src/Parsers/Access/ASTCreateQuotaQuery.cpp
+++ b/src/Parsers/Access/ASTCreateQuotaQuery.cpp
@@ -170,6 +170,12 @@ void ASTCreateQuotaQuery::formatImpl(const FormatSettings & settings, FormatStat
         settings.ostr << (settings.hilite ? hilite_keyword : "") << " OR REPLACE" << (settings.hilite ? hilite_none : "");
 
     formatNames(names, settings);
+
+    if (!storage_name.empty())
+        settings.ostr << (settings.hilite ? IAST::hilite_keyword : "")
+                    << " IN " << (settings.hilite ? IAST::hilite_none : "")
+                    << backQuoteIfNeed(storage_name);
+
     formatOnCluster(settings);
 
     if (!new_name.empty())
diff --git a/src/Parsers/Access/ASTCreateQuotaQuery.h b/src/Parsers/Access/ASTCreateQuotaQuery.h
index f5eb59800ec..aecbbb00f9a 100644
--- a/src/Parsers/Access/ASTCreateQuotaQuery.h
+++ b/src/Parsers/Access/ASTCreateQuotaQuery.h
@@ -38,6 +38,7 @@ public:
     Strings names;
     String new_name;
     std::optional<QuotaKeyType> key_type;
+    String storage_name;
 
     struct Limits
     {
diff --git a/src/Parsers/Access/ASTCreateRoleQuery.cpp b/src/Parsers/Access/ASTCreateRoleQuery.cpp
index 14946f2d3cd..eeeb34c97e4 100644
--- a/src/Parsers/Access/ASTCreateRoleQuery.cpp
+++ b/src/Parsers/Access/ASTCreateRoleQuery.cpp
@@ -74,7 +74,7 @@ void ASTCreateRoleQuery::formatImpl(const FormatSettings & format, FormatState &
 
     if (!storage_name.empty())
         format.ostr << (format.hilite ? IAST::hilite_keyword : "")
-                    << " AT " << (format.hilite ? IAST::hilite_none : "")
+                    << " IN " << (format.hilite ? IAST::hilite_none : "")
                     << backQuoteIfNeed(storage_name);
 
     formatOnCluster(format);
diff --git a/src/Parsers/Access/ASTCreateRowPolicyQuery.cpp b/src/Parsers/Access/ASTCreateRowPolicyQuery.cpp
index ca888be2cfe..a423581e300 100644
--- a/src/Parsers/Access/ASTCreateRowPolicyQuery.cpp
+++ b/src/Parsers/Access/ASTCreateRowPolicyQuery.cpp
@@ -168,6 +168,11 @@ void ASTCreateRowPolicyQuery::formatImpl(const FormatSettings & settings, Format
     settings.ostr << " ";
     names->format(settings);
 
+    if (!storage_name.empty())
+        settings.ostr << (settings.hilite ? IAST::hilite_keyword : "")
+                    << " IN " << (settings.hilite ? IAST::hilite_none : "")
+                    << backQuoteIfNeed(storage_name);
+    
     formatOnCluster(settings);
     assert(names->cluster.empty());
 
diff --git a/src/Parsers/Access/ASTCreateRowPolicyQuery.h b/src/Parsers/Access/ASTCreateRowPolicyQuery.h
index 32d98fab822..3f2418e7287 100644
--- a/src/Parsers/Access/ASTCreateRowPolicyQuery.h
+++ b/src/Parsers/Access/ASTCreateRowPolicyQuery.h
@@ -35,6 +35,7 @@ public:
     bool if_exists = false;
     bool if_not_exists = false;
     bool or_replace = false;
+    String storage_name;
 
     std::shared_ptr<ASTRowPolicyNames> names;
     String new_short_name;
diff --git a/src/Parsers/Access/ASTCreateSettingsProfileQuery.cpp b/src/Parsers/Access/ASTCreateSettingsProfileQuery.cpp
index 56ddef433ef..3379486d789 100644
--- a/src/Parsers/Access/ASTCreateSettingsProfileQuery.cpp
+++ b/src/Parsers/Access/ASTCreateSettingsProfileQuery.cpp
@@ -81,6 +81,12 @@ void ASTCreateSettingsProfileQuery::formatImpl(const FormatSettings & format, Fo
         format.ostr << (format.hilite ? hilite_keyword : "") << " OR REPLACE" << (format.hilite ? hilite_none : "");
 
     formatNames(names, format);
+
+    if (!storage_name.empty())
+        format.ostr << (format.hilite ? IAST::hilite_keyword : "")
+                    << " IN " << (format.hilite ? IAST::hilite_none : "")
+                    << backQuoteIfNeed(storage_name);
+
     formatOnCluster(format);
 
     if (!new_name.empty())
diff --git a/src/Parsers/Access/ASTCreateSettingsProfileQuery.h b/src/Parsers/Access/ASTCreateSettingsProfileQuery.h
index c1a64998f29..be01aae1e26 100644
--- a/src/Parsers/Access/ASTCreateSettingsProfileQuery.h
+++ b/src/Parsers/Access/ASTCreateSettingsProfileQuery.h
@@ -28,6 +28,7 @@ public:
     bool if_exists = false;
     bool if_not_exists = false;
     bool or_replace = false;
+    String storage_name;
 
     Strings names;
     String new_name;
diff --git a/src/Parsers/Access/ASTCreateUserQuery.cpp b/src/Parsers/Access/ASTCreateUserQuery.cpp
index 0611545adf0..dbb4c1c6d57 100644
--- a/src/Parsers/Access/ASTCreateUserQuery.cpp
+++ b/src/Parsers/Access/ASTCreateUserQuery.cpp
@@ -208,6 +208,11 @@ void ASTCreateUserQuery::formatImpl(const FormatSettings & format, FormatState &
     format.ostr << " ";
     names->format(format);
 
+    if (!storage_name.empty())
+        format.ostr << (format.hilite ? IAST::hilite_keyword : "")
+                    << " IN " << (format.hilite ? IAST::hilite_none : "")
+                    << backQuoteIfNeed(storage_name);
+    
     formatOnCluster(format);
 
     if (new_name)
diff --git a/src/Parsers/Access/ASTCreateUserQuery.h b/src/Parsers/Access/ASTCreateUserQuery.h
index 62ddbfd0040..06940e67d91 100644
--- a/src/Parsers/Access/ASTCreateUserQuery.h
+++ b/src/Parsers/Access/ASTCreateUserQuery.h
@@ -45,6 +45,7 @@ public:
 
     std::shared_ptr<ASTUserNamesWithHost> names;
     std::optional<String> new_name;
+    String storage_name;
 
     std::shared_ptr<ASTAuthenticationData> auth_data;
 
diff --git a/src/Parsers/Access/ASTDropAccessEntityQuery.cpp b/src/Parsers/Access/ASTDropAccessEntityQuery.cpp
index 02b29f3fd2c..bcd7105d0e9 100644
--- a/src/Parsers/Access/ASTDropAccessEntityQuery.cpp
+++ b/src/Parsers/Access/ASTDropAccessEntityQuery.cpp
@@ -54,8 +54,8 @@ void ASTDropAccessEntityQuery::formatImpl(const FormatSettings & settings, Forma
         formatNames(names, settings);
 
     if (!storage_name.empty())
-        settings.ostr << (settings.hilite ? IAST::hilite_keyword : "")
-                      << " FROM " << (settings.hilite ? IAST::hilite_none : "")
+        settings.ostr << (settings.hilite ? hilite_keyword : "")
+                      << " FROM " << (settings.hilite ? hilite_none : "")
                       << backQuoteIfNeed(storage_name);
 
     formatOnCluster(settings);
diff --git a/src/Parsers/Access/ASTMoveAccessEntityQuery.cpp b/src/Parsers/Access/ASTMoveAccessEntityQuery.cpp
new file mode 100644
index 00000000000..285f07854c0
--- /dev/null
+++ b/src/Parsers/Access/ASTMoveAccessEntityQuery.cpp
@@ -0,0 +1,64 @@
+#include <Parsers/Access/ASTMoveAccessEntityQuery.h>
+#include <Parsers/Access/ASTRowPolicyName.h>
+#include <Common/quoteString.h>
+#include <IO/Operators.h>
+
+
+namespace DB
+{
+namespace
+{
+    void formatNames(const Strings & names, const IAST::FormatSettings & settings)
+    {
+        bool need_comma = false;
+        for (const auto & name : names)
+        {
+            if (std::exchange(need_comma, true))
+                settings.ostr << ',';
+            settings.ostr << ' ' << backQuoteIfNeed(name);
+        }
+    }
+}
+
+String ASTMoveAccessEntityQuery::getID(char) const
+{
+    return String("MOVE ") + toString(type) + " query";
+}
+
+ASTPtr ASTMoveAccessEntityQuery::clone() const
+{
+    auto res = std::make_shared<ASTMoveAccessEntityQuery>(*this);
+
+    if (row_policy_names)
+        res->row_policy_names = std::static_pointer_cast<ASTRowPolicyNames>(row_policy_names->clone());
+
+    return res;
+}
+
+void ASTMoveAccessEntityQuery::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const
+{
+    settings.ostr << (settings.hilite ? hilite_keyword : "")
+                  << "MOVE " << AccessEntityTypeInfo::get(type).name
+                  << (settings.hilite ? hilite_none : "");
+
+    if (type == AccessEntityType::ROW_POLICY)
+    {
+        settings.ostr << " ";
+        row_policy_names->format(settings);
+    }
+    else
+        formatNames(names, settings);
+
+    settings.ostr << (settings.hilite ? hilite_keyword : "")
+                  << " TO " << (settings.hilite ? hilite_none : "")
+                  << backQuoteIfNeed(storage_name);
+
+    formatOnCluster(settings);
+}
+
+void ASTMoveAccessEntityQuery::replaceEmptyDatabase(const String & current_database) const
+{
+    if (row_policy_names)
+        row_policy_names->replaceEmptyDatabase(current_database);
+}
+}
diff --git a/src/Parsers/Access/ASTMoveAccessEntityQuery.h b/src/Parsers/Access/ASTMoveAccessEntityQuery.h
new file mode 100644
index 00000000000..aa2b3b0f98c
--- /dev/null
+++ b/src/Parsers/Access/ASTMoveAccessEntityQuery.h
@@ -0,0 +1,32 @@
+#pragma once
+
+#include <Parsers/IAST.h>
+#include <Parsers/ASTQueryWithOnCluster.h>
+#include <Access/Common/AccessEntityType.h>
+
+
+namespace DB
+{
+class ASTRowPolicyNames;
+
+/** MOVE {USER | ROLE | QUOTA | [ROW] POLICY | [SETTINGS] PROFILE} [IF EXISTS] name [,...] [ON [database.]table [,...]] TO storage_name
+  */
+class ASTMoveAccessEntityQuery : public IAST, public ASTQueryWithOnCluster
+{
+public:
+    AccessEntityType type;
+    Strings names;
+    std::shared_ptr<ASTRowPolicyNames> row_policy_names;
+
+    String storage_name;
+
+    String getID(char) const override;
+    ASTPtr clone() const override;
+    void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override;
+    ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams &) const override { return removeOnCluster<ASTMoveAccessEntityQuery>(clone()); }
+
+    void replaceEmptyDatabase(const String & current_database) const;
+
+    QueryKind getQueryKind() const override { return QueryKind::Move; }
+};
+}
diff --git a/src/Parsers/Access/ParserCreateQuotaQuery.cpp b/src/Parsers/Access/ParserCreateQuotaQuery.cpp
index 2330ba7771c..05c2820b012 100644
--- a/src/Parsers/Access/ParserCreateQuotaQuery.cpp
+++ b/src/Parsers/Access/ParserCreateQuotaQuery.cpp
@@ -5,6 +5,7 @@
 #include <Parsers/Access/ASTRolesOrUsersSet.h>
 #include <Parsers/Access/ParserCreateQuotaQuery.h>
 #include <Parsers/Access/ParserRolesOrUsersSet.h>
+#include <Parsers/Access/parseUserName.h>
 #include <Parsers/CommonParsers.h>
 #include <Parsers/ExpressionElementParsers.h>
 #include <Parsers/ExpressionListParsers.h>
@@ -288,6 +289,7 @@ bool ParserCreateQuotaQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
     std::optional<QuotaKeyType> key_type;
     std::vector<ASTCreateQuotaQuery::Limits> all_limits;
     String cluster;
+    String storage_name;
 
     while (true)
     {
@@ -310,6 +312,9 @@ bool ParserCreateQuotaQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
         if (cluster.empty() && parseOnCluster(pos, expected, cluster))
             continue;
 
+        if (storage_name.empty() && ParserKeyword{"IN"}.ignore(pos, expected) && parseStorageName(pos, expected, storage_name))
+            continue;
+
         break;
     }
 
@@ -332,6 +337,7 @@ bool ParserCreateQuotaQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
     query->key_type = key_type;
     query->all_limits = std::move(all_limits);
     query->roles = std::move(roles);
+    query->storage_name = std::move(storage_name);
 
     return true;
 }
diff --git a/src/Parsers/Access/ParserCreateRoleQuery.cpp b/src/Parsers/Access/ParserCreateRoleQuery.cpp
index de9e5baed18..b548c07ec70 100644
--- a/src/Parsers/Access/ParserCreateRoleQuery.cpp
+++ b/src/Parsers/Access/ParserCreateRoleQuery.cpp
@@ -93,9 +93,6 @@ bool ParserCreateRoleQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
     String cluster;
     String storage_name;
 
-    if (ParserKeyword{"AT"}.ignore(pos, expected))
-        parseStorageName(pos, expected, storage_name);
-
     while (true)
     {
         if (alter && new_name.empty() && (names.size() == 1) && parseRenameTo(pos, expected, new_name))
@@ -114,6 +111,9 @@ bool ParserCreateRoleQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
         if (cluster.empty() && parseOnCluster(pos, expected, cluster))
             continue;
 
+        if (storage_name.empty() && ParserKeyword{"IN"}.ignore(pos, expected) && parseStorageName(pos, expected, storage_name))
+            continue;
+
         break;
     }
 
diff --git a/src/Parsers/Access/ParserCreateRowPolicyQuery.cpp b/src/Parsers/Access/ParserCreateRowPolicyQuery.cpp
index 2c25fc14e7d..b2177ca4ade 100644
--- a/src/Parsers/Access/ParserCreateRowPolicyQuery.cpp
+++ b/src/Parsers/Access/ParserCreateRowPolicyQuery.cpp
@@ -4,6 +4,7 @@
 #include <Parsers/Access/ASTRowPolicyName.h>
 #include <Parsers/Access/ParserRolesOrUsersSet.h>
 #include <Parsers/Access/ParserRowPolicyName.h>
+#include <Parsers/Access/parseUserName.h>
 #include <Parsers/ExpressionListParsers.h>
 #include <Parsers/ExpressionElementParsers.h>
 #include <Parsers/parseDatabaseAndTableName.h>
@@ -245,6 +246,7 @@ bool ParserCreateRowPolicyQuery::parseImpl(Pos & pos, ASTPtr & node, Expected &
     String new_short_name;
     std::optional<bool> is_restrictive;
     std::vector<std::pair<RowPolicyFilterType, ASTPtr>> filters;
+    String storage_name;
 
     while (true)
     {
@@ -271,6 +273,9 @@ bool ParserCreateRowPolicyQuery::parseImpl(Pos & pos, ASTPtr & node, Expected &
         if (cluster.empty() && parseOnCluster(pos, expected, cluster))
             continue;
 
+        if (storage_name.empty() && ParserKeyword{"IN"}.ignore(pos, expected) && parseStorageName(pos, expected, storage_name))
+            continue;
+
         break;
     }
 
@@ -294,6 +299,7 @@ bool ParserCreateRowPolicyQuery::parseImpl(Pos & pos, ASTPtr & node, Expected &
     query->is_restrictive = is_restrictive;
     query->filters = std::move(filters);
     query->roles = std::move(roles);
+    query->storage_name = std::move(storage_name);
 
     return true;
 }
diff --git a/src/Parsers/Access/ParserCreateSettingsProfileQuery.cpp b/src/Parsers/Access/ParserCreateSettingsProfileQuery.cpp
index c58a3035dc6..634ff68427a 100644
--- a/src/Parsers/Access/ParserCreateSettingsProfileQuery.cpp
+++ b/src/Parsers/Access/ParserCreateSettingsProfileQuery.cpp
@@ -4,6 +4,7 @@
 #include <Parsers/Access/ASTSettingsProfileElement.h>
 #include <Parsers/Access/ParserSettingsProfileElement.h>
 #include <Parsers/Access/ParserRolesOrUsersSet.h>
+#include <Parsers/Access/parseUserName.h>
 #include <Parsers/ASTLiteral.h>
 #include <Parsers/CommonParsers.h>
 #include <Parsers/ExpressionElementParsers.h>
@@ -111,6 +112,7 @@ bool ParserCreateSettingsProfileQuery::parseImpl(Pos & pos, ASTPtr & node, Expec
     String new_name;
     std::shared_ptr<ASTSettingsProfileElements> settings;
     String cluster;
+    String storage_name;
 
     while (true)
     {
@@ -130,6 +132,9 @@ bool ParserCreateSettingsProfileQuery::parseImpl(Pos & pos, ASTPtr & node, Expec
         if (cluster.empty() && parseOnCluster(pos, expected, cluster))
             continue;
 
+        if (storage_name.empty() && ParserKeyword{"IN"}.ignore(pos, expected) && parseStorageName(pos, expected, storage_name))
+            continue;
+
         break;
     }
 
@@ -152,6 +157,7 @@ bool ParserCreateSettingsProfileQuery::parseImpl(Pos & pos, ASTPtr & node, Expec
     query->new_name = std::move(new_name);
     query->settings = std::move(settings);
     query->to_roles = std::move(to_roles);
+    query->storage_name = std::move(storage_name);
 
     return true;
 }
diff --git a/src/Parsers/Access/ParserCreateUserQuery.cpp b/src/Parsers/Access/ParserCreateUserQuery.cpp
index 0344fb99c04..d95d41296dc 100644
--- a/src/Parsers/Access/ParserCreateUserQuery.cpp
+++ b/src/Parsers/Access/ParserCreateUserQuery.cpp
@@ -414,6 +414,7 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
     std::shared_ptr<ASTRolesOrUsersSet> grantees;
     std::shared_ptr<ASTDatabaseOrNone> default_database;
     String cluster;
+    String storage_name;
 
     while (true)
     {
@@ -480,6 +481,9 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
             }
         }
 
+        if (storage_name.empty() && ParserKeyword{"IN"}.ignore(pos, expected) && parseStorageName(pos, expected, storage_name))
+            continue;
+
         break;
     }
 
@@ -514,6 +518,7 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
     query->settings = std::move(settings);
     query->grantees = std::move(grantees);
     query->default_database = std::move(default_database);
+    query->storage_name = std::move(storage_name);
 
     if (query->auth_data)
         query->children.push_back(query->auth_data);
diff --git a/src/Parsers/Access/ParserMoveAccessEntityQuery.cpp b/src/Parsers/Access/ParserMoveAccessEntityQuery.cpp
new file mode 100644
index 00000000000..6ac41b77b84
--- /dev/null
+++ b/src/Parsers/Access/ParserMoveAccessEntityQuery.cpp
@@ -0,0 +1,93 @@
+#include <Parsers/Access/ParserMoveAccessEntityQuery.h>
+#include <Parsers/Access/ASTMoveAccessEntityQuery.h>
+#include <Parsers/Access/ParserRowPolicyName.h>
+#include <Parsers/Access/ASTRowPolicyName.h>
+#include <Parsers/Access/parseUserName.h>
+#include <Parsers/CommonParsers.h>
+#include <Parsers/parseIdentifierOrStringLiteral.h>
+#include <base/range.h>
+
+
+namespace DB
+{
+namespace
+{
+    bool parseEntityType(IParserBase::Pos & pos, Expected & expected, AccessEntityType & type)
+    {
+        for (auto i : collections::range(AccessEntityType::MAX))
+        {
+            const auto & type_info = AccessEntityTypeInfo::get(i);
+            if (ParserKeyword{type_info.name}.ignore(pos, expected)
+                || (!type_info.alias.empty() && ParserKeyword{type_info.alias}.ignore(pos, expected)))
+            {
+                type = i;
+                return true;
+            }
+        }
+        return false;
+    }
+
+
+    bool parseOnCluster(IParserBase::Pos & pos, Expected & expected, String & cluster)
+    {
+        return IParserBase::wrapParseImpl(pos, [&]
+        {
+            return ParserKeyword{"ON"}.ignore(pos, expected) && ASTQueryWithOnCluster::parse(pos, cluster, expected);
+        });
+    }
+}
+
+
+bool ParserMoveAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
+{
+    if (!ParserKeyword{"MOVE"}.ignore(pos, expected))
+        return false;
+
+    AccessEntityType type;
+    if (!parseEntityType(pos, expected, type))
+        return false;
+
+    Strings names;
+    std::shared_ptr<ASTRowPolicyNames> row_policy_names;
+    String storage_name;
+    String cluster;
+
+    if ((type == AccessEntityType::USER) || (type == AccessEntityType::ROLE))
+    {
+        if (!parseUserNames(pos, expected, names))
+            return false;
+    }
+    else if (type == AccessEntityType::ROW_POLICY)
+    {
+        ParserRowPolicyNames parser;
+        ASTPtr ast;
+        parser.allowOnCluster();
+        if (!parser.parse(pos, ast, expected))
+            return false;
+        row_policy_names = typeid_cast<std::shared_ptr<ASTRowPolicyNames>>(ast);
+        cluster = std::exchange(row_policy_names->cluster, "");
+    }
+    else
+    {
+        if (!parseIdentifiersOrStringLiterals(pos, expected, names))
+            return false;
+    }
+
+    if (!ParserKeyword{"TO"}.ignore(pos, expected) || !parseStorageName(pos, expected, storage_name))
+        return false;
+
+    if (cluster.empty())
+        parseOnCluster(pos, expected, cluster);
+
+    auto query = std::make_shared<ASTMoveAccessEntityQuery>();
+    node = query;
+
+    query->type = type;
+    query->cluster = std::move(cluster);
+    query->names = std::move(names);
+    query->row_policy_names = std::move(row_policy_names);
+    query->storage_name = std::move(storage_name);
+
+    return true;
+}
+}
diff --git a/src/Parsers/Access/ParserMoveAccessEntityQuery.h b/src/Parsers/Access/ParserMoveAccessEntityQuery.h
new file mode 100644
index 00000000000..3accec883dd
--- /dev/null
+++ b/src/Parsers/Access/ParserMoveAccessEntityQuery.h
@@ -0,0 +1,17 @@
+#pragma once
+
+#include <Parsers/IParserBase.h>
+
+
+namespace DB
+{
+/** Parses queries like
+  * MOVE {USER | ROLE | QUOTA | [ROW] POLICY | [SETTINGS] PROFILE} [IF EXISTS] name [,...] [ON [database.]table [,...]] TO storage_name
+  */
+class ParserMoveAccessEntityQuery : public IParserBase
+{
+protected:
+    const char * getName() const override { return "MOVE access entity query"; }
+    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
+};
+}
diff --git a/src/Parsers/Access/parseUserName.h b/src/Parsers/Access/parseUserName.h
index 717911bf2f1..64fed2ce21c 100644
--- a/src/Parsers/Access/parseUserName.h
+++ b/src/Parsers/Access/parseUserName.h
@@ -2,6 +2,7 @@
 
 #include <Core/Types.h>
 #include <Parsers/IParser.h>
+#include <Parsers/parseIdentifierOrStringLiteral.h>
 
 
 namespace DB
@@ -36,7 +37,7 @@ inline bool parseRoleNames(IParser::Pos & pos, Expected & expected, Strings & ro
 
 inline bool parseStorageName(IParser::Pos & pos, Expected & expected, String & storage_name)
 {
-    return parseUserName(pos, expected, storage_name);
+    return parseIdentifierOrStringLiteral(pos, expected, storage_name);
 }
 
 }
diff --git a/src/Parsers/IAST.h b/src/Parsers/IAST.h
index aa5302a15b9..92fb8ed275b 100644
--- a/src/Parsers/IAST.h
+++ b/src/Parsers/IAST.h
@@ -290,6 +290,7 @@ public:
         Alter,
         Grant,
         Revoke,
+        Move,
         System,
         Set,
         Use,
diff --git a/src/Parsers/ParserQuery.cpp b/src/Parsers/ParserQuery.cpp
index 77e7b58e6b1..7ed69940bed 100644
--- a/src/Parsers/ParserQuery.cpp
+++ b/src/Parsers/ParserQuery.cpp
@@ -27,6 +27,7 @@
 #include <Parsers/Access/ParserCreateUserQuery.h>
 #include <Parsers/Access/ParserDropAccessEntityQuery.h>
 #include <Parsers/Access/ParserGrantQuery.h>
+#include <Parsers/Access/ParserMoveAccessEntityQuery.h>
 #include <Parsers/Access/ParserSetRoleQuery.h>
 
 
@@ -54,6 +55,7 @@ bool ParserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
     ParserCreateIndexQuery create_index_p;
     ParserDropIndexQuery drop_index_p;
     ParserDropAccessEntityQuery drop_access_entity_p;
+    ParserMoveAccessEntityQuery move_access_entity_p;
     ParserGrantQuery grant_p;
     ParserSetRoleQuery set_role_p;
     ParserExternalDDLQuery external_ddl_p;
@@ -80,6 +82,7 @@ bool ParserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
         || create_index_p.parse(pos, node, expected)
         || drop_index_p.parse(pos, node, expected)
         || drop_access_entity_p.parse(pos, node, expected)
+        || move_access_entity_p.parse(pos, node, expected)
         || grant_p.parse(pos, node, expected)
         || external_ddl_p.parse(pos, node, expected)
         || transaction_control_p.parse(pos, node, expected)
diff --git a/tests/integration/test_grant_and_revoke/test.py b/tests/integration/test_grant_and_revoke/test.py
index ee5d4b5df93..8e040a18a88 100644
--- a/tests/integration/test_grant_and_revoke/test.py
+++ b/tests/integration/test_grant_and_revoke/test.py
@@ -471,7 +471,7 @@ def test_introspection():
         [
             [
                 "A",
-                "local directory",
+                "local_directory",
                 "no_password",
                 "{}",
                 "['::/0']",
@@ -484,7 +484,7 @@ def test_introspection():
             ],
             [
                 "B",
-                "local directory",
+                "local_directory",
                 "no_password",
                 "{}",
                 "['::/0']",
diff --git a/tests/integration/test_multi_access_storage_role_management/configs/memory.xml b/tests/integration/test_multi_access_storage_role_management/configs/memory.xml
new file mode 100644
index 00000000000..c2f60c8bbc1
--- /dev/null
+++ b/tests/integration/test_multi_access_storage_role_management/configs/memory.xml
@@ -0,0 +1,5 @@
+<clickhouse>
+    <user_directories>
+        <memory/>
+    </user_directories>
+</clickhouse>
diff --git a/tests/integration/test_multi_access_storage_role_management/configs/roles.xml b/tests/integration/test_multi_access_storage_role_management/configs/roles.xml
index 672c7e24728..faaf1ee27ae 100644
--- a/tests/integration/test_multi_access_storage_role_management/configs/roles.xml
+++ b/tests/integration/test_multi_access_storage_role_management/configs/roles.xml
@@ -1,8 +1,4 @@
 <clickhouse>
-    <user_directories>
-        <memory/>
-    </user_directories>
-
     <roles>
         <default_role>
             <grants>
diff --git a/tests/integration/test_multi_access_storage_role_management/test.py b/tests/integration/test_multi_access_storage_role_management/test.py
index d9a4a73c56b..fb4bbd6b718 100644
--- a/tests/integration/test_multi_access_storage_role_management/test.py
+++ b/tests/integration/test_multi_access_storage_role_management/test.py
@@ -8,6 +8,7 @@ cluster = ClickHouseCluster(__file__)
 node = cluster.add_instance(
     "node",
     stay_alive=True,
+    main_configs=["configs/memory.xml"]
 )
 
 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
@@ -28,6 +29,68 @@ def started_cluster():
         cluster.shutdown()
 
 
+def execute_test_for_access_type(access_type: str, system_table_name: str):
+    node.query(f"CREATE {access_type} test1 IN local_directory")
+    node.query(f"CREATE {access_type} test2 IN local_directory")
+    node.query(f"CREATE {access_type} test3 IN local_directory")
+
+    node.query(f"CREATE {access_type} test4 IN memory")
+    node.query(f"CREATE {access_type} test5 IN memory")
+    node.query(f"CREATE {access_type} test6 IN memory")
+
+    # Already exists
+    with pytest.raises(QueryRuntimeException):
+        node.query(f"CREATE {access_type} test1 IN memory")
+
+    node.query(f"MOVE {access_type} test1 TO memory")
+    assert node.query(f"SELECT storage FROM system.{system_table_name} WHERE name = 'test1'") == TSV(["memory"])
+
+    node.query(f"MOVE {access_type} test2 TO local_directory")
+    assert node.query(f"SELECT storage FROM system.{system_table_name} WHERE name = 'test2'") == TSV(["local_directory"])
+
+    node.query(f"MOVE {access_type} test2,test3 TO memory")
+    assert node.query(f"SELECT storage FROM system.{system_table_name} WHERE name = 'test2'") == TSV(["memory"])
+    assert node.query(f"SELECT storage FROM system.{system_table_name} WHERE name = 'test3'") == TSV(["memory"])
+
+    node.query(f"MOVE {access_type} test4,test5 TO local_directory")
+
+    # Different storages
+    with pytest.raises(QueryRuntimeException):
+        node.query(f"MOVE {access_type} test4,test1 TO memory")
+
+    # Doesn't exist
+    with pytest.raises(QueryRuntimeException):
+        node.query(f"MOVE {access_type} test7 TO local_directory")
+
+    # Storage doesn't exist
+    with pytest.raises(QueryRuntimeException):
+        node.query(f"MOVE {access_type} test6 TO non_existing_storage")
+
+    # Unwriteable storage
+    with pytest.raises(QueryRuntimeException):
+        node.query(f"MOVE {access_type} test6 TO users_xml")
+
+
+def test_roles():
+    execute_test_for_access_type("ROLE", "roles")
+
+
+def test_users():
+    execute_test_for_access_type("USER", "users")
+
+
+def test_settings_profiles():
+    execute_test_for_access_type("SETTINGS PROFILE", "settings_profiles")
+
+
+def test_quotas():
+    execute_test_for_access_type("QUOTA", "quotas")
+
+
+def test_row_policies():
+    execute_test_for_access_type("ROW POLICY", "row_policies")
+
+
 def test_role_from_different_storages():
     node.query("CREATE ROLE default_role")
     node.query("GRANT SELECT ON system.* TO default_role")
@@ -72,9 +135,9 @@ def test_role_from_different_storages():
 
     # Already exists
     with pytest.raises(QueryRuntimeException):
-        node.query("CREATE ROLE default_role AT memory")
+        node.query("CREATE ROLE default_role IN memory")
 
-    node.query("CREATE ROLE other_role AT memory")
+    node.query("CREATE ROLE other_role IN memory")
 
     assert node.query(
         "SELECT storage FROM system.roles WHERE name = 'other_role'"
diff --git a/tests/integration/test_role/test.py b/tests/integration/test_role/test.py
index 9a352f43eab..43a402d9fda 100644
--- a/tests/integration/test_role/test.py
+++ b/tests/integration/test_role/test.py
@@ -287,7 +287,7 @@ def test_introspection():
 
     assert instance.query(
         "SELECT name, storage from system.roles WHERE name IN ('R1', 'R2') ORDER BY name"
-    ) == TSV([["R1", "local directory"], ["R2", "local directory"]])
+    ) == TSV([["R1", "local_directory"], ["R2", "local_directory"]])
 
     assert instance.query(
         "SELECT * from system.grants WHERE user_name IN ('A', 'B') OR role_name IN ('R1', 'R2') ORDER BY user_name, role_name, access_type, database, table, column, is_partial_revoke, grant_option"
diff --git a/tests/integration/test_settings_profile/test.py b/tests/integration/test_settings_profile/test.py
index 13acc3f1042..8770cc82064 100644
--- a/tests/integration/test_settings_profile/test.py
+++ b/tests/integration/test_settings_profile/test.py
@@ -88,7 +88,7 @@ def test_smoke():
         )
     )
     assert system_settings_profile("xyz") == [
-        ["xyz", "local directory", 1, 0, "['robin']", "[]"]
+        ["xyz", "local_directory", 1, 0, "['robin']", "[]"]
     ]
     assert system_settings_profile_elements(profile_name="xyz") == [
         [
@@ -120,7 +120,7 @@ def test_smoke():
     instance.query("SET max_memory_usage = 80000000", user="robin")
     instance.query("SET max_memory_usage = 120000000", user="robin")
     assert system_settings_profile("xyz") == [
-        ["xyz", "local directory", 1, 0, "[]", "[]"]
+        ["xyz", "local_directory", 1, 0, "[]", "[]"]
     ]
     assert system_settings_profile_elements(user_name="robin") == []
 
@@ -201,7 +201,7 @@ def test_settings_from_granted_role():
         )
     )
     assert system_settings_profile("xyz") == [
-        ["xyz", "local directory", 2, 0, "[]", "[]"]
+        ["xyz", "local_directory", 2, 0, "[]", "[]"]
     ]
     assert system_settings_profile_elements(profile_name="xyz") == [
         [
@@ -276,7 +276,7 @@ def test_settings_from_granted_role():
         )
     )
     assert system_settings_profile("xyz") == [
-        ["xyz", "local directory", 2, 0, "['worker']", "[]"]
+        ["xyz", "local_directory", 2, 0, "['worker']", "[]"]
     ]
 
     instance.query("ALTER SETTINGS PROFILE xyz TO NONE")
@@ -293,7 +293,7 @@ def test_settings_from_granted_role():
     )
     instance.query("SET max_memory_usage = 120000000", user="robin")
     assert system_settings_profile("xyz") == [
-        ["xyz", "local directory", 2, 0, "[]", "[]"]
+        ["xyz", "local_directory", 2, 0, "[]", "[]"]
     ]
 
 
@@ -323,7 +323,7 @@ def test_inheritance():
     )
 
     assert system_settings_profile("xyz") == [
-        ["xyz", "local directory", 1, 0, "[]", "[]"]
+        ["xyz", "local_directory", 1, 0, "[]", "[]"]
     ]
     assert system_settings_profile_elements(profile_name="xyz") == [
         [
@@ -340,7 +340,7 @@ def test_inheritance():
         ]
     ]
     assert system_settings_profile("alpha") == [
-        ["alpha", "local directory", 1, 0, "['robin']", "[]"]
+        ["alpha", "local_directory", 1, 0, "['robin']", "[]"]
     ]
     assert system_settings_profile_elements(profile_name="alpha") == [
         ["alpha", "\\N", "\\N", 0, "\\N", "\\N", "\\N", "\\N", "\\N", "xyz"]
diff --git a/tests/integration/test_user_directories/test.py b/tests/integration/test_user_directories/test.py
index 45afb86f464..704fb30b2fd 100644
--- a/tests/integration/test_user_directories/test.py
+++ b/tests/integration/test_user_directories/test.py
@@ -38,14 +38,14 @@ def test_old_style():
     assert node.query("SELECT * FROM system.user_directories") == TSV(
         [
             [
-                "users.xml",
-                "users.xml",
+                "users_xml",
+                "users_xml",
                 '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users2.xml"}',
                 1,
             ],
             [
-                "local directory",
-                "local directory",
+                "local_directory",
+                "local_directory",
                 '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access2\\\\/"}',
                 2,
             ],
@@ -62,20 +62,20 @@ def test_local_directories():
     assert node.query("SELECT * FROM system.user_directories") == TSV(
         [
             [
-                "users.xml",
-                "users.xml",
+                "users_xml",
+                "users_xml",
                 '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users3.xml"}',
                 1,
             ],
             [
-                "local directory",
-                "local directory",
+                "local_directory",
+                "local_directory",
                 '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access3\\\\/"}',
                 2,
             ],
             [
                 "local directory (ro)",
-                "local directory",
+                "local_directory",
                 '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access3-ro\\\\/","readonly":true}',
                 3,
             ],
@@ -92,8 +92,8 @@ def test_relative_path():
     assert node.query("SELECT * FROM system.user_directories") == TSV(
         [
             [
-                "users.xml",
-                "users.xml",
+                "users_xml",
+                "users_xml",
                 '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users4.xml"}',
                 1,
             ]
@@ -110,8 +110,8 @@ def test_memory():
     assert node.query("SELECT * FROM system.user_directories") == TSV(
         [
             [
-                "users.xml",
-                "users.xml",
+                "users_xml",
+                "users_xml",
                 '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users5.xml"}',
                 1,
             ],
@@ -129,20 +129,20 @@ def test_mixed_style():
     assert node.query("SELECT * FROM system.user_directories") == TSV(
         [
             [
-                "users.xml",
-                "users.xml",
+                "users_xml",
+                "users_xml",
                 '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users6.xml"}',
                 1,
             ],
             [
-                "local directory",
-                "local directory",
+                "local_directory",
+                "local_directory",
                 '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access6\\\\/"}',
                 2,
             ],
             [
-                "local directory",
-                "local directory",
+                "local_directory",
+                "local_directory",
                 '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access6a\\\\/"}',
                 3,
             ],
@@ -160,14 +160,14 @@ def test_duplicates():
     assert node.query("SELECT * FROM system.user_directories") == TSV(
         [
             [
-                "users.xml",
-                "users.xml",
+                "users_xml",
+                "users_xml",
                 '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users7.xml"}',
                 1,
             ],
             [
-                "local directory",
-                "local directory",
+                "local_directory",
+                "local_directory",
                 '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access7\\\\/"}',
                 2,
             ],

From a8c08b34a6ca179d2b89da87d920916978bf4129 Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Fri, 21 Jul 2023 02:39:21 +0000
Subject: [PATCH 023/249] Automatic style fix

---
 .../test.py                                   | 20 ++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/tests/integration/test_multi_access_storage_role_management/test.py b/tests/integration/test_multi_access_storage_role_management/test.py
index fb4bbd6b718..4c2696158ec 100644
--- a/tests/integration/test_multi_access_storage_role_management/test.py
+++ b/tests/integration/test_multi_access_storage_role_management/test.py
@@ -6,9 +6,7 @@ from helpers.client import QueryRuntimeException
 
 cluster = ClickHouseCluster(__file__)
 node = cluster.add_instance(
-    "node",
-    stay_alive=True,
-    main_configs=["configs/memory.xml"]
+    "node", stay_alive=True, main_configs=["configs/memory.xml"]
 )
 
 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
@@ -43,14 +41,22 @@ def execute_test_for_access_type(access_type: str, system_table_name: str):
         node.query(f"CREATE {access_type} test1 IN memory")
 
     node.query(f"MOVE {access_type} test1 TO memory")
-    assert node.query(f"SELECT storage FROM system.{system_table_name} WHERE name = 'test1'") == TSV(["memory"])
+    assert node.query(
+        f"SELECT storage FROM system.{system_table_name} WHERE name = 'test1'"
+    ) == TSV(["memory"])
 
     node.query(f"MOVE {access_type} test2 TO local_directory")
-    assert node.query(f"SELECT storage FROM system.{system_table_name} WHERE name = 'test2'") == TSV(["local_directory"])
+    assert node.query(
+        f"SELECT storage FROM system.{system_table_name} WHERE name = 'test2'"
+    ) == TSV(["local_directory"])
 
     node.query(f"MOVE {access_type} test2,test3 TO memory")
-    assert node.query(f"SELECT storage FROM system.{system_table_name} WHERE name = 'test2'") == TSV(["memory"])
-    assert node.query(f"SELECT storage FROM system.{system_table_name} WHERE name = 'test3'") == TSV(["memory"])
+    assert node.query(
+        f"SELECT storage FROM system.{system_table_name} WHERE name = 'test2'"
+    ) == TSV(["memory"])
+    assert node.query(
+        f"SELECT storage FROM system.{system_table_name} WHERE name = 'test3'"
+    ) == TSV(["memory"])
 
     node.query(f"MOVE {access_type} test4,test5 TO local_directory")
 

From 3398355e4649beaa2b0a3180aa687c364c245e81 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 22 Jul 2023 02:04:30 +0200
Subject: [PATCH 024/249] Add something with unclear purpose

---
 docker/README.md        | 2 +-
 tests/ci/docker_test.py | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docker/README.md b/docker/README.md
index ec52ddd143e..76fdbf0f3f0 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -2,4 +2,4 @@
 
 This directory contain Dockerfiles for `clickhouse-client` and `clickhouse-server`. They are updated in each release.
 
-Also there is bunch of images for testing and CI. They are listed in `images.json` file and updated on each commit to master. If you need to add another image, place information about it into `images.json`.
+Also, there is a bunch of images for testing and CI. They are listed in `images.json` file and updated on each commit to master. If you need to add another image, place information about it into `images.json`.
diff --git a/tests/ci/docker_test.py b/tests/ci/docker_test.py
index d5d27f73694..61319041b9f 100644
--- a/tests/ci/docker_test.py
+++ b/tests/ci/docker_test.py
@@ -40,6 +40,7 @@ class TestDockerImageCheck(unittest.TestCase):
             [
                 di.DockerImage("docker/test/base", "clickhouse/test-base", False),
                 di.DockerImage("docker/docs/builder", "clickhouse/docs-builder", True),
+                di.DockerImage("docker/test/sqltest", "clickhouse/sqltest", True),
                 di.DockerImage(
                     "docker/test/stateless",
                     "clickhouse/stateless-test",

From c3c8ea9c27669a548e2404e2ed6c2e8f3c734dc9 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 22 Jul 2023 17:30:55 +0200
Subject: [PATCH 025/249] Do something

---
 tests/ci/docker_test.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tests/ci/docker_test.py b/tests/ci/docker_test.py
index 61319041b9f..8542de412d2 100644
--- a/tests/ci/docker_test.py
+++ b/tests/ci/docker_test.py
@@ -40,7 +40,12 @@ class TestDockerImageCheck(unittest.TestCase):
             [
                 di.DockerImage("docker/test/base", "clickhouse/test-base", False),
                 di.DockerImage("docker/docs/builder", "clickhouse/docs-builder", True),
-                di.DockerImage("docker/test/sqltest", "clickhouse/sqltest", True),
+                di.DockerImage(
+                    "docker/test/sqltest",
+                    "clickhouse/sqltest",
+                    True,
+                    "clickhouse/test-base",  # type: ignore
+                ),
                 di.DockerImage(
                     "docker/test/stateless",
                     "clickhouse/stateless-test",

From fbe9f8d0f653cedd30640c42968df1ac8cf78dff Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Mon, 24 Jul 2023 14:12:01 +0000
Subject: [PATCH 026/249] fix recalculation of skip indexes and projcetion in
 ALTER DELETE

---
 src/Interpreters/MutationsInterpreter.cpp     | 89 +++++++++++------
 src/Interpreters/MutationsInterpreter.h       |  3 +-
 src/Storages/MergeTree/IMergeTreeDataPart.cpp |  6 ++
 src/Storages/MergeTree/IMergeTreeDataPart.h   |  3 +
 src/Storages/MergeTree/MutateTask.cpp         | 98 +++++++++++++++++--
 src/Storages/StorageInMemoryMetadata.cpp      |  6 +-
 src/Storages/StorageInMemoryMetadata.h        |  4 +-
 ...alter_delete_indexes_projections.reference |  6 ++
 ...02832_alter_delete_indexes_projections.sql | 26 +++++
 9 files changed, 201 insertions(+), 40 deletions(-)
 create mode 100644 tests/queries/0_stateless/02832_alter_delete_indexes_projections.reference
 create mode 100644 tests/queries/0_stateless/02832_alter_delete_indexes_projections.sql

diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp
index 25c52ad8925..1a7cbb45999 100644
--- a/src/Interpreters/MutationsInterpreter.cpp
+++ b/src/Interpreters/MutationsInterpreter.cpp
@@ -113,13 +113,14 @@ QueryTreeNodePtr prepareQueryAffectedQueryTree(const std::vector<MutationCommand
 ColumnDependencies getAllColumnDependencies(
     const StorageMetadataPtr & metadata_snapshot,
     const NameSet & updated_columns,
-    const std::function<bool(const String & file_name)> & has_index_or_projection)
+    const StorageInMemoryMetadata::HasDependencyCallback & has_dependency)
 {
     NameSet new_updated_columns = updated_columns;
     ColumnDependencies dependencies;
+
     while (!new_updated_columns.empty())
     {
-        auto new_dependencies = metadata_snapshot->getColumnDependencies(new_updated_columns, true, has_index_or_projection);
+        auto new_dependencies = metadata_snapshot->getColumnDependencies(new_updated_columns, true, has_dependency);
         new_updated_columns.clear();
         for (const auto & dependency : new_dependencies)
         {
@@ -292,9 +293,14 @@ bool MutationsInterpreter::Source::materializeTTLRecalculateOnly() const
     return data && data->getSettings()->materialize_ttl_recalculate_only;
 }
 
-bool MutationsInterpreter::Source::hasIndexOrProjection(const String & file_name) const
+bool MutationsInterpreter::Source::hasSecondaryIndex(const String & name) const
 {
-    return part && part->checksums.has(file_name);
+    return part && part->hasSecondaryIndex(name);
+}
+
+bool MutationsInterpreter::Source::hasProjection(const String & name) const
+{
+    return part && part->hasProjection(name);
 }
 
 static Names getAvailableColumnsWithVirtuals(StorageMetadataPtr metadata_snapshot, const IStorage & storage)
@@ -533,13 +539,24 @@ void MutationsInterpreter::prepare(bool dry_run)
         validateUpdateColumns(source, metadata_snapshot, updated_columns, column_to_affected_materialized);
     }
 
-    std::function<bool(const String & file_name)> has_index_or_projection
-        = [&](const String & file_name) { return source.hasIndexOrProjection(file_name); };
+    StorageInMemoryMetadata::HasDependencyCallback has_dependency =
+        [&](const String & name, ColumnDependency::Kind kind)
+    {
+        if (kind == ColumnDependency::PROJECTION)
+            return source.hasProjection(name);
+
+        if (kind == ColumnDependency::SKIP_INDEX)
+            return source.hasSecondaryIndex(name);
+
+        return true;
+    };
 
     if (settings.recalculate_dependencies_of_updated_columns)
-        dependencies = getAllColumnDependencies(metadata_snapshot, updated_columns, has_index_or_projection);
+        dependencies = getAllColumnDependencies(metadata_snapshot, updated_columns, has_dependency);
 
+    bool has_alter_delete = false;
     std::vector<String> read_columns;
+
     /// First, break a sequence of commands into stages.
     for (auto & command : commands)
     {
@@ -558,6 +575,7 @@ void MutationsInterpreter::prepare(bool dry_run)
                 predicate = makeASTFunction("isZeroOrNull", predicate);
 
             stages.back().filters.push_back(predicate);
+            has_alter_delete = true;
         }
         else if (command.type == MutationCommand::UPDATE)
         {
@@ -692,8 +710,7 @@ void MutationsInterpreter::prepare(bool dry_run)
             if (it == std::cend(indices_desc))
                 throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown index: {}", command.index_name);
 
-            if (!source.hasIndexOrProjection("skp_idx_" + it->name + ".idx")
-                && !source.hasIndexOrProjection("skp_idx_" + it->name + ".idx2"))
+            if (!source.hasSecondaryIndex(it->name))
             {
                 auto query = (*it).expression_list_ast->clone();
                 auto syntax_result = TreeRewriter(context).analyze(query, all_columns);
@@ -707,7 +724,7 @@ void MutationsInterpreter::prepare(bool dry_run)
         {
             mutation_kind.set(MutationKind::MUTATE_INDEX_PROJECTION);
             const auto & projection = projections_desc.get(command.projection_name);
-            if (!source.hasIndexOrProjection(projection.getDirectoryName()))
+            if (!source.hasProjection(projection.name))
             {
                 for (const auto & column : projection.required_columns)
                     dependencies.emplace(column, ColumnDependency::PROJECTION);
@@ -732,7 +749,7 @@ void MutationsInterpreter::prepare(bool dry_run)
                 // just recalculate ttl_infos without remove expired data
                 auto all_columns_vec = all_columns.getNames();
                 auto new_dependencies = metadata_snapshot->getColumnDependencies(
-                    NameSet(all_columns_vec.begin(), all_columns_vec.end()), false, has_index_or_projection);
+                    NameSet(all_columns_vec.begin(), all_columns_vec.end()), false, has_dependency);
                 for (const auto & dependency : new_dependencies)
                 {
                     if (dependency.kind == ColumnDependency::TTL_EXPRESSION)
@@ -758,7 +775,7 @@ void MutationsInterpreter::prepare(bool dry_run)
 
                 auto all_columns_vec = all_columns.getNames();
                 auto all_dependencies = getAllColumnDependencies(
-                    metadata_snapshot, NameSet(all_columns_vec.begin(), all_columns_vec.end()), has_index_or_projection);
+                    metadata_snapshot, NameSet(all_columns_vec.begin(), all_columns_vec.end()), has_dependency);
 
                 for (const auto & dependency : all_dependencies)
                 {
@@ -767,7 +784,7 @@ void MutationsInterpreter::prepare(bool dry_run)
                 }
 
                 /// Recalc only skip indices and projections of columns which could be updated by TTL.
-                auto new_dependencies = metadata_snapshot->getColumnDependencies(new_updated_columns, true, has_index_or_projection);
+                auto new_dependencies = metadata_snapshot->getColumnDependencies(new_updated_columns, true, has_dependency);
                 for (const auto & dependency : new_dependencies)
                 {
                     if (dependency.kind == ColumnDependency::SKIP_INDEX || dependency.kind == ColumnDependency::PROJECTION)
@@ -861,30 +878,44 @@ void MutationsInterpreter::prepare(bool dry_run)
 
     for (const auto & index : metadata_snapshot->getSecondaryIndices())
     {
-        if (source.hasIndexOrProjection("skp_idx_" + index.name + ".idx") || source.hasIndexOrProjection("skp_idx_" + index.name + ".idx2"))
+        if (!source.hasSecondaryIndex(index.name))
+            continue;
+
+        if (has_alter_delete)
         {
-            const auto & index_cols = index.expression->getRequiredColumns();
-            bool changed = std::any_of(
-                index_cols.begin(),
-                index_cols.end(),
-                [&](const auto & col) { return updated_columns.contains(col) || changed_columns.contains(col); });
-            if (changed)
-                materialized_indices.insert(index.name);
+            materialized_indices.insert(index.name);
+            continue;
         }
+
+        const auto & index_cols = index.expression->getRequiredColumns();
+        bool changed = std::any_of(
+            index_cols.begin(),
+            index_cols.end(),
+            [&](const auto & col) { return updated_columns.contains(col) || changed_columns.contains(col); });
+
+        if (changed)
+            materialized_indices.insert(index.name);
     }
 
     for (const auto & projection : metadata_snapshot->getProjections())
     {
-        if (source.hasIndexOrProjection(projection.getDirectoryName()))
+        if (!source.hasProjection(projection.name))
+            continue;
+
+        if (has_alter_delete)
         {
-            const auto & projection_cols = projection.required_columns;
-            bool changed = std::any_of(
-                projection_cols.begin(),
-                projection_cols.end(),
-                [&](const auto & col) { return updated_columns.contains(col) || changed_columns.contains(col); });
-            if (changed)
-                materialized_projections.insert(projection.name);
+            materialized_projections.insert(projection.name);
+            continue;
         }
+
+        const auto & projection_cols = projection.required_columns;
+        bool changed = std::any_of(
+            projection_cols.begin(),
+            projection_cols.end(),
+            [&](const auto & col) { return updated_columns.contains(col) || changed_columns.contains(col); });
+
+        if (changed)
+            materialized_projections.insert(projection.name);
     }
 
     /// Stages might be empty when we materialize skip indices or projections which don't add any
diff --git a/src/Interpreters/MutationsInterpreter.h b/src/Interpreters/MutationsInterpreter.h
index d783b503531..9b4caaae231 100644
--- a/src/Interpreters/MutationsInterpreter.h
+++ b/src/Interpreters/MutationsInterpreter.h
@@ -120,7 +120,8 @@ public:
         bool supportsLightweightDelete() const;
         bool hasLightweightDeleteMask() const;
         bool materializeTTLRecalculateOnly() const;
-        bool hasIndexOrProjection(const String & file_name) const;
+        bool hasSecondaryIndex(const String & name) const;
+        bool hasProjection(const String & name) const;
 
         void read(
             Stage & first_stage,
diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp
index 7050a98a4bc..1ab8dc7fb05 100644
--- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp
+++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp
@@ -1983,6 +1983,12 @@ IndexSize IMergeTreeDataPart::getSecondaryIndexSize(const String & secondary_ind
     return ColumnSize{};
 }
 
+bool IMergeTreeDataPart::hasSecondaryIndex(const String & index_name) const
+{
+    auto file_name = INDEX_FILE_PREFIX + index_name;
+    return checksums.has(file_name + ".idx") || checksums.has(file_name + ".idx2");
+}
+
 void IMergeTreeDataPart::accumulateColumnSizes(ColumnToSize & column_to_size) const
 {
     for (const auto & [column_name, size] : columns_sizes)
diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h
index af6906e004d..bfb472ca50d 100644
--- a/src/Storages/MergeTree/IMergeTreeDataPart.h
+++ b/src/Storages/MergeTree/IMergeTreeDataPart.h
@@ -122,6 +122,9 @@ public:
     /// Otherwise return information about secondary index size on disk.
     IndexSize getSecondaryIndexSize(const String & secondary_index_name) const;
 
+    /// Returns true if there is materialized index with specified name in part.
+    bool hasSecondaryIndex(const String & index_name) const;
+
     /// Return information about column size on disk for all columns in part
     ColumnSize getTotalColumnsSize() const { return total_columns_size; }
 
diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp
index 491c36433ca..fa5ad9858e8 100644
--- a/src/Storages/MergeTree/MutateTask.cpp
+++ b/src/Storages/MergeTree/MutateTask.cpp
@@ -69,6 +69,7 @@ static void splitAndModifyMutationCommands(
     {
         NameSet mutated_columns;
         NameSet dropped_columns;
+        NameSet extra_columns_for_indices_and_projections;
 
         for (const auto & command : commands)
         {
@@ -85,6 +86,41 @@ static void splitAndModifyMutationCommands(
 
                 if (command.type == MutationCommand::Type::MATERIALIZE_COLUMN)
                     mutated_columns.emplace(command.column_name);
+
+                if (command.type == MutationCommand::Type::MATERIALIZE_INDEX)
+                {
+                    const auto & all_indices = metadata_snapshot->getSecondaryIndices();
+                    for (const auto & index : all_indices)
+                    {
+                        if (index.name == command.index_name)
+                        {
+                            auto required_columns = index.expression->getRequiredColumns();
+                            for (const auto & column : required_columns)
+                            {
+                                if (!part_columns.has(column))
+                                    extra_columns_for_indices_and_projections.insert(column);
+                            }
+                            break;
+                        }
+                    }
+                }
+
+                if (command.type == MutationCommand::Type::MATERIALIZE_PROJECTION)
+                {
+                    const auto & all_projections = metadata_snapshot->getProjections();
+                    for (const auto & projection : all_projections)
+                    {
+                        if (projection.name == command.projection_name)
+                        {
+                            for (const auto & column : projection.required_columns)
+                            {
+                                if (!part_columns.has(column))
+                                    extra_columns_for_indices_and_projections.insert(column);
+                            }
+                            break;
+                        }
+                    }
+                }
             }
             else if (command.type == MutationCommand::Type::DROP_INDEX || command.type == MutationCommand::Type::DROP_PROJECTION)
             {
@@ -187,6 +223,25 @@ static void splitAndModifyMutationCommands(
                 });
             }
         }
+
+        for (const auto & column_name : extra_columns_for_indices_and_projections)
+        {
+            if (mutated_columns.contains(column_name))
+                continue;
+
+            auto data_type = metadata_snapshot->getColumns().getColumn(
+                GetColumnsOptions::AllPhysical,
+                column_name).type;
+
+            for_interpreter.push_back(
+                MutationCommand
+                {
+                    .type = MutationCommand::Type::READ_COLUMN,
+                    .column_name = column_name,
+                    .data_type = std::move(data_type),
+                }
+            );
+        }
     }
     else
     {
@@ -453,6 +508,7 @@ static ExecuteTTLType shouldExecuteTTL(const StorageMetadataPtr & metadata_snaps
 /// Return set of indices which should be recalculated during mutation also
 /// wraps input stream into additional expression stream
 static std::set<MergeTreeIndexPtr> getIndicesToRecalculate(
+    const MergeTreeDataPartPtr & source_part,
     QueryPipelineBuilder & builder,
     const StorageMetadataPtr & metadata_snapshot,
     ContextPtr context,
@@ -463,10 +519,15 @@ static std::set<MergeTreeIndexPtr> getIndicesToRecalculate(
     std::set<MergeTreeIndexPtr> indices_to_recalc;
     ASTPtr indices_recalc_expr_list = std::make_shared<ASTExpressionList>();
     const auto & indices = metadata_snapshot->getSecondaryIndices();
+    bool is_full_part_storage = isFullPartStorage(source_part->getDataPartStorage());
 
     for (const auto & index : indices)
     {
-        if (materialized_indices.contains(index.name))
+        bool need_recalculate =
+            materialized_indices.contains(index.name)
+            || (!is_full_part_storage && source_part->hasSecondaryIndex(index.name));
+
+        if (need_recalculate)
         {
             if (indices_to_recalc.insert(index_factory.get(index)).second)
             {
@@ -496,15 +557,23 @@ static std::set<MergeTreeIndexPtr> getIndicesToRecalculate(
 }
 
 static std::set<ProjectionDescriptionRawPtr> getProjectionsToRecalculate(
+    const MergeTreeDataPartPtr & source_part,
     const StorageMetadataPtr & metadata_snapshot,
     const NameSet & materialized_projections)
 {
     std::set<ProjectionDescriptionRawPtr> projections_to_recalc;
+    bool is_full_part_storage = isFullPartStorage(source_part->getDataPartStorage());
+
     for (const auto & projection : metadata_snapshot->getProjections())
     {
-        if (materialized_projections.contains(projection.name))
+        bool need_recalculate =
+            materialized_projections.contains(projection.name)
+            || (!is_full_part_storage && source_part->hasProjection(projection.name));
+
+        if (need_recalculate)
             projections_to_recalc.insert(&projection);
     }
+
     return projections_to_recalc;
 }
 
@@ -1279,14 +1348,20 @@ private:
                 removed_indices.insert(command.column_name);
         }
 
+        bool is_full_part_storage = isFullPartStorage(ctx->new_data_part->getDataPartStorage());
         const auto & indices = ctx->metadata_snapshot->getSecondaryIndices();
+
         MergeTreeIndices skip_indices;
         for (const auto & idx : indices)
         {
             if (removed_indices.contains(idx.name))
                 continue;
 
-            if (ctx->materialized_indices.contains(idx.name))
+            bool need_recalculate =
+                ctx->materialized_indices.contains(idx.name)
+                || (!is_full_part_storage && ctx->source_part->hasSecondaryIndex(idx.name));
+
+            if (need_recalculate)
             {
                 skip_indices.push_back(MergeTreeIndexFactory::instance().get(idx));
             }
@@ -1319,7 +1394,11 @@ private:
             if (removed_projections.contains(projection.name))
                 continue;
 
-            if (ctx->materialized_projections.contains(projection.name))
+            bool need_recalculate =
+                ctx->materialized_projections.contains(projection.name)
+                || (!is_full_part_storage && ctx->source_part->hasProjection(projection.name));
+
+            if (need_recalculate)
             {
                 ctx->projections_to_build.push_back(&projection);
             }
@@ -1920,9 +1999,16 @@ bool MutateTask::prepare()
     else /// TODO: check that we modify only non-key columns in this case.
     {
         ctx->indices_to_recalc = MutationHelpers::getIndicesToRecalculate(
-            ctx->mutating_pipeline_builder, ctx->metadata_snapshot, ctx->context, ctx->materialized_indices);
+            ctx->source_part,
+            ctx->mutating_pipeline_builder,
+            ctx->metadata_snapshot,
+            ctx->context,
+            ctx->materialized_indices);
 
-        ctx->projections_to_recalc = MutationHelpers::getProjectionsToRecalculate(ctx->metadata_snapshot, ctx->materialized_projections);
+        ctx->projections_to_recalc = MutationHelpers::getProjectionsToRecalculate(
+            ctx->source_part,
+            ctx->metadata_snapshot,
+            ctx->materialized_projections);
 
         ctx->files_to_skip = MutationHelpers::collectFilesToSkip(
             ctx->source_part,
diff --git a/src/Storages/StorageInMemoryMetadata.cpp b/src/Storages/StorageInMemoryMetadata.cpp
index afe75349864..b1da509c9b0 100644
--- a/src/Storages/StorageInMemoryMetadata.cpp
+++ b/src/Storages/StorageInMemoryMetadata.cpp
@@ -239,7 +239,7 @@ bool StorageInMemoryMetadata::hasAnyGroupByTTL() const
 ColumnDependencies StorageInMemoryMetadata::getColumnDependencies(
     const NameSet & updated_columns,
     bool include_ttl_target,
-    const std::function<bool(const String & file_name)> & has_indice_or_projection) const
+    const HasDependencyCallback & has_dependency) const
 {
     if (updated_columns.empty())
         return {};
@@ -268,13 +268,13 @@ ColumnDependencies StorageInMemoryMetadata::getColumnDependencies(
 
     for (const auto & index : getSecondaryIndices())
     {
-        if (has_indice_or_projection("skp_idx_" + index.name + ".idx") || has_indice_or_projection("skp_idx_" + index.name + ".idx2"))
+        if (has_dependency(index.name, ColumnDependency::SKIP_INDEX))
             add_dependent_columns(index.expression, indices_columns);
     }
 
     for (const auto & projection : getProjections())
     {
-        if (has_indice_or_projection(projection.getDirectoryName()))
+        if (has_dependency(projection.getDirectoryName(), ColumnDependency::PROJECTION))
             add_dependent_columns(&projection, projections_columns);
     }
 
diff --git a/src/Storages/StorageInMemoryMetadata.h b/src/Storages/StorageInMemoryMetadata.h
index 4ed7eb8bf29..30b2b303492 100644
--- a/src/Storages/StorageInMemoryMetadata.h
+++ b/src/Storages/StorageInMemoryMetadata.h
@@ -147,12 +147,14 @@ struct StorageInMemoryMetadata
     TTLDescriptions getGroupByTTLs() const;
     bool hasAnyGroupByTTL() const;
 
+    using HasDependencyCallback = std::function<bool(const String &, ColumnDependency::Kind)>;
+
     /// Returns columns, which will be needed to calculate dependencies (skip indices, projections,
     /// TTL expressions) if we update @updated_columns set of columns.
     ColumnDependencies getColumnDependencies(
         const NameSet & updated_columns,
         bool include_ttl_target,
-        const std::function<bool(const String & file_name)> & has_indice_or_projection) const;
+        const HasDependencyCallback & has_dependency) const;
 
     /// Block with ordinary + materialized columns.
     Block getSampleBlock() const;
diff --git a/tests/queries/0_stateless/02832_alter_delete_indexes_projections.reference b/tests/queries/0_stateless/02832_alter_delete_indexes_projections.reference
new file mode 100644
index 00000000000..f14acdf9e6d
--- /dev/null
+++ b/tests/queries/0_stateless/02832_alter_delete_indexes_projections.reference
@@ -0,0 +1,6 @@
+2
+0
+3355402240
+3355402240
+3321851904
+3321851904
diff --git a/tests/queries/0_stateless/02832_alter_delete_indexes_projections.sql b/tests/queries/0_stateless/02832_alter_delete_indexes_projections.sql
new file mode 100644
index 00000000000..b87230e57d1
--- /dev/null
+++ b/tests/queries/0_stateless/02832_alter_delete_indexes_projections.sql
@@ -0,0 +1,26 @@
+set mutations_sync = 2;
+
+drop table if exists t_delete_skip_index;
+
+create table t_delete_skip_index (x UInt32, y String, index i y type minmax granularity 3) engine = MergeTree order by tuple();
+insert into t_delete_skip_index select number, toString(number) from numbers(8192 * 10);
+
+select count() from t_delete_skip_index where y in (4, 5);
+alter table t_delete_skip_index delete where x < 8192;
+select count() from t_delete_skip_index where y in (4, 5);
+
+drop table if exists t_delete_skip_index;
+drop table if exists t_delete_projection;
+
+create table t_delete_projection (x UInt32, y UInt64, projection p (select sum(y))) engine = MergeTree order by tuple();
+insert into t_delete_projection select number, toString(number) from numbers(8192 * 10);
+
+select sum(y) from t_delete_projection settings optimize_use_projections = 0;
+select sum(y) from t_delete_projection settings optimize_use_projections = 0, force_optimize_projection = 1;
+
+alter table t_delete_projection delete where x < 8192;
+
+select sum(y) from t_delete_projection settings optimize_use_projections = 0;
+select sum(y) from t_delete_projection settings optimize_use_projections = 0, force_optimize_projection = 1;
+
+drop table if exists t_delete_projection;

From 393481181d5fb7c878af2f1e0bbbd5d9c8d56807 Mon Sep 17 00:00:00 2001
From: pufit <pufit@clickhouse.com>
Date: Mon, 24 Jul 2023 13:35:07 -0400
Subject: [PATCH 027/249] Remove Trailing whitespaces

---
 src/Parsers/Access/ASTCreateRowPolicyQuery.cpp | 2 +-
 src/Parsers/Access/ASTCreateUserQuery.cpp      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Parsers/Access/ASTCreateRowPolicyQuery.cpp b/src/Parsers/Access/ASTCreateRowPolicyQuery.cpp
index a423581e300..e95959703ee 100644
--- a/src/Parsers/Access/ASTCreateRowPolicyQuery.cpp
+++ b/src/Parsers/Access/ASTCreateRowPolicyQuery.cpp
@@ -172,7 +172,7 @@ void ASTCreateRowPolicyQuery::formatImpl(const FormatSettings & settings, Format
         settings.ostr << (settings.hilite ? IAST::hilite_keyword : "")
                     << " IN " << (settings.hilite ? IAST::hilite_none : "")
                     << backQuoteIfNeed(storage_name);
-    
+
     formatOnCluster(settings);
     assert(names->cluster.empty());
 
diff --git a/src/Parsers/Access/ASTCreateUserQuery.cpp b/src/Parsers/Access/ASTCreateUserQuery.cpp
index dbb4c1c6d57..b7290c16013 100644
--- a/src/Parsers/Access/ASTCreateUserQuery.cpp
+++ b/src/Parsers/Access/ASTCreateUserQuery.cpp
@@ -212,7 +212,7 @@ void ASTCreateUserQuery::formatImpl(const FormatSettings & format, FormatState &
         format.ostr << (format.hilite ? IAST::hilite_keyword : "")
                     << " IN " << (format.hilite ? IAST::hilite_none : "")
                     << backQuoteIfNeed(storage_name);
-    
+
     formatOnCluster(format);
 
     if (new_name)

From a5d7391fbf953d6395ecc5adba48a59e1c248811 Mon Sep 17 00:00:00 2001
From: taiyang-li <654010905@qq.com>
Date: Tue, 25 Jul 2023 15:34:38 +0800
Subject: [PATCH 028/249] add new function startsWithUTF8 and endsWithUTF8

---
 .../functions/string-functions.md             | 28 +++++++
 src/Functions/FunctionStartsEndsWith.h        | 77 ++++++++++++++++---
 src/Functions/endsWithUTF8.cpp                | 16 ++++
 src/Functions/startsWithUTF8.cpp              | 16 ++++
 .../02833_starts_ends_with_utf8.reference     | 29 +++++++
 .../02833_starts_ends_with_utf8.sql           | 19 +++++
 6 files changed, 175 insertions(+), 10 deletions(-)
 create mode 100644 src/Functions/endsWithUTF8.cpp
 create mode 100644 src/Functions/startsWithUTF8.cpp
 create mode 100644 tests/queries/0_stateless/02833_starts_ends_with_utf8.reference
 create mode 100644 tests/queries/0_stateless/02833_starts_ends_with_utf8.sql

diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md
index 9890d257e84..a398a7c0993 100644
--- a/docs/en/sql-reference/functions/string-functions.md
+++ b/docs/en/sql-reference/functions/string-functions.md
@@ -693,6 +693,22 @@ Returns whether string `str` ends with `suffix`.
 endsWith(str, suffix)
 ```
 
+## endsWithUTF8
+
+Returns whether string `str` ends with `suffix`, assuming that both strings contain svalid UTF-8 encoded text. If this assumption is violated, no exception is thrown and the result is undefined.
+
+**Syntax**
+
+```sql
+endsWithUTF8(str, suffix)
+```
+
+**Example**
+
+``` sql
+SELECT endsWithUTF8('中国', '国');
+```
+
 ## startsWith
 
 Returns whether string `str` starts with `prefix`.
@@ -709,6 +725,18 @@ startsWith(str, prefix)
 SELECT startsWith('Spider-Man', 'Spi');
 ```
 
+## startsWithUTF8
+
+Returns whether string `str` starts with `prefix`, assuming that both string contains valid UTF-8 encoded text. If this assumption is violated, no exception is thrown and the result is undefined.
+
+
+**Example**
+
+``` sql
+SELECT startsWithUTF8('中国', '中');
+```
+
+
 ## trim
 
 Removes the specified characters from the start or end of a string. If not specified otherwise, the function removes whitespace (ASCII-character 32).
diff --git a/src/Functions/FunctionStartsEndsWith.h b/src/Functions/FunctionStartsEndsWith.h
index b15bab47ae0..3565af326fe 100644
--- a/src/Functions/FunctionStartsEndsWith.h
+++ b/src/Functions/FunctionStartsEndsWith.h
@@ -28,10 +28,23 @@ namespace ErrorCodes
 struct NameStartsWith
 {
     static constexpr auto name = "startsWith";
+    static constexpr auto is_utf8 = false;
 };
 struct NameEndsWith
 {
     static constexpr auto name = "endsWith";
+    static constexpr auto is_utf8 = false;
+};
+
+struct NameStartsWithUTF8
+{
+    static constexpr auto name = "startsWithUTF8";
+    static constexpr auto is_utf8 = true;
+};
+struct NameEndsWithUTF8
+{
+    static constexpr auto name = "endsWithUTF8";
+    static constexpr auto is_utf8 = true;
 };
 
 DECLARE_MULTITARGET_CODE(
@@ -41,6 +54,7 @@ class FunctionStartsEndsWith : public IFunction
 {
 public:
     static constexpr auto name = Name::name;
+    static constexpr auto is_utf8 = Name::is_utf8;
 
     String getName() const override
     {
@@ -64,7 +78,8 @@ public:
 
     DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
     {
-        if (isStringOrFixedString(arguments[0]) && isStringOrFixedString(arguments[1]))
+        if (!is_utf8 && isStringOrFixedString(arguments[0]) && isStringOrFixedString(arguments[1])
+            || isString(arguments[0]) && isString(arguments[1]))
             return std::make_shared<DataTypeUInt8>();
 
         if (isArray(arguments[0]) && isArray(arguments[1]))
@@ -78,8 +93,11 @@ public:
     ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         auto data_type = arguments[0].type;
-        if (isStringOrFixedString(*data_type))
+
+        if (!is_utf8 && isStringOrFixedString(*data_type))
             return executeImplString(arguments, {}, input_rows_count);
+        if (is_utf8 && isString(*data_type))
+            return executeImplStringUTF8(arguments, {}, input_rows_count);
         if (isArray(data_type))
             return executeImplArray(arguments, {}, input_rows_count);
         return {};
@@ -131,7 +149,6 @@ private:
         typename ColumnVector<UInt8>::Container & vec_res = col_res->getData();
 
         vec_res.resize(input_rows_count);
-
         if (const ColumnString * haystack = checkAndGetColumn<ColumnString>(haystack_column))
             dispatch<StringSource>(StringSource(*haystack), needle_column, vec_res);
         else if (const ColumnFixedString * haystack_fixed = checkAndGetColumn<ColumnFixedString>(haystack_column))
@@ -146,6 +163,26 @@ private:
         return col_res;
     }
 
+    ColumnPtr executeImplStringUTF8(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const
+    {
+        const IColumn * haystack_column = arguments[0].column.get();
+        const IColumn * needle_column = arguments[1].column.get();
+
+        auto col_res = ColumnVector<UInt8>::create();
+        typename ColumnVector<UInt8>::Container & vec_res = col_res->getData();
+
+        vec_res.resize(input_rows_count);
+        if (const ColumnString * haystack = checkAndGetColumn<ColumnString>(haystack_column))
+            dispatchUTF8<UTF8StringSource>(UTF8StringSource(*haystack), needle_column, vec_res);
+        else if (const ColumnConst * haystack_const = checkAndGetColumnConst<ColumnString>(haystack_column))
+            dispatchUTF8<ConstSource<UTF8StringSource>>(ConstSource<UTF8StringSource>(*haystack_const), needle_column, vec_res);
+        else
+            throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal combination of columns as arguments of function {}", getName());
+
+        return col_res;
+    }
+
+
     template <typename HaystackSource>
     void dispatch(HaystackSource haystack_source, const IColumn * needle_column, PaddedPODArray<UInt8> & res_data) const
     {
@@ -161,6 +198,17 @@ private:
             throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal combination of columns as arguments of function {}", getName());
     }
 
+    template <typename HaystackSource>
+    void dispatchUTF8(HaystackSource haystack_source, const IColumn * needle_column, PaddedPODArray<UInt8> & res_data) const
+    {
+        if (const ColumnString * needle = checkAndGetColumn<ColumnString>(needle_column))
+            execute<HaystackSource, UTF8StringSource>(haystack_source, UTF8StringSource(*needle), res_data);
+        else if (const ColumnConst * needle_const = checkAndGetColumnConst<ColumnString>(needle_column))
+            execute<HaystackSource, ConstSource<UTF8StringSource>>(haystack_source, ConstSource<UTF8StringSource>(*needle_const), res_data);
+        else
+            throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal combination of columns as arguments of function {}", getName());
+    }
+
     template <typename HaystackSource, typename NeedleSource>
     static void execute(HaystackSource haystack_source, NeedleSource needle_source, PaddedPODArray<UInt8> & res_data)
     {
@@ -172,18 +220,27 @@ private:
             auto needle = needle_source.getWhole();
 
             if (needle.size > haystack.size)
-            {
                 res_data[row_num] = false;
-            }
             else
             {
-                if constexpr (std::is_same_v<Name, NameStartsWith>)
-                {
+                if constexpr (std::is_same_v<Name, NameStartsWith>) /// startsWith
                     res_data[row_num] = StringRef(haystack.data, needle.size) == StringRef(needle.data, needle.size);
-                }
-                else    /// endsWith
-                {
+                else if constexpr (std::is_same_v<Name, NameEndsWith>) /// endsWith
                     res_data[row_num] = StringRef(haystack.data + haystack.size - needle.size, needle.size) == StringRef(needle.data, needle.size);
+                else /// startsWithUTF8 or endsWithUTF8
+                {
+                    auto length = UTF8::countCodePoints(needle.data, needle.size);
+
+                    if constexpr (std::is_same_v<Name, NameStartsWithUTF8>)
+                    {
+                        auto slice = haystack_source.getSliceFromLeft(0, length);
+                        res_data[row_num] = StringRef(slice.data, slice.size) == StringRef(needle.data, needle.size);
+                    }
+                    else
+                    {
+                        auto slice = haystack_source.getSliceFromRight(length);
+                        res_data[row_num] = StringRef(slice.data, slice.size) == StringRef(needle.data, needle.size);
+                    }
                 }
             }
 
diff --git a/src/Functions/endsWithUTF8.cpp b/src/Functions/endsWithUTF8.cpp
new file mode 100644
index 00000000000..377683ce47c
--- /dev/null
+++ b/src/Functions/endsWithUTF8.cpp
@@ -0,0 +1,16 @@
+#include <DataTypes/DataTypeString.h>
+#include <Functions/FunctionFactory.h>
+#include <Functions/FunctionStartsEndsWith.h>
+
+
+namespace DB
+{
+
+using FunctionEndsWithUTF8 = FunctionStartsEndsWith<NameEndsWithUTF8>;
+
+REGISTER_FUNCTION(EndsWithUTF8)
+{
+    factory.registerFunction<FunctionEndsWithUTF8>();
+}
+
+}
diff --git a/src/Functions/startsWithUTF8.cpp b/src/Functions/startsWithUTF8.cpp
new file mode 100644
index 00000000000..e623a55f4dc
--- /dev/null
+++ b/src/Functions/startsWithUTF8.cpp
@@ -0,0 +1,16 @@
+#include <DataTypes/DataTypeString.h>
+#include <Functions/FunctionFactory.h>
+#include <Functions/FunctionStartsEndsWith.h>
+
+
+namespace DB
+{
+
+using FunctionStartsWithUTF8 = FunctionStartsEndsWith<NameStartsWithUTF8>;
+
+REGISTER_FUNCTION(StartsWithUTF8)
+{
+    factory.registerFunction<FunctionStartsWithUTF8>();
+}
+
+}
diff --git a/tests/queries/0_stateless/02833_starts_ends_with_utf8.reference b/tests/queries/0_stateless/02833_starts_ends_with_utf8.reference
new file mode 100644
index 00000000000..ca2a5bc50f8
--- /dev/null
+++ b/tests/queries/0_stateless/02833_starts_ends_with_utf8.reference
@@ -0,0 +1,29 @@
+-- { echoOn }
+select startsWithUTF8('富强民主文明和谐', '富强');
+1
+select startsWithUTF8('富强民主文明和谐', '\xe5');
+0
+select startsWithUTF8('富强民主文明和谐', '');
+1
+SELECT startsWithUTF8('123', '123');
+1
+SELECT startsWithUTF8('123', '12');
+1
+SELECT startsWithUTF8('123', '1234');
+0
+SELECT startsWithUTF8('123', '');
+1
+select endsWithUTF8('富强民主文明和谐', '和谐');
+1
+select endsWithUTF8('富强民主文明和谐', '\x90');
+0
+select endsWithUTF8('富强民主文明和谐', '');
+1
+SELECT endsWithUTF8('123', '3');
+1
+SELECT endsWithUTF8('123', '23');
+1
+SELECT endsWithUTF8('123', '32');
+0
+SELECT endsWithUTF8('123', '');
+1
diff --git a/tests/queries/0_stateless/02833_starts_ends_with_utf8.sql b/tests/queries/0_stateless/02833_starts_ends_with_utf8.sql
new file mode 100644
index 00000000000..3a783dc280e
--- /dev/null
+++ b/tests/queries/0_stateless/02833_starts_ends_with_utf8.sql
@@ -0,0 +1,19 @@
+-- { echoOn }
+select startsWithUTF8('富强民主文明和谐', '富强');
+select startsWithUTF8('富强民主文明和谐', '\xe5');
+select startsWithUTF8('富强民主文明和谐', '');
+
+SELECT startsWithUTF8('123', '123');
+SELECT startsWithUTF8('123', '12');
+SELECT startsWithUTF8('123', '1234');
+SELECT startsWithUTF8('123', '');
+
+select endsWithUTF8('富强民主文明和谐', '和谐');
+select endsWithUTF8('富强民主文明和谐', '\x90');
+select endsWithUTF8('富强民主文明和谐', '');
+
+SELECT endsWithUTF8('123', '3');
+SELECT endsWithUTF8('123', '23');
+SELECT endsWithUTF8('123', '32');
+SELECT endsWithUTF8('123', '');
+-- { echoOff }

From acb33c1d810982291e7f208340dfba50d7f6c327 Mon Sep 17 00:00:00 2001
From: taiyang-li <654010905@qq.com>
Date: Tue, 25 Jul 2023 15:39:49 +0800
Subject: [PATCH 029/249] fix code style

---
 src/Functions/FunctionStartsEndsWith.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Functions/FunctionStartsEndsWith.h b/src/Functions/FunctionStartsEndsWith.h
index 3565af326fe..7671129fcfc 100644
--- a/src/Functions/FunctionStartsEndsWith.h
+++ b/src/Functions/FunctionStartsEndsWith.h
@@ -41,6 +41,7 @@ struct NameStartsWithUTF8
     static constexpr auto name = "startsWithUTF8";
     static constexpr auto is_utf8 = true;
 };
+
 struct NameEndsWithUTF8
 {
     static constexpr auto name = "endsWithUTF8";

From 23897b823d7a98c122e385ed9452f8e930132825 Mon Sep 17 00:00:00 2001
From: taiyang-li <654010905@qq.com>
Date: Tue, 25 Jul 2023 20:41:16 +0800
Subject: [PATCH 030/249] fix code style

---
 docs/en/sql-reference/functions/string-functions.md             | 2 +-
 .../02415_all_new_functions_must_be_documented.reference        | 2 ++
 utils/check-style/aspell-ignore/en/aspell-dict.txt              | 2 ++
 3 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md
index a398a7c0993..f36de0dc0d8 100644
--- a/docs/en/sql-reference/functions/string-functions.md
+++ b/docs/en/sql-reference/functions/string-functions.md
@@ -695,7 +695,7 @@ endsWith(str, suffix)
 
 ## endsWithUTF8
 
-Returns whether string `str` ends with `suffix`, assuming that both strings contain svalid UTF-8 encoded text. If this assumption is violated, no exception is thrown and the result is undefined.
+Returns whether string `str` ends with `suffix`, assuming that both strings contain valid UTF-8 encoded text. If this assumption is violated, no exception is thrown and the result is undefined.
 
 **Syntax**
 
diff --git a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference
index 247df64f741..d70214e9930 100644
--- a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference
+++ b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference
@@ -263,6 +263,7 @@ encodeURLComponent
 encodeURLFormComponent
 encodeXMLComponent
 endsWith
+endsWithUTF8
 equals
 erf
 erfc
@@ -665,6 +666,7 @@ splitByString
 splitByWhitespace
 sqrt
 startsWith
+startsWithUTF8
 subBitmap
 substring
 substringUTF8
diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt
index 65271830555..3cc25108eb1 100644
--- a/utils/check-style/aspell-ignore/en/aspell-dict.txt
+++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt
@@ -1394,6 +1394,7 @@ encodings
 encryptions
 endian
 endsWith
+endsWithUTF8
 enum
 enum's
 enums
@@ -2200,6 +2201,7 @@ src
 stacktrace
 stacktraces
 startsWith
+startsWithUTF8
 statbox
 stateful
 stddev

From 27f3c5cfa27f9aef5df563a9fd90089963afe26c Mon Sep 17 00:00:00 2001
From: taiyang-li <654010905@qq.com>
Date: Wed, 26 Jul 2023 10:53:07 +0800
Subject: [PATCH 031/249] change as requested

---
 docs/en/sql-reference/functions/string-functions.md        | 4 ++--
 src/Functions/endsWithUTF8.cpp                             | 7 ++++++-
 src/Functions/startsWithUTF8.cpp                           | 7 ++++++-
 .../02415_all_new_functions_must_be_documented.reference   | 2 --
 4 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md
index f36de0dc0d8..b1ffb32e7cd 100644
--- a/docs/en/sql-reference/functions/string-functions.md
+++ b/docs/en/sql-reference/functions/string-functions.md
@@ -695,7 +695,7 @@ endsWith(str, suffix)
 
 ## endsWithUTF8
 
-Returns whether string `str` ends with `suffix`, assuming that both strings contain valid UTF-8 encoded text. If this assumption is violated, no exception is thrown and the result is undefined.
+Returns whether string `str` ends with `suffix`, the difference between `endsWithUTF8` and `endsWith` is that `endsWithUTF8` match `str` and `suffix` by UTF-8 characters.
 
 **Syntax**
 
@@ -727,7 +727,7 @@ SELECT startsWith('Spider-Man', 'Spi');
 
 ## startsWithUTF8
 
-Returns whether string `str` starts with `prefix`, assuming that both string contains valid UTF-8 encoded text. If this assumption is violated, no exception is thrown and the result is undefined.
+Returns whether string `str` starts with `prefix`, the difference between `startsWithUTF8` and `startsWith` is that `startsWithUTF8` match `str` and `suffix` by UTF-8 characters.
 
 
 **Example**
diff --git a/src/Functions/endsWithUTF8.cpp b/src/Functions/endsWithUTF8.cpp
index 377683ce47c..1b042452298 100644
--- a/src/Functions/endsWithUTF8.cpp
+++ b/src/Functions/endsWithUTF8.cpp
@@ -10,7 +10,12 @@ using FunctionEndsWithUTF8 = FunctionStartsEndsWith<NameEndsWithUTF8>;
 
 REGISTER_FUNCTION(EndsWithUTF8)
 {
-    factory.registerFunction<FunctionEndsWithUTF8>();
+    factory.registerFunction<FunctionEndsWithUTF8>(FunctionDocumentation{
+        .description = R"(
+Returns whether string `str` ends with `suffix`, the difference between `endsWithUTF8` and `endsWith` is that `endsWithUTF8` match `str` and `suffix` by UTF-8 characters.
+        )",
+        .examples{{"endsWithUTF8", "select endsWithUTF8('富强民主文明和谐', '富强');", ""}},
+        .categories{"String"}});
 }
 
 }
diff --git a/src/Functions/startsWithUTF8.cpp b/src/Functions/startsWithUTF8.cpp
index e623a55f4dc..3f46916d760 100644
--- a/src/Functions/startsWithUTF8.cpp
+++ b/src/Functions/startsWithUTF8.cpp
@@ -10,7 +10,12 @@ using FunctionStartsWithUTF8 = FunctionStartsEndsWith<NameStartsWithUTF8>;
 
 REGISTER_FUNCTION(StartsWithUTF8)
 {
-    factory.registerFunction<FunctionStartsWithUTF8>();
+    factory.registerFunction<FunctionStartsWithUTF8>(FunctionDocumentation{
+        .description = R"(
+Returns whether string `str` starts with `prefix`, the difference between `startsWithUTF8` and `startsWith` is that `startsWithUTF8` match `str` and `suffix` by UTF-8 characters.
+        )",
+        .examples{{"startsWithUTF8", "select startsWithUTF8('富强民主文明和谐', '富强');", ""}},
+        .categories{"String"}});
 }
 
 }
diff --git a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference
index d70214e9930..247df64f741 100644
--- a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference
+++ b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference
@@ -263,7 +263,6 @@ encodeURLComponent
 encodeURLFormComponent
 encodeXMLComponent
 endsWith
-endsWithUTF8
 equals
 erf
 erfc
@@ -666,7 +665,6 @@ splitByString
 splitByWhitespace
 sqrt
 startsWith
-startsWithUTF8
 subBitmap
 substring
 substringUTF8

From 21e12ea0842578043fd4686e16c935052480bb80 Mon Sep 17 00:00:00 2001
From: pufit <pufit@clickhouse.com>
Date: Wed, 26 Jul 2023 00:53:45 -0400
Subject: [PATCH 032/249] Fix tests

---
 src/Access/IAccessStorage.cpp                         | 2 +-
 tests/integration/test_role/test.py                   | 2 +-
 tests/queries/0_stateless/01292_create_user.reference | 8 ++++----
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp
index 11bc97f6cc3..db73db900aa 100644
--- a/src/Access/IAccessStorage.cpp
+++ b/src/Access/IAccessStorage.cpp
@@ -180,7 +180,7 @@ UUID IAccessStorage::insert(const AccessEntityPtr & entity)
 
 std::optional<UUID> IAccessStorage::insert(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists)
 {
-    return *insert(entity, replace_if_exists, throw_if_exists, /* set_id = */ std::nullopt);
+    return insert(entity, replace_if_exists, throw_if_exists, /* set_id = */ std::nullopt);
 }
 
 std::optional<UUID> IAccessStorage::insert(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, std::optional<UUID> set_id)
diff --git a/tests/integration/test_role/test.py b/tests/integration/test_role/test.py
index 43a402d9fda..0e1bbea9cea 100644
--- a/tests/integration/test_role/test.py
+++ b/tests/integration/test_role/test.py
@@ -301,7 +301,7 @@ def test_introspection():
     )
 
     assert instance.query(
-        "SELECT * from system.role_grants WHERE user_name IN ('A', 'B') OR role_name IN ('R1', 'R2') ORDER BY user_name, role_name, granted_role_name"
+        "SELECT user_name, role_name, granted_role_name, granted_role_is_default, with_admin_option from system.role_grants WHERE user_name IN ('A', 'B') OR role_name IN ('R1', 'R2') ORDER BY user_name, role_name, granted_role_name"
     ) == TSV([["A", "\\N", "R1", 1, 0], ["B", "\\N", "R2", 1, 1]])
 
     assert instance.query(
diff --git a/tests/queries/0_stateless/01292_create_user.reference b/tests/queries/0_stateless/01292_create_user.reference
index eb89a5ed38c..b7c30b304bf 100644
--- a/tests/queries/0_stateless/01292_create_user.reference
+++ b/tests/queries/0_stateless/01292_create_user.reference
@@ -106,10 +106,10 @@ CREATE USER u2_01292 DEFAULT ROLE r1_01292, r2_01292 SETTINGS readonly = 1
 CREATE USER u3_01292 HOST LIKE \'%.%.myhost.com\' DEFAULT ROLE r1_01292, r2_01292
 CREATE USER u4_01292 HOST LIKE \'%.%.myhost.com\' DEFAULT ROLE r1_01292, r2_01292
 -- system.users
-u1_01292	local directory	plaintext_password	{}	[]	['localhost']	[]	[]	1	[]	[]
-u2_01292	local directory	no_password	{}	[]	[]	[]	['%.%.myhost.com']	0	[]	[]
-u3_01292	local directory	sha256_password	{}	['192.169.1.1','192.168.0.0/16']	['localhost']	[]	[]	0	['r1_01292']	[]
-u4_01292	local directory	double_sha1_password	{}	['::/0']	[]	[]	[]	1	[]	['r1_01292']
+u1_01292	local_directory	plaintext_password	{}	[]	['localhost']	[]	[]	1	[]	[]
+u2_01292	local_directory	no_password	{}	[]	[]	[]	['%.%.myhost.com']	0	[]	[]
+u3_01292	local_directory	sha256_password	{}	['192.169.1.1','192.168.0.0/16']	['localhost']	[]	[]	0	['r1_01292']	[]
+u4_01292	local_directory	double_sha1_password	{}	['::/0']	[]	[]	[]	1	[]	['r1_01292']
 -- system.settings_profile_elements
 \N	u1_01292	\N	0	readonly	1	\N	\N	\N	\N
 \N	u2_01292	\N	0	\N	\N	\N	\N	\N	default

From d83eabcf772d942c18c8d8b2dccb43aaad1bf235 Mon Sep 17 00:00:00 2001
From: taiyang-li <654010905@qq.com>
Date: Wed, 26 Jul 2023 17:39:45 +0800
Subject: [PATCH 033/249] fix spell

---
 utils/check-style/aspell-ignore/en/aspell-dict.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt
index 3cc25108eb1..240238cab12 100644
--- a/utils/check-style/aspell-ignore/en/aspell-dict.txt
+++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt
@@ -1394,7 +1394,7 @@ encodings
 encryptions
 endian
 endsWith
-endsWithUTF8
+endsWithUTF
 enum
 enum's
 enums
@@ -2201,7 +2201,7 @@ src
 stacktrace
 stacktraces
 startsWith
-startsWithUTF8
+startsWithUTF
 statbox
 stateful
 stddev

From 13d9952227162b63ae5e4ae98b0a40677c169b97 Mon Sep 17 00:00:00 2001
From: pufit <pufit@clickhouse.com>
Date: Thu, 27 Jul 2023 00:38:51 -0400
Subject: [PATCH 034/249] add id to insertImpl

---
 src/Access/AccessControl.cpp                  | 10 +++++---
 src/Access/AccessControl.h                    |  2 +-
 src/Access/DiskAccessStorage.cpp              | 16 +++---------
 src/Access/DiskAccessStorage.h                |  3 +--
 src/Access/IAccessStorage.cpp                 | 25 ++++++++++++-------
 src/Access/IAccessStorage.h                   | 11 ++++++--
 src/Access/MemoryAccessStorage.cpp            | 14 ++---------
 src/Access/MemoryAccessStorage.h              |  7 +-----
 src/Access/MultipleAccessStorage.cpp          | 11 ++++----
 src/Access/MultipleAccessStorage.h            |  2 +-
 src/Access/ReplicatedAccessStorage.cpp        | 16 +++---------
 src/Access/ReplicatedAccessStorage.h          |  3 +--
 src/Parsers/Access/ParserCreateQuotaQuery.cpp |  4 +--
 src/Parsers/Access/ParserCreateRoleQuery.cpp  |  3 ++-
 .../Access/ParserCreateRowPolicyQuery.cpp     |  4 +--
 .../ParserCreateSettingsProfileQuery.cpp      |  4 +--
 src/Parsers/Access/ParserCreateUserQuery.cpp  |  3 ++-
 .../Access/ParserDropAccessEntityQuery.cpp    |  3 ++-
 .../Access/ParserMoveAccessEntityQuery.cpp    |  3 ++-
 src/Parsers/Access/parseUserName.h            |  5 ----
 20 files changed, 64 insertions(+), 85 deletions(-)

diff --git a/src/Access/AccessControl.cpp b/src/Access/AccessControl.cpp
index 22b00d5dde7..3759b6c8919 100644
--- a/src/Access/AccessControl.cpp
+++ b/src/Access/AccessControl.cpp
@@ -525,12 +525,14 @@ scope_guard AccessControl::subscribeForChanges(const std::vector<UUID> & ids, co
     return changes_notifier->subscribeForChanges(ids, handler);
 }
 
-std::optional<UUID> AccessControl::insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, std::optional<UUID> set_id)
+bool AccessControl::insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists)
 {
-    auto id = MultipleAccessStorage::insertImpl(entity, replace_if_exists, throw_if_exists, set_id);
-    if (id)
+    if (MultipleAccessStorage::insertImpl(id, entity, replace_if_exists, throw_if_exists))
+    {
         changes_notifier->sendNotifications();
-    return id;
+        return true;
+    }
+    return false;
 }
 
 bool AccessControl::removeImpl(const UUID & id, bool throw_if_not_exists)
diff --git a/src/Access/AccessControl.h b/src/Access/AccessControl.h
index 85feaa70462..60b40775a56 100644
--- a/src/Access/AccessControl.h
+++ b/src/Access/AccessControl.h
@@ -229,7 +229,7 @@ private:
     class CustomSettingsPrefixes;
     class PasswordComplexityRules;
 
-    std::optional<UUID> insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, std::optional<UUID> set_id) override;
+    bool insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override;
     bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
     bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;
 
diff --git a/src/Access/DiskAccessStorage.cpp b/src/Access/DiskAccessStorage.cpp
index acbd7b12569..190c7567b85 100644
--- a/src/Access/DiskAccessStorage.cpp
+++ b/src/Access/DiskAccessStorage.cpp
@@ -498,20 +498,10 @@ std::optional<std::pair<String, AccessEntityType>> DiskAccessStorage::readNameWi
 }
 
 
-std::optional<UUID> DiskAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, std::optional<UUID> set_id)
-{
-    UUID id = set_id ? *set_id : generateRandomID();
-    if (insertWithID(id, new_entity, replace_if_exists, throw_if_exists, /* write_on_disk= */ true))
-        return id;
-
-    return std::nullopt;
-}
-
-
-bool DiskAccessStorage::insertWithID(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, bool write_on_disk)
+bool DiskAccessStorage::insertImpl(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists)
 {
     std::lock_guard lock{mutex};
-    return insertNoLock(id, new_entity, replace_if_exists, throw_if_exists, write_on_disk);
+    return insertNoLock(id, new_entity, replace_if_exists, throw_if_exists, /* write_on_disk = */ true);
 }
 
 
@@ -745,7 +735,7 @@ void DiskAccessStorage::restoreFromBackup(RestorerFromBackup & restorer)
     restorer.addDataRestoreTask([this, my_entities = std::move(entities), replace_if_exists, throw_if_exists]
     {
         for (const auto & [id, entity] : my_entities)
-            insertWithID(id, entity, replace_if_exists, throw_if_exists, /* write_on_disk= */ true);
+            insert(id, entity, replace_if_exists, throw_if_exists);
     });
 }
 
diff --git a/src/Access/DiskAccessStorage.h b/src/Access/DiskAccessStorage.h
index 3541c3228e3..5d94008b34f 100644
--- a/src/Access/DiskAccessStorage.h
+++ b/src/Access/DiskAccessStorage.h
@@ -39,7 +39,7 @@ private:
     std::vector<UUID> findAllImpl(AccessEntityType type) const override;
     AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override;
     std::optional<std::pair<String, AccessEntityType>> readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const override;
-    std::optional<UUID> insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, std::optional<UUID> set_id) override;
+    bool insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override;
     bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
     bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;
 
@@ -53,7 +53,6 @@ private:
     void listsWritingThreadFunc() TSA_NO_THREAD_SAFETY_ANALYSIS;
     void stopListsWritingThread();
 
-    bool insertWithID(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, bool write_on_disk);
     bool insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, bool write_on_disk) TSA_REQUIRES(mutex);
     bool updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists, bool write_on_disk) TSA_REQUIRES(mutex);
     bool removeNoLock(const UUID & id, bool throw_if_not_exists, bool write_on_disk) TSA_REQUIRES(mutex);
diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp
index db73db900aa..468c93c1116 100644
--- a/src/Access/IAccessStorage.cpp
+++ b/src/Access/IAccessStorage.cpp
@@ -180,14 +180,21 @@ UUID IAccessStorage::insert(const AccessEntityPtr & entity)
 
 std::optional<UUID> IAccessStorage::insert(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists)
 {
-    return insert(entity, replace_if_exists, throw_if_exists, /* set_id = */ std::nullopt);
+    auto id = generateRandomID();
+
+    if (insert(id, entity, replace_if_exists, throw_if_exists))
+        return id;
+
+    return std::nullopt;
 }
 
-std::optional<UUID> IAccessStorage::insert(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, std::optional<UUID> set_id)
+
+bool IAccessStorage::insert(const DB::UUID & id, const DB::AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists)
 {
-    return insertImpl(entity, replace_if_exists, throw_if_exists, set_id);
+    return insertImpl(id, entity, replace_if_exists, throw_if_exists);
 }
 
+
 std::vector<UUID> IAccessStorage::insert(const std::vector<AccessEntityPtr> & multiple_entities, bool replace_if_exists, bool throw_if_exists)
 {
     return insert(multiple_entities, /* ids = */ {}, replace_if_exists, throw_if_exists);
@@ -216,16 +223,16 @@ std::vector<UUID> IAccessStorage::insert(const std::vector<AccessEntityPtr> & mu
         {
             const auto & entity = multiple_entities[i];
 
-            std::optional<UUID> id;
+            UUID id;
             if (!ids.empty())
                 id = ids[i];
+            else
+                id = generateRandomID();
 
-            auto new_id = insertImpl(entity, replace_if_exists, throw_if_exists, id);
-
-            if (new_id)
+            if (insertImpl(id, entity, replace_if_exists, throw_if_exists))
             {
                 successfully_inserted.push_back(entity);
-                new_ids.push_back(*new_id);
+                new_ids.push_back(id);
             }
         }
         return new_ids;
@@ -274,7 +281,7 @@ std::vector<UUID> IAccessStorage::insertOrReplace(const std::vector<AccessEntity
 }
 
 
-std::optional<UUID> IAccessStorage::insertImpl(const AccessEntityPtr & entity, bool, bool, std::optional<UUID>)
+bool IAccessStorage::insertImpl(const UUID &, const AccessEntityPtr & entity, bool, bool)
 {
     if (isReadOnly())
         throwReadonlyCannotInsert(entity->getType(), entity->getName());
diff --git a/src/Access/IAccessStorage.h b/src/Access/IAccessStorage.h
index 894220e95ad..b82537df639 100644
--- a/src/Access/IAccessStorage.h
+++ b/src/Access/IAccessStorage.h
@@ -3,6 +3,8 @@
 #include <Access/IAccessEntity.h>
 #include <Core/Types.h>
 #include <Core/UUID.h>
+#include <Parsers/IParser.h>
+#include <Parsers/parseIdentifierOrStringLiteral.h>
 #include <functional>
 #include <optional>
 #include <vector>
@@ -132,7 +134,7 @@ public:
     /// Throws an exception if the specified name already exists.
     UUID insert(const AccessEntityPtr & entity);
     std::optional<UUID> insert(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists);
-    std::optional<UUID> insert(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, std::optional<UUID> set_id);
+    bool insert(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists);
     std::vector<UUID> insert(const std::vector<AccessEntityPtr> & multiple_entities, bool replace_if_exists = false, bool throw_if_exists = true);
     std::vector<UUID> insert(const std::vector<AccessEntityPtr> & multiple_entities, const std::vector<UUID> & ids, bool replace_if_exists = false, bool throw_if_exists = true);
 
@@ -185,7 +187,7 @@ protected:
     virtual std::vector<UUID> findAllImpl(AccessEntityType type) const = 0;
     virtual AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const = 0;
     virtual std::optional<std::pair<String, AccessEntityType>> readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const;
-    virtual std::optional<UUID> insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, std::optional<UUID> set_id);
+    virtual bool insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists);
     virtual bool removeImpl(const UUID & id, bool throw_if_not_exists);
     virtual bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists);
     virtual std::optional<UUID> authenticateImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool throw_if_user_not_exists, bool allow_no_password, bool allow_plaintext_password) const;
@@ -284,4 +286,9 @@ std::vector<std::pair<UUID, std::shared_ptr<const EntityClassT>>> IAccessStorage
     return entities;
 }
 
+inline bool parseAccessStorageName(IParser::Pos & pos, Expected & expected, String & storage_name)
+{
+    return parseIdentifierOrStringLiteral(pos, expected, storage_name);
+}
+
 }
diff --git a/src/Access/MemoryAccessStorage.cpp b/src/Access/MemoryAccessStorage.cpp
index 16f3b3793a0..999e2c41d0b 100644
--- a/src/Access/MemoryAccessStorage.cpp
+++ b/src/Access/MemoryAccessStorage.cpp
@@ -63,17 +63,7 @@ AccessEntityPtr MemoryAccessStorage::readImpl(const UUID & id, bool throw_if_not
 }
 
 
-std::optional<UUID> MemoryAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, std::optional<UUID> set_id)
-{
-    UUID id = set_id ? *set_id : generateRandomID();
-    if (insertWithID(id, new_entity, replace_if_exists, throw_if_exists))
-        return id;
-
-    return std::nullopt;
-}
-
-
-bool MemoryAccessStorage::insertWithID(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists)
+bool MemoryAccessStorage::insertImpl(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists)
 {
     std::lock_guard lock{mutex};
     return insertNoLock(id, new_entity, replace_if_exists, throw_if_exists);
@@ -300,7 +290,7 @@ void MemoryAccessStorage::restoreFromBackup(RestorerFromBackup & restorer)
     restorer.addDataRestoreTask([this, my_entities = std::move(entities), replace_if_exists, throw_if_exists]
     {
         for (const auto & [id, entity] : my_entities)
-            insertWithID(id, entity, replace_if_exists, throw_if_exists);
+            insert(id, entity, replace_if_exists, throw_if_exists);
     });
 }
 
diff --git a/src/Access/MemoryAccessStorage.h b/src/Access/MemoryAccessStorage.h
index 7c19ab9c4ab..f877860041a 100644
--- a/src/Access/MemoryAccessStorage.h
+++ b/src/Access/MemoryAccessStorage.h
@@ -22,11 +22,6 @@ public:
 
     const char * getStorageType() const override { return STORAGE_TYPE; }
 
-    /// Inserts an entity with a specified ID.
-    /// If `replace_if_exists == true` it can replace an existing entry with such ID and also remove an existing entry
-    /// with such name & type.
-    bool insertWithID(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists);
-
     /// Removes all entities except the specified list `ids_to_keep`.
     /// The function skips IDs not contained in the storage.
     void removeAllExcept(const std::vector<UUID> & ids_to_keep);
@@ -44,7 +39,7 @@ private:
     std::optional<UUID> findImpl(AccessEntityType type, const String & name) const override;
     std::vector<UUID> findAllImpl(AccessEntityType type) const override;
     AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override;
-    std::optional<UUID> insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, std::optional<UUID> set_id) override;
+    bool insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override;
     bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
     bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;
 
diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp
index 169dc889509..e6665349ae6 100644
--- a/src/Access/MultipleAccessStorage.cpp
+++ b/src/Access/MultipleAccessStorage.cpp
@@ -316,7 +316,7 @@ void MultipleAccessStorage::reload(ReloadMode reload_mode)
 }
 
 
-std::optional<UUID> MultipleAccessStorage::insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, std::optional<UUID> set_id)
+bool MultipleAccessStorage::insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists)
 {
     std::shared_ptr<IAccessStorage> storage_for_insertion;
 
@@ -339,13 +339,14 @@ std::optional<UUID> MultipleAccessStorage::insertImpl(const AccessEntityPtr & en
             getStorageName());
     }
 
-    auto id = storage_for_insertion->insert(entity, replace_if_exists, throw_if_exists, set_id);
-    if (id)
+    if (storage_for_insertion->insert(id, entity, replace_if_exists, throw_if_exists))
     {
         std::lock_guard lock{mutex};
-        ids_cache.set(*id, storage_for_insertion);
+        ids_cache.set(id, storage_for_insertion);
+        return true;
     }
-    return id;
+
+    return false;
 }
 
 
diff --git a/src/Access/MultipleAccessStorage.h b/src/Access/MultipleAccessStorage.h
index 1d95296b796..069d414f601 100644
--- a/src/Access/MultipleAccessStorage.h
+++ b/src/Access/MultipleAccessStorage.h
@@ -63,7 +63,7 @@ protected:
     std::vector<UUID> findAllImpl(AccessEntityType type) const override;
     AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override;
     std::optional<std::pair<String, AccessEntityType>> readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const override;
-    std::optional<UUID> insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, std::optional<UUID> set_id) override;
+    bool insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override;
     bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
     bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;
     std::optional<UUID> authenticateImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool throw_if_user_not_exists, bool allow_no_password, bool allow_plaintext_password) const override;
diff --git a/src/Access/ReplicatedAccessStorage.cpp b/src/Access/ReplicatedAccessStorage.cpp
index efa9dbbdfaa..275672da115 100644
--- a/src/Access/ReplicatedAccessStorage.cpp
+++ b/src/Access/ReplicatedAccessStorage.cpp
@@ -108,17 +108,7 @@ static void retryOnZooKeeperUserError(size_t attempts, Func && function)
     }
 }
 
-std::optional<UUID> ReplicatedAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, std::optional<UUID> set_id)
-{
-    const UUID id = set_id ? *set_id : generateRandomID();
-    if (insertWithID(id, new_entity, replace_if_exists, throw_if_exists))
-        return id;
-
-    return std::nullopt;
-}
-
-
-bool ReplicatedAccessStorage::insertWithID(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists)
+bool ReplicatedAccessStorage::insertImpl(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists)
 {
     const AccessEntityTypeInfo type_info = AccessEntityTypeInfo::get(new_entity->getType());
     const String & name = new_entity->getName();
@@ -619,7 +609,7 @@ AccessEntityPtr ReplicatedAccessStorage::tryReadEntityFromZooKeeper(const zkutil
 void ReplicatedAccessStorage::setEntityNoLock(const UUID & id, const AccessEntityPtr & entity)
 {
     LOG_DEBUG(getLogger(), "Setting id {} to entity named {}", toString(id), entity->getName());
-    memory_storage.insertWithID(id, entity, /* replace_if_exists= */ true, /* throw_if_exists= */ false);
+    memory_storage.insert(id, entity, /* replace_if_exists= */ true, /* throw_if_exists= */ false);
 }
 
 
@@ -711,7 +701,7 @@ void ReplicatedAccessStorage::restoreFromBackup(RestorerFromBackup & restorer)
     restorer.addDataRestoreTask([this, my_entities = std::move(entities), replace_if_exists, throw_if_exists]
     {
         for (const auto & [id, entity] : my_entities)
-            insertWithID(id, entity, replace_if_exists, throw_if_exists);
+            insert(id, entity, replace_if_exists, throw_if_exists);
     });
 }
 
diff --git a/src/Access/ReplicatedAccessStorage.h b/src/Access/ReplicatedAccessStorage.h
index 0a0c53d3bc4..cddb20860f7 100644
--- a/src/Access/ReplicatedAccessStorage.h
+++ b/src/Access/ReplicatedAccessStorage.h
@@ -46,11 +46,10 @@ private:
     std::unique_ptr<ThreadFromGlobalPool> watching_thread;
     std::shared_ptr<ConcurrentBoundedQueue<UUID>> watched_queue;
 
-    std::optional<UUID> insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, std::optional<UUID> set_id) override;
+    bool insertImpl(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists) override;
     bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
     bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;
 
-    bool insertWithID(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists);
     bool insertZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists);
     bool removeZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, bool throw_if_not_exists);
     bool updateZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists);
diff --git a/src/Parsers/Access/ParserCreateQuotaQuery.cpp b/src/Parsers/Access/ParserCreateQuotaQuery.cpp
index 05c2820b012..718498ed1d1 100644
--- a/src/Parsers/Access/ParserCreateQuotaQuery.cpp
+++ b/src/Parsers/Access/ParserCreateQuotaQuery.cpp
@@ -1,11 +1,11 @@
 #include <IO/ReadHelpers.h>
+#include <Access/IAccessStorage.h>
 #include <Parsers/ASTIdentifier_fwd.h>
 #include <Parsers/ASTLiteral.h>
 #include <Parsers/Access/ASTCreateQuotaQuery.h>
 #include <Parsers/Access/ASTRolesOrUsersSet.h>
 #include <Parsers/Access/ParserCreateQuotaQuery.h>
 #include <Parsers/Access/ParserRolesOrUsersSet.h>
-#include <Parsers/Access/parseUserName.h>
 #include <Parsers/CommonParsers.h>
 #include <Parsers/ExpressionElementParsers.h>
 #include <Parsers/ExpressionListParsers.h>
@@ -312,7 +312,7 @@ bool ParserCreateQuotaQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
         if (cluster.empty() && parseOnCluster(pos, expected, cluster))
             continue;
 
-        if (storage_name.empty() && ParserKeyword{"IN"}.ignore(pos, expected) && parseStorageName(pos, expected, storage_name))
+        if (storage_name.empty() && ParserKeyword{"IN"}.ignore(pos, expected) && parseAccessStorageName(pos, expected, storage_name))
             continue;
 
         break;
diff --git a/src/Parsers/Access/ParserCreateRoleQuery.cpp b/src/Parsers/Access/ParserCreateRoleQuery.cpp
index b548c07ec70..99a97f6901d 100644
--- a/src/Parsers/Access/ParserCreateRoleQuery.cpp
+++ b/src/Parsers/Access/ParserCreateRoleQuery.cpp
@@ -1,3 +1,4 @@
+#include <Access/IAccessStorage.h>
 #include <Parsers/Access/ParserCreateRoleQuery.h>
 #include <Parsers/Access/ASTCreateRoleQuery.h>
 #include <Parsers/Access/ASTSettingsProfileElement.h>
@@ -111,7 +112,7 @@ bool ParserCreateRoleQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
         if (cluster.empty() && parseOnCluster(pos, expected, cluster))
             continue;
 
-        if (storage_name.empty() && ParserKeyword{"IN"}.ignore(pos, expected) && parseStorageName(pos, expected, storage_name))
+        if (storage_name.empty() && ParserKeyword{"IN"}.ignore(pos, expected) && parseAccessStorageName(pos, expected, storage_name))
             continue;
 
         break;
diff --git a/src/Parsers/Access/ParserCreateRowPolicyQuery.cpp b/src/Parsers/Access/ParserCreateRowPolicyQuery.cpp
index b2177ca4ade..f9e9466e35d 100644
--- a/src/Parsers/Access/ParserCreateRowPolicyQuery.cpp
+++ b/src/Parsers/Access/ParserCreateRowPolicyQuery.cpp
@@ -1,10 +1,10 @@
+#include <Access/IAccessStorage.h>
 #include <Parsers/Access/ParserCreateRowPolicyQuery.h>
 #include <Parsers/Access/ASTCreateRowPolicyQuery.h>
 #include <Parsers/Access/ASTRolesOrUsersSet.h>
 #include <Parsers/Access/ASTRowPolicyName.h>
 #include <Parsers/Access/ParserRolesOrUsersSet.h>
 #include <Parsers/Access/ParserRowPolicyName.h>
-#include <Parsers/Access/parseUserName.h>
 #include <Parsers/ExpressionListParsers.h>
 #include <Parsers/ExpressionElementParsers.h>
 #include <Parsers/parseDatabaseAndTableName.h>
@@ -273,7 +273,7 @@ bool ParserCreateRowPolicyQuery::parseImpl(Pos & pos, ASTPtr & node, Expected &
         if (cluster.empty() && parseOnCluster(pos, expected, cluster))
             continue;
 
-        if (storage_name.empty() && ParserKeyword{"IN"}.ignore(pos, expected) && parseStorageName(pos, expected, storage_name))
+        if (storage_name.empty() && ParserKeyword{"IN"}.ignore(pos, expected) && parseAccessStorageName(pos, expected, storage_name))
             continue;
 
         break;
diff --git a/src/Parsers/Access/ParserCreateSettingsProfileQuery.cpp b/src/Parsers/Access/ParserCreateSettingsProfileQuery.cpp
index 634ff68427a..fcb6ebd8806 100644
--- a/src/Parsers/Access/ParserCreateSettingsProfileQuery.cpp
+++ b/src/Parsers/Access/ParserCreateSettingsProfileQuery.cpp
@@ -1,10 +1,10 @@
+#include <Access/IAccessStorage.h>
 #include <Parsers/Access/ParserCreateSettingsProfileQuery.h>
 #include <Parsers/Access/ASTCreateSettingsProfileQuery.h>
 #include <Parsers/Access/ASTRolesOrUsersSet.h>
 #include <Parsers/Access/ASTSettingsProfileElement.h>
 #include <Parsers/Access/ParserSettingsProfileElement.h>
 #include <Parsers/Access/ParserRolesOrUsersSet.h>
-#include <Parsers/Access/parseUserName.h>
 #include <Parsers/ASTLiteral.h>
 #include <Parsers/CommonParsers.h>
 #include <Parsers/ExpressionElementParsers.h>
@@ -132,7 +132,7 @@ bool ParserCreateSettingsProfileQuery::parseImpl(Pos & pos, ASTPtr & node, Expec
         if (cluster.empty() && parseOnCluster(pos, expected, cluster))
             continue;
 
-        if (storage_name.empty() && ParserKeyword{"IN"}.ignore(pos, expected) && parseStorageName(pos, expected, storage_name))
+        if (storage_name.empty() && ParserKeyword{"IN"}.ignore(pos, expected) && parseAccessStorageName(pos, expected, storage_name))
             continue;
 
         break;
diff --git a/src/Parsers/Access/ParserCreateUserQuery.cpp b/src/Parsers/Access/ParserCreateUserQuery.cpp
index d95d41296dc..577b6772ab2 100644
--- a/src/Parsers/Access/ParserCreateUserQuery.cpp
+++ b/src/Parsers/Access/ParserCreateUserQuery.cpp
@@ -1,3 +1,4 @@
+#include <Access/IAccessStorage.h>
 #include <Parsers/Access/ParserCreateUserQuery.h>
 #include <Parsers/Access/ASTCreateUserQuery.h>
 #include <Parsers/Access/ASTRolesOrUsersSet.h>
@@ -481,7 +482,7 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
             }
         }
 
-        if (storage_name.empty() && ParserKeyword{"IN"}.ignore(pos, expected) && parseStorageName(pos, expected, storage_name))
+        if (storage_name.empty() && ParserKeyword{"IN"}.ignore(pos, expected) && parseAccessStorageName(pos, expected, storage_name))
             continue;
 
         break;
diff --git a/src/Parsers/Access/ParserDropAccessEntityQuery.cpp b/src/Parsers/Access/ParserDropAccessEntityQuery.cpp
index d5ecdf27449..7fd34a6f8e4 100644
--- a/src/Parsers/Access/ParserDropAccessEntityQuery.cpp
+++ b/src/Parsers/Access/ParserDropAccessEntityQuery.cpp
@@ -1,3 +1,4 @@
+#include <Access/IAccessStorage.h>
 #include <Parsers/Access/ParserDropAccessEntityQuery.h>
 #include <Parsers/Access/ASTDropAccessEntityQuery.h>
 #include <Parsers/Access/ParserRowPolicyName.h>
@@ -78,7 +79,7 @@ bool ParserDropAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expected &
     }
 
     if (ParserKeyword{"FROM"}.ignore(pos, expected))
-        parseStorageName(pos, expected, storage_name);
+        parseAccessStorageName(pos, expected, storage_name);
 
     if (cluster.empty())
         parseOnCluster(pos, expected, cluster);
diff --git a/src/Parsers/Access/ParserMoveAccessEntityQuery.cpp b/src/Parsers/Access/ParserMoveAccessEntityQuery.cpp
index 6ac41b77b84..798e200bfb3 100644
--- a/src/Parsers/Access/ParserMoveAccessEntityQuery.cpp
+++ b/src/Parsers/Access/ParserMoveAccessEntityQuery.cpp
@@ -1,3 +1,4 @@
+#include <Access/IAccessStorage.h>
 #include <Parsers/Access/ParserMoveAccessEntityQuery.h>
 #include <Parsers/Access/ASTMoveAccessEntityQuery.h>
 #include <Parsers/Access/ParserRowPolicyName.h>
@@ -73,7 +74,7 @@ bool ParserMoveAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expected &
             return false;
     }
 
-    if (!ParserKeyword{"TO"}.ignore(pos, expected) || !parseStorageName(pos, expected, storage_name))
+    if (!ParserKeyword{"TO"}.ignore(pos, expected) || !parseAccessStorageName(pos, expected, storage_name))
         return false;
 
     if (cluster.empty())
diff --git a/src/Parsers/Access/parseUserName.h b/src/Parsers/Access/parseUserName.h
index 64fed2ce21c..10c548d8826 100644
--- a/src/Parsers/Access/parseUserName.h
+++ b/src/Parsers/Access/parseUserName.h
@@ -35,9 +35,4 @@ inline bool parseRoleNames(IParser::Pos & pos, Expected & expected, Strings & ro
     return parseUserNames(pos, expected, role_names);
 }
 
-inline bool parseStorageName(IParser::Pos & pos, Expected & expected, String & storage_name)
-{
-    return parseIdentifierOrStringLiteral(pos, expected, storage_name);
-}
-
 }

From 8c14c4b5e8925c6793c04cced5ebe64668ba3944 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Thu, 27 Jul 2023 14:59:28 +0000
Subject: [PATCH 035/249] Remove unneeded readBinary() specializations + update
 docs

The default instantiation

    template <typename T>
    requires is_arithmetic_v<T>
    inline void readBinary(T & x, ReadBuffer & buf) { readPODBinary(x, buf); }

covers Int32, Int128, Int256, UInt32, UInt128 and UInt256 cases already.
---
 docs/en/sql-reference/data-types/uuid.md | 24 +++++++++++-------------
 src/IO/ReadHelpers.h                     |  6 ------
 2 files changed, 11 insertions(+), 19 deletions(-)

diff --git a/docs/en/sql-reference/data-types/uuid.md b/docs/en/sql-reference/data-types/uuid.md
index b0f19f0d8be..40f756b9588 100644
--- a/docs/en/sql-reference/data-types/uuid.md
+++ b/docs/en/sql-reference/data-types/uuid.md
@@ -6,42 +6,42 @@ sidebar_label: UUID
 
 # UUID
 
-A universally unique identifier (UUID) is a 16-byte number used to identify records. For detailed information about the UUID, see [Wikipedia](https://en.wikipedia.org/wiki/Universally_unique_identifier).
+A Universally Unique Identifier (UUID) is a 16-byte value used to identify records. For detailed information about UUIDs, see [Wikipedia](https://en.wikipedia.org/wiki/Universally_unique_identifier).
 
-The example of UUID type value is represented below:
+While different UUID variants exist (see [here](https://datatracker.ietf.org/doc/html/draft-ietf-uuidrev-rfc4122bis)), ClickHouse does not validate that inserted UUIDs conform to a particular variant. UUIDs are internally treated as a sequence of 16 random bytes with [8-4-4-4-12 representation](https://en.wikipedia.org/wiki/Universally_unique_identifier#Textual_representation) at SQL level.
+
+Example UUID value:
 
 ``` text
 61f0c404-5cb3-11e7-907b-a6006ad3dba0
 ```
 
-If you do not specify the UUID column value when inserting a new record, the UUID value is filled with zero:
+The default UUID is all-zero. It is used, for example, when a new record is inserted but no value for a UUID column is specified:
 
 ``` text
 00000000-0000-0000-0000-000000000000
 ```
 
-## How to Generate
+## Generating UUIDs
 
-To generate the UUID value, ClickHouse provides the [generateUUIDv4](../../sql-reference/functions/uuid-functions.md) function.
+ClickHouse provides the [generateUUIDv4](../../sql-reference/functions/uuid-functions.md) function to generate random UUID version 4 values.
 
 ## Usage Example
 
 **Example 1**
 
-This example demonstrates creating a table with the UUID type column and inserting a value into the table.
+This example demonstrates the creation of a table with a UUID column and the insertion of a value into the table.
 
 ``` sql
 CREATE TABLE t_uuid (x UUID, y String) ENGINE=TinyLog
-```
 
-``` sql
 INSERT INTO t_uuid SELECT generateUUIDv4(), 'Example 1'
-```
 
-``` sql
 SELECT * FROM t_uuid
 ```
 
+Result:
+
 ``` text
 ┌────────────────────────────────────x─┬─y─────────┐
 │ 417ddc5d-e556-4d27-95dd-a34d84e46a50 │ Example 1 │
@@ -50,13 +50,11 @@ SELECT * FROM t_uuid
 
 **Example 2**
 
-In this example, the UUID column value is not specified when inserting a new record.
+In this example, no UUID column value is specified when the record is inserted, i.e. the default UUID value is inserted:
 
 ``` sql
 INSERT INTO t_uuid (y) VALUES ('Example 2')
-```
 
-``` sql
 SELECT * FROM t_uuid
 ```
 
diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h
index 2636898c1b3..1fe08a738c2 100644
--- a/src/IO/ReadHelpers.h
+++ b/src/IO/ReadHelpers.h
@@ -1096,12 +1096,6 @@ inline void readBinary(bool & x, ReadBuffer & buf)
 }
 
 inline void readBinary(String & x, ReadBuffer & buf) { readStringBinary(x, buf); }
-inline void readBinary(Int32 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
-inline void readBinary(Int128 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
-inline void readBinary(Int256 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
-inline void readBinary(UInt32 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
-inline void readBinary(UInt128 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
-inline void readBinary(UInt256 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
 inline void readBinary(Decimal32 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
 inline void readBinary(Decimal64 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
 inline void readBinary(Decimal128 & x, ReadBuffer & buf) { readPODBinary(x, buf); }

From 07147ef88805ca09b5cd7120ecf60d71c1d94d55 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Thu, 27 Jul 2023 15:24:28 +0000
Subject: [PATCH 036/249] Remove remainders of legacy setting
 'allow_experimental_query_cache'

---
 src/Core/Settings.h                                             | 1 -
 ...726_set_allow_experimental_query_cache_as_obsolete.reference | 0
 .../02726_set_allow_experimental_query_cache_as_obsolete.sql    | 2 --
 3 files changed, 3 deletions(-)
 delete mode 100644 tests/queries/0_stateless/02726_set_allow_experimental_query_cache_as_obsolete.reference
 delete mode 100644 tests/queries/0_stateless/02726_set_allow_experimental_query_cache_as_obsolete.sql

diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index c69d132ea25..e7fd28476d9 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -808,7 +808,6 @@ class IColumn;
     MAKE_OBSOLETE(M, UInt64, merge_tree_clear_old_parts_interval_seconds, 1) \
     MAKE_OBSOLETE(M, UInt64, partial_merge_join_optimizations, 0) \
     MAKE_OBSOLETE(M, MaxThreads, max_alter_threads, 0) \
-    MAKE_OBSOLETE(M, Bool, allow_experimental_query_cache, true) \
     /* moved to config.xml: see also src/Core/ServerSettings.h */ \
     MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, background_buffer_flush_schedule_pool_size, 16) \
     MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, background_pool_size, 16) \
diff --git a/tests/queries/0_stateless/02726_set_allow_experimental_query_cache_as_obsolete.reference b/tests/queries/0_stateless/02726_set_allow_experimental_query_cache_as_obsolete.reference
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/queries/0_stateless/02726_set_allow_experimental_query_cache_as_obsolete.sql b/tests/queries/0_stateless/02726_set_allow_experimental_query_cache_as_obsolete.sql
deleted file mode 100644
index 244ba4e959a..00000000000
--- a/tests/queries/0_stateless/02726_set_allow_experimental_query_cache_as_obsolete.sql
+++ /dev/null
@@ -1,2 +0,0 @@
-SET allow_experimental_query_cache = 0;
-SET allow_experimental_query_cache = 1;

From b98cb7fa145d1a92c2c78421be1eeb8fe8353d53 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Thu, 27 Jul 2023 17:50:39 +0000
Subject: [PATCH 037/249] Fix build

---
 src/IO/ReadHelpers.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h
index 1fe08a738c2..bbb1c517d7c 100644
--- a/src/IO/ReadHelpers.h
+++ b/src/IO/ReadHelpers.h
@@ -1101,6 +1101,9 @@ inline void readBinary(Decimal64 & x, ReadBuffer & buf) { readPODBinary(x, buf);
 inline void readBinary(Decimal128 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
 inline void readBinary(Decimal256 & x, ReadBuffer & buf) { readPODBinary(x.value, buf); }
 inline void readBinary(LocalDate & x, ReadBuffer & buf) { readPODBinary(x, buf); }
+inline void readBinary(UUID & x, ReadBuffer & buf) { readPODBinary(x, buf); }
+inline void readBinary(IPv4 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
+inline void readBinary(IPv6 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
 
 inline void readBinary(StackTrace::FramePointers & x, ReadBuffer & buf) { readPODBinary(x, buf); }
 

From 9dbb106dc14a085013140427401a235ae2115e47 Mon Sep 17 00:00:00 2001
From: pufit <pufit@clickhouse.com>
Date: Thu, 27 Jul 2023 23:37:09 -0400
Subject: [PATCH 038/249] Fix tests, docs

---
 .../sql-reference/statements/create/quota.md  |  1 +
 .../sql-reference/statements/create/role.md   |  1 +
 .../statements/create/row-policy.md           |  1 +
 .../statements/create/settings-profile.md     |  1 +
 .../sql-reference/statements/create/user.md   |  1 +
 docs/en/sql-reference/statements/drop.md      | 10 +++---
 docs/en/sql-reference/statements/move.md      | 32 +++++++++++++++++
 src/Access/IAccessStorage.cpp                 | 15 +++++---
 src/Access/MultipleAccessStorage.cpp          |  4 +--
 .../InterpreterMoveAccessEntityQuery.cpp      |  7 ++--
 .../test.py                                   | 11 +++---
 tests/integration/test_quota/test.py          | 36 +++++++++----------
 12 files changed, 81 insertions(+), 39 deletions(-)
 create mode 100644 docs/en/sql-reference/statements/move.md

diff --git a/docs/en/sql-reference/statements/create/quota.md b/docs/en/sql-reference/statements/create/quota.md
index c69285171ab..a6ced870c18 100644
--- a/docs/en/sql-reference/statements/create/quota.md
+++ b/docs/en/sql-reference/statements/create/quota.md
@@ -11,6 +11,7 @@ Syntax:
 
 ``` sql
 CREATE QUOTA [IF NOT EXISTS | OR REPLACE] name [ON CLUSTER cluster_name]
+    [IN access_storage_type]
     [KEYED BY {user_name | ip_address | client_key | client_key,user_name | client_key,ip_address} | NOT KEYED]
     [FOR [RANDOMIZED] INTERVAL number {second | minute | hour | day | week | month | quarter | year}
         {MAX { {queries | query_selects | query_inserts | errors | result_rows | result_bytes | read_rows | read_bytes | execution_time} = number } [,...] |
diff --git a/docs/en/sql-reference/statements/create/role.md b/docs/en/sql-reference/statements/create/role.md
index 9b14e220e1f..4b6fffe4f60 100644
--- a/docs/en/sql-reference/statements/create/role.md
+++ b/docs/en/sql-reference/statements/create/role.md
@@ -11,6 +11,7 @@ Syntax:
 
 ``` sql
 CREATE ROLE [IF NOT EXISTS | OR REPLACE] name1 [ON CLUSTER cluster_name1] [, name2 [ON CLUSTER cluster_name2] ...]
+    [IN access_storage_type]
     [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...]
 ```
 
diff --git a/docs/en/sql-reference/statements/create/row-policy.md b/docs/en/sql-reference/statements/create/row-policy.md
index 83bb2e6bb9a..cd7718793bd 100644
--- a/docs/en/sql-reference/statements/create/row-policy.md
+++ b/docs/en/sql-reference/statements/create/row-policy.md
@@ -16,6 +16,7 @@ Syntax:
 ``` sql
 CREATE [ROW] POLICY [IF NOT EXISTS | OR REPLACE] policy_name1 [ON CLUSTER cluster_name1] ON [db1.]table1|db1.*
         [, policy_name2 [ON CLUSTER cluster_name2] ON [db2.]table2|db2.* ...]
+    [IN access_storage_type]
     [FOR SELECT] USING condition
     [AS {PERMISSIVE | RESTRICTIVE}]
     [TO {role1 [, role2 ...] | ALL | ALL EXCEPT role1 [, role2 ...]}]
diff --git a/docs/en/sql-reference/statements/create/settings-profile.md b/docs/en/sql-reference/statements/create/settings-profile.md
index 8e221a4d82f..d8afce9d6de 100644
--- a/docs/en/sql-reference/statements/create/settings-profile.md
+++ b/docs/en/sql-reference/statements/create/settings-profile.md
@@ -12,6 +12,7 @@ Syntax:
 ``` sql
 CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] name1 [ON CLUSTER cluster_name1]
         [, name2 [ON CLUSTER cluster_name2] ...]
+    [IN access_storage_type]
     [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | INHERIT 'profile_name'] [,...]
 ```
 
diff --git a/docs/en/sql-reference/statements/create/user.md b/docs/en/sql-reference/statements/create/user.md
index d168be63c36..11d4eae7bc8 100644
--- a/docs/en/sql-reference/statements/create/user.md
+++ b/docs/en/sql-reference/statements/create/user.md
@@ -14,6 +14,7 @@ CREATE USER [IF NOT EXISTS | OR REPLACE] name1 [ON CLUSTER cluster_name1]
         [, name2 [ON CLUSTER cluster_name2] ...]
     [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name'}]
     [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
+    [IN access_storage_type]
     [DEFAULT ROLE role [,...]]
     [DEFAULT DATABASE database | NONE]
     [GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]]
diff --git a/docs/en/sql-reference/statements/drop.md b/docs/en/sql-reference/statements/drop.md
index b6208c2fd52..245fd22e57c 100644
--- a/docs/en/sql-reference/statements/drop.md
+++ b/docs/en/sql-reference/statements/drop.md
@@ -49,7 +49,7 @@ Deletes a user.
 Syntax:
 
 ``` sql
-DROP USER [IF EXISTS] name [,...] [ON CLUSTER cluster_name]
+DROP USER [IF EXISTS] name [,...] [ON CLUSTER cluster_name] [FROM access_storage_type]
 ```
 
 ## DROP ROLE
@@ -59,7 +59,7 @@ Deletes a role. The deleted role is revoked from all the entities where it was a
 Syntax:
 
 ``` sql
-DROP ROLE [IF EXISTS] name [,...] [ON CLUSTER cluster_name]
+DROP ROLE [IF EXISTS] name [,...] [ON CLUSTER cluster_name] [FROM access_storage_type]
 ```
 
 ## DROP ROW POLICY
@@ -69,7 +69,7 @@ Deletes a row policy. Deleted row policy is revoked from all the entities where
 Syntax:
 
 ``` sql
-DROP [ROW] POLICY [IF EXISTS] name [,...] ON [database.]table [,...] [ON CLUSTER cluster_name]
+DROP [ROW] POLICY [IF EXISTS] name [,...] ON [database.]table [,...] [ON CLUSTER cluster_name] [FROM access_storage_type]
 ```
 
 ## DROP QUOTA
@@ -79,7 +79,7 @@ Deletes a quota. The deleted quota is revoked from all the entities where it was
 Syntax:
 
 ``` sql
-DROP QUOTA [IF EXISTS] name [,...] [ON CLUSTER cluster_name]
+DROP QUOTA [IF EXISTS] name [,...] [ON CLUSTER cluster_name] [FROM access_storage_type]
 ```
 
 ## DROP SETTINGS PROFILE
@@ -89,7 +89,7 @@ Deletes a settings profile. The deleted settings profile is revoked from all the
 Syntax:
 
 ``` sql
-DROP [SETTINGS] PROFILE [IF EXISTS] name [,...] [ON CLUSTER cluster_name]
+DROP [SETTINGS] PROFILE [IF EXISTS] name [,...] [ON CLUSTER cluster_name] [FROM access_storage_type]
 ```
 
 ## DROP VIEW
diff --git a/docs/en/sql-reference/statements/move.md b/docs/en/sql-reference/statements/move.md
new file mode 100644
index 00000000000..fac738ff711
--- /dev/null
+++ b/docs/en/sql-reference/statements/move.md
@@ -0,0 +1,32 @@
+---
+slug: /en/sql-reference/statements/move
+sidebar_position: 54
+sidebar_label: MOVE
+---
+
+# MOVE access entity statement
+
+This statement allows to move an access entity from one access storage to another.
+
+Syntax:
+
+```sql
+MOVE {USER, ROLE, QUOTA, SETTINGS PROFILE, ROW POLICY} name1 [, name2, ...] TO access_storage_type
+```
+
+Currently, there are five access storages in ClickHouse:
+ - `local_directory`
+ - `memory`
+ - `replicated`
+ - `users_xml` (ro)
+ - `ldap` (ro)
+
+Examples:
+
+```sql
+MOVE USER test TO local_directory
+```
+
+```sql
+MOVE ROLE test TO memory
+```
diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp
index 468c93c1116..ec25fd505f0 100644
--- a/src/Access/IAccessStorage.cpp
+++ b/src/Access/IAccessStorage.cpp
@@ -202,16 +202,21 @@ std::vector<UUID> IAccessStorage::insert(const std::vector<AccessEntityPtr> & mu
 
 std::vector<UUID> IAccessStorage::insert(const std::vector<AccessEntityPtr> & multiple_entities, const std::vector<UUID> & ids, bool replace_if_exists, bool throw_if_exists)
 {
-    if (!ids.empty())
-        assert(multiple_entities.size() == ids.size());
+    assert(ids.empty() || (multiple_entities.size() == ids.size()));
 
     if (multiple_entities.empty())
         return {};
 
     if (multiple_entities.size() == 1)
     {
-        if (auto id = insert(multiple_entities[0], replace_if_exists, throw_if_exists))
-            return {*id};
+        UUID id;
+        if (!ids.empty())
+            id = ids[0];
+        else
+            id = generateRandomID();
+
+        if (insert(id, multiple_entities[0], replace_if_exists, throw_if_exists))
+            return {id};
         return {};
     }
 
@@ -229,7 +234,7 @@ std::vector<UUID> IAccessStorage::insert(const std::vector<AccessEntityPtr> & mu
             else
                 id = generateRandomID();
 
-            if (insertImpl(id, entity, replace_if_exists, throw_if_exists))
+            if (insert(id, entity, replace_if_exists, throw_if_exists))
             {
                 successfully_inserted.push_back(entity);
                 new_ids.push_back(id);
diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp
index e6665349ae6..c6c9ede9e1c 100644
--- a/src/Access/MultipleAccessStorage.cpp
+++ b/src/Access/MultipleAccessStorage.cpp
@@ -230,8 +230,8 @@ StoragePtr MultipleAccessStorage::findExcludingStorage(AccessEntityType type, co
 
 void MultipleAccessStorage::moveAccessEntities(const std::vector<UUID> & ids, const String & source_storage_name, const String & destination_storage_name)
 {
-    auto source_storage = findStorageByName(source_storage_name);
-    auto destination_storage = findStorageByName(destination_storage_name);
+    auto source_storage = getStorageByName(source_storage_name);
+    auto destination_storage = getStorageByName(destination_storage_name);
 
     auto to_move = source_storage->read(ids);
     source_storage->remove(ids);
diff --git a/src/Interpreters/Access/InterpreterMoveAccessEntityQuery.cpp b/src/Interpreters/Access/InterpreterMoveAccessEntityQuery.cpp
index ac7ff4efcb6..49e90783a59 100644
--- a/src/Interpreters/Access/InterpreterMoveAccessEntityQuery.cpp
+++ b/src/Interpreters/Access/InterpreterMoveAccessEntityQuery.cpp
@@ -28,12 +28,9 @@ BlockIO InterpreterMoveAccessEntityQuery::execute()
 
     std::vector<UUID> ids;
     if (query.type == AccessEntityType::ROW_POLICY)
-        ids = access_control.find(query.type, query.row_policy_names->toStrings());
+        ids = access_control.getIDs(query.type, query.row_policy_names->toStrings());
     else
-        ids = access_control.find(query.type, query.names);
-
-    if (ids.empty())
-        return {};
+        ids = access_control.getIDs(query.type, query.names);
 
     /// Validate that all entities are from the same storage.
     const auto source_storage = access_control.findStorage(ids.front());
diff --git a/tests/integration/test_multi_access_storage_role_management/test.py b/tests/integration/test_multi_access_storage_role_management/test.py
index 4c2696158ec..77aea411394 100644
--- a/tests/integration/test_multi_access_storage_role_management/test.py
+++ b/tests/integration/test_multi_access_storage_role_management/test.py
@@ -76,6 +76,13 @@ def execute_test_for_access_type(access_type: str, system_table_name: str):
     with pytest.raises(QueryRuntimeException):
         node.query(f"MOVE {access_type} test6 TO users_xml")
 
+    node.query(f"DROP {access_type} test1")
+    node.query(f"DROP {access_type} test2")
+    node.query(f"DROP {access_type} test3")
+    node.query(f"DROP {access_type} test4")
+    node.query(f"DROP {access_type} test5")
+    node.query(f"DROP {access_type} test6")
+
 
 def test_roles():
     execute_test_for_access_type("ROLE", "roles")
@@ -93,10 +100,6 @@ def test_quotas():
     execute_test_for_access_type("QUOTA", "quotas")
 
 
-def test_row_policies():
-    execute_test_for_access_type("ROW POLICY", "row_policies")
-
-
 def test_role_from_different_storages():
     node.query("CREATE ROLE default_role")
     node.query("GRANT SELECT ON system.* TO default_role")
diff --git a/tests/integration/test_quota/test.py b/tests/integration/test_quota/test.py
index 4f98b9a0d0d..cec14b0af73 100644
--- a/tests/integration/test_quota/test.py
+++ b/tests/integration/test_quota/test.py
@@ -105,7 +105,7 @@ def test_quota_from_users_xml():
             [
                 "myQuota",
                 "e651da9c-a748-8703-061a-7e5e5096dae7",
-                "users.xml",
+                "users_xml",
                 "['user_name']",
                 [31556952],
                 0,
@@ -252,7 +252,7 @@ def test_simpliest_quota():
             [
                 "myQuota",
                 "e651da9c-a748-8703-061a-7e5e5096dae7",
-                "users.xml",
+                "users_xml",
                 "['user_name']",
                 "[]",
                 0,
@@ -326,7 +326,7 @@ def test_tracking_quota():
             [
                 "myQuota",
                 "e651da9c-a748-8703-061a-7e5e5096dae7",
-                "users.xml",
+                "users_xml",
                 "['user_name']",
                 "[31556952]",
                 0,
@@ -446,7 +446,7 @@ def test_exceed_quota():
             [
                 "myQuota",
                 "e651da9c-a748-8703-061a-7e5e5096dae7",
-                "users.xml",
+                "users_xml",
                 "['user_name']",
                 "[31556952]",
                 0,
@@ -523,7 +523,7 @@ def test_exceed_quota():
             [
                 "myQuota",
                 "e651da9c-a748-8703-061a-7e5e5096dae7",
-                "users.xml",
+                "users_xml",
                 "['user_name']",
                 "[31556952]",
                 0,
@@ -613,7 +613,7 @@ def test_add_remove_interval():
             [
                 "myQuota",
                 "e651da9c-a748-8703-061a-7e5e5096dae7",
-                "users.xml",
+                "users_xml",
                 "['user_name']",
                 [31556952],
                 0,
@@ -675,7 +675,7 @@ def test_add_remove_interval():
             [
                 "myQuota",
                 "e651da9c-a748-8703-061a-7e5e5096dae7",
-                "users.xml",
+                "users_xml",
                 "['user_name']",
                 "[31556952,63113904]",
                 0,
@@ -824,7 +824,7 @@ def test_add_remove_interval():
             [
                 "myQuota",
                 "e651da9c-a748-8703-061a-7e5e5096dae7",
-                "users.xml",
+                "users_xml",
                 "['user_name']",
                 [31556952],
                 0,
@@ -914,7 +914,7 @@ def test_add_remove_interval():
             [
                 "myQuota",
                 "e651da9c-a748-8703-061a-7e5e5096dae7",
-                "users.xml",
+                "users_xml",
                 "['user_name']",
                 "[]",
                 0,
@@ -986,7 +986,7 @@ def test_add_remove_interval():
             [
                 "myQuota",
                 "e651da9c-a748-8703-061a-7e5e5096dae7",
-                "users.xml",
+                "users_xml",
                 "['user_name']",
                 [31556952],
                 0,
@@ -1048,7 +1048,7 @@ def test_add_remove_quota():
             [
                 "myQuota",
                 "e651da9c-a748-8703-061a-7e5e5096dae7",
-                "users.xml",
+                "users_xml",
                 "['user_name']",
                 [31556952],
                 0,
@@ -1111,7 +1111,7 @@ def test_add_remove_quota():
             [
                 "myQuota",
                 "e651da9c-a748-8703-061a-7e5e5096dae7",
-                "users.xml",
+                "users_xml",
                 "['user_name']",
                 "[31556952]",
                 0,
@@ -1121,7 +1121,7 @@ def test_add_remove_quota():
             [
                 "myQuota2",
                 "4590510c-4d13-bf21-ec8a-c2187b092e73",
-                "users.xml",
+                "users_xml",
                 "['client_key','user_name']",
                 "[3600,2629746]",
                 0,
@@ -1214,7 +1214,7 @@ def test_add_remove_quota():
             [
                 "myQuota",
                 "e651da9c-a748-8703-061a-7e5e5096dae7",
-                "users.xml",
+                "users_xml",
                 "['user_name']",
                 "[31556952]",
                 0,
@@ -1283,7 +1283,7 @@ def test_add_remove_quota():
             [
                 "myQuota",
                 "e651da9c-a748-8703-061a-7e5e5096dae7",
-                "users.xml",
+                "users_xml",
                 "['user_name']",
                 "[31556952]",
                 0,
@@ -1346,7 +1346,7 @@ def test_reload_users_xml_by_timer():
             [
                 "myQuota",
                 "e651da9c-a748-8703-061a-7e5e5096dae7",
-                "users.xml",
+                "users_xml",
                 "['user_name']",
                 "[31556952]",
                 0,
@@ -1385,7 +1385,7 @@ def test_reload_users_xml_by_timer():
             [
                 "myQuota",
                 "e651da9c-a748-8703-061a-7e5e5096dae7",
-                "users.xml",
+                "users_xml",
                 ["user_name"],
                 "[31556952]",
                 0,
@@ -1554,7 +1554,7 @@ def test_query_inserts():
             [
                 "myQuota",
                 "e651da9c-a748-8703-061a-7e5e5096dae7",
-                "users.xml",
+                "users_xml",
                 "['user_name']",
                 [31556952],
                 0,

From f3a31c20fe8fd4a0ba57586bd36828af8581cf5d Mon Sep 17 00:00:00 2001
From: pufit <pufit@clickhouse.com>
Date: Fri, 28 Jul 2023 10:53:02 -0400
Subject: [PATCH 039/249] Update test_row_policy.py

---
 .../test_row_policy.py                           | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/tests/integration/test_disabled_access_control_improvements/test_row_policy.py b/tests/integration/test_disabled_access_control_improvements/test_row_policy.py
index 509b4de1a37..b620e88e7eb 100644
--- a/tests/integration/test_disabled_access_control_improvements/test_row_policy.py
+++ b/tests/integration/test_disabled_access_control_improvements/test_row_policy.py
@@ -76,7 +76,7 @@ def test_introspection():
             "mydb",
             "filtered_table1",
             "6068883a-0e9d-f802-7e22-0144f8e66d3c",
-            "users.xml",
+            "users_xml",
             "1",
             0,
             0,
@@ -89,7 +89,7 @@ def test_introspection():
             "mydb",
             "filtered_table2",
             "c019e957-c60b-d54e-cc52-7c90dac5fb01",
-            "users.xml",
+            "users_xml",
             "1",
             0,
             0,
@@ -102,7 +102,7 @@ def test_introspection():
             "mydb",
             "filtered_table3",
             "4cb080d0-44e8-dbef-6026-346655143628",
-            "users.xml",
+            "users_xml",
             "1",
             0,
             0,
@@ -115,7 +115,7 @@ def test_introspection():
             "mydb",
             "local",
             "5b23c389-7e18-06bf-a6bc-dd1afbbc0a97",
-            "users.xml",
+            "users_xml",
             "a = 1",
             0,
             0,
@@ -128,7 +128,7 @@ def test_introspection():
             "mydb",
             "filtered_table1",
             "9e8a8f62-4965-2b5e-8599-57c7b99b3549",
-            "users.xml",
+            "users_xml",
             "a = 1",
             0,
             0,
@@ -141,7 +141,7 @@ def test_introspection():
             "mydb",
             "filtered_table2",
             "cffae79d-b9bf-a2ef-b798-019c18470b25",
-            "users.xml",
+            "users_xml",
             "a + b < 1 or c - d > 5",
             0,
             0,
@@ -154,7 +154,7 @@ def test_introspection():
             "mydb",
             "filtered_table3",
             "12fc5cef-e3da-3940-ec79-d8be3911f42b",
-            "users.xml",
+            "users_xml",
             "c = 1",
             0,
             0,
@@ -167,7 +167,7 @@ def test_introspection():
             "mydb",
             "local",
             "cdacaeb5-1d97-f99d-2bb0-4574f290629c",
-            "users.xml",
+            "users_xml",
             "1",
             0,
             0,

From f42cfddee7966496836da44e26d27fe208d43ac9 Mon Sep 17 00:00:00 2001
From: pufit <pufit@clickhouse.com>
Date: Fri, 28 Jul 2023 20:28:56 -0400
Subject: [PATCH 040/249] Update test.py

---
 tests/integration/test_row_policy/test.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/integration/test_row_policy/test.py b/tests/integration/test_row_policy/test.py
index 1933823f5d2..acf544c98aa 100644
--- a/tests/integration/test_row_policy/test.py
+++ b/tests/integration/test_row_policy/test.py
@@ -420,7 +420,7 @@ def test_introspection():
             "mydb",
             "local",
             "5b23c389-7e18-06bf-a6bc-dd1afbbc0a97",
-            "users.xml",
+            "users_xml",
             "a = 1",
             0,
             0,
@@ -433,7 +433,7 @@ def test_introspection():
             "mydb",
             "filtered_table1",
             "9e8a8f62-4965-2b5e-8599-57c7b99b3549",
-            "users.xml",
+            "users_xml",
             "a = 1",
             0,
             0,
@@ -446,7 +446,7 @@ def test_introspection():
             "mydb",
             "filtered_table2",
             "cffae79d-b9bf-a2ef-b798-019c18470b25",
-            "users.xml",
+            "users_xml",
             "a + b < 1 or c - d > 5",
             0,
             0,
@@ -459,7 +459,7 @@ def test_introspection():
             "mydb",
             "filtered_table3",
             "12fc5cef-e3da-3940-ec79-d8be3911f42b",
-            "users.xml",
+            "users_xml",
             "c = 1",
             0,
             0,

From 1ee1ae120e921a355a752b332aae25b18b1cf89d Mon Sep 17 00:00:00 2001
From: Yakov Olkhovskiy <yakov@clickhouse.com>
Date: Sat, 29 Jul 2023 02:03:21 +0000
Subject: [PATCH 041/249] add modulo, intDiv, intDivOrZero for tuple

---
 src/Functions/FunctionBinaryArithmetic.h      | 24 ++++++++++++++++---
 src/Functions/IsOperation.h                   |  2 +-
 src/Functions/vectorFunctions.cpp             | 21 ++++++++++++++++
 .../0_stateless/02841_tuple_modulo.reference  |  4 ++++
 .../0_stateless/02841_tuple_modulo.sql        |  4 ++++
 5 files changed, 51 insertions(+), 4 deletions(-)
 create mode 100644 tests/queries/0_stateless/02841_tuple_modulo.reference
 create mode 100644 tests/queries/0_stateless/02841_tuple_modulo.sql

diff --git a/src/Functions/FunctionBinaryArithmetic.h b/src/Functions/FunctionBinaryArithmetic.h
index bf3b33d13ff..408a16236e8 100644
--- a/src/Functions/FunctionBinaryArithmetic.h
+++ b/src/Functions/FunctionBinaryArithmetic.h
@@ -742,6 +742,9 @@ class FunctionBinaryArithmetic : public IFunction
     static constexpr bool is_multiply = IsOperation<Op>::multiply;
     static constexpr bool is_division = IsOperation<Op>::division;
     static constexpr bool is_bit_hamming_distance = IsOperation<Op>::bit_hamming_distance;
+    static constexpr bool is_modulo = IsOperation<Op>::modulo;
+    static constexpr bool is_div_int = IsOperation<Op>::div_int;
+    static constexpr bool is_div_int_or_zero = IsOperation<Op>::div_int_or_zero;
 
     ContextPtr context;
     bool check_decimal_overflow = true;
@@ -951,13 +954,28 @@ class FunctionBinaryArithmetic : public IFunction
                                                                   "argument of numeric type cannot be first", name);
 
         std::string function_name;
-        if (is_multiply)
+        if constexpr (is_multiply)
         {
             function_name = "tupleMultiplyByNumber";
         }
-        else
+        else // is_division
         {
-            function_name = "tupleDivideByNumber";
+            if constexpr (is_modulo)
+            {
+                function_name = "tupleModuloByNumber";
+            }
+            else if constexpr (is_div_int)
+            {
+                function_name = "tupleIntDivByNumber";
+            }
+            else if constexpr (is_div_int_or_zero)
+            {
+                function_name = "tupleIntDivOrZeroByNumber";
+            }
+            else
+            {
+                function_name = "tupleDivideByNumber";
+            }
         }
 
         return FunctionFactory::instance().get(function_name, context);
diff --git a/src/Functions/IsOperation.h b/src/Functions/IsOperation.h
index 0c54901579e..8ea53c865ce 100644
--- a/src/Functions/IsOperation.h
+++ b/src/Functions/IsOperation.h
@@ -60,7 +60,7 @@ struct IsOperation
 
     static constexpr bool bit_hamming_distance = IsSameOperation<Op, BitHammingDistanceImpl>::value;
 
-    static constexpr bool division = div_floating || div_int || div_int_or_zero;
+    static constexpr bool division = div_floating || div_int || div_int_or_zero || modulo;
 
     static constexpr bool allow_decimal = plus || minus || multiply || division || least || greatest;
 };
diff --git a/src/Functions/vectorFunctions.cpp b/src/Functions/vectorFunctions.cpp
index d53d39e2f3b..35ba49e4545 100644
--- a/src/Functions/vectorFunctions.cpp
+++ b/src/Functions/vectorFunctions.cpp
@@ -23,6 +23,9 @@ struct PlusName { static constexpr auto name = "plus"; };
 struct MinusName { static constexpr auto name = "minus"; };
 struct MultiplyName { static constexpr auto name = "multiply"; };
 struct DivideName { static constexpr auto name = "divide"; };
+struct ModuloName { static constexpr auto name = "modulo"; };
+struct IntDivName { static constexpr auto name = "intDiv"; };
+struct IntDivOrZeroName { static constexpr auto name = "intDivOrZero"; };
 
 struct L1Label { static constexpr auto name = "1"; };
 struct L2Label { static constexpr auto name = "2"; };
@@ -141,6 +144,12 @@ using FunctionTupleMultiply = FunctionTupleOperator<MultiplyName>;
 
 using FunctionTupleDivide = FunctionTupleOperator<DivideName>;
 
+using FunctionTupleModulo = FunctionTupleOperator<ModuloName>;
+
+using FunctionTupleIntDiv = FunctionTupleOperator<IntDivName>;
+
+using FunctionTupleIntDivOrZero = FunctionTupleOperator<IntDivOrZeroName>;
+
 class FunctionTupleNegate : public ITupleFunction
 {
 public:
@@ -297,6 +306,12 @@ using FunctionTupleMultiplyByNumber = FunctionTupleOperatorByNumber<MultiplyName
 
 using FunctionTupleDivideByNumber = FunctionTupleOperatorByNumber<DivideName>;
 
+using FunctionTupleModuloByNumber = FunctionTupleOperatorByNumber<ModuloName>;
+
+using FunctionTupleIntDivByNumber = FunctionTupleOperatorByNumber<IntDivName>;
+
+using FunctionTupleIntDivOrZeroByNumber = FunctionTupleOperatorByNumber<IntDivOrZeroName>;
+
 class FunctionDotProduct : public ITupleFunction
 {
 public:
@@ -1563,6 +1578,9 @@ REGISTER_FUNCTION(VectorFunctions)
     factory.registerAlias("vectorDifference", FunctionTupleMinus::name, FunctionFactory::CaseInsensitive);
     factory.registerFunction<FunctionTupleMultiply>();
     factory.registerFunction<FunctionTupleDivide>();
+    factory.registerFunction<FunctionTupleModulo>();
+    factory.registerFunction<FunctionTupleIntDiv>();
+    factory.registerFunction<FunctionTupleIntDivOrZero>();
     factory.registerFunction<FunctionTupleNegate>();
 
     factory.registerFunction<FunctionAddTupleOfIntervals>(FunctionDocumentation
@@ -1626,6 +1644,9 @@ If the types of the first interval (or the interval in the tuple) and the second
 
     factory.registerFunction<FunctionTupleMultiplyByNumber>();
     factory.registerFunction<FunctionTupleDivideByNumber>();
+    factory.registerFunction<FunctionTupleModuloByNumber>();
+    factory.registerFunction<FunctionTupleIntDivByNumber>();
+    factory.registerFunction<FunctionTupleIntDivOrZeroByNumber>();
 
     factory.registerFunction<TupleOrArrayFunctionDotProduct>();
     factory.registerAlias("scalarProduct", TupleOrArrayFunctionDotProduct::name, FunctionFactory::CaseInsensitive);
diff --git a/tests/queries/0_stateless/02841_tuple_modulo.reference b/tests/queries/0_stateless/02841_tuple_modulo.reference
new file mode 100644
index 00000000000..6e6f07d0683
--- /dev/null
+++ b/tests/queries/0_stateless/02841_tuple_modulo.reference
@@ -0,0 +1,4 @@
+(1,0)
+(2,2)
+(2,2)
+(0,0)
diff --git a/tests/queries/0_stateless/02841_tuple_modulo.sql b/tests/queries/0_stateless/02841_tuple_modulo.sql
new file mode 100644
index 00000000000..56bacf87967
--- /dev/null
+++ b/tests/queries/0_stateless/02841_tuple_modulo.sql
@@ -0,0 +1,4 @@
+SELECT (5,4) % 2;
+SELECT intDiv((5,4), 2);
+SELECT intDivOrZero((5,4), 2);
+SELECT intDivOrZero((5,4), 0);

From 6f2404b79209bf332e54e83b86eff2f677bbd2bc Mon Sep 17 00:00:00 2001
From: pufit <pufit@clickhouse.com>
Date: Fri, 28 Jul 2023 22:18:53 -0400
Subject: [PATCH 042/249] Additional info in exception

---
 src/Access/MultipleAccessStorage.cpp | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp
index c6c9ede9e1c..0550c140c17 100644
--- a/src/Access/MultipleAccessStorage.cpp
+++ b/src/Access/MultipleAccessStorage.cpp
@@ -234,17 +234,32 @@ void MultipleAccessStorage::moveAccessEntities(const std::vector<UUID> & ids, co
     auto destination_storage = getStorageByName(destination_storage_name);
 
     auto to_move = source_storage->read(ids);
-    source_storage->remove(ids);
+    bool need_rollback = false;
 
     try
     {
+        source_storage->remove(ids);
+        need_rollback = true;
         destination_storage->insert(to_move, ids);
     }
     catch (Exception & e)
     {
-        e.addMessage("while moving access entities");
+        String message;
+
+        bool need_comma = false;
+        for (const auto & entity : to_move)
+        {
+            if (std::exchange(need_comma, true))
+                message += ", ";
+
+            message += entity->formatTypeWithName();
+        }
+
+        e.addMessage("while moving {} from {} to {}", message, source_storage_name, destination_storage_name);
+
+        if (need_rollback)
+            source_storage->insert(to_move, ids);
 
-        source_storage->insert(to_move, ids);
         throw;
     }
 }

From a64d6b11f29c3eac5eedf4e48fc118510d4e2a01 Mon Sep 17 00:00:00 2001
From: Yakov Olkhovskiy <yakov@clickhouse.com>
Date: Sat, 29 Jul 2023 04:55:19 +0000
Subject: [PATCH 043/249] add functions to undocumented

---
 .../02415_all_new_functions_must_be_documented.reference    | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference
index 61a2e4e9f02..b57372ffa8d 100644
--- a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference
+++ b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference
@@ -885,7 +885,13 @@ tupleDivide
 tupleDivideByNumber
 tupleElement
 tupleHammingDistance
+tupleIntDiv
+tupleIntDivByNumber
+tupleIntDivOrZero
+tupleIntDivOrZeroByNumber
 tupleMinus
+tupleModulo
+tupleModuloByNumber
 tupleMultiply
 tupleMultiplyByNumber
 tupleNegate

From 12069b2cff6c4ae77cc674f02ce4e04987c9515b Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 29 Jul 2023 23:25:44 +0200
Subject: [PATCH 044/249] Allow yaml configs in clickhouse-client

---
 src/Common/Config/configReadClient.cpp | 51 ++++++++++++++++++--------
 1 file changed, 36 insertions(+), 15 deletions(-)

diff --git a/src/Common/Config/configReadClient.cpp b/src/Common/Config/configReadClient.cpp
index 44d338c07af..fea055618bb 100644
--- a/src/Common/Config/configReadClient.cpp
+++ b/src/Common/Config/configReadClient.cpp
@@ -6,36 +6,57 @@
 #include <iostream>
 #include <base/types.h>
 
+
 namespace fs = std::filesystem;
 
 namespace DB
 {
 
-bool safeFsExists(const String & path)
-{
-    std::error_code ec;
-    return fs::exists(path, ec);
-}
-
 bool configReadClient(Poco::Util::LayeredConfiguration & config, const std::string & home_path)
 {
     std::string config_path;
-    if (config.has("config-file"))
-        config_path = config.getString("config-file");
-    else if (safeFsExists("./clickhouse-client.xml"))
-        config_path = "./clickhouse-client.xml";
-    else if (!home_path.empty() && safeFsExists(home_path + "/.clickhouse-client/config.xml"))
-        config_path = home_path + "/.clickhouse-client/config.xml";
-    else if (safeFsExists("/etc/clickhouse-client/config.xml"))
-        config_path = "/etc/clickhouse-client/config.xml";
 
-    if (!config_path.empty())
+    bool found = false;
+    if (config.has("config-file"))
+    {
+        found = true;
+        config_path = config.getString("config-file");
+    }
+    else
+    {
+        std::vector<std::string> names;
+        names.emplace_back("./clickhouse-client");
+        if (!home_path.empty())
+            names.emplace_back(home_path + "/.clickhouse-client/config");
+        names.emplace_back("/etc/clickhouse-client/config");
+
+        for (const auto & name : names)
+        {
+            for (const auto & extension : {".xml", ".yaml", ".yml"})
+            {
+                config_path = name + extension;
+
+                std::error_code ec;
+                if (fs::exists(config_path, ec))
+                {
+                    found = true;
+                    break;
+                }
+            }
+            if (found)
+                break;
+        }
+    }
+
+    if (found)
     {
         ConfigProcessor config_processor(config_path);
         auto loaded_config = config_processor.loadConfig();
         config.add(loaded_config.configuration);
         return true;
     }
+
     return false;
 }
+
 }

From fdfe64f1b0330522180811b83e8db8436839c829 Mon Sep 17 00:00:00 2001
From: taiyang-li <654010905@qq.com>
Date: Mon, 31 Jul 2023 12:07:50 +0800
Subject: [PATCH 045/249] change as request

---
 .../functions/string-functions.md             | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md
index b1ffb32e7cd..fff37257917 100644
--- a/docs/en/sql-reference/functions/string-functions.md
+++ b/docs/en/sql-reference/functions/string-functions.md
@@ -706,7 +706,15 @@ endsWithUTF8(str, suffix)
 **Example**
 
 ``` sql
-SELECT endsWithUTF8('中国', '国');
+SELECT endsWithUTF8('中国', '\xbd'), endsWith('中国', '\xbd')
+```
+
+Result:
+
+```result
+┌─endsWithUTF8('中国', '½')─┬─endsWith('中国', '½')─┐
+│                        0 │                    1 │
+└──────────────────────────┴──────────────────────┘
 ```
 
 ## startsWith
@@ -733,9 +741,16 @@ Returns whether string `str` starts with `prefix`, the difference between `start
 **Example**
 
 ``` sql
-SELECT startsWithUTF8('中国', '中');
+SELECT startsWithUTF8('中国', '\xe4'), startsWith('中国', '\xe4')
 ```
 
+Result:
+
+```result
+┌─startsWithUTF8('中国', '⥩─┬─startsWith('中国', '⥩─┐
+│                          0 │                      1 │
+└────────────────────────────┴────────────────────────┘
+```
 
 ## trim
 

From fb8502ba7671f75cb420ab5937cced224f5be7c6 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Mon, 31 Jul 2023 12:20:27 +0000
Subject: [PATCH 046/249] do not test upper bounds for throttlers

---
 tests/integration/test_throttling/test.py | 45 ++++++++++++-----------
 1 file changed, 23 insertions(+), 22 deletions(-)

diff --git a/tests/integration/test_throttling/test.py b/tests/integration/test_throttling/test.py
index 62640394a85..526c34ac916 100644
--- a/tests/integration/test_throttling/test.py
+++ b/tests/integration/test_throttling/test.py
@@ -117,7 +117,8 @@ def assert_took(took, should_took):
     # we need to decrease the lower limit because the server limits could
     # be enforced by throttling some server background IO instead of query IO
     # and we have no control over it
-    assert took >= should_took[0] * 0.85 and took < should_took[1]
+    # Note that throttler does not apply any restrictions on upper bound, so we can only tell how much time required "at least", not "at most"
+    assert took >= should_took * 0.85
 
 
 @pytest.mark.parametrize(
@@ -132,7 +133,7 @@ def assert_took(took, should_took):
             None,
             None,
             None,
-            (0, 3),
+            0,
             id="no_local_throttling",
         ),
         # reading 1e6*8 bytes with 1M default bandwith should take (8-1)/1=7 seconds
@@ -142,7 +143,7 @@ def assert_took(took, should_took):
             "user",
             "max_backup_bandwidth",
             "1M",
-            (7, 14),
+            7,
             id="user_local_throttling",
         ),
         # reading 1e6*8 bytes with 2M default bandwith should take (8-2)/2=3 seconds
@@ -152,7 +153,7 @@ def assert_took(took, should_took):
             "server",
             "max_backup_bandwidth_for_server",
             "2M",
-            (3, 7),
+            3,
             id="server_local_throttling",
         ),
         #
@@ -164,7 +165,7 @@ def assert_took(took, should_took):
             None,
             None,
             None,
-            (0, 3),
+            0,
             id="no_remote_to_local_throttling",
         ),
         # reading 1e6*8 bytes with 1M default bandwith should take (8-1)/1=7 seconds
@@ -184,7 +185,7 @@ def assert_took(took, should_took):
             "server",
             "max_backup_bandwidth_for_server",
             "2M",
-            (3, 7),
+            3,
             id="server_remote_to_local_throttling",
         ),
         #
@@ -196,7 +197,7 @@ def assert_took(took, should_took):
             None,
             None,
             None,
-            (0, 3),
+            0,
             id="no_remote_to_remote_throttling",
         ),
         # No throttling for S3-to-S3, uses native copy
@@ -206,7 +207,7 @@ def assert_took(took, should_took):
             "user",
             "max_backup_bandwidth",
             "1M",
-            (0, 3),
+            0,
             id="user_remote_to_remote_throttling",
         ),
         # No throttling for S3-to-S3, uses native copy
@@ -216,7 +217,7 @@ def assert_took(took, should_took):
             "server",
             "max_backup_bandwidth_for_server",
             "2M",
-            (0, 3),
+            0,
             id="server_remote_to_remote_throttling",
         ),
         #
@@ -233,7 +234,7 @@ def assert_took(took, should_took):
             None,
             None,
             None,
-            (0, 3),
+            0,
             id="no_local_to_remote_throttling",
         ),
         # reading 1e6*8 bytes with 1M default bandwith should take (8-1)/1=7 seconds, but for S3Client it is 2x more
@@ -243,7 +244,7 @@ def assert_took(took, should_took):
             "user",
             "max_backup_bandwidth",
             "1M",
-            (7 * 3, 7 * 4 - 1),
+            7 * 3,
             id="user_local_to_remote_throttling",
         ),
         # reading 1e6*8 bytes with 2M default bandwith should take (8-2)/2=3 seconds, but for S3Client it is 2x more
@@ -253,7 +254,7 @@ def assert_took(took, should_took):
             "server",
             "max_backup_bandwidth_for_server",
             "2M",
-            (3 * 3, 3 * 5),
+            3 * 3,
             id="server_local_to_remote_throttling",
         ),
     ],
@@ -306,7 +307,7 @@ def test_backup_throttling_override():
             "user",
             "max_local_read_bandwidth",
             "1M",
-            (7, 14),
+            7,
             id="user_local_throttling",
         ),
         # reading 1e6*8 bytes with 2M default bandwith should take (8-2)/2=3 seconds
@@ -315,7 +316,7 @@ def test_backup_throttling_override():
             "server",
             "max_local_read_bandwidth_for_server",
             "2M",
-            (3, 7),
+            3,
             id="server_local_throttling",
         ),
         #
@@ -328,7 +329,7 @@ def test_backup_throttling_override():
             "user",
             "max_remote_read_network_bandwidth",
             "1M",
-            (7, 14),
+            7,
             id="user_remote_throttling",
         ),
         # reading 1e6*8 bytes with 2M default bandwith should take (8-2)/2=3 seconds
@@ -337,7 +338,7 @@ def test_backup_throttling_override():
             "server",
             "max_remote_read_network_bandwidth_for_server",
             "2M",
-            (3, 7),
+            3,
             id="server_remote_throttling",
         ),
     ],
@@ -368,7 +369,7 @@ def test_read_throttling(policy, mode, setting, value, should_took):
             "user",
             "max_local_write_bandwidth",
             "1M",
-            (7, 14),
+            7,
             id="local_user_throttling",
         ),
         # reading 1e6*8 bytes with 2M default bandwith should take (8-2)/2=3 seconds
@@ -377,29 +378,29 @@ def test_read_throttling(policy, mode, setting, value, should_took):
             "server",
             "max_local_write_bandwidth_for_server",
             "2M",
-            (3, 7),
+            3,
             id="local_server_throttling",
         ),
         #
         # Remote
         #
         pytest.param("s3", None, None, None, (0, 3), id="no_remote_throttling"),
-        # writeing 1e6*8 bytes with 1M default bandwith should take (8-1)/1=7 seconds
+        # writing 1e6*8 bytes with 1M default bandwith should take (8-1)/1=7 seconds
         pytest.param(
             "s3",
             "user",
             "max_remote_write_network_bandwidth",
             "1M",
-            (7, 14),
+            7,
             id="user_remote_throttling",
         ),
-        # writeing 1e6*8 bytes with 2M default bandwith should take (8-2)/2=3 seconds
+        # writing 1e6*8 bytes with 2M default bandwith should take (8-2)/2=3 seconds
         pytest.param(
             "s3",
             "server",
             "max_remote_write_network_bandwidth_for_server",
             "2M",
-            (3, 7),
+            3,
             id="server_remote_throttling",
         ),
     ],

From a0ff04e0e7cc7bd911d80a01aee34846ec26bb3c Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 1 Aug 2023 08:03:09 +0200
Subject: [PATCH 047/249] Add a test

---
 .../02834_client_yaml_configs.reference       |  3 +++
 .../0_stateless/02834_client_yaml_configs.sh  | 21 +++++++++++++++++++
 2 files changed, 24 insertions(+)
 create mode 100644 tests/queries/0_stateless/02834_client_yaml_configs.reference
 create mode 100755 tests/queries/0_stateless/02834_client_yaml_configs.sh

diff --git a/tests/queries/0_stateless/02834_client_yaml_configs.reference b/tests/queries/0_stateless/02834_client_yaml_configs.reference
new file mode 100644
index 00000000000..302360f2570
--- /dev/null
+++ b/tests/queries/0_stateless/02834_client_yaml_configs.reference
@@ -0,0 +1,3 @@
+31337
+31337
+31337
diff --git a/tests/queries/0_stateless/02834_client_yaml_configs.sh b/tests/queries/0_stateless/02834_client_yaml_configs.sh
new file mode 100755
index 00000000000..f17186328b4
--- /dev/null
+++ b/tests/queries/0_stateless/02834_client_yaml_configs.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CUR_DIR"/../shell_config.sh
+
+pushd "${CLICKHOUSE_TMP}" > /dev/null
+
+echo "max_block_size: 31337" > clickhouse-client.yaml
+${CLICKHOUSE_CLIENT} --query "SELECT getSetting('max_block_size')"
+rm clickhouse-client.yaml
+
+echo "max_block_size: 31337" > clickhouse-client.yml
+${CLICKHOUSE_CLIENT} --query "SELECT getSetting('max_block_size')"
+rm clickhouse-client.yml
+
+echo "<clickhouse><max_block_size>31337</max_block_size></clickhouse>" > clickhouse-client.xml
+${CLICKHOUSE_CLIENT} --query "SELECT getSetting('max_block_size')"
+rm clickhouse-client.xml
+
+popd > /dev/null

From 628786e7eb6a55d5f5b2881a0fc34c3edcf7d0e8 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 1 Aug 2023 10:52:06 +0300
Subject: [PATCH 048/249] Update 02834_client_yaml_configs.sh

---
 tests/queries/0_stateless/02834_client_yaml_configs.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/queries/0_stateless/02834_client_yaml_configs.sh b/tests/queries/0_stateless/02834_client_yaml_configs.sh
index f17186328b4..5a6d26808a3 100755
--- a/tests/queries/0_stateless/02834_client_yaml_configs.sh
+++ b/tests/queries/0_stateless/02834_client_yaml_configs.sh
@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+# Tags: no-fasttest
 
 CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

From 8b37abfa598b80bf9a1281d519f09d953e18523e Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 1 Aug 2023 11:16:00 +0300
Subject: [PATCH 049/249] Update 02834_client_yaml_configs.sh

---
 tests/queries/0_stateless/02834_client_yaml_configs.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/02834_client_yaml_configs.sh b/tests/queries/0_stateless/02834_client_yaml_configs.sh
index 5a6d26808a3..dbb40d33e0a 100755
--- a/tests/queries/0_stateless/02834_client_yaml_configs.sh
+++ b/tests/queries/0_stateless/02834_client_yaml_configs.sh
@@ -5,7 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 
-pushd "${CLICKHOUSE_TMP}" > /dev/null
+pushd "${CLICKHOUSE_TMP}" || exit > /dev/null
 
 echo "max_block_size: 31337" > clickhouse-client.yaml
 ${CLICKHOUSE_CLIENT} --query "SELECT getSetting('max_block_size')"
@@ -19,4 +19,4 @@ echo "<clickhouse><max_block_size>31337</max_block_size></clickhouse>" > clickho
 ${CLICKHOUSE_CLIENT} --query "SELECT getSetting('max_block_size')"
 rm clickhouse-client.xml
 
-popd > /dev/null
+popd || exit > /dev/null

From 513d3fd3c1c1484cc908987f135f153f59d36405 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Tue, 1 Aug 2023 09:15:35 +0000
Subject: [PATCH 050/249] fix more places

---
 tests/integration/test_throttling/test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/integration/test_throttling/test.py b/tests/integration/test_throttling/test.py
index 526c34ac916..91b472513c7 100644
--- a/tests/integration/test_throttling/test.py
+++ b/tests/integration/test_throttling/test.py
@@ -175,7 +175,7 @@ def assert_took(took, should_took):
             "user",
             "max_backup_bandwidth",
             "1M",
-            (7, 14),
+            7,
             id="user_remote_to_local_throttling",
         ),
         # reading 1e6*8 bytes with 2M default bandwith should take (8-2)/2=3 seconds
@@ -291,7 +291,7 @@ def test_backup_throttling_override():
         },
     )
     # reading 1e6*8 bytes with 500Ki default bandwith should take (8-0.5)/0.5=15 seconds
-    assert_took(took, (15, 20))
+    assert_took(took, 15)
 
 
 @pytest.mark.parametrize(

From 135790c0d6caed1075399cab69a5036c71860cd8 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Tue, 1 Aug 2023 09:44:08 +0000
Subject: [PATCH 051/249] more fixes

---
 tests/integration/test_throttling/test.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/integration/test_throttling/test.py b/tests/integration/test_throttling/test.py
index 91b472513c7..a27bb472ea8 100644
--- a/tests/integration/test_throttling/test.py
+++ b/tests/integration/test_throttling/test.py
@@ -300,7 +300,7 @@ def test_backup_throttling_override():
         #
         # Local
         #
-        pytest.param("default", None, None, None, (0, 3), id="no_local_throttling"),
+        pytest.param("default", None, None, None, 0, id="no_local_throttling"),
         # reading 1e6*8 bytes with 1M default bandwith should take (8-1)/1=7 seconds
         pytest.param(
             "default",
@@ -322,7 +322,7 @@ def test_backup_throttling_override():
         #
         # Remote
         #
-        pytest.param("s3", None, None, None, (0, 3), id="no_remote_throttling"),
+        pytest.param("s3", None, None, None, 0, id="no_remote_throttling"),
         # reading 1e6*8 bytes with 1M default bandwith should take (8-1)/1=7 seconds
         pytest.param(
             "s3",
@@ -362,7 +362,7 @@ def test_read_throttling(policy, mode, setting, value, should_took):
         #
         # Local
         #
-        pytest.param("default", None, None, None, (0, 3), id="no_local_throttling"),
+        pytest.param("default", None, None, None, 0, id="no_local_throttling"),
         # reading 1e6*8 bytes with 1M default bandwith should take (8-1)/1=7 seconds
         pytest.param(
             "default",
@@ -384,7 +384,7 @@ def test_read_throttling(policy, mode, setting, value, should_took):
         #
         # Remote
         #
-        pytest.param("s3", None, None, None, (0, 3), id="no_remote_throttling"),
+        pytest.param("s3", None, None, None, 0, id="no_remote_throttling"),
         # writing 1e6*8 bytes with 1M default bandwith should take (8-1)/1=7 seconds
         pytest.param(
             "s3",

From 8646eefc9cfeedc840c14bc7571c0c6490679a4e Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 1 Aug 2023 17:05:24 +0300
Subject: [PATCH 052/249] Update docker_test.py

---
 tests/ci/docker_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/ci/docker_test.py b/tests/ci/docker_test.py
index 8542de412d2..bdbee92d2d3 100644
--- a/tests/ci/docker_test.py
+++ b/tests/ci/docker_test.py
@@ -43,7 +43,7 @@ class TestDockerImageCheck(unittest.TestCase):
                 di.DockerImage(
                     "docker/test/sqltest",
                     "clickhouse/sqltest",
-                    True,
+                    False,
                     "clickhouse/test-base",  # type: ignore
                 ),
                 di.DockerImage(

From 07790ab485012f844a7efafa1729ec901df79c0b Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 1 Aug 2023 17:40:55 +0300
Subject: [PATCH 053/249] Update
 tests/queries/0_stateless/02834_client_yaml_configs.sh
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
---
 tests/queries/0_stateless/02834_client_yaml_configs.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/02834_client_yaml_configs.sh b/tests/queries/0_stateless/02834_client_yaml_configs.sh
index dbb40d33e0a..d1c5a40c04f 100755
--- a/tests/queries/0_stateless/02834_client_yaml_configs.sh
+++ b/tests/queries/0_stateless/02834_client_yaml_configs.sh
@@ -11,11 +11,11 @@ echo "max_block_size: 31337" > clickhouse-client.yaml
 ${CLICKHOUSE_CLIENT} --query "SELECT getSetting('max_block_size')"
 rm clickhouse-client.yaml
 
-echo "max_block_size: 31337" > clickhouse-client.yml
+echo "max_block_size: 31338" > clickhouse-client.yml
 ${CLICKHOUSE_CLIENT} --query "SELECT getSetting('max_block_size')"
 rm clickhouse-client.yml
 
-echo "<clickhouse><max_block_size>31337</max_block_size></clickhouse>" > clickhouse-client.xml
+echo "<clickhouse><max_block_size>31339</max_block_size></clickhouse>" > clickhouse-client.xml
 ${CLICKHOUSE_CLIENT} --query "SELECT getSetting('max_block_size')"
 rm clickhouse-client.xml
 

From 93d766ba74f22f7de2dbbc3f196be06af91614d3 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 1 Aug 2023 17:41:00 +0300
Subject: [PATCH 054/249] Update
 tests/queries/0_stateless/02834_client_yaml_configs.reference
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
---
 tests/queries/0_stateless/02834_client_yaml_configs.reference | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/02834_client_yaml_configs.reference b/tests/queries/0_stateless/02834_client_yaml_configs.reference
index 302360f2570..b2eddb19e52 100644
--- a/tests/queries/0_stateless/02834_client_yaml_configs.reference
+++ b/tests/queries/0_stateless/02834_client_yaml_configs.reference
@@ -1,3 +1,3 @@
 31337
-31337
-31337
+31338
+31339

From 17c4abce10c03bafc5316cde3f80a163fa56a944 Mon Sep 17 00:00:00 2001
From: Kenji Noguchi <knoguchi@edg.io>
Date: Tue, 1 Aug 2023 10:44:43 -0700
Subject: [PATCH 055/249] CVE-2016-2183: disable 3DES

---
 base/poco/NetSSL_OpenSSL/include/Poco/Net/Context.h         | 6 +++---
 base/poco/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h      | 2 +-
 base/poco/NetSSL_OpenSSL/src/Context.cpp                    | 2 +-
 .../operations/server-configuration-parameters/settings.md  | 2 +-
 .../operations/server-configuration-parameters/settings.md  | 2 +-
 .../operations/server-configuration-parameters/settings.md  | 2 +-
 6 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/base/poco/NetSSL_OpenSSL/include/Poco/Net/Context.h b/base/poco/NetSSL_OpenSSL/include/Poco/Net/Context.h
index 65917ac9dd4..c19eecf5c73 100644
--- a/base/poco/NetSSL_OpenSSL/include/Poco/Net/Context.h
+++ b/base/poco/NetSSL_OpenSSL/include/Poco/Net/Context.h
@@ -146,7 +146,7 @@ namespace Net
 
             std::string cipherList;
             /// Specifies the supported ciphers in OpenSSL notation.
-            /// Defaults to "ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH".
+            /// Defaults to "ALL:!ADH:!LOW:!EXP:!MD5:!3DES:@STRENGTH".
 
             std::string dhParamsFile;
             /// Specifies a file containing Diffie-Hellman parameters.
@@ -172,7 +172,7 @@ namespace Net
             VerificationMode verificationMode = VERIFY_RELAXED,
             int verificationDepth = 9,
             bool loadDefaultCAs = false,
-            const std::string & cipherList = "ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH");
+            const std::string & cipherList = "ALL:!ADH:!LOW:!EXP:!MD5:!3DES:@STRENGTH");
         /// Creates a Context.
         ///
         ///   * usage specifies whether the context is used by a client or server.
@@ -200,7 +200,7 @@ namespace Net
             VerificationMode verificationMode = VERIFY_RELAXED,
             int verificationDepth = 9,
             bool loadDefaultCAs = false,
-            const std::string & cipherList = "ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH");
+            const std::string & cipherList = "ALL:!ADH:!LOW:!EXP:!MD5:!3DES:@STRENGTH");
         /// Creates a Context.
         ///
         ///   * usage specifies whether the context is used by a client or server.
diff --git a/base/poco/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h b/base/poco/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h
index 21a1ed685e5..e4037c87927 100644
--- a/base/poco/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h
+++ b/base/poco/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h
@@ -76,7 +76,7 @@ namespace Net
     ///            <verificationMode>none|relaxed|strict|once</verificationMode>
     ///            <verificationDepth>1..9</verificationDepth>
     ///            <loadDefaultCAFile>true|false</loadDefaultCAFile>
-    ///            <cipherList>ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH</cipherList>
+    ///            <cipherList>ALL:!ADH:!LOW:!EXP:!MD5:!3DES:@STRENGTH</cipherList>
     ///            <preferServerCiphers>true|false</preferServerCiphers>
     ///            <privateKeyPassphraseHandler>
     ///                <name>KeyFileHandler</name>
diff --git a/base/poco/NetSSL_OpenSSL/src/Context.cpp b/base/poco/NetSSL_OpenSSL/src/Context.cpp
index ca220c40a33..d0bab902b89 100644
--- a/base/poco/NetSSL_OpenSSL/src/Context.cpp
+++ b/base/poco/NetSSL_OpenSSL/src/Context.cpp
@@ -41,7 +41,7 @@ Context::Params::Params():
 	verificationMode(VERIFY_RELAXED),
 	verificationDepth(9),
 	loadDefaultCAs(false),
-	cipherList("ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH")
+	cipherList("ALL:!ADH:!LOW:!EXP:!MD5:!3DES:@STRENGTH")
 {
 }
 
diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md
index 5187ccce789..a7637082496 100644
--- a/docs/en/operations/server-configuration-parameters/settings.md
+++ b/docs/en/operations/server-configuration-parameters/settings.md
@@ -1640,7 +1640,7 @@ Keys for server/client settings:
 - verificationMode (default: relaxed) – The method for checking the node’s certificates. Details are in the description of the [Context](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/Context.h) class. Possible values: `none`, `relaxed`, `strict`, `once`.
 - verificationDepth (default: 9) – The maximum length of the verification chain. Verification will fail if the certificate chain length exceeds the set value.
 - loadDefaultCAFile (default: true) – Wether built-in CA certificates for OpenSSL will be used. ClickHouse assumes that builtin CA certificates are in the file `/etc/ssl/cert.pem` (resp. the directory `/etc/ssl/certs`) or in file (resp. directory) specified by the environment variable `SSL_CERT_FILE` (resp. `SSL_CERT_DIR`).
-- cipherList (default: `ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH`) - Supported OpenSSL encryptions.
+- cipherList (default: `ALL:!ADH:!LOW:!EXP:!MD5:!3DES:@STRENGTH`) - Supported OpenSSL encryptions.
 - cacheSessions (default: false) – Enables or disables caching sessions. Must be used in combination with `sessionIdContext`. Acceptable values: `true`, `false`.
 - sessionIdContext (default: `${application.name}`) – A unique set of random characters that the server appends to each generated identifier. The length of the string must not exceed `SSL_MAX_SSL_SESSION_ID_LENGTH`. This parameter is always recommended since it helps avoid problems both if the server caches the session and if the client requested caching. Default value: `${application.name}`.
 - sessionCacheSize (default: [1024\*20](https://github.com/ClickHouse/boringssl/blob/master/include/openssl/ssl.h#L1978)) – The maximum number of sessions that the server caches. A value of 0 means unlimited sessions.
diff --git a/docs/ru/operations/server-configuration-parameters/settings.md b/docs/ru/operations/server-configuration-parameters/settings.md
index 81a696bcfc1..7b026244624 100644
--- a/docs/ru/operations/server-configuration-parameters/settings.md
+++ b/docs/ru/operations/server-configuration-parameters/settings.md
@@ -1106,7 +1106,7 @@ ClickHouse использует потоки из глобального пул
 -   verificationMode - Способ проверки сертификатов узла. Подробности находятся в описании класса [Context](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/Context.h). Допустимые значения: `none`, `relaxed`, `strict`, `once`.
 -   verificationDepth - Максимальная длина верификационной цепи. Верификация завершится ошибкой, если длина цепи сертификатов превысит установленное значение.
 -   loadDefaultCAFile - Признак того, что будут использоваться встроенные CA-сертификаты для OpenSSL. Допустимые значения: `true`, `false`. \|
--   cipherList - Поддерживаемые OpenSSL-шифры. Например, `ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH`.
+-   cipherList - Поддерживаемые OpenSSL-шифры. Например, `ALL:!ADH:!LOW:!EXP:!MD5:!3DES:@STRENGTH`.
 -   cacheSessions - Включение/выключение кеширования сессии. Использовать обязательно вместе с `sessionIdContext`. Допустимые значения: `true`, `false`.
 -   sessionIdContext - Уникальный набор произвольных символов, которые сервер добавляет к каждому сгенерированному идентификатору. Длина строки не должна превышать `SSL_MAX_SSL_SESSION_ID_LENGTH`. Рекомендуется к использованию всегда, поскольку позволяет избежать проблем как в случае, если сервер кеширует сессию, так и если клиент затребовал кеширование. По умолчанию `${application.name}`.
 -   sessionCacheSize - Максимальное количество сессий, которые кэширует сервер. По умолчанию - 1024\*20. 0 - неограниченное количество сессий.
diff --git a/docs/zh/operations/server-configuration-parameters/settings.md b/docs/zh/operations/server-configuration-parameters/settings.md
index f6106d8734e..8e2cb389f04 100644
--- a/docs/zh/operations/server-configuration-parameters/settings.md
+++ b/docs/zh/operations/server-configuration-parameters/settings.md
@@ -455,7 +455,7 @@ SSL客户端/服务器配置。
 -   verificationMode – The method for checking the node’s certificates. Details are in the description of the [A.背景](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/Context.h) 同学们 可能的值: `none`, `relaxed`, `strict`, `once`.
 -   verificationDepth – The maximum length of the verification chain. Verification will fail if the certificate chain length exceeds the set value.
 -   loadDefaultCAFile – Indicates that built-in CA certificates for OpenSSL will be used. Acceptable values: `true`, `false`. \|
--   cipherList – Supported OpenSSL encryptions. For example: `ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH`.
+-   cipherList – Supported OpenSSL encryptions. For example: `ALL:!ADH:!LOW:!EXP:!MD5:!3DES:@STRENGTH`.
 -   cacheSessions – Enables or disables caching sessions. Must be used in combination with `sessionIdContext`. 可接受的值: `true`, `false`.
 -   sessionIdContext – A unique set of random characters that the server appends to each generated identifier. The length of the string must not exceed `SSL_MAX_SSL_SESSION_ID_LENGTH`. 始终建议使用此参数，因为如果服务器缓存会话，以及客户端请求缓存，它有助于避免出现问题。 默认值: `${application.name}`.
 -   sessionCacheSize – The maximum number of sessions that the server caches. Default value: 1024\*20. 0 – Unlimited sessions.

From 0641dfd7e640bfd08d8533d41cb8ada524dd7b3a Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 1 Aug 2023 22:33:04 +0200
Subject: [PATCH 056/249] Add missing modification

---
 tests/ci/ci_config.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py
index 517e40fd2d6..be8364f8874 100644
--- a/tests/ci/ci_config.py
+++ b/tests/ci/ci_config.py
@@ -402,6 +402,9 @@ CI_CONFIG = {
         "Sqllogic test (release)": {
             "required_build": "package_release",
         },
+        "SQLTest": {
+            "required_build": "package_release",
+        },
     },
 }  # type: dict
 

From 469dd7f30033b4e7870d6999825f8a16fe106e59 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 2 Aug 2023 19:06:31 +0200
Subject: [PATCH 057/249] Add the docs

---
 docs/en/interfaces/cli.md | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/docs/en/interfaces/cli.md b/docs/en/interfaces/cli.md
index 8779dd1a544..36afb94433a 100644
--- a/docs/en/interfaces/cli.md
+++ b/docs/en/interfaces/cli.md
@@ -323,9 +323,9 @@ clickhouse-client clickhouse://192.168.1.15,192.168.1.25
 `clickhouse-client` uses the first existing file of the following:
 
 - Defined in the `--config-file` parameter.
-- `./clickhouse-client.xml`
-- `~/.clickhouse-client/config.xml`
-- `/etc/clickhouse-client/config.xml`
+- `./clickhouse-client.xml`, `.yaml`, `.yml`
+- `~/.clickhouse-client/config.xml`, `.yaml`, `.yml`
+- `/etc/clickhouse-client/config.xml`, `.yaml`, `.yml`
 
 Example of a config file:
 
@@ -342,6 +342,17 @@ Example of a config file:
 </config>
 ```
 
+Or the same config in a YAML format:
+ 
+```yaml
+user: username
+password: 'password'
+secure: true
+openSSL:
+  client:
+    caConfig: '/etc/ssl/cert.pem'
+```
+
 ### Query ID Format {#query-id-format}
 
 In interactive mode `clickhouse-client` shows query ID for every query. By default, the ID is formatted like this:

From b4f0d0bc2edb5865128e16dd90db00d25df8747e Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 2 Aug 2023 21:05:39 +0300
Subject: [PATCH 058/249] Update pull_request.yml

---
 .github/workflows/pull_request.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index 1d515633f61..6ec8c003491 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -5194,7 +5194,7 @@ jobs:
           cat >> "$GITHUB_ENV" << 'EOF'
           TEMP_PATH=${{runner.temp}}/sqltest
           REPORTS_PATH=${{runner.temp}}/reports_dir
-          CHECK_NAME="SQLTest"
+          CHECK_NAME=SQLTest
           REPO_COPY=${{runner.temp}}/sqltest/ClickHouse
           EOF
       - name: Download json reports

From 5ec6a4695f2911758e8d64ee52e34149c4940112 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 2 Aug 2023 23:22:15 +0200
Subject: [PATCH 059/249] Fix log paths

---
 tests/ci/sqltest.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/tests/ci/sqltest.py b/tests/ci/sqltest.py
index b752d4e4aee..7385716850f 100644
--- a/tests/ci/sqltest.py
+++ b/tests/ci/sqltest.py
@@ -110,11 +110,10 @@ def main():
     s3_prefix = f"{pr_info.number}/{pr_info.sha}/sqltest_{check_name_lower}/"
     paths = {
         "run.log": run_log_path,
-        "main.log": os.path.join(workspace_path, "main.log"),
-        "server.log.zst": os.path.join(workspace_path, "server.log.zst"),
-        "server.err.log.zst": os.path.join(workspace_path, "server.err.log.zst"),
-        "report.html": os.path.join(workspace_path, "report.html"),
-        "test.log": os.path.join(workspace_path, "test.log"),
+        "server.log.zst": os.path.join(workspace_path, "/var/log/clickhouse-server/server.log.zst"),
+        "server.err.log.zst": os.path.join(workspace_path, "/var/log/clickhouse-server/server.err.log.zst"),
+        "report.html": os.path.join(workspace_path, "/report.html"),
+        "test.log": os.path.join(workspace_path, "/test.log"),
     }
 
     s3_helper = S3Helper()

From 619af0c2c66255e8c8aa4f8fd62b96c94fe35ee3 Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Wed, 2 Aug 2023 21:46:23 +0000
Subject: [PATCH 060/249] Automatic style fix

---
 tests/ci/sqltest.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tests/ci/sqltest.py b/tests/ci/sqltest.py
index 7385716850f..498448a7928 100644
--- a/tests/ci/sqltest.py
+++ b/tests/ci/sqltest.py
@@ -110,8 +110,12 @@ def main():
     s3_prefix = f"{pr_info.number}/{pr_info.sha}/sqltest_{check_name_lower}/"
     paths = {
         "run.log": run_log_path,
-        "server.log.zst": os.path.join(workspace_path, "/var/log/clickhouse-server/server.log.zst"),
-        "server.err.log.zst": os.path.join(workspace_path, "/var/log/clickhouse-server/server.err.log.zst"),
+        "server.log.zst": os.path.join(
+            workspace_path, "/var/log/clickhouse-server/server.log.zst"
+        ),
+        "server.err.log.zst": os.path.join(
+            workspace_path, "/var/log/clickhouse-server/server.err.log.zst"
+        ),
         "report.html": os.path.join(workspace_path, "/report.html"),
         "test.log": os.path.join(workspace_path, "/test.log"),
     }

From 99c7f7b48ca61af6077fd99431194bde24f761b6 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 2 Aug 2023 23:50:19 +0200
Subject: [PATCH 061/249] Fix the test

---
 tests/queries/0_stateless/02834_client_yaml_configs.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/queries/0_stateless/02834_client_yaml_configs.sh b/tests/queries/0_stateless/02834_client_yaml_configs.sh
index d1c5a40c04f..66d3df8829e 100755
--- a/tests/queries/0_stateless/02834_client_yaml_configs.sh
+++ b/tests/queries/0_stateless/02834_client_yaml_configs.sh
@@ -1,11 +1,11 @@
 #!/usr/bin/env bash
-# Tags: no-fasttest
+# Tags: no-fasttest, no-random-settings
 
 CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 
-pushd "${CLICKHOUSE_TMP}" || exit > /dev/null
+pushd "${CLICKHOUSE_TMP}" > /dev/null || exit
 
 echo "max_block_size: 31337" > clickhouse-client.yaml
 ${CLICKHOUSE_CLIENT} --query "SELECT getSetting('max_block_size')"
@@ -19,4 +19,4 @@ echo "<clickhouse><max_block_size>31339</max_block_size></clickhouse>" > clickho
 ${CLICKHOUSE_CLIENT} --query "SELECT getSetting('max_block_size')"
 rm clickhouse-client.xml
 
-popd || exit > /dev/null
+popd > /dev/null || exit

From dadf4678ee9d678459c96cd13b8231e33dc02cc3 Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Thu, 3 Aug 2023 12:50:40 +0000
Subject: [PATCH 062/249] Automatic style fix

---
 tests/integration/test_dictionary_custom_settings/test.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tests/integration/test_dictionary_custom_settings/test.py b/tests/integration/test_dictionary_custom_settings/test.py
index 6b5ea643998..eb394da8bb6 100644
--- a/tests/integration/test_dictionary_custom_settings/test.py
+++ b/tests/integration/test_dictionary_custom_settings/test.py
@@ -82,5 +82,9 @@ def test_work(start_cluster):
 
     # It is possible that the HTTP server takes long time to start accepting connections
 
-    assert_eq_with_retry(instance, "SELECT dictGetString('test_http', 'first', toUInt64(1))", "\\'a\n")
-    assert_eq_with_retry(instance, "SELECT dictGetString('test_http', 'second', toUInt64(1))", '"b\n')
+    assert_eq_with_retry(
+        instance, "SELECT dictGetString('test_http', 'first', toUInt64(1))", "\\'a\n"
+    )
+    assert_eq_with_retry(
+        instance, "SELECT dictGetString('test_http', 'second', toUInt64(1))", '"b\n'
+    )

From 65e8ee8fb7d30b0765c8bfd3d58d9def64e002bb Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 3 Aug 2023 15:28:18 +0200
Subject: [PATCH 063/249] Fix paths

---
 docker/test/sqltest/run.sh | 4 +++-
 tests/ci/sqltest.py        | 8 ++++----
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/docker/test/sqltest/run.sh b/docker/test/sqltest/run.sh
index 42aeef9df15..cba1c1dab1f 100755
--- a/docker/test/sqltest/run.sh
+++ b/docker/test/sqltest/run.sh
@@ -42,8 +42,10 @@ done
 # Run the test
 pushd sqltest/standards/2016/
 /test.py
-mv report.html test.log /
+mv report.html test.log /workspace
 popd
 
 zstd --threads=0 /var/log/clickhouse-server/clickhouse-server.log
 zstd --threads=0 /var/log/clickhouse-server/clickhouse-server.err.log
+
+mv /var/log/clickhouse-server/clickhouse-server.log.zst /var/log/clickhouse-server/clickhouse-server.err.log.zst /workspace
diff --git a/tests/ci/sqltest.py b/tests/ci/sqltest.py
index 7385716850f..d76baf22bb3 100644
--- a/tests/ci/sqltest.py
+++ b/tests/ci/sqltest.py
@@ -110,10 +110,10 @@ def main():
     s3_prefix = f"{pr_info.number}/{pr_info.sha}/sqltest_{check_name_lower}/"
     paths = {
         "run.log": run_log_path,
-        "server.log.zst": os.path.join(workspace_path, "/var/log/clickhouse-server/server.log.zst"),
-        "server.err.log.zst": os.path.join(workspace_path, "/var/log/clickhouse-server/server.err.log.zst"),
-        "report.html": os.path.join(workspace_path, "/report.html"),
-        "test.log": os.path.join(workspace_path, "/test.log"),
+        "server.log.zst": os.path.join(workspace_path, "/workspace/server.log.zst"),
+        "server.err.log.zst": os.path.join(workspace_path, "/workspace/server.err.log.zst"),
+        "report.html": os.path.join(workspace_path, "/workspace/report.html"),
+        "test.log": os.path.join(workspace_path, "/workspace/test.log"),
     }
 
     s3_helper = S3Helper()

From a96b0457e04543876b35ca1a5223ea6007bbf9bb Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Thu, 3 Aug 2023 13:43:45 +0000
Subject: [PATCH 064/249] Automatic style fix

---
 tests/ci/sqltest.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/ci/sqltest.py b/tests/ci/sqltest.py
index d76baf22bb3..69af21f1761 100644
--- a/tests/ci/sqltest.py
+++ b/tests/ci/sqltest.py
@@ -111,7 +111,9 @@ def main():
     paths = {
         "run.log": run_log_path,
         "server.log.zst": os.path.join(workspace_path, "/workspace/server.log.zst"),
-        "server.err.log.zst": os.path.join(workspace_path, "/workspace/server.err.log.zst"),
+        "server.err.log.zst": os.path.join(
+            workspace_path, "/workspace/server.err.log.zst"
+        ),
         "report.html": os.path.join(workspace_path, "/workspace/report.html"),
         "test.log": os.path.join(workspace_path, "/workspace/test.log"),
     }

From a0fa3cc73bbbec2879873608607fc18f51a6556a Mon Sep 17 00:00:00 2001
From: Maksim Kita <kitaetoya@gmail.com>
Date: Thu, 3 Aug 2023 18:55:15 +0300
Subject: [PATCH 065/249] EXPLAIN actions for JOIN step

---
 src/Interpreters/ConcurrentHashJoin.h         |   1 +
 src/Interpreters/DirectJoin.h                 |   1 +
 src/Interpreters/FullSortingMergeJoin.h       |   1 +
 src/Interpreters/GraceHashJoin.h              |   1 +
 src/Interpreters/HashJoin.h                   |   1 +
 src/Interpreters/IJoin.h                      |   2 +
 src/Interpreters/JoinSwitcher.h               |   1 +
 src/Interpreters/MergeJoin.h                  |   1 +
 src/Interpreters/TableJoin.h                  |   2 +-
 src/Processors/QueryPlan/JoinStep.cpp         |  33 +++++
 src/Processors/QueryPlan/JoinStep.h           |   3 +
 .../02835_join_step_explain.reference         | 116 ++++++++++++++++++
 .../0_stateless/02835_join_step_explain.sql   |  31 +++++
 13 files changed, 193 insertions(+), 1 deletion(-)
 create mode 100644 tests/queries/0_stateless/02835_join_step_explain.reference
 create mode 100644 tests/queries/0_stateless/02835_join_step_explain.sql

diff --git a/src/Interpreters/ConcurrentHashJoin.h b/src/Interpreters/ConcurrentHashJoin.h
index 1283879971d..85e0c5a0ae7 100644
--- a/src/Interpreters/ConcurrentHashJoin.h
+++ b/src/Interpreters/ConcurrentHashJoin.h
@@ -36,6 +36,7 @@ public:
     explicit ConcurrentHashJoin(ContextPtr context_, std::shared_ptr<TableJoin> table_join_, size_t slots_, const Block & right_sample_block, bool any_take_last_row_ = false);
     ~ConcurrentHashJoin() override = default;
 
+    std::string getName() const override { return "ConcurrentHashJoin"; }
     const TableJoin & getTableJoin() const override { return *table_join; }
     bool addBlockToJoin(const Block & block, bool check_limits) override;
     void checkTypesOfKeys(const Block & block) const override;
diff --git a/src/Interpreters/DirectJoin.h b/src/Interpreters/DirectJoin.h
index e55ac278705..5f664314818 100644
--- a/src/Interpreters/DirectJoin.h
+++ b/src/Interpreters/DirectJoin.h
@@ -30,6 +30,7 @@ public:
         std::shared_ptr<const IKeyValueEntity> storage_,
         const Block & right_sample_block_with_storage_column_names_);
 
+    std::string getName() const override { return "DirectKeyValueJoin"; }
     virtual const TableJoin & getTableJoin() const override { return *table_join; }
 
     virtual bool addBlockToJoin(const Block &, bool) override;
diff --git a/src/Interpreters/FullSortingMergeJoin.h b/src/Interpreters/FullSortingMergeJoin.h
index a6b53a51c04..3fc9f8920ed 100644
--- a/src/Interpreters/FullSortingMergeJoin.h
+++ b/src/Interpreters/FullSortingMergeJoin.h
@@ -28,6 +28,7 @@ public:
         LOG_TRACE(&Poco::Logger::get("FullSortingMergeJoin"), "Will use full sorting merge join");
     }
 
+    std::string getName() const override { return "FullSortingMergeJoin"; }
     const TableJoin & getTableJoin() const override { return *table_join; }
 
     bool addBlockToJoin(const Block & /* block */, bool /* check_limits */) override
diff --git a/src/Interpreters/GraceHashJoin.h b/src/Interpreters/GraceHashJoin.h
index ce519892b0e..44949440467 100644
--- a/src/Interpreters/GraceHashJoin.h
+++ b/src/Interpreters/GraceHashJoin.h
@@ -60,6 +60,7 @@ public:
 
     ~GraceHashJoin() override;
 
+    std::string getName() const override { return "GraceHashJoin"; }
     const TableJoin & getTableJoin() const override { return *table_join; }
 
     void initialize(const Block & sample_block) override;
diff --git a/src/Interpreters/HashJoin.h b/src/Interpreters/HashJoin.h
index 56dea98c1f1..9f55945816c 100644
--- a/src/Interpreters/HashJoin.h
+++ b/src/Interpreters/HashJoin.h
@@ -151,6 +151,7 @@ public:
 
     ~HashJoin() override;
 
+    std::string getName() const override { return "HashJoin"; }
     const TableJoin & getTableJoin() const override { return *table_join; }
 
     /** Add block of data from right hand of JOIN to the map.
diff --git a/src/Interpreters/IJoin.h b/src/Interpreters/IJoin.h
index 97b119bd795..493a5dd2126 100644
--- a/src/Interpreters/IJoin.h
+++ b/src/Interpreters/IJoin.h
@@ -48,6 +48,8 @@ class IJoin
 public:
     virtual ~IJoin() = default;
 
+    virtual std::string getName() const = 0;
+
     virtual const TableJoin & getTableJoin() const = 0;
 
     /// Add block of data from right hand of JOIN.
diff --git a/src/Interpreters/JoinSwitcher.h b/src/Interpreters/JoinSwitcher.h
index fb5066b2d04..1d2ebc6b456 100644
--- a/src/Interpreters/JoinSwitcher.h
+++ b/src/Interpreters/JoinSwitcher.h
@@ -18,6 +18,7 @@ class JoinSwitcher : public IJoin
 public:
     JoinSwitcher(std::shared_ptr<TableJoin> table_join_, const Block & right_sample_block_);
 
+    std::string getName() const override { return "JoinSwitcher"; }
     const TableJoin & getTableJoin() const override { return *table_join; }
 
     /// Add block of data from right hand of JOIN into current join object.
diff --git a/src/Interpreters/MergeJoin.h b/src/Interpreters/MergeJoin.h
index 03a661c5b8a..98fae1d419f 100644
--- a/src/Interpreters/MergeJoin.h
+++ b/src/Interpreters/MergeJoin.h
@@ -22,6 +22,7 @@ class MergeJoin : public IJoin
 public:
     MergeJoin(std::shared_ptr<TableJoin> table_join_, const Block & right_sample_block);
 
+    std::string getName() const override { return "PartialMergeJoin"; }
     const TableJoin & getTableJoin() const override { return *table_join; }
     bool addBlockToJoin(const Block & block, bool check_limits) override;
     void checkTypesOfKeys(const Block & block) const override;
diff --git a/src/Interpreters/TableJoin.h b/src/Interpreters/TableJoin.h
index 5d14a57759f..75626764bda 100644
--- a/src/Interpreters/TableJoin.h
+++ b/src/Interpreters/TableJoin.h
@@ -331,7 +331,7 @@ public:
         const ColumnsWithTypeAndName & right_sample_columns);
 
     void setAsofInequality(ASOFJoinInequality inequality) { asof_inequality = inequality; }
-    ASOFJoinInequality getAsofInequality() { return asof_inequality; }
+    ASOFJoinInequality getAsofInequality() const { return asof_inequality; }
 
     ASTPtr leftKeysList() const;
     ASTPtr rightKeysList() const; /// For ON syntax only
diff --git a/src/Processors/QueryPlan/JoinStep.cpp b/src/Processors/QueryPlan/JoinStep.cpp
index 33fa7955e0d..858a01a437d 100644
--- a/src/Processors/QueryPlan/JoinStep.cpp
+++ b/src/Processors/QueryPlan/JoinStep.cpp
@@ -2,6 +2,9 @@
 #include <QueryPipeline/QueryPipelineBuilder.h>
 #include <Processors/Transforms/JoiningTransform.h>
 #include <Interpreters/IJoin.h>
+#include <Interpreters/TableJoin.h>
+#include <IO/Operators.h>
+#include <Common/JSONBuilder.h>
 #include <Common/typeid_cast.h>
 
 namespace DB
@@ -62,6 +65,36 @@ void JoinStep::describePipeline(FormatSettings & settings) const
     IQueryPlanStep::describePipeline(processors, settings);
 }
 
+void JoinStep::describeActions(FormatSettings & settings) const
+{
+    String prefix(settings.offset, ' ');
+
+    const auto & table_join = join->getTableJoin();
+    settings.out << prefix << "Kind: " << toString(table_join.kind()) << '\n';
+    settings.out << prefix << "Strictness: " << toString(table_join.strictness()) << '\n';
+    settings.out << prefix << "Type: " << join->getName() << '\n';
+
+    if (table_join.strictness() == JoinStrictness::Asof)
+        settings.out << prefix << "ASOF inequality: " << toString(table_join.getAsofInequality()) << '\n';
+
+    if (!table_join.getClauses().empty())
+        settings.out << prefix << "Clauses: " << table_join.formatClauses(table_join.getClauses(), true /*short_format*/) << '\n';
+}
+
+void JoinStep::describeActions(JSONBuilder::JSONMap & map) const
+{
+    const auto & table_join = join->getTableJoin();
+    map.add("Kind", toString(table_join.kind()));
+    map.add("Strictness", toString(table_join.strictness()));
+    map.add("Type", join->getName());
+
+    if (table_join.strictness() == JoinStrictness::Asof)
+        map.add("ASOF inequality", toString(table_join.getAsofInequality()));
+
+    if (!table_join.getClauses().empty())
+        map.add("Clauses", table_join.formatClauses(table_join.getClauses(), true /*short_format*/));
+}
+
 void JoinStep::updateInputStream(const DataStream & new_input_stream_, size_t idx)
 {
     if (idx == 0)
diff --git a/src/Processors/QueryPlan/JoinStep.h b/src/Processors/QueryPlan/JoinStep.h
index e7185f36588..369ee9bec8b 100644
--- a/src/Processors/QueryPlan/JoinStep.h
+++ b/src/Processors/QueryPlan/JoinStep.h
@@ -27,6 +27,9 @@ public:
 
     void describePipeline(FormatSettings & settings) const override;
 
+    void describeActions(JSONBuilder::JSONMap & map) const override;
+    void describeActions(FormatSettings & settings) const override;
+
     const JoinPtr & getJoin() const { return join; }
     bool allowPushDownToRight() const;
 
diff --git a/tests/queries/0_stateless/02835_join_step_explain.reference b/tests/queries/0_stateless/02835_join_step_explain.reference
new file mode 100644
index 00000000000..77c91b3d7e9
--- /dev/null
+++ b/tests/queries/0_stateless/02835_join_step_explain.reference
@@ -0,0 +1,116 @@
+Expression ((Project names + (Projection + DROP unused columns after JOIN)))
+Header: id UInt64
+        value_1 String
+        rhs.id UInt64
+        rhs.value_1 String
+Actions: INPUT : 0 -> id_0 UInt64 : 0
+         INPUT : 1 -> value_1_1 String : 1
+         INPUT : 2 -> value_1_3 String : 2
+         INPUT : 3 -> id_2 UInt64 : 3
+         ALIAS id_0 :: 0 -> id UInt64 : 4
+         ALIAS value_1_1 :: 1 -> value_1 String : 0
+         ALIAS value_1_3 :: 2 -> rhs.value_1 String : 1
+         ALIAS id_2 :: 3 -> rhs.id UInt64 : 2
+Positions: 4 0 2 1
+  Join (JOIN FillRightFirst)
+  Header: id_0 UInt64
+          value_1_1 String
+          value_1_3 String
+          id_2 UInt64
+  Kind: INNER
+  Strictness: ALL
+  Type: HashJoin
+  Clauses: [(id_0) = (id_2)]
+    Expression ((JOIN actions + Change column names to column identifiers))
+    Header: id_0 UInt64
+            value_1_1 String
+    Actions: INPUT : 0 -> id UInt64 : 0
+             INPUT : 1 -> value_1 String : 1
+             ALIAS id :: 0 -> id_0 UInt64 : 2
+             ALIAS value_1 :: 1 -> value_1_1 String : 0
+    Positions: 2 0
+      ReadFromMergeTree (default.test_table_1)
+      Header: id UInt64
+              value_1 String
+      ReadType: Default
+      Parts: 1
+      Granules: 1
+    Expression ((JOIN actions + Change column names to column identifiers))
+    Header: id_2 UInt64
+            value_1_3 String
+    Actions: INPUT : 0 -> id UInt64 : 0
+             INPUT : 1 -> value_1 String : 1
+             ALIAS id :: 0 -> id_2 UInt64 : 2
+             ALIAS value_1 :: 1 -> value_1_3 String : 0
+    Positions: 2 0
+      ReadFromMergeTree (default.test_table_2)
+      Header: id UInt64
+              value_1 String
+      ReadType: Default
+      Parts: 1
+      Granules: 1
+--
+Expression ((Project names + (Projection + DROP unused columns after JOIN)))
+Header: id UInt64
+        value_1 String
+        rhs.id UInt64
+        rhs.value_1 String
+Actions: INPUT : 0 -> id_0 UInt64 : 0
+         INPUT : 1 -> value_1_1 String : 1
+         INPUT :: 2 -> value_2_4 UInt64 : 2
+         INPUT : 3 -> value_1_3 String : 3
+         INPUT :: 4 -> value_2_5 UInt64 : 4
+         INPUT : 5 -> id_2 UInt64 : 5
+         ALIAS id_0 :: 0 -> id UInt64 : 6
+         ALIAS value_1_1 :: 1 -> value_1 String : 0
+         ALIAS value_1_3 :: 3 -> rhs.value_1 String : 1
+         ALIAS id_2 :: 5 -> rhs.id UInt64 : 3
+Positions: 6 0 3 1
+  Join (JOIN FillRightFirst)
+  Header: id_0 UInt64
+          value_1_1 String
+          value_2_4 UInt64
+          value_1_3 String
+          value_2_5 UInt64
+          id_2 UInt64
+  Kind: INNER
+  Strictness: ASOF
+  Type: HashJoin
+  ASOF inequality: LESS
+  Clauses: [(id_0, value_2_4) = (id_2, value_2_5)]
+    Expression ((JOIN actions + Change column names to column identifiers))
+    Header: id_0 UInt64
+            value_1_1 String
+            value_2_4 UInt64
+    Actions: INPUT : 0 -> id UInt64 : 0
+             INPUT : 1 -> value_1 String : 1
+             INPUT : 2 -> value_2 UInt64 : 2
+             ALIAS id :: 0 -> id_0 UInt64 : 3
+             ALIAS value_1 :: 1 -> value_1_1 String : 0
+             ALIAS value_2 :: 2 -> value_2_4 UInt64 : 1
+    Positions: 3 0 1
+      ReadFromMergeTree (default.test_table_1)
+      Header: id UInt64
+              value_1 String
+              value_2 UInt64
+      ReadType: Default
+      Parts: 1
+      Granules: 1
+    Expression ((JOIN actions + Change column names to column identifiers))
+    Header: id_2 UInt64
+            value_1_3 String
+            value_2_5 UInt64
+    Actions: INPUT : 0 -> id UInt64 : 0
+             INPUT : 1 -> value_1 String : 1
+             INPUT : 2 -> value_2 UInt64 : 2
+             ALIAS id :: 0 -> id_2 UInt64 : 3
+             ALIAS value_1 :: 1 -> value_1_3 String : 0
+             ALIAS value_2 :: 2 -> value_2_5 UInt64 : 1
+    Positions: 3 0 1
+      ReadFromMergeTree (default.test_table_2)
+      Header: id UInt64
+              value_1 String
+              value_2 UInt64
+      ReadType: Default
+      Parts: 1
+      Granules: 1
diff --git a/tests/queries/0_stateless/02835_join_step_explain.sql b/tests/queries/0_stateless/02835_join_step_explain.sql
new file mode 100644
index 00000000000..d0475fa14b6
--- /dev/null
+++ b/tests/queries/0_stateless/02835_join_step_explain.sql
@@ -0,0 +1,31 @@
+SET allow_experimental_analyzer = 1;
+
+DROP TABLE IF EXISTS test_table_1;
+CREATE TABLE test_table_1
+(
+    id UInt64,
+    value_1 String,
+    value_2 UInt64
+) ENGINE=MergeTree ORDER BY id;
+
+DROP TABLE IF EXISTS test_table_2;
+CREATE TABLE test_table_2
+(
+    id UInt64,
+    value_1 String,
+    value_2 UInt64
+) ENGINE=MergeTree ORDER BY id;
+
+INSERT INTO test_table_1 VALUES (0, 'Value', 0);
+INSERT INTO test_table_2 VALUES (0, 'Value', 0);
+
+EXPLAIN header = 1, actions = 1 SELECT lhs.id, lhs.value_1, rhs.id, rhs.value_1
+FROM test_table_1 AS lhs INNER JOIN test_table_2 AS rhs ON lhs.id = rhs.id;
+
+SELECT '--';
+
+EXPLAIN header = 1, actions = 1 SELECT lhs.id, lhs.value_1, rhs.id, rhs.value_1
+FROM test_table_1 AS lhs ASOF JOIN test_table_2 AS rhs ON lhs.id = rhs.id AND lhs.value_2 < rhs.value_2;
+
+DROP TABLE test_table_1;
+DROP TABLE test_table_2;

From 388feb953d97b244b162c433e2983cbe4e2d73f3 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 3 Aug 2023 19:28:59 +0200
Subject: [PATCH 066/249] Fix paths

---
 tests/ci/sqltest.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/ci/sqltest.py b/tests/ci/sqltest.py
index d76baf22bb3..be22a1c9312 100644
--- a/tests/ci/sqltest.py
+++ b/tests/ci/sqltest.py
@@ -110,10 +110,10 @@ def main():
     s3_prefix = f"{pr_info.number}/{pr_info.sha}/sqltest_{check_name_lower}/"
     paths = {
         "run.log": run_log_path,
-        "server.log.zst": os.path.join(workspace_path, "/workspace/server.log.zst"),
-        "server.err.log.zst": os.path.join(workspace_path, "/workspace/server.err.log.zst"),
-        "report.html": os.path.join(workspace_path, "/workspace/report.html"),
-        "test.log": os.path.join(workspace_path, "/workspace/test.log"),
+        "server.log.zst": os.path.join(workspace_path, "server.log.zst"),
+        "server.err.log.zst": os.path.join(workspace_path, "server.err.log.zst"),
+        "report.html": os.path.join(workspace_path, "report.html"),
+        "test.log": os.path.join(workspace_path, "test.log"),
     }
 
     s3_helper = S3Helper()

From 6dc74fb5ccf0c6ef6538e60835667b57b6d4f060 Mon Sep 17 00:00:00 2001
From: ltrk2 <107155950+ltrk2@users.noreply.github.com>
Date: Thu, 3 Aug 2023 12:38:43 -0700
Subject: [PATCH 067/249] Implement big-endian support for transform

---
 .../Serializations/SerializationArray.cpp          |  4 ++--
 .../Serializations/SerializationNumber.cpp         |  8 ++++----
 src/Functions/transform.cpp                        | 14 ++++++++------
 3 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/src/DataTypes/Serializations/SerializationArray.cpp b/src/DataTypes/Serializations/SerializationArray.cpp
index cedcca870dd..e01c1aea0e9 100644
--- a/src/DataTypes/Serializations/SerializationArray.cpp
+++ b/src/DataTypes/Serializations/SerializationArray.cpp
@@ -129,7 +129,7 @@ namespace
         for (size_t i = offset; i < end; ++i)
         {
             ColumnArray::Offset current_offset = offset_values[i];
-            writeIntBinary(current_offset - prev_offset, ostr);
+            writeBinaryLittleEndian(current_offset - prev_offset, ostr);
             prev_offset = current_offset;
         }
     }
@@ -145,7 +145,7 @@ namespace
         while (i < initial_size + limit && !istr.eof())
         {
             ColumnArray::Offset current_size = 0;
-            readIntBinary(current_size, istr);
+            readBinaryLittleEndian(current_size, istr);
 
             if (unlikely(current_size > MAX_ARRAY_SIZE))
                 throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Array size is too large: {}", current_size);
diff --git a/src/DataTypes/Serializations/SerializationNumber.cpp b/src/DataTypes/Serializations/SerializationNumber.cpp
index 8cabaec753d..0294a1c8a67 100644
--- a/src/DataTypes/Serializations/SerializationNumber.cpp
+++ b/src/DataTypes/Serializations/SerializationNumber.cpp
@@ -106,28 +106,28 @@ void SerializationNumber<T>::serializeBinary(const Field & field, WriteBuffer &
 {
     /// ColumnVector<T>::ValueType is a narrower type. For example, UInt8, when the Field type is UInt64
     typename ColumnVector<T>::ValueType x = static_cast<typename ColumnVector<T>::ValueType>(field.get<FieldType>());
-    writeBinary(x, ostr);
+    writeBinaryLittleEndian(x, ostr);
 }
 
 template <typename T>
 void SerializationNumber<T>::deserializeBinary(Field & field, ReadBuffer & istr, const FormatSettings &) const
 {
     typename ColumnVector<T>::ValueType x;
-    readBinary(x, istr);
+    readBinaryLittleEndian(x, istr);
     field = NearestFieldType<FieldType>(x);
 }
 
 template <typename T>
 void SerializationNumber<T>::serializeBinary(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
 {
-    writeBinary(assert_cast<const ColumnVector<T> &>(column).getData()[row_num], ostr);
+    writeBinaryLittleEndian(assert_cast<const ColumnVector<T> &>(column).getData()[row_num], ostr);
 }
 
 template <typename T>
 void SerializationNumber<T>::deserializeBinary(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
 {
     typename ColumnVector<T>::ValueType x;
-    readBinary(x, istr);
+    readBinaryLittleEndian(x, istr);
     assert_cast<ColumnVector<T> &>(column).getData().push_back(x);
 }
 
diff --git a/src/Functions/transform.cpp b/src/Functions/transform.cpp
index e03701327b1..16326dd5a44 100644
--- a/src/Functions/transform.cpp
+++ b/src/Functions/transform.cpp
@@ -764,9 +764,8 @@ namespace
             }
 
             /// Note: Doesn't check the duplicates in the `from` array.
-
-            WhichDataType which(from_type);
-            if (isNativeNumber(which) || which.isDecimal32() || which.isDecimal64())
+            /// Field may be of Float type, but for the purpose of bitwise equality we can treat them as UInt64
+            if (WhichDataType which(from_type); isNativeNumber(which) || which.isDecimal32() || which.isDecimal64())
             {
                 cache.table_num_to_idx = std::make_unique<Cache::NumToIdx>();
                 auto & table = *cache.table_num_to_idx;
@@ -774,10 +773,13 @@ namespace
                 {
                     if (applyVisitor(FieldVisitorAccurateEquals(), (*cache.from_column)[i], (*from_column_uncasted)[i]))
                     {
-                        /// Field may be of Float type, but for the purpose of bitwise equality we can treat them as UInt64
-                        StringRef ref = cache.from_column->getDataAt(i);
                         UInt64 key = 0;
-                        memcpy(&key, ref.data, ref.size);
+                        auto * dst = reinterpret_cast<char *>(&key);
+                        const auto ref = cache.from_column->getDataAt(i);
+                        if constexpr (std::endian::native == std::endian::big)
+                            dst += sizeof(key) - ref.size;
+
+                        memcpy(dst, ref.data, ref.size);
                         table[key] = i;
                     }
                 }

From 06631f6a86a2e099ef69f807592ec2eb70b5aa35 Mon Sep 17 00:00:00 2001
From: ltrk2 <107155950+ltrk2@users.noreply.github.com>
Date: Fri, 4 Aug 2023 09:27:15 -0700
Subject: [PATCH 068/249] Make hasTokenOrNull return null on empty needle

---
 src/Functions/HasTokenImpl.h | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/Functions/HasTokenImpl.h b/src/Functions/HasTokenImpl.h
index ab6b6399486..661b8ae9753 100644
--- a/src/Functions/HasTokenImpl.h
+++ b/src/Functions/HasTokenImpl.h
@@ -39,9 +39,6 @@ struct HasTokenImpl
         if (start_pos != nullptr)
             throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function '{}' does not support start_pos argument", name);
 
-        if (pattern.empty())
-            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Needle cannot be empty, because empty string isn't a token");
-
         if (haystack_offsets.empty())
             return;
 
@@ -49,7 +46,7 @@ struct HasTokenImpl
         const UInt8 * const end = haystack_data.data() + haystack_data.size();
         const UInt8 * pos = begin;
 
-        if (!std::none_of(pattern.begin(), pattern.end(), isTokenSeparator))
+        if (const auto has_separator = std::any_of(pattern.cbegin(), pattern.cend(), isTokenSeparator); has_separator || pattern.empty())
         {
             if (res_null)
             {
@@ -57,8 +54,12 @@ struct HasTokenImpl
                 std::ranges::fill(res_null->getData(), true);
                 return;
             }
-            else
+            else if (has_separator)
                 throw Exception(ErrorCodes::BAD_ARGUMENTS, "Needle must not contain whitespace or separator characters");
+            else if (pattern.empty())
+                throw Exception(ErrorCodes::BAD_ARGUMENTS, "Needle cannot be empty, because empty string isn't a token");
+            else
+                throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected internal state");
         }
 
         size_t pattern_size = pattern.size();

From 054562233143594b1ebd83d248567151bfa9adb2 Mon Sep 17 00:00:00 2001
From: ltrk2 <107155950+ltrk2@users.noreply.github.com>
Date: Fri, 4 Aug 2023 13:03:15 -0700
Subject: [PATCH 069/249] Add functional tests for hasTokenOrNull

---
 tests/queries/0_stateless/02816_has_token_empty.reference | 4 ++++
 tests/queries/0_stateless/02816_has_token_empty.sql       | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/tests/queries/0_stateless/02816_has_token_empty.reference b/tests/queries/0_stateless/02816_has_token_empty.reference
index aa47d0d46d4..8435d77c5fe 100644
--- a/tests/queries/0_stateless/02816_has_token_empty.reference
+++ b/tests/queries/0_stateless/02816_has_token_empty.reference
@@ -1,2 +1,6 @@
 0
+\N
+\N
 0
+\N
+\N
diff --git a/tests/queries/0_stateless/02816_has_token_empty.sql b/tests/queries/0_stateless/02816_has_token_empty.sql
index e5d6156debd..3e00959126b 100644
--- a/tests/queries/0_stateless/02816_has_token_empty.sql
+++ b/tests/queries/0_stateless/02816_has_token_empty.sql
@@ -2,6 +2,10 @@ SELECT hasTokenCaseInsensitive('K(G', ''); -- { serverError BAD_ARGUMENTS }
 SELECT hasTokenCaseInsensitive('Hello', ''); -- { serverError BAD_ARGUMENTS }
 SELECT hasTokenCaseInsensitive('', ''); -- { serverError BAD_ARGUMENTS }
 SELECT hasTokenCaseInsensitive('', 'Hello');
+SELECT hasTokenCaseInsensitiveOrNull('Hello', '');
+SELECT hasTokenCaseInsensitiveOrNull('', '');
 SELECT hasToken('Hello', ''); -- { serverError BAD_ARGUMENTS }
 SELECT hasToken('', 'Hello');
 SELECT hasToken('', ''); -- { serverError BAD_ARGUMENTS }
+SELECT hasTokenOrNull('', '');
+SELECT hasTokenOrNull('Hello', '');

From 264bff8c9fcf25e206b86c88e1c3a6f2a5caf8b6 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 5 Aug 2023 01:43:54 +0200
Subject: [PATCH 070/249] Fix a comment

---
 base/poco/Data/include/Poco/Data/TypeHandler.h | 2 +-
 programs/format/Format.cpp                     | 6 ++++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/base/poco/Data/include/Poco/Data/TypeHandler.h b/base/poco/Data/include/Poco/Data/TypeHandler.h
index 34f88e986f7..e7633de7018 100644
--- a/base/poco/Data/include/Poco/Data/TypeHandler.h
+++ b/base/poco/Data/include/Poco/Data/TypeHandler.h
@@ -97,7 +97,7 @@ namespace Data
     ///
     ///		static void extract(std::size_t pos, Person& obj, const Person& defVal, AbstractExtractor::Ptr pExt)
     ///		{
-    ///			// defVal is the default person we should use if we encunter NULL entries, so we take the individual fields
+    ///			// defVal is the default person we should use if we encounter NULL entries, so we take the individual fields
     ///			// as defaults. You can do more complex checking, ie return defVal if only one single entry of the fields is null etc...
     ///			poco_assert_dbg (!pExt.isNull());
     ///			std::string lastName;
diff --git a/programs/format/Format.cpp b/programs/format/Format.cpp
index 43c66a32302..d7d61bbcd3b 100644
--- a/programs/format/Format.cpp
+++ b/programs/format/Format.cpp
@@ -163,13 +163,15 @@ int mainEntryClickHouseFormat(int argc, char ** argv)
             {
                 ASTPtr res = parseQueryAndMovePosition(
                     parser, pos, end, "query", multiple, cmd_settings.max_query_size, cmd_settings.max_parser_depth);
-                /// For insert query with data(INSERT INTO ... VALUES ...), will lead to format fail,
-                /// should throw exception early and make exception message more readable.
+
+                /// For insert query with data(INSERT INTO ... VALUES ...), that will lead to the formatting failure,
+                /// we should throw an exception early, and make exception message more readable.
                 if (const auto * insert_query = res->as<ASTInsertQuery>(); insert_query && insert_query->data)
                 {
                     throw Exception(DB::ErrorCodes::INVALID_FORMAT_INSERT_QUERY_WITH_DATA,
                         "Can't format ASTInsertQuery with data, since data will be lost");
                 }
+
                 if (!quiet)
                 {
                     if (!backslash)

From d479c125784010ed516188e948fa6554e72078c7 Mon Sep 17 00:00:00 2001
From: Maksim Kita <kitaetoya@gmail.com>
Date: Sat, 5 Aug 2023 14:11:22 +0300
Subject: [PATCH 071/249] Fixed tests

---
 tests/queries/0_stateless/01655_plan_optimizations.reference | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/queries/0_stateless/01655_plan_optimizations.reference b/tests/queries/0_stateless/01655_plan_optimizations.reference
index be42a656c66..54ca55d2068 100644
--- a/tests/queries/0_stateless/01655_plan_optimizations.reference
+++ b/tests/queries/0_stateless/01655_plan_optimizations.reference
@@ -168,19 +168,23 @@ Filter
 3
 > one condition of filter is pushed down before LEFT JOIN
 Join
+Join
 Filter column: notEquals(number, 1)
 Join
 > (analyzer) one condition of filter is pushed down before LEFT JOIN
 Join
+Join
 Filter column: notEquals(number_0, 1_UInt8)
 0	0
 3	3
 > one condition of filter is pushed down before INNER JOIN
 Join
+Join
 Filter column: notEquals(number, 1)
 Join
 > (analyzer) one condition of filter is pushed down before INNER JOIN
 Join
+Join
 Filter column: notEquals(number_0, 1_UInt8)
 3	3
 > filter is pushed down before UNION

From 9de95879655cb0ebbd895b0e95198e6ad759f09a Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 5 Aug 2023 13:54:36 +0200
Subject: [PATCH 072/249] Something with tests

---
 tests/integration/test_zookeeper_config/test.py           | 7 +++----
 tests/queries/0_stateless/02263_format_insert_settings.sh | 2 ++
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/tests/integration/test_zookeeper_config/test.py b/tests/integration/test_zookeeper_config/test.py
index 5e36a97b3ca..34ac2cfa572 100644
--- a/tests/integration/test_zookeeper_config/test.py
+++ b/tests/integration/test_zookeeper_config/test.py
@@ -58,9 +58,7 @@ def test_chroot_with_same_root(started_cluster):
             node.query("INSERT INTO simple VALUES ({0}, {0})".format(i))
 
     # Replication might take time
-
     assert_eq_with_retry(node1, "select count() from simple", "2\n")
-
     assert_eq_with_retry(node2, "select count() from simple", "2\n")
 
 
@@ -78,5 +76,6 @@ def test_chroot_with_different_root(started_cluster):
         for j in range(2):  # Second insert to test deduplication
             node.query("INSERT INTO simple_different VALUES ({0}, {0})".format(i))
 
-    assert node1.query("select count() from simple_different").strip() == "1"
-    assert node3.query("select count() from simple_different").strip() == "1"
+    # Replication might take time
+    assert_eq_with_retry(node1, "select count() from simple_different", "1\n")
+    assert_eq_with_retry(node3, "select count() from simple_different", "1\n")
diff --git a/tests/queries/0_stateless/02263_format_insert_settings.sh b/tests/queries/0_stateless/02263_format_insert_settings.sh
index 3d5f780a38c..efb3d39ab6c 100755
--- a/tests/queries/0_stateless/02263_format_insert_settings.sh
+++ b/tests/queries/0_stateless/02263_format_insert_settings.sh
@@ -23,10 +23,12 @@ function run_format_both()
 
 # NOTE: that those queries may work slow, due to stack trace obtaining
 run_format 'insert into foo settings max_threads=1' 2> >(grep -m1 -o "Syntax error (query): failed at position .* (end of query):")
+
 # compatibility
 run_format 'insert into foo format tsv settings max_threads=1' 2> >(grep -m1 -F -o "Can't format ASTInsertQuery with data, since data will be lost.")
 run_format_both 'insert into foo format tsv settings max_threads=1' --allow_settings_after_format_in_insert
 run_format 'insert into foo settings max_threads=1 format tsv settings max_threads=1' --allow_settings_after_format_in_insert 2> >(grep -m1 -F -o "You have SETTINGS before and after FORMAT")
+
 # and via server (since this is a separate code path)
 $CLICKHOUSE_CLIENT -q 'drop table if exists data_02263'
 $CLICKHOUSE_CLIENT -q 'create table data_02263 (key Int) engine=Memory()'

From 10af8fcca4fedf6347fe724ed68f1e60b53b2b1a Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 5 Aug 2023 14:05:06 +0200
Subject: [PATCH 073/249] Fix Python

---
 tests/integration/test_zookeeper_config/test.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/integration/test_zookeeper_config/test.py b/tests/integration/test_zookeeper_config/test.py
index 34ac2cfa572..3b5ae7186fb 100644
--- a/tests/integration/test_zookeeper_config/test.py
+++ b/tests/integration/test_zookeeper_config/test.py
@@ -58,8 +58,8 @@ def test_chroot_with_same_root(started_cluster):
             node.query("INSERT INTO simple VALUES ({0}, {0})".format(i))
 
     # Replication might take time
-    assert_eq_with_retry(node1, "select count() from simple", "2\n")
-    assert_eq_with_retry(node2, "select count() from simple", "2\n")
+    assert_eq_with_retry(node1, "select count() from simple", "2")
+    assert_eq_with_retry(node2, "select count() from simple", "2")
 
 
 def test_chroot_with_different_root(started_cluster):
@@ -77,5 +77,5 @@ def test_chroot_with_different_root(started_cluster):
             node.query("INSERT INTO simple_different VALUES ({0}, {0})".format(i))
 
     # Replication might take time
-    assert_eq_with_retry(node1, "select count() from simple_different", "1\n")
-    assert_eq_with_retry(node3, "select count() from simple_different", "1\n")
+    assert_eq_with_retry(node1, "select count() from simple_different", "1")
+    assert_eq_with_retry(node3, "select count() from simple_different", "1")

From aa57fb6602bd0a815fe512b82df8b18893d268eb Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 5 Aug 2023 14:07:55 +0200
Subject: [PATCH 074/249] Maybe better, but not sure

---
 tests/integration/test_zookeeper_config/test.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/integration/test_zookeeper_config/test.py b/tests/integration/test_zookeeper_config/test.py
index 3b5ae7186fb..4008df3f84f 100644
--- a/tests/integration/test_zookeeper_config/test.py
+++ b/tests/integration/test_zookeeper_config/test.py
@@ -57,8 +57,9 @@ def test_chroot_with_same_root(started_cluster):
         for j in range(2):  # Second insert to test deduplication
             node.query("INSERT INTO simple VALUES ({0}, {0})".format(i))
 
-    # Replication might take time
+    node1.query("SYSTEM SYNC REPLICA simple")
     assert_eq_with_retry(node1, "select count() from simple", "2")
+    node2.query("SYSTEM SYNC REPLICA simple")
     assert_eq_with_retry(node2, "select count() from simple", "2")
 
 
@@ -76,6 +77,7 @@ def test_chroot_with_different_root(started_cluster):
         for j in range(2):  # Second insert to test deduplication
             node.query("INSERT INTO simple_different VALUES ({0}, {0})".format(i))
 
-    # Replication might take time
+    node1.query("SYSTEM SYNC REPLICA simple_different")
     assert_eq_with_retry(node1, "select count() from simple_different", "1")
+    node2.query("SYSTEM SYNC REPLICA simple_different")
     assert_eq_with_retry(node3, "select count() from simple_different", "1")

From 0a6d9881a72c9476ffeb6263f924150bf3116f29 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 5 Aug 2023 16:52:44 +0200
Subject: [PATCH 075/249] Maybe better, but not sure

---
 tests/integration/test_zookeeper_config/test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/test_zookeeper_config/test.py b/tests/integration/test_zookeeper_config/test.py
index 4008df3f84f..0c0f77ec597 100644
--- a/tests/integration/test_zookeeper_config/test.py
+++ b/tests/integration/test_zookeeper_config/test.py
@@ -79,5 +79,5 @@ def test_chroot_with_different_root(started_cluster):
 
     node1.query("SYSTEM SYNC REPLICA simple_different")
     assert_eq_with_retry(node1, "select count() from simple_different", "1")
-    node2.query("SYSTEM SYNC REPLICA simple_different")
+    node3.query("SYSTEM SYNC REPLICA simple_different")
     assert_eq_with_retry(node3, "select count() from simple_different", "1")

From afb1a55bf3789196e2b9f96aea5318ea0a9f9219 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 6 Aug 2023 00:56:33 +0200
Subject: [PATCH 076/249] Remove whitespaces

---
 src/Interpreters/SystemLog.h | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h
index 437b1b2a6bb..c4c2b0b2499 100644
--- a/src/Interpreters/SystemLog.h
+++ b/src/Interpreters/SystemLog.h
@@ -131,8 +131,6 @@ protected:
     using Base::queue;
 
 private:
-
-
     /* Saving thread data */
     const StorageID table_id;
     const String storage_def;

From 7e90df67a746fb12a9d4ecb6944beeab198d869d Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 6 Aug 2023 00:58:26 +0200
Subject: [PATCH 077/249] Remove whitespaces

---
 src/Interpreters/SystemLog.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h
index c4c2b0b2499..cf287ad7775 100644
--- a/src/Interpreters/SystemLog.h
+++ b/src/Interpreters/SystemLog.h
@@ -111,7 +111,7 @@ public:
       *   and new table get created - as if previous table was not exist.
       */
     SystemLog(ContextPtr context_,
-              const SystemLogSettings& settings_,
+              const SystemLogSettings & settings_,
               std::shared_ptr<SystemLogQueue<LogElement>> queue_ = nullptr);
 
     /** Append a record into log.

From 06d45cfa81c2e21963a1a82bb7fabc1bcb9b115b Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 6 Aug 2023 01:29:31 +0200
Subject: [PATCH 078/249] Allow creating system logs at startup

---
 programs/server/config.xml     | 3 +++
 src/Interpreters/SystemLog.cpp | 6 ++++++
 src/Interpreters/SystemLog.h   | 3 +++
 3 files changed, 12 insertions(+)

diff --git a/programs/server/config.xml b/programs/server/config.xml
index 14b8954fc39..3d1c92b135c 100644
--- a/programs/server/config.xml
+++ b/programs/server/config.xml
@@ -1037,6 +1037,9 @@
 
         <!-- example of using a different storage policy for a system table -->
         <!-- storage_policy>local_ssd</storage_policy -->
+
+        <!-- Create the table at startup or lazily on first record (the default is lazy) -->
+        <create_at_startup>false</create_at_startup>
     </query_log>
 
     <!-- Trace log. Stores stack traces collected by query profilers.
diff --git a/src/Interpreters/SystemLog.cpp b/src/Interpreters/SystemLog.cpp
index 12f3b9744cb..ddb8560c094 100644
--- a/src/Interpreters/SystemLog.cpp
+++ b/src/Interpreters/SystemLog.cpp
@@ -129,9 +129,12 @@ std::shared_ptr<TSystemLog> createSystemLog(
               "Creating {}.{} from {}", default_database_name, default_table_name, config_prefix);
 
     SystemLogSettings log_settings;
+
     log_settings.queue_settings.database = config.getString(config_prefix + ".database", default_database_name);
     log_settings.queue_settings.table = config.getString(config_prefix + ".table", default_table_name);
 
+    log_settings.create_at_startup = config.getBool(config_prefix + ".create_at_startup", false);
+
     if (log_settings.queue_settings.database != default_database_name)
     {
         /// System tables must be loaded before other tables, but loading order is undefined for all databases except `system`
@@ -381,6 +384,9 @@ SystemLog<LogElement>::SystemLog(
     , create_query(serializeAST(*getCreateTableQuery()))
 {
     assert(settings_.queue_settings.database == DatabaseCatalog::SYSTEM_DATABASE);
+
+    if (settings_.create_at_startup)
+        prepareTable();
 }
 
 template <typename LogElement>
diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h
index 437b1b2a6bb..a1401707568 100644
--- a/src/Interpreters/SystemLog.h
+++ b/src/Interpreters/SystemLog.h
@@ -93,6 +93,9 @@ struct SystemLogSettings
     SystemLogQueueSettings queue_settings;
 
     String engine;
+
+    /// Create the table eagerly at startup or lazily on first record.
+    bool create_at_startup = false;
 };
 
 template <typename LogElement>

From 292a5de8164992bb4d859fb3fe69a593dcda7bca Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 6 Aug 2023 01:51:23 +0200
Subject: [PATCH 079/249] Create them in tests

---
 .../config.d/create_logs_at_startup.yaml      |  1 +
 .../config.d/create_logs_at_startup.yaml      | 34 +++++++++++++++++++
 tests/config/install.sh                       |  1 +
 3 files changed, 36 insertions(+)
 create mode 120000 programs/server/config.d/create_logs_at_startup.yaml
 create mode 100644 tests/config/config.d/create_logs_at_startup.yaml

diff --git a/programs/server/config.d/create_logs_at_startup.yaml b/programs/server/config.d/create_logs_at_startup.yaml
new file mode 120000
index 00000000000..a09d2783a9e
--- /dev/null
+++ b/programs/server/config.d/create_logs_at_startup.yaml
@@ -0,0 +1 @@
+../../../tests/config/config.d/create_logs_at_startup.yaml
\ No newline at end of file
diff --git a/tests/config/config.d/create_logs_at_startup.yaml b/tests/config/config.d/create_logs_at_startup.yaml
new file mode 100644
index 00000000000..e9cc94925de
--- /dev/null
+++ b/tests/config/config.d/create_logs_at_startup.yaml
@@ -0,0 +1,34 @@
+query_log:
+  create_at_startup: 1
+trace_log:
+  create_at_startup: 1
+query_thread_log:
+  create_at_startup: 1
+query_views_log:
+  create_at_startup: 1
+part_log:
+  create_at_startup: 1
+text_log:
+  create_at_startup: 1
+metric_log:
+  create_at_startup: 1
+asynchronous_metric_log:
+  create_at_startup: 1
+opentelemetry_span_log:
+  create_at_startup: 1
+crash_log:
+  create_at_startup: 1
+processors_profile_log:
+  create_at_startup: 1
+asynchronous_insert_log:
+  create_at_startup: 1
+zookeeper_log:
+  create_at_startup: 1
+transactions_info_log:
+  create_at_startup: 1
+session_log:
+  create_at_startup: 1
+filesystem_cache_log:
+  create_at_startup: 1
+filesystem_read_prefetches_log:
+  create_at_startup: 1
diff --git a/tests/config/install.sh b/tests/config/install.sh
index 503da780061..44b515965f4 100755
--- a/tests/config/install.sh
+++ b/tests/config/install.sh
@@ -60,6 +60,7 @@ ln -sf $SRC_PATH/config.d/compressed_marks_and_index.xml $DEST_SERVER_PATH/confi
 ln -sf $SRC_PATH/config.d/disable_s3_env_credentials.xml $DEST_SERVER_PATH/config.d/
 ln -sf $SRC_PATH/config.d/enable_wait_for_shutdown_replicated_tables.xml $DEST_SERVER_PATH/config.d/
 ln -sf $SRC_PATH/config.d/backups.xml $DEST_SERVER_PATH/config.d/
+ln -sf $SRC_PATH/config.d/create_logs_at_startup.yaml $DEST_SERVER_PATH/config.d/
 
 # Not supported with fasttest.
 if [ "${DEST_SERVER_PATH}" = "/etc/clickhouse-server" ]

From 32a038cfd7952601b0f4b9dd2fd95a31c0df0924 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 6 Aug 2023 02:52:36 +0200
Subject: [PATCH 080/249] YAML is mandatory

---
 docker/test/fasttest/run.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh
index 60e6199aaa4..477bbdb6919 100755
--- a/docker/test/fasttest/run.sh
+++ b/docker/test/fasttest/run.sh
@@ -148,6 +148,7 @@ function clone_submodules
             contrib/liburing
             contrib/libfiu
             contrib/incbin
+            contrib/yaml-cpp
         )
 
         git submodule sync
@@ -170,6 +171,7 @@ function run_cmake
         "-DENABLE_SIMDJSON=1"
         "-DENABLE_JEMALLOC=1"
         "-DENABLE_LIBURING=1"
+        "-DENABLE_YAML_CPP=1"
     )
 
     export CCACHE_DIR="$FASTTEST_WORKSPACE/ccache"

From 8b7f8ad910ab76af78eaca1a9c6e4b5b2d830fd4 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 6 Aug 2023 02:53:11 +0200
Subject: [PATCH 081/249] Add a script to export logs in CI

---
 utils/export-logs-in-ci/setup.sh | 58 ++++++++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)
 create mode 100755 utils/export-logs-in-ci/setup.sh

diff --git a/utils/export-logs-in-ci/setup.sh b/utils/export-logs-in-ci/setup.sh
new file mode 100755
index 00000000000..d7b5f1e5d37
--- /dev/null
+++ b/utils/export-logs-in-ci/setup.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+
+# This script sets up export of system log tables to a remote server.
+# Remote tables are created if not exist, and augmented with extra columns,
+# and their names will contain a hash of the table structure,
+# which allows exporting tables from servers of different versions.
+
+# Pre-configured destination cluster, where to export the data
+CLUSTER=${CLUSTER:=system_logs_export}
+
+EXTRA_COLUMNS=${EXTRA_COLUMNS:="pull_request_number UInt32, commit_sha String, check_start_time DateTime, check_name LowCardinality(String), instance_type LowCardinality(String), "}
+EXTRA_COLUMNS_EXPRESSION=${EXTRA_COLUMNS_EXPRESSION:="0 AS pull_request_number, '' AS commit_sha, now() AS check_start_time, '' AS check_name, '' AS instance_type"}
+EXTRA_ORDER_BY_COLUMNS=${EXTRA_ORDER_BY_COLUMNS:="check_name, "}
+
+CONNECTION_PARAMETERS=${CONNECTION_PARAMETERS:=""}
+
+# For each system log table:
+clickhouse-client --query "SHOW TABLES FROM system LIKE '%\\_log'" | while read table
+do
+    # Calculate hash of its structure:
+    hash=$(clickhouse-client --query "
+        SELECT sipHash64(groupArray((name, type)))
+        FROM (SELECT name, type FROM system.columns
+            WHERE database = 'system' AND table = '$table'
+            ORDER BY position)
+        ")
+
+    # Create the destination table with adapted name and structure:
+    statement=$(clickhouse-client --format TSVRaw --query "SHOW CREATE TABLE system.${table}" | sed -r -e '
+        s/^\($/('"$EXTRA_COLUMNS"'/;
+        s/ORDER BY \(/ORDER BY ('"$EXTRA_ORDER_BY_COLUMNS"'/;
+        s/^CREATE TABLE system\.\w+_log$/CREATE TABLE IF NOT EXISTS '"$table"'_'"$hash"'/;
+        /^TTL /d
+        ')
+
+    echo "Creating destination table ${table}_${hash}" >&2
+
+    echo "$statement" | clickhouse-client $CONNECTION_PARAMETERS
+
+    echo "Creating table system.${table}_sender" >&2
+
+    # Create Distributed table and materialized view to watch on the original table:
+    clickhouse-client --query "
+        CREATE TABLE system.${table}_sender
+        ENGINE = Distributed(${CLUSTER}, default, ${table}_${hash})
+        EMPTY AS
+        SELECT ${EXTRA_COLUMNS_EXPRESSION}, *
+        FROM system.${table}
+    "
+
+    echo "Creating materialized view system.${table}_watcher" >&2
+
+    clickhouse-client --query "
+        CREATE MATERIALIZED VIEW system.${table}_watcher TO system.${table}_sender AS
+        SELECT ${EXTRA_COLUMNS_EXPRESSION}, *
+        FROM system.${table}
+    "
+done

From f1eee292c9bb8af138b879526df9345ac28383fd Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 6 Aug 2023 04:38:04 +0200
Subject: [PATCH 082/249] Enable export (first try)

---
 .github/workflows/master.yml       |  3 +++
 .github/workflows/pull_request.yml |  3 +++
 docker/test/stateless/run.sh       | 35 ++++++++++++++++++++++++++++++
 tests/ci/functional_test_check.py  | 28 +++++++++++++++++++-----
 4 files changed, 63 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index ae1862e327f..f25611e8666 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -3,6 +3,9 @@ name: MasterCI
 env:
   # Force the stdout and stderr streams to be unbuffered
   PYTHONUNBUFFERED: 1
+  # Export system tables to ClickHouse Cloud
+  CLICKHOUSE_CI_LOGS_HOST: ${{ secrets.CLICKHOUSE_CI_LOGS_HOST }}
+  CLICKHOUSE_CI_LOGS_PASSWORD: ${{ secrets.CLICKHOUSE_CI_LOGS_PASSWORD }}
 
 on: # yamllint disable-line rule:truthy
   push:
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index d97b9975c3c..25927954809 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -3,6 +3,9 @@ name: PullRequestCI
 env:
   # Force the stdout and stderr streams to be unbuffered
   PYTHONUNBUFFERED: 1
+  # Export system tables to ClickHouse Cloud
+  CLICKHOUSE_CI_LOGS_HOST: ${{ secrets.CLICKHOUSE_CI_LOGS_HOST }}
+  CLICKHOUSE_CI_LOGS_PASSWORD: ${{ secrets.CLICKHOUSE_CI_LOGS_PASSWORD }}
 
 on:  # yamllint disable-line rule:truthy
   pull_request:
diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh
index 3694fb7c2f6..321515cc469 100755
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@@ -36,6 +36,22 @@ fi
 ./setup_minio.sh stateless
 ./setup_hdfs_minicluster.sh
 
+# Setup a cluster for logs export to ClickHouse Cloud
+# Note: these variables are provided to the Docker run command by the Python script in tests/ci
+if [ -n "${CLICKHOUSE_CI_LOGS_HOST}" ]
+then
+    echo "
+    remote_servers:
+        system_logs_export:
+            shard:
+                replica:
+                    secure: 1
+                    user: ci
+                    host: '${CLICKHOUSE_CI_LOGS_HOST}'
+                    password: '${CLICKHOUSE_CI_LOGS_PASSWORD}'
+    " > /etc/clickhouse-server/config.d/system_logs_export.yaml
+fi
+
 # For flaky check we also enable thread fuzzer
 if [ "$NUM_TRIES" -gt "1" ]; then
     export THREAD_FUZZER_CPU_TIME_PERIOD_US=1000
@@ -92,6 +108,25 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]
     MAX_RUN_TIME=$((MAX_RUN_TIME != 0 ? MAX_RUN_TIME : 9000))    # set to 2.5 hours if 0 (unlimited)
 fi
 
+
+# Wait for the server to start
+while true
+do
+    clickhouse-client --query "SELECT 1" && break
+    sleep 1
+done
+
+# Initialize export of system logs to ClickHouse Cloud
+if [ -n "${CLICKHOUSE_CI_LOGS_HOST}" ]
+then
+    EXTRA_COLUMNS_EXPRESSION="$PULL_REQUEST_NUMBER AS pull_request_number, '$COMMIT_SHA' AS commit_sha, '$CHECK_START_TIME' AS check_start_time, '$CHECK_NAME' AS check_name, '$INSTANCE_TYPE' AS instance_type"
+    # TODO: Check if the password will appear in the logs.
+    CONNECTION_PARAMETERS="--secure --user ci --host '${CLICKHOUSE_CI_LOGS_HOST}' --password '${CLICKHOUSE_CI_LOGS_PASSWORD}'"
+
+    /repository/utils/export-logs-in-ci/setup.sh
+fi
+
+# TODO what is this? Remove it.
 sleep 5
 
 attach_gdb_to_clickhouse || true  # FIXME: to not break old builds, clean on 2023-09-01
diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py
index b773d1eddd9..0cc4c2264f2 100644
--- a/tests/ci/functional_test_check.py
+++ b/tests/ci/functional_test_check.py
@@ -17,6 +17,7 @@ from build_download_helper import download_all_deb_packages
 from clickhouse_helper import (
     ClickHouseHelper,
     prepare_tests_results_for_clickhouse,
+    get_instance_type
 )
 from commit_status_helper import (
     NotSet,
@@ -73,9 +74,11 @@ def get_image_name(check_name):
 
 
 def get_run_command(
+    pr_info,
+    check_start_time,
     check_name,
     builds_path,
-    repo_tests_path,
+    repo_path,
     result_path,
     server_log_path,
     kill_timeout,
@@ -105,16 +108,29 @@ def get_run_command(
 
     envs += [f"-e {e}" for e in additional_envs]
 
+    instance_type = get_instance_type()
+
+    envs += [
+        "-e CLICKHOUSE_CI_LOGS_HOST",
+        "-e CLICKHOUSE_CI_LOGS_PASSWORD",
+        f"-e PULL_REQUEST_NUMBER='{pr_info.number}'",
+        f"-e COMMIT_SHA='{pr_info.sha}'",
+        f"-e CHECK_START_TIME='{check_start_time}'",
+        f"-e CHECK_NAME='{check_name}'",
+        f"-e INSTANCE_TYPE='{instance_type}'",
+    ]
+
     env_str = " ".join(envs)
     volume_with_broken_test = (
-        f"--volume={repo_tests_path}/analyzer_tech_debt.txt:/analyzer_tech_debt.txt"
+        f"--volume={repo_path}/tests/analyzer_tech_debt.txt:/analyzer_tech_debt.txt"
         if "analyzer" in check_name
         else ""
     )
 
     return (
         f"docker run --volume={builds_path}:/package_folder "
-        f"--volume={repo_tests_path}:/usr/share/clickhouse-test "
+        f"--volume={repo_path}/tests:/usr/share/clickhouse-test "
+        f"--volume={repo_path}:/repository "
         f"{volume_with_broken_test} "
         f"--volume={result_path}:/test_output --volume={server_log_path}:/var/log/clickhouse-server "
         f"--cap-add=SYS_PTRACE {env_str} {additional_options_str} {image}"
@@ -306,8 +322,6 @@ def main():
     image_name = get_image_name(check_name)
     docker_image = get_image_with_version(reports_path, image_name)
 
-    repo_tests_path = os.path.join(repo_path, "tests")
-
     packages_path = os.path.join(temp_path, "packages")
     if not os.path.exists(packages_path):
         os.makedirs(packages_path)
@@ -334,9 +348,11 @@ def main():
         additional_envs.append("GLOBAL_TAGS=no-random-settings")
 
     run_command = get_run_command(
+        pr_info,
+        stopwatch.start_time_str,
         check_name,
         packages_path,
-        repo_tests_path,
+        repo_path,
         result_path,
         server_log_path,
         kill_timeout,

From c9b9bedc4e20f4f1c5456a12f3b2fd9bba106cb9 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 6 Aug 2023 04:43:41 +0200
Subject: [PATCH 083/249] Fix lock order inversion

---
 src/Interpreters/Context.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp
index a210a9efbc7..d6031e26ede 100644
--- a/src/Interpreters/Context.cpp
+++ b/src/Interpreters/Context.cpp
@@ -3182,7 +3182,6 @@ void Context::setCluster(const String & cluster_name, const std::shared_ptr<Clus
 
 void Context::initializeSystemLogs()
 {
-    auto lock = getLock();
     shared->system_logs = std::make_unique<SystemLogs>(getGlobalContext(), getConfigRef());
 }
 

From 3e599acdb11783fafa8a70ce796120d2f74630f8 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 6 Aug 2023 04:43:41 +0200
Subject: [PATCH 084/249] Fix lock order inversion

---
 src/Interpreters/Context.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp
index a210a9efbc7..d6031e26ede 100644
--- a/src/Interpreters/Context.cpp
+++ b/src/Interpreters/Context.cpp
@@ -3182,7 +3182,6 @@ void Context::setCluster(const String & cluster_name, const std::shared_ptr<Clus
 
 void Context::initializeSystemLogs()
 {
-    auto lock = getLock();
     shared->system_logs = std::make_unique<SystemLogs>(getGlobalContext(), getConfigRef());
 }
 

From 2edda961d2a428e5b98f52fb80adcf7a263ed274 Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Sun, 6 Aug 2023 02:55:26 +0000
Subject: [PATCH 085/249] Automatic style fix

---
 tests/ci/functional_test_check.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py
index 0cc4c2264f2..5497a978878 100644
--- a/tests/ci/functional_test_check.py
+++ b/tests/ci/functional_test_check.py
@@ -17,7 +17,7 @@ from build_download_helper import download_all_deb_packages
 from clickhouse_helper import (
     ClickHouseHelper,
     prepare_tests_results_for_clickhouse,
-    get_instance_type
+    get_instance_type,
 )
 from commit_status_helper import (
     NotSet,

From f0a632f1f1157ef70a6e262692dfb4e97d9e13a3 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 6 Aug 2023 05:17:39 +0200
Subject: [PATCH 086/249] Fix shellcheck

---
 docker/test/stateless/run.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh
index 321515cc469..0f629c4f5d5 100755
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@@ -119,9 +119,9 @@ done
 # Initialize export of system logs to ClickHouse Cloud
 if [ -n "${CLICKHOUSE_CI_LOGS_HOST}" ]
 then
-    EXTRA_COLUMNS_EXPRESSION="$PULL_REQUEST_NUMBER AS pull_request_number, '$COMMIT_SHA' AS commit_sha, '$CHECK_START_TIME' AS check_start_time, '$CHECK_NAME' AS check_name, '$INSTANCE_TYPE' AS instance_type"
+    export EXTRA_COLUMNS_EXPRESSION="$PULL_REQUEST_NUMBER AS pull_request_number, '$COMMIT_SHA' AS commit_sha, '$CHECK_START_TIME' AS check_start_time, '$CHECK_NAME' AS check_name, '$INSTANCE_TYPE' AS instance_type"
     # TODO: Check if the password will appear in the logs.
-    CONNECTION_PARAMETERS="--secure --user ci --host '${CLICKHOUSE_CI_LOGS_HOST}' --password '${CLICKHOUSE_CI_LOGS_PASSWORD}'"
+    export CONNECTION_PARAMETERS="--secure --user ci --host '${CLICKHOUSE_CI_LOGS_HOST}' --password '${CLICKHOUSE_CI_LOGS_PASSWORD}'"
 
     /repository/utils/export-logs-in-ci/setup.sh
 fi

From 695516ae74487a619a416a5868cbbfd2b788b505 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 6 Aug 2023 12:51:37 +0200
Subject: [PATCH 087/249] Maybe fix bash

---
 docker/test/stateless/run.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh
index 0f629c4f5d5..a711c213d51 100755
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@@ -121,7 +121,7 @@ if [ -n "${CLICKHOUSE_CI_LOGS_HOST}" ]
 then
     export EXTRA_COLUMNS_EXPRESSION="$PULL_REQUEST_NUMBER AS pull_request_number, '$COMMIT_SHA' AS commit_sha, '$CHECK_START_TIME' AS check_start_time, '$CHECK_NAME' AS check_name, '$INSTANCE_TYPE' AS instance_type"
     # TODO: Check if the password will appear in the logs.
-    export CONNECTION_PARAMETERS="--secure --user ci --host '${CLICKHOUSE_CI_LOGS_HOST}' --password '${CLICKHOUSE_CI_LOGS_PASSWORD}'"
+    export CONNECTION_PARAMETERS="--secure --user ci --host ${CLICKHOUSE_CI_LOGS_HOST} --password ${CLICKHOUSE_CI_LOGS_PASSWORD}"
 
     /repository/utils/export-logs-in-ci/setup.sh
 fi

From fbb2be21ccfdc6bf43f42fe517bc0358c2d7f3b7 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 6 Aug 2023 13:04:35 +0200
Subject: [PATCH 088/249] Slightly better

---
 docker/test/stateless/run.sh      |  5 +++++
 tests/ci/functional_test_check.py | 14 ++++++++++++++
 2 files changed, 19 insertions(+)

diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh
index a711c213d51..290222c9d26 100755
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@@ -124,6 +124,11 @@ then
     export CONNECTION_PARAMETERS="--secure --user ci --host ${CLICKHOUSE_CI_LOGS_HOST} --password ${CLICKHOUSE_CI_LOGS_PASSWORD}"
 
     /repository/utils/export-logs-in-ci/setup.sh
+
+    # Unset variables after use
+    export CONNECTION_PARAMETERS=''
+    export CLICKHOUSE_CI_LOGS_HOST=''
+    export CLICKHOUSE_CI_LOGS_PASSWORD=''
 fi
 
 # TODO what is this? Remove it.
diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py
index 5497a978878..b3ad1caaeab 100644
--- a/tests/ci/functional_test_check.py
+++ b/tests/ci/functional_test_check.py
@@ -384,6 +384,20 @@ def main():
 
     ch_helper = ClickHouseHelper()
 
+    # Cleanup run log from the credentials of CI logs database.
+    # Note: a malicious user can still print them by splitting the value into parts.
+    # But we will be warned when a malicious user modifies CI script.
+    # Although they can also print them from inside tests.
+    # Nevertheless, the credentials of the CI logs have limited scope
+    # and does not provide access to sensitive info.
+
+    ci_logs_host = os.getenv("CLICKHOUSE_CI_LOGS_HOST");
+    ci_logs_password = os.getenv("CLICKHOUSE_CI_LOGS_PASSWORD");
+    subprocess.check_call(
+        f"sed -i -r -e 's!{ci_logs_host}!CLICKHOUSE_CI_LOGS_HOST!g; s!{ci_logs_password}!CLICKHOUSE_CI_LOGS_PASSWORD!g;' '{run_log_path}'",
+        shell=True
+    )
+
     report_url = upload_results(
         s3_helper,
         pr_info.number,

From 9428040d05d914fc4d19e9b0f7692743a971db6a Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 6 Aug 2023 13:07:48 +0200
Subject: [PATCH 089/249] Fix hung check in stress test

---
 tests/ci/stress.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/tests/ci/stress.py b/tests/ci/stress.py
index 60ddeb85d35..eb829cf519c 100755
--- a/tests/ci/stress.py
+++ b/tests/ci/stress.py
@@ -202,13 +202,14 @@ def prepare_for_hung_check(drop_databases):
     call(
         make_query_command(
             """
-    select sleepEachRow((
-        select maxOrDefault(300 - elapsed) + 1
-        from system.processes
-        where query not like '%from system.processes%' and elapsed < 300
+    SELECT sleepEachRow((
+        SELECT maxOrDefault(300 - elapsed) + 1
+        FROM system.processes
+        WHERE query NOT LIKE '%FROM system.processes%' AND elapsed < 300
     ) / 300)
-    from numbers(300)
-    format Null
+    FROM numbers(300)
+    FORMAT Null
+    SETTINGS function_sleep_max_microseconds_per_block = 0
     """
         ),
         shell=True,

From 167f90e7f65fcb13545c61bcb8d06723d82337b5 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 6 Aug 2023 13:28:21 +0200
Subject: [PATCH 090/249] Maybe better

---
 programs/server/Server.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp
index e6d5837dd0e..da5b80426f0 100644
--- a/programs/server/Server.cpp
+++ b/programs/server/Server.cpp
@@ -1650,9 +1650,9 @@ try
         database_catalog.initializeAndLoadTemporaryDatabase();
         loadMetadataSystem(global_context);
         maybeConvertSystemDatabase(global_context);
-        startupSystemTables();
         /// After attaching system databases we can initialize system log.
         global_context->initializeSystemLogs();
+        startupSystemTables();
         global_context->setSystemZooKeeperLogAfterInitializationIfNeeded();
         /// Build loggers before tables startup to make log messages from tables
         /// attach available in system.text_log

From 333ed6425cc30bb9bc01bbfbe4aa17e1654e9890 Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Sun, 6 Aug 2023 11:30:50 +0000
Subject: [PATCH 091/249] Automatic style fix

---
 tests/ci/functional_test_check.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py
index b3ad1caaeab..fdea5be9a6c 100644
--- a/tests/ci/functional_test_check.py
+++ b/tests/ci/functional_test_check.py
@@ -391,11 +391,11 @@ def main():
     # Nevertheless, the credentials of the CI logs have limited scope
     # and does not provide access to sensitive info.
 
-    ci_logs_host = os.getenv("CLICKHOUSE_CI_LOGS_HOST");
-    ci_logs_password = os.getenv("CLICKHOUSE_CI_LOGS_PASSWORD");
+    ci_logs_host = os.getenv("CLICKHOUSE_CI_LOGS_HOST")
+    ci_logs_password = os.getenv("CLICKHOUSE_CI_LOGS_PASSWORD")
     subprocess.check_call(
         f"sed -i -r -e 's!{ci_logs_host}!CLICKHOUSE_CI_LOGS_HOST!g; s!{ci_logs_password}!CLICKHOUSE_CI_LOGS_PASSWORD!g;' '{run_log_path}'",
-        shell=True
+        shell=True,
     )
 
     report_url = upload_results(

From ce2fcc8f336c4e2038d75baf25ca874950d85c3c Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 6 Aug 2023 14:48:20 +0200
Subject: [PATCH 092/249] Fix wording

---
 src/Access/ContextAccess.cpp                  |  8 +--
 .../Access/InterpreterGrantQuery.cpp          |  2 +-
 .../InterpreterKillQueryQuery.cpp             |  4 +-
 src/Interpreters/InterpreterSelectQuery.cpp   |  2 +-
 src/Planner/PlannerJoinTree.cpp               |  2 +-
 .../test_access_for_functions/test.py         |  6 +-
 .../test_backup_restore_new/test.py           | 16 ++---
 .../test_backup_restore_on_cluster/test.py    | 12 ++--
 .../test_select_from_system_tables.py         |  4 +-
 .../integration/test_grant_and_revoke/test.py |  6 +-
 .../test_named_collections/test.py            |  8 +--
 tests/integration/test_row_policy/test.py     | 18 +++---
 .../test_from_system_tables.py                | 10 ++--
 .../test_select_access_rights/test_main.py    | 60 +++++++++----------
 .../integration/test_settings_profile/test.py | 10 ++--
 tests/integration/test_storage_s3/test.py     |  8 +--
 tests/integration/test_storage_url/test.py    |  6 +-
 .../test.py                                   |  4 +-
 .../02250_ON_CLUSTER_grant.reference          |  2 +-
 .../0_stateless/02250_ON_CLUSTER_grant.sh     |  2 +-
 .../02561_temporary_table_grants.sh           |  8 +--
 21 files changed, 99 insertions(+), 99 deletions(-)

diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp
index e277c49e39d..18469140f59 100644
--- a/src/Access/ContextAccess.cpp
+++ b/src/Access/ContextAccess.cpp
@@ -550,12 +550,12 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
             return access_denied(ErrorCodes::ACCESS_DENIED,
                 "{}: Not enough privileges. "
                 "The required privileges have been granted, but without grant option. "
-                "To execute this query it's necessary to have grant {} WITH GRANT OPTION",
+                "To execute this query, it's necessary to have the grant {} WITH GRANT OPTION",
                 AccessRightsElement{flags, args...}.toStringWithoutOptions());
         }
 
         return access_denied(ErrorCodes::ACCESS_DENIED,
-            "{}: Not enough privileges. To execute this query it's necessary to have grant {}",
+            "{}: Not enough privileges. To execute this query, it's necessary to have the grant {}",
             AccessRightsElement{flags, args...}.toStringWithoutOptions() + (grant_option ? " WITH GRANT OPTION" : ""));
     }
 
@@ -756,11 +756,11 @@ bool ContextAccess::checkAdminOptionImplHelper(const Container & role_ids, const
                 show_error(ErrorCodes::ACCESS_DENIED,
                            "Not enough privileges. "
                            "Role {} is granted, but without ADMIN option. "
-                           "To execute this query it's necessary to have the role {} granted with ADMIN option.",
+                           "to execute this query, it's necessary to have the role {} granted with ADMIN option.",
                            backQuote(*role_name), backQuoteIfNeed(*role_name));
             else
                 show_error(ErrorCodes::ACCESS_DENIED, "Not enough privileges. "
-                           "To execute this query it's necessary to have the role {} granted with ADMIN option.",
+                           "to execute this query, it's necessary to have the role {} granted with ADMIN option.",
                            backQuoteIfNeed(*role_name));
         }
 
diff --git a/src/Interpreters/Access/InterpreterGrantQuery.cpp b/src/Interpreters/Access/InterpreterGrantQuery.cpp
index 77474d68795..1a8268b9b1b 100644
--- a/src/Interpreters/Access/InterpreterGrantQuery.cpp
+++ b/src/Interpreters/Access/InterpreterGrantQuery.cpp
@@ -139,7 +139,7 @@ namespace
         /// For example, to execute
         /// GRANT ALL ON mydb.* TO role1
         /// REVOKE ALL ON *.* FROM role1
-        /// the current user needs to have grants only on the 'mydb' database.
+        /// the current user needs to have the grants only on the 'mydb' database.
         AccessRights all_granted_access;
         for (const auto & id : grantees_from_query)
         {
diff --git a/src/Interpreters/InterpreterKillQueryQuery.cpp b/src/Interpreters/InterpreterKillQueryQuery.cpp
index 3330159aff5..e7dbd34c543 100644
--- a/src/Interpreters/InterpreterKillQueryQuery.cpp
+++ b/src/Interpreters/InterpreterKillQueryQuery.cpp
@@ -296,7 +296,7 @@ BlockIO InterpreterKillQueryQuery::execute()
 
         if (res_columns[0]->empty() && access_denied)
             throw Exception(ErrorCodes::ACCESS_DENIED, "Not allowed to kill mutation. "
-                "To execute this query it's necessary to have the grant {}", required_access_rights.toString());
+                "to execute this query, it's necessary to have the grant {}", required_access_rights.toString());
 
         res_io.pipeline = QueryPipeline(Pipe(std::make_shared<SourceFromSingleChunk>(header.cloneWithColumns(std::move(res_columns)))));
 
@@ -360,7 +360,7 @@ BlockIO InterpreterKillQueryQuery::execute()
 
         if (res_columns[0]->empty() && access_denied)
             throw Exception(ErrorCodes::ACCESS_DENIED, "Not allowed to kill move partition. "
-                "To execute this query it's necessary to have the grant {}", required_access_rights.toString());
+                "to execute this query, it's necessary to have the grant {}", required_access_rights.toString());
 
         res_io.pipeline = QueryPipeline(Pipe(std::make_shared<SourceFromSingleChunk>(header.cloneWithColumns(std::move(res_columns)))));
 
diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp
index 20fca1b1e76..fb04cc77955 100644
--- a/src/Interpreters/InterpreterSelectQuery.cpp
+++ b/src/Interpreters/InterpreterSelectQuery.cpp
@@ -299,7 +299,7 @@ void checkAccessRightsForSelect(
         }
         throw Exception(
             ErrorCodes::ACCESS_DENIED,
-            "{}: Not enough privileges. To execute this query it's necessary to have grant SELECT for at least one column on {}",
+            "{}: Not enough privileges. To execute this query, it's necessary to have the grant SELECT for at least one column on {}",
             context->getUserName(),
             table_id.getFullTableName());
     }
diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp
index c118fccded4..56a48ce8328 100644
--- a/src/Planner/PlannerJoinTree.cpp
+++ b/src/Planner/PlannerJoinTree.cpp
@@ -101,7 +101,7 @@ void checkAccessRights(const TableNode & table_node, const Names & column_names,
         }
 
         throw Exception(ErrorCodes::ACCESS_DENIED,
-            "{}: Not enough privileges. To execute this query it's necessary to have grant SELECT for at least one column on {}",
+            "{}: Not enough privileges. To execute this query, it's necessary to have the grant SELECT for at least one column on {}",
             query_context->getUserName(),
             storage_id.getFullTableName());
     }
diff --git a/tests/integration/test_access_for_functions/test.py b/tests/integration/test_access_for_functions/test.py
index be4d71502d2..5069468110c 100644
--- a/tests/integration/test_access_for_functions/test.py
+++ b/tests/integration/test_access_for_functions/test.py
@@ -22,7 +22,7 @@ def test_access_rights_for_function():
     instance.query("CREATE USER A")
     instance.query("CREATE USER B")
     assert (
-        "it's necessary to have grant CREATE FUNCTION ON *.*"
+        "it's necessary to have the grant CREATE FUNCTION ON *.*"
         in instance.query_and_get_error(create_function_query, user="A")
     )
 
@@ -32,7 +32,7 @@ def test_access_rights_for_function():
     assert instance.query("SELECT MySum(1, 2)") == "3\n"
 
     assert (
-        "it's necessary to have grant DROP FUNCTION ON *.*"
+        "it's necessary to have the grant DROP FUNCTION ON *.*"
         in instance.query_and_get_error("DROP FUNCTION MySum", user="B")
     )
 
@@ -44,7 +44,7 @@ def test_access_rights_for_function():
 
     instance.query("REVOKE CREATE FUNCTION ON *.* FROM A")
     assert (
-        "it's necessary to have grant CREATE FUNCTION ON *.*"
+        "it's necessary to have the grant CREATE FUNCTION ON *.*"
         in instance.query_and_get_error(create_function_query, user="A")
     )
 
diff --git a/tests/integration/test_backup_restore_new/test.py b/tests/integration/test_backup_restore_new/test.py
index c19cca4126a..a6ba0c8918f 100644
--- a/tests/integration/test_backup_restore_new/test.py
+++ b/tests/integration/test_backup_restore_new/test.py
@@ -882,7 +882,7 @@ def test_required_privileges():
     instance.query("CREATE USER u1")
 
     backup_name = new_backup_name()
-    expected_error = "necessary to have grant BACKUP ON test.table"
+    expected_error = "necessary to have the grant BACKUP ON test.table"
     assert expected_error in instance.query_and_get_error(
         f"BACKUP TABLE test.table TO {backup_name}", user="u1"
     )
@@ -890,12 +890,12 @@ def test_required_privileges():
     instance.query("GRANT BACKUP ON test.table TO u1")
     instance.query(f"BACKUP TABLE test.table TO {backup_name}", user="u1")
 
-    expected_error = "necessary to have grant INSERT, CREATE TABLE ON test.table"
+    expected_error = "necessary to have the grant INSERT, CREATE TABLE ON test.table"
     assert expected_error in instance.query_and_get_error(
         f"RESTORE TABLE test.table FROM {backup_name}", user="u1"
     )
 
-    expected_error = "necessary to have grant INSERT, CREATE TABLE ON test.table2"
+    expected_error = "necessary to have the grant INSERT, CREATE TABLE ON test.table2"
     assert expected_error in instance.query_and_get_error(
         f"RESTORE TABLE test.table AS test.table2 FROM {backup_name}", user="u1"
     )
@@ -907,7 +907,7 @@ def test_required_privileges():
 
     instance.query("DROP TABLE test.table")
 
-    expected_error = "necessary to have grant INSERT, CREATE TABLE ON test.table"
+    expected_error = "necessary to have the grant INSERT, CREATE TABLE ON test.table"
     assert expected_error in instance.query_and_get_error(
         f"RESTORE ALL FROM {backup_name}", user="u1"
     )
@@ -1014,14 +1014,14 @@ def test_system_users_required_privileges():
 
     backup_name = new_backup_name()
 
-    expected_error = "necessary to have grant BACKUP ON system.users"
+    expected_error = "necessary to have the grant BACKUP ON system.users"
     assert expected_error in instance.query_and_get_error(
         f"BACKUP TABLE system.users, TABLE system.roles TO {backup_name}", user="u2"
     )
 
     instance.query("GRANT BACKUP ON system.users TO u2")
 
-    expected_error = "necessary to have grant BACKUP ON system.roles"
+    expected_error = "necessary to have the grant BACKUP ON system.roles"
     assert expected_error in instance.query_and_get_error(
         f"BACKUP TABLE system.users, TABLE system.roles TO {backup_name}", user="u2"
     )
@@ -1035,7 +1035,7 @@ def test_system_users_required_privileges():
     instance.query("DROP ROLE r1")
 
     expected_error = (
-        "necessary to have grant CREATE USER, CREATE ROLE, ROLE ADMIN ON *.*"
+        "necessary to have the grant CREATE USER, CREATE ROLE, ROLE ADMIN ON *.*"
     )
     assert expected_error in instance.query_and_get_error(
         f"RESTORE ALL FROM {backup_name}", user="u2"
@@ -1043,7 +1043,7 @@ def test_system_users_required_privileges():
 
     instance.query("GRANT CREATE USER, CREATE ROLE, ROLE ADMIN ON *.* TO u2")
 
-    expected_error = "necessary to have grant SELECT ON test.* WITH GRANT OPTION"
+    expected_error = "necessary to have the grant SELECT ON test.* WITH GRANT OPTION"
     assert expected_error in instance.query_and_get_error(
         f"RESTORE ALL FROM {backup_name}", user="u2"
     )
diff --git a/tests/integration/test_backup_restore_on_cluster/test.py b/tests/integration/test_backup_restore_on_cluster/test.py
index 39496b8a5c8..366b3279cf9 100644
--- a/tests/integration/test_backup_restore_on_cluster/test.py
+++ b/tests/integration/test_backup_restore_on_cluster/test.py
@@ -561,7 +561,7 @@ def test_required_privileges():
     node1.query("GRANT CLUSTER ON *.* TO u1")
 
     backup_name = new_backup_name()
-    expected_error = "necessary to have grant BACKUP ON default.tbl"
+    expected_error = "necessary to have the grant BACKUP ON default.tbl"
     assert expected_error in node1.query_and_get_error(
         f"BACKUP TABLE tbl ON CLUSTER 'cluster' TO {backup_name}", user="u1"
     )
@@ -571,7 +571,7 @@ def test_required_privileges():
 
     node1.query(f"DROP TABLE tbl ON CLUSTER 'cluster' SYNC")
 
-    expected_error = "necessary to have grant INSERT, CREATE TABLE ON default.tbl2"
+    expected_error = "necessary to have the grant INSERT, CREATE TABLE ON default.tbl2"
     assert expected_error in node1.query_and_get_error(
         f"RESTORE TABLE tbl AS tbl2 ON CLUSTER 'cluster' FROM {backup_name}", user="u1"
     )
@@ -587,7 +587,7 @@ def test_required_privileges():
     node1.query(f"DROP TABLE tbl2 ON CLUSTER 'cluster' SYNC")
     node1.query("REVOKE ALL FROM u1")
 
-    expected_error = "necessary to have grant INSERT, CREATE TABLE ON default.tbl"
+    expected_error = "necessary to have the grant INSERT, CREATE TABLE ON default.tbl"
     assert expected_error in node1.query_and_get_error(
         f"RESTORE ALL ON CLUSTER 'cluster' FROM {backup_name}", user="u1"
     )
@@ -607,7 +607,7 @@ def test_system_users():
     node1.query("CREATE USER u2 SETTINGS allow_backup=false")
     node1.query("GRANT CLUSTER ON *.* TO u2")
 
-    expected_error = "necessary to have grant BACKUP ON system.users"
+    expected_error = "necessary to have the grant BACKUP ON system.users"
     assert expected_error in node1.query_and_get_error(
         f"BACKUP TABLE system.users ON CLUSTER 'cluster' TO {backup_name}", user="u2"
     )
@@ -619,14 +619,14 @@ def test_system_users():
 
     node1.query("DROP USER u1")
 
-    expected_error = "necessary to have grant CREATE USER ON *.*"
+    expected_error = "necessary to have the grant CREATE USER ON *.*"
     assert expected_error in node1.query_and_get_error(
         f"RESTORE TABLE system.users ON CLUSTER 'cluster' FROM {backup_name}", user="u2"
     )
 
     node1.query("GRANT CREATE USER ON *.* TO u2")
 
-    expected_error = "necessary to have grant SELECT ON default.tbl WITH GRANT OPTION"
+    expected_error = "necessary to have the grant SELECT ON default.tbl WITH GRANT OPTION"
     assert expected_error in node1.query_and_get_error(
         f"RESTORE TABLE system.users ON CLUSTER 'cluster' FROM {backup_name}", user="u2"
     )
diff --git a/tests/integration/test_disabled_access_control_improvements/test_select_from_system_tables.py b/tests/integration/test_disabled_access_control_improvements/test_select_from_system_tables.py
index 5d760c9fc2c..894464fd813 100644
--- a/tests/integration/test_disabled_access_control_improvements/test_select_from_system_tables.py
+++ b/tests/integration/test_disabled_access_control_improvements/test_select_from_system_tables.py
@@ -43,7 +43,7 @@ def test_system_db():
     assert node.query("SELECT count() FROM system.tables WHERE name='table2'") == "1\n"
 
     assert node.query("SELECT count()>0 FROM system.settings", user="another") == "1\n"
-    expected_error = "necessary to have grant SHOW USERS ON *.*"
+    expected_error = "necessary to have the grant SHOW USERS ON *.*"
     assert expected_error in node.query_and_get_error(
         "SELECT count()>0 FROM system.users", user="another"
     )
@@ -62,7 +62,7 @@ def test_system_db():
     )
 
     assert node.query("SELECT count()>0 FROM system.settings", user="sqluser") == "1\n"
-    expected_error = "necessary to have grant SHOW USERS ON *.*"
+    expected_error = "necessary to have the grant SHOW USERS ON *.*"
     assert expected_error in node.query_and_get_error(
         "SELECT count()>0 FROM system.users", user="sqluser"
     )
diff --git a/tests/integration/test_grant_and_revoke/test.py b/tests/integration/test_grant_and_revoke/test.py
index 8a8eb11d91f..a1db5a25207 100644
--- a/tests/integration/test_grant_and_revoke/test.py
+++ b/tests/integration/test_grant_and_revoke/test.py
@@ -449,7 +449,7 @@ def test_introspection():
         ]
     )
 
-    expected_error = "necessary to have grant SHOW USERS"
+    expected_error = "necessary to have the grant SHOW USERS"
     assert expected_error in instance.query_and_get_error("SHOW GRANTS FOR B", user="A")
 
     expected_access1 = (
@@ -556,7 +556,7 @@ def test_grant_with_replace_option():
     )
 
     expected_error = (
-        "it's necessary to have grant INSERT ON test.table WITH GRANT OPTION"
+        "it's necessary to have the grant INSERT ON test.table WITH GRANT OPTION"
     )
     assert expected_error in instance.query_and_get_error(
         "GRANT INSERT ON test.table TO B WITH REPLACE OPTION", user="A"
@@ -568,7 +568,7 @@ def test_grant_with_replace_option():
 
     instance.query("GRANT INSERT ON test.table TO A WITH GRANT OPTION")
     expected_error = (
-        "it's necessary to have grant SELECT ON test.table WITH GRANT OPTION"
+        "it's necessary to have the grant SELECT ON test.table WITH GRANT OPTION"
     )
     assert expected_error in instance.query_and_get_error(
         "GRANT INSERT ON test.table TO B WITH REPLACE OPTION", user="A"
diff --git a/tests/integration/test_named_collections/test.py b/tests/integration/test_named_collections/test.py
index 5f5657fad54..dee1209f2aa 100644
--- a/tests/integration/test_named_collections/test.py
+++ b/tests/integration/test_named_collections/test.py
@@ -344,7 +344,7 @@ def test_granular_access_create_alter_drop_query(cluster):
     )
 
     assert (
-        "DB::Exception: kek: Not enough privileges. To execute this query it's necessary to have grant CREATE NAMED COLLECTION"
+        "DB::Exception: kek: Not enough privileges. to execute this query, it's necessary to have the grant CREATE NAMED COLLECTION"
         in node.query_and_get_error(
             "CREATE NAMED COLLECTION collection2 AS key1=1, key2='value2'", user="kek"
         )
@@ -370,7 +370,7 @@ def test_granular_access_create_alter_drop_query(cluster):
     )
 
     assert (
-        "DB::Exception: kek: Not enough privileges. To execute this query it's necessary to have grant ALTER NAMED COLLECTION"
+        "DB::Exception: kek: Not enough privileges. to execute this query, it's necessary to have the grant ALTER NAMED COLLECTION"
         in node.query_and_get_error(
             "ALTER NAMED COLLECTION collection2 SET key1=2", user="kek"
         )
@@ -385,14 +385,14 @@ def test_granular_access_create_alter_drop_query(cluster):
     )
     node.query("REVOKE alter named collection ON collection2 FROM kek")
     assert (
-        "DB::Exception: kek: Not enough privileges. To execute this query it's necessary to have grant ALTER NAMED COLLECTION"
+        "DB::Exception: kek: Not enough privileges. to execute this query, it's necessary to have the grant ALTER NAMED COLLECTION"
         in node.query_and_get_error(
             "ALTER NAMED COLLECTION collection2 SET key1=3", user="kek"
         )
     )
 
     assert (
-        "DB::Exception: kek: Not enough privileges. To execute this query it's necessary to have grant DROP NAMED COLLECTION"
+        "DB::Exception: kek: Not enough privileges. to execute this query, it's necessary to have the grant DROP NAMED COLLECTION"
         in node.query_and_get_error("DROP NAMED COLLECTION collection2", user="kek")
     )
     node.query("GRANT drop named collection ON collection2 TO kek")
diff --git a/tests/integration/test_row_policy/test.py b/tests/integration/test_row_policy/test.py
index 1933823f5d2..7fb39aece63 100644
--- a/tests/integration/test_row_policy/test.py
+++ b/tests/integration/test_row_policy/test.py
@@ -636,7 +636,7 @@ def test_grant_create_row_policy():
     assert node.query("SHOW POLICIES") == ""
     node.query("CREATE USER X")
 
-    expected_error = "necessary to have grant CREATE ROW POLICY ON mydb.filtered_table1"
+    expected_error = "necessary to have the grant CREATE ROW POLICY ON mydb.filtered_table1"
     assert expected_error in node.query_and_get_error(
         "CREATE POLICY pA ON mydb.filtered_table1 FOR SELECT USING a<b", user="X"
     )
@@ -644,12 +644,12 @@ def test_grant_create_row_policy():
     node.query(
         "CREATE POLICY pA ON mydb.filtered_table1 FOR SELECT USING a<b", user="X"
     )
-    expected_error = "necessary to have grant CREATE ROW POLICY ON mydb.filtered_table2"
+    expected_error = "necessary to have the grant CREATE ROW POLICY ON mydb.filtered_table2"
     assert expected_error in node.query_and_get_error(
         "CREATE POLICY pA ON mydb.filtered_table2 FOR SELECT USING a<b", user="X"
     )
 
-    expected_error = "necessary to have grant ALTER ROW POLICY ON mydb.filtered_table1"
+    expected_error = "necessary to have the grant ALTER ROW POLICY ON mydb.filtered_table1"
     assert expected_error in node.query_and_get_error(
         "ALTER POLICY pA ON mydb.filtered_table1 FOR SELECT USING a==b", user="X"
     )
@@ -657,25 +657,25 @@ def test_grant_create_row_policy():
     node.query(
         "ALTER POLICY pA ON mydb.filtered_table1 FOR SELECT USING a==b", user="X"
     )
-    expected_error = "necessary to have grant ALTER ROW POLICY ON mydb.filtered_table2"
+    expected_error = "necessary to have the grant ALTER ROW POLICY ON mydb.filtered_table2"
     assert expected_error in node.query_and_get_error(
         "ALTER POLICY pA ON mydb.filtered_table2 FOR SELECT USING a==b", user="X"
     )
 
-    expected_error = "necessary to have grant DROP ROW POLICY ON mydb.filtered_table1"
+    expected_error = "necessary to have the grant DROP ROW POLICY ON mydb.filtered_table1"
     assert expected_error in node.query_and_get_error(
         "DROP POLICY pA ON mydb.filtered_table1", user="X"
     )
     node.query("GRANT DROP POLICY ON mydb.filtered_table1 TO X")
     node.query("DROP POLICY pA ON mydb.filtered_table1", user="X")
-    expected_error = "necessary to have grant DROP ROW POLICY ON mydb.filtered_table2"
+    expected_error = "necessary to have the grant DROP ROW POLICY ON mydb.filtered_table2"
     assert expected_error in node.query_and_get_error(
         "DROP POLICY pA ON mydb.filtered_table2", user="X"
     )
 
     node.query("REVOKE ALL ON *.* FROM X")
 
-    expected_error = "necessary to have grant CREATE ROW POLICY ON mydb.filtered_table1"
+    expected_error = "necessary to have the grant CREATE ROW POLICY ON mydb.filtered_table1"
     assert expected_error in node.query_and_get_error(
         "CREATE POLICY pA ON mydb.filtered_table1 FOR SELECT USING a<b", user="X"
     )
@@ -684,7 +684,7 @@ def test_grant_create_row_policy():
         "CREATE POLICY pA ON mydb.filtered_table1 FOR SELECT USING a<b", user="X"
     )
 
-    expected_error = "necessary to have grant ALTER ROW POLICY ON mydb.filtered_table1"
+    expected_error = "necessary to have the grant ALTER ROW POLICY ON mydb.filtered_table1"
     assert expected_error in node.query_and_get_error(
         "ALTER POLICY pA ON mydb.filtered_table1 FOR SELECT USING a==b", user="X"
     )
@@ -693,7 +693,7 @@ def test_grant_create_row_policy():
         "ALTER POLICY pA ON mydb.filtered_table1 FOR SELECT USING a==b", user="X"
     )
 
-    expected_error = "necessary to have grant DROP ROW POLICY ON mydb.filtered_table1"
+    expected_error = "necessary to have the grant DROP ROW POLICY ON mydb.filtered_table1"
     assert expected_error in node.query_and_get_error(
         "DROP POLICY pA ON mydb.filtered_table1", user="X"
     )
diff --git a/tests/integration/test_select_access_rights/test_from_system_tables.py b/tests/integration/test_select_access_rights/test_from_system_tables.py
index cd29183561a..a4dfa662601 100644
--- a/tests/integration/test_select_access_rights/test_from_system_tables.py
+++ b/tests/integration/test_select_access_rights/test_from_system_tables.py
@@ -44,14 +44,14 @@ def test_system_db():
     assert node.query("SELECT count()>0 FROM system.settings", user="another") == "1\n"
 
     expected_error = (
-        "necessary to have grant SELECT for at least one column on system.users"
+        "necessary to have the grant SELECT for at least one column on system.users"
     )
     assert expected_error in node.query_and_get_error(
         "SELECT count()>0 FROM system.users", user="another"
     )
 
     expected_error = (
-        "necessary to have grant SELECT for at least one column on system.clusters"
+        "necessary to have the grant SELECT for at least one column on system.clusters"
     )
     assert expected_error in node.query_and_get_error(
         "SELECT count()>0 FROM system.clusters", user="another"
@@ -72,14 +72,14 @@ def test_system_db():
     assert node.query("SELECT count()>0 FROM system.settings", user="sqluser") == "1\n"
 
     expected_error = (
-        "necessary to have grant SELECT for at least one column on system.users"
+        "necessary to have the grant SELECT for at least one column on system.users"
     )
     assert expected_error in node.query_and_get_error(
         "SELECT count()>0 FROM system.users", user="sqluser"
     )
 
     expected_error = (
-        "necessary to have grant SELECT for at least one column on system.clusters"
+        "necessary to have the grant SELECT for at least one column on system.clusters"
     )
     assert node.query_and_get_error(
         "SELECT count()>0 FROM system.clusters", user="sqluser"
@@ -138,7 +138,7 @@ def test_information_schema():
     )
 
     expected_error = (
-        "necessary to have grant SELECT(table_name) ON information_schema.tables"
+        "necessary to have the grant SELECT(table_name) ON information_schema.tables"
     )
     assert expected_error in node.query_and_get_error(
         "SELECT count() FROM information_schema.tables WHERE table_name='table1'",
diff --git a/tests/integration/test_select_access_rights/test_main.py b/tests/integration/test_select_access_rights/test_main.py
index 76940cdadb4..eedecc2d30c 100644
--- a/tests/integration/test_select_access_rights/test_main.py
+++ b/tests/integration/test_select_access_rights/test_main.py
@@ -30,7 +30,7 @@ def test_select_single_column():
 
     select_query = "SELECT a FROM table1"
     assert (
-        "it's necessary to have grant SELECT(a) ON default.table1"
+        "it's necessary to have the grant SELECT(a) ON default.table1"
         in instance.query_and_get_error(select_query, user="A")
     )
 
@@ -39,7 +39,7 @@ def test_select_single_column():
 
     instance.query("REVOKE SELECT(a) ON default.table1 FROM A")
     assert (
-        "it's necessary to have grant SELECT(a) ON default.table1"
+        "it's necessary to have the grant SELECT(a) ON default.table1"
         in instance.query_and_get_error(select_query, user="A")
     )
 
@@ -51,7 +51,7 @@ def test_select_single_column_with_table_grant():
 
     select_query = "SELECT a FROM table1"
     assert (
-        "it's necessary to have grant SELECT(a) ON default.table1"
+        "it's necessary to have the grant SELECT(a) ON default.table1"
         in instance.query_and_get_error(select_query, user="A")
     )
 
@@ -60,7 +60,7 @@ def test_select_single_column_with_table_grant():
 
     instance.query("REVOKE SELECT(a) ON default.table1 FROM A")
     assert (
-        "it's necessary to have grant SELECT(a) ON default.table1"
+        "it's necessary to have the grant SELECT(a) ON default.table1"
         in instance.query_and_get_error(select_query, user="A")
     )
 
@@ -72,19 +72,19 @@ def test_select_all_columns():
 
     select_query = "SELECT * FROM table1"
     assert (
-        "it's necessary to have grant SELECT(d, a, b) ON default.table1"
+        "it's necessary to have the grant SELECT(d, a, b) ON default.table1"
         in instance.query_and_get_error(select_query, user="A")
     )
 
     instance.query("GRANT SELECT(d) ON default.table1 TO A")
     assert (
-        "it's necessary to have grant SELECT(d, a, b) ON default.table1"
+        "it's necessary to have the grant SELECT(d, a, b) ON default.table1"
         in instance.query_and_get_error(select_query, user="A")
     )
 
     instance.query("GRANT SELECT(a) ON default.table1 TO A")
     assert (
-        "it's necessary to have grant SELECT(d, a, b) ON default.table1"
+        "it's necessary to have the grant SELECT(d, a, b) ON default.table1"
         in instance.query_and_get_error(select_query, user="A")
     )
 
@@ -99,7 +99,7 @@ def test_select_all_columns_with_table_grant():
 
     select_query = "SELECT * FROM table1"
     assert (
-        "it's necessary to have grant SELECT(d, a, b) ON default.table1"
+        "it's necessary to have the grant SELECT(d, a, b) ON default.table1"
         in instance.query_and_get_error(select_query, user="A")
     )
 
@@ -114,7 +114,7 @@ def test_alias():
 
     select_query = "SELECT x, y, x + y AS s FROM table1"
     assert (
-        "it's necessary to have grant SELECT(x, y) ON default.table1"
+        "it's necessary to have the grant SELECT(x, y) ON default.table1"
         in instance.query_and_get_error(select_query, user="A")
     )
 
@@ -129,7 +129,7 @@ def test_alias_columns():
 
     select_query = "SELECT * FROM table1"
     assert (
-        "it's necessary to have grant SELECT(x, y) ON default.table1"
+        "it's necessary to have the grant SELECT(x, y) ON default.table1"
         in instance.query_and_get_error(select_query, user="A")
     )
 
@@ -138,7 +138,7 @@ def test_alias_columns():
 
     select_query = "SELECT s FROM table1"
     assert (
-        "it's necessary to have grant SELECT(s) ON default.table1"
+        "it's necessary to have the grant SELECT(s) ON default.table1"
         in instance.query_and_get_error(select_query, user="A")
     )
 
@@ -156,7 +156,7 @@ def test_materialized_columns():
 
     select_query = "SELECT * FROM table1"
     assert (
-        "it's necessary to have grant SELECT(x, y) ON default.table1"
+        "it's necessary to have the grant SELECT(x, y) ON default.table1"
         in instance.query_and_get_error(select_query, user="A")
     )
 
@@ -165,7 +165,7 @@ def test_materialized_columns():
 
     select_query = "SELECT p FROM table1"
     assert (
-        "it's necessary to have grant SELECT(p) ON default.table1"
+        "it's necessary to have the grant SELECT(p) ON default.table1"
         in instance.query_and_get_error(select_query, user="A")
     )
 
@@ -186,13 +186,13 @@ def test_select_join():
 
     select_query = "SELECT * FROM table1 JOIN table2 USING(d)"
     assert (
-        "it's necessary to have grant SELECT(d, x, y) ON default.table2"
+        "it's necessary to have the grant SELECT(d, x, y) ON default.table2"
         in instance.query_and_get_error(select_query, user="A")
     )
 
     instance.query("GRANT SELECT(d, x, y) ON default.table2 TO A")
     assert (
-        "it's necessary to have grant SELECT(d, a, b) ON default.table1"
+        "it's necessary to have the grant SELECT(d, a, b) ON default.table1"
         in instance.query_and_get_error(select_query, user="A")
     )
 
@@ -201,7 +201,7 @@ def test_select_join():
 
     instance.query("REVOKE SELECT ON default.table2 FROM A")
     assert (
-        "it's necessary to have grant SELECT(d, x, y) ON default.table2"
+        "it's necessary to have the grant SELECT(d, x, y) ON default.table2"
         in instance.query_and_get_error(select_query, user="A")
     )
 
@@ -216,13 +216,13 @@ def test_select_union():
 
     select_query = "SELECT * FROM table1 UNION ALL SELECT * FROM table2"
     assert (
-        "it's necessary to have grant SELECT(a, b) ON default.table1"
+        "it's necessary to have the grant SELECT(a, b) ON default.table1"
         in instance.query_and_get_error(select_query, user="A")
     )
 
     instance.query("GRANT SELECT(a, b) ON default.table1 TO A")
     assert (
-        "it's necessary to have grant SELECT(a, b) ON default.table2"
+        "it's necessary to have the grant SELECT(a, b) ON default.table2"
         in instance.query_and_get_error(select_query, user="A")
     )
 
@@ -231,7 +231,7 @@ def test_select_union():
 
     instance.query("REVOKE SELECT ON default.table1 FROM A")
     assert (
-        "it's necessary to have grant SELECT(a, b) ON default.table1"
+        "it's necessary to have the grant SELECT(a, b) ON default.table1"
         in instance.query_and_get_error(select_query, user="A")
     )
 
@@ -243,7 +243,7 @@ def test_select_count():
 
     select_query = "SELECT count() FROM table1"
     assert (
-        "it's necessary to have grant SELECT for at least one column on default.table1"
+        "it's necessary to have the grant SELECT for at least one column on default.table1"
         in instance.query_and_get_error(select_query, user="A")
     )
 
@@ -252,7 +252,7 @@ def test_select_count():
 
     instance.query("REVOKE SELECT(x) ON default.table1 FROM A")
     assert (
-        "it's necessary to have grant SELECT for at least one column on default.table1"
+        "it's necessary to have the grant SELECT for at least one column on default.table1"
         in instance.query_and_get_error(select_query, user="A")
     )
 
@@ -261,7 +261,7 @@ def test_select_count():
 
     instance.query("REVOKE SELECT(y) ON default.table1 FROM A")
     assert (
-        "it's necessary to have grant SELECT for at least one column on default.table1"
+        "it's necessary to have the grant SELECT for at least one column on default.table1"
         in instance.query_and_get_error(select_query, user="A")
     )
 
@@ -279,7 +279,7 @@ def test_select_where():
 
     select_query = "SELECT a FROM table1 WHERE b = 0"
     assert (
-        "it's necessary to have grant SELECT(a, b) ON default.table1"
+        "it's necessary to have the grant SELECT(a, b) ON default.table1"
         in instance.query_and_get_error(select_query, user="A")
     )
 
@@ -288,7 +288,7 @@ def test_select_where():
 
     instance.query("REVOKE SELECT ON default.table1 FROM A")
     assert (
-        "it's necessary to have grant SELECT(a, b) ON default.table1"
+        "it's necessary to have the grant SELECT(a, b) ON default.table1"
         in instance.query_and_get_error(select_query, user="A")
     )
 
@@ -306,7 +306,7 @@ def test_select_prewhere():
 
     select_query = "SELECT a FROM table1 PREWHERE b = 0"
     assert (
-        "it's necessary to have grant SELECT(a, b) ON default.table1"
+        "it's necessary to have the grant SELECT(a, b) ON default.table1"
         in instance.query_and_get_error(select_query, user="A")
     )
 
@@ -315,7 +315,7 @@ def test_select_prewhere():
 
     instance.query("REVOKE SELECT ON default.table1 FROM A")
     assert (
-        "it's necessary to have grant SELECT(a, b) ON default.table1"
+        "it's necessary to have the grant SELECT(a, b) ON default.table1"
         in instance.query_and_get_error(select_query, user="A")
     )
 
@@ -334,11 +334,11 @@ def test_select_with_row_policy():
     select_query = "SELECT a FROM table1"
     select_query2 = "SELECT count() FROM table1"
     assert (
-        "it's necessary to have grant SELECT(a) ON default.table1"
+        "it's necessary to have the grant SELECT(a) ON default.table1"
         in instance.query_and_get_error(select_query, user="A")
     )
     assert (
-        "it's necessary to have grant SELECT for at least one column on default.table1"
+        "it's necessary to have the grant SELECT for at least one column on default.table1"
         in instance.query_and_get_error(select_query2, user="A")
     )
 
@@ -348,10 +348,10 @@ def test_select_with_row_policy():
 
     instance.query("REVOKE SELECT(a) ON default.table1 FROM A")
     assert (
-        "it's necessary to have grant SELECT(a) ON default.table1"
+        "it's necessary to have the grant SELECT(a) ON default.table1"
         in instance.query_and_get_error(select_query, user="A")
     )
     assert (
-        "it's necessary to have grant SELECT for at least one column on default.table1"
+        "it's necessary to have the grant SELECT for at least one column on default.table1"
         in instance.query_and_get_error(select_query2, user="A")
     )
diff --git a/tests/integration/test_settings_profile/test.py b/tests/integration/test_settings_profile/test.py
index 13acc3f1042..feccebc988d 100644
--- a/tests/integration/test_settings_profile/test.py
+++ b/tests/integration/test_settings_profile/test.py
@@ -589,10 +589,10 @@ def test_function_current_profiles():
 
 
 def test_allow_ddl():
-    assert "it's necessary to have grant" in instance.query_and_get_error(
+    assert "it's necessary to have the grant" in instance.query_and_get_error(
         "CREATE TABLE tbl(a Int32) ENGINE=Log", user="robin"
     )
-    assert "it's necessary to have grant" in instance.query_and_get_error(
+    assert "it's necessary to have the grant" in instance.query_and_get_error(
         "GRANT CREATE ON tbl TO robin", user="robin"
     )
     assert "DDL queries are prohibited" in instance.query_and_get_error(
@@ -615,10 +615,10 @@ def test_allow_introspection():
     assert "Introspection functions are disabled" in instance.query_and_get_error(
         "SELECT demangle('a')"
     )
-    assert "it's necessary to have grant" in instance.query_and_get_error(
+    assert "it's necessary to have the grant" in instance.query_and_get_error(
         "SELECT demangle('a')", user="robin"
     )
-    assert "it's necessary to have grant" in instance.query_and_get_error(
+    assert "it's necessary to have the grant" in instance.query_and_get_error(
         "SELECT demangle('a')",
         user="robin",
         settings={"allow_introspection_functions": 1},
@@ -659,7 +659,7 @@ def test_allow_introspection():
         "REVOKE demangle ON *.* FROM robin",
         settings={"allow_introspection_functions": 1},
     )
-    assert "it's necessary to have grant" in instance.query_and_get_error(
+    assert "it's necessary to have the grant" in instance.query_and_get_error(
         "SELECT demangle('a')", user="robin"
     )
 
diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py
index e3eb3ae455b..295f8ca18a1 100644
--- a/tests/integration/test_storage_s3/test.py
+++ b/tests/integration/test_storage_s3/test.py
@@ -944,7 +944,7 @@ def test_predefined_connection_configuration(started_cluster):
         f"CREATE TABLE {name} (id UInt32) ENGINE = S3(s3_conf1, format='CSV')"
     )
     assert (
-        "To execute this query it's necessary to have grant NAMED COLLECTION ON s3_conf1"
+        "to execute this query, it's necessary to have the grant NAMED COLLECTION ON s3_conf1"
         in error
     )
     error = instance.query_and_get_error(
@@ -952,7 +952,7 @@ def test_predefined_connection_configuration(started_cluster):
         user="user",
     )
     assert (
-        "To execute this query it's necessary to have grant NAMED COLLECTION ON s3_conf1"
+        "to execute this query, it's necessary to have the grant NAMED COLLECTION ON s3_conf1"
         in error
     )
 
@@ -973,12 +973,12 @@ def test_predefined_connection_configuration(started_cluster):
 
     error = instance.query_and_get_error("SELECT * FROM s3(no_collection)")
     assert (
-        "To execute this query it's necessary to have grant NAMED COLLECTION ON no_collection"
+        "to execute this query, it's necessary to have the grant NAMED COLLECTION ON no_collection"
         in error
     )
     error = instance.query_and_get_error("SELECT * FROM s3(no_collection)", user="user")
     assert (
-        "To execute this query it's necessary to have grant NAMED COLLECTION ON no_collection"
+        "to execute this query, it's necessary to have the grant NAMED COLLECTION ON no_collection"
         in error
     )
     instance = started_cluster.instances["dummy"]  # has named collection access
diff --git a/tests/integration/test_storage_url/test.py b/tests/integration/test_storage_url/test.py
index 7f359078967..7ff7a871413 100644
--- a/tests/integration/test_storage_url/test.py
+++ b/tests/integration/test_storage_url/test.py
@@ -68,13 +68,13 @@ def test_url_cluster_with_named_collection():
 def test_table_function_url_access_rights():
     node1.query("CREATE USER OR REPLACE u1")
 
-    expected_error = "necessary to have grant CREATE TEMPORARY TABLE, URL ON *.*"
+    expected_error = "necessary to have the grant CREATE TEMPORARY TABLE, URL ON *.*"
     assert expected_error in node1.query_and_get_error(
         f"SELECT * FROM url('http://nginx:80/test_1', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32')",
         user="u1",
     )
 
-    expected_error = "necessary to have grant CREATE TEMPORARY TABLE, URL ON *.*"
+    expected_error = "necessary to have the grant CREATE TEMPORARY TABLE, URL ON *.*"
     assert expected_error in node1.query_and_get_error(
         f"SELECT * FROM url('http://nginx:80/test_1', 'TSV')", user="u1"
     )
@@ -89,7 +89,7 @@ def test_table_function_url_access_rights():
         user="u1",
     ) == TSV([["column1", "UInt32"], ["column2", "UInt32"], ["column3", "UInt32"]])
 
-    expected_error = "necessary to have grant URL ON *.*"
+    expected_error = "necessary to have the grant URL ON *.*"
     assert expected_error in node1.query_and_get_error(
         f"DESCRIBE TABLE url('http://nginx:80/test_1', 'TSV')", user="u1"
     )
diff --git a/tests/integration/test_table_functions_access_rights/test.py b/tests/integration/test_table_functions_access_rights/test.py
index 09a05122c07..b1d1a291bc5 100644
--- a/tests/integration/test_table_functions_access_rights/test.py
+++ b/tests/integration/test_table_functions_access_rights/test.py
@@ -40,7 +40,7 @@ def test_merge():
 
     instance.query("CREATE USER A")
     assert (
-        "it's necessary to have grant CREATE TEMPORARY TABLE ON *.*"
+        "it's necessary to have the grant CREATE TEMPORARY TABLE ON *.*"
         in instance.query_and_get_error(select_query, user="A")
     )
 
@@ -62,7 +62,7 @@ def test_merge():
     instance.query("GRANT SELECT ON default.table1 TO A")
     instance.query("GRANT INSERT ON default.table2 TO A")
     assert (
-        "it's necessary to have grant SELECT ON default.table2"
+        "it's necessary to have the grant SELECT ON default.table2"
         in instance.query_and_get_error(select_query, user="A")
     )
 
diff --git a/tests/queries/0_stateless/02250_ON_CLUSTER_grant.reference b/tests/queries/0_stateless/02250_ON_CLUSTER_grant.reference
index d43b658d5d4..07cc24628e9 100644
--- a/tests/queries/0_stateless/02250_ON_CLUSTER_grant.reference
+++ b/tests/queries/0_stateless/02250_ON_CLUSTER_grant.reference
@@ -1,3 +1,3 @@
 with_on_cluster_02250_ON_CLUSTER_grant_default
 without_on_cluster_02250_ON_CLUSTER_grant_default
-Not enough privileges. To execute this query it's necessary to have grant CLUSTER ON *.*. (ACCESS_DENIED)
+Not enough privileges. to execute this query, it's necessary to have the grant CLUSTER ON *.*. (ACCESS_DENIED)
diff --git a/tests/queries/0_stateless/02250_ON_CLUSTER_grant.sh b/tests/queries/0_stateless/02250_ON_CLUSTER_grant.sh
index 6129d0d39c6..4627bb81b19 100755
--- a/tests/queries/0_stateless/02250_ON_CLUSTER_grant.sh
+++ b/tests/queries/0_stateless/02250_ON_CLUSTER_grant.sh
@@ -27,5 +27,5 @@ echo "with_on_cluster_$CLICKHOUSE_TEST_UNIQUE_NAME"
 $CLICKHOUSE_CLIENT --user "with_on_cluster_$CLICKHOUSE_TEST_UNIQUE_NAME" -q "CREATE DATABASE IF NOT EXISTS db_with_on_cluster_$CLICKHOUSE_TEST_UNIQUE_NAME ON CLUSTER test_shard_localhost" >/dev/null
 echo "without_on_cluster_$CLICKHOUSE_TEST_UNIQUE_NAME"
 $CLICKHOUSE_CLIENT --user "without_on_cluster_$CLICKHOUSE_TEST_UNIQUE_NAME" -q "CREATE DATABASE IF NOT EXISTS db_without_on_cluster_$CLICKHOUSE_TEST_UNIQUE_NAME ON CLUSTER test_shard_localhost" |& {
-    grep -m1 -F -o "Not enough privileges. To execute this query it's necessary to have grant CLUSTER ON *.*. (ACCESS_DENIED)"
+    grep -m1 -F -o "Not enough privileges. to execute this query, it's necessary to have the grant CLUSTER ON *.*. (ACCESS_DENIED)"
 }
diff --git a/tests/queries/0_stateless/02561_temporary_table_grants.sh b/tests/queries/0_stateless/02561_temporary_table_grants.sh
index 6e0c96786e8..965e9687950 100755
--- a/tests/queries/0_stateless/02561_temporary_table_grants.sh
+++ b/tests/queries/0_stateless/02561_temporary_table_grants.sh
@@ -10,24 +10,24 @@ user=user_$CLICKHOUSE_TEST_UNIQUE_NAME
 $CLICKHOUSE_CLIENT --query "DROP USER IF EXISTS $user"
 $CLICKHOUSE_CLIENT --query "CREATE USER $user IDENTIFIED WITH PLAINTEXT_PASSWORD BY 'hello'"
 
-$CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_memory_02561(name String)" 2>&1 | grep -F "Not enough privileges. To execute this query it's necessary to have grant CREATE TEMPORARY TABLE" > /dev/null && echo "OK"
+$CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_memory_02561(name String)" 2>&1 | grep -F "Not enough privileges. to execute this query, it's necessary to have the grant CREATE TEMPORARY TABLE" > /dev/null && echo "OK"
 
 $CLICKHOUSE_CLIENT --query "GRANT CREATE TEMPORARY TABLE ON *.* TO $user"
 $CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_memory_02561(name String)"
 
-$CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_merge_tree_02561(name String) ENGINE = MergeTree() ORDER BY name" 2>&1 | grep -F "Not enough privileges. To execute this query it's necessary to have grant CREATE ARBITRARY TEMPORARY TABLE" > /dev/null && echo "OK"
+$CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_merge_tree_02561(name String) ENGINE = MergeTree() ORDER BY name" 2>&1 | grep -F "Not enough privileges. to execute this query, it's necessary to have the grant CREATE ARBITRARY TEMPORARY TABLE" > /dev/null && echo "OK"
 
 $CLICKHOUSE_CLIENT --query "GRANT CREATE ARBITRARY TEMPORARY TABLE ON *.* TO $user"
 
 $CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_merge_tree_02561(name String) ENGINE = MergeTree() ORDER BY name"
 
-$CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_file_02561(name String) ENGINE = File(TabSeparated)" 2>&1 | grep -F "Not enough privileges. To execute this query it's necessary to have grant FILE" > /dev/null && echo "OK"
+$CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_file_02561(name String) ENGINE = File(TabSeparated)" 2>&1 | grep -F "Not enough privileges. to execute this query, it's necessary to have the grant FILE" > /dev/null && echo "OK"
 
 $CLICKHOUSE_CLIENT --query "GRANT FILE ON *.* TO $user"
 
 $CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_file_02561(name String) ENGINE = File(TabSeparated)"
 
-$CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_url_02561(name String) ENGINE = URL('http://127.0.0.1:8123?query=select+12', 'RawBLOB')" 2>&1 | grep -F "Not enough privileges. To execute this query it's necessary to have grant URL" > /dev/null && echo "OK"
+$CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_url_02561(name String) ENGINE = URL('http://127.0.0.1:8123?query=select+12', 'RawBLOB')" 2>&1 | grep -F "Not enough privileges. to execute this query, it's necessary to have the grant URL" > /dev/null && echo "OK"
 
 $CLICKHOUSE_CLIENT --query "GRANT URL ON *.* TO $user"
 

From 5b0b5284c5c26fbc0a95e10e93adcbe2421fecdd Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Sun, 6 Aug 2023 12:57:48 +0000
Subject: [PATCH 093/249] Automatic style fix

---
 .../test_backup_restore_on_cluster/test.py    |  4 ++-
 tests/integration/test_row_policy/test.py     | 36 ++++++++++++++-----
 2 files changed, 30 insertions(+), 10 deletions(-)

diff --git a/tests/integration/test_backup_restore_on_cluster/test.py b/tests/integration/test_backup_restore_on_cluster/test.py
index 366b3279cf9..a9ebbeb66c2 100644
--- a/tests/integration/test_backup_restore_on_cluster/test.py
+++ b/tests/integration/test_backup_restore_on_cluster/test.py
@@ -626,7 +626,9 @@ def test_system_users():
 
     node1.query("GRANT CREATE USER ON *.* TO u2")
 
-    expected_error = "necessary to have the grant SELECT ON default.tbl WITH GRANT OPTION"
+    expected_error = (
+        "necessary to have the grant SELECT ON default.tbl WITH GRANT OPTION"
+    )
     assert expected_error in node1.query_and_get_error(
         f"RESTORE TABLE system.users ON CLUSTER 'cluster' FROM {backup_name}", user="u2"
     )
diff --git a/tests/integration/test_row_policy/test.py b/tests/integration/test_row_policy/test.py
index 7fb39aece63..e6fbf228556 100644
--- a/tests/integration/test_row_policy/test.py
+++ b/tests/integration/test_row_policy/test.py
@@ -636,7 +636,9 @@ def test_grant_create_row_policy():
     assert node.query("SHOW POLICIES") == ""
     node.query("CREATE USER X")
 
-    expected_error = "necessary to have the grant CREATE ROW POLICY ON mydb.filtered_table1"
+    expected_error = (
+        "necessary to have the grant CREATE ROW POLICY ON mydb.filtered_table1"
+    )
     assert expected_error in node.query_and_get_error(
         "CREATE POLICY pA ON mydb.filtered_table1 FOR SELECT USING a<b", user="X"
     )
@@ -644,12 +646,16 @@ def test_grant_create_row_policy():
     node.query(
         "CREATE POLICY pA ON mydb.filtered_table1 FOR SELECT USING a<b", user="X"
     )
-    expected_error = "necessary to have the grant CREATE ROW POLICY ON mydb.filtered_table2"
+    expected_error = (
+        "necessary to have the grant CREATE ROW POLICY ON mydb.filtered_table2"
+    )
     assert expected_error in node.query_and_get_error(
         "CREATE POLICY pA ON mydb.filtered_table2 FOR SELECT USING a<b", user="X"
     )
 
-    expected_error = "necessary to have the grant ALTER ROW POLICY ON mydb.filtered_table1"
+    expected_error = (
+        "necessary to have the grant ALTER ROW POLICY ON mydb.filtered_table1"
+    )
     assert expected_error in node.query_and_get_error(
         "ALTER POLICY pA ON mydb.filtered_table1 FOR SELECT USING a==b", user="X"
     )
@@ -657,25 +663,33 @@ def test_grant_create_row_policy():
     node.query(
         "ALTER POLICY pA ON mydb.filtered_table1 FOR SELECT USING a==b", user="X"
     )
-    expected_error = "necessary to have the grant ALTER ROW POLICY ON mydb.filtered_table2"
+    expected_error = (
+        "necessary to have the grant ALTER ROW POLICY ON mydb.filtered_table2"
+    )
     assert expected_error in node.query_and_get_error(
         "ALTER POLICY pA ON mydb.filtered_table2 FOR SELECT USING a==b", user="X"
     )
 
-    expected_error = "necessary to have the grant DROP ROW POLICY ON mydb.filtered_table1"
+    expected_error = (
+        "necessary to have the grant DROP ROW POLICY ON mydb.filtered_table1"
+    )
     assert expected_error in node.query_and_get_error(
         "DROP POLICY pA ON mydb.filtered_table1", user="X"
     )
     node.query("GRANT DROP POLICY ON mydb.filtered_table1 TO X")
     node.query("DROP POLICY pA ON mydb.filtered_table1", user="X")
-    expected_error = "necessary to have the grant DROP ROW POLICY ON mydb.filtered_table2"
+    expected_error = (
+        "necessary to have the grant DROP ROW POLICY ON mydb.filtered_table2"
+    )
     assert expected_error in node.query_and_get_error(
         "DROP POLICY pA ON mydb.filtered_table2", user="X"
     )
 
     node.query("REVOKE ALL ON *.* FROM X")
 
-    expected_error = "necessary to have the grant CREATE ROW POLICY ON mydb.filtered_table1"
+    expected_error = (
+        "necessary to have the grant CREATE ROW POLICY ON mydb.filtered_table1"
+    )
     assert expected_error in node.query_and_get_error(
         "CREATE POLICY pA ON mydb.filtered_table1 FOR SELECT USING a<b", user="X"
     )
@@ -684,7 +698,9 @@ def test_grant_create_row_policy():
         "CREATE POLICY pA ON mydb.filtered_table1 FOR SELECT USING a<b", user="X"
     )
 
-    expected_error = "necessary to have the grant ALTER ROW POLICY ON mydb.filtered_table1"
+    expected_error = (
+        "necessary to have the grant ALTER ROW POLICY ON mydb.filtered_table1"
+    )
     assert expected_error in node.query_and_get_error(
         "ALTER POLICY pA ON mydb.filtered_table1 FOR SELECT USING a==b", user="X"
     )
@@ -693,7 +709,9 @@ def test_grant_create_row_policy():
         "ALTER POLICY pA ON mydb.filtered_table1 FOR SELECT USING a==b", user="X"
     )
 
-    expected_error = "necessary to have the grant DROP ROW POLICY ON mydb.filtered_table1"
+    expected_error = (
+        "necessary to have the grant DROP ROW POLICY ON mydb.filtered_table1"
+    )
     assert expected_error in node.query_and_get_error(
         "DROP POLICY pA ON mydb.filtered_table1", user="X"
     )

From 93a10980e9ee0d9a9160d9686630ebb5f8c7943e Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 6 Aug 2023 16:27:45 +0200
Subject: [PATCH 094/249] Fix review comments

---
 src/Access/ContextAccess.cpp                   | 4 ++--
 src/Interpreters/InterpreterKillQueryQuery.cpp | 4 ++--
 tests/integration/test_storage_s3/test.py      | 8 ++++----
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp
index 18469140f59..c3aeec15cdd 100644
--- a/src/Access/ContextAccess.cpp
+++ b/src/Access/ContextAccess.cpp
@@ -756,11 +756,11 @@ bool ContextAccess::checkAdminOptionImplHelper(const Container & role_ids, const
                 show_error(ErrorCodes::ACCESS_DENIED,
                            "Not enough privileges. "
                            "Role {} is granted, but without ADMIN option. "
-                           "to execute this query, it's necessary to have the role {} granted with ADMIN option.",
+                           "To execute this query, it's necessary to have the role {} granted with ADMIN option.",
                            backQuote(*role_name), backQuoteIfNeed(*role_name));
             else
                 show_error(ErrorCodes::ACCESS_DENIED, "Not enough privileges. "
-                           "to execute this query, it's necessary to have the role {} granted with ADMIN option.",
+                           "To execute this query, it's necessary to have the role {} granted with ADMIN option.",
                            backQuoteIfNeed(*role_name));
         }
 
diff --git a/src/Interpreters/InterpreterKillQueryQuery.cpp b/src/Interpreters/InterpreterKillQueryQuery.cpp
index e7dbd34c543..76b2fdf1f58 100644
--- a/src/Interpreters/InterpreterKillQueryQuery.cpp
+++ b/src/Interpreters/InterpreterKillQueryQuery.cpp
@@ -296,7 +296,7 @@ BlockIO InterpreterKillQueryQuery::execute()
 
         if (res_columns[0]->empty() && access_denied)
             throw Exception(ErrorCodes::ACCESS_DENIED, "Not allowed to kill mutation. "
-                "to execute this query, it's necessary to have the grant {}", required_access_rights.toString());
+                "To execute this query, it's necessary to have the grant {}", required_access_rights.toString());
 
         res_io.pipeline = QueryPipeline(Pipe(std::make_shared<SourceFromSingleChunk>(header.cloneWithColumns(std::move(res_columns)))));
 
@@ -360,7 +360,7 @@ BlockIO InterpreterKillQueryQuery::execute()
 
         if (res_columns[0]->empty() && access_denied)
             throw Exception(ErrorCodes::ACCESS_DENIED, "Not allowed to kill move partition. "
-                "to execute this query, it's necessary to have the grant {}", required_access_rights.toString());
+                "To execute this query, it's necessary to have the grant {}", required_access_rights.toString());
 
         res_io.pipeline = QueryPipeline(Pipe(std::make_shared<SourceFromSingleChunk>(header.cloneWithColumns(std::move(res_columns)))));
 
diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py
index 295f8ca18a1..9fb794648cb 100644
--- a/tests/integration/test_storage_s3/test.py
+++ b/tests/integration/test_storage_s3/test.py
@@ -944,7 +944,7 @@ def test_predefined_connection_configuration(started_cluster):
         f"CREATE TABLE {name} (id UInt32) ENGINE = S3(s3_conf1, format='CSV')"
     )
     assert (
-        "to execute this query, it's necessary to have the grant NAMED COLLECTION ON s3_conf1"
+        "To execute this query, it's necessary to have the grant NAMED COLLECTION ON s3_conf1"
         in error
     )
     error = instance.query_and_get_error(
@@ -952,7 +952,7 @@ def test_predefined_connection_configuration(started_cluster):
         user="user",
     )
     assert (
-        "to execute this query, it's necessary to have the grant NAMED COLLECTION ON s3_conf1"
+        "To execute this query, it's necessary to have the grant NAMED COLLECTION ON s3_conf1"
         in error
     )
 
@@ -973,12 +973,12 @@ def test_predefined_connection_configuration(started_cluster):
 
     error = instance.query_and_get_error("SELECT * FROM s3(no_collection)")
     assert (
-        "to execute this query, it's necessary to have the grant NAMED COLLECTION ON no_collection"
+        "To execute this query, it's necessary to have the grant NAMED COLLECTION ON no_collection"
         in error
     )
     error = instance.query_and_get_error("SELECT * FROM s3(no_collection)", user="user")
     assert (
-        "to execute this query, it's necessary to have the grant NAMED COLLECTION ON no_collection"
+        "To execute this query, it's necessary to have the grant NAMED COLLECTION ON no_collection"
         in error
     )
     instance = started_cluster.instances["dummy"]  # has named collection access

From 674767df16530db954583cb1fd8a6eecf349c813 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 6 Aug 2023 16:29:15 +0200
Subject: [PATCH 095/249] Fix review comments

---
 tests/integration/test_named_collections/test.py          | 8 ++++----
 .../queries/0_stateless/02250_ON_CLUSTER_grant.reference  | 2 +-
 tests/queries/0_stateless/02250_ON_CLUSTER_grant.sh       | 2 +-
 tests/queries/0_stateless/02561_temporary_table_grants.sh | 8 ++++----
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/tests/integration/test_named_collections/test.py b/tests/integration/test_named_collections/test.py
index dee1209f2aa..cbb8c94c701 100644
--- a/tests/integration/test_named_collections/test.py
+++ b/tests/integration/test_named_collections/test.py
@@ -344,7 +344,7 @@ def test_granular_access_create_alter_drop_query(cluster):
     )
 
     assert (
-        "DB::Exception: kek: Not enough privileges. to execute this query, it's necessary to have the grant CREATE NAMED COLLECTION"
+        "DB::Exception: kek: Not enough privileges. To execute this query, it's necessary to have the grant CREATE NAMED COLLECTION"
         in node.query_and_get_error(
             "CREATE NAMED COLLECTION collection2 AS key1=1, key2='value2'", user="kek"
         )
@@ -370,7 +370,7 @@ def test_granular_access_create_alter_drop_query(cluster):
     )
 
     assert (
-        "DB::Exception: kek: Not enough privileges. to execute this query, it's necessary to have the grant ALTER NAMED COLLECTION"
+        "DB::Exception: kek: Not enough privileges. To execute this query, it's necessary to have the grant ALTER NAMED COLLECTION"
         in node.query_and_get_error(
             "ALTER NAMED COLLECTION collection2 SET key1=2", user="kek"
         )
@@ -385,14 +385,14 @@ def test_granular_access_create_alter_drop_query(cluster):
     )
     node.query("REVOKE alter named collection ON collection2 FROM kek")
     assert (
-        "DB::Exception: kek: Not enough privileges. to execute this query, it's necessary to have the grant ALTER NAMED COLLECTION"
+        "DB::Exception: kek: Not enough privileges. To execute this query, it's necessary to have the grant ALTER NAMED COLLECTION"
         in node.query_and_get_error(
             "ALTER NAMED COLLECTION collection2 SET key1=3", user="kek"
         )
     )
 
     assert (
-        "DB::Exception: kek: Not enough privileges. to execute this query, it's necessary to have the grant DROP NAMED COLLECTION"
+        "DB::Exception: kek: Not enough privileges. To execute this query, it's necessary to have the grant DROP NAMED COLLECTION"
         in node.query_and_get_error("DROP NAMED COLLECTION collection2", user="kek")
     )
     node.query("GRANT drop named collection ON collection2 TO kek")
diff --git a/tests/queries/0_stateless/02250_ON_CLUSTER_grant.reference b/tests/queries/0_stateless/02250_ON_CLUSTER_grant.reference
index 07cc24628e9..f78c46e54f4 100644
--- a/tests/queries/0_stateless/02250_ON_CLUSTER_grant.reference
+++ b/tests/queries/0_stateless/02250_ON_CLUSTER_grant.reference
@@ -1,3 +1,3 @@
 with_on_cluster_02250_ON_CLUSTER_grant_default
 without_on_cluster_02250_ON_CLUSTER_grant_default
-Not enough privileges. to execute this query, it's necessary to have the grant CLUSTER ON *.*. (ACCESS_DENIED)
+Not enough privileges. To execute this query, it's necessary to have the grant CLUSTER ON *.*. (ACCESS_DENIED)
diff --git a/tests/queries/0_stateless/02250_ON_CLUSTER_grant.sh b/tests/queries/0_stateless/02250_ON_CLUSTER_grant.sh
index 4627bb81b19..66417e9694a 100755
--- a/tests/queries/0_stateless/02250_ON_CLUSTER_grant.sh
+++ b/tests/queries/0_stateless/02250_ON_CLUSTER_grant.sh
@@ -27,5 +27,5 @@ echo "with_on_cluster_$CLICKHOUSE_TEST_UNIQUE_NAME"
 $CLICKHOUSE_CLIENT --user "with_on_cluster_$CLICKHOUSE_TEST_UNIQUE_NAME" -q "CREATE DATABASE IF NOT EXISTS db_with_on_cluster_$CLICKHOUSE_TEST_UNIQUE_NAME ON CLUSTER test_shard_localhost" >/dev/null
 echo "without_on_cluster_$CLICKHOUSE_TEST_UNIQUE_NAME"
 $CLICKHOUSE_CLIENT --user "without_on_cluster_$CLICKHOUSE_TEST_UNIQUE_NAME" -q "CREATE DATABASE IF NOT EXISTS db_without_on_cluster_$CLICKHOUSE_TEST_UNIQUE_NAME ON CLUSTER test_shard_localhost" |& {
-    grep -m1 -F -o "Not enough privileges. to execute this query, it's necessary to have the grant CLUSTER ON *.*. (ACCESS_DENIED)"
+    grep -m1 -F -o "Not enough privileges. To execute this query, it's necessary to have the grant CLUSTER ON *.*. (ACCESS_DENIED)"
 }
diff --git a/tests/queries/0_stateless/02561_temporary_table_grants.sh b/tests/queries/0_stateless/02561_temporary_table_grants.sh
index 965e9687950..33784f1d536 100755
--- a/tests/queries/0_stateless/02561_temporary_table_grants.sh
+++ b/tests/queries/0_stateless/02561_temporary_table_grants.sh
@@ -10,24 +10,24 @@ user=user_$CLICKHOUSE_TEST_UNIQUE_NAME
 $CLICKHOUSE_CLIENT --query "DROP USER IF EXISTS $user"
 $CLICKHOUSE_CLIENT --query "CREATE USER $user IDENTIFIED WITH PLAINTEXT_PASSWORD BY 'hello'"
 
-$CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_memory_02561(name String)" 2>&1 | grep -F "Not enough privileges. to execute this query, it's necessary to have the grant CREATE TEMPORARY TABLE" > /dev/null && echo "OK"
+$CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_memory_02561(name String)" 2>&1 | grep -F "Not enough privileges. To execute this query, it's necessary to have the grant CREATE TEMPORARY TABLE" > /dev/null && echo "OK"
 
 $CLICKHOUSE_CLIENT --query "GRANT CREATE TEMPORARY TABLE ON *.* TO $user"
 $CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_memory_02561(name String)"
 
-$CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_merge_tree_02561(name String) ENGINE = MergeTree() ORDER BY name" 2>&1 | grep -F "Not enough privileges. to execute this query, it's necessary to have the grant CREATE ARBITRARY TEMPORARY TABLE" > /dev/null && echo "OK"
+$CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_merge_tree_02561(name String) ENGINE = MergeTree() ORDER BY name" 2>&1 | grep -F "Not enough privileges. To execute this query, it's necessary to have the grant CREATE ARBITRARY TEMPORARY TABLE" > /dev/null && echo "OK"
 
 $CLICKHOUSE_CLIENT --query "GRANT CREATE ARBITRARY TEMPORARY TABLE ON *.* TO $user"
 
 $CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_merge_tree_02561(name String) ENGINE = MergeTree() ORDER BY name"
 
-$CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_file_02561(name String) ENGINE = File(TabSeparated)" 2>&1 | grep -F "Not enough privileges. to execute this query, it's necessary to have the grant FILE" > /dev/null && echo "OK"
+$CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_file_02561(name String) ENGINE = File(TabSeparated)" 2>&1 | grep -F "Not enough privileges. To execute this query, it's necessary to have the grant FILE" > /dev/null && echo "OK"
 
 $CLICKHOUSE_CLIENT --query "GRANT FILE ON *.* TO $user"
 
 $CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_file_02561(name String) ENGINE = File(TabSeparated)"
 
-$CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_url_02561(name String) ENGINE = URL('http://127.0.0.1:8123?query=select+12', 'RawBLOB')" 2>&1 | grep -F "Not enough privileges. to execute this query, it's necessary to have the grant URL" > /dev/null && echo "OK"
+$CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_url_02561(name String) ENGINE = URL('http://127.0.0.1:8123?query=select+12', 'RawBLOB')" 2>&1 | grep -F "Not enough privileges. To execute this query, it's necessary to have the grant URL" > /dev/null && echo "OK"
 
 $CLICKHOUSE_CLIENT --query "GRANT URL ON *.* TO $user"
 

From 8f6bde529b3e17bc59c9a38696f6eaeb75f6d33d Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 6 Aug 2023 22:35:45 +0200
Subject: [PATCH 096/249] Fix a test and remove another

---
 .../01293_system_distribution_queue.sql       |  2 +-
 .../01460_DistributedFilesToInsert.reference  | 12 -----
 .../01460_DistributedFilesToInsert.sql        | 45 -------------------
 3 files changed, 1 insertion(+), 58 deletions(-)
 delete mode 100644 tests/queries/0_stateless/01460_DistributedFilesToInsert.reference
 delete mode 100644 tests/queries/0_stateless/01460_DistributedFilesToInsert.sql

diff --git a/tests/queries/0_stateless/01293_system_distribution_queue.sql b/tests/queries/0_stateless/01293_system_distribution_queue.sql
index 9997f18f61d..f14c0d64a7d 100644
--- a/tests/queries/0_stateless/01293_system_distribution_queue.sql
+++ b/tests/queries/0_stateless/01293_system_distribution_queue.sql
@@ -8,7 +8,7 @@ create table null_01293 (key Int) engine=Null();
 create table dist_01293 as null_01293 engine=Distributed(test_cluster_two_shards, currentDatabase(), null_01293, key);
 
 -- no rows, since no active monitor
-select * from system.distribution_queue;
+select * from system.distribution_queue where database = currentDatabase();
 
 select 'INSERT';
 system stop distributed sends dist_01293;
diff --git a/tests/queries/0_stateless/01460_DistributedFilesToInsert.reference b/tests/queries/0_stateless/01460_DistributedFilesToInsert.reference
deleted file mode 100644
index 7a123a39978..00000000000
--- a/tests/queries/0_stateless/01460_DistributedFilesToInsert.reference
+++ /dev/null
@@ -1,12 +0,0 @@
-INSERT
-0
-0
-STOP/START DISTRIBUTED SENDS
-1
-0
-FLUSH DISTRIBUTED
-1
-0
-DROP TABLE
-1
-0
diff --git a/tests/queries/0_stateless/01460_DistributedFilesToInsert.sql b/tests/queries/0_stateless/01460_DistributedFilesToInsert.sql
deleted file mode 100644
index 9b7a98f30bc..00000000000
--- a/tests/queries/0_stateless/01460_DistributedFilesToInsert.sql
+++ /dev/null
@@ -1,45 +0,0 @@
--- Tags: no-parallel, no-fasttest
--- Tag no-fasttest: Look at DistributedFilesToInsert, so cannot run in parallel.
-
--- otherwise SYSTEM STOP DISTRIBUTED SENDS does not makes any effect (for localhost)
--- (i.e. no .bin files and hence no sending is required)
-set prefer_localhost_replica=0;
-
-drop table if exists data_01460;
-drop table if exists dist_01460;
-
-create table data_01460 as system.one engine=Null();
-create table dist_01460 as data_01460 engine=Distributed(test_shard_localhost, currentDatabase(), data_01460) settings monitor_sleep_time_ms=50;
-
-select 'INSERT';
-select value from system.metrics where metric = 'DistributedFilesToInsert';
-insert into dist_01460 select * from system.one;
-select sleep(1) format Null; -- monitor_sleep_time_ms
-select value from system.metrics where metric = 'DistributedFilesToInsert';
-
-select 'STOP/START DISTRIBUTED SENDS';
-system stop distributed sends dist_01460;
-insert into dist_01460 select * from system.one;
-select sleep(1) format Null; -- monitor_sleep_time_ms
-select value from system.metrics where metric = 'DistributedFilesToInsert';
-system start distributed sends dist_01460;
-select sleep(1) format Null; -- monitor_sleep_time_ms
-select value from system.metrics where metric = 'DistributedFilesToInsert';
-
-select 'FLUSH DISTRIBUTED';
-system stop distributed sends dist_01460;
-insert into dist_01460 select * from system.one;
-select sleep(1) format Null; -- monitor_sleep_time_ms
-select value from system.metrics where metric = 'DistributedFilesToInsert';
-system flush distributed dist_01460;
-select value from system.metrics where metric = 'DistributedFilesToInsert';
-
-select 'DROP TABLE';
-system stop distributed sends dist_01460;
-insert into dist_01460 select * from system.one;
-select sleep(1) format Null; -- monitor_sleep_time_ms
-select value from system.metrics where metric = 'DistributedFilesToInsert';
-drop table dist_01460;
-select value from system.metrics where metric = 'DistributedFilesToInsert';
-
-drop table data_01460;

From 4afce1c4adefa336413dda1ebfd8a9689a6a3f29 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 7 Aug 2023 02:25:47 +0200
Subject: [PATCH 097/249] Don not produce warning if a system table was created
 on startup

---
 src/Databases/DatabaseAtomic.cpp          | 12 ++++++++++--
 src/Interpreters/InterpreterRenameQuery.h |  2 +-
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp
index 0f65069db35..bbab279c0e2 100644
--- a/src/Databases/DatabaseAtomic.cpp
+++ b/src/Databases/DatabaseAtomic.cpp
@@ -471,8 +471,16 @@ void DatabaseAtomic::tryCreateSymlink(const String & table_name, const String &
     {
         String link = path_to_table_symlinks + escapeForFileName(table_name);
         fs::path data = fs::canonical(getContext()->getPath()) / actual_data_path;
-        if (!if_data_path_exist || fs::exists(data))
-            fs::create_directory_symlink(data, link);
+
+        /// If it already points where needed.
+        std::error_code ec;
+        if (fs::equivalent(data, link, ec))
+            return;
+
+        if (if_data_path_exist && !fs::exists(data))
+            return;
+
+        fs::create_directory_symlink(data, link);
     }
     catch (...)
     {
diff --git a/src/Interpreters/InterpreterRenameQuery.h b/src/Interpreters/InterpreterRenameQuery.h
index 25c707d9962..79a6c0ddef4 100644
--- a/src/Interpreters/InterpreterRenameQuery.h
+++ b/src/Interpreters/InterpreterRenameQuery.h
@@ -10,7 +10,7 @@ namespace DB
 class AccessRightsElements;
 class DDLGuard;
 
-/// To avoid deadlocks, we must acquire locks for tables in same order in any different RENAMES.
+/// To avoid deadlocks, we must acquire locks for tables in same order in any different RENAMEs.
 struct UniqueTableName
 {
     String database_name;

From 74a4c9755cf315fc957639eeb6a15e1c3b4c5818 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 7 Aug 2023 02:25:47 +0200
Subject: [PATCH 098/249] Don not produce warning if a system table was created
 on startup

---
 src/Databases/DatabaseAtomic.cpp          | 12 ++++++++++--
 src/Interpreters/InterpreterRenameQuery.h |  2 +-
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp
index 0f65069db35..bbab279c0e2 100644
--- a/src/Databases/DatabaseAtomic.cpp
+++ b/src/Databases/DatabaseAtomic.cpp
@@ -471,8 +471,16 @@ void DatabaseAtomic::tryCreateSymlink(const String & table_name, const String &
     {
         String link = path_to_table_symlinks + escapeForFileName(table_name);
         fs::path data = fs::canonical(getContext()->getPath()) / actual_data_path;
-        if (!if_data_path_exist || fs::exists(data))
-            fs::create_directory_symlink(data, link);
+
+        /// If it already points where needed.
+        std::error_code ec;
+        if (fs::equivalent(data, link, ec))
+            return;
+
+        if (if_data_path_exist && !fs::exists(data))
+            return;
+
+        fs::create_directory_symlink(data, link);
     }
     catch (...)
     {
diff --git a/src/Interpreters/InterpreterRenameQuery.h b/src/Interpreters/InterpreterRenameQuery.h
index 25c707d9962..79a6c0ddef4 100644
--- a/src/Interpreters/InterpreterRenameQuery.h
+++ b/src/Interpreters/InterpreterRenameQuery.h
@@ -10,7 +10,7 @@ namespace DB
 class AccessRightsElements;
 class DDLGuard;
 
-/// To avoid deadlocks, we must acquire locks for tables in same order in any different RENAMES.
+/// To avoid deadlocks, we must acquire locks for tables in same order in any different RENAMEs.
 struct UniqueTableName
 {
     String database_name;

From c3edb57953d41192f3c39034e105f56db000b035 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 7 Aug 2023 02:29:29 +0200
Subject: [PATCH 099/249] Remove old util

---
 utils/CMakeLists.txt                 |   1 -
 utils/wikistat-loader/CMakeLists.txt |   2 -
 utils/wikistat-loader/main.cpp       | 225 ---------------------------
 3 files changed, 228 deletions(-)
 delete mode 100644 utils/wikistat-loader/CMakeLists.txt
 delete mode 100644 utils/wikistat-loader/main.cpp

diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt
index 47dd2fc9f2d..4e1184cc9a5 100644
--- a/utils/CMakeLists.txt
+++ b/utils/CMakeLists.txt
@@ -20,7 +20,6 @@ if (ENABLE_UTILS)
     add_subdirectory (zookeeper-cli)
     add_subdirectory (zookeeper-dump-tree)
     add_subdirectory (zookeeper-remove-by-list)
-    add_subdirectory (wikistat-loader)
     add_subdirectory (check-marks)
     add_subdirectory (checksum-for-compressed-block)
     add_subdirectory (check-mysql-binlog)
diff --git a/utils/wikistat-loader/CMakeLists.txt b/utils/wikistat-loader/CMakeLists.txt
deleted file mode 100644
index fc5416dea2e..00000000000
--- a/utils/wikistat-loader/CMakeLists.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-clickhouse_add_executable (wikistat-loader main.cpp ${SRCS})
-target_link_libraries (wikistat-loader PRIVATE clickhouse_common_io boost::program_options)
diff --git a/utils/wikistat-loader/main.cpp b/utils/wikistat-loader/main.cpp
deleted file mode 100644
index 493f1df05da..00000000000
--- a/utils/wikistat-loader/main.cpp
+++ /dev/null
@@ -1,225 +0,0 @@
-#include <boost/program_options.hpp>
-
-#include <base/hex.h>
-#include <IO/ReadBuffer.h>
-#include <IO/WriteBuffer.h>
-#include <IO/ReadHelpers.h>
-#include <IO/WriteHelpers.h>
-#include <IO/ReadBufferFromFileDescriptor.h>
-#include <IO/WriteBufferFromFileDescriptor.h>
-
-
-/** Reads uncompressed wikistat data from stdin,
-  *  and writes transformed data in tsv format,
-  *  ready to be loaded into ClickHouse.
-  *
-  * Input data has format:
-  *
-  * aa Wikipedia 1 17224
-  * aa.b Main_Page 2 21163
-  *
-  * project, optional subproject, path, hits, total size in bytes.
-  */
-
-
-template <bool break_at_dot>
-static void readString(std::string & s, DB::ReadBuffer & buf)
-{
-    s.clear();
-
-    while (!buf.eof())
-    {
-        const char * next_pos;
-
-        if (break_at_dot)
-            next_pos = find_first_symbols<' ', '\n', '.'>(buf.position(), buf.buffer().end());
-        else
-            next_pos = find_first_symbols<' ', '\n'>(buf.position(), buf.buffer().end());
-
-        s.append(buf.position(), next_pos - buf.position());
-        buf.position() += next_pos - buf.position();
-
-        if (!buf.hasPendingData())
-            continue;
-
-        if (*buf.position() == ' ' || *buf.position() == '\n' || (break_at_dot && *buf.position() == '.'))
-            return;
-    }
-}
-
-
-/** Reads path before whitespace and decodes %xx sequences (to more compact and handy representation),
-  *  except %2F '/', %26 '&', %3D '=', %3F '?', %23 '#' (to not break structure of URL).
-  */
-static void readPath(std::string & s, DB::ReadBuffer & buf)
-{
-    s.clear();
-
-    while (!buf.eof())
-    {
-        const char * next_pos = find_first_symbols<' ', '\n', '%'>(buf.position(), buf.buffer().end());
-
-        s.append(buf.position(), next_pos - buf.position());
-        buf.position() += next_pos - buf.position();
-
-        if (!buf.hasPendingData())
-            continue;
-
-        if (*buf.position() == ' ' || *buf.position() == '\n')
-            return;
-
-        if (*buf.position() == '%')
-        {
-            ++buf.position();
-
-            char c1;
-            char c2;
-
-            if (buf.eof() || *buf.position() == ' ')
-                break;
-
-            DB::readChar(c1, buf);
-
-            if (buf.eof() || *buf.position() == ' ')
-                break;
-
-            DB::readChar(c2, buf);
-
-            if ((c1 == '2' && (c2 == 'f' || c2 == '6' || c2 == '3' || c2 == 'F'))
-                || (c1 == '3' && (c2 == 'd' || c2 == 'f' || c2 == 'D' || c2 == 'F')))
-            {
-                s += '%';
-                s += c1;
-                s += c2;
-            }
-            else
-                s += static_cast<char>(static_cast<UInt8>(unhex(c1)) * 16 + static_cast<UInt8>(unhex(c2)));
-        }
-    }
-}
-
-
-static void skipUntilNewline(DB::ReadBuffer & buf)
-{
-    while (!buf.eof())
-    {
-        const char * next_pos = find_first_symbols<'\n'>(buf.position(), buf.buffer().end());
-
-        buf.position() += next_pos - buf.position();
-
-        if (!buf.hasPendingData())
-            continue;
-
-        if (*buf.position() == '\n')
-        {
-            ++buf.position();
-            return;
-        }
-    }
-}
-
-
-namespace DB
-{
-    namespace ErrorCodes
-    {
-        extern const int CANNOT_PARSE_INPUT_ASSERTION_FAILED;
-    }
-}
-
-
-int main(int argc, char ** argv)
-try
-{
-    boost::program_options::options_description desc("Allowed options");
-    desc.add_options()
-        ("help,h", "produce help message")
-        ("time", boost::program_options::value<std::string>()->required(),
-            "time of data in YYYY-MM-DD hh:mm:ss form")
-    ;
-
-    boost::program_options::variables_map options;
-    boost::program_options::store(boost::program_options::parse_command_line(argc, argv, desc), options);
-
-    if (options.count("help"))
-    {
-        std::cout << "Reads uncompressed wikistat data from stdin and writes transformed data in tsv format." << std::endl;
-        std::cout << "Usage: " << argv[0] << " --time='YYYY-MM-DD hh:00:00' < in > out" << std::endl;
-        std::cout << desc << std::endl;
-        return 1;
-    }
-
-    std::string time_str = options.at("time").as<std::string>();
-    LocalDateTime time(time_str);
-    LocalDate date(time_str);
-
-    DB::ReadBufferFromFileDescriptor in(STDIN_FILENO);
-    DB::WriteBufferFromFileDescriptor out(STDOUT_FILENO);
-
-    std::string project;
-    std::string subproject;
-    std::string path;
-    UInt64 hits = 0;
-    UInt64 size = 0;
-
-    size_t row_num = 0;
-    while (!in.eof())
-    {
-        try
-        {
-            ++row_num;
-            readString<true>(project, in);
-
-            if (in.eof())
-                break;
-
-            if (*in.position() == '.')
-                readString<false>(subproject, in);
-            else
-                subproject.clear();
-
-            DB::assertChar(' ', in);
-            readPath(path, in);
-            DB::assertChar(' ', in);
-            DB::readIntText(hits, in);
-            DB::assertChar(' ', in);
-            DB::readIntText(size, in);
-            DB::assertChar('\n', in);
-        }
-        catch (const DB::Exception & e)
-        {
-            /// Sometimes, input data has errors. For example, look at first lines in pagecounts-20130210-130000.gz
-            /// To save rest of data, just skip lines with errors.
-            if (e.code() == DB::ErrorCodes::CANNOT_PARSE_INPUT_ASSERTION_FAILED)
-            {
-                std::cerr << "At row " << row_num << ": " << DB::getCurrentExceptionMessage(false) << '\n';
-                skipUntilNewline(in);
-                continue;
-            }
-            else
-                throw;
-        }
-
-        DB::writeText(date, out);
-        DB::writeChar('\t', out);
-        DB::writeText(time, out);
-        DB::writeChar('\t', out);
-        DB::writeText(project, out);
-        DB::writeChar('\t', out);
-        DB::writeText(subproject, out);
-        DB::writeChar('\t', out);
-        DB::writeText(path, out);
-        DB::writeChar('\t', out);
-        DB::writeText(hits, out);
-        DB::writeChar('\t', out);
-        DB::writeText(size, out);
-        DB::writeChar('\n', out);
-    }
-
-    return 0;
-}
-catch (...)
-{
-    std::cerr << DB::getCurrentExceptionMessage(true) << '\n';
-    throw;
-}

From 9a70b0d41771b83c05283b0e35ec450f36a5be1f Mon Sep 17 00:00:00 2001
From: Maksim Kita <kitaetoya@gmail.com>
Date: Mon, 7 Aug 2023 11:06:25 +0300
Subject: [PATCH 100/249] Fixed tests

---
 src/Processors/QueryPlan/JoinStep.cpp                     | 8 ++++----
 .../queries/0_stateless/02835_join_step_explain.reference | 8 ++++----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/Processors/QueryPlan/JoinStep.cpp b/src/Processors/QueryPlan/JoinStep.cpp
index 858a01a437d..63a5eeb51d2 100644
--- a/src/Processors/QueryPlan/JoinStep.cpp
+++ b/src/Processors/QueryPlan/JoinStep.cpp
@@ -70,9 +70,9 @@ void JoinStep::describeActions(FormatSettings & settings) const
     String prefix(settings.offset, ' ');
 
     const auto & table_join = join->getTableJoin();
-    settings.out << prefix << "Kind: " << toString(table_join.kind()) << '\n';
+    settings.out << prefix << "Type: " << toString(table_join.kind()) << '\n';
     settings.out << prefix << "Strictness: " << toString(table_join.strictness()) << '\n';
-    settings.out << prefix << "Type: " << join->getName() << '\n';
+    settings.out << prefix << "Algorithm: " << join->getName() << '\n';
 
     if (table_join.strictness() == JoinStrictness::Asof)
         settings.out << prefix << "ASOF inequality: " << toString(table_join.getAsofInequality()) << '\n';
@@ -84,9 +84,9 @@ void JoinStep::describeActions(FormatSettings & settings) const
 void JoinStep::describeActions(JSONBuilder::JSONMap & map) const
 {
     const auto & table_join = join->getTableJoin();
-    map.add("Kind", toString(table_join.kind()));
+    map.add("Type", toString(table_join.kind()));
     map.add("Strictness", toString(table_join.strictness()));
-    map.add("Type", join->getName());
+    map.add("Algorithm", join->getName());
 
     if (table_join.strictness() == JoinStrictness::Asof)
         map.add("ASOF inequality", toString(table_join.getAsofInequality()));
diff --git a/tests/queries/0_stateless/02835_join_step_explain.reference b/tests/queries/0_stateless/02835_join_step_explain.reference
index 77c91b3d7e9..0cc2e802682 100644
--- a/tests/queries/0_stateless/02835_join_step_explain.reference
+++ b/tests/queries/0_stateless/02835_join_step_explain.reference
@@ -17,9 +17,9 @@ Positions: 4 0 2 1
           value_1_1 String
           value_1_3 String
           id_2 UInt64
-  Kind: INNER
+  Type: INNER
   Strictness: ALL
-  Type: HashJoin
+  Algorithm: HashJoin
   Clauses: [(id_0) = (id_2)]
     Expression ((JOIN actions + Change column names to column identifiers))
     Header: id_0 UInt64
@@ -73,9 +73,9 @@ Positions: 6 0 3 1
           value_1_3 String
           value_2_5 UInt64
           id_2 UInt64
-  Kind: INNER
+  Type: INNER
   Strictness: ASOF
-  Type: HashJoin
+  Algorithm: HashJoin
   ASOF inequality: LESS
   Clauses: [(id_0, value_2_4) = (id_2, value_2_5)]
     Expression ((JOIN actions + Change column names to column identifiers))

From d37283e9fb23c4065d13d3b4a97ecbb810943b30 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Mon, 7 Aug 2023 10:46:55 +0200
Subject: [PATCH 101/249] Fix query_id in part_log with async flush queries

---
 src/Interpreters/AsynchronousInsertQueue.cpp  |  4 +++-
 ...02790_async_queries_in_query_log.reference | 24 +++++++++++++++++++
 .../02790_async_queries_in_query_log.sh       | 18 ++++++++++++--
 3 files changed, 43 insertions(+), 3 deletions(-)

diff --git a/src/Interpreters/AsynchronousInsertQueue.cpp b/src/Interpreters/AsynchronousInsertQueue.cpp
index 4f15192f46c..c8a3f77556a 100644
--- a/src/Interpreters/AsynchronousInsertQueue.cpp
+++ b/src/Interpreters/AsynchronousInsertQueue.cpp
@@ -460,7 +460,6 @@ try
     const auto * log = &Poco::Logger::get("AsynchronousInsertQueue");
     const auto & insert_query = assert_cast<const ASTInsertQuery &>(*key.query);
     auto insert_context = Context::createCopy(global_context);
-    DB::CurrentThread::QueryScope query_scope_holder(insert_context);
     bool internal = false; // To enable logging this query
     bool async_insert = true;
 
@@ -482,6 +481,9 @@ try
     insert_context->setInitialQueryStartTime(query_start_time);
     insert_context->setCurrentQueryId(insert_query_id);
     insert_context->setInitialQueryId(insert_query_id);
+
+    DB::CurrentThread::QueryScope query_scope_holder(insert_context);
+
     size_t log_queries_cut_to_length = insert_context->getSettingsRef().log_queries_cut_to_length;
     String query_for_logging = insert_query.hasSecretParts()
         ? insert_query.formatForLogging(log_queries_cut_to_length)
diff --git a/tests/queries/0_stateless/02790_async_queries_in_query_log.reference b/tests/queries/0_stateless/02790_async_queries_in_query_log.reference
index e202a38f068..aa18817f4e6 100644
--- a/tests/queries/0_stateless/02790_async_queries_in_query_log.reference
+++ b/tests/queries/0_stateless/02790_async_queries_in_query_log.reference
@@ -36,6 +36,14 @@ exception_code: 0
 
 system.query_views_log
 
+system.part_log
+Row 1:
+──────
+database:     default
+table:        async_insert_landing
+partition_id: all
+rows:         4
+
 system.query_log
 Row 1:
 ──────
@@ -85,6 +93,14 @@ written_bytes:  0
 status:         QueryFinish
 exception_code: 0
 
+system.part_log
+Row 1:
+──────
+database:     default
+table:        async_insert_landing
+partition_id: all
+rows:         3
+
 system.query_log
 Row 1:
 ──────
@@ -133,3 +149,11 @@ written_rows:   0
 written_bytes:  0
 status:         Exc*****onWhileProcessing
 exception_code: 395
+
+system.part_log
+Row 1:
+──────
+database:     default
+table:        async_insert_landing
+partition_id: all
+rows:         3
diff --git a/tests/queries/0_stateless/02790_async_queries_in_query_log.sh b/tests/queries/0_stateless/02790_async_queries_in_query_log.sh
index d1c8fe122cb..1ff97031acb 100755
--- a/tests/queries/0_stateless/02790_async_queries_in_query_log.sh
+++ b/tests/queries/0_stateless/02790_async_queries_in_query_log.sh
@@ -28,7 +28,7 @@ function print_flush_query_logs()
       FROM system.query_log
       WHERE
           event_date >= yesterday()
-      AND initial_query_id = (SELECT flush_query_id FROM system.asynchronous_insert_log WHERE query_id = '$1')
+      AND initial_query_id = (SELECT flush_query_id FROM system.asynchronous_insert_log WHERE event_date >= yesterday() AND query_id = '$1')
       -- AND current_database = currentDatabase() -- Just to silence style check: this is not ok for this test since the query uses default values
       ORDER BY type DESC
       FORMAT Vertical"
@@ -50,7 +50,21 @@ function print_flush_query_logs()
       FROM system.query_views_log
       WHERE
           event_date >= yesterday()
-      AND initial_query_id = (SELECT flush_query_id FROM system.asynchronous_insert_log WHERE query_id = '$1')
+      AND initial_query_id = (SELECT flush_query_id FROM system.asynchronous_insert_log WHERE event_date >= yesterday() AND query_id = '$1')
+      FORMAT Vertical"
+
+    echo ""
+    echo "system.part_log"
+    ${CLICKHOUSE_CLIENT} -q "
+      SELECT
+          database,
+          table,
+          partition_id,
+          rows
+      FROM system.part_log
+      WHERE
+          event_date >= yesterday()
+      AND query_id = (SELECT flush_query_id FROM system.asynchronous_insert_log WHERE event_date >= yesterday() AND query_id = '$1')
       FORMAT Vertical"
 }
 

From 8e6f7b4d0bf554abe2f79b4d442fbe871755b307 Mon Sep 17 00:00:00 2001
From: pufit <pufit@clickhouse.com>
Date: Mon, 7 Aug 2023 12:08:31 -0400
Subject: [PATCH 102/249] Fix typo, fix tests, `touch` command.

---
 .../utilities/clickhouse-keeper-client.md     |  5 +-
 programs/keeper-client/Commands.cpp           | 19 +++++-
 programs/keeper-client/Commands.h             | 17 ++++-
 programs/keeper-client/KeeperClient.cpp       |  3 +-
 tests/integration/test_keeper_client/test.py  | 65 ++++++++++---------
 5 files changed, 69 insertions(+), 40 deletions(-)

diff --git a/docs/en/operations/utilities/clickhouse-keeper-client.md b/docs/en/operations/utilities/clickhouse-keeper-client.md
index 37eb0bb71ff..443bf19de4c 100644
--- a/docs/en/operations/utilities/clickhouse-keeper-client.md
+++ b/docs/en/operations/utilities/clickhouse-keeper-client.md
@@ -45,7 +45,8 @@ keeper foo bar
 -   `ls [path]` -- Lists the nodes for the given path (default: cwd)
 -   `cd [path]` -- Change the working path (default `.`)
 -   `set <path> <value> [version]` -- Updates the node's value. Only update if version matches (default: -1)
--   `create <path> <value>` -- Creates new node
+-   `create <path> <value> [mode]` -- Creates new node with the set value
+-   `touch <path>` -- Creates new node with an empty string as value. Doesn't throw an exception if the node already exists
 -   `get <path>` -- Returns the node's value
 -   `remove <path>` -- Remove the node
 -   `rmr <path>` -- Recursively deletes path. Confirmation required
@@ -53,5 +54,5 @@ keeper foo bar
 -   `help` -- Prints this message
 -   `get_stat [path]` -- Returns the node's stat (default `.`)
 -   `find_super_nodes <threshold> [path]` -- Finds nodes with number of children larger than some threshold for the given path (default `.`)
--   `delete_stable_backups` -- Deletes ClickHouse nodes used for backups that are now inactive
+-   `delete_stale_backups` -- Deletes ClickHouse nodes used for backups that are now inactive
 -   `find_big_family [path] [n]` -- Returns the top n nodes with the biggest family in the subtree (default path = `.` and n = 10)
diff --git a/programs/keeper-client/Commands.cpp b/programs/keeper-client/Commands.cpp
index fd0a00d59db..b60b543ffce 100644
--- a/programs/keeper-client/Commands.cpp
+++ b/programs/keeper-client/Commands.cpp
@@ -126,6 +126,21 @@ void CreateCommand::execute(const ASTKeeperQuery * query, KeeperClient * client)
         static_cast<int>(query->args[2].safeGet<Int64>()));
 }
 
+bool TouchCommand::parse(IParser::Pos & pos, std::shared_ptr<ASTKeeperQuery> & node, Expected & expected) const
+{
+    String arg;
+    if (!parseKeeperPath(pos, expected, arg))
+        return false;
+    node->args.push_back(std::move(arg));
+
+    return true;
+}
+
+void TouchCommand::execute(const ASTKeeperQuery * query, KeeperClient * client) const
+{
+    client->zookeeper->createIfNotExists(client->getAbsolutePath(query->args[0].safeGet<String>()), "");
+}
+
 bool GetCommand::parse(IParser::Pos & pos, std::shared_ptr<ASTKeeperQuery> & node, Expected & expected) const
 {
     String arg;
@@ -215,12 +230,12 @@ void FindSuperNodes::execute(const ASTKeeperQuery * query, KeeperClient * client
     }
 }
 
-bool DeleteStableBackups::parse(IParser::Pos & /* pos */, std::shared_ptr<ASTKeeperQuery> & /* node */, Expected & /* expected */) const
+bool DeleteStaleBackups::parse(IParser::Pos & /* pos */, std::shared_ptr<ASTKeeperQuery> & /* node */, Expected & /* expected */) const
 {
     return true;
 }
 
-void DeleteStableBackups::execute(const ASTKeeperQuery * /* query */, KeeperClient * client) const
+void DeleteStaleBackups::execute(const ASTKeeperQuery * /* query */, KeeperClient * client) const
 {
     client->askConfirmation(
         "You are going to delete all inactive backups in /clickhouse/backups.",
diff --git a/programs/keeper-client/Commands.h b/programs/keeper-client/Commands.h
index 093920cb10d..7d64fef1738 100644
--- a/programs/keeper-client/Commands.h
+++ b/programs/keeper-client/Commands.h
@@ -76,7 +76,18 @@ class CreateCommand : public IKeeperClientCommand
 
     void execute(const ASTKeeperQuery * query, KeeperClient * client) const override;
 
-    String getHelpMessage() const override { return "{} <path> <value> -- Creates new node"; }
+    String getHelpMessage() const override { return "{} <path> <value> [mode] -- Creates new node with the set value"; }
+};
+
+class TouchCommand : public IKeeperClientCommand
+{
+    String getName() const override { return "touch"; }
+
+    bool parse(IParser::Pos & pos, std::shared_ptr<ASTKeeperQuery> & node, Expected & expected) const override;
+
+    void execute(const ASTKeeperQuery * query, KeeperClient * client) const override;
+
+    String getHelpMessage() const override { return "{} <path> -- Creates new node with an empty string as value. Doesn't throw an exception if the node already exists"; }
 };
 
 class GetCommand : public IKeeperClientCommand
@@ -115,9 +126,9 @@ class FindSuperNodes : public IKeeperClientCommand
     }
 };
 
-class DeleteStableBackups : public IKeeperClientCommand
+class DeleteStaleBackups : public IKeeperClientCommand
 {
-    String getName() const override { return "delete_stable_backups"; }
+    String getName() const override { return "delete_stale_backups"; }
 
     bool parse(IParser::Pos & pos, std::shared_ptr<ASTKeeperQuery> & node, Expected & expected) const override;
 
diff --git a/programs/keeper-client/KeeperClient.cpp b/programs/keeper-client/KeeperClient.cpp
index 561a1f41f7a..22d17c7a263 100644
--- a/programs/keeper-client/KeeperClient.cpp
+++ b/programs/keeper-client/KeeperClient.cpp
@@ -176,10 +176,11 @@ void KeeperClient::initialize(Poco::Util::Application & /* self */)
         std::make_shared<CDCommand>(),
         std::make_shared<SetCommand>(),
         std::make_shared<CreateCommand>(),
+        std::make_shared<TouchCommand>(),
         std::make_shared<GetCommand>(),
         std::make_shared<GetStatCommand>(),
         std::make_shared<FindSuperNodes>(),
-        std::make_shared<DeleteStableBackups>(),
+        std::make_shared<DeleteStaleBackups>(),
         std::make_shared<FindBigFamily>(),
         std::make_shared<RMCommand>(),
         std::make_shared<RMRCommand>(),
diff --git a/tests/integration/test_keeper_client/test.py b/tests/integration/test_keeper_client/test.py
index 3187ce10d2a..8f7056a5afd 100644
--- a/tests/integration/test_keeper_client/test.py
+++ b/tests/integration/test_keeper_client/test.py
@@ -42,17 +42,17 @@ def keeper_query(query: str):
 
 def test_big_family():
     command = keeper_query(
-        "create test_big_family foo;"
-        "create test_big_family/1 foo;"
-        "create test_big_family/1/1 foo;"
-        "create test_big_family/1/2 foo;"
-        "create test_big_family/1/3 foo;"
-        "create test_big_family/1/4 foo;"
-        "create test_big_family/1/5 foo;"
-        "create test_big_family/2 foo;"
-        "create test_big_family/2/1 foo;"
-        "create test_big_family/2/2 foo;"
-        "create test_big_family/2/3 foo;"
+        "touch test_big_family;"
+        "touch test_big_family/1;"
+        "touch test_big_family/1/1;"
+        "touch test_big_family/1/2;"
+        "touch test_big_family/1/3;"
+        "touch test_big_family/1/4;"
+        "touch test_big_family/1/5;"
+        "touch test_big_family/2;"
+        "touch test_big_family/2/1;"
+        "touch test_big_family/2/2;"
+        "touch test_big_family/2/3;"
         "find_big_family test_big_family;"
     )
 
@@ -82,18 +82,18 @@ def test_big_family():
 
 def test_find_super_nodes():
     command = keeper_query(
-        "create test_find_super_nodes foo;"
-        "create test_find_super_nodes/1 foo;"
-        "create test_find_super_nodes/1/1 foo;"
-        "create test_find_super_nodes/1/2 foo;"
-        "create test_find_super_nodes/1/3 foo;"
-        "create test_find_super_nodes/1/4 foo;"
-        "create test_find_super_nodes/1/5 foo;"
-        "create test_find_super_nodes/2 foo;"
-        "create test_find_super_nodes/2/1 foo;"
-        "create test_find_super_nodes/2/2 foo;"
-        "create test_find_super_nodes/2/3 foo;"
-        "create test_find_super_nodes/2/4 foo;"
+        "touch test_find_super_nodes;"
+        "touch test_find_super_nodes/1;"
+        "touch test_find_super_nodes/1/1;"
+        "touch test_find_super_nodes/1/2;"
+        "touch test_find_super_nodes/1/3;"
+        "touch test_find_super_nodes/1/4;"
+        "touch test_find_super_nodes/1/5;"
+        "touch test_find_super_nodes/2;"
+        "touch test_find_super_nodes/2/1;"
+        "touch test_find_super_nodes/2/2;"
+        "touch test_find_super_nodes/2/3;"
+        "touch test_find_super_nodes/2/4;"
         "cd test_find_super_nodes;"
         "find_super_nodes 4;"
     )
@@ -106,16 +106,17 @@ def test_find_super_nodes():
     )
 
 
-def test_delete_stable_backups():
+def test_delete_stale_backups():
     command = keeper_query(
-        "create /clickhouse/backups foo;"
-        "create /clickhouse/backups/1 foo;"
-        "create /clickhouse/backups/1/stage foo;"
-        "create /clickhouse/backups/1/stage/alive123 foo;"
-        "create /clickhouse/backups/2 foo;"
-        "create /clickhouse/backups/2/stage foo;"
-        "create /clickhouse/backups/2/stage/dead123 foo;"
-        "delete_stable_backups;"
+        "touch /clickhouse;"
+        "touch /clickhouse/backups;"
+        "touch /clickhouse/backups/1;"
+        "touch /clickhouse/backups/1/stage;"
+        "touch /clickhouse/backups/1/stage/alive123;"
+        "touch /clickhouse/backups/2;"
+        "touch /clickhouse/backups/2/stage;"
+        "touch /clickhouse/backups/2/stage/dead123;"
+        "delete_stale_backups;"
         "y;"
         "ls clickhouse/backups;"
     )

From 473fb60e6fb4e27aa712435482bf49109ae3d64f Mon Sep 17 00:00:00 2001
From: Smita Kulkarni <Smita.Kulkarni@clickhouse.com>
Date: Mon, 7 Aug 2023 18:56:18 +0200
Subject: [PATCH 103/249] Added retry for TransportException for azure read &
 download

---
 .../IO/ReadBufferFromAzureBlobStorage.cpp     | 48 +++++++++++++------
 1 file changed, 34 insertions(+), 14 deletions(-)

diff --git a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp
index 129bb97be09..15b632a54d4 100644
--- a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp
+++ b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp
@@ -102,6 +102,19 @@ bool ReadBufferFromAzureBlobStorage::nextImpl()
     size_t bytes_read = 0;
 
     size_t sleep_time_with_backoff_milliseconds = 100;
+
+    auto handle_exception = [&, this](const auto & e, size_t i)
+    {
+        LOG_INFO(log, "Exception caught during Azure Read for file {} at attempt {}: {}", path, i, e.Message);
+        if (i + 1 == max_single_read_retries)
+            throw;
+
+        sleepForMilliseconds(sleep_time_with_backoff_milliseconds);
+        sleep_time_with_backoff_milliseconds *= 2;
+        initialized = false;
+        initialize();
+    };
+
     for (size_t i = 0; i < max_single_read_retries; ++i)
     {
         try
@@ -111,16 +124,13 @@ bool ReadBufferFromAzureBlobStorage::nextImpl()
                 read_settings.remote_throttler->add(bytes_read, ProfileEvents::RemoteReadThrottlerBytes, ProfileEvents::RemoteReadThrottlerSleepMicroseconds);
             break;
         }
+        catch (const Azure::Core::Http::TransportException & e)
+        {
+            handle_exception(e, i);
+        }
         catch (const Azure::Storage::StorageException & e)
         {
-            LOG_INFO(log, "Exception caught during Azure Read for file {} at attempt {}: {}", path, i, e.Message);
-            if (i + 1 == max_single_read_retries)
-                throw;
-
-            sleepForMilliseconds(sleep_time_with_backoff_milliseconds);
-            sleep_time_with_backoff_milliseconds *= 2;
-            initialized = false;
-            initialize();
+            handle_exception(e, i);
         }
     }
 
@@ -211,6 +221,17 @@ void ReadBufferFromAzureBlobStorage::initialize()
         blob_client = std::make_unique<Azure::Storage::Blobs::BlobClient>(blob_container_client->GetBlobClient(path));
 
     size_t sleep_time_with_backoff_milliseconds = 100;
+
+    auto handle_exception = [&, this](const auto & e, size_t i)
+    {
+        LOG_INFO(log, "Exception caught during Azure Download for file {} at offset {} at attempt {} : {}", path, offset, i + 1, e.Message);
+        if (i + 1 == max_single_download_retries)
+            throw;
+
+        sleepForMilliseconds(sleep_time_with_backoff_milliseconds);
+        sleep_time_with_backoff_milliseconds *= 2;
+    };
+
     for (size_t i = 0; i < max_single_download_retries; ++i)
     {
         try
@@ -219,14 +240,13 @@ void ReadBufferFromAzureBlobStorage::initialize()
             data_stream = std::move(download_response.Value.BodyStream);
             break;
         }
+        catch (const Azure::Core::Http::TransportException & e)
+        {
+            handle_exception(e, i);
+        }
         catch (const Azure::Core::RequestFailedException & e)
         {
-            LOG_INFO(log, "Exception caught during Azure Download for file {} at offset {} at attempt {} : {}", path, offset, i + 1, e.Message);
-            if (i + 1 == max_single_download_retries)
-                throw;
-
-            sleepForMilliseconds(sleep_time_with_backoff_milliseconds);
-            sleep_time_with_backoff_milliseconds *= 2;
+            handle_exception(e,i);
         }
     }
 

From e1206dbaab112745c2e5e94eba982d3cd5eb5610 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 7 Aug 2023 18:57:30 +0200
Subject: [PATCH 104/249] Add a comment from @tavplubix

---
 programs/server/Server.cpp | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp
index da5b80426f0..4a8cf197fbd 100644
--- a/programs/server/Server.cpp
+++ b/programs/server/Server.cpp
@@ -1650,9 +1650,12 @@ try
         database_catalog.initializeAndLoadTemporaryDatabase();
         loadMetadataSystem(global_context);
         maybeConvertSystemDatabase(global_context);
+        /// This has to be done before the initialization of system logs,
+        /// otherwise there is a race condition between the system database initialization
+        /// and creation of new tables in the database.
+        startupSystemTables();
         /// After attaching system databases we can initialize system log.
         global_context->initializeSystemLogs();
-        startupSystemTables();
         global_context->setSystemZooKeeperLogAfterInitializationIfNeeded();
         /// Build loggers before tables startup to make log messages from tables
         /// attach available in system.text_log

From 109c8d8be6b3fdac2293d0a6e01b31882fc006d7 Mon Sep 17 00:00:00 2001
From: pufit <pufit@clickhouse.com>
Date: Mon, 7 Aug 2023 13:35:35 -0400
Subject: [PATCH 105/249] Add no-parallel tag to tests

---
 tests/queries/0_stateless/01293_create_role.sql             | 2 ++
 tests/queries/0_stateless/01294_create_settings_profile.sql | 2 ++
 tests/queries/0_stateless/01295_create_row_policy.sql       | 2 ++
 3 files changed, 6 insertions(+)

diff --git a/tests/queries/0_stateless/01293_create_role.sql b/tests/queries/0_stateless/01293_create_role.sql
index f22edfeec3e..fd75d62964d 100644
--- a/tests/queries/0_stateless/01293_create_role.sql
+++ b/tests/queries/0_stateless/01293_create_role.sql
@@ -1,3 +1,5 @@
+-- Tags: no-parallel
+
 DROP ROLE IF EXISTS r1_01293, r2_01293, r3_01293, r4_01293, r5_01293, r6_01293, r7_01293, r8_01293, r9_01293;
 DROP ROLE IF EXISTS r2_01293_renamed;
 DROP ROLE IF EXISTS r1_01293@'%', 'r2_01293@%.myhost.com';
diff --git a/tests/queries/0_stateless/01294_create_settings_profile.sql b/tests/queries/0_stateless/01294_create_settings_profile.sql
index 565b4e70367..f71eefa6975 100644
--- a/tests/queries/0_stateless/01294_create_settings_profile.sql
+++ b/tests/queries/0_stateless/01294_create_settings_profile.sql
@@ -1,3 +1,5 @@
+-- Tags: no-parallel
+
 DROP SETTINGS PROFILE IF EXISTS s1_01294, s2_01294, s3_01294, s4_01294, s5_01294, s6_01294, s7_01294, s8_01294, s9_01294, s10_01294;
 DROP SETTINGS PROFILE IF EXISTS s2_01294_renamed;
 DROP USER IF EXISTS u1_01294;
diff --git a/tests/queries/0_stateless/01295_create_row_policy.sql b/tests/queries/0_stateless/01295_create_row_policy.sql
index b484d0ea0f3..5ccd815c89a 100644
--- a/tests/queries/0_stateless/01295_create_row_policy.sql
+++ b/tests/queries/0_stateless/01295_create_row_policy.sql
@@ -1,3 +1,5 @@
+-- Tags: no-parallel
+
 DROP ROW POLICY IF EXISTS p1_01295, p2_01295, p3_01295, p4_01295, p5_01295, p6_01295, p7_01295, p8_01295, p9_01295, p10_01295 ON db.table;
 DROP ROW POLICY IF EXISTS p2_01295_renamed ON db.table;
 DROP ROW POLICY IF EXISTS p3_01295 ON db.table, db2.table2;

From 27559b0349fd194c1cb7f167318b19a67dce371f Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 7 Aug 2023 23:09:34 +0200
Subject: [PATCH 106/249] Adjust a test

---
 tests/ci/clickhouse_helper.py                        | 2 +-
 tests/queries/0_stateless/01161_all_system_tables.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/ci/clickhouse_helper.py b/tests/ci/clickhouse_helper.py
index 7f98f6e52d2..7abc33bddbd 100644
--- a/tests/ci/clickhouse_helper.py
+++ b/tests/ci/clickhouse_helper.py
@@ -61,7 +61,7 @@ class ClickHouseHelper:
             )
 
             if response.status_code >= 500:
-                # A retriable error
+                # A retryable error
                 time.sleep(1)
                 continue
 
diff --git a/tests/queries/0_stateless/01161_all_system_tables.sh b/tests/queries/0_stateless/01161_all_system_tables.sh
index 47316a6a805..3ba59f9a424 100755
--- a/tests/queries/0_stateless/01161_all_system_tables.sh
+++ b/tests/queries/0_stateless/01161_all_system_tables.sh
@@ -19,7 +19,7 @@ function run_selects()
     thread_num=$1
     readarray -t tables_arr < <(${CLICKHOUSE_CLIENT} -q "SELECT database || '.' || name FROM system.tables
     WHERE database in ('system', 'information_schema', 'INFORMATION_SCHEMA') and name != 'zookeeper' and name != 'models'
-    AND sipHash64(name || toString($RAND)) % $THREADS = $thread_num")
+    AND sipHash64(name || toString($RAND)) % $THREADS = $thread_num AND name NOT LIKE '%\\_sender' AND name NOT LIKE '%\\_watcher'")
 
     for t in "${tables_arr[@]}"
     do

From 5dd99db3692f9e2fb7264275c43385e274467674 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 8 Aug 2023 03:55:06 +0200
Subject: [PATCH 107/249] Add diagnostic info about file name during schema
 inference

---
 src/Formats/ReadSchemaUtils.cpp                  | 16 ++++++++++++----
 src/Processors/Formats/IRowInputFormat.cpp       | 16 +++++++++++++++-
 ...le_diagnostics_while_reading_header.reference |  2 ++
 ...2836_file_diagnostics_while_reading_header.sh | 11 +++++++++++
 4 files changed, 40 insertions(+), 5 deletions(-)
 create mode 100644 tests/queries/0_stateless/02836_file_diagnostics_while_reading_header.reference
 create mode 100755 tests/queries/0_stateless/02836_file_diagnostics_while_reading_header.sh

diff --git a/src/Formats/ReadSchemaUtils.cpp b/src/Formats/ReadSchemaUtils.cpp
index cbfa05bfba6..d183442ba44 100644
--- a/src/Formats/ReadSchemaUtils.cpp
+++ b/src/Formats/ReadSchemaUtils.cpp
@@ -1,13 +1,11 @@
-#include <DataTypes/DataTypeArray.h>
-#include <DataTypes/DataTypeLowCardinality.h>
 #include <DataTypes/DataTypeMap.h>
-#include <DataTypes/DataTypeNullable.h>
-#include <DataTypes/DataTypeTuple.h>
 #include <Formats/ReadSchemaUtils.h>
 #include <Interpreters/Context.h>
 #include <Processors/Formats/ISchemaReader.h>
 #include <Storages/IStorage.h>
 #include <Common/assert_cast.h>
+#include <IO/WithFileName.h>
+
 
 namespace DB
 {
@@ -53,6 +51,7 @@ ColumnsDescription readSchemaFromFormat(
     bool retry,
     ContextPtr & context,
     std::unique_ptr<ReadBuffer> & buf)
+try
 {
     NamesAndTypesList names_and_types;
     if (FormatFactory::instance().checkIfFormatHasExternalSchemaReader(format_name))
@@ -209,12 +208,21 @@ ColumnsDescription readSchemaFromFormat(
             ErrorCodes::BAD_ARGUMENTS,
             "{} file format doesn't support schema inference. You must specify the structure manually",
             format_name);
+
     /// Some formats like CSVWithNames can contain empty column names. We don't support empty column names and further processing can fail with an exception. Let's just remove columns with empty names from the structure.
     names_and_types.erase(
         std::remove_if(names_and_types.begin(), names_and_types.end(), [](const NameAndTypePair & pair) { return pair.name.empty(); }),
         names_and_types.end());
     return ColumnsDescription(names_and_types);
 }
+catch (Exception & e)
+{
+    auto file_name = getFileNameFromReadBuffer(*buf);
+    if (!file_name.empty())
+        e.addMessage(fmt::format("(in file/uri {})", file_name));
+    throw;
+}
+
 
 ColumnsDescription readSchemaFromFormat(
     const String & format_name,
diff --git a/src/Processors/Formats/IRowInputFormat.cpp b/src/Processors/Formats/IRowInputFormat.cpp
index 8bcf293033b..7395d9b916a 100644
--- a/src/Processors/Formats/IRowInputFormat.cpp
+++ b/src/Processors/Formats/IRowInputFormat.cpp
@@ -86,7 +86,21 @@ void IRowInputFormat::logError()
 Chunk IRowInputFormat::generate()
 {
     if (total_rows == 0)
-        readPrefix();
+    {
+        try
+        {
+            readPrefix();
+        }
+        catch (Exception & e)
+        {
+            auto file_name = getFileNameFromReadBuffer(getReadBuffer());
+            if (!file_name.empty())
+                e.addMessage(fmt::format("(in file/uri {})", file_name));
+
+            e.addMessage("(while reading header)");
+            throw;
+        }
+    }
 
     const Block & header = getPort().getHeader();
 
diff --git a/tests/queries/0_stateless/02836_file_diagnostics_while_reading_header.reference b/tests/queries/0_stateless/02836_file_diagnostics_while_reading_header.reference
new file mode 100644
index 00000000000..6829f972684
--- /dev/null
+++ b/tests/queries/0_stateless/02836_file_diagnostics_while_reading_header.reference
@@ -0,0 +1,2 @@
+in file/uri
+test.csv
diff --git a/tests/queries/0_stateless/02836_file_diagnostics_while_reading_header.sh b/tests/queries/0_stateless/02836_file_diagnostics_while_reading_header.sh
new file mode 100755
index 00000000000..d1b5ffa2af8
--- /dev/null
+++ b/tests/queries/0_stateless/02836_file_diagnostics_while_reading_header.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+FILENAME="${CLICKHOUSE_TMP}/test.csv"
+
+printf 'Bad\rHeader\n123\n' > "${FILENAME}"
+${CLICKHOUSE_LOCAL} --query "SELECT * FROM file('${CLICKHOUSE_TMP}/t*e*s*t.csv')" 2>&1 | grep -o -P 'in file/uri|test\.csv'
+rm "${FILENAME}"

From 1994bc0d05d209bed85bcc66ef2892d2c708ce3a Mon Sep 17 00:00:00 2001
From: Smita Kulkarni <Smita.Kulkarni@clickhouse.com>
Date: Tue, 8 Aug 2023 11:08:43 +0200
Subject: [PATCH 108/249] Improvements to disallow concurrency of backup
 restore function, added wait_for_fail_backup & wait_for_fail_restore

---
 .../test_disallow_concurrency.py              | 184 +++++++++---------
 1 file changed, 92 insertions(+), 92 deletions(-)

diff --git a/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py b/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py
index 5c3f06a9d9d..17832846f92 100644
--- a/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py
+++ b/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py
@@ -5,6 +5,7 @@ import time
 import concurrent
 from helpers.cluster import ClickHouseCluster
 from helpers.test_tools import TSV, assert_eq_with_retry
+import re
 
 cluster = ClickHouseCluster(__file__)
 
@@ -110,6 +111,82 @@ def create_and_fill_table():
         nodes[i].query(f"INSERT INTO tbl SELECT number FROM numbers(40000000)")
 
 
+def wait_for_fail_backup(node, backup_id, backup_name):
+    expected_errors = [
+        "Concurrent backups not supported",
+        f"Backup {backup_name} already exists",
+    ]
+    status = (
+        node
+        .query(f"SELECT status FROM system.backups WHERE id == '{backup_id}'")
+        .rstrip("\n")
+    )
+    # It is possible that the second backup was picked up first, and then the async backup
+    if status == "BACKUP_FAILED":
+        error = (
+            node
+            .query(f"SELECT error FROM system.backups WHERE id == '{backup_id}'")
+            .rstrip("\n")
+        )
+        assert any([expected_error in error for expected_error in expected_errors])
+        return
+    elif status == "CREATING_BACKUP":
+        assert_eq_with_retry(
+            node,
+            f"SELECT status FROM system.backups WHERE id = '{backup_id}'",
+            "BACKUP_FAILED",
+            sleep_time=2,
+            retry_count=50,
+        )
+        error = (
+            node
+            .query(f"SELECT error FROM system.backups WHERE id == '{backup_id}'")
+            .rstrip("\n")
+        )
+        assert re.search(f"Backup {backup_name} already exists",error)
+        return
+    else:
+        assert False, "Concurrent backups both passed, when one is expected to fail"
+
+
+def wait_for_fail_restore(node, restore_id):
+    expected_errors = [
+        "Concurrent restores not supported",
+        "Cannot restore the table default.tbl because it already contains some data",
+    ]
+    status = (
+        node
+        .query(f"SELECT status FROM system.backups WHERE id == '{restore_id}'")
+        .rstrip("\n")
+    )
+    # It is possible that the second backup was picked up first, and then the async backup
+    if status == "RESTORE_FAILED":
+        error = (
+            node
+            .query(f"SELECT error FROM system.backups WHERE id == '{restore_id}'")
+            .rstrip("\n")
+        )
+        assert any([expected_error in error for expected_error in expected_errors])
+        return
+    elif status == "RESTORING":
+        assert_eq_with_retry(
+            node,
+            f"SELECT status FROM system.backups WHERE id = '{backup_id}'",
+            "RESTORE_FAILED",
+            sleep_time=2,
+            retry_count=50,
+        )
+        error = (
+            node
+            .query(f"SELECT error FROM system.backups WHERE id == '{backup_id}'")
+            .rstrip("\n")
+        )
+        assert re.search("Cannot restore the table default.tbl because it already contains some data",error)
+        return
+    else:
+        assert False, "Concurrent restores both passed, when one is expected to fail"
+
+
 # All the tests have concurrent backup/restores with same backup names
 # The same works with different backup names too. Since concurrency
 # check comes before backup name check, separate tests are not added for different names
@@ -137,33 +214,13 @@ def test_concurrent_backups_on_same_node():
         f"BACKUP TABLE tbl ON CLUSTER 'cluster' TO {backup_name}"
     )
 
-    if not error:
-        status = (
-            nodes[0]
-            .query(f"SELECT status FROM system.backups WHERE id == '{id}'")
-            .rstrip("\n")
-        )
-        # It is possible that the second backup was picked up first, and then the async backup
-        if status == "BACKUP_FAILED":
-            return
-        elif status == "CREATING_BACKUP":
-            assert_eq_with_retry(
-                nodes[0],
-                f"SELECT status FROM system.backups WHERE id = '{id}'",
-                "BACKUP_FAILED",
-                sleep_time=2,
-                retry_count=50,
-            )
-            return
-        else:
-            raise Exception(
-                "Concurrent backups both passed, when one is expected to fail"
-            )
-
     expected_errors = [
         "Concurrent backups not supported",
         f"Backup {backup_name} already exists",
     ]
+    if not error:
+        wait_for_fail_backup(nodes[0],id,backup_name)
+
     assert any([expected_error in error for expected_error in expected_errors])
 
     assert_eq_with_retry(
@@ -207,33 +264,14 @@ def test_concurrent_backups_on_different_nodes():
         f"BACKUP TABLE tbl ON CLUSTER 'cluster' TO {backup_name}"
     )
 
-    if not error:
-        status = (
-            nodes[1]
-            .query(f"SELECT status FROM system.backups WHERE id == '{id}'")
-            .rstrip("\n")
-        )
-        # It is possible that the second backup was picked up first, and then the async backup
-        if status == "BACKUP_FAILED":
-            return
-        elif status == "CREATING_BACKUP":
-            assert_eq_with_retry(
-                nodes[1],
-                f"SELECT status FROM system.backups WHERE id = '{id}'",
-                "BACKUP_FAILED",
-                sleep_time=2,
-                retry_count=50,
-            )
-            return
-        else:
-            raise Exception(
-                "Concurrent backups both passed, when one is expected to fail"
-            )
-
     expected_errors = [
         "Concurrent backups not supported",
         f"Backup {backup_name} already exists",
     ]
+
+    if not error:
+        wait_for_fail_backup(nodes[1],id,backup_name)
+
     assert any([expected_error in error for expected_error in expected_errors])
 
     assert_eq_with_retry(
@@ -276,33 +314,14 @@ def test_concurrent_restores_on_same_node():
         f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name}"
     )
 
-    if not error:
-        status = (
-            nodes[0]
-            .query(f"SELECT status FROM system.backups WHERE id == '{restore_id}'")
-            .rstrip("\n")
-        )
-        # It is possible that the second backup was picked up first, and then the async backup
-        if status == "RESTORE_FAILED":
-            return
-        elif status == "RESTORING":
-            assert_eq_with_retry(
-                nodes[0],
-                f"SELECT status FROM system.backups WHERE id == '{restore_id}'",
-                "RESTORE_FAILED",
-                sleep_time=2,
-                retry_count=50,
-            )
-            return
-        else:
-            raise Exception(
-                "Concurrent restores both passed, when one is expected to fail"
-            )
-
     expected_errors = [
         "Concurrent restores not supported",
         "Cannot restore the table default.tbl because it already contains some data",
     ]
+
+    if not error:
+        wait_for_fail_restore(nodes[0],restore_id)
+
     assert any([expected_error in error for expected_error in expected_errors])
 
     assert_eq_with_retry(
@@ -345,33 +364,14 @@ def test_concurrent_restores_on_different_node():
         f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name}"
     )
 
-    if not error:
-        status = (
-            nodes[0]
-            .query(f"SELECT status FROM system.backups WHERE id == '{restore_id}'")
-            .rstrip("\n")
-        )
-        # It is possible that the second backup was picked up first, and then the async backup
-        if status == "RESTORE_FAILED":
-            return
-        elif status == "RESTORING":
-            assert_eq_with_retry(
-                nodes[0],
-                f"SELECT status FROM system.backups WHERE id == '{restore_id}'",
-                "RESTORE_FAILED",
-                sleep_time=2,
-                retry_count=50,
-            )
-            return
-        else:
-            raise Exception(
-                "Concurrent restores both passed, when one is expected to fail"
-            )
-
     expected_errors = [
         "Concurrent restores not supported",
         "Cannot restore the table default.tbl because it already contains some data",
     ]
+
+    if not error:
+        wait_for_fail_restore(nodes[0],restore_id)
+
     assert any([expected_error in error for expected_error in expected_errors])
 
     assert_eq_with_retry(

From 201b4fd661ab3763f9ec9c9933cc40c6ada4a0d3 Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Tue, 8 Aug 2023 09:32:44 +0000
Subject: [PATCH 109/249] Automatic style fix

---
 .../test_disallow_concurrency.py              | 61 ++++++++-----------
 1 file changed, 26 insertions(+), 35 deletions(-)

diff --git a/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py b/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py
index 17832846f92..c9f20333654 100644
--- a/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py
+++ b/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py
@@ -116,18 +116,14 @@ def wait_for_fail_backup(node, backup_id, backup_name):
         "Concurrent backups not supported",
         f"Backup {backup_name} already exists",
     ]
-    status = (
-        node
-        .query(f"SELECT status FROM system.backups WHERE id == '{backup_id}'")
-        .rstrip("\n")
-    )
+    status = node.query(
+        f"SELECT status FROM system.backups WHERE id == '{backup_id}'"
+    ).rstrip("\n")
     # It is possible that the second backup was picked up first, and then the async backup
     if status == "BACKUP_FAILED":
-        error = (
-            node
-            .query(f"SELECT error FROM system.backups WHERE id == '{backup_id}'")
-            .rstrip("\n")
-        )
+        error = node.query(
+            f"SELECT error FROM system.backups WHERE id == '{backup_id}'"
+        ).rstrip("\n")
         assert any([expected_error in error for expected_error in expected_errors])
         return
     elif status == "CREATING_BACKUP":
@@ -138,12 +134,10 @@ def wait_for_fail_backup(node, backup_id, backup_name):
             sleep_time=2,
             retry_count=50,
         )
-        error = (
-            node
-            .query(f"SELECT error FROM system.backups WHERE id == '{backup_id}'")
-            .rstrip("\n")
-        )
-        assert re.search(f"Backup {backup_name} already exists",error)
+        error = node.query(
+            f"SELECT error FROM system.backups WHERE id == '{backup_id}'"
+        ).rstrip("\n")
+        assert re.search(f"Backup {backup_name} already exists", error)
         return
     else:
         assert False, "Concurrent backups both passed, when one is expected to fail"
@@ -154,18 +148,14 @@ def wait_for_fail_restore(node, restore_id):
         "Concurrent restores not supported",
         "Cannot restore the table default.tbl because it already contains some data",
     ]
-    status = (
-        node
-        .query(f"SELECT status FROM system.backups WHERE id == '{restore_id}'")
-        .rstrip("\n")
-    )
+    status = node.query(
+        f"SELECT status FROM system.backups WHERE id == '{restore_id}'"
+    ).rstrip("\n")
     # It is possible that the second backup was picked up first, and then the async backup
     if status == "RESTORE_FAILED":
-        error = (
-            node
-            .query(f"SELECT error FROM system.backups WHERE id == '{restore_id}'")
-            .rstrip("\n")
-        )
+        error = node.query(
+            f"SELECT error FROM system.backups WHERE id == '{restore_id}'"
+        ).rstrip("\n")
         assert any([expected_error in error for expected_error in expected_errors])
         return
     elif status == "RESTORING":
@@ -176,12 +166,13 @@ def wait_for_fail_restore(node, restore_id):
             sleep_time=2,
             retry_count=50,
         )
-        error = (
-            node
-            .query(f"SELECT error FROM system.backups WHERE id == '{backup_id}'")
-            .rstrip("\n")
+        error = node.query(
+            f"SELECT error FROM system.backups WHERE id == '{backup_id}'"
+        ).rstrip("\n")
+        assert re.search(
+            "Cannot restore the table default.tbl because it already contains some data",
+            error,
         )
-        assert re.search("Cannot restore the table default.tbl because it already contains some data",error)
         return
     else:
         assert False, "Concurrent restores both passed, when one is expected to fail"
@@ -219,7 +210,7 @@ def test_concurrent_backups_on_same_node():
         f"Backup {backup_name} already exists",
     ]
     if not error:
-        wait_for_fail_backup(nodes[0],id,backup_name)
+        wait_for_fail_backup(nodes[0], id, backup_name)
 
     assert any([expected_error in error for expected_error in expected_errors])
 
@@ -270,7 +261,7 @@ def test_concurrent_backups_on_different_nodes():
     ]
 
     if not error:
-        wait_for_fail_backup(nodes[1],id,backup_name)
+        wait_for_fail_backup(nodes[1], id, backup_name)
 
     assert any([expected_error in error for expected_error in expected_errors])
 
@@ -320,7 +311,7 @@ def test_concurrent_restores_on_same_node():
     ]
 
     if not error:
-        wait_for_fail_restore(nodes[0],restore_id)
+        wait_for_fail_restore(nodes[0], restore_id)
 
     assert any([expected_error in error for expected_error in expected_errors])
 
@@ -370,7 +361,7 @@ def test_concurrent_restores_on_different_node():
     ]
 
     if not error:
-        wait_for_fail_restore(nodes[0],restore_id)
+        wait_for_fail_restore(nodes[0], restore_id)
 
     assert any([expected_error in error for expected_error in expected_errors])
 

From a882ef295f5fdad5a805aa9ed6761ba1129ff081 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Fri, 28 Jul 2023 16:00:01 +0000
Subject: [PATCH 110/249] Query CHECK TABLE takes care about progress and
 cancellation

---
 src/Common/FileChecker.cpp                  |  22 ++-
 src/Common/FileChecker.h                    |   1 +
 src/Interpreters/InterpreterCheckQuery.cpp  | 172 +++++++++++++++-----
 src/Storages/CheckResults.h                 |   5 +
 src/Storages/IStorage.cpp                   |   8 +
 src/Storages/IStorage.h                     |   4 +-
 src/Storages/StorageLog.cpp                 |   4 +-
 src/Storages/StorageLog.h                   |   2 +-
 src/Storages/StorageMergeTree.cpp           |  20 ++-
 src/Storages/StorageMergeTree.h             |   2 +-
 src/Storages/StorageProxy.h                 |   3 +-
 src/Storages/StorageReplicatedMergeTree.cpp |  13 +-
 src/Storages/StorageReplicatedMergeTree.h   |   2 +-
 src/Storages/StorageStripeLog.cpp           |   5 +-
 src/Storages/StorageStripeLog.h             |   2 +-
 15 files changed, 196 insertions(+), 69 deletions(-)

diff --git a/src/Common/FileChecker.cpp b/src/Common/FileChecker.cpp
index 876bc4e641c..dc3800eb29a 100644
--- a/src/Common/FileChecker.cpp
+++ b/src/Common/FileChecker.cpp
@@ -84,10 +84,20 @@ size_t FileChecker::getTotalSize() const
 
 CheckResults FileChecker::check() const
 {
-    if (map.empty())
-        return {};
-
     CheckResults results;
+    auto callback = [&results](const CheckResult & result, size_t) -> bool
+    {
+        results.push_back(result);
+        return true;
+    };
+    check(callback);
+    return results;
+}
+
+void FileChecker::check(CheckDataCallback check_callback) const
+{
+    if (map.empty())
+        return;
 
     for (const auto & name_size : map)
     {
@@ -101,14 +111,12 @@ CheckResults FileChecker::check() const
             String failure_message = exists
                 ? ("Size of " + path + " is wrong. Size is " + toString(real_size) + " but should be " + toString(name_size.second))
                 : ("File " + path + " doesn't exist");
-            results.emplace_back(name, false, failure_message);
+            check_callback(CheckResult(name, false, failure_message), map.size());
             break;
         }
 
-        results.emplace_back(name, true, "");
+        check_callback(CheckResult(name, true, ""), map.size());
     }
-
-    return results;
 }
 
 void FileChecker::repair()
diff --git a/src/Common/FileChecker.h b/src/Common/FileChecker.h
index bb0383e4b56..28dc17a4610 100644
--- a/src/Common/FileChecker.h
+++ b/src/Common/FileChecker.h
@@ -29,6 +29,7 @@ public:
 
     /// Check the files whose parameters are specified in sizes.json
     CheckResults check() const;
+    void check(CheckDataCallback check_callback) const;
 
     /// Truncate files that have excessive size to the expected size.
     /// Throw exception if the file size is less than expected.
diff --git a/src/Interpreters/InterpreterCheckQuery.cpp b/src/Interpreters/InterpreterCheckQuery.cpp
index 333aed84873..00d2749eaf9 100644
--- a/src/Interpreters/InterpreterCheckQuery.cpp
+++ b/src/Interpreters/InterpreterCheckQuery.cpp
@@ -8,6 +8,7 @@
 #include <DataTypes/DataTypeString.h>
 #include <Columns/ColumnsNumber.h>
 #include <Common/typeid_cast.h>
+#include <Interpreters/ProcessList.h>
 #include <algorithm>
 
 
@@ -17,19 +18,139 @@ namespace DB
 namespace
 {
 
-NamesAndTypes getBlockStructure()
+Block getBlockFromCheckResult(const CheckResults & check_results, bool check_query_single_value_result)
 {
-    return {
+    if (check_query_single_value_result)
+    {
+        bool result = std::all_of(check_results.begin(), check_results.end(), [] (const CheckResult & res) { return res.success; });
+        return Block{{ColumnUInt8::create(1, static_cast<UInt8>(result)), std::make_shared<DataTypeUInt8>(), "result"}};
+    }
+
+    NamesAndTypes block_structure = NamesAndTypes{
         {"part_path", std::make_shared<DataTypeString>()},
         {"is_passed", std::make_shared<DataTypeUInt8>()},
         {"message", std::make_shared<DataTypeString>()},
     };
+    auto path_column = block_structure[0].type->createColumn();
+    auto is_passed_column = block_structure[1].type->createColumn();
+    auto message_column = block_structure[2].type->createColumn();
+
+    for (const auto & check_result : check_results)
+    {
+        path_column->insert(check_result.fs_path);
+        is_passed_column->insert(static_cast<UInt8>(check_result.success));
+        message_column->insert(check_result.failure_message);
+    }
+
+    return Block({
+        {std::move(path_column), block_structure[0].type, block_structure[0].name},
+        {std::move(is_passed_column), block_structure[1].type, block_structure[1].name},
+        {std::move(message_column), block_structure[2].type, block_structure[2].name},
+    });
 }
 
+class TableCheckResultSource : public ISource
+{
+
+public:
+    explicit TableCheckResultSource(const ASTPtr & query_ptr_, StoragePtr table_, bool check_query_single_value_result_, ContextPtr context_)
+        : ISource(getBlockFromCheckResult({}, check_query_single_value_result_).cloneEmpty())
+        , query_ptr(query_ptr_)
+        , table(table_)
+        , context(context_)
+        , check_query_single_value_result(check_query_single_value_result_)
+    {
+        worker_result = std::async(std::launch::async, [this]{ worker(); });
+    }
+
+    String getName() const override { return "TableCheckResultSource"; }
+
+protected:
+
+    std::optional<Chunk> tryGenerate() override
+    {
+
+        if (is_check_completed)
+            return {};
+
+        auto status = worker_result.wait_for(std::chrono::milliseconds(100));
+        is_check_completed = (status == std::future_status::ready);
+
+        if (is_check_completed)
+        {
+            worker_result.get();
+            auto result_block = getBlockFromCheckResult(check_results, check_query_single_value_result);
+            check_results.clear();
+            return Chunk(result_block.getColumns(), result_block.rows());
+        }
+
+        std::lock_guard lock(mutex);
+        progress(progress_rows, 0);
+        progress_rows = 0;
+
+        if (check_query_single_value_result || check_results.empty())
+        {
+            return Chunk();
+        }
+
+        auto result_block = getBlockFromCheckResult(check_results, check_query_single_value_result);
+        check_results.clear();
+        return Chunk(result_block.getColumns(), result_block.rows());
+    }
+
+private:
+    void worker()
+    {
+        table->checkData(query_ptr, context,
+            [this](const CheckResult & check_result, size_t new_total_rows)
+            {
+                if (isCancelled())
+                    return false;
+
+                std::lock_guard lock(mutex);
+                if (new_total_rows > total_rows)
+                {
+                    addTotalRowsApprox(new_total_rows - total_rows);
+                    total_rows = new_total_rows;
+                }
+                progress_rows++;
+
+                if (!check_result.success)
+                {
+                    LOG_WARNING(&Poco::Logger::get("InterpreterCheckQuery"),
+                        "Check query for table {} failed, path {}, reason: {}",
+                        table->getStorageID().getNameForLogs(),
+                        check_result.fs_path,
+                        check_result.failure_message);
+                }
+
+                check_results.push_back(check_result);
+
+                bool should_continue = check_result.success || !check_query_single_value_result;
+                return should_continue;
+            });
+    }
+
+    ASTPtr query_ptr;
+    StoragePtr table;
+    ContextPtr context;
+    bool check_query_single_value_result;
+
+    std::future<void> worker_result;
+
+    std::mutex mutex;
+    CheckResults check_results;
+    size_t progress_rows = 0;
+    size_t total_rows = 0;
+
+    bool is_check_completed = false;
+};
+
 }
 
-
-InterpreterCheckQuery::InterpreterCheckQuery(const ASTPtr & query_ptr_, ContextPtr context_) : WithContext(context_), query_ptr(query_ptr_)
+InterpreterCheckQuery::InterpreterCheckQuery(const ASTPtr & query_ptr_, ContextPtr context_)
+    : WithContext(context_)
+    , query_ptr(query_ptr_)
 {
 }
 
@@ -37,43 +158,18 @@ InterpreterCheckQuery::InterpreterCheckQuery(const ASTPtr & query_ptr_, ContextP
 BlockIO InterpreterCheckQuery::execute()
 {
     const auto & check = query_ptr->as<ASTCheckQuery &>();
-    auto table_id = getContext()->resolveStorageID(check, Context::ResolveOrdinary);
+    const auto & context = getContext();
+    auto table_id = context->resolveStorageID(check, Context::ResolveOrdinary);
 
-    getContext()->checkAccess(AccessType::SHOW_TABLES, table_id);
-    StoragePtr table = DatabaseCatalog::instance().getTable(table_id, getContext());
-    auto check_results = table->checkData(query_ptr, getContext());
-
-    Block block;
-    if (getContext()->getSettingsRef().check_query_single_value_result)
-    {
-        bool result = std::all_of(check_results.begin(), check_results.end(), [] (const CheckResult & res) { return res.success; });
-        auto column = ColumnUInt8::create();
-        column->insertValue(static_cast<UInt64>(result));
-        block = Block{{std::move(column), std::make_shared<DataTypeUInt8>(), "result"}};
-    }
-    else
-    {
-        auto block_structure = getBlockStructure();
-        auto path_column = block_structure[0].type->createColumn();
-        auto is_passed_column = block_structure[1].type->createColumn();
-        auto message_column = block_structure[2].type->createColumn();
-
-        for (const auto & check_result : check_results)
-        {
-            path_column->insert(check_result.fs_path);
-            is_passed_column->insert(static_cast<UInt8>(check_result.success));
-            message_column->insert(check_result.failure_message);
-        }
-
-        block = Block({
-            {std::move(path_column), block_structure[0].type, block_structure[0].name},
-            {std::move(is_passed_column), block_structure[1].type, block_structure[1].name},
-            {std::move(message_column), block_structure[2].type, block_structure[2].name}});
-    }
+    context->checkAccess(AccessType::SHOW_TABLES, table_id);
+    StoragePtr table = DatabaseCatalog::instance().getTable(table_id, context);
 
     BlockIO res;
-    res.pipeline = QueryPipeline(std::make_shared<SourceFromSingleChunk>(std::move(block)));
-
+    {
+        bool check_query_single_value_result = context->getSettingsRef().check_query_single_value_result;
+        auto result_source = std::make_shared<TableCheckResultSource>(query_ptr, table, check_query_single_value_result, context);
+        res.pipeline = QueryPipeline(result_source);
+    }
     return res;
 }
 
diff --git a/src/Storages/CheckResults.h b/src/Storages/CheckResults.h
index b342b014fa4..b84262b50c9 100644
--- a/src/Storages/CheckResults.h
+++ b/src/Storages/CheckResults.h
@@ -22,6 +22,11 @@ struct CheckResult
     {}
 };
 
+/// Process single result of checkData
+/// Second argument is an estimated number of check results
+/// Return true to continue checking, false to stop
+using CheckDataCallback = std::function<bool(const CheckResult &, size_t)>;
+
 using CheckResults = std::vector<CheckResult>;
 
 }
diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp
index ae7659e074f..467a34ca855 100644
--- a/src/Storages/IStorage.cpp
+++ b/src/Storages/IStorage.cpp
@@ -273,6 +273,14 @@ bool IStorage::isStaticStorage() const
     return false;
 }
 
+CheckResults IStorage::checkData(const ASTPtr & query, ContextPtr context)
+{
+    CheckResults results;
+    auto callback = [&](const CheckResult & result, size_t) { results.push_back(result); return true;};
+    checkData(query, context, callback);
+    return results;
+}
+
 void IStorage::adjustCreateQueryForBackup(ASTPtr &) const
 {
 }
diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h
index ec92f57aeda..48ebf6b1242 100644
--- a/src/Storages/IStorage.h
+++ b/src/Storages/IStorage.h
@@ -596,7 +596,9 @@ public:
     virtual bool mayBenefitFromIndexForIn(const ASTPtr & /* left_in_operand */, ContextPtr /* query_context */, const StorageMetadataPtr & /* metadata_snapshot */) const { return false; }
 
     /// Checks validity of the data
-    virtual CheckResults checkData(const ASTPtr & /* query */, ContextPtr /* context */) { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Check query is not supported for {} storage", getName()); }
+    virtual CheckResults checkData(const ASTPtr & /* query */, ContextPtr /* context */);
+
+    virtual void checkData(const ASTPtr & /* query */, ContextPtr /* context */, CheckDataCallback /* callback */) { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Check query is not supported for {} storage", getName()); }
 
     /// Checks that table could be dropped right now
     /// Otherwise - throws an exception with detailed information.
diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp
index 87aa71f3e8d..bb58e05d756 100644
--- a/src/Storages/StorageLog.cpp
+++ b/src/Storages/StorageLog.cpp
@@ -866,13 +866,13 @@ SinkToStoragePtr StorageLog::write(const ASTPtr & /*query*/, const StorageMetada
     return std::make_shared<LogSink>(*this, metadata_snapshot, std::move(lock));
 }
 
-CheckResults StorageLog::checkData(const ASTPtr & /* query */, ContextPtr local_context)
+void StorageLog::checkData(const ASTPtr & /* query */, ContextPtr local_context, CheckDataCallback check_callback)
 {
     ReadLock lock{rwlock, getLockTimeout(local_context)};
     if (!lock)
         throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Lock timeout exceeded");
 
-    return file_checker.check();
+    file_checker.check(check_callback);
 }
 
 
diff --git a/src/Storages/StorageLog.h b/src/Storages/StorageLog.h
index f1d05ed39ac..f2539a6c48c 100644
--- a/src/Storages/StorageLog.h
+++ b/src/Storages/StorageLog.h
@@ -59,7 +59,7 @@ public:
 
     void rename(const String & new_path_to_table_data, const StorageID & new_table_id) override;
 
-    CheckResults checkData(const ASTPtr & query, ContextPtr local_context) override;
+    void checkData(const ASTPtr & query, ContextPtr local_context, CheckDataCallback check_callback) override;
 
     void truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr, TableExclusiveLockHolder &) override;
 
diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp
index ad9013d9f13..4754412a0de 100644
--- a/src/Storages/StorageMergeTree.cpp
+++ b/src/Storages/StorageMergeTree.cpp
@@ -2197,9 +2197,8 @@ void StorageMergeTree::onActionLockRemove(StorageActionBlockType action_type)
         background_moves_assignee.trigger();
 }
 
-CheckResults StorageMergeTree::checkData(const ASTPtr & query, ContextPtr local_context)
+void StorageMergeTree::checkData(const ASTPtr & query, ContextPtr local_context, CheckDataCallback check_callback)
 {
-    CheckResults results;
     DataPartsVector data_parts;
     if (const auto & check_query = query->as<ASTCheckQuery &>(); check_query.partition)
     {
@@ -2224,12 +2223,16 @@ CheckResults StorageMergeTree::checkData(const ASTPtr & query, ContextPtr local_
                 part_mutable.writeChecksums(part->checksums, local_context->getWriteSettings());
 
                 part->checkMetadata();
-                results.emplace_back(part->name, true, "Checksums recounted and written to disk.");
+                bool should_continue = check_callback(CheckResult(part->name, true, "Checksums recounted and written to disk."), data_parts.size());
+                if (!should_continue)
+                    break;
             }
             catch (const Exception & ex)
             {
                 tryLogCurrentException(log, __PRETTY_FUNCTION__);
-                results.emplace_back(part->name, false, "Check of part finished with error: '" + ex.message() + "'");
+                bool should_continue = check_callback(CheckResult(part->name, false, "Check of part finished with error: '" + ex.message() + "'"), data_parts.size());
+                if (!should_continue)
+                    break;
             }
         }
         else
@@ -2238,15 +2241,18 @@ CheckResults StorageMergeTree::checkData(const ASTPtr & query, ContextPtr local_
             {
                 checkDataPart(part, true);
                 part->checkMetadata();
-                results.emplace_back(part->name, true, "");
+                bool should_continue = check_callback(CheckResult(part->name, true, ""), data_parts.size());
+                if (!should_continue)
+                    break;
             }
             catch (const Exception & ex)
             {
-                results.emplace_back(part->name, false, ex.message());
+                bool should_continue = check_callback(CheckResult(part->name, false, ex.message()), data_parts.size());
+                if (!should_continue)
+                    break;
             }
         }
     }
-    return results;
 }
 
 
diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h
index c77e5140d75..6b1b1b7b415 100644
--- a/src/Storages/StorageMergeTree.h
+++ b/src/Storages/StorageMergeTree.h
@@ -108,7 +108,7 @@ public:
 
     void onActionLockRemove(StorageActionBlockType action_type) override;
 
-    CheckResults checkData(const ASTPtr & query, ContextPtr context) override;
+    void checkData(const ASTPtr & query, ContextPtr context, CheckDataCallback check_callback) override;
 
     bool scheduleDataProcessingJob(BackgroundJobsAssignee & assignee) override;
 
diff --git a/src/Storages/StorageProxy.h b/src/Storages/StorageProxy.h
index 21ed4b91c62..5339656782e 100644
--- a/src/Storages/StorageProxy.h
+++ b/src/Storages/StorageProxy.h
@@ -149,7 +149,8 @@ public:
         return getNested()->mayBenefitFromIndexForIn(left_in_operand, query_context, metadata_snapshot);
     }
 
-    CheckResults checkData(const ASTPtr & query, ContextPtr context) override { return getNested()->checkData(query, context); }
+    void checkData(const ASTPtr & query, ContextPtr context, CheckDataCallback check_callback) override { getNested()->checkData(query, context, check_callback); }
+
     void checkTableCanBeDropped() const override { getNested()->checkTableCanBeDropped(); }
     bool storesDataOnDisk() const override { return getNested()->storesDataOnDisk(); }
     Strings getDataPaths() const override { return getNested()->getDataPaths(); }
diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index 7fce373e26b..71fbd009d70 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -8481,9 +8481,8 @@ void StorageReplicatedMergeTree::enqueuePartForCheck(const String & part_name, t
     part_check_thread.enqueuePart(part_name, delay_to_check_seconds);
 }
 
-CheckResults StorageReplicatedMergeTree::checkData(const ASTPtr & query, ContextPtr local_context)
+void StorageReplicatedMergeTree::checkData(const ASTPtr & query, ContextPtr local_context, CheckDataCallback check_callback)
 {
-    CheckResults results;
     DataPartsVector data_parts;
     if (const auto & check_query = query->as<ASTCheckQuery &>(); check_query.partition)
     {
@@ -8500,17 +8499,19 @@ CheckResults StorageReplicatedMergeTree::checkData(const ASTPtr & query, Context
         {
             try
             {
-                results.push_back(part_check_thread.checkPartAndFix(part->name));
+                bool should_continue = check_callback(part_check_thread.checkPartAndFix(part->name), data_parts.size());
+                if (!should_continue)
+                    break;
             }
             catch (const Exception & ex)
             {
                 tryLogCurrentException(log, __PRETTY_FUNCTION__);
-                results.emplace_back(part->name, false, "Check of part finished with error: '" + ex.message() + "'");
+                bool should_continue = check_callback(CheckResult(part->name, false, "Check of part finished with error: '" + ex.message() + "'"), data_parts.size());
+                if (!should_continue)
+                    break;
             }
         }
     }
-
-    return results;
 }
 
 
diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h
index 78ef39f032f..6f9853122cb 100644
--- a/src/Storages/StorageReplicatedMergeTree.h
+++ b/src/Storages/StorageReplicatedMergeTree.h
@@ -230,7 +230,7 @@ public:
     /// Add a part to the queue of parts whose data you want to check in the background thread.
     void enqueuePartForCheck(const String & part_name, time_t delay_to_check_seconds = 0);
 
-    CheckResults checkData(const ASTPtr & query, ContextPtr context) override;
+    void checkData(const ASTPtr & query, ContextPtr context, CheckDataCallback check_callback) override;
 
     /// Checks ability to use granularity
     bool canUseAdaptiveGranularity() const override;
diff --git a/src/Storages/StorageStripeLog.cpp b/src/Storages/StorageStripeLog.cpp
index 0bfef5ed5e5..5a3a007035c 100644
--- a/src/Storages/StorageStripeLog.cpp
+++ b/src/Storages/StorageStripeLog.cpp
@@ -403,14 +403,13 @@ SinkToStoragePtr StorageStripeLog::write(const ASTPtr & /*query*/, const Storage
     return std::make_shared<StripeLogSink>(*this, metadata_snapshot, std::move(lock));
 }
 
-
-CheckResults StorageStripeLog::checkData(const ASTPtr & /* query */, ContextPtr local_context)
+void StorageStripeLog::checkData(const ASTPtr & /* query */, ContextPtr local_context, CheckDataCallback check_callback)
 {
     ReadLock lock{rwlock, getLockTimeout(local_context)};
     if (!lock)
         throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Lock timeout exceeded");
 
-    return file_checker.check();
+    file_checker.check(check_callback);
 }
 
 
diff --git a/src/Storages/StorageStripeLog.h b/src/Storages/StorageStripeLog.h
index f889a1de71b..61ce49f43da 100644
--- a/src/Storages/StorageStripeLog.h
+++ b/src/Storages/StorageStripeLog.h
@@ -53,7 +53,7 @@ public:
 
     void rename(const String & new_path_to_table_data, const StorageID & new_table_id) override;
 
-    CheckResults checkData(const ASTPtr & query, ContextPtr ocal_context) override;
+    void checkData(const ASTPtr & query, ContextPtr ocal_context, CheckDataCallback check_callback) override;
 
     bool storesDataOnDisk() const override { return true; }
     Strings getDataPaths() const override { return {DB::fullPath(disk, table_path)}; }

From 1aedc4e89292b39d29367fc41b3485c893c08be3 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Tue, 1 Aug 2023 10:57:59 +0000
Subject: [PATCH 111/249] Build proper pipeline for CHECK TABLE

---
 src/Common/FileChecker.cpp                  |  52 ++---
 src/Common/FileChecker.h                    |  38 +++-
 src/Interpreters/InterpreterCheckQuery.cpp  | 219 +++++++++++---------
 src/Storages/CheckResults.h                 |   7 -
 src/Storages/IStorage.cpp                   |  13 +-
 src/Storages/IStorage.h                     |  41 +++-
 src/Storages/StorageLog.cpp                 |   9 +-
 src/Storages/StorageLog.h                   |  16 +-
 src/Storages/StorageMergeTree.cpp           |  32 +--
 src/Storages/StorageMergeTree.h             |  29 ++-
 src/Storages/StorageProxy.h                 |   3 +-
 src/Storages/StorageReplicatedMergeTree.cpp |  38 ++--
 src/Storages/StorageReplicatedMergeTree.h   |  31 ++-
 src/Storages/StorageStripeLog.cpp           |   9 +-
 src/Storages/StorageStripeLog.h             |  17 +-
 15 files changed, 373 insertions(+), 181 deletions(-)

diff --git a/src/Common/FileChecker.cpp b/src/Common/FileChecker.cpp
index dc3800eb29a..122ea83835d 100644
--- a/src/Common/FileChecker.cpp
+++ b/src/Common/FileChecker.cpp
@@ -82,41 +82,35 @@ size_t FileChecker::getTotalSize() const
 }
 
 
-CheckResults FileChecker::check() const
+FileChecker::DataValidationTasksPtr FileChecker::getDataValidationTasks()
 {
-    CheckResults results;
-    auto callback = [&results](const CheckResult & result, size_t) -> bool
-    {
-        results.push_back(result);
-        return true;
-    };
-    check(callback);
-    return results;
+    return std::make_unique<DataValidationTasks>(map);
 }
 
-void FileChecker::check(CheckDataCallback check_callback) const
+CheckResult FileChecker::checkNextEntry(DataValidationTasksPtr & check_data_tasks, bool & has_nothing_to_do) const
 {
-    if (map.empty())
-        return;
-
-    for (const auto & name_size : map)
+    String name;
+    size_t expected_size;
+    bool is_finished = check_data_tasks->next(name, expected_size);
+    if (is_finished)
     {
-        const String & name = name_size.first;
-        String path = parentPath(files_info_path) + name;
-        bool exists = fileReallyExists(path);
-        auto real_size = exists ? getRealFileSize(path) : 0;  /// No race condition assuming no one else is working with these files.
-
-        if (real_size != name_size.second)
-        {
-            String failure_message = exists
-                ? ("Size of " + path + " is wrong. Size is " + toString(real_size) + " but should be " + toString(name_size.second))
-                : ("File " + path + " doesn't exist");
-            check_callback(CheckResult(name, false, failure_message), map.size());
-            break;
-        }
-
-        check_callback(CheckResult(name, true, ""), map.size());
+        has_nothing_to_do = true;
+        return {};
     }
+
+    String path = parentPath(files_info_path) + name;
+    bool exists = fileReallyExists(path);
+    auto real_size = exists ? getRealFileSize(path) : 0;  /// No race condition assuming no one else is working with these files.
+
+    if (real_size != expected_size)
+    {
+        String failure_message = exists
+            ? ("Size of " + path + " is wrong. Size is " + toString(real_size) + " but should be " + toString(expected_size))
+            : ("File " + path + " doesn't exist");
+        return CheckResult(name, false, failure_message);
+    }
+
+    return CheckResult(name, true, "");
 }
 
 void FileChecker::repair()
diff --git a/src/Common/FileChecker.h b/src/Common/FileChecker.h
index 28dc17a4610..8ffc310b84d 100644
--- a/src/Common/FileChecker.h
+++ b/src/Common/FileChecker.h
@@ -3,6 +3,7 @@
 #include <Storages/CheckResults.h>
 #include <map>
 #include <base/types.h>
+#include <mutex>
 
 namespace Poco { class Logger; }
 
@@ -28,8 +29,11 @@ public:
     bool empty() const { return map.empty(); }
 
     /// Check the files whose parameters are specified in sizes.json
-    CheckResults check() const;
-    void check(CheckDataCallback check_callback) const;
+    /// See comment in IStorage::checkDataNext
+    struct DataValidationTasks;
+    using DataValidationTasksPtr = std::unique_ptr<DataValidationTasks>;
+    DataValidationTasksPtr getDataValidationTasks();
+    CheckResult checkNextEntry(DataValidationTasksPtr & check_data_tasks, bool & has_nothing_to_do) const;
 
     /// Truncate files that have excessive size to the expected size.
     /// Throw exception if the file size is less than expected.
@@ -42,6 +46,36 @@ public:
     /// Returns total size of all files.
     size_t getTotalSize() const;
 
+    struct DataValidationTasks
+    {
+        DataValidationTasks(const std::map<String, size_t> & map_)
+            : map(map_), it(map.begin())
+        {}
+
+        bool next(String & out_name, size_t & out_size)
+        {
+            std::lock_guard lock(mutex);
+            if (it == map.end())
+                return true;
+            out_name = it->first;
+            out_size = it->second;
+            ++it;
+            return false;
+        }
+
+        size_t size() const
+        {
+            std::lock_guard lock(mutex);
+            return std::distance(it, map.end());
+        }
+
+        const std::map<String, size_t> & map;
+
+        mutable std::mutex mutex;
+        using Iterator = std::map<String, size_t>::const_iterator;
+        Iterator it;
+    };
+
 private:
     void load();
 
diff --git a/src/Interpreters/InterpreterCheckQuery.cpp b/src/Interpreters/InterpreterCheckQuery.cpp
index 00d2749eaf9..113a5d9fc90 100644
--- a/src/Interpreters/InterpreterCheckQuery.cpp
+++ b/src/Interpreters/InterpreterCheckQuery.cpp
@@ -11,139 +11,133 @@
 #include <Interpreters/ProcessList.h>
 #include <algorithm>
 
+#include <Columns/IColumn.h>
+
+#include <Processors/ResizeProcessor.h>
+#include <Processors/IAccumulatingTransform.h>
+
 
 namespace DB
 {
 
+namespace ErrorCodes
+{
+    extern const int LOGICAL_ERROR;
+}
+
 namespace
 {
 
-Block getBlockFromCheckResult(const CheckResults & check_results, bool check_query_single_value_result)
+Block getSingleValueBlock(UInt8 value)
 {
-    if (check_query_single_value_result)
-    {
-        bool result = std::all_of(check_results.begin(), check_results.end(), [] (const CheckResult & res) { return res.success; });
-        return Block{{ColumnUInt8::create(1, static_cast<UInt8>(result)), std::make_shared<DataTypeUInt8>(), "result"}};
-    }
+    return Block{{ColumnUInt8::create(1, value), std::make_shared<DataTypeUInt8>(), "result"}};
+}
 
-    NamesAndTypes block_structure = NamesAndTypes{
+Block getHeaderForCheckResult()
+{
+    auto names_and_types = NamesAndTypes{
         {"part_path", std::make_shared<DataTypeString>()},
         {"is_passed", std::make_shared<DataTypeUInt8>()},
         {"message", std::make_shared<DataTypeString>()},
     };
-    auto path_column = block_structure[0].type->createColumn();
-    auto is_passed_column = block_structure[1].type->createColumn();
-    auto message_column = block_structure[2].type->createColumn();
-
-    for (const auto & check_result : check_results)
-    {
-        path_column->insert(check_result.fs_path);
-        is_passed_column->insert(static_cast<UInt8>(check_result.success));
-        message_column->insert(check_result.failure_message);
-    }
 
     return Block({
-        {std::move(path_column), block_structure[0].type, block_structure[0].name},
-        {std::move(is_passed_column), block_structure[1].type, block_structure[1].name},
-        {std::move(message_column), block_structure[2].type, block_structure[2].name},
+        {names_and_types[0].type->createColumn(), names_and_types[0].type, names_and_types[0].name},
+        {names_and_types[1].type->createColumn(), names_and_types[1].type, names_and_types[1].name},
+        {names_and_types[2].type->createColumn(), names_and_types[2].type, names_and_types[2].name},
     });
 }
 
-class TableCheckResultSource : public ISource
+Chunk getChunkFromCheckResult(const CheckResult & check_result)
+{
+    MutableColumns columns = getHeaderForCheckResult().cloneEmptyColumns();
+    columns[0]->insert(check_result.fs_path);
+    columns[1]->insert(static_cast<UInt8>(check_result.success));
+    columns[2]->insert(check_result.failure_message);
+    return Chunk(std::move(columns), 1);
+}
+
+class TableCheckWorkerProcessor : public ISource
 {
 
 public:
-    explicit TableCheckResultSource(const ASTPtr & query_ptr_, StoragePtr table_, bool check_query_single_value_result_, ContextPtr context_)
-        : ISource(getBlockFromCheckResult({}, check_query_single_value_result_).cloneEmpty())
-        , query_ptr(query_ptr_)
+    TableCheckWorkerProcessor(IStorage::DataValidationTasksPtr check_data_tasks_, StoragePtr table_)
+        : ISource(getHeaderForCheckResult())
         , table(table_)
-        , context(context_)
-        , check_query_single_value_result(check_query_single_value_result_)
+        , check_data_tasks(check_data_tasks_)
     {
-        worker_result = std::async(std::launch::async, [this]{ worker(); });
     }
 
-    String getName() const override { return "TableCheckResultSource"; }
+    String getName() const override { return "TableCheckWorkerProcessor"; }
 
 protected:
 
     std::optional<Chunk> tryGenerate() override
     {
-
-        if (is_check_completed)
+        bool has_nothing_to_do = false;
+        auto check_result = table->checkDataNext(check_data_tasks, has_nothing_to_do);
+        if (has_nothing_to_do)
             return {};
 
-        auto status = worker_result.wait_for(std::chrono::milliseconds(100));
-        is_check_completed = (status == std::future_status::ready);
+        /// We can omit manual `progess` call, ISource will may count it automatically by returned chunk
+        /// However, we want to report only rows in progress
+        progress(1, 0);
 
-        if (is_check_completed)
+        if (!check_result.success)
         {
-            worker_result.get();
-            auto result_block = getBlockFromCheckResult(check_results, check_query_single_value_result);
-            check_results.clear();
-            return Chunk(result_block.getColumns(), result_block.rows());
+            LOG_WARNING(&Poco::Logger::get("InterpreterCheckQuery"),
+                "Check query for table {} failed, path {}, reason: {}",
+                table->getStorageID().getNameForLogs(),
+                check_result.fs_path,
+                check_result.failure_message);
         }
 
-        std::lock_guard lock(mutex);
-        progress(progress_rows, 0);
-        progress_rows = 0;
-
-        if (check_query_single_value_result || check_results.empty())
-        {
-            return Chunk();
-        }
-
-        auto result_block = getBlockFromCheckResult(check_results, check_query_single_value_result);
-        check_results.clear();
-        return Chunk(result_block.getColumns(), result_block.rows());
+        return getChunkFromCheckResult(check_result);
     }
 
 private:
-    void worker()
+    StoragePtr table;
+    IStorage::DataValidationTasksPtr check_data_tasks;
+};
+
+class TableCheckResultEmitter : public IAccumulatingTransform
+{
+public:
+    TableCheckResultEmitter() : IAccumulatingTransform(getHeaderForCheckResult(), getSingleValueBlock(1).cloneEmpty()) {}
+
+    String getName() const override { return "TableCheckResultEmitter"; }
+
+    void consume(Chunk chunk) override
     {
-        table->checkData(query_ptr, context,
-            [this](const CheckResult & check_result, size_t new_total_rows)
+        if (result_value == 0)
+            return;
+
+        auto columns = chunk.getColumns();
+        if (columns.size() != 3)
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Wrong number of columns: {}", columns.size());
+
+        const auto * col = checkAndGetColumn<ColumnUInt8>(columns[1].get());
+        for (size_t i = 0; i < col->size(); ++i)
+        {
+            if (col->getElement(i) == 0)
             {
-                if (isCancelled())
-                    return false;
-
-                std::lock_guard lock(mutex);
-                if (new_total_rows > total_rows)
-                {
-                    addTotalRowsApprox(new_total_rows - total_rows);
-                    total_rows = new_total_rows;
-                }
-                progress_rows++;
-
-                if (!check_result.success)
-                {
-                    LOG_WARNING(&Poco::Logger::get("InterpreterCheckQuery"),
-                        "Check query for table {} failed, path {}, reason: {}",
-                        table->getStorageID().getNameForLogs(),
-                        check_result.fs_path,
-                        check_result.failure_message);
-                }
-
-                check_results.push_back(check_result);
-
-                bool should_continue = check_result.success || !check_query_single_value_result;
-                return should_continue;
-            });
+                result_value = 0;
+                return;
+            }
+        }
     }
 
-    ASTPtr query_ptr;
-    StoragePtr table;
-    ContextPtr context;
-    bool check_query_single_value_result;
+    Chunk generate() override
+    {
+        if (is_valuer_emitted.exchange(true))
+            return {};
+        auto block = getSingleValueBlock(result_value);
+        return Chunk(block.getColumns(), block.rows());
+    }
 
-    std::future<void> worker_result;
-
-    std::mutex mutex;
-    CheckResults check_results;
-    size_t progress_rows = 0;
-    size_t total_rows = 0;
-
-    bool is_check_completed = false;
+private:
+    std::atomic<UInt8> result_value{1};
+    std::atomic_bool is_valuer_emitted{false};
 };
 
 }
@@ -154,7 +148,6 @@ InterpreterCheckQuery::InterpreterCheckQuery(const ASTPtr & query_ptr_, ContextP
 {
 }
 
-
 BlockIO InterpreterCheckQuery::execute()
 {
     const auto & check = query_ptr->as<ASTCheckQuery &>();
@@ -164,11 +157,51 @@ BlockIO InterpreterCheckQuery::execute()
     context->checkAccess(AccessType::SHOW_TABLES, table_id);
     StoragePtr table = DatabaseCatalog::instance().getTable(table_id, context);
 
+    auto check_data_tasks = table->getCheckTaskList(query_ptr, context);
+
+    const auto & settings = context->getSettingsRef();
+
     BlockIO res;
     {
-        bool check_query_single_value_result = context->getSettingsRef().check_query_single_value_result;
-        auto result_source = std::make_shared<TableCheckResultSource>(query_ptr, table, check_query_single_value_result, context);
-        res.pipeline = QueryPipeline(result_source);
+        auto processors = std::make_shared<Processors>();
+
+        std::vector<OutputPort *> worker_ports;
+
+        size_t num_streams = std::max<size_t>(1, settings.max_threads);
+
+        for (size_t i = 0; i < num_streams; ++i)
+        {
+            auto worker_processor = std::make_shared<TableCheckWorkerProcessor>(check_data_tasks, table);
+            if (i == 0)
+                worker_processor->addTotalRowsApprox(check_data_tasks->size());
+            worker_ports.emplace_back(&worker_processor->getPort());
+            processors->emplace_back(worker_processor);
+        }
+
+        OutputPort * resize_outport;
+        {
+            auto resize_processor = std::make_shared<ResizeProcessor>(getHeaderForCheckResult(), worker_ports.size(), 1);
+
+            auto & resize_inputs = resize_processor->getInputs();
+            auto resize_inport_it = resize_inputs.begin();
+            for (size_t i = 0; i < worker_ports.size(); ++i, ++resize_inport_it)
+                connect(*worker_ports[i], *resize_inport_it);
+
+            resize_outport = &resize_processor->getOutputs().front();
+            processors->emplace_back(resize_processor);
+        }
+
+        if (settings.check_query_single_value_result)
+        {
+            auto emitter_processor = std::make_shared<TableCheckResultEmitter>();
+            auto input_port = &emitter_processor->getInputPort();
+            processors->emplace_back(emitter_processor);
+
+            connect(*resize_outport, *input_port);
+        }
+
+        res.pipeline = QueryPipeline(Pipe(std::move(processors)));
+        res.pipeline.setNumThreads(num_streams);
     }
     return res;
 }
diff --git a/src/Storages/CheckResults.h b/src/Storages/CheckResults.h
index b84262b50c9..2e4652fea29 100644
--- a/src/Storages/CheckResults.h
+++ b/src/Storages/CheckResults.h
@@ -22,11 +22,4 @@ struct CheckResult
     {}
 };
 
-/// Process single result of checkData
-/// Second argument is an estimated number of check results
-/// Return true to continue checking, false to stop
-using CheckDataCallback = std::function<bool(const CheckResult &, size_t)>;
-
-using CheckResults = std::vector<CheckResult>;
-
 }
diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp
index 467a34ca855..85299f63165 100644
--- a/src/Storages/IStorage.cpp
+++ b/src/Storages/IStorage.cpp
@@ -273,12 +273,15 @@ bool IStorage::isStaticStorage() const
     return false;
 }
 
-CheckResults IStorage::checkData(const ASTPtr & query, ContextPtr context)
+IStorage::DataValidationTasksPtr IStorage::getCheckTaskList(const ASTPtr & /* query */, ContextPtr /* context */)
 {
-    CheckResults results;
-    auto callback = [&](const CheckResult & result, size_t) { results.push_back(result); return true;};
-    checkData(query, context, callback);
-    return results;
+    throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Check query is not supported for {} storage", getName());
+}
+
+CheckResult IStorage::checkDataNext(DataValidationTasksPtr & /* check_task_list */, bool & has_nothing_to_do)
+{
+    has_nothing_to_do = true;
+    return {};
 }
 
 void IStorage::adjustCreateQueryForBackup(ASTPtr &) const
diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h
index 48ebf6b1242..b1e20c55782 100644
--- a/src/Storages/IStorage.h
+++ b/src/Storages/IStorage.h
@@ -595,10 +595,45 @@ public:
     /// Provides a hint that the storage engine may evaluate the IN-condition by using an index.
     virtual bool mayBenefitFromIndexForIn(const ASTPtr & /* left_in_operand */, ContextPtr /* query_context */, const StorageMetadataPtr & /* metadata_snapshot */) const { return false; }
 
-    /// Checks validity of the data
-    virtual CheckResults checkData(const ASTPtr & /* query */, ContextPtr /* context */);
 
-    virtual void checkData(const ASTPtr & /* query */, ContextPtr /* context */, CheckDataCallback /* callback */) { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Check query is not supported for {} storage", getName()); }
+    /** A list of tasks to check a validity of data.
+      * Each IStorage implementation may interpret this task in its own way.
+      * E.g. for some storages it to check data it need to check a list of files in filesystem, for others it can be a list of parts.
+      * Also it may hold resources (e.g. locks) required during check.
+      */
+    struct DataValidationTasksBase
+    {
+        /// Number of entries left to check.
+        /// It decreases after each call to checkDataNext().
+        virtual size_t size() const = 0;
+        virtual ~DataValidationTasksBase() = default;
+    };
+
+    using DataValidationTasksPtr = std::shared_ptr<DataValidationTasksBase>;
+
+    virtual DataValidationTasksPtr getCheckTaskList(const ASTPtr & /* query */, ContextPtr /* context */);
+
+    /** Executes one task from the list.
+      * If no tasks left, sets has_nothing_to_do to true.
+      * Note: Function `checkDataNext` is accessing `check_task_list` thread-safely,
+      *   and can be called simultaneously for the same `getCheckTaskList` result
+      *   to process different tasks in parallel.
+      * Usage:
+      *
+      * auto check_task_list = storage.getCheckTaskList(query, context);
+      * size_t total_tasks = check_task_list->size();
+      * while (true)
+      * {
+      *     size_t tasks_left = check_task_list->size();
+      *     std::cout << "Checking data: " << (total_tasks - tasks_left) << " / " << total_tasks << " tasks done." << std::endl;
+      *     bool has_nothing_to_do = false;
+      *     auto result = storage.checkDataNext(check_task_list, has_nothing_to_do);
+      *     if (has_nothing_to_do)
+      *         break;
+      *     doSomething(result);
+      * }
+      */
+    virtual CheckResult checkDataNext(DataValidationTasksPtr & check_task_list, bool & has_nothing_to_do);
 
     /// Checks that table could be dropped right now
     /// Otherwise - throws an exception with detailed information.
diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp
index bb58e05d756..6fb78692725 100644
--- a/src/Storages/StorageLog.cpp
+++ b/src/Storages/StorageLog.cpp
@@ -866,15 +866,18 @@ SinkToStoragePtr StorageLog::write(const ASTPtr & /*query*/, const StorageMetada
     return std::make_shared<LogSink>(*this, metadata_snapshot, std::move(lock));
 }
 
-void StorageLog::checkData(const ASTPtr & /* query */, ContextPtr local_context, CheckDataCallback check_callback)
+IStorage::DataValidationTasksPtr StorageLog::getCheckTaskList(const ASTPtr & /* query */, ContextPtr local_context)
 {
     ReadLock lock{rwlock, getLockTimeout(local_context)};
     if (!lock)
         throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Lock timeout exceeded");
-
-    file_checker.check(check_callback);
+    return std::make_unique<DataValidationTasks>(file_checker.getDataValidationTasks(), std::move(lock));
 }
 
+CheckResult StorageLog::checkDataNext(DataValidationTasksPtr & check_task_list, bool & has_nothing_to_do)
+{
+    return file_checker.checkNextEntry(static_cast<DataValidationTasks *>(check_task_list.get())->file_checker_tasks, has_nothing_to_do);
+}
 
 IStorage::ColumnSizeByName StorageLog::getColumnSizes() const
 {
diff --git a/src/Storages/StorageLog.h b/src/Storages/StorageLog.h
index f2539a6c48c..95f95088aa2 100644
--- a/src/Storages/StorageLog.h
+++ b/src/Storages/StorageLog.h
@@ -59,7 +59,8 @@ public:
 
     void rename(const String & new_path_to_table_data, const StorageID & new_table_id) override;
 
-    void checkData(const ASTPtr & query, ContextPtr local_context, CheckDataCallback check_callback) override;
+    DataValidationTasksPtr getCheckTaskList(const ASTPtr & query, ContextPtr context) override;
+    CheckResult checkDataNext(DataValidationTasksPtr & check_task_list, bool & has_nothing_to_do) override;
 
     void truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr, TableExclusiveLockHolder &) override;
 
@@ -142,6 +143,19 @@ private:
     std::atomic<UInt64> total_rows = 0;
     std::atomic<UInt64> total_bytes = 0;
 
+    struct DataValidationTasks : public IStorage::DataValidationTasksBase
+    {
+        DataValidationTasks(FileChecker::DataValidationTasksPtr file_checker_tasks_, ReadLock && lock_)
+            : file_checker_tasks(std::move(file_checker_tasks_)), lock(std::move(lock_))
+        {}
+
+        size_t size() const override { return file_checker_tasks->size(); }
+
+        FileChecker::DataValidationTasksPtr file_checker_tasks;
+        /// Lock to prevent table modification while checking
+        ReadLock lock;
+    };
+
     FileChecker file_checker;
 
     const size_t max_compress_block_size;
diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp
index 4754412a0de..4593ed6c84a 100644
--- a/src/Storages/StorageMergeTree.cpp
+++ b/src/Storages/StorageMergeTree.cpp
@@ -2197,7 +2197,7 @@ void StorageMergeTree::onActionLockRemove(StorageActionBlockType action_type)
         background_moves_assignee.trigger();
 }
 
-void StorageMergeTree::checkData(const ASTPtr & query, ContextPtr local_context, CheckDataCallback check_callback)
+IStorage::DataValidationTasksPtr StorageMergeTree::getCheckTaskList(const ASTPtr & query, ContextPtr local_context)
 {
     DataPartsVector data_parts;
     if (const auto & check_query = query->as<ASTCheckQuery &>(); check_query.partition)
@@ -2208,7 +2208,14 @@ void StorageMergeTree::checkData(const ASTPtr & query, ContextPtr local_context,
     else
         data_parts = getVisibleDataPartsVector(local_context);
 
-    for (auto & part : data_parts)
+    return std::make_unique<DataValidationTasks>(std::move(data_parts), local_context);
+}
+
+CheckResult StorageMergeTree::checkDataNext(DataValidationTasksPtr & check_task_list, bool & has_nothing_to_do)
+{
+    auto local_context = static_cast<DataValidationTasks *>(check_task_list.get())->context;
+
+    if (auto part = static_cast<DataValidationTasks *>(check_task_list.get())->next())
     {
         /// If the checksums file is not present, calculate the checksums and write them to disk.
         static constexpr auto checksums_path = "checksums.txt";
@@ -2223,16 +2230,12 @@ void StorageMergeTree::checkData(const ASTPtr & query, ContextPtr local_context,
                 part_mutable.writeChecksums(part->checksums, local_context->getWriteSettings());
 
                 part->checkMetadata();
-                bool should_continue = check_callback(CheckResult(part->name, true, "Checksums recounted and written to disk."), data_parts.size());
-                if (!should_continue)
-                    break;
+                return CheckResult(part->name, true, "Checksums recounted and written to disk.");
             }
             catch (const Exception & ex)
             {
                 tryLogCurrentException(log, __PRETTY_FUNCTION__);
-                bool should_continue = check_callback(CheckResult(part->name, false, "Check of part finished with error: '" + ex.message() + "'"), data_parts.size());
-                if (!should_continue)
-                    break;
+                return CheckResult(part->name, false, "Check of part finished with error: '" + ex.message() + "'");
             }
         }
         else
@@ -2241,18 +2244,19 @@ void StorageMergeTree::checkData(const ASTPtr & query, ContextPtr local_context,
             {
                 checkDataPart(part, true);
                 part->checkMetadata();
-                bool should_continue = check_callback(CheckResult(part->name, true, ""), data_parts.size());
-                if (!should_continue)
-                    break;
+                return CheckResult(part->name, true, "");
             }
             catch (const Exception & ex)
             {
-                bool should_continue = check_callback(CheckResult(part->name, false, ex.message()), data_parts.size());
-                if (!should_continue)
-                    break;
+                return CheckResult(part->name, false, ex.message());
             }
         }
     }
+    else
+    {
+        has_nothing_to_do = true;
+        return {};
+    }
 }
 
 
diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h
index 6b1b1b7b415..ec4796e4941 100644
--- a/src/Storages/StorageMergeTree.h
+++ b/src/Storages/StorageMergeTree.h
@@ -108,7 +108,8 @@ public:
 
     void onActionLockRemove(StorageActionBlockType action_type) override;
 
-    void checkData(const ASTPtr & query, ContextPtr context, CheckDataCallback check_callback) override;
+    DataValidationTasksPtr getCheckTaskList(const ASTPtr & query, ContextPtr context) override;
+    CheckResult checkDataNext(DataValidationTasksPtr & check_task_list, bool & has_nothing_to_do) override;
 
     bool scheduleDataProcessingJob(BackgroundJobsAssignee & assignee) override;
 
@@ -278,6 +279,32 @@ private:
     friend class MergePlainMergeTreeTask;
     friend class MutatePlainMergeTreeTask;
 
+    struct DataValidationTasks : public IStorage::DataValidationTasksBase
+    {
+        DataValidationTasks(DataPartsVector && parts_, ContextPtr context_)
+            : parts(std::move(parts_)), it(parts.begin()), context(std::move(context_))
+        {}
+
+        DataPartPtr next()
+        {
+            std::lock_guard lock(mutex);
+            if (it == parts.end())
+                return nullptr;
+            return *(it++);
+        }
+
+        size_t size() const override
+        {
+            std::lock_guard lock(mutex);
+            return std::distance(it, parts.end());
+        }
+
+        mutable std::mutex mutex;
+        DataPartsVector parts;
+        DataPartsVector::const_iterator it;
+
+        ContextPtr context;
+    };
 
 protected:
     std::map<int64_t, MutationCommands> getAlterMutationCommandsForPart(const DataPartPtr & part) const override;
diff --git a/src/Storages/StorageProxy.h b/src/Storages/StorageProxy.h
index 5339656782e..ea908bea032 100644
--- a/src/Storages/StorageProxy.h
+++ b/src/Storages/StorageProxy.h
@@ -149,7 +149,8 @@ public:
         return getNested()->mayBenefitFromIndexForIn(left_in_operand, query_context, metadata_snapshot);
     }
 
-    void checkData(const ASTPtr & query, ContextPtr context, CheckDataCallback check_callback) override { getNested()->checkData(query, context, check_callback); }
+    DataValidationTasksPtr getCheckTaskList(const ASTPtr & query, ContextPtr context) override { return getNested()->getCheckTaskList(query, context); }
+    CheckResult checkDataNext(DataValidationTasksPtr & check_task_list, bool & has_nothing_to_do) override { return getNested()->checkDataNext(check_task_list, has_nothing_to_do); }
 
     void checkTableCanBeDropped() const override { getNested()->checkTableCanBeDropped(); }
     bool storesDataOnDisk() const override { return getNested()->storesDataOnDisk(); }
diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index 71fbd009d70..b04d824e323 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -8481,7 +8481,7 @@ void StorageReplicatedMergeTree::enqueuePartForCheck(const String & part_name, t
     part_check_thread.enqueuePart(part_name, delay_to_check_seconds);
 }
 
-void StorageReplicatedMergeTree::checkData(const ASTPtr & query, ContextPtr local_context, CheckDataCallback check_callback)
+IStorage::DataValidationTasksPtr StorageReplicatedMergeTree::getCheckTaskList(const ASTPtr & query, ContextPtr local_context)
 {
     DataPartsVector data_parts;
     if (const auto & check_query = query->as<ASTCheckQuery &>(); check_query.partition)
@@ -8492,25 +8492,29 @@ void StorageReplicatedMergeTree::checkData(const ASTPtr & query, ContextPtr loca
     else
         data_parts = getVisibleDataPartsVector(local_context);
 
-    {
-        auto part_check_lock = part_check_thread.pausePartsCheck();
+    auto part_check_lock = part_check_thread.pausePartsCheck();
+    return std::make_unique<DataValidationTasks>(std::move(data_parts), std::move(part_check_lock));
+}
 
-        for (auto & part : data_parts)
+CheckResult StorageReplicatedMergeTree::checkDataNext(DataValidationTasksPtr & check_task_list, bool & has_nothing_to_do)
+{
+
+    if (auto part = static_cast<DataValidationTasks *>(check_task_list.get())->next())
+    {
+        try
         {
-            try
-            {
-                bool should_continue = check_callback(part_check_thread.checkPartAndFix(part->name), data_parts.size());
-                if (!should_continue)
-                    break;
-            }
-            catch (const Exception & ex)
-            {
-                tryLogCurrentException(log, __PRETTY_FUNCTION__);
-                bool should_continue = check_callback(CheckResult(part->name, false, "Check of part finished with error: '" + ex.message() + "'"), data_parts.size());
-                if (!should_continue)
-                    break;
-            }
+            return CheckResult(part_check_thread.checkPartAndFix(part->name));
         }
+        catch (const Exception & ex)
+        {
+            tryLogCurrentException(log, __PRETTY_FUNCTION__);
+            return CheckResult(part->name, false, "Check of part finished with error: '" + ex.message() + "'");
+        }
+    }
+    else
+    {
+        has_nothing_to_do = true;
+        return {};
     }
 }
 
diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h
index 6f9853122cb..2bc18aa3b0a 100644
--- a/src/Storages/StorageReplicatedMergeTree.h
+++ b/src/Storages/StorageReplicatedMergeTree.h
@@ -230,7 +230,8 @@ public:
     /// Add a part to the queue of parts whose data you want to check in the background thread.
     void enqueuePartForCheck(const String & part_name, time_t delay_to_check_seconds = 0);
 
-    void checkData(const ASTPtr & query, ContextPtr context, CheckDataCallback check_callback) override;
+    DataValidationTasksPtr getCheckTaskList(const ASTPtr & query, ContextPtr context) override;
+    CheckResult checkDataNext(DataValidationTasksPtr & check_task_list, bool & has_nothing_to_do) override;
 
     /// Checks ability to use granularity
     bool canUseAdaptiveGranularity() const override;
@@ -990,6 +991,34 @@ private:
     bool waitZeroCopyLockToDisappear(const ZeroCopyLock & lock, size_t milliseconds_to_wait) override;
 
     void startupImpl(bool from_attach_thread);
+
+    struct DataValidationTasks : public IStorage::DataValidationTasksBase
+    {
+        explicit DataValidationTasks(DataPartsVector && parts_, std::unique_lock<std::mutex> && parts_check_lock_)
+            : parts_check_lock(std::move(parts_check_lock_)), parts(std::move(parts_)), it(parts.begin())
+        {}
+
+        DataPartPtr next()
+        {
+            std::lock_guard lock(mutex);
+            if (it == parts.end())
+                return nullptr;
+            return *(it++);
+        }
+
+        size_t size() const override
+        {
+            std::lock_guard lock(mutex);
+            return std::distance(it, parts.end());
+        }
+
+        std::unique_lock<std::mutex> parts_check_lock;
+
+        mutable std::mutex mutex;
+        DataPartsVector parts;
+        DataPartsVector::const_iterator it;
+    };
+
 };
 
 String getPartNamePossiblyFake(MergeTreeDataFormatVersion format_version, const MergeTreePartInfo & part_info);
diff --git a/src/Storages/StorageStripeLog.cpp b/src/Storages/StorageStripeLog.cpp
index 5a3a007035c..c242830e657 100644
--- a/src/Storages/StorageStripeLog.cpp
+++ b/src/Storages/StorageStripeLog.cpp
@@ -403,15 +403,18 @@ SinkToStoragePtr StorageStripeLog::write(const ASTPtr & /*query*/, const Storage
     return std::make_shared<StripeLogSink>(*this, metadata_snapshot, std::move(lock));
 }
 
-void StorageStripeLog::checkData(const ASTPtr & /* query */, ContextPtr local_context, CheckDataCallback check_callback)
+IStorage::DataValidationTasksPtr StorageStripeLog::getCheckTaskList(const ASTPtr & /* query */, ContextPtr local_context)
 {
     ReadLock lock{rwlock, getLockTimeout(local_context)};
     if (!lock)
         throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Lock timeout exceeded");
-
-    file_checker.check(check_callback);
+    return std::make_unique<DataValidationTasks>(file_checker.getDataValidationTasks(), std::move(lock));
 }
 
+CheckResult StorageStripeLog::checkDataNext(DataValidationTasksPtr & check_task_list, bool & has_nothing_to_do)
+{
+    return file_checker.checkNextEntry(static_cast<DataValidationTasks *>(check_task_list.get())->file_checker_tasks, has_nothing_to_do);
+}
 
 void StorageStripeLog::truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr, TableExclusiveLockHolder &)
 {
diff --git a/src/Storages/StorageStripeLog.h b/src/Storages/StorageStripeLog.h
index 61ce49f43da..5d4e2fcbd3a 100644
--- a/src/Storages/StorageStripeLog.h
+++ b/src/Storages/StorageStripeLog.h
@@ -53,7 +53,8 @@ public:
 
     void rename(const String & new_path_to_table_data, const StorageID & new_table_id) override;
 
-    void checkData(const ASTPtr & query, ContextPtr ocal_context, CheckDataCallback check_callback) override;
+    DataValidationTasksPtr getCheckTaskList(const ASTPtr & query, ContextPtr context) override;
+    CheckResult checkDataNext(DataValidationTasksPtr & check_task_list, bool & has_nothing_to_do) override;
 
     bool storesDataOnDisk() const override { return true; }
     Strings getDataPaths() const override { return {DB::fullPath(disk, table_path)}; }
@@ -93,6 +94,20 @@ private:
     const DiskPtr disk;
     String table_path;
 
+    struct DataValidationTasks : public IStorage::DataValidationTasksBase
+    {
+        DataValidationTasks(FileChecker::DataValidationTasksPtr file_checker_tasks_, ReadLock && lock_)
+            : file_checker_tasks(std::move(file_checker_tasks_)), lock(std::move(lock_))
+        {}
+
+        size_t size() const override { return file_checker_tasks->size(); }
+
+        FileChecker::DataValidationTasksPtr file_checker_tasks;
+
+        /// Lock to prevent table modification while checking
+        ReadLock lock;
+    };
+
     String data_file_path;
     String index_file_path;
     FileChecker file_checker;

From 11554fe9c0efae873fb112fbde718cd33217f4ed Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Tue, 1 Aug 2023 12:39:26 +0000
Subject: [PATCH 112/249] Upd tests for CHECK TABLE

---
 .../0_stateless/00063_check_query.reference   |  1 +
 .../queries/0_stateless/00063_check_query.sql |  3 ++
 .../queries/0_stateless/00961_check_table.sql | 12 ++++----
 ...1042_check_query_and_last_granule_size.sql | 12 ++++----
 ...02235_check_table_sparse_serialization.sql |  3 +-
 .../02841_check_table_progress.reference      |  2 ++
 .../0_stateless/02841_check_table_progress.sh | 29 +++++++++++++++++++
 7 files changed, 48 insertions(+), 14 deletions(-)
 create mode 100644 tests/queries/0_stateless/02841_check_table_progress.reference
 create mode 100755 tests/queries/0_stateless/02841_check_table_progress.sh

diff --git a/tests/queries/0_stateless/00063_check_query.reference b/tests/queries/0_stateless/00063_check_query.reference
index 6ed281c757a..e8183f05f5d 100644
--- a/tests/queries/0_stateless/00063_check_query.reference
+++ b/tests/queries/0_stateless/00063_check_query.reference
@@ -1,2 +1,3 @@
 1
 1
+1
diff --git a/tests/queries/0_stateless/00063_check_query.sql b/tests/queries/0_stateless/00063_check_query.sql
index e7362074a05..263cf94fb4a 100644
--- a/tests/queries/0_stateless/00063_check_query.sql
+++ b/tests/queries/0_stateless/00063_check_query.sql
@@ -8,6 +8,9 @@ INSERT INTO check_query_tiny_log VALUES (1, 'A'), (2, 'B'), (3, 'C');
 
 CHECK TABLE check_query_tiny_log;
 
+-- Settings and FORMAT are supported
+CHECK TABLE check_query_tiny_log SETTINGS max_threads = 16;
+CHECK TABLE check_query_tiny_log FORMAT Null SETTINGS max_threads = 8, check_query_single_value_result = 0;
 
 DROP TABLE IF EXISTS check_query_log;
 
diff --git a/tests/queries/0_stateless/00961_check_table.sql b/tests/queries/0_stateless/00961_check_table.sql
index 0e0b2c3b483..079acc8cdbb 100644
--- a/tests/queries/0_stateless/00961_check_table.sql
+++ b/tests/queries/0_stateless/00961_check_table.sql
@@ -3,29 +3,29 @@ DROP TABLE IF EXISTS mt_table;
 
 CREATE TABLE mt_table (d Date, key UInt64, data String) ENGINE = MergeTree() PARTITION BY toYYYYMM(d) ORDER BY key;
 
-CHECK TABLE mt_table;
+CHECK TABLE mt_table SETTINGS max_threads = 1;
 
 INSERT INTO mt_table VALUES (toDate('2019-01-02'), 1, 'Hello'), (toDate('2019-01-02'), 2, 'World');
 
-CHECK TABLE mt_table;
+CHECK TABLE mt_table SETTINGS max_threads = 1;
 
 INSERT INTO mt_table VALUES (toDate('2019-01-02'), 3, 'quick'), (toDate('2019-01-02'), 4, 'brown');
 
 SELECT '========';
 
-CHECK TABLE mt_table;
+CHECK TABLE mt_table SETTINGS max_threads = 1;
 
 OPTIMIZE TABLE mt_table FINAL;
 
 SELECT '========';
 
-CHECK TABLE mt_table;
+CHECK TABLE mt_table SETTINGS max_threads = 1;
 
 SELECT '========';
 
 INSERT INTO mt_table VALUES (toDate('2019-02-03'), 5, '!'), (toDate('2019-02-03'), 6, '?');
 
-CHECK TABLE mt_table;
+CHECK TABLE mt_table SETTINGS max_threads = 1;
 
 SELECT '========';
 
@@ -33,6 +33,6 @@ INSERT INTO mt_table VALUES (toDate('2019-02-03'), 7, 'jump'), (toDate('2019-02-
 
 OPTIMIZE TABLE mt_table FINAL;
 
-CHECK TABLE mt_table PARTITION 201902;
+CHECK TABLE mt_table PARTITION 201902 SETTINGS max_threads = 1;
 
 DROP TABLE IF EXISTS mt_table;
diff --git a/tests/queries/0_stateless/01042_check_query_and_last_granule_size.sql b/tests/queries/0_stateless/01042_check_query_and_last_granule_size.sql
index b66aff8384d..eccb2d25878 100644
--- a/tests/queries/0_stateless/01042_check_query_and_last_granule_size.sql
+++ b/tests/queries/0_stateless/01042_check_query_and_last_granule_size.sql
@@ -7,11 +7,11 @@ CREATE TABLE check_query_test (SomeKey UInt64, SomeValue String) ENGINE = MergeT
 -- Rows in this table are short, so granularity will be 8192.
 INSERT INTO check_query_test SELECT number, toString(number) FROM system.numbers LIMIT 81920;
 
-CHECK TABLE check_query_test;
+CHECK TABLE check_query_test SETTINGS max_threads = 1;
 
 OPTIMIZE TABLE check_query_test;
 
-CHECK TABLE check_query_test;
+CHECK TABLE check_query_test SETTINGS max_threads = 1;
 
 DROP TABLE IF EXISTS check_query_test;
 
@@ -21,18 +21,18 @@ CREATE TABLE check_query_test_non_adaptive (SomeKey UInt64, SomeValue String) EN
 
 INSERT INTO check_query_test_non_adaptive SELECT number, toString(number) FROM system.numbers LIMIT 81920;
 
-CHECK TABLE check_query_test_non_adaptive;
+CHECK TABLE check_query_test_non_adaptive SETTINGS max_threads = 1;
 
 OPTIMIZE TABLE check_query_test_non_adaptive;
 
-CHECK TABLE check_query_test_non_adaptive;
+CHECK TABLE check_query_test_non_adaptive SETTINGS max_threads = 1;
 
 INSERT INTO check_query_test_non_adaptive SELECT number, toString(number) FROM system.numbers LIMIT 77;
 
-CHECK TABLE check_query_test_non_adaptive;
+CHECK TABLE check_query_test_non_adaptive SETTINGS max_threads = 1;
 
 OPTIMIZE TABLE check_query_test_non_adaptive;
 
-CHECK TABLE check_query_test_non_adaptive;
+CHECK TABLE check_query_test_non_adaptive SETTINGS max_threads = 1;
 
 DROP TABLE IF EXISTS check_query_test_non_adaptive;
diff --git a/tests/queries/0_stateless/02235_check_table_sparse_serialization.sql b/tests/queries/0_stateless/02235_check_table_sparse_serialization.sql
index 0ac97404c46..625be63e0c0 100644
--- a/tests/queries/0_stateless/02235_check_table_sparse_serialization.sql
+++ b/tests/queries/0_stateless/02235_check_table_sparse_serialization.sql
@@ -12,7 +12,6 @@ SELECT name, column, serialization_kind FROM system.parts_columns
 WHERE database = currentDatabase() AND table = 't_sparse_02235'
 ORDER BY name, column;
 
-SET check_query_single_value_result = 0;
-CHECK TABLE t_sparse_02235;
+CHECK TABLE t_sparse_02235 SETTINGS check_query_single_value_result = 0, max_threads = 1;
 
 DROP TABLE t_sparse_02235;
diff --git a/tests/queries/0_stateless/02841_check_table_progress.reference b/tests/queries/0_stateless/02841_check_table_progress.reference
new file mode 100644
index 00000000000..541dab48def
--- /dev/null
+++ b/tests/queries/0_stateless/02841_check_table_progress.reference
@@ -0,0 +1,2 @@
+Ok
+Ok
diff --git a/tests/queries/0_stateless/02841_check_table_progress.sh b/tests/queries/0_stateless/02841_check_table_progress.sh
new file mode 100755
index 00000000000..166386b999b
--- /dev/null
+++ b/tests/queries/0_stateless/02841_check_table_progress.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS t0";
+${CLICKHOUSE_CLIENT} -q "CREATE TABLE t0 (x UInt64, val String) ENGINE = MergeTree ORDER BY x PARTITION BY x % 100";
+${CLICKHOUSE_CLIENT} -q "INSERT INTO t0 SELECT sipHash64(number), randomPrintableASCII(1000) FROM numbers(1000)";
+
+
+# Check that we have at least 3 different values for read_rows
+UNIQUE_VALUES=$(
+    ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0" -d @- <<< "CHECK TABLE t0" -v |& {
+        grep -F -e X-ClickHouse-Progress: -e X-ClickHouse-Summary:  | grep -o '"read_rows"\s*:\s*"[0-9]*"'
+    } | uniq | wc -l
+)
+
+[ "$UNIQUE_VALUES" -ge "3" ] && echo "Ok" || echo "Fail: got $UNIQUE_VALUES"
+
+
+# Check that we have we have at least 100 total_rows_to_read (at least one check task per partition)
+MAX_TOTAL_VALUE=$(
+    ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0" -d @- <<< "CHECK TABLE t0" -v |& {
+        grep -F -e X-ClickHouse-Progress: -e X-ClickHouse-Summary:  | grep -o '"total_rows_to_read"\s*:\s*"[0-9]*"' | grep -o '[0-9]*'
+    } | sort -n | tail -1
+)
+
+[ "$MAX_TOTAL_VALUE" -ge "100" ] && echo "Ok" || echo "Fail: got $MAX_TOTAL_VALUE"

From aeee50466cb237c0458c552a8e15d957dce013bc Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Tue, 1 Aug 2023 12:39:49 +0000
Subject: [PATCH 113/249] Upd doc for CHECK TABLE

---
 .../sql-reference/statements/check-table.md   | 105 +++++++++++++-----
 1 file changed, 79 insertions(+), 26 deletions(-)

diff --git a/docs/en/sql-reference/statements/check-table.md b/docs/en/sql-reference/statements/check-table.md
index 0209d59b018..588be841755 100644
--- a/docs/en/sql-reference/statements/check-table.md
+++ b/docs/en/sql-reference/statements/check-table.md
@@ -5,19 +5,38 @@ sidebar_label: CHECK TABLE
 title: "CHECK TABLE Statement"
 ---
 
-Checks if the data in the table is corrupted.
+The `CHECK TABLE` query in ClickHouse is used to perform a validation check on a specific table or its partitions. It ensures the integrity of the data by verifying the checksums and other internal data structures.
 
-``` sql
-CHECK TABLE [db.]name [PARTITION partition_expr]
+Particularly it compares actual file sizes with the expected values which are stored on the server. If the file sizes do not match the stored values, it means the data is corrupted. This can be caused, for example, by a system crash during query execution.
+
+:::note
+The `CHECK TABLE`` query may read all the data in the table and hold some resources, making it resource-intensive.
+Consider the potential impact on performance and resource utilization before executing this query.
+:::
+
+## Syntax
+
+The basic syntax of the query is as follows:
+
+```sql
+CHECK TABLE table_name [PARTITION partition_expression] [FORMAT format] [SETTINGS check_query_single_value_result = (0|1) [, other_settings]  ]
 ```
 
-The `CHECK TABLE` query compares actual file sizes with the expected values which are stored on the server. If the file sizes do not match the stored values, it means the data is corrupted. This can be caused, for example, by a system crash during query execution.
+- `table_name`: Specifies the name of the table that you want to check.
+- `partition_expression`: (Optional) If you want to check a specific partition of the table, you can use this expression to specify the partition.
+- `FORMAT format`: (Optional) Allows you to specify the output format of the result.
+- `SETTINGS`: (Optional) Allows additional settings.
+	- **`check_query_single_value_result`**: (Optional) This setting allows you to toggle between a detailed result (`0`) or a summarized result (`1`).
+	- Other settings (e.g. `max_threads` can be applied as well).
 
-The query response contains the `result` column with a single row. The row has a value of
-[Boolean](../../sql-reference/data-types/boolean.md) type:
 
-- 0 - The data in the table is corrupted.
-- 1 - The data maintains integrity.
+The query response depends on the value of contains `check_query_single_value_result` setting.
+In case of `check_query_single_value_result = 1` only `result` column with a single row is returned. Value inside this row is `1` if the integrity check is passed and `0` if data is corrupted.
+
+With `check_query_single_value_result = 0` the query returns the following columns:
+    - `part_path`: Indicates the path to the data part or file name.
+    - `is_passed`: Returns 1 if the check for this part is successful, 0 otherwise.
+    - `message`: Any additional messages related to the check, such as errors or success messages.
 
 The `CHECK TABLE` query supports the following table engines:
 
@@ -26,30 +45,15 @@ The `CHECK TABLE` query supports the following table engines:
 - [StripeLog](../../engines/table-engines/log-family/stripelog.md)
 - [MergeTree family](../../engines/table-engines/mergetree-family/mergetree.md)
 
-Performed over the tables with another table engines causes an exception.
+Performed over the tables with another table engines causes an `NOT_IMPLEMETED` exception.
 
 Engines from the `*Log` family do not provide automatic data recovery on failure. Use the `CHECK TABLE` query to track data loss in a timely manner.
 
-## Checking the MergeTree Family Tables
+## Examples
 
-For `MergeTree` family engines, if [check_query_single_value_result](../../operations/settings/settings.md#check_query_single_value_result) = 0, the `CHECK TABLE` query shows a check status for every individual data part of a table on the local server.
+By default `CHECK TABLE` query shows the general table check status:
 
 ```sql
-SET check_query_single_value_result = 0;
-CHECK TABLE test_table;
-```
-
-```text
-┌─part_path─┬─is_passed─┬─message─┐
-│ all_1_4_1 │         1 │         │
-│ all_1_4_2 │         1 │         │
-└───────────┴───────────┴─────────┘
-```
-
-If `check_query_single_value_result` = 1, the `CHECK TABLE` query shows the general table check status.
-
-```sql
-SET check_query_single_value_result = 1;
 CHECK TABLE test_table;
 ```
 
@@ -59,6 +63,55 @@ CHECK TABLE test_table;
 └────────┘
 ```
 
+If you want to see the check status for every individual data part you may use `check_query_single_value_result` setting.
+
+Also, to check a specific partition of the table, you can use the `PARTITION` keyword.
+
+```sql
+CHECK TABLE t0 PARTITION ID '201003'
+FORMAT PrettyCompactMonoBlock
+SETTINGS check_query_single_value_result = 0
+```
+
+Output:
+
+```text
+┌─part_path────┬─is_passed─┬─message─┐
+│ 201003_7_7_0 │         1 │         │
+│ 201003_3_3_0 │         1 │         │
+└──────────────┴───────────┴─────────┘
+```
+
+### Receiving a 'Corrupted' Result
+
+:::warning
+Disclaimer: The procedure described here, including the manual manipulating or removing files directly from the data directory, is for experimental or development environments only. Do **not** attempt this on a production server, as it may lead to data loss or other unintended consequences.
+:::
+
+Remove the existing checksum file:
+
+```bash
+rm /var/lib/clickhouse-server/data/default/t0/201003_3_3_0/checksums.txt
+```
+
+```sql
+CHECK TABLE t0 PARTITION ID '201003'
+FORMAT PrettyCompactMonoBlock
+SETTINGS check_query_single_value_result = 0
+
+
+Output:
+
+```text
+┌─part_path────┬─is_passed─┬─message──────────────────────────────────┐
+│ 201003_7_7_0 │         1 │                                          │
+│ 201003_3_3_0 │         1 │ Checksums recounted and written to disk. │
+└──────────────┴───────────┴──────────────────────────────────────────┘
+```
+
+If the checksums.txt file is missing, it can be restored. It will be recalculated and rewritten during the execution of the CHECK TABLE command for the specific partition, and the status will still be reported as 'success.'"
+
+
 ## If the Data Is Corrupted
 
 If the table is corrupted, you can copy the non-corrupted data to another table. To do this:

From 358ef26385fd3b56233babae4b9ab335c16cf6dd Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Fri, 4 Aug 2023 09:22:00 +0000
Subject: [PATCH 114/249] small fixes for CHECK TABLE

---
 src/Interpreters/InterpreterCheckQuery.cpp  | 6 +++---
 src/Storages/StorageLog.cpp                 | 2 +-
 src/Storages/StorageMergeTree.cpp           | 6 +++---
 src/Storages/StorageReplicatedMergeTree.cpp | 2 +-
 src/Storages/StorageStripeLog.cpp           | 2 +-
 5 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/Interpreters/InterpreterCheckQuery.cpp b/src/Interpreters/InterpreterCheckQuery.cpp
index 113a5d9fc90..bd530654dd2 100644
--- a/src/Interpreters/InterpreterCheckQuery.cpp
+++ b/src/Interpreters/InterpreterCheckQuery.cpp
@@ -129,7 +129,7 @@ public:
 
     Chunk generate() override
     {
-        if (is_valuer_emitted.exchange(true))
+        if (is_value_emitted.exchange(true))
             return {};
         auto block = getSingleValueBlock(result_value);
         return Chunk(block.getColumns(), block.rows());
@@ -137,7 +137,7 @@ public:
 
 private:
     std::atomic<UInt8> result_value{1};
-    std::atomic_bool is_valuer_emitted{false};
+    std::atomic_bool is_value_emitted{false};
 };
 
 }
@@ -194,7 +194,7 @@ BlockIO InterpreterCheckQuery::execute()
         if (settings.check_query_single_value_result)
         {
             auto emitter_processor = std::make_shared<TableCheckResultEmitter>();
-            auto input_port = &emitter_processor->getInputPort();
+            auto * input_port = &emitter_processor->getInputPort();
             processors->emplace_back(emitter_processor);
 
             connect(*resize_outport, *input_port);
diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp
index 6fb78692725..0e9f83e886a 100644
--- a/src/Storages/StorageLog.cpp
+++ b/src/Storages/StorageLog.cpp
@@ -876,7 +876,7 @@ IStorage::DataValidationTasksPtr StorageLog::getCheckTaskList(const ASTPtr & /*
 
 CheckResult StorageLog::checkDataNext(DataValidationTasksPtr & check_task_list, bool & has_nothing_to_do)
 {
-    return file_checker.checkNextEntry(static_cast<DataValidationTasks *>(check_task_list.get())->file_checker_tasks, has_nothing_to_do);
+    return file_checker.checkNextEntry(assert_cast<DataValidationTasks *>(check_task_list.get())->file_checker_tasks, has_nothing_to_do);
 }
 
 IStorage::ColumnSizeByName StorageLog::getColumnSizes() const
diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp
index 4593ed6c84a..97fc7a6731f 100644
--- a/src/Storages/StorageMergeTree.cpp
+++ b/src/Storages/StorageMergeTree.cpp
@@ -2213,9 +2213,9 @@ IStorage::DataValidationTasksPtr StorageMergeTree::getCheckTaskList(const ASTPtr
 
 CheckResult StorageMergeTree::checkDataNext(DataValidationTasksPtr & check_task_list, bool & has_nothing_to_do)
 {
-    auto local_context = static_cast<DataValidationTasks *>(check_task_list.get())->context;
-
-    if (auto part = static_cast<DataValidationTasks *>(check_task_list.get())->next())
+    auto * data_validation_tasks = assert_cast<DataValidationTasks *>(check_task_list.get());
+    auto local_context = data_validation_tasks->context;
+    if (auto part = data_validation_tasks->next())
     {
         /// If the checksums file is not present, calculate the checksums and write them to disk.
         static constexpr auto checksums_path = "checksums.txt";
diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index b04d824e323..c08f1ebcc48 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -8499,7 +8499,7 @@ IStorage::DataValidationTasksPtr StorageReplicatedMergeTree::getCheckTaskList(co
 CheckResult StorageReplicatedMergeTree::checkDataNext(DataValidationTasksPtr & check_task_list, bool & has_nothing_to_do)
 {
 
-    if (auto part = static_cast<DataValidationTasks *>(check_task_list.get())->next())
+    if (auto part = assert_cast<DataValidationTasks *>(check_task_list.get())->next())
     {
         try
         {
diff --git a/src/Storages/StorageStripeLog.cpp b/src/Storages/StorageStripeLog.cpp
index c242830e657..a3cbff96199 100644
--- a/src/Storages/StorageStripeLog.cpp
+++ b/src/Storages/StorageStripeLog.cpp
@@ -413,7 +413,7 @@ IStorage::DataValidationTasksPtr StorageStripeLog::getCheckTaskList(const ASTPtr
 
 CheckResult StorageStripeLog::checkDataNext(DataValidationTasksPtr & check_task_list, bool & has_nothing_to_do)
 {
-    return file_checker.checkNextEntry(static_cast<DataValidationTasks *>(check_task_list.get())->file_checker_tasks, has_nothing_to_do);
+    return file_checker.checkNextEntry(assert_cast<DataValidationTasks *>(check_task_list.get())->file_checker_tasks, has_nothing_to_do);
 }
 
 void StorageStripeLog::truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr, TableExclusiveLockHolder &)

From 50715ca4fc2682e810ec7cc3d74212fec8098e49 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Mon, 7 Aug 2023 11:57:11 +0000
Subject: [PATCH 115/249] fix doc links

---
 docs/en/sql-reference/statements/check-table.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/sql-reference/statements/check-table.md b/docs/en/sql-reference/statements/check-table.md
index 588be841755..45fc17b044a 100644
--- a/docs/en/sql-reference/statements/check-table.md
+++ b/docs/en/sql-reference/statements/check-table.md
@@ -117,6 +117,6 @@ If the checksums.txt file is missing, it can be restored. It will be recalculate
 If the table is corrupted, you can copy the non-corrupted data to another table. To do this:
 
 1.  Create a new table with the same structure as damaged table. To do this execute the query `CREATE TABLE <new_table_name> AS <damaged_table_name>`.
-2.  Set the [max_threads](../../operations/settings/settings.md#settings-max_threads) value to 1 to process the next query in a single thread. To do this run the query `SET max_threads = 1`.
+2.  Set the [max_threads](/docs/en/operations/settings/settings.md/#settings-max_threads) value to 1 to process the next query in a single thread. To do this run the query `SET max_threads = 1`.
 3.  Execute the query `INSERT INTO <new_table_name> SELECT * FROM <damaged_table_name>`. This request copies the non-corrupted data from the damaged table to another table. Only the data before the corrupted part will be copied.
 4.  Restart the `clickhouse-client` to reset the `max_threads` value.

From 1183dac293d5d474b8a9bbad34819d1ac14def44 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Tue, 8 Aug 2023 09:45:23 +0000
Subject: [PATCH 116/249] fix doc links

---
 docs/en/sql-reference/statements/check-table.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/sql-reference/statements/check-table.md b/docs/en/sql-reference/statements/check-table.md
index 45fc17b044a..db8c32249ef 100644
--- a/docs/en/sql-reference/statements/check-table.md
+++ b/docs/en/sql-reference/statements/check-table.md
@@ -117,6 +117,6 @@ If the checksums.txt file is missing, it can be restored. It will be recalculate
 If the table is corrupted, you can copy the non-corrupted data to another table. To do this:
 
 1.  Create a new table with the same structure as damaged table. To do this execute the query `CREATE TABLE <new_table_name> AS <damaged_table_name>`.
-2.  Set the [max_threads](/docs/en/operations/settings/settings.md/#settings-max_threads) value to 1 to process the next query in a single thread. To do this run the query `SET max_threads = 1`.
+2.  Set the `max_threads` value to 1 to process the next query in a single thread. To do this run the query `SET max_threads = 1`.
 3.  Execute the query `INSERT INTO <new_table_name> SELECT * FROM <damaged_table_name>`. This request copies the non-corrupted data from the damaged table to another table. Only the data before the corrupted part will be copied.
 4.  Restart the `clickhouse-client` to reset the `max_threads` value.

From 52c7a29309bd5a739ea2611c30f9ae38f1c59eb7 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Tue, 8 Aug 2023 11:02:25 +0000
Subject: [PATCH 117/249] use unique table names in test_quorum_inserts

---
 tests/integration/test_quorum_inserts/test.py | 121 ++++++++----------
 1 file changed, 51 insertions(+), 70 deletions(-)

diff --git a/tests/integration/test_quorum_inserts/test.py b/tests/integration/test_quorum_inserts/test.py
index 4dbd530dd17..581a02c154e 100644
--- a/tests/integration/test_quorum_inserts/test.py
+++ b/tests/integration/test_quorum_inserts/test.py
@@ -147,12 +147,16 @@ def test_drop_replica_and_achieve_quorum(started_cluster):
 
 @pytest.mark.parametrize(("add_new_data"), [False, True])
 def test_insert_quorum_with_drop_partition(started_cluster, add_new_data):
-    zero.query(
-        "DROP TABLE IF EXISTS test_quorum_insert_with_drop_partition ON CLUSTER cluster"
+    # use different table names for easier disambiguation in logs between runs (you may also check uuid though, but not always convenient)
+    table_name = (
+        "test_quorum_insert_with_drop_partition_new_data"
+        if add_new_data
+        else "test_quorum_insert_with_drop_partition"
     )
+    zero.query(f"DROP TABLE IF EXISTS {table_name} ON CLUSTER cluster")
 
     create_query = (
-        "CREATE TABLE test_quorum_insert_with_drop_partition ON CLUSTER cluster "
+        f"CREATE TABLE {table_name} ON CLUSTER cluster "
         "(a Int8, d Date) "
         "Engine = ReplicatedMergeTree "
         "PARTITION BY d ORDER BY a "
@@ -161,34 +165,28 @@ def test_insert_quorum_with_drop_partition(started_cluster, add_new_data):
     print("Create Replicated table with three replicas")
     zero.query(create_query)
 
-    print("Stop fetches for test_quorum_insert_with_drop_partition at first replica.")
-    first.query("SYSTEM STOP FETCHES test_quorum_insert_with_drop_partition")
+    print(f"Stop fetches for {table_name} at first replica.")
+    first.query(f"SYSTEM STOP FETCHES {table_name}")
 
     print("Insert with quorum. (zero and second)")
-    zero.query(
-        "INSERT INTO test_quorum_insert_with_drop_partition(a,d) VALUES(1, '2011-01-01')"
-    )
+    zero.query(f"INSERT INTO {table_name}(a,d) VALUES(1, '2011-01-01')")
 
     print("Drop partition.")
-    zero.query(
-        "ALTER TABLE test_quorum_insert_with_drop_partition DROP PARTITION '2011-01-01'"
-    )
+    zero.query(f"ALTER TABLE {table_name} DROP PARTITION '2011-01-01'")
 
     if add_new_data:
         print("Insert to deleted partition")
-        zero.query(
-            "INSERT INTO test_quorum_insert_with_drop_partition(a,d) VALUES(2, '2011-01-01')"
-        )
+        zero.query(f"INSERT INTO {table_name}(a,d) VALUES(2, '2011-01-01')")
 
-    print("Resume fetches for test_quorum_insert_with_drop_partition at first replica.")
-    first.query("SYSTEM START FETCHES test_quorum_insert_with_drop_partition")
+    print(f"Resume fetches for {table_name} at first replica.")
+    first.query(f"SYSTEM START FETCHES {table_name}")
 
     print("Sync first replica with others.")
-    first.query("SYSTEM SYNC REPLICA test_quorum_insert_with_drop_partition")
+    first.query(f"SYSTEM SYNC REPLICA {table_name}")
 
     assert "20110101" not in first.query(
-        """
-    WITH (SELECT toString(uuid) FROM system.tables WHERE name = 'test_quorum_insert_with_drop_partition') AS uuid,
+        f"""
+    WITH (SELECT toString(uuid) FROM system.tables WHERE name = '{table_name}') AS uuid,
          '/clickhouse/tables/' || uuid || '/0/quorum/last_part' AS p
     SELECT * FROM system.zookeeper WHERE path = p FORMAT Vertical
     """
@@ -196,43 +194,42 @@ def test_insert_quorum_with_drop_partition(started_cluster, add_new_data):
 
     print("Select from updated partition.")
     if add_new_data:
+        assert TSV("2\t2011-01-01\n") == TSV(zero.query(f"SELECT * FROM {table_name}"))
         assert TSV("2\t2011-01-01\n") == TSV(
-            zero.query("SELECT * FROM test_quorum_insert_with_drop_partition")
-        )
-        assert TSV("2\t2011-01-01\n") == TSV(
-            second.query("SELECT * FROM test_quorum_insert_with_drop_partition")
+            second.query(f"SELECT * FROM {table_name}")
         )
     else:
-        assert TSV("") == TSV(
-            zero.query("SELECT * FROM test_quorum_insert_with_drop_partition")
-        )
-        assert TSV("") == TSV(
-            second.query("SELECT * FROM test_quorum_insert_with_drop_partition")
-        )
+        assert TSV("") == TSV(zero.query(f"SELECT * FROM {table_name}"))
+        assert TSV("") == TSV(second.query(f"SELECT * FROM {table_name}"))
 
-    zero.query(
-        "DROP TABLE IF EXISTS test_quorum_insert_with_drop_partition ON CLUSTER cluster"
-    )
+    zero.query(f"DROP TABLE IF EXISTS {table_name} ON CLUSTER cluster")
 
 
 @pytest.mark.parametrize(("add_new_data"), [False, True])
 def test_insert_quorum_with_move_partition(started_cluster, add_new_data):
-    zero.query(
-        "DROP TABLE IF EXISTS test_insert_quorum_with_move_partition_source ON CLUSTER cluster"
+    # use different table names for easier disambiguation in logs between runs (you may also check uuid though, but not always convenient)
+    source_table_name = (
+        "test_insert_quorum_with_move_partition_source_new_data"
+        if add_new_data
+        else "test_insert_quorum_with_move_partition_source"
     )
-    zero.query(
-        "DROP TABLE IF EXISTS test_insert_quorum_with_move_partition_destination ON CLUSTER cluster"
+    destination_table_name = (
+        "test_insert_quorum_with_move_partition_destination_new_data"
+        if add_new_data
+        else "test_insert_quorum_with_move_partition_destination"
     )
+    zero.query(f"DROP TABLE IF EXISTS {source_table_name} ON CLUSTER cluster")
+    zero.query(f"DROP TABLE IF EXISTS {destination_table_name} ON CLUSTER cluster")
 
     create_source = (
-        "CREATE TABLE test_insert_quorum_with_move_partition_source ON CLUSTER cluster "
+        f"CREATE TABLE {source_table_name} ON CLUSTER cluster "
         "(a Int8, d Date) "
         "Engine = ReplicatedMergeTree "
         "PARTITION BY d ORDER BY a "
     )
 
     create_destination = (
-        "CREATE TABLE test_insert_quorum_with_move_partition_destination ON CLUSTER cluster "
+        f"CREATE TABLE {destination_table_name} ON CLUSTER cluster "
         "(a Int8, d Date) "
         "Engine = ReplicatedMergeTree "
         "PARTITION BY d ORDER BY a "
@@ -244,38 +241,30 @@ def test_insert_quorum_with_move_partition(started_cluster, add_new_data):
     print("Create destination Replicated table with three replicas")
     zero.query(create_destination)
 
-    print(
-        "Stop fetches for test_insert_quorum_with_move_partition_source at first replica."
-    )
-    first.query("SYSTEM STOP FETCHES test_insert_quorum_with_move_partition_source")
+    print(f"Stop fetches for {source_table_name} at first replica.")
+    first.query(f"SYSTEM STOP FETCHES {source_table_name}")
 
     print("Insert with quorum. (zero and second)")
-    zero.query(
-        "INSERT INTO test_insert_quorum_with_move_partition_source(a,d) VALUES(1, '2011-01-01')"
-    )
+    zero.query(f"INSERT INTO {source_table_name}(a,d) VALUES(1, '2011-01-01')")
 
     print("Drop partition.")
     zero.query(
-        "ALTER TABLE test_insert_quorum_with_move_partition_source MOVE PARTITION '2011-01-01' TO TABLE test_insert_quorum_with_move_partition_destination"
+        f"ALTER TABLE {source_table_name} MOVE PARTITION '2011-01-01' TO TABLE {destination_table_name}"
     )
 
     if add_new_data:
         print("Insert to deleted partition")
-        zero.query(
-            "INSERT INTO test_insert_quorum_with_move_partition_source(a,d) VALUES(2, '2011-01-01')"
-        )
+        zero.query(f"INSERT INTO {source_table_name}(a,d) VALUES(2, '2011-01-01')")
 
-    print(
-        "Resume fetches for test_insert_quorum_with_move_partition_source at first replica."
-    )
-    first.query("SYSTEM START FETCHES test_insert_quorum_with_move_partition_source")
+    print(f"Resume fetches for {source_table_name} at first replica.")
+    first.query(f"SYSTEM START FETCHES {source_table_name}")
 
     print("Sync first replica with others.")
-    first.query("SYSTEM SYNC REPLICA test_insert_quorum_with_move_partition_source")
+    first.query(f"SYSTEM SYNC REPLICA {source_table_name}")
 
     assert "20110101" not in first.query(
-        """
-    WITH (SELECT toString(uuid) FROM system.tables WHERE name = 'test_insert_quorum_with_move_partition_source') AS uuid,
+        f"""
+    WITH (SELECT toString(uuid) FROM system.tables WHERE name = '{source_table_name}') AS uuid,
          '/clickhouse/tables/' || uuid || '/0/quorum/last_part' AS p
     SELECT * FROM system.zookeeper WHERE path = p FORMAT Vertical
     """
@@ -284,25 +273,17 @@ def test_insert_quorum_with_move_partition(started_cluster, add_new_data):
     print("Select from updated partition.")
     if add_new_data:
         assert TSV("2\t2011-01-01\n") == TSV(
-            zero.query("SELECT * FROM test_insert_quorum_with_move_partition_source")
+            zero.query(f"SELECT * FROM {source_table_name}")
         )
         assert TSV("2\t2011-01-01\n") == TSV(
-            second.query("SELECT * FROM test_insert_quorum_with_move_partition_source")
+            second.query(f"SELECT * FROM {source_table_name}")
         )
     else:
-        assert TSV("") == TSV(
-            zero.query("SELECT * FROM test_insert_quorum_with_move_partition_source")
-        )
-        assert TSV("") == TSV(
-            second.query("SELECT * FROM test_insert_quorum_with_move_partition_source")
-        )
+        assert TSV("") == TSV(zero.query(f"SELECT * FROM {source_table_name}"))
+        assert TSV("") == TSV(second.query(f"SELECT * FROM {source_table_name}"))
 
-    zero.query(
-        "DROP TABLE IF EXISTS test_insert_quorum_with_move_partition_source ON CLUSTER cluster"
-    )
-    zero.query(
-        "DROP TABLE IF EXISTS test_insert_quorum_with_move_partition_destination ON CLUSTER cluster"
-    )
+    zero.query(f"DROP TABLE IF EXISTS {source_table_name} ON CLUSTER cluster")
+    zero.query(f"DROP TABLE IF EXISTS {destination_table_name} ON CLUSTER cluster")
 
 
 def test_insert_quorum_with_ttl(started_cluster):

From 60ab34f6b0a5344380ea6ce6c017b08b60522574 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Tue, 8 Aug 2023 11:06:49 +0000
Subject: [PATCH 118/249] sync second replica in test_insert_quorum

---
 tests/integration/test_quorum_inserts/test.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tests/integration/test_quorum_inserts/test.py b/tests/integration/test_quorum_inserts/test.py
index 581a02c154e..1276a6079f0 100644
--- a/tests/integration/test_quorum_inserts/test.py
+++ b/tests/integration/test_quorum_inserts/test.py
@@ -192,6 +192,9 @@ def test_insert_quorum_with_drop_partition(started_cluster, add_new_data):
     """
     )
 
+    # Sync second replica not to have `REPLICA_IS_NOT_IN_QUORUM` error
+    second.query(f"SYSTEM SYNC REPLICA {table_name}")
+
     print("Select from updated partition.")
     if add_new_data:
         assert TSV("2\t2011-01-01\n") == TSV(zero.query(f"SELECT * FROM {table_name}"))
@@ -270,6 +273,9 @@ def test_insert_quorum_with_move_partition(started_cluster, add_new_data):
     """
     )
 
+    # Sync second replica not to have `REPLICA_IS_NOT_IN_QUORUM` error
+    second.query(f"SYSTEM SYNC REPLICA {source_table_name}")
+
     print("Select from updated partition.")
     if add_new_data:
         assert TSV("2\t2011-01-01\n") == TSV(

From b6ce7253670a363fce779be41e9bef6d1079978b Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Tue, 8 Aug 2023 14:04:46 +0000
Subject: [PATCH 119/249] fix projection recalc

---
 src/Interpreters/MutationsInterpreter.cpp |  9 ++--
 src/Storages/MergeTree/MutateTask.cpp     | 55 -----------------------
 src/Storages/StorageInMemoryMetadata.cpp  |  2 +-
 3 files changed, 6 insertions(+), 60 deletions(-)

diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp
index 1a7cbb45999..e50f8488cac 100644
--- a/src/Interpreters/MutationsInterpreter.cpp
+++ b/src/Interpreters/MutationsInterpreter.cpp
@@ -748,8 +748,9 @@ void MutationsInterpreter::prepare(bool dry_run)
             {
                 // just recalculate ttl_infos without remove expired data
                 auto all_columns_vec = all_columns.getNames();
-                auto new_dependencies = metadata_snapshot->getColumnDependencies(
-                    NameSet(all_columns_vec.begin(), all_columns_vec.end()), false, has_dependency);
+                auto all_columns_set = NameSet(all_columns_vec.begin(), all_columns_vec.end());
+                auto new_dependencies = metadata_snapshot->getColumnDependencies(all_columns_set, false, has_dependency);
+
                 for (const auto & dependency : new_dependencies)
                 {
                     if (dependency.kind == ColumnDependency::TTL_EXPRESSION)
@@ -774,8 +775,8 @@ void MutationsInterpreter::prepare(bool dry_run)
                 }
 
                 auto all_columns_vec = all_columns.getNames();
-                auto all_dependencies = getAllColumnDependencies(
-                    metadata_snapshot, NameSet(all_columns_vec.begin(), all_columns_vec.end()), has_dependency);
+                auto all_columns_set = NameSet(all_columns_vec.begin(), all_columns_vec.end());
+                auto all_dependencies = getAllColumnDependencies(metadata_snapshot, all_columns_set, has_dependency);
 
                 for (const auto & dependency : all_dependencies)
                 {
diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp
index fa5ad9858e8..5143d9d5bb8 100644
--- a/src/Storages/MergeTree/MutateTask.cpp
+++ b/src/Storages/MergeTree/MutateTask.cpp
@@ -69,7 +69,6 @@ static void splitAndModifyMutationCommands(
     {
         NameSet mutated_columns;
         NameSet dropped_columns;
-        NameSet extra_columns_for_indices_and_projections;
 
         for (const auto & command : commands)
         {
@@ -86,41 +85,6 @@ static void splitAndModifyMutationCommands(
 
                 if (command.type == MutationCommand::Type::MATERIALIZE_COLUMN)
                     mutated_columns.emplace(command.column_name);
-
-                if (command.type == MutationCommand::Type::MATERIALIZE_INDEX)
-                {
-                    const auto & all_indices = metadata_snapshot->getSecondaryIndices();
-                    for (const auto & index : all_indices)
-                    {
-                        if (index.name == command.index_name)
-                        {
-                            auto required_columns = index.expression->getRequiredColumns();
-                            for (const auto & column : required_columns)
-                            {
-                                if (!part_columns.has(column))
-                                    extra_columns_for_indices_and_projections.insert(column);
-                            }
-                            break;
-                        }
-                    }
-                }
-
-                if (command.type == MutationCommand::Type::MATERIALIZE_PROJECTION)
-                {
-                    const auto & all_projections = metadata_snapshot->getProjections();
-                    for (const auto & projection : all_projections)
-                    {
-                        if (projection.name == command.projection_name)
-                        {
-                            for (const auto & column : projection.required_columns)
-                            {
-                                if (!part_columns.has(column))
-                                    extra_columns_for_indices_and_projections.insert(column);
-                            }
-                            break;
-                        }
-                    }
-                }
             }
             else if (command.type == MutationCommand::Type::DROP_INDEX || command.type == MutationCommand::Type::DROP_PROJECTION)
             {
@@ -223,25 +187,6 @@ static void splitAndModifyMutationCommands(
                 });
             }
         }
-
-        for (const auto & column_name : extra_columns_for_indices_and_projections)
-        {
-            if (mutated_columns.contains(column_name))
-                continue;
-
-            auto data_type = metadata_snapshot->getColumns().getColumn(
-                GetColumnsOptions::AllPhysical,
-                column_name).type;
-
-            for_interpreter.push_back(
-                MutationCommand
-                {
-                    .type = MutationCommand::Type::READ_COLUMN,
-                    .column_name = column_name,
-                    .data_type = std::move(data_type),
-                }
-            );
-        }
     }
     else
     {
diff --git a/src/Storages/StorageInMemoryMetadata.cpp b/src/Storages/StorageInMemoryMetadata.cpp
index b1da509c9b0..af285a953dc 100644
--- a/src/Storages/StorageInMemoryMetadata.cpp
+++ b/src/Storages/StorageInMemoryMetadata.cpp
@@ -274,7 +274,7 @@ ColumnDependencies StorageInMemoryMetadata::getColumnDependencies(
 
     for (const auto & projection : getProjections())
     {
-        if (has_dependency(projection.getDirectoryName(), ColumnDependency::PROJECTION))
+        if (has_dependency(projection.name, ColumnDependency::PROJECTION))
             add_dependent_columns(&projection, projections_columns);
     }
 

From 263730820751e89a0f0ea76b9ba04a005f25c934 Mon Sep 17 00:00:00 2001
From: Dmitry Novik <n0vik@clickhouse.com>
Date: Tue, 8 Aug 2023 19:54:07 +0000
Subject: [PATCH 120/249] Analyzer: fix test_system_flush_logs

---
 tests/analyzer_integration_broken_tests.txt      | 1 -
 tests/integration/test_system_flush_logs/test.py | 3 ++-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/analyzer_integration_broken_tests.txt b/tests/analyzer_integration_broken_tests.txt
index 91e2f1aa568..b9cca4bf677 100644
--- a/tests/analyzer_integration_broken_tests.txt
+++ b/tests/analyzer_integration_broken_tests.txt
@@ -114,7 +114,6 @@ test_quota/test.py::test_tracking_quota
 test_quota/test.py::test_users_xml_is_readonly
 test_mysql_database_engine/test.py::test_mysql_ddl_for_mysql_database
 test_profile_events_s3/test.py::test_profile_events
-test_system_flush_logs/test.py::test_system_logs[system.text_log-0]
 test_user_defined_object_persistence/test.py::test_persistence
 test_settings_profile/test.py::test_show_profiles
 test_sql_user_defined_functions_on_cluster/test.py::test_sql_user_defined_functions_on_cluster
diff --git a/tests/integration/test_system_flush_logs/test.py b/tests/integration/test_system_flush_logs/test.py
index bf225ac30f8..186dbc217c5 100644
--- a/tests/integration/test_system_flush_logs/test.py
+++ b/tests/integration/test_system_flush_logs/test.py
@@ -45,7 +45,8 @@ def test_system_logs(flush_logs, table, exists):
     if exists:
         node.query(q)
     else:
-        assert "Table {} doesn't exist".format(table) in node.query_and_get_error(q)
+        response = node.query_and_get_error(q)
+        assert "Table {} doesn't exist".format(table) in response or "Unknown table expression identifier '{}'".format(table) in response
 
 
 # Logic is tricky, let's check that there is no hang in case of message queue

From 4b6ade18b2a7dd43e06ba838460a9b76d313af10 Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Tue, 8 Aug 2023 20:06:59 +0000
Subject: [PATCH 121/249] Automatic style fix

---
 tests/integration/test_system_flush_logs/test.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/integration/test_system_flush_logs/test.py b/tests/integration/test_system_flush_logs/test.py
index 186dbc217c5..5bf3eff8b2d 100644
--- a/tests/integration/test_system_flush_logs/test.py
+++ b/tests/integration/test_system_flush_logs/test.py
@@ -46,7 +46,10 @@ def test_system_logs(flush_logs, table, exists):
         node.query(q)
     else:
         response = node.query_and_get_error(q)
-        assert "Table {} doesn't exist".format(table) in response or "Unknown table expression identifier '{}'".format(table) in response
+        assert (
+            "Table {} doesn't exist".format(table) in response
+            or "Unknown table expression identifier '{}'".format(table) in response
+        )
 
 
 # Logic is tricky, let's check that there is no hang in case of message queue

From 1ab99e5fdd2a52551a7f4f049a3feb06023079c9 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 8 Aug 2023 22:38:52 +0200
Subject: [PATCH 122/249] Replace old docs with symlinks

---
 .../example-datasets/wikistat.md              | 33 +------------------
 .../example-datasets/wikistat.md              | 33 +------------------
 src/Coordination/FourLetterCommand.cpp        |  1 -
 3 files changed, 2 insertions(+), 65 deletions(-)
 mode change 100644 => 120000 docs/ru/getting-started/example-datasets/wikistat.md
 mode change 100644 => 120000 docs/zh/getting-started/example-datasets/wikistat.md

diff --git a/docs/ru/getting-started/example-datasets/wikistat.md b/docs/ru/getting-started/example-datasets/wikistat.md
deleted file mode 100644
index 479616d667b..00000000000
--- a/docs/ru/getting-started/example-datasets/wikistat.md
+++ /dev/null
@@ -1,32 +0,0 @@
----
-slug: /ru/getting-started/example-datasets/wikistat
-sidebar_position: 17
-sidebar_label: WikiStat
----
-
-# WikiStat {#wikistat}
-
-См: http://dumps.wikimedia.org/other/pagecounts-raw/
-
-Создание таблицы:
-
-``` sql
-CREATE TABLE wikistat
-(
-    date Date,
-    time DateTime,
-    project String,
-    subproject String,
-    path String,
-    hits UInt64,
-    size UInt64
-) ENGINE = MergeTree(date, (path, time), 8192);
-```
-
-Загрузка данных:
-
-``` bash
-$ for i in {2007..2016}; do for j in {01..12}; do echo $i-$j >&2; curl -sSL "http://dumps.wikimedia.org/other/pagecounts-raw/$i/$i-$j/" | grep -oE 'pagecounts-[0-9]+-[0-9]+\.gz'; done; done | sort | uniq | tee links.txt
-$ cat links.txt | while read link; do wget http://dumps.wikimedia.org/other/pagecounts-raw/$(echo $link | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})[0-9]{2}-[0-9]+\.gz/\1/')/$(echo $link | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})[0-9]{2}-[0-9]+\.gz/\1-\2/')/$link; done
-$ ls -1 /opt/wikistat/ | grep gz | while read i; do echo $i; gzip -cd /opt/wikistat/$i | ./wikistat-loader --time="$(echo -n $i | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})([0-9]{2})-([0-9]{2})([0-9]{2})([0-9]{2})\.gz/\1-\2-\3 \4-00-00/')" | clickhouse-client --query="INSERT INTO wikistat FORMAT TabSeparated"; done
-```
diff --git a/docs/ru/getting-started/example-datasets/wikistat.md b/docs/ru/getting-started/example-datasets/wikistat.md
new file mode 120000
index 00000000000..2d429d00984
--- /dev/null
+++ b/docs/ru/getting-started/example-datasets/wikistat.md
@@ -0,0 +1 @@
+../../../en/getting-started/example-datasets/wikistat.md
\ No newline at end of file
diff --git a/docs/zh/getting-started/example-datasets/wikistat.md b/docs/zh/getting-started/example-datasets/wikistat.md
deleted file mode 100644
index 4ce13b0f1d3..00000000000
--- a/docs/zh/getting-started/example-datasets/wikistat.md
+++ /dev/null
@@ -1,32 +0,0 @@
----
-slug: /zh/getting-started/example-datasets/wikistat
-sidebar_position: 17
-sidebar_label: WikiStat
----
-
-# WikiStat {#wikistat}
-
-参考: http://dumps.wikimedia.org/other/pagecounts-raw/
-
-创建表结构：
-
-``` sql
-CREATE TABLE wikistat
-(
-    date Date,
-    time DateTime,
-    project String,
-    subproject String,
-    path String,
-    hits UInt64,
-    size UInt64
-) ENGINE = MergeTree(date, (path, time), 8192);
-```
-
-加载数据：
-
-``` bash
-$ for i in {2007..2016}; do for j in {01..12}; do echo $i-$j >&2; curl -sSL "http://dumps.wikimedia.org/other/pagecounts-raw/$i/$i-$j/" | grep -oE 'pagecounts-[0-9]+-[0-9]+\.gz'; done; done | sort | uniq | tee links.txt
-$ cat links.txt | while read link; do wget http://dumps.wikimedia.org/other/pagecounts-raw/$(echo $link | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})[0-9]{2}-[0-9]+\.gz/\1/')/$(echo $link | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})[0-9]{2}-[0-9]+\.gz/\1-\2/')/$link; done
-$ ls -1 /opt/wikistat/ | grep gz | while read i; do echo $i; gzip -cd /opt/wikistat/$i | ./wikistat-loader --time="$(echo -n $i | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})([0-9]{2})-([0-9]{2})([0-9]{2})([0-9]{2})\.gz/\1-\2-\3 \4-00-00/')" | clickhouse-client --query="INSERT INTO wikistat FORMAT TabSeparated"; done
-```
diff --git a/docs/zh/getting-started/example-datasets/wikistat.md b/docs/zh/getting-started/example-datasets/wikistat.md
new file mode 120000
index 00000000000..2d429d00984
--- /dev/null
+++ b/docs/zh/getting-started/example-datasets/wikistat.md
@@ -0,0 +1 @@
+../../../en/getting-started/example-datasets/wikistat.md
\ No newline at end of file
diff --git a/src/Coordination/FourLetterCommand.cpp b/src/Coordination/FourLetterCommand.cpp
index d1ff03482b6..379a24e5aab 100644
--- a/src/Coordination/FourLetterCommand.cpp
+++ b/src/Coordination/FourLetterCommand.cpp
@@ -577,7 +577,6 @@ String FeatureFlagsCommand::run()
     }
 
     return ret.str();
-
 }
 
 }

From 040d6665464a65e3aebb83fe3d2b5830890183fa Mon Sep 17 00:00:00 2001
From: Igor Nikonov <igor@clickhouse.com>
Date: Tue, 8 Aug 2023 21:57:27 +0000
Subject: [PATCH 123/249] Fix: 00838_unique_index test with analyzer

---
 src/Storages/MergeTree/MergeTreeIndexSet.cpp | 5 ++---
 tests/analyzer_tech_debt.txt                 | 1 -
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeIndexSet.cpp b/src/Storages/MergeTree/MergeTreeIndexSet.cpp
index 05ea7d15603..651dd731e97 100644
--- a/src/Storages/MergeTree/MergeTreeIndexSet.cpp
+++ b/src/Storages/MergeTree/MergeTreeIndexSet.cpp
@@ -457,11 +457,10 @@ const ActionsDAG::Node * MergeTreeIndexConditionSet::operatorFromDAG(const Actio
         if (arguments_size != 1)
             return nullptr;
 
-        auto bit_wrapper_function = FunctionFactory::instance().get("__bitWrapperFunc", context);
-        const auto & bit_wrapper_func_node = result_dag->addFunction(bit_wrapper_function, {arguments[0]}, {});
+        const ActionsDAG::Node * argument = &traverseDAG(*arguments[0], result_dag, context, node_to_result_node);
 
         auto bit_swap_last_two_function = FunctionFactory::instance().get("__bitSwapLastTwo", context);
-        return &result_dag->addFunction(bit_swap_last_two_function, {&bit_wrapper_func_node}, {});
+        return &result_dag->addFunction(bit_swap_last_two_function, {argument}, {});
     }
     else if (function_name == "and" || function_name == "indexHint" || function_name == "or")
     {
diff --git a/tests/analyzer_tech_debt.txt b/tests/analyzer_tech_debt.txt
index c8f2bb9f43d..7b61de546fe 100644
--- a/tests/analyzer_tech_debt.txt
+++ b/tests/analyzer_tech_debt.txt
@@ -7,7 +7,6 @@
 00725_memory_tracking
 00754_distributed_optimize_skip_select_on_unused_shards
 00754_distributed_optimize_skip_select_on_unused_shards_with_prewhere
-00838_unique_index
 00927_asof_joins
 00940_order_by_read_in_order_query_plan
 00945_bloom_filter_index

From 3296daad96e3df8b1f610458f46ea11afe4a586c Mon Sep 17 00:00:00 2001
From: Igor Nikonov <igor@clickhouse.com>
Date: Tue, 8 Aug 2023 22:27:12 +0000
Subject: [PATCH 124/249] Don't build AST filter with enabled analyzer

---
 src/Storages/MergeTree/MergeTreeIndexSet.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeIndexSet.cpp b/src/Storages/MergeTree/MergeTreeIndexSet.cpp
index 651dd731e97..eaa695c7f7a 100644
--- a/src/Storages/MergeTree/MergeTreeIndexSet.cpp
+++ b/src/Storages/MergeTree/MergeTreeIndexSet.cpp
@@ -256,10 +256,6 @@ MergeTreeIndexConditionSet::MergeTreeIndexConditionSet(
         if (!key_columns.contains(name))
             key_columns.insert(name);
 
-    ASTPtr ast_filter_node = buildFilterNode(query_info.query);
-    if (!ast_filter_node)
-        return;
-
     if (context->getSettingsRef().allow_experimental_analyzer)
     {
         if (!query_info.filter_actions_dag)
@@ -280,6 +276,10 @@ MergeTreeIndexConditionSet::MergeTreeIndexConditionSet(
     }
     else
     {
+        ASTPtr ast_filter_node = buildFilterNode(query_info.query);
+        if (!ast_filter_node)
+            return;
+
         if (checkASTUseless(ast_filter_node))
             return;
 

From 5561e3e198b91a7596188d98ba2f213e0405614b Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 9 Aug 2023 01:44:39 +0200
Subject: [PATCH 125/249] Remove garbage and speed up Debug and Tidy builds

---
 CMakeLists.txt                                                | 3 ---
 src/DataTypes/tests/gtest_DataType_deserializeAsText.cpp      | 2 --
 src/Functions/array/arrayEnumerateRanked.h                    | 1 -
 src/Functions/extractAllGroups.h                              | 2 --
 .../keyvaluepair/tests/gtest_extractKeyValuePairs.cpp         | 4 +---
 src/Processors/Formats/Impl/ODBCDriver2BlockOutputFormat.cpp  | 3 ---
 6 files changed, 1 insertion(+), 14 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 157f8c9fac0..55bcf5fbf3c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -208,9 +208,6 @@ option(OMIT_HEAVY_DEBUG_SYMBOLS
     "Do not generate debugger info for heavy modules (ClickHouse functions and dictionaries, some contrib)"
     ${OMIT_HEAVY_DEBUG_SYMBOLS_DEFAULT})
 
-if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")
-    set(USE_DEBUG_HELPERS ON)
-endif()
 option(USE_DEBUG_HELPERS "Enable debug helpers" ${USE_DEBUG_HELPERS})
 
 option(BUILD_STANDALONE_KEEPER "Build keeper as small standalone binary" OFF)
diff --git a/src/DataTypes/tests/gtest_DataType_deserializeAsText.cpp b/src/DataTypes/tests/gtest_DataType_deserializeAsText.cpp
index b755bd109d0..0373e55a62d 100644
--- a/src/DataTypes/tests/gtest_DataType_deserializeAsText.cpp
+++ b/src/DataTypes/tests/gtest_DataType_deserializeAsText.cpp
@@ -1,8 +1,6 @@
 #include <Columns/IColumn.h>
 #include <Core/Field.h>
 #include <DataTypes/DataTypeFactory.h>
-#include <DataTypes/IDataType.h>
-#include <DataTypes/getMostSubtype.h>
 #include <Formats/FormatSettings.h>
 #include <IO/ReadBuffer.h>
 
diff --git a/src/Functions/array/arrayEnumerateRanked.h b/src/Functions/array/arrayEnumerateRanked.h
index 0733f1e2d43..2e28a0f455d 100644
--- a/src/Functions/array/arrayEnumerateRanked.h
+++ b/src/Functions/array/arrayEnumerateRanked.h
@@ -13,7 +13,6 @@
 #include <Common/ColumnsHashing.h>
 #include <Common/HashTable/ClearableHashMap.h>
 
-// for better debug: #include <Core/iostream_debug_helpers.h>
 
 /** The function will enumerate distinct values of the passed multidimensional arrays looking inside at the specified depths.
   * This is very unusual function made as a special order for our dear customer - Metrica web analytics system.
diff --git a/src/Functions/extractAllGroups.h b/src/Functions/extractAllGroups.h
index 3a7987be93e..f9637e59461 100644
--- a/src/Functions/extractAllGroups.h
+++ b/src/Functions/extractAllGroups.h
@@ -14,8 +14,6 @@
 #include <string>
 #include <vector>
 
-#include <Core/iostream_debug_helpers.h>
-
 
 namespace DB
 {
diff --git a/src/Functions/keyvaluepair/tests/gtest_extractKeyValuePairs.cpp b/src/Functions/keyvaluepair/tests/gtest_extractKeyValuePairs.cpp
index 507d9c0e5c7..55a08023cbd 100644
--- a/src/Functions/keyvaluepair/tests/gtest_extractKeyValuePairs.cpp
+++ b/src/Functions/keyvaluepair/tests/gtest_extractKeyValuePairs.cpp
@@ -9,11 +9,11 @@
 #include <Formats/FormatSettings.h>
 #include <IO/WriteBufferFromString.h>
 
-#include <ostream>
 #include <gtest/gtest.h>
 #include <initializer_list>
 #include <Core/iostream_debug_helpers.h>
 
+
 namespace
 {
 using namespace DB;
@@ -174,5 +174,3 @@ INSTANTIATE_TEST_SUITE_P(InvalidEscapeSeqInValue, extractKVPairKeyValuePairExtra
         }
     )
 );
-
-
diff --git a/src/Processors/Formats/Impl/ODBCDriver2BlockOutputFormat.cpp b/src/Processors/Formats/Impl/ODBCDriver2BlockOutputFormat.cpp
index 43294355f2f..c8c00086e8c 100644
--- a/src/Processors/Formats/Impl/ODBCDriver2BlockOutputFormat.cpp
+++ b/src/Processors/Formats/Impl/ODBCDriver2BlockOutputFormat.cpp
@@ -3,9 +3,6 @@
 #include <Processors/Formats/Impl/ODBCDriver2BlockOutputFormat.h>
 #include <IO/WriteBuffer.h>
 #include <IO/WriteHelpers.h>
-
-
-#include <Core/iostream_debug_helpers.h>
 #include <DataTypes/DataTypeLowCardinality.h>
 
 

From aa757490bdaae835ef5beb466f05154689879f86 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 9 Aug 2023 02:19:02 +0200
Subject: [PATCH 126/249] Ditch tons of garbage

---
 base/base/JSON.cpp                            |  2 --
 base/base/wide_integer_impl.h                 |  1 -
 base/poco/Data/ODBC/src/Unicode_UNIXODBC.cpp  |  1 -
 base/poco/Foundation/src/Task.cpp             |  1 -
 base/poco/JSON/src/Object.cpp                 |  1 -
 base/poco/Net/src/HTTPClientSession.cpp       |  1 -
 programs/disks/ICommand.cpp                   |  2 ++
 src/Access/MemoryAccessStorage.h              |  1 +
 src/AggregateFunctions/tests/gtest_ranks.cpp  |  1 -
 src/Analyzer/Passes/FuseFunctionsPass.cpp     |  3 +++
 src/Analyzer/examples/query_analyzer.cpp      |  1 -
 src/Client/ConnectionParameters.cpp           |  1 -
 src/Client/ConnectionString.cpp               |  1 -
 src/Columns/ColumnObject.cpp                  |  4 ++--
 src/Columns/tests/gtest_column_sparse.cpp     |  1 +
 src/Common/Config/configReadClient.cpp        |  1 -
 src/Common/CounterInFile.h                    |  1 -
 src/Common/DateLUTImpl.cpp                    |  1 -
 src/Common/EventNotifier.h                    |  2 +-
 src/Common/FST.cpp                            |  1 -
 src/Common/SpaceSaving.h                      |  2 +-
 src/Common/StudentTTest.cpp                   |  1 -
 src/Common/ThreadPool.cpp                     |  1 -
 src/Common/UnicodeBar.cpp                     |  1 -
 src/Common/VersionNumber.h                    |  7 +------
 .../integer_hash_tables_and_hashes.cpp        |  1 -
 src/Common/examples/average.cpp               |  1 -
 src/Common/examples/chaos_sanitizer.cpp       |  1 -
 src/Common/examples/compact_array.cpp         |  2 +-
 src/Common/tests/gtest_hash_table.cpp         |  2 +-
 src/Common/tests/gtest_lru_cache.cpp          |  1 -
 src/Common/tests/gtest_lru_hash_map.cpp       |  1 -
 src/Common/tests/gtest_lru_resource_cache.cpp |  1 -
 src/Common/tests/gtest_slru_cache.cpp         |  1 -
 src/Common/tests/gtest_thread_pool_limit.cpp  |  1 -
 src/Common/tests/gtest_thread_pool_loop.cpp   |  1 -
 .../gtest_thread_pool_schedule_exception.cpp  |  1 -
 .../CompressionCodecDeflateQpl.cpp            |  4 +++-
 .../fuzzers/compressed_buffer_fuzzer.cpp      |  1 -
 .../fuzzers/delta_decompress_fuzzer.cpp       |  1 -
 .../double_delta_decompress_fuzzer.cpp        |  1 -
 .../fuzzers/encrypted_decompress_fuzzer.cpp   |  1 -
 .../fuzzers/lz4_decompress_fuzzer.cpp         |  1 -
 src/Coordination/SnapshotableHashTable.h      |  2 +-
 src/Coordination/SummingStateMachine.cpp      |  1 -
 src/Coordination/pathUtils.cpp                |  1 -
 src/Core/MySQL/MySQLCharset.cpp               |  1 -
 src/Core/fuzzers/names_and_types_fuzzer.cpp   |  1 -
 src/Daemon/BaseDaemon.h                       |  1 -
 src/Daemon/SentryWriter.cpp                   |  2 --
 src/Dictionaries/HashedDictionary.cpp         |  1 +
 src/Dictionaries/PolygonDictionaryUtils.h     |  1 +
 src/Disks/IO/ReadBufferFromRemoteFSGather.cpp |  1 -
 src/Functions/array/arrayShuffle.cpp          |  1 +
 src/Functions/evalMLMethod.cpp                |  1 -
 .../keyvaluepair/impl/StateHandler.h          |  1 -
 src/Functions/translate.cpp                   |  2 ++
 src/IO/Archives/LibArchiveReader.h            |  1 -
 src/IO/HTTPCommon.h                           |  1 -
 src/IO/ReadBufferFromIStream.h                |  2 --
 src/IO/S3/SessionAwareIOStream.h              |  2 +-
 src/IO/StdStreamFromReadBuffer.h              |  1 -
 src/IO/VarInt.h                               |  1 -
 src/IO/WriteBuffer.h                          |  1 -
 src/IO/WriteBufferFromOStream.h               |  2 +-
 src/IO/examples/read_buffer_from_hdfs.cpp     |  1 -
 src/Interpreters/Context.cpp                  | 19 -------------------
 src/Interpreters/GraceHashJoin.cpp            |  7 +++----
 .../InterpreterKillQueryQuery.cpp             |  1 -
 src/Loggers/OwnSplitChannel.cpp               |  1 -
 src/Parsers/ASTSelectWithUnionQuery.cpp       |  1 -
 .../Executors/StreamingFormatExecutor.cpp     |  1 -
 src/Processors/IAccumulatingTransform.cpp     |  1 -
 src/Processors/ResizeProcessor.cpp            |  1 -
 .../Transforms/CountingTransform.cpp          |  1 -
 .../Transforms/ExceptionKeepingTransform.cpp  |  1 -
 .../Transforms/SquashingChunksTransform.cpp   |  1 -
 src/Server/HTTP/HTTPServerResponse.h          |  2 +-
 src/Storages/MergeTree/GinIndexStore.cpp      |  1 -
 .../fuzzers/mergetree_checksum_fuzzer.cpp     |  1 -
 80 files changed, 29 insertions(+), 102 deletions(-)

diff --git a/base/base/JSON.cpp b/base/base/JSON.cpp
index 4c6d97b4444..0b43be38149 100644
--- a/base/base/JSON.cpp
+++ b/base/base/JSON.cpp
@@ -7,8 +7,6 @@
 #include <base/find_symbols.h>
 #include <base/preciseExp10.h>
 
-#include <iostream>
-
 #define JSON_MAX_DEPTH 100
 
 
diff --git a/base/base/wide_integer_impl.h b/base/base/wide_integer_impl.h
index 411841e6d9f..278d62d38a1 100644
--- a/base/base/wide_integer_impl.h
+++ b/base/base/wide_integer_impl.h
@@ -12,7 +12,6 @@
 #include <tuple>
 #include <limits>
 
-#include <boost/multiprecision/cpp_bin_float.hpp>
 #include <boost/math/special_functions/fpclassify.hpp>
 
 // NOLINTBEGIN(*)
diff --git a/base/poco/Data/ODBC/src/Unicode_UNIXODBC.cpp b/base/poco/Data/ODBC/src/Unicode_UNIXODBC.cpp
index 4caf097c28a..1c5555f8cf3 100644
--- a/base/poco/Data/ODBC/src/Unicode_UNIXODBC.cpp
+++ b/base/poco/Data/ODBC/src/Unicode_UNIXODBC.cpp
@@ -19,7 +19,6 @@
 #include "Poco/UTF16Encoding.h"
 #include "Poco/Buffer.h"
 #include "Poco/Exception.h"
-#include <iostream>
 
 
 using Poco::Buffer;
diff --git a/base/poco/Foundation/src/Task.cpp b/base/poco/Foundation/src/Task.cpp
index a850ae37eff..4303d50d6e8 100644
--- a/base/poco/Foundation/src/Task.cpp
+++ b/base/poco/Foundation/src/Task.cpp
@@ -16,7 +16,6 @@
 #include "Poco/TaskManager.h"
 #include "Poco/Exception.h"
 
-#include <iostream>
 #include <array>
 
 
diff --git a/base/poco/JSON/src/Object.cpp b/base/poco/JSON/src/Object.cpp
index 7fca65c5b01..b041f570934 100644
--- a/base/poco/JSON/src/Object.cpp
+++ b/base/poco/JSON/src/Object.cpp
@@ -14,7 +14,6 @@
 
 #include "Poco/JSON/Object.h"
 #include <iostream>
-#include <sstream>
 
 
 using Poco::Dynamic::Var;
diff --git a/base/poco/Net/src/HTTPClientSession.cpp b/base/poco/Net/src/HTTPClientSession.cpp
index c5697b556d1..2712c0c452e 100644
--- a/base/poco/Net/src/HTTPClientSession.cpp
+++ b/base/poco/Net/src/HTTPClientSession.cpp
@@ -26,7 +26,6 @@
 #include "Poco/CountingStream.h"
 #include "Poco/RegularExpression.h"
 #include <sstream>
-#include <iostream>
 
 
 using Poco::NumberFormatter;
diff --git a/programs/disks/ICommand.cpp b/programs/disks/ICommand.cpp
index 52d1a2196a9..86188fb6db1 100644
--- a/programs/disks/ICommand.cpp
+++ b/programs/disks/ICommand.cpp
@@ -1,4 +1,6 @@
 #include "ICommand.h"
+#include <iostream>
+
 
 namespace DB
 {
diff --git a/src/Access/MemoryAccessStorage.h b/src/Access/MemoryAccessStorage.h
index b63132147da..deb5a30f4b3 100644
--- a/src/Access/MemoryAccessStorage.h
+++ b/src/Access/MemoryAccessStorage.h
@@ -6,6 +6,7 @@
 #include <memory>
 #include <mutex>
 #include <unordered_map>
+#include <boost/container/flat_set.hpp>
 
 
 namespace DB
diff --git a/src/AggregateFunctions/tests/gtest_ranks.cpp b/src/AggregateFunctions/tests/gtest_ranks.cpp
index b29271cbec7..99c7fbd26d7 100644
--- a/src/AggregateFunctions/tests/gtest_ranks.cpp
+++ b/src/AggregateFunctions/tests/gtest_ranks.cpp
@@ -2,7 +2,6 @@
 #include <IO/ReadBufferFromString.h>
 #include <Common/PODArray.h>
 #include <AggregateFunctions/StatCommon.h>
-#include <iostream>
 
 #include <gtest/gtest.h>
 
diff --git a/src/Analyzer/Passes/FuseFunctionsPass.cpp b/src/Analyzer/Passes/FuseFunctionsPass.cpp
index 2cb7afa4ad6..7b27bd70ce3 100644
--- a/src/Analyzer/Passes/FuseFunctionsPass.cpp
+++ b/src/Analyzer/Passes/FuseFunctionsPass.cpp
@@ -14,6 +14,9 @@
 #include <Analyzer/FunctionNode.h>
 #include <Analyzer/HashUtils.h>
 
+#include <numeric>
+
+
 namespace DB
 {
 
diff --git a/src/Analyzer/examples/query_analyzer.cpp b/src/Analyzer/examples/query_analyzer.cpp
index 5a20b46b346..869113b10f8 100644
--- a/src/Analyzer/examples/query_analyzer.cpp
+++ b/src/Analyzer/examples/query_analyzer.cpp
@@ -1,4 +1,3 @@
-#include <iostream>
 
 int main(int argc, char ** argv)
 {
diff --git a/src/Client/ConnectionParameters.cpp b/src/Client/ConnectionParameters.cpp
index 8c7a4e830fe..1f90e757b73 100644
--- a/src/Client/ConnectionParameters.cpp
+++ b/src/Client/ConnectionParameters.cpp
@@ -1,6 +1,5 @@
 #include "ConnectionParameters.h"
 #include <fstream>
-#include <iostream>
 #include <Core/Defines.h>
 #include <Core/Protocol.h>
 #include <Core/Types.h>
diff --git a/src/Client/ConnectionString.cpp b/src/Client/ConnectionString.cpp
index 8150ae98c8b..0893ef1af00 100644
--- a/src/Client/ConnectionString.cpp
+++ b/src/Client/ConnectionString.cpp
@@ -6,7 +6,6 @@
 #include <Poco/URI.h>
 
 #include <array>
-#include <iostream>
 #include <string>
 #include <unordered_map>
 
diff --git a/src/Columns/ColumnObject.cpp b/src/Columns/ColumnObject.cpp
index 07872774559..2fb6d2c3028 100644
--- a/src/Columns/ColumnObject.cpp
+++ b/src/Columns/ColumnObject.cpp
@@ -2,17 +2,17 @@
 #include <Columns/ColumnObject.h>
 #include <Columns/ColumnsNumber.h>
 #include <Columns/ColumnArray.h>
-#include <Columns/ColumnSparse.h>
 #include <DataTypes/ObjectUtils.h>
 #include <DataTypes/getLeastSupertype.h>
 #include <DataTypes/DataTypeNothing.h>
 #include <DataTypes/DataTypeNullable.h>
 #include <DataTypes/DataTypeFactory.h>
-#include <DataTypes/NestedUtils.h>
 #include <Interpreters/castColumn.h>
 #include <Interpreters/convertFieldToType.h>
 #include <Common/HashTable/HashSet.h>
 #include <Processors/Transforms/ColumnGathererTransform.h>
+#include <numeric>
+
 
 namespace DB
 {
diff --git a/src/Columns/tests/gtest_column_sparse.cpp b/src/Columns/tests/gtest_column_sparse.cpp
index 6062ea51941..c3450ff91b4 100644
--- a/src/Columns/tests/gtest_column_sparse.cpp
+++ b/src/Columns/tests/gtest_column_sparse.cpp
@@ -10,6 +10,7 @@
 
 #include <Common/FieldVisitors.h>
 
+
 using namespace DB;
 static pcg64 rng(randomSeed());
 
diff --git a/src/Common/Config/configReadClient.cpp b/src/Common/Config/configReadClient.cpp
index 44d338c07af..9f137124be0 100644
--- a/src/Common/Config/configReadClient.cpp
+++ b/src/Common/Config/configReadClient.cpp
@@ -3,7 +3,6 @@
 #include <Poco/Util/LayeredConfiguration.h>
 #include "ConfigProcessor.h"
 #include <filesystem>
-#include <iostream>
 #include <base/types.h>
 
 namespace fs = std::filesystem;
diff --git a/src/Common/CounterInFile.h b/src/Common/CounterInFile.h
index d52f4a10050..fe3b74173f6 100644
--- a/src/Common/CounterInFile.h
+++ b/src/Common/CounterInFile.h
@@ -4,7 +4,6 @@
 #include <sys/file.h>
 
 #include <string>
-#include <iostream>
 #include <mutex>
 #include <filesystem>
 
diff --git a/src/Common/DateLUTImpl.cpp b/src/Common/DateLUTImpl.cpp
index d5e04238ef9..4c21d9c9783 100644
--- a/src/Common/DateLUTImpl.cpp
+++ b/src/Common/DateLUTImpl.cpp
@@ -10,7 +10,6 @@
 #include <chrono>
 #include <cstring>
 #include <memory>
-#include <iostream>
 
 
 /// Embedded timezones.
diff --git a/src/Common/EventNotifier.h b/src/Common/EventNotifier.h
index edf7622512d..1d34f7ad9b1 100644
--- a/src/Common/EventNotifier.h
+++ b/src/Common/EventNotifier.h
@@ -7,11 +7,11 @@
 #include <map>
 #include <memory>
 #include <utility>
-#include <iostream>
 
 #include <base/types.h>
 #include <Common/HashTable/Hash.h>
 
+
 namespace DB
 {
 
diff --git a/src/Common/FST.cpp b/src/Common/FST.cpp
index 30e10610eab..af2acc859e7 100644
--- a/src/Common/FST.cpp
+++ b/src/Common/FST.cpp
@@ -1,7 +1,6 @@
 #include "FST.h"
 #include <algorithm>
 #include <cassert>
-#include <iostream>
 #include <memory>
 #include <vector>
 #include <Common/Exception.h>
diff --git a/src/Common/SpaceSaving.h b/src/Common/SpaceSaving.h
index 3452402ba82..5d3e8fa321b 100644
--- a/src/Common/SpaceSaving.h
+++ b/src/Common/SpaceSaving.h
@@ -1,6 +1,5 @@
 #pragma once
 
-#include <iostream>
 #include <vector>
 
 #include <boost/range/adaptor/reversed.hpp>
@@ -19,6 +18,7 @@
 #include <IO/ReadHelpers.h>
 #include <IO/VarInt.h>
 
+
 /*
  * Implementation of the Filtered Space-Saving for TopK streaming analysis.
  *   http://www.l2f.inesc-id.pt/~fmmb/wiki/uploads/Work/misnis.ref0a.pdf
diff --git a/src/Common/StudentTTest.cpp b/src/Common/StudentTTest.cpp
index 59842488fd0..03159dca2ca 100644
--- a/src/Common/StudentTTest.cpp
+++ b/src/Common/StudentTTest.cpp
@@ -1,7 +1,6 @@
 #include "StudentTTest.h"
 
 #include <cmath>
-#include <iostream>
 #include <iomanip>
 #include <sstream>
 #include <stdexcept>
diff --git a/src/Common/ThreadPool.cpp b/src/Common/ThreadPool.cpp
index cc88594d84f..4a5bdeffcee 100644
--- a/src/Common/ThreadPool.cpp
+++ b/src/Common/ThreadPool.cpp
@@ -6,7 +6,6 @@
 #include <Common/noexcept_scope.h>
 
 #include <cassert>
-#include <iostream>
 #include <type_traits>
 
 #include <Poco/Util/Application.h>
diff --git a/src/Common/UnicodeBar.cpp b/src/Common/UnicodeBar.cpp
index bad39d8080c..253d720e8a1 100644
--- a/src/Common/UnicodeBar.cpp
+++ b/src/Common/UnicodeBar.cpp
@@ -7,7 +7,6 @@
 #include <Common/UnicodeBar.h>
 #include <Common/NaNUtils.h>
 
-#include <iostream>
 
 namespace DB
 {
diff --git a/src/Common/VersionNumber.h b/src/Common/VersionNumber.h
index 94bf234c1da..050c94b9574 100644
--- a/src/Common/VersionNumber.h
+++ b/src/Common/VersionNumber.h
@@ -2,9 +2,9 @@
 
 #include <string>
 #include <vector>
-#include <iostream>
 #include <base/types.h>
 
+
 namespace DB
 {
 
@@ -27,11 +27,6 @@ struct VersionNumber
 
     std::string toString() const;
 
-    friend std::ostream & operator<<(std::ostream & os, const VersionNumber & v)
-    {
-        return os << v.toString();
-    }
-
 private:
     using Components = std::vector<Int64>;
     Components components;
diff --git a/src/Common/benchmarks/integer_hash_tables_and_hashes.cpp b/src/Common/benchmarks/integer_hash_tables_and_hashes.cpp
index c245fc471cc..2a4e366f1c5 100644
--- a/src/Common/benchmarks/integer_hash_tables_and_hashes.cpp
+++ b/src/Common/benchmarks/integer_hash_tables_and_hashes.cpp
@@ -1,7 +1,6 @@
 #include <benchmark/benchmark.h>
 
 #include <iomanip>
-#include <iostream>
 #include <random>
 #include <vector>
 
diff --git a/src/Common/examples/average.cpp b/src/Common/examples/average.cpp
index f281abdced2..cd4f9321cf0 100644
--- a/src/Common/examples/average.cpp
+++ b/src/Common/examples/average.cpp
@@ -1,4 +1,3 @@
-#include <iostream>
 #include <string>
 #include <bit>
 
diff --git a/src/Common/examples/chaos_sanitizer.cpp b/src/Common/examples/chaos_sanitizer.cpp
index 0b7d21aa33e..76e22411a2e 100644
--- a/src/Common/examples/chaos_sanitizer.cpp
+++ b/src/Common/examples/chaos_sanitizer.cpp
@@ -6,7 +6,6 @@
 
 #include <IO/ReadHelpers.h>
 
-#include <Common/Exception.h>
 #include <Common/ThreadFuzzer.h>
 
 
diff --git a/src/Common/examples/compact_array.cpp b/src/Common/examples/compact_array.cpp
index 58c4ea3be1e..9e16372deff 100644
--- a/src/Common/examples/compact_array.cpp
+++ b/src/Common/examples/compact_array.cpp
@@ -4,11 +4,11 @@
 #include <filesystem>
 #include <string>
 #include <iostream>
-#include <fstream>
 #include <stdexcept>
 #include <cstdlib>
 #include <unistd.h>
 
+
 namespace fs = std::filesystem;
 
 static std::string createTmpPath(const std::string & filename)
diff --git a/src/Common/tests/gtest_hash_table.cpp b/src/Common/tests/gtest_hash_table.cpp
index 0221a682577..72941126cfd 100644
--- a/src/Common/tests/gtest_hash_table.cpp
+++ b/src/Common/tests/gtest_hash_table.cpp
@@ -1,5 +1,5 @@
 #include <iomanip>
-#include <iostream>
+#include <numeric>
 
 #include <Interpreters/AggregationCommon.h>
 
diff --git a/src/Common/tests/gtest_lru_cache.cpp b/src/Common/tests/gtest_lru_cache.cpp
index 1185dd58e5e..0360b5b3324 100644
--- a/src/Common/tests/gtest_lru_cache.cpp
+++ b/src/Common/tests/gtest_lru_cache.cpp
@@ -1,5 +1,4 @@
 #include <iomanip>
-#include <iostream>
 #include <gtest/gtest.h>
 #include <Common/CacheBase.h>
 
diff --git a/src/Common/tests/gtest_lru_hash_map.cpp b/src/Common/tests/gtest_lru_hash_map.cpp
index f45a503be43..cbb3d302efc 100644
--- a/src/Common/tests/gtest_lru_hash_map.cpp
+++ b/src/Common/tests/gtest_lru_hash_map.cpp
@@ -1,5 +1,4 @@
 #include <iomanip>
-#include <iostream>
 
 #include <Common/HashTable/LRUHashMap.h>
 
diff --git a/src/Common/tests/gtest_lru_resource_cache.cpp b/src/Common/tests/gtest_lru_resource_cache.cpp
index f88eded531e..bc037824ff8 100644
--- a/src/Common/tests/gtest_lru_resource_cache.cpp
+++ b/src/Common/tests/gtest_lru_resource_cache.cpp
@@ -1,5 +1,4 @@
 #include <iomanip>
-#include <iostream>
 #include <gtest/gtest.h>
 #include <Common/LRUResourceCache.h>
 
diff --git a/src/Common/tests/gtest_slru_cache.cpp b/src/Common/tests/gtest_slru_cache.cpp
index 52549592f0e..ed04f427d9d 100644
--- a/src/Common/tests/gtest_slru_cache.cpp
+++ b/src/Common/tests/gtest_slru_cache.cpp
@@ -1,5 +1,4 @@
 #include <iomanip>
-#include <iostream>
 #include <gtest/gtest.h>
 #include <Common/CacheBase.h>
 
diff --git a/src/Common/tests/gtest_thread_pool_limit.cpp b/src/Common/tests/gtest_thread_pool_limit.cpp
index 17f79d17894..b47c8cdad18 100644
--- a/src/Common/tests/gtest_thread_pool_limit.cpp
+++ b/src/Common/tests/gtest_thread_pool_limit.cpp
@@ -1,5 +1,4 @@
 #include <atomic>
-#include <iostream>
 #include <Common/ThreadPool.h>
 #include <Common/CurrentMetrics.h>
 
diff --git a/src/Common/tests/gtest_thread_pool_loop.cpp b/src/Common/tests/gtest_thread_pool_loop.cpp
index 556c39df949..170a888ff72 100644
--- a/src/Common/tests/gtest_thread_pool_loop.cpp
+++ b/src/Common/tests/gtest_thread_pool_loop.cpp
@@ -1,5 +1,4 @@
 #include <atomic>
-#include <iostream>
 #include <Common/ThreadPool.h>
 #include <Common/CurrentMetrics.h>
 
diff --git a/src/Common/tests/gtest_thread_pool_schedule_exception.cpp b/src/Common/tests/gtest_thread_pool_schedule_exception.cpp
index 5dbad00848d..d8e00b5314c 100644
--- a/src/Common/tests/gtest_thread_pool_schedule_exception.cpp
+++ b/src/Common/tests/gtest_thread_pool_schedule_exception.cpp
@@ -1,4 +1,3 @@
-#include <iostream>
 #include <stdexcept>
 #include <Common/ThreadPool.h>
 #include <Common/CurrentMetrics.h>
diff --git a/src/Compression/CompressionCodecDeflateQpl.cpp b/src/Compression/CompressionCodecDeflateQpl.cpp
index 5dcd6008b51..0737e523ba0 100644
--- a/src/Compression/CompressionCodecDeflateQpl.cpp
+++ b/src/Compression/CompressionCodecDeflateQpl.cpp
@@ -1,15 +1,17 @@
 #ifdef ENABLE_QPL_COMPRESSION
+
 #include <cstdio>
 #include <thread>
 #include <Compression/CompressionCodecDeflateQpl.h>
 #include <Compression/CompressionFactory.h>
 #include <Compression/CompressionInfo.h>
-#include <Parsers/ASTIdentifier.h>
 #include <Poco/Logger.h>
 #include <Common/logger_useful.h>
 #include "libaccel_config.h"
 #include <Common/MemorySanitizer.h>
 #include <base/scope_guard.h>
+#include <immintrin.h>
+
 
 namespace DB
 {
diff --git a/src/Compression/fuzzers/compressed_buffer_fuzzer.cpp b/src/Compression/fuzzers/compressed_buffer_fuzzer.cpp
index 1f669696fb9..bdab11ef8ab 100644
--- a/src/Compression/fuzzers/compressed_buffer_fuzzer.cpp
+++ b/src/Compression/fuzzers/compressed_buffer_fuzzer.cpp
@@ -1,4 +1,3 @@
-#include <iostream>
 #include <IO/ReadBufferFromMemory.h>
 #include <Compression/CompressedReadBuffer.h>
 #include <Common/Exception.h>
diff --git a/src/Compression/fuzzers/delta_decompress_fuzzer.cpp b/src/Compression/fuzzers/delta_decompress_fuzzer.cpp
index b039777da15..eaef1d1896c 100644
--- a/src/Compression/fuzzers/delta_decompress_fuzzer.cpp
+++ b/src/Compression/fuzzers/delta_decompress_fuzzer.cpp
@@ -1,4 +1,3 @@
-#include <iostream>
 #include <string>
 
 #include <Compression/ICompressionCodec.h>
diff --git a/src/Compression/fuzzers/double_delta_decompress_fuzzer.cpp b/src/Compression/fuzzers/double_delta_decompress_fuzzer.cpp
index f9822daa3bd..c32120bacbf 100644
--- a/src/Compression/fuzzers/double_delta_decompress_fuzzer.cpp
+++ b/src/Compression/fuzzers/double_delta_decompress_fuzzer.cpp
@@ -1,4 +1,3 @@
-#include <iostream>
 #include <string>
 
 #include <Compression/ICompressionCodec.h>
diff --git a/src/Compression/fuzzers/encrypted_decompress_fuzzer.cpp b/src/Compression/fuzzers/encrypted_decompress_fuzzer.cpp
index 3e3d0e164fe..eb95c83e1d7 100644
--- a/src/Compression/fuzzers/encrypted_decompress_fuzzer.cpp
+++ b/src/Compression/fuzzers/encrypted_decompress_fuzzer.cpp
@@ -1,6 +1,5 @@
 #include <cstddef>
 #include <cstdint>
-#include <iostream>
 #include <memory>
 #include <string>
 
diff --git a/src/Compression/fuzzers/lz4_decompress_fuzzer.cpp b/src/Compression/fuzzers/lz4_decompress_fuzzer.cpp
index 85c4c9bd329..f03fc716c2c 100644
--- a/src/Compression/fuzzers/lz4_decompress_fuzzer.cpp
+++ b/src/Compression/fuzzers/lz4_decompress_fuzzer.cpp
@@ -1,4 +1,3 @@
-#include <iostream>
 #include <string>
 
 #include <Compression/ICompressionCodec.h>
diff --git a/src/Coordination/SnapshotableHashTable.h b/src/Coordination/SnapshotableHashTable.h
index 7db546bd4c8..093126237ef 100644
--- a/src/Coordination/SnapshotableHashTable.h
+++ b/src/Coordination/SnapshotableHashTable.h
@@ -6,7 +6,7 @@
 #include <unordered_map>
 #include <list>
 #include <atomic>
-#include <iostream>
+
 
 namespace DB
 {
diff --git a/src/Coordination/SummingStateMachine.cpp b/src/Coordination/SummingStateMachine.cpp
index 4c21f1d8658..b70509fe6f0 100644
--- a/src/Coordination/SummingStateMachine.cpp
+++ b/src/Coordination/SummingStateMachine.cpp
@@ -1,5 +1,4 @@
 #include <Coordination/SummingStateMachine.h>
-#include <iostream>
 #include <cstring>
 
 namespace DB
diff --git a/src/Coordination/pathUtils.cpp b/src/Coordination/pathUtils.cpp
index afa42b4a639..25f8e25cf06 100644
--- a/src/Coordination/pathUtils.cpp
+++ b/src/Coordination/pathUtils.cpp
@@ -1,5 +1,4 @@
 #include <Coordination/pathUtils.h>
-#include <iostream>
 
 namespace DB
 {
diff --git a/src/Core/MySQL/MySQLCharset.cpp b/src/Core/MySQL/MySQLCharset.cpp
index 869941ebd84..787e4edcf88 100644
--- a/src/Core/MySQL/MySQLCharset.cpp
+++ b/src/Core/MySQL/MySQLCharset.cpp
@@ -1,6 +1,5 @@
 #include "MySQLCharset.h"
 #include "config.h"
-#include <iostream>
 #include <Common/Exception.h>
 
 #if USE_ICU
diff --git a/src/Core/fuzzers/names_and_types_fuzzer.cpp b/src/Core/fuzzers/names_and_types_fuzzer.cpp
index cc4a2920c66..94f0872fff6 100644
--- a/src/Core/fuzzers/names_and_types_fuzzer.cpp
+++ b/src/Core/fuzzers/names_and_types_fuzzer.cpp
@@ -1,4 +1,3 @@
-#include <iostream>
 #include <Core/NamesAndTypes.h>
 #include <IO/ReadBufferFromMemory.h>
 
diff --git a/src/Daemon/BaseDaemon.h b/src/Daemon/BaseDaemon.h
index 7aa1e8ad1a0..952cf61d8e0 100644
--- a/src/Daemon/BaseDaemon.h
+++ b/src/Daemon/BaseDaemon.h
@@ -2,7 +2,6 @@
 
 #include <sys/types.h>
 #include <unistd.h>
-#include <iostream>
 #include <memory>
 #include <functional>
 #include <optional>
diff --git a/src/Daemon/SentryWriter.cpp b/src/Daemon/SentryWriter.cpp
index e38d339d088..942d1e306ae 100644
--- a/src/Daemon/SentryWriter.cpp
+++ b/src/Daemon/SentryWriter.cpp
@@ -3,7 +3,6 @@
 #include <Poco/Util/Application.h>
 #include <Poco/Util/LayeredConfiguration.h>
 
-#include <base/defines.h>
 #include <base/getFQDNOrHostName.h>
 #include <base/getMemoryAmount.h>
 #include <Common/logger_useful.h>
@@ -13,7 +12,6 @@
 #include <Common/StackTrace.h>
 #include <Common/getNumberOfPhysicalCPUCores.h>
 #include <Core/ServerUUID.h>
-#include <base/hex.h>
 
 #include "config.h"
 #include "config_version.h"
diff --git a/src/Dictionaries/HashedDictionary.cpp b/src/Dictionaries/HashedDictionary.cpp
index 5f25600db8f..e336ca80856 100644
--- a/src/Dictionaries/HashedDictionary.cpp
+++ b/src/Dictionaries/HashedDictionary.cpp
@@ -25,6 +25,7 @@
 #include <Dictionaries/HierarchyDictionariesUtils.h>
 #include <Dictionaries/HashedDictionaryCollectionTraits.h>
 
+
 namespace CurrentMetrics
 {
     extern const Metric HashedDictionaryThreads;
diff --git a/src/Dictionaries/PolygonDictionaryUtils.h b/src/Dictionaries/PolygonDictionaryUtils.h
index 94b8b961577..0238ef0b2b9 100644
--- a/src/Dictionaries/PolygonDictionaryUtils.h
+++ b/src/Dictionaries/PolygonDictionaryUtils.h
@@ -13,6 +13,7 @@
 
 #include <numeric>
 
+
 namespace DB
 {
 
diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp
index cc75f18bc0d..dd33395fbfa 100644
--- a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp
+++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp
@@ -2,7 +2,6 @@
 
 #include <IO/SeekableReadBuffer.h>
 
-#include <iostream>
 #include <Disks/IO/CachedOnDiskReadBufferFromFile.h>
 #include <Disks/ObjectStorages/Cached/CachedObjectStorage.h>
 #include <IO/ReadSettings.h>
diff --git a/src/Functions/array/arrayShuffle.cpp b/src/Functions/array/arrayShuffle.cpp
index 7c20e195098..faa5ae47b29 100644
--- a/src/Functions/array/arrayShuffle.cpp
+++ b/src/Functions/array/arrayShuffle.cpp
@@ -16,6 +16,7 @@
 #include <algorithm>
 #include <numeric>
 
+
 namespace DB
 {
 
diff --git a/src/Functions/evalMLMethod.cpp b/src/Functions/evalMLMethod.cpp
index 346c8249905..4d5657f0aab 100644
--- a/src/Functions/evalMLMethod.cpp
+++ b/src/Functions/evalMLMethod.cpp
@@ -5,7 +5,6 @@
 #include <Columns/ColumnAggregateFunction.h>
 #include <Common/typeid_cast.h>
 
-#include <iostream>
 
 #include <Common/PODArray.h>
 
diff --git a/src/Functions/keyvaluepair/impl/StateHandler.h b/src/Functions/keyvaluepair/impl/StateHandler.h
index 27c1a0b44be..178974e9d36 100644
--- a/src/Functions/keyvaluepair/impl/StateHandler.h
+++ b/src/Functions/keyvaluepair/impl/StateHandler.h
@@ -2,7 +2,6 @@
 
 #include <string_view>
 
-#include <iostream>
 
 namespace DB
 {
diff --git a/src/Functions/translate.cpp b/src/Functions/translate.cpp
index 83779eee23c..836cb4de2f3 100644
--- a/src/Functions/translate.cpp
+++ b/src/Functions/translate.cpp
@@ -6,6 +6,8 @@
 #include <Common/StringUtils/StringUtils.h>
 #include <Common/UTF8Helpers.h>
 #include <Common/HashTable/HashMap.h>
+#include <numeric>
+
 
 namespace DB
 {
diff --git a/src/IO/Archives/LibArchiveReader.h b/src/IO/Archives/LibArchiveReader.h
index 700e8f70d04..ef50d9de56e 100644
--- a/src/IO/Archives/LibArchiveReader.h
+++ b/src/IO/Archives/LibArchiveReader.h
@@ -4,7 +4,6 @@
 
 #include <IO/Archives/IArchiveReader.h>
 
-#include <iostream>
 
 namespace DB
 {
diff --git a/src/IO/HTTPCommon.h b/src/IO/HTTPCommon.h
index 082491b2851..f10fd748200 100644
--- a/src/IO/HTTPCommon.h
+++ b/src/IO/HTTPCommon.h
@@ -1,6 +1,5 @@
 #pragma once
 
-#include <iostream>
 #include <memory>
 #include <mutex>
 
diff --git a/src/IO/ReadBufferFromIStream.h b/src/IO/ReadBufferFromIStream.h
index 67cc60c053f..8c3f62728b5 100644
--- a/src/IO/ReadBufferFromIStream.h
+++ b/src/IO/ReadBufferFromIStream.h
@@ -1,7 +1,5 @@
 #pragma once
 
-#include <iostream>
-
 #include <IO/ReadBuffer.h>
 #include <IO/BufferWithOwnMemory.h>
 
diff --git a/src/IO/S3/SessionAwareIOStream.h b/src/IO/S3/SessionAwareIOStream.h
index f7e42f99f51..babe52545d1 100644
--- a/src/IO/S3/SessionAwareIOStream.h
+++ b/src/IO/S3/SessionAwareIOStream.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#include <iostream>
+#include <iosfwd>
 
 
 namespace DB::S3
diff --git a/src/IO/StdStreamFromReadBuffer.h b/src/IO/StdStreamFromReadBuffer.h
index eae939a28b5..ff327dc342e 100644
--- a/src/IO/StdStreamFromReadBuffer.h
+++ b/src/IO/StdStreamFromReadBuffer.h
@@ -1,7 +1,6 @@
 #pragma once
 
 #include <IO/StdStreamBufFromReadBuffer.h>
-#include <iostream>
 #include <memory>
 
 
diff --git a/src/IO/VarInt.h b/src/IO/VarInt.h
index 2a2743e3407..8d10055a3df 100644
--- a/src/IO/VarInt.h
+++ b/src/IO/VarInt.h
@@ -1,6 +1,5 @@
 #pragma once
 
-#include <iostream>
 #include <base/types.h>
 #include <base/defines.h>
 #include <IO/ReadBuffer.h>
diff --git a/src/IO/WriteBuffer.h b/src/IO/WriteBuffer.h
index ea032302235..d29ca6d5c6c 100644
--- a/src/IO/WriteBuffer.h
+++ b/src/IO/WriteBuffer.h
@@ -2,7 +2,6 @@
 
 #include <algorithm>
 #include <memory>
-#include <iostream>
 #include <cassert>
 #include <cstring>
 
diff --git a/src/IO/WriteBufferFromOStream.h b/src/IO/WriteBufferFromOStream.h
index 5a933739cb1..3f9d3ee3d92 100644
--- a/src/IO/WriteBufferFromOStream.h
+++ b/src/IO/WriteBufferFromOStream.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#include <iostream>
+#include <iosfwd>
 
 #include <IO/WriteBuffer.h>
 #include <IO/BufferWithOwnMemory.h>
diff --git a/src/IO/examples/read_buffer_from_hdfs.cpp b/src/IO/examples/read_buffer_from_hdfs.cpp
index da4e5298681..977dd2ae227 100644
--- a/src/IO/examples/read_buffer_from_hdfs.cpp
+++ b/src/IO/examples/read_buffer_from_hdfs.cpp
@@ -1,4 +1,3 @@
-#include <iostream>
 #include <memory>
 #include <string>
 #include <IO/WriteBufferFromFile.h>
diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp
index a210a9efbc7..8f98759f592 100644
--- a/src/Interpreters/Context.cpp
+++ b/src/Interpreters/Context.cpp
@@ -2,14 +2,10 @@
 #include <set>
 #include <optional>
 #include <memory>
-#include <Poco/Mutex.h>
 #include <Poco/UUID.h>
-#include <Poco/Net/IPAddress.h>
 #include <Poco/Util/Application.h>
 #include <Common/Macros.h>
-#include <Common/escapeForFileName.h>
 #include <Common/EventNotifier.h>
-#include <Common/setThreadName.h>
 #include <Common/Stopwatch.h>
 #include <Common/formatReadable.h>
 #include <Common/Throttler.h>
@@ -17,12 +13,10 @@
 #include <Common/FieldVisitorToString.h>
 #include <Common/getMultipleKeysFromConfig.h>
 #include <Coordination/KeeperDispatcher.h>
-#include <Compression/ICompressionCodec.h>
 #include <Core/BackgroundSchedulePool.h>
 #include <Formats/FormatFactory.h>
 #include <Databases/IDatabase.h>
 #include <Server/ServerType.h>
-#include <Storages/IStorage.h>
 #include <Storages/MarkCache.h>
 #include <Storages/MergeTree/MergeList.h>
 #include <Storages/MergeTree/MovesList.h>
@@ -34,8 +28,6 @@
 #include <Disks/DiskLocal.h>
 #include <Disks/ObjectStorages/DiskObjectStorage.h>
 #include <Disks/ObjectStorages/IObjectStorage.h>
-#include <Disks/IO/ThreadPoolRemoteFSReader.h>
-#include <Disks/IO/ThreadPoolReader.h>
 #include <Disks/StoragePolicy.h>
 #include <IO/SynchronousReader.h>
 #include <TableFunctions/TableFunctionFactory.h>
@@ -44,7 +36,6 @@
 #include <Interpreters/TemporaryDataOnDisk.h>
 #include <Interpreters/Cache/QueryCache.h>
 #include <Interpreters/Cache/FileCacheFactory.h>
-#include <Interpreters/Cache/FileCache.h>
 #include <Interpreters/SessionTracker.h>
 #include <Core/ServerSettings.h>
 #include <Interpreters/PreparedSets.h>
@@ -56,7 +47,6 @@
 #include <Access/EnabledRowPolicies.h>
 #include <Access/QuotaUsage.h>
 #include <Access/User.h>
-#include <Access/Credentials.h>
 #include <Access/SettingsProfile.h>
 #include <Access/SettingsProfilesInfo.h>
 #include <Access/SettingsConstraintsAndProfileIDs.h>
@@ -70,7 +60,6 @@
 #include <Functions/UserDefined/ExternalUserDefinedExecutableFunctionsLoader.h>
 #include <Functions/UserDefined/IUserDefinedSQLObjectsLoader.h>
 #include <Functions/UserDefined/createUserDefinedSQLObjectsLoader.h>
-#include <Interpreters/ExpressionActions.h>
 #include <Interpreters/ProcessList.h>
 #include <Interpreters/InterserverCredentials.h>
 #include <Interpreters/Cluster.h>
@@ -87,8 +76,6 @@
 #include <IO/MMappedFileCache.h>
 #include <IO/WriteSettings.h>
 #include <Parsers/ASTCreateQuery.h>
-#include <Parsers/ParserCreateQuery.h>
-#include <Parsers/parseQuery.h>
 #include <Parsers/ASTAsterisk.h>
 #include <Parsers/ASTIdentifier.h>
 #include <Common/StackTrace.h>
@@ -98,14 +85,12 @@
 #include <Common/ZooKeeper/ZooKeeper.h>
 #include <Common/ShellCommand.h>
 #include <Common/logger_useful.h>
-#include <base/EnumReflection.h>
 #include <Common/RemoteHostFilter.h>
 #include <Common/HTTPHeaderFilter.h>
 #include <Interpreters/AsynchronousInsertQueue.h>
 #include <Interpreters/DatabaseCatalog.h>
 #include <Interpreters/JIT/CompiledExpressionCache.h>
 #include <Storages/MergeTree/BackgroundJobsAssignee.h>
-#include <Storages/MergeTree/MergeTreeBackgroundExecutor.h>
 #include <Storages/MergeTree/MergeTreeDataPartUUID.h>
 #include <Storages/MergeTree/MergeTreeMetadataCache.h>
 #include <Interpreters/SynonymsExtensions.h>
@@ -118,12 +103,8 @@
 #include <Parsers/ASTFunction.h>
 #include <Parsers/FunctionParameterValuesVisitor.h>
 #include <Parsers/ASTSelectWithUnionQuery.h>
-#include <base/find_symbols.h>
 #include <Interpreters/InterpreterSelectWithUnionQuery.h>
 
-#if USE_ROCKSDB
-#include <rocksdb/table.h>
-#endif
 
 namespace fs = std::filesystem;
 
diff --git a/src/Interpreters/GraceHashJoin.cpp b/src/Interpreters/GraceHashJoin.cpp
index 5d72cf20740..89ea3a326cc 100644
--- a/src/Interpreters/GraceHashJoin.cpp
+++ b/src/Interpreters/GraceHashJoin.cpp
@@ -1,15 +1,11 @@
 #include <Interpreters/GraceHashJoin.h>
 #include <Interpreters/HashJoin.h>
 #include <Interpreters/TableJoin.h>
-#include <Interpreters/Context.h>
 
 #include <Formats/NativeWriter.h>
 #include <Interpreters/TemporaryDataOnDisk.h>
 
 #include <Compression/CompressedWriteBuffer.h>
-#include <Core/ProtocolDefines.h>
-#include <Disks/IVolume.h>
-#include <Disks/TemporaryFileOnDisk.h>
 #include <Common/logger_useful.h>
 #include <Common/thread_local_rng.h>
 
@@ -18,6 +14,9 @@
 
 #include <Formats/formatBlock.h>
 
+#include <numeric>
+
+
 namespace CurrentMetrics
 {
     extern const Metric TemporaryFilesForJoin;
diff --git a/src/Interpreters/InterpreterKillQueryQuery.cpp b/src/Interpreters/InterpreterKillQueryQuery.cpp
index 3330159aff5..82fe2f57a0e 100644
--- a/src/Interpreters/InterpreterKillQueryQuery.cpp
+++ b/src/Interpreters/InterpreterKillQueryQuery.cpp
@@ -24,7 +24,6 @@
 #include <Storages/IStorage.h>
 #include <Common/quoteString.h>
 #include <thread>
-#include <iostream>
 #include <cstddef>
 
 
diff --git a/src/Loggers/OwnSplitChannel.cpp b/src/Loggers/OwnSplitChannel.cpp
index cdf8402745f..618d453c238 100644
--- a/src/Loggers/OwnSplitChannel.cpp
+++ b/src/Loggers/OwnSplitChannel.cpp
@@ -1,7 +1,6 @@
 #include "OwnSplitChannel.h"
 #include "OwnFormattingChannel.h"
 
-#include <iostream>
 #include <Core/Block.h>
 #include <Interpreters/InternalTextLogsQueue.h>
 #include <Interpreters/TextLog.h>
diff --git a/src/Parsers/ASTSelectWithUnionQuery.cpp b/src/Parsers/ASTSelectWithUnionQuery.cpp
index 9550752b1f3..48b4ae3c38d 100644
--- a/src/Parsers/ASTSelectWithUnionQuery.cpp
+++ b/src/Parsers/ASTSelectWithUnionQuery.cpp
@@ -5,7 +5,6 @@
 #include <IO/Operators.h>
 #include <Parsers/ASTSelectQuery.h>
 
-#include <iostream>
 
 namespace DB
 {
diff --git a/src/Processors/Executors/StreamingFormatExecutor.cpp b/src/Processors/Executors/StreamingFormatExecutor.cpp
index 2223721439e..46818989032 100644
--- a/src/Processors/Executors/StreamingFormatExecutor.cpp
+++ b/src/Processors/Executors/StreamingFormatExecutor.cpp
@@ -1,6 +1,5 @@
 #include <Processors/Executors/StreamingFormatExecutor.h>
 #include <Processors/Transforms/AddingDefaultsTransform.h>
-#include <iostream>
 
 namespace DB
 {
diff --git a/src/Processors/IAccumulatingTransform.cpp b/src/Processors/IAccumulatingTransform.cpp
index 00d5b2ee089..ea3c3c2c1b0 100644
--- a/src/Processors/IAccumulatingTransform.cpp
+++ b/src/Processors/IAccumulatingTransform.cpp
@@ -1,5 +1,4 @@
 #include <Processors/IAccumulatingTransform.h>
-#include <iostream>
 
 namespace DB
 {
diff --git a/src/Processors/ResizeProcessor.cpp b/src/Processors/ResizeProcessor.cpp
index 8167fae9baf..3a8c6fb2bff 100644
--- a/src/Processors/ResizeProcessor.cpp
+++ b/src/Processors/ResizeProcessor.cpp
@@ -1,5 +1,4 @@
 #include <Processors/ResizeProcessor.h>
-#include <iostream>
 
 namespace DB
 {
diff --git a/src/Processors/Transforms/CountingTransform.cpp b/src/Processors/Transforms/CountingTransform.cpp
index 646256d60c0..3dfb9fe178f 100644
--- a/src/Processors/Transforms/CountingTransform.cpp
+++ b/src/Processors/Transforms/CountingTransform.cpp
@@ -1,4 +1,3 @@
-#include <iostream>
 
 #include <Interpreters/ProcessList.h>
 #include <Processors/Transforms/CountingTransform.h>
diff --git a/src/Processors/Transforms/ExceptionKeepingTransform.cpp b/src/Processors/Transforms/ExceptionKeepingTransform.cpp
index 3c40c078225..b50f66b0240 100644
--- a/src/Processors/Transforms/ExceptionKeepingTransform.cpp
+++ b/src/Processors/Transforms/ExceptionKeepingTransform.cpp
@@ -2,7 +2,6 @@
 #include <Common/ThreadStatus.h>
 #include <Common/Stopwatch.h>
 #include <base/scope_guard.h>
-#include <iostream>
 
 namespace DB
 {
diff --git a/src/Processors/Transforms/SquashingChunksTransform.cpp b/src/Processors/Transforms/SquashingChunksTransform.cpp
index e89aec31655..7de9538e435 100644
--- a/src/Processors/Transforms/SquashingChunksTransform.cpp
+++ b/src/Processors/Transforms/SquashingChunksTransform.cpp
@@ -1,5 +1,4 @@
 #include <Processors/Transforms/SquashingChunksTransform.h>
-#include <iostream>
 
 namespace DB
 {
diff --git a/src/Server/HTTP/HTTPServerResponse.h b/src/Server/HTTP/HTTPServerResponse.h
index f5b7a70dc79..236a56e2323 100644
--- a/src/Server/HTTP/HTTPServerResponse.h
+++ b/src/Server/HTTP/HTTPServerResponse.h
@@ -5,9 +5,9 @@
 #include <Poco/Net/HTTPServerSession.h>
 #include <Poco/Net/HTTPResponse.h>
 
-#include <iostream>
 #include <memory>
 
+
 namespace DB
 {
 
diff --git a/src/Storages/MergeTree/GinIndexStore.cpp b/src/Storages/MergeTree/GinIndexStore.cpp
index 91e831270d4..5b798ecc8a5 100644
--- a/src/Storages/MergeTree/GinIndexStore.cpp
+++ b/src/Storages/MergeTree/GinIndexStore.cpp
@@ -11,7 +11,6 @@
 #include <IO/WriteHelpers.h>
 #include <vector>
 #include <unordered_map>
-#include <iostream>
 #include <numeric>
 #include <algorithm>
 
diff --git a/src/Storages/fuzzers/mergetree_checksum_fuzzer.cpp b/src/Storages/fuzzers/mergetree_checksum_fuzzer.cpp
index 9a5a68f09cc..0f2ce8a2e44 100644
--- a/src/Storages/fuzzers/mergetree_checksum_fuzzer.cpp
+++ b/src/Storages/fuzzers/mergetree_checksum_fuzzer.cpp
@@ -1,4 +1,3 @@
-#include <iostream>
 
 #include <IO/ReadBufferFromMemory.h>
 #include <IO/WriteBufferFromFileDescriptor.h>

From d8a231e0cbcad7b1dc1697fb2733f5bae0b145de Mon Sep 17 00:00:00 2001
From: Yakov Olkhovskiy <yakov@clickhouse.com>
Date: Wed, 9 Aug 2023 00:44:01 +0000
Subject: [PATCH 127/249] Add Linux s390x build to CI

---
 .github/workflows/master.yml       | 43 ++++++++++++++++++++++++++++++
 .github/workflows/pull_request.yml | 42 +++++++++++++++++++++++++++++
 docker/packager/packager           |  7 +++++
 tests/ci/ci_config.py              |  6 +++++
 4 files changed, 98 insertions(+)

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index f25611e8666..284d94b5c5a 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -895,6 +895,48 @@ jobs:
           docker ps --quiet | xargs --no-run-if-empty docker kill ||:
           docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
+  BuilderBinS390X:
+    needs: [DockerHubPush]
+    runs-on: [self-hosted, builder]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/build_check
+          IMAGES_PATH=${{runner.temp}}/images_path
+          REPO_COPY=${{runner.temp}}/build_check/ClickHouse
+          CACHES_PATH=${{runner.temp}}/../ccaches
+          BUILD_NAME=binary_s390x
+          EOF
+      - name: Download changed images
+        uses: actions/download-artifact@v3
+        with:
+          name: changed_images
+          path: ${{ env.IMAGES_PATH }}
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+          submodules: true
+          fetch-depth: 0 # otherwise we will have no info about contributors
+      - name: Build
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
+      - name: Upload build URLs to artifacts
+        if: ${{ success() || failure() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
+      - name: Cleanup
+        if: always()
+        run: |
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+          sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
 ############################################################################################
 ##################################### Docker images  #######################################
 ############################################################################################
@@ -978,6 +1020,7 @@ jobs:
       - BuilderBinFreeBSD
       - BuilderBinPPC64
       - BuilderBinRISCV64
+      - BuilderBinS390X
       - BuilderBinAmd64Compat
       - BuilderBinAarch64V80Compat
       - BuilderBinClangTidy
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index 25927954809..bd3e5156a77 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -955,6 +955,47 @@ jobs:
           docker ps --quiet | xargs --no-run-if-empty docker kill ||:
           docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
+  BuilderBinS390X:
+    needs: [DockerHubPush, FastTest, StyleCheck]
+    runs-on: [self-hosted, builder]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/build_check
+          IMAGES_PATH=${{runner.temp}}/images_path
+          REPO_COPY=${{runner.temp}}/build_check/ClickHouse
+          CACHES_PATH=${{runner.temp}}/../ccaches
+          BUILD_NAME=binary_s390x
+          EOF
+      - name: Download changed images
+        uses: actions/download-artifact@v3
+        with:
+          name: changed_images
+          path: ${{ env.IMAGES_PATH }}
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+          submodules: true
+      - name: Build
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
+      - name: Upload build URLs to artifacts
+        if: ${{ success() || failure() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
+      - name: Cleanup
+        if: always()
+        run: |
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+          sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
 ############################################################################################
 ##################################### Docker images  #######################################
 ############################################################################################
@@ -1037,6 +1078,7 @@ jobs:
       - BuilderBinFreeBSD
       - BuilderBinPPC64
       - BuilderBinRISCV64
+      - BuilderBinS390X
       - BuilderBinAmd64Compat
       - BuilderBinAarch64V80Compat
       - BuilderBinClangTidy
diff --git a/docker/packager/packager b/docker/packager/packager
index 9f949f63eda..5a970a1d79d 100755
--- a/docker/packager/packager
+++ b/docker/packager/packager
@@ -142,6 +142,7 @@ def parse_env_variables(
     FREEBSD_SUFFIX = "-freebsd"
     PPC_SUFFIX = "-ppc64le"
     RISCV_SUFFIX = "-riscv64"
+    S390X_SUFFIX = "-s390x"
     AMD64_COMPAT_SUFFIX = "-amd64-compat"
 
     result = []
@@ -216,6 +217,11 @@ def parse_env_variables(
         cmake_flags.append(
             "-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-riscv64.cmake"
         )
+    elif is_cross_s390x:
+        cc = compiler[: -len(S390X_SUFFIX)]
+        cmake_flags.append(
+            "-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-s390x.cmake"
+        )
     elif is_amd64_compat:
         cc = compiler[: -len(AMD64_COMPAT_SUFFIX)]
         result.append("DEB_ARCH=amd64")
@@ -377,6 +383,7 @@ def parse_args() -> argparse.Namespace:
             "clang-16-aarch64-v80compat",
             "clang-16-ppc64le",
             "clang-16-riscv64",
+            "clang-16-s390x",
             "clang-16-amd64-compat",
             "clang-16-freebsd",
         ),
diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py
index 979f7c35c2c..206037bdfdf 100644
--- a/tests/ci/ci_config.py
+++ b/tests/ci/ci_config.py
@@ -177,6 +177,11 @@ CI_CONFIG = CiConfig(
             package_type="binary",
             static_binary_name="riscv64",
         ),
+        "binary_s390x": BuildConfig(
+            compiler="clang-16-s390x",
+            package_type="binary",
+            static_binary_name="s390x",
+        ),
     },
     builds_report_config={
         "ClickHouse build check": [
@@ -198,6 +203,7 @@ CI_CONFIG = CiConfig(
             "binary_darwin_aarch64",
             "binary_ppc64le",
             "binary_riscv64",
+            "binary_s390x",
             "binary_amd64_compat",
         ],
     },

From 1c9cff551e4b9fe3e221cb5f7a12dcc01ff9ed90 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 9 Aug 2023 02:58:12 +0200
Subject: [PATCH 128/249] Continue on garbage removal

---
 base/base/EnumReflection.h                             |  1 +
 src/Common/LoggingFormatStringHelpers.h                |  3 +++
 .../NamedCollections/NamedCollectionConfiguration.cpp  |  2 ++
 src/Core/Field.cpp                                     |  8 ++++++++
 src/Core/Field.h                                       |  5 ++---
 src/Core/SettingsFields.h                              |  2 ++
 src/IO/S3/Credentials.cpp                              | 10 ++++++----
 7 files changed, 24 insertions(+), 7 deletions(-)

diff --git a/base/base/EnumReflection.h b/base/base/EnumReflection.h
index 0d1f8ae0a40..e3208f16a75 100644
--- a/base/base/EnumReflection.h
+++ b/base/base/EnumReflection.h
@@ -3,6 +3,7 @@
 #include <magic_enum.hpp>
 #include <fmt/format.h>
 
+
 template <class T> concept is_enum = std::is_enum_v<T>;
 
 namespace detail
diff --git a/src/Common/LoggingFormatStringHelpers.h b/src/Common/LoggingFormatStringHelpers.h
index 6dffd202807..06320509c17 100644
--- a/src/Common/LoggingFormatStringHelpers.h
+++ b/src/Common/LoggingFormatStringHelpers.h
@@ -1,4 +1,5 @@
 #pragma once
+
 #include <base/defines.h>
 #include <base/types.h>
 #include <fmt/format.h>
@@ -6,6 +7,8 @@
 #include <unordered_map>
 #include <Poco/Logger.h>
 #include <Poco/Message.h>
+#include <base/EnumReflection.h>
+
 
 struct PreformattedMessage;
 consteval void formatStringCheckArgsNumImpl(std::string_view str, size_t nargs);
diff --git a/src/Common/NamedCollections/NamedCollectionConfiguration.cpp b/src/Common/NamedCollections/NamedCollectionConfiguration.cpp
index 1c42b001ceb..d9bf2349a5a 100644
--- a/src/Common/NamedCollections/NamedCollectionConfiguration.cpp
+++ b/src/Common/NamedCollections/NamedCollectionConfiguration.cpp
@@ -3,6 +3,8 @@
 #include <Common/Exception.h>
 #include <Common/SettingsChanges.h>
 #include <Common/FieldVisitorToString.h>
+#include <magic_enum.hpp>
+
 
 namespace DB
 {
diff --git a/src/Core/Field.cpp b/src/Core/Field.cpp
index 90dcb2e5f8e..89faaed8a72 100644
--- a/src/Core/Field.cpp
+++ b/src/Core/Field.cpp
@@ -9,16 +9,24 @@
 #include <Common/FieldVisitorDump.h>
 #include <Common/FieldVisitorToString.h>
 #include <Common/FieldVisitorWriteBinary.h>
+#include <base/EnumReflection.h>
 
 
 namespace DB
 {
+
 namespace ErrorCodes
 {
     extern const int CANNOT_RESTORE_FROM_FIELD_DUMP;
     extern const int DECIMAL_OVERFLOW;
 }
 
+/// Keep in mind, that "magic_enum" is very expensive for compiler.
+std::string_view Field::getTypeName() const
+{
+    return magic_enum::enum_name(which);
+}
+
 inline Field getBinaryValue(UInt8 type, ReadBuffer & buf)
 {
     switch (type)
diff --git a/src/Core/Field.h b/src/Core/Field.h
index ef1bd9a895d..6666e66e8d5 100644
--- a/src/Core/Field.h
+++ b/src/Core/Field.h
@@ -15,8 +15,7 @@
 #include <Core/UUID.h>
 #include <base/IPv4andIPv6.h>
 #include <base/DayNum.h>
-#include <base/strong_typedef.h>
-#include <base/EnumReflection.h>
+
 
 namespace DB
 {
@@ -449,7 +448,7 @@ public:
 
     Types::Which getType() const { return which; }
 
-    constexpr std::string_view getTypeName() const { return magic_enum::enum_name(which); }
+    std::string_view getTypeName() const;
 
     bool isNull() const { return which == Types::Null; }
     template <typename T>
diff --git a/src/Core/SettingsFields.h b/src/Core/SettingsFields.h
index 32fffd3af06..db14f91512f 100644
--- a/src/Core/SettingsFields.h
+++ b/src/Core/SettingsFields.h
@@ -10,6 +10,8 @@
 #include <chrono>
 #include <unordered_map>
 #include <string_view>
+#include <magic_enum.hpp>
+
 
 namespace DB
 {
diff --git a/src/IO/S3/Credentials.cpp b/src/IO/S3/Credentials.cpp
index 84550d3f7c5..d59bc4fd115 100644
--- a/src/IO/S3/Credentials.cpp
+++ b/src/IO/S3/Credentials.cpp
@@ -14,26 +14,26 @@
 #    include <Common/logger_useful.h>
 
 #    include <IO/S3/PocoHTTPClient.h>
-#    include <IO/S3/PocoHTTPClientFactory.h>
 #    include <IO/S3/Client.h>
-#    include <IO/S3Common.h>
 
 #    include <fstream>
+#    include <base/EnumReflection.h>
 
 
 namespace DB
 {
+
 namespace ErrorCodes
 {
     extern const int AWS_ERROR;
 }
-}
 
-namespace DB::S3
+namespace S3
 {
 
 namespace
 {
+
 bool areCredentialsEmptyOrExpired(const Aws::Auth::AWSCredentials & credentials, uint64_t expiration_window_seconds)
 {
     if (credentials.IsEmpty())
@@ -569,4 +569,6 @@ S3CredentialsProviderChain::S3CredentialsProviderChain(
 
 }
 
+}
+
 #endif

From 2f6e66c8e3809d80bcab0afe7bf5d9d03761640d Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 9 Aug 2023 03:19:24 +0200
Subject: [PATCH 129/249] Remove unnecessary templates

---
 src/Functions/array/arrayAUC.cpp | 122 +++++++++++++++++++++++--------
 1 file changed, 90 insertions(+), 32 deletions(-)

diff --git a/src/Functions/array/arrayAUC.cpp b/src/Functions/array/arrayAUC.cpp
index 4d2b8175f5b..a0aca7a563b 100644
--- a/src/Functions/array/arrayAUC.cpp
+++ b/src/Functions/array/arrayAUC.cpp
@@ -1,7 +1,9 @@
-#include <algorithm>
 #include <DataTypes/DataTypesNumber.h>
+#include <DataTypes/DataTypeArray.h>
+#include <Columns/ColumnVector.h>
+#include <Columns/ColumnArray.h>
+#include <Functions/FunctionHelpers.h>
 #include <Functions/FunctionFactory.h>
-#include "arrayScalarProduct.h"
 
 
 namespace DB
@@ -10,6 +12,8 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+    extern const int ILLEGAL_COLUMN;
+    extern const int BAD_ARGUMENTS;
 }
 
 
@@ -70,44 +74,32 @@ namespace ErrorCodes
   * The "curve" will be present by a line that moves one step either towards right or top on each threshold change.
   */
 
-
-struct NameArrayAUC
-{
-    static constexpr auto name = "arrayAUC";
-};
-
-
-class ArrayAUCImpl
+class FunctionArrayAUC : public IFunction
 {
 public:
-    using ResultType = Float64;
+    static constexpr auto name = "arrayAUC";
+    static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionArrayAUC>(); }
 
-    static DataTypePtr getReturnType(const DataTypePtr & /* score_type */, const DataTypePtr & label_type)
-    {
-        if (!(isNumber(label_type) || isEnum(label_type)))
-            throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "{} label must have numeric type.", std::string(NameArrayAUC::name));
-
-        return std::make_shared<DataTypeNumber<ResultType>>();
-    }
-
-    template <typename ResultType, typename T, typename U>
-    static ResultType apply(
-        const T * scores,
-        const U * labels,
-        size_t size)
+private:
+    static Float64 apply(
+        const IColumn & data1,
+        const IColumn & data2,
+        ColumnArray::Offset current_offset,
+        ColumnArray::Offset next_offset)
     {
         struct ScoreLabel
         {
-            T score;
+            Float64 score;
             bool label;
         };
 
+        size_t size = next_offset - current_offset;
         PODArrayWithStackMemory<ScoreLabel, 1024> sorted_labels(size);
 
         for (size_t i = 0; i < size; ++i)
         {
-            bool label = labels[i] > 0;
-            sorted_labels[i].score = scores[i];
+            bool label = data2.getFloat64(current_offset + i) > 0;
+            sorted_labels[i].score = data1.getFloat64(current_offset + i);
             sorted_labels[i].label = label;
         }
 
@@ -129,18 +121,84 @@ public:
         /// Then divide the area to the area of rectangle.
 
         if (count_positive == 0 || count_positive == size)
-            return std::numeric_limits<ResultType>::quiet_NaN();
+            return std::numeric_limits<Float64>::quiet_NaN();
 
-        return static_cast<ResultType>(area) / count_positive / (size - count_positive);
+        return static_cast<Float64>(area) / count_positive / (size - count_positive);
+    }
+
+    static void vector(
+        const IColumn & data1,
+        const IColumn & data2,
+        const ColumnArray::Offsets & offsets,
+        PaddedPODArray<Float64> & result)
+    {
+        size_t size = offsets.size();
+        result.resize(size);
+
+        ColumnArray::Offset current_offset = 0;
+        for (size_t i = 0; i < size; ++i)
+        {
+            auto next_offset = offsets[i];
+            result[i] = apply(data1, data2, current_offset, next_offset);
+            current_offset = next_offset;
+        }
+    }
+
+public:
+    String getName() const override { return name; }
+    size_t getNumberOfArguments() const override { return 2; }
+    bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override { return false; }
+
+    DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
+    {
+        // Basic type check
+        std::vector<DataTypePtr> nested_types(2, nullptr);
+        for (size_t i = 0; i < getNumberOfArguments(); ++i)
+        {
+            const DataTypeArray * array_type = checkAndGetDataType<DataTypeArray>(arguments[i].get());
+            if (!array_type)
+                throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "All arguments for function {} must be an array.", getName());
+
+            const auto & nested_type = array_type->getNestedType();
+            if (!isNativeNumber(nested_type) && !isEnum(nested_type))
+                throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "{} cannot process values of type {}",
+                                getName(), nested_type->getName());
+            nested_types[i] = nested_type;
+        }
+
+        return std::make_shared<DataTypeFloat64>();
+    }
+
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t) const override
+    {
+        ColumnPtr col1 = arguments[0].column->convertToFullColumnIfConst();
+        ColumnPtr col2 = arguments[1].column->convertToFullColumnIfConst();
+
+        const ColumnArray * col_array1 = checkAndGetColumn<ColumnArray>(col1.get());
+        const ColumnArray * col_array2 = checkAndGetColumn<ColumnArray>(col2.get());
+        if (!col_array1 || !col_array2)
+            throw Exception(ErrorCodes::ILLEGAL_COLUMN,
+                "Illegal column {} of first argument of function {}", arguments[0].column->getName(), getName());
+
+        if (!col_array1->hasEqualOffsets(*col_array2))
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Array arguments for function {} must have equal sizes", getName());
+
+        auto col_res = ColumnVector<Float64>::create();
+
+        vector(
+            col_array1->getData(),
+            col_array2->getData(),
+            col_array1->getOffsets(),
+            col_res->getData());
+
+        return col_res;
     }
 };
 
 
-/// auc(array_score, array_label) - Calculate AUC with array of score and label
-using FunctionArrayAUC = FunctionArrayScalarProduct<ArrayAUCImpl, NameArrayAUC>;
-
 REGISTER_FUNCTION(ArrayAUC)
 {
     factory.registerFunction<FunctionArrayAUC>();
 }
+
 }

From 4d02a924e60979c688c27822ab785bd1aca62068 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 9 Aug 2023 03:29:07 +0200
Subject: [PATCH 130/249] Fix AArch64

---
 base/base/wide_integer_impl.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/base/base/wide_integer_impl.h b/base/base/wide_integer_impl.h
index 278d62d38a1..fc4e9e551ca 100644
--- a/base/base/wide_integer_impl.h
+++ b/base/base/wide_integer_impl.h
@@ -21,6 +21,7 @@
 #define CONSTEXPR_FROM_DOUBLE constexpr
 using FromDoubleIntermediateType = long double;
 #else
+#include <boost/multiprecision/cpp_bin_float.hpp>
 /// `wide_integer_from_builtin` can't be constexpr with non-literal `cpp_bin_float_double_extended`
 #define CONSTEXPR_FROM_DOUBLE
 using FromDoubleIntermediateType = boost::multiprecision::cpp_bin_float_double_extended;

From b5273e75bc747e171d87c96f9db53560b76bf9a3 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 9 Aug 2023 03:29:07 +0200
Subject: [PATCH 131/249] Fix AArch64

---
 base/base/wide_integer_impl.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/base/base/wide_integer_impl.h b/base/base/wide_integer_impl.h
index 278d62d38a1..fc4e9e551ca 100644
--- a/base/base/wide_integer_impl.h
+++ b/base/base/wide_integer_impl.h
@@ -21,6 +21,7 @@
 #define CONSTEXPR_FROM_DOUBLE constexpr
 using FromDoubleIntermediateType = long double;
 #else
+#include <boost/multiprecision/cpp_bin_float.hpp>
 /// `wide_integer_from_builtin` can't be constexpr with non-literal `cpp_bin_float_double_extended`
 #define CONSTEXPR_FROM_DOUBLE
 using FromDoubleIntermediateType = boost::multiprecision::cpp_bin_float_double_extended;

From 29221188baaa6b601824f47319ad888462dbf296 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 9 Aug 2023 04:07:31 +0200
Subject: [PATCH 132/249] Fix error

---
 src/Formats/ReadSchemaUtils.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/Formats/ReadSchemaUtils.cpp b/src/Formats/ReadSchemaUtils.cpp
index d183442ba44..b40fb20dadf 100644
--- a/src/Formats/ReadSchemaUtils.cpp
+++ b/src/Formats/ReadSchemaUtils.cpp
@@ -217,6 +217,8 @@ try
 }
 catch (Exception & e)
 {
+    if (!buf)
+        throw;
     auto file_name = getFileNameFromReadBuffer(*buf);
     if (!file_name.empty())
         e.addMessage(fmt::format("(in file/uri {})", file_name));

From d6d6569dc72948b18ae59bcc593df942f387aaac Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 9 Aug 2023 04:28:24 +0200
Subject: [PATCH 133/249] Fix upgrade check

---
 src/Common/SystemLogBase.h     |  4 ++++
 src/Interpreters/SystemLog.cpp | 11 +++++++----
 src/Interpreters/SystemLog.h   |  2 +-
 3 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/src/Common/SystemLogBase.h b/src/Common/SystemLogBase.h
index 28ccc91b014..1f5832e7aac 100644
--- a/src/Common/SystemLogBase.h
+++ b/src/Common/SystemLogBase.h
@@ -33,13 +33,17 @@
 
 namespace Poco
 {
+
 class Logger;
+
 namespace Util
 {
     class AbstractConfiguration;
 }
+
 }
 
+
 namespace DB
 {
 
diff --git a/src/Interpreters/SystemLog.cpp b/src/Interpreters/SystemLog.cpp
index ddb8560c094..6c5576b07d7 100644
--- a/src/Interpreters/SystemLog.cpp
+++ b/src/Interpreters/SystemLog.cpp
@@ -386,7 +386,7 @@ SystemLog<LogElement>::SystemLog(
     assert(settings_.queue_settings.database == DatabaseCatalog::SYSTEM_DATABASE);
 
     if (settings_.create_at_startup)
-        prepareTable();
+        prepareTable(true);
 }
 
 template <typename LogElement>
@@ -442,7 +442,7 @@ void SystemLog<LogElement>::savingThreadFunction()
             {
                 if (should_prepare_tables_anyway)
                 {
-                    prepareTable();
+                    prepareTable(false);
                     LOG_TRACE(log, "Table created (force)");
 
                     queue->confirm(to_flush_end);
@@ -474,7 +474,7 @@ void SystemLog<LogElement>::flushImpl(const std::vector<LogElement> & to_flush,
         /// flush. This is done to allow user to drop the table at any moment
         /// (new empty table will be created automatically). BTW, flush method
         /// is called from single thread.
-        prepareTable();
+        prepareTable(false);
 
         ColumnsWithTypeAndName log_element_columns;
         auto log_element_names_and_types = LogElement::getNamesAndTypes();
@@ -524,13 +524,16 @@ void SystemLog<LogElement>::flushImpl(const std::vector<LogElement> & to_flush,
 
 
 template <typename LogElement>
-void SystemLog<LogElement>::prepareTable()
+void SystemLog<LogElement>::prepareTable(bool if_not_exists)
 {
     String description = table_id.getNameForLogs();
 
     auto table = DatabaseCatalog::instance().tryGetTable(table_id, getContext());
     if (table)
     {
+        if (if_not_exists)
+            return;
+
         if (old_create_query.empty())
         {
             old_create_query = serializeAST(*getCreateTableQueryClean(table_id, getContext()));
diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h
index a1401707568..bc7ce8d1040 100644
--- a/src/Interpreters/SystemLog.h
+++ b/src/Interpreters/SystemLog.h
@@ -147,7 +147,7 @@ private:
       * Renames old table if its structure is not suitable.
       * This cannot be done in constructor to avoid deadlock while renaming a table under locked Context when SystemLog object is created.
       */
-    void prepareTable() override;
+    void prepareTable(bool if_not_exists) override;
 
     void savingThreadFunction() override;
 

From c6c6d1ed9c69a6e443c709bc37cc120d29ae03fd Mon Sep 17 00:00:00 2001
From: Yakov Olkhovskiy <yakov@clickhouse.com>
Date: Wed, 9 Aug 2023 02:42:04 +0000
Subject: [PATCH 134/249] fix

---
 docker/packager/packager | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker/packager/packager b/docker/packager/packager
index 5a970a1d79d..0d11358ea7c 100755
--- a/docker/packager/packager
+++ b/docker/packager/packager
@@ -156,6 +156,7 @@ def parse_env_variables(
     is_cross_arm_v80compat = compiler.endswith(ARM_V80COMPAT_SUFFIX)
     is_cross_ppc = compiler.endswith(PPC_SUFFIX)
     is_cross_riscv = compiler.endswith(RISCV_SUFFIX)
+    is_cross_s390x = compiler.endswith(S390X_SUFFIX)
     is_cross_freebsd = compiler.endswith(FREEBSD_SUFFIX)
     is_amd64_compat = compiler.endswith(AMD64_COMPAT_SUFFIX)
 

From fa9abc5038957ab2af34dc933825473231164ff4 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 9 Aug 2023 05:02:50 +0200
Subject: [PATCH 135/249] Better usage of ccache

---
 programs/keeper/Keeper.cpp                    |  2 +-
 programs/server/Server.cpp                    |  2 +-
 src/CMakeLists.txt                            |  3 ++-
 src/Client/ClientBase.cpp                     |  2 +-
 src/Client/Connection.cpp                     |  6 ++---
 src/Common/config_version.cpp.in              |  3 +++
 src/Common/config_version.h.in                | 22 ++-----------------
 src/Coordination/FourLetterCommand.cpp        |  2 +-
 src/Coordination/FourLetterCommand.h          |  4 +++-
 src/Interpreters/ClientInfo.cpp               | 10 ++++-----
 .../QueryPlan/DistributedCreateLocalPlan.cpp  |  2 +-
 src/Server/TCPHandler.cpp                     |  8 +++----
 .../System/StorageSystemBuildOptions.cpp.in   |  1 -
 13 files changed, 27 insertions(+), 40 deletions(-)
 create mode 100644 src/Common/config_version.cpp.in

diff --git a/programs/keeper/Keeper.cpp b/programs/keeper/Keeper.cpp
index 49009fffb90..1723c274fdb 100644
--- a/programs/keeper/Keeper.cpp
+++ b/programs/keeper/Keeper.cpp
@@ -150,7 +150,7 @@ int Keeper::run()
     }
     if (config().hasOption("version"))
     {
-        std::cout << DBMS_NAME << " keeper version " << VERSION_STRING << VERSION_OFFICIAL << "." << std::endl;
+        std::cout << VERSION_NAME << " keeper version " << VERSION_STRING << VERSION_OFFICIAL << "." << std::endl;
         return 0;
     }
 
diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp
index e6d5837dd0e..47607ec0630 100644
--- a/programs/server/Server.cpp
+++ b/programs/server/Server.cpp
@@ -389,7 +389,7 @@ int Server::run()
     }
     if (config().hasOption("version"))
     {
-        std::cout << DBMS_NAME << " server version " << VERSION_STRING << VERSION_OFFICIAL << "." << std::endl;
+        std::cout << VERSION_NAME << " server version " << VERSION_STRING << VERSION_OFFICIAL << "." << std::endl;
         return 0;
     }
     return Application::run(); // NOLINT
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index ca38a447c70..ddb6fcebd23 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -19,6 +19,7 @@ message (STATUS "Will build ${VERSION_FULL} revision ${VERSION_REVISION} ${VERSI
 include (configure_config.cmake)
 configure_file (Common/config.h.in ${CONFIG_INCLUDE_PATH}/config.h)
 configure_file (Common/config_version.h.in ${CONFIG_INCLUDE_PATH}/config_version.h)
+configure_file (Common/config_version.cpp.in ${CONFIG_INCLUDE_PATH}/config_version.cpp)
 
 if (USE_DEBUG_HELPERS)
     get_target_property(MAGIC_ENUM_INCLUDE_DIR ch_contrib::magic_enum INTERFACE_INCLUDE_DIRECTORIES)
@@ -150,7 +151,7 @@ else()
     message(STATUS "StorageFileLog is only supported on Linux")
 endif ()
 
-list (APPEND clickhouse_common_io_sources ${CONFIG_BUILD})
+list (APPEND clickhouse_common_io_sources ${CONFIG_INCLUDE_PATH}/config_version.cpp)
 
 list (APPEND dbms_sources Functions/IFunction.cpp Functions/FunctionFactory.cpp Functions/FunctionHelpers.cpp Functions/extractTimeZoneFromFunctionArguments.cpp Functions/FunctionsLogical.cpp Functions/indexHint.cpp)
 list (APPEND dbms_headers Functions/IFunction.h Functions/FunctionFactory.h Functions/FunctionHelpers.h Functions/extractTimeZoneFromFunctionArguments.h Functions/FunctionsLogical.h Functions/indexHint.h)
diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp
index a72de2645d4..c7288d4793a 100644
--- a/src/Client/ClientBase.cpp
+++ b/src/Client/ClientBase.cpp
@@ -2505,7 +2505,7 @@ void ClientBase::clearTerminal()
 
 void ClientBase::showClientVersion()
 {
-    std::cout << DBMS_NAME << " " + getName() + " version " << VERSION_STRING << VERSION_OFFICIAL << "." << std::endl;
+    std::cout << VERSION_NAME << " " + getName() + " version " << VERSION_STRING << VERSION_OFFICIAL << "." << std::endl;
 }
 
 namespace
diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp
index cac5600fbcb..3e12e60be08 100644
--- a/src/Client/Connection.cpp
+++ b/src/Client/Connection.cpp
@@ -280,9 +280,9 @@ void Connection::sendHello()
                         "Parameters 'default_database', 'user' and 'password' must not contain ASCII control characters");
 
     writeVarUInt(Protocol::Client::Hello, *out);
-    writeStringBinary((DBMS_NAME " ") + client_name, *out);
-    writeVarUInt(DBMS_VERSION_MAJOR, *out);
-    writeVarUInt(DBMS_VERSION_MINOR, *out);
+    writeStringBinary((VERSION_NAME " ") + client_name, *out);
+    writeVarUInt(VERSION_MAJOR, *out);
+    writeVarUInt(VERSION_MINOR, *out);
     // NOTE For backward compatibility of the protocol, client cannot send its version_patch.
     writeVarUInt(DBMS_TCP_PROTOCOL_VERSION, *out);
     writeStringBinary(default_database, *out);
diff --git a/src/Common/config_version.cpp.in b/src/Common/config_version.cpp.in
new file mode 100644
index 00000000000..f31e82bf582
--- /dev/null
+++ b/src/Common/config_version.cpp.in
@@ -0,0 +1,3 @@
+/// This file was autogenerated by CMake
+
+const char * VERSION_GITHASH = "@VERSION_GITHASH@";
diff --git a/src/Common/config_version.h.in b/src/Common/config_version.h.in
index 7c768bbfad9..aa8ddeeb860 100644
--- a/src/Common/config_version.h.in
+++ b/src/Common/config_version.h.in
@@ -6,7 +6,6 @@
 // only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes.
 #cmakedefine VERSION_REVISION @VERSION_REVISION@
 #cmakedefine VERSION_NAME "@VERSION_NAME@"
-#define DBMS_NAME VERSION_NAME
 #cmakedefine VERSION_MAJOR @VERSION_MAJOR@
 #cmakedefine VERSION_MINOR @VERSION_MINOR@
 #cmakedefine VERSION_PATCH @VERSION_PATCH@
@@ -15,27 +14,10 @@
 #cmakedefine VERSION_OFFICIAL "@VERSION_OFFICIAL@"
 #cmakedefine VERSION_FULL "@VERSION_FULL@"
 #cmakedefine VERSION_DESCRIBE "@VERSION_DESCRIBE@"
-#cmakedefine VERSION_GITHASH "@VERSION_GITHASH@"
 #cmakedefine VERSION_INTEGER @VERSION_INTEGER@
-#cmakedefine VERSION_DATE @VERSION_DATE@
 
-#if defined(VERSION_MAJOR)
-#define DBMS_VERSION_MAJOR VERSION_MAJOR
-#else
-#define DBMS_VERSION_MAJOR 0
-#endif
-
-#if defined(VERSION_MINOR)
-#define DBMS_VERSION_MINOR VERSION_MINOR
-#else
-#define DBMS_VERSION_MINOR 0
-#endif
-
-#if defined(VERSION_PATCH)
-#define DBMS_VERSION_PATCH VERSION_PATCH
-#else
-#define DBMS_VERSION_PATCH 0
-#endif
+/// These fields are frequently changing and we don't want to have them in the header file to allow caching.
+extern const char * VERSION_GITHASH;
 
 #if !defined(VERSION_OFFICIAL)
 #   define VERSION_OFFICIAL ""
diff --git a/src/Coordination/FourLetterCommand.cpp b/src/Coordination/FourLetterCommand.cpp
index d1ff03482b6..6ac22e8bd15 100644
--- a/src/Coordination/FourLetterCommand.cpp
+++ b/src/Coordination/FourLetterCommand.cpp
@@ -466,7 +466,7 @@ String EnviCommand::run()
 
     StringBuffer buf;
     buf << "Environment:\n";
-    buf << "clickhouse.keeper.version=" << (String(VERSION_DESCRIBE) + "-" + VERSION_GITHASH) << '\n';
+    buf << "clickhouse.keeper.version=" << VERSION_DESCRIBE << '-' << VERSION_GITHASH << '\n';
 
     buf << "host.name=" << Environment::nodeName() << '\n';
     buf << "os.name=" << Environment::osDisplayName() << '\n';
diff --git a/src/Coordination/FourLetterCommand.h b/src/Coordination/FourLetterCommand.h
index eb2cf9419ae..0520da06b6d 100644
--- a/src/Coordination/FourLetterCommand.h
+++ b/src/Coordination/FourLetterCommand.h
@@ -9,8 +9,10 @@
 
 #include "config_version.h"
 
+
 namespace DB
 {
+
 struct IFourLetterCommand;
 using FourLetterCommandPtr = std::shared_ptr<DB::IFourLetterCommand>;
 
@@ -43,7 +45,7 @@ public:
     using Commands = std::unordered_map<int32_t, FourLetterCommandPtr>;
     using AllowList = std::vector<int32_t>;
 
-    ///represent '*' which is used in allow list
+    /// Represents '*' which is used in allow list.
     static constexpr int32_t ALLOW_LIST_ALL = 0;
 
     bool isKnown(int32_t code);
diff --git a/src/Interpreters/ClientInfo.cpp b/src/Interpreters/ClientInfo.cpp
index 413ff1db6bc..6c09b327ca1 100644
--- a/src/Interpreters/ClientInfo.cpp
+++ b/src/Interpreters/ClientInfo.cpp
@@ -194,9 +194,9 @@ void ClientInfo::setInitialQuery()
     query_kind = QueryKind::INITIAL_QUERY;
     fillOSUserHostNameAndVersionInfo();
     if (client_name.empty())
-        client_name = DBMS_NAME;
+        client_name = VERSION_NAME;
     else
-        client_name = (DBMS_NAME " ") + client_name;
+        client_name = (VERSION_NAME " ") + client_name;
 }
 
 
@@ -210,9 +210,9 @@ void ClientInfo::fillOSUserHostNameAndVersionInfo()
 
     client_hostname = getFQDNOrHostName();
 
-    client_version_major = DBMS_VERSION_MAJOR;
-    client_version_minor = DBMS_VERSION_MINOR;
-    client_version_patch = DBMS_VERSION_PATCH;
+    client_version_major = VERSION_MAJOR;
+    client_version_minor = VERSION_MINOR;
+    client_version_patch = VERSION_PATCH;
     client_tcp_protocol_version = DBMS_TCP_PROTOCOL_VERSION;
 }
 
diff --git a/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp b/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp
index b251eec2d28..2ff50ca4fe3 100644
--- a/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp
+++ b/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp
@@ -75,7 +75,7 @@ std::unique_ptr<QueryPlan> createLocalPlan(
         new_context->setClientInterface(ClientInfo::Interface::LOCAL);
         new_context->setQueryKind(ClientInfo::QueryKind::SECONDARY_QUERY);
         new_context->setReplicaInfo(true, replica_count, replica_num);
-        new_context->setConnectionClientVersion(DBMS_VERSION_MAJOR, DBMS_VERSION_MINOR, DBMS_VERSION_PATCH, DBMS_TCP_PROTOCOL_VERSION);
+        new_context->setConnectionClientVersion(VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH, DBMS_TCP_PROTOCOL_VERSION);
         new_context->setParallelReplicasGroupUUID(group_uuid);
         new_context->setMergeTreeAllRangesCallback([coordinator](InitialAllRangesAnnouncement announcement)
         {
diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp
index 5f3a7614eee..cd17c2bcb55 100644
--- a/src/Server/TCPHandler.cpp
+++ b/src/Server/TCPHandler.cpp
@@ -1303,16 +1303,16 @@ void TCPHandler::receiveUnexpectedHello()
 void TCPHandler::sendHello()
 {
     writeVarUInt(Protocol::Server::Hello, *out);
-    writeStringBinary(DBMS_NAME, *out);
-    writeVarUInt(DBMS_VERSION_MAJOR, *out);
-    writeVarUInt(DBMS_VERSION_MINOR, *out);
+    writeStringBinary(VERSION_NAME, *out);
+    writeVarUInt(VERSION_MAJOR, *out);
+    writeVarUInt(VERSION_MINOR, *out);
     writeVarUInt(DBMS_TCP_PROTOCOL_VERSION, *out);
     if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_SERVER_TIMEZONE)
         writeStringBinary(DateLUT::instance().getTimeZone(), *out);
     if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME)
         writeStringBinary(server_display_name, *out);
     if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_VERSION_PATCH)
-        writeVarUInt(DBMS_VERSION_PATCH, *out);
+        writeVarUInt(VERSION_PATCH, *out);
     if (client_tcp_protocol_version >= DBMS_MIN_PROTOCOL_VERSION_WITH_PASSWORD_COMPLEXITY_RULES)
     {
         auto rules = server.context()->getAccessControl().getPasswordComplexityRules();
diff --git a/src/Storages/System/StorageSystemBuildOptions.cpp.in b/src/Storages/System/StorageSystemBuildOptions.cpp.in
index 4e7a25d7726..796b134ba56 100644
--- a/src/Storages/System/StorageSystemBuildOptions.cpp.in
+++ b/src/Storages/System/StorageSystemBuildOptions.cpp.in
@@ -8,7 +8,6 @@ const char * auto_config_build[]
     "SYSTEM", "@CMAKE_SYSTEM_NAME@",
     "VERSION_GITHASH", "@VERSION_GITHASH@",
     "VERSION_REVISION", "@VERSION_REVISION@",
-    "VERSION_DATE", "@VERSION_DATE@",
     "BUILD_TYPE", "@CMAKE_BUILD_TYPE@",
     "SYSTEM_PROCESSOR", "@CMAKE_SYSTEM_PROCESSOR@",
     "CMAKE_VERSION", "@CMAKE_VERSION@",

From 922542b4973510bc8c6cd1c09d69117c92b9ad60 Mon Sep 17 00:00:00 2001
From: yariks5s <yaroslav.briukhovetskyi@clickhouse.com>
Date: Wed, 9 Aug 2023 09:11:55 +0000
Subject: [PATCH 136/249] Improved efficiency

---
 src/Functions/FunctionBinaryArithmetic.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Functions/FunctionBinaryArithmetic.h b/src/Functions/FunctionBinaryArithmetic.h
index 723abc7c308..08baf6b23a5 100644
--- a/src/Functions/FunctionBinaryArithmetic.h
+++ b/src/Functions/FunctionBinaryArithmetic.h
@@ -1180,7 +1180,7 @@ class FunctionBinaryArithmetic : public IFunction
         /// Unpacking non-const arrays and checking sizes of them.
         for (auto offset_index = 0U; offset_index < left_offsets.size(); ++offset_index)
         {
-            if (left_offsets[offset_index] != right_offsets[offset_index])
+            if (right_array_col->hasEqualOffsets(*left_array_col))
             {
                 throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH,
                 "Cannot apply operation for arrays of different sizes. Size of the first argument: {}, size of the second argument: {}",

From 8ab4f8c9cfaa6665967ccd6c26cb216a35c2943f Mon Sep 17 00:00:00 2001
From: yariks5s <yaroslav.briukhovetskyi@clickhouse.com>
Date: Wed, 9 Aug 2023 09:12:39 +0000
Subject: [PATCH 137/249] Improved efficiency

---
 src/Functions/FunctionBinaryArithmetic.h | 18 +++---------------
 1 file changed, 3 insertions(+), 15 deletions(-)

diff --git a/src/Functions/FunctionBinaryArithmetic.h b/src/Functions/FunctionBinaryArithmetic.h
index 08baf6b23a5..86a5ce29d9c 100644
--- a/src/Functions/FunctionBinaryArithmetic.h
+++ b/src/Functions/FunctionBinaryArithmetic.h
@@ -1173,21 +1173,8 @@ class FunctionBinaryArithmetic : public IFunction
 
         const auto * left_array_col = typeid_cast<const ColumnArray *>(arguments[0].column.get());
         const auto * right_array_col = typeid_cast<const ColumnArray *>(arguments[1].column.get());
-        const auto & left_offsets = left_array_col->getOffsets();
-        const auto & right_offsets = right_array_col->getOffsets();
-
-        chassert(left_offsets.size() == right_offsets.size() && "Unexpected difference in number of offsets");
-        /// Unpacking non-const arrays and checking sizes of them.
-        for (auto offset_index = 0U; offset_index < left_offsets.size(); ++offset_index)
-        {
-            if (right_array_col->hasEqualOffsets(*left_array_col))
-            {
-                throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH,
-                "Cannot apply operation for arrays of different sizes. Size of the first argument: {}, size of the second argument: {}",
-                *left_array_col->getOffsets().data(),
-                *right_array_col ->getOffsets().data());
-            }
-        }
+        if (!left_array_col->hasEqualOffsets(*right_array_col))
+            throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, "Two arguments for function {} must have equal sizes", getName());
 
         const auto & left_array_type = typeid_cast<const DataTypeArray *>(arguments[0].type.get())->getNestedType();
         new_arguments[0] = {left_array_col->getDataPtr(), left_array_type, arguments[0].name};
@@ -1198,6 +1185,7 @@ class FunctionBinaryArithmetic : public IFunction
         result_array_type = typeid_cast<const DataTypeArray *>(result_type.get())->getNestedType();
 
         size_t rows_count = 0;
+        const auto & left_offsets = left_array_col->getOffsets();
         if (!left_offsets.empty())
             rows_count = left_offsets.back();
         auto res = executeImpl(new_arguments, result_array_type, rows_count);

From dc4be338959ac307ef1e71570c8d1cda5e08ed45 Mon Sep 17 00:00:00 2001
From: Smita Kulkarni <Smita.Kulkarni@clickhouse.com>
Date: Wed, 9 Aug 2023 14:55:44 +0200
Subject: [PATCH 138/249] Updated logs to include retries

---
 src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp
index 15b632a54d4..b60c4856672 100644
--- a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp
+++ b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp
@@ -105,7 +105,7 @@ bool ReadBufferFromAzureBlobStorage::nextImpl()
 
     auto handle_exception = [&, this](const auto & e, size_t i)
     {
-        LOG_INFO(log, "Exception caught during Azure Read for file {} at attempt {}: {}", path, i, e.Message);
+        LOG_INFO(log, "Exception caught during Azure Read for file {} at attempt {}/{}: {}", path, i + 1, max_single_read_retries, e.Message);
         if (i + 1 == max_single_read_retries)
             throw;
 
@@ -224,7 +224,7 @@ void ReadBufferFromAzureBlobStorage::initialize()
 
     auto handle_exception = [&, this](const auto & e, size_t i)
     {
-        LOG_INFO(log, "Exception caught during Azure Download for file {} at offset {} at attempt {} : {}", path, offset, i + 1, e.Message);
+        LOG_INFO(log, "Exception caught during Azure Download for file {} at offset {} at attempt {}/{}: {}", path, offset, i + 1, max_single_read_retries, e.Message);
         if (i + 1 == max_single_download_retries)
             throw;
 

From 9953050b620b8be4a134829a20e84934097521dc Mon Sep 17 00:00:00 2001
From: Smita Kulkarni <Smita.Kulkarni@clickhouse.com>
Date: Wed, 9 Aug 2023 14:57:17 +0200
Subject: [PATCH 139/249] Fixed logs for download

---
 src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp
index b60c4856672..fc0dbee373a 100644
--- a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp
+++ b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp
@@ -224,7 +224,7 @@ void ReadBufferFromAzureBlobStorage::initialize()
 
     auto handle_exception = [&, this](const auto & e, size_t i)
     {
-        LOG_INFO(log, "Exception caught during Azure Download for file {} at offset {} at attempt {}/{}: {}", path, offset, i + 1, max_single_read_retries, e.Message);
+        LOG_INFO(log, "Exception caught during Azure Download for file {} at offset {} at attempt {}/{}: {}", path, offset, i + 1, max_single_download_retriesg, e.Message);
         if (i + 1 == max_single_download_retries)
             throw;
 

From 964c407d482f5e3f6ff6094c7707f4fe8924e84d Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 9 Aug 2023 16:11:17 +0300
Subject: [PATCH 140/249] Revert "Revert "Upload build time-trace data to CI
 database""

---
 .github/workflows/backport_branches.yml       |  3 +
 .github/workflows/master.yml                  |  3 +
 .github/workflows/pull_request.yml            |  3 +
 .github/workflows/release_branches.yml        |  3 +
 docker/packager/binary/Dockerfile             |  1 +
 docker/packager/binary/build.sh               | 13 +++-
 docker/packager/packager                      |  7 +-
 src/Storages/StorageFile.cpp                  |  4 +-
 tests/ci/build_check.py                       | 78 ++++++++++++++++++-
 .../prepare-time-trace/prepare-time-trace.sh  |  3 +-
 10 files changed, 107 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/backport_branches.yml b/.github/workflows/backport_branches.yml
index d69168b01ee..1df8cea4ad5 100644
--- a/.github/workflows/backport_branches.yml
+++ b/.github/workflows/backport_branches.yml
@@ -3,6 +3,9 @@ name: BackportPR
 env:
   # Force the stdout and stderr streams to be unbuffered
   PYTHONUNBUFFERED: 1
+  # Export system tables to ClickHouse Cloud
+  CLICKHOUSE_CI_LOGS_HOST: ${{ secrets.CLICKHOUSE_CI_LOGS_HOST }}
+  CLICKHOUSE_CI_LOGS_PASSWORD: ${{ secrets.CLICKHOUSE_CI_LOGS_PASSWORD }}
 
 on: # yamllint disable-line rule:truthy
   push:
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index ae1862e327f..f25611e8666 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -3,6 +3,9 @@ name: MasterCI
 env:
   # Force the stdout and stderr streams to be unbuffered
   PYTHONUNBUFFERED: 1
+  # Export system tables to ClickHouse Cloud
+  CLICKHOUSE_CI_LOGS_HOST: ${{ secrets.CLICKHOUSE_CI_LOGS_HOST }}
+  CLICKHOUSE_CI_LOGS_PASSWORD: ${{ secrets.CLICKHOUSE_CI_LOGS_PASSWORD }}
 
 on: # yamllint disable-line rule:truthy
   push:
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index d97b9975c3c..25927954809 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -3,6 +3,9 @@ name: PullRequestCI
 env:
   # Force the stdout and stderr streams to be unbuffered
   PYTHONUNBUFFERED: 1
+  # Export system tables to ClickHouse Cloud
+  CLICKHOUSE_CI_LOGS_HOST: ${{ secrets.CLICKHOUSE_CI_LOGS_HOST }}
+  CLICKHOUSE_CI_LOGS_PASSWORD: ${{ secrets.CLICKHOUSE_CI_LOGS_PASSWORD }}
 
 on:  # yamllint disable-line rule:truthy
   pull_request:
diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml
index 21284815583..e5527177aa5 100644
--- a/.github/workflows/release_branches.yml
+++ b/.github/workflows/release_branches.yml
@@ -3,6 +3,9 @@ name: ReleaseBranchCI
 env:
   # Force the stdout and stderr streams to be unbuffered
   PYTHONUNBUFFERED: 1
+  # Export system tables to ClickHouse Cloud
+  CLICKHOUSE_CI_LOGS_HOST: ${{ secrets.CLICKHOUSE_CI_LOGS_HOST }}
+  CLICKHOUSE_CI_LOGS_PASSWORD: ${{ secrets.CLICKHOUSE_CI_LOGS_PASSWORD }}
 
 on: # yamllint disable-line rule:truthy
   push:
diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile
index 99e748c41d4..15e0bad9355 100644
--- a/docker/packager/binary/Dockerfile
+++ b/docker/packager/binary/Dockerfile
@@ -101,6 +101,7 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \
         python3-boto3 \
         yasm \
         zstd \
+        jq \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists
 
diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh
index 38b2ef75d24..e36212bfeef 100755
--- a/docker/packager/binary/build.sh
+++ b/docker/packager/binary/build.sh
@@ -59,7 +59,7 @@ if [ "$BUILD_MUSL_KEEPER" == "1" ]
 then
     # build keeper with musl separately
     # and without rust bindings
-    cmake --debug-trycompile -DENABLE_RUST=OFF -DBUILD_STANDALONE_KEEPER=1 -DENABLE_CLICKHOUSE_KEEPER=1 -DCMAKE_VERBOSE_MAKEFILE=1 -DUSE_MUSL=1 -LA -DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-x86_64-musl.cmake "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
+    cmake --debug-trycompile -DENABLE_RUST=OFF -DBUILD_STANDALONE_KEEPER=1 -DENABLE_CLICKHOUSE_KEEPER=1 -DCMAKE_VERBOSE_MAKEFILE=1 -DUSE_MUSL=1 -LA -DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-x86_64-musl.cmake "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 -DENABLE_BUILD_PROFILING=1 "${CMAKE_FLAGS[@]}" ..
     # shellcheck disable=SC2086 # No quotes because I want it to expand to nothing if empty.
     ninja $NINJA_FLAGS clickhouse-keeper
 
@@ -74,10 +74,10 @@ then
     rm -f CMakeCache.txt
 
     # Build the rest of binaries
-    cmake --debug-trycompile -DBUILD_STANDALONE_KEEPER=0 -DCREATE_KEEPER_SYMLINK=0 -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
+    cmake --debug-trycompile -DBUILD_STANDALONE_KEEPER=0 -DCREATE_KEEPER_SYMLINK=0 -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 -DENABLE_BUILD_PROFILING=1 "${CMAKE_FLAGS[@]}" ..
 else
     # Build everything
-    cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
+    cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 -DENABLE_BUILD_PROFILING=1 "${CMAKE_FLAGS[@]}" ..
 fi
 
 # No quotes because I want it to expand to nothing if empty.
@@ -181,4 +181,11 @@ then
     tar -cv -I pixz -f /output/ccache.log.txz "$CCACHE_LOGFILE"
 fi
 
+# Prepare profile info (time-trace)
+mkdir -p profile-tmp
+../utils/prepare-time-trace/prepare-time-trace.sh . profile-tmp
+find profile-tmp -type f -print0 | xargs -0 cat > /profile/profile.json
+
+wc -c /profile/profile.json
+
 ls -l /output
diff --git a/docker/packager/packager b/docker/packager/packager
index b6b9682f418..9f949f63eda 100755
--- a/docker/packager/packager
+++ b/docker/packager/packager
@@ -78,11 +78,14 @@ def run_docker_image_with_env(
     image_name: str,
     as_root: bool,
     output_dir: Path,
+    profile_dir: Path,
     env_variables: List[str],
     ch_root: Path,
     ccache_dir: Optional[Path],
 ):
     output_dir.mkdir(parents=True, exist_ok=True)
+    profile_dir.mkdir(parents=True, exist_ok=True)
+
     env_part = " -e ".join(env_variables)
     if env_part:
         env_part = " -e " + env_part
@@ -103,7 +106,7 @@ def run_docker_image_with_env(
 
     cmd = (
         f"docker run --network=host --user={user} --rm {ccache_mount}"
-        f"--volume={output_dir}:/output --volume={ch_root}:/build {env_part} "
+        f"--volume={output_dir}:/output --volume={ch_root}:/build --volume={profile_dir}:/profile {env_part} "
         f"{interactive} {image_name}"
     )
 
@@ -361,6 +364,7 @@ def parse_args() -> argparse.Namespace:
         help="ClickHouse git repository",
     )
     parser.add_argument("--output-dir", type=dir_name, required=True)
+    parser.add_argument("--profile-dir", type=dir_name, required=True)
     parser.add_argument("--debug-build", action="store_true")
 
     parser.add_argument(
@@ -488,6 +492,7 @@ def main():
         image_with_version,
         args.as_root,
         args.output_dir,
+        args.profile_dir,
         env_prepared,
         ch_root,
         args.ccache_dir,
diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp
index e3908c75a58..5c03540de9a 100644
--- a/src/Storages/StorageFile.cpp
+++ b/src/Storages/StorageFile.cpp
@@ -396,9 +396,9 @@ std::unique_ptr<ReadBuffer> createReadBuffer(
                 throw Exception(ErrorCodes::CANNOT_COMPILE_REGEXP,
                     "Cannot compile regex from glob ({}): {}", current_path, matcher->error());
 
-            return reader->readFile([matcher = std::move(matcher)](const std::string & path)
+            return reader->readFile([my_matcher = std::move(matcher)](const std::string & path)
             {
-                return re2::RE2::FullMatch(path, *matcher);
+                return re2::RE2::FullMatch(path, *my_matcher);
             });
         }
         else
diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py
index 592c27c2c68..5e22557edd8 100644
--- a/tests/ci/build_check.py
+++ b/tests/ci/build_check.py
@@ -1,12 +1,14 @@
 #!/usr/bin/env python3
 
+from typing import List, Tuple
 import subprocess
 import logging
 import json
 import os
 import sys
 import time
-from typing import List, Tuple
+import urllib.parse
+import requests  # type: ignore
 
 from ci_config import CI_CONFIG, BuildConfig
 from docker_pull_helper import get_image_with_version
@@ -30,6 +32,7 @@ from version_helper import (
 from clickhouse_helper import (
     ClickHouseHelper,
     prepare_tests_results_for_clickhouse,
+    get_instance_type,
 )
 from stopwatch import Stopwatch
 
@@ -51,6 +54,7 @@ def get_packager_cmd(
     build_config: BuildConfig,
     packager_path: str,
     output_path: str,
+    profile_path: str,
     build_version: str,
     image_version: str,
     official: bool,
@@ -59,8 +63,8 @@ def get_packager_cmd(
     comp = build_config.compiler
     cmake_flags = "-DENABLE_CLICKHOUSE_SELF_EXTRACTING=1"
     cmd = (
-        f"cd {packager_path} && CMAKE_FLAGS='{cmake_flags}' ./packager --output-dir={output_path} "
-        f"--package-type={package_type} --compiler={comp}"
+        f"cd {packager_path} && CMAKE_FLAGS='{cmake_flags}' ./packager --output-dir={output_path} --profile-dir={profile_path}"
+        f" --package-type={package_type} --compiler={comp}"
     )
 
     if build_config.debug_build:
@@ -286,10 +290,15 @@ def main():
     if not os.path.exists(build_output_path):
         os.makedirs(build_output_path)
 
+    build_profile_path = os.path.join(TEMP_PATH, f"{build_name}_profile")
+    if not os.path.exists(build_profile_path):
+        os.makedirs(build_profile_path)
+
     packager_cmd = get_packager_cmd(
         build_config,
         os.path.join(REPO_COPY, "docker/packager"),
         build_output_path,
+        build_profile_path,
         version.string,
         image_version,
         official_flag,
@@ -360,6 +369,69 @@ def main():
 
     upload_master_static_binaries(pr_info, build_config, s3_helper, build_output_path)
 
+    # Upload profile data
+
+    instance_type = get_instance_type()
+    query = urllib.parse.quote(
+        f"""
+        INSERT INTO build_time_trace
+        (
+            pull_request_number,
+            commit_sha,
+            check_start_time,
+            check_name,
+            instance_type,
+            file,
+            library,
+            time,
+            pid,
+            tid,
+            ph,
+            ts,
+            dur,
+            cat,
+            name,
+            detail,
+            count,
+            avgMs,
+            args_name
+        )
+        SELECT {pr_info.number}, '{pr_info.sha}', '{stopwatch.start_time_str}', '{build_name}', '{instance_type}', *
+        FROM input('
+            file String,
+            library String,
+            time DateTime64(6),
+            pid UInt32,
+            tid UInt32,
+            ph String,
+            ts UInt64,
+            dur UInt64,
+            cat String,
+            name String,
+            detail String,
+            count UInt64,
+            avgMs UInt64,
+            args_name String')
+        FORMAT JSONCompactEachRow
+    """
+    )
+    clickhouse_ci_logs_host = os.getenv("CLICKHOUSE_CI_LOGS_HOST")
+    maybe_clickhouse_ci_logs_password: str = (
+        os.getenv("CLICKHOUSE_CI_LOGS_PASSWORD") or ""
+    )
+    url = f"https://{clickhouse_ci_logs_host}/?query={query}"
+    file_path = os.path.join(build_profile_path, "profile.json")
+    file_size = os.path.getsize(file_path)
+
+    print(
+        f"::notice ::Log Uploading profile data, path: {file_path}, size: {file_size}, query: {query}"
+    )
+
+    with open(file_path, "rb") as file:
+        requests.post(url, data=file, auth=("ci", maybe_clickhouse_ci_logs_password))
+
+    # Upload statistics to CI database
+
     ch_helper = ClickHouseHelper()
     prepared_events = prepare_tests_results_for_clickhouse(
         pr_info,
diff --git a/utils/prepare-time-trace/prepare-time-trace.sh b/utils/prepare-time-trace/prepare-time-trace.sh
index 300a32b0fd1..7cacdec8c94 100755
--- a/utils/prepare-time-trace/prepare-time-trace.sh
+++ b/utils/prepare-time-trace/prepare-time-trace.sh
@@ -35,7 +35,6 @@ ENGINE = MergeTree ORDER BY (date, file, name, args_name);
 
 INPUT_DIR=$1
 OUTPUT_DIR=$2
-EXTRA_COLUMN_VALUES=$3
 
 find "$INPUT_DIR" -name '*.json' | grep -P '\.(c|cpp|cc|cxx)\.json$' | xargs -P $(nproc) -I{} bash -c "
 
@@ -43,7 +42,7 @@ find "$INPUT_DIR" -name '*.json' | grep -P '\.(c|cpp|cc|cxx)\.json$' | xargs -P
     LIBRARY_NAME=\$(echo '{}' | sed -r -e 's!^.*/CMakeFiles/([^/]+)\.dir/.*\$!\1!')
     START_TIME=\$(jq '.beginningOfTime' '{}')
 
-    jq -c '.traceEvents[] | [${EXTRA_COLUMN_VALUES} \"'\"\$ORIGINAL_FILENAME\"'\", \"'\"\$LIBRARY_NAME\"'\", '\$START_TIME', .pid, .tid, .ph, .ts, .dur, .cat, .name, .args.detail, .args.count, .args[\"avg ms\"], .args.name]' '{}' > \"${OUTPUT_DIR}/\$\$\"
+    jq -c '.traceEvents[] | [\"'\"\$ORIGINAL_FILENAME\"'\", \"'\"\$LIBRARY_NAME\"'\", '\$START_TIME', .pid, .tid, .ph, .ts, .dur, .cat, .name, .args.detail, .args.count, .args[\"avg ms\"], .args.name]' '{}' > \"${OUTPUT_DIR}/\$\$\"
 "
 
 # Now you can upload it as follows:

From f90e80e27df2904dc01668f4ccc1fbe317624d6a Mon Sep 17 00:00:00 2001
From: Vitaliy <vitaliyf@users.noreply.github.com>
Date: Wed, 9 Aug 2023 09:20:46 -0400
Subject: [PATCH 141/249] docs: Fixed invalid MySQL named collection examples

It's not valid to specify both on_duplicate_clause and replace_query, plus on_duplicate_clause
was supposed to be an SQL clause and not a boolean/integer.
---
 docs/en/operations/named-collections.md | 2 --
 docs/ru/operations/named-collections.md | 1 -
 2 files changed, 3 deletions(-)

diff --git a/docs/en/operations/named-collections.md b/docs/en/operations/named-collections.md
index 02f52b6f8bf..6ed72152c1e 100644
--- a/docs/en/operations/named-collections.md
+++ b/docs/en/operations/named-collections.md
@@ -169,7 +169,6 @@ host = '127.0.0.1',
 port = 3306,
 database = 'test',
 connection_pool_size = 8,
-on_duplicate_clause = 1,
 replace_query = 1
 ```
 
@@ -185,7 +184,6 @@ replace_query = 1
             <port>3306</port>
             <database>test</database>
             <connection_pool_size>8</connection_pool_size>
-            <on_duplicate_clause>1</on_duplicate_clause>
             <replace_query>1</replace_query>
         </mymysql>
     </named_collections>
diff --git a/docs/ru/operations/named-collections.md b/docs/ru/operations/named-collections.md
index ba6b47116ad..48ee7c9f15d 100644
--- a/docs/ru/operations/named-collections.md
+++ b/docs/ru/operations/named-collections.md
@@ -88,7 +88,6 @@ SELECT * FROM s3_engine_table LIMIT 3;
             <port>3306</port>
             <database>test</database>
             <connection_pool_size>8</connection_pool_size>
-            <on_duplicate_clause>1</on_duplicate_clause>
             <replace_query>1</replace_query>
         </mymysql>
     </named_collections>

From aadd675ed777b39db6e7e578b980d66b47b1a62b Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 9 Aug 2023 15:28:18 +0200
Subject: [PATCH 142/249] Fix build

---
 src/Common/SystemLogBase.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Common/SystemLogBase.h b/src/Common/SystemLogBase.h
index 1f5832e7aac..e05fae69549 100644
--- a/src/Common/SystemLogBase.h
+++ b/src/Common/SystemLogBase.h
@@ -56,7 +56,7 @@ public:
 
     //// force -- force table creation (used for SYSTEM FLUSH LOGS)
     virtual void flush(bool force = false) = 0; /// NOLINT
-    virtual void prepareTable() = 0;
+    virtual void prepareTable(bool if_not_exists) = 0;
 
     /// Start the background thread.
     virtual void startup() = 0;

From 52e6cf36b28794e2c84ad8466b81cff8019e88e4 Mon Sep 17 00:00:00 2001
From: ltrk2 <107155950+ltrk2@users.noreply.github.com>
Date: Wed, 9 Aug 2023 06:43:27 -0700
Subject: [PATCH 143/249] Fix style issue

---
 src/Functions/HasTokenImpl.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Functions/HasTokenImpl.h b/src/Functions/HasTokenImpl.h
index 661b8ae9753..a4ff49859cc 100644
--- a/src/Functions/HasTokenImpl.h
+++ b/src/Functions/HasTokenImpl.h
@@ -13,6 +13,7 @@ namespace ErrorCodes
     extern const int BAD_ARGUMENTS;
     extern const int ILLEGAL_COLUMN;
     extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+    extern const int LOGICAL_ERROR;
 }
 
 /** Token search the string, means that needle must be surrounded by some separator chars, like whitespace or puctuation.

From c013c5ff438b173c50299488552cf9441b02cbca Mon Sep 17 00:00:00 2001
From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com>
Date: Wed, 9 Aug 2023 16:02:45 +0200
Subject: [PATCH 144/249] Enable hedged requests under tsan

---
 src/Core/SettingsQuirks.cpp | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/src/Core/SettingsQuirks.cpp b/src/Core/SettingsQuirks.cpp
index 37a0f2db3e2..1a79c23d955 100644
--- a/src/Core/SettingsQuirks.cpp
+++ b/src/Core/SettingsQuirks.cpp
@@ -72,12 +72,6 @@ void applySettingsQuirks(Settings & settings, Poco::Logger * log)
         }
     }
 
-#if defined(THREAD_SANITIZER)
-    settings.use_hedged_requests.value = false;
-    if (log)
-        LOG_WARNING(log, "use_hedged_requests has been disabled for the build with Thread Sanitizer, because they are using fibers, leading to a failed assertion inside TSan");
-#endif
-
     if (!queryProfilerWorks())
     {
         if (settings.query_profiler_real_time_period_ns)

From 901901a76271e64637bc7c33f60dc38677998de3 Mon Sep 17 00:00:00 2001
From: Smita Kulkarni <Smita.Kulkarni@clickhouse.com>
Date: Wed, 9 Aug 2023 16:50:40 +0200
Subject: [PATCH 145/249] Fixed typo

---
 src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp
index fc0dbee373a..cd8502c913f 100644
--- a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp
+++ b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp
@@ -224,7 +224,7 @@ void ReadBufferFromAzureBlobStorage::initialize()
 
     auto handle_exception = [&, this](const auto & e, size_t i)
     {
-        LOG_INFO(log, "Exception caught during Azure Download for file {} at offset {} at attempt {}/{}: {}", path, offset, i + 1, max_single_download_retriesg, e.Message);
+        LOG_INFO(log, "Exception caught during Azure Download for file {} at offset {} at attempt {}/{}: {}", path, offset, i + 1, max_single_download_retries, e.Message);
         if (i + 1 == max_single_download_retries)
             throw;
 

From 5341c5c8dd90adccbe6cb850819e3d155e51426b Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Wed, 9 Aug 2023 16:00:10 +0200
Subject: [PATCH 146/249] Use CMAKE_FLAGS and args for cmake profiler

---
 docker/packager/binary/build.sh |  6 +++---
 docker/packager/packager        | 16 +++++++++++-----
 tests/ci/build_check.py         |  1 +
 3 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh
index e36212bfeef..9277ac0491e 100755
--- a/docker/packager/binary/build.sh
+++ b/docker/packager/binary/build.sh
@@ -59,7 +59,7 @@ if [ "$BUILD_MUSL_KEEPER" == "1" ]
 then
     # build keeper with musl separately
     # and without rust bindings
-    cmake --debug-trycompile -DENABLE_RUST=OFF -DBUILD_STANDALONE_KEEPER=1 -DENABLE_CLICKHOUSE_KEEPER=1 -DCMAKE_VERBOSE_MAKEFILE=1 -DUSE_MUSL=1 -LA -DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-x86_64-musl.cmake "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 -DENABLE_BUILD_PROFILING=1 "${CMAKE_FLAGS[@]}" ..
+    cmake --debug-trycompile -DENABLE_RUST=OFF -DBUILD_STANDALONE_KEEPER=1 -DENABLE_CLICKHOUSE_KEEPER=1 -DCMAKE_VERBOSE_MAKEFILE=1 -DUSE_MUSL=1 -LA -DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-x86_64-musl.cmake "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
     # shellcheck disable=SC2086 # No quotes because I want it to expand to nothing if empty.
     ninja $NINJA_FLAGS clickhouse-keeper
 
@@ -74,10 +74,10 @@ then
     rm -f CMakeCache.txt
 
     # Build the rest of binaries
-    cmake --debug-trycompile -DBUILD_STANDALONE_KEEPER=0 -DCREATE_KEEPER_SYMLINK=0 -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 -DENABLE_BUILD_PROFILING=1 "${CMAKE_FLAGS[@]}" ..
+    cmake --debug-trycompile -DBUILD_STANDALONE_KEEPER=0 -DCREATE_KEEPER_SYMLINK=0 -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
 else
     # Build everything
-    cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 -DENABLE_BUILD_PROFILING=1 "${CMAKE_FLAGS[@]}" ..
+    cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
 fi
 
 # No quotes because I want it to expand to nothing if empty.
diff --git a/docker/packager/packager b/docker/packager/packager
index 9f949f63eda..5a710a8c57c 100755
--- a/docker/packager/packager
+++ b/docker/packager/packager
@@ -22,7 +22,7 @@ def check_image_exists_locally(image_name: str) -> bool:
         output = subprocess.check_output(
             f"docker images -q {image_name} 2> /dev/null", shell=True
         )
-        return output != ""
+        return output != b""
     except subprocess.CalledProcessError:
         return False
 
@@ -46,7 +46,7 @@ def build_image(image_name: str, filepath: Path) -> None:
     )
 
 
-def pre_build(repo_path: Path, env_variables: List[str]):
+def pre_build(repo_path: Path, env_variables: List[str]) -> None:
     if "WITH_PERFORMANCE=1" in env_variables:
         current_branch = subprocess.check_output(
             "git branch --show-current", shell=True, encoding="utf-8"
@@ -82,7 +82,7 @@ def run_docker_image_with_env(
     env_variables: List[str],
     ch_root: Path,
     ccache_dir: Optional[Path],
-):
+) -> None:
     output_dir.mkdir(parents=True, exist_ok=True)
     profile_dir.mkdir(parents=True, exist_ok=True)
 
@@ -132,9 +132,10 @@ def parse_env_variables(
     version: str,
     official: bool,
     additional_pkgs: bool,
+    with_profiler: bool,
     with_coverage: bool,
     with_binaries: str,
-):
+) -> List[str]:
     DARWIN_SUFFIX = "-darwin"
     DARWIN_ARM_SUFFIX = "-darwin-aarch64"
     ARM_SUFFIX = "-aarch64"
@@ -325,6 +326,9 @@ def parse_env_variables(
         # utils are not included into clickhouse-bundle, so build everything
         build_target = "all"
 
+    if with_profiler:
+        cmake_flags.append("-DENABLE_BUILD_PROFILING=1")
+
     if with_coverage:
         cmake_flags.append("-DWITH_COVERAGE=1")
 
@@ -420,6 +424,7 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--version")
     parser.add_argument("--official", action="store_true")
     parser.add_argument("--additional-pkgs", action="store_true")
+    parser.add_argument("--with-profiler", action="store_true")
     parser.add_argument("--with-coverage", action="store_true")
     parser.add_argument(
         "--with-binaries", choices=("programs", "tests", ""), default=""
@@ -455,7 +460,7 @@ def parse_args() -> argparse.Namespace:
     return args
 
 
-def main():
+def main() -> None:
     logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
     args = parse_args()
 
@@ -483,6 +488,7 @@ def main():
         args.version,
         args.official,
         args.additional_pkgs,
+        args.with_profiler,
         args.with_coverage,
         args.with_binaries,
     )
diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py
index 5e22557edd8..139c995c7ec 100644
--- a/tests/ci/build_check.py
+++ b/tests/ci/build_check.py
@@ -82,6 +82,7 @@ def get_packager_cmd(
         cmd += " --additional-pkgs"
 
     cmd += f" --docker-image-version={image_version}"
+    cmd += " --with-profiler"
     cmd += f" --version={build_version}"
 
     if _can_export_binaries(build_config):

From 140534683090d9bd4ee67e618f13802638b4f170 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Wed, 9 Aug 2023 18:56:19 +0200
Subject: [PATCH 147/249] Rewrite build_check to use pathlib.Path

---
 tests/ci/build_check.py | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py
index 139c995c7ec..999c8b6d74f 100644
--- a/tests/ci/build_check.py
+++ b/tests/ci/build_check.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+from pathlib import Path
 from typing import List, Tuple
 import subprocess
 import logging
@@ -53,7 +54,7 @@ def _can_export_binaries(build_config: BuildConfig) -> bool:
 def get_packager_cmd(
     build_config: BuildConfig,
     packager_path: str,
-    output_path: str,
+    output_path: Path,
     profile_path: str,
     build_version: str,
     image_version: str,
@@ -95,13 +96,13 @@ def get_packager_cmd(
 
 
 def build_clickhouse(
-    packager_cmd: str, logs_path: str, build_output_path: str
-) -> Tuple[str, bool]:
-    build_log_path = os.path.join(logs_path, BUILD_LOG_NAME)
+    packager_cmd: str, logs_path: Path, build_output_path: Path
+) -> Tuple[Path, bool]:
+    build_log_path = logs_path / BUILD_LOG_NAME
     success = False
     with TeePopen(packager_cmd, build_log_path) as process:
         retcode = process.wait()
-        if os.path.exists(build_output_path):
+        if build_output_path.exists():
             build_results = os.listdir(build_output_path)
         else:
             build_results = []
@@ -222,7 +223,7 @@ def upload_master_static_binaries(
     pr_info: PRInfo,
     build_config: BuildConfig,
     s3_helper: S3Helper,
-    build_output_path: str,
+    build_output_path: Path,
 ) -> None:
     """Upload binary artifacts to a static S3 links"""
     static_binary_name = build_config.static_binary_name
@@ -234,7 +235,7 @@ def upload_master_static_binaries(
         return
 
     s3_path = "/".join((pr_info.base_ref, static_binary_name, "clickhouse"))
-    binary = os.path.join(build_output_path, "clickhouse")
+    binary = build_output_path / "clickhouse"
     url = s3_helper.upload_build_file_to_s3(binary, s3_path)
     print(f"::notice ::Binary static URL: {url}")
 
@@ -247,8 +248,8 @@ def main():
 
     build_config = CI_CONFIG.build_config[build_name]
 
-    if not os.path.exists(TEMP_PATH):
-        os.makedirs(TEMP_PATH)
+    temp_path = Path(TEMP_PATH)
+    os.makedirs(temp_path, exist_ok=True)
 
     pr_info = PRInfo()
 
@@ -287,9 +288,8 @@ def main():
 
     logging.info("Build short name %s", build_name)
 
-    build_output_path = os.path.join(TEMP_PATH, build_name)
-    if not os.path.exists(build_output_path):
-        os.makedirs(build_output_path)
+    build_output_path = temp_path / build_name
+    os.makedirs(build_output_path, exist_ok=True)
 
     build_profile_path = os.path.join(TEMP_PATH, f"{build_name}_profile")
     if not os.path.exists(build_profile_path):
@@ -330,8 +330,8 @@ def main():
 
     # FIXME performance
     performance_urls = []
-    performance_path = os.path.join(build_output_path, "performance.tar.zst")
-    if os.path.exists(performance_path):
+    performance_path = build_output_path / "performance.tar.zst"
+    if performance_path.exists():
         performance_urls.append(
             s3_helper.upload_build_file_to_s3(performance_path, s3_performance_path)
         )
@@ -354,9 +354,9 @@ def main():
 
     print("::notice ::Build URLs: {}".format("\n".join(build_urls)))
 
-    if os.path.exists(log_path):
+    if log_path.exists():
         log_url = s3_helper.upload_build_file_to_s3(
-            log_path, s3_path_prefix + "/" + os.path.basename(log_path)
+            log_path, s3_path_prefix + "/" + log_path.name
         )
         logging.info("Log url %s", log_url)
     else:

From da01d756f9c17d27b35bfcb3381b479b5c1e5a7f Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 9 Aug 2023 20:20:25 +0200
Subject: [PATCH 148/249] Addressed review comments

---
 src/Functions/array/arrayAUC.cpp | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/src/Functions/array/arrayAUC.cpp b/src/Functions/array/arrayAUC.cpp
index a0aca7a563b..caf929ba038 100644
--- a/src/Functions/array/arrayAUC.cpp
+++ b/src/Functions/array/arrayAUC.cpp
@@ -82,8 +82,8 @@ public:
 
 private:
     static Float64 apply(
-        const IColumn & data1,
-        const IColumn & data2,
+        const IColumn & scores,
+        const IColumn & labels,
         ColumnArray::Offset current_offset,
         ColumnArray::Offset next_offset)
     {
@@ -98,8 +98,8 @@ private:
 
         for (size_t i = 0; i < size; ++i)
         {
-            bool label = data2.getFloat64(current_offset + i) > 0;
-            sorted_labels[i].score = data1.getFloat64(current_offset + i);
+            bool label = labels.getFloat64(current_offset + i) > 0;
+            sorted_labels[i].score = scores.getFloat64(current_offset + i);
             sorted_labels[i].label = label;
         }
 
@@ -151,8 +151,6 @@ public:
 
     DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
     {
-        // Basic type check
-        std::vector<DataTypePtr> nested_types(2, nullptr);
         for (size_t i = 0; i < getNumberOfArguments(); ++i)
         {
             const DataTypeArray * array_type = checkAndGetDataType<DataTypeArray>(arguments[i].get());
@@ -163,7 +161,6 @@ public:
             if (!isNativeNumber(nested_type) && !isEnum(nested_type))
                 throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "{} cannot process values of type {}",
                                 getName(), nested_type->getName());
-            nested_types[i] = nested_type;
         }
 
         return std::make_shared<DataTypeFloat64>();
@@ -175,11 +172,15 @@ public:
         ColumnPtr col2 = arguments[1].column->convertToFullColumnIfConst();
 
         const ColumnArray * col_array1 = checkAndGetColumn<ColumnArray>(col1.get());
-        const ColumnArray * col_array2 = checkAndGetColumn<ColumnArray>(col2.get());
-        if (!col_array1 || !col_array2)
+        if (!col_array1)
             throw Exception(ErrorCodes::ILLEGAL_COLUMN,
                 "Illegal column {} of first argument of function {}", arguments[0].column->getName(), getName());
 
+        const ColumnArray * col_array2 = checkAndGetColumn<ColumnArray>(col2.get());
+        if (!col_array2)
+            throw Exception(ErrorCodes::ILLEGAL_COLUMN,
+                "Illegal column {} of second argument of function {}", arguments[1].column->getName(), getName());
+
         if (!col_array1->hasEqualOffsets(*col_array2))
             throw Exception(ErrorCodes::BAD_ARGUMENTS, "Array arguments for function {} must have equal sizes", getName());
 

From 6a3267ad3770187ed5d253e952c02f282cd1cb3f Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 9 Aug 2023 20:21:51 +0200
Subject: [PATCH 149/249] A fix for clang-17

---
 src/Storages/StorageFile.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp
index e3908c75a58..5c03540de9a 100644
--- a/src/Storages/StorageFile.cpp
+++ b/src/Storages/StorageFile.cpp
@@ -396,9 +396,9 @@ std::unique_ptr<ReadBuffer> createReadBuffer(
                 throw Exception(ErrorCodes::CANNOT_COMPILE_REGEXP,
                     "Cannot compile regex from glob ({}): {}", current_path, matcher->error());
 
-            return reader->readFile([matcher = std::move(matcher)](const std::string & path)
+            return reader->readFile([my_matcher = std::move(matcher)](const std::string & path)
             {
-                return re2::RE2::FullMatch(path, *matcher);
+                return re2::RE2::FullMatch(path, *my_matcher);
             });
         }
         else

From 535450a0505034a46237cda97276248f40ce74c4 Mon Sep 17 00:00:00 2001
From: Yakov Olkhovskiy <yakov@clickhouse.com>
Date: Wed, 9 Aug 2023 18:40:00 +0000
Subject: [PATCH 150/249] different linker?

---
 cmake/linux/toolchain-s390x.cmake | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cmake/linux/toolchain-s390x.cmake b/cmake/linux/toolchain-s390x.cmake
index b85d4253b89..c1cbb849bd3 100644
--- a/cmake/linux/toolchain-s390x.cmake
+++ b/cmake/linux/toolchain-s390x.cmake
@@ -20,6 +20,8 @@ set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}/s390x-linux-gnu/libc")
 set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
 set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
 set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
+set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=s390x-linux-gnu-ld")
+set (CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fuse-ld=s390x-linux-gnu-ld")
 
 set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
 set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)

From 239d198cf2049688790fc7607a61fbf44eb85329 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Wed, 9 Aug 2023 21:11:21 +0200
Subject: [PATCH 151/249] Add insert_file to ClickHouseHelper, make
 insert_json_str public

---
 tests/ci/clickhouse_helper.py | 71 ++++++++++++++++++++++++++++++++---
 1 file changed, 66 insertions(+), 5 deletions(-)

diff --git a/tests/ci/clickhouse_helper.py b/tests/ci/clickhouse_helper.py
index 7f98f6e52d2..dbf26288469 100644
--- a/tests/ci/clickhouse_helper.py
+++ b/tests/ci/clickhouse_helper.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
-from typing import List
+from pathlib import Path
+from typing import Dict, List, Optional
 import json
 import logging
 import time
@@ -16,18 +17,78 @@ class InsertException(Exception):
 
 
 class ClickHouseHelper:
-    def __init__(self, url=None):
+    def __init__(
+        self, url: Optional[str] = None, auth: Optional[Dict[str, str]] = None
+    ):
         if url is None:
             url = get_parameter_from_ssm("clickhouse-test-stat-url")
 
         self.url = url
-        self.auth = {
+        self.auth = auth or {
             "X-ClickHouse-User": get_parameter_from_ssm("clickhouse-test-stat-login"),
             "X-ClickHouse-Key": get_parameter_from_ssm("clickhouse-test-stat-password"),
         }
 
     @staticmethod
-    def _insert_json_str_info_impl(url, auth, db, table, json_str):
+    def insert_file(
+        url: str,
+        auth: Optional[Dict[str, str]],
+        query: str,
+        file: Path,
+        additional_options: Optional[Dict[str, str]] = None,
+    ) -> None:
+        params = {
+            "query": query,
+            "date_time_input_format": "best_effort",
+            "send_logs_level": "warning",
+        }
+        if additional_options:
+            for k, v in additional_options.items():
+                params[k] = v
+
+        with open(file, "rb") as data_fd:
+            for i in range(5):
+                try:
+                    response = requests.post(
+                        url, params=params, data=data_fd, headers=auth
+                    )
+                except Exception as e:
+                    error = f"Received exception while sending data to {url} on {i} attempt: {e}"
+                    logging.warning(error)
+                    continue
+
+                logging.info("Response content '%s'", response.content)
+
+                if response.ok:
+                    break
+
+                error = (
+                    "Cannot insert data into clickhouse at try "
+                    + str(i)
+                    + ": HTTP code "
+                    + str(response.status_code)
+                    + ": '"
+                    + str(response.text)
+                    + "'"
+                )
+
+                if response.status_code >= 500:
+                    # A retriable error
+                    time.sleep(1)
+                    continue
+
+                logging.info(
+                    "Request headers '%s', body '%s'",
+                    response.request.headers,
+                    response.request.body,
+                )
+
+                raise InsertException(error)
+            else:
+                raise InsertException(error)
+
+    @staticmethod
+    def insert_json_str(url, auth, db, table, json_str):
         params = {
             "database": db,
             "query": f"INSERT INTO {table} FORMAT JSONEachRow",
@@ -76,7 +137,7 @@ class ClickHouseHelper:
             raise InsertException(error)
 
     def _insert_json_str_info(self, db, table, json_str):
-        self._insert_json_str_info_impl(self.url, self.auth, db, table, json_str)
+        self.insert_json_str(self.url, self.auth, db, table, json_str)
 
     def insert_event_into(self, db, table, event, safe=True):
         event_str = json.dumps(event)

From c7b045fdde6a2108ee47036be770b8556df42b32 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Wed, 9 Aug 2023 21:12:01 +0200
Subject: [PATCH 152/249] Process build profiles in the python script, not in
 docker

---
 docker/packager/binary/Dockerfile |   1 -
 docker/packager/binary/build.sh   |   7 --
 docker/packager/packager          |   6 +-
 tests/ci/build_check.py           | 144 +++++++++++++++---------------
 4 files changed, 73 insertions(+), 85 deletions(-)

diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile
index 15e0bad9355..99e748c41d4 100644
--- a/docker/packager/binary/Dockerfile
+++ b/docker/packager/binary/Dockerfile
@@ -101,7 +101,6 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \
         python3-boto3 \
         yasm \
         zstd \
-        jq \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists
 
diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh
index 9277ac0491e..38b2ef75d24 100755
--- a/docker/packager/binary/build.sh
+++ b/docker/packager/binary/build.sh
@@ -181,11 +181,4 @@ then
     tar -cv -I pixz -f /output/ccache.log.txz "$CCACHE_LOGFILE"
 fi
 
-# Prepare profile info (time-trace)
-mkdir -p profile-tmp
-../utils/prepare-time-trace/prepare-time-trace.sh . profile-tmp
-find profile-tmp -type f -print0 | xargs -0 cat > /profile/profile.json
-
-wc -c /profile/profile.json
-
 ls -l /output
diff --git a/docker/packager/packager b/docker/packager/packager
index 5a710a8c57c..c0baacccef2 100755
--- a/docker/packager/packager
+++ b/docker/packager/packager
@@ -78,13 +78,11 @@ def run_docker_image_with_env(
     image_name: str,
     as_root: bool,
     output_dir: Path,
-    profile_dir: Path,
     env_variables: List[str],
     ch_root: Path,
     ccache_dir: Optional[Path],
 ) -> None:
     output_dir.mkdir(parents=True, exist_ok=True)
-    profile_dir.mkdir(parents=True, exist_ok=True)
 
     env_part = " -e ".join(env_variables)
     if env_part:
@@ -106,7 +104,7 @@ def run_docker_image_with_env(
 
     cmd = (
         f"docker run --network=host --user={user} --rm {ccache_mount}"
-        f"--volume={output_dir}:/output --volume={ch_root}:/build --volume={profile_dir}:/profile {env_part} "
+        f"--volume={output_dir}:/output --volume={ch_root}:/build {env_part} "
         f"{interactive} {image_name}"
     )
 
@@ -368,7 +366,6 @@ def parse_args() -> argparse.Namespace:
         help="ClickHouse git repository",
     )
     parser.add_argument("--output-dir", type=dir_name, required=True)
-    parser.add_argument("--profile-dir", type=dir_name, required=True)
     parser.add_argument("--debug-build", action="store_true")
 
     parser.add_argument(
@@ -498,7 +495,6 @@ def main() -> None:
         image_with_version,
         args.as_root,
         args.output_dir,
-        args.profile_dir,
         env_prepared,
         ch_root,
         args.ccache_dir,
diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py
index 999c8b6d74f..9e6370c2e6e 100644
--- a/tests/ci/build_check.py
+++ b/tests/ci/build_check.py
@@ -8,8 +8,6 @@ import json
 import os
 import sys
 import time
-import urllib.parse
-import requests  # type: ignore
 
 from ci_config import CI_CONFIG, BuildConfig
 from docker_pull_helper import get_image_with_version
@@ -21,12 +19,12 @@ from env_helper import (
     S3_DOWNLOAD,
     TEMP_PATH,
 )
+from git_helper import Git, git_runner
 from pr_info import PRInfo
 from s3_helper import S3Helper
 from tee_popen import TeePopen
 from version_helper import (
     ClickHouseVersion,
-    Git,
     get_version_from_repo,
     update_version_local,
 )
@@ -55,7 +53,6 @@ def get_packager_cmd(
     build_config: BuildConfig,
     packager_path: str,
     output_path: Path,
-    profile_path: str,
     build_version: str,
     image_version: str,
     official: bool,
@@ -64,8 +61,8 @@ def get_packager_cmd(
     comp = build_config.compiler
     cmake_flags = "-DENABLE_CLICKHOUSE_SELF_EXTRACTING=1"
     cmd = (
-        f"cd {packager_path} && CMAKE_FLAGS='{cmake_flags}' ./packager --output-dir={output_path} --profile-dir={profile_path}"
-        f" --package-type={package_type} --compiler={comp}"
+        f"cd {packager_path} && CMAKE_FLAGS='{cmake_flags}' ./packager "
+        f"--output-dir={output_path} --package-type={package_type} --compiler={comp}"
     )
 
     if build_config.debug_build:
@@ -291,15 +288,10 @@ def main():
     build_output_path = temp_path / build_name
     os.makedirs(build_output_path, exist_ok=True)
 
-    build_profile_path = os.path.join(TEMP_PATH, f"{build_name}_profile")
-    if not os.path.exists(build_profile_path):
-        os.makedirs(build_profile_path)
-
     packager_cmd = get_packager_cmd(
         build_config,
         os.path.join(REPO_COPY, "docker/packager"),
         build_output_path,
-        build_profile_path,
         version.string,
         image_version,
         official_flag,
@@ -307,9 +299,8 @@ def main():
 
     logging.info("Going to run packager with %s", packager_cmd)
 
-    logs_path = os.path.join(TEMP_PATH, "build_log")
-    if not os.path.exists(logs_path):
-        os.makedirs(logs_path)
+    logs_path = temp_path / "build_log"
+    os.makedirs(logs_path, exist_ok=True)
 
     start = time.time()
     log_path, success = build_clickhouse(packager_cmd, logs_path, build_output_path)
@@ -371,69 +362,78 @@ def main():
     upload_master_static_binaries(pr_info, build_config, s3_helper, build_output_path)
 
     # Upload profile data
+    ch_helper = ClickHouseHelper()
 
-    instance_type = get_instance_type()
-    query = urllib.parse.quote(
-        f"""
-        INSERT INTO build_time_trace
-        (
-            pull_request_number,
-            commit_sha,
-            check_start_time,
-            check_name,
-            instance_type,
-            file,
-            library,
-            time,
-            pid,
-            tid,
-            ph,
-            ts,
-            dur,
-            cat,
-            name,
-            detail,
-            count,
-            avgMs,
-            args_name
+    clickhouse_ci_logs_host = os.getenv("CLICKHOUSE_CI_LOGS_HOST", "")
+    if clickhouse_ci_logs_host:
+        instance_type = get_instance_type()
+        query = f"""INSERT INTO build_time_trace
+(
+    pull_request_number,
+    commit_sha,
+    check_start_time,
+    check_name,
+    instance_type,
+    file,
+    library,
+    time,
+    pid,
+    tid,
+    ph,
+    ts,
+    dur,
+    cat,
+    name,
+    detail,
+    count,
+    avgMs,
+    args_name
+)
+SELECT {pr_info.number}, '{pr_info.sha}', '{stopwatch.start_time_str}', '{build_name}', '{instance_type}', *
+FROM input('
+    file String,
+    library String,
+    time DateTime64(6),
+    pid UInt32,
+    tid UInt32,
+    ph String,
+    ts UInt64,
+    dur UInt64,
+    cat String,
+    name String,
+    detail String,
+    count UInt64,
+    avgMs UInt64,
+    args_name String')
+FORMAT JSONCompactEachRow"""
+
+        auth = {
+            "X-ClickHouse-User": "ci",
+            "X-ClickHouse-Key": os.getenv("CLICKHOUSE_CI_LOGS_PASSWORD", ""),
+        }
+        url = f"https://{clickhouse_ci_logs_host}/"
+        profiles_dir = temp_path / "profiles_source"
+        os.makedirs(profiles_dir, exist_ok=True)
+        logging.info("Processing profile JSON files from {GIT_REPO_ROOT}/build_docker")
+        git_runner(
+            "./utils/prepare-time-trace/prepare-time-trace.sh "
+            f"build_docker {profiles_dir.absolute()}"
         )
-        SELECT {pr_info.number}, '{pr_info.sha}', '{stopwatch.start_time_str}', '{build_name}', '{instance_type}', *
-        FROM input('
-            file String,
-            library String,
-            time DateTime64(6),
-            pid UInt32,
-            tid UInt32,
-            ph String,
-            ts UInt64,
-            dur UInt64,
-            cat String,
-            name String,
-            detail String,
-            count UInt64,
-            avgMs UInt64,
-            args_name String')
-        FORMAT JSONCompactEachRow
-    """
-    )
-    clickhouse_ci_logs_host = os.getenv("CLICKHOUSE_CI_LOGS_HOST")
-    maybe_clickhouse_ci_logs_password: str = (
-        os.getenv("CLICKHOUSE_CI_LOGS_PASSWORD") or ""
-    )
-    url = f"https://{clickhouse_ci_logs_host}/?query={query}"
-    file_path = os.path.join(build_profile_path, "profile.json")
-    file_size = os.path.getsize(file_path)
+        profile_data_file = temp_path / "profile.json"
+        with open(profile_data_file, "wb") as profile_fd:
+            for profile_sourse in os.listdir(profiles_dir):
+                with open(profiles_dir / profile_sourse, "rb") as ps_fd:
+                    profile_fd.write(ps_fd.read())
 
-    print(
-        f"::notice ::Log Uploading profile data, path: {file_path}, size: {file_size}, query: {query}"
-    )
-
-    with open(file_path, "rb") as file:
-        requests.post(url, data=file, auth=("ci", maybe_clickhouse_ci_logs_password))
+        logging.info(
+            "::notice ::Log Uploading profile data, path: %s, size: %s, query: %s",
+            profile_data_file,
+            profile_data_file.stat().st_size,
+            query,
+        )
+        ch_helper.insert_file(url, auth, query, profile_data_file)
 
     # Upload statistics to CI database
-
-    ch_helper = ClickHouseHelper()
     prepared_events = prepare_tests_results_for_clickhouse(
         pr_info,
         [],

From 5403507950373378cc94cba43fdd733f2dd0e48f Mon Sep 17 00:00:00 2001
From: Nikolay Degterinsky <evillique@gmail.com>
Date: Wed, 9 Aug 2023 20:52:09 +0000
Subject: [PATCH 153/249] Add tupleConcat function

---
 docs/en/sql-reference/data-types/tuple.md     |   2 +-
 .../functions/array-functions.md              |   3 +-
 .../functions/tuple-functions.md              |  23 ++++
 src/Functions/concat.cpp                      |   4 +
 src/Functions/tupleConcat.cpp                 | 102 ++++++++++++++++++
 .../0_stateless/02833_tuple_concat.reference  |   6 ++
 .../0_stateless/02833_tuple_concat.sql        |  23 ++++
 7 files changed, 160 insertions(+), 3 deletions(-)
 create mode 100644 src/Functions/tupleConcat.cpp
 create mode 100644 tests/queries/0_stateless/02833_tuple_concat.reference
 create mode 100644 tests/queries/0_stateless/02833_tuple_concat.sql

diff --git a/docs/en/sql-reference/data-types/tuple.md b/docs/en/sql-reference/data-types/tuple.md
index dfe0eda2e21..3add754e6b6 100644
--- a/docs/en/sql-reference/data-types/tuple.md
+++ b/docs/en/sql-reference/data-types/tuple.md
@@ -4,7 +4,7 @@ sidebar_position: 54
 sidebar_label: Tuple(T1, T2, ...)
 ---
 
-# Tuple(t1, T2, …)
+# Tuple(T1, T2, …)
 
 A tuple of elements, each having an individual [type](../../sql-reference/data-types/index.md#data_types). Tuple must contain at least one element.
 
diff --git a/docs/en/sql-reference/functions/array-functions.md b/docs/en/sql-reference/functions/array-functions.md
index 44d385312d0..f66994ed5a6 100644
--- a/docs/en/sql-reference/functions/array-functions.md
+++ b/docs/en/sql-reference/functions/array-functions.md
@@ -183,9 +183,8 @@ arrayConcat(arrays)
 **Arguments**
 
 - `arrays` – Arbitrary number of arguments of [Array](../../sql-reference/data-types/array.md) type.
-    **Example**
 
-<!-- -->
+**Example**
 
 ``` sql
 SELECT arrayConcat([1, 2], [3, 4], [5, 6]) AS res
diff --git a/docs/en/sql-reference/functions/tuple-functions.md b/docs/en/sql-reference/functions/tuple-functions.md
index 7ed2deaeda6..88e4ac03fdb 100644
--- a/docs/en/sql-reference/functions/tuple-functions.md
+++ b/docs/en/sql-reference/functions/tuple-functions.md
@@ -559,6 +559,29 @@ Result:
 └────────────────────────────┘
 ```
 
+## tupleConcat
+
+Combines tuples passed as arguments.
+
+``` sql
+tupleConcat(tuples)
+```
+
+**Arguments**
+
+- `tuples` – Arbitrary number of arguments of [Tuple](../../sql-reference/data-types/tuple.md) type.
+
+**Example**
+
+``` sql
+SELECT tupleConcat((1, 2), (3, 4), (true, false)) AS res
+```
+
+``` text
+┌─res──────────────────┐
+│ (1,2,3,4,true,false) │
+└──────────────────────┘
+```
 
 ## Distance functions
 
diff --git a/src/Functions/concat.cpp b/src/Functions/concat.cpp
index 8288d872f18..9eb222d8c09 100644
--- a/src/Functions/concat.cpp
+++ b/src/Functions/concat.cpp
@@ -208,6 +208,10 @@ public:
         {
             return FunctionFactory::instance().getImpl("mapConcat", context)->build(arguments);
         }
+        else if (isTuple(arguments.at(0).type))
+        {
+            return FunctionFactory::instance().getImpl("tupleConcat", context)->build(arguments);
+        }
         else
             return std::make_unique<FunctionToFunctionBaseAdaptor>(
                 FunctionConcat::create(context), collections::map<DataTypes>(arguments, [](const auto & elem) { return elem.type; }), return_type);
diff --git a/src/Functions/tupleConcat.cpp b/src/Functions/tupleConcat.cpp
new file mode 100644
index 00000000000..0556f4181e6
--- /dev/null
+++ b/src/Functions/tupleConcat.cpp
@@ -0,0 +1,102 @@
+#include <Columns/ColumnTuple.h>
+#include <DataTypes/DataTypeTuple.h>
+#include <Functions/FunctionFactory.h>
+#include <Functions/FunctionHelpers.h>
+#include <Functions/IFunction.h>
+
+namespace DB
+{
+namespace ErrorCodes
+{
+    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+    extern const int ILLEGAL_COLUMN;
+}
+
+/// tupleConcat(tup1, ...) - concatenate tuples.
+class FunctionTupleConcat : public IFunction
+{
+public:
+    static constexpr auto name = "tupleConcat";
+    static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionTupleConcat>(); }
+
+    String getName() const override { return name; }
+
+    bool isVariadic() const override { return true; }
+
+    size_t getNumberOfArguments() const override { return 0; }
+
+    bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
+
+    bool useDefaultImplementationForConstants() const override { return true; }
+
+    DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
+    {
+        if (arguments.empty())
+            throw Exception(
+                ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
+                "Function {} requires at least one argument.",
+                getName());
+
+        DataTypes tuple_arg_types;
+
+        for (const auto arg_idx : collections::range(0, arguments.size()))
+        {
+            const auto * arg = arguments[arg_idx].get();
+            if (!isTuple(arg))
+                throw Exception(
+                    ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+                    "Illegal type {} of argument {} of function {}",
+                    arg->getName(),
+                    arg_idx + 1,
+                    getName());
+
+            const auto * type = checkAndGetDataType<DataTypeTuple>(arg);
+            for (const auto & elem : type->getElements())
+                tuple_arg_types.push_back(elem);
+        }
+
+        return std::make_shared<DataTypeTuple>(tuple_arg_types);
+    }
+
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t) const override
+    {
+        const size_t num_arguments = arguments.size();
+        Columns columns;
+
+        for (size_t i = 0; i < num_arguments; i++)
+        {
+            const DataTypeTuple * arg_type = checkAndGetDataType<DataTypeTuple>(arguments[i].type.get());
+
+            if (!arg_type)
+                throw Exception(
+                    ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+                    "Illegal type {} of argument {} of function {}",
+                    arguments[i].type->getName(),
+                    i + 1,
+                    getName());
+
+            ColumnPtr arg_col = arguments[i].column->convertToFullColumnIfConst();
+            const ColumnTuple * tuple_col = checkAndGetColumn<ColumnTuple>(arg_col.get());
+
+            if (!tuple_col)
+                throw Exception(
+                    ErrorCodes::ILLEGAL_COLUMN,
+                    "Illegal column {} of argument of function {}",
+                    arguments[i].column->getName(),
+                    getName());
+
+            for (const auto & inner_col : tuple_col->getColumns())
+                columns.push_back(inner_col);
+        }
+
+        return ColumnTuple::create(columns);
+    }
+};
+
+REGISTER_FUNCTION(TupleConcat)
+{
+    factory.registerFunction<FunctionTupleConcat>();
+}
+
+}
diff --git a/tests/queries/0_stateless/02833_tuple_concat.reference b/tests/queries/0_stateless/02833_tuple_concat.reference
new file mode 100644
index 00000000000..2c865f13ffc
--- /dev/null
+++ b/tests/queries/0_stateless/02833_tuple_concat.reference
@@ -0,0 +1,6 @@
+(1,'y',2,'n')
+(1,'y',2,'n',3,'n')
+(1,2,3,'a','b','c','2020-10-08','2020-11-08')	1	2	3	a	b	c	2020-10-08	2020-11-08
+(1,2,1,2)	1	2	1	2
+(1,2,3,4)	1	2	3	4
+(3,4,1,2)	3	4	1	2
diff --git a/tests/queries/0_stateless/02833_tuple_concat.sql b/tests/queries/0_stateless/02833_tuple_concat.sql
new file mode 100644
index 00000000000..df43e08d595
--- /dev/null
+++ b/tests/queries/0_stateless/02833_tuple_concat.sql
@@ -0,0 +1,23 @@
+SELECT tupleConcat(); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
+SELECT tupleConcat((1, 'y'), 1); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
+
+SELECT tupleConcat((1, 'y'), (2, 'n'));
+SELECT tupleConcat((1, 'y'), (2, 'n'), (3, 'n'));
+
+WITH (1,2,3) || ('a','b','c') || ('2020-10-08'::Date, '2020-11-08'::Date) AS t
+SELECT t, t.1, t.2, t.3, t.4, t.5, t.6, t.7, t.8;
+
+DROP TABLE IF EXISTS t_02833;
+CREATE TABLE t_02833 (tup Tuple(a UInt64, b UInt64)) ENGINE=Log;
+INSERT INTO t_02833 VALUES ((1, 2));
+
+WITH (tup || tup) AS res
+SELECT res, res.1, res.2, res.3, res.4 FROM t_02833;
+
+WITH (tup || (3, 4)) AS res
+SELECT res, res.1, res.2, res.3, res.4 FROM t_02833;
+
+WITH ((3, 4) || tup) AS res
+SELECT res, res.1, res.2, res.3, res.4 FROM t_02833;
+
+DROP TABLE t_02833;

From 0498b48c70aa60b49ae5a2f6ee702f564921062d Mon Sep 17 00:00:00 2001
From: Yakov Olkhovskiy <yakov@clickhouse.com>
Date: Wed, 9 Aug 2023 21:01:58 +0000
Subject: [PATCH 154/249] different linker

---
 cmake/linux/toolchain-s390x.cmake | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cmake/linux/toolchain-s390x.cmake b/cmake/linux/toolchain-s390x.cmake
index c1cbb849bd3..f5e1cf660c9 100644
--- a/cmake/linux/toolchain-s390x.cmake
+++ b/cmake/linux/toolchain-s390x.cmake
@@ -22,6 +22,8 @@ set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
 set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
 set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=s390x-linux-gnu-ld")
 set (CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fuse-ld=s390x-linux-gnu-ld")
+set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=s390x-linux-gnu-ld")
+set (CMAKE_STATIC_LINKER_FLAGS "${CMAKE_STATIC_LINKER_FLAGS} -fuse-ld=s390x-linux-gnu-ld")
 
 set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
 set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)

From 481792181e6ac9515db966817d123afb99d9971f Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 9 Aug 2023 23:12:39 +0200
Subject: [PATCH 155/249] Remove garbage

---
 .../test_cluster_copier/test_two_nodes.py     |   21 -
 .../test_cassandra.py                         |   99 -
 .../test.py                                   |   68 -
 .../test_default_reading.py                   |    1 -
 .../test_default_string.py                    |    1 -
 .../test_dict_get.py                          |    1 -
 .../test_dict_get_or_default.py               |    1 -
 .../test_extreme_deduplication/test.py        |   70 -
 tests/integration/test_hive_query/__init__.py |    0
 .../test_hive_query/configs/config.xml        |   32 -
 .../test_hive_query/configs/hdfs-site.xml     |    6 -
 .../test_hive_query/data/prepare_hive_data.sh |   15 -
 tests/integration/test_hive_query/test.py     |  498 -----
 .../__init__.py                               |    0
 .../configs/host_regexp.xml                   |   11 -
 .../configs/listen_host.xml                   |    5 -
 .../coredns_config/Corefile                   |    8 -
 .../coredns_config/example.com                |    1 -
 .../test.py                                   |  115 -
 .../test.py                                   |   68 -
 .../test.py                                   |   78 -
 .../test_mutations_with_merge_tree/test.py    |  194 --
 .../test_polymorphic_parts/test.py            |   48 -
 .../__init__.py                               |    0
 .../configs/config.d/storage_conf.xml         |   93 -
 .../test.py                                   |  322 ---
 .../test_reverse_dns_query/__init__.py        |    0
 .../test_reverse_dns_query/configs/config.xml |    3 -
 .../configs/listen_host.xml                   |    5 -
 .../configs/reverse_dns_function.xml          |    3 -
 .../coredns_config/Corefile                   |    4 -
 .../test_reverse_dns_query/test.py            |   56 -
 .../test_s3_zero_copy_replication/test.py     |   90 -
 .../test_s3_zero_copy_ttl/__init__.py         |    0
 .../configs/max_delayed_streams.xml           |    9 -
 .../test_s3_zero_copy_ttl/configs/s3.xml      |   39 -
 .../integration/test_s3_zero_copy_ttl/test.py |   94 -
 .../test_ttl_move_memory_usage.py             |   81 -
 .../test_vertical_merge_memory_usage.py       |   46 -
 .../integration/test_storage_nats/__init__.py |    0
 .../clickhouse_path/format_schemas/nats.proto |    6 -
 .../test_storage_nats/configs/macros.xml      |    7 -
 .../configs/named_collection.xml              |   13 -
 .../test_storage_nats/configs/nats.xml        |    6 -
 .../test_storage_nats/configs/users.xml       |    7 -
 .../test_storage_nats/nats_certs.sh           |   13 -
 .../integration/test_storage_nats/nats_pb2.py |   37 -
 tests/integration/test_storage_nats/test.py   | 1875 -----------------
 .../test_nats_json.reference                  |   50 -
 .../__init__.py                               |    0
 .../configs/log_conf.xml                      |   11 -
 .../test_storage_postgresql_replica/test.py   |  780 -------
 .../test_postgresql_replica.reference         |   50 -
 .../integration/test_storage_rabbitmq/test.py |   63 -
 tests/integration/test_ttl_move/test.py       |  100 -
 55 files changed, 5204 deletions(-)
 delete mode 100644 tests/integration/test_dictionaries_all_layouts_separate_sources/test_cassandra.py
 delete mode 100644 tests/integration/test_hive_query/__init__.py
 delete mode 100644 tests/integration/test_hive_query/configs/config.xml
 delete mode 100644 tests/integration/test_hive_query/configs/hdfs-site.xml
 delete mode 100755 tests/integration/test_hive_query/data/prepare_hive_data.sh
 delete mode 100644 tests/integration/test_hive_query/test.py
 delete mode 100644 tests/integration/test_host_regexp_multiple_ptr_records/__init__.py
 delete mode 100644 tests/integration/test_host_regexp_multiple_ptr_records/configs/host_regexp.xml
 delete mode 100644 tests/integration/test_host_regexp_multiple_ptr_records/configs/listen_host.xml
 delete mode 100644 tests/integration/test_host_regexp_multiple_ptr_records/coredns_config/Corefile
 delete mode 100644 tests/integration/test_host_regexp_multiple_ptr_records/coredns_config/example.com
 delete mode 100644 tests/integration/test_host_regexp_multiple_ptr_records/test.py
 delete mode 100644 tests/integration/test_replicated_merge_tree_hdfs_zero_copy/__init__.py
 delete mode 100644 tests/integration/test_replicated_merge_tree_hdfs_zero_copy/configs/config.d/storage_conf.xml
 delete mode 100644 tests/integration/test_replicated_merge_tree_hdfs_zero_copy/test.py
 delete mode 100644 tests/integration/test_reverse_dns_query/__init__.py
 delete mode 100644 tests/integration/test_reverse_dns_query/configs/config.xml
 delete mode 100644 tests/integration/test_reverse_dns_query/configs/listen_host.xml
 delete mode 100644 tests/integration/test_reverse_dns_query/configs/reverse_dns_function.xml
 delete mode 100644 tests/integration/test_reverse_dns_query/coredns_config/Corefile
 delete mode 100644 tests/integration/test_reverse_dns_query/test.py
 delete mode 100644 tests/integration/test_s3_zero_copy_ttl/__init__.py
 delete mode 100644 tests/integration/test_s3_zero_copy_ttl/configs/max_delayed_streams.xml
 delete mode 100644 tests/integration/test_s3_zero_copy_ttl/configs/s3.xml
 delete mode 100644 tests/integration/test_s3_zero_copy_ttl/test.py
 delete mode 100644 tests/integration/test_s3_zero_copy_ttl/test_ttl_move_memory_usage.py
 delete mode 100644 tests/integration/test_s3_zero_copy_ttl/test_vertical_merge_memory_usage.py
 delete mode 100644 tests/integration/test_storage_nats/__init__.py
 delete mode 100755 tests/integration/test_storage_nats/clickhouse_path/format_schemas/nats.proto
 delete mode 100644 tests/integration/test_storage_nats/configs/macros.xml
 delete mode 100644 tests/integration/test_storage_nats/configs/named_collection.xml
 delete mode 100644 tests/integration/test_storage_nats/configs/nats.xml
 delete mode 100644 tests/integration/test_storage_nats/configs/users.xml
 delete mode 100755 tests/integration/test_storage_nats/nats_certs.sh
 delete mode 100644 tests/integration/test_storage_nats/nats_pb2.py
 delete mode 100644 tests/integration/test_storage_nats/test.py
 delete mode 100644 tests/integration/test_storage_nats/test_nats_json.reference
 delete mode 100644 tests/integration/test_storage_postgresql_replica/__init__.py
 delete mode 100644 tests/integration/test_storage_postgresql_replica/configs/log_conf.xml
 delete mode 100644 tests/integration/test_storage_postgresql_replica/test.py
 delete mode 100644 tests/integration/test_storage_postgresql_replica/test_postgresql_replica.reference

diff --git a/tests/integration/test_cluster_copier/test_two_nodes.py b/tests/integration/test_cluster_copier/test_two_nodes.py
index 2b6fcf6cac2..1bd3561f24f 100644
--- a/tests/integration/test_cluster_copier/test_two_nodes.py
+++ b/tests/integration/test_cluster_copier/test_two_nodes.py
@@ -595,24 +595,3 @@ def execute_task(started_cluster, task, cmd_options):
         task.check()
     finally:
         zk.delete(task.zk_task_path, recursive=True)
-
-
-# Tests
-@pytest.mark.skip(reason="Too flaky :(")
-def test_different_schema(started_cluster):
-    execute_task(started_cluster, TaskWithDifferentSchema(started_cluster), [])
-
-
-@pytest.mark.skip(reason="Too flaky :(")
-def test_ttl_columns(started_cluster):
-    execute_task(started_cluster, TaskTTL(started_cluster), [])
-
-
-@pytest.mark.skip(reason="Too flaky :(")
-def test_skip_index(started_cluster):
-    execute_task(started_cluster, TaskSkipIndex(started_cluster), [])
-
-
-@pytest.mark.skip(reason="Too flaky :(")
-def test_ttl_move_to_volume(started_cluster):
-    execute_task(started_cluster, TaskTTLMoveToVolume(started_cluster), [])
diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_cassandra.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_cassandra.py
deleted file mode 100644
index 90287e19bd0..00000000000
--- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_cassandra.py
+++ /dev/null
@@ -1,99 +0,0 @@
-import os
-import math
-import pytest
-
-# FIXME This test is too flaky
-# https://github.com/ClickHouse/ClickHouse/issues/33006
-
-pytestmark = pytest.mark.skip
-
-from .common import *
-
-from helpers.cluster import ClickHouseCluster
-from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout
-from helpers.external_sources import SourceCassandra
-
-SOURCE = None
-cluster = None
-node = None
-simple_tester = None
-complex_tester = None
-ranged_tester = None
-test_name = "cassandra"
-
-
-def setup_module(module):
-    global cluster
-    global node
-    global simple_tester
-    global complex_tester
-    global ranged_tester
-
-    cluster = ClickHouseCluster(__file__)
-
-    SOURCE = SourceCassandra(
-        "Cassandra",
-        None,
-        cluster.cassandra_port,
-        cluster.cassandra_host,
-        cluster.cassandra_port,
-        "",
-        "",
-    )
-
-    simple_tester = SimpleLayoutTester(test_name)
-    simple_tester.cleanup()
-    simple_tester.create_dictionaries(SOURCE)
-
-    complex_tester = ComplexLayoutTester(test_name)
-    complex_tester.create_dictionaries(SOURCE)
-
-    ranged_tester = RangedLayoutTester(test_name)
-    ranged_tester.create_dictionaries(SOURCE)
-    # Since that all .xml configs were created
-
-    main_configs = []
-    main_configs.append(os.path.join("configs", "disable_ssl_verification.xml"))
-
-    dictionaries = simple_tester.list_dictionaries()
-
-    node = cluster.add_instance(
-        "cass_node",
-        main_configs=main_configs,
-        dictionaries=dictionaries,
-        with_cassandra=True,
-    )
-
-
-def teardown_module(module):
-    simple_tester.cleanup()
-
-
-@pytest.fixture(scope="module")
-def started_cluster():
-    try:
-        cluster.start()
-
-        simple_tester.prepare(cluster)
-        complex_tester.prepare(cluster)
-        ranged_tester.prepare(cluster)
-
-        yield cluster
-
-    finally:
-        cluster.shutdown()
-
-
-@pytest.mark.parametrize("layout_name", sorted(LAYOUTS_SIMPLE))
-def test_simple(started_cluster, layout_name):
-    simple_tester.execute(layout_name, node)
-
-
-@pytest.mark.parametrize("layout_name", sorted(LAYOUTS_COMPLEX))
-def test_complex(started_cluster, layout_name):
-    complex_tester.execute(layout_name, node)
-
-
-@pytest.mark.parametrize("layout_name", sorted(LAYOUTS_RANGED))
-def test_ranged(started_cluster, layout_name):
-    ranged_tester.execute(layout_name, node)
diff --git a/tests/integration/test_dictionaries_complex_key_cache_string/test.py b/tests/integration/test_dictionaries_complex_key_cache_string/test.py
index ae9cd4e7c91..bca7122a2a8 100644
--- a/tests/integration/test_dictionaries_complex_key_cache_string/test.py
+++ b/tests/integration/test_dictionaries_complex_key_cache_string/test.py
@@ -27,71 +27,3 @@ def started_cluster():
         yield cluster
     finally:
         cluster.shutdown()
-
-
-@pytest.mark.skip(reason="SSD cache test can run on disk only")
-@pytest.mark.parametrize("type", ["memory", "ssd"])
-def test_memory_consumption(started_cluster, type):
-    node = started_cluster.instances[f"node_{type}"]
-    node.query(
-        "insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format(
-            "w" * 8
-        )
-    )
-    node.query(
-        "insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format(
-            "x" * 16
-        )
-    )
-    node.query(
-        "insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format(
-            "y" * 32
-        )
-    )
-    node.query(
-        "insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format(
-            "z" * 64
-        )
-    )
-
-    # Fill dictionary
-    node.query(
-        "select dictGetString('radars', 'client_id', tuple(toString(number))) from numbers(0, 5000)"
-    )
-
-    allocated_first = int(
-        node.query(
-            "select bytes_allocated from system.dictionaries where name = 'radars'"
-        ).strip()
-    )
-
-    alloc_array = []
-    for i in range(5):
-        node.query(
-            "select dictGetString('radars', 'client_id', tuple(toString(number))) from numbers(0, 5000)"
-        )
-
-        allocated = int(
-            node.query(
-                "select bytes_allocated from system.dictionaries where name = 'radars'"
-            ).strip()
-        )
-        alloc_array.append(allocated)
-
-    # size doesn't grow
-    assert all(allocated_first >= a for a in alloc_array)
-
-    for i in range(5):
-        node.query(
-            "select dictGetString('radars', 'client_id', tuple(toString(number))) from numbers(0, 5000)"
-        )
-
-        allocated = int(
-            node.query(
-                "select bytes_allocated from system.dictionaries where name = 'radars'"
-            ).strip()
-        )
-        alloc_array.append(allocated)
-
-    # size doesn't grow
-    assert all(allocated_first >= a for a in alloc_array)
diff --git a/tests/integration/test_dictionary_allow_read_expired_keys/test_default_reading.py b/tests/integration/test_dictionary_allow_read_expired_keys/test_default_reading.py
index 85c45d5df3c..921fb4e4154 100644
--- a/tests/integration/test_dictionary_allow_read_expired_keys/test_default_reading.py
+++ b/tests/integration/test_dictionary_allow_read_expired_keys/test_default_reading.py
@@ -38,7 +38,6 @@ def started_cluster():
         cluster.shutdown()
 
 
-# @pytest.mark.skip(reason="debugging")
 def test_default_reading(started_cluster):
     assert None != dictionary_node.get_process_pid(
         "clickhouse"
diff --git a/tests/integration/test_dictionary_allow_read_expired_keys/test_default_string.py b/tests/integration/test_dictionary_allow_read_expired_keys/test_default_string.py
index 92d681698bc..6eedf63f95c 100644
--- a/tests/integration/test_dictionary_allow_read_expired_keys/test_default_string.py
+++ b/tests/integration/test_dictionary_allow_read_expired_keys/test_default_string.py
@@ -53,7 +53,6 @@ def started_cluster():
         cluster.shutdown()
 
 
-# @pytest.mark.skip(reason="debugging")
 def test_return_real_values(started_cluster):
     assert None != dictionary_node.get_process_pid(
         "clickhouse"
diff --git a/tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get.py b/tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get.py
index 05f638ec337..8e45af44640 100644
--- a/tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get.py
+++ b/tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get.py
@@ -38,7 +38,6 @@ def started_cluster():
         cluster.shutdown()
 
 
-# @pytest.mark.skip(reason="debugging")
 def test_simple_dict_get(started_cluster):
     assert None != dictionary_node.get_process_pid(
         "clickhouse"
diff --git a/tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get_or_default.py b/tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get_or_default.py
index 1da8fd3325a..8d6242f4711 100644
--- a/tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get_or_default.py
+++ b/tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get_or_default.py
@@ -38,7 +38,6 @@ def started_cluster():
         cluster.shutdown()
 
 
-# @pytest.mark.skip(reason="debugging")
 def test_simple_dict_get_or_default(started_cluster):
     assert None != dictionary_node.get_process_pid(
         "clickhouse"
diff --git a/tests/integration/test_extreme_deduplication/test.py b/tests/integration/test_extreme_deduplication/test.py
index 03451933171..3632369154a 100644
--- a/tests/integration/test_extreme_deduplication/test.py
+++ b/tests/integration/test_extreme_deduplication/test.py
@@ -71,73 +71,3 @@ def test_deduplication_window_in_seconds(started_cluster):
     assert TSV.toMat(node.query("SELECT count() FROM simple"))[0][0] == "3"
 
     node1.query("""DROP TABLE simple ON CLUSTER test_cluster""")
-
-
-# Currently this test just reproduce incorrect behavior that sould be fixed
-@pytest.mark.skip(reason="Flapping test")
-def test_deduplication_works_in_case_of_intensive_inserts(started_cluster):
-    inserters = []
-    fetchers = []
-
-    node1.query(
-        """
-        CREATE TABLE simple ON CLUSTER test_cluster (date Date, id UInt32)
-        ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/simple', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY id"""
-    )
-
-    node1.query("INSERT INTO simple VALUES (0, 0)")
-
-    for node in nodes:
-        host = node.ip_address
-
-        inserters.append(
-            CommandRequest(
-                ["/bin/bash"],
-                timeout=10,
-                stdin="""
-set -e
-for i in `seq 1000`; do
-    {} --host {} -q "INSERT INTO simple VALUES (0, 0)"
-done
-""".format(
-                    cluster.get_client_cmd(), host
-                ),
-            )
-        )
-
-        fetchers.append(
-            CommandRequest(
-                ["/bin/bash"],
-                timeout=10,
-                stdin="""
-set -e
-for i in `seq 1000`; do
-    res=`{} --host {} -q "SELECT count() FROM simple"`
-    if [[ $? -ne 0 || $res -ne 1 ]]; then
-        echo "Selected $res elements! Host: {}" 1>&2
-        exit -1
-    fi;
-done
-""".format(
-                    cluster.get_client_cmd(), host, node.name
-                ),
-            )
-        )
-
-    # There were not errors during INSERTs
-    for inserter in inserters:
-        try:
-            inserter.get_answer()
-        except QueryTimeoutExceedException:
-            # Only timeout is accepted
-            pass
-
-    # There were not errors during SELECTs
-    for fetcher in fetchers:
-        try:
-            fetcher.get_answer()
-        except QueryTimeoutExceedException:
-            # Only timeout is accepted
-            pass
-
-    node1.query("""DROP TABLE simple ON CLUSTER test_cluster""")
diff --git a/tests/integration/test_hive_query/__init__.py b/tests/integration/test_hive_query/__init__.py
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/integration/test_hive_query/configs/config.xml b/tests/integration/test_hive_query/configs/config.xml
deleted file mode 100644
index 7de1391e56c..00000000000
--- a/tests/integration/test_hive_query/configs/config.xml
+++ /dev/null
@@ -1,32 +0,0 @@
-<clickhouse>
-    <remote_servers  >
-        <!-- Test only shard config for testing distributed storage -->
-        <simple>
-            <shard>
-                <replica>
-                    <host>localhost</host>
-                    <port>9000</port>
-                </replica>
-            </shard>
-        </simple>
-    </remote_servers>
-
-    <local_cache_for_remote_fs>
-        <enable>true</enable>
-        <root_dir>/tmp/clickhouse_local_cache,/tmp/clickhouse_local_cache1</root_dir>
-        <limit_size>207374182400</limit_size>
-        <bytes_read_befor_flush>1048576</bytes_read_befor_flush>
-    </local_cache_for_remote_fs>
-    
-    <hdfs>
-	    <libhdfs3_conf>/etc/clickhouse-server/extra_conf.d/hdfs-site.xml</libhdfs3_conf>
-    </hdfs>
-
-    <metric_log>
-        <database>system</database>
-        <table>metric_log</table>
-        <flush_interval_milliseconds>1000</flush_interval_milliseconds>
-        <collect_interval_milliseconds>1000</collect_interval_milliseconds>
-    </metric_log>
-
-</clickhouse>
diff --git a/tests/integration/test_hive_query/configs/hdfs-site.xml b/tests/integration/test_hive_query/configs/hdfs-site.xml
deleted file mode 100644
index 82c525ea414..00000000000
--- a/tests/integration/test_hive_query/configs/hdfs-site.xml
+++ /dev/null
@@ -1,6 +0,0 @@
-<configuration>
-    <property>
-        <name>dfs.replication</name>
-        <value>1</value>
-    </property>
-</configuration>
diff --git a/tests/integration/test_hive_query/data/prepare_hive_data.sh b/tests/integration/test_hive_query/data/prepare_hive_data.sh
deleted file mode 100755
index 495ea201870..00000000000
--- a/tests/integration/test_hive_query/data/prepare_hive_data.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/bash
-hive -e "create database test"
-
-hive -e "drop table if exists test.demo; create table test.demo(id string, score int) PARTITIONED BY(day string) ROW FORMAT SERDE   'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'  STORED AS INPUTFORMAT  'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' OUTPUTFORMAT  'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'; create table test.demo_orc(id string, score int) PARTITIONED BY(day string) ROW FORMAT SERDE   'org.apache.hadoop.hive.ql.io.orc.OrcSerde'  STORED AS INPUTFORMAT  'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' OUTPUTFORMAT  'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat'; "
-hive -e "drop table if exists test.parquet_demo; create table test.parquet_demo(id string, score int) PARTITIONED BY(day string, hour string) ROW FORMAT SERDE   'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'  STORED AS INPUTFORMAT  'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' OUTPUTFORMAT  'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'"
-hive -e "drop table if exists test.demo_text; create table test.demo_text(id string, score int, day string)row format delimited fields terminated by ','; load data local inpath '/demo_data.txt' into table test.demo_text "
-hive -e "set hive.exec.dynamic.partition.mode=nonstrict;insert into test.demo partition(day) select * from test.demo_text; insert into test.demo_orc partition(day) select * from test.demo_text"
-
-hive -e "set hive.exec.dynamic.partition.mode=nonstrict;insert into test.parquet_demo partition(day, hour) select id, score, day, '00' as hour from test.demo;"
-hive -e "set hive.exec.dynamic.partition.mode=nonstrict;insert into test.parquet_demo partition(day, hour) select id, score, day, '01' as hour from test.demo;"
-
-hive -e "drop table if exists test.test_hive_types; CREATE TABLE test.test_hive_types( f_tinyint tinyint, f_smallint smallint, f_int int, f_integer int, f_bigint bigint, f_float float, f_double double, f_decimal decimal(10,0), f_timestamp timestamp, f_date date, f_string string, f_varchar varchar(100), f_char char(100), f_bool boolean, f_array_int array<int>, f_array_string array<string>, f_array_float array<float>, f_map_int map<string, int>, f_map_string map<string, string>, f_map_float map<string, float>, f_struct struct<a:string, b:int, c:float, d: struct<x:int, y:string>>) PARTITIONED BY( day string) ROW FORMAT SERDE   'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'  STORED AS INPUTFORMAT  'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' OUTPUTFORMAT  'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat';"
-
-hive -e "insert into test.test_hive_types partition(day='2022-02-20') select 1, 2, 3, 4, 5, 6.11, 7.22, 8.333, '2022-02-20 14:47:04', '2022-02-20', 'hello world', 'hello world', 'hello world', true, array(1,2,3), array('hello world', 'hello world'), array(float(1.1),float(1.2)), map('a', 100, 'b', 200, 'c', 300), map('a', 'aa', 'b', 'bb', 'c', 'cc'), map('a', float(111.1), 'b', float(222.2), 'c', float(333.3)), named_struct('a', 'aaa', 'b', 200, 'c', float(333.3), 'd', named_struct('x', 10, 'y', 'xyz')); insert into test.test_hive_types partition(day='2022-02-19') select 1, 2, 3, 4, 5, 6.11, 7.22, 8.333, '2022-02-19 14:47:04', '2022-02-19', 'hello world', 'hello world', 'hello world', true, array(1,2,3), array('hello world', 'hello world'), array(float(1.1),float(1.2)), map('a', 100, 'b', 200, 'c', 300), map('a', 'aa', 'b', 'bb', 'c', 'cc'), map('a', float(111.1), 'b', float(222.2), 'c', float(333.3)), named_struct('a', 'aaa', 'b', 200, 'c', float(333.3), 'd', named_struct('x', 11, 'y', 'abc'));"
-
diff --git a/tests/integration/test_hive_query/test.py b/tests/integration/test_hive_query/test.py
deleted file mode 100644
index 791ae03f9f6..00000000000
--- a/tests/integration/test_hive_query/test.py
+++ /dev/null
@@ -1,498 +0,0 @@
-import pytest
-
-# FIXME This test is too flaky
-# https://github.com/ClickHouse/ClickHouse/issues/43541
-
-pytestmark = pytest.mark.skip
-
-import logging
-import os
-
-import time
-from helpers.cluster import ClickHouseCluster
-from helpers.test_tools import TSV
-
-logging.getLogger().setLevel(logging.INFO)
-logging.getLogger().addHandler(logging.StreamHandler())
-
-SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
-
-
-@pytest.fixture(scope="module")
-def started_cluster():
-    try:
-        cluster = ClickHouseCluster(__file__)
-        cluster.add_instance(
-            "h0_0_0",
-            main_configs=["configs/config.xml"],
-            extra_configs=["configs/hdfs-site.xml", "data/prepare_hive_data.sh"],
-            with_hive=True,
-        )
-
-        logging.info("Starting cluster ...")
-        cluster.start()
-        cluster.copy_file_to_container(
-            "roottesthivequery_hdfs1_1",
-            "/ClickHouse/tests/integration/test_hive_query/data/prepare_hive_data.sh",
-            "/prepare_hive_data.sh",
-        )
-        cluster.exec_in_container(
-            "roottesthivequery_hdfs1_1", ["bash", "-c", "bash /prepare_hive_data.sh"]
-        )
-        yield cluster
-    finally:
-        cluster.shutdown()
-
-
-def test_create_parquet_table(started_cluster):
-    logging.info("Start testing creating hive table ...")
-    node = started_cluster.instances["h0_0_0"]
-    test_passed = False
-    for i in range(10):
-        node.query("set input_format_parquet_allow_missing_columns = true")
-        result = node.query(
-            """
-DROP TABLE IF EXISTS default.demo_parquet;
-CREATE TABLE default.demo_parquet (`id` Nullable(String), `score` Nullable(Int32), `day` Nullable(String)) ENGINE = Hive('thrift://hivetest:9083', 'test', 'demo') PARTITION BY(day)
-            """
-        )
-        logging.info("create result {}".format(result))
-        if result.strip() == "":
-            test_passed = True
-            break
-        time.sleep(60)
-    assert test_passed
-
-
-def test_create_parquet_table_1(started_cluster):
-    logging.info("Start testing creating hive table ...")
-    node = started_cluster.instances["h0_0_0"]
-    for i in range(10):
-        node.query("set input_format_parquet_allow_missing_columns = true")
-        result = node.query(
-            """
-DROP TABLE IF EXISTS default.demo_parquet_parts;
-CREATE TABLE default.demo_parquet_parts (`id` Nullable(String), `score` Nullable(Int32), `day` Nullable(String), `hour` String) ENGINE = Hive('thrift://hivetest:9083', 'test', 'parquet_demo') PARTITION BY(day, hour);
-            """
-        )
-        logging.info("create result {}".format(result))
-        if result.strip() == "":
-            test_passed = True
-            break
-        time.sleep(60)
-    assert test_passed
-
-
-def test_create_orc_table(started_cluster):
-    logging.info("Start testing creating hive table ...")
-    node = started_cluster.instances["h0_0_0"]
-    test_passed = False
-    for i in range(10):
-        result = node.query(
-            """
-    DROP TABLE IF EXISTS default.demo_orc;
-    CREATE TABLE default.demo_orc (`id` Nullable(String), `score` Nullable(Int32), `day` Nullable(String)) ENGINE = Hive('thrift://hivetest:9083', 'test', 'demo_orc') PARTITION BY(day)
-            """
-        )
-        logging.info("create result {}".format(result))
-        if result.strip() == "":
-            test_passed = True
-            break
-        time.sleep(60)
-
-    assert test_passed
-
-
-def test_create_text_table(started_cluster):
-    logging.info("Start testing creating hive table ...")
-    node = started_cluster.instances["h0_0_0"]
-    result = node.query(
-        """
-    DROP TABLE IF EXISTS default.demo_text;
-    CREATE TABLE default.demo_text (`id` Nullable(String), `score` Nullable(Int32), `day` Nullable(String)) ENGINE = Hive('thrift://hivetest:9083', 'test', 'demo_text') PARTITION BY (tuple())
-            """
-    )
-    logging.info("create result {}".format(result))
-
-    assert result.strip() == ""
-
-
-def test_parquet_groupby(started_cluster):
-    logging.info("Start testing groupby ...")
-    node = started_cluster.instances["h0_0_0"]
-    result = node.query(
-        """
-    SELECT day, count(*) FROM default.demo_parquet group by day order by day
-            """
-    )
-    expected_result = """2021-11-01	1
-2021-11-05	2
-2021-11-11	1
-2021-11-16	2
-"""
-    assert result == expected_result
-
-
-def test_parquet_in_filter(started_cluster):
-    logging.info("Start testing groupby ...")
-    node = started_cluster.instances["h0_0_0"]
-    result = node.query(
-        """
-    SELECT count(*) FROM default.demo_parquet_parts where day = '2021-11-05' and hour in ('00')
-            """
-    )
-    expected_result = """2
-"""
-    logging.info("query result:{}".format(result))
-    assert result == expected_result
-
-
-def test_orc_groupby(started_cluster):
-    logging.info("Start testing groupby ...")
-    node = started_cluster.instances["h0_0_0"]
-    result = node.query(
-        """
-    DROP TABLE IF EXISTS default.demo_orc;
-    CREATE TABLE default.demo_orc (`id` Nullable(String), `score` Nullable(Int32), `day` Nullable(String)) ENGINE = Hive('thrift://hivetest:9083', 'test', 'demo_orc') PARTITION BY(day);
-    SELECT day, count(*) FROM default.demo_orc group by day order by day
-            """
-    )
-    expected_result = """2021-11-01	1
-2021-11-05	2
-2021-11-11	1
-2021-11-16	2
-"""
-    assert result == expected_result
-
-
-@pytest.mark.parametrize(
-    "table,use_local_cache_for_remote_storage,enable_orc_file_minmax_index,enable_orc_stripe_minmax_index",
-    [
-        pytest.param(
-            "demo_orc_no_cache_no_index",
-            "false",
-            "false",
-            "false",
-            id="demo_orc_no_cache_no_index",
-        ),
-        pytest.param(
-            "demo_orc_with_cache_no_index",
-            "true",
-            "false",
-            "false",
-            id="demo_orc_with_cache_no_index",
-        ),
-        pytest.param(
-            "demo_orc_no_cache_file_index",
-            "false",
-            "true",
-            "false",
-            id="demo_orc_no_cache_file_index",
-        ),
-        pytest.param(
-            "demo_orc_with_cache_file_index",
-            "true",
-            "true",
-            "false",
-            id="demo_orc_with_cache_file_index",
-        ),
-        pytest.param(
-            "demo_orc_no_cache_stripe_index",
-            "false",
-            "true",
-            "true",
-            id="demo_orc_no_cache_stripe_index",
-        ),
-        pytest.param(
-            "demo_orc_with_cache_stripe_index",
-            "true",
-            "true",
-            "true",
-            id="demo_orc_with_cache_stripe_index",
-        ),
-    ],
-)
-def test_orc_minmax_index(
-    started_cluster,
-    table,
-    use_local_cache_for_remote_storage,
-    enable_orc_file_minmax_index,
-    enable_orc_stripe_minmax_index,
-):
-    node = started_cluster.instances["h0_0_0"]
-    result = node.query(
-        """
-        DROP TABLE IF EXISTS default.{table};
-        CREATE TABLE default.{table} (`id` Nullable(String), `score` Nullable(Int32), `day` Nullable(String)) ENGINE = Hive('thrift://hivetest:9083', 'test', 'demo_orc') PARTITION BY(day)
-        SETTINGS enable_orc_file_minmax_index = {enable_orc_file_minmax_index}, enable_orc_stripe_minmax_index = {enable_orc_stripe_minmax_index};
-    """.format(
-            table=table,
-            enable_orc_file_minmax_index=enable_orc_file_minmax_index,
-            enable_orc_stripe_minmax_index=enable_orc_stripe_minmax_index,
-        )
-    )
-    assert result.strip() == ""
-
-    for i in range(2):
-        result = node.query(
-            """
-            SELECT day, id, score FROM default.{table} where day >= '2021-11-05' and day <= '2021-11-16' and score >= 15 and score <= 30 order by day, id
-            SETTINGS use_local_cache_for_remote_storage = {use_local_cache_for_remote_storage}
-            """.format(
-                table=table,
-                use_local_cache_for_remote_storage=use_local_cache_for_remote_storage,
-            )
-        )
-
-        assert (
-            result
-            == """2021-11-05	abd	15
-2021-11-16	aaa	22
-"""
-        )
-
-
-@pytest.mark.parametrize(
-    "table,use_local_cache_for_remote_storage,enable_parquet_rowgroup_minmax_index",
-    [
-        pytest.param(
-            "demo_parquet_no_cache_no_index",
-            "false",
-            "false",
-            id="demo_parquet_no_cache_no_index",
-        ),
-        pytest.param(
-            "demo_parquet_with_cache_no_index",
-            "true",
-            "false",
-            id="demo_parquet_with_cache_no_index",
-        ),
-        pytest.param(
-            "demo_parquet_no_cache_rowgroup_index",
-            "false",
-            "true",
-            id="demo_parquet_no_cache_rowgroup_index",
-        ),
-        pytest.param(
-            "demo_parquet_with_cache_rowgroup_index",
-            "true",
-            "true",
-            id="demo_parquet_with_cache_rowgroup_index",
-        ),
-    ],
-)
-def test_parquet_minmax_index(
-    started_cluster,
-    table,
-    use_local_cache_for_remote_storage,
-    enable_parquet_rowgroup_minmax_index,
-):
-    node = started_cluster.instances["h0_0_0"]
-    result = node.query(
-        """
-        DROP TABLE IF EXISTS default.{table};
-        CREATE TABLE default.{table} (`id` Nullable(String), `score` Nullable(Int32), `day` Nullable(String)) ENGINE = Hive('thrift://hivetest:9083', 'test', 'demo') PARTITION BY(day)
-        SETTINGS enable_parquet_rowgroup_minmax_index = {enable_parquet_rowgroup_minmax_index}
-    """.format(
-            table=table,
-            enable_parquet_rowgroup_minmax_index=enable_parquet_rowgroup_minmax_index,
-        )
-    )
-    assert result.strip() == ""
-
-    for i in range(2):
-        result = node.query(
-            """
-            SELECT day, id, score FROM default.{table} where day >= '2021-11-05' and day <= '2021-11-16' and score >= 15 and score <= 30 order by day, id
-            SETTINGS use_local_cache_for_remote_storage = {use_local_cache_for_remote_storage}
-            """.format(
-                table=table,
-                use_local_cache_for_remote_storage=use_local_cache_for_remote_storage,
-            )
-        )
-
-        assert (
-            result
-            == """2021-11-05	abd	15
-2021-11-16	aaa	22
-"""
-        )
-
-
-def test_hive_columns_prunning(started_cluster):
-    logging.info("Start testing groupby ...")
-    node = started_cluster.instances["h0_0_0"]
-    result = node.query(
-        """
-    SELECT count(*) FROM default.demo_parquet_parts where day = '2021-11-05'
-            """
-    )
-    expected_result = """4
-"""
-    logging.info("query result:{}".format(result))
-    assert result == expected_result
-
-
-def test_text_count(started_cluster):
-    node = started_cluster.instances["h0_0_0"]
-    result = node.query(
-        """
-    DROP TABLE IF EXISTS default.demo_orc;
-    CREATE TABLE default.demo_orc (`id` Nullable(String), `score` Nullable(Int32), `day` Nullable(String)) ENGINE = Hive('thrift://hivetest:9083', 'test', 'demo_orc') PARTITION BY(day);
-    SELECT day, count(*) FROM default.demo_orc group by day order by day SETTINGS format_csv_delimiter = '\x01'
-            """
-    )
-    expected_result = """2021-11-01	1
-2021-11-05	2
-2021-11-11	1
-2021-11-16	2
-"""
-    assert result == expected_result
-
-
-def test_parquet_groupby_with_cache(started_cluster):
-    logging.info("Start testing groupby ...")
-    node = started_cluster.instances["h0_0_0"]
-    result = node.query(
-        """
-    SELECT day, count(*) FROM default.demo_parquet group by day order by day
-            """
-    )
-    expected_result = """2021-11-01	1
-2021-11-05	2
-2021-11-11	1
-2021-11-16	2
-"""
-    assert result == expected_result
-
-
-def test_parquet_groupby_by_hive_function(started_cluster):
-    logging.info("Start testing groupby ...")
-    node = started_cluster.instances["h0_0_0"]
-    result = node.query(
-        """
-    SELECT day, count(*) FROM hive('thrift://hivetest:9083', 'test', 'demo', '`id` Nullable(String), `score` Nullable(Int32), `day` Nullable(String)', 'day') group by day order by day
-            """
-    )
-    expected_result = """2021-11-01	1
-2021-11-05	2
-2021-11-11	1
-2021-11-16	2
-"""
-    assert result == expected_result
-
-
-def test_cache_read_bytes(started_cluster):
-    node = started_cluster.instances["h0_0_0"]
-    result = node.query(
-        """
-    CREATE TABLE IF NOT EXISTS default.demo_parquet_1 (`id` Nullable(String), `score` Nullable(Int32), `day` Nullable(String)) ENGINE = Hive('thrift://hivetest:9083', 'test', 'demo') PARTITION BY(day)
-            """
-    )
-    test_passed = False
-    for i in range(10):
-        result = node.query(
-            """
-    SELECT * FROM default.demo_parquet_1 settings input_format_parquet_allow_missing_columns = true
-            """
-        )
-        node.query("system flush logs")
-        result = node.query(
-            "select sum(ProfileEvent_ExternalDataSourceLocalCacheReadBytes)  from system.metric_log where ProfileEvent_ExternalDataSourceLocalCacheReadBytes > 0"
-        )
-        if result.strip() == "0":
-            logging.info("ProfileEvent_ExternalDataSourceLocalCacheReadBytes == 0")
-            time.sleep(10)
-            continue
-        test_passed = True
-        break
-    assert test_passed
-
-
-def test_cache_dir_use(started_cluster):
-    node = started_cluster.instances["h0_0_0"]
-    result0 = node.exec_in_container(
-        ["bash", "-c", "ls /tmp/clickhouse_local_cache | wc -l"]
-    )
-    result1 = node.exec_in_container(
-        ["bash", "-c", "ls /tmp/clickhouse_local_cache1 | wc -l"]
-    )
-    assert result0 != "0" and result1 != "0"
-
-
-def test_hive_struct_type(started_cluster):
-    node = started_cluster.instances["h0_0_0"]
-    result = node.query(
-        """
-        CREATE TABLE IF NOT EXISTS default.test_hive_types (`f_tinyint` Int8, `f_smallint` Int16, `f_int` Int32, `f_integer` Int32, `f_bigint` Int64, `f_float` Float32, `f_double` Float64, `f_decimal` Float64, `f_timestamp` DateTime, `f_date` Date, `f_string` String, `f_varchar` String, `f_char` String, `f_bool` Boolean, `f_array_int` Array(Int32), `f_array_string` Array(String), `f_array_float` Array(Float32), `f_map_int` Map(String, Int32), `f_map_string` Map(String, String), `f_map_float` Map(String, Float32), `f_struct` Tuple(a String, b Int32, c Float32, d Tuple(x Int32, y String)), `day` String) ENGINE = Hive('thrift://hivetest:9083', 'test', 'test_hive_types') PARTITION BY (day)
-        """
-    )
-    result = node.query(
-        """
-    SELECT * FROM default.test_hive_types WHERE day = '2022-02-20' SETTINGS input_format_parquet_import_nested=1
-        """
-    )
-    expected_result = """1	2	3	4	5	6.11	7.22	8	2022-02-20 14:47:04	2022-02-20	hello world	hello world	hello world	true	[1,2,3]	['hello world','hello world']	[1.1,1.2]	{'a':100,'b':200,'c':300}	{'a':'aa','b':'bb','c':'cc'}	{'a':111.1,'b':222.2,'c':333.3}	('aaa',200,333.3,(10,'xyz'))	2022-02-20"""
-    assert result.strip() == expected_result
-
-    result = node.query(
-        """
-    SELECT day, f_struct.a, f_struct.d.x FROM default.test_hive_types WHERE day = '2022-02-20' SETTINGS input_format_parquet_import_nested=1
-        """
-    )
-    expected_result = """2022-02-20	aaa	10"""
-
-
-def test_table_alter_add(started_cluster):
-    node = started_cluster.instances["h0_0_0"]
-    result = node.query("DROP TABLE IF EXISTS default.demo_parquet_1")
-    result = node.query(
-        """
-CREATE TABLE IF NOT EXISTS default.demo_parquet_1 (`score` Nullable(Int32), `day` Nullable(String)) ENGINE = Hive('thrift://hivetest:9083', 'test', 'demo') PARTITION BY(day)
-        """
-    )
-    result = node.query(
-        """
-ALTER TABLE default.demo_parquet_1 ADD COLUMN id Nullable(String) FIRST
-        """
-    )
-    result = node.query("""DESC default.demo_parquet_1 FORMAT TSV""")
-
-    expected_result = "id\tNullable(String)\t\t\t\t\t\nscore\tNullable(Int32)\t\t\t\t\t\nday\tNullable(String)"
-    assert result.strip() == expected_result
-
-
-def test_table_alter_drop(started_cluster):
-    node = started_cluster.instances["h0_0_0"]
-    result = node.query("DROP TABLE IF EXISTS default.demo_parquet_1")
-    result = node.query(
-        """
-CREATE TABLE IF NOT EXISTS default.demo_parquet_1 (`id` Nullable(String), `score` Nullable(Int32), `day` Nullable(String)) ENGINE = Hive('thrift://hivetest:9083', 'test', 'demo') PARTITION BY(day)
-        """
-    )
-    result = node.query(
-        """
-ALTER TABLE default.demo_parquet_1 DROP COLUMN id
-        """
-    )
-
-    result = node.query("""DESC default.demo_parquet_1 FORMAT TSV""")
-    expected_result = """score\tNullable(Int32)\t\t\t\t\t\nday\tNullable(String)"""
-    assert result.strip() == expected_result
-
-
-def test_table_alter_comment(started_cluster):
-    node = started_cluster.instances["h0_0_0"]
-    result = node.query("DROP TABLE IF EXISTS default.demo_parquet_1")
-    result = node.query(
-        """
-CREATE TABLE IF NOT EXISTS default.demo_parquet_1 (`id` Nullable(String), `score` Nullable(Int32), `day` Nullable(String)) ENGINE = Hive('thrift://hivetest:9083', 'test', 'demo') PARTITION BY(day)
-        """
-    )
-
-    result = node.query(
-        """ALTER TABLE default.demo_parquet_1 COMMENT COLUMN id 'Text comment'"""
-    )
-    result = node.query("""DESC default.demo_parquet_1 FORMAT TSV""")
-    expected_result = """id\tNullable(String)\t\t\tText comment\t\t\nscore\tNullable(Int32)\t\t\t\t\t\nday\tNullable(String)"""
-    assert result.strip() == expected_result
diff --git a/tests/integration/test_host_regexp_multiple_ptr_records/__init__.py b/tests/integration/test_host_regexp_multiple_ptr_records/__init__.py
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/integration/test_host_regexp_multiple_ptr_records/configs/host_regexp.xml b/tests/integration/test_host_regexp_multiple_ptr_records/configs/host_regexp.xml
deleted file mode 100644
index 7a2141e6c7e..00000000000
--- a/tests/integration/test_host_regexp_multiple_ptr_records/configs/host_regexp.xml
+++ /dev/null
@@ -1,11 +0,0 @@
-<yandex>
-    <users>
-        <test_dns>
-            <password/>
-            <networks>
-                <host_regexp>test1\.example\.com$</host_regexp>
-            </networks>
-            <profile>default</profile>
-        </test_dns>
-    </users>
-</yandex>
\ No newline at end of file
diff --git a/tests/integration/test_host_regexp_multiple_ptr_records/configs/listen_host.xml b/tests/integration/test_host_regexp_multiple_ptr_records/configs/listen_host.xml
deleted file mode 100644
index 58ef55cd3f3..00000000000
--- a/tests/integration/test_host_regexp_multiple_ptr_records/configs/listen_host.xml
+++ /dev/null
@@ -1,5 +0,0 @@
-<yandex>
-    <listen_host>::</listen_host>
-    <listen_host>0.0.0.0</listen_host>
-    <listen_try>1</listen_try>
-</yandex>
diff --git a/tests/integration/test_host_regexp_multiple_ptr_records/coredns_config/Corefile b/tests/integration/test_host_regexp_multiple_ptr_records/coredns_config/Corefile
deleted file mode 100644
index 3edf37dafa5..00000000000
--- a/tests/integration/test_host_regexp_multiple_ptr_records/coredns_config/Corefile
+++ /dev/null
@@ -1,8 +0,0 @@
-. {
-    hosts /example.com {
-        reload "20ms"
-        fallthrough
-    }
-    forward . 127.0.0.11
-    log
-}
diff --git a/tests/integration/test_host_regexp_multiple_ptr_records/coredns_config/example.com b/tests/integration/test_host_regexp_multiple_ptr_records/coredns_config/example.com
deleted file mode 100644
index 9beb415c290..00000000000
--- a/tests/integration/test_host_regexp_multiple_ptr_records/coredns_config/example.com
+++ /dev/null
@@ -1 +0,0 @@
-filled in runtime, but needs to exist in order to be volume mapped in docker
\ No newline at end of file
diff --git a/tests/integration/test_host_regexp_multiple_ptr_records/test.py b/tests/integration/test_host_regexp_multiple_ptr_records/test.py
deleted file mode 100644
index 36f76140a2c..00000000000
--- a/tests/integration/test_host_regexp_multiple_ptr_records/test.py
+++ /dev/null
@@ -1,115 +0,0 @@
-import pytest
-
-# FIXME This test is too flaky
-# https://github.com/ClickHouse/ClickHouse/issues/51471
-
-pytestmark = pytest.mark.skip
-
-import socket
-from helpers.cluster import ClickHouseCluster, get_docker_compose_path, run_and_check
-from time import sleep
-import os
-
-DOCKER_COMPOSE_PATH = get_docker_compose_path()
-SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
-
-cluster = ClickHouseCluster(__file__)
-
-ch_server = cluster.add_instance(
-    "clickhouse-server",
-    with_coredns=True,
-    main_configs=["configs/listen_host.xml"],
-    user_configs=["configs/host_regexp.xml"],
-    ipv6_address="2001:3984:3989::1:1111",
-)
-
-client = cluster.add_instance(
-    "clickhouse-client",
-    ipv6_address="2001:3984:3989::1:1112",
-)
-
-
-@pytest.fixture(scope="module")
-def started_cluster():
-    global cluster
-    try:
-        cluster.start()
-        yield cluster
-
-    finally:
-        cluster.shutdown()
-
-
-def check_ptr_record(ip, hostname):
-    try:
-        host, aliaslist, ipaddrlist = socket.gethostbyaddr(ip)
-        if hostname.lower() == host.lower():
-            return True
-    except socket.herror:
-        pass
-    return False
-
-
-def setup_dns_server(ip):
-    domains_string = "test3.example.com test2.example.com test1.example.com"
-    example_file_path = f'{ch_server.env_variables["COREDNS_CONFIG_DIR"]}/example.com'
-    run_and_check(f"echo '{ip} {domains_string}' > {example_file_path}", shell=True)
-
-    # DNS server takes time to reload the configuration.
-    for try_num in range(10):
-        if all(check_ptr_record(ip, host) for host in domains_string.split()):
-            break
-        sleep(1)
-
-
-def setup_ch_server(dns_server_ip):
-    ch_server.exec_in_container(
-        (["bash", "-c", f"echo 'nameserver {dns_server_ip}' > /etc/resolv.conf"])
-    )
-    ch_server.exec_in_container(
-        (["bash", "-c", "echo 'options ndots:0' >> /etc/resolv.conf"])
-    )
-    ch_server.query("SYSTEM DROP DNS CACHE")
-
-
-def build_endpoint_v4(ip):
-    return f"'http://{ip}:8123/?query=SELECT+1&user=test_dns'"
-
-
-def build_endpoint_v6(ip):
-    return build_endpoint_v4(f"[{ip}]")
-
-
-def test_host_regexp_multiple_ptr_v4_fails_with_wrong_resolution(started_cluster):
-    server_ip = cluster.get_instance_ip("clickhouse-server")
-    random_ip = "9.9.9.9"
-    dns_server_ip = cluster.get_instance_ip(cluster.coredns_host)
-
-    setup_dns_server(random_ip)
-    setup_ch_server(dns_server_ip)
-
-    endpoint = build_endpoint_v4(server_ip)
-
-    assert "1\n" != client.exec_in_container(["bash", "-c", f"curl {endpoint}"])
-
-
-def test_host_regexp_multiple_ptr_v4(started_cluster):
-    server_ip = cluster.get_instance_ip("clickhouse-server")
-    client_ip = cluster.get_instance_ip("clickhouse-client")
-    dns_server_ip = cluster.get_instance_ip(cluster.coredns_host)
-
-    setup_dns_server(client_ip)
-    setup_ch_server(dns_server_ip)
-
-    endpoint = build_endpoint_v4(server_ip)
-
-    assert "1\n" == client.exec_in_container(["bash", "-c", f"curl {endpoint}"])
-
-
-def test_host_regexp_multiple_ptr_v6(started_cluster):
-    setup_dns_server(client.ipv6_address)
-    setup_ch_server(cluster.get_instance_global_ipv6(cluster.coredns_host))
-
-    endpoint = build_endpoint_v6(ch_server.ipv6_address)
-
-    assert "1\n" == client.exec_in_container(["bash", "-c", f"curl -6 {endpoint}"])
diff --git a/tests/integration/test_insert_into_distributed_through_materialized_view/test.py b/tests/integration/test_insert_into_distributed_through_materialized_view/test.py
index a5f92002450..b1eb0df2d43 100644
--- a/tests/integration/test_insert_into_distributed_through_materialized_view/test.py
+++ b/tests/integration/test_insert_into_distributed_through_materialized_view/test.py
@@ -108,74 +108,6 @@ def test_reconnect(started_cluster):
         assert remote.query("SELECT count(*) FROM local1").strip() == "3"
 
 
-@pytest.mark.skip(reason="Flapping test")
-def test_inserts_batching(started_cluster):
-    instance = instance_test_inserts_batching
-
-    with PartitionManager() as pm:
-        pm.partition_instances(instance, remote)
-
-        instance.query("INSERT INTO local2_source(d, x) VALUES ('2000-01-01', 1)")
-        # Sleep a bit so that this INSERT forms a batch of its own.
-        time.sleep(0.2)
-
-        instance.query("INSERT INTO local2_source(x, d) VALUES (2, '2000-01-01')")
-
-        for i in range(3, 7):
-            instance.query(
-                "INSERT INTO local2_source(d, x) VALUES ('2000-01-01', {})".format(i)
-            )
-
-        for i in range(7, 9):
-            instance.query(
-                "INSERT INTO local2_source(x, d) VALUES ({}, '2000-01-01')".format(i)
-            )
-
-        instance.query("INSERT INTO local2_source(d, x) VALUES ('2000-01-01', 9)")
-
-        # After ALTER the structure of the saved blocks will be different
-        instance.query("DROP TABLE local2_view")
-        instance.query("ALTER TABLE distributed ADD COLUMN s String")
-
-        # Memory Engine doesn't support ALTER so we just DROP/CREATE everything
-        instance.query("DROP TABLE  local2_source")
-        instance.query(
-            "CREATE TABLE local2_source (d Date, x UInt32, s String) ENGINE = Memory"
-        )
-        instance.query(
-            "CREATE MATERIALIZED VIEW local2_view to distributed AS SELECT d,x,s FROM local2_source"
-        )
-
-        for i in range(10, 13):
-            instance.query(
-                "INSERT INTO local2_source(d, x) VALUES ('2000-01-01', {})".format(i)
-            )
-
-    time.sleep(1.0)
-
-    result = remote.query(
-        "SELECT _part, groupArray(x) FROM local2 GROUP BY _part ORDER BY _part"
-    )
-
-    # Explanation: as merges are turned off on remote instance, active parts in local2 table correspond 1-to-1
-    # to inserted blocks.
-    # Batches of max 3 rows are formed as min_insert_block_size_rows = 3.
-    # Blocks:
-    # 1. Failed batch that is retried with the same contents.
-    # 2. Full batch of inserts regardless of the order of columns thanks to the view.
-    # 3. Full batch of inserts regardless order of columns thanks to the view.
-    # 4. Full batch of inserts after ALTER (that have different block structure).
-    # 5. What was left to insert before ALTER.
-    expected = """\
-20000101_20000101_1_1_0	[1]
-20000101_20000101_2_2_0	[2,3,4]
-20000101_20000101_3_3_0	[5,6,7]
-20000101_20000101_4_4_0	[10,11,12]
-20000101_20000101_5_5_0	[8,9]
-"""
-    assert TSV(result) == TSV(expected)
-
-
 def test_inserts_local(started_cluster):
     instance = instance_test_inserts_local_cluster
     instance.query("INSERT INTO local_source VALUES ('2000-01-01', 1)")
diff --git a/tests/integration/test_keeper_reconfig_replace_leader_in_one_command/test.py b/tests/integration/test_keeper_reconfig_replace_leader_in_one_command/test.py
index 1ec44d8a002..b099d0513e1 100644
--- a/tests/integration/test_keeper_reconfig_replace_leader_in_one_command/test.py
+++ b/tests/integration/test_keeper_reconfig_replace_leader_in_one_command/test.py
@@ -41,81 +41,3 @@ def started_cluster():
 
 def get_fake_zk(node):
     return ku.get_fake_zk(cluster, node)
-
-
-@pytest.mark.skip(reason="test is flaky because changes are not properly waited for")
-def test_reconfig_replace_leader_in_one_command(started_cluster):
-    """
-    Remove leader from a cluster of 3 and add a new node to this cluster in a single command
-    """
-
-    zk1 = get_fake_zk(node1)
-    config = ku.get_config_str(zk1)
-
-    assert len(config.split("\n")) == 3
-    assert "node1" in config
-    assert "node2" in config
-    assert "node3" in config
-    assert "node4" not in config
-
-    for i in range(100):
-        zk1.create(f"/test_four_{i}", b"somedata")
-
-    zk2 = get_fake_zk(node2)
-    zk2.sync("/test_four_0")
-    ku.wait_configs_equal(config, zk2)
-
-    zk3 = get_fake_zk(node3)
-    zk3.sync("/test_four_0")
-    ku.wait_configs_equal(config, zk3)
-
-    for i in range(100):
-        assert zk2.exists(f"/test_four_{i}") is not None
-        assert zk3.exists(f"/test_four_{i}") is not None
-
-    assert ku.is_leader(cluster, node1)
-    node4.start_clickhouse()
-    config, _ = zk2.reconfig(
-        joining="server.4=node4:9234", leaving="1", new_members=None
-    )
-    config = config.decode("utf-8")
-
-    print("After removing 1 and adding 4", config)
-    assert len(config.split("\n")) == 3
-    assert "node1" not in config
-    assert "node2" in config
-    assert "node3" in config
-    assert "node4" in config
-
-    ku.wait_until_connected(cluster, node4)
-    time.sleep(1)
-
-    zk4 = get_fake_zk(node4)
-    zk4.sync("/test_four_0")
-    ku.wait_configs_equal(config, zk4)
-
-    for i in range(100):
-        assert zk4.exists(f"test_four_{i}") is not None
-        zk4.create(f"/test_four_{100 + i}", b"somedata")
-
-    with pytest.raises(Exception):
-        zk1.stop()
-        zk1.close()
-        zk1 = get_fake_zk(node1)
-        zk1.sync("/test_four_0")
-
-    zk2.stop()
-    zk2.close()
-    zk2 = get_fake_zk(node2)
-    zk2.sync("/test_four_0")
-    ku.wait_configs_equal(config, zk2)
-
-    zk3.stop()
-    zk3.close()
-    zk3 = get_fake_zk(node3)
-    zk3.sync("/test_four_0")
-    ku.wait_configs_equal(config, zk3)
-
-    for i in range(200):
-        assert zk2.exists(f"test_four_{i}") is not None
-        assert zk3.exists(f"test_four_{i}") is not None
diff --git a/tests/integration/test_mutations_with_merge_tree/test.py b/tests/integration/test_mutations_with_merge_tree/test.py
index e7d7abc8c7c..0b4a750b38e 100644
--- a/tests/integration/test_mutations_with_merge_tree/test.py
+++ b/tests/integration/test_mutations_with_merge_tree/test.py
@@ -27,111 +27,6 @@ def started_cluster():
         cluster.shutdown()
 
 
-@pytest.mark.skip(reason="Skipping mutations in partition does not work")
-def test_mutations_in_partition_background(started_cluster):
-    try:
-        numbers = 100
-
-        name = "test_mutations_in_partition"
-        instance_test_mutations.query(
-            f"""CREATE TABLE {name} (date Date, a UInt64, b String) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY a"""
-        )
-        instance_test_mutations.query(
-            f"""INSERT INTO {name} SELECT '2019-07-29' AS date, number, toString(number) FROM numbers({numbers})"""
-        )
-
-        for i in range(0, numbers, 3):
-            instance_test_mutations.query(
-                f"""ALTER TABLE {name} DELETE IN PARTITION {i} WHERE a = {i}"""
-            )
-
-        for i in range(1, numbers, 3):
-            instance_test_mutations.query(
-                f"""ALTER TABLE {name} UPDATE b = 'changed' IN PARTITION {i} WHERE a = {i} """
-            )
-
-        def count_and_changed():
-            return instance_test_mutations.query(
-                f"SELECT count(), countIf(b == 'changed') FROM {name} SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT CSV"
-            ).splitlines()
-
-        all_done = False
-        for wait_times_for_mutation in range(
-            100
-        ):  # wait for replication 80 seconds max
-            time.sleep(0.8)
-
-            if count_and_changed() == ["66,33"]:
-                all_done = True
-                break
-
-        print(
-            instance_test_mutations.query(
-                f"SELECT mutation_id, command, parts_to_do, is_done, latest_failed_part, latest_fail_reason, parts_to_do_names FROM system.mutations WHERE table = '{name}' SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT TSVWithNames"
-            )
-        )
-
-        assert (count_and_changed(), all_done) == (["66,33"], True)
-        assert instance_test_mutations.query(
-            f"SELECT count(), sum(is_done) FROM system.mutations WHERE table = '{name}' SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT CSV"
-        ).splitlines() == ["67,67"]
-
-    finally:
-        instance_test_mutations.query(f"""DROP TABLE {name}""")
-
-
-@pytest.mark.skip(reason="Skipping mutations in partition does not work")
-@pytest.mark.parametrize("sync", [("last",), ("all",)])
-def test_mutations_in_partition_sync(started_cluster, sync):
-    try:
-        numbers = 10
-
-        name = "test_mutations_in_partition_sync"
-        instance_test_mutations.query(
-            f"""CREATE TABLE {name} (date Date, a UInt64, b String) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY a"""
-        )
-        instance_test_mutations.query(
-            f"""INSERT INTO {name} SELECT '2019-07-29' AS date, number, toString(number) FROM numbers({numbers})"""
-        )
-
-        for i in range(0, numbers, 3):
-            instance_test_mutations.query(
-                f"""ALTER TABLE {name} DELETE IN PARTITION {i} WHERE a = {i}"""
-                + (" SETTINGS mutations_sync = 1" if sync == "all" else "")
-            )
-
-        for reverse_index, i in reversed(
-            list(enumerate(reversed(range(1, numbers, 3))))
-        ):
-            instance_test_mutations.query(
-                f"""ALTER TABLE {name} UPDATE b = 'changed' IN PARTITION {i} WHERE a = {i}"""
-                + (
-                    " SETTINGS mutations_sync = 1"
-                    if not reverse_index or sync == "all"
-                    else ""
-                )
-            )
-
-        def count_and_changed():
-            return instance_test_mutations.query(
-                f"SELECT count(), countIf(b == 'changed') FROM {name} SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT CSV"
-            ).splitlines()
-
-        print(
-            instance_test_mutations.query(
-                f"SELECT mutation_id, command, parts_to_do, is_done, latest_failed_part, latest_fail_reason FROM system.mutations WHERE table = '{name}' SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT TSVWithNames"
-            )
-        )
-
-        assert count_and_changed() == ["6,3"]
-        assert instance_test_mutations.query(
-            f"SELECT count(), sum(is_done) FROM system.mutations WHERE table = '{name}' SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT CSV"
-        ).splitlines() == ["7,7"]
-
-    finally:
-        instance_test_mutations.query(f"""DROP TABLE {name}""")
-
-
 def test_mutations_with_merge_background_task(started_cluster):
     instance_test_mutations.query(
         """SYSTEM STOP MERGES test_mutations_with_ast_elements"""
@@ -190,92 +85,3 @@ def test_mutations_with_truncate_table(started_cluster):
         ).rstrip()
         == "0"
     )
-
-
-@pytest.mark.skip(reason="Skipping mutations in partition does not work")
-def test_mutations_will_not_hang_for_non_existing_parts_sync(started_cluster):
-    try:
-        numbers = 100
-
-        name = "test_mutations_will_not_hang_for_non_existing_parts_sync"
-        instance_test_mutations.query(
-            f"""CREATE TABLE {name} (date Date, a UInt64, b String) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY a"""
-        )
-        instance_test_mutations.query(
-            f"""INSERT INTO {name} SELECT '2019-07-29' AS date, number, toString(number) FROM numbers({numbers})"""
-        )
-
-        for i in range(0, numbers, 3):
-            instance_test_mutations.query(
-                f"""ALTER TABLE {name} DELETE IN PARTITION {i+1000} WHERE a = {i} SETTINGS mutations_sync = 1"""
-            )
-
-        def count():
-            return instance_test_mutations.query(
-                f"SELECT count() FROM {name} SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT CSV"
-            ).splitlines()
-
-        print(
-            instance_test_mutations.query(
-                f"SELECT mutation_id, command, parts_to_do, is_done, latest_failed_part, latest_fail_reason, parts_to_do_names FROM system.mutations WHERE table = '{name}' SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT TSVWithNames"
-            )
-        )
-
-        assert count() == [f"{numbers}"]
-        assert instance_test_mutations.query(
-            f"SELECT count(), sum(is_done) FROM system.mutations WHERE table = '{name}' SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT CSV"
-        ).splitlines() == [f"34,34"]
-
-    finally:
-        instance_test_mutations.query(f"""DROP TABLE {name}""")
-
-
-@pytest.mark.skip(reason="Skipping mutations in partition does not work")
-def test_mutations_will_not_hang_for_non_existing_parts_async(started_cluster):
-    try:
-        numbers = 100
-
-        name = "test_mutations_will_not_hang_for_non_existing_parts_async"
-        instance_test_mutations.query(
-            f"""CREATE TABLE {name} (date Date, a UInt64, b String) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY a"""
-        )
-        instance_test_mutations.query(
-            f"""INSERT INTO {name} SELECT '2019-07-29' AS date, number, toString(number) FROM numbers({numbers})"""
-        )
-
-        for i in range(0, numbers, 3):
-            instance_test_mutations.query(
-                f"""ALTER TABLE {name} DELETE IN PARTITION {i+1000} WHERE a = {i}"""
-            )
-
-        def count():
-            return instance_test_mutations.query(
-                f"SELECT count() FROM {name} SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT CSV"
-            ).splitlines()
-
-        def count_and_sum_is_done():
-            return instance_test_mutations.query(
-                f"SELECT count(), sum(is_done) FROM system.mutations WHERE table = '{name}' SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT CSV"
-            ).splitlines()
-
-        all_done = False
-        for wait_times_for_mutation in range(
-            100
-        ):  # wait for replication 80 seconds max
-            time.sleep(0.8)
-
-            if count_and_sum_is_done() == ["34,34"]:
-                all_done = True
-                break
-
-        print(
-            instance_test_mutations.query(
-                f"SELECT mutation_id, command, parts_to_do, is_done, latest_failed_part, latest_fail_reason, parts_to_do_names FROM system.mutations WHERE table = '{name}' SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT TSVWithNames"
-            )
-        )
-
-        assert count() == [f"{numbers}"]
-        assert count_and_sum_is_done() == ["34,34"]
-
-    finally:
-        instance_test_mutations.query(f"""DROP TABLE {name}""")
diff --git a/tests/integration/test_polymorphic_parts/test.py b/tests/integration/test_polymorphic_parts/test.py
index 48bcca7d505..debb509de90 100644
--- a/tests/integration/test_polymorphic_parts/test.py
+++ b/tests/integration/test_polymorphic_parts/test.py
@@ -414,54 +414,6 @@ def start_cluster_diff_versions():
         cluster.shutdown()
 
 
-@pytest.mark.skip(reason="compatability is temporary broken")
-def test_polymorphic_parts_diff_versions(start_cluster_diff_versions):
-    # Check that replication with Wide parts works between different versions.
-
-    node_old = node7
-    node_new = node8
-
-    insert_random_data("polymorphic_table", node7, 100)
-    node8.query("SYSTEM SYNC REPLICA polymorphic_table", timeout=20)
-
-    assert node8.query("SELECT count() FROM polymorphic_table") == "100\n"
-    assert (
-        node8.query(
-            "SELECT DISTINCT part_type FROM system.parts WHERE table = 'polymorphic_table' and active"
-        )
-        == "Wide\n"
-    )
-
-
-@pytest.mark.skip(reason="compatability is temporary broken")
-def test_polymorphic_parts_diff_versions_2(start_cluster_diff_versions):
-    # Replication doesn't work on old version if part is created in compact format, because
-    #  this version doesn't know anything about it. It's considered to be ok.
-
-    node_old = node7
-    node_new = node8
-
-    insert_random_data("polymorphic_table_2", node_new, 100)
-
-    assert node_new.query("SELECT count() FROM polymorphic_table_2") == "100\n"
-    assert node_old.query("SELECT count() FROM polymorphic_table_2") == "0\n"
-    with pytest.raises(Exception):
-        node_old.query("SYSTEM SYNC REPLICA polymorphic_table_2", timeout=3)
-
-    node_old.restart_with_latest_version(fix_metadata=True)
-
-    node_old.query("SYSTEM SYNC REPLICA polymorphic_table_2", timeout=20)
-
-    # Works after update
-    assert node_old.query("SELECT count() FROM polymorphic_table_2") == "100\n"
-    assert (
-        node_old.query(
-            "SELECT DISTINCT part_type FROM system.parts WHERE table = 'polymorphic_table_2' and active"
-        )
-        == "Compact\n"
-    )
-
-
 def test_polymorphic_parts_non_adaptive(start_cluster):
     node1.query("SYSTEM STOP MERGES")
     node2.query("SYSTEM STOP MERGES")
diff --git a/tests/integration/test_replicated_merge_tree_hdfs_zero_copy/__init__.py b/tests/integration/test_replicated_merge_tree_hdfs_zero_copy/__init__.py
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/integration/test_replicated_merge_tree_hdfs_zero_copy/configs/config.d/storage_conf.xml b/tests/integration/test_replicated_merge_tree_hdfs_zero_copy/configs/config.d/storage_conf.xml
deleted file mode 100644
index bb4aba94e0b..00000000000
--- a/tests/integration/test_replicated_merge_tree_hdfs_zero_copy/configs/config.d/storage_conf.xml
+++ /dev/null
@@ -1,93 +0,0 @@
-<clickhouse>
-    <storage_configuration>
-        <disks>
-            <hdfs1>
-                <type>hdfs</type>
-                <endpoint>hdfs://hdfs1:9000/clickhouse1/</endpoint>
-                <!-- FIXME: chicken and egg problem with current cluster.py -->
-                <skip_access_check>true</skip_access_check>
-            </hdfs1>
-            <hdfs1_again>
-                <type>hdfs</type>
-                <endpoint>hdfs://hdfs1:9000/clickhouse1/</endpoint>
-                <!-- FIXME: chicken and egg problem with current cluster.py -->
-                <skip_access_check>true</skip_access_check>
-            </hdfs1_again>
-            <hdfs2>
-                <type>hdfs</type>
-                <endpoint>hdfs://hdfs1:9000/clickhouse2/</endpoint>
-                <!-- FIXME: chicken and egg problem with current cluster.py -->
-                <skip_access_check>true</skip_access_check>
-            </hdfs2>
-        </disks>
-        <policies>
-            <hdfs_only>
-                <volumes>
-                    <main>
-                        <disk>hdfs1</disk>
-                    </main>
-                </volumes>
-            </hdfs_only>
-            <hybrid>
-                <volumes>
-                    <main>
-                        <disk>default</disk>
-                    </main>
-                    <external>
-                        <disk>hdfs1</disk>
-                    </external>
-                </volumes>
-                <move_factor>0.0</move_factor>
-            </hybrid>
-            <tiered>
-                <volumes>
-                    <main>
-                        <disk>hdfs2</disk>
-                    </main>
-                    <external>
-                        <disk>hdfs1</disk>
-                    </external>
-                </volumes>
-            </tiered>
-            <tiered_copy>
-                <volumes>
-                    <main>
-                        <disk>hdfs1_again</disk>
-                    </main>
-                    <external>
-                        <disk>hdfs1</disk>
-                    </external>
-                </volumes>
-            </tiered_copy>
-        </policies>
-    </storage_configuration>
-
-    <merge_tree>
-        <min_bytes_for_wide_part>1024000</min_bytes_for_wide_part>
-        <old_parts_lifetime>1</old_parts_lifetime>
-        <allow_remote_fs_zero_copy_replication>true</allow_remote_fs_zero_copy_replication>
-    </merge_tree>
-
-    <remote_servers>
-        <test_cluster>
-            <shard>
-                <replica>
-                    <host>node1</host>
-                    <port>9000</port>
-                </replica>
-            </shard>
-            <shard>
-                <replica>
-                    <host>node2</host>
-                    <port>9000</port>
-                </replica>
-            </shard>
-        </test_cluster>
-    </remote_servers>
-
-    <macros>
-        <cluster>test_cluster</cluster>
-        <shard>1</shard>
-    </macros>
-    <allow_remove_stale_moving_parts>true</allow_remove_stale_moving_parts>
-</clickhouse>
diff --git a/tests/integration/test_replicated_merge_tree_hdfs_zero_copy/test.py b/tests/integration/test_replicated_merge_tree_hdfs_zero_copy/test.py
deleted file mode 100644
index eb3d62eb718..00000000000
--- a/tests/integration/test_replicated_merge_tree_hdfs_zero_copy/test.py
+++ /dev/null
@@ -1,322 +0,0 @@
-import pytest
-
-# FIXME This test is too flaky
-# https://github.com/ClickHouse/ClickHouse/issues/42561
-
-pytestmark = pytest.mark.skip
-
-import logging
-from string import Template
-import time
-
-from helpers.cluster import ClickHouseCluster
-from helpers.test_tools import assert_eq_with_retry
-
-from pyhdfs import HdfsClient
-
-SHARDS = 2
-FILES_OVERHEAD_PER_TABLE = 1  # format_version.txt
-FILES_OVERHEAD_PER_PART_COMPACT = 7
-
-
-def wait_for_hdfs_objects(cluster, fp, expected, num_tries=30):
-    fs = HdfsClient(hosts=cluster.hdfs_ip)
-    while num_tries > 0:
-        num_hdfs_objects = len(fs.listdir(fp))
-        if num_hdfs_objects == expected:
-            break
-        num_tries -= 1
-        time.sleep(1)
-    assert len(fs.listdir(fp)) == expected
-
-
-@pytest.fixture(scope="module")
-def cluster():
-    try:
-        cluster = ClickHouseCluster(__file__)
-        cluster.add_instance(
-            "node1",
-            main_configs=["configs/config.d/storage_conf.xml"],
-            macros={"replica": "node1"},
-            with_zookeeper=True,
-            with_hdfs=True,
-        )
-        cluster.add_instance(
-            "node2",
-            main_configs=["configs/config.d/storage_conf.xml"],
-            macros={"replica": "node2"},
-            with_zookeeper=True,
-            with_hdfs=True,
-        )
-        logging.info("Starting cluster...")
-        cluster.start()
-        if cluster.instances["node1"].is_debug_build():
-            # https://github.com/ClickHouse/ClickHouse/issues/27814
-            pytest.skip(
-                "libhdfs3 calls rand function which does not pass harmful check in debug build"
-            )
-        logging.info("Cluster started")
-
-        fs = HdfsClient(hosts=cluster.hdfs_ip)
-        fs.mkdirs("/clickhouse1")
-        fs.mkdirs("/clickhouse2")
-        logging.info("Created HDFS directory")
-
-        yield cluster
-    finally:
-        cluster.shutdown()
-
-
-def test_hdfs_zero_copy_replication_insert(cluster):
-    node1 = cluster.instances["node1"]
-    node2 = cluster.instances["node2"]
-    try:
-        node1.query(
-            """
-            CREATE TABLE hdfs_test ON CLUSTER test_cluster (dt DateTime, id Int64)
-            ENGINE=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/hdfs_test', '{replica}')
-            ORDER BY (dt, id)
-            SETTINGS storage_policy='hdfs_only'
-            """
-        )
-        wait_for_hdfs_objects(
-            cluster, "/clickhouse1", SHARDS * FILES_OVERHEAD_PER_TABLE
-        )
-
-        node1.query("INSERT INTO hdfs_test VALUES (now() - INTERVAL 3 DAY, 10)")
-        node2.query("SYSTEM SYNC REPLICA hdfs_test", timeout=30)
-        assert node1.query("SELECT count() FROM hdfs_test FORMAT Values") == "(1)"
-        assert node2.query("SELECT count() FROM hdfs_test FORMAT Values") == "(1)"
-        assert (
-            node1.query("SELECT id FROM hdfs_test ORDER BY dt FORMAT Values") == "(10)"
-        )
-        assert (
-            node2.query("SELECT id FROM hdfs_test ORDER BY dt FORMAT Values") == "(10)"
-        )
-        assert (
-            node1.query(
-                "SELECT partition_id,disk_name FROM system.parts WHERE table='hdfs_test' FORMAT Values"
-            )
-            == "('all','hdfs1')"
-        )
-        assert (
-            node2.query(
-                "SELECT partition_id,disk_name FROM system.parts WHERE table='hdfs_test' FORMAT Values"
-            )
-            == "('all','hdfs1')"
-        )
-        wait_for_hdfs_objects(
-            cluster,
-            "/clickhouse1",
-            SHARDS * FILES_OVERHEAD_PER_TABLE + FILES_OVERHEAD_PER_PART_COMPACT,
-        )
-    finally:
-        node1.query("DROP TABLE IF EXISTS hdfs_test SYNC")
-        node2.query("DROP TABLE IF EXISTS hdfs_test SYNC")
-
-
-@pytest.mark.parametrize(
-    ("storage_policy", "init_objects"),
-    [("hybrid", 0), ("tiered", 0), ("tiered_copy", FILES_OVERHEAD_PER_TABLE)],
-)
-def test_hdfs_zero_copy_replication_single_move(cluster, storage_policy, init_objects):
-    node1 = cluster.instances["node1"]
-    try:
-        node1.query(
-            Template(
-                """
-            CREATE TABLE single_node_move_test (dt DateTime, id Int64)
-            ENGINE=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/single_node_move_test', '{replica}')
-            ORDER BY (dt, id)
-            SETTINGS storage_policy='$policy',temporary_directories_lifetime=1
-            """
-            ).substitute(policy=storage_policy)
-        )
-        wait_for_hdfs_objects(cluster, "/clickhouse1", init_objects)
-
-        node1.query(
-            "INSERT INTO single_node_move_test VALUES (now() - INTERVAL 3 DAY, 10), (now() - INTERVAL 1 DAY, 11)"
-        )
-        assert (
-            node1.query(
-                "SELECT id FROM single_node_move_test ORDER BY dt FORMAT Values"
-            )
-            == "(10),(11)"
-        )
-
-        node1.query(
-            "ALTER TABLE single_node_move_test MOVE PARTITION ID 'all' TO VOLUME 'external'"
-        )
-        assert (
-            node1.query(
-                "SELECT partition_id,disk_name FROM system.parts WHERE table='single_node_move_test' FORMAT Values"
-            )
-            == "('all','hdfs1')"
-        )
-        assert (
-            node1.query(
-                "SELECT id FROM single_node_move_test ORDER BY dt FORMAT Values"
-            )
-            == "(10),(11)"
-        )
-        wait_for_hdfs_objects(
-            cluster, "/clickhouse1", init_objects + FILES_OVERHEAD_PER_PART_COMPACT
-        )
-
-        node1.query(
-            "ALTER TABLE single_node_move_test MOVE PARTITION ID 'all' TO VOLUME 'main'"
-        )
-        assert (
-            node1.query(
-                "SELECT id FROM single_node_move_test ORDER BY dt FORMAT Values"
-            )
-            == "(10),(11)"
-        )
-    finally:
-        node1.query("DROP TABLE IF EXISTS single_node_move_test SYNC")
-
-
-@pytest.mark.parametrize(
-    ("storage_policy", "init_objects"),
-    [("hybrid", 0), ("tiered", 0), ("tiered_copy", SHARDS * FILES_OVERHEAD_PER_TABLE)],
-)
-def test_hdfs_zero_copy_replication_move(cluster, storage_policy, init_objects):
-    node1 = cluster.instances["node1"]
-    node2 = cluster.instances["node2"]
-    try:
-        node1.query(
-            Template(
-                """
-            CREATE TABLE move_test ON CLUSTER test_cluster (dt DateTime, id Int64)
-            ENGINE=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/move_test', '{replica}')
-            ORDER BY (dt, id)
-            SETTINGS storage_policy='$policy'
-            """
-            ).substitute(policy=storage_policy)
-        )
-        wait_for_hdfs_objects(cluster, "/clickhouse1", init_objects)
-
-        node1.query(
-            "INSERT INTO move_test VALUES (now() - INTERVAL 3 DAY, 10), (now() - INTERVAL 1 DAY, 11)"
-        )
-        node2.query("SYSTEM SYNC REPLICA move_test", timeout=30)
-
-        assert (
-            node1.query("SELECT id FROM move_test ORDER BY dt FORMAT Values")
-            == "(10),(11)"
-        )
-        assert (
-            node2.query("SELECT id FROM move_test ORDER BY dt FORMAT Values")
-            == "(10),(11)"
-        )
-
-        node1.query(
-            "ALTER TABLE move_test MOVE PARTITION ID 'all' TO VOLUME 'external'"
-        )
-        wait_for_hdfs_objects(
-            cluster, "/clickhouse1", init_objects + FILES_OVERHEAD_PER_PART_COMPACT
-        )
-
-        node2.query(
-            "ALTER TABLE move_test MOVE PARTITION ID 'all' TO VOLUME 'external'"
-        )
-        assert (
-            node1.query(
-                "SELECT partition_id,disk_name FROM system.parts WHERE table='move_test' FORMAT Values"
-            )
-            == "('all','hdfs1')"
-        )
-        assert (
-            node2.query(
-                "SELECT partition_id,disk_name FROM system.parts WHERE table='move_test' FORMAT Values"
-            )
-            == "('all','hdfs1')"
-        )
-        assert (
-            node1.query("SELECT id FROM move_test ORDER BY dt FORMAT Values")
-            == "(10),(11)"
-        )
-        assert (
-            node2.query("SELECT id FROM move_test ORDER BY dt FORMAT Values")
-            == "(10),(11)"
-        )
-        wait_for_hdfs_objects(
-            cluster, "/clickhouse1", init_objects + FILES_OVERHEAD_PER_PART_COMPACT
-        )
-    finally:
-        node1.query("DROP TABLE IF EXISTS move_test SYNC")
-        node2.query("DROP TABLE IF EXISTS move_test SYNC")
-
-
-@pytest.mark.parametrize(("storage_policy"), ["hybrid", "tiered", "tiered_copy"])
-def test_hdfs_zero_copy_with_ttl_move(cluster, storage_policy):
-    node1 = cluster.instances["node1"]
-    node2 = cluster.instances["node2"]
-    try:
-        node1.query(
-            Template(
-                """
-            CREATE TABLE ttl_move_test ON CLUSTER test_cluster (dt DateTime, id Int64)
-            ENGINE=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/ttl_move_test', '{replica}')
-            ORDER BY (dt, id)
-            TTL dt + INTERVAL 2 DAY TO VOLUME 'external'
-            SETTINGS storage_policy='$policy'
-            """
-            ).substitute(policy=storage_policy)
-        )
-
-        node1.query("INSERT INTO ttl_move_test VALUES (now() - INTERVAL 3 DAY, 10)")
-        node1.query("INSERT INTO ttl_move_test VALUES (now() - INTERVAL 1 DAY, 11)")
-
-        node1.query("OPTIMIZE TABLE ttl_move_test FINAL")
-        node2.query("SYSTEM SYNC REPLICA ttl_move_test", timeout=30)
-
-        assert_eq_with_retry(node1, "SELECT count() FROM ttl_move_test", "2")
-        assert_eq_with_retry(node2, "SELECT count() FROM ttl_move_test", "2")
-        assert (
-            node1.query("SELECT id FROM ttl_move_test ORDER BY id FORMAT Values")
-            == "(10),(11)"
-        )
-        assert (
-            node2.query("SELECT id FROM ttl_move_test ORDER BY id FORMAT Values")
-            == "(10),(11)"
-        )
-    finally:
-        node1.query("DROP TABLE IF EXISTS ttl_move_test SYNC")
-        node2.query("DROP TABLE IF EXISTS ttl_move_test SYNC")
-
-
-def test_hdfs_zero_copy_with_ttl_delete(cluster):
-    node1 = cluster.instances["node1"]
-    node2 = cluster.instances["node2"]
-    try:
-        node1.query(
-            """
-            CREATE TABLE ttl_delete_test ON CLUSTER test_cluster (dt DateTime, id Int64)
-            ENGINE=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/ttl_delete_test', '{replica}')
-            ORDER BY (dt, id)
-            TTL dt + INTERVAL 2 DAY
-            SETTINGS storage_policy='tiered'
-            """
-        )
-
-        node1.query("INSERT INTO ttl_delete_test VALUES (now() - INTERVAL 3 DAY, 10)")
-        node1.query("INSERT INTO ttl_delete_test VALUES (now() - INTERVAL 1 DAY, 11)")
-
-        node1.query("OPTIMIZE TABLE ttl_delete_test FINAL")
-        node2.query("SYSTEM SYNC REPLICA ttl_delete_test", timeout=30)
-
-        assert_eq_with_retry(node1, "SELECT count() FROM ttl_delete_test", "1")
-        assert_eq_with_retry(node2, "SELECT count() FROM ttl_delete_test", "1")
-
-        assert (
-            node1.query("SELECT id FROM ttl_delete_test ORDER BY id FORMAT Values")
-            == "(11)"
-        )
-        assert (
-            node2.query("SELECT id FROM ttl_delete_test ORDER BY id FORMAT Values")
-            == "(11)"
-        )
-    finally:
-        node1.query("DROP TABLE IF EXISTS ttl_delete_test SYNC")
-        node2.query("DROP TABLE IF EXISTS ttl_delete_test SYNC")
diff --git a/tests/integration/test_reverse_dns_query/__init__.py b/tests/integration/test_reverse_dns_query/__init__.py
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/integration/test_reverse_dns_query/configs/config.xml b/tests/integration/test_reverse_dns_query/configs/config.xml
deleted file mode 100644
index 5ce55afa2a7..00000000000
--- a/tests/integration/test_reverse_dns_query/configs/config.xml
+++ /dev/null
@@ -1,3 +0,0 @@
-<clickhouse>
-    <disable_internal_dns_cache>1</disable_internal_dns_cache>
-</clickhouse>
diff --git a/tests/integration/test_reverse_dns_query/configs/listen_host.xml b/tests/integration/test_reverse_dns_query/configs/listen_host.xml
deleted file mode 100644
index 58ef55cd3f3..00000000000
--- a/tests/integration/test_reverse_dns_query/configs/listen_host.xml
+++ /dev/null
@@ -1,5 +0,0 @@
-<yandex>
-    <listen_host>::</listen_host>
-    <listen_host>0.0.0.0</listen_host>
-    <listen_try>1</listen_try>
-</yandex>
diff --git a/tests/integration/test_reverse_dns_query/configs/reverse_dns_function.xml b/tests/integration/test_reverse_dns_query/configs/reverse_dns_function.xml
deleted file mode 100644
index 35d0a07c6a6..00000000000
--- a/tests/integration/test_reverse_dns_query/configs/reverse_dns_function.xml
+++ /dev/null
@@ -1,3 +0,0 @@
-<clickhouse>
-    <allow_reverse_dns_query_function>1</allow_reverse_dns_query_function>
-</clickhouse>
diff --git a/tests/integration/test_reverse_dns_query/coredns_config/Corefile b/tests/integration/test_reverse_dns_query/coredns_config/Corefile
deleted file mode 100644
index 84d297f7cdf..00000000000
--- a/tests/integration/test_reverse_dns_query/coredns_config/Corefile
+++ /dev/null
@@ -1,4 +0,0 @@
-. {
-    forward . 127.0.0.11
-    log
-}
diff --git a/tests/integration/test_reverse_dns_query/test.py b/tests/integration/test_reverse_dns_query/test.py
deleted file mode 100644
index 6a39bd7b586..00000000000
--- a/tests/integration/test_reverse_dns_query/test.py
+++ /dev/null
@@ -1,56 +0,0 @@
-import pytest
-
-# FIXME This test is broken
-# https://github.com/ClickHouse/ClickHouse/issues/53194
-
-pytestmark = pytest.mark.skip
-
-from helpers.cluster import ClickHouseCluster, get_docker_compose_path, run_and_check
-from time import sleep
-import os
-
-DOCKER_COMPOSE_PATH = get_docker_compose_path()
-SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
-
-cluster = ClickHouseCluster(__file__)
-
-ch_server = cluster.add_instance(
-    "clickhouse-server",
-    with_coredns=True,
-    main_configs=[
-        "configs/config.xml",
-        "configs/reverse_dns_function.xml",
-        "configs/listen_host.xml",
-    ],
-)
-
-
-@pytest.fixture(scope="module")
-def started_cluster():
-    global cluster
-    try:
-        cluster.start()
-        yield cluster
-
-    finally:
-        cluster.shutdown()
-
-
-def setup_ch_server(dns_server_ip):
-    ch_server.exec_in_container(
-        (["bash", "-c", f"echo 'nameserver {dns_server_ip}' > /etc/resolv.conf"])
-    )
-    ch_server.exec_in_container(
-        (["bash", "-c", "echo 'options ndots:0' >> /etc/resolv.conf"])
-    )
-    ch_server.query("SYSTEM DROP DNS CACHE")
-
-
-def test_reverse_dns_query(started_cluster):
-    dns_server_ip = cluster.get_instance_ip(cluster.coredns_host)
-
-    setup_ch_server(dns_server_ip)
-
-    for _ in range(0, 200):
-        response = ch_server.query("select reverseDNSQuery('2001:4860:4860::8888')")
-        assert response == "['dns.google']\n"
diff --git a/tests/integration/test_s3_zero_copy_replication/test.py b/tests/integration/test_s3_zero_copy_replication/test.py
index 2a4e0eece08..0ca81a27802 100644
--- a/tests/integration/test_s3_zero_copy_replication/test.py
+++ b/tests/integration/test_s3_zero_copy_replication/test.py
@@ -153,96 +153,6 @@ def test_s3_zero_copy_replication(started_cluster, policy):
     node2.query("DROP TABLE IF EXISTS s3_test SYNC")
 
 
-@pytest.mark.skip(reason="Test is flaky (and never was stable)")
-def test_s3_zero_copy_on_hybrid_storage(started_cluster):
-    node1 = cluster.instances["node1"]
-    node2 = cluster.instances["node2"]
-
-    node1.query(
-        """
-        CREATE TABLE hybrid_test ON CLUSTER test_cluster (id UInt32, value String)
-        ENGINE=ReplicatedMergeTree('/clickhouse/tables/hybrid_test', '{}')
-        ORDER BY id
-        SETTINGS storage_policy='hybrid',temporary_directories_lifetime=1
-        """.format(
-            "{replica}"
-        )
-    )
-
-    node1.query("INSERT INTO hybrid_test VALUES (0,'data'),(1,'data')")
-    node2.query("SYSTEM SYNC REPLICA hybrid_test", timeout=30)
-
-    assert (
-        node1.query("SELECT * FROM hybrid_test ORDER BY id FORMAT Values")
-        == "(0,'data'),(1,'data')"
-    )
-    assert (
-        node2.query("SELECT * FROM hybrid_test ORDER BY id FORMAT Values")
-        == "(0,'data'),(1,'data')"
-    )
-
-    assert (
-        node1.query(
-            "SELECT partition_id,disk_name FROM system.parts WHERE table='hybrid_test' FORMAT Values"
-        )
-        == "('all','default')"
-    )
-    assert (
-        node2.query(
-            "SELECT partition_id,disk_name FROM system.parts WHERE table='hybrid_test' FORMAT Values"
-        )
-        == "('all','default')"
-    )
-
-    node1.query("ALTER TABLE hybrid_test MOVE PARTITION ID 'all' TO DISK 's31'")
-
-    assert (
-        node1.query(
-            "SELECT partition_id,disk_name FROM system.parts WHERE table='hybrid_test' FORMAT Values"
-        )
-        == "('all','s31')"
-    )
-    assert (
-        node2.query(
-            "SELECT partition_id,disk_name FROM system.parts WHERE table='hybrid_test' FORMAT Values"
-        )
-        == "('all','default')"
-    )
-
-    # Total objects in S3
-    s3_objects = get_large_objects_count(cluster, size=0)
-
-    node2.query("ALTER TABLE hybrid_test MOVE PARTITION ID 'all' TO DISK 's31'")
-
-    assert (
-        node1.query(
-            "SELECT partition_id,disk_name FROM system.parts WHERE table='hybrid_test' FORMAT Values"
-        )
-        == "('all','s31')"
-    )
-    assert (
-        node2.query(
-            "SELECT partition_id,disk_name FROM system.parts WHERE table='hybrid_test' FORMAT Values"
-        )
-        == "('all','s31')"
-    )
-
-    # Check that after moving partition on node2 no new obects on s3
-    wait_for_large_objects_count(cluster, s3_objects, size=0)
-
-    assert (
-        node1.query("SELECT * FROM hybrid_test ORDER BY id FORMAT Values")
-        == "(0,'data'),(1,'data')"
-    )
-    assert (
-        node2.query("SELECT * FROM hybrid_test ORDER BY id FORMAT Values")
-        == "(0,'data'),(1,'data')"
-    )
-
-    node1.query("DROP TABLE IF EXISTS hybrid_test SYNC")
-    node2.query("DROP TABLE IF EXISTS hybrid_test SYNC")
-
-
 def insert_data_time(node, table, number_of_mb, time, start=0):
     values = ",".join(
         f"({x},{time})"
diff --git a/tests/integration/test_s3_zero_copy_ttl/__init__.py b/tests/integration/test_s3_zero_copy_ttl/__init__.py
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/integration/test_s3_zero_copy_ttl/configs/max_delayed_streams.xml b/tests/integration/test_s3_zero_copy_ttl/configs/max_delayed_streams.xml
deleted file mode 100644
index 54f7152690b..00000000000
--- a/tests/integration/test_s3_zero_copy_ttl/configs/max_delayed_streams.xml
+++ /dev/null
@@ -1,9 +0,0 @@
-<clickhouse>
-    <!-- Profiles of settings. -->
-    <profiles>
-        <!-- Default settings. -->
-        <default>
-            <max_insert_delayed_streams_for_parallel_write>10</max_insert_delayed_streams_for_parallel_write>
-        </default>
-    </profiles>
-</clickhouse>
diff --git a/tests/integration/test_s3_zero_copy_ttl/configs/s3.xml b/tests/integration/test_s3_zero_copy_ttl/configs/s3.xml
deleted file mode 100644
index 7bb7fa875e4..00000000000
--- a/tests/integration/test_s3_zero_copy_ttl/configs/s3.xml
+++ /dev/null
@@ -1,39 +0,0 @@
-<clickhouse>
-  <storage_configuration>
-    <disks>
-        <s3_disk>
-            <type>s3</type>
-            <endpoint>http://minio1:9001/root/data/</endpoint>
-            <access_key_id>minio</access_key_id>
-            <secret_access_key>minio123</secret_access_key>
-        </s3_disk>
-    </disks>
-
-    <policies>
-        <s3_and_default>
-            <volumes>
-                <main>
-                    <disk>default</disk>
-                </main>
-                <external>
-                    <disk>s3_disk</disk>
-                </external>
-            </volumes>
-        </s3_and_default>
-        <s3>
-            <volumes>
-                <main>
-                    <disk>s3_disk</disk>
-                </main>
-            </volumes>
-        </s3>
-    </policies>
-  </storage_configuration>
-
-  <merge_tree>
-    <allow_remote_fs_zero_copy_replication>true</allow_remote_fs_zero_copy_replication>
-    <ratio_of_defaults_for_sparse_serialization>1.0</ratio_of_defaults_for_sparse_serialization>
-  </merge_tree>
-
-  <allow_remove_stale_moving_parts>true</allow_remove_stale_moving_parts>
-</clickhouse>
diff --git a/tests/integration/test_s3_zero_copy_ttl/test.py b/tests/integration/test_s3_zero_copy_ttl/test.py
deleted file mode 100644
index 04bff4a44fb..00000000000
--- a/tests/integration/test_s3_zero_copy_ttl/test.py
+++ /dev/null
@@ -1,94 +0,0 @@
-#!/usr/bin/env python3
-import time
-
-import pytest
-from helpers.cluster import ClickHouseCluster
-
-cluster = ClickHouseCluster(__file__)
-node1 = cluster.add_instance(
-    "node1", main_configs=["configs/s3.xml"], with_minio=True, with_zookeeper=True
-)
-node2 = cluster.add_instance(
-    "node2", main_configs=["configs/s3.xml"], with_minio=True, with_zookeeper=True
-)
-node3 = cluster.add_instance(
-    "node3", main_configs=["configs/s3.xml"], with_minio=True, with_zookeeper=True
-)
-
-
-@pytest.fixture(scope="module")
-def started_cluster():
-    try:
-        cluster.start()
-
-        yield cluster
-    finally:
-        cluster.shutdown()
-
-
-def test_ttl_move_and_s3(started_cluster):
-    for i, node in enumerate([node1, node2, node3]):
-        node.query(
-            """
-            CREATE TABLE s3_test_with_ttl (date DateTime, id UInt32, value String)
-            ENGINE=ReplicatedMergeTree('/clickhouse/tables/s3_test', '{}')
-            ORDER BY id
-            PARTITION BY id
-            TTL date TO DISK 's3_disk'
-            SETTINGS storage_policy='s3_and_default', temporary_directories_lifetime=1
-            """.format(
-                i
-            )
-        )
-
-    node1.query("SYSTEM STOP MOVES s3_test_with_ttl")
-
-    node2.query("SYSTEM STOP MOVES s3_test_with_ttl")
-
-    for i in range(30):
-        if i % 2 == 0:
-            node = node1
-        else:
-            node = node2
-
-        node.query(
-            f"INSERT INTO s3_test_with_ttl SELECT now() + 5, {i}, randomPrintableASCII(1048570)"
-        )
-
-    node1.query("SYSTEM SYNC REPLICA s3_test_with_ttl")
-    node2.query("SYSTEM SYNC REPLICA s3_test_with_ttl")
-    node3.query("SYSTEM SYNC REPLICA s3_test_with_ttl")
-
-    assert node1.query("SELECT COUNT() FROM s3_test_with_ttl") == "30\n"
-    assert node2.query("SELECT COUNT() FROM s3_test_with_ttl") == "30\n"
-
-    node1.query("SYSTEM START MOVES s3_test_with_ttl")
-    node2.query("SYSTEM START MOVES s3_test_with_ttl")
-
-    assert node1.query("SELECT COUNT() FROM s3_test_with_ttl") == "30\n"
-    assert node2.query("SELECT COUNT() FROM s3_test_with_ttl") == "30\n"
-
-    for attempt in reversed(range(5)):
-        time.sleep(5)
-
-        print(
-            node1.query(
-                "SELECT * FROM system.parts WHERE table = 's3_test_with_ttl' FORMAT Vertical"
-            )
-        )
-
-        minio = cluster.minio_client
-        objects = minio.list_objects(cluster.minio_bucket, "data/", recursive=True)
-        counter = 0
-        for obj in objects:
-            print(f"Objectname: {obj.object_name}, metadata: {obj.metadata}")
-            counter += 1
-
-        print(f"Total objects: {counter}")
-
-        if counter == 330:
-            break
-
-        print(f"Attempts remaining: {attempt}")
-
-    assert counter == 330
diff --git a/tests/integration/test_s3_zero_copy_ttl/test_ttl_move_memory_usage.py b/tests/integration/test_s3_zero_copy_ttl/test_ttl_move_memory_usage.py
deleted file mode 100644
index 5fbe426074f..00000000000
--- a/tests/integration/test_s3_zero_copy_ttl/test_ttl_move_memory_usage.py
+++ /dev/null
@@ -1,81 +0,0 @@
-#!/usr/bin/env python3
-import time
-
-import pytest
-
-# FIXME This test is too flaky
-# https://github.com/ClickHouse/ClickHouse/issues/45887
-
-pytestmark = pytest.mark.skip
-
-from helpers.cluster import ClickHouseCluster
-
-
-single_node_cluster = ClickHouseCluster(__file__)
-small_node = single_node_cluster.add_instance(
-    "small_node", main_configs=["configs/s3.xml"], with_minio=True
-)
-
-
-@pytest.fixture(scope="module")
-def started_single_node_cluster():
-    try:
-        single_node_cluster.start()
-
-        yield single_node_cluster
-    finally:
-        single_node_cluster.shutdown()
-
-
-def test_move_and_s3_memory_usage(started_single_node_cluster):
-    if small_node.is_built_with_sanitizer() or small_node.is_debug_build():
-        pytest.skip("Disabled for debug and sanitizers. Too slow.")
-
-    small_node.query(
-        "CREATE TABLE s3_test_with_ttl (x UInt32, a String codec(NONE), b String codec(NONE), c String codec(NONE), d String codec(NONE), e String codec(NONE)) engine = MergeTree order by x partition by x SETTINGS storage_policy='s3_and_default'"
-    )
-
-    for _ in range(10):
-        small_node.query(
-            "insert into s3_test_with_ttl select 0, repeat('a', 100), repeat('b', 100), repeat('c', 100), repeat('d', 100), repeat('e', 100) from zeros(400000) settings max_block_size = 8192, max_insert_block_size=10000000, min_insert_block_size_rows=10000000"
-        )
-
-    # After this, we should have 5 columns per 10 * 100 * 400000 ~ 400 MB; total ~2G data in partition
-    small_node.query(
-        "optimize table s3_test_with_ttl final",
-        settings={
-            "send_logs_level": "error",
-            "allow_prefetched_read_pool_for_remote_filesystem": 0,
-        },
-    )
-
-    small_node.query("system flush logs")
-    # Will take memory usage from metric_log.
-    # It is easier then specifying total memory limit (insert queries can hit this limit).
-    small_node.query("truncate table system.metric_log")
-
-    small_node.query(
-        "alter table s3_test_with_ttl move partition 0 to volume 'external'",
-        settings={
-            "send_logs_level": "error",
-            "allow_prefetched_read_pool_for_remote_filesystem": 0,
-        },
-    )
-    small_node.query("system flush logs")
-    max_usage = small_node.query(
-        """
-        select max(m.val - am.val * 4096) from
-        (select toStartOfMinute(event_time) as time, max(CurrentMetric_MemoryTracking) as val from system.metric_log group by time) as m join
-        (select toStartOfMinute(event_time) as time, min(value) as val from system.asynchronous_metric_log where metric='jemalloc.arenas.all.pdirty' group by time) as am using time;"""
-    )
-    # 3G limit is a big one. However, we can hit it anyway with parallel s3 writes enabled.
-    # Also actual value can be bigger because of memory drift.
-    # Increase it a little bit if test fails.
-    assert int(max_usage) < 3e9
-    res = small_node.query(
-        "select * from system.errors where last_error_message like '%Memory limit%' limit 1",
-        settings={
-            "allow_prefetched_read_pool_for_remote_filesystem": 0,
-        },
-    )
-    assert res == ""
diff --git a/tests/integration/test_s3_zero_copy_ttl/test_vertical_merge_memory_usage.py b/tests/integration/test_s3_zero_copy_ttl/test_vertical_merge_memory_usage.py
deleted file mode 100644
index fb9f3eb67b9..00000000000
--- a/tests/integration/test_s3_zero_copy_ttl/test_vertical_merge_memory_usage.py
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/usr/bin/env python3
-import time
-
-import pytest
-from helpers.cluster import ClickHouseCluster
-
-
-single_node_cluster = ClickHouseCluster(__file__)
-small_node = single_node_cluster.add_instance(
-    "small_node",
-    main_configs=["configs/s3.xml"],
-    user_configs=["configs/max_delayed_streams.xml"],
-    with_minio=True,
-)
-
-
-@pytest.fixture(scope="module")
-def started_single_node_cluster():
-    try:
-        single_node_cluster.start()
-
-        yield single_node_cluster
-    finally:
-        single_node_cluster.shutdown()
-
-
-def test_vertical_merge_memory_usage(started_single_node_cluster):
-    if small_node.is_built_with_sanitizer() or small_node.is_debug_build():
-        pytest.skip("Disabled for debug and sanitizers. Too slow.")
-
-    small_node.query(
-        "create table tvm2 (c0 UInt64, c1 UInt64, c2 UInt64, c3 UInt64, c4 UInt64, c5 UInt64, c6 UInt64, c7 UInt64, c8 UInt64, c9 UInt64, c10 UInt64, c11 UInt64, c12 UInt64, c13 UInt64, c14 UInt64, c15 UInt64, c16 UInt64, c17 UInt64, c18 UInt64, c19 UInt64, c20 UInt64, c21 UInt64, c22 UInt64, c23 UInt64, c24 UInt64, c25 UInt64, c26 UInt64, c27 UInt64, c28 UInt64, c29 UInt64, c30 UInt64, c31 UInt64, c32 UInt64, c33 UInt64, c34 UInt64, c35 UInt64, c36 UInt64, c37 UInt64, c38 UInt64, c39 UInt64, c40 UInt64, c41 UInt64, c42 UInt64, c43 UInt64, c44 UInt64, c45 UInt64, c46 UInt64, c47 UInt64, c48 UInt64, c49 UInt64, c50 UInt64, c51 UInt64, c52 UInt64, c53 UInt64, c54 UInt64, c55 UInt64, c56 UInt64, c57 UInt64, c58 UInt64, c59 UInt64, c60 UInt64, c61 UInt64, c62 UInt64, c63 UInt64, c64 UInt64, c65 UInt64, c66 UInt64, c67 UInt64, c68 UInt64, c69 UInt64, c70 UInt64, c71 UInt64, c72 UInt64, c73 UInt64, c74 UInt64, c75 UInt64, c76 UInt64, c77 UInt64, c78 UInt64, c79 UInt64, c80 UInt64, c81 UInt64, c82 UInt64, c83 UInt64, c84 UInt64, c85 UInt64, c86 UInt64, c87 UInt64, c88 UInt64, c89 UInt64, c90 UInt64, c91 UInt64, c92 UInt64, c93 UInt64, c94 UInt64, c95 UInt64, c96 UInt64, c97 UInt64, c98 UInt64, c99 UInt64, c100 UInt64, c101 UInt64, c102 UInt64, c103 UInt64, c104 UInt64, c105 UInt64, c106 UInt64, c107 UInt64, c108 UInt64, c109 UInt64, c110 UInt64, c111 UInt64, c112 UInt64, c113 UInt64, c114 UInt64, c115 UInt64, c116 UInt64, c117 UInt64, c118 UInt64, c119 UInt64, c120 UInt64, c121 UInt64, c122 UInt64, c123 UInt64, c124 UInt64, c125 UInt64, c126 UInt64, c127 UInt64, c128 UInt64, c129 UInt64, c130 UInt64, c131 UInt64, c132 UInt64, c133 UInt64, c134 UInt64, c135 UInt64, c136 UInt64, c137 UInt64, c138 UInt64, c139 UInt64, c140 UInt64, c141 UInt64, c142 UInt64, c143 UInt64, c144 UInt64, c145 UInt64, c146 UInt64, c147 UInt64, c148 UInt64, c149 UInt64, c150 UInt64, c151 UInt64, c152 UInt64, c153 UInt64, c154 UInt64, c155 UInt64, c156 UInt64, c157 UInt64, c158 UInt64, c159 UInt64, c160 UInt64, c161 UInt64, c162 UInt64, c163 UInt64, c164 UInt64, c165 UInt64, c166 UInt64, c167 UInt64, c168 UInt64, c169 UInt64, c170 UInt64, c171 UInt64, c172 UInt64, c173 UInt64, c174 UInt64, c175 UInt64, c176 UInt64, c177 UInt64, c178 UInt64, c179 UInt64, c180 UInt64, c181 UInt64, c182 UInt64, c183 UInt64, c184 UInt64, c185 UInt64, c186 UInt64, c187 UInt64, c188 UInt64, c189 UInt64, c190 UInt64, c191 UInt64, c192 UInt64, c193 UInt64, c194 UInt64, c195 UInt64, c196 UInt64, c197 UInt64, c198 UInt64, c199 UInt64, c200 UInt64, c201 UInt64, c202 UInt64, c203 UInt64, c204 UInt64, c205 UInt64, c206 UInt64, c207 UInt64, c208 UInt64, c209 UInt64, c210 UInt64, c211 UInt64, c212 UInt64, c213 UInt64, c214 UInt64, c215 UInt64, c216 UInt64, c217 UInt64, c218 UInt64, c219 UInt64, c220 UInt64, c221 UInt64, c222 UInt64, c223 UInt64, c224 UInt64, c225 UInt64, c226 UInt64, c227 UInt64, c228 UInt64, c229 UInt64, c230 UInt64, c231 UInt64, c232 UInt64, c233 UInt64, c234 UInt64, c235 UInt64, c236 UInt64, c237 UInt64, c238 UInt64, c239 UInt64, c240 UInt64, c241 UInt64, c242 UInt64, c243 UInt64, c244 UInt64, c245 UInt64, c246 UInt64, c247 UInt64, c248 UInt64, c249 UInt64, c250 UInt64, c251 UInt64, c252 UInt64, c253 UInt64, c254 UInt64, c255 UInt64, c256 UInt64, c257 UInt64, c258 UInt64, c259 UInt64, c260 UInt64, c261 UInt64, c262 UInt64, c263 UInt64, c264 UInt64, c265 UInt64, c266 UInt64, c267 UInt64, c268 UInt64, c269 UInt64, c270 UInt64, c271 UInt64, c272 UInt64, c273 UInt64, c274 UInt64, c275 UInt64, c276 UInt64, c277 UInt64, c278 UInt64, c279 UInt64, c280 UInt64, c281 UInt64, c282 UInt64, c283 UInt64, c284 UInt64, c285 UInt64, c286 UInt64, c287 UInt64, c288 UInt64, c289 UInt64, c290 UInt64, c291 UInt64, c292 UInt64, c293 UInt64, c294 UInt64, c295 UInt64, c296 UInt64, c297 UInt64, c298 UInt64, c299 UInt64) engine = MergeTree order by tuple() settings min_rows_for_wide_part = 10, min_bytes_for_wide_part=0, storage_policy = 's3', vertical_merge_algorithm_min_rows_to_activate=1"
-    )
-
-    small_node.query(
-        "insert into tvm2 select number + 0, number + 1, number + 2, number + 3, number + 4, number + 5, number + 6, number + 7, number + 8, number + 9, number + 10, number + 11, number + 12, number + 13, number + 14, number + 15, number + 16, number + 17, number + 18, number + 19, number + 20, number + 21, number + 22, number + 23, number + 24, number + 25, number + 26, number + 27, number + 28, number + 29, number + 30, number + 31, number + 32, number + 33, number + 34, number + 35, number + 36, number + 37, number + 38, number + 39, number + 40, number + 41, number + 42, number + 43, number + 44, number + 45, number + 46, number + 47, number + 48, number + 49, number + 50, number + 51, number + 52, number + 53, number + 54, number + 55, number + 56, number + 57, number + 58, number + 59, number + 60, number + 61, number + 62, number + 63, number + 64, number + 65, number + 66, number + 67, number + 68, number + 69, number + 70, number + 71, number + 72, number + 73, number + 74, number + 75, number + 76, number + 77, number + 78, number + 79, number + 80, number + 81, number + 82, number + 83, number + 84, number + 85, number + 86, number + 87, number + 88, number + 89, number + 90, number + 91, number + 92, number + 93, number + 94, number + 95, number + 96, number + 97, number + 98, number + 99, number + 100, number + 101, number + 102, number + 103, number + 104, number + 105, number + 106, number + 107, number + 108, number + 109, number + 110, number + 111, number + 112, number + 113, number + 114, number + 115, number + 116, number + 117, number + 118, number + 119, number + 120, number + 121, number + 122, number + 123, number + 124, number + 125, number + 126, number + 127, number + 128, number + 129, number + 130, number + 131, number + 132, number + 133, number + 134, number + 135, number + 136, number + 137, number + 138, number + 139, number + 140, number + 141, number + 142, number + 143, number + 144, number + 145, number + 146, number + 147, number + 148, number + 149, number + 150, number + 151, number + 152, number + 153, number + 154, number + 155, number + 156, number + 157, number + 158, number + 159, number + 160, number + 161, number + 162, number + 163, number + 164, number + 165, number + 166, number + 167, number + 168, number + 169, number + 170, number + 171, number + 172, number + 173, number + 174, number + 175, number + 176, number + 177, number + 178, number + 179, number + 180, number + 181, number + 182, number + 183, number + 184, number + 185, number + 186, number + 187, number + 188, number + 189, number + 190, number + 191, number + 192, number + 193, number + 194, number + 195, number + 196, number + 197, number + 198, number + 199, number + 200, number + 201, number + 202, number + 203, number + 204, number + 205, number + 206, number + 207, number + 208, number + 209, number + 210, number + 211, number + 212, number + 213, number + 214, number + 215, number + 216, number + 217, number + 218, number + 219, number + 220, number + 221, number + 222, number + 223, number + 224, number + 225, number + 226, number + 227, number + 228, number + 229, number + 230, number + 231, number + 232, number + 233, number + 234, number + 235, number + 236, number + 237, number + 238, number + 239, number + 240, number + 241, number + 242, number + 243, number + 244, number + 245, number + 246, number + 247, number + 248, number + 249, number + 250, number + 251, number + 252, number + 253, number + 254, number + 255, number + 256, number + 257, number + 258, number + 259, number + 260, number + 261, number + 262, number + 263, number + 264, number + 265, number + 266, number + 267, number + 268, number + 269, number + 270, number + 271, number + 272, number + 273, number + 274, number + 275, number + 276, number + 277, number + 278, number + 279, number + 280, number + 281, number + 282, number + 283, number + 284, number + 285, number + 286, number + 287, number + 288, number + 289, number + 290, number + 291, number + 292, number + 293, number + 294, number + 295, number + 296, number + 297, number + 298, number + 299 from numbers(20)"
-    )
-    small_node.query("optimize table tvm2 final")
-    small_node.query("system flush logs")
-
-    # Should be about 25M
-    res = small_node.query(
-        "select formatReadableSize(peak_memory_usage), * from system.part_log where table = 'tvm2' and database = currentDatabase() and event_date >= today() - 1 and event_type = 'MergeParts' and peak_memory_usage > 100000000 format Vertical"
-    )
-
-    assert res == ""
diff --git a/tests/integration/test_storage_nats/__init__.py b/tests/integration/test_storage_nats/__init__.py
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/integration/test_storage_nats/clickhouse_path/format_schemas/nats.proto b/tests/integration/test_storage_nats/clickhouse_path/format_schemas/nats.proto
deleted file mode 100755
index 090ed917cdd..00000000000
--- a/tests/integration/test_storage_nats/clickhouse_path/format_schemas/nats.proto
+++ /dev/null
@@ -1,6 +0,0 @@
-syntax = "proto3";
-
-  message ProtoKeyValue {
-   uint64 key = 1;
-   string value = 2;
- }
diff --git a/tests/integration/test_storage_nats/configs/macros.xml b/tests/integration/test_storage_nats/configs/macros.xml
deleted file mode 100644
index 4aa547e049e..00000000000
--- a/tests/integration/test_storage_nats/configs/macros.xml
+++ /dev/null
@@ -1,7 +0,0 @@
-<clickhouse>
-  <macros>
-    <nats_url>nats1:4444</nats_url>
-    <nats_subjects>macro</nats_subjects>
-    <nats_format>JSONEachRow</nats_format>
-  </macros>
-</clickhouse>
diff --git a/tests/integration/test_storage_nats/configs/named_collection.xml b/tests/integration/test_storage_nats/configs/named_collection.xml
deleted file mode 100644
index 15817f321f0..00000000000
--- a/tests/integration/test_storage_nats/configs/named_collection.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-<clickhouse>
-  <named_collections>
-    <nats1>
-        <nats_url>nats1:4444</nats_url>
-        <nats_subjects>named</nats_subjects>
-        <nats_format>JSONEachRow</nats_format>
-        <nats_skip_broken_messages>111</nats_skip_broken_messages>
-        <nats_num_consumers>12</nats_num_consumers>
-        <nats_username>click</nats_username>
-        <nats_password>house</nats_password>
-    </nats1>
-  </named_collections>
-</clickhouse>
diff --git a/tests/integration/test_storage_nats/configs/nats.xml b/tests/integration/test_storage_nats/configs/nats.xml
deleted file mode 100644
index 0a8be9122ad..00000000000
--- a/tests/integration/test_storage_nats/configs/nats.xml
+++ /dev/null
@@ -1,6 +0,0 @@
-<clickhouse>
-    <nats>
-        <user>click</user>
-        <password>house</password>
-    </nats>
-</clickhouse>
diff --git a/tests/integration/test_storage_nats/configs/users.xml b/tests/integration/test_storage_nats/configs/users.xml
deleted file mode 100644
index 2cef0a6de3c..00000000000
--- a/tests/integration/test_storage_nats/configs/users.xml
+++ /dev/null
@@ -1,7 +0,0 @@
-<clickhouse>
-    <profiles>
-        <default>
-            <stream_like_engine_allow_direct_select>1</stream_like_engine_allow_direct_select>
-        </default>
-    </profiles>
-</clickhouse>
diff --git a/tests/integration/test_storage_nats/nats_certs.sh b/tests/integration/test_storage_nats/nats_certs.sh
deleted file mode 100755
index 689221c39e4..00000000000
--- a/tests/integration/test_storage_nats/nats_certs.sh
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/bin/bash
-set -euxo pipefail
-
-mkdir -p "${NATS_CERT_DIR}/ca"
-mkdir -p "${NATS_CERT_DIR}/nats"
-openssl req -newkey rsa:4096 -x509 -days 365 -nodes -batch -keyout "${NATS_CERT_DIR}/ca/ca-key.pem" -out "${NATS_CERT_DIR}/ca/ca-cert.pem" -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=ca"
-openssl req -newkey rsa:4096 -nodes -batch -keyout "${NATS_CERT_DIR}/nats/server-key.pem" -out "${NATS_CERT_DIR}/nats/server-req.pem" -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=server"
-openssl x509 -req -days 365 -in "${NATS_CERT_DIR}/nats/server-req.pem" -CA "${NATS_CERT_DIR}/ca/ca-cert.pem" -CAkey "${NATS_CERT_DIR}/ca/ca-key.pem" -CAcreateserial -out "${NATS_CERT_DIR}/nats/server-cert.pem" -extfile <(
-cat <<-EOF
-subjectAltName = DNS:localhost, DNS:nats1
-EOF
-)
-rm -f "${NATS_CERT_DIR}/nats/server-req.pem"
diff --git a/tests/integration/test_storage_nats/nats_pb2.py b/tests/integration/test_storage_nats/nats_pb2.py
deleted file mode 100644
index e9e5cb72363..00000000000
--- a/tests/integration/test_storage_nats/nats_pb2.py
+++ /dev/null
@@ -1,37 +0,0 @@
-# -*- coding: utf-8 -*-
-# Generated by the protocol buffer compiler.  DO NOT EDIT!
-# source: clickhouse_path/format_schemas/nats.proto
-"""Generated protocol buffer code."""
-from google.protobuf import descriptor as _descriptor
-from google.protobuf import descriptor_pool as _descriptor_pool
-from google.protobuf import message as _message
-from google.protobuf import reflection as _reflection
-from google.protobuf import symbol_database as _symbol_database
-
-# @@protoc_insertion_point(imports)
-
-_sym_db = _symbol_database.Default()
-
-
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
-    b'\n)clickhouse_path/format_schemas/nats.proto"+\n\rProtoKeyValue\x12\x0b\n\x03key\x18\x01 \x01(\x04\x12\r\n\x05value\x18\x02 \x01(\tb\x06proto3'
-)
-
-
-_PROTOKEYVALUE = DESCRIPTOR.message_types_by_name["ProtoKeyValue"]
-ProtoKeyValue = _reflection.GeneratedProtocolMessageType(
-    "ProtoKeyValue",
-    (_message.Message,),
-    {
-        "DESCRIPTOR": _PROTOKEYVALUE,
-        "__module__": "clickhouse_path.format_schemas.nats_pb2"
-        # @@protoc_insertion_point(class_scope:ProtoKeyValue)
-    },
-)
-_sym_db.RegisterMessage(ProtoKeyValue)
-
-if _descriptor._USE_C_DESCRIPTORS == False:
-    DESCRIPTOR._options = None
-    _PROTOKEYVALUE._serialized_start = 45
-    _PROTOKEYVALUE._serialized_end = 88
-# @@protoc_insertion_point(module_scope)
diff --git a/tests/integration/test_storage_nats/test.py b/tests/integration/test_storage_nats/test.py
deleted file mode 100644
index 4d7e4cf813d..00000000000
--- a/tests/integration/test_storage_nats/test.py
+++ /dev/null
@@ -1,1875 +0,0 @@
-import pytest
-
-# FIXME This test is too flaky
-# https://github.com/ClickHouse/ClickHouse/issues/39185
-
-pytestmark = pytest.mark.skip
-
-import json
-import os.path as p
-import random
-import subprocess
-import threading
-import logging
-import time
-from random import randrange
-import math
-
-import asyncio
-from google.protobuf.internal.encoder import _VarintBytes
-from helpers.client import QueryRuntimeException
-from helpers.cluster import ClickHouseCluster, check_nats_is_available, nats_connect_ssl
-from helpers.test_tools import TSV
-
-from . import nats_pb2
-
-cluster = ClickHouseCluster(__file__)
-instance = cluster.add_instance(
-    "instance",
-    main_configs=[
-        "configs/nats.xml",
-        "configs/macros.xml",
-        "configs/named_collection.xml",
-    ],
-    user_configs=["configs/users.xml"],
-    with_nats=True,
-    clickhouse_path_dir="clickhouse_path",
-)
-
-
-# Helpers
-
-
-def wait_nats_to_start(nats_port, ssl_ctx=None, timeout=180):
-    start = time.time()
-    while time.time() - start < timeout:
-        try:
-            if asyncio.run(check_nats_is_available(nats_port, ssl_ctx=ssl_ctx)):
-                logging.debug("NATS is available")
-                return
-            time.sleep(0.5)
-        except Exception as ex:
-            logging.debug("Can't connect to NATS " + str(ex))
-            time.sleep(0.5)
-
-
-def nats_check_result(result, check=False, ref_file="test_nats_json.reference"):
-    fpath = p.join(p.dirname(__file__), ref_file)
-    with open(fpath) as reference:
-        if check:
-            assert TSV(result) == TSV(reference)
-        else:
-            return TSV(result) == TSV(reference)
-
-
-def kill_nats(nats_id):
-    p = subprocess.Popen(("docker", "stop", nats_id), stdout=subprocess.PIPE)
-    p.communicate()
-    return p.returncode == 0
-
-
-def revive_nats(nats_id, nats_port):
-    p = subprocess.Popen(("docker", "start", nats_id), stdout=subprocess.PIPE)
-    p.communicate()
-    wait_nats_to_start(nats_port)
-
-
-# Fixtures
-
-
-@pytest.fixture(scope="module")
-def nats_cluster():
-    try:
-        cluster.start()
-        logging.debug("nats_id is {}".format(instance.cluster.nats_docker_id))
-        instance.query("CREATE DATABASE test")
-
-        yield cluster
-
-    finally:
-        cluster.shutdown()
-
-
-@pytest.fixture(autouse=True)
-def nats_setup_teardown():
-    print("NATS is available - running test")
-    yield  # run test
-    instance.query("DROP DATABASE test SYNC")
-    instance.query("CREATE DATABASE test")
-
-
-# Tests
-
-
-async def nats_produce_messages(cluster_inst, subject, messages=(), bytes=None):
-    nc = await nats_connect_ssl(
-        cluster_inst.nats_port,
-        user="click",
-        password="house",
-        ssl_ctx=cluster_inst.nats_ssl_context,
-    )
-    logging.debug("NATS connection status: " + str(nc.is_connected))
-
-    for message in messages:
-        await nc.publish(subject, message.encode())
-    if bytes is not None:
-        await nc.publish(subject, bytes)
-    logging.debug("Finished publising to " + subject)
-
-    await nc.close()
-    return messages
-
-
-def check_table_is_ready(instance, table_name):
-    try:
-        instance.query("SELECT * FROM {}".format(table_name))
-        return True
-    except Exception:
-        return False
-
-
-def test_nats_select(nats_cluster):
-    instance.query(
-        """
-        CREATE TABLE test.nats (key UInt64, value UInt64)
-            ENGINE = NATS
-            SETTINGS nats_url = 'nats1:4444',
-                     nats_subjects = 'select',
-                     nats_format = 'JSONEachRow',
-                     nats_row_delimiter = '\\n';
-        """
-    )
-    while not check_table_is_ready(instance, "test.nats"):
-        logging.debug("Table test.nats is not yet ready")
-        time.sleep(0.5)
-
-    messages = []
-    for i in range(50):
-        messages.append(json.dumps({"key": i, "value": i}))
-    asyncio.run(nats_produce_messages(nats_cluster, "select", messages))
-
-    # The order of messages in select * from test.nats is not guaranteed, so sleep to collect everything in one select
-    time.sleep(1)
-
-    result = ""
-    while True:
-        result += instance.query(
-            "SELECT * FROM test.nats ORDER BY key", ignore_error=True
-        )
-        if nats_check_result(result):
-            break
-
-    nats_check_result(result, True)
-
-
-def test_nats_select_empty(nats_cluster):
-    instance.query(
-        """
-        CREATE TABLE test.nats (key UInt64, value UInt64)
-            ENGINE = NATS
-            SETTINGS nats_url = 'nats1:4444',
-                     nats_subjects = 'empty',
-                     nats_format = 'TSV',
-                     nats_row_delimiter = '\\n';
-        """
-    )
-
-    assert int(instance.query("SELECT count() FROM test.nats")) == 0
-
-
-def test_nats_json_without_delimiter(nats_cluster):
-    instance.query(
-        """
-        CREATE TABLE test.nats (key UInt64, value UInt64)
-            ENGINE = NATS
-            SETTINGS nats_url = 'nats1:4444',
-                     nats_subjects = 'json',
-                     nats_format = 'JSONEachRow';
-        """
-    )
-    while not check_table_is_ready(instance, "test.nats"):
-        logging.debug("Table test.nats is not yet ready")
-        time.sleep(0.5)
-
-    messages = ""
-    for i in range(25):
-        messages += json.dumps({"key": i, "value": i}) + "\n"
-
-    all_messages = [messages]
-    asyncio.run(nats_produce_messages(nats_cluster, "json", all_messages))
-
-    messages = ""
-    for i in range(25, 50):
-        messages += json.dumps({"key": i, "value": i}) + "\n"
-    all_messages = [messages]
-    asyncio.run(nats_produce_messages(nats_cluster, "json", all_messages))
-
-    time.sleep(1)
-
-    result = ""
-    time_limit_sec = 60
-    deadline = time.monotonic() + time_limit_sec
-
-    while time.monotonic() < deadline:
-        result += instance.query(
-            "SELECT * FROM test.nats ORDER BY key", ignore_error=True
-        )
-        if nats_check_result(result):
-            break
-
-    nats_check_result(result, True)
-
-
-def test_nats_csv_with_delimiter(nats_cluster):
-    instance.query(
-        """
-        CREATE TABLE test.nats (key UInt64, value UInt64)
-            ENGINE = NATS
-            SETTINGS nats_url = 'nats1:4444',
-                     nats_subjects = 'csv',
-                     nats_format = 'CSV',
-                     nats_row_delimiter = '\\n';
-        """
-    )
-    while not check_table_is_ready(instance, "test.nats"):
-        logging.debug("Table test.nats is not yet ready")
-        time.sleep(0.5)
-
-    messages = []
-    for i in range(50):
-        messages.append("{i}, {i}".format(i=i))
-
-    asyncio.run(nats_produce_messages(nats_cluster, "csv", messages))
-
-    time.sleep(1)
-
-    result = ""
-    for _ in range(60):
-        result += instance.query(
-            "SELECT * FROM test.nats ORDER BY key", ignore_error=True
-        )
-        if nats_check_result(result):
-            break
-
-    nats_check_result(result, True)
-
-
-def test_nats_tsv_with_delimiter(nats_cluster):
-    instance.query(
-        """
-        CREATE TABLE test.nats (key UInt64, value UInt64)
-            ENGINE = NATS
-            SETTINGS nats_url = 'nats1:4444',
-                     nats_subjects = 'tsv',
-                     nats_format = 'TSV',
-                     nats_row_delimiter = '\\n';
-        CREATE TABLE test.view (key UInt64, value UInt64)
-            ENGINE = MergeTree()
-            ORDER BY key;
-        CREATE MATERIALIZED VIEW test.consumer TO test.view AS
-            SELECT * FROM test.nats;
-        """
-    )
-    while not check_table_is_ready(instance, "test.nats"):
-        logging.debug("Table test.nats is not yet ready")
-        time.sleep(0.5)
-
-    messages = []
-    for i in range(50):
-        messages.append("{i}\t{i}".format(i=i))
-
-    asyncio.run(nats_produce_messages(nats_cluster, "tsv", messages))
-
-    result = ""
-    for _ in range(60):
-        result = instance.query("SELECT * FROM test.view ORDER BY key")
-        if nats_check_result(result):
-            break
-
-    nats_check_result(result, True)
-
-
-#
-
-
-def test_nats_macros(nats_cluster):
-    instance.query(
-        """
-        CREATE TABLE test.nats (key UInt64, value UInt64)
-            ENGINE = NATS
-            SETTINGS nats_url = '{nats_url}',
-                     nats_subjects = '{nats_subjects}',
-                     nats_format = '{nats_format}'
-        """
-    )
-    while not check_table_is_ready(instance, "test.nats"):
-        logging.debug("Table test.nats is not yet ready")
-        time.sleep(0.5)
-
-    message = ""
-    for i in range(50):
-        message += json.dumps({"key": i, "value": i}) + "\n"
-    asyncio.run(nats_produce_messages(nats_cluster, "macro", [message]))
-
-    time.sleep(1)
-
-    result = ""
-    for _ in range(60):
-        result += instance.query(
-            "SELECT * FROM test.nats ORDER BY key", ignore_error=True
-        )
-        if nats_check_result(result):
-            break
-
-    nats_check_result(result, True)
-
-
-def test_nats_materialized_view(nats_cluster):
-    instance.query(
-        """
-        CREATE TABLE test.nats (key UInt64, value UInt64)
-            ENGINE = NATS
-            SETTINGS nats_url = 'nats1:4444',
-                     nats_subjects = 'mv',
-                     nats_format = 'JSONEachRow',
-                     nats_row_delimiter = '\\n';
-        CREATE TABLE test.view (key UInt64, value UInt64)
-            ENGINE = MergeTree()
-            ORDER BY key;
-        CREATE MATERIALIZED VIEW test.consumer TO test.view AS
-            SELECT * FROM test.nats;
-
-        CREATE TABLE test.view2 (key UInt64, value UInt64)
-            ENGINE = MergeTree()
-            ORDER BY key;
-        CREATE MATERIALIZED VIEW test.consumer2 TO test.view2 AS
-            SELECT * FROM test.nats group by (key, value);
-    """
-    )
-    while not check_table_is_ready(instance, "test.nats"):
-        logging.debug("Table test.nats is not yet ready")
-        time.sleep(0.5)
-
-    messages = []
-    for i in range(50):
-        messages.append(json.dumps({"key": i, "value": i}))
-
-    asyncio.run(nats_produce_messages(nats_cluster, "mv", messages))
-
-    time_limit_sec = 60
-    deadline = time.monotonic() + time_limit_sec
-
-    while time.monotonic() < deadline:
-        result = instance.query("SELECT * FROM test.view ORDER BY key")
-        if nats_check_result(result):
-            break
-
-    nats_check_result(result, True)
-
-    deadline = time.monotonic() + time_limit_sec
-
-    while time.monotonic() < deadline:
-        result = instance.query("SELECT * FROM test.view2 ORDER BY key")
-        if nats_check_result(result):
-            break
-
-    nats_check_result(result, True)
-
-
-def test_nats_materialized_view_with_subquery(nats_cluster):
-    instance.query(
-        """
-        CREATE TABLE test.nats (key UInt64, value UInt64)
-            ENGINE = NATS
-            SETTINGS nats_url = 'nats1:4444',
-                     nats_subjects = 'mvsq',
-                     nats_format = 'JSONEachRow',
-                     nats_row_delimiter = '\\n';
-        CREATE TABLE test.view (key UInt64, value UInt64)
-            ENGINE = MergeTree()
-            ORDER BY key;
-        CREATE MATERIALIZED VIEW test.consumer TO test.view AS
-            SELECT * FROM (SELECT * FROM test.nats);
-    """
-    )
-    while not check_table_is_ready(instance, "test.nats"):
-        logging.debug("Table test.nats is not yet ready")
-        time.sleep(0.5)
-
-    messages = []
-    for i in range(50):
-        messages.append(json.dumps({"key": i, "value": i}))
-    asyncio.run(nats_produce_messages(nats_cluster, "mvsq", messages))
-
-    time_limit_sec = 60
-    deadline = time.monotonic() + time_limit_sec
-
-    while time.monotonic() < deadline:
-        result = instance.query("SELECT * FROM test.view ORDER BY key")
-        if nats_check_result(result):
-            break
-
-    nats_check_result(result, True)
-
-
-def test_nats_many_materialized_views(nats_cluster):
-    instance.query(
-        """
-        DROP TABLE IF EXISTS test.view1;
-        DROP TABLE IF EXISTS test.view2;
-        DROP TABLE IF EXISTS test.consumer1;
-        DROP TABLE IF EXISTS test.consumer2;
-        CREATE TABLE test.nats (key UInt64, value UInt64)
-            ENGINE = NATS
-            SETTINGS nats_url = 'nats1:4444',
-                     nats_subjects = 'mmv',
-                     nats_format = 'JSONEachRow',
-                     nats_row_delimiter = '\\n';
-        CREATE TABLE test.view1 (key UInt64, value UInt64)
-            ENGINE = MergeTree()
-            ORDER BY key;
-        CREATE TABLE test.view2 (key UInt64, value UInt64)
-            ENGINE = MergeTree()
-            ORDER BY key;
-        CREATE MATERIALIZED VIEW test.consumer1 TO test.view1 AS
-            SELECT * FROM test.nats;
-        CREATE MATERIALIZED VIEW test.consumer2 TO test.view2 AS
-            SELECT * FROM test.nats;
-    """
-    )
-    while not check_table_is_ready(instance, "test.nats"):
-        logging.debug("Table test.nats is not yet ready")
-        time.sleep(0.5)
-
-    messages = []
-    for i in range(50):
-        messages.append(json.dumps({"key": i, "value": i}))
-    asyncio.run(nats_produce_messages(nats_cluster, "mmv", messages))
-
-    time_limit_sec = 60
-    deadline = time.monotonic() + time_limit_sec
-
-    while time.monotonic() < deadline:
-        result1 = instance.query("SELECT * FROM test.view1 ORDER BY key")
-        result2 = instance.query("SELECT * FROM test.view2 ORDER BY key")
-        if nats_check_result(result1) and nats_check_result(result2):
-            break
-
-    instance.query(
-        """
-        DROP TABLE test.consumer1;
-        DROP TABLE test.consumer2;
-        DROP TABLE test.view1;
-        DROP TABLE test.view2;
-    """
-    )
-
-    nats_check_result(result1, True)
-    nats_check_result(result2, True)
-
-
-def test_nats_protobuf(nats_cluster):
-    instance.query(
-        """
-        CREATE TABLE test.nats (key UInt64, value String)
-            ENGINE = NATS
-            SETTINGS nats_url = 'nats1:4444',
-                     nats_subjects = 'pb',
-                     nats_format = 'Protobuf',
-                     nats_schema = 'nats.proto:ProtoKeyValue';
-        CREATE TABLE test.view (key UInt64, value UInt64)
-            ENGINE = MergeTree()
-            ORDER BY key;
-        CREATE MATERIALIZED VIEW test.consumer TO test.view AS
-            SELECT * FROM test.nats;
-        """
-    )
-    while not check_table_is_ready(instance, "test.nats"):
-        logging.debug("Table test.nats is not yet ready")
-        time.sleep(0.5)
-
-    data = b""
-    for i in range(0, 20):
-        msg = nats_pb2.ProtoKeyValue()
-        msg.key = i
-        msg.value = str(i)
-        serialized_msg = msg.SerializeToString()
-        data = data + _VarintBytes(len(serialized_msg)) + serialized_msg
-    asyncio.run(nats_produce_messages(nats_cluster, "pb", bytes=data))
-    data = b""
-    for i in range(20, 21):
-        msg = nats_pb2.ProtoKeyValue()
-        msg.key = i
-        msg.value = str(i)
-        serialized_msg = msg.SerializeToString()
-        data = data + _VarintBytes(len(serialized_msg)) + serialized_msg
-    asyncio.run(nats_produce_messages(nats_cluster, "pb", bytes=data))
-    data = b""
-    for i in range(21, 50):
-        msg = nats_pb2.ProtoKeyValue()
-        msg.key = i
-        msg.value = str(i)
-        serialized_msg = msg.SerializeToString()
-        data = data + _VarintBytes(len(serialized_msg)) + serialized_msg
-    asyncio.run(nats_produce_messages(nats_cluster, "pb", bytes=data))
-
-    result = ""
-    time_limit_sec = 60
-    deadline = time.monotonic() + time_limit_sec
-
-    while time.monotonic() < deadline:
-        result = instance.query("SELECT * FROM test.view ORDER BY key")
-        if nats_check_result(result):
-            break
-
-    nats_check_result(result, True)
-
-
-def test_nats_big_message(nats_cluster):
-    # Create batchs of messages of size ~100Kb
-    nats_messages = 1000
-    batch_messages = 1000
-    messages = [
-        json.dumps({"key": i, "value": "x" * 100}) * batch_messages
-        for i in range(nats_messages)
-    ]
-
-    instance.query(
-        """
-        CREATE TABLE test.nats (key UInt64, value String)
-            ENGINE = NATS
-            SETTINGS nats_url = 'nats1:4444',
-                     nats_subjects = 'big',
-                     nats_format = 'JSONEachRow';
-        CREATE TABLE test.view (key UInt64, value String)
-            ENGINE = MergeTree
-            ORDER BY key;
-        CREATE MATERIALIZED VIEW test.consumer TO test.view AS
-            SELECT * FROM test.nats;
-    """
-    )
-    while not check_table_is_ready(instance, "test.nats"):
-        logging.debug("Table test.nats is not yet ready")
-        time.sleep(0.5)
-
-    asyncio.run(nats_produce_messages(nats_cluster, "big", messages))
-
-    while True:
-        result = instance.query("SELECT count() FROM test.view")
-        if int(result) == batch_messages * nats_messages:
-            break
-
-    assert (
-        int(result) == nats_messages * batch_messages
-    ), "ClickHouse lost some messages: {}".format(result)
-
-
-def test_nats_mv_combo(nats_cluster):
-    NUM_MV = 5
-    NUM_CONSUMERS = 4
-
-    instance.query(
-        """
-        CREATE TABLE test.nats (key UInt64, value UInt64)
-            ENGINE = NATS
-            SETTINGS nats_url = 'nats1:4444',
-                     nats_subjects = 'combo',
-                     nats_num_consumers = {},
-                     nats_format = 'JSONEachRow',
-                     nats_row_delimiter = '\\n';
-    """.format(
-            NUM_CONSUMERS
-        )
-    )
-    while not check_table_is_ready(instance, "test.nats"):
-        logging.debug("Table test.nats is not yet ready")
-        time.sleep(0.5)
-
-    for mv_id in range(NUM_MV):
-        instance.query(
-            """
-            DROP TABLE IF EXISTS test.combo_{0};
-            DROP TABLE IF EXISTS test.combo_{0}_mv;
-            CREATE TABLE test.combo_{0} (key UInt64, value UInt64)
-                ENGINE = MergeTree()
-                ORDER BY key;
-            CREATE MATERIALIZED VIEW test.combo_{0}_mv TO test.combo_{0} AS
-                SELECT * FROM test.nats;
-        """.format(
-                mv_id
-            )
-        )
-
-    time.sleep(2)
-
-    i = [0]
-    messages_num = 10000
-
-    def produce():
-        messages = []
-        for _ in range(messages_num):
-            messages.append(json.dumps({"key": i[0], "value": i[0]}))
-            i[0] += 1
-        asyncio.run(nats_produce_messages(nats_cluster, "combo", messages))
-
-    threads = []
-    threads_num = 20
-
-    for _ in range(threads_num):
-        threads.append(threading.Thread(target=produce))
-    for thread in threads:
-        time.sleep(random.uniform(0, 1))
-        thread.start()
-
-    while True:
-        result = 0
-        for mv_id in range(NUM_MV):
-            result += int(
-                instance.query("SELECT count() FROM test.combo_{0}".format(mv_id))
-            )
-        if int(result) == messages_num * threads_num * NUM_MV:
-            break
-        time.sleep(1)
-
-    for thread in threads:
-        thread.join()
-
-    for mv_id in range(NUM_MV):
-        instance.query(
-            """
-            DROP TABLE test.combo_{0}_mv;
-            DROP TABLE test.combo_{0};
-        """.format(
-                mv_id
-            )
-        )
-
-    assert (
-        int(result) == messages_num * threads_num * NUM_MV
-    ), "ClickHouse lost some messages: {}".format(result)
-
-
-def test_nats_insert(nats_cluster):
-    instance.query(
-        """
-        CREATE TABLE test.nats (key UInt64, value UInt64)
-            ENGINE = NATS
-            SETTINGS nats_url = 'nats1:4444',
-                     nats_subjects = 'insert',
-                     nats_format = 'TSV',
-                     nats_row_delimiter = '\\n';
-    """
-    )
-    while not check_table_is_ready(instance, "test.nats"):
-        logging.debug("Table test.nats is not yet ready")
-        time.sleep(0.5)
-
-    values = []
-    for i in range(50):
-        values.append("({i}, {i})".format(i=i))
-    values = ",".join(values)
-
-    insert_messages = []
-
-    async def sub_to_nats():
-        nc = await nats_connect_ssl(
-            nats_cluster.nats_port,
-            user="click",
-            password="house",
-            ssl_ctx=nats_cluster.nats_ssl_context,
-        )
-        sub = await nc.subscribe("insert")
-        await sub.unsubscribe(50)
-        async for msg in sub.messages:
-            insert_messages.append(msg.data.decode())
-
-        await sub.drain()
-        await nc.drain()
-
-    def run_sub():
-        asyncio.run(sub_to_nats())
-
-    thread = threading.Thread(target=run_sub)
-    thread.start()
-    time.sleep(1)
-
-    while True:
-        try:
-            instance.query("INSERT INTO test.nats VALUES {}".format(values))
-            break
-        except QueryRuntimeException as e:
-            if "Local: Timed out." in str(e):
-                continue
-            else:
-                raise
-    thread.join()
-
-    result = "\n".join(insert_messages)
-    nats_check_result(result, True)
-
-
-def test_nats_many_subjects_insert_wrong(nats_cluster):
-    instance.query(
-        """
-        CREATE TABLE test.nats (key UInt64, value UInt64)
-            ENGINE = NATS
-            SETTINGS nats_url = 'nats1:4444',
-                     nats_subjects = 'insert1,insert2.>,insert3.*.foo',
-                     nats_format = 'TSV',
-                     nats_row_delimiter = '\\n';
-    """
-    )
-    while not check_table_is_ready(instance, "test.nats"):
-        logging.debug("Table test.nats is not yet ready")
-        time.sleep(0.5)
-
-    values = []
-    for i in range(50):
-        values.append("({i}, {i})".format(i=i))
-    values = ",".join(values)
-
-    # no subject specified
-    instance.query_and_get_error("INSERT INTO test.nats VALUES {}".format(values))
-
-    # can't insert into wildcard subjects
-    instance.query_and_get_error(
-        "INSERT INTO test.nats SETTINGS stream_like_engine_insert_queue='insert2.>' VALUES {}".format(
-            values
-        )
-    )
-    instance.query_and_get_error(
-        "INSERT INTO test.nats SETTINGS stream_like_engine_insert_queue='insert3.*.foo' VALUES {}".format(
-            values
-        )
-    )
-
-    # specified subject is not among engine's subjects
-    instance.query_and_get_error(
-        "INSERT INTO test.nats SETTINGS stream_like_engine_insert_queue='insert4' VALUES {}".format(
-            values
-        )
-    )
-    instance.query_and_get_error(
-        "INSERT INTO test.nats SETTINGS stream_like_engine_insert_queue='insert3.foo.baz' VALUES {}".format(
-            values
-        )
-    )
-    instance.query_and_get_error(
-        "INSERT INTO test.nats SETTINGS stream_like_engine_insert_queue='foo.insert2' VALUES {}".format(
-            values
-        )
-    )
-
-
-def test_nats_many_subjects_insert_right(nats_cluster):
-    instance.query(
-        """
-        CREATE TABLE test.nats (key UInt64, value UInt64)
-            ENGINE = NATS
-            SETTINGS nats_url = 'nats1:4444',
-                     nats_subjects = 'right_insert1,right_insert2',
-                     nats_format = 'TSV',
-                     nats_row_delimiter = '\\n';
-    """
-    )
-    while not check_table_is_ready(instance, "test.nats"):
-        logging.debug("Table test.nats is not yet ready")
-        time.sleep(0.5)
-
-    values = []
-    for i in range(50):
-        values.append("({i}, {i})".format(i=i))
-    values = ",".join(values)
-
-    insert_messages = []
-
-    async def sub_to_nats():
-        nc = await nats_connect_ssl(
-            nats_cluster.nats_port,
-            user="click",
-            password="house",
-            ssl_ctx=nats_cluster.nats_ssl_context,
-        )
-        sub = await nc.subscribe("right_insert1")
-        await sub.unsubscribe(50)
-        async for msg in sub.messages:
-            insert_messages.append(msg.data.decode())
-
-        await sub.drain()
-        await nc.drain()
-
-    def run_sub():
-        asyncio.run(sub_to_nats())
-
-    thread = threading.Thread(target=run_sub)
-    thread.start()
-    time.sleep(1)
-
-    while True:
-        try:
-            instance.query(
-                "INSERT INTO test.nats SETTINGS stream_like_engine_insert_queue='right_insert1' VALUES {}".format(
-                    values
-                )
-            )
-            break
-        except QueryRuntimeException as e:
-            if "Local: Timed out." in str(e):
-                continue
-            else:
-                raise
-    thread.join()
-
-    result = "\n".join(insert_messages)
-    nats_check_result(result, True)
-
-
-def test_nats_many_inserts(nats_cluster):
-    instance.query(
-        """
-        DROP TABLE IF EXISTS test.nats_many;
-        DROP TABLE IF EXISTS test.nats_consume;
-        DROP TABLE IF EXISTS test.view_many;
-        DROP TABLE IF EXISTS test.consumer_many;
-        CREATE TABLE test.nats_many (key UInt64, value UInt64)
-            ENGINE = NATS
-            SETTINGS nats_url = 'nats1:4444',
-                     nats_subjects = 'many_inserts',
-                     nats_format = 'TSV',
-                     nats_row_delimiter = '\\n';
-        CREATE TABLE test.nats_consume (key UInt64, value UInt64)
-            ENGINE = NATS
-            SETTINGS nats_url = 'nats1:4444',
-                     nats_subjects = 'many_inserts',
-                     nats_format = 'TSV',
-                     nats_row_delimiter = '\\n';
-        CREATE TABLE test.view_many (key UInt64, value UInt64)
-            ENGINE = MergeTree
-            ORDER BY key;
-        CREATE MATERIALIZED VIEW test.consumer_many TO test.view_many AS
-            SELECT * FROM test.nats_consume;
-    """
-    )
-    while not check_table_is_ready(instance, "test.nats_consume"):
-        logging.debug("Table test.nats_consume is not yet ready")
-        time.sleep(0.5)
-
-    messages_num = 10000
-    values = []
-    for i in range(messages_num):
-        values.append("({i}, {i})".format(i=i))
-    values = ",".join(values)
-
-    def insert():
-        while True:
-            try:
-                instance.query("INSERT INTO test.nats_many VALUES {}".format(values))
-                break
-            except QueryRuntimeException as e:
-                if "Local: Timed out." in str(e):
-                    continue
-                else:
-                    raise
-
-    threads = []
-    threads_num = 10
-    for _ in range(threads_num):
-        threads.append(threading.Thread(target=insert))
-    for thread in threads:
-        time.sleep(random.uniform(0, 1))
-        thread.start()
-
-    for thread in threads:
-        thread.join()
-
-    time_limit_sec = 300
-    deadline = time.monotonic() + time_limit_sec
-
-    while time.monotonic() < deadline:
-        result = instance.query("SELECT count() FROM test.view_many")
-        print(result, messages_num * threads_num)
-        if int(result) >= messages_num * threads_num:
-            break
-        time.sleep(1)
-
-    instance.query(
-        """
-        DROP TABLE test.nats_consume;
-        DROP TABLE test.nats_many;
-        DROP TABLE test.consumer_many;
-        DROP TABLE test.view_many;
-    """
-    )
-
-    assert (
-        int(result) == messages_num * threads_num
-    ), "ClickHouse lost some messages or got duplicated ones. Total count: {}".format(
-        result
-    )
-
-
-def test_nats_overloaded_insert(nats_cluster):
-    instance.query(
-        """
-        DROP TABLE IF EXISTS test.view_overload;
-        DROP TABLE IF EXISTS test.consumer_overload;
-        DROP TABLE IF EXISTS test.nats_consume;
-        CREATE TABLE test.nats_consume (key UInt64, value UInt64)
-            ENGINE = NATS
-            SETTINGS nats_url = 'nats1:4444',
-                     nats_subjects = 'over',
-                     nats_num_consumers = 5,
-                     nats_max_block_size = 10000,
-                     nats_format = 'TSV',
-                     nats_row_delimiter = '\\n';
-        CREATE TABLE test.nats_overload (key UInt64, value UInt64)
-            ENGINE = NATS
-            SETTINGS nats_url = 'nats1:4444',
-                     nats_subjects = 'over',
-                     nats_format = 'TSV',
-                     nats_row_delimiter = '\\n';
-        CREATE TABLE test.view_overload (key UInt64, value UInt64)
-            ENGINE = MergeTree
-            ORDER BY key
-            SETTINGS old_parts_lifetime=5, cleanup_delay_period=2, cleanup_delay_period_random_add=3,
-            cleanup_thread_preferred_points_per_iteration=0;
-        CREATE MATERIALIZED VIEW test.consumer_overload TO test.view_overload AS
-            SELECT * FROM test.nats_consume;
-    """
-    )
-    while not check_table_is_ready(instance, "test.nats_consume"):
-        logging.debug("Table test.nats_consume is not yet ready")
-        time.sleep(0.5)
-
-    messages_num = 100000
-
-    def insert():
-        values = []
-        for i in range(messages_num):
-            values.append("({i}, {i})".format(i=i))
-        values = ",".join(values)
-
-        while True:
-            try:
-                instance.query(
-                    "INSERT INTO test.nats_overload VALUES {}".format(values)
-                )
-                break
-            except QueryRuntimeException as e:
-                if "Local: Timed out." in str(e):
-                    continue
-                else:
-                    raise
-
-    threads = []
-    threads_num = 5
-    for _ in range(threads_num):
-        threads.append(threading.Thread(target=insert))
-    for thread in threads:
-        time.sleep(random.uniform(0, 1))
-        thread.start()
-
-    time_limit_sec = 300
-    deadline = time.monotonic() + time_limit_sec
-
-    while time.monotonic() < deadline:
-        result = instance.query("SELECT count() FROM test.view_overload")
-        time.sleep(1)
-        if int(result) >= messages_num * threads_num:
-            break
-
-    instance.query(
-        """
-        DROP TABLE test.consumer_overload;
-        DROP TABLE test.view_overload;
-        DROP TABLE test.nats_consume;
-        DROP TABLE test.nats_overload;
-    """
-    )
-
-    for thread in threads:
-        thread.join()
-
-    assert (
-        int(result) == messages_num * threads_num
-    ), "ClickHouse lost some messages or got duplicated ones. Total count: {}".format(
-        result
-    )
-
-
-def test_nats_virtual_column(nats_cluster):
-    instance.query(
-        """
-        CREATE TABLE test.nats_virtuals (key UInt64, value UInt64)
-            ENGINE = NATS
-            SETTINGS nats_url = 'nats1:4444',
-                     nats_subjects = 'virtuals',
-                     nats_format = 'JSONEachRow';
-        CREATE MATERIALIZED VIEW test.view Engine=Log AS
-        SELECT value, key, _subject FROM test.nats_virtuals;
-    """
-    )
-    while not check_table_is_ready(instance, "test.nats_virtuals"):
-        logging.debug("Table test.nats_virtuals is not yet ready")
-        time.sleep(0.5)
-
-    message_num = 10
-    i = 0
-    messages = []
-    for _ in range(message_num):
-        messages.append(json.dumps({"key": i, "value": i}))
-        i += 1
-
-    asyncio.run(nats_produce_messages(nats_cluster, "virtuals", messages))
-
-    while True:
-        result = instance.query("SELECT count() FROM test.view")
-        time.sleep(1)
-        if int(result) == message_num:
-            break
-
-    result = instance.query(
-        """
-        SELECT key, value, _subject
-        FROM test.view ORDER BY key
-    """
-    )
-
-    expected = """\
-0	0	virtuals
-1	1	virtuals
-2	2	virtuals
-3	3	virtuals
-4	4	virtuals
-5	5	virtuals
-6	6	virtuals
-7	7	virtuals
-8	8	virtuals
-9	9	virtuals
-"""
-
-    instance.query(
-        """
-        DROP TABLE test.nats_virtuals;
-        DROP TABLE test.view;
-    """
-    )
-
-    assert TSV(result) == TSV(expected)
-
-
-def test_nats_virtual_column_with_materialized_view(nats_cluster):
-    instance.query(
-        """
-        CREATE TABLE test.nats_virtuals_mv (key UInt64, value UInt64)
-            ENGINE = NATS
-            SETTINGS nats_url = 'nats1:4444',
-                     nats_subjects = 'virtuals_mv',
-                     nats_format = 'JSONEachRow';
-        CREATE TABLE test.view (key UInt64, value UInt64, subject String) ENGINE = MergeTree()
-            ORDER BY key;
-        CREATE MATERIALIZED VIEW test.consumer TO test.view AS
-        SELECT *, _subject as subject
-        FROM test.nats_virtuals_mv;
-    """
-    )
-    while not check_table_is_ready(instance, "test.nats_virtuals_mv"):
-        logging.debug("Table test.nats_virtuals_mv is not yet ready")
-        time.sleep(0.5)
-
-    message_num = 10
-    i = 0
-    messages = []
-    for _ in range(message_num):
-        messages.append(json.dumps({"key": i, "value": i}))
-        i += 1
-
-    asyncio.run(nats_produce_messages(nats_cluster, "virtuals_mv", messages))
-
-    while True:
-        result = instance.query("SELECT count() FROM test.view")
-        time.sleep(1)
-        if int(result) == message_num:
-            break
-
-    result = instance.query("SELECT key, value, subject FROM test.view ORDER BY key")
-    expected = """\
-0	0	virtuals_mv
-1	1	virtuals_mv
-2	2	virtuals_mv
-3	3	virtuals_mv
-4	4	virtuals_mv
-5	5	virtuals_mv
-6	6	virtuals_mv
-7	7	virtuals_mv
-8	8	virtuals_mv
-9	9	virtuals_mv
-"""
-
-    instance.query(
-        """
-        DROP TABLE test.consumer;
-        DROP TABLE test.view;
-        DROP TABLE test.nats_virtuals_mv
-    """
-    )
-
-    assert TSV(result) == TSV(expected)
-
-
-def test_nats_many_consumers_to_each_queue(nats_cluster):
-    instance.query(
-        """
-        DROP TABLE IF EXISTS test.destination;
-        CREATE TABLE test.destination(key UInt64, value UInt64)
-        ENGINE = MergeTree()
-        ORDER BY key;
-    """
-    )
-
-    num_tables = 4
-    for table_id in range(num_tables):
-        print(("Setting up table {}".format(table_id)))
-        instance.query(
-            """
-            DROP TABLE IF EXISTS test.many_consumers_{0};
-            DROP TABLE IF EXISTS test.many_consumers_{0}_mv;
-            CREATE TABLE test.many_consumers_{0} (key UInt64, value UInt64)
-                ENGINE = NATS
-                SETTINGS nats_url = 'nats1:4444',
-                         nats_subjects = 'many_consumers',
-                         nats_num_consumers = 2,
-                         nats_queue_group = 'many_consumers',
-                         nats_format = 'JSONEachRow',
-                         nats_row_delimiter = '\\n';
-            CREATE MATERIALIZED VIEW test.many_consumers_{0}_mv TO test.destination AS
-            SELECT key, value FROM test.many_consumers_{0};
-        """.format(
-                table_id
-            )
-        )
-        while not check_table_is_ready(
-            instance, "test.many_consumers_{}".format(table_id)
-        ):
-            logging.debug(
-                "Table test.many_consumers_{} is not yet ready".format(table_id)
-            )
-            time.sleep(0.5)
-
-    i = [0]
-    messages_num = 1000
-
-    def produce():
-        messages = []
-        for _ in range(messages_num):
-            messages.append(json.dumps({"key": i[0], "value": i[0]}))
-            i[0] += 1
-        asyncio.run(nats_produce_messages(nats_cluster, "many_consumers", messages))
-
-    threads = []
-    threads_num = 20
-
-    for _ in range(threads_num):
-        threads.append(threading.Thread(target=produce))
-    for thread in threads:
-        time.sleep(random.uniform(0, 1))
-        thread.start()
-
-    result1 = ""
-    while True:
-        result1 = instance.query("SELECT count() FROM test.destination")
-        time.sleep(1)
-        if int(result1) == messages_num * threads_num:
-            break
-
-    for thread in threads:
-        thread.join()
-
-    for consumer_id in range(num_tables):
-        instance.query(
-            """
-            DROP TABLE test.many_consumers_{0};
-            DROP TABLE test.many_consumers_{0}_mv;
-        """.format(
-                consumer_id
-            )
-        )
-
-    instance.query(
-        """
-        DROP TABLE test.destination;
-    """
-    )
-
-    assert (
-        int(result1) == messages_num * threads_num
-    ), "ClickHouse lost some messages: {}".format(result1)
-
-
-def test_nats_restore_failed_connection_without_losses_on_write(nats_cluster):
-    instance.query(
-        """
-        DROP TABLE IF EXISTS test.consume;
-        CREATE TABLE test.view (key UInt64, value UInt64)
-            ENGINE = MergeTree
-            ORDER BY key;
-        CREATE TABLE test.consume (key UInt64, value UInt64)
-            ENGINE = NATS
-            SETTINGS nats_url = 'nats1:4444',
-                     nats_subjects = 'producer_reconnect',
-                     nats_format = 'JSONEachRow',
-                     nats_num_consumers = 2,
-                     nats_row_delimiter = '\\n';
-        CREATE MATERIALIZED VIEW test.consumer TO test.view AS
-            SELECT * FROM test.consume;
-        DROP TABLE IF EXISTS test.producer_reconnect;
-        CREATE TABLE test.producer_reconnect (key UInt64, value UInt64)
-            ENGINE = NATS
-            SETTINGS nats_url = 'nats1:4444',
-                     nats_subjects = 'producer_reconnect',
-                     nats_format = 'JSONEachRow',
-                     nats_row_delimiter = '\\n';
-    """
-    )
-    while not check_table_is_ready(instance, "test.consume"):
-        logging.debug("Table test.consume is not yet ready")
-        time.sleep(0.5)
-
-    messages_num = 100000
-    values = []
-    for i in range(messages_num):
-        values.append("({i}, {i})".format(i=i))
-    values = ",".join(values)
-
-    while True:
-        try:
-            instance.query(
-                "INSERT INTO test.producer_reconnect VALUES {}".format(values)
-            )
-            break
-        except QueryRuntimeException as e:
-            if "Local: Timed out." in str(e):
-                continue
-            else:
-                raise
-
-    while int(instance.query("SELECT count() FROM test.view")) == 0:
-        time.sleep(0.1)
-
-    kill_nats(nats_cluster.nats_docker_id)
-    time.sleep(4)
-    revive_nats(nats_cluster.nats_docker_id, nats_cluster.nats_port)
-
-    while True:
-        result = instance.query("SELECT count(DISTINCT key) FROM test.view")
-        time.sleep(1)
-        if int(result) == messages_num:
-            break
-
-    instance.query(
-        """
-        DROP TABLE test.consume;
-        DROP TABLE test.producer_reconnect;
-    """
-    )
-
-    assert int(result) == messages_num, "ClickHouse lost some messages: {}".format(
-        result
-    )
-
-
-def test_nats_no_connection_at_startup_1(nats_cluster):
-    # no connection when table is initialized
-    nats_cluster.pause_container("nats1")
-    instance.query_and_get_error(
-        """
-        CREATE TABLE test.cs (key UInt64, value UInt64)
-            ENGINE = NATS
-            SETTINGS nats_url = 'nats1:4444',
-                     nats_subjects = 'cs',
-                     nats_format = 'JSONEachRow',
-                     nats_num_consumers = '5',
-                     nats_row_delimiter = '\\n';
-    """
-    )
-    nats_cluster.unpause_container("nats1")
-
-
-def test_nats_no_connection_at_startup_2(nats_cluster):
-    instance.query(
-        """
-        CREATE TABLE test.cs (key UInt64, value UInt64)
-            ENGINE = NATS
-            SETTINGS nats_url = 'nats1:4444',
-                     nats_subjects = 'cs',
-                     nats_format = 'JSONEachRow',
-                     nats_num_consumers = '5',
-                     nats_row_delimiter = '\\n';
-        CREATE TABLE test.view (key UInt64, value UInt64)
-            ENGINE = MergeTree
-            ORDER BY key;
-        CREATE MATERIALIZED VIEW test.consumer TO test.view AS
-            SELECT * FROM test.cs;
-    """
-    )
-
-    instance.query("DETACH TABLE test.cs")
-    nats_cluster.pause_container("nats1")
-    instance.query("ATTACH TABLE test.cs")
-    nats_cluster.unpause_container("nats1")
-    while not check_table_is_ready(instance, "test.cs"):
-        logging.debug("Table test.cs is not yet ready")
-        time.sleep(0.5)
-
-    messages_num = 1000
-    messages = []
-    for i in range(messages_num):
-        messages.append(json.dumps({"key": i, "value": i}))
-    asyncio.run(nats_produce_messages(nats_cluster, "cs", messages))
-
-    for _ in range(20):
-        result = instance.query("SELECT count() FROM test.view")
-        time.sleep(1)
-        if int(result) == messages_num:
-            break
-
-    instance.query(
-        """
-        DROP TABLE test.consumer;
-        DROP TABLE test.cs;
-    """
-    )
-
-    assert int(result) == messages_num, "ClickHouse lost some messages: {}".format(
-        result
-    )
-
-
-def test_nats_format_factory_settings(nats_cluster):
-    instance.query(
-        """
-        CREATE TABLE test.format_settings (
-            id String, date DateTime
-        ) ENGINE = NATS
-            SETTINGS nats_url = 'nats1:4444',
-                     nats_subjects = 'format_settings',
-                     nats_format = 'JSONEachRow',
-                     date_time_input_format = 'best_effort';
-        """
-    )
-    while not check_table_is_ready(instance, "test.format_settings"):
-        logging.debug("Table test.format_settings is not yet ready")
-        time.sleep(0.5)
-
-    message = json.dumps(
-        {"id": "format_settings_test", "date": "2021-01-19T14:42:33.1829214Z"}
-    )
-    expected = instance.query(
-        """SELECT parseDateTimeBestEffort(CAST('2021-01-19T14:42:33.1829214Z', 'String'))"""
-    )
-
-    asyncio.run(nats_produce_messages(nats_cluster, "format_settings", [message]))
-
-    while True:
-        result = instance.query("SELECT date FROM test.format_settings")
-        if result == expected:
-            break
-
-    instance.query(
-        """
-        CREATE TABLE test.view (
-            id String, date DateTime
-        ) ENGINE = MergeTree ORDER BY id;
-        CREATE MATERIALIZED VIEW test.consumer TO test.view AS
-            SELECT * FROM test.format_settings;
-        """
-    )
-
-    asyncio.run(nats_produce_messages(nats_cluster, "format_settings", [message]))
-    while True:
-        result = instance.query("SELECT date FROM test.view")
-        if result == expected:
-            break
-
-    instance.query(
-        """
-        DROP TABLE test.consumer;
-        DROP TABLE test.format_settings;
-    """
-    )
-
-    assert result == expected
-
-
-def test_nats_bad_args(nats_cluster):
-    instance.query_and_get_error(
-        """
-        CREATE TABLE test.drop (key UInt64, value UInt64)
-            ENGINE = NATS
-            SETTINGS nats_url = 'nats1:4444',
-                     nats_secure = true,
-                     nats_format = 'JSONEachRow';
-    """
-    )
-
-
-def test_nats_drop_mv(nats_cluster):
-    instance.query(
-        """
-        CREATE TABLE test.nats (key UInt64, value UInt64)
-            ENGINE = NATS
-            SETTINGS nats_url = 'nats1:4444',
-                     nats_subjects = 'mv',
-                     nats_format = 'JSONEachRow';
-        CREATE TABLE test.view (key UInt64, value UInt64)
-            ENGINE = MergeTree()
-            ORDER BY key;
-        CREATE MATERIALIZED VIEW test.consumer TO test.view AS
-            SELECT * FROM test.nats;
-    """
-    )
-    while not check_table_is_ready(instance, "test.nats"):
-        logging.debug("Table test.nats is not yet ready")
-        time.sleep(0.5)
-
-    messages = []
-    for i in range(20):
-        messages.append(json.dumps({"key": i, "value": i}))
-    asyncio.run(nats_produce_messages(nats_cluster, "mv", messages))
-
-    instance.query("DROP VIEW test.consumer")
-    messages = []
-    for i in range(20, 40):
-        messages.append(json.dumps({"key": i, "value": i}))
-    asyncio.run(nats_produce_messages(nats_cluster, "mv", messages))
-
-    instance.query(
-        """
-        CREATE MATERIALIZED VIEW test.consumer TO test.view AS
-            SELECT * FROM test.nats;
-    """
-    )
-    messages = []
-    for i in range(40, 50):
-        messages.append(json.dumps({"key": i, "value": i}))
-    asyncio.run(nats_produce_messages(nats_cluster, "mv", messages))
-
-    while True:
-        result = instance.query("SELECT * FROM test.view ORDER BY key")
-        if nats_check_result(result):
-            break
-
-    nats_check_result(result, True)
-
-    instance.query("DROP VIEW test.consumer")
-    messages = []
-    for i in range(50, 60):
-        messages.append(json.dumps({"key": i, "value": i}))
-    asyncio.run(nats_produce_messages(nats_cluster, "mv", messages))
-
-    count = 0
-    while True:
-        count = int(instance.query("SELECT count() FROM test.nats"))
-        if count:
-            break
-
-    assert count > 0
-
-
-def test_nats_predefined_configuration(nats_cluster):
-    instance.query(
-        """
-        CREATE TABLE test.nats (key UInt64, value UInt64)
-            ENGINE = NATS(nats1) """
-    )
-    while not check_table_is_ready(instance, "test.nats"):
-        logging.debug("Table test.nats is not yet ready")
-        time.sleep(0.5)
-
-    asyncio.run(
-        nats_produce_messages(
-            nats_cluster, "named", [json.dumps({"key": 1, "value": 2})]
-        )
-    )
-    while True:
-        result = instance.query(
-            "SELECT * FROM test.nats ORDER BY key", ignore_error=True
-        )
-        if result == "1\t2\n":
-            break
-
-
-def test_format_with_prefix_and_suffix(nats_cluster):
-    instance.query(
-        """
-        DROP TABLE IF EXISTS test.nats;
-        
-        CREATE TABLE test.nats (key UInt64, value UInt64)
-            ENGINE = NATS
-            SETTINGS nats_url = 'nats1:4444',
-                     nats_subjects = 'custom',
-                     nats_format = 'CustomSeparated';
-    """
-    )
-    while not check_table_is_ready(instance, "test.nats"):
-        logging.debug("Table test.nats is not yet ready")
-        time.sleep(0.5)
-
-    insert_messages = []
-
-    async def sub_to_nats():
-        nc = await nats_connect_ssl(
-            nats_cluster.nats_port,
-            user="click",
-            password="house",
-            ssl_ctx=nats_cluster.nats_ssl_context,
-        )
-        sub = await nc.subscribe("custom")
-        await sub.unsubscribe(2)
-        async for msg in sub.messages:
-            insert_messages.append(msg.data.decode())
-
-        await sub.drain()
-        await nc.drain()
-
-    def run_sub():
-        asyncio.run(sub_to_nats())
-
-    thread = threading.Thread(target=run_sub)
-    thread.start()
-    time.sleep(1)
-
-    instance.query(
-        "INSERT INTO test.nats select number*10 as key, number*100 as value from numbers(2) settings format_custom_result_before_delimiter='<prefix>\n', format_custom_result_after_delimiter='<suffix>\n'"
-    )
-
-    thread.join()
-
-    assert (
-        "".join(insert_messages)
-        == "<prefix>\n0\t0\n<suffix>\n<prefix>\n10\t100\n<suffix>\n"
-    )
-
-
-def test_max_rows_per_message(nats_cluster):
-    instance.query(
-        """
-        DROP TABLE IF EXISTS test.view;
-        DROP TABLE IF EXISTS test.nats;
-           
-        CREATE TABLE test.nats (key UInt64, value UInt64)
-            ENGINE = NATS
-            SETTINGS nats_url = 'nats1:4444',
-                     nats_subjects = 'custom1',
-                     nats_format = 'CustomSeparated',
-                     nats_max_rows_per_message = 3,
-                     format_custom_result_before_delimiter = '<prefix>\n',
-                     format_custom_result_after_delimiter = '<suffix>\n';
-        
-        CREATE MATERIALIZED VIEW test.view Engine=Log AS
-        SELECT key, value FROM test.nats;
-    """
-    )
-    while not check_table_is_ready(instance, "test.nats"):
-        logging.debug("Table test.nats is not yet ready")
-        time.sleep(0.5)
-
-    num_rows = 5
-
-    insert_messages = []
-
-    async def sub_to_nats():
-        nc = await nats_connect_ssl(
-            nats_cluster.nats_port,
-            user="click",
-            password="house",
-            ssl_ctx=nats_cluster.nats_ssl_context,
-        )
-        sub = await nc.subscribe("custom1")
-        await sub.unsubscribe(2)
-        async for msg in sub.messages:
-            insert_messages.append(msg.data.decode())
-
-        await sub.drain()
-        await nc.drain()
-
-    def run_sub():
-        asyncio.run(sub_to_nats())
-
-    thread = threading.Thread(target=run_sub)
-    thread.start()
-    time.sleep(1)
-
-    instance.query(
-        f"INSERT INTO test.nats select number*10 as key, number*100 as value from numbers({num_rows}) settings format_custom_result_before_delimiter='<prefix>\n', format_custom_result_after_delimiter='<suffix>\n'"
-    )
-
-    thread.join()
-
-    assert (
-        "".join(insert_messages)
-        == "<prefix>\n0\t0\n10\t100\n20\t200\n<suffix>\n<prefix>\n30\t300\n40\t400\n<suffix>\n"
-    )
-
-    attempt = 0
-    rows = 0
-    while attempt < 100:
-        rows = int(instance.query("SELECT count() FROM test.view"))
-        if rows == num_rows:
-            break
-        attempt += 1
-
-    assert rows == num_rows
-
-    result = instance.query("SELECT * FROM test.view")
-    assert result == "0\t0\n10\t100\n20\t200\n30\t300\n40\t400\n"
-
-
-def test_row_based_formats(nats_cluster):
-    num_rows = 10
-
-    for format_name in [
-        "TSV",
-        "TSVWithNamesAndTypes",
-        "TSKV",
-        "CSV",
-        "CSVWithNamesAndTypes",
-        "CustomSeparatedWithNamesAndTypes",
-        "Values",
-        "JSON",
-        "JSONEachRow",
-        "JSONCompactEachRow",
-        "JSONCompactEachRowWithNamesAndTypes",
-        "JSONObjectEachRow",
-        "Avro",
-        "RowBinary",
-        "RowBinaryWithNamesAndTypes",
-        "MsgPack",
-    ]:
-        print(format_name)
-
-        instance.query(
-            f"""
-            DROP TABLE IF EXISTS test.view;
-            DROP TABLE IF EXISTS test.nats;
-               
-            CREATE TABLE test.nats (key UInt64, value UInt64)
-                ENGINE = NATS
-                SETTINGS nats_url = 'nats1:4444',
-                         nats_subjects = '{format_name}',
-                         nats_format = '{format_name}';      
-    
-            CREATE MATERIALIZED VIEW test.view Engine=Log AS
-            SELECT key, value FROM test.nats;
-        """
-        )
-
-        while not check_table_is_ready(instance, "test.nats"):
-            logging.debug("Table test.nats is not yet ready")
-            time.sleep(0.5)
-
-        insert_messages = 0
-
-        async def sub_to_nats():
-            nc = await nats_connect_ssl(
-                nats_cluster.nats_port,
-                user="click",
-                password="house",
-                ssl_ctx=nats_cluster.nats_ssl_context,
-            )
-            sub = await nc.subscribe(format_name)
-            await sub.unsubscribe(2)
-            async for msg in sub.messages:
-                nonlocal insert_messages
-                insert_messages += 1
-
-            await sub.drain()
-            await nc.drain()
-
-        def run_sub():
-            asyncio.run(sub_to_nats())
-
-        thread = threading.Thread(target=run_sub)
-        thread.start()
-        time.sleep(1)
-
-        instance.query(
-            f"INSERT INTO test.nats select number*10 as key, number*100 as value from numbers({num_rows})"
-        )
-
-        thread.join()
-
-        assert insert_messages == 2
-
-        attempt = 0
-        rows = 0
-        while attempt < 100:
-            rows = int(instance.query("SELECT count() FROM test.view"))
-            if rows == num_rows:
-                break
-            attempt += 1
-
-        assert rows == num_rows
-
-        expected = ""
-        for i in range(num_rows):
-            expected += str(i * 10) + "\t" + str(i * 100) + "\n"
-
-        result = instance.query("SELECT * FROM test.view")
-        assert result == expected
-
-
-def test_block_based_formats_1(nats_cluster):
-    instance.query(
-        """
-        DROP TABLE IF EXISTS test.nats;
-        
-        CREATE TABLE test.nats (key UInt64, value UInt64)
-            ENGINE = NATS
-            SETTINGS nats_url = 'nats1:4444',
-                     nats_subjects = 'PrettySpace',
-                     nats_format = 'PrettySpace';
-    """
-    )
-
-    insert_messages = []
-
-    async def sub_to_nats():
-        nc = await nats_connect_ssl(
-            nats_cluster.nats_port,
-            user="click",
-            password="house",
-            ssl_ctx=nats_cluster.nats_ssl_context,
-        )
-        sub = await nc.subscribe("PrettySpace")
-        await sub.unsubscribe(3)
-        async for msg in sub.messages:
-            insert_messages.append(msg.data.decode())
-
-        await sub.drain()
-        await nc.drain()
-
-    def run_sub():
-        asyncio.run(sub_to_nats())
-
-    thread = threading.Thread(target=run_sub)
-    thread.start()
-    time.sleep(1)
-
-    attempt = 0
-    while attempt < 100:
-        try:
-            instance.query(
-                "INSERT INTO test.nats SELECT number * 10 as key, number * 100 as value FROM numbers(5) settings max_block_size=2, optimize_trivial_insert_select=0;"
-            )
-            break
-        except Exception:
-            logging.debug("Table test.nats is not yet ready")
-            time.sleep(0.5)
-            attempt += 1
-    thread.join()
-
-    data = []
-    for message in insert_messages:
-        splitted = message.split("\n")
-        assert splitted[0] == " \x1b[1mkey\x1b[0m   \x1b[1mvalue\x1b[0m"
-        assert splitted[1] == ""
-        assert splitted[-1] == ""
-        data += [line.split() for line in splitted[2:-1]]
-
-    assert data == [
-        ["0", "0"],
-        ["10", "100"],
-        ["20", "200"],
-        ["30", "300"],
-        ["40", "400"],
-    ]
-
-
-def test_block_based_formats_2(nats_cluster):
-    num_rows = 100
-
-    for format_name in [
-        "JSONColumns",
-        "Native",
-        "Arrow",
-        "Parquet",
-        "ORC",
-        "JSONCompactColumns",
-    ]:
-        print(format_name)
-
-        instance.query(
-            f"""
-            DROP TABLE IF EXISTS test.view;
-            DROP TABLE IF EXISTS test.nats;
-               
-            CREATE TABLE test.nats (key UInt64, value UInt64)
-                ENGINE = NATS
-                SETTINGS nats_url = 'nats1:4444',
-                         nats_subjects = '{format_name}',
-                         nats_format = '{format_name}';      
-    
-            CREATE MATERIALIZED VIEW test.view Engine=Log AS
-            SELECT key, value FROM test.nats;
-        """
-        )
-
-        while not check_table_is_ready(instance, "test.nats"):
-            logging.debug("Table test.nats is not yet ready")
-            time.sleep(0.5)
-
-        insert_messages = 0
-
-        async def sub_to_nats():
-            nc = await nats_connect_ssl(
-                nats_cluster.nats_port,
-                user="click",
-                password="house",
-                ssl_ctx=nats_cluster.nats_ssl_context,
-            )
-            sub = await nc.subscribe(format_name)
-            await sub.unsubscribe(9)
-            async for msg in sub.messages:
-                nonlocal insert_messages
-                insert_messages += 1
-
-            await sub.drain()
-            await nc.drain()
-
-        def run_sub():
-            asyncio.run(sub_to_nats())
-
-        thread = threading.Thread(target=run_sub)
-        thread.start()
-        time.sleep(1)
-
-        instance.query(
-            f"INSERT INTO test.nats SELECT number * 10 as key, number * 100 as value FROM numbers({num_rows}) settings max_block_size=12, optimize_trivial_insert_select=0;"
-        )
-
-        thread.join()
-
-        assert insert_messages == 9
-
-        attempt = 0
-        rows = 0
-        while attempt < 100:
-            rows = int(instance.query("SELECT count() FROM test.view"))
-            if rows == num_rows:
-                break
-            attempt += 1
-
-        assert rows == num_rows
-
-        result = instance.query("SELECT * FROM test.view ORDER by key")
-        expected = ""
-        for i in range(num_rows):
-            expected += str(i * 10) + "\t" + str(i * 100) + "\n"
-        assert result == expected
-
-
-if __name__ == "__main__":
-    cluster.start()
-    input("Cluster created, press any key to destroy...")
-    cluster.shutdown()
diff --git a/tests/integration/test_storage_nats/test_nats_json.reference b/tests/integration/test_storage_nats/test_nats_json.reference
deleted file mode 100644
index 959bb2aad74..00000000000
--- a/tests/integration/test_storage_nats/test_nats_json.reference
+++ /dev/null
@@ -1,50 +0,0 @@
-0	0
-1	1
-2	2
-3	3
-4	4
-5	5
-6	6
-7	7
-8	8
-9	9
-10	10
-11	11
-12	12
-13	13
-14	14
-15	15
-16	16
-17	17
-18	18
-19	19
-20	20
-21	21
-22	22
-23	23
-24	24
-25	25
-26	26
-27	27
-28	28
-29	29
-30	30
-31	31
-32	32
-33	33
-34	34
-35	35
-36	36
-37	37
-38	38
-39	39
-40	40
-41	41
-42	42
-43	43
-44	44
-45	45
-46	46
-47	47
-48	48
-49	49
diff --git a/tests/integration/test_storage_postgresql_replica/__init__.py b/tests/integration/test_storage_postgresql_replica/__init__.py
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/integration/test_storage_postgresql_replica/configs/log_conf.xml b/tests/integration/test_storage_postgresql_replica/configs/log_conf.xml
deleted file mode 100644
index 27c7107ce5e..00000000000
--- a/tests/integration/test_storage_postgresql_replica/configs/log_conf.xml
+++ /dev/null
@@ -1,11 +0,0 @@
-<clickhouse>
-    <logger>
-        <level>trace</level>
-        <log>/var/log/clickhouse-server/log.log</log>
-        <errorlog>/var/log/clickhouse-server/log.err.log</errorlog>
-        <size>1000M</size>
-        <count>10</count>
-        <stderr>/var/log/clickhouse-server/stderr.log</stderr>
-        <stdout>/var/log/clickhouse-server/stdout.log</stdout>
-    </logger>
-</clickhouse>
diff --git a/tests/integration/test_storage_postgresql_replica/test.py b/tests/integration/test_storage_postgresql_replica/test.py
deleted file mode 100644
index 66495700102..00000000000
--- a/tests/integration/test_storage_postgresql_replica/test.py
+++ /dev/null
@@ -1,780 +0,0 @@
-import pytest
-
-# FIXME Tests with MaterializedPostgresSQL are temporarily disabled
-# https://github.com/ClickHouse/ClickHouse/issues/36898
-# https://github.com/ClickHouse/ClickHouse/issues/38677
-# https://github.com/ClickHouse/ClickHouse/pull/39272#issuecomment-1190087190
-
-pytestmark = pytest.mark.skip
-
-import time
-import psycopg2
-import os.path as p
-
-from helpers.cluster import ClickHouseCluster
-from helpers.test_tools import assert_eq_with_retry
-from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
-from helpers.test_tools import TSV
-
-import threading
-
-cluster = ClickHouseCluster(__file__)
-instance = cluster.add_instance(
-    "instance",
-    main_configs=["configs/log_conf.xml"],
-    with_postgres=True,
-    stay_alive=True,
-)
-
-postgres_table_template = """
-    CREATE TABLE IF NOT EXISTS {} (
-    key Integer NOT NULL, value Integer, PRIMARY KEY(key))
-    """
-
-queries = [
-    "INSERT INTO {} select i, i from generate_series(0, 10000) as t(i);",
-    "DELETE FROM {} WHERE (value*value) % 3 = 0;",
-    "UPDATE {} SET value = value + 125 WHERE key % 2 = 0;",
-    "UPDATE {} SET key=key+20000 WHERE key%2=0",
-    "INSERT INTO {} select i, i from generate_series(40000, 50000) as t(i);",
-    "DELETE FROM {} WHERE key % 10 = 0;",
-    "UPDATE {} SET value = value + 101 WHERE key % 2 = 1;",
-    "UPDATE {} SET key=key+80000 WHERE key%2=1",
-    "DELETE FROM {} WHERE value % 2 = 0;",
-    "UPDATE {} SET value = value + 2000 WHERE key % 5 = 0;",
-    "INSERT INTO {} select i, i from generate_series(200000, 250000) as t(i);",
-    "DELETE FROM {} WHERE value % 3 = 0;",
-    "UPDATE {} SET value = value * 2 WHERE key % 3 = 0;",
-    "UPDATE {} SET key=key+500000 WHERE key%2=1",
-    "INSERT INTO {} select i, i from generate_series(1000000, 1050000) as t(i);",
-    "DELETE FROM {} WHERE value % 9 = 2;",
-    "UPDATE {} SET key=key+10000000",
-    "UPDATE {} SET value = value + 2  WHERE key % 3 = 1;",
-    "DELETE FROM {} WHERE value%5 = 0;",
-]
-
-
-@pytest.mark.timeout(30)
-def check_tables_are_synchronized(
-    table_name, order_by="key", postgres_database="postgres_database"
-):
-    while True:
-        expected = instance.query(
-            "select * from {}.{} order by {};".format(
-                postgres_database, table_name, order_by
-            )
-        )
-        result = instance.query(
-            "select * from test.{} order by {};".format(table_name, order_by)
-        )
-        if result == expected:
-            break
-
-    assert result == expected
-
-
-def get_postgres_conn(
-    ip, port, database=False, auto_commit=True, database_name="postgres_database"
-):
-    if database == True:
-        conn_string = "host={} port={} dbname='{}' user='postgres' password='mysecretpassword'".format(
-            ip, port, database_name
-        )
-    else:
-        conn_string = (
-            "host={} port={} user='postgres' password='mysecretpassword'".format(
-                ip, port
-            )
-        )
-
-    conn = psycopg2.connect(conn_string)
-    if auto_commit:
-        conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
-        conn.autocommit = True
-    return conn
-
-
-def create_postgres_db(cursor, name):
-    cursor.execute("CREATE DATABASE {}".format(name))
-
-
-def create_clickhouse_postgres_db(ip, port, name="postgres_database"):
-    instance.query(
-        """
-            CREATE DATABASE {}
-            ENGINE = PostgreSQL('{}:{}', '{}', 'postgres', 'mysecretpassword')""".format(
-            name, ip, port, name
-        )
-    )
-
-
-def create_materialized_table(ip, port, table_name="postgresql_replica"):
-    instance.query(
-        f"""
-        CREATE TABLE test.{table_name} (key Int64, value Int64)
-            ENGINE = MaterializedPostgreSQL(
-            '{ip}:{port}', 'postgres_database', '{table_name}', 'postgres', 'mysecretpassword')
-            PRIMARY KEY key; """
-    )
-
-
-def create_postgres_table(cursor, table_name, replica_identity_full=False):
-    cursor.execute("DROP TABLE IF EXISTS {}".format(table_name))
-    cursor.execute(postgres_table_template.format(table_name))
-    if replica_identity_full:
-        cursor.execute("ALTER TABLE {} REPLICA IDENTITY FULL;".format(table_name))
-
-
-def postgresql_replica_check_result(
-    result, check=False, ref_file="test_postgresql_replica.reference"
-):
-    fpath = p.join(p.dirname(__file__), ref_file)
-    with open(fpath) as reference:
-        if check:
-            assert TSV(result) == TSV(reference)
-        else:
-            return TSV(result) == TSV(reference)
-
-
-@pytest.fixture(scope="module")
-def started_cluster():
-    try:
-        cluster.start()
-        conn = get_postgres_conn(ip=cluster.postgres_ip, port=cluster.postgres_port)
-        cursor = conn.cursor()
-        create_postgres_db(cursor, "postgres_database")
-        create_clickhouse_postgres_db(
-            ip=cluster.postgres_ip, port=cluster.postgres_port
-        )
-
-        instance.query("CREATE DATABASE test")
-        yield cluster
-
-    finally:
-        cluster.shutdown()
-
-
-@pytest.mark.timeout(320)
-def test_initial_load_from_snapshot(started_cluster):
-    conn = get_postgres_conn(
-        ip=started_cluster.postgres_ip,
-        port=started_cluster.postgres_port,
-        database=True,
-    )
-    cursor = conn.cursor()
-    create_postgres_table(cursor, "postgresql_replica")
-    instance.query(
-        "INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(50)"
-    )
-
-    instance.query("DROP TABLE IF EXISTS test.postgresql_replica")
-    create_materialized_table(
-        ip=started_cluster.postgres_ip, port=started_cluster.postgres_port
-    )
-
-    result = instance.query("SELECT * FROM test.postgresql_replica ORDER BY key;")
-    while postgresql_replica_check_result(result) == False:
-        time.sleep(0.2)
-        result = instance.query("SELECT * FROM test.postgresql_replica ORDER BY key;")
-
-    cursor.execute("DROP TABLE postgresql_replica;")
-    postgresql_replica_check_result(result, True)
-    instance.query(f"DROP TABLE test.postgresql_replica SYNC")
-
-
-@pytest.mark.timeout(320)
-def test_no_connection_at_startup(started_cluster):
-    conn = get_postgres_conn(
-        ip=started_cluster.postgres_ip,
-        port=started_cluster.postgres_port,
-        database=True,
-    )
-    cursor = conn.cursor()
-    create_postgres_table(cursor, "postgresql_replica")
-    instance.query(
-        "INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(50)"
-    )
-
-    instance.query("DROP TABLE IF EXISTS test.postgresql_replica")
-    create_materialized_table(
-        ip=started_cluster.postgres_ip, port=started_cluster.postgres_port
-    )
-    time.sleep(3)
-
-    instance.query("DETACH TABLE test.postgresql_replica")
-    started_cluster.pause_container("postgres1")
-
-    instance.query("ATTACH TABLE test.postgresql_replica")
-    time.sleep(3)
-    started_cluster.unpause_container("postgres1")
-
-    result = instance.query("SELECT count() FROM test.postgresql_replica;")
-    while int(result) == 0:
-        time.sleep(0.5)
-        result = instance.query("SELECT count() FROM test.postgresql_replica;")
-
-    result = instance.query("SELECT * FROM test.postgresql_replica ORDER BY key;")
-    cursor.execute("DROP TABLE postgresql_replica;")
-    postgresql_replica_check_result(result, True)
-    instance.query(f"DROP TABLE test.postgresql_replica SYNC")
-
-
-@pytest.mark.timeout(320)
-def test_detach_attach_is_ok(started_cluster):
-    conn = get_postgres_conn(
-        ip=started_cluster.postgres_ip,
-        port=started_cluster.postgres_port,
-        database=True,
-    )
-    cursor = conn.cursor()
-    create_postgres_table(cursor, "postgresql_replica")
-    instance.query(
-        "INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(50)"
-    )
-
-    instance.query("DROP TABLE IF EXISTS test.postgresql_replica")
-    create_materialized_table(
-        ip=started_cluster.postgres_ip, port=started_cluster.postgres_port
-    )
-
-    result = instance.query("SELECT count() FROM test.postgresql_replica;")
-    while int(result) == 0:
-        time.sleep(0.2)
-        result = instance.query("SELECT count() FROM test.postgresql_replica;")
-
-    result = instance.query("SELECT * FROM test.postgresql_replica ORDER BY key;")
-    postgresql_replica_check_result(result, True)
-
-    instance.query("DETACH TABLE test.postgresql_replica")
-    instance.query("ATTACH TABLE test.postgresql_replica")
-
-    result = instance.query("SELECT * FROM test.postgresql_replica ORDER BY key;")
-    while postgresql_replica_check_result(result) == False:
-        time.sleep(0.5)
-        result = instance.query("SELECT * FROM test.postgresql_replica ORDER BY key;")
-
-    cursor.execute("DROP TABLE postgresql_replica;")
-    postgresql_replica_check_result(result, True)
-    instance.query(f"DROP TABLE test.postgresql_replica SYNC")
-
-
-@pytest.mark.timeout(320)
-def test_replicating_insert_queries(started_cluster):
-    conn = get_postgres_conn(
-        ip=started_cluster.postgres_ip,
-        port=started_cluster.postgres_port,
-        database=True,
-    )
-    cursor = conn.cursor()
-    create_postgres_table(cursor, "postgresql_replica")
-
-    instance.query(
-        "INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(10)"
-    )
-
-    instance.query("DROP TABLE IF EXISTS test.postgresql_replica")
-    create_materialized_table(
-        ip=started_cluster.postgres_ip, port=started_cluster.postgres_port
-    )
-
-    result = instance.query("SELECT count() FROM test.postgresql_replica;")
-    while int(result) != 10:
-        time.sleep(0.2)
-        result = instance.query("SELECT count() FROM test.postgresql_replica;")
-
-    instance.query(
-        "INSERT INTO postgres_database.postgresql_replica SELECT 10 + number, 10 + number from numbers(10)"
-    )
-    instance.query(
-        "INSERT INTO postgres_database.postgresql_replica SELECT 20 + number, 20 + number from numbers(10)"
-    )
-
-    result = instance.query("SELECT count() FROM test.postgresql_replica;")
-    while int(result) != 30:
-        time.sleep(0.2)
-        result = instance.query("SELECT count() FROM test.postgresql_replica;")
-
-    instance.query(
-        "INSERT INTO postgres_database.postgresql_replica SELECT 30 + number, 30 + number from numbers(10)"
-    )
-    instance.query(
-        "INSERT INTO postgres_database.postgresql_replica SELECT 40 + number, 40 + number from numbers(10)"
-    )
-
-    result = instance.query("SELECT count() FROM test.postgresql_replica;")
-    while int(result) != 50:
-        time.sleep(0.2)
-        result = instance.query("SELECT count() FROM test.postgresql_replica;")
-
-    result = instance.query("SELECT * FROM test.postgresql_replica ORDER BY key;")
-    cursor.execute("DROP TABLE postgresql_replica;")
-    postgresql_replica_check_result(result, True)
-    instance.query(f"DROP TABLE test.postgresql_replica SYNC")
-
-
-@pytest.mark.timeout(320)
-def test_replicating_delete_queries(started_cluster):
-    conn = get_postgres_conn(
-        ip=started_cluster.postgres_ip,
-        port=started_cluster.postgres_port,
-        database=True,
-    )
-    cursor = conn.cursor()
-    create_postgres_table(cursor, "postgresql_replica")
-
-    instance.query(
-        "INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(50)"
-    )
-
-    instance.query("DROP TABLE IF EXISTS test.postgresql_replica")
-    create_materialized_table(
-        ip=started_cluster.postgres_ip, port=started_cluster.postgres_port
-    )
-
-    result = instance.query("SELECT * FROM test.postgresql_replica ORDER BY key;")
-    while postgresql_replica_check_result(result) == False:
-        time.sleep(0.2)
-        result = instance.query("SELECT * FROM test.postgresql_replica ORDER BY key;")
-
-    instance.query(
-        "INSERT INTO postgres_database.postgresql_replica SELECT 50 + number, 50 + number from numbers(50)"
-    )
-
-    result = instance.query("SELECT count() FROM test.postgresql_replica;")
-    while int(result) != 100:
-        time.sleep(0.5)
-        result = instance.query("SELECT count() FROM test.postgresql_replica;")
-
-    cursor.execute("DELETE FROM postgresql_replica WHERE key > 49;")
-
-    result = instance.query("SELECT * FROM test.postgresql_replica ORDER BY key;")
-    while postgresql_replica_check_result(result) == False:
-        time.sleep(0.5)
-        result = instance.query("SELECT * FROM test.postgresql_replica ORDER BY key;")
-
-    cursor.execute("DROP TABLE postgresql_replica;")
-    postgresql_replica_check_result(result, True)
-
-
-@pytest.mark.timeout(320)
-def test_replicating_update_queries(started_cluster):
-    conn = get_postgres_conn(
-        ip=started_cluster.postgres_ip,
-        port=started_cluster.postgres_port,
-        database=True,
-    )
-    cursor = conn.cursor()
-    create_postgres_table(cursor, "postgresql_replica")
-
-    instance.query(
-        "INSERT INTO postgres_database.postgresql_replica SELECT number, number + 10 from numbers(50)"
-    )
-
-    instance.query("DROP TABLE IF EXISTS test.postgresql_replica")
-    create_materialized_table(
-        ip=started_cluster.postgres_ip, port=started_cluster.postgres_port
-    )
-
-    result = instance.query("SELECT count() FROM test.postgresql_replica;")
-    while int(result) != 50:
-        time.sleep(0.2)
-        result = instance.query("SELECT count() FROM test.postgresql_replica;")
-
-    cursor.execute("UPDATE postgresql_replica SET value = value - 10;")
-
-    result = instance.query("SELECT * FROM test.postgresql_replica ORDER BY key;")
-    while postgresql_replica_check_result(result) == False:
-        time.sleep(0.5)
-        result = instance.query("SELECT * FROM test.postgresql_replica ORDER BY key;")
-
-    cursor.execute("DROP TABLE postgresql_replica;")
-    postgresql_replica_check_result(result, True)
-
-
-@pytest.mark.timeout(320)
-def test_resume_from_written_version(started_cluster):
-    conn = get_postgres_conn(
-        ip=started_cluster.postgres_ip,
-        port=started_cluster.postgres_port,
-        database=True,
-    )
-    cursor = conn.cursor()
-    create_postgres_table(cursor, "postgresql_replica")
-    instance.query(
-        "INSERT INTO postgres_database.postgresql_replica SELECT number, number + 10 from numbers(50)"
-    )
-
-    instance.query("DROP TABLE IF EXISTS test.postgresql_replica")
-    create_materialized_table(
-        ip=started_cluster.postgres_ip, port=started_cluster.postgres_port
-    )
-
-    result = instance.query("SELECT count() FROM test.postgresql_replica;")
-    while int(result) != 50:
-        time.sleep(0.2)
-        result = instance.query("SELECT count() FROM test.postgresql_replica;")
-
-    instance.query(
-        "INSERT INTO postgres_database.postgresql_replica SELECT 50 + number, 50 + number from numbers(50)"
-    )
-
-    result = instance.query("SELECT count() FROM test.postgresql_replica;")
-    while int(result) != 100:
-        time.sleep(0.2)
-        result = instance.query("SELECT count() FROM test.postgresql_replica;")
-
-    instance.query("DETACH TABLE test.postgresql_replica")
-
-    cursor.execute("DELETE FROM postgresql_replica WHERE key > 49;")
-    cursor.execute("UPDATE postgresql_replica SET value = value - 10;")
-
-    instance.query("ATTACH TABLE test.postgresql_replica")
-
-    result = instance.query("SELECT * FROM test.postgresql_replica ORDER BY key;")
-    while postgresql_replica_check_result(result) == False:
-        time.sleep(0.5)
-        result = instance.query("SELECT * FROM test.postgresql_replica ORDER BY key;")
-
-    cursor.execute("DROP TABLE postgresql_replica;")
-    postgresql_replica_check_result(result, True)
-
-
-@pytest.mark.timeout(320)
-def test_many_replication_messages(started_cluster):
-    conn = get_postgres_conn(
-        ip=started_cluster.postgres_ip,
-        port=started_cluster.postgres_port,
-        database=True,
-    )
-    cursor = conn.cursor()
-    create_postgres_table(cursor, "postgresql_replica")
-    instance.query(
-        "INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(100000)"
-    )
-
-    instance.query("DROP TABLE IF EXISTS test.postgresql_replica")
-    create_materialized_table(
-        ip=started_cluster.postgres_ip, port=started_cluster.postgres_port
-    )
-
-    result = instance.query("SELECT count() FROM test.postgresql_replica;")
-    while int(result) != 100000:
-        time.sleep(0.2)
-        result = instance.query("SELECT count() FROM test.postgresql_replica;")
-    print("SYNC OK")
-
-    instance.query(
-        "INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(100000, 100000)"
-    )
-
-    result = instance.query("SELECT count() FROM test.postgresql_replica;")
-    while int(result) != 200000:
-        time.sleep(1)
-        result = instance.query("SELECT count() FROM test.postgresql_replica;")
-    print("INSERT OK")
-
-    result = instance.query("SELECT key FROM test.postgresql_replica ORDER BY key;")
-    expected = instance.query("SELECT number from numbers(200000)")
-    assert result == expected
-
-    cursor.execute("UPDATE postgresql_replica SET value = key + 1 WHERE key < 100000;")
-
-    result = instance.query(
-        "SELECT key FROM test.postgresql_replica WHERE value = key + 1 ORDER BY key;"
-    )
-    expected = instance.query("SELECT number from numbers(100000)")
-
-    while result != expected:
-        time.sleep(1)
-        result = instance.query(
-            "SELECT key FROM test.postgresql_replica WHERE value = key + 1 ORDER BY key;"
-        )
-    print("UPDATE OK")
-
-    cursor.execute("DELETE FROM postgresql_replica WHERE key % 2 = 1;")
-    cursor.execute("DELETE FROM postgresql_replica WHERE key != value;")
-
-    result = instance.query(
-        "SELECT count() FROM (SELECT * FROM test.postgresql_replica);"
-    )
-    while int(result) != 50000:
-        time.sleep(1)
-        result = instance.query(
-            "SELECT count() FROM (SELECT * FROM test.postgresql_replica);"
-        )
-    print("DELETE OK")
-
-    cursor.execute("DROP TABLE postgresql_replica;")
-
-
-@pytest.mark.timeout(320)
-def test_connection_loss(started_cluster):
-    conn = get_postgres_conn(
-        ip=started_cluster.postgres_ip,
-        port=started_cluster.postgres_port,
-        database=True,
-    )
-    cursor = conn.cursor()
-    create_postgres_table(cursor, "postgresql_replica")
-    instance.query(
-        "INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(50)"
-    )
-
-    instance.query("DROP TABLE IF EXISTS test.postgresql_replica")
-    create_materialized_table(
-        ip=started_cluster.postgres_ip, port=started_cluster.postgres_port
-    )
-
-    i = 50
-    while i < 100000:
-        instance.query(
-            "INSERT INTO postgres_database.postgresql_replica SELECT {} + number, number from numbers(10000)".format(
-                i
-            )
-        )
-        i += 10000
-
-    started_cluster.pause_container("postgres1")
-
-    result = instance.query("SELECT count() FROM test.postgresql_replica;")
-    print(int(result))
-    time.sleep(6)
-
-    started_cluster.unpause_container("postgres1")
-
-    result = instance.query("SELECT count() FROM test.postgresql_replica;")
-    while int(result) < 100050:
-        time.sleep(1)
-        result = instance.query("SELECT count() FROM test.postgresql_replica;")
-
-    cursor.execute("DROP TABLE postgresql_replica;")
-    assert int(result) == 100050
-
-
-@pytest.mark.timeout(320)
-def test_clickhouse_restart(started_cluster):
-    pytest.skip("Temporary disabled (FIXME)")
-    conn = get_postgres_conn(
-        ip=started_cluster.postgres_ip,
-        port=started_cluster.postgres_port,
-        database=True,
-    )
-    cursor = conn.cursor()
-    create_postgres_table(cursor, "postgresql_replica")
-    instance.query(
-        "INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(50)"
-    )
-
-    instance.query("DROP TABLE IF EXISTS test.postgresql_replica")
-    create_materialized_table(
-        ip=started_cluster.postgres_ip, port=started_cluster.postgres_port
-    )
-
-    i = 50
-    while i < 100000:
-        instance.query(
-            "INSERT INTO postgres_database.postgresql_replica SELECT {} + number, number from numbers(10000)".format(
-                i
-            )
-        )
-        i += 10000
-
-    instance.restart_clickhouse()
-
-    result = instance.query("SELECT count() FROM test.postgresql_replica;")
-    while int(result) < 100050:
-        time.sleep(1)
-        result = instance.query("SELECT count() FROM test.postgresql_replica;")
-
-    cursor.execute("DROP TABLE postgresql_replica;")
-    print(result)
-    assert int(result) == 100050
-
-
-def test_rename_table(started_cluster):
-    conn = get_postgres_conn(
-        ip=started_cluster.postgres_ip,
-        port=started_cluster.postgres_port,
-        database=True,
-    )
-    cursor = conn.cursor()
-    create_postgres_table(cursor, "postgresql_replica")
-
-    instance.query("DROP TABLE IF EXISTS test.postgresql_replica")
-    create_materialized_table(
-        ip=started_cluster.postgres_ip, port=started_cluster.postgres_port
-    )
-
-    instance.query(
-        "INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(25)"
-    )
-
-    result = instance.query("SELECT count() FROM test.postgresql_replica;")
-    while int(result) != 25:
-        time.sleep(0.5)
-        result = instance.query("SELECT count() FROM test.postgresql_replica;")
-
-    instance.query(
-        "RENAME TABLE test.postgresql_replica TO test.postgresql_replica_renamed"
-    )
-    assert (
-        int(instance.query("SELECT count() FROM test.postgresql_replica_renamed;"))
-        == 25
-    )
-
-    instance.query(
-        "INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(25, 25)"
-    )
-
-    result = instance.query("SELECT count() FROM test.postgresql_replica_renamed;")
-    while int(result) != 50:
-        time.sleep(0.5)
-        result = instance.query("SELECT count() FROM test.postgresql_replica_renamed;")
-
-    result = instance.query(
-        "SELECT * FROM test.postgresql_replica_renamed ORDER BY key;"
-    )
-    postgresql_replica_check_result(result, True)
-    cursor.execute("DROP TABLE postgresql_replica;")
-    instance.query("DROP TABLE IF EXISTS test.postgresql_replica_renamed")
-
-
-def test_virtual_columns(started_cluster):
-    conn = get_postgres_conn(
-        ip=started_cluster.postgres_ip,
-        port=started_cluster.postgres_port,
-        database=True,
-    )
-    cursor = conn.cursor()
-    create_postgres_table(cursor, "postgresql_replica")
-
-    instance.query("DROP TABLE IF EXISTS test.postgresql_replica")
-    create_materialized_table(
-        ip=started_cluster.postgres_ip, port=started_cluster.postgres_port
-    )
-
-    instance.query(
-        "INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(10)"
-    )
-    result = instance.query("SELECT count() FROM test.postgresql_replica;")
-    while int(result) != 10:
-        time.sleep(0.5)
-        result = instance.query("SELECT count() FROM test.postgresql_replica;")
-
-    # just check that it works, no check with `expected` because _version is taken as LSN, which will be different each time.
-    result = instance.query(
-        "SELECT key, value, _sign, _version FROM test.postgresql_replica;"
-    )
-    print(result)
-    cursor.execute("DROP TABLE postgresql_replica;")
-    instance.query(f"DROP TABLE test.postgresql_replica SYNC")
-
-
-def test_abrupt_connection_loss_while_heavy_replication(started_cluster):
-    instance.query("DROP DATABASE IF EXISTS test_database")
-    conn = get_postgres_conn(
-        ip=started_cluster.postgres_ip,
-        port=started_cluster.postgres_port,
-        database=True,
-    )
-    cursor = conn.cursor()
-    table_name = "postgresql_replica"
-    create_postgres_table(cursor, table_name)
-
-    instance.query(f"DROP TABLE IF EXISTS test.{table_name}")
-    create_materialized_table(
-        ip=started_cluster.postgres_ip, port=started_cluster.postgres_port
-    )
-
-    for i in range(len(queries)):
-        query = queries[i].format(table_name)
-        cursor.execute(query)
-        print("query {}".format(query.format(table_name)))
-
-    started_cluster.pause_container("postgres1")
-
-    result = instance.query("SELECT count() FROM test.postgresql_replica")
-    print(result)  # Just debug
-
-    started_cluster.unpause_container("postgres1")
-
-    check_tables_are_synchronized("postgresql_replica")
-
-    result = instance.query("SELECT count() FROM test.postgresql_replica")
-    print(result)  # Just debug
-    instance.query(f"DROP TABLE test.postgresql_replica SYNC")
-
-
-def test_abrupt_server_restart_while_heavy_replication(started_cluster):
-    # FIXME (kssenii) temporary disabled
-    if instance.is_built_with_sanitizer():
-        pytest.skip("Temporary disabled (FIXME)")
-
-    conn = get_postgres_conn(
-        ip=started_cluster.postgres_ip,
-        port=started_cluster.postgres_port,
-        database=True,
-    )
-    cursor = conn.cursor()
-    table_name = "postgresql_replica_697"
-    create_postgres_table(cursor, table_name)
-
-    instance.query(f"INSERT INTO postgres_database.{table_name} SELECT -1, 1")
-    instance.query(f"DROP TABLE IF EXISTS test.{table_name} SYNC")
-    create_materialized_table(
-        ip=started_cluster.postgres_ip,
-        port=started_cluster.postgres_port,
-        table_name=table_name,
-    )
-
-    n = 1
-    while int(instance.query(f"select count() from test.{table_name}")) != 1:
-        sleep(1)
-        n += 1
-        if n > 10:
-            break
-
-    for query in queries:
-        cursor.execute(query.format(table_name))
-        print("query {}".format(query.format(table_name)))
-
-    instance.restart_clickhouse()
-
-    result = instance.query(f"SELECT count() FROM test.{table_name}")
-    print(result)  # Just debug
-
-    check_tables_are_synchronized(table_name)
-
-    result = instance.query(f"SELECT count() FROM test.{table_name}")
-    print(result)  # Just debug
-    instance.query(f"DROP TABLE test.{table_name} SYNC")
-
-
-def test_drop_table_immediately(started_cluster):
-    conn = get_postgres_conn(
-        ip=started_cluster.postgres_ip,
-        port=started_cluster.postgres_port,
-        database=True,
-    )
-    cursor = conn.cursor()
-    create_postgres_table(cursor, "postgresql_replica")
-    instance.query(
-        "INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(100000)"
-    )
-
-    instance.query("DROP TABLE IF EXISTS test.postgresql_replica")
-    create_materialized_table(
-        ip=started_cluster.postgres_ip, port=started_cluster.postgres_port
-    )
-    instance.query("DROP TABLE test.postgresql_replica")
-    create_materialized_table(
-        ip=started_cluster.postgres_ip, port=started_cluster.postgres_port
-    )
-    check_tables_are_synchronized("postgresql_replica")
-    instance.query(f"DROP TABLE test.postgresql_replica SYNC")
-
-
-if __name__ == "__main__":
-    cluster.start()
-    input("Cluster created, press any key to destroy...")
-    cluster.shutdown()
diff --git a/tests/integration/test_storage_postgresql_replica/test_postgresql_replica.reference b/tests/integration/test_storage_postgresql_replica/test_postgresql_replica.reference
deleted file mode 100644
index 959bb2aad74..00000000000
--- a/tests/integration/test_storage_postgresql_replica/test_postgresql_replica.reference
+++ /dev/null
@@ -1,50 +0,0 @@
-0	0
-1	1
-2	2
-3	3
-4	4
-5	5
-6	6
-7	7
-8	8
-9	9
-10	10
-11	11
-12	12
-13	13
-14	14
-15	15
-16	16
-17	17
-18	18
-19	19
-20	20
-21	21
-22	22
-23	23
-24	24
-25	25
-26	26
-27	27
-28	28
-29	29
-30	30
-31	31
-32	32
-33	33
-34	34
-35	35
-36	36
-37	37
-38	38
-39	39
-40	40
-41	41
-42	42
-43	43
-44	44
-45	45
-46	46
-47	47
-48	48
-49	49
diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py
index 751279f5e5a..80d2050b394 100644
--- a/tests/integration/test_storage_rabbitmq/test.py
+++ b/tests/integration/test_storage_rabbitmq/test.py
@@ -511,69 +511,6 @@ def test_rabbitmq_many_materialized_views(rabbitmq_cluster):
     rabbitmq_check_result(result2, True)
 
 
-@pytest.mark.skip(reason="clichouse_path with rabbitmq.proto fails to be exported")
-def test_rabbitmq_protobuf(rabbitmq_cluster):
-    instance.query(
-        """
-        CREATE TABLE test.rabbitmq (key UInt64, value String)
-            ENGINE = RabbitMQ
-            SETTINGS rabbitmq_host_port = 'rabbitmq1:5672',
-                     rabbitmq_exchange_name = 'pb',
-                     rabbitmq_format = 'Protobuf',
-                     rabbitmq_flush_interval_ms=1000,
-                     rabbitmq_max_block_size=100,
-                     rabbitmq_schema = 'rabbitmq.proto:KeyValueProto';
-        CREATE TABLE test.view (key UInt64, value UInt64)
-            ENGINE = MergeTree()
-            ORDER BY key;
-        CREATE MATERIALIZED VIEW test.consumer TO test.view AS
-            SELECT * FROM test.rabbitmq;
-        """
-    )
-
-    credentials = pika.PlainCredentials("root", "clickhouse")
-    parameters = pika.ConnectionParameters(
-        rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials
-    )
-    connection = pika.BlockingConnection(parameters)
-    channel = connection.channel()
-
-    data = ""
-    for i in range(0, 20):
-        msg = rabbitmq_pb2.KeyValueProto()
-        msg.key = i
-        msg.value = str(i)
-        serialized_msg = msg.SerializeToString()
-        data = data + _VarintBytes(len(serialized_msg)) + serialized_msg
-    channel.basic_publish(exchange="pb", routing_key="", body=data)
-    data = ""
-    for i in range(20, 21):
-        msg = rabbitmq_pb2.KeyValueProto()
-        msg.key = i
-        msg.value = str(i)
-        serialized_msg = msg.SerializeToString()
-        data = data + _VarintBytes(len(serialized_msg)) + serialized_msg
-    channel.basic_publish(exchange="pb", routing_key="", body=data)
-    data = ""
-    for i in range(21, 50):
-        msg = rabbitmq_pb2.KeyValueProto()
-        msg.key = i
-        msg.value = str(i)
-        serialized_msg = msg.SerializeToString()
-        data = data + _VarintBytes(len(serialized_msg)) + serialized_msg
-    channel.basic_publish(exchange="pb", routing_key="", body=data)
-
-    connection.close()
-
-    result = ""
-    while True:
-        result = instance.query("SELECT * FROM test.view ORDER BY key")
-        if rabbitmq_check_result(result):
-            break
-
-    rabbitmq_check_result(result, True)
-
-
 def test_rabbitmq_big_message(rabbitmq_cluster):
     # Create batchs of messages of size ~100Kb
     rabbitmq_messages = 1000
diff --git a/tests/integration/test_ttl_move/test.py b/tests/integration/test_ttl_move/test.py
index a2f28e21666..c1c076277bb 100644
--- a/tests/integration/test_ttl_move/test.py
+++ b/tests/integration/test_ttl_move/test.py
@@ -1529,106 +1529,6 @@ def test_concurrent_alter_with_ttl_move(started_cluster, name, engine):
         node1.query("DROP TABLE IF EXISTS {name} SYNC".format(name=name))
 
 
-@pytest.mark.skip(reason="Flacky test")
-@pytest.mark.parametrize(
-    "name,positive",
-    [
-        pytest.param("test_double_move_while_select_negative", 0, id="negative"),
-        pytest.param("test_double_move_while_select_positive", 1, id="positive"),
-    ],
-)
-def test_double_move_while_select(started_cluster, name, positive):
-    name = unique_table_name(name)
-
-    try:
-        node1.query(
-            """
-            CREATE TABLE {name} (
-                n Int64,
-                s String
-            ) ENGINE = MergeTree
-            ORDER BY tuple()
-            PARTITION BY n
-            SETTINGS storage_policy='small_jbod_with_external',temporary_directories_lifetime=1
-        """.format(
-                name=name
-            )
-        )
-
-        node1.query(
-            "INSERT INTO {name} VALUES (1, randomPrintableASCII(10*1024*1024))".format(
-                name=name
-            )
-        )
-
-        parts = node1.query(
-            "SELECT name FROM system.parts WHERE table = '{name}' AND active = 1".format(
-                name=name
-            )
-        ).splitlines()
-        assert len(parts) == 1
-
-        node1.query(
-            "ALTER TABLE {name} MOVE PART '{part}' TO DISK 'external'".format(
-                name=name, part=parts[0]
-            )
-        )
-
-        def long_select():
-            if positive:
-                node1.query(
-                    "SELECT sleep(3), sleep(2), sleep(1), n FROM {name}".format(
-                        name=name
-                    )
-                )
-
-        thread = threading.Thread(target=long_select)
-        thread.start()
-
-        time.sleep(1)
-
-        node1.query(
-            "ALTER TABLE {name} MOVE PART '{part}' TO DISK 'jbod1'".format(
-                name=name, part=parts[0]
-            )
-        )
-
-        # Fill jbod1 to force ClickHouse to make move of partition 1 to external.
-        node1.query(
-            "INSERT INTO {name} VALUES (2, randomPrintableASCII(9*1024*1024))".format(
-                name=name
-            )
-        )
-        node1.query(
-            "INSERT INTO {name} VALUES (3, randomPrintableASCII(9*1024*1024))".format(
-                name=name
-            )
-        )
-        node1.query(
-            "INSERT INTO {name} VALUES (4, randomPrintableASCII(9*1024*1024))".format(
-                name=name
-            )
-        )
-
-        wait_parts_mover(node1, name, retry_count=40)
-
-        # If SELECT locked old part on external, move shall fail.
-        assert node1.query(
-            "SELECT disk_name FROM system.parts WHERE table = '{name}' AND active = 1 AND name = '{part}'".format(
-                name=name, part=parts[0]
-            )
-        ).splitlines() == ["jbod1" if positive else "external"]
-
-        thread.join()
-
-        assert node1.query(
-            "SELECT n FROM {name} ORDER BY n".format(name=name)
-        ).splitlines() == ["1", "2", "3", "4"]
-
-    finally:
-        node1.query("DROP TABLE IF EXISTS {name} SYNC".format(name=name))
-
-
 @pytest.mark.parametrize(
     "name,engine,positive",
     [

From f2c3000a93e70d3eb6994a79bfccc84c5ba275e4 Mon Sep 17 00:00:00 2001
From: Nikolay Degterinsky <evillique@gmail.com>
Date: Wed, 9 Aug 2023 21:40:06 +0000
Subject: [PATCH 156/249] Improve exception in ALTER query

---
 src/Storages/MergeTree/MergeTreeData.cpp                  | 4 ++++
 tests/queries/0_stateless/02834_alter_exception.reference | 0
 tests/queries/0_stateless/02834_alter_exception.sql       | 4 ++++
 3 files changed, 8 insertions(+)
 create mode 100644 tests/queries/0_stateless/02834_alter_exception.reference
 create mode 100644 tests/queries/0_stateless/02834_alter_exception.sql

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 6179c70ca57..8938a3b6fbd 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -3166,6 +3166,10 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context
             }
         }
 
+        if (command.type == AlterCommand::MODIFY_QUERY)
+            throw Exception(ErrorCodes::NOT_IMPLEMENTED,
+                            "ALTER MODIFY QUERY is not supported by MergeTree engines family");
+
         if (command.type == AlterCommand::MODIFY_ORDER_BY && !is_custom_partitioned)
         {
             throw Exception(ErrorCodes::BAD_ARGUMENTS,
diff --git a/tests/queries/0_stateless/02834_alter_exception.reference b/tests/queries/0_stateless/02834_alter_exception.reference
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/queries/0_stateless/02834_alter_exception.sql b/tests/queries/0_stateless/02834_alter_exception.sql
new file mode 100644
index 00000000000..d42f40fcbf7
--- /dev/null
+++ b/tests/queries/0_stateless/02834_alter_exception.sql
@@ -0,0 +1,4 @@
+DROP TABLE IF EXISTS alter_02834;
+CREATE TABLE alter_02834 (a UInt64) ENGINE=MergeTree() ORDER BY a;
+ALTER TABLE alter_02834 MODIFY QUERY SELECT a FROM alter_02834; -- { serverError NOT_IMPLEMENTED }
+DROP TABLE alter_02834;

From ce30b93b34348700351e0d462edb5baa6ddb5685 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Wed, 9 Aug 2023 23:46:45 +0200
Subject: [PATCH 157/249] Move repeated code for ClickHouseHelper to
 _insert_post

---
 tests/ci/clickhouse_helper.py | 63 ++++++++---------------------------
 1 file changed, 14 insertions(+), 49 deletions(-)

diff --git a/tests/ci/clickhouse_helper.py b/tests/ci/clickhouse_helper.py
index dbf26288469..b165cda37e8 100644
--- a/tests/ci/clickhouse_helper.py
+++ b/tests/ci/clickhouse_helper.py
@@ -47,45 +47,9 @@ class ClickHouseHelper:
                 params[k] = v
 
         with open(file, "rb") as data_fd:
-            for i in range(5):
-                try:
-                    response = requests.post(
-                        url, params=params, data=data_fd, headers=auth
-                    )
-                except Exception as e:
-                    error = f"Received exception while sending data to {url} on {i} attempt: {e}"
-                    logging.warning(error)
-                    continue
-
-                logging.info("Response content '%s'", response.content)
-
-                if response.ok:
-                    break
-
-                error = (
-                    "Cannot insert data into clickhouse at try "
-                    + str(i)
-                    + ": HTTP code "
-                    + str(response.status_code)
-                    + ": '"
-                    + str(response.text)
-                    + "'"
-                )
-
-                if response.status_code >= 500:
-                    # A retriable error
-                    time.sleep(1)
-                    continue
-
-                logging.info(
-                    "Request headers '%s', body '%s'",
-                    response.request.headers,
-                    response.request.body,
-                )
-
-                raise InsertException(error)
-            else:
-                raise InsertException(error)
+            ClickHouseHelper._insert_post(
+                url, params=params, data=data_fd, headers=auth
+            )
 
     @staticmethod
     def insert_json_str(url, auth, db, table, json_str):
@@ -95,12 +59,18 @@ class ClickHouseHelper:
             "date_time_input_format": "best_effort",
             "send_logs_level": "warning",
         }
+        ClickHouseHelper._insert_post(url, params=params, data=json_str, headers=auth)
+
+    @staticmethod
+    def _insert_post(*args, **kwargs):
+        url = ""
+        if args:
+            url = args[0]
+        url = kwargs.get("url", url)
 
         for i in range(5):
             try:
-                response = requests.post(
-                    url, params=params, data=json_str, headers=auth
-                )
+                response = requests.post(*args, **kwargs)
             except Exception as e:
                 error = f"Received exception while sending data to {url} on {i} attempt: {e}"
                 logging.warning(error)
@@ -112,13 +82,8 @@ class ClickHouseHelper:
                 break
 
             error = (
-                "Cannot insert data into clickhouse at try "
-                + str(i)
-                + ": HTTP code "
-                + str(response.status_code)
-                + ": '"
-                + str(response.text)
-                + "'"
+                f"Cannot insert data into clickhouse at try {i}: HTTP code "
+                f"{response.status_code}: '{response.text}'"
             )
 
             if response.status_code >= 500:

From 8c63088a31d2b1d829a356f810392f9cf1484e93 Mon Sep 17 00:00:00 2001
From: Nikolay Degterinsky <evillique@gmail.com>
Date: Wed, 9 Aug 2023 21:54:58 +0000
Subject: [PATCH 158/249] Fixes

---
 .../02415_all_new_functions_must_be_documented.reference         | 1 +
 utils/check-style/aspell-ignore/en/aspell-dict.txt               | 1 +
 2 files changed, 2 insertions(+)

diff --git a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference
index 61a2e4e9f02..231f268ba57 100644
--- a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference
+++ b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference
@@ -881,6 +881,7 @@ tumble
 tumbleEnd
 tumbleStart
 tuple
+tupleConcat
 tupleDivide
 tupleDivideByNumber
 tupleElement
diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt
index a314815e2c4..1a7e8e16bd1 100644
--- a/utils/check-style/aspell-ignore/en/aspell-dict.txt
+++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt
@@ -2421,6 +2421,7 @@ tsv
 tui
 tumbleEnd
 tumbleStart
+tupleConcat
 tupleDivide
 tupleDivideByNumber
 tupleElement

From a79f2f140d25657a248f0d96bbe0241065a4260d Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 10 Aug 2023 00:19:24 +0200
Subject: [PATCH 159/249] Fix bad test `00417_kill_query`

---
 .../0_stateless/00417_kill_query.reference    |  4 +--
 tests/queries/0_stateless/00417_kill_query.sh | 25 ++++++++++++-------
 2 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/tests/queries/0_stateless/00417_kill_query.reference b/tests/queries/0_stateless/00417_kill_query.reference
index 1a3b47964c0..0dd1d78ac82 100644
--- a/tests/queries/0_stateless/00417_kill_query.reference
+++ b/tests/queries/0_stateless/00417_kill_query.reference
@@ -1,2 +1,2 @@
-SELECT sleep(1) FROM system.numbers LIMIT 30
-SELECT sleep(1) FROM system.numbers LIMIT 31
+SELECT sleep(1) FROM system.numbers LIMIT 300
+SELECT sleep(1) FROM system.numbers LIMIT 301
diff --git a/tests/queries/0_stateless/00417_kill_query.sh b/tests/queries/0_stateless/00417_kill_query.sh
index cd5b788a147..e64e93fd4a0 100755
--- a/tests/queries/0_stateless/00417_kill_query.sh
+++ b/tests/queries/0_stateless/00417_kill_query.sh
@@ -7,19 +7,26 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-QUERY_FIELND_NUM=4
+QUERY_FIELD_NUM=4
 
-$CLICKHOUSE_CLIENT --max_block_size=1 -q "SELECT sleep(1) FROM system.numbers LIMIT 30" &>/dev/null &
-sleep 1
-$CLICKHOUSE_CLIENT -q "KILL QUERY WHERE current_database = '${CLICKHOUSE_DATABASE}' and query LIKE 'SELECT sleep(%' AND (elapsed >= 0.) SYNC" | cut -f $QUERY_FIELND_NUM
+$CLICKHOUSE_CLIENT --max_block_size=1 -q "SELECT sleep(1) FROM system.numbers LIMIT 300" &>/dev/null &
 
-# 31 is for the query to be different from the previous one 
-$CLICKHOUSE_CLIENT --max_block_size=1 -q "SELECT sleep(1) FROM system.numbers LIMIT 31" &>/dev/null &
-sleep 1
-$CLICKHOUSE_CLIENT -q "KILL QUERY WHERE current_database = '${CLICKHOUSE_DATABASE}' and query = 'SELECT sleep(1) FROM system.numbers LIMIT 31' ASYNC" | cut -f $QUERY_FIELND_NUM
+while true
+do
+    $CLICKHOUSE_CLIENT --query "KILL QUERY WHERE current_database = '${CLICKHOUSE_DATABASE}' AND query LIKE 'SELECT sleep(%' AND (elapsed >= 0.) SYNC" | cut -f $QUERY_FIELD_NUM | grep '.' && break
+    sleep 0.1
+done
+
+# 31 is for the query to be different from the previous one
+$CLICKHOUSE_CLIENT --max_block_size=1 -q "SELECT sleep(1) FROM system.numbers LIMIT 301" &>/dev/null &
+
+while true
+do
+    $CLICKHOUSE_CLIENT --query "KILL QUERY WHERE current_database = '${CLICKHOUSE_DATABASE}' AND query = 'SELECT sleep(1) FROM system.numbers LIMIT 301' ASYNC" | cut -f $QUERY_FIELD_NUM | grep '.' && break
+    sleep 0.1
+done
 
 $CLICKHOUSE_CLIENT -q "KILL QUERY WHERE 0 ASYNC"
 $CLICKHOUSE_CLIENT -q "KILL QUERY WHERE 0 FORMAT TabSeparated"
 $CLICKHOUSE_CLIENT -q "KILL QUERY WHERE 0 SYNC FORMAT TabSeparated"
 $CLICKHOUSE_CLIENT -q "KILL QUERY WHERE 1 TEST" &>/dev/null
-

From 2802508c07917dc0b1d9b2d9ba91752c21a59cdf Mon Sep 17 00:00:00 2001
From: Yakov Olkhovskiy <yakov@clickhouse.com>
Date: Wed, 9 Aug 2023 22:27:49 +0000
Subject: [PATCH 160/249] different linker

---
 cmake/linux/toolchain-s390x.cmake | 1 -
 1 file changed, 1 deletion(-)

diff --git a/cmake/linux/toolchain-s390x.cmake b/cmake/linux/toolchain-s390x.cmake
index f5e1cf660c9..349d7fc83a6 100644
--- a/cmake/linux/toolchain-s390x.cmake
+++ b/cmake/linux/toolchain-s390x.cmake
@@ -23,7 +23,6 @@ set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
 set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=s390x-linux-gnu-ld")
 set (CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fuse-ld=s390x-linux-gnu-ld")
 set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=s390x-linux-gnu-ld")
-set (CMAKE_STATIC_LINKER_FLAGS "${CMAKE_STATIC_LINKER_FLAGS} -fuse-ld=s390x-linux-gnu-ld")
 
 set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
 set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)

From 087cc5bfed6736fc31f29e20a16f8161e40018c6 Mon Sep 17 00:00:00 2001
From: Yakov Olkhovskiy <yakov@clickhouse.com>
Date: Wed, 9 Aug 2023 23:18:27 +0000
Subject: [PATCH 161/249] different linker

---
 cmake/linux/toolchain-s390x.cmake | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cmake/linux/toolchain-s390x.cmake b/cmake/linux/toolchain-s390x.cmake
index 349d7fc83a6..188d9ed75b7 100644
--- a/cmake/linux/toolchain-s390x.cmake
+++ b/cmake/linux/toolchain-s390x.cmake
@@ -20,9 +20,9 @@ set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}/s390x-linux-gnu/libc")
 set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
 set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
 set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
-set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=s390x-linux-gnu-ld")
-set (CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fuse-ld=s390x-linux-gnu-ld")
-set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=s390x-linux-gnu-ld")
+set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=gold")
+set (CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fuse-ld=gold")
+set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=gold")
 
 set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
 set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)

From 65d3778b031dab996556e7e95597d7802daf3432 Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Tue, 8 Aug 2023 20:15:04 +0000
Subject: [PATCH 162/249] Fix warning in test_replicated_database

---
 tests/integration/test_replicated_database/test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/test_replicated_database/test.py b/tests/integration/test_replicated_database/test.py
index 1235f7d34df..d0a04f40b69 100644
--- a/tests/integration/test_replicated_database/test.py
+++ b/tests/integration/test_replicated_database/test.py
@@ -1226,7 +1226,7 @@ def test_force_synchronous_settings(started_cluster):
 
     def select_func():
         dummy_node.query(
-            "SELECT sleepEachRow(1) FROM test_force_synchronous_settings.t"
+            "SELECT sleepEachRow(1) FROM test_force_synchronous_settings.t SETTINGS function_sleep_max_microseconds_per_block = 0"
         )
 
     select_thread = threading.Thread(target=select_func)

From eb61074e070587edb048fb02f26852dd3a2fd398 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 10 Aug 2023 02:37:23 +0200
Subject: [PATCH 163/249] Fix test `02428_delete_with_settings`

---
 tests/queries/0_stateless/02428_delete_with_settings.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02428_delete_with_settings.sql b/tests/queries/0_stateless/02428_delete_with_settings.sql
index 071a3f74184..618c08608fc 100644
--- a/tests/queries/0_stateless/02428_delete_with_settings.sql
+++ b/tests/queries/0_stateless/02428_delete_with_settings.sql
@@ -1,5 +1,5 @@
 drop table if exists test;
-create table test (id Int32, key String) engine=MergeTree() order by tuple();
+create table test (id Int32, key String) engine=MergeTree() order by tuple() settings index_granularity = 8192, index_granularity_bytes = '10Mi';
 insert into test select number, toString(number) from numbers(1000000);
 delete from test where id % 2 = 0 SETTINGS mutations_sync=0;
 select count() from test;

From a9b760a15bf284dbe20882ddc696bdf28efed061 Mon Sep 17 00:00:00 2001
From: Yakov Olkhovskiy <yakov@clickhouse.com>
Date: Thu, 10 Aug 2023 00:40:06 +0000
Subject: [PATCH 164/249] different linker

---
 cmake/linux/toolchain-s390x.cmake | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cmake/linux/toolchain-s390x.cmake b/cmake/linux/toolchain-s390x.cmake
index 188d9ed75b7..23d14d6ed26 100644
--- a/cmake/linux/toolchain-s390x.cmake
+++ b/cmake/linux/toolchain-s390x.cmake
@@ -20,9 +20,9 @@ set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}/s390x-linux-gnu/libc")
 set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
 set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
 set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
-set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=gold")
-set (CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fuse-ld=gold")
-set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=gold")
+set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=bfd")
+set (CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fuse-ld=bfd")
+set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=bfd")
 
 set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
 set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)

From c071f6755d70a5ce9f1924fc44a905cc3122da5d Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 10 Aug 2023 04:10:03 +0300
Subject: [PATCH 165/249] Revert "Added new tests for session_log and fixed the
 inconsistency between login and logout."

---
 src/Interpreters/Session.cpp                  |  44 +--
 src/Interpreters/Session.h                    |   2 -
 src/Interpreters/SessionLog.cpp               |  14 +-
 src/Interpreters/SessionLog.h                 |   9 +-
 src/Server/HTTPHandler.cpp                    |   3 +-
 .../test.py                                   |   5 +-
 tests/integration/test_session_log/.gitignore |   1 -
 .../integration/test_session_log/__init__.py  |   0
 .../test_session_log/configs/log.xml          |   9 -
 .../test_session_log/configs/ports.xml        |   9 -
 .../test_session_log/configs/session_log.xml  |   9 -
 .../test_session_log/configs/users.xml        |  23 --
 .../protos/clickhouse_grpc.proto              |   1 -
 tests/integration/test_session_log/test.py    | 289 ------------------
 .../02833_concurrrent_sessions.reference      |  34 ---
 .../0_stateless/02833_concurrrent_sessions.sh | 138 ---------
 .../02834_remote_session_log.reference        |  13 -
 .../0_stateless/02834_remote_session_log.sh   |  56 ----
 .../02835_drop_user_during_session.reference  |   8 -
 .../02835_drop_user_during_session.sh         | 113 -------
 20 files changed, 27 insertions(+), 753 deletions(-)
 delete mode 100644 tests/integration/test_session_log/.gitignore
 delete mode 100644 tests/integration/test_session_log/__init__.py
 delete mode 100644 tests/integration/test_session_log/configs/log.xml
 delete mode 100644 tests/integration/test_session_log/configs/ports.xml
 delete mode 100644 tests/integration/test_session_log/configs/session_log.xml
 delete mode 100644 tests/integration/test_session_log/configs/users.xml
 delete mode 120000 tests/integration/test_session_log/protos/clickhouse_grpc.proto
 delete mode 100644 tests/integration/test_session_log/test.py
 delete mode 100644 tests/queries/0_stateless/02833_concurrrent_sessions.reference
 delete mode 100755 tests/queries/0_stateless/02833_concurrrent_sessions.sh
 delete mode 100644 tests/queries/0_stateless/02834_remote_session_log.reference
 delete mode 100755 tests/queries/0_stateless/02834_remote_session_log.sh
 delete mode 100644 tests/queries/0_stateless/02835_drop_user_during_session.reference
 delete mode 100755 tests/queries/0_stateless/02835_drop_user_during_session.sh

diff --git a/src/Interpreters/Session.cpp b/src/Interpreters/Session.cpp
index 0a6435cff75..de2a779b740 100644
--- a/src/Interpreters/Session.cpp
+++ b/src/Interpreters/Session.cpp
@@ -520,8 +520,6 @@ ContextMutablePtr Session::makeSessionContext()
         {},
         session_context->getSettingsRef().max_sessions_for_user);
 
-    recordLoginSucess(session_context);
-
     return session_context;
 }
 
@@ -584,8 +582,6 @@ ContextMutablePtr Session::makeSessionContext(const String & session_name_, std:
         { session_name_ },
         max_sessions_for_user);
 
-    recordLoginSucess(session_context);
-
     return session_context;
 }
 
@@ -659,38 +655,24 @@ ContextMutablePtr Session::makeQueryContextImpl(const ClientInfo * client_info_t
     if (user_id)
         user = query_context->getUser();
 
-    /// Interserver does not create session context
-    recordLoginSucess(query_context);
+    if (!notified_session_log_about_login)
+    {
+        if (auto session_log = getSessionLog())
+        {
+            session_log->addLoginSuccess(
+                    auth_id,
+                    named_session ? std::optional<std::string>(named_session->key.second) : std::nullopt,
+                    *query_context,
+                    user);
+
+            notified_session_log_about_login = true;
+        }
+    }
 
     return query_context;
 }
 
 
-void Session::recordLoginSucess(ContextPtr login_context) const
-{
-    if (notified_session_log_about_login)
-        return;
-
-    if (!login_context)
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "Session or query context must be created");
-
-    if (auto session_log = getSessionLog())
-    {
-        const auto & settings   = login_context->getSettingsRef();
-        const auto access       = login_context->getAccess();
-
-        session_log->addLoginSuccess(auth_id,
-                                     named_session ? named_session->key.second : "",
-                                     settings,
-                                     access,
-                                     getClientInfo(),
-                                     user);
-
-        notified_session_log_about_login = true;
-    }
-}
-
-
 void Session::releaseSessionID()
 {
     if (!named_session)
diff --git a/src/Interpreters/Session.h b/src/Interpreters/Session.h
index 81ef987b428..51c0e3c71fa 100644
--- a/src/Interpreters/Session.h
+++ b/src/Interpreters/Session.h
@@ -97,8 +97,6 @@ public:
 private:
     std::shared_ptr<SessionLog> getSessionLog() const;
     ContextMutablePtr makeQueryContextImpl(const ClientInfo * client_info_to_copy, ClientInfo * client_info_to_move) const;
-    void recordLoginSucess(ContextPtr login_context) const;
-
 
     mutable bool notified_session_log_about_login = false;
     const UUID auth_id;
diff --git a/src/Interpreters/SessionLog.cpp b/src/Interpreters/SessionLog.cpp
index b847eaf9824..0a8a7fc18c5 100644
--- a/src/Interpreters/SessionLog.cpp
+++ b/src/Interpreters/SessionLog.cpp
@@ -199,13 +199,12 @@ void SessionLogElement::appendToBlock(MutableColumns & columns) const
     columns[i++]->insertData(auth_failure_reason.data(), auth_failure_reason.length());
 }
 
-void SessionLog::addLoginSuccess(const UUID & auth_id,
-                                 const String & session_id,
-                                 const Settings & settings,
-                                 const ContextAccessPtr & access,
-                                 const ClientInfo & client_info,
-                                 const UserPtr & login_user)
+void SessionLog::addLoginSuccess(const UUID & auth_id, std::optional<String> session_id, const Context & login_context, const UserPtr & login_user)
 {
+    const auto access = login_context.getAccess();
+    const auto & settings = login_context.getSettingsRef();
+    const auto & client_info = login_context.getClientInfo();
+
     DB::SessionLogElement log_entry(auth_id, SESSION_LOGIN_SUCCESS);
     log_entry.client_info = client_info;
 
@@ -216,7 +215,8 @@ void SessionLog::addLoginSuccess(const UUID & auth_id,
     }
     log_entry.external_auth_server = login_user ? login_user->auth_data.getLDAPServerName() : "";
 
-    log_entry.session_id = session_id;
+    if (session_id)
+        log_entry.session_id = *session_id;
 
     if (const auto roles_info = access->getRolesInfo())
         log_entry.roles = roles_info->getCurrentRolesNames();
diff --git a/src/Interpreters/SessionLog.h b/src/Interpreters/SessionLog.h
index 8757bc12270..1282ac09c4d 100644
--- a/src/Interpreters/SessionLog.h
+++ b/src/Interpreters/SessionLog.h
@@ -20,7 +20,6 @@ enum SessionLogElementType : int8_t
 class ContextAccess;
 struct User;
 using UserPtr = std::shared_ptr<const User>;
-using ContextAccessPtr = std::shared_ptr<const ContextAccess>;
 
 /** A struct which will be inserted as row into session_log table.
   *
@@ -73,13 +72,7 @@ class SessionLog : public SystemLog<SessionLogElement>
     using SystemLog<SessionLogElement>::SystemLog;
 
 public:
-    void addLoginSuccess(const UUID & auth_id,
-                         const String & session_id,
-                         const Settings & settings,
-                         const ContextAccessPtr & access,
-                         const ClientInfo & client_info,
-                         const UserPtr & login_user);
-
+    void addLoginSuccess(const UUID & auth_id, std::optional<String> session_id, const Context & login_context, const UserPtr & login_user);
     void addLoginFailure(const UUID & auth_id, const ClientInfo & info, const std::optional<String> & user, const Exception & reason);
     void addLogOut(const UUID & auth_id, const UserPtr & login_user, const ClientInfo & client_info);
 };
diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp
index 10d5e7a0242..ff5690a3b07 100644
--- a/src/Server/HTTPHandler.cpp
+++ b/src/Server/HTTPHandler.cpp
@@ -561,7 +561,8 @@ void HTTPHandler::processQuery(
         session->makeSessionContext();
     }
 
-    auto context = session->makeQueryContext();
+    auto client_info = session->getClientInfo();
+    auto context = session->makeQueryContext(std::move(client_info));
 
     /// This parameter is used to tune the behavior of output formats (such as Native) for compatibility.
     if (params.has("client_protocol_version"))
diff --git a/tests/integration/test_profile_max_sessions_for_user/test.py b/tests/integration/test_profile_max_sessions_for_user/test.py
index 65587933fed..2930262f63e 100755
--- a/tests/integration/test_profile_max_sessions_for_user/test.py
+++ b/tests/integration/test_profile_max_sessions_for_user/test.py
@@ -27,7 +27,10 @@ proto_dir = os.path.join(SCRIPT_DIR, "./protos")
 gen_dir = os.path.join(SCRIPT_DIR, "./_gen")
 os.makedirs(gen_dir, exist_ok=True)
 run_and_check(
-    f"python3 -m grpc_tools.protoc -I{proto_dir} --python_out={gen_dir} --grpc_python_out={gen_dir} {proto_dir}/clickhouse_grpc.proto",
+    "python3 -m grpc_tools.protoc -I{proto_dir} --python_out={gen_dir} --grpc_python_out={gen_dir} \
+    {proto_dir}/clickhouse_grpc.proto".format(
+        proto_dir=proto_dir, gen_dir=gen_dir
+    ),
     shell=True,
 )
 
diff --git a/tests/integration/test_session_log/.gitignore b/tests/integration/test_session_log/.gitignore
deleted file mode 100644
index edf565ec632..00000000000
--- a/tests/integration/test_session_log/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-_gen
diff --git a/tests/integration/test_session_log/__init__.py b/tests/integration/test_session_log/__init__.py
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/integration/test_session_log/configs/log.xml b/tests/integration/test_session_log/configs/log.xml
deleted file mode 100644
index 7a079b81e69..00000000000
--- a/tests/integration/test_session_log/configs/log.xml
+++ /dev/null
@@ -1,9 +0,0 @@
- <clickhouse>
-     <logger>
-         <level>trace</level>
-         <log>/var/log/clickhouse-server/clickhouse-server.log</log>
-         <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
-         <size>1000M</size>
-         <count>10</count>
-     </logger>
- </clickhouse>
\ No newline at end of file
diff --git a/tests/integration/test_session_log/configs/ports.xml b/tests/integration/test_session_log/configs/ports.xml
deleted file mode 100644
index fbaefc16b3a..00000000000
--- a/tests/integration/test_session_log/configs/ports.xml
+++ /dev/null
@@ -1,9 +0,0 @@
- <clickhouse>
-    <postgresql_port>5433</postgresql_port>
-    <mysql_port>9001</mysql_port>
-    <grpc_port>9100</grpc_port>
-    <grpc replace="replace">
-        <!-- Enable if you want very detailed logs -->
-        <verbose_logs>false</verbose_logs>
-    </grpc>
- </clickhouse>
\ No newline at end of file
diff --git a/tests/integration/test_session_log/configs/session_log.xml b/tests/integration/test_session_log/configs/session_log.xml
deleted file mode 100644
index a0e4e3e2216..00000000000
--- a/tests/integration/test_session_log/configs/session_log.xml
+++ /dev/null
@@ -1,9 +0,0 @@
-<clickhouse>
-    <session_log>
-        <database>system</database>
-        <table>session_log</table>
-
-        <partition_by>toYYYYMM(event_date)</partition_by>
-        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
-    </session_log>
-</clickhouse>
diff --git a/tests/integration/test_session_log/configs/users.xml b/tests/integration/test_session_log/configs/users.xml
deleted file mode 100644
index 0416dfadc8a..00000000000
--- a/tests/integration/test_session_log/configs/users.xml
+++ /dev/null
@@ -1,23 +0,0 @@
-<clickhouse>
-    <profiles>
-        <default>
-            <function_sleep_max_microseconds_per_block>0</function_sleep_max_microseconds_per_block>
-        </default>
-    </profiles>
-    <users>
-        <default>
-        </default>
-        <mysql_user>
-            <password>pass</password>
-        </mysql_user>
-        <postgres_user>
-            <password>pass</password>
-        </postgres_user>
-        <grpc_user>
-            <password>pass</password>
-        </grpc_user>
-        <parallel_user>
-            <password>pass</password>
-        </parallel_user>
-    </users>
-</clickhouse>
\ No newline at end of file
diff --git a/tests/integration/test_session_log/protos/clickhouse_grpc.proto b/tests/integration/test_session_log/protos/clickhouse_grpc.proto
deleted file mode 120000
index 25d15f11e3b..00000000000
--- a/tests/integration/test_session_log/protos/clickhouse_grpc.proto
+++ /dev/null
@@ -1 +0,0 @@
-../../../../src/Server/grpc_protos/clickhouse_grpc.proto
\ No newline at end of file
diff --git a/tests/integration/test_session_log/test.py b/tests/integration/test_session_log/test.py
deleted file mode 100644
index dbb39993ce3..00000000000
--- a/tests/integration/test_session_log/test.py
+++ /dev/null
@@ -1,289 +0,0 @@
-import os
-
-import grpc
-import pymysql.connections
-import psycopg2 as py_psql
-import pytest
-import random
-import sys
-import threading
-
-from helpers.cluster import ClickHouseCluster, run_and_check
-
-POSTGRES_SERVER_PORT = 5433
-MYSQL_SERVER_PORT = 9001
-GRPC_PORT = 9100
-SESSION_LOG_MATCHING_FIELDS = "auth_id, auth_type, client_version_major, client_version_minor, client_version_patch, interface"
-
-SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
-DEFAULT_ENCODING = "utf-8"
-
-# Use grpcio-tools to generate *pb2.py files from *.proto.
-proto_dir = os.path.join(SCRIPT_DIR, "./protos")
-gen_dir = os.path.join(SCRIPT_DIR, "./_gen")
-os.makedirs(gen_dir, exist_ok=True)
-run_and_check(
-    f"python3 -m grpc_tools.protoc -I{proto_dir} --python_out={gen_dir} --grpc_python_out={gen_dir} {proto_dir}/clickhouse_grpc.proto",
-    shell=True,
-)
-
-sys.path.append(gen_dir)
-
-import clickhouse_grpc_pb2
-import clickhouse_grpc_pb2_grpc
-
-cluster = ClickHouseCluster(__file__)
-instance = cluster.add_instance(
-    "node",
-    main_configs=[
-        "configs/ports.xml",
-        "configs/log.xml",
-        "configs/session_log.xml",
-    ],
-    user_configs=["configs/users.xml"],
-    # Bug in TSAN reproduces in this test https://github.com/grpc/grpc/issues/29550#issuecomment-1188085387
-    env_variables={
-        "TSAN_OPTIONS": "report_atomic_races=0 " + os.getenv("TSAN_OPTIONS", default="")
-    },
-)
-
-
-def grpc_get_url():
-    return f"{instance.ip_address}:{GRPC_PORT}"
-
-
-def grpc_create_insecure_channel():
-    channel = grpc.insecure_channel(grpc_get_url())
-    grpc.channel_ready_future(channel).result(timeout=2)
-    return channel
-
-
-session_id_counter = 0
-
-
-def next_session_id():
-    global session_id_counter
-    session_id = session_id_counter
-    session_id_counter += 1
-    return str(session_id)
-
-
-def grpc_query(query, user_, pass_, raise_exception):
-    try:
-        query_info = clickhouse_grpc_pb2.QueryInfo(
-            query=query,
-            session_id=next_session_id(),
-            user_name=user_,
-            password=pass_,
-        )
-        channel = grpc_create_insecure_channel()
-        stub = clickhouse_grpc_pb2_grpc.ClickHouseStub(channel)
-        result = stub.ExecuteQuery(query_info)
-        if result and result.HasField("exception"):
-            raise Exception(result.exception.display_text)
-
-        return result.output.decode(DEFAULT_ENCODING)
-    except Exception:
-        assert raise_exception
-
-
-def postgres_query(query, user_, pass_, raise_exception):
-    try:
-        client = py_psql.connect(
-            host=instance.ip_address,
-            port=POSTGRES_SERVER_PORT,
-            user=user_,
-            password=pass_,
-            database="default",
-        )
-        cursor = client.cursor()
-        cursor.execute(query)
-        cursor.fetchall()
-    except Exception:
-        assert raise_exception
-
-
-def mysql_query(query, user_, pass_, raise_exception):
-    try:
-        client = pymysql.connections.Connection(
-            host=instance.ip_address,
-            user=user_,
-            password=pass_,
-            database="default",
-            port=MYSQL_SERVER_PORT,
-        )
-        cursor = client.cursor(pymysql.cursors.DictCursor)
-        if raise_exception:
-            with pytest.raises(Exception):
-                cursor.execute(query)
-        else:
-            cursor.execute(query)
-        cursor.fetchall()
-    except Exception:
-        assert raise_exception
-
-
-@pytest.fixture(scope="module")
-def started_cluster():
-    try:
-        cluster.start()
-        yield cluster
-    finally:
-        cluster.shutdown()
-
-
-def test_grpc_session(started_cluster):
-    grpc_query("SELECT 1", "grpc_user", "pass", False)
-    grpc_query("SELECT 2", "grpc_user", "wrong_pass", True)
-    grpc_query("SELECT 3", "wrong_grpc_user", "pass", True)
-
-    instance.query("SYSTEM FLUSH LOGS")
-    login_success_records = instance.query(
-        "SELECT user, client_port <> 0,  client_address <> toIPv6('::') FROM system.session_log WHERE user='grpc_user' AND type = 'LoginSuccess'"
-    )
-    assert login_success_records == "grpc_user\t1\t1\n"
-    logout_records = instance.query(
-        "SELECT user, client_port <> 0,  client_address <> toIPv6('::') FROM system.session_log WHERE user='grpc_user' AND type = 'Logout'"
-    )
-    assert logout_records == "grpc_user\t1\t1\n"
-    login_failure_records = instance.query(
-        "SELECT user, client_port <> 0,  client_address <> toIPv6('::') FROM system.session_log WHERE user='grpc_user' AND type = 'LoginFailure'"
-    )
-    assert login_failure_records == "grpc_user\t1\t1\n"
-    logins_and_logouts = instance.query(
-        f"SELECT COUNT(*) FROM (SELECT {SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = 'grpc_user' AND type = 'LoginSuccess' INTERSECT SELECT {SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = 'grpc_user' AND type = 'Logout')"
-    )
-    assert logins_and_logouts == "1\n"
-
-
-def test_mysql_session(started_cluster):
-    mysql_query("SELECT 1", "mysql_user", "pass", False)
-    mysql_query("SELECT 2", "mysql_user", "wrong_pass", True)
-    mysql_query("SELECT 3", "wrong_mysql_user", "pass", True)
-
-    instance.query("SYSTEM FLUSH LOGS")
-    login_success_records = instance.query(
-        "SELECT user, client_port <> 0,  client_address <> toIPv6('::') FROM system.session_log WHERE user='mysql_user' AND type = 'LoginSuccess'"
-    )
-    assert login_success_records == "mysql_user\t1\t1\n"
-    logout_records = instance.query(
-        "SELECT user, client_port <> 0,  client_address <> toIPv6('::') FROM system.session_log WHERE user='mysql_user' AND type = 'Logout'"
-    )
-    assert logout_records == "mysql_user\t1\t1\n"
-    login_failure_records = instance.query(
-        "SELECT user, client_port <> 0,  client_address <> toIPv6('::') FROM system.session_log WHERE user='mysql_user' AND type = 'LoginFailure'"
-    )
-    assert login_failure_records == "mysql_user\t1\t1\n"
-    logins_and_logouts = instance.query(
-        f"SELECT COUNT(*) FROM (SELECT {SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = 'mysql_user' AND type = 'LoginSuccess' INTERSECT SELECT {SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = 'mysql_user' AND type = 'Logout')"
-    )
-    assert logins_and_logouts == "1\n"
-
-
-def test_postgres_session(started_cluster):
-    postgres_query("SELECT 1", "postgres_user", "pass", False)
-    postgres_query("SELECT 2", "postgres_user", "wrong_pass", True)
-    postgres_query("SELECT 3", "wrong_postgres_user", "pass", True)
-
-    instance.query("SYSTEM FLUSH LOGS")
-    login_success_records = instance.query(
-        "SELECT user, client_port <> 0,  client_address <> toIPv6('::') FROM system.session_log WHERE user='postgres_user' AND type = 'LoginSuccess'"
-    )
-    assert login_success_records == "postgres_user\t1\t1\n"
-    logout_records = instance.query(
-        "SELECT user, client_port <> 0,  client_address <> toIPv6('::') FROM system.session_log WHERE user='postgres_user' AND type = 'Logout'"
-    )
-    assert logout_records == "postgres_user\t1\t1\n"
-    login_failure_records = instance.query(
-        "SELECT user, client_port <> 0,  client_address <> toIPv6('::') FROM system.session_log WHERE user='postgres_user' AND type = 'LoginFailure'"
-    )
-    assert login_failure_records == "postgres_user\t1\t1\n"
-    logins_and_logouts = instance.query(
-        f"SELECT COUNT(*) FROM (SELECT {SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = 'postgres_user' AND type = 'LoginSuccess' INTERSECT SELECT {SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = 'postgres_user' AND type = 'Logout')"
-    )
-    assert logins_and_logouts == "1\n"
-
-
-def test_parallel_sessions(started_cluster):
-    thread_list = []
-    for _ in range(10):
-        # Sleep time does not significantly matter here,
-        # test should pass even without sleeping.
-        for function in [postgres_query, grpc_query, mysql_query]:
-            thread = threading.Thread(
-                target=function,
-                args=(
-                    f"SELECT sleep({random.uniform(0.03, 0.04)})",
-                    "parallel_user",
-                    "pass",
-                    False,
-                ),
-            )
-            thread.start()
-            thread_list.append(thread)
-            thread = threading.Thread(
-                target=function,
-                args=(
-                    f"SELECT sleep({random.uniform(0.03, 0.04)})",
-                    "parallel_user",
-                    "wrong_pass",
-                    True,
-                ),
-            )
-            thread.start()
-            thread_list.append(thread)
-            thread = threading.Thread(
-                target=function,
-                args=(
-                    f"SELECT sleep({random.uniform(0.03, 0.04)})",
-                    "wrong_parallel_user",
-                    "pass",
-                    True,
-                ),
-            )
-            thread.start()
-            thread_list.append(thread)
-
-    for thread in thread_list:
-        thread.join()
-
-    instance.query("SYSTEM FLUSH LOGS")
-    port_0_sessions = instance.query(
-        f"SELECT COUNT(*) FROM system.session_log WHERE user = 'parallel_user'"
-    )
-    assert port_0_sessions == "90\n"
-
-    port_0_sessions = instance.query(
-        f"SELECT COUNT(*) FROM system.session_log WHERE user = 'parallel_user' AND client_port = 0"
-    )
-    assert port_0_sessions == "0\n"
-
-    address_0_sessions = instance.query(
-        f"SELECT COUNT(*) FROM system.session_log WHERE user = 'parallel_user' AND client_address = toIPv6('::')"
-    )
-    assert address_0_sessions == "0\n"
-
-    grpc_sessions = instance.query(
-        f"SELECT COUNT(*) FROM system.session_log WHERE user = 'parallel_user' AND interface = 'gRPC'"
-    )
-    assert grpc_sessions == "30\n"
-
-    mysql_sessions = instance.query(
-        f"SELECT COUNT(*) FROM system.session_log WHERE user = 'parallel_user' AND interface = 'MySQL'"
-    )
-    assert mysql_sessions == "30\n"
-
-    postgres_sessions = instance.query(
-        f"SELECT COUNT(*) FROM system.session_log WHERE user = 'parallel_user' AND interface = 'PostgreSQL'"
-    )
-    assert postgres_sessions == "30\n"
-
-    logins_and_logouts = instance.query(
-        f"SELECT COUNT(*) FROM (SELECT {SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = 'parallel_user' AND type = 'LoginSuccess' INTERSECT SELECT {SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = 'parallel_user' AND type = 'Logout')"
-    )
-    assert logins_and_logouts == "30\n"
-
-    logout_failure_sessions = instance.query(
-        f"SELECT COUNT(*) FROM system.session_log  WHERE user = 'parallel_user' AND type = 'LoginFailure'"
-    )
-    assert logout_failure_sessions == "30\n"
diff --git a/tests/queries/0_stateless/02833_concurrrent_sessions.reference b/tests/queries/0_stateless/02833_concurrrent_sessions.reference
deleted file mode 100644
index bfe507e8eac..00000000000
--- a/tests/queries/0_stateless/02833_concurrrent_sessions.reference
+++ /dev/null
@@ -1,34 +0,0 @@
-sessions:
-150
-port_0_sessions:
-0
-address_0_sessions:
-0
-tcp_sessions
-60
-http_sessions
-30
-http_with_session_id_sessions
-30
-my_sql_sessions
-30
-Corresponding LoginSuccess/Logout
-10
-LoginFailure
-10
-Corresponding LoginSuccess/Logout
-10
-LoginFailure
-10
-Corresponding LoginSuccess/Logout
-10
-LoginFailure
-10
-Corresponding LoginSuccess/Logout
-10
-LoginFailure
-10
-Corresponding LoginSuccess/Logout
-10
-LoginFailure
-10
diff --git a/tests/queries/0_stateless/02833_concurrrent_sessions.sh b/tests/queries/0_stateless/02833_concurrrent_sessions.sh
deleted file mode 100755
index d1d571c6985..00000000000
--- a/tests/queries/0_stateless/02833_concurrrent_sessions.sh
+++ /dev/null
@@ -1,138 +0,0 @@
-#!/usr/bin/env bash
-# Tags: no-fasttest, long
-
-CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# shellcheck source=../shell_config.sh
-. "$CURDIR"/../shell_config.sh
-
-readonly PID=$$
-
-# Each user uses a separate thread.
-readonly TCP_USERS=( "02833_TCP_USER_${PID}"_{1,2} ) # 2 concurrent TCP users
-readonly HTTP_USERS=( "02833_HTTP_USER_${PID}" )
-readonly HTTP_WITH_SESSION_ID_SESSION_USERS=( "02833_HTTP_WITH_SESSION_ID_USER_${PID}" )
-readonly MYSQL_USERS=( "02833_MYSQL_USER_${PID}")
-readonly ALL_USERS=( "${TCP_USERS[@]}" "${HTTP_USERS[@]}" "${HTTP_WITH_SESSION_ID_SESSION_USERS[@]}" "${MYSQL_USERS[@]}" )
-
-readonly TCP_USERS_SQL_COLLECTION_STRING="$( echo "${TCP_USERS[*]}" | sed "s/[^[:space:]]\+/'&'/g" | sed 's/[[:space:]]/,/g' )"
-readonly HTTP_USERS_SQL_COLLECTION_STRING="$( echo "${HTTP_USERS[*]}" | sed "s/[^[:space:]]\+/'&'/g" | sed 's/[[:space:]]/,/g' )"
-readonly HTTP_WITH_SESSION_ID_USERS_SQL_COLLECTION_STRING="$( echo "${HTTP_WITH_SESSION_ID_SESSION_USERS[*]}" | sed "s/[^[:space:]]\+/'&'/g" | sed 's/[[:space:]]/,/g' )"
-readonly MYSQL_USERS_SQL_COLLECTION_STRING="$( echo "${MYSQL_USERS[*]}" | sed "s/[^[:space:]]\+/'&'/g" | sed 's/[[:space:]]/,/g' )"
-readonly ALL_USERS_SQL_COLLECTION_STRING="$( echo "${ALL_USERS[*]}" | sed "s/[^[:space:]]\+/'&'/g" | sed 's/[[:space:]]/,/g' )"
-
-readonly SESSION_LOG_MATCHING_FIELDS="auth_id, auth_type, client_version_major, client_version_minor, client_version_patch, interface"
-
-for user in "${ALL_USERS[@]}"; do
-    ${CLICKHOUSE_CLIENT} -q "CREATE USER IF NOT EXISTS ${user} IDENTIFIED WITH plaintext_password BY 'pass'"
-    ${CLICKHOUSE_CLIENT} -q "GRANT SELECT ON system.* TO ${user}"
-    ${CLICKHOUSE_CLIENT} -q "GRANT SELECT ON INFORMATION_SCHEMA.* TO ${user}";
-done
-
-# All <type>_session functions execute in separate threads.
-# These functions try to create a session with successful login and logout.
-# Sleep a small, random amount of time to make concurrency more intense.
-# and try to login with an invalid password.
-function tcp_session() 
-{
-    local user=$1
-    local i=0
-    while (( (i++) < 10 )); do
-        # login logout
-        ${CLICKHOUSE_CLIENT} -q "SELECT 1, sleep(0.01${RANDOM})" --user="${user}" --password="pass"
-        # login failure
-        ${CLICKHOUSE_CLIENT} -q "SELECT 2" --user="${user}" --password 'invalid'
-    done
-}
-
-function http_session() 
-{
-    local user=$1
-    local i=0
-    while (( (i++) < 10 )); do
-        # login logout
-        ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&user=${user}&password=pass" -d "SELECT 3, sleep(0.01${RANDOM})"
-
-        # login failure
-        ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&user=${user}&password=wrong" -d "SELECT 4"
-    done
-}
-
-function http_with_session_id_session() 
-{
-    local user=$1
-    local i=0
-    while (( (i++) < 10 )); do
-        # login logout
-        ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=${user}&user=${user}&password=pass" -d "SELECT 5, sleep 0.01${RANDOM}"
-
-        # login failure
-        ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=${user}&user=${user}&password=wrong" -d "SELECT 6"
-    done
-}
-
-function mysql_session() 
-{
-    local user=$1
-    local i=0
-    while (( (i++) < 10 )); do
-        # login logout
-        ${CLICKHOUSE_CLIENT} -q "SELECT 1, sleep(0.01${RANDOM}) FROM mysql('127.0.0.1:9004', 'system', 'one', '${user}', 'pass')"
-
-        # login failure
-        ${CLICKHOUSE_CLIENT} -q "SELECT 1 FROM mysql('127.0.0.1:9004', 'system', 'one', '${user}', 'wrong', SETTINGS connection_max_tries=1)"
-    done
-}
-
-${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS"
-${CLICKHOUSE_CLIENT} -q "DELETE FROM system.session_log WHERE user IN (${ALL_USERS_SQL_COLLECTION_STRING})"
-
-export -f tcp_session;
-export -f http_session;
-export -f http_with_session_id_session;
-export -f mysql_session;
-
-for user in "${TCP_USERS[@]}"; do
-    timeout 60s bash -c "tcp_session ${user}" >/dev/null 2>&1 &
-done
-
-for user in "${HTTP_USERS[@]}"; do
-    timeout 60s bash -c "http_session ${user}" >/dev/null 2>&1 &
-done
-
-for user in "${HTTP_WITH_SESSION_ID_SESSION_USERS[@]}"; do
-    timeout 60s bash -c "http_with_session_id_session ${user}" >/dev/null 2>&1 &
-done
-
-for user in "${MYSQL_USERS[@]}"; do
-    timeout 60s bash -c "mysql_session ${user}" >/dev/null 2>&1 &
-done
-
-wait
-
-${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS"
-
-echo "sessions:" 
-${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user IN (${ALL_USERS_SQL_COLLECTION_STRING})"
-
-echo "port_0_sessions:" 
-${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user IN (${ALL_USERS_SQL_COLLECTION_STRING}) AND client_port = 0"
-
-echo "address_0_sessions:"
-${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user IN (${ALL_USERS_SQL_COLLECTION_STRING}) AND client_address = toIPv6('::')"
-
-echo "tcp_sessions"
-${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user IN (${TCP_USERS_SQL_COLLECTION_STRING}) AND interface = 'TCP'"
-echo "http_sessions"
-${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user IN (${HTTP_USERS_SQL_COLLECTION_STRING}) AND interface = 'HTTP'"
-echo "http_with_session_id_sessions"
-${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user IN (${HTTP_WITH_SESSION_ID_USERS_SQL_COLLECTION_STRING}) AND interface = 'HTTP'"
-echo "my_sql_sessions"
-${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user IN (${MYSQL_USERS_SQL_COLLECTION_STRING}) AND interface = 'MySQL'"
-
-for user in "${ALL_USERS[@]}"; do
-    ${CLICKHOUSE_CLIENT} -q "DROP USER ${user}"
-    echo "Corresponding LoginSuccess/Logout" 
-    ${CLICKHOUSE_CLIENT} -q "SELECT COUNT(*) FROM (SELECT ${SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = '${user}' AND type = 'LoginSuccess' INTERSECT SELECT ${SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = '${user}' AND type = 'Logout')"
-    echo "LoginFailure"
-    ${CLICKHOUSE_CLIENT} -q "SELECT COUNT(*) FROM system.session_log WHERE user = '${user}' AND type = 'LoginFailure'" 
- done
diff --git a/tests/queries/0_stateless/02834_remote_session_log.reference b/tests/queries/0_stateless/02834_remote_session_log.reference
deleted file mode 100644
index e2680982ab0..00000000000
--- a/tests/queries/0_stateless/02834_remote_session_log.reference
+++ /dev/null
@@ -1,13 +0,0 @@
-0
-0
-0
-0
-client_port 0 connections:
-0
-client_address '::' connections:
-0
-login failures:
-0
-TCP Login and logout count is equal
-HTTP Login and logout count is equal
-MySQL Login and logout count is equal
diff --git a/tests/queries/0_stateless/02834_remote_session_log.sh b/tests/queries/0_stateless/02834_remote_session_log.sh
deleted file mode 100755
index 64f20a70cb1..00000000000
--- a/tests/queries/0_stateless/02834_remote_session_log.sh
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/usr/bin/env bash
-# Tags: no-fasttest 
-
-CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# shellcheck source=../shell_config.sh
-. "$CURDIR"/../shell_config.sh
-
-readonly PID=$$
-readonly TEST_USER=$"02834_USER_${PID}"
-readonly SESSION_LOG_MATCHING_FIELDS="auth_id, auth_type, client_version_major, client_version_minor, client_version_patch, interface"
-
-${CLICKHOUSE_CLIENT} -q "CREATE USER IF NOT EXISTS ${TEST_USER} IDENTIFIED WITH plaintext_password BY 'pass'"
-${CLICKHOUSE_CLIENT} -q "GRANT SELECT ON INFORMATION_SCHEMA.* TO ${TEST_USER}"
-${CLICKHOUSE_CLIENT} -q "GRANT SELECT ON system.* TO ${TEST_USER}"
-${CLICKHOUSE_CLIENT} -q "GRANT CREATE TEMPORARY TABLE, MYSQL, REMOTE ON *.* TO ${TEST_USER}"
-
-${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS"
-${CLICKHOUSE_CLIENT} -q "DELETE FROM system.session_log WHERE user = '${TEST_USER}'"
-
-${CLICKHOUSE_CURL} -sS -X POST "${CLICKHOUSE_URL}&user=${TEST_USER}&password=pass" \
-   -d "SELECT * FROM remote('127.0.0.1:${CLICKHOUSE_PORT_TCP}', 'system', 'one', '${TEST_USER}', 'pass')"
-
-${CLICKHOUSE_CURL} -sS -X POST "${CLICKHOUSE_URL}&user=${TEST_USER}&password=pass" \
-    -d "SELECT * FROM mysql('127.0.0.1:9004', 'system', 'one', '${TEST_USER}', 'pass')"
-
-${CLICKHOUSE_CLIENT} -q "SELECT * FROM remote('127.0.0.1:${CLICKHOUSE_PORT_TCP}', 'system', 'one', '${TEST_USER}', 'pass')" -u "${TEST_USER}" --password "pass"
-${CLICKHOUSE_CLIENT} -q "SELECT * FROM mysql('127.0.0.1:9004', 'system', 'one', '${TEST_USER}', 'pass')" -u "${TEST_USER}" --password "pass"
-
-${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS"
-
-echo "client_port 0 connections:"
-${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user = '${TEST_USER}' and client_port = 0"
-
-echo "client_address '::' connections:"
-${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user = '${TEST_USER}' and client_address = toIPv6('::')"
-
-echo "login failures:"
-${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user = '${TEST_USER}' and type = 'LoginFailure'"
-
-# remote(...) function sometimes reuses old cached sessions for query execution.
-# This makes LoginSuccess/Logout entries count unstable, but success and logouts must always match.
-
-for interface in 'TCP' 'HTTP' 'MySQL'
-do
-    LOGIN_COUNT=`${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user = '${TEST_USER}' AND type = 'LoginSuccess' AND interface = '${interface}'"`
-    CORRESPONDING_LOGOUT_RECORDS_COUNT=`${CLICKHOUSE_CLIENT} -q "SELECT COUNT(*) FROM (SELECT ${SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = '${TEST_USER}' AND type = 'LoginSuccess' AND interface = '${interface}' INTERSECT SELECT ${SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = '${TEST_USER}' AND type = 'Logout' AND interface = '${interface}')"`
-
-    if [ "$LOGIN_COUNT" == "$CORRESPONDING_LOGOUT_RECORDS_COUNT" ]; then
-        echo "${interface} Login and logout count is equal"
-    else
-        TOTAL_LOGOUT_COUNT=`${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user = '${TEST_USER}' AND type = 'Logout' AND interface = '${interface}'"`
-        echo "${interface} Login count ${LOGIN_COUNT} != corresponding logout count ${CORRESPONDING_LOGOUT_RECORDS_COUNT}. TOTAL_LOGOUT_COUNT ${TOTAL_LOGOUT_COUNT}"
-    fi
-done
-
-${CLICKHOUSE_CLIENT} -q "DROP USER ${TEST_USER}"
diff --git a/tests/queries/0_stateless/02835_drop_user_during_session.reference b/tests/queries/0_stateless/02835_drop_user_during_session.reference
deleted file mode 100644
index 7252faab8c6..00000000000
--- a/tests/queries/0_stateless/02835_drop_user_during_session.reference
+++ /dev/null
@@ -1,8 +0,0 @@
-port_0_sessions:
-0
-address_0_sessions:
-0
-Corresponding LoginSuccess/Logout
-9
-LoginFailure
-0
diff --git a/tests/queries/0_stateless/02835_drop_user_during_session.sh b/tests/queries/0_stateless/02835_drop_user_during_session.sh
deleted file mode 100755
index e848e1fe90d..00000000000
--- a/tests/queries/0_stateless/02835_drop_user_during_session.sh
+++ /dev/null
@@ -1,113 +0,0 @@
-#!/usr/bin/env bash
-
-CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# shellcheck source=../shell_config.sh
-. "$CURDIR"/../shell_config.sh
-
-readonly PID=$$
-
-readonly TEST_USER="02835_USER_${PID}"
-readonly TEST_ROLE="02835_ROLE_${PID}"
-readonly TEST_PROFILE="02835_PROFILE_${PID}"
-readonly SESSION_LOG_MATCHING_FIELDS="auth_id, auth_type, client_version_major, client_version_minor, client_version_patch, interface"
-
-function tcp_session()
-{
-    local user=$1
-    ${CLICKHOUSE_CLIENT} -q "SELECT COUNT(*) FROM system.numbers" --user="${user}"
-}
-
-function http_session()
-{
-    local user=$1
-    ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&user=${user}&password=pass" -d "SELECT COUNT(*) FROM system.numbers"
-}
-
-function http_with_session_id_session() 
-{
-    local user=$1
-    ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&user=${user}&password=pass" -d "SELECT COUNT(*) FROM system.numbers"
-}
-
-# Busy-waits until user $1, specified amount of queries ($2) will run simultaneously.
-function wait_for_queries_start()
-{
-    local user=$1
-    local queries_count=$2
-    # 10 seconds waiting
-    counter=0 retries=100
-    while [[ $counter -lt $retries ]]; do
-        result=$($CLICKHOUSE_CLIENT --query "SELECT COUNT(*) FROM system.processes WHERE user = '${user}'")
-        if [[ $result == "${queries_count}" ]]; then
-            break;
-        fi
-        sleep 0.1
-        ((++counter))
-    done
-}
-
-${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS"
-${CLICKHOUSE_CLIENT} -q "DELETE FROM system.session_log WHERE user = '${TEST_USER}'"
-
-# DROP USE CASE
-${CLICKHOUSE_CLIENT} -q "CREATE USER IF NOT EXISTS ${TEST_USER}"
-${CLICKHOUSE_CLIENT} -q "GRANT SELECT ON system.numbers TO ${TEST_USER}"
-
-export -f tcp_session;
-export -f http_session;
-export -f http_with_session_id_session;
-
-timeout 10s bash -c "tcp_session ${TEST_USER}" >/dev/null 2>&1 &
-timeout 10s bash -c "http_session ${TEST_USER}" >/dev/null 2>&1 &
-timeout 10s bash -c "http_with_session_id_session ${TEST_USER}" >/dev/null 2>&1 &
-
-wait_for_queries_start $TEST_USER 3
-${CLICKHOUSE_CLIENT} -q "DROP USER ${TEST_USER}"
-${CLICKHOUSE_CLIENT} -q "KILL QUERY WHERE user = '${TEST_USER}' SYNC" >/dev/null &
-
-wait
-
-# DROP ROLE CASE
-${CLICKHOUSE_CLIENT} -q "CREATE ROLE IF NOT EXISTS ${TEST_ROLE}"
-${CLICKHOUSE_CLIENT} -q "CREATE USER ${TEST_USER} DEFAULT ROLE ${TEST_ROLE}"
-${CLICKHOUSE_CLIENT} -q "GRANT SELECT ON system.numbers TO ${TEST_USER}"
-
-timeout 10s bash -c "tcp_session ${TEST_USER}" >/dev/null 2>&1 &
-timeout 10s bash -c "http_session ${TEST_USER}" >/dev/null 2>&1 &
-timeout 10s bash -c "http_with_session_id_session ${TEST_USER}" >/dev/null 2>&1 &
-
-wait_for_queries_start $TEST_USER 3
-${CLICKHOUSE_CLIENT} -q "DROP ROLE ${TEST_ROLE}"
-${CLICKHOUSE_CLIENT} -q "DROP USER ${TEST_USER}"
-
-${CLICKHOUSE_CLIENT} -q "KILL QUERY WHERE user = '${TEST_USER}' SYNC" >/dev/null &
-
-wait
-
-# DROP PROFILE CASE
-${CLICKHOUSE_CLIENT} -q "CREATE SETTINGS PROFILE IF NOT EXISTS '${TEST_PROFILE}'"
-${CLICKHOUSE_CLIENT} -q "CREATE USER ${TEST_USER} SETTINGS PROFILE '${TEST_PROFILE}'"
-${CLICKHOUSE_CLIENT} -q "GRANT SELECT ON system.numbers TO ${TEST_USER}"
-
-timeout 10s bash -c "tcp_session ${TEST_USER}" >/dev/null 2>&1 &
-timeout 10s bash -c "http_session ${TEST_USER}" >/dev/null 2>&1 &
-timeout 10s bash -c "http_with_session_id_session ${TEST_USER}" >/dev/null 2>&1 &
-
-wait_for_queries_start $TEST_USER 3
-${CLICKHOUSE_CLIENT} -q "DROP SETTINGS PROFILE '${TEST_PROFILE}'"
-${CLICKHOUSE_CLIENT} -q "DROP USER ${TEST_USER}"
-
-${CLICKHOUSE_CLIENT} -q "KILL QUERY WHERE user = '${TEST_USER}' SYNC" >/dev/null &
-
-wait
-
-${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS"
-
-echo "port_0_sessions:" 
-${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user = '${TEST_USER}' AND client_port = 0"
-echo "address_0_sessions:"
-${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user = '${TEST_USER}' AND client_address = toIPv6('::')"
-echo "Corresponding LoginSuccess/Logout" 
-${CLICKHOUSE_CLIENT} -q "SELECT COUNT(*) FROM (SELECT ${SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = '${TEST_USER}' AND type = 'LoginSuccess' INTERSECT SELECT ${SESSION_LOG_MATCHING_FIELDS}, FROM system.session_log WHERE user = '${TEST_USER}' AND type = 'Logout')"
-echo "LoginFailure"
-${CLICKHOUSE_CLIENT} -q "SELECT COUNT(*) FROM system.session_log WHERE user = '${TEST_USER}' AND type = 'LoginFailure'" 

From 6edf4a777f33a84db9c18b08741207251fa5fc3d Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 10 Aug 2023 03:26:11 +0200
Subject: [PATCH 166/249] Remove unrecognizable garbage from the performance
 test

---
 docker/test/performance-comparison/compare.sh | 64 ++++++-------------
 1 file changed, 21 insertions(+), 43 deletions(-)

diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh
index 798d2a40b12..c0422816787 100755
--- a/docker/test/performance-comparison/compare.sh
+++ b/docker/test/performance-comparison/compare.sh
@@ -666,8 +666,8 @@ create view partial_query_times as select * from
 -- queries with new functions added in the tested PR).
 create table partial_queries_report engine File(TSV, 'report/partial-queries-report.tsv')
     settings output_format_decimal_trailing_zeros = 1
-    as select toDecimal64(time_median, 3) time,
-        toDecimal64(time_stddev / time_median, 3) relative_time_stddev,
+    as select round(time_median, 3) time,
+        round(time_stddev / time_median, 3) relative_time_stddev,
         test, query_index, query_display_name
     from partial_query_times
     join query_display_names using (test, query_index)
@@ -746,21 +746,21 @@ create table changed_perf_report engine File(TSV, 'report/changed-perf.tsv')
         -- of NaN to decimal.
         (left > right ? left / right : right / left) as times_change_float,
         isFinite(times_change_float) as times_change_finite,
-        toDecimal64(times_change_finite ? times_change_float : 1., 3) as times_change_decimal,
+        round(times_change_finite ? times_change_float : 1., 3) as times_change_decimal,
         times_change_finite
             ? (left > right ? '-' : '+') || toString(times_change_decimal) || 'x'
             : '--' as times_change_str
     select
-        toDecimal64(left, 3), toDecimal64(right, 3), times_change_str,
-        toDecimal64(diff, 3), toDecimal64(stat_threshold, 3),
+        round(left, 3), round(right, 3), times_change_str,
+        round(diff, 3), round(stat_threshold, 3),
         changed_fail, test, query_index, query_display_name
     from queries where changed_show order by abs(diff) desc;
 
 create table unstable_queries_report engine File(TSV, 'report/unstable-queries.tsv')
     settings output_format_decimal_trailing_zeros = 1
     as select
-        toDecimal64(left, 3), toDecimal64(right, 3), toDecimal64(diff, 3),
-        toDecimal64(stat_threshold, 3), unstable_fail, test, query_index, query_display_name
+        round(left, 3), round(right, 3), round(diff, 3),
+        round(stat_threshold, 3), unstable_fail, test, query_index, query_display_name
     from queries where unstable_show order by stat_threshold desc;
 
 
@@ -792,8 +792,8 @@ create table test_perf_changes_report engine File(TSV, 'report/test-perf-changes
     settings output_format_decimal_trailing_zeros = 1
     as with
         (times_speedup >= 1
-            ? '-' || toString(toDecimal64(times_speedup, 3)) || 'x'
-            : '+' || toString(toDecimal64(1 / times_speedup, 3)) || 'x')
+            ? '-' || toString(round(times_speedup, 3)) || 'x'
+            : '+' || toString(round(1 / times_speedup, 3)) || 'x')
         as times_speedup_str
     select test, times_speedup_str, queries, bad, changed, unstable
     -- Not sure what's the precedence of UNION ALL vs WHERE & ORDER BY, hence all
@@ -818,10 +818,10 @@ create view total_client_time_per_query as select *
 
 create table slow_on_client_report engine File(TSV, 'report/slow-on-client.tsv')
     settings output_format_decimal_trailing_zeros = 1
-    as select client, server, toDecimal64(client/server, 3) p,
+    as select client, server, round(client/server, 3) p,
         test, query_display_name
     from total_client_time_per_query left join query_display_names using (test, query_index)
-    where p > toDecimal64(1.02, 3) order by p desc;
+    where p > round(1.02, 3) order by p desc;
 
 create table wall_clock_time_per_test engine Memory as select *
     from file('wall-clock-times.tsv', TSV, 'test text, real float, user float, system float');
@@ -902,12 +902,12 @@ create table test_times_report engine File(TSV, 'report/test-times.tsv')
     settings output_format_decimal_trailing_zeros = 1
     as select
         test,
-        toDecimal64(real, 3),
-        toDecimal64(total_client_time, 3),
+        round(real, 3),
+        round(total_client_time, 3),
         queries,
-        toDecimal64(query_max, 3),
-        toDecimal64(avg_real_per_query, 3),
-        toDecimal64(query_min, 3),
+        round(query_max, 3),
+        round(avg_real_per_query, 3),
+        round(query_min, 3),
         runs
     from (
         select * from test_times_view
@@ -926,14 +926,14 @@ create table all_tests_report engine File(TSV, 'report/all-queries.tsv')
         -- of NaN to decimal.
         (left > right ? left / right : right / left) as times_change_float,
         isFinite(times_change_float) as times_change_finite,
-        toDecimal64(times_change_finite ? times_change_float : 1., 3) as times_change_decimal,
+        round(times_change_finite ? times_change_float : 1., 3) as times_change_decimal,
         times_change_finite
             ? (left > right ? '-' : '+') || toString(times_change_decimal) || 'x'
             : '--' as times_change_str
     select changed_fail, unstable_fail,
-        toDecimal64(left, 3), toDecimal64(right, 3), times_change_str,
-        toDecimal64(isFinite(diff) ? diff : 0, 3),
-        toDecimal64(isFinite(stat_threshold) ? stat_threshold : 0, 3),
+        round(left, 3), round(right, 3), times_change_str,
+        round(isFinite(diff) ? diff : 0, 3),
+        round(isFinite(stat_threshold) ? stat_threshold : 0, 3),
         test, query_index, query_display_name
     from queries order by test, query_index;
 
@@ -1044,27 +1044,6 @@ create table unstable_run_traces engine File(TSVWithNamesAndTypes,
     order by count() desc
     ;
 
-create table metric_devation engine File(TSVWithNamesAndTypes,
-        'report/metric-deviation.$version.tsv')
-    settings output_format_decimal_trailing_zeros = 1
-    -- first goes the key used to split the file with grep
-    as select test, query_index, query_display_name,
-        toDecimal64(d, 3) d, q, metric
-    from (
-        select
-            test, query_index,
-            (q[3] - q[1])/q[2] d,
-            quantilesExact(0, 0.5, 1)(value) q, metric
-        from (select * from unstable_run_metrics
-            union all select * from unstable_run_traces
-            union all select * from unstable_run_metrics_2) mm
-        group by test, query_index, metric
-        having isFinite(d) and d > 0.5 and q[3] > 5
-    ) metrics
-    left join query_display_names using (test, query_index)
-    order by test, query_index, d desc
-    ;
-
 create table stacks engine File(TSV, 'report/stacks.$version.tsv') as
     select
         -- first goes the key used to split the file with grep
@@ -1175,7 +1154,7 @@ create table metrics engine File(TSV, 'metrics/metrics.tsv') as
 create table changes engine File(TSV, 'metrics/changes.tsv')
     settings output_format_decimal_trailing_zeros = 1
     as select metric, left, right,
-        toDecimal64(diff, 3), toDecimal64(times_diff, 3)
+        round(diff, 3), round(times_diff, 3)
     from (
         select metric, median(left) as left, median(right) as right,
             (right - left) / left diff,
@@ -1226,7 +1205,6 @@ create table ci_checks engine File(TSVWithNamesAndTypes, 'ci-checks.tsv')
         '$SHA_TO_TEST' :: LowCardinality(String) AS commit_sha,
         '${CLICKHOUSE_PERFORMANCE_COMPARISON_CHECK_NAME:-Performance}' :: LowCardinality(String) AS check_name,
         '$(sed -n 's/.*<!--status: \(.*\)-->/\1/p' report.html)' :: LowCardinality(String) AS check_status,
-        -- TODO toDateTime() can't parse output of 'date', so no time for now.
         (($(date +%s) - $CHPC_CHECK_START_TIMESTAMP) * 1000) :: UInt64 AS check_duration_ms,
         fromUnixTimestamp($CHPC_CHECK_START_TIMESTAMP) check_start_time,
         test_name :: LowCardinality(String) AS test_name ,

From f2621e70aed39f2bf1a061b4820714f84bf7324a Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 10 Aug 2023 03:27:07 +0200
Subject: [PATCH 167/249] Remove unrecognizable garbage from the performance
 test

---
 docker/test/performance-comparison/compare.sh | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh
index c0422816787..f949e66ab17 100755
--- a/docker/test/performance-comparison/compare.sh
+++ b/docker/test/performance-comparison/compare.sh
@@ -665,7 +665,6 @@ create view partial_query_times as select * from
 -- Report for backward-incompatible ('partial') queries that we could only run on the new server (e.g.
 -- queries with new functions added in the tested PR).
 create table partial_queries_report engine File(TSV, 'report/partial-queries-report.tsv')
-    settings output_format_decimal_trailing_zeros = 1
     as select round(time_median, 3) time,
         round(time_stddev / time_median, 3) relative_time_stddev,
         test, query_index, query_display_name
@@ -739,7 +738,6 @@ create table queries engine File(TSVWithNamesAndTypes, 'report/queries.tsv')
     ;
 
 create table changed_perf_report engine File(TSV, 'report/changed-perf.tsv')
-    settings output_format_decimal_trailing_zeros = 1
     as with
         -- server_time is sometimes reported as zero (if it's less than 1 ms),
         -- so we have to work around this to not get an error about conversion
@@ -757,7 +755,6 @@ create table changed_perf_report engine File(TSV, 'report/changed-perf.tsv')
     from queries where changed_show order by abs(diff) desc;
 
 create table unstable_queries_report engine File(TSV, 'report/unstable-queries.tsv')
-    settings output_format_decimal_trailing_zeros = 1
     as select
         round(left, 3), round(right, 3), round(diff, 3),
         round(stat_threshold, 3), unstable_fail, test, query_index, query_display_name
@@ -789,7 +786,6 @@ create view total_speedup as
     ;
 
 create table test_perf_changes_report engine File(TSV, 'report/test-perf-changes.tsv')
-    settings output_format_decimal_trailing_zeros = 1
     as with
         (times_speedup >= 1
             ? '-' || toString(round(times_speedup, 3)) || 'x'
@@ -817,7 +813,6 @@ create view total_client_time_per_query as select *
         'test text, query_index int, client float, server float');
 
 create table slow_on_client_report engine File(TSV, 'report/slow-on-client.tsv')
-    settings output_format_decimal_trailing_zeros = 1
     as select client, server, round(client/server, 3) p,
         test, query_display_name
     from total_client_time_per_query left join query_display_names using (test, query_index)
@@ -899,7 +894,6 @@ create view test_times_view_total as
     ;
 
 create table test_times_report engine File(TSV, 'report/test-times.tsv')
-    settings output_format_decimal_trailing_zeros = 1
     as select
         test,
         round(real, 3),
@@ -919,7 +913,6 @@ create table test_times_report engine File(TSV, 'report/test-times.tsv')
 
 -- report for all queries page, only main metric
 create table all_tests_report engine File(TSV, 'report/all-queries.tsv')
-    settings output_format_decimal_trailing_zeros = 1
     as with
         -- server_time is sometimes reported as zero (if it's less than 1 ms),
         -- so we have to work around this to not get an error about conversion
@@ -1152,7 +1145,6 @@ create table metrics engine File(TSV, 'metrics/metrics.tsv') as
 
 -- Show metrics that have changed
 create table changes engine File(TSV, 'metrics/changes.tsv')
-    settings output_format_decimal_trailing_zeros = 1
     as select metric, left, right,
         round(diff, 3), round(times_diff, 3)
     from (

From 3c6d140c179c29ce890f190529e29d1e9ec3db8b Mon Sep 17 00:00:00 2001
From: Nikolay Degterinsky <evillique@gmail.com>
Date: Thu, 10 Aug 2023 01:31:59 +0000
Subject: [PATCH 168/249] Fix sigle quote escaping in PostgreSQL engine

---
 src/Storages/StoragePostgreSQL.cpp                |  5 +++++
 tests/integration/test_storage_postgresql/test.py | 14 ++++++++++++++
 2 files changed, 19 insertions(+)

diff --git a/src/Storages/StoragePostgreSQL.cpp b/src/Storages/StoragePostgreSQL.cpp
index 3551ee36819..11558b39ad3 100644
--- a/src/Storages/StoragePostgreSQL.cpp
+++ b/src/Storages/StoragePostgreSQL.cpp
@@ -45,6 +45,7 @@
 
 #include <Databases/PostgreSQL/fetchPostgreSQLTableStructure.h>
 
+#include <boost/algorithm/string/replace.hpp>
 
 namespace DB
 {
@@ -123,6 +124,10 @@ Pipe StoragePostgreSQL::read(
         column_names_,
         storage_snapshot->metadata->getColumns().getOrdinary(),
         IdentifierQuotingStyle::DoubleQuotes, remote_table_schema, remote_table_name, context_);
+
+    /// Single quotes in PostgreSQL are escaped through repetition
+    boost::replace_all(query, "\\'", "''");
+
     LOG_TRACE(log, "Query: {}", query);
 
     Block sample_block;
diff --git a/tests/integration/test_storage_postgresql/test.py b/tests/integration/test_storage_postgresql/test.py
index 686eb1ea751..3a36d050f17 100644
--- a/tests/integration/test_storage_postgresql/test.py
+++ b/tests/integration/test_storage_postgresql/test.py
@@ -726,6 +726,20 @@ def test_auto_close_connection(started_cluster):
     assert count == 2
 
 
+def test_single_quotes(started_cluster):
+    cursor = started_cluster.postgres_conn.cursor()
+    cursor.execute(f"DROP TABLE IF EXISTS single_quote_fails")
+    cursor.execute(f"CREATE TABLE single_quote_fails(text varchar(255))")
+    node1.query(
+        "CREATE TABLE default.single_quote_fails (text String) ENGINE = PostgreSQL('postgres1:5432', 'postgres', 'single_quote_fails', 'postgres', 'mysecretpassword')"
+    )
+    node1.query("SELECT * FROM single_quote_fails WHERE text = ''''")
+    node1.query("SELECT * FROM single_quote_fails WHERE text = '\\''")
+    node1.query("SELECT * FROM single_quote_fails WHERE text like '%a''a%'")
+    node1.query("SELECT * FROM single_quote_fails WHERE text like '%a\\'a%'")
+    cursor.execute(f"DROP TABLE single_quote_fails")
+
+
 if __name__ == "__main__":
     cluster.start()
     input("Cluster created, press any key to destroy...")

From 4ba5273939804ed2f639cc4748eaaa7f3b9d4af2 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 10 Aug 2023 03:56:27 +0200
Subject: [PATCH 169/249] Disable randomization in `02273_full_sort_join`

---
 tests/queries/0_stateless/02273_full_sort_join.sql.j2 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02273_full_sort_join.sql.j2 b/tests/queries/0_stateless/02273_full_sort_join.sql.j2
index 6500306356c..2a13e11e116 100644
--- a/tests/queries/0_stateless/02273_full_sort_join.sql.j2
+++ b/tests/queries/0_stateless/02273_full_sort_join.sql.j2
@@ -1,4 +1,4 @@
--- Tags: long
+-- Tags: long, no-random-settings
 
 DROP TABLE IF EXISTS t1;
 DROP TABLE IF EXISTS t2;

From 0b138fdf16eeeebef564a20928e83e6a7b1d7122 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 10 Aug 2023 04:15:19 +0200
Subject: [PATCH 170/249] Remove outdated Dockerfile

---
 docker/client/Dockerfile | 34 ----------------------------------
 docker/client/README.md  |  7 -------
 2 files changed, 41 deletions(-)
 delete mode 100644 docker/client/Dockerfile
 delete mode 100644 docker/client/README.md

diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile
deleted file mode 100644
index 1c185daec75..00000000000
--- a/docker/client/Dockerfile
+++ /dev/null
@@ -1,34 +0,0 @@
-FROM ubuntu:18.04
-
-# ARG for quick switch to a given ubuntu mirror
-ARG apt_archive="http://archive.ubuntu.com"
-RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
-
-ARG repository="deb https://repo.clickhouse.com/deb/stable/ main/"
-ARG version=22.1.1.*
-
-RUN apt-get update \
-    && apt-get install --yes --no-install-recommends \
-        apt-transport-https \
-        ca-certificates \
-        dirmngr \
-        gnupg \
-    && mkdir -p /etc/apt/sources.list.d \
-    && apt-key adv --keyserver keyserver.ubuntu.com --recv E0C56BD4 \
-    && echo $repository > /etc/apt/sources.list.d/clickhouse.list \
-    && apt-get update \
-    && env DEBIAN_FRONTEND=noninteractive \
-        apt-get install --allow-unauthenticated --yes --no-install-recommends \
-            clickhouse-client=$version \
-            clickhouse-common-static=$version \
-            locales \
-            tzdata \
-    && rm -rf /var/lib/apt/lists/* /var/cache/debconf \
-    && apt-get clean
-
-RUN locale-gen en_US.UTF-8
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8
-
-ENTRYPOINT ["/usr/bin/clickhouse-client"]
diff --git a/docker/client/README.md b/docker/client/README.md
deleted file mode 100644
index bbcc7d60794..00000000000
--- a/docker/client/README.md
+++ /dev/null
@@ -1,7 +0,0 @@
-# ClickHouse Client Docker Image
-
-For more information see [ClickHouse Server Docker Image](https://hub.docker.com/r/clickhouse/clickhouse-server/).
-
-## License
-
-View [license information](https://github.com/ClickHouse/ClickHouse/blob/master/LICENSE) for the software contained in this image.

From fc3970e3ff46d6795ac811d51edb29c5b6128a2d Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 10 Aug 2023 04:30:03 +0200
Subject: [PATCH 171/249] Update readme

---
 docker/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/README.md b/docker/README.md
index ec52ddd143e..bae02201add 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -1,5 +1,5 @@
 ## ClickHouse Dockerfiles
 
-This directory contain Dockerfiles for `clickhouse-client` and `clickhouse-server`. They are updated in each release.
+This directory contain Dockerfiles for `clickhouse-server`. They are updated in each release.
 
 Also there is bunch of images for testing and CI. They are listed in `images.json` file and updated on each commit to master. If you need to add another image, place information about it into `images.json`.

From 2a89587300a408610591cf251546a6c59affc034 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 10 Aug 2023 04:32:39 +0200
Subject: [PATCH 172/249] Address review comments

---
 docker/test/stateless/run.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh
index 290222c9d26..0ef25741b18 100755
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@@ -109,8 +109,8 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]
 fi
 
 
-# Wait for the server to start
-while true
+# Wait for the server to start, but not for too long.
+for _ in {1..100}
 do
     clickhouse-client --query "SELECT 1" && break
     sleep 1

From 8fba4d2c4a0bf6b9f9061bf19980f98158ef6846 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 10 Aug 2023 04:41:19 +0200
Subject: [PATCH 173/249] Protection if the repository is old

---
 docker/test/stateless/run.sh | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh
index 0ef25741b18..6c930fa583b 100755
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@@ -117,7 +117,7 @@ do
 done
 
 # Initialize export of system logs to ClickHouse Cloud
-if [ -n "${CLICKHOUSE_CI_LOGS_HOST}" ]
+if [ -n "${CLICKHOUSE_CI_LOGS_HOST}" -a -f /repository/utils/export-logs-in-ci/setup.sh ]
 then
     export EXTRA_COLUMNS_EXPRESSION="$PULL_REQUEST_NUMBER AS pull_request_number, '$COMMIT_SHA' AS commit_sha, '$CHECK_START_TIME' AS check_start_time, '$CHECK_NAME' AS check_name, '$INSTANCE_TYPE' AS instance_type"
     # TODO: Check if the password will appear in the logs.
@@ -131,9 +131,6 @@ then
     export CLICKHOUSE_CI_LOGS_PASSWORD=''
 fi
 
-# TODO what is this? Remove it.
-sleep 5
-
 attach_gdb_to_clickhouse || true  # FIXME: to not break old builds, clean on 2023-09-01
 
 function fn_exists() {

From 0ba3fea7fefa1237434e1cca36b8d8ac11fb73e5 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 10 Aug 2023 04:44:20 +0200
Subject: [PATCH 174/249] Apply recommendations

---
 docker/test/stateless/Dockerfile           | 2 ++
 docker/test/stateless/run.sh               | 4 ++--
 docker/test/stateless/setup_export_logs.sh | 1 +
 3 files changed, 5 insertions(+), 2 deletions(-)
 create mode 120000 docker/test/stateless/setup_export_logs.sh

diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile
index 86ca7924f74..48d26c84ed8 100644
--- a/docker/test/stateless/Dockerfile
+++ b/docker/test/stateless/Dockerfile
@@ -87,4 +87,6 @@ RUN npm install -g azurite \
 COPY run.sh /
 COPY setup_minio.sh /
 COPY setup_hdfs_minicluster.sh /
+COPY setup_export_logs.sh /
+
 CMD ["/bin/bash", "/run.sh"]
diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh
index 6c930fa583b..015ba7d7423 100755
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@@ -117,13 +117,13 @@ do
 done
 
 # Initialize export of system logs to ClickHouse Cloud
-if [ -n "${CLICKHOUSE_CI_LOGS_HOST}" -a -f /repository/utils/export-logs-in-ci/setup.sh ]
+if [ -n "${CLICKHOUSE_CI_LOGS_HOST}" ]
 then
     export EXTRA_COLUMNS_EXPRESSION="$PULL_REQUEST_NUMBER AS pull_request_number, '$COMMIT_SHA' AS commit_sha, '$CHECK_START_TIME' AS check_start_time, '$CHECK_NAME' AS check_name, '$INSTANCE_TYPE' AS instance_type"
     # TODO: Check if the password will appear in the logs.
     export CONNECTION_PARAMETERS="--secure --user ci --host ${CLICKHOUSE_CI_LOGS_HOST} --password ${CLICKHOUSE_CI_LOGS_PASSWORD}"
 
-    /repository/utils/export-logs-in-ci/setup.sh
+    ./setup_export_logs.sh
 
     # Unset variables after use
     export CONNECTION_PARAMETERS=''
diff --git a/docker/test/stateless/setup_export_logs.sh b/docker/test/stateless/setup_export_logs.sh
new file mode 120000
index 00000000000..b6e2f981cd4
--- /dev/null
+++ b/docker/test/stateless/setup_export_logs.sh
@@ -0,0 +1 @@
+../../../utils/export-logs-in-ci/setup.sh
\ No newline at end of file

From e4fbca2e2e7e21229e2a68254570c7effeaddbf9 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 10 Aug 2023 04:47:44 +0200
Subject: [PATCH 175/249] Apply review recommendations

---
 tests/ci/functional_test_check.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py
index fdea5be9a6c..c84884bcfaf 100644
--- a/tests/ci/functional_test_check.py
+++ b/tests/ci/functional_test_check.py
@@ -130,7 +130,6 @@ def get_run_command(
     return (
         f"docker run --volume={builds_path}:/package_folder "
         f"--volume={repo_path}/tests:/usr/share/clickhouse-test "
-        f"--volume={repo_path}:/repository "
         f"{volume_with_broken_test} "
         f"--volume={result_path}:/test_output --volume={server_log_path}:/var/log/clickhouse-server "
         f"--cap-add=SYS_PTRACE {env_str} {additional_options_str} {image}"

From 931d75ec571882c86374b5a2cdfa7c4b1fe07012 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 10 Aug 2023 05:05:17 +0200
Subject: [PATCH 176/249] Maybe better

---
 src/Core/Field.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Core/Field.cpp b/src/Core/Field.cpp
index 89faaed8a72..de5f7ba9c92 100644
--- a/src/Core/Field.cpp
+++ b/src/Core/Field.cpp
@@ -9,7 +9,7 @@
 #include <Common/FieldVisitorDump.h>
 #include <Common/FieldVisitorToString.h>
 #include <Common/FieldVisitorWriteBinary.h>
-#include <base/EnumReflection.h>
+#include <magic_enum.hpp>
 
 
 namespace DB

From 7a6d438b88c03004c105c614749fc7cbab69da42 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 10 Aug 2023 05:11:37 +0200
Subject: [PATCH 177/249] Remove magic_enum

---
 src/Core/Field.cpp | 64 ++++++++++++++++++++++++----------------------
 src/Core/Field.h   |  2 +-
 2 files changed, 34 insertions(+), 32 deletions(-)

diff --git a/src/Core/Field.cpp b/src/Core/Field.cpp
index de5f7ba9c92..1fcf663a744 100644
--- a/src/Core/Field.cpp
+++ b/src/Core/Field.cpp
@@ -9,9 +9,10 @@
 #include <Common/FieldVisitorDump.h>
 #include <Common/FieldVisitorToString.h>
 #include <Common/FieldVisitorWriteBinary.h>
-#include <magic_enum.hpp>
 
 
+using namespace std::literals;
+
 namespace DB
 {
 
@@ -21,12 +22,6 @@ namespace ErrorCodes
     extern const int DECIMAL_OVERFLOW;
 }
 
-/// Keep in mind, that "magic_enum" is very expensive for compiler.
-std::string_view Field::getTypeName() const
-{
-    return magic_enum::enum_name(which);
-}
-
 inline Field getBinaryValue(UInt8 type, ReadBuffer & buf)
 {
     switch (type)
@@ -590,34 +585,41 @@ String toString(const Field & x)
         x);
 }
 
-String fieldTypeToString(Field::Types::Which type)
+std::string_view fieldTypeToString(Field::Types::Which type)
 {
     switch (type)
     {
-        case Field::Types::Which::Null: return "Null";
-        case Field::Types::Which::Array: return "Array";
-        case Field::Types::Which::Tuple: return "Tuple";
-        case Field::Types::Which::Map: return "Map";
-        case Field::Types::Which::Object: return "Object";
-        case Field::Types::Which::AggregateFunctionState: return "AggregateFunctionState";
-        case Field::Types::Which::Bool: return "Bool";
-        case Field::Types::Which::String: return "String";
-        case Field::Types::Which::Decimal32: return "Decimal32";
-        case Field::Types::Which::Decimal64: return "Decimal64";
-        case Field::Types::Which::Decimal128: return "Decimal128";
-        case Field::Types::Which::Decimal256: return "Decimal256";
-        case Field::Types::Which::Float64: return "Float64";
-        case Field::Types::Which::Int64: return "Int64";
-        case Field::Types::Which::Int128: return "Int128";
-        case Field::Types::Which::Int256: return "Int256";
-        case Field::Types::Which::UInt64: return "UInt64";
-        case Field::Types::Which::UInt128: return "UInt128";
-        case Field::Types::Which::UInt256: return "UInt256";
-        case Field::Types::Which::UUID: return "UUID";
-        case Field::Types::Which::IPv4: return "IPv4";
-        case Field::Types::Which::IPv6: return "IPv6";
-        case Field::Types::Which::CustomType: return "CustomType";
+        case Field::Types::Which::Null: return "Null"sv;
+        case Field::Types::Which::Array: return "Array"sv;
+        case Field::Types::Which::Tuple: return "Tuple"sv;
+        case Field::Types::Which::Map: return "Map"sv;
+        case Field::Types::Which::Object: return "Object"sv;
+        case Field::Types::Which::AggregateFunctionState: return "AggregateFunctionState"sv;
+        case Field::Types::Which::Bool: return "Bool"sv;
+        case Field::Types::Which::String: return "String"sv;
+        case Field::Types::Which::Decimal32: return "Decimal32"sv;
+        case Field::Types::Which::Decimal64: return "Decimal64"sv;
+        case Field::Types::Which::Decimal128: return "Decimal128"sv;
+        case Field::Types::Which::Decimal256: return "Decimal256"sv;
+        case Field::Types::Which::Float64: return "Float64"sv;
+        case Field::Types::Which::Int64: return "Int64"sv;
+        case Field::Types::Which::Int128: return "Int128"sv;
+        case Field::Types::Which::Int256: return "Int256"sv;
+        case Field::Types::Which::UInt64: return "UInt64"sv;
+        case Field::Types::Which::UInt128: return "UInt128"sv;
+        case Field::Types::Which::UInt256: return "UInt256"sv;
+        case Field::Types::Which::UUID: return "UUID"sv;
+        case Field::Types::Which::IPv4: return "IPv4"sv;
+        case Field::Types::Which::IPv6: return "IPv6"sv;
+        case Field::Types::Which::CustomType: return "CustomType"sv;
     }
 }
 
+/// Keep in mind, that "magic_enum" is very expensive for compiler, that's why we don't use it.
+std::string_view Field::getTypeName() const
+{
+    return fieldTypeToString(which);
+}
+
+
 }
diff --git a/src/Core/Field.h b/src/Core/Field.h
index 6666e66e8d5..12542ca0bf1 100644
--- a/src/Core/Field.h
+++ b/src/Core/Field.h
@@ -1004,7 +1004,7 @@ void writeFieldText(const Field & x, WriteBuffer & buf);
 
 String toString(const Field & x);
 
-String fieldTypeToString(Field::Types::Which type);
+std::string_view fieldTypeToString(Field::Types::Which type);
 
 }
 

From b8be7eef044052a24d0c00873d150d8de3a5ea4f Mon Sep 17 00:00:00 2001
From: Alexey Gerasimchuck <a.gerasimchuck@arenadata.io>
Date: Thu, 10 Aug 2023 03:45:59 +0000
Subject: [PATCH 178/249] Added new tests for session_log and bugfix

---
 src/Interpreters/Session.cpp                  |  44 ++-
 src/Interpreters/Session.h                    |   2 +
 src/Interpreters/SessionLog.cpp               |  14 +-
 src/Interpreters/SessionLog.h                 |   9 +-
 src/Server/HTTPHandler.cpp                    |   3 +-
 tests/integration/test_session_log/.gitignore |   1 +
 .../integration/test_session_log/__init__.py  |   0
 .../test_session_log/configs/log.xml          |   9 +
 .../test_session_log/configs/ports.xml        |   9 +
 .../test_session_log/configs/session_log.xml  |   9 +
 .../test_session_log/configs/users.xml        |  23 ++
 .../protos/clickhouse_grpc.proto              |   1 +
 tests/integration/test_session_log/test.py    | 289 ++++++++++++++++++
 .../02833_concurrrent_sessions.reference      |  34 +++
 .../0_stateless/02833_concurrrent_sessions.sh | 138 +++++++++
 .../02834_remote_session_log.reference        |  13 +
 .../0_stateless/02834_remote_session_log.sh   |  56 ++++
 .../02835_drop_user_during_session.reference  |   8 +
 .../02835_drop_user_during_session.sh         | 114 +++++++
 19 files changed, 753 insertions(+), 23 deletions(-)
 create mode 100644 tests/integration/test_session_log/.gitignore
 create mode 100644 tests/integration/test_session_log/__init__.py
 create mode 100644 tests/integration/test_session_log/configs/log.xml
 create mode 100644 tests/integration/test_session_log/configs/ports.xml
 create mode 100644 tests/integration/test_session_log/configs/session_log.xml
 create mode 100644 tests/integration/test_session_log/configs/users.xml
 create mode 120000 tests/integration/test_session_log/protos/clickhouse_grpc.proto
 create mode 100644 tests/integration/test_session_log/test.py
 create mode 100644 tests/queries/0_stateless/02833_concurrrent_sessions.reference
 create mode 100755 tests/queries/0_stateless/02833_concurrrent_sessions.sh
 create mode 100644 tests/queries/0_stateless/02834_remote_session_log.reference
 create mode 100755 tests/queries/0_stateless/02834_remote_session_log.sh
 create mode 100644 tests/queries/0_stateless/02835_drop_user_during_session.reference
 create mode 100755 tests/queries/0_stateless/02835_drop_user_during_session.sh

diff --git a/src/Interpreters/Session.cpp b/src/Interpreters/Session.cpp
index de2a779b740..0a6435cff75 100644
--- a/src/Interpreters/Session.cpp
+++ b/src/Interpreters/Session.cpp
@@ -520,6 +520,8 @@ ContextMutablePtr Session::makeSessionContext()
         {},
         session_context->getSettingsRef().max_sessions_for_user);
 
+    recordLoginSucess(session_context);
+
     return session_context;
 }
 
@@ -582,6 +584,8 @@ ContextMutablePtr Session::makeSessionContext(const String & session_name_, std:
         { session_name_ },
         max_sessions_for_user);
 
+    recordLoginSucess(session_context);
+
     return session_context;
 }
 
@@ -655,24 +659,38 @@ ContextMutablePtr Session::makeQueryContextImpl(const ClientInfo * client_info_t
     if (user_id)
         user = query_context->getUser();
 
-    if (!notified_session_log_about_login)
-    {
-        if (auto session_log = getSessionLog())
-        {
-            session_log->addLoginSuccess(
-                    auth_id,
-                    named_session ? std::optional<std::string>(named_session->key.second) : std::nullopt,
-                    *query_context,
-                    user);
-
-            notified_session_log_about_login = true;
-        }
-    }
+    /// Interserver does not create session context
+    recordLoginSucess(query_context);
 
     return query_context;
 }
 
 
+void Session::recordLoginSucess(ContextPtr login_context) const
+{
+    if (notified_session_log_about_login)
+        return;
+
+    if (!login_context)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Session or query context must be created");
+
+    if (auto session_log = getSessionLog())
+    {
+        const auto & settings   = login_context->getSettingsRef();
+        const auto access       = login_context->getAccess();
+
+        session_log->addLoginSuccess(auth_id,
+                                     named_session ? named_session->key.second : "",
+                                     settings,
+                                     access,
+                                     getClientInfo(),
+                                     user);
+
+        notified_session_log_about_login = true;
+    }
+}
+
+
 void Session::releaseSessionID()
 {
     if (!named_session)
diff --git a/src/Interpreters/Session.h b/src/Interpreters/Session.h
index 51c0e3c71fa..81ef987b428 100644
--- a/src/Interpreters/Session.h
+++ b/src/Interpreters/Session.h
@@ -97,6 +97,8 @@ public:
 private:
     std::shared_ptr<SessionLog> getSessionLog() const;
     ContextMutablePtr makeQueryContextImpl(const ClientInfo * client_info_to_copy, ClientInfo * client_info_to_move) const;
+    void recordLoginSucess(ContextPtr login_context) const;
+
 
     mutable bool notified_session_log_about_login = false;
     const UUID auth_id;
diff --git a/src/Interpreters/SessionLog.cpp b/src/Interpreters/SessionLog.cpp
index 0a8a7fc18c5..b847eaf9824 100644
--- a/src/Interpreters/SessionLog.cpp
+++ b/src/Interpreters/SessionLog.cpp
@@ -199,12 +199,13 @@ void SessionLogElement::appendToBlock(MutableColumns & columns) const
     columns[i++]->insertData(auth_failure_reason.data(), auth_failure_reason.length());
 }
 
-void SessionLog::addLoginSuccess(const UUID & auth_id, std::optional<String> session_id, const Context & login_context, const UserPtr & login_user)
+void SessionLog::addLoginSuccess(const UUID & auth_id,
+                                 const String & session_id,
+                                 const Settings & settings,
+                                 const ContextAccessPtr & access,
+                                 const ClientInfo & client_info,
+                                 const UserPtr & login_user)
 {
-    const auto access = login_context.getAccess();
-    const auto & settings = login_context.getSettingsRef();
-    const auto & client_info = login_context.getClientInfo();
-
     DB::SessionLogElement log_entry(auth_id, SESSION_LOGIN_SUCCESS);
     log_entry.client_info = client_info;
 
@@ -215,8 +216,7 @@ void SessionLog::addLoginSuccess(const UUID & auth_id, std::optional<String> ses
     }
     log_entry.external_auth_server = login_user ? login_user->auth_data.getLDAPServerName() : "";
 
-    if (session_id)
-        log_entry.session_id = *session_id;
+    log_entry.session_id = session_id;
 
     if (const auto roles_info = access->getRolesInfo())
         log_entry.roles = roles_info->getCurrentRolesNames();
diff --git a/src/Interpreters/SessionLog.h b/src/Interpreters/SessionLog.h
index 1282ac09c4d..8757bc12270 100644
--- a/src/Interpreters/SessionLog.h
+++ b/src/Interpreters/SessionLog.h
@@ -20,6 +20,7 @@ enum SessionLogElementType : int8_t
 class ContextAccess;
 struct User;
 using UserPtr = std::shared_ptr<const User>;
+using ContextAccessPtr = std::shared_ptr<const ContextAccess>;
 
 /** A struct which will be inserted as row into session_log table.
   *
@@ -72,7 +73,13 @@ class SessionLog : public SystemLog<SessionLogElement>
     using SystemLog<SessionLogElement>::SystemLog;
 
 public:
-    void addLoginSuccess(const UUID & auth_id, std::optional<String> session_id, const Context & login_context, const UserPtr & login_user);
+    void addLoginSuccess(const UUID & auth_id,
+                         const String & session_id,
+                         const Settings & settings,
+                         const ContextAccessPtr & access,
+                         const ClientInfo & client_info,
+                         const UserPtr & login_user);
+
     void addLoginFailure(const UUID & auth_id, const ClientInfo & info, const std::optional<String> & user, const Exception & reason);
     void addLogOut(const UUID & auth_id, const UserPtr & login_user, const ClientInfo & client_info);
 };
diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp
index ff5690a3b07..10d5e7a0242 100644
--- a/src/Server/HTTPHandler.cpp
+++ b/src/Server/HTTPHandler.cpp
@@ -561,8 +561,7 @@ void HTTPHandler::processQuery(
         session->makeSessionContext();
     }
 
-    auto client_info = session->getClientInfo();
-    auto context = session->makeQueryContext(std::move(client_info));
+    auto context = session->makeQueryContext();
 
     /// This parameter is used to tune the behavior of output formats (such as Native) for compatibility.
     if (params.has("client_protocol_version"))
diff --git a/tests/integration/test_session_log/.gitignore b/tests/integration/test_session_log/.gitignore
new file mode 100644
index 00000000000..edf565ec632
--- /dev/null
+++ b/tests/integration/test_session_log/.gitignore
@@ -0,0 +1 @@
+_gen
diff --git a/tests/integration/test_session_log/__init__.py b/tests/integration/test_session_log/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/integration/test_session_log/configs/log.xml b/tests/integration/test_session_log/configs/log.xml
new file mode 100644
index 00000000000..7a079b81e69
--- /dev/null
+++ b/tests/integration/test_session_log/configs/log.xml
@@ -0,0 +1,9 @@
+ <clickhouse>
+     <logger>
+         <level>trace</level>
+         <log>/var/log/clickhouse-server/clickhouse-server.log</log>
+         <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
+         <size>1000M</size>
+         <count>10</count>
+     </logger>
+ </clickhouse>
\ No newline at end of file
diff --git a/tests/integration/test_session_log/configs/ports.xml b/tests/integration/test_session_log/configs/ports.xml
new file mode 100644
index 00000000000..fbaefc16b3a
--- /dev/null
+++ b/tests/integration/test_session_log/configs/ports.xml
@@ -0,0 +1,9 @@
+ <clickhouse>
+    <postgresql_port>5433</postgresql_port>
+    <mysql_port>9001</mysql_port>
+    <grpc_port>9100</grpc_port>
+    <grpc replace="replace">
+        <!-- Enable if you want very detailed logs -->
+        <verbose_logs>false</verbose_logs>
+    </grpc>
+ </clickhouse>
\ No newline at end of file
diff --git a/tests/integration/test_session_log/configs/session_log.xml b/tests/integration/test_session_log/configs/session_log.xml
new file mode 100644
index 00000000000..a0e4e3e2216
--- /dev/null
+++ b/tests/integration/test_session_log/configs/session_log.xml
@@ -0,0 +1,9 @@
+<clickhouse>
+    <session_log>
+        <database>system</database>
+        <table>session_log</table>
+
+        <partition_by>toYYYYMM(event_date)</partition_by>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </session_log>
+</clickhouse>
diff --git a/tests/integration/test_session_log/configs/users.xml b/tests/integration/test_session_log/configs/users.xml
new file mode 100644
index 00000000000..0416dfadc8a
--- /dev/null
+++ b/tests/integration/test_session_log/configs/users.xml
@@ -0,0 +1,23 @@
+<clickhouse>
+    <profiles>
+        <default>
+            <function_sleep_max_microseconds_per_block>0</function_sleep_max_microseconds_per_block>
+        </default>
+    </profiles>
+    <users>
+        <default>
+        </default>
+        <mysql_user>
+            <password>pass</password>
+        </mysql_user>
+        <postgres_user>
+            <password>pass</password>
+        </postgres_user>
+        <grpc_user>
+            <password>pass</password>
+        </grpc_user>
+        <parallel_user>
+            <password>pass</password>
+        </parallel_user>
+    </users>
+</clickhouse>
\ No newline at end of file
diff --git a/tests/integration/test_session_log/protos/clickhouse_grpc.proto b/tests/integration/test_session_log/protos/clickhouse_grpc.proto
new file mode 120000
index 00000000000..25d15f11e3b
--- /dev/null
+++ b/tests/integration/test_session_log/protos/clickhouse_grpc.proto
@@ -0,0 +1 @@
+../../../../src/Server/grpc_protos/clickhouse_grpc.proto
\ No newline at end of file
diff --git a/tests/integration/test_session_log/test.py b/tests/integration/test_session_log/test.py
new file mode 100644
index 00000000000..dbb39993ce3
--- /dev/null
+++ b/tests/integration/test_session_log/test.py
@@ -0,0 +1,289 @@
+import os
+
+import grpc
+import pymysql.connections
+import psycopg2 as py_psql
+import pytest
+import random
+import sys
+import threading
+
+from helpers.cluster import ClickHouseCluster, run_and_check
+
+POSTGRES_SERVER_PORT = 5433
+MYSQL_SERVER_PORT = 9001
+GRPC_PORT = 9100
+SESSION_LOG_MATCHING_FIELDS = "auth_id, auth_type, client_version_major, client_version_minor, client_version_patch, interface"
+
+SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
+DEFAULT_ENCODING = "utf-8"
+
+# Use grpcio-tools to generate *pb2.py files from *.proto.
+proto_dir = os.path.join(SCRIPT_DIR, "./protos")
+gen_dir = os.path.join(SCRIPT_DIR, "./_gen")
+os.makedirs(gen_dir, exist_ok=True)
+run_and_check(
+    f"python3 -m grpc_tools.protoc -I{proto_dir} --python_out={gen_dir} --grpc_python_out={gen_dir} {proto_dir}/clickhouse_grpc.proto",
+    shell=True,
+)
+
+sys.path.append(gen_dir)
+
+import clickhouse_grpc_pb2
+import clickhouse_grpc_pb2_grpc
+
+cluster = ClickHouseCluster(__file__)
+instance = cluster.add_instance(
+    "node",
+    main_configs=[
+        "configs/ports.xml",
+        "configs/log.xml",
+        "configs/session_log.xml",
+    ],
+    user_configs=["configs/users.xml"],
+    # Bug in TSAN reproduces in this test https://github.com/grpc/grpc/issues/29550#issuecomment-1188085387
+    env_variables={
+        "TSAN_OPTIONS": "report_atomic_races=0 " + os.getenv("TSAN_OPTIONS", default="")
+    },
+)
+
+
+def grpc_get_url():
+    return f"{instance.ip_address}:{GRPC_PORT}"
+
+
+def grpc_create_insecure_channel():
+    channel = grpc.insecure_channel(grpc_get_url())
+    grpc.channel_ready_future(channel).result(timeout=2)
+    return channel
+
+
+session_id_counter = 0
+
+
+def next_session_id():
+    global session_id_counter
+    session_id = session_id_counter
+    session_id_counter += 1
+    return str(session_id)
+
+
+def grpc_query(query, user_, pass_, raise_exception):
+    try:
+        query_info = clickhouse_grpc_pb2.QueryInfo(
+            query=query,
+            session_id=next_session_id(),
+            user_name=user_,
+            password=pass_,
+        )
+        channel = grpc_create_insecure_channel()
+        stub = clickhouse_grpc_pb2_grpc.ClickHouseStub(channel)
+        result = stub.ExecuteQuery(query_info)
+        if result and result.HasField("exception"):
+            raise Exception(result.exception.display_text)
+
+        return result.output.decode(DEFAULT_ENCODING)
+    except Exception:
+        assert raise_exception
+
+
+def postgres_query(query, user_, pass_, raise_exception):
+    try:
+        client = py_psql.connect(
+            host=instance.ip_address,
+            port=POSTGRES_SERVER_PORT,
+            user=user_,
+            password=pass_,
+            database="default",
+        )
+        cursor = client.cursor()
+        cursor.execute(query)
+        cursor.fetchall()
+    except Exception:
+        assert raise_exception
+
+
+def mysql_query(query, user_, pass_, raise_exception):
+    try:
+        client = pymysql.connections.Connection(
+            host=instance.ip_address,
+            user=user_,
+            password=pass_,
+            database="default",
+            port=MYSQL_SERVER_PORT,
+        )
+        cursor = client.cursor(pymysql.cursors.DictCursor)
+        if raise_exception:
+            with pytest.raises(Exception):
+                cursor.execute(query)
+        else:
+            cursor.execute(query)
+        cursor.fetchall()
+    except Exception:
+        assert raise_exception
+
+
+@pytest.fixture(scope="module")
+def started_cluster():
+    try:
+        cluster.start()
+        yield cluster
+    finally:
+        cluster.shutdown()
+
+
+def test_grpc_session(started_cluster):
+    grpc_query("SELECT 1", "grpc_user", "pass", False)
+    grpc_query("SELECT 2", "grpc_user", "wrong_pass", True)
+    grpc_query("SELECT 3", "wrong_grpc_user", "pass", True)
+
+    instance.query("SYSTEM FLUSH LOGS")
+    login_success_records = instance.query(
+        "SELECT user, client_port <> 0,  client_address <> toIPv6('::') FROM system.session_log WHERE user='grpc_user' AND type = 'LoginSuccess'"
+    )
+    assert login_success_records == "grpc_user\t1\t1\n"
+    logout_records = instance.query(
+        "SELECT user, client_port <> 0,  client_address <> toIPv6('::') FROM system.session_log WHERE user='grpc_user' AND type = 'Logout'"
+    )
+    assert logout_records == "grpc_user\t1\t1\n"
+    login_failure_records = instance.query(
+        "SELECT user, client_port <> 0,  client_address <> toIPv6('::') FROM system.session_log WHERE user='grpc_user' AND type = 'LoginFailure'"
+    )
+    assert login_failure_records == "grpc_user\t1\t1\n"
+    logins_and_logouts = instance.query(
+        f"SELECT COUNT(*) FROM (SELECT {SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = 'grpc_user' AND type = 'LoginSuccess' INTERSECT SELECT {SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = 'grpc_user' AND type = 'Logout')"
+    )
+    assert logins_and_logouts == "1\n"
+
+
+def test_mysql_session(started_cluster):
+    mysql_query("SELECT 1", "mysql_user", "pass", False)
+    mysql_query("SELECT 2", "mysql_user", "wrong_pass", True)
+    mysql_query("SELECT 3", "wrong_mysql_user", "pass", True)
+
+    instance.query("SYSTEM FLUSH LOGS")
+    login_success_records = instance.query(
+        "SELECT user, client_port <> 0,  client_address <> toIPv6('::') FROM system.session_log WHERE user='mysql_user' AND type = 'LoginSuccess'"
+    )
+    assert login_success_records == "mysql_user\t1\t1\n"
+    logout_records = instance.query(
+        "SELECT user, client_port <> 0,  client_address <> toIPv6('::') FROM system.session_log WHERE user='mysql_user' AND type = 'Logout'"
+    )
+    assert logout_records == "mysql_user\t1\t1\n"
+    login_failure_records = instance.query(
+        "SELECT user, client_port <> 0,  client_address <> toIPv6('::') FROM system.session_log WHERE user='mysql_user' AND type = 'LoginFailure'"
+    )
+    assert login_failure_records == "mysql_user\t1\t1\n"
+    logins_and_logouts = instance.query(
+        f"SELECT COUNT(*) FROM (SELECT {SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = 'mysql_user' AND type = 'LoginSuccess' INTERSECT SELECT {SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = 'mysql_user' AND type = 'Logout')"
+    )
+    assert logins_and_logouts == "1\n"
+
+
+def test_postgres_session(started_cluster):
+    postgres_query("SELECT 1", "postgres_user", "pass", False)
+    postgres_query("SELECT 2", "postgres_user", "wrong_pass", True)
+    postgres_query("SELECT 3", "wrong_postgres_user", "pass", True)
+
+    instance.query("SYSTEM FLUSH LOGS")
+    login_success_records = instance.query(
+        "SELECT user, client_port <> 0,  client_address <> toIPv6('::') FROM system.session_log WHERE user='postgres_user' AND type = 'LoginSuccess'"
+    )
+    assert login_success_records == "postgres_user\t1\t1\n"
+    logout_records = instance.query(
+        "SELECT user, client_port <> 0,  client_address <> toIPv6('::') FROM system.session_log WHERE user='postgres_user' AND type = 'Logout'"
+    )
+    assert logout_records == "postgres_user\t1\t1\n"
+    login_failure_records = instance.query(
+        "SELECT user, client_port <> 0,  client_address <> toIPv6('::') FROM system.session_log WHERE user='postgres_user' AND type = 'LoginFailure'"
+    )
+    assert login_failure_records == "postgres_user\t1\t1\n"
+    logins_and_logouts = instance.query(
+        f"SELECT COUNT(*) FROM (SELECT {SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = 'postgres_user' AND type = 'LoginSuccess' INTERSECT SELECT {SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = 'postgres_user' AND type = 'Logout')"
+    )
+    assert logins_and_logouts == "1\n"
+
+
+def test_parallel_sessions(started_cluster):
+    thread_list = []
+    for _ in range(10):
+        # Sleep time does not significantly matter here,
+        # test should pass even without sleeping.
+        for function in [postgres_query, grpc_query, mysql_query]:
+            thread = threading.Thread(
+                target=function,
+                args=(
+                    f"SELECT sleep({random.uniform(0.03, 0.04)})",
+                    "parallel_user",
+                    "pass",
+                    False,
+                ),
+            )
+            thread.start()
+            thread_list.append(thread)
+            thread = threading.Thread(
+                target=function,
+                args=(
+                    f"SELECT sleep({random.uniform(0.03, 0.04)})",
+                    "parallel_user",
+                    "wrong_pass",
+                    True,
+                ),
+            )
+            thread.start()
+            thread_list.append(thread)
+            thread = threading.Thread(
+                target=function,
+                args=(
+                    f"SELECT sleep({random.uniform(0.03, 0.04)})",
+                    "wrong_parallel_user",
+                    "pass",
+                    True,
+                ),
+            )
+            thread.start()
+            thread_list.append(thread)
+
+    for thread in thread_list:
+        thread.join()
+
+    instance.query("SYSTEM FLUSH LOGS")
+    port_0_sessions = instance.query(
+        f"SELECT COUNT(*) FROM system.session_log WHERE user = 'parallel_user'"
+    )
+    assert port_0_sessions == "90\n"
+
+    port_0_sessions = instance.query(
+        f"SELECT COUNT(*) FROM system.session_log WHERE user = 'parallel_user' AND client_port = 0"
+    )
+    assert port_0_sessions == "0\n"
+
+    address_0_sessions = instance.query(
+        f"SELECT COUNT(*) FROM system.session_log WHERE user = 'parallel_user' AND client_address = toIPv6('::')"
+    )
+    assert address_0_sessions == "0\n"
+
+    grpc_sessions = instance.query(
+        f"SELECT COUNT(*) FROM system.session_log WHERE user = 'parallel_user' AND interface = 'gRPC'"
+    )
+    assert grpc_sessions == "30\n"
+
+    mysql_sessions = instance.query(
+        f"SELECT COUNT(*) FROM system.session_log WHERE user = 'parallel_user' AND interface = 'MySQL'"
+    )
+    assert mysql_sessions == "30\n"
+
+    postgres_sessions = instance.query(
+        f"SELECT COUNT(*) FROM system.session_log WHERE user = 'parallel_user' AND interface = 'PostgreSQL'"
+    )
+    assert postgres_sessions == "30\n"
+
+    logins_and_logouts = instance.query(
+        f"SELECT COUNT(*) FROM (SELECT {SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = 'parallel_user' AND type = 'LoginSuccess' INTERSECT SELECT {SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = 'parallel_user' AND type = 'Logout')"
+    )
+    assert logins_and_logouts == "30\n"
+
+    logout_failure_sessions = instance.query(
+        f"SELECT COUNT(*) FROM system.session_log  WHERE user = 'parallel_user' AND type = 'LoginFailure'"
+    )
+    assert logout_failure_sessions == "30\n"
diff --git a/tests/queries/0_stateless/02833_concurrrent_sessions.reference b/tests/queries/0_stateless/02833_concurrrent_sessions.reference
new file mode 100644
index 00000000000..bfe507e8eac
--- /dev/null
+++ b/tests/queries/0_stateless/02833_concurrrent_sessions.reference
@@ -0,0 +1,34 @@
+sessions:
+150
+port_0_sessions:
+0
+address_0_sessions:
+0
+tcp_sessions
+60
+http_sessions
+30
+http_with_session_id_sessions
+30
+my_sql_sessions
+30
+Corresponding LoginSuccess/Logout
+10
+LoginFailure
+10
+Corresponding LoginSuccess/Logout
+10
+LoginFailure
+10
+Corresponding LoginSuccess/Logout
+10
+LoginFailure
+10
+Corresponding LoginSuccess/Logout
+10
+LoginFailure
+10
+Corresponding LoginSuccess/Logout
+10
+LoginFailure
+10
diff --git a/tests/queries/0_stateless/02833_concurrrent_sessions.sh b/tests/queries/0_stateless/02833_concurrrent_sessions.sh
new file mode 100755
index 00000000000..26b48462a76
--- /dev/null
+++ b/tests/queries/0_stateless/02833_concurrrent_sessions.sh
@@ -0,0 +1,138 @@
+#!/usr/bin/env bash
+# Tags: no-fasttest, no-debug
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+readonly PID=$$
+
+# Each user uses a separate thread.
+readonly TCP_USERS=( "02833_TCP_USER_${PID}"_{1,2} ) # 2 concurrent TCP users
+readonly HTTP_USERS=( "02833_HTTP_USER_${PID}" )
+readonly HTTP_WITH_SESSION_ID_SESSION_USERS=( "02833_HTTP_WITH_SESSION_ID_USER_${PID}" )
+readonly MYSQL_USERS=( "02833_MYSQL_USER_${PID}")
+readonly ALL_USERS=( "${TCP_USERS[@]}" "${HTTP_USERS[@]}" "${HTTP_WITH_SESSION_ID_SESSION_USERS[@]}" "${MYSQL_USERS[@]}" )
+
+readonly TCP_USERS_SQL_COLLECTION_STRING="$( echo "${TCP_USERS[*]}" | sed "s/[^[:space:]]\+/'&'/g" | sed 's/[[:space:]]/,/g' )"
+readonly HTTP_USERS_SQL_COLLECTION_STRING="$( echo "${HTTP_USERS[*]}" | sed "s/[^[:space:]]\+/'&'/g" | sed 's/[[:space:]]/,/g' )"
+readonly HTTP_WITH_SESSION_ID_USERS_SQL_COLLECTION_STRING="$( echo "${HTTP_WITH_SESSION_ID_SESSION_USERS[*]}" | sed "s/[^[:space:]]\+/'&'/g" | sed 's/[[:space:]]/,/g' )"
+readonly MYSQL_USERS_SQL_COLLECTION_STRING="$( echo "${MYSQL_USERS[*]}" | sed "s/[^[:space:]]\+/'&'/g" | sed 's/[[:space:]]/,/g' )"
+readonly ALL_USERS_SQL_COLLECTION_STRING="$( echo "${ALL_USERS[*]}" | sed "s/[^[:space:]]\+/'&'/g" | sed 's/[[:space:]]/,/g' )"
+
+readonly SESSION_LOG_MATCHING_FIELDS="auth_id, auth_type, client_version_major, client_version_minor, client_version_patch, interface"
+
+for user in "${ALL_USERS[@]}"; do
+    ${CLICKHOUSE_CLIENT} -q "CREATE USER IF NOT EXISTS ${user} IDENTIFIED WITH plaintext_password BY 'pass'"
+    ${CLICKHOUSE_CLIENT} -q "GRANT SELECT ON system.* TO ${user}"
+    ${CLICKHOUSE_CLIENT} -q "GRANT SELECT ON INFORMATION_SCHEMA.* TO ${user}";
+done
+
+# All <type>_session functions execute in separate threads.
+# These functions try to create a session with successful login and logout.
+# Sleep a small, random amount of time to make concurrency more intense.
+# and try to login with an invalid password.
+function tcp_session() 
+{
+    local user=$1
+    local i=0
+    while (( (i++) < 10 )); do
+        # login logout
+        ${CLICKHOUSE_CLIENT} -q "SELECT 1, sleep(0.01${RANDOM})" --user="${user}" --password="pass"
+        # login failure
+        ${CLICKHOUSE_CLIENT} -q "SELECT 2" --user="${user}" --password 'invalid'
+    done
+}
+
+function http_session() 
+{
+    local user=$1
+    local i=0
+    while (( (i++) < 10 )); do
+        # login logout
+        ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&user=${user}&password=pass" -d "SELECT 3, sleep(0.01${RANDOM})"
+
+        # login failure
+        ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&user=${user}&password=wrong" -d "SELECT 4"
+    done
+}
+
+function http_with_session_id_session() 
+{
+    local user=$1
+    local i=0
+    while (( (i++) < 10 )); do
+        # login logout
+        ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=${user}&user=${user}&password=pass" -d "SELECT 5, sleep 0.01${RANDOM}"
+
+        # login failure
+        ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=${user}&user=${user}&password=wrong" -d "SELECT 6"
+    done
+}
+
+function mysql_session() 
+{
+    local user=$1
+    local i=0
+    while (( (i++) < 10 )); do
+        # login logout
+        ${CLICKHOUSE_CLIENT} -q "SELECT 1, sleep(0.01${RANDOM}) FROM mysql('127.0.0.1:9004', 'system', 'one', '${user}', 'pass')"
+
+        # login failure
+        ${CLICKHOUSE_CLIENT} -q "SELECT 1 FROM mysql('127.0.0.1:9004', 'system', 'one', '${user}', 'wrong', SETTINGS connection_max_tries=1)"
+    done
+}
+
+${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS"
+${CLICKHOUSE_CLIENT} -q "DELETE FROM system.session_log WHERE user IN (${ALL_USERS_SQL_COLLECTION_STRING})"
+
+export -f tcp_session;
+export -f http_session;
+export -f http_with_session_id_session;
+export -f mysql_session;
+
+for user in "${TCP_USERS[@]}"; do
+    timeout 60s bash -c "tcp_session ${user}" >/dev/null 2>&1 &
+done
+
+for user in "${HTTP_USERS[@]}"; do
+    timeout 60s bash -c "http_session ${user}" >/dev/null 2>&1 &
+done
+
+for user in "${HTTP_WITH_SESSION_ID_SESSION_USERS[@]}"; do
+    timeout 60s bash -c "http_with_session_id_session ${user}" >/dev/null 2>&1 &
+done
+
+for user in "${MYSQL_USERS[@]}"; do
+    timeout 60s bash -c "mysql_session ${user}" >/dev/null 2>&1 &
+done
+
+wait
+
+${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS"
+
+echo "sessions:" 
+${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user IN (${ALL_USERS_SQL_COLLECTION_STRING})"
+
+echo "port_0_sessions:" 
+${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user IN (${ALL_USERS_SQL_COLLECTION_STRING}) AND client_port = 0"
+
+echo "address_0_sessions:"
+${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user IN (${ALL_USERS_SQL_COLLECTION_STRING}) AND client_address = toIPv6('::')"
+
+echo "tcp_sessions"
+${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user IN (${TCP_USERS_SQL_COLLECTION_STRING}) AND interface = 'TCP'"
+echo "http_sessions"
+${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user IN (${HTTP_USERS_SQL_COLLECTION_STRING}) AND interface = 'HTTP'"
+echo "http_with_session_id_sessions"
+${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user IN (${HTTP_WITH_SESSION_ID_USERS_SQL_COLLECTION_STRING}) AND interface = 'HTTP'"
+echo "my_sql_sessions"
+${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user IN (${MYSQL_USERS_SQL_COLLECTION_STRING}) AND interface = 'MySQL'"
+
+for user in "${ALL_USERS[@]}"; do
+    ${CLICKHOUSE_CLIENT} -q "DROP USER ${user}"
+    echo "Corresponding LoginSuccess/Logout" 
+    ${CLICKHOUSE_CLIENT} -q "SELECT COUNT(*) FROM (SELECT ${SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = '${user}' AND type = 'LoginSuccess' INTERSECT SELECT ${SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = '${user}' AND type = 'Logout')"
+    echo "LoginFailure"
+    ${CLICKHOUSE_CLIENT} -q "SELECT COUNT(*) FROM system.session_log WHERE user = '${user}' AND type = 'LoginFailure'" 
+ done
diff --git a/tests/queries/0_stateless/02834_remote_session_log.reference b/tests/queries/0_stateless/02834_remote_session_log.reference
new file mode 100644
index 00000000000..e2680982ab0
--- /dev/null
+++ b/tests/queries/0_stateless/02834_remote_session_log.reference
@@ -0,0 +1,13 @@
+0
+0
+0
+0
+client_port 0 connections:
+0
+client_address '::' connections:
+0
+login failures:
+0
+TCP Login and logout count is equal
+HTTP Login and logout count is equal
+MySQL Login and logout count is equal
diff --git a/tests/queries/0_stateless/02834_remote_session_log.sh b/tests/queries/0_stateless/02834_remote_session_log.sh
new file mode 100755
index 00000000000..3bedfb6c9ee
--- /dev/null
+++ b/tests/queries/0_stateless/02834_remote_session_log.sh
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# Tags: no-fasttest
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+readonly PID=$$
+readonly TEST_USER=$"02834_USER_${PID}"
+readonly SESSION_LOG_MATCHING_FIELDS="auth_id, auth_type, client_version_major, client_version_minor, client_version_patch, interface"
+
+${CLICKHOUSE_CLIENT} -q "CREATE USER IF NOT EXISTS ${TEST_USER} IDENTIFIED WITH plaintext_password BY 'pass'"
+${CLICKHOUSE_CLIENT} -q "GRANT SELECT ON INFORMATION_SCHEMA.* TO ${TEST_USER}"
+${CLICKHOUSE_CLIENT} -q "GRANT SELECT ON system.* TO ${TEST_USER}"
+${CLICKHOUSE_CLIENT} -q "GRANT CREATE TEMPORARY TABLE, MYSQL, REMOTE ON *.* TO ${TEST_USER}"
+
+${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS"
+${CLICKHOUSE_CLIENT} -q "DELETE FROM system.session_log WHERE user = '${TEST_USER}'"
+
+${CLICKHOUSE_CURL} -sS -X POST "${CLICKHOUSE_URL}&user=${TEST_USER}&password=pass" \
+   -d "SELECT * FROM remote('127.0.0.1:${CLICKHOUSE_PORT_TCP}', 'system', 'one', '${TEST_USER}', 'pass')"
+
+${CLICKHOUSE_CURL} -sS -X POST "${CLICKHOUSE_URL}&user=${TEST_USER}&password=pass" \
+    -d "SELECT * FROM mysql('127.0.0.1:9004', 'system', 'one', '${TEST_USER}', 'pass')"
+
+${CLICKHOUSE_CLIENT} -q "SELECT * FROM remote('127.0.0.1:${CLICKHOUSE_PORT_TCP}', 'system', 'one', '${TEST_USER}', 'pass')" -u "${TEST_USER}" --password "pass"
+${CLICKHOUSE_CLIENT} -q "SELECT * FROM mysql('127.0.0.1:9004', 'system', 'one', '${TEST_USER}', 'pass')" -u "${TEST_USER}" --password "pass"
+
+${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS"
+
+echo "client_port 0 connections:"
+${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user = '${TEST_USER}' and client_port = 0"
+
+echo "client_address '::' connections:"
+${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user = '${TEST_USER}' and client_address = toIPv6('::')"
+
+echo "login failures:"
+${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user = '${TEST_USER}' and type = 'LoginFailure'"
+
+# remote(...) function sometimes reuses old cached sessions for query execution.
+# This makes LoginSuccess/Logout entries count unstable, but success and logouts must always match.
+
+for interface in 'TCP' 'HTTP' 'MySQL'
+do
+    LOGIN_COUNT=`${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user = '${TEST_USER}' AND type = 'LoginSuccess' AND interface = '${interface}'"`
+    CORRESPONDING_LOGOUT_RECORDS_COUNT=`${CLICKHOUSE_CLIENT} -q "SELECT COUNT(*) FROM (SELECT ${SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = '${TEST_USER}' AND type = 'LoginSuccess' AND interface = '${interface}' INTERSECT SELECT ${SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = '${TEST_USER}' AND type = 'Logout' AND interface = '${interface}')"`
+
+    if [ "$LOGIN_COUNT" == "$CORRESPONDING_LOGOUT_RECORDS_COUNT" ]; then
+        echo "${interface} Login and logout count is equal"
+    else
+        TOTAL_LOGOUT_COUNT=`${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user = '${TEST_USER}' AND type = 'Logout' AND interface = '${interface}'"`
+        echo "${interface} Login count ${LOGIN_COUNT} != corresponding logout count ${CORRESPONDING_LOGOUT_RECORDS_COUNT}. TOTAL_LOGOUT_COUNT ${TOTAL_LOGOUT_COUNT}"
+    fi
+done
+
+${CLICKHOUSE_CLIENT} -q "DROP USER ${TEST_USER}"
diff --git a/tests/queries/0_stateless/02835_drop_user_during_session.reference b/tests/queries/0_stateless/02835_drop_user_during_session.reference
new file mode 100644
index 00000000000..7252faab8c6
--- /dev/null
+++ b/tests/queries/0_stateless/02835_drop_user_during_session.reference
@@ -0,0 +1,8 @@
+port_0_sessions:
+0
+address_0_sessions:
+0
+Corresponding LoginSuccess/Logout
+9
+LoginFailure
+0
diff --git a/tests/queries/0_stateless/02835_drop_user_during_session.sh b/tests/queries/0_stateless/02835_drop_user_during_session.sh
new file mode 100755
index 00000000000..347ebd22f96
--- /dev/null
+++ b/tests/queries/0_stateless/02835_drop_user_during_session.sh
@@ -0,0 +1,114 @@
+#!/usr/bin/env bash
+# Tags: no-debug
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+readonly PID=$$
+
+readonly TEST_USER="02835_USER_${PID}"
+readonly TEST_ROLE="02835_ROLE_${PID}"
+readonly TEST_PROFILE="02835_PROFILE_${PID}"
+readonly SESSION_LOG_MATCHING_FIELDS="auth_id, auth_type, client_version_major, client_version_minor, client_version_patch, interface"
+
+function tcp_session()
+{
+    local user=$1
+    ${CLICKHOUSE_CLIENT} -q "SELECT COUNT(*) FROM system.numbers" --user="${user}"
+}
+
+function http_session()
+{
+    local user=$1
+    ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&user=${user}&password=pass" -d "SELECT COUNT(*) FROM system.numbers"
+}
+
+function http_with_session_id_session() 
+{
+    local user=$1
+    ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&user=${user}&password=pass" -d "SELECT COUNT(*) FROM system.numbers"
+}
+
+# Busy-waits until user $1, specified amount of queries ($2) will run simultaneously.
+function wait_for_queries_start()
+{
+    local user=$1
+    local queries_count=$2
+    # 10 seconds waiting
+    counter=0 retries=100
+    while [[ $counter -lt $retries ]]; do
+        result=$($CLICKHOUSE_CLIENT --query "SELECT COUNT(*) FROM system.processes WHERE user = '${user}'")
+        if [[ $result == "${queries_count}" ]]; then
+            break;
+        fi
+        sleep 0.1
+        ((++counter))
+    done
+}
+
+${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS"
+${CLICKHOUSE_CLIENT} -q "DELETE FROM system.session_log WHERE user = '${TEST_USER}'"
+
+# DROP USE CASE
+${CLICKHOUSE_CLIENT} -q "CREATE USER IF NOT EXISTS ${TEST_USER}"
+${CLICKHOUSE_CLIENT} -q "GRANT SELECT ON system.numbers TO ${TEST_USER}"
+
+export -f tcp_session;
+export -f http_session;
+export -f http_with_session_id_session;
+
+timeout 10s bash -c "tcp_session ${TEST_USER}" >/dev/null 2>&1 &
+timeout 10s bash -c "http_session ${TEST_USER}" >/dev/null 2>&1 &
+timeout 10s bash -c "http_with_session_id_session ${TEST_USER}" >/dev/null 2>&1 &
+
+wait_for_queries_start $TEST_USER 3
+${CLICKHOUSE_CLIENT} -q "DROP USER ${TEST_USER}"
+${CLICKHOUSE_CLIENT} -q "KILL QUERY WHERE user = '${TEST_USER}' SYNC" >/dev/null &
+
+wait
+
+# DROP ROLE CASE
+${CLICKHOUSE_CLIENT} -q "CREATE ROLE IF NOT EXISTS ${TEST_ROLE}"
+${CLICKHOUSE_CLIENT} -q "CREATE USER ${TEST_USER} DEFAULT ROLE ${TEST_ROLE}"
+${CLICKHOUSE_CLIENT} -q "GRANT SELECT ON system.numbers TO ${TEST_USER}"
+
+timeout 10s bash -c "tcp_session ${TEST_USER}" >/dev/null 2>&1 &
+timeout 10s bash -c "http_session ${TEST_USER}" >/dev/null 2>&1 &
+timeout 10s bash -c "http_with_session_id_session ${TEST_USER}" >/dev/null 2>&1 &
+
+wait_for_queries_start $TEST_USER 3
+${CLICKHOUSE_CLIENT} -q "DROP ROLE ${TEST_ROLE}"
+${CLICKHOUSE_CLIENT} -q "DROP USER ${TEST_USER}"
+
+${CLICKHOUSE_CLIENT} -q "KILL QUERY WHERE user = '${TEST_USER}' SYNC" >/dev/null &
+
+wait
+
+# DROP PROFILE CASE
+${CLICKHOUSE_CLIENT} -q "CREATE SETTINGS PROFILE IF NOT EXISTS '${TEST_PROFILE}'"
+${CLICKHOUSE_CLIENT} -q "CREATE USER ${TEST_USER} SETTINGS PROFILE '${TEST_PROFILE}'"
+${CLICKHOUSE_CLIENT} -q "GRANT SELECT ON system.numbers TO ${TEST_USER}"
+
+timeout 10s bash -c "tcp_session ${TEST_USER}" >/dev/null 2>&1 &
+timeout 10s bash -c "http_session ${TEST_USER}" >/dev/null 2>&1 &
+timeout 10s bash -c "http_with_session_id_session ${TEST_USER}" >/dev/null 2>&1 &
+
+wait_for_queries_start $TEST_USER 3
+${CLICKHOUSE_CLIENT} -q "DROP SETTINGS PROFILE '${TEST_PROFILE}'"
+${CLICKHOUSE_CLIENT} -q "DROP USER ${TEST_USER}"
+
+${CLICKHOUSE_CLIENT} -q "KILL QUERY WHERE user = '${TEST_USER}' SYNC" >/dev/null &
+
+wait
+
+${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS"
+
+echo "port_0_sessions:" 
+${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user = '${TEST_USER}' AND client_port = 0"
+echo "address_0_sessions:"
+${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user = '${TEST_USER}' AND client_address = toIPv6('::')"
+echo "Corresponding LoginSuccess/Logout" 
+${CLICKHOUSE_CLIENT} -q "SELECT COUNT(*) FROM (SELECT ${SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = '${TEST_USER}' AND type = 'LoginSuccess' INTERSECT SELECT ${SESSION_LOG_MATCHING_FIELDS}, FROM system.session_log WHERE user = '${TEST_USER}' AND type = 'Logout')"
+echo "LoginFailure"
+${CLICKHOUSE_CLIENT} -q "SELECT COUNT(*) FROM system.session_log WHERE user = '${TEST_USER}' AND type = 'LoginFailure'" 

From 7b0036dee3ef050c060ee2aee0ab9e6faf927e41 Mon Sep 17 00:00:00 2001
From: Alexey Gerasimchuck <a.gerasimchuck@arenadata.io>
Date: Thu, 10 Aug 2023 04:23:40 +0000
Subject: [PATCH 179/249] moved notified_session_log_about_login=true line

---
 src/Interpreters/Session.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Interpreters/Session.cpp b/src/Interpreters/Session.cpp
index 0a6435cff75..f8bd70afdb6 100644
--- a/src/Interpreters/Session.cpp
+++ b/src/Interpreters/Session.cpp
@@ -685,9 +685,9 @@ void Session::recordLoginSucess(ContextPtr login_context) const
                                      access,
                                      getClientInfo(),
                                      user);
-
-        notified_session_log_about_login = true;
     }
+
+    notified_session_log_about_login = true;
 }
 
 

From d499f8030a34d5076c7a9fc97d94a5f04b2898d6 Mon Sep 17 00:00:00 2001
From: Alexey Gerasimchuck <a.gerasimchuck@arenadata.io>
Date: Thu, 10 Aug 2023 05:20:00 +0000
Subject: [PATCH 180/249] reverted change in test

---
 tests/integration/test_profile_max_sessions_for_user/test.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/tests/integration/test_profile_max_sessions_for_user/test.py b/tests/integration/test_profile_max_sessions_for_user/test.py
index 2930262f63e..65587933fed 100755
--- a/tests/integration/test_profile_max_sessions_for_user/test.py
+++ b/tests/integration/test_profile_max_sessions_for_user/test.py
@@ -27,10 +27,7 @@ proto_dir = os.path.join(SCRIPT_DIR, "./protos")
 gen_dir = os.path.join(SCRIPT_DIR, "./_gen")
 os.makedirs(gen_dir, exist_ok=True)
 run_and_check(
-    "python3 -m grpc_tools.protoc -I{proto_dir} --python_out={gen_dir} --grpc_python_out={gen_dir} \
-    {proto_dir}/clickhouse_grpc.proto".format(
-        proto_dir=proto_dir, gen_dir=gen_dir
-    ),
+    f"python3 -m grpc_tools.protoc -I{proto_dir} --python_out={gen_dir} --grpc_python_out={gen_dir} {proto_dir}/clickhouse_grpc.proto",
     shell=True,
 )
 

From 7321f5e543387aa65e20fe12421ee692eb05aa64 Mon Sep 17 00:00:00 2001
From: Nikolay Degterinsky <evillique@gmail.com>
Date: Thu, 10 Aug 2023 06:32:28 +0000
Subject: [PATCH 181/249] Better

---
 src/IO/WriteHelpers.h                         | 15 ++++++++---
 src/Parsers/ASTLiteral.cpp                    | 26 +++++++++++++++++--
 src/Parsers/IAST.h                            |  7 ++++-
 src/Parsers/LiteralEscapingStyle.h            | 14 ++++++++++
 src/Storages/StorageMySQL.cpp                 |  1 +
 src/Storages/StoragePostgreSQL.cpp            |  6 +----
 src/Storages/StorageSQLite.cpp                |  1 +
 src/Storages/StorageXDBC.cpp                  |  1 +
 .../transformQueryForExternalDatabase.cpp     |  7 ++++-
 .../transformQueryForExternalDatabase.h       |  1 +
 .../test_storage_postgresql/test.py           | 20 +++++++-------
 11 files changed, 78 insertions(+), 21 deletions(-)
 create mode 100644 src/Parsers/LiteralEscapingStyle.h

diff --git a/src/IO/WriteHelpers.h b/src/IO/WriteHelpers.h
index aa4c9b17e48..d092c7b8ea5 100644
--- a/src/IO/WriteHelpers.h
+++ b/src/IO/WriteHelpers.h
@@ -304,9 +304,10 @@ inline void writeJSONString(const char * begin, const char * end, WriteBuffer &
 /** Will escape quote_character and a list of special characters('\b', '\f', '\n', '\r', '\t', '\0', '\\').
  *   - when escape_quote_with_quote is true, use backslash to escape list of special characters,
  *      and use quote_character to escape quote_character. such as: 'hello''world'
- *   - otherwise use backslash to escape list of special characters and quote_character
+ *     otherwise use backslash to escape list of special characters and quote_character
+ *   - when escape_backslash_with_backslash is true, backslash is escaped with another backslash
  */
-template <char quote_character, bool escape_quote_with_quote = false>
+template <char quote_character, bool escape_quote_with_quote = false, bool escape_backslash_with_backslash = true>
 void writeAnyEscapedString(const char * begin, const char * end, WriteBuffer & buf)
 {
     const char * pos = begin;
@@ -360,7 +361,8 @@ void writeAnyEscapedString(const char * begin, const char * end, WriteBuffer & b
                     writeChar('0', buf);
                     break;
                 case '\\':
-                    writeChar('\\', buf);
+                    if constexpr (escape_backslash_with_backslash)
+                        writeChar('\\', buf);
                     writeChar('\\', buf);
                     break;
                 default:
@@ -466,6 +468,13 @@ inline void writeQuotedString(std::string_view ref, WriteBuffer & buf)
     writeAnyQuotedString<'\''>(ref.data(), ref.data() + ref.size(), buf);
 }
 
+inline void writeQuotedStringPostgreSQL(std::string_view ref, WriteBuffer & buf)
+{
+    writeChar('\'', buf);
+    writeAnyEscapedString<'\'', true, false>(ref.data(), ref.data() + ref.size(), buf);
+    writeChar('\'', buf);
+}
+
 inline void writeDoubleQuotedString(const String & s, WriteBuffer & buf)
 {
     writeAnyQuotedString<'"'>(s, buf);
diff --git a/src/Parsers/ASTLiteral.cpp b/src/Parsers/ASTLiteral.cpp
index 5c76f6f33bf..4a9a3d8df5b 100644
--- a/src/Parsers/ASTLiteral.cpp
+++ b/src/Parsers/ASTLiteral.cpp
@@ -93,7 +93,7 @@ void ASTLiteral::appendColumnNameImpl(WriteBuffer & ostr) const
 
 void ASTLiteral::appendColumnNameImplLegacy(WriteBuffer & ostr) const
 {
-     /// 100 - just arbitrary value.
+    /// 100 - just arbitrary value.
     constexpr auto min_elements_for_hashing = 100;
 
     /// Special case for very large arrays. Instead of listing all elements, will use hash of them.
@@ -118,9 +118,31 @@ void ASTLiteral::appendColumnNameImplLegacy(WriteBuffer & ostr) const
     }
 }
 
+/// Use different rules for escaping backslashes and quotes
+class FieldVisitorToStringPostgreSQL : public StaticVisitor<String>
+{
+public:
+    template<typename T>
+    String operator() (const T & x) const { return visitor(x); }
+
+private:
+    FieldVisitorToString visitor;
+};
+
+template<>
+String FieldVisitorToStringPostgreSQL::operator() (const String & x) const
+{
+    WriteBufferFromOwnString wb;
+    writeQuotedStringPostgreSQL(x, wb);
+    return wb.str();
+}
+
 void ASTLiteral::formatImplWithoutAlias(const FormatSettings & settings, IAST::FormatState &, IAST::FormatStateStacked) const
 {
-    settings.ostr << applyVisitor(FieldVisitorToString(), value);
+    if (settings.literal_escaping_style == LiteralEscapingStyle::Regular)
+        settings.ostr << applyVisitor(FieldVisitorToString(), value);
+    else
+        settings.ostr << applyVisitor(FieldVisitorToStringPostgreSQL(), value);
 }
 
 }
diff --git a/src/Parsers/IAST.h b/src/Parsers/IAST.h
index d217876459f..58bc9702142 100644
--- a/src/Parsers/IAST.h
+++ b/src/Parsers/IAST.h
@@ -3,6 +3,7 @@
 #include <base/types.h>
 #include <Parsers/IAST_fwd.h>
 #include <Parsers/IdentifierQuotingStyle.h>
+#include <Parsers/LiteralEscapingStyle.h>
 #include <Common/Exception.h>
 #include <Common/TypePromotion.h>
 #include <IO/WriteBufferFromString.h>
@@ -197,6 +198,7 @@ public:
         IdentifierQuotingStyle identifier_quoting_style;
         bool show_secrets; /// Show secret parts of the AST (e.g. passwords, encryption keys).
         char nl_or_ws; /// Newline or whitespace.
+        LiteralEscapingStyle literal_escaping_style;
 
         explicit FormatSettings(
             WriteBuffer & ostr_,
@@ -204,7 +206,8 @@ public:
             bool hilite_ = false,
             bool always_quote_identifiers_ = false,
             IdentifierQuotingStyle identifier_quoting_style_ = IdentifierQuotingStyle::Backticks,
-            bool show_secrets_ = true)
+            bool show_secrets_ = true,
+            LiteralEscapingStyle literal_escaping_style_ = LiteralEscapingStyle::Regular)
             : ostr(ostr_)
             , one_line(one_line_)
             , hilite(hilite_)
@@ -212,6 +215,7 @@ public:
             , identifier_quoting_style(identifier_quoting_style_)
             , show_secrets(show_secrets_)
             , nl_or_ws(one_line ? ' ' : '\n')
+            , literal_escaping_style(literal_escaping_style_)
         {
         }
 
@@ -223,6 +227,7 @@ public:
             , identifier_quoting_style(other.identifier_quoting_style)
             , show_secrets(other.show_secrets)
             , nl_or_ws(other.nl_or_ws)
+            , literal_escaping_style(other.literal_escaping_style)
         {
         }
 
diff --git a/src/Parsers/LiteralEscapingStyle.h b/src/Parsers/LiteralEscapingStyle.h
new file mode 100644
index 00000000000..10d4d84a85d
--- /dev/null
+++ b/src/Parsers/LiteralEscapingStyle.h
@@ -0,0 +1,14 @@
+#pragma once
+
+
+namespace DB
+{
+
+/// Method to escape single quotes.
+enum class LiteralEscapingStyle
+{
+    Regular,         /// Escape backslashes with backslash (\\) and quotes with backslash (\')
+    PostgreSQL,      /// Do not escape backslashes (\), escape quotes with quote ('')
+};
+
+}
diff --git a/src/Storages/StorageMySQL.cpp b/src/Storages/StorageMySQL.cpp
index b0a220eb1d2..76a439eabaf 100644
--- a/src/Storages/StorageMySQL.cpp
+++ b/src/Storages/StorageMySQL.cpp
@@ -104,6 +104,7 @@ Pipe StorageMySQL::read(
         column_names_,
         storage_snapshot->metadata->getColumns().getOrdinary(),
         IdentifierQuotingStyle::BackticksMySQL,
+        LiteralEscapingStyle::Regular,
         remote_database_name,
         remote_table_name,
         context_);
diff --git a/src/Storages/StoragePostgreSQL.cpp b/src/Storages/StoragePostgreSQL.cpp
index 11558b39ad3..f233d4ff213 100644
--- a/src/Storages/StoragePostgreSQL.cpp
+++ b/src/Storages/StoragePostgreSQL.cpp
@@ -123,11 +123,7 @@ Pipe StoragePostgreSQL::read(
         query_info_,
         column_names_,
         storage_snapshot->metadata->getColumns().getOrdinary(),
-        IdentifierQuotingStyle::DoubleQuotes, remote_table_schema, remote_table_name, context_);
-
-    /// Single quotes in PostgreSQL are escaped through repetition
-    boost::replace_all(query, "\\'", "''");
-
+        IdentifierQuotingStyle::DoubleQuotes, LiteralEscapingStyle::PostgreSQL, remote_table_schema, remote_table_name, context_);
     LOG_TRACE(log, "Query: {}", query);
 
     Block sample_block;
diff --git a/src/Storages/StorageSQLite.cpp b/src/Storages/StorageSQLite.cpp
index d5ae6f2383f..d5db5763da9 100644
--- a/src/Storages/StorageSQLite.cpp
+++ b/src/Storages/StorageSQLite.cpp
@@ -91,6 +91,7 @@ Pipe StorageSQLite::read(
         column_names,
         storage_snapshot->metadata->getColumns().getOrdinary(),
         IdentifierQuotingStyle::DoubleQuotes,
+        LiteralEscapingStyle::Regular,
         "",
         remote_table_name,
         context_);
diff --git a/src/Storages/StorageXDBC.cpp b/src/Storages/StorageXDBC.cpp
index b532d1c91f0..1715cde9d1e 100644
--- a/src/Storages/StorageXDBC.cpp
+++ b/src/Storages/StorageXDBC.cpp
@@ -79,6 +79,7 @@ std::function<void(std::ostream &)> StorageXDBC::getReadPOSTDataCallback(
         column_names,
         columns_description.getOrdinary(),
         bridge_helper->getIdentifierQuotingStyle(),
+        LiteralEscapingStyle::Regular,
         remote_database_name,
         remote_table_name,
         local_context);
diff --git a/src/Storages/transformQueryForExternalDatabase.cpp b/src/Storages/transformQueryForExternalDatabase.cpp
index 375510e62bf..84a696a1e9c 100644
--- a/src/Storages/transformQueryForExternalDatabase.cpp
+++ b/src/Storages/transformQueryForExternalDatabase.cpp
@@ -258,6 +258,7 @@ String transformQueryForExternalDatabaseImpl(
     Names used_columns,
     const NamesAndTypesList & available_columns,
     IdentifierQuotingStyle identifier_quoting_style,
+    LiteralEscapingStyle literal_escaping_style,
     const String & database,
     const String & table,
     ContextPtr context)
@@ -337,7 +338,8 @@ String transformQueryForExternalDatabaseImpl(
     IAST::FormatSettings settings(
             out, /*one_line*/ true, /*hilite*/ false,
             /*always_quote_identifiers*/ identifier_quoting_style != IdentifierQuotingStyle::None,
-            /*identifier_quoting_style*/ identifier_quoting_style);
+            /*identifier_quoting_style*/ identifier_quoting_style, /*show_secrets_*/ true,
+            /*literal_escaping_style*/ literal_escaping_style);
 
     select->format(settings);
 
@@ -351,6 +353,7 @@ String transformQueryForExternalDatabase(
     const Names & column_names,
     const NamesAndTypesList & available_columns,
     IdentifierQuotingStyle identifier_quoting_style,
+    LiteralEscapingStyle literal_escaping_style,
     const String & database,
     const String & table,
     ContextPtr context)
@@ -375,6 +378,7 @@ String transformQueryForExternalDatabase(
             column_names,
             available_columns,
             identifier_quoting_style,
+            literal_escaping_style,
             database,
             table,
             context);
@@ -386,6 +390,7 @@ String transformQueryForExternalDatabase(
         query_info.syntax_analyzer_result->requiredSourceColumns(),
         available_columns,
         identifier_quoting_style,
+        literal_escaping_style,
         database,
         table,
         context);
diff --git a/src/Storages/transformQueryForExternalDatabase.h b/src/Storages/transformQueryForExternalDatabase.h
index 0f2b0a5822f..fb6af21907e 100644
--- a/src/Storages/transformQueryForExternalDatabase.h
+++ b/src/Storages/transformQueryForExternalDatabase.h
@@ -31,6 +31,7 @@ String transformQueryForExternalDatabase(
     const Names & column_names,
     const NamesAndTypesList & available_columns,
     IdentifierQuotingStyle identifier_quoting_style,
+    LiteralEscapingStyle literal_escaping_style,
     const String & database,
     const String & table,
     ContextPtr context);
diff --git a/tests/integration/test_storage_postgresql/test.py b/tests/integration/test_storage_postgresql/test.py
index 3a36d050f17..d4f8fab3a82 100644
--- a/tests/integration/test_storage_postgresql/test.py
+++ b/tests/integration/test_storage_postgresql/test.py
@@ -726,18 +726,20 @@ def test_auto_close_connection(started_cluster):
     assert count == 2
 
 
-def test_single_quotes(started_cluster):
+def test_literal_escaping(started_cluster):
     cursor = started_cluster.postgres_conn.cursor()
-    cursor.execute(f"DROP TABLE IF EXISTS single_quote_fails")
-    cursor.execute(f"CREATE TABLE single_quote_fails(text varchar(255))")
+    cursor.execute(f"DROP TABLE IF EXISTS escaping")
+    cursor.execute(f"CREATE TABLE escaping(text varchar(255))")
     node1.query(
-        "CREATE TABLE default.single_quote_fails (text String) ENGINE = PostgreSQL('postgres1:5432', 'postgres', 'single_quote_fails', 'postgres', 'mysecretpassword')"
+        "CREATE TABLE default.escaping (text String) ENGINE = PostgreSQL('postgres1:5432', 'postgres', 'escaping', 'postgres', 'mysecretpassword')"
     )
-    node1.query("SELECT * FROM single_quote_fails WHERE text = ''''")
-    node1.query("SELECT * FROM single_quote_fails WHERE text = '\\''")
-    node1.query("SELECT * FROM single_quote_fails WHERE text like '%a''a%'")
-    node1.query("SELECT * FROM single_quote_fails WHERE text like '%a\\'a%'")
-    cursor.execute(f"DROP TABLE single_quote_fails")
+    node1.query("SELECT * FROM escaping WHERE text = ''''")  # ' -> ''
+    node1.query("SELECT * FROM escaping WHERE text = '\\''")  # ' -> ''
+    node1.query("SELECT * FROM escaping WHERE text = '\\\\\\''")  # \' -> \''
+    node1.query("SELECT * FROM escaping WHERE text = '\\\\\\''")  # \' -> \''
+    node1.query("SELECT * FROM escaping WHERE text like '%a''a%'")  # %a'a% -> %a''a%
+    node1.query("SELECT * FROM escaping WHERE text like '%a\\'a%'")  # %a'a% -> %a''a%
+    cursor.execute(f"DROP TABLE escaping")
 
 
 if __name__ == "__main__":

From 5cdeacf4cf61c0ac228eb63b7178392a6436d41c Mon Sep 17 00:00:00 2001
From: Ruslan Mardugalliamov <ruslan.mardugalliamov@gmail.com>
Date: Sun, 6 Aug 2023 15:33:36 -0400
Subject: [PATCH 182/249] Add hints for HTTP handlers

Add hints to HTTP handlers to help users avoid
misspellings. For example, if a user mistakenly writes
`/dashboad` instead of `/dashboard`, they will now get a
hint that /dashboard is the correct handler.

This change will improve the user experience by making it
easier for users to find the correct handlers.

 #47662
---
 programs/keeper/CMakeLists.txt                |  1 +
 src/Server/HTTPHandlerFactory.cpp             |  4 ++++
 src/Server/HTTPPathHints.cpp                  | 16 ++++++++++++++
 src/Server/HTTPPathHints.h                    | 22 +++++++++++++++++++
 src/Server/HTTPRequestHandlerFactoryMain.cpp  |  2 +-
 src/Server/HTTPRequestHandlerFactoryMain.h    |  4 ++++
 src/Server/NotFoundHandler.cpp                |  3 ++-
 src/Server/NotFoundHandler.h                  |  3 +++
 ...ggest_http_page_in_error_message.reference |  4 ++++
 ...2842_suggest_http_page_in_error_message.sh | 12 ++++++++++
 10 files changed, 69 insertions(+), 2 deletions(-)
 create mode 100644 src/Server/HTTPPathHints.cpp
 create mode 100644 src/Server/HTTPPathHints.h
 create mode 100644 tests/queries/0_stateless/02842_suggest_http_page_in_error_message.reference
 create mode 100755 tests/queries/0_stateless/02842_suggest_http_page_in_error_message.sh

diff --git a/programs/keeper/CMakeLists.txt b/programs/keeper/CMakeLists.txt
index 43a8d84b513..a43a312ba54 100644
--- a/programs/keeper/CMakeLists.txt
+++ b/programs/keeper/CMakeLists.txt
@@ -57,6 +57,7 @@ if (BUILD_STANDALONE_KEEPER)
 
         ${CMAKE_CURRENT_SOURCE_DIR}/../../src/IO/ReadBuffer.cpp
 
+        ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Server/HTTPPathHints.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Server/KeeperTCPHandler.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Server/TCPServer.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Server/NotFoundHandler.cpp
diff --git a/src/Server/HTTPHandlerFactory.cpp b/src/Server/HTTPHandlerFactory.cpp
index 78e374ee9e0..1c911034da1 100644
--- a/src/Server/HTTPHandlerFactory.cpp
+++ b/src/Server/HTTPHandlerFactory.cpp
@@ -132,21 +132,25 @@ void addCommonDefaultHandlersFactory(HTTPRequestHandlerFactoryMain & factory, IS
     auto ping_handler = std::make_shared<HandlingRuleHTTPHandlerFactory<StaticRequestHandler>>(server, ping_response_expression);
     ping_handler->attachStrictPath("/ping");
     ping_handler->allowGetAndHeadRequest();
+    factory.addPathToHints("/ping");
     factory.addHandler(ping_handler);
 
     auto replicas_status_handler = std::make_shared<HandlingRuleHTTPHandlerFactory<ReplicasStatusHandler>>(server);
     replicas_status_handler->attachNonStrictPath("/replicas_status");
     replicas_status_handler->allowGetAndHeadRequest();
+    factory.addPathToHints("/replicas_status");
     factory.addHandler(replicas_status_handler);
 
     auto play_handler = std::make_shared<HandlingRuleHTTPHandlerFactory<WebUIRequestHandler>>(server);
     play_handler->attachNonStrictPath("/play");
     play_handler->allowGetAndHeadRequest();
+    factory.addPathToHints("/play");
     factory.addHandler(play_handler);
 
     auto dashboard_handler = std::make_shared<HandlingRuleHTTPHandlerFactory<WebUIRequestHandler>>(server);
     dashboard_handler->attachNonStrictPath("/dashboard");
     dashboard_handler->allowGetAndHeadRequest();
+    factory.addPathToHints("/dashboard");
     factory.addHandler(dashboard_handler);
 
     auto js_handler = std::make_shared<HandlingRuleHTTPHandlerFactory<WebUIRequestHandler>>(server);
diff --git a/src/Server/HTTPPathHints.cpp b/src/Server/HTTPPathHints.cpp
new file mode 100644
index 00000000000..51ef3eabffe
--- /dev/null
+++ b/src/Server/HTTPPathHints.cpp
@@ -0,0 +1,16 @@
+#include <Server/HTTPPathHints.h>
+
+namespace DB
+{
+
+void HTTPPathHints::add(const String & http_path)
+{
+    http_paths.push_back(http_path);
+}
+
+std::vector<String> HTTPPathHints::getAllRegisteredNames() const
+{
+    return http_paths;
+}
+
+}
diff --git a/src/Server/HTTPPathHints.h b/src/Server/HTTPPathHints.h
new file mode 100644
index 00000000000..708816ebf07
--- /dev/null
+++ b/src/Server/HTTPPathHints.h
@@ -0,0 +1,22 @@
+#pragma once
+
+#include <base/types.h>
+
+#include <Common/NamePrompter.h>
+
+namespace DB
+{
+
+class HTTPPathHints : public IHints<1, HTTPPathHints>
+{
+public:
+    std::vector<String> getAllRegisteredNames() const override;
+    void add(const String & http_path);
+
+private:
+    std::vector<String> http_paths;
+};
+
+using HTTPPathHintsPtr = std::shared_ptr<HTTPPathHints>;
+
+}
diff --git a/src/Server/HTTPRequestHandlerFactoryMain.cpp b/src/Server/HTTPRequestHandlerFactoryMain.cpp
index 61a2909d30f..5481bcd5083 100644
--- a/src/Server/HTTPRequestHandlerFactoryMain.cpp
+++ b/src/Server/HTTPRequestHandlerFactoryMain.cpp
@@ -29,7 +29,7 @@ std::unique_ptr<HTTPRequestHandler> HTTPRequestHandlerFactoryMain::createRequest
         || request.getMethod() == Poco::Net::HTTPRequest::HTTP_HEAD
         || request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST)
     {
-        return std::unique_ptr<HTTPRequestHandler>(new NotFoundHandler);
+        return std::unique_ptr<HTTPRequestHandler>(new NotFoundHandler(hints.getHints(request.getURI())));
     }
 
     return nullptr;
diff --git a/src/Server/HTTPRequestHandlerFactoryMain.h b/src/Server/HTTPRequestHandlerFactoryMain.h
index b0e57bd6b3b..07b278d831c 100644
--- a/src/Server/HTTPRequestHandlerFactoryMain.h
+++ b/src/Server/HTTPRequestHandlerFactoryMain.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <Server/HTTP/HTTPRequestHandlerFactory.h>
+#include <Server/HTTPPathHints.h>
 
 #include <vector>
 
@@ -15,11 +16,14 @@ public:
 
     void addHandler(HTTPRequestHandlerFactoryPtr child_factory) { child_factories.emplace_back(child_factory); }
 
+    void addPathToHints(const std::string & http_path) { hints.add(http_path); }
+
     std::unique_ptr<HTTPRequestHandler> createRequestHandler(const HTTPServerRequest & request) override;
 
 private:
     Poco::Logger * log;
     std::string name;
+    HTTPPathHints hints;
 
     std::vector<HTTPRequestHandlerFactoryPtr> child_factories;
 };
diff --git a/src/Server/NotFoundHandler.cpp b/src/Server/NotFoundHandler.cpp
index 3181708b9b7..5b1db508551 100644
--- a/src/Server/NotFoundHandler.cpp
+++ b/src/Server/NotFoundHandler.cpp
@@ -10,7 +10,8 @@ void NotFoundHandler::handleRequest(HTTPServerRequest & request, HTTPServerRespo
     try
     {
         response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_NOT_FOUND);
-        *response.send() << "There is no handle " << request.getURI() << "\n\n"
+        *response.send() << "There is no handle " << request.getURI()
+                         << (!hints.empty() ? fmt::format(". Maybe you meant {}.", hints.front()) : "") << "\n\n"
                          << "Use / or /ping for health checks.\n"
                          << "Or /replicas_status for more sophisticated health checks.\n\n"
                          << "Send queries from your program with POST method or GET /?query=...\n\n"
diff --git a/src/Server/NotFoundHandler.h b/src/Server/NotFoundHandler.h
index 749ac388c4d..1cbfcd57f8f 100644
--- a/src/Server/NotFoundHandler.h
+++ b/src/Server/NotFoundHandler.h
@@ -9,7 +9,10 @@ namespace DB
 class NotFoundHandler : public HTTPRequestHandler
 {
 public:
+    NotFoundHandler(std::vector<std::string> hints_) : hints(std::move(hints_)) {}
     void handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) override;
+private:
+    std::vector<std::string> hints;
 };
 
 }
diff --git a/tests/queries/0_stateless/02842_suggest_http_page_in_error_message.reference b/tests/queries/0_stateless/02842_suggest_http_page_in_error_message.reference
new file mode 100644
index 00000000000..0025187be30
--- /dev/null
+++ b/tests/queries/0_stateless/02842_suggest_http_page_in_error_message.reference
@@ -0,0 +1,4 @@
+There is no handle /sashboards. Maybe you meant /dashboard
+There is no handle /sashboard. Maybe you meant /dashboard
+There is no handle /sashboarb. Maybe you meant /dashboard
+There is no handle /sashboaxb. Maybe you meant /dashboard
diff --git a/tests/queries/0_stateless/02842_suggest_http_page_in_error_message.sh b/tests/queries/0_stateless/02842_suggest_http_page_in_error_message.sh
new file mode 100755
index 00000000000..cf69c742777
--- /dev/null
+++ b/tests/queries/0_stateless/02842_suggest_http_page_in_error_message.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+export CLICKHOUSE_URL="${CLICKHOUSE_PORT_HTTP_PROTO}://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/"
+
+${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}sashboards" | grep -o ".* Maybe you meant /dashboard"
+${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}sashboard"  | grep -o ".* Maybe you meant /dashboard"
+${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}sashboarb"  | grep -o ".* Maybe you meant /dashboard"
+${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}sashboaxb"  | grep -o ".* Maybe you meant /dashboard"

From 4b30900fed3dcafa015a7d6dc4d6d91ea362a966 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@gmail.com>
Date: Thu, 10 Aug 2023 14:44:16 +0300
Subject: [PATCH 183/249] Revert "Improve CHECK TABLE system query"

---
 .../sql-reference/statements/check-table.md   | 107 +++-------
 src/Common/FileChecker.cpp                    |  42 ++--
 src/Common/FileChecker.h                      |  37 +---
 src/Interpreters/InterpreterCheckQuery.cpp    | 201 ++++--------------
 src/Storages/CheckResults.h                   |   2 +
 src/Storages/IStorage.cpp                     |  11 -
 src/Storages/IStorage.h                       |  41 +---
 src/Storages/StorageLog.cpp                   |   9 +-
 src/Storages/StorageLog.h                     |  16 +-
 src/Storages/StorageMergeTree.cpp             |  26 +--
 src/Storages/StorageMergeTree.h               |  29 +--
 src/Storages/StorageProxy.h                   |   4 +-
 src/Storages/StorageReplicatedMergeTree.cpp   |  37 ++--
 src/Storages/StorageReplicatedMergeTree.h     |  31 +--
 src/Storages/StorageStripeLog.cpp             |  10 +-
 src/Storages/StorageStripeLog.h               |  17 +-
 .../0_stateless/00063_check_query.reference   |   1 -
 .../queries/0_stateless/00063_check_query.sql |   3 -
 .../queries/0_stateless/00961_check_table.sql |  12 +-
 ...1042_check_query_and_last_granule_size.sql |  12 +-
 ...02235_check_table_sparse_serialization.sql |   3 +-
 .../02841_check_table_progress.reference      |   2 -
 .../0_stateless/02841_check_table_progress.sh |  29 ---
 23 files changed, 138 insertions(+), 544 deletions(-)
 delete mode 100644 tests/queries/0_stateless/02841_check_table_progress.reference
 delete mode 100755 tests/queries/0_stateless/02841_check_table_progress.sh

diff --git a/docs/en/sql-reference/statements/check-table.md b/docs/en/sql-reference/statements/check-table.md
index db8c32249ef..0209d59b018 100644
--- a/docs/en/sql-reference/statements/check-table.md
+++ b/docs/en/sql-reference/statements/check-table.md
@@ -5,38 +5,19 @@ sidebar_label: CHECK TABLE
 title: "CHECK TABLE Statement"
 ---
 
-The `CHECK TABLE` query in ClickHouse is used to perform a validation check on a specific table or its partitions. It ensures the integrity of the data by verifying the checksums and other internal data structures.
+Checks if the data in the table is corrupted.
 
-Particularly it compares actual file sizes with the expected values which are stored on the server. If the file sizes do not match the stored values, it means the data is corrupted. This can be caused, for example, by a system crash during query execution.
-
-:::note
-The `CHECK TABLE`` query may read all the data in the table and hold some resources, making it resource-intensive.
-Consider the potential impact on performance and resource utilization before executing this query.
-:::
-
-## Syntax
-
-The basic syntax of the query is as follows:
-
-```sql
-CHECK TABLE table_name [PARTITION partition_expression] [FORMAT format] [SETTINGS check_query_single_value_result = (0|1) [, other_settings]  ]
+``` sql
+CHECK TABLE [db.]name [PARTITION partition_expr]
 ```
 
-- `table_name`: Specifies the name of the table that you want to check.
-- `partition_expression`: (Optional) If you want to check a specific partition of the table, you can use this expression to specify the partition.
-- `FORMAT format`: (Optional) Allows you to specify the output format of the result.
-- `SETTINGS`: (Optional) Allows additional settings.
-	- **`check_query_single_value_result`**: (Optional) This setting allows you to toggle between a detailed result (`0`) or a summarized result (`1`).
-	- Other settings (e.g. `max_threads` can be applied as well).
+The `CHECK TABLE` query compares actual file sizes with the expected values which are stored on the server. If the file sizes do not match the stored values, it means the data is corrupted. This can be caused, for example, by a system crash during query execution.
 
+The query response contains the `result` column with a single row. The row has a value of
+[Boolean](../../sql-reference/data-types/boolean.md) type:
 
-The query response depends on the value of contains `check_query_single_value_result` setting.
-In case of `check_query_single_value_result = 1` only `result` column with a single row is returned. Value inside this row is `1` if the integrity check is passed and `0` if data is corrupted.
-
-With `check_query_single_value_result = 0` the query returns the following columns:
-    - `part_path`: Indicates the path to the data part or file name.
-    - `is_passed`: Returns 1 if the check for this part is successful, 0 otherwise.
-    - `message`: Any additional messages related to the check, such as errors or success messages.
+- 0 - The data in the table is corrupted.
+- 1 - The data maintains integrity.
 
 The `CHECK TABLE` query supports the following table engines:
 
@@ -45,15 +26,30 @@ The `CHECK TABLE` query supports the following table engines:
 - [StripeLog](../../engines/table-engines/log-family/stripelog.md)
 - [MergeTree family](../../engines/table-engines/mergetree-family/mergetree.md)
 
-Performed over the tables with another table engines causes an `NOT_IMPLEMETED` exception.
+Performed over the tables with another table engines causes an exception.
 
 Engines from the `*Log` family do not provide automatic data recovery on failure. Use the `CHECK TABLE` query to track data loss in a timely manner.
 
-## Examples
+## Checking the MergeTree Family Tables
 
-By default `CHECK TABLE` query shows the general table check status:
+For `MergeTree` family engines, if [check_query_single_value_result](../../operations/settings/settings.md#check_query_single_value_result) = 0, the `CHECK TABLE` query shows a check status for every individual data part of a table on the local server.
 
 ```sql
+SET check_query_single_value_result = 0;
+CHECK TABLE test_table;
+```
+
+```text
+┌─part_path─┬─is_passed─┬─message─┐
+│ all_1_4_1 │         1 │         │
+│ all_1_4_2 │         1 │         │
+└───────────┴───────────┴─────────┘
+```
+
+If `check_query_single_value_result` = 1, the `CHECK TABLE` query shows the general table check status.
+
+```sql
+SET check_query_single_value_result = 1;
 CHECK TABLE test_table;
 ```
 
@@ -63,60 +59,11 @@ CHECK TABLE test_table;
 └────────┘
 ```
 
-If you want to see the check status for every individual data part you may use `check_query_single_value_result` setting.
-
-Also, to check a specific partition of the table, you can use the `PARTITION` keyword.
-
-```sql
-CHECK TABLE t0 PARTITION ID '201003'
-FORMAT PrettyCompactMonoBlock
-SETTINGS check_query_single_value_result = 0
-```
-
-Output:
-
-```text
-┌─part_path────┬─is_passed─┬─message─┐
-│ 201003_7_7_0 │         1 │         │
-│ 201003_3_3_0 │         1 │         │
-└──────────────┴───────────┴─────────┘
-```
-
-### Receiving a 'Corrupted' Result
-
-:::warning
-Disclaimer: The procedure described here, including the manual manipulating or removing files directly from the data directory, is for experimental or development environments only. Do **not** attempt this on a production server, as it may lead to data loss or other unintended consequences.
-:::
-
-Remove the existing checksum file:
-
-```bash
-rm /var/lib/clickhouse-server/data/default/t0/201003_3_3_0/checksums.txt
-```
-
-```sql
-CHECK TABLE t0 PARTITION ID '201003'
-FORMAT PrettyCompactMonoBlock
-SETTINGS check_query_single_value_result = 0
-
-
-Output:
-
-```text
-┌─part_path────┬─is_passed─┬─message──────────────────────────────────┐
-│ 201003_7_7_0 │         1 │                                          │
-│ 201003_3_3_0 │         1 │ Checksums recounted and written to disk. │
-└──────────────┴───────────┴──────────────────────────────────────────┘
-```
-
-If the checksums.txt file is missing, it can be restored. It will be recalculated and rewritten during the execution of the CHECK TABLE command for the specific partition, and the status will still be reported as 'success.'"
-
-
 ## If the Data Is Corrupted
 
 If the table is corrupted, you can copy the non-corrupted data to another table. To do this:
 
 1.  Create a new table with the same structure as damaged table. To do this execute the query `CREATE TABLE <new_table_name> AS <damaged_table_name>`.
-2.  Set the `max_threads` value to 1 to process the next query in a single thread. To do this run the query `SET max_threads = 1`.
+2.  Set the [max_threads](../../operations/settings/settings.md#settings-max_threads) value to 1 to process the next query in a single thread. To do this run the query `SET max_threads = 1`.
 3.  Execute the query `INSERT INTO <new_table_name> SELECT * FROM <damaged_table_name>`. This request copies the non-corrupted data from the damaged table to another table. Only the data before the corrupted part will be copied.
 4.  Restart the `clickhouse-client` to reset the `max_threads` value.
diff --git a/src/Common/FileChecker.cpp b/src/Common/FileChecker.cpp
index 122ea83835d..876bc4e641c 100644
--- a/src/Common/FileChecker.cpp
+++ b/src/Common/FileChecker.cpp
@@ -82,35 +82,33 @@ size_t FileChecker::getTotalSize() const
 }
 
 
-FileChecker::DataValidationTasksPtr FileChecker::getDataValidationTasks()
+CheckResults FileChecker::check() const
 {
-    return std::make_unique<DataValidationTasks>(map);
-}
-
-CheckResult FileChecker::checkNextEntry(DataValidationTasksPtr & check_data_tasks, bool & has_nothing_to_do) const
-{
-    String name;
-    size_t expected_size;
-    bool is_finished = check_data_tasks->next(name, expected_size);
-    if (is_finished)
-    {
-        has_nothing_to_do = true;
+    if (map.empty())
         return {};
-    }
 
-    String path = parentPath(files_info_path) + name;
-    bool exists = fileReallyExists(path);
-    auto real_size = exists ? getRealFileSize(path) : 0;  /// No race condition assuming no one else is working with these files.
+    CheckResults results;
 
-    if (real_size != expected_size)
+    for (const auto & name_size : map)
     {
-        String failure_message = exists
-            ? ("Size of " + path + " is wrong. Size is " + toString(real_size) + " but should be " + toString(expected_size))
-            : ("File " + path + " doesn't exist");
-        return CheckResult(name, false, failure_message);
+        const String & name = name_size.first;
+        String path = parentPath(files_info_path) + name;
+        bool exists = fileReallyExists(path);
+        auto real_size = exists ? getRealFileSize(path) : 0;  /// No race condition assuming no one else is working with these files.
+
+        if (real_size != name_size.second)
+        {
+            String failure_message = exists
+                ? ("Size of " + path + " is wrong. Size is " + toString(real_size) + " but should be " + toString(name_size.second))
+                : ("File " + path + " doesn't exist");
+            results.emplace_back(name, false, failure_message);
+            break;
+        }
+
+        results.emplace_back(name, true, "");
     }
 
-    return CheckResult(name, true, "");
+    return results;
 }
 
 void FileChecker::repair()
diff --git a/src/Common/FileChecker.h b/src/Common/FileChecker.h
index 8ffc310b84d..bb0383e4b56 100644
--- a/src/Common/FileChecker.h
+++ b/src/Common/FileChecker.h
@@ -3,7 +3,6 @@
 #include <Storages/CheckResults.h>
 #include <map>
 #include <base/types.h>
-#include <mutex>
 
 namespace Poco { class Logger; }
 
@@ -29,11 +28,7 @@ public:
     bool empty() const { return map.empty(); }
 
     /// Check the files whose parameters are specified in sizes.json
-    /// See comment in IStorage::checkDataNext
-    struct DataValidationTasks;
-    using DataValidationTasksPtr = std::unique_ptr<DataValidationTasks>;
-    DataValidationTasksPtr getDataValidationTasks();
-    CheckResult checkNextEntry(DataValidationTasksPtr & check_data_tasks, bool & has_nothing_to_do) const;
+    CheckResults check() const;
 
     /// Truncate files that have excessive size to the expected size.
     /// Throw exception if the file size is less than expected.
@@ -46,36 +41,6 @@ public:
     /// Returns total size of all files.
     size_t getTotalSize() const;
 
-    struct DataValidationTasks
-    {
-        DataValidationTasks(const std::map<String, size_t> & map_)
-            : map(map_), it(map.begin())
-        {}
-
-        bool next(String & out_name, size_t & out_size)
-        {
-            std::lock_guard lock(mutex);
-            if (it == map.end())
-                return true;
-            out_name = it->first;
-            out_size = it->second;
-            ++it;
-            return false;
-        }
-
-        size_t size() const
-        {
-            std::lock_guard lock(mutex);
-            return std::distance(it, map.end());
-        }
-
-        const std::map<String, size_t> & map;
-
-        mutable std::mutex mutex;
-        using Iterator = std::map<String, size_t>::const_iterator;
-        Iterator it;
-    };
-
 private:
     void load();
 
diff --git a/src/Interpreters/InterpreterCheckQuery.cpp b/src/Interpreters/InterpreterCheckQuery.cpp
index bd530654dd2..333aed84873 100644
--- a/src/Interpreters/InterpreterCheckQuery.cpp
+++ b/src/Interpreters/InterpreterCheckQuery.cpp
@@ -8,201 +8,72 @@
 #include <DataTypes/DataTypeString.h>
 #include <Columns/ColumnsNumber.h>
 #include <Common/typeid_cast.h>
-#include <Interpreters/ProcessList.h>
 #include <algorithm>
 
-#include <Columns/IColumn.h>
-
-#include <Processors/ResizeProcessor.h>
-#include <Processors/IAccumulatingTransform.h>
-
 
 namespace DB
 {
 
-namespace ErrorCodes
-{
-    extern const int LOGICAL_ERROR;
-}
-
 namespace
 {
 
-Block getSingleValueBlock(UInt8 value)
+NamesAndTypes getBlockStructure()
 {
-    return Block{{ColumnUInt8::create(1, value), std::make_shared<DataTypeUInt8>(), "result"}};
-}
-
-Block getHeaderForCheckResult()
-{
-    auto names_and_types = NamesAndTypes{
+    return {
         {"part_path", std::make_shared<DataTypeString>()},
         {"is_passed", std::make_shared<DataTypeUInt8>()},
         {"message", std::make_shared<DataTypeString>()},
     };
-
-    return Block({
-        {names_and_types[0].type->createColumn(), names_and_types[0].type, names_and_types[0].name},
-        {names_and_types[1].type->createColumn(), names_and_types[1].type, names_and_types[1].name},
-        {names_and_types[2].type->createColumn(), names_and_types[2].type, names_and_types[2].name},
-    });
 }
 
-Chunk getChunkFromCheckResult(const CheckResult & check_result)
-{
-    MutableColumns columns = getHeaderForCheckResult().cloneEmptyColumns();
-    columns[0]->insert(check_result.fs_path);
-    columns[1]->insert(static_cast<UInt8>(check_result.success));
-    columns[2]->insert(check_result.failure_message);
-    return Chunk(std::move(columns), 1);
-}
-
-class TableCheckWorkerProcessor : public ISource
-{
-
-public:
-    TableCheckWorkerProcessor(IStorage::DataValidationTasksPtr check_data_tasks_, StoragePtr table_)
-        : ISource(getHeaderForCheckResult())
-        , table(table_)
-        , check_data_tasks(check_data_tasks_)
-    {
-    }
-
-    String getName() const override { return "TableCheckWorkerProcessor"; }
-
-protected:
-
-    std::optional<Chunk> tryGenerate() override
-    {
-        bool has_nothing_to_do = false;
-        auto check_result = table->checkDataNext(check_data_tasks, has_nothing_to_do);
-        if (has_nothing_to_do)
-            return {};
-
-        /// We can omit manual `progess` call, ISource will may count it automatically by returned chunk
-        /// However, we want to report only rows in progress
-        progress(1, 0);
-
-        if (!check_result.success)
-        {
-            LOG_WARNING(&Poco::Logger::get("InterpreterCheckQuery"),
-                "Check query for table {} failed, path {}, reason: {}",
-                table->getStorageID().getNameForLogs(),
-                check_result.fs_path,
-                check_result.failure_message);
-        }
-
-        return getChunkFromCheckResult(check_result);
-    }
-
-private:
-    StoragePtr table;
-    IStorage::DataValidationTasksPtr check_data_tasks;
-};
-
-class TableCheckResultEmitter : public IAccumulatingTransform
-{
-public:
-    TableCheckResultEmitter() : IAccumulatingTransform(getHeaderForCheckResult(), getSingleValueBlock(1).cloneEmpty()) {}
-
-    String getName() const override { return "TableCheckResultEmitter"; }
-
-    void consume(Chunk chunk) override
-    {
-        if (result_value == 0)
-            return;
-
-        auto columns = chunk.getColumns();
-        if (columns.size() != 3)
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Wrong number of columns: {}", columns.size());
-
-        const auto * col = checkAndGetColumn<ColumnUInt8>(columns[1].get());
-        for (size_t i = 0; i < col->size(); ++i)
-        {
-            if (col->getElement(i) == 0)
-            {
-                result_value = 0;
-                return;
-            }
-        }
-    }
-
-    Chunk generate() override
-    {
-        if (is_value_emitted.exchange(true))
-            return {};
-        auto block = getSingleValueBlock(result_value);
-        return Chunk(block.getColumns(), block.rows());
-    }
-
-private:
-    std::atomic<UInt8> result_value{1};
-    std::atomic_bool is_value_emitted{false};
-};
-
 }
 
-InterpreterCheckQuery::InterpreterCheckQuery(const ASTPtr & query_ptr_, ContextPtr context_)
-    : WithContext(context_)
-    , query_ptr(query_ptr_)
+
+InterpreterCheckQuery::InterpreterCheckQuery(const ASTPtr & query_ptr_, ContextPtr context_) : WithContext(context_), query_ptr(query_ptr_)
 {
 }
 
+
 BlockIO InterpreterCheckQuery::execute()
 {
     const auto & check = query_ptr->as<ASTCheckQuery &>();
-    const auto & context = getContext();
-    auto table_id = context->resolveStorageID(check, Context::ResolveOrdinary);
+    auto table_id = getContext()->resolveStorageID(check, Context::ResolveOrdinary);
 
-    context->checkAccess(AccessType::SHOW_TABLES, table_id);
-    StoragePtr table = DatabaseCatalog::instance().getTable(table_id, context);
+    getContext()->checkAccess(AccessType::SHOW_TABLES, table_id);
+    StoragePtr table = DatabaseCatalog::instance().getTable(table_id, getContext());
+    auto check_results = table->checkData(query_ptr, getContext());
 
-    auto check_data_tasks = table->getCheckTaskList(query_ptr, context);
+    Block block;
+    if (getContext()->getSettingsRef().check_query_single_value_result)
+    {
+        bool result = std::all_of(check_results.begin(), check_results.end(), [] (const CheckResult & res) { return res.success; });
+        auto column = ColumnUInt8::create();
+        column->insertValue(static_cast<UInt64>(result));
+        block = Block{{std::move(column), std::make_shared<DataTypeUInt8>(), "result"}};
+    }
+    else
+    {
+        auto block_structure = getBlockStructure();
+        auto path_column = block_structure[0].type->createColumn();
+        auto is_passed_column = block_structure[1].type->createColumn();
+        auto message_column = block_structure[2].type->createColumn();
 
-    const auto & settings = context->getSettingsRef();
+        for (const auto & check_result : check_results)
+        {
+            path_column->insert(check_result.fs_path);
+            is_passed_column->insert(static_cast<UInt8>(check_result.success));
+            message_column->insert(check_result.failure_message);
+        }
+
+        block = Block({
+            {std::move(path_column), block_structure[0].type, block_structure[0].name},
+            {std::move(is_passed_column), block_structure[1].type, block_structure[1].name},
+            {std::move(message_column), block_structure[2].type, block_structure[2].name}});
+    }
 
     BlockIO res;
-    {
-        auto processors = std::make_shared<Processors>();
+    res.pipeline = QueryPipeline(std::make_shared<SourceFromSingleChunk>(std::move(block)));
 
-        std::vector<OutputPort *> worker_ports;
-
-        size_t num_streams = std::max<size_t>(1, settings.max_threads);
-
-        for (size_t i = 0; i < num_streams; ++i)
-        {
-            auto worker_processor = std::make_shared<TableCheckWorkerProcessor>(check_data_tasks, table);
-            if (i == 0)
-                worker_processor->addTotalRowsApprox(check_data_tasks->size());
-            worker_ports.emplace_back(&worker_processor->getPort());
-            processors->emplace_back(worker_processor);
-        }
-
-        OutputPort * resize_outport;
-        {
-            auto resize_processor = std::make_shared<ResizeProcessor>(getHeaderForCheckResult(), worker_ports.size(), 1);
-
-            auto & resize_inputs = resize_processor->getInputs();
-            auto resize_inport_it = resize_inputs.begin();
-            for (size_t i = 0; i < worker_ports.size(); ++i, ++resize_inport_it)
-                connect(*worker_ports[i], *resize_inport_it);
-
-            resize_outport = &resize_processor->getOutputs().front();
-            processors->emplace_back(resize_processor);
-        }
-
-        if (settings.check_query_single_value_result)
-        {
-            auto emitter_processor = std::make_shared<TableCheckResultEmitter>();
-            auto * input_port = &emitter_processor->getInputPort();
-            processors->emplace_back(emitter_processor);
-
-            connect(*resize_outport, *input_port);
-        }
-
-        res.pipeline = QueryPipeline(Pipe(std::move(processors)));
-        res.pipeline.setNumThreads(num_streams);
-    }
     return res;
 }
 
diff --git a/src/Storages/CheckResults.h b/src/Storages/CheckResults.h
index 2e4652fea29..b342b014fa4 100644
--- a/src/Storages/CheckResults.h
+++ b/src/Storages/CheckResults.h
@@ -22,4 +22,6 @@ struct CheckResult
     {}
 };
 
+using CheckResults = std::vector<CheckResult>;
+
 }
diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp
index 85299f63165..ae7659e074f 100644
--- a/src/Storages/IStorage.cpp
+++ b/src/Storages/IStorage.cpp
@@ -273,17 +273,6 @@ bool IStorage::isStaticStorage() const
     return false;
 }
 
-IStorage::DataValidationTasksPtr IStorage::getCheckTaskList(const ASTPtr & /* query */, ContextPtr /* context */)
-{
-    throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Check query is not supported for {} storage", getName());
-}
-
-CheckResult IStorage::checkDataNext(DataValidationTasksPtr & /* check_task_list */, bool & has_nothing_to_do)
-{
-    has_nothing_to_do = true;
-    return {};
-}
-
 void IStorage::adjustCreateQueryForBackup(ASTPtr &) const
 {
 }
diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h
index b1e20c55782..ec92f57aeda 100644
--- a/src/Storages/IStorage.h
+++ b/src/Storages/IStorage.h
@@ -595,45 +595,8 @@ public:
     /// Provides a hint that the storage engine may evaluate the IN-condition by using an index.
     virtual bool mayBenefitFromIndexForIn(const ASTPtr & /* left_in_operand */, ContextPtr /* query_context */, const StorageMetadataPtr & /* metadata_snapshot */) const { return false; }
 
-
-    /** A list of tasks to check a validity of data.
-      * Each IStorage implementation may interpret this task in its own way.
-      * E.g. for some storages it to check data it need to check a list of files in filesystem, for others it can be a list of parts.
-      * Also it may hold resources (e.g. locks) required during check.
-      */
-    struct DataValidationTasksBase
-    {
-        /// Number of entries left to check.
-        /// It decreases after each call to checkDataNext().
-        virtual size_t size() const = 0;
-        virtual ~DataValidationTasksBase() = default;
-    };
-
-    using DataValidationTasksPtr = std::shared_ptr<DataValidationTasksBase>;
-
-    virtual DataValidationTasksPtr getCheckTaskList(const ASTPtr & /* query */, ContextPtr /* context */);
-
-    /** Executes one task from the list.
-      * If no tasks left, sets has_nothing_to_do to true.
-      * Note: Function `checkDataNext` is accessing `check_task_list` thread-safely,
-      *   and can be called simultaneously for the same `getCheckTaskList` result
-      *   to process different tasks in parallel.
-      * Usage:
-      *
-      * auto check_task_list = storage.getCheckTaskList(query, context);
-      * size_t total_tasks = check_task_list->size();
-      * while (true)
-      * {
-      *     size_t tasks_left = check_task_list->size();
-      *     std::cout << "Checking data: " << (total_tasks - tasks_left) << " / " << total_tasks << " tasks done." << std::endl;
-      *     bool has_nothing_to_do = false;
-      *     auto result = storage.checkDataNext(check_task_list, has_nothing_to_do);
-      *     if (has_nothing_to_do)
-      *         break;
-      *     doSomething(result);
-      * }
-      */
-    virtual CheckResult checkDataNext(DataValidationTasksPtr & check_task_list, bool & has_nothing_to_do);
+    /// Checks validity of the data
+    virtual CheckResults checkData(const ASTPtr & /* query */, ContextPtr /* context */) { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Check query is not supported for {} storage", getName()); }
 
     /// Checks that table could be dropped right now
     /// Otherwise - throws an exception with detailed information.
diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp
index 0e9f83e886a..87aa71f3e8d 100644
--- a/src/Storages/StorageLog.cpp
+++ b/src/Storages/StorageLog.cpp
@@ -866,18 +866,15 @@ SinkToStoragePtr StorageLog::write(const ASTPtr & /*query*/, const StorageMetada
     return std::make_shared<LogSink>(*this, metadata_snapshot, std::move(lock));
 }
 
-IStorage::DataValidationTasksPtr StorageLog::getCheckTaskList(const ASTPtr & /* query */, ContextPtr local_context)
+CheckResults StorageLog::checkData(const ASTPtr & /* query */, ContextPtr local_context)
 {
     ReadLock lock{rwlock, getLockTimeout(local_context)};
     if (!lock)
         throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Lock timeout exceeded");
-    return std::make_unique<DataValidationTasks>(file_checker.getDataValidationTasks(), std::move(lock));
+
+    return file_checker.check();
 }
 
-CheckResult StorageLog::checkDataNext(DataValidationTasksPtr & check_task_list, bool & has_nothing_to_do)
-{
-    return file_checker.checkNextEntry(assert_cast<DataValidationTasks *>(check_task_list.get())->file_checker_tasks, has_nothing_to_do);
-}
 
 IStorage::ColumnSizeByName StorageLog::getColumnSizes() const
 {
diff --git a/src/Storages/StorageLog.h b/src/Storages/StorageLog.h
index 95f95088aa2..f1d05ed39ac 100644
--- a/src/Storages/StorageLog.h
+++ b/src/Storages/StorageLog.h
@@ -59,8 +59,7 @@ public:
 
     void rename(const String & new_path_to_table_data, const StorageID & new_table_id) override;
 
-    DataValidationTasksPtr getCheckTaskList(const ASTPtr & query, ContextPtr context) override;
-    CheckResult checkDataNext(DataValidationTasksPtr & check_task_list, bool & has_nothing_to_do) override;
+    CheckResults checkData(const ASTPtr & query, ContextPtr local_context) override;
 
     void truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr, TableExclusiveLockHolder &) override;
 
@@ -143,19 +142,6 @@ private:
     std::atomic<UInt64> total_rows = 0;
     std::atomic<UInt64> total_bytes = 0;
 
-    struct DataValidationTasks : public IStorage::DataValidationTasksBase
-    {
-        DataValidationTasks(FileChecker::DataValidationTasksPtr file_checker_tasks_, ReadLock && lock_)
-            : file_checker_tasks(std::move(file_checker_tasks_)), lock(std::move(lock_))
-        {}
-
-        size_t size() const override { return file_checker_tasks->size(); }
-
-        FileChecker::DataValidationTasksPtr file_checker_tasks;
-        /// Lock to prevent table modification while checking
-        ReadLock lock;
-    };
-
     FileChecker file_checker;
 
     const size_t max_compress_block_size;
diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp
index 97fc7a6731f..ad9013d9f13 100644
--- a/src/Storages/StorageMergeTree.cpp
+++ b/src/Storages/StorageMergeTree.cpp
@@ -2197,8 +2197,9 @@ void StorageMergeTree::onActionLockRemove(StorageActionBlockType action_type)
         background_moves_assignee.trigger();
 }
 
-IStorage::DataValidationTasksPtr StorageMergeTree::getCheckTaskList(const ASTPtr & query, ContextPtr local_context)
+CheckResults StorageMergeTree::checkData(const ASTPtr & query, ContextPtr local_context)
 {
+    CheckResults results;
     DataPartsVector data_parts;
     if (const auto & check_query = query->as<ASTCheckQuery &>(); check_query.partition)
     {
@@ -2208,14 +2209,7 @@ IStorage::DataValidationTasksPtr StorageMergeTree::getCheckTaskList(const ASTPtr
     else
         data_parts = getVisibleDataPartsVector(local_context);
 
-    return std::make_unique<DataValidationTasks>(std::move(data_parts), local_context);
-}
-
-CheckResult StorageMergeTree::checkDataNext(DataValidationTasksPtr & check_task_list, bool & has_nothing_to_do)
-{
-    auto * data_validation_tasks = assert_cast<DataValidationTasks *>(check_task_list.get());
-    auto local_context = data_validation_tasks->context;
-    if (auto part = data_validation_tasks->next())
+    for (auto & part : data_parts)
     {
         /// If the checksums file is not present, calculate the checksums and write them to disk.
         static constexpr auto checksums_path = "checksums.txt";
@@ -2230,12 +2224,12 @@ CheckResult StorageMergeTree::checkDataNext(DataValidationTasksPtr & check_task_
                 part_mutable.writeChecksums(part->checksums, local_context->getWriteSettings());
 
                 part->checkMetadata();
-                return CheckResult(part->name, true, "Checksums recounted and written to disk.");
+                results.emplace_back(part->name, true, "Checksums recounted and written to disk.");
             }
             catch (const Exception & ex)
             {
                 tryLogCurrentException(log, __PRETTY_FUNCTION__);
-                return CheckResult(part->name, false, "Check of part finished with error: '" + ex.message() + "'");
+                results.emplace_back(part->name, false, "Check of part finished with error: '" + ex.message() + "'");
             }
         }
         else
@@ -2244,19 +2238,15 @@ CheckResult StorageMergeTree::checkDataNext(DataValidationTasksPtr & check_task_
             {
                 checkDataPart(part, true);
                 part->checkMetadata();
-                return CheckResult(part->name, true, "");
+                results.emplace_back(part->name, true, "");
             }
             catch (const Exception & ex)
             {
-                return CheckResult(part->name, false, ex.message());
+                results.emplace_back(part->name, false, ex.message());
             }
         }
     }
-    else
-    {
-        has_nothing_to_do = true;
-        return {};
-    }
+    return results;
 }
 
 
diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h
index ec4796e4941..c77e5140d75 100644
--- a/src/Storages/StorageMergeTree.h
+++ b/src/Storages/StorageMergeTree.h
@@ -108,8 +108,7 @@ public:
 
     void onActionLockRemove(StorageActionBlockType action_type) override;
 
-    DataValidationTasksPtr getCheckTaskList(const ASTPtr & query, ContextPtr context) override;
-    CheckResult checkDataNext(DataValidationTasksPtr & check_task_list, bool & has_nothing_to_do) override;
+    CheckResults checkData(const ASTPtr & query, ContextPtr context) override;
 
     bool scheduleDataProcessingJob(BackgroundJobsAssignee & assignee) override;
 
@@ -279,32 +278,6 @@ private:
     friend class MergePlainMergeTreeTask;
     friend class MutatePlainMergeTreeTask;
 
-    struct DataValidationTasks : public IStorage::DataValidationTasksBase
-    {
-        DataValidationTasks(DataPartsVector && parts_, ContextPtr context_)
-            : parts(std::move(parts_)), it(parts.begin()), context(std::move(context_))
-        {}
-
-        DataPartPtr next()
-        {
-            std::lock_guard lock(mutex);
-            if (it == parts.end())
-                return nullptr;
-            return *(it++);
-        }
-
-        size_t size() const override
-        {
-            std::lock_guard lock(mutex);
-            return std::distance(it, parts.end());
-        }
-
-        mutable std::mutex mutex;
-        DataPartsVector parts;
-        DataPartsVector::const_iterator it;
-
-        ContextPtr context;
-    };
 
 protected:
     std::map<int64_t, MutationCommands> getAlterMutationCommandsForPart(const DataPartPtr & part) const override;
diff --git a/src/Storages/StorageProxy.h b/src/Storages/StorageProxy.h
index ea908bea032..21ed4b91c62 100644
--- a/src/Storages/StorageProxy.h
+++ b/src/Storages/StorageProxy.h
@@ -149,9 +149,7 @@ public:
         return getNested()->mayBenefitFromIndexForIn(left_in_operand, query_context, metadata_snapshot);
     }
 
-    DataValidationTasksPtr getCheckTaskList(const ASTPtr & query, ContextPtr context) override { return getNested()->getCheckTaskList(query, context); }
-    CheckResult checkDataNext(DataValidationTasksPtr & check_task_list, bool & has_nothing_to_do) override { return getNested()->checkDataNext(check_task_list, has_nothing_to_do); }
-
+    CheckResults checkData(const ASTPtr & query, ContextPtr context) override { return getNested()->checkData(query, context); }
     void checkTableCanBeDropped() const override { getNested()->checkTableCanBeDropped(); }
     bool storesDataOnDisk() const override { return getNested()->storesDataOnDisk(); }
     Strings getDataPaths() const override { return getNested()->getDataPaths(); }
diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index c08f1ebcc48..7fce373e26b 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -8481,8 +8481,9 @@ void StorageReplicatedMergeTree::enqueuePartForCheck(const String & part_name, t
     part_check_thread.enqueuePart(part_name, delay_to_check_seconds);
 }
 
-IStorage::DataValidationTasksPtr StorageReplicatedMergeTree::getCheckTaskList(const ASTPtr & query, ContextPtr local_context)
+CheckResults StorageReplicatedMergeTree::checkData(const ASTPtr & query, ContextPtr local_context)
 {
+    CheckResults results;
     DataPartsVector data_parts;
     if (const auto & check_query = query->as<ASTCheckQuery &>(); check_query.partition)
     {
@@ -8492,30 +8493,24 @@ IStorage::DataValidationTasksPtr StorageReplicatedMergeTree::getCheckTaskList(co
     else
         data_parts = getVisibleDataPartsVector(local_context);
 
-    auto part_check_lock = part_check_thread.pausePartsCheck();
-    return std::make_unique<DataValidationTasks>(std::move(data_parts), std::move(part_check_lock));
-}
-
-CheckResult StorageReplicatedMergeTree::checkDataNext(DataValidationTasksPtr & check_task_list, bool & has_nothing_to_do)
-{
-
-    if (auto part = assert_cast<DataValidationTasks *>(check_task_list.get())->next())
     {
-        try
+        auto part_check_lock = part_check_thread.pausePartsCheck();
+
+        for (auto & part : data_parts)
         {
-            return CheckResult(part_check_thread.checkPartAndFix(part->name));
-        }
-        catch (const Exception & ex)
-        {
-            tryLogCurrentException(log, __PRETTY_FUNCTION__);
-            return CheckResult(part->name, false, "Check of part finished with error: '" + ex.message() + "'");
+            try
+            {
+                results.push_back(part_check_thread.checkPartAndFix(part->name));
+            }
+            catch (const Exception & ex)
+            {
+                tryLogCurrentException(log, __PRETTY_FUNCTION__);
+                results.emplace_back(part->name, false, "Check of part finished with error: '" + ex.message() + "'");
+            }
         }
     }
-    else
-    {
-        has_nothing_to_do = true;
-        return {};
-    }
+
+    return results;
 }
 
 
diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h
index 2bc18aa3b0a..78ef39f032f 100644
--- a/src/Storages/StorageReplicatedMergeTree.h
+++ b/src/Storages/StorageReplicatedMergeTree.h
@@ -230,8 +230,7 @@ public:
     /// Add a part to the queue of parts whose data you want to check in the background thread.
     void enqueuePartForCheck(const String & part_name, time_t delay_to_check_seconds = 0);
 
-    DataValidationTasksPtr getCheckTaskList(const ASTPtr & query, ContextPtr context) override;
-    CheckResult checkDataNext(DataValidationTasksPtr & check_task_list, bool & has_nothing_to_do) override;
+    CheckResults checkData(const ASTPtr & query, ContextPtr context) override;
 
     /// Checks ability to use granularity
     bool canUseAdaptiveGranularity() const override;
@@ -991,34 +990,6 @@ private:
     bool waitZeroCopyLockToDisappear(const ZeroCopyLock & lock, size_t milliseconds_to_wait) override;
 
     void startupImpl(bool from_attach_thread);
-
-    struct DataValidationTasks : public IStorage::DataValidationTasksBase
-    {
-        explicit DataValidationTasks(DataPartsVector && parts_, std::unique_lock<std::mutex> && parts_check_lock_)
-            : parts_check_lock(std::move(parts_check_lock_)), parts(std::move(parts_)), it(parts.begin())
-        {}
-
-        DataPartPtr next()
-        {
-            std::lock_guard lock(mutex);
-            if (it == parts.end())
-                return nullptr;
-            return *(it++);
-        }
-
-        size_t size() const override
-        {
-            std::lock_guard lock(mutex);
-            return std::distance(it, parts.end());
-        }
-
-        std::unique_lock<std::mutex> parts_check_lock;
-
-        mutable std::mutex mutex;
-        DataPartsVector parts;
-        DataPartsVector::const_iterator it;
-    };
-
 };
 
 String getPartNamePossiblyFake(MergeTreeDataFormatVersion format_version, const MergeTreePartInfo & part_info);
diff --git a/src/Storages/StorageStripeLog.cpp b/src/Storages/StorageStripeLog.cpp
index a3cbff96199..0bfef5ed5e5 100644
--- a/src/Storages/StorageStripeLog.cpp
+++ b/src/Storages/StorageStripeLog.cpp
@@ -403,18 +403,16 @@ SinkToStoragePtr StorageStripeLog::write(const ASTPtr & /*query*/, const Storage
     return std::make_shared<StripeLogSink>(*this, metadata_snapshot, std::move(lock));
 }
 
-IStorage::DataValidationTasksPtr StorageStripeLog::getCheckTaskList(const ASTPtr & /* query */, ContextPtr local_context)
+
+CheckResults StorageStripeLog::checkData(const ASTPtr & /* query */, ContextPtr local_context)
 {
     ReadLock lock{rwlock, getLockTimeout(local_context)};
     if (!lock)
         throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Lock timeout exceeded");
-    return std::make_unique<DataValidationTasks>(file_checker.getDataValidationTasks(), std::move(lock));
+
+    return file_checker.check();
 }
 
-CheckResult StorageStripeLog::checkDataNext(DataValidationTasksPtr & check_task_list, bool & has_nothing_to_do)
-{
-    return file_checker.checkNextEntry(assert_cast<DataValidationTasks *>(check_task_list.get())->file_checker_tasks, has_nothing_to_do);
-}
 
 void StorageStripeLog::truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr, TableExclusiveLockHolder &)
 {
diff --git a/src/Storages/StorageStripeLog.h b/src/Storages/StorageStripeLog.h
index 5d4e2fcbd3a..f889a1de71b 100644
--- a/src/Storages/StorageStripeLog.h
+++ b/src/Storages/StorageStripeLog.h
@@ -53,8 +53,7 @@ public:
 
     void rename(const String & new_path_to_table_data, const StorageID & new_table_id) override;
 
-    DataValidationTasksPtr getCheckTaskList(const ASTPtr & query, ContextPtr context) override;
-    CheckResult checkDataNext(DataValidationTasksPtr & check_task_list, bool & has_nothing_to_do) override;
+    CheckResults checkData(const ASTPtr & query, ContextPtr ocal_context) override;
 
     bool storesDataOnDisk() const override { return true; }
     Strings getDataPaths() const override { return {DB::fullPath(disk, table_path)}; }
@@ -94,20 +93,6 @@ private:
     const DiskPtr disk;
     String table_path;
 
-    struct DataValidationTasks : public IStorage::DataValidationTasksBase
-    {
-        DataValidationTasks(FileChecker::DataValidationTasksPtr file_checker_tasks_, ReadLock && lock_)
-            : file_checker_tasks(std::move(file_checker_tasks_)), lock(std::move(lock_))
-        {}
-
-        size_t size() const override { return file_checker_tasks->size(); }
-
-        FileChecker::DataValidationTasksPtr file_checker_tasks;
-
-        /// Lock to prevent table modification while checking
-        ReadLock lock;
-    };
-
     String data_file_path;
     String index_file_path;
     FileChecker file_checker;
diff --git a/tests/queries/0_stateless/00063_check_query.reference b/tests/queries/0_stateless/00063_check_query.reference
index e8183f05f5d..6ed281c757a 100644
--- a/tests/queries/0_stateless/00063_check_query.reference
+++ b/tests/queries/0_stateless/00063_check_query.reference
@@ -1,3 +1,2 @@
 1
 1
-1
diff --git a/tests/queries/0_stateless/00063_check_query.sql b/tests/queries/0_stateless/00063_check_query.sql
index 263cf94fb4a..e7362074a05 100644
--- a/tests/queries/0_stateless/00063_check_query.sql
+++ b/tests/queries/0_stateless/00063_check_query.sql
@@ -8,9 +8,6 @@ INSERT INTO check_query_tiny_log VALUES (1, 'A'), (2, 'B'), (3, 'C');
 
 CHECK TABLE check_query_tiny_log;
 
--- Settings and FORMAT are supported
-CHECK TABLE check_query_tiny_log SETTINGS max_threads = 16;
-CHECK TABLE check_query_tiny_log FORMAT Null SETTINGS max_threads = 8, check_query_single_value_result = 0;
 
 DROP TABLE IF EXISTS check_query_log;
 
diff --git a/tests/queries/0_stateless/00961_check_table.sql b/tests/queries/0_stateless/00961_check_table.sql
index 079acc8cdbb..0e0b2c3b483 100644
--- a/tests/queries/0_stateless/00961_check_table.sql
+++ b/tests/queries/0_stateless/00961_check_table.sql
@@ -3,29 +3,29 @@ DROP TABLE IF EXISTS mt_table;
 
 CREATE TABLE mt_table (d Date, key UInt64, data String) ENGINE = MergeTree() PARTITION BY toYYYYMM(d) ORDER BY key;
 
-CHECK TABLE mt_table SETTINGS max_threads = 1;
+CHECK TABLE mt_table;
 
 INSERT INTO mt_table VALUES (toDate('2019-01-02'), 1, 'Hello'), (toDate('2019-01-02'), 2, 'World');
 
-CHECK TABLE mt_table SETTINGS max_threads = 1;
+CHECK TABLE mt_table;
 
 INSERT INTO mt_table VALUES (toDate('2019-01-02'), 3, 'quick'), (toDate('2019-01-02'), 4, 'brown');
 
 SELECT '========';
 
-CHECK TABLE mt_table SETTINGS max_threads = 1;
+CHECK TABLE mt_table;
 
 OPTIMIZE TABLE mt_table FINAL;
 
 SELECT '========';
 
-CHECK TABLE mt_table SETTINGS max_threads = 1;
+CHECK TABLE mt_table;
 
 SELECT '========';
 
 INSERT INTO mt_table VALUES (toDate('2019-02-03'), 5, '!'), (toDate('2019-02-03'), 6, '?');
 
-CHECK TABLE mt_table SETTINGS max_threads = 1;
+CHECK TABLE mt_table;
 
 SELECT '========';
 
@@ -33,6 +33,6 @@ INSERT INTO mt_table VALUES (toDate('2019-02-03'), 7, 'jump'), (toDate('2019-02-
 
 OPTIMIZE TABLE mt_table FINAL;
 
-CHECK TABLE mt_table PARTITION 201902 SETTINGS max_threads = 1;
+CHECK TABLE mt_table PARTITION 201902;
 
 DROP TABLE IF EXISTS mt_table;
diff --git a/tests/queries/0_stateless/01042_check_query_and_last_granule_size.sql b/tests/queries/0_stateless/01042_check_query_and_last_granule_size.sql
index eccb2d25878..b66aff8384d 100644
--- a/tests/queries/0_stateless/01042_check_query_and_last_granule_size.sql
+++ b/tests/queries/0_stateless/01042_check_query_and_last_granule_size.sql
@@ -7,11 +7,11 @@ CREATE TABLE check_query_test (SomeKey UInt64, SomeValue String) ENGINE = MergeT
 -- Rows in this table are short, so granularity will be 8192.
 INSERT INTO check_query_test SELECT number, toString(number) FROM system.numbers LIMIT 81920;
 
-CHECK TABLE check_query_test SETTINGS max_threads = 1;
+CHECK TABLE check_query_test;
 
 OPTIMIZE TABLE check_query_test;
 
-CHECK TABLE check_query_test SETTINGS max_threads = 1;
+CHECK TABLE check_query_test;
 
 DROP TABLE IF EXISTS check_query_test;
 
@@ -21,18 +21,18 @@ CREATE TABLE check_query_test_non_adaptive (SomeKey UInt64, SomeValue String) EN
 
 INSERT INTO check_query_test_non_adaptive SELECT number, toString(number) FROM system.numbers LIMIT 81920;
 
-CHECK TABLE check_query_test_non_adaptive SETTINGS max_threads = 1;
+CHECK TABLE check_query_test_non_adaptive;
 
 OPTIMIZE TABLE check_query_test_non_adaptive;
 
-CHECK TABLE check_query_test_non_adaptive SETTINGS max_threads = 1;
+CHECK TABLE check_query_test_non_adaptive;
 
 INSERT INTO check_query_test_non_adaptive SELECT number, toString(number) FROM system.numbers LIMIT 77;
 
-CHECK TABLE check_query_test_non_adaptive SETTINGS max_threads = 1;
+CHECK TABLE check_query_test_non_adaptive;
 
 OPTIMIZE TABLE check_query_test_non_adaptive;
 
-CHECK TABLE check_query_test_non_adaptive SETTINGS max_threads = 1;
+CHECK TABLE check_query_test_non_adaptive;
 
 DROP TABLE IF EXISTS check_query_test_non_adaptive;
diff --git a/tests/queries/0_stateless/02235_check_table_sparse_serialization.sql b/tests/queries/0_stateless/02235_check_table_sparse_serialization.sql
index 625be63e0c0..0ac97404c46 100644
--- a/tests/queries/0_stateless/02235_check_table_sparse_serialization.sql
+++ b/tests/queries/0_stateless/02235_check_table_sparse_serialization.sql
@@ -12,6 +12,7 @@ SELECT name, column, serialization_kind FROM system.parts_columns
 WHERE database = currentDatabase() AND table = 't_sparse_02235'
 ORDER BY name, column;
 
-CHECK TABLE t_sparse_02235 SETTINGS check_query_single_value_result = 0, max_threads = 1;
+SET check_query_single_value_result = 0;
+CHECK TABLE t_sparse_02235;
 
 DROP TABLE t_sparse_02235;
diff --git a/tests/queries/0_stateless/02841_check_table_progress.reference b/tests/queries/0_stateless/02841_check_table_progress.reference
deleted file mode 100644
index 541dab48def..00000000000
--- a/tests/queries/0_stateless/02841_check_table_progress.reference
+++ /dev/null
@@ -1,2 +0,0 @@
-Ok
-Ok
diff --git a/tests/queries/0_stateless/02841_check_table_progress.sh b/tests/queries/0_stateless/02841_check_table_progress.sh
deleted file mode 100755
index 166386b999b..00000000000
--- a/tests/queries/0_stateless/02841_check_table_progress.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/usr/bin/env bash
-
-CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# shellcheck source=../shell_config.sh
-. "$CURDIR"/../shell_config.sh
-
-${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS t0";
-${CLICKHOUSE_CLIENT} -q "CREATE TABLE t0 (x UInt64, val String) ENGINE = MergeTree ORDER BY x PARTITION BY x % 100";
-${CLICKHOUSE_CLIENT} -q "INSERT INTO t0 SELECT sipHash64(number), randomPrintableASCII(1000) FROM numbers(1000)";
-
-
-# Check that we have at least 3 different values for read_rows
-UNIQUE_VALUES=$(
-    ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0" -d @- <<< "CHECK TABLE t0" -v |& {
-        grep -F -e X-ClickHouse-Progress: -e X-ClickHouse-Summary:  | grep -o '"read_rows"\s*:\s*"[0-9]*"'
-    } | uniq | wc -l
-)
-
-[ "$UNIQUE_VALUES" -ge "3" ] && echo "Ok" || echo "Fail: got $UNIQUE_VALUES"
-
-
-# Check that we have we have at least 100 total_rows_to_read (at least one check task per partition)
-MAX_TOTAL_VALUE=$(
-    ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0" -d @- <<< "CHECK TABLE t0" -v |& {
-        grep -F -e X-ClickHouse-Progress: -e X-ClickHouse-Summary:  | grep -o '"total_rows_to_read"\s*:\s*"[0-9]*"' | grep -o '[0-9]*'
-    } | sort -n | tail -1
-)
-
-[ "$MAX_TOTAL_VALUE" -ge "100" ] && echo "Ok" || echo "Fail: got $MAX_TOTAL_VALUE"

From 1377d86ed9d6aaf17878b8c7d2960a0053b1111d Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 10 Aug 2023 17:01:09 +0300
Subject: [PATCH 184/249] Update src/Functions/array/arrayAUC.cpp
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
---
 src/Functions/array/arrayAUC.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Functions/array/arrayAUC.cpp b/src/Functions/array/arrayAUC.cpp
index caf929ba038..b7bd7dcc0ad 100644
--- a/src/Functions/array/arrayAUC.cpp
+++ b/src/Functions/array/arrayAUC.cpp
@@ -127,8 +127,8 @@ private:
     }
 
     static void vector(
-        const IColumn & data1,
-        const IColumn & data2,
+        const IColumn & scores,
+        const IColumn & labels,
         const ColumnArray::Offsets & offsets,
         PaddedPODArray<Float64> & result)
     {

From 635e7e74a86b80519544167eeb2d771a612d6a34 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Thu, 10 Aug 2023 16:52:57 +0200
Subject: [PATCH 185/249] add garbage

---
 .../test_s3_zero_copy_ttl/__init__.py         |  0
 .../configs/max_delayed_streams.xml           |  9 ++
 .../test_s3_zero_copy_ttl/configs/s3.xml      | 39 ++++++++
 .../integration/test_s3_zero_copy_ttl/test.py | 94 +++++++++++++++++++
 .../test_vertical_merge_memory_usage.py       | 46 +++++++++
 5 files changed, 188 insertions(+)
 create mode 100644 tests/integration/test_s3_zero_copy_ttl/__init__.py
 create mode 100644 tests/integration/test_s3_zero_copy_ttl/configs/max_delayed_streams.xml
 create mode 100644 tests/integration/test_s3_zero_copy_ttl/configs/s3.xml
 create mode 100644 tests/integration/test_s3_zero_copy_ttl/test.py
 create mode 100644 tests/integration/test_s3_zero_copy_ttl/test_vertical_merge_memory_usage.py

diff --git a/tests/integration/test_s3_zero_copy_ttl/__init__.py b/tests/integration/test_s3_zero_copy_ttl/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/integration/test_s3_zero_copy_ttl/configs/max_delayed_streams.xml b/tests/integration/test_s3_zero_copy_ttl/configs/max_delayed_streams.xml
new file mode 100644
index 00000000000..54f7152690b
--- /dev/null
+++ b/tests/integration/test_s3_zero_copy_ttl/configs/max_delayed_streams.xml
@@ -0,0 +1,9 @@
+<clickhouse>
+    <!-- Profiles of settings. -->
+    <profiles>
+        <!-- Default settings. -->
+        <default>
+            <max_insert_delayed_streams_for_parallel_write>10</max_insert_delayed_streams_for_parallel_write>
+        </default>
+    </profiles>
+</clickhouse>
diff --git a/tests/integration/test_s3_zero_copy_ttl/configs/s3.xml b/tests/integration/test_s3_zero_copy_ttl/configs/s3.xml
new file mode 100644
index 00000000000..7bb7fa875e4
--- /dev/null
+++ b/tests/integration/test_s3_zero_copy_ttl/configs/s3.xml
@@ -0,0 +1,39 @@
+<clickhouse>
+  <storage_configuration>
+    <disks>
+        <s3_disk>
+            <type>s3</type>
+            <endpoint>http://minio1:9001/root/data/</endpoint>
+            <access_key_id>minio</access_key_id>
+            <secret_access_key>minio123</secret_access_key>
+        </s3_disk>
+    </disks>
+
+    <policies>
+        <s3_and_default>
+            <volumes>
+                <main>
+                    <disk>default</disk>
+                </main>
+                <external>
+                    <disk>s3_disk</disk>
+                </external>
+            </volumes>
+        </s3_and_default>
+        <s3>
+            <volumes>
+                <main>
+                    <disk>s3_disk</disk>
+                </main>
+            </volumes>
+        </s3>
+    </policies>
+  </storage_configuration>
+
+  <merge_tree>
+    <allow_remote_fs_zero_copy_replication>true</allow_remote_fs_zero_copy_replication>
+    <ratio_of_defaults_for_sparse_serialization>1.0</ratio_of_defaults_for_sparse_serialization>
+  </merge_tree>
+
+  <allow_remove_stale_moving_parts>true</allow_remove_stale_moving_parts>
+</clickhouse>
diff --git a/tests/integration/test_s3_zero_copy_ttl/test.py b/tests/integration/test_s3_zero_copy_ttl/test.py
new file mode 100644
index 00000000000..04bff4a44fb
--- /dev/null
+++ b/tests/integration/test_s3_zero_copy_ttl/test.py
@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+import time
+
+import pytest
+from helpers.cluster import ClickHouseCluster
+
+cluster = ClickHouseCluster(__file__)
+node1 = cluster.add_instance(
+    "node1", main_configs=["configs/s3.xml"], with_minio=True, with_zookeeper=True
+)
+node2 = cluster.add_instance(
+    "node2", main_configs=["configs/s3.xml"], with_minio=True, with_zookeeper=True
+)
+node3 = cluster.add_instance(
+    "node3", main_configs=["configs/s3.xml"], with_minio=True, with_zookeeper=True
+)
+
+
+@pytest.fixture(scope="module")
+def started_cluster():
+    try:
+        cluster.start()
+
+        yield cluster
+    finally:
+        cluster.shutdown()
+
+
+def test_ttl_move_and_s3(started_cluster):
+    for i, node in enumerate([node1, node2, node3]):
+        node.query(
+            """
+            CREATE TABLE s3_test_with_ttl (date DateTime, id UInt32, value String)
+            ENGINE=ReplicatedMergeTree('/clickhouse/tables/s3_test', '{}')
+            ORDER BY id
+            PARTITION BY id
+            TTL date TO DISK 's3_disk'
+            SETTINGS storage_policy='s3_and_default', temporary_directories_lifetime=1
+            """.format(
+                i
+            )
+        )
+
+    node1.query("SYSTEM STOP MOVES s3_test_with_ttl")
+
+    node2.query("SYSTEM STOP MOVES s3_test_with_ttl")
+
+    for i in range(30):
+        if i % 2 == 0:
+            node = node1
+        else:
+            node = node2
+
+        node.query(
+            f"INSERT INTO s3_test_with_ttl SELECT now() + 5, {i}, randomPrintableASCII(1048570)"
+        )
+
+    node1.query("SYSTEM SYNC REPLICA s3_test_with_ttl")
+    node2.query("SYSTEM SYNC REPLICA s3_test_with_ttl")
+    node3.query("SYSTEM SYNC REPLICA s3_test_with_ttl")
+
+    assert node1.query("SELECT COUNT() FROM s3_test_with_ttl") == "30\n"
+    assert node2.query("SELECT COUNT() FROM s3_test_with_ttl") == "30\n"
+
+    node1.query("SYSTEM START MOVES s3_test_with_ttl")
+    node2.query("SYSTEM START MOVES s3_test_with_ttl")
+
+    assert node1.query("SELECT COUNT() FROM s3_test_with_ttl") == "30\n"
+    assert node2.query("SELECT COUNT() FROM s3_test_with_ttl") == "30\n"
+
+    for attempt in reversed(range(5)):
+        time.sleep(5)
+
+        print(
+            node1.query(
+                "SELECT * FROM system.parts WHERE table = 's3_test_with_ttl' FORMAT Vertical"
+            )
+        )
+
+        minio = cluster.minio_client
+        objects = minio.list_objects(cluster.minio_bucket, "data/", recursive=True)
+        counter = 0
+        for obj in objects:
+            print(f"Objectname: {obj.object_name}, metadata: {obj.metadata}")
+            counter += 1
+
+        print(f"Total objects: {counter}")
+
+        if counter == 330:
+            break
+
+        print(f"Attempts remaining: {attempt}")
+
+    assert counter == 330
diff --git a/tests/integration/test_s3_zero_copy_ttl/test_vertical_merge_memory_usage.py b/tests/integration/test_s3_zero_copy_ttl/test_vertical_merge_memory_usage.py
new file mode 100644
index 00000000000..fb9f3eb67b9
--- /dev/null
+++ b/tests/integration/test_s3_zero_copy_ttl/test_vertical_merge_memory_usage.py
@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+import time
+
+import pytest
+from helpers.cluster import ClickHouseCluster
+
+
+single_node_cluster = ClickHouseCluster(__file__)
+small_node = single_node_cluster.add_instance(
+    "small_node",
+    main_configs=["configs/s3.xml"],
+    user_configs=["configs/max_delayed_streams.xml"],
+    with_minio=True,
+)
+
+
+@pytest.fixture(scope="module")
+def started_single_node_cluster():
+    try:
+        single_node_cluster.start()
+
+        yield single_node_cluster
+    finally:
+        single_node_cluster.shutdown()
+
+
+def test_vertical_merge_memory_usage(started_single_node_cluster):
+    if small_node.is_built_with_sanitizer() or small_node.is_debug_build():
+        pytest.skip("Disabled for debug and sanitizers. Too slow.")
+
+    small_node.query(
+        "create table tvm2 (c0 UInt64, c1 UInt64, c2 UInt64, c3 UInt64, c4 UInt64, c5 UInt64, c6 UInt64, c7 UInt64, c8 UInt64, c9 UInt64, c10 UInt64, c11 UInt64, c12 UInt64, c13 UInt64, c14 UInt64, c15 UInt64, c16 UInt64, c17 UInt64, c18 UInt64, c19 UInt64, c20 UInt64, c21 UInt64, c22 UInt64, c23 UInt64, c24 UInt64, c25 UInt64, c26 UInt64, c27 UInt64, c28 UInt64, c29 UInt64, c30 UInt64, c31 UInt64, c32 UInt64, c33 UInt64, c34 UInt64, c35 UInt64, c36 UInt64, c37 UInt64, c38 UInt64, c39 UInt64, c40 UInt64, c41 UInt64, c42 UInt64, c43 UInt64, c44 UInt64, c45 UInt64, c46 UInt64, c47 UInt64, c48 UInt64, c49 UInt64, c50 UInt64, c51 UInt64, c52 UInt64, c53 UInt64, c54 UInt64, c55 UInt64, c56 UInt64, c57 UInt64, c58 UInt64, c59 UInt64, c60 UInt64, c61 UInt64, c62 UInt64, c63 UInt64, c64 UInt64, c65 UInt64, c66 UInt64, c67 UInt64, c68 UInt64, c69 UInt64, c70 UInt64, c71 UInt64, c72 UInt64, c73 UInt64, c74 UInt64, c75 UInt64, c76 UInt64, c77 UInt64, c78 UInt64, c79 UInt64, c80 UInt64, c81 UInt64, c82 UInt64, c83 UInt64, c84 UInt64, c85 UInt64, c86 UInt64, c87 UInt64, c88 UInt64, c89 UInt64, c90 UInt64, c91 UInt64, c92 UInt64, c93 UInt64, c94 UInt64, c95 UInt64, c96 UInt64, c97 UInt64, c98 UInt64, c99 UInt64, c100 UInt64, c101 UInt64, c102 UInt64, c103 UInt64, c104 UInt64, c105 UInt64, c106 UInt64, c107 UInt64, c108 UInt64, c109 UInt64, c110 UInt64, c111 UInt64, c112 UInt64, c113 UInt64, c114 UInt64, c115 UInt64, c116 UInt64, c117 UInt64, c118 UInt64, c119 UInt64, c120 UInt64, c121 UInt64, c122 UInt64, c123 UInt64, c124 UInt64, c125 UInt64, c126 UInt64, c127 UInt64, c128 UInt64, c129 UInt64, c130 UInt64, c131 UInt64, c132 UInt64, c133 UInt64, c134 UInt64, c135 UInt64, c136 UInt64, c137 UInt64, c138 UInt64, c139 UInt64, c140 UInt64, c141 UInt64, c142 UInt64, c143 UInt64, c144 UInt64, c145 UInt64, c146 UInt64, c147 UInt64, c148 UInt64, c149 UInt64, c150 UInt64, c151 UInt64, c152 UInt64, c153 UInt64, c154 UInt64, c155 UInt64, c156 UInt64, c157 UInt64, c158 UInt64, c159 UInt64, c160 UInt64, c161 UInt64, c162 UInt64, c163 UInt64, c164 UInt64, c165 UInt64, c166 UInt64, c167 UInt64, c168 UInt64, c169 UInt64, c170 UInt64, c171 UInt64, c172 UInt64, c173 UInt64, c174 UInt64, c175 UInt64, c176 UInt64, c177 UInt64, c178 UInt64, c179 UInt64, c180 UInt64, c181 UInt64, c182 UInt64, c183 UInt64, c184 UInt64, c185 UInt64, c186 UInt64, c187 UInt64, c188 UInt64, c189 UInt64, c190 UInt64, c191 UInt64, c192 UInt64, c193 UInt64, c194 UInt64, c195 UInt64, c196 UInt64, c197 UInt64, c198 UInt64, c199 UInt64, c200 UInt64, c201 UInt64, c202 UInt64, c203 UInt64, c204 UInt64, c205 UInt64, c206 UInt64, c207 UInt64, c208 UInt64, c209 UInt64, c210 UInt64, c211 UInt64, c212 UInt64, c213 UInt64, c214 UInt64, c215 UInt64, c216 UInt64, c217 UInt64, c218 UInt64, c219 UInt64, c220 UInt64, c221 UInt64, c222 UInt64, c223 UInt64, c224 UInt64, c225 UInt64, c226 UInt64, c227 UInt64, c228 UInt64, c229 UInt64, c230 UInt64, c231 UInt64, c232 UInt64, c233 UInt64, c234 UInt64, c235 UInt64, c236 UInt64, c237 UInt64, c238 UInt64, c239 UInt64, c240 UInt64, c241 UInt64, c242 UInt64, c243 UInt64, c244 UInt64, c245 UInt64, c246 UInt64, c247 UInt64, c248 UInt64, c249 UInt64, c250 UInt64, c251 UInt64, c252 UInt64, c253 UInt64, c254 UInt64, c255 UInt64, c256 UInt64, c257 UInt64, c258 UInt64, c259 UInt64, c260 UInt64, c261 UInt64, c262 UInt64, c263 UInt64, c264 UInt64, c265 UInt64, c266 UInt64, c267 UInt64, c268 UInt64, c269 UInt64, c270 UInt64, c271 UInt64, c272 UInt64, c273 UInt64, c274 UInt64, c275 UInt64, c276 UInt64, c277 UInt64, c278 UInt64, c279 UInt64, c280 UInt64, c281 UInt64, c282 UInt64, c283 UInt64, c284 UInt64, c285 UInt64, c286 UInt64, c287 UInt64, c288 UInt64, c289 UInt64, c290 UInt64, c291 UInt64, c292 UInt64, c293 UInt64, c294 UInt64, c295 UInt64, c296 UInt64, c297 UInt64, c298 UInt64, c299 UInt64) engine = MergeTree order by tuple() settings min_rows_for_wide_part = 10, min_bytes_for_wide_part=0, storage_policy = 's3', vertical_merge_algorithm_min_rows_to_activate=1"
+    )
+
+    small_node.query(
+        "insert into tvm2 select number + 0, number + 1, number + 2, number + 3, number + 4, number + 5, number + 6, number + 7, number + 8, number + 9, number + 10, number + 11, number + 12, number + 13, number + 14, number + 15, number + 16, number + 17, number + 18, number + 19, number + 20, number + 21, number + 22, number + 23, number + 24, number + 25, number + 26, number + 27, number + 28, number + 29, number + 30, number + 31, number + 32, number + 33, number + 34, number + 35, number + 36, number + 37, number + 38, number + 39, number + 40, number + 41, number + 42, number + 43, number + 44, number + 45, number + 46, number + 47, number + 48, number + 49, number + 50, number + 51, number + 52, number + 53, number + 54, number + 55, number + 56, number + 57, number + 58, number + 59, number + 60, number + 61, number + 62, number + 63, number + 64, number + 65, number + 66, number + 67, number + 68, number + 69, number + 70, number + 71, number + 72, number + 73, number + 74, number + 75, number + 76, number + 77, number + 78, number + 79, number + 80, number + 81, number + 82, number + 83, number + 84, number + 85, number + 86, number + 87, number + 88, number + 89, number + 90, number + 91, number + 92, number + 93, number + 94, number + 95, number + 96, number + 97, number + 98, number + 99, number + 100, number + 101, number + 102, number + 103, number + 104, number + 105, number + 106, number + 107, number + 108, number + 109, number + 110, number + 111, number + 112, number + 113, number + 114, number + 115, number + 116, number + 117, number + 118, number + 119, number + 120, number + 121, number + 122, number + 123, number + 124, number + 125, number + 126, number + 127, number + 128, number + 129, number + 130, number + 131, number + 132, number + 133, number + 134, number + 135, number + 136, number + 137, number + 138, number + 139, number + 140, number + 141, number + 142, number + 143, number + 144, number + 145, number + 146, number + 147, number + 148, number + 149, number + 150, number + 151, number + 152, number + 153, number + 154, number + 155, number + 156, number + 157, number + 158, number + 159, number + 160, number + 161, number + 162, number + 163, number + 164, number + 165, number + 166, number + 167, number + 168, number + 169, number + 170, number + 171, number + 172, number + 173, number + 174, number + 175, number + 176, number + 177, number + 178, number + 179, number + 180, number + 181, number + 182, number + 183, number + 184, number + 185, number + 186, number + 187, number + 188, number + 189, number + 190, number + 191, number + 192, number + 193, number + 194, number + 195, number + 196, number + 197, number + 198, number + 199, number + 200, number + 201, number + 202, number + 203, number + 204, number + 205, number + 206, number + 207, number + 208, number + 209, number + 210, number + 211, number + 212, number + 213, number + 214, number + 215, number + 216, number + 217, number + 218, number + 219, number + 220, number + 221, number + 222, number + 223, number + 224, number + 225, number + 226, number + 227, number + 228, number + 229, number + 230, number + 231, number + 232, number + 233, number + 234, number + 235, number + 236, number + 237, number + 238, number + 239, number + 240, number + 241, number + 242, number + 243, number + 244, number + 245, number + 246, number + 247, number + 248, number + 249, number + 250, number + 251, number + 252, number + 253, number + 254, number + 255, number + 256, number + 257, number + 258, number + 259, number + 260, number + 261, number + 262, number + 263, number + 264, number + 265, number + 266, number + 267, number + 268, number + 269, number + 270, number + 271, number + 272, number + 273, number + 274, number + 275, number + 276, number + 277, number + 278, number + 279, number + 280, number + 281, number + 282, number + 283, number + 284, number + 285, number + 286, number + 287, number + 288, number + 289, number + 290, number + 291, number + 292, number + 293, number + 294, number + 295, number + 296, number + 297, number + 298, number + 299 from numbers(20)"
+    )
+    small_node.query("optimize table tvm2 final")
+    small_node.query("system flush logs")
+
+    # Should be about 25M
+    res = small_node.query(
+        "select formatReadableSize(peak_memory_usage), * from system.part_log where table = 'tvm2' and database = currentDatabase() and event_date >= today() - 1 and event_type = 'MergeParts' and peak_memory_usage > 100000000 format Vertical"
+    )
+
+    assert res == ""

From 708fd914bc708345fcc46fadd19b47eceddc786f Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sat, 5 Aug 2023 13:47:57 +0200
Subject: [PATCH 186/249] Fix 02263_format_insert_settings flakiness

I guess the problem was with the async nature of the process
substitution ("2> >(cmd)"), let's avoid using this feature of bash.

CI: https://s3.amazonaws.com/clickhouse-test-reports/52683/b98cb7fa145d1a92c2c78421be1eeb8fe8353d53/stateless_tests__aarch64_.html
Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
Co-authored-by: Alexey Milovidov <milovidov@clickhouse.com>

Update 02263_format_insert_settings.sh
---
 .../02263_format_insert_settings.reference           |  6 +++---
 .../0_stateless/02263_format_insert_settings.sh      | 12 ++++++------
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/tests/queries/0_stateless/02263_format_insert_settings.reference b/tests/queries/0_stateless/02263_format_insert_settings.reference
index 721e7960875..e2d1ec3980e 100644
--- a/tests/queries/0_stateless/02263_format_insert_settings.reference
+++ b/tests/queries/0_stateless/02263_format_insert_settings.reference
@@ -1,6 +1,6 @@
-insert into foo settings max_threads=1
+[multi] insert into foo settings max_threads=1
 Syntax error (query): failed at position 40 (end of query):
-insert into foo format tsv settings max_threads=1
+[multi] insert into foo format tsv settings max_threads=1
 Can't format ASTInsertQuery with data, since data will be lost.
 [multi] insert into foo format tsv settings max_threads=1
 INSERT INTO foo
@@ -8,7 +8,7 @@ SETTINGS max_threads = 1
 FORMAT tsv
 [oneline] insert into foo format tsv settings max_threads=1
 INSERT INTO foo SETTINGS max_threads = 1 FORMAT tsv
-insert into foo settings max_threads=1 format tsv settings max_threads=1
+[multi] insert into foo settings max_threads=1 format tsv settings max_threads=1
 You have SETTINGS before and after FORMAT
 Cannot parse input: expected '\n' before: 'settings max_threads=1 1'
 1
diff --git a/tests/queries/0_stateless/02263_format_insert_settings.sh b/tests/queries/0_stateless/02263_format_insert_settings.sh
index efb3d39ab6c..8b156ffec83 100755
--- a/tests/queries/0_stateless/02263_format_insert_settings.sh
+++ b/tests/queries/0_stateless/02263_format_insert_settings.sh
@@ -8,7 +8,7 @@ function run_format()
 {
     local q="$1" && shift
 
-    echo "$q"
+    echo "[multi] $q"
     $CLICKHOUSE_FORMAT "$@" <<<"$q"
 }
 function run_format_both()
@@ -22,20 +22,20 @@ function run_format_both()
 }
 
 # NOTE: that those queries may work slow, due to stack trace obtaining
-run_format 'insert into foo settings max_threads=1' 2> >(grep -m1 -o "Syntax error (query): failed at position .* (end of query):")
+run_format 'insert into foo settings max_threads=1' |& grep --max-count 2 --only-matching -e "Syntax error (query): failed at position .* (end of query):" -e '^\[.*$'
 
 # compatibility
-run_format 'insert into foo format tsv settings max_threads=1' 2> >(grep -m1 -F -o "Can't format ASTInsertQuery with data, since data will be lost.")
+run_format 'insert into foo format tsv settings max_threads=1' |& grep --max-count 2 --only-matching -e "Can't format ASTInsertQuery with data, since data will be lost." -e '^\[.*$'
 run_format_both 'insert into foo format tsv settings max_threads=1' --allow_settings_after_format_in_insert
-run_format 'insert into foo settings max_threads=1 format tsv settings max_threads=1' --allow_settings_after_format_in_insert 2> >(grep -m1 -F -o "You have SETTINGS before and after FORMAT")
+run_format 'insert into foo settings max_threads=1 format tsv settings max_threads=1' --allow_settings_after_format_in_insert |& grep --max-count 2 --only-matching -e "You have SETTINGS before and after FORMAT" -e '^\[.*$'
 
 # and via server (since this is a separate code path)
 $CLICKHOUSE_CLIENT -q 'drop table if exists data_02263'
 $CLICKHOUSE_CLIENT -q 'create table data_02263 (key Int) engine=Memory()'
-$CLICKHOUSE_CLIENT -q 'insert into data_02263 format TSV settings max_threads=1 1' 2> >(grep -m1 -F -o "Cannot parse input: expected '\n' before: 'settings max_threads=1 1'")
+$CLICKHOUSE_CLIENT -q 'insert into data_02263 format TSV settings max_threads=1 1' |& grep --max-count 1 -F --only-matching "Cannot parse input: expected '\n' before: 'settings max_threads=1 1'"
 $CLICKHOUSE_CLIENT --allow_settings_after_format_in_insert=1 -q 'insert into data_02263 format TSV settings max_threads=1 1'
 $CLICKHOUSE_CLIENT -q 'select * from data_02263'
-$CLICKHOUSE_CLIENT --allow_settings_after_format_in_insert=1 -q 'insert into data_02263 settings max_threads=1 format tsv settings max_threads=1' 2> >(grep -m1 -F -o "You have SETTINGS before and after FORMAT")
+$CLICKHOUSE_CLIENT --allow_settings_after_format_in_insert=1 -q 'insert into data_02263 settings max_threads=1 format tsv settings max_threads=1' |& grep --max-count 1 -F --only-matching "You have SETTINGS before and after FORMAT"
 $CLICKHOUSE_CLIENT -q 'drop table data_02263'
 
 run_format_both 'insert into foo values'

From cda633a1f66d68f282722d9cdfa7ab445a50f9cc Mon Sep 17 00:00:00 2001
From: Yakov Olkhovskiy <yakov@clickhouse.com>
Date: Thu, 10 Aug 2023 15:48:51 +0000
Subject: [PATCH 187/249] different linker

---
 cmake/linux/toolchain-s390x.cmake | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cmake/linux/toolchain-s390x.cmake b/cmake/linux/toolchain-s390x.cmake
index 23d14d6ed26..4d4d1fab473 100644
--- a/cmake/linux/toolchain-s390x.cmake
+++ b/cmake/linux/toolchain-s390x.cmake
@@ -20,9 +20,9 @@ set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}/s390x-linux-gnu/libc")
 set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
 set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
 set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
-set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=bfd")
-set (CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fuse-ld=bfd")
-set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=bfd")
+set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=mold")
+set (CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fuse-ld=mold")
+set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=mold")
 
 set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
 set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)

From 4f0be777c5e186f3ebeabd5c1d8fd9adc2fe761b Mon Sep 17 00:00:00 2001
From: Nikolay Degterinsky <evillique@gmail.com>
Date: Thu, 10 Aug 2023 13:18:32 +0000
Subject: [PATCH 188/249] Fix build

---
 .../tests/gtest_transform_query_for_external_database.cpp   | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/Storages/tests/gtest_transform_query_for_external_database.cpp b/src/Storages/tests/gtest_transform_query_for_external_database.cpp
index 5c1442ece11..749a154c19d 100644
--- a/src/Storages/tests/gtest_transform_query_for_external_database.cpp
+++ b/src/Storages/tests/gtest_transform_query_for_external_database.cpp
@@ -127,7 +127,8 @@ static void checkOld(
     std::string transformed_query = transformQueryForExternalDatabase(
         query_info,
         query_info.syntax_analyzer_result->requiredSourceColumns(),
-        state.getColumns(0), IdentifierQuotingStyle::DoubleQuotes, "test", "table", state.context);
+        state.getColumns(0), IdentifierQuotingStyle::DoubleQuotes,
+        LiteralEscapingStyle::Regular, "test", "table", state.context);
 
     EXPECT_EQ(transformed_query, expected) << query;
 }
@@ -180,7 +181,8 @@ static void checkNewAnalyzer(
     query_info.table_expression = findTableExpression(query_node->getJoinTree(), "table");
 
     std::string transformed_query = transformQueryForExternalDatabase(
-        query_info, column_names, state.getColumns(0), IdentifierQuotingStyle::DoubleQuotes, "test", "table", state.context);
+        query_info, column_names, state.getColumns(0), IdentifierQuotingStyle::DoubleQuotes,
+        LiteralEscapingStyle::Regular, "test", "table", state.context);
 
     EXPECT_EQ(transformed_query, expected) << query;
 }

From 3acb10b8bc08077c9ac39149517709ddc2c3163a Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 10 Aug 2023 20:51:03 +0200
Subject: [PATCH 189/249] Inhibit randomization in
 `00906_low_cardinality_cache`

---
 tests/ci/stress.py                                        | 2 +-
 tests/queries/0_stateless/00906_low_cardinality_cache.sql | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/ci/stress.py b/tests/ci/stress.py
index eb829cf519c..2c566144f2c 100755
--- a/tests/ci/stress.py
+++ b/tests/ci/stress.py
@@ -302,7 +302,7 @@ if __name__ == "__main__":
             have_long_running_queries = prepare_for_hung_check(args.drop_databases)
         except Exception as ex:
             have_long_running_queries = True
-            logging.error("Failed to prepare for hung check %s", str(ex))
+            logging.error("Failed to prepare for hung check: %s", str(ex))
         logging.info("Checking if some queries hung")
         cmd = " ".join(
             [
diff --git a/tests/queries/0_stateless/00906_low_cardinality_cache.sql b/tests/queries/0_stateless/00906_low_cardinality_cache.sql
index cd2ceabcf6d..55eacd0db44 100644
--- a/tests/queries/0_stateless/00906_low_cardinality_cache.sql
+++ b/tests/queries/0_stateless/00906_low_cardinality_cache.sql
@@ -1,5 +1,5 @@
 drop table if exists lc_00906;
-create table lc_00906 (b LowCardinality(String)) engine=MergeTree order by b;
+create table lc_00906 (b LowCardinality(String)) engine=MergeTree order by b SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
 insert into lc_00906 select '0123456789' from numbers(100000000);
 select count(), b from lc_00906 group by b;
 drop table if exists lc_00906;

From 3aca2408548bc149f933379506250e49238a24de Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 10 Aug 2023 20:59:33 +0200
Subject: [PATCH 190/249] Change the default of max_concurrent_queries from 100
 to 1000

---
 programs/server/config.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/server/config.xml b/programs/server/config.xml
index 14b8954fc39..85cdda63558 100644
--- a/programs/server/config.xml
+++ b/programs/server/config.xml
@@ -317,7 +317,7 @@
     <concurrent_threads_soft_limit_ratio_to_cores>0</concurrent_threads_soft_limit_ratio_to_cores>
 
     <!-- Maximum number of concurrent queries. -->
-    <max_concurrent_queries>100</max_concurrent_queries>
+    <max_concurrent_queries>1000</max_concurrent_queries>
 
     <!-- Maximum memory usage (resident set size) for server process.
          Zero value or unset means default. Default is "max_server_memory_usage_to_ram_ratio" of available physical RAM.

From 9781a8cee9cb04d18b435bc8c8bef4cf34fd2a09 Mon Sep 17 00:00:00 2001
From: Arthur Passos <arthur.ti@outlook.com>
Date: Thu, 10 Aug 2023 16:45:29 -0300
Subject: [PATCH 191/249] bring back dns tests, fix them by pining coredns
 version and modifying test_reverse_dns_query

---
 .../runner/compose/docker_compose_coredns.yml |   2 +-
 .../__init__.py                               |   0
 .../configs/host_regexp.xml                   |  11 ++
 .../configs/listen_host.xml                   |   5 +
 .../coredns_config/Corefile                   |   8 ++
 .../coredns_config/example.com                |   1 +
 .../test.py                                   | 109 ++++++++++++++++++
 .../__init__.py                               |   0
 .../configs/config.xml                        |   4 +
 .../configs/host_regexp.xml                   |  11 ++
 .../configs/listen_host.xml                   |   5 +
 .../coredns_config/Corefile                   |   8 ++
 .../coredns_config/example.com                |   1 +
 .../scripts/stress_test.py                    |  62 ++++++++++
 .../test.py                                   |  88 ++++++++++++++
 .../test_reverse_dns_query/__init__.py        |   0
 .../test_reverse_dns_query/configs/config.xml |   3 +
 .../configs/listen_host.xml                   |   5 +
 .../configs/reverse_dns_function.xml          |   3 +
 .../coredns_config/Corefile                   |   8 ++
 .../coredns_config/example.com                |   1 +
 .../test_reverse_dns_query/test.py            |  74 ++++++++++++
 22 files changed, 408 insertions(+), 1 deletion(-)
 create mode 100644 tests/integration/test_host_regexp_multiple_ptr_records/__init__.py
 create mode 100644 tests/integration/test_host_regexp_multiple_ptr_records/configs/host_regexp.xml
 create mode 100644 tests/integration/test_host_regexp_multiple_ptr_records/configs/listen_host.xml
 create mode 100644 tests/integration/test_host_regexp_multiple_ptr_records/coredns_config/Corefile
 create mode 100644 tests/integration/test_host_regexp_multiple_ptr_records/coredns_config/example.com
 create mode 100644 tests/integration/test_host_regexp_multiple_ptr_records/test.py
 create mode 100644 tests/integration/test_host_regexp_multiple_ptr_records_concurrent/__init__.py
 create mode 100644 tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/config.xml
 create mode 100644 tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/host_regexp.xml
 create mode 100644 tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/listen_host.xml
 create mode 100644 tests/integration/test_host_regexp_multiple_ptr_records_concurrent/coredns_config/Corefile
 create mode 100644 tests/integration/test_host_regexp_multiple_ptr_records_concurrent/coredns_config/example.com
 create mode 100644 tests/integration/test_host_regexp_multiple_ptr_records_concurrent/scripts/stress_test.py
 create mode 100644 tests/integration/test_host_regexp_multiple_ptr_records_concurrent/test.py
 create mode 100644 tests/integration/test_reverse_dns_query/__init__.py
 create mode 100644 tests/integration/test_reverse_dns_query/configs/config.xml
 create mode 100644 tests/integration/test_reverse_dns_query/configs/listen_host.xml
 create mode 100644 tests/integration/test_reverse_dns_query/configs/reverse_dns_function.xml
 create mode 100644 tests/integration/test_reverse_dns_query/coredns_config/Corefile
 create mode 100644 tests/integration/test_reverse_dns_query/coredns_config/example.com
 create mode 100644 tests/integration/test_reverse_dns_query/test.py

diff --git a/docker/test/integration/runner/compose/docker_compose_coredns.yml b/docker/test/integration/runner/compose/docker_compose_coredns.yml
index b329d4e0a46..e4736e04846 100644
--- a/docker/test/integration/runner/compose/docker_compose_coredns.yml
+++ b/docker/test/integration/runner/compose/docker_compose_coredns.yml
@@ -2,7 +2,7 @@ version: "2.3"
 
 services:
   coredns:
-    image: coredns/coredns:latest
+    image: coredns/coredns:1.9.3 # :latest broke this test
     restart: always
     volumes:
       - ${COREDNS_CONFIG_DIR}/example.com:/example.com
diff --git a/tests/integration/test_host_regexp_multiple_ptr_records/__init__.py b/tests/integration/test_host_regexp_multiple_ptr_records/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/integration/test_host_regexp_multiple_ptr_records/configs/host_regexp.xml b/tests/integration/test_host_regexp_multiple_ptr_records/configs/host_regexp.xml
new file mode 100644
index 00000000000..6ca61fb3c44
--- /dev/null
+++ b/tests/integration/test_host_regexp_multiple_ptr_records/configs/host_regexp.xml
@@ -0,0 +1,11 @@
+<clickhouse>
+    <users>
+        <test_dns>
+            <password/>
+            <networks>
+                <host_regexp>test1\.example\.com$</host_regexp>
+            </networks>
+            <profile>default</profile>
+        </test_dns>
+    </users>
+</clickhouse>
\ No newline at end of file
diff --git a/tests/integration/test_host_regexp_multiple_ptr_records/configs/listen_host.xml b/tests/integration/test_host_regexp_multiple_ptr_records/configs/listen_host.xml
new file mode 100644
index 00000000000..9c27c612f63
--- /dev/null
+++ b/tests/integration/test_host_regexp_multiple_ptr_records/configs/listen_host.xml
@@ -0,0 +1,5 @@
+<clickhouse>
+    <listen_host>::</listen_host>
+    <listen_host>0.0.0.0</listen_host>
+    <listen_try>1</listen_try>
+</clickhouse>
diff --git a/tests/integration/test_host_regexp_multiple_ptr_records/coredns_config/Corefile b/tests/integration/test_host_regexp_multiple_ptr_records/coredns_config/Corefile
new file mode 100644
index 00000000000..3edf37dafa5
--- /dev/null
+++ b/tests/integration/test_host_regexp_multiple_ptr_records/coredns_config/Corefile
@@ -0,0 +1,8 @@
+. {
+    hosts /example.com {
+        reload "20ms"
+        fallthrough
+    }
+    forward . 127.0.0.11
+    log
+}
diff --git a/tests/integration/test_host_regexp_multiple_ptr_records/coredns_config/example.com b/tests/integration/test_host_regexp_multiple_ptr_records/coredns_config/example.com
new file mode 100644
index 00000000000..6c6e4cbee2e
--- /dev/null
+++ b/tests/integration/test_host_regexp_multiple_ptr_records/coredns_config/example.com
@@ -0,0 +1 @@
+filled in runtime, but needs to exist in order to be volume mapped in docker
diff --git a/tests/integration/test_host_regexp_multiple_ptr_records/test.py b/tests/integration/test_host_regexp_multiple_ptr_records/test.py
new file mode 100644
index 00000000000..82ae0b6c527
--- /dev/null
+++ b/tests/integration/test_host_regexp_multiple_ptr_records/test.py
@@ -0,0 +1,109 @@
+import pytest
+import socket
+from helpers.cluster import ClickHouseCluster, get_docker_compose_path, run_and_check
+from time import sleep
+import os
+
+DOCKER_COMPOSE_PATH = get_docker_compose_path()
+SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
+
+cluster = ClickHouseCluster(__file__)
+
+ch_server = cluster.add_instance(
+    "clickhouse-server",
+    with_coredns=True,
+    main_configs=["configs/listen_host.xml"],
+    user_configs=["configs/host_regexp.xml"],
+    ipv6_address="2001:3984:3989::1:1111",
+)
+
+client = cluster.add_instance(
+    "clickhouse-client",
+    ipv6_address="2001:3984:3989::1:1112",
+)
+
+
+@pytest.fixture(scope="module")
+def started_cluster():
+    global cluster
+    try:
+        cluster.start()
+        yield cluster
+
+    finally:
+        cluster.shutdown()
+
+
+def check_ptr_record(ip, hostname):
+    try:
+        host, aliaslist, ipaddrlist = socket.gethostbyaddr(ip)
+        if hostname.lower() == host.lower():
+            return True
+    except socket.herror:
+        pass
+    return False
+
+
+def setup_dns_server(ip):
+    domains_string = "test3.example.com test2.example.com test1.example.com"
+    example_file_path = f'{ch_server.env_variables["COREDNS_CONFIG_DIR"]}/example.com'
+    run_and_check(f"echo '{ip} {domains_string}' > {example_file_path}", shell=True)
+
+    # DNS server takes time to reload the configuration.
+    for try_num in range(10):
+        if all(check_ptr_record(ip, host) for host in domains_string.split()):
+            break
+        sleep(1)
+
+
+def setup_ch_server(dns_server_ip):
+    ch_server.exec_in_container(
+        (["bash", "-c", f"echo 'nameserver {dns_server_ip}' > /etc/resolv.conf"])
+    )
+    ch_server.exec_in_container(
+        (["bash", "-c", "echo 'options ndots:0' >> /etc/resolv.conf"])
+    )
+    ch_server.query("SYSTEM DROP DNS CACHE")
+
+
+def build_endpoint_v4(ip):
+    return f"'http://{ip}:8123/?query=SELECT+1&user=test_dns'"
+
+
+def build_endpoint_v6(ip):
+    return build_endpoint_v4(f"[{ip}]")
+
+
+def test_host_regexp_multiple_ptr_v4_fails_with_wrong_resolution(started_cluster):
+    server_ip = cluster.get_instance_ip("clickhouse-server")
+    random_ip = "9.9.9.9"
+    dns_server_ip = cluster.get_instance_ip(cluster.coredns_host)
+
+    setup_dns_server(random_ip)
+    setup_ch_server(dns_server_ip)
+
+    endpoint = build_endpoint_v4(server_ip)
+
+    assert "1\n" != client.exec_in_container(["bash", "-c", f"curl {endpoint}"])
+
+
+def test_host_regexp_multiple_ptr_v4(started_cluster):
+    server_ip = cluster.get_instance_ip("clickhouse-server")
+    client_ip = cluster.get_instance_ip("clickhouse-client")
+    dns_server_ip = cluster.get_instance_ip(cluster.coredns_host)
+
+    setup_dns_server(client_ip)
+    setup_ch_server(dns_server_ip)
+
+    endpoint = build_endpoint_v4(server_ip)
+
+    assert "1\n" == client.exec_in_container(["bash", "-c", f"curl {endpoint}"])
+
+
+def test_host_regexp_multiple_ptr_v6(started_cluster):
+    setup_dns_server(client.ipv6_address)
+    setup_ch_server(cluster.get_instance_global_ipv6(cluster.coredns_host))
+
+    endpoint = build_endpoint_v6(ch_server.ipv6_address)
+
+    assert "1\n" == client.exec_in_container(["bash", "-c", f"curl -6 {endpoint}"])
diff --git a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/__init__.py b/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/config.xml b/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/config.xml
new file mode 100644
index 00000000000..42a1f962705
--- /dev/null
+++ b/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/config.xml
@@ -0,0 +1,4 @@
+<clickhouse>
+    <disable_internal_dns_cache>1</disable_internal_dns_cache>
+    <max_concurrent_queries>250</max_concurrent_queries>
+</clickhouse>
diff --git a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/host_regexp.xml b/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/host_regexp.xml
new file mode 100644
index 00000000000..9329c8dbde2
--- /dev/null
+++ b/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/host_regexp.xml
@@ -0,0 +1,11 @@
+<clickhouse>
+    <users>
+        <test_dns>
+            <password/>
+            <networks>
+                <host_regexp>test1\.example\.com$</host_regexp>
+            </networks>
+            <profile>default</profile>
+        </test_dns>
+    </users>
+</clickhouse>
diff --git a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/listen_host.xml b/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/listen_host.xml
new file mode 100644
index 00000000000..9c27c612f63
--- /dev/null
+++ b/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/configs/listen_host.xml
@@ -0,0 +1,5 @@
+<clickhouse>
+    <listen_host>::</listen_host>
+    <listen_host>0.0.0.0</listen_host>
+    <listen_try>1</listen_try>
+</clickhouse>
diff --git a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/coredns_config/Corefile b/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/coredns_config/Corefile
new file mode 100644
index 00000000000..3edf37dafa5
--- /dev/null
+++ b/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/coredns_config/Corefile
@@ -0,0 +1,8 @@
+. {
+    hosts /example.com {
+        reload "20ms"
+        fallthrough
+    }
+    forward . 127.0.0.11
+    log
+}
diff --git a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/coredns_config/example.com b/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/coredns_config/example.com
new file mode 100644
index 00000000000..9beb415c290
--- /dev/null
+++ b/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/coredns_config/example.com
@@ -0,0 +1 @@
+filled in runtime, but needs to exist in order to be volume mapped in docker
\ No newline at end of file
diff --git a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/scripts/stress_test.py b/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/scripts/stress_test.py
new file mode 100644
index 00000000000..fe69d72c1c7
--- /dev/null
+++ b/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/scripts/stress_test.py
@@ -0,0 +1,62 @@
+import pycurl
+import threading
+from io import BytesIO
+import sys
+
+client_ip = sys.argv[1]
+server_ip = sys.argv[2]
+
+mutex = threading.Lock()
+success_counter = 0
+number_of_threads = 100
+number_of_iterations = 100
+
+
+def perform_request():
+    buffer = BytesIO()
+    crl = pycurl.Curl()
+    crl.setopt(pycurl.INTERFACE, client_ip)
+    crl.setopt(crl.WRITEDATA, buffer)
+    crl.setopt(crl.URL, f"http://{server_ip}:8123/?query=select+1&user=test_dns")
+
+    crl.perform()
+
+    # End curl session
+    crl.close()
+
+    str_response = buffer.getvalue().decode("iso-8859-1")
+    expected_response = "1\n"
+
+    mutex.acquire()
+
+    global success_counter
+
+    if str_response == expected_response:
+        success_counter += 1
+
+    mutex.release()
+
+
+def perform_multiple_requests(n):
+    for request_number in range(n):
+        perform_request()
+
+
+threads = []
+
+
+for i in range(number_of_threads):
+    thread = threading.Thread(
+        target=perform_multiple_requests, args=(number_of_iterations,)
+    )
+    thread.start()
+    threads.append(thread)
+
+for thread in threads:
+    thread.join()
+
+
+if success_counter == number_of_threads * number_of_iterations:
+    exit(0)
+
+exit(1)
diff --git a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/test.py b/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/test.py
new file mode 100644
index 00000000000..d73e8813e79
--- /dev/null
+++ b/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/test.py
@@ -0,0 +1,88 @@
+import pytest
+import socket
+from helpers.cluster import ClickHouseCluster, get_docker_compose_path, run_and_check
+from time import sleep
+import os
+
+DOCKER_COMPOSE_PATH = get_docker_compose_path()
+SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
+
+cluster = ClickHouseCluster(__file__)
+
+ch_server = cluster.add_instance(
+    "clickhouse-server",
+    with_coredns=True,
+    main_configs=["configs/config.xml", "configs/listen_host.xml"],
+    user_configs=["configs/host_regexp.xml"],
+)
+
+client = cluster.add_instance(
+    "clickhouse-client",
+)
+
+
+@pytest.fixture(scope="module")
+def started_cluster():
+    global cluster
+    try:
+        cluster.start()
+        yield cluster
+
+    finally:
+        cluster.shutdown()
+
+
+def check_ptr_record(ip, hostname):
+    try:
+        host, aliaslist, ipaddrlist = socket.gethostbyaddr(ip)
+        if hostname.lower() == host.lower():
+            return True
+    except socket.herror:
+        pass
+    return False
+
+
+def setup_dns_server(ip):
+    domains_string = "test3.example.com test2.example.com test1.example.com"
+    example_file_path = f'{ch_server.env_variables["COREDNS_CONFIG_DIR"]}/example.com'
+    run_and_check(f"echo '{ip} {domains_string}' > {example_file_path}", shell=True)
+
+    # DNS server takes time to reload the configuration.
+    for try_num in range(10):
+        if all(check_ptr_record(ip, host) for host in domains_string.split()):
+            break
+        sleep(1)
+
+
+def setup_ch_server(dns_server_ip):
+    ch_server.exec_in_container(
+        (["bash", "-c", f"echo 'nameserver {dns_server_ip}' > /etc/resolv.conf"])
+    )
+    ch_server.exec_in_container(
+        (["bash", "-c", "echo 'options ndots:0' >> /etc/resolv.conf"])
+    )
+    ch_server.query("SYSTEM DROP DNS CACHE")
+
+
+def build_endpoint_v4(ip):
+    return f"'http://{ip}:8123/?query=SELECT+1&user=test_dns'"
+
+
+def build_endpoint_v6(ip):
+    return build_endpoint_v4(f"[{ip}]")
+
+
+def test_host_regexp_multiple_ptr_v4(started_cluster):
+    server_ip = cluster.get_instance_ip("clickhouse-server")
+    client_ip = cluster.get_instance_ip("clickhouse-client")
+    dns_server_ip = cluster.get_instance_ip(cluster.coredns_host)
+
+    setup_dns_server(client_ip)
+    setup_ch_server(dns_server_ip)
+
+    current_dir = os.path.dirname(__file__)
+    client.copy_file_to_container(
+        os.path.join(current_dir, "scripts", "stress_test.py"), "stress_test.py"
+    )
+
+    client.exec_in_container(["python3", f"stress_test.py", client_ip, server_ip])
diff --git a/tests/integration/test_reverse_dns_query/__init__.py b/tests/integration/test_reverse_dns_query/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/integration/test_reverse_dns_query/configs/config.xml b/tests/integration/test_reverse_dns_query/configs/config.xml
new file mode 100644
index 00000000000..5ce55afa2a7
--- /dev/null
+++ b/tests/integration/test_reverse_dns_query/configs/config.xml
@@ -0,0 +1,3 @@
+<clickhouse>
+    <disable_internal_dns_cache>1</disable_internal_dns_cache>
+</clickhouse>
diff --git a/tests/integration/test_reverse_dns_query/configs/listen_host.xml b/tests/integration/test_reverse_dns_query/configs/listen_host.xml
new file mode 100644
index 00000000000..9c27c612f63
--- /dev/null
+++ b/tests/integration/test_reverse_dns_query/configs/listen_host.xml
@@ -0,0 +1,5 @@
+<clickhouse>
+    <listen_host>::</listen_host>
+    <listen_host>0.0.0.0</listen_host>
+    <listen_try>1</listen_try>
+</clickhouse>
diff --git a/tests/integration/test_reverse_dns_query/configs/reverse_dns_function.xml b/tests/integration/test_reverse_dns_query/configs/reverse_dns_function.xml
new file mode 100644
index 00000000000..35d0a07c6a6
--- /dev/null
+++ b/tests/integration/test_reverse_dns_query/configs/reverse_dns_function.xml
@@ -0,0 +1,3 @@
+<clickhouse>
+    <allow_reverse_dns_query_function>1</allow_reverse_dns_query_function>
+</clickhouse>
diff --git a/tests/integration/test_reverse_dns_query/coredns_config/Corefile b/tests/integration/test_reverse_dns_query/coredns_config/Corefile
new file mode 100644
index 00000000000..3edf37dafa5
--- /dev/null
+++ b/tests/integration/test_reverse_dns_query/coredns_config/Corefile
@@ -0,0 +1,8 @@
+. {
+    hosts /example.com {
+        reload "20ms"
+        fallthrough
+    }
+    forward . 127.0.0.11
+    log
+}
diff --git a/tests/integration/test_reverse_dns_query/coredns_config/example.com b/tests/integration/test_reverse_dns_query/coredns_config/example.com
new file mode 100644
index 00000000000..6c6e4cbee2e
--- /dev/null
+++ b/tests/integration/test_reverse_dns_query/coredns_config/example.com
@@ -0,0 +1 @@
+filled in runtime, but needs to exist in order to be volume mapped in docker
diff --git a/tests/integration/test_reverse_dns_query/test.py b/tests/integration/test_reverse_dns_query/test.py
new file mode 100644
index 00000000000..1b94885939e
--- /dev/null
+++ b/tests/integration/test_reverse_dns_query/test.py
@@ -0,0 +1,74 @@
+import pytest
+import socket
+from helpers.cluster import ClickHouseCluster, get_docker_compose_path, run_and_check
+from time import sleep
+import os
+
+DOCKER_COMPOSE_PATH = get_docker_compose_path()
+SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
+
+cluster = ClickHouseCluster(__file__)
+
+ch_server = cluster.add_instance(
+    "clickhouse-server",
+    with_coredns=True,
+    main_configs=[
+        "configs/config.xml",
+        "configs/reverse_dns_function.xml",
+        "configs/listen_host.xml",
+    ],
+)
+
+
+@pytest.fixture(scope="module")
+def started_cluster():
+    global cluster
+    try:
+        cluster.start()
+        yield cluster
+
+    finally:
+        cluster.shutdown()
+
+
+def check_ptr_record(ip, hostname):
+    try:
+        host, aliaslist, ipaddrlist = socket.gethostbyaddr(ip)
+        if hostname.lower() == host.lower():
+            return True
+    except socket.herror:
+        pass
+    return False
+
+
+def setup_dns_server(ip):
+    domains_string = "test.example.com"
+    example_file_path = f'{ch_server.env_variables["COREDNS_CONFIG_DIR"]}/example.com'
+    run_and_check(f"echo '{ip} {domains_string}' > {example_file_path}", shell=True)
+
+    # DNS server takes time to reload the configuration.
+    for try_num in range(10):
+        if all(check_ptr_record(ip, host) for host in domains_string.split()):
+            break
+        sleep(1)
+
+
+def setup_ch_server(dns_server_ip):
+    ch_server.exec_in_container(
+        (["bash", "-c", f"echo 'nameserver {dns_server_ip}' > /etc/resolv.conf"])
+    )
+    ch_server.exec_in_container(
+        (["bash", "-c", "echo 'options ndots:0' >> /etc/resolv.conf"])
+    )
+    ch_server.query("SYSTEM DROP DNS CACHE")
+
+
+def test_reverse_dns_query(started_cluster):
+    dns_server_ip = cluster.get_instance_ip(cluster.coredns_host)
+    random_ipv6 = "4ae8:fa0f:ee1d:68c5:0b76:1b79:7ae6:1549" # https://commentpicker.com/ip-address-generator.php
+    setup_dns_server(random_ipv6)
+    setup_ch_server(dns_server_ip)
+
+    for _ in range(0, 200):
+        response = ch_server.query(f"select reverseDNSQuery('{random_ipv6}')")
+        assert response == "['test.example.com']\n"

From 6957d320a8abb166cefd20ca9e4b4dbb50f45486 Mon Sep 17 00:00:00 2001
From: Arthur Passos <arthur.ti@outlook.com>
Date: Thu, 10 Aug 2023 17:00:21 -0300
Subject: [PATCH 192/249] trigger ci


From 18447022e61042ff6311d114dcb4ed027ed07a12 Mon Sep 17 00:00:00 2001
From: Yakov Olkhovskiy <yakov@clickhouse.com>
Date: Thu, 10 Aug 2023 20:01:53 +0000
Subject: [PATCH 193/249] add mold 2.0 to docker

---
 docker/packager/binary/Dockerfile | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile
index 15e0bad9355..394ace2b220 100644
--- a/docker/packager/binary/Dockerfile
+++ b/docker/packager/binary/Dockerfile
@@ -108,6 +108,14 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \
 # Download toolchain and SDK for Darwin
 RUN curl -sL -O https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX11.0.sdk.tar.xz
 
+# Download and install mold 2.0 for s390x build
+RUN curl -Lo /tmp/mold.tar.gz "https://github.com/rui314/mold/releases/download/v2.0.0/mold-2.0.0-x86_64-linux.tar.gz" \
+    && mkdir /tmp/mold \
+    && tar -xzf /tmp/mold.tar.gz -C /tmp/mold \
+    && cp /tmp/mold/mold*/bin/* /usr/bin \
+    && rm -rf /tmp/mold \
+    && rm /tmp/mold.tar.gz
+
 # Architecture of the image when BuildKit/buildx is used
 ARG TARGETARCH
 ARG NFPM_VERSION=2.20.0

From cbd81b3758fb09db91fa5c0e0a864b2c66333cec Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Thu, 10 Aug 2023 14:29:25 +0200
Subject: [PATCH 194/249] Partially revert changes from #52395

---
 docker/packager/binary/Dockerfile | 27 -------------------------
 docker/packager/binary/rust       |  1 -
 rust/.dockerignore                |  4 ----
 rust/.gitignore                   |  4 ----
 tests/ci/docker_images_check.py   | 33 ++-----------------------------
 tests/ci/docker_test.py           | 12 ++++-------
 6 files changed, 6 insertions(+), 75 deletions(-)
 delete mode 120000 docker/packager/binary/rust
 delete mode 100644 rust/.dockerignore
 delete mode 100644 rust/.gitignore

diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile
index 99e748c41d4..897bcd24d04 100644
--- a/docker/packager/binary/Dockerfile
+++ b/docker/packager/binary/Dockerfile
@@ -58,33 +58,6 @@ RUN curl https://sh.rustup.rs -sSf | bash -s -- -y && \
     rustup target add aarch64-apple-darwin && \
     rustup target add powerpc64le-unknown-linux-gnu
 
-# Create vendor cache for cargo.
-#
-# Note, that the config.toml for the root is used, you will not be able to
-# install any other crates, except those which had been vendored (since if
-# there is "replace-with" for some source, then cargo will not look to other
-# remotes except this).
-#
-# Notes for the command itself:
-# - --chown is required to preserve the rights
-# - unstable-options for -C
-# - chmod is required to fix the permissions, since builds are running from a different user
-# - copy of the Cargo.lock is required for proper dependencies versions
-# - cargo vendor --sync is requried to overcome [1] bug.
-#
-#   [1]: https://github.com/rust-lang/wg-cargo-std-aware/issues/23
-COPY --chown=root:root /rust /rust/packages
-RUN cargo -Z unstable-options -C /rust/packages vendor > $CARGO_HOME/config.toml && \
-    cp "$(rustc --print=sysroot)"/lib/rustlib/src/rust/Cargo.lock "$(rustc --print=sysroot)"/lib/rustlib/src/rust/library/test/ && \
-    cargo -Z unstable-options -C /rust/packages vendor --sync "$(rustc --print=sysroot)"/lib/rustlib/src/rust/library/test/Cargo.toml && \
-    rm "$(rustc --print=sysroot)"/lib/rustlib/src/rust/library/test/Cargo.lock && \
-    sed -i "s#\"vendor\"#\"/rust/vendor\"#" $CARGO_HOME/config.toml && \
-    cat $CARGO_HOME/config.toml && \
-    mv /rust/packages/vendor /rust/vendor && \
-    chmod -R o=r+X /rust/vendor && \
-    ls -R -l /rust/packages && \
-    rm -r /rust/packages
-
 # NOTE: Seems like gcc-11 is too new for ubuntu20 repository
 # A cross-linker for RISC-V 64 (we need it, because LLVM's LLD does not work):
 RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \
diff --git a/docker/packager/binary/rust b/docker/packager/binary/rust
deleted file mode 120000
index 742dc49e9ac..00000000000
--- a/docker/packager/binary/rust
+++ /dev/null
@@ -1 +0,0 @@
-../../../rust
\ No newline at end of file
diff --git a/rust/.dockerignore b/rust/.dockerignore
deleted file mode 100644
index 6b761aa401c..00000000000
--- a/rust/.dockerignore
+++ /dev/null
@@ -1,4 +0,0 @@
-# Just in case ignore any cargo stuff (and just in case someone will run this
-# docker build locally with build context using folder root):
-target
-vendor
diff --git a/rust/.gitignore b/rust/.gitignore
deleted file mode 100644
index f850cd563c9..00000000000
--- a/rust/.gitignore
+++ /dev/null
@@ -1,4 +0,0 @@
-# This is for tar --exclude-vcs-ignores (and just in case someone will run
-# docker build locally with build context created via tar):
-target
-vendor
diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py
index fff2975cea4..16a58a90dcf 100644
--- a/tests/ci/docker_images_check.py
+++ b/tests/ci/docker_images_check.py
@@ -8,7 +8,6 @@ import shutil
 import subprocess
 import time
 import sys
-from glob import glob
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Set, Tuple, Union
 
@@ -32,17 +31,6 @@ TEMP_PATH = os.path.join(RUNNER_TEMP, "docker_images_check")
 ImagesDict = Dict[str, dict]
 
 
-# workaround for mypy issue [1]:
-#
-#    "Argument 1 to "map" has incompatible type overloaded function" [1]
-#
-#  [1]: https://github.com/python/mypy/issues/9864
-#
-# NOTE: simply lambda will do the trick as well, but pylint will not like it
-def realpath(*args, **kwargs):
-    return os.path.realpath(*args, **kwargs)
-
-
 class DockerImage:
     def __init__(
         self,
@@ -123,23 +111,8 @@ def get_changed_docker_images(
     changed_images = []
 
     for dockerfile_dir, image_description in images_dict.items():
-        source_dir = GITHUB_WORKSPACE.rstrip("/") + "/"
-        dockerfile_files = glob(f"{source_dir}/{dockerfile_dir}/**", recursive=True)
-        # resolve symlinks
-        dockerfile_files = list(map(realpath, dockerfile_files))
-        # trim prefix to get relative path again, to match with files_changed
-        dockerfile_files = list(map(lambda x: x[len(source_dir) :], dockerfile_files))
-        logging.info(
-            "Docker %s (source_dir=%s) build context for PR %s @ %s: %s",
-            dockerfile_dir,
-            source_dir,
-            pr_info.number,
-            pr_info.sha,
-            str(dockerfile_files),
-        )
-
         for f in files_changed:
-            if f in dockerfile_files:
+            if f.startswith(dockerfile_dir):
                 name = image_description["name"]
                 only_amd64 = image_description.get("only_amd64", False)
                 logging.info(
@@ -272,8 +245,6 @@ def build_and_push_one_image(
         cache_from = f"{cache_from} --cache-from type=registry,ref={image.repo}:{tag}"
 
     cmd = (
-        # tar is requried to follow symlinks, since docker-build cannot do this
-        f"tar -v --exclude-vcs-ignores --show-transformed-names --transform 's#{image.full_path.lstrip('/')}#./#' --dereference --create {image.full_path} | "
         "docker buildx build --builder default "
         f"--label build-url={GITHUB_RUN_URL} "
         f"{from_tag_arg}"
@@ -283,7 +254,7 @@ def build_and_push_one_image(
         f"{cache_from} "
         f"--cache-to type=inline,mode=max "
         f"{push_arg}"
-        f"--progress plain -"
+        f"--progress plain {image.full_path}"
     )
     logging.info("Docker command to run: %s", cmd)
     with TeePopen(cmd, build_log) as proc:
diff --git a/tests/ci/docker_test.py b/tests/ci/docker_test.py
index 40c51702868..1cbd4cf9222 100644
--- a/tests/ci/docker_test.py
+++ b/tests/ci/docker_test.py
@@ -126,13 +126,12 @@ class TestDockerImageCheck(unittest.TestCase):
         mock_popen.assert_called_once()
         mock_machine.assert_not_called()
         self.assertIn(
-            "tar -v --exclude-vcs-ignores --show-transformed-names --transform 's#path#./#' --dereference --create path | "
             f"docker buildx build --builder default --label build-url={GITHUB_RUN_URL} "
             "--build-arg FROM_TAG=version "
             f"--build-arg CACHE_INVALIDATOR={GITHUB_RUN_URL} "
             "--tag name:version --cache-from type=registry,ref=name:version "
             "--cache-from type=registry,ref=name:latest "
-            "--cache-to type=inline,mode=max --push --progress plain -",
+            "--cache-to type=inline,mode=max --push --progress plain path",
             mock_popen.call_args.args,
         )
         self.assertTrue(result)
@@ -144,13 +143,12 @@ class TestDockerImageCheck(unittest.TestCase):
         mock_popen.assert_called_once()
         mock_machine.assert_not_called()
         self.assertIn(
-            "tar -v --exclude-vcs-ignores --show-transformed-names --transform 's#path#./#' --dereference --create path | "
             f"docker buildx build --builder default --label build-url={GITHUB_RUN_URL} "
             "--build-arg FROM_TAG=version2 "
             f"--build-arg CACHE_INVALIDATOR={GITHUB_RUN_URL} "
             "--tag name:version2 --cache-from type=registry,ref=name:version2 "
             "--cache-from type=registry,ref=name:latest "
-            "--cache-to type=inline,mode=max --progress plain -",
+            "--cache-to type=inline,mode=max --progress plain path",
             mock_popen.call_args.args,
         )
         self.assertTrue(result)
@@ -162,12 +160,11 @@ class TestDockerImageCheck(unittest.TestCase):
         mock_popen.assert_called_once()
         mock_machine.assert_not_called()
         self.assertIn(
-            "tar -v --exclude-vcs-ignores --show-transformed-names --transform 's#path#./#' --dereference --create path | "
             f"docker buildx build --builder default --label build-url={GITHUB_RUN_URL} "
             f"--build-arg CACHE_INVALIDATOR={GITHUB_RUN_URL} "
             "--tag name:version2 --cache-from type=registry,ref=name:version2 "
             "--cache-from type=registry,ref=name:latest "
-            "--cache-to type=inline,mode=max --progress plain -",
+            "--cache-to type=inline,mode=max --progress plain path",
             mock_popen.call_args.args,
         )
         self.assertFalse(result)
@@ -181,14 +178,13 @@ class TestDockerImageCheck(unittest.TestCase):
         mock_popen.assert_called_once()
         mock_machine.assert_not_called()
         self.assertIn(
-            "tar -v --exclude-vcs-ignores --show-transformed-names --transform 's#path#./#' --dereference --create path | "
             f"docker buildx build --builder default --label build-url={GITHUB_RUN_URL} "
             f"--build-arg CACHE_INVALIDATOR={GITHUB_RUN_URL} "
             "--tag name:version2 --cache-from type=registry,ref=name:version2 "
             "--cache-from type=registry,ref=name:latest "
             "--cache-from type=registry,ref=name:cached-version "
             "--cache-from type=registry,ref=name:another-cached "
-            "--cache-to type=inline,mode=max --progress plain -",
+            "--cache-to type=inline,mode=max --progress plain path",
             mock_popen.call_args.args,
         )
         self.assertFalse(result)

From dbd3fdd437ff0f8a7728413bde2f62484af8c21b Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Wed, 9 Aug 2023 13:43:10 +0200
Subject: [PATCH 195/249] Clean ccache_utils a little bit

---
 tests/ci/ccache_utils.py | 62 ++++++----------------------------------
 1 file changed, 9 insertions(+), 53 deletions(-)

diff --git a/tests/ci/ccache_utils.py b/tests/ci/ccache_utils.py
index e8522127132..82620647cdd 100644
--- a/tests/ci/ccache_utils.py
+++ b/tests/ci/ccache_utils.py
@@ -1,14 +1,11 @@
 #!/usr/bin/env python3
 
 import logging
-import time
-import sys
 import os
 import shutil
 from pathlib import Path
 
-import requests  # type: ignore
-
+from build_download_helper import download_build_with_progress
 from compress_files import decompress_fast, compress_fast
 from env_helper import S3_DOWNLOAD, S3_BUILDS_BUCKET
 from s3_helper import S3Helper
@@ -16,56 +13,15 @@ from s3_helper import S3Helper
 DOWNLOAD_RETRIES_COUNT = 5
 
 
-def dowload_file_with_progress(url, path):
-    logging.info("Downloading from %s to temp path %s", url, path)
-    for i in range(DOWNLOAD_RETRIES_COUNT):
-        try:
-            with open(path, "wb") as f:
-                response = requests.get(url, stream=True)
-                response.raise_for_status()
-                total_length = response.headers.get("content-length")
-                if total_length is None or int(total_length) == 0:
-                    logging.info(
-                        "No content-length, will download file without progress"
-                    )
-                    f.write(response.content)
-                else:
-                    dl = 0
-                    total_length = int(total_length)
-                    logging.info("Content length is %ld bytes", total_length)
-                    for data in response.iter_content(chunk_size=4096):
-                        dl += len(data)
-                        f.write(data)
-                        if sys.stdout.isatty():
-                            done = int(50 * dl / total_length)
-                            percent = int(100 * float(dl) / total_length)
-                            eq_str = "=" * done
-                            space_str = " " * (50 - done)
-                            sys.stdout.write(f"\r[{eq_str}{space_str}] {percent}%")
-                            sys.stdout.flush()
-            break
-        except Exception as ex:
-            sys.stdout.write("\n")
-            time.sleep(3)
-            logging.info("Exception while downloading %s, retry %s", ex, i + 1)
-            if os.path.exists(path):
-                os.remove(path)
-    else:
-        raise Exception(f"Cannot download dataset from {url}, all retries exceeded")
-
-    sys.stdout.write("\n")
-    logging.info("Downloading finished")
-
-
 def get_ccache_if_not_exists(
-    path_to_ccache_dir: str,
+    path_to_ccache_dir: Path,
     s3_helper: S3Helper,
     current_pr_number: int,
-    temp_path: str,
+    temp_path: Path,
     release_pr: int,
 ) -> int:
     """returns: number of PR for downloaded PR. -1 if ccache not found"""
-    ccache_name = os.path.basename(path_to_ccache_dir)
+    ccache_name = path_to_ccache_dir.name
     cache_found = False
     prs_to_check = [current_pr_number]
     # Release PR is either 0 or defined
@@ -94,11 +50,11 @@ def get_ccache_if_not_exists(
 
         logging.info("Found ccache on path %s", obj)
         url = f"{S3_DOWNLOAD}/{S3_BUILDS_BUCKET}/{obj}"
-        compressed_cache = os.path.join(temp_path, os.path.basename(obj))
-        dowload_file_with_progress(url, compressed_cache)
+        compressed_cache = temp_path / os.path.basename(obj)
+        download_build_with_progress(url, compressed_cache)
 
-        path_to_decompress = str(Path(path_to_ccache_dir).parent)
-        if not os.path.exists(path_to_decompress):
+        path_to_decompress = path_to_ccache_dir.parent
+        if not path_to_decompress.exists():
             os.makedirs(path_to_decompress)
 
         if os.path.exists(path_to_ccache_dir):
@@ -106,7 +62,7 @@ def get_ccache_if_not_exists(
             logging.info("Ccache already exists, removing it")
 
         logging.info("Decompressing cache to path %s", path_to_decompress)
-        decompress_fast(compressed_cache, path_to_decompress)
+        decompress_fast(compressed_cache.as_posix(), path_to_decompress.as_posix())
         logging.info("Files on path %s", os.listdir(path_to_decompress))
         cache_found = True
         ccache_pr = pr_number

From ed7ade255a8c1a122c2204fa3b307a8dfcf5cccc Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Thu, 10 Aug 2023 16:17:00 +0200
Subject: [PATCH 196/249] Use sccache for rust builds

---
 rust/.cargo/config.toml.in | 1 +
 rust/CMakeLists.txt        | 7 +++++++
 2 files changed, 8 insertions(+)

diff --git a/rust/.cargo/config.toml.in b/rust/.cargo/config.toml.in
index db83145d449..ef2dc7896d6 100644
--- a/rust/.cargo/config.toml.in
+++ b/rust/.cargo/config.toml.in
@@ -5,6 +5,7 @@ CXXFLAGS = "@RUST_CXXFLAGS@"
 [build]
 rustflags = @RUSTFLAGS@
 rustdocflags = @RUSTFLAGS@
+@RUSTCWRAPPER@
 
 [unstable]
 @RUST_CARGO_BUILD_STD@
diff --git a/rust/CMakeLists.txt b/rust/CMakeLists.txt
index ca0886cb300..6aa25e95679 100644
--- a/rust/CMakeLists.txt
+++ b/rust/CMakeLists.txt
@@ -14,6 +14,13 @@ macro(configure_rustc)
         set(RUST_CFLAGS "${RUST_CFLAGS} --sysroot ${CMAKE_SYSROOT}")
     endif()
 
+    if(CCACHE_EXECUTABLE MATCHES "/sccache$")
+        message(STATUS "Using RUSTC_WRAPPER: ${CCACHE_EXECUTABLE}")
+        set(RUSTCWRAPPER "rustc-wrapper = \"${CCACHE_EXECUTABLE}\"")
+    else()
+        set(RUSTCWRAPPER "")
+    endif()
+
     set(RUSTFLAGS "[]")
     set(RUST_CARGO_BUILD_STD "")
     # For more info: https://doc.rust-lang.org/beta/unstable-book/compiler-flags/sanitizer.html#memorysanitizer

From 8656f9090b976ba8acacff7a70ca46313b472ed9 Mon Sep 17 00:00:00 2001
From: Arthur Passos <arthur.ti@outlook.com>
Date: Thu, 10 Aug 2023 17:22:01 -0300
Subject: [PATCH 197/249] black

---
 tests/integration/test_reverse_dns_query/test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/test_reverse_dns_query/test.py b/tests/integration/test_reverse_dns_query/test.py
index 1b94885939e..00c3956f74f 100644
--- a/tests/integration/test_reverse_dns_query/test.py
+++ b/tests/integration/test_reverse_dns_query/test.py
@@ -65,7 +65,7 @@ def setup_ch_server(dns_server_ip):
 
 def test_reverse_dns_query(started_cluster):
     dns_server_ip = cluster.get_instance_ip(cluster.coredns_host)
-    random_ipv6 = "4ae8:fa0f:ee1d:68c5:0b76:1b79:7ae6:1549" # https://commentpicker.com/ip-address-generator.php
+    random_ipv6 = "4ae8:fa0f:ee1d:68c5:0b76:1b79:7ae6:1549"  # https://commentpicker.com/ip-address-generator.php
     setup_dns_server(random_ipv6)
     setup_ch_server(dns_server_ip)
 

From 18a291a4c789ed60b246cacba7dc2ecb1ba06738 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Thu, 10 Aug 2023 19:12:09 +0200
Subject: [PATCH 198/249] Upload cargo cache for rust to S3 and reuse

---
 docker/packager/packager          | 12 +++++-
 tests/ci/build_check.py           | 11 +++++
 tests/ci/build_download_helper.py | 10 ++++-
 tests/ci/ccache_utils.py          | 68 ++++++++++++++++++++++++++++++-
 4 files changed, 97 insertions(+), 4 deletions(-)

diff --git a/docker/packager/packager b/docker/packager/packager
index c0baacccef2..2d50d7efd4b 100755
--- a/docker/packager/packager
+++ b/docker/packager/packager
@@ -80,9 +80,11 @@ def run_docker_image_with_env(
     output_dir: Path,
     env_variables: List[str],
     ch_root: Path,
+    cargo_cache_dir: Path,
     ccache_dir: Optional[Path],
 ) -> None:
     output_dir.mkdir(parents=True, exist_ok=True)
+    cargo_cache_dir.mkdir(parents=True, exist_ok=True)
 
     env_part = " -e ".join(env_variables)
     if env_part:
@@ -105,7 +107,7 @@ def run_docker_image_with_env(
     cmd = (
         f"docker run --network=host --user={user} --rm {ccache_mount}"
         f"--volume={output_dir}:/output --volume={ch_root}:/build {env_part} "
-        f"{interactive} {image_name}"
+        f"--volume={cargo_cache_dir}:/rust/cargo/registry {interactive} {image_name}"
     )
 
     logging.info("Will build ClickHouse pkg with cmd: '%s'", cmd)
@@ -417,6 +419,13 @@ def parse_args() -> argparse.Namespace:
         action="store_true",
         help="if set, the build fails on errors writing cache to S3",
     )
+    parser.add_argument(
+        "--cargo-cache-dir",
+        default=Path(os.getenv("CARGO_HOME", "") or Path.home() / ".cargo")
+        / "registry",
+        type=dir_name,
+        help="a directory to preserve the rust cargo crates",
+    )
     parser.add_argument("--force-build-image", action="store_true")
     parser.add_argument("--version")
     parser.add_argument("--official", action="store_true")
@@ -497,6 +506,7 @@ def main() -> None:
         args.output_dir,
         env_prepared,
         ch_root,
+        args.cargo_cache_dir,
         args.ccache_dir,
     )
     logging.info("Output placed into %s", args.output_dir)
diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py
index 9e6370c2e6e..3e853c0ce41 100644
--- a/tests/ci/build_check.py
+++ b/tests/ci/build_check.py
@@ -10,6 +10,7 @@ import sys
 import time
 
 from ci_config import CI_CONFIG, BuildConfig
+from ccache_utils import CargoCache
 from docker_pull_helper import get_image_with_version
 from env_helper import (
     GITHUB_JOB,
@@ -53,6 +54,7 @@ def get_packager_cmd(
     build_config: BuildConfig,
     packager_path: str,
     output_path: Path,
+    cargo_cache_dir: Path,
     build_version: str,
     image_version: str,
     official: bool,
@@ -75,6 +77,7 @@ def get_packager_cmd(
     cmd += " --cache=sccache"
     cmd += " --s3-rw-access"
     cmd += f" --s3-bucket={S3_BUILDS_BUCKET}"
+    cmd += f" --cargo-cache-dir={cargo_cache_dir}"
 
     if build_config.additional_pkgs:
         cmd += " --additional-pkgs"
@@ -287,11 +290,16 @@ def main():
 
     build_output_path = temp_path / build_name
     os.makedirs(build_output_path, exist_ok=True)
+    cargo_cache = CargoCache(
+        temp_path / "cargo_cache" / "registry", temp_path, s3_helper
+    )
+    cargo_cache.download()
 
     packager_cmd = get_packager_cmd(
         build_config,
         os.path.join(REPO_COPY, "docker/packager"),
         build_output_path,
+        cargo_cache.directory,
         version.string,
         image_version,
         official_flag,
@@ -309,6 +317,9 @@ def main():
         f"sudo chown -R ubuntu:ubuntu {build_output_path}", shell=True
     )
     logging.info("Build finished with %s, log path %s", success, log_path)
+    if success:
+        cargo_cache.upload()
+
     if not success:
         # We check if docker works, because if it's down, it's infrastructure
         try:
diff --git a/tests/ci/build_download_helper.py b/tests/ci/build_download_helper.py
index ec4cf8f9bfa..a6fda749494 100644
--- a/tests/ci/build_download_helper.py
+++ b/tests/ci/build_download_helper.py
@@ -16,6 +16,10 @@ from ci_config import CI_CONFIG
 DOWNLOAD_RETRIES_COUNT = 5
 
 
+class DownloadException(Exception):
+    pass
+
+
 def get_with_retries(
     url: str,
     retries: int = DOWNLOAD_RETRIES_COUNT,
@@ -149,7 +153,9 @@ def download_build_with_progress(url: str, path: Path) -> None:
             if os.path.exists(path):
                 os.remove(path)
     else:
-        raise Exception(f"Cannot download dataset from {url}, all retries exceeded")
+        raise DownloadException(
+            f"Cannot download dataset from {url}, all retries exceeded"
+        )
 
     if sys.stdout.isatty():
         sys.stdout.write("\n")
@@ -174,7 +180,7 @@ def download_builds_filter(
     print(urls)
 
     if not urls:
-        raise Exception("No build URLs found")
+        raise DownloadException("No build URLs found")
 
     download_builds(result_path, urls, filter_fn)
 
diff --git a/tests/ci/ccache_utils.py b/tests/ci/ccache_utils.py
index 82620647cdd..d3ca4777134 100644
--- a/tests/ci/ccache_utils.py
+++ b/tests/ci/ccache_utils.py
@@ -3,11 +3,15 @@
 import logging
 import os
 import shutil
+from hashlib import md5
 from pathlib import Path
 
-from build_download_helper import download_build_with_progress
+import requests  # type: ignore
+
+from build_download_helper import download_build_with_progress, DownloadException
 from compress_files import decompress_fast, compress_fast
 from env_helper import S3_DOWNLOAD, S3_BUILDS_BUCKET
+from git_helper import git_runner
 from s3_helper import S3Helper
 
 DOWNLOAD_RETRIES_COUNT = 5
@@ -90,3 +94,65 @@ def upload_ccache(path_to_ccache_dir, s3_helper, current_pr_number, temp_path):
     logging.info("Will upload %s to path %s", compressed_cache_path, s3_path)
     s3_helper.upload_build_file_to_s3(compressed_cache_path, s3_path)
     logging.info("Upload finished")
+
+
+class CargoCache:
+    PREFIX = "ccache/cargo_cache"
+
+    def __init__(
+        self,
+        directory: Path,
+        temp_path: Path,
+        s3_helper: S3Helper,
+    ):
+        self._cargo_lock_file = Path(git_runner.cwd) / "rust" / "Cargo.lock"
+        self.lock_hash = md5(self._cargo_lock_file.read_bytes()).hexdigest()
+        self.directory = directory
+        self.archive_name = f"Cargo_cache_{self.lock_hash}.tar.zst"
+        self.temp_path = temp_path
+        self.s3_helper = s3_helper
+        self._url = (
+            f"{S3_DOWNLOAD}/{S3_BUILDS_BUCKET}/{self.PREFIX}/{self.archive_name}"
+        )
+        self._force_upload_cache = False
+
+    def download(self):
+        logging.info("Searching rust cache for Cargo.lock md5 %s", self.lock_hash)
+        compressed_cache = self.temp_path / self.archive_name
+        try:
+            download_build_with_progress(self._url, compressed_cache)
+        except DownloadException:
+            logging.warning("Unable downloading cargo cache, creating empty directory")
+            self.directory.mkdir(parents=True, exist_ok=True)
+            return
+
+        # decompress the cache and check if the necessary directory is there
+        self.directory.parent.mkdir(parents=True, exist_ok=True)
+        decompress_fast(compressed_cache.as_posix(), self.directory.parent.as_posix())
+        if not self.directory.exists():
+            logging.warning(
+                "The cargo cache archive was successfully downloaded and "
+                "decompressed, but %s does not exitst. Creating empty one",
+                self.directory,
+            )
+            logging.info("Cache for Cargo.lock md5 %s will be uploaded", self.lock_hash)
+            self.directory.mkdir(parents=True, exist_ok=True)
+
+    def upload(self):
+        if not self._force_upload_cache:
+            cache_response = requests.head(self._url)
+            if cache_response.status_code == 200:
+                logging.info(
+                    "Remote cargo cache %s already exist, won't reupload", self._url
+                )
+                return
+
+        logging.info("Compressing cargo cache")
+        archive_path = self.directory.parent / self.archive_name
+        compress_fast(
+            self.directory.as_posix(),
+            archive_path.as_posix(),
+        )
+        s3_path = f"{self.PREFIX}/{self.archive_name}"
+        logging.info("Uploading %s to S3 path %s", archive_path, s3_path)
+        self.s3_helper.upload_build_file_to_s3(archive_path, s3_path)

From c8d5deda837f2a1c290ee31f6ffdc22007394e67 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Thu, 10 Aug 2023 22:09:05 +0200
Subject: [PATCH 199/249] Add typing to compress_files

---
 tests/ci/compress_files.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tests/ci/compress_files.py b/tests/ci/compress_files.py
index 8d52d030b84..670a12d9693 100644
--- a/tests/ci/compress_files.py
+++ b/tests/ci/compress_files.py
@@ -3,8 +3,10 @@ import subprocess
 import logging
 import os
 
+from typing import Optional
 
-def compress_file_fast(path, archive_path):
+
+def compress_file_fast(path: str, archive_path: str) -> None:
     if archive_path.endswith(".zst"):
         subprocess.check_call(f"zstd < {path} > {archive_path}", shell=True)
     elif os.path.exists("/usr/bin/pigz"):
@@ -13,7 +15,7 @@ def compress_file_fast(path, archive_path):
         subprocess.check_call(f"gzip < {path} > {archive_path}", shell=True)
 
 
-def compress_fast(path, archive_path, exclude=None):
+def compress_fast(path: str, archive_path: str, exclude: Optional[str] = None) -> None:
     program_part = ""
     if archive_path.endswith(".zst"):
         logging.info("zstd will be used for compression")
@@ -43,7 +45,7 @@ def compress_fast(path, archive_path, exclude=None):
     subprocess.check_call(cmd, shell=True)
 
 
-def decompress_fast(archive_path, result_path=None):
+def decompress_fast(archive_path: str, result_path: Optional[str] = None) -> None:
     program_part = ""
     if archive_path.endswith(".zst"):
         logging.info(

From dcb1d3ed000cccf0ee69a2e5d8d3cac6f4e0f888 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Thu, 10 Aug 2023 22:41:41 +0200
Subject: [PATCH 200/249] Rewrite compress_file to pathlib.Path

---
 tests/ci/ccache_utils.py   | 24 ++++++++++++------------
 tests/ci/compress_files.py | 35 +++++++++++++++++++----------------
 tests/ci/install_check.py  |  5 +----
 tests/ci/jepsen_check.py   |  9 +++++----
 tests/ci/s3_helper.py      |  6 +++++-
 5 files changed, 42 insertions(+), 37 deletions(-)

diff --git a/tests/ci/ccache_utils.py b/tests/ci/ccache_utils.py
index d3ca4777134..e726bf80b92 100644
--- a/tests/ci/ccache_utils.py
+++ b/tests/ci/ccache_utils.py
@@ -66,7 +66,7 @@ def get_ccache_if_not_exists(
             logging.info("Ccache already exists, removing it")
 
         logging.info("Decompressing cache to path %s", path_to_decompress)
-        decompress_fast(compressed_cache.as_posix(), path_to_decompress.as_posix())
+        decompress_fast(compressed_cache, path_to_decompress)
         logging.info("Files on path %s", os.listdir(path_to_decompress))
         cache_found = True
         ccache_pr = pr_number
@@ -82,15 +82,18 @@ def get_ccache_if_not_exists(
     return ccache_pr
 
 
-def upload_ccache(path_to_ccache_dir, s3_helper, current_pr_number, temp_path):
+def upload_ccache(
+    path_to_ccache_dir: Path,
+    s3_helper: S3Helper,
+    current_pr_number: int,
+    temp_path: Path,
+) -> None:
     logging.info("Uploading cache %s for pr %s", path_to_ccache_dir, current_pr_number)
-    ccache_name = os.path.basename(path_to_ccache_dir)
-    compressed_cache_path = os.path.join(temp_path, ccache_name + ".tar.zst")
+    ccache_name = path_to_ccache_dir.name
+    compressed_cache_path = temp_path / f"{ccache_name}.tar.zst"
     compress_fast(path_to_ccache_dir, compressed_cache_path)
 
-    s3_path = (
-        str(current_pr_number) + "/ccaches/" + os.path.basename(compressed_cache_path)
-    )
+    s3_path = f"{current_pr_number}/ccaches/{compressed_cache_path.name}"
     logging.info("Will upload %s to path %s", compressed_cache_path, s3_path)
     s3_helper.upload_build_file_to_s3(compressed_cache_path, s3_path)
     logging.info("Upload finished")
@@ -128,7 +131,7 @@ class CargoCache:
 
         # decompress the cache and check if the necessary directory is there
         self.directory.parent.mkdir(parents=True, exist_ok=True)
-        decompress_fast(compressed_cache.as_posix(), self.directory.parent.as_posix())
+        decompress_fast(compressed_cache, self.directory.parent)
         if not self.directory.exists():
             logging.warning(
                 "The cargo cache archive was successfully downloaded and "
@@ -149,10 +152,7 @@ class CargoCache:
 
         logging.info("Compressing cargo cache")
         archive_path = self.directory.parent / self.archive_name
-        compress_fast(
-            self.directory.as_posix(),
-            archive_path.as_posix(),
-        )
+        compress_fast(self.directory, archive_path)
         s3_path = f"{self.PREFIX}/{self.archive_name}"
         logging.info("Uploading %s to S3 path %s", archive_path, s3_path)
         self.s3_helper.upload_build_file_to_s3(archive_path, s3_path)
diff --git a/tests/ci/compress_files.py b/tests/ci/compress_files.py
index 670a12d9693..d8e691ce3b4 100644
--- a/tests/ci/compress_files.py
+++ b/tests/ci/compress_files.py
@@ -1,26 +1,31 @@
 #!/usr/bin/env python3
 import subprocess
 import logging
-import os
 
+from pathlib import Path
 from typing import Optional
 
 
-def compress_file_fast(path: str, archive_path: str) -> None:
-    if archive_path.endswith(".zst"):
+PIGZ = Path("/usr/bin/pigz")
+
+
+def compress_file_fast(path: Path, archive_path: Path) -> None:
+    if archive_path.suffix == ".zst":
         subprocess.check_call(f"zstd < {path} > {archive_path}", shell=True)
-    elif os.path.exists("/usr/bin/pigz"):
+    elif PIGZ.exists():
         subprocess.check_call(f"pigz < {path} > {archive_path}", shell=True)
     else:
         subprocess.check_call(f"gzip < {path} > {archive_path}", shell=True)
 
 
-def compress_fast(path: str, archive_path: str, exclude: Optional[str] = None) -> None:
+def compress_fast(
+    path: Path, archive_path: Path, exclude: Optional[Path] = None
+) -> None:
     program_part = ""
-    if archive_path.endswith(".zst"):
+    if archive_path.suffix == ".zst":
         logging.info("zstd will be used for compression")
         program_part = "--use-compress-program='zstd --threads=0'"
-    elif os.path.exists("/usr/bin/pigz"):
+    elif PIGZ.exists():
         logging.info("pigz found, will compress and decompress faster")
         program_part = "--use-compress-program='pigz'"
     else:
@@ -34,27 +39,25 @@ def compress_fast(path: str, archive_path: str, exclude: Optional[str] = None) -
     else:
         exclude_part = f"--exclude {exclude}"
 
-    fname = os.path.basename(path)
-    if os.path.isfile(path):
-        path = os.path.dirname(path)
-    else:
-        path += "/.."
+    fname = path.name
 
-    cmd = f"tar {program_part} {exclude_part} -cf {archive_path} -C {path} {fname}"
+    cmd = (
+        f"tar {program_part} {exclude_part} -cf {archive_path} -C {path.parent} {fname}"
+    )
     logging.debug("compress_fast cmd: %s", cmd)
     subprocess.check_call(cmd, shell=True)
 
 
-def decompress_fast(archive_path: str, result_path: Optional[str] = None) -> None:
+def decompress_fast(archive_path: Path, result_path: Optional[Path] = None) -> None:
     program_part = ""
-    if archive_path.endswith(".zst"):
+    if archive_path.suffix == ".zst":
         logging.info(
             "zstd will be used for decompression ('%s' -> '%s')",
             archive_path,
             result_path,
         )
         program_part = "--use-compress-program='zstd --threads=0'"
-    elif os.path.exists("/usr/bin/pigz"):
+    elif PIGZ.exists():
         logging.info(
             "pigz found, will compress and decompress faster ('%s' -> '%s')",
             archive_path,
diff --git a/tests/ci/install_check.py b/tests/ci/install_check.py
index 73e1a6ef739..010b0dab408 100644
--- a/tests/ci/install_check.py
+++ b/tests/ci/install_check.py
@@ -196,10 +196,7 @@ def test_install(image: DockerImage, tests: Dict[str, str]) -> TestResults:
                 status = FAIL
             copy2(log_file, LOGS_PATH)
             archive_path = TEMP_PATH / f"{container_name}-{retry}.tar.gz"
-            compress_fast(
-                LOGS_PATH.as_posix(),
-                archive_path.as_posix(),
-            )
+            compress_fast(LOGS_PATH, archive_path)
             logs.append(archive_path)
 
         subprocess.check_call(f"docker kill -s 9 {container_id}", shell=True)
diff --git a/tests/ci/jepsen_check.py b/tests/ci/jepsen_check.py
index c21fafa2605..54eb70536a8 100644
--- a/tests/ci/jepsen_check.py
+++ b/tests/ci/jepsen_check.py
@@ -1,11 +1,12 @@
 #!/usr/bin/env python3
 
-import time
+import argparse
 import logging
 import os
 import sys
+import time
 
-import argparse
+from pathlib import Path
 
 import boto3  # type: ignore
 import requests  # type: ignore
@@ -268,8 +269,8 @@ if __name__ == "__main__":
             description = "Found invalid analysis (ﾉಥ益ಥ）ﾉ ┻━┻"
 
         compress_fast(
-            os.path.join(result_path, "store"),
-            os.path.join(result_path, "jepsen_store.tar.zst"),
+            Path(result_path) / "store",
+            Path(result_path) / "jepsen_store.tar.zst",
         )
         additional_data.append(os.path.join(result_path, "jepsen_store.tar.zst"))
     except Exception as ex:
diff --git a/tests/ci/s3_helper.py b/tests/ci/s3_helper.py
index 9ee0a431294..e21e03cc8b6 100644
--- a/tests/ci/s3_helper.py
+++ b/tests/ci/s3_helper.py
@@ -102,7 +102,11 @@ class S3Helper:
                     file_path,
                     file_path + ".zst",
                 )
-                compress_file_fast(file_path, file_path + ".zst")
+                # FIXME: rewrite S3 to Path
+                _file_path = Path(file_path)
+                compress_file_fast(
+                    _file_path, _file_path.with_suffix(_file_path.suffix + ".zst")
+                )
                 file_path += ".zst"
                 s3_path += ".zst"
             else:

From ad7c60e2161cb5ac18462ca4ccdcf32fe0422fbc Mon Sep 17 00:00:00 2001
From: Yakov Olkhovskiy <yakov@clickhouse.com>
Date: Thu, 10 Aug 2023 21:08:47 +0000
Subject: [PATCH 201/249] fix

---
 docker/packager/binary/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile
index 394ace2b220..7b7d8c59229 100644
--- a/docker/packager/binary/Dockerfile
+++ b/docker/packager/binary/Dockerfile
@@ -112,7 +112,7 @@ RUN curl -sL -O https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/M
 RUN curl -Lo /tmp/mold.tar.gz "https://github.com/rui314/mold/releases/download/v2.0.0/mold-2.0.0-x86_64-linux.tar.gz" \
     && mkdir /tmp/mold \
     && tar -xzf /tmp/mold.tar.gz -C /tmp/mold \
-    && cp /tmp/mold/mold*/bin/* /usr/bin \
+    && cp -r /tmp/mold/mold*/* /usr \
     && rm -rf /tmp/mold \
     && rm /tmp/mold.tar.gz
 

From f32c9a84d95c8b7aecad0e7d17d9e3e00473dee5 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Thu, 10 Aug 2023 23:28:46 +0200
Subject: [PATCH 202/249] fix

---
 .../0_stateless/01169_alter_partition_isolation_stress.sh | 8 ++++----
 .../01169_old_alter_partition_isolation_stress.sh         | 8 ++++----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/tests/queries/0_stateless/01169_alter_partition_isolation_stress.sh b/tests/queries/0_stateless/01169_alter_partition_isolation_stress.sh
index f2348c29146..d4884cbf457 100755
--- a/tests/queries/0_stateless/01169_alter_partition_isolation_stress.sh
+++ b/tests/queries/0_stateless/01169_alter_partition_isolation_stress.sh
@@ -242,10 +242,10 @@ wait ||:
 
 wait_for_queries_to_finish 40
 
-$CLICKHOUSE_CLIENT -q "SELECT type, count(n) = countDistinct(n) FROM merge(currentDatabase(), '') GROUP BY type ORDER BY type"
-$CLICKHOUSE_CLIENT -q "SELECT DISTINCT arraySort(groupArrayIf(n, type=1)) = arraySort(groupArrayIf(n, type=2)) FROM merge(currentDatabase(), '') GROUP BY _table ORDER BY _table"
-$CLICKHOUSE_CLIENT -q "SELECT count(n), sum(n) FROM merge(currentDatabase(), '') WHERE type=4"
-$CLICKHOUSE_CLIENT -q "SELECT type, count(n) == max(n), sum(n) == max(n)*(max(n)+1)/2 FROM merge(currentDatabase(), '') WHERE type IN (1, 2) GROUP BY type ORDER BY type"
+$CLICKHOUSE_CLIENT --implicit_transaction=1 --throw_on_unsupported_query_inside_transaction=0 -q "SELECT type, count(n) = countDistinct(n) FROM merge(currentDatabase(), '') GROUP BY type ORDER BY type"
+$CLICKHOUSE_CLIENT --implicit_transaction=1 --throw_on_unsupported_query_inside_transaction=0 -q "SELECT DISTINCT arraySort(groupArrayIf(n, type=1)) = arraySort(groupArrayIf(n, type=2)) FROM merge(currentDatabase(), '') GROUP BY _table ORDER BY _table"
+$CLICKHOUSE_CLIENT --implicit_transaction=1 --throw_on_unsupported_query_inside_transaction=0 -q "SELECT count(n), sum(n) FROM merge(currentDatabase(), '') WHERE type=4"
+$CLICKHOUSE_CLIENT --implicit_transaction=1 --throw_on_unsupported_query_inside_transaction=0 -q "SELECT type, count(n) == max(n), sum(n) == max(n)*(max(n)+1)/2 FROM merge(currentDatabase(), '') WHERE type IN (1, 2) GROUP BY type ORDER BY type"
 
 $CLICKHOUSE_CLIENT --query "DROP TABLE src";
 $CLICKHOUSE_CLIENT --query "DROP TABLE dst";
diff --git a/tests/queries/0_stateless/01169_old_alter_partition_isolation_stress.sh b/tests/queries/0_stateless/01169_old_alter_partition_isolation_stress.sh
index 32ad78dead6..0d2016952d4 100755
--- a/tests/queries/0_stateless/01169_old_alter_partition_isolation_stress.sh
+++ b/tests/queries/0_stateless/01169_old_alter_partition_isolation_stress.sh
@@ -113,10 +113,10 @@ kill -TERM $PID_2
 wait
 wait_for_queries_to_finish
 
-$CLICKHOUSE_CLIENT -q "SELECT type, count(n) = countDistinct(n) FROM merge(currentDatabase(), '') GROUP BY type ORDER BY type"
-$CLICKHOUSE_CLIENT -q "SELECT DISTINCT arraySort(groupArrayIf(n, type=1)) = arraySort(groupArrayIf(n, type=2)) FROM merge(currentDatabase(), '') GROUP BY _table ORDER BY _table"
-$CLICKHOUSE_CLIENT -q "SELECT count(n), sum(n) FROM merge(currentDatabase(), '') WHERE type=4"
-$CLICKHOUSE_CLIENT -q "SELECT type, count(n) == max(n), sum(n) == max(n)*(max(n)+1)/2 FROM merge(currentDatabase(), '') WHERE type IN (1, 2) GROUP BY type ORDER BY type"
+$CLICKHOUSE_CLIENT --implicit_transaction=1 --throw_on_unsupported_query_inside_transaction=0 -q "SELECT type, count(n) = countDistinct(n) FROM merge(currentDatabase(), '') GROUP BY type ORDER BY type"
+$CLICKHOUSE_CLIENT --implicit_transaction=1 --throw_on_unsupported_query_inside_transaction=0 -q "SELECT DISTINCT arraySort(groupArrayIf(n, type=1)) = arraySort(groupArrayIf(n, type=2)) FROM merge(currentDatabase(), '') GROUP BY _table ORDER BY _table"
+$CLICKHOUSE_CLIENT --implicit_transaction=1 --throw_on_unsupported_query_inside_transaction=0 -q "SELECT count(n), sum(n) FROM merge(currentDatabase(), '') WHERE type=4"
+$CLICKHOUSE_CLIENT --implicit_transaction=1 --throw_on_unsupported_query_inside_transaction=0 -q "SELECT type, count(n) == max(n), sum(n) == max(n)*(max(n)+1)/2 FROM merge(currentDatabase(), '') WHERE type IN (1, 2) GROUP BY type ORDER BY type"
 
 
 $CLICKHOUSE_CLIENT --query "DROP TABLE src";

From 45655928d1913d5610c9124fb9a6350cd594f4c2 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 10 Aug 2023 23:39:06 +0200
Subject: [PATCH 203/249] clickhouse-benchmark: connect in parallel

---
 programs/benchmark/Benchmark.cpp | 131 ++++++-------------------------
 1 file changed, 24 insertions(+), 107 deletions(-)

diff --git a/programs/benchmark/Benchmark.cpp b/programs/benchmark/Benchmark.cpp
index 466a0c194f7..da4f01e4f4a 100644
--- a/programs/benchmark/Benchmark.cpp
+++ b/programs/benchmark/Benchmark.cpp
@@ -1,8 +1,6 @@
 #include <unistd.h>
 #include <cstdlib>
-#include <fcntl.h>
 #include <csignal>
-#include <ctime>
 #include <iostream>
 #include <fstream>
 #include <iomanip>
@@ -18,9 +16,7 @@
 #include <Common/Exception.h>
 #include <Common/randomSeed.h>
 #include <Common/clearPasswordFromCommandLine.h>
-#include <Core/Types.h>
 #include <IO/ReadBufferFromFileDescriptor.h>
-#include <IO/WriteBufferFromFileDescriptor.h>
 #include <IO/WriteBufferFromFile.h>
 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
@@ -79,7 +75,6 @@ public:
             bool randomize_,
             size_t max_iterations_,
             double max_time_,
-            const String & json_path_,
             size_t confidence_,
             const String & query_id_,
             const String & query_to_execute_,
@@ -98,7 +93,6 @@ public:
         cumulative(cumulative_),
         max_iterations(max_iterations_),
         max_time(max_time_),
-        json_path(json_path_),
         confidence(confidence_),
         query_id(query_id_),
         query_to_execute(query_to_execute_),
@@ -165,9 +159,6 @@ public:
 
     int main(const std::vector<std::string> &) override
     {
-        if (!json_path.empty() && fs::exists(json_path)) /// Clear file with previous results
-            fs::remove(json_path);
-
         readQueries();
         runBenchmark();
         return 0;
@@ -197,7 +188,6 @@ private:
     bool cumulative;
     size_t max_iterations;
     double max_time;
-    String json_path;
     size_t confidence;
     String query_id;
     String query_to_execute;
@@ -226,26 +216,23 @@ private:
         size_t read_bytes = 0;
         size_t result_rows = 0;
         size_t result_bytes = 0;
-        double work_time = 0;
 
         using Sampler = ReservoirSampler<double>;
         Sampler sampler {1 << 16};
 
-        void add(double seconds, size_t read_rows_inc, size_t read_bytes_inc, size_t result_rows_inc, size_t result_bytes_inc)
+        void add(double duration, size_t read_rows_inc, size_t read_bytes_inc, size_t result_rows_inc, size_t result_bytes_inc)
         {
             ++queries;
-            work_time += seconds;
             read_rows += read_rows_inc;
             read_bytes += read_bytes_inc;
             result_rows += result_rows_inc;
             result_bytes += result_bytes_inc;
-            sampler.insert(seconds);
+            sampler.insert(duration);
         }
 
         void clear()
         {
             queries = 0;
-            work_time = 0;
             read_rows = 0;
             read_bytes = 0;
             result_rows = 0;
@@ -331,10 +318,13 @@ private:
                 return false;
             }
 
-            if (delay > 0 && delay_watch.elapsedSeconds() > delay)
+            double seconds = delay_watch.elapsedSeconds();
+            if (delay > 0 && seconds > delay)
             {
                 printNumberOfQueriesExecuted(queries_executed);
-                cumulative ? report(comparison_info_total) : report(comparison_info_per_interval);
+                cumulative
+                    ? report(comparison_info_total, total_watch.elapsedSeconds())
+                    : report(comparison_info_per_interval, seconds);
                 delay_watch.restart();
             }
         }
@@ -350,16 +340,7 @@ private:
         try
         {
             for (size_t i = 0; i < concurrency; ++i)
-            {
-                EntryPtrs connection_entries;
-                connection_entries.reserve(connections.size());
-
-                for (const auto & connection : connections)
-                    connection_entries.emplace_back(std::make_shared<Entry>(
-                            connection->get(ConnectionTimeouts::getTCPTimeoutsWithoutFailover(settings))));
-
-                pool.scheduleOrThrowOnError([this, connection_entries]() mutable { thread(connection_entries); });
-            }
+                pool.scheduleOrThrowOnError([this]() mutable { thread(); });
         }
         catch (...)
         {
@@ -389,21 +370,18 @@ private:
         pool.wait();
         total_watch.stop();
 
-        if (!json_path.empty())
-            reportJSON(comparison_info_total, json_path);
-
         printNumberOfQueriesExecuted(queries_executed);
-        report(comparison_info_total);
+        report(comparison_info_total, total_watch.elapsedSeconds());
     }
 
 
-    void thread(EntryPtrs & connection_entries)
+    void thread()
     {
         Query query;
 
         /// Randomly choosing connection index
         pcg64 generator(randomSeed());
-        std::uniform_int_distribution<size_t> distribution(0, connection_entries.size() - 1);
+        std::uniform_int_distribution<size_t> distribution(0, connections.size() - 1);
 
         /// In these threads we do not accept INT signal.
         sigset_t sig_set;
@@ -423,15 +401,13 @@ private:
                 extracted = queue.tryPop(query, 100);
 
                 if (shutdown || (max_iterations && queries_executed == max_iterations))
-                {
                     return;
-                }
             }
 
             const auto connection_index = distribution(generator);
             try
             {
-                execute(connection_entries, query, connection_index);
+                execute(query, connection_index);
                 consecutive_errors = 0;
             }
             catch (...)
@@ -460,17 +436,18 @@ private:
         }
     }
 
-    void execute(EntryPtrs & connection_entries, Query & query, size_t connection_index)
+    void execute(Query & query, size_t connection_index)
     {
         Stopwatch watch;
 
-        Connection & connection = **connection_entries[connection_index];
+        ConnectionPool::Entry entry = connections[connection_index]->get(
+            ConnectionTimeouts::getTCPTimeoutsWithoutFailover(settings));
 
         if (reconnect)
-            connection.disconnect();
+            entry->disconnect();
 
         RemoteQueryExecutor executor(
-            connection, query, {}, global_context, nullptr, Scalars(), Tables(), query_processing_stage);
+            *entry, query, {}, global_context, nullptr, Scalars(), Tables(), query_processing_stage);
         if (!query_id.empty())
             executor.setQueryId(query_id);
 
@@ -485,19 +462,19 @@ private:
 
         executor.finish();
 
-        double seconds = (display_client_side_time || progress.elapsed_ns == 0)
+        double duration = (display_client_side_time || progress.elapsed_ns == 0)
             ? watch.elapsedSeconds()
             : progress.elapsed_ns / 1e9;
 
         std::lock_guard lock(mutex);
 
         size_t info_index = round_robin ? 0 : connection_index;
-        comparison_info_per_interval[info_index]->add(seconds, progress.read_rows, progress.read_bytes, info.rows, info.bytes);
-        comparison_info_total[info_index]->add(seconds, progress.read_rows, progress.read_bytes, info.rows, info.bytes);
-        t_test.add(info_index, seconds);
+        comparison_info_per_interval[info_index]->add(duration, progress.read_rows, progress.read_bytes, info.rows, info.bytes);
+        comparison_info_total[info_index]->add(duration, progress.read_rows, progress.read_bytes, info.rows, info.bytes);
+        t_test.add(info_index, duration);
     }
 
-    void report(MultiStats & infos)
+    void report(MultiStats & infos, double seconds)
     {
         std::lock_guard lock(mutex);
 
@@ -510,8 +487,6 @@ private:
             if (0 == info->queries)
                 return;
 
-            double seconds = info->work_time / concurrency;
-
             std::string connection_description = connections[i]->getDescription();
             if (round_robin)
             {
@@ -525,10 +500,10 @@ private:
             }
             std::cerr
                     << connection_description << ", "
-                    << "queries " << info->queries << ", ";
+                    << "queries: " << info->queries << ", ";
             if (info->errors)
             {
-                std::cerr << "errors " << info->errors << ", ";
+                std::cerr << "errors: " << info->errors << ", ";
             }
             std::cerr
                     << "QPS: " << (info->queries / seconds) << ", "
@@ -567,62 +542,6 @@ private:
         }
     }
 
-    void reportJSON(MultiStats & infos, const std::string & filename)
-    {
-        WriteBufferFromFile json_out(filename);
-
-        std::lock_guard lock(mutex);
-
-        auto print_key_value = [&](auto key, auto value, bool with_comma = true)
-        {
-            json_out << double_quote << key << ": " << value << (with_comma ? ",\n" : "\n");
-        };
-
-        auto print_percentile = [&json_out](Stats & info, auto percent, bool with_comma = true)
-        {
-            json_out << "\"" << percent << "\": " << info.sampler.quantileNearest(percent / 100.0) << (with_comma ? ",\n" : "\n");
-        };
-
-        json_out << "{\n";
-
-        for (size_t i = 0; i < infos.size(); ++i)
-        {
-            const auto & info = infos[i];
-
-            json_out << double_quote << connections[i]->getDescription() << ": {\n";
-            json_out << double_quote << "statistics" << ": {\n";
-
-            double seconds = info->work_time / concurrency;
-
-            print_key_value("QPS", info->queries.load() / seconds);
-            print_key_value("RPS", info->read_rows / seconds);
-            print_key_value("MiBPS", info->read_bytes / seconds / 1048576);
-            print_key_value("RPS_result", info->result_rows / seconds);
-            print_key_value("MiBPS_result", info->result_bytes / seconds / 1048576);
-            print_key_value("num_queries", info->queries.load());
-            print_key_value("num_errors", info->errors, false);
-
-            json_out << "},\n";
-            json_out << double_quote << "query_time_percentiles" << ": {\n";
-
-            if (info->queries != 0)
-            {
-                for (int percent = 0; percent <= 90; percent += 10)
-                    print_percentile(*info, percent);
-
-                print_percentile(*info, 95);
-                print_percentile(*info, 99);
-                print_percentile(*info, 99.9);
-                print_percentile(*info, 99.99, false);
-            }
-
-            json_out << "}\n";
-            json_out << (i == infos.size() - 1 ? "}\n" : "},\n");
-        }
-
-        json_out << "}\n";
-    }
-
 public:
 
     ~Benchmark() override
@@ -675,7 +594,6 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv)
             ("iterations,i",  value<size_t>()->default_value(0),                "amount of queries to be executed")
             ("timelimit,t",   value<double>()->default_value(0.),               "stop launch of queries after specified time limit")
             ("randomize,r",                                                     "randomize order of execution")
-            ("json",          value<std::string>()->default_value(""),          "write final report to specified file in JSON format")
             ("host,h",        value<Strings>()->multitoken(),                   "list of hosts")
             ("port",          value<Ports>()->multitoken(),                     "list of ports")
             ("roundrobin",    "Instead of comparing queries for different --host/--port just pick one random --host/--port for every query and send query to it.")
@@ -739,7 +657,6 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv)
             options.count("randomize"),
             options["iterations"].as<size_t>(),
             options["timelimit"].as<double>(),
-            options["json"].as<std::string>(),
             options["confidence"].as<size_t>(),
             options["query_id"].as<std::string>(),
             options["query"].as<std::string>(),

From 5fa4cb934543e72848a2735140df59b5a898e592 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 11 Aug 2023 00:59:48 +0300
Subject: [PATCH 204/249] Revert "#2 Added new tests for session_log and fixed
 the inconsistency between login and logout."

---
 src/Interpreters/Session.cpp                  |  40 +--
 src/Interpreters/Session.h                    |   2 -
 src/Interpreters/SessionLog.cpp               |  14 +-
 src/Interpreters/SessionLog.h                 |   9 +-
 src/Server/HTTPHandler.cpp                    |   3 +-
 .../test.py                                   |   5 +-
 tests/integration/test_session_log/.gitignore |   1 -
 .../integration/test_session_log/__init__.py  |   0
 .../test_session_log/configs/log.xml          |   9 -
 .../test_session_log/configs/ports.xml        |   9 -
 .../test_session_log/configs/session_log.xml  |   9 -
 .../test_session_log/configs/users.xml        |  23 --
 .../protos/clickhouse_grpc.proto              |   1 -
 tests/integration/test_session_log/test.py    | 289 ------------------
 .../02833_concurrrent_sessions.reference      |  34 ---
 .../0_stateless/02833_concurrrent_sessions.sh | 138 ---------
 .../02834_remote_session_log.reference        |  13 -
 .../0_stateless/02834_remote_session_log.sh   |  56 ----
 .../02835_drop_user_during_session.reference  |   8 -
 .../02835_drop_user_during_session.sh         | 114 -------
 20 files changed, 25 insertions(+), 752 deletions(-)
 delete mode 100644 tests/integration/test_session_log/.gitignore
 delete mode 100644 tests/integration/test_session_log/__init__.py
 delete mode 100644 tests/integration/test_session_log/configs/log.xml
 delete mode 100644 tests/integration/test_session_log/configs/ports.xml
 delete mode 100644 tests/integration/test_session_log/configs/session_log.xml
 delete mode 100644 tests/integration/test_session_log/configs/users.xml
 delete mode 120000 tests/integration/test_session_log/protos/clickhouse_grpc.proto
 delete mode 100644 tests/integration/test_session_log/test.py
 delete mode 100644 tests/queries/0_stateless/02833_concurrrent_sessions.reference
 delete mode 100755 tests/queries/0_stateless/02833_concurrrent_sessions.sh
 delete mode 100644 tests/queries/0_stateless/02834_remote_session_log.reference
 delete mode 100755 tests/queries/0_stateless/02834_remote_session_log.sh
 delete mode 100644 tests/queries/0_stateless/02835_drop_user_during_session.reference
 delete mode 100755 tests/queries/0_stateless/02835_drop_user_during_session.sh

diff --git a/src/Interpreters/Session.cpp b/src/Interpreters/Session.cpp
index f8bd70afdb6..de2a779b740 100644
--- a/src/Interpreters/Session.cpp
+++ b/src/Interpreters/Session.cpp
@@ -520,8 +520,6 @@ ContextMutablePtr Session::makeSessionContext()
         {},
         session_context->getSettingsRef().max_sessions_for_user);
 
-    recordLoginSucess(session_context);
-
     return session_context;
 }
 
@@ -584,8 +582,6 @@ ContextMutablePtr Session::makeSessionContext(const String & session_name_, std:
         { session_name_ },
         max_sessions_for_user);
 
-    recordLoginSucess(session_context);
-
     return session_context;
 }
 
@@ -659,35 +655,21 @@ ContextMutablePtr Session::makeQueryContextImpl(const ClientInfo * client_info_t
     if (user_id)
         user = query_context->getUser();
 
-    /// Interserver does not create session context
-    recordLoginSucess(query_context);
-
-    return query_context;
-}
-
-
-void Session::recordLoginSucess(ContextPtr login_context) const
-{
-    if (notified_session_log_about_login)
-        return;
-
-    if (!login_context)
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "Session or query context must be created");
-
-    if (auto session_log = getSessionLog())
+    if (!notified_session_log_about_login)
     {
-        const auto & settings   = login_context->getSettingsRef();
-        const auto access       = login_context->getAccess();
+        if (auto session_log = getSessionLog())
+        {
+            session_log->addLoginSuccess(
+                    auth_id,
+                    named_session ? std::optional<std::string>(named_session->key.second) : std::nullopt,
+                    *query_context,
+                    user);
 
-        session_log->addLoginSuccess(auth_id,
-                                     named_session ? named_session->key.second : "",
-                                     settings,
-                                     access,
-                                     getClientInfo(),
-                                     user);
+            notified_session_log_about_login = true;
+        }
     }
 
-    notified_session_log_about_login = true;
+    return query_context;
 }
 
 
diff --git a/src/Interpreters/Session.h b/src/Interpreters/Session.h
index 81ef987b428..51c0e3c71fa 100644
--- a/src/Interpreters/Session.h
+++ b/src/Interpreters/Session.h
@@ -97,8 +97,6 @@ public:
 private:
     std::shared_ptr<SessionLog> getSessionLog() const;
     ContextMutablePtr makeQueryContextImpl(const ClientInfo * client_info_to_copy, ClientInfo * client_info_to_move) const;
-    void recordLoginSucess(ContextPtr login_context) const;
-
 
     mutable bool notified_session_log_about_login = false;
     const UUID auth_id;
diff --git a/src/Interpreters/SessionLog.cpp b/src/Interpreters/SessionLog.cpp
index b847eaf9824..0a8a7fc18c5 100644
--- a/src/Interpreters/SessionLog.cpp
+++ b/src/Interpreters/SessionLog.cpp
@@ -199,13 +199,12 @@ void SessionLogElement::appendToBlock(MutableColumns & columns) const
     columns[i++]->insertData(auth_failure_reason.data(), auth_failure_reason.length());
 }
 
-void SessionLog::addLoginSuccess(const UUID & auth_id,
-                                 const String & session_id,
-                                 const Settings & settings,
-                                 const ContextAccessPtr & access,
-                                 const ClientInfo & client_info,
-                                 const UserPtr & login_user)
+void SessionLog::addLoginSuccess(const UUID & auth_id, std::optional<String> session_id, const Context & login_context, const UserPtr & login_user)
 {
+    const auto access = login_context.getAccess();
+    const auto & settings = login_context.getSettingsRef();
+    const auto & client_info = login_context.getClientInfo();
+
     DB::SessionLogElement log_entry(auth_id, SESSION_LOGIN_SUCCESS);
     log_entry.client_info = client_info;
 
@@ -216,7 +215,8 @@ void SessionLog::addLoginSuccess(const UUID & auth_id,
     }
     log_entry.external_auth_server = login_user ? login_user->auth_data.getLDAPServerName() : "";
 
-    log_entry.session_id = session_id;
+    if (session_id)
+        log_entry.session_id = *session_id;
 
     if (const auto roles_info = access->getRolesInfo())
         log_entry.roles = roles_info->getCurrentRolesNames();
diff --git a/src/Interpreters/SessionLog.h b/src/Interpreters/SessionLog.h
index 8757bc12270..1282ac09c4d 100644
--- a/src/Interpreters/SessionLog.h
+++ b/src/Interpreters/SessionLog.h
@@ -20,7 +20,6 @@ enum SessionLogElementType : int8_t
 class ContextAccess;
 struct User;
 using UserPtr = std::shared_ptr<const User>;
-using ContextAccessPtr = std::shared_ptr<const ContextAccess>;
 
 /** A struct which will be inserted as row into session_log table.
   *
@@ -73,13 +72,7 @@ class SessionLog : public SystemLog<SessionLogElement>
     using SystemLog<SessionLogElement>::SystemLog;
 
 public:
-    void addLoginSuccess(const UUID & auth_id,
-                         const String & session_id,
-                         const Settings & settings,
-                         const ContextAccessPtr & access,
-                         const ClientInfo & client_info,
-                         const UserPtr & login_user);
-
+    void addLoginSuccess(const UUID & auth_id, std::optional<String> session_id, const Context & login_context, const UserPtr & login_user);
     void addLoginFailure(const UUID & auth_id, const ClientInfo & info, const std::optional<String> & user, const Exception & reason);
     void addLogOut(const UUID & auth_id, const UserPtr & login_user, const ClientInfo & client_info);
 };
diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp
index 10d5e7a0242..ff5690a3b07 100644
--- a/src/Server/HTTPHandler.cpp
+++ b/src/Server/HTTPHandler.cpp
@@ -561,7 +561,8 @@ void HTTPHandler::processQuery(
         session->makeSessionContext();
     }
 
-    auto context = session->makeQueryContext();
+    auto client_info = session->getClientInfo();
+    auto context = session->makeQueryContext(std::move(client_info));
 
     /// This parameter is used to tune the behavior of output formats (such as Native) for compatibility.
     if (params.has("client_protocol_version"))
diff --git a/tests/integration/test_profile_max_sessions_for_user/test.py b/tests/integration/test_profile_max_sessions_for_user/test.py
index 65587933fed..2930262f63e 100755
--- a/tests/integration/test_profile_max_sessions_for_user/test.py
+++ b/tests/integration/test_profile_max_sessions_for_user/test.py
@@ -27,7 +27,10 @@ proto_dir = os.path.join(SCRIPT_DIR, "./protos")
 gen_dir = os.path.join(SCRIPT_DIR, "./_gen")
 os.makedirs(gen_dir, exist_ok=True)
 run_and_check(
-    f"python3 -m grpc_tools.protoc -I{proto_dir} --python_out={gen_dir} --grpc_python_out={gen_dir} {proto_dir}/clickhouse_grpc.proto",
+    "python3 -m grpc_tools.protoc -I{proto_dir} --python_out={gen_dir} --grpc_python_out={gen_dir} \
+    {proto_dir}/clickhouse_grpc.proto".format(
+        proto_dir=proto_dir, gen_dir=gen_dir
+    ),
     shell=True,
 )
 
diff --git a/tests/integration/test_session_log/.gitignore b/tests/integration/test_session_log/.gitignore
deleted file mode 100644
index edf565ec632..00000000000
--- a/tests/integration/test_session_log/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-_gen
diff --git a/tests/integration/test_session_log/__init__.py b/tests/integration/test_session_log/__init__.py
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/integration/test_session_log/configs/log.xml b/tests/integration/test_session_log/configs/log.xml
deleted file mode 100644
index 7a079b81e69..00000000000
--- a/tests/integration/test_session_log/configs/log.xml
+++ /dev/null
@@ -1,9 +0,0 @@
- <clickhouse>
-     <logger>
-         <level>trace</level>
-         <log>/var/log/clickhouse-server/clickhouse-server.log</log>
-         <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
-         <size>1000M</size>
-         <count>10</count>
-     </logger>
- </clickhouse>
\ No newline at end of file
diff --git a/tests/integration/test_session_log/configs/ports.xml b/tests/integration/test_session_log/configs/ports.xml
deleted file mode 100644
index fbaefc16b3a..00000000000
--- a/tests/integration/test_session_log/configs/ports.xml
+++ /dev/null
@@ -1,9 +0,0 @@
- <clickhouse>
-    <postgresql_port>5433</postgresql_port>
-    <mysql_port>9001</mysql_port>
-    <grpc_port>9100</grpc_port>
-    <grpc replace="replace">
-        <!-- Enable if you want very detailed logs -->
-        <verbose_logs>false</verbose_logs>
-    </grpc>
- </clickhouse>
\ No newline at end of file
diff --git a/tests/integration/test_session_log/configs/session_log.xml b/tests/integration/test_session_log/configs/session_log.xml
deleted file mode 100644
index a0e4e3e2216..00000000000
--- a/tests/integration/test_session_log/configs/session_log.xml
+++ /dev/null
@@ -1,9 +0,0 @@
-<clickhouse>
-    <session_log>
-        <database>system</database>
-        <table>session_log</table>
-
-        <partition_by>toYYYYMM(event_date)</partition_by>
-        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
-    </session_log>
-</clickhouse>
diff --git a/tests/integration/test_session_log/configs/users.xml b/tests/integration/test_session_log/configs/users.xml
deleted file mode 100644
index 0416dfadc8a..00000000000
--- a/tests/integration/test_session_log/configs/users.xml
+++ /dev/null
@@ -1,23 +0,0 @@
-<clickhouse>
-    <profiles>
-        <default>
-            <function_sleep_max_microseconds_per_block>0</function_sleep_max_microseconds_per_block>
-        </default>
-    </profiles>
-    <users>
-        <default>
-        </default>
-        <mysql_user>
-            <password>pass</password>
-        </mysql_user>
-        <postgres_user>
-            <password>pass</password>
-        </postgres_user>
-        <grpc_user>
-            <password>pass</password>
-        </grpc_user>
-        <parallel_user>
-            <password>pass</password>
-        </parallel_user>
-    </users>
-</clickhouse>
\ No newline at end of file
diff --git a/tests/integration/test_session_log/protos/clickhouse_grpc.proto b/tests/integration/test_session_log/protos/clickhouse_grpc.proto
deleted file mode 120000
index 25d15f11e3b..00000000000
--- a/tests/integration/test_session_log/protos/clickhouse_grpc.proto
+++ /dev/null
@@ -1 +0,0 @@
-../../../../src/Server/grpc_protos/clickhouse_grpc.proto
\ No newline at end of file
diff --git a/tests/integration/test_session_log/test.py b/tests/integration/test_session_log/test.py
deleted file mode 100644
index dbb39993ce3..00000000000
--- a/tests/integration/test_session_log/test.py
+++ /dev/null
@@ -1,289 +0,0 @@
-import os
-
-import grpc
-import pymysql.connections
-import psycopg2 as py_psql
-import pytest
-import random
-import sys
-import threading
-
-from helpers.cluster import ClickHouseCluster, run_and_check
-
-POSTGRES_SERVER_PORT = 5433
-MYSQL_SERVER_PORT = 9001
-GRPC_PORT = 9100
-SESSION_LOG_MATCHING_FIELDS = "auth_id, auth_type, client_version_major, client_version_minor, client_version_patch, interface"
-
-SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
-DEFAULT_ENCODING = "utf-8"
-
-# Use grpcio-tools to generate *pb2.py files from *.proto.
-proto_dir = os.path.join(SCRIPT_DIR, "./protos")
-gen_dir = os.path.join(SCRIPT_DIR, "./_gen")
-os.makedirs(gen_dir, exist_ok=True)
-run_and_check(
-    f"python3 -m grpc_tools.protoc -I{proto_dir} --python_out={gen_dir} --grpc_python_out={gen_dir} {proto_dir}/clickhouse_grpc.proto",
-    shell=True,
-)
-
-sys.path.append(gen_dir)
-
-import clickhouse_grpc_pb2
-import clickhouse_grpc_pb2_grpc
-
-cluster = ClickHouseCluster(__file__)
-instance = cluster.add_instance(
-    "node",
-    main_configs=[
-        "configs/ports.xml",
-        "configs/log.xml",
-        "configs/session_log.xml",
-    ],
-    user_configs=["configs/users.xml"],
-    # Bug in TSAN reproduces in this test https://github.com/grpc/grpc/issues/29550#issuecomment-1188085387
-    env_variables={
-        "TSAN_OPTIONS": "report_atomic_races=0 " + os.getenv("TSAN_OPTIONS", default="")
-    },
-)
-
-
-def grpc_get_url():
-    return f"{instance.ip_address}:{GRPC_PORT}"
-
-
-def grpc_create_insecure_channel():
-    channel = grpc.insecure_channel(grpc_get_url())
-    grpc.channel_ready_future(channel).result(timeout=2)
-    return channel
-
-
-session_id_counter = 0
-
-
-def next_session_id():
-    global session_id_counter
-    session_id = session_id_counter
-    session_id_counter += 1
-    return str(session_id)
-
-
-def grpc_query(query, user_, pass_, raise_exception):
-    try:
-        query_info = clickhouse_grpc_pb2.QueryInfo(
-            query=query,
-            session_id=next_session_id(),
-            user_name=user_,
-            password=pass_,
-        )
-        channel = grpc_create_insecure_channel()
-        stub = clickhouse_grpc_pb2_grpc.ClickHouseStub(channel)
-        result = stub.ExecuteQuery(query_info)
-        if result and result.HasField("exception"):
-            raise Exception(result.exception.display_text)
-
-        return result.output.decode(DEFAULT_ENCODING)
-    except Exception:
-        assert raise_exception
-
-
-def postgres_query(query, user_, pass_, raise_exception):
-    try:
-        client = py_psql.connect(
-            host=instance.ip_address,
-            port=POSTGRES_SERVER_PORT,
-            user=user_,
-            password=pass_,
-            database="default",
-        )
-        cursor = client.cursor()
-        cursor.execute(query)
-        cursor.fetchall()
-    except Exception:
-        assert raise_exception
-
-
-def mysql_query(query, user_, pass_, raise_exception):
-    try:
-        client = pymysql.connections.Connection(
-            host=instance.ip_address,
-            user=user_,
-            password=pass_,
-            database="default",
-            port=MYSQL_SERVER_PORT,
-        )
-        cursor = client.cursor(pymysql.cursors.DictCursor)
-        if raise_exception:
-            with pytest.raises(Exception):
-                cursor.execute(query)
-        else:
-            cursor.execute(query)
-        cursor.fetchall()
-    except Exception:
-        assert raise_exception
-
-
-@pytest.fixture(scope="module")
-def started_cluster():
-    try:
-        cluster.start()
-        yield cluster
-    finally:
-        cluster.shutdown()
-
-
-def test_grpc_session(started_cluster):
-    grpc_query("SELECT 1", "grpc_user", "pass", False)
-    grpc_query("SELECT 2", "grpc_user", "wrong_pass", True)
-    grpc_query("SELECT 3", "wrong_grpc_user", "pass", True)
-
-    instance.query("SYSTEM FLUSH LOGS")
-    login_success_records = instance.query(
-        "SELECT user, client_port <> 0,  client_address <> toIPv6('::') FROM system.session_log WHERE user='grpc_user' AND type = 'LoginSuccess'"
-    )
-    assert login_success_records == "grpc_user\t1\t1\n"
-    logout_records = instance.query(
-        "SELECT user, client_port <> 0,  client_address <> toIPv6('::') FROM system.session_log WHERE user='grpc_user' AND type = 'Logout'"
-    )
-    assert logout_records == "grpc_user\t1\t1\n"
-    login_failure_records = instance.query(
-        "SELECT user, client_port <> 0,  client_address <> toIPv6('::') FROM system.session_log WHERE user='grpc_user' AND type = 'LoginFailure'"
-    )
-    assert login_failure_records == "grpc_user\t1\t1\n"
-    logins_and_logouts = instance.query(
-        f"SELECT COUNT(*) FROM (SELECT {SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = 'grpc_user' AND type = 'LoginSuccess' INTERSECT SELECT {SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = 'grpc_user' AND type = 'Logout')"
-    )
-    assert logins_and_logouts == "1\n"
-
-
-def test_mysql_session(started_cluster):
-    mysql_query("SELECT 1", "mysql_user", "pass", False)
-    mysql_query("SELECT 2", "mysql_user", "wrong_pass", True)
-    mysql_query("SELECT 3", "wrong_mysql_user", "pass", True)
-
-    instance.query("SYSTEM FLUSH LOGS")
-    login_success_records = instance.query(
-        "SELECT user, client_port <> 0,  client_address <> toIPv6('::') FROM system.session_log WHERE user='mysql_user' AND type = 'LoginSuccess'"
-    )
-    assert login_success_records == "mysql_user\t1\t1\n"
-    logout_records = instance.query(
-        "SELECT user, client_port <> 0,  client_address <> toIPv6('::') FROM system.session_log WHERE user='mysql_user' AND type = 'Logout'"
-    )
-    assert logout_records == "mysql_user\t1\t1\n"
-    login_failure_records = instance.query(
-        "SELECT user, client_port <> 0,  client_address <> toIPv6('::') FROM system.session_log WHERE user='mysql_user' AND type = 'LoginFailure'"
-    )
-    assert login_failure_records == "mysql_user\t1\t1\n"
-    logins_and_logouts = instance.query(
-        f"SELECT COUNT(*) FROM (SELECT {SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = 'mysql_user' AND type = 'LoginSuccess' INTERSECT SELECT {SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = 'mysql_user' AND type = 'Logout')"
-    )
-    assert logins_and_logouts == "1\n"
-
-
-def test_postgres_session(started_cluster):
-    postgres_query("SELECT 1", "postgres_user", "pass", False)
-    postgres_query("SELECT 2", "postgres_user", "wrong_pass", True)
-    postgres_query("SELECT 3", "wrong_postgres_user", "pass", True)
-
-    instance.query("SYSTEM FLUSH LOGS")
-    login_success_records = instance.query(
-        "SELECT user, client_port <> 0,  client_address <> toIPv6('::') FROM system.session_log WHERE user='postgres_user' AND type = 'LoginSuccess'"
-    )
-    assert login_success_records == "postgres_user\t1\t1\n"
-    logout_records = instance.query(
-        "SELECT user, client_port <> 0,  client_address <> toIPv6('::') FROM system.session_log WHERE user='postgres_user' AND type = 'Logout'"
-    )
-    assert logout_records == "postgres_user\t1\t1\n"
-    login_failure_records = instance.query(
-        "SELECT user, client_port <> 0,  client_address <> toIPv6('::') FROM system.session_log WHERE user='postgres_user' AND type = 'LoginFailure'"
-    )
-    assert login_failure_records == "postgres_user\t1\t1\n"
-    logins_and_logouts = instance.query(
-        f"SELECT COUNT(*) FROM (SELECT {SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = 'postgres_user' AND type = 'LoginSuccess' INTERSECT SELECT {SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = 'postgres_user' AND type = 'Logout')"
-    )
-    assert logins_and_logouts == "1\n"
-
-
-def test_parallel_sessions(started_cluster):
-    thread_list = []
-    for _ in range(10):
-        # Sleep time does not significantly matter here,
-        # test should pass even without sleeping.
-        for function in [postgres_query, grpc_query, mysql_query]:
-            thread = threading.Thread(
-                target=function,
-                args=(
-                    f"SELECT sleep({random.uniform(0.03, 0.04)})",
-                    "parallel_user",
-                    "pass",
-                    False,
-                ),
-            )
-            thread.start()
-            thread_list.append(thread)
-            thread = threading.Thread(
-                target=function,
-                args=(
-                    f"SELECT sleep({random.uniform(0.03, 0.04)})",
-                    "parallel_user",
-                    "wrong_pass",
-                    True,
-                ),
-            )
-            thread.start()
-            thread_list.append(thread)
-            thread = threading.Thread(
-                target=function,
-                args=(
-                    f"SELECT sleep({random.uniform(0.03, 0.04)})",
-                    "wrong_parallel_user",
-                    "pass",
-                    True,
-                ),
-            )
-            thread.start()
-            thread_list.append(thread)
-
-    for thread in thread_list:
-        thread.join()
-
-    instance.query("SYSTEM FLUSH LOGS")
-    port_0_sessions = instance.query(
-        f"SELECT COUNT(*) FROM system.session_log WHERE user = 'parallel_user'"
-    )
-    assert port_0_sessions == "90\n"
-
-    port_0_sessions = instance.query(
-        f"SELECT COUNT(*) FROM system.session_log WHERE user = 'parallel_user' AND client_port = 0"
-    )
-    assert port_0_sessions == "0\n"
-
-    address_0_sessions = instance.query(
-        f"SELECT COUNT(*) FROM system.session_log WHERE user = 'parallel_user' AND client_address = toIPv6('::')"
-    )
-    assert address_0_sessions == "0\n"
-
-    grpc_sessions = instance.query(
-        f"SELECT COUNT(*) FROM system.session_log WHERE user = 'parallel_user' AND interface = 'gRPC'"
-    )
-    assert grpc_sessions == "30\n"
-
-    mysql_sessions = instance.query(
-        f"SELECT COUNT(*) FROM system.session_log WHERE user = 'parallel_user' AND interface = 'MySQL'"
-    )
-    assert mysql_sessions == "30\n"
-
-    postgres_sessions = instance.query(
-        f"SELECT COUNT(*) FROM system.session_log WHERE user = 'parallel_user' AND interface = 'PostgreSQL'"
-    )
-    assert postgres_sessions == "30\n"
-
-    logins_and_logouts = instance.query(
-        f"SELECT COUNT(*) FROM (SELECT {SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = 'parallel_user' AND type = 'LoginSuccess' INTERSECT SELECT {SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = 'parallel_user' AND type = 'Logout')"
-    )
-    assert logins_and_logouts == "30\n"
-
-    logout_failure_sessions = instance.query(
-        f"SELECT COUNT(*) FROM system.session_log  WHERE user = 'parallel_user' AND type = 'LoginFailure'"
-    )
-    assert logout_failure_sessions == "30\n"
diff --git a/tests/queries/0_stateless/02833_concurrrent_sessions.reference b/tests/queries/0_stateless/02833_concurrrent_sessions.reference
deleted file mode 100644
index bfe507e8eac..00000000000
--- a/tests/queries/0_stateless/02833_concurrrent_sessions.reference
+++ /dev/null
@@ -1,34 +0,0 @@
-sessions:
-150
-port_0_sessions:
-0
-address_0_sessions:
-0
-tcp_sessions
-60
-http_sessions
-30
-http_with_session_id_sessions
-30
-my_sql_sessions
-30
-Corresponding LoginSuccess/Logout
-10
-LoginFailure
-10
-Corresponding LoginSuccess/Logout
-10
-LoginFailure
-10
-Corresponding LoginSuccess/Logout
-10
-LoginFailure
-10
-Corresponding LoginSuccess/Logout
-10
-LoginFailure
-10
-Corresponding LoginSuccess/Logout
-10
-LoginFailure
-10
diff --git a/tests/queries/0_stateless/02833_concurrrent_sessions.sh b/tests/queries/0_stateless/02833_concurrrent_sessions.sh
deleted file mode 100755
index 26b48462a76..00000000000
--- a/tests/queries/0_stateless/02833_concurrrent_sessions.sh
+++ /dev/null
@@ -1,138 +0,0 @@
-#!/usr/bin/env bash
-# Tags: no-fasttest, no-debug
-
-CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# shellcheck source=../shell_config.sh
-. "$CURDIR"/../shell_config.sh
-
-readonly PID=$$
-
-# Each user uses a separate thread.
-readonly TCP_USERS=( "02833_TCP_USER_${PID}"_{1,2} ) # 2 concurrent TCP users
-readonly HTTP_USERS=( "02833_HTTP_USER_${PID}" )
-readonly HTTP_WITH_SESSION_ID_SESSION_USERS=( "02833_HTTP_WITH_SESSION_ID_USER_${PID}" )
-readonly MYSQL_USERS=( "02833_MYSQL_USER_${PID}")
-readonly ALL_USERS=( "${TCP_USERS[@]}" "${HTTP_USERS[@]}" "${HTTP_WITH_SESSION_ID_SESSION_USERS[@]}" "${MYSQL_USERS[@]}" )
-
-readonly TCP_USERS_SQL_COLLECTION_STRING="$( echo "${TCP_USERS[*]}" | sed "s/[^[:space:]]\+/'&'/g" | sed 's/[[:space:]]/,/g' )"
-readonly HTTP_USERS_SQL_COLLECTION_STRING="$( echo "${HTTP_USERS[*]}" | sed "s/[^[:space:]]\+/'&'/g" | sed 's/[[:space:]]/,/g' )"
-readonly HTTP_WITH_SESSION_ID_USERS_SQL_COLLECTION_STRING="$( echo "${HTTP_WITH_SESSION_ID_SESSION_USERS[*]}" | sed "s/[^[:space:]]\+/'&'/g" | sed 's/[[:space:]]/,/g' )"
-readonly MYSQL_USERS_SQL_COLLECTION_STRING="$( echo "${MYSQL_USERS[*]}" | sed "s/[^[:space:]]\+/'&'/g" | sed 's/[[:space:]]/,/g' )"
-readonly ALL_USERS_SQL_COLLECTION_STRING="$( echo "${ALL_USERS[*]}" | sed "s/[^[:space:]]\+/'&'/g" | sed 's/[[:space:]]/,/g' )"
-
-readonly SESSION_LOG_MATCHING_FIELDS="auth_id, auth_type, client_version_major, client_version_minor, client_version_patch, interface"
-
-for user in "${ALL_USERS[@]}"; do
-    ${CLICKHOUSE_CLIENT} -q "CREATE USER IF NOT EXISTS ${user} IDENTIFIED WITH plaintext_password BY 'pass'"
-    ${CLICKHOUSE_CLIENT} -q "GRANT SELECT ON system.* TO ${user}"
-    ${CLICKHOUSE_CLIENT} -q "GRANT SELECT ON INFORMATION_SCHEMA.* TO ${user}";
-done
-
-# All <type>_session functions execute in separate threads.
-# These functions try to create a session with successful login and logout.
-# Sleep a small, random amount of time to make concurrency more intense.
-# and try to login with an invalid password.
-function tcp_session() 
-{
-    local user=$1
-    local i=0
-    while (( (i++) < 10 )); do
-        # login logout
-        ${CLICKHOUSE_CLIENT} -q "SELECT 1, sleep(0.01${RANDOM})" --user="${user}" --password="pass"
-        # login failure
-        ${CLICKHOUSE_CLIENT} -q "SELECT 2" --user="${user}" --password 'invalid'
-    done
-}
-
-function http_session() 
-{
-    local user=$1
-    local i=0
-    while (( (i++) < 10 )); do
-        # login logout
-        ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&user=${user}&password=pass" -d "SELECT 3, sleep(0.01${RANDOM})"
-
-        # login failure
-        ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&user=${user}&password=wrong" -d "SELECT 4"
-    done
-}
-
-function http_with_session_id_session() 
-{
-    local user=$1
-    local i=0
-    while (( (i++) < 10 )); do
-        # login logout
-        ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=${user}&user=${user}&password=pass" -d "SELECT 5, sleep 0.01${RANDOM}"
-
-        # login failure
-        ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=${user}&user=${user}&password=wrong" -d "SELECT 6"
-    done
-}
-
-function mysql_session() 
-{
-    local user=$1
-    local i=0
-    while (( (i++) < 10 )); do
-        # login logout
-        ${CLICKHOUSE_CLIENT} -q "SELECT 1, sleep(0.01${RANDOM}) FROM mysql('127.0.0.1:9004', 'system', 'one', '${user}', 'pass')"
-
-        # login failure
-        ${CLICKHOUSE_CLIENT} -q "SELECT 1 FROM mysql('127.0.0.1:9004', 'system', 'one', '${user}', 'wrong', SETTINGS connection_max_tries=1)"
-    done
-}
-
-${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS"
-${CLICKHOUSE_CLIENT} -q "DELETE FROM system.session_log WHERE user IN (${ALL_USERS_SQL_COLLECTION_STRING})"
-
-export -f tcp_session;
-export -f http_session;
-export -f http_with_session_id_session;
-export -f mysql_session;
-
-for user in "${TCP_USERS[@]}"; do
-    timeout 60s bash -c "tcp_session ${user}" >/dev/null 2>&1 &
-done
-
-for user in "${HTTP_USERS[@]}"; do
-    timeout 60s bash -c "http_session ${user}" >/dev/null 2>&1 &
-done
-
-for user in "${HTTP_WITH_SESSION_ID_SESSION_USERS[@]}"; do
-    timeout 60s bash -c "http_with_session_id_session ${user}" >/dev/null 2>&1 &
-done
-
-for user in "${MYSQL_USERS[@]}"; do
-    timeout 60s bash -c "mysql_session ${user}" >/dev/null 2>&1 &
-done
-
-wait
-
-${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS"
-
-echo "sessions:" 
-${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user IN (${ALL_USERS_SQL_COLLECTION_STRING})"
-
-echo "port_0_sessions:" 
-${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user IN (${ALL_USERS_SQL_COLLECTION_STRING}) AND client_port = 0"
-
-echo "address_0_sessions:"
-${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user IN (${ALL_USERS_SQL_COLLECTION_STRING}) AND client_address = toIPv6('::')"
-
-echo "tcp_sessions"
-${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user IN (${TCP_USERS_SQL_COLLECTION_STRING}) AND interface = 'TCP'"
-echo "http_sessions"
-${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user IN (${HTTP_USERS_SQL_COLLECTION_STRING}) AND interface = 'HTTP'"
-echo "http_with_session_id_sessions"
-${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user IN (${HTTP_WITH_SESSION_ID_USERS_SQL_COLLECTION_STRING}) AND interface = 'HTTP'"
-echo "my_sql_sessions"
-${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user IN (${MYSQL_USERS_SQL_COLLECTION_STRING}) AND interface = 'MySQL'"
-
-for user in "${ALL_USERS[@]}"; do
-    ${CLICKHOUSE_CLIENT} -q "DROP USER ${user}"
-    echo "Corresponding LoginSuccess/Logout" 
-    ${CLICKHOUSE_CLIENT} -q "SELECT COUNT(*) FROM (SELECT ${SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = '${user}' AND type = 'LoginSuccess' INTERSECT SELECT ${SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = '${user}' AND type = 'Logout')"
-    echo "LoginFailure"
-    ${CLICKHOUSE_CLIENT} -q "SELECT COUNT(*) FROM system.session_log WHERE user = '${user}' AND type = 'LoginFailure'" 
- done
diff --git a/tests/queries/0_stateless/02834_remote_session_log.reference b/tests/queries/0_stateless/02834_remote_session_log.reference
deleted file mode 100644
index e2680982ab0..00000000000
--- a/tests/queries/0_stateless/02834_remote_session_log.reference
+++ /dev/null
@@ -1,13 +0,0 @@
-0
-0
-0
-0
-client_port 0 connections:
-0
-client_address '::' connections:
-0
-login failures:
-0
-TCP Login and logout count is equal
-HTTP Login and logout count is equal
-MySQL Login and logout count is equal
diff --git a/tests/queries/0_stateless/02834_remote_session_log.sh b/tests/queries/0_stateless/02834_remote_session_log.sh
deleted file mode 100755
index 3bedfb6c9ee..00000000000
--- a/tests/queries/0_stateless/02834_remote_session_log.sh
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/usr/bin/env bash
-# Tags: no-fasttest
-
-CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# shellcheck source=../shell_config.sh
-. "$CURDIR"/../shell_config.sh
-
-readonly PID=$$
-readonly TEST_USER=$"02834_USER_${PID}"
-readonly SESSION_LOG_MATCHING_FIELDS="auth_id, auth_type, client_version_major, client_version_minor, client_version_patch, interface"
-
-${CLICKHOUSE_CLIENT} -q "CREATE USER IF NOT EXISTS ${TEST_USER} IDENTIFIED WITH plaintext_password BY 'pass'"
-${CLICKHOUSE_CLIENT} -q "GRANT SELECT ON INFORMATION_SCHEMA.* TO ${TEST_USER}"
-${CLICKHOUSE_CLIENT} -q "GRANT SELECT ON system.* TO ${TEST_USER}"
-${CLICKHOUSE_CLIENT} -q "GRANT CREATE TEMPORARY TABLE, MYSQL, REMOTE ON *.* TO ${TEST_USER}"
-
-${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS"
-${CLICKHOUSE_CLIENT} -q "DELETE FROM system.session_log WHERE user = '${TEST_USER}'"
-
-${CLICKHOUSE_CURL} -sS -X POST "${CLICKHOUSE_URL}&user=${TEST_USER}&password=pass" \
-   -d "SELECT * FROM remote('127.0.0.1:${CLICKHOUSE_PORT_TCP}', 'system', 'one', '${TEST_USER}', 'pass')"
-
-${CLICKHOUSE_CURL} -sS -X POST "${CLICKHOUSE_URL}&user=${TEST_USER}&password=pass" \
-    -d "SELECT * FROM mysql('127.0.0.1:9004', 'system', 'one', '${TEST_USER}', 'pass')"
-
-${CLICKHOUSE_CLIENT} -q "SELECT * FROM remote('127.0.0.1:${CLICKHOUSE_PORT_TCP}', 'system', 'one', '${TEST_USER}', 'pass')" -u "${TEST_USER}" --password "pass"
-${CLICKHOUSE_CLIENT} -q "SELECT * FROM mysql('127.0.0.1:9004', 'system', 'one', '${TEST_USER}', 'pass')" -u "${TEST_USER}" --password "pass"
-
-${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS"
-
-echo "client_port 0 connections:"
-${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user = '${TEST_USER}' and client_port = 0"
-
-echo "client_address '::' connections:"
-${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user = '${TEST_USER}' and client_address = toIPv6('::')"
-
-echo "login failures:"
-${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user = '${TEST_USER}' and type = 'LoginFailure'"
-
-# remote(...) function sometimes reuses old cached sessions for query execution.
-# This makes LoginSuccess/Logout entries count unstable, but success and logouts must always match.
-
-for interface in 'TCP' 'HTTP' 'MySQL'
-do
-    LOGIN_COUNT=`${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user = '${TEST_USER}' AND type = 'LoginSuccess' AND interface = '${interface}'"`
-    CORRESPONDING_LOGOUT_RECORDS_COUNT=`${CLICKHOUSE_CLIENT} -q "SELECT COUNT(*) FROM (SELECT ${SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = '${TEST_USER}' AND type = 'LoginSuccess' AND interface = '${interface}' INTERSECT SELECT ${SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = '${TEST_USER}' AND type = 'Logout' AND interface = '${interface}')"`
-
-    if [ "$LOGIN_COUNT" == "$CORRESPONDING_LOGOUT_RECORDS_COUNT" ]; then
-        echo "${interface} Login and logout count is equal"
-    else
-        TOTAL_LOGOUT_COUNT=`${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user = '${TEST_USER}' AND type = 'Logout' AND interface = '${interface}'"`
-        echo "${interface} Login count ${LOGIN_COUNT} != corresponding logout count ${CORRESPONDING_LOGOUT_RECORDS_COUNT}. TOTAL_LOGOUT_COUNT ${TOTAL_LOGOUT_COUNT}"
-    fi
-done
-
-${CLICKHOUSE_CLIENT} -q "DROP USER ${TEST_USER}"
diff --git a/tests/queries/0_stateless/02835_drop_user_during_session.reference b/tests/queries/0_stateless/02835_drop_user_during_session.reference
deleted file mode 100644
index 7252faab8c6..00000000000
--- a/tests/queries/0_stateless/02835_drop_user_during_session.reference
+++ /dev/null
@@ -1,8 +0,0 @@
-port_0_sessions:
-0
-address_0_sessions:
-0
-Corresponding LoginSuccess/Logout
-9
-LoginFailure
-0
diff --git a/tests/queries/0_stateless/02835_drop_user_during_session.sh b/tests/queries/0_stateless/02835_drop_user_during_session.sh
deleted file mode 100755
index 347ebd22f96..00000000000
--- a/tests/queries/0_stateless/02835_drop_user_during_session.sh
+++ /dev/null
@@ -1,114 +0,0 @@
-#!/usr/bin/env bash
-# Tags: no-debug
-
-CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# shellcheck source=../shell_config.sh
-. "$CURDIR"/../shell_config.sh
-
-readonly PID=$$
-
-readonly TEST_USER="02835_USER_${PID}"
-readonly TEST_ROLE="02835_ROLE_${PID}"
-readonly TEST_PROFILE="02835_PROFILE_${PID}"
-readonly SESSION_LOG_MATCHING_FIELDS="auth_id, auth_type, client_version_major, client_version_minor, client_version_patch, interface"
-
-function tcp_session()
-{
-    local user=$1
-    ${CLICKHOUSE_CLIENT} -q "SELECT COUNT(*) FROM system.numbers" --user="${user}"
-}
-
-function http_session()
-{
-    local user=$1
-    ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&user=${user}&password=pass" -d "SELECT COUNT(*) FROM system.numbers"
-}
-
-function http_with_session_id_session() 
-{
-    local user=$1
-    ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&user=${user}&password=pass" -d "SELECT COUNT(*) FROM system.numbers"
-}
-
-# Busy-waits until user $1, specified amount of queries ($2) will run simultaneously.
-function wait_for_queries_start()
-{
-    local user=$1
-    local queries_count=$2
-    # 10 seconds waiting
-    counter=0 retries=100
-    while [[ $counter -lt $retries ]]; do
-        result=$($CLICKHOUSE_CLIENT --query "SELECT COUNT(*) FROM system.processes WHERE user = '${user}'")
-        if [[ $result == "${queries_count}" ]]; then
-            break;
-        fi
-        sleep 0.1
-        ((++counter))
-    done
-}
-
-${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS"
-${CLICKHOUSE_CLIENT} -q "DELETE FROM system.session_log WHERE user = '${TEST_USER}'"
-
-# DROP USE CASE
-${CLICKHOUSE_CLIENT} -q "CREATE USER IF NOT EXISTS ${TEST_USER}"
-${CLICKHOUSE_CLIENT} -q "GRANT SELECT ON system.numbers TO ${TEST_USER}"
-
-export -f tcp_session;
-export -f http_session;
-export -f http_with_session_id_session;
-
-timeout 10s bash -c "tcp_session ${TEST_USER}" >/dev/null 2>&1 &
-timeout 10s bash -c "http_session ${TEST_USER}" >/dev/null 2>&1 &
-timeout 10s bash -c "http_with_session_id_session ${TEST_USER}" >/dev/null 2>&1 &
-
-wait_for_queries_start $TEST_USER 3
-${CLICKHOUSE_CLIENT} -q "DROP USER ${TEST_USER}"
-${CLICKHOUSE_CLIENT} -q "KILL QUERY WHERE user = '${TEST_USER}' SYNC" >/dev/null &
-
-wait
-
-# DROP ROLE CASE
-${CLICKHOUSE_CLIENT} -q "CREATE ROLE IF NOT EXISTS ${TEST_ROLE}"
-${CLICKHOUSE_CLIENT} -q "CREATE USER ${TEST_USER} DEFAULT ROLE ${TEST_ROLE}"
-${CLICKHOUSE_CLIENT} -q "GRANT SELECT ON system.numbers TO ${TEST_USER}"
-
-timeout 10s bash -c "tcp_session ${TEST_USER}" >/dev/null 2>&1 &
-timeout 10s bash -c "http_session ${TEST_USER}" >/dev/null 2>&1 &
-timeout 10s bash -c "http_with_session_id_session ${TEST_USER}" >/dev/null 2>&1 &
-
-wait_for_queries_start $TEST_USER 3
-${CLICKHOUSE_CLIENT} -q "DROP ROLE ${TEST_ROLE}"
-${CLICKHOUSE_CLIENT} -q "DROP USER ${TEST_USER}"
-
-${CLICKHOUSE_CLIENT} -q "KILL QUERY WHERE user = '${TEST_USER}' SYNC" >/dev/null &
-
-wait
-
-# DROP PROFILE CASE
-${CLICKHOUSE_CLIENT} -q "CREATE SETTINGS PROFILE IF NOT EXISTS '${TEST_PROFILE}'"
-${CLICKHOUSE_CLIENT} -q "CREATE USER ${TEST_USER} SETTINGS PROFILE '${TEST_PROFILE}'"
-${CLICKHOUSE_CLIENT} -q "GRANT SELECT ON system.numbers TO ${TEST_USER}"
-
-timeout 10s bash -c "tcp_session ${TEST_USER}" >/dev/null 2>&1 &
-timeout 10s bash -c "http_session ${TEST_USER}" >/dev/null 2>&1 &
-timeout 10s bash -c "http_with_session_id_session ${TEST_USER}" >/dev/null 2>&1 &
-
-wait_for_queries_start $TEST_USER 3
-${CLICKHOUSE_CLIENT} -q "DROP SETTINGS PROFILE '${TEST_PROFILE}'"
-${CLICKHOUSE_CLIENT} -q "DROP USER ${TEST_USER}"
-
-${CLICKHOUSE_CLIENT} -q "KILL QUERY WHERE user = '${TEST_USER}' SYNC" >/dev/null &
-
-wait
-
-${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS"
-
-echo "port_0_sessions:" 
-${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user = '${TEST_USER}' AND client_port = 0"
-echo "address_0_sessions:"
-${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user = '${TEST_USER}' AND client_address = toIPv6('::')"
-echo "Corresponding LoginSuccess/Logout" 
-${CLICKHOUSE_CLIENT} -q "SELECT COUNT(*) FROM (SELECT ${SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = '${TEST_USER}' AND type = 'LoginSuccess' INTERSECT SELECT ${SESSION_LOG_MATCHING_FIELDS}, FROM system.session_log WHERE user = '${TEST_USER}' AND type = 'Logout')"
-echo "LoginFailure"
-${CLICKHOUSE_CLIENT} -q "SELECT COUNT(*) FROM system.session_log WHERE user = '${TEST_USER}' AND type = 'LoginFailure'" 

From b9638c6387d46e85abe55a333049efb0d2c757e7 Mon Sep 17 00:00:00 2001
From: Yakov Olkhovskiy <yakov@clickhouse.com>
Date: Thu, 10 Aug 2023 22:16:30 +0000
Subject: [PATCH 205/249] path to libc

---
 cmake/linux/toolchain-s390x.cmake | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cmake/linux/toolchain-s390x.cmake b/cmake/linux/toolchain-s390x.cmake
index 4d4d1fab473..03314a9d2ba 100644
--- a/cmake/linux/toolchain-s390x.cmake
+++ b/cmake/linux/toolchain-s390x.cmake
@@ -20,9 +20,9 @@ set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}/s390x-linux-gnu/libc")
 set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
 set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
 set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
-set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=mold")
-set (CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fuse-ld=mold")
-set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=mold")
+set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=mold -Wl,-L/usr/lib/x86_64-linux-gnu")
+set (CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fuse-ld=mold -Wl,-L/usr/lib/x86_64-linux-gnu")
+set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=mold -Wl,-L/usr/lib/x86_64-linux-gnu")
 
 set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
 set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)

From ea8ae28c7dfeae6c3641c8617152750aa0d94be2 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 11 Aug 2023 00:40:18 +0200
Subject: [PATCH 206/249] Fix build with clang-17

---
 src/Core/examples/coro.cpp | 194 -------------------------------------
 1 file changed, 194 deletions(-)
 delete mode 100644 src/Core/examples/coro.cpp

diff --git a/src/Core/examples/coro.cpp b/src/Core/examples/coro.cpp
deleted file mode 100644
index fbccc261e9d..00000000000
--- a/src/Core/examples/coro.cpp
+++ /dev/null
@@ -1,194 +0,0 @@
-#include <cassert>
-#include <iostream>
-#include <string>
-#include <optional>
-
-#include <Common/Exception.h>
-#include <Common/logger_useful.h>
-#include <Poco/ConsoleChannel.h>
-#include <Poco/Logger.h>
-#include <Poco/AutoPtr.h>
-
-#if defined(__clang__)
-#include <experimental/coroutine>
-
-namespace std // NOLINT(cert-dcl58-cpp)
-{
-    using namespace experimental::coroutines_v1; // NOLINT(cert-dcl58-cpp)
-}
-
-#if __has_warning("-Wdeprecated-experimental-coroutine")
-#pragma clang diagnostic push
-#pragma clang diagnostic ignored "-Wdeprecated-experimental-coroutine"
-#endif
-
-#else
-#include <coroutine>
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wzero-as-null-pointer-constant"
-#endif
-
-
-template <typename T>
-struct suspend_value // NOLINT(readability-identifier-naming)
-{
-    constexpr bool await_ready() const noexcept { return true; } // NOLINT(readability-identifier-naming)
-    constexpr void await_suspend(std::coroutine_handle<>) const noexcept {} // NOLINT(readability-identifier-naming)
-    constexpr T await_resume() const noexcept // NOLINT(readability-identifier-naming)
-    {
-        std::cout << "  ret " << val << std::endl;
-        return val;
-    }
-
-    T val;
-};
-
-template <typename T>
-struct Task
-{
-    struct promise_type // NOLINT(readability-identifier-naming)
-    {
-        using coro_handle = std::coroutine_handle<promise_type>;
-        auto get_return_object() { return coro_handle::from_promise(*this); } // NOLINT(readability-identifier-naming)
-        auto initial_suspend() { return std::suspend_never(); } // NOLINT(readability-identifier-naming)
-        auto final_suspend() noexcept { return suspend_value<T>{*r->value}; } // NOLINT(readability-identifier-naming)
-        //void return_void() {}
-        void return_value(T value_) { r->value = value_; } // NOLINT(readability-identifier-naming)
-        void unhandled_exception() // NOLINT(readability-identifier-naming)
-        {
-            DB::tryLogCurrentException("Logger");
-            r->exception = std::current_exception(); // NOLINT(bugprone-throw-keyword-missing)
-        }
-
-        explicit promise_type(std::string tag_) : tag(tag_) {}
-        ~promise_type() { std::cout << "~promise_type " << tag << std::endl; }
-        std::string tag;
-        coro_handle next;
-        Task * r = nullptr;
-    };
-
-    using coro_handle = std::coroutine_handle<promise_type>;
-
-    bool await_ready() const noexcept { return false; } // NOLINT(readability-identifier-naming)
-    void await_suspend(coro_handle g) noexcept // NOLINT(readability-identifier-naming)
-    {
-        std::cout << "  await_suspend " << my.promise().tag << std::endl;
-        std::cout << "  g tag " << g.promise().tag << std::endl;
-        g.promise().next = my;
-    }
-    T await_resume() noexcept // NOLINT(readability-identifier-naming)
-    {
-        std::cout << "  await_res " << my.promise().tag << std::endl;
-        return *value;
-    }
-
-    Task(coro_handle handle) : my(handle), tag(handle.promise().tag) // NOLINT(google-explicit-constructor)
-    {
-        assert(handle);
-        my.promise().r = this;
-        std::cout << "    Task " << tag << std::endl;
-    }
-    Task(Task &) = delete;
-    Task(Task &&rhs) noexcept : my(rhs.my), tag(rhs.tag)
-    {
-        rhs.my = {};
-        std::cout << "    Task&& " << tag << std::endl;
-    }
-    static bool resumeImpl(Task *r)
-    {
-        if (r->value)
-            return false;
-
-        auto & next = r->my.promise().next;
-
-        if (next)
-        {
-            if (resumeImpl(next.promise().r))
-                return true;
-            next = {};
-        }
-
-        if (!r->value)
-        {
-            r->my.resume();
-            if (r->exception)
-                std::rethrow_exception(r->exception);
-        }
-        return !r->value;
-    }
-
-    bool resume()
-    {
-        return resumeImpl(this);
-    }
-
-    T res()
-    {
-        return *value;
-    }
-
-    ~Task()
-    {
-        std::cout << "    ~Task " << tag << std::endl;
-    }
-
-private:
-    coro_handle my;
-    std::string tag;
-    std::optional<T> value;
-    std::exception_ptr exception;
-};
-
-Task<int> boo([[maybe_unused]] std::string tag)
-{
-    std::cout << "x" << std::endl;
-    co_await std::suspend_always();
-    std::cout << StackTrace().toString();
-    std::cout << "y" << std::endl;
-    co_return 1;
-}
-
-Task<int> bar([[maybe_unused]] std::string tag)
-{
-    std::cout << "a" << std::endl;
-    int res1 = co_await boo("boo1");
-    std::cout << "b " << res1 << std::endl;
-    int res2 = co_await boo("boo2");
-    if (res2 == 1)
-        throw DB::Exception(1, "hello");
-    std::cout << "c " << res2 << std::endl;
-    co_return res1 + res2;  // 1 + 1 = 2
-}
-
-Task<int> foo([[maybe_unused]] std::string tag)
-{
-    std::cout << "Hello" << std::endl;
-    auto res1 = co_await bar("bar1");
-    std::cout << "Coro " << res1 << std::endl;
-    auto res2 = co_await bar("bar2");
-    std::cout << "World " << res2 << std::endl;
-    co_return res1 * res2; // 2 * 2 = 4
-}
-
-int main()
-{
-    Poco::AutoPtr<Poco::ConsoleChannel> app_channel(new Poco::ConsoleChannel(std::cerr));
-    Poco::Logger::root().setChannel(app_channel);
-    Poco::Logger::root().setLevel("trace");
-
-    LOG_INFO(&Poco::Logger::get(""), "Starting");
-
-    try
-    {
-        auto t = foo("foo");
-        std::cout << ".. started" << std::endl;
-        while (t.resume())
-            std::cout << ".. yielded" << std::endl;
-        std::cout << ".. done: " << t.res() << std::endl;
-    }
-    catch (DB::Exception & e)
-    {
-        std::cout << "Got exception " << e.what() << std::endl;
-        std::cout << e.getStackTraceString() << std::endl;
-    }
-}

From ea62629d83ee487e64b1ba3c051211103f258069 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 11 Aug 2023 00:40:26 +0200
Subject: [PATCH 207/249] Fix linking of examples

---
 src/Core/examples/CMakeLists.txt    | 3 ---
 src/Parsers/examples/CMakeLists.txt | 4 ++--
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/src/Core/examples/CMakeLists.txt b/src/Core/examples/CMakeLists.txt
index 868173e0e31..2326eada96d 100644
--- a/src/Core/examples/CMakeLists.txt
+++ b/src/Core/examples/CMakeLists.txt
@@ -9,6 +9,3 @@ target_link_libraries (string_ref_hash PRIVATE clickhouse_common_io)
 
 clickhouse_add_executable (mysql_protocol mysql_protocol.cpp)
 target_link_libraries (mysql_protocol PRIVATE dbms)
-
-clickhouse_add_executable (coro coro.cpp)
-target_link_libraries (coro PRIVATE clickhouse_common_io)
diff --git a/src/Parsers/examples/CMakeLists.txt b/src/Parsers/examples/CMakeLists.txt
index e411574bd65..261f234081c 100644
--- a/src/Parsers/examples/CMakeLists.txt
+++ b/src/Parsers/examples/CMakeLists.txt
@@ -4,7 +4,7 @@ clickhouse_add_executable(lexer lexer.cpp ${SRCS})
 target_link_libraries(lexer PRIVATE clickhouse_parsers)
 
 clickhouse_add_executable(select_parser select_parser.cpp ${SRCS} "../../Server/ServerType.cpp")
-target_link_libraries(select_parser PRIVATE clickhouse_parsers)
+target_link_libraries(select_parser PRIVATE dbms)
 
 clickhouse_add_executable(create_parser create_parser.cpp ${SRCS} "../../Server/ServerType.cpp")
-target_link_libraries(create_parser PRIVATE clickhouse_parsers)
+target_link_libraries(create_parser PRIVATE dbms)

From 3193c6d4ccc7ef02857f14825e1a781e8c2e83c2 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 11 Aug 2023 00:55:57 +0200
Subject: [PATCH 208/249] Fix build

---
 src/Functions/array/arrayAUC.cpp | 2 +-
 src/Server/ServerType.cpp        | 2 +-
 src/Server/ServerType.h          | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/Functions/array/arrayAUC.cpp b/src/Functions/array/arrayAUC.cpp
index b7bd7dcc0ad..499fe4ce7b2 100644
--- a/src/Functions/array/arrayAUC.cpp
+++ b/src/Functions/array/arrayAUC.cpp
@@ -139,7 +139,7 @@ private:
         for (size_t i = 0; i < size; ++i)
         {
             auto next_offset = offsets[i];
-            result[i] = apply(data1, data2, current_offset, next_offset);
+            result[i] = apply(scores, labels, current_offset, next_offset);
             current_offset = next_offset;
         }
     }
diff --git a/src/Server/ServerType.cpp b/src/Server/ServerType.cpp
index 4952cd1bd24..29ba7224c75 100644
--- a/src/Server/ServerType.cpp
+++ b/src/Server/ServerType.cpp
@@ -2,10 +2,10 @@
 
 #include <vector>
 #include <algorithm>
-#include <base/types.h>
 
 #include <magic_enum.hpp>
 
+
 namespace DB
 {
 
diff --git a/src/Server/ServerType.h b/src/Server/ServerType.h
index 1fab492222a..eafe4f941dd 100644
--- a/src/Server/ServerType.h
+++ b/src/Server/ServerType.h
@@ -1,13 +1,13 @@
 #pragma once
 
 #include <base/types.h>
+
 namespace DB
 {
 
 class ServerType
 {
 public:
-
     enum Type
     {
         TCP,

From 396db7ab3cee77955173de76f355e1d08c1ec755 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 11 Aug 2023 01:01:43 +0200
Subject: [PATCH 209/249] Fix test
 `00002_log_and_exception_messages_formatting`

---
 .../00002_log_and_exception_messages_formatting.sql           | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql
index 86fe01dc0e3..54daeb928a3 100644
--- a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql
+++ b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql
@@ -36,7 +36,9 @@ create temporary table known_short_messages (s String) as select * from (select
 'Database {} doesn''t exist', 'Dictionary ({}) not found', 'Unknown table function {}',
 'Unknown format {}', 'Unknown explain kind ''{}''', 'Unknown setting {}', 'Unknown input format {}',
 'Unknown identifier: ''{}''', 'User name is empty', 'Expected function, got: {}',
-'Attempt to read after eof', 'String size is too big ({}), maximum: {}', 'API mode: {}'
+'Attempt to read after eof', 'String size is too big ({}), maximum: {}', 'API mode: {}',
+'Processed: {}%', 'Creating {}: {}', 'Table {}.{} doesn''t exist', 'Invalid cache key hex: {}',
+'User has been dropped', 'Illegal type {} of argument of function {}. Should be DateTime or DateTime64'
 ] as arr) array join arr;
 
 -- Check that we don't have too many short meaningless message patterns.

From c98e38611fccca5acfaf466a7ce22e9ae2bc0a6a Mon Sep 17 00:00:00 2001
From: Yakov Olkhovskiy <yakov@clickhouse.com>
Date: Thu, 10 Aug 2023 23:36:47 +0000
Subject: [PATCH 210/249] path to libc

---
 cmake/linux/toolchain-s390x.cmake | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cmake/linux/toolchain-s390x.cmake b/cmake/linux/toolchain-s390x.cmake
index 03314a9d2ba..945eb9affa4 100644
--- a/cmake/linux/toolchain-s390x.cmake
+++ b/cmake/linux/toolchain-s390x.cmake
@@ -20,9 +20,9 @@ set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}/s390x-linux-gnu/libc")
 set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
 set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
 set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
-set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=mold -Wl,-L/usr/lib/x86_64-linux-gnu")
-set (CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fuse-ld=mold -Wl,-L/usr/lib/x86_64-linux-gnu")
-set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=mold -Wl,-L/usr/lib/x86_64-linux-gnu")
+set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=mold -Wl,-L${CMAKE_SYSROOT}/usr/lib64")
+set (CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fuse-ld=mold -Wl,-L${CMAKE_SYSROOT}/usr/lib64")
+set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=mold -Wl,-L${CMAKE_SYSROOT}/usr/lib64")
 
 set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
 set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)

From 3b9d2c708096e51b106acffbfd3cc2c4ac8606e0 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 11 Aug 2023 03:06:07 +0200
Subject: [PATCH 211/249] Simplification

---
 .../config.d/create_logs_at_startup.yaml      |  1 -
 programs/server/config.xml                    |  3 -
 src/Common/SystemLogBase.h                    |  2 +-
 src/Interpreters/Context.cpp                  | 16 ++++
 src/Interpreters/Context.h                    |  3 +
 src/Interpreters/InterpreterSystemQuery.cpp   | 82 ++++++++-----------
 src/Interpreters/SystemLog.cpp                | 14 +---
 src/Interpreters/SystemLog.h                  |  5 +-
 .../config.d/create_logs_at_startup.yaml      | 34 --------
 tests/config/install.sh                       |  1 -
 10 files changed, 57 insertions(+), 104 deletions(-)
 delete mode 120000 programs/server/config.d/create_logs_at_startup.yaml
 delete mode 100644 tests/config/config.d/create_logs_at_startup.yaml

diff --git a/programs/server/config.d/create_logs_at_startup.yaml b/programs/server/config.d/create_logs_at_startup.yaml
deleted file mode 120000
index a09d2783a9e..00000000000
--- a/programs/server/config.d/create_logs_at_startup.yaml
+++ /dev/null
@@ -1 +0,0 @@
-../../../tests/config/config.d/create_logs_at_startup.yaml
\ No newline at end of file
diff --git a/programs/server/config.xml b/programs/server/config.xml
index 3d1c92b135c..14b8954fc39 100644
--- a/programs/server/config.xml
+++ b/programs/server/config.xml
@@ -1037,9 +1037,6 @@
 
         <!-- example of using a different storage policy for a system table -->
         <!-- storage_policy>local_ssd</storage_policy -->
-
-        <!-- Create the table at startup or lazily on first record (the default is lazy) -->
-        <create_at_startup>false</create_at_startup>
     </query_log>
 
     <!-- Trace log. Stores stack traces collected by query profilers.
diff --git a/src/Common/SystemLogBase.h b/src/Common/SystemLogBase.h
index e05fae69549..1f5832e7aac 100644
--- a/src/Common/SystemLogBase.h
+++ b/src/Common/SystemLogBase.h
@@ -56,7 +56,7 @@ public:
 
     //// force -- force table creation (used for SYSTEM FLUSH LOGS)
     virtual void flush(bool force = false) = 0; /// NOLINT
-    virtual void prepareTable(bool if_not_exists) = 0;
+    virtual void prepareTable() = 0;
 
     /// Start the background thread.
     virtual void startup() = 0;
diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp
index ce737aa8730..b8a700f5311 100644
--- a/src/Interpreters/Context.cpp
+++ b/src/Interpreters/Context.cpp
@@ -3163,6 +3163,12 @@ void Context::setCluster(const String & cluster_name, const std::shared_ptr<Clus
 
 void Context::initializeSystemLogs()
 {
+    /// It is required, because the initialization of system logs can be also
+    /// triggered from another thread, that is launched while initializing the system logs,
+    /// for example, system.filesystem_cache_log will be triggered by parts loading
+    /// of any other table if it is stored on a disk with cache.
+    auto lock = getLock();
+
     shared->system_logs = std::make_unique<SystemLogs>(getGlobalContext(), getConfigRef());
 }
 
@@ -3363,6 +3369,16 @@ std::shared_ptr<AsynchronousInsertLog> Context::getAsynchronousInsertLog() const
     return shared->system_logs->asynchronous_insert_log;
 }
 
+std::vector<ISystemLog *> Context::getSystemLogs() const
+{
+    auto lock = getLock();
+
+    if (!shared->system_logs)
+        return {};
+
+    return shared->system_logs->logs;
+}
+
 CompressionCodecPtr Context::chooseCompressionCodec(size_t part_size, double part_size_ratio) const
 {
     auto lock = getLock();
diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h
index 28427fd5bda..07e0cac79e3 100644
--- a/src/Interpreters/Context.h
+++ b/src/Interpreters/Context.h
@@ -86,6 +86,7 @@ struct Progress;
 struct FileProgress;
 class Clusters;
 class QueryCache;
+class ISystemLog;
 class QueryLog;
 class QueryThreadLog;
 class QueryViewsLog;
@@ -1020,6 +1021,8 @@ public:
     std::shared_ptr<FilesystemReadPrefetchesLog> getFilesystemReadPrefetchesLog() const;
     std::shared_ptr<AsynchronousInsertLog> getAsynchronousInsertLog() const;
 
+    std::vector<ISystemLog *> getSystemLogs() const;
+
     /// Returns an object used to log operations with parts if it possible.
     /// Provide table name to make required checks.
     std::shared_ptr<PartLog> getPartLog(const String & part_database) const;
diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp
index 9c77b40f803..1ea68dad043 100644
--- a/src/Interpreters/InterpreterSystemQuery.cpp
+++ b/src/Interpreters/InterpreterSystemQuery.cpp
@@ -103,40 +103,34 @@ namespace ActionLocks
 namespace
 {
 
-ExecutionStatus getOverallExecutionStatusOfCommands()
+/// Sequentially tries to execute all commands and throws exception with info about failed commands
+void executeCommandsAndThrowIfError(std::vector<std::function<void()>> commands)
 {
-    return ExecutionStatus(0);
-}
+    ExecutionStatus result(0);
+    for (auto & command : commands)
+    {
+        try
+        {
+            command();
+        }
+        catch (...)
+        {
+            ExecutionStatus current_result = ExecutionStatus::fromCurrentException();
 
-/// Consequently tries to execute all commands and generates final exception message for failed commands
-template <typename Callable, typename ... Callables>
-ExecutionStatus getOverallExecutionStatusOfCommands(Callable && command, Callables && ... commands)
-{
-    ExecutionStatus status_head(0);
-    try
-    {
-        command();
-    }
-    catch (...)
-    {
-        status_head = ExecutionStatus::fromCurrentException();
+            if (result.code == 0)
+                result.code = current_result.code;
+
+            if (!current_result.message.empty())
+            {
+                if (!result.message.empty())
+                    result.message += '\n';
+                result.message += current_result.message;
+            }
+        }
     }
 
-    ExecutionStatus status_tail = getOverallExecutionStatusOfCommands(std::forward<Callables>(commands)...);
-
-    auto res_status = status_head.code != 0 ? status_head.code : status_tail.code;
-    auto res_message = status_head.message + (status_tail.message.empty() ? "" : ("\n" + status_tail.message));
-
-    return ExecutionStatus(res_status, res_message);
-}
-
-/// Consequently tries to execute all commands and throws exception with info about failed commands
-template <typename ... Callables>
-void executeCommandsAndThrowIfError(Callables && ... commands)
-{
-    auto status = getOverallExecutionStatusOfCommands(std::forward<Callables>(commands)...);
-    if (status.code != 0)
-        throw Exception::createDeprecated(status.message, status.code);
+    if (result.code != 0)
+        throw Exception::createDeprecated(result.message, result.code);
 }
 
 
@@ -425,10 +419,10 @@ BlockIO InterpreterSystemQuery::execute()
         case Type::RELOAD_DICTIONARIES:
         {
             getContext()->checkAccess(AccessType::SYSTEM_RELOAD_DICTIONARY);
-            executeCommandsAndThrowIfError(
+            executeCommandsAndThrowIfError({
                 [&] { system_context->getExternalDictionariesLoader().reloadAllTriedToLoad(); },
                 [&] { system_context->getEmbeddedDictionaries().reload(); }
-            );
+            });
             ExternalDictionariesLoader::resetAll();
             break;
         }
@@ -557,23 +551,13 @@ BlockIO InterpreterSystemQuery::execute()
         case Type::FLUSH_LOGS:
         {
             getContext()->checkAccess(AccessType::SYSTEM_FLUSH_LOGS);
-            executeCommandsAndThrowIfError(
-                [&] { if (auto query_log = getContext()->getQueryLog()) query_log->flush(true); },
-                [&] { if (auto part_log = getContext()->getPartLog("")) part_log->flush(true); },
-                [&] { if (auto query_thread_log = getContext()->getQueryThreadLog()) query_thread_log->flush(true); },
-                [&] { if (auto trace_log = getContext()->getTraceLog()) trace_log->flush(true); },
-                [&] { if (auto text_log = getContext()->getTextLog()) text_log->flush(true); },
-                [&] { if (auto metric_log = getContext()->getMetricLog()) metric_log->flush(true); },
-                [&] { if (auto asynchronous_metric_log = getContext()->getAsynchronousMetricLog()) asynchronous_metric_log->flush(true); },
-                [&] { if (auto opentelemetry_span_log = getContext()->getOpenTelemetrySpanLog()) opentelemetry_span_log->flush(true); },
-                [&] { if (auto query_views_log = getContext()->getQueryViewsLog()) query_views_log->flush(true); },
-                [&] { if (auto zookeeper_log = getContext()->getZooKeeperLog()) zookeeper_log->flush(true); },
-                [&] { if (auto session_log = getContext()->getSessionLog()) session_log->flush(true); },
-                [&] { if (auto transactions_info_log = getContext()->getTransactionsInfoLog()) transactions_info_log->flush(true); },
-                [&] { if (auto processors_profile_log = getContext()->getProcessorsProfileLog()) processors_profile_log->flush(true); },
-                [&] { if (auto cache_log = getContext()->getFilesystemCacheLog()) cache_log->flush(true); },
-                [&] { if (auto asynchronous_insert_log = getContext()->getAsynchronousInsertLog()) asynchronous_insert_log->flush(true); }
-            );
+
+            std::vector<std::function<void()>> commands;
+            auto logs = getContext()->getSystemLogs();
+            for (auto * system_log : logs)
+                commands.push_back([system_log] { system_log->flush(true); });
+
+            executeCommandsAndThrowIfError(commands);
             break;
         }
         case Type::STOP_LISTEN:
diff --git a/src/Interpreters/SystemLog.cpp b/src/Interpreters/SystemLog.cpp
index 6c5576b07d7..23a5a96584c 100644
--- a/src/Interpreters/SystemLog.cpp
+++ b/src/Interpreters/SystemLog.cpp
@@ -133,8 +133,6 @@ std::shared_ptr<TSystemLog> createSystemLog(
     log_settings.queue_settings.database = config.getString(config_prefix + ".database", default_database_name);
     log_settings.queue_settings.table = config.getString(config_prefix + ".table", default_table_name);
 
-    log_settings.create_at_startup = config.getBool(config_prefix + ".create_at_startup", false);
-
     if (log_settings.queue_settings.database != default_database_name)
     {
         /// System tables must be loaded before other tables, but loading order is undefined for all databases except `system`
@@ -384,9 +382,6 @@ SystemLog<LogElement>::SystemLog(
     , create_query(serializeAST(*getCreateTableQuery()))
 {
     assert(settings_.queue_settings.database == DatabaseCatalog::SYSTEM_DATABASE);
-
-    if (settings_.create_at_startup)
-        prepareTable(true);
 }
 
 template <typename LogElement>
@@ -442,7 +437,7 @@ void SystemLog<LogElement>::savingThreadFunction()
             {
                 if (should_prepare_tables_anyway)
                 {
-                    prepareTable(false);
+                    prepareTable();
                     LOG_TRACE(log, "Table created (force)");
 
                     queue->confirm(to_flush_end);
@@ -474,7 +469,7 @@ void SystemLog<LogElement>::flushImpl(const std::vector<LogElement> & to_flush,
         /// flush. This is done to allow user to drop the table at any moment
         /// (new empty table will be created automatically). BTW, flush method
         /// is called from single thread.
-        prepareTable(false);
+        prepareTable();
 
         ColumnsWithTypeAndName log_element_columns;
         auto log_element_names_and_types = LogElement::getNamesAndTypes();
@@ -524,16 +519,13 @@ void SystemLog<LogElement>::flushImpl(const std::vector<LogElement> & to_flush,
 
 
 template <typename LogElement>
-void SystemLog<LogElement>::prepareTable(bool if_not_exists)
+void SystemLog<LogElement>::prepareTable()
 {
     String description = table_id.getNameForLogs();
 
     auto table = DatabaseCatalog::instance().tryGetTable(table_id, getContext());
     if (table)
     {
-        if (if_not_exists)
-            return;
-
         if (old_create_query.empty())
         {
             old_create_query = serializeAST(*getCreateTableQueryClean(table_id, getContext()));
diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h
index ad130f17ce5..cf287ad7775 100644
--- a/src/Interpreters/SystemLog.h
+++ b/src/Interpreters/SystemLog.h
@@ -93,9 +93,6 @@ struct SystemLogSettings
     SystemLogQueueSettings queue_settings;
 
     String engine;
-
-    /// Create the table eagerly at startup or lazily on first record.
-    bool create_at_startup = false;
 };
 
 template <typename LogElement>
@@ -145,7 +142,7 @@ private:
       * Renames old table if its structure is not suitable.
       * This cannot be done in constructor to avoid deadlock while renaming a table under locked Context when SystemLog object is created.
       */
-    void prepareTable(bool if_not_exists) override;
+    void prepareTable() override;
 
     void savingThreadFunction() override;
 
diff --git a/tests/config/config.d/create_logs_at_startup.yaml b/tests/config/config.d/create_logs_at_startup.yaml
deleted file mode 100644
index e9cc94925de..00000000000
--- a/tests/config/config.d/create_logs_at_startup.yaml
+++ /dev/null
@@ -1,34 +0,0 @@
-query_log:
-  create_at_startup: 1
-trace_log:
-  create_at_startup: 1
-query_thread_log:
-  create_at_startup: 1
-query_views_log:
-  create_at_startup: 1
-part_log:
-  create_at_startup: 1
-text_log:
-  create_at_startup: 1
-metric_log:
-  create_at_startup: 1
-asynchronous_metric_log:
-  create_at_startup: 1
-opentelemetry_span_log:
-  create_at_startup: 1
-crash_log:
-  create_at_startup: 1
-processors_profile_log:
-  create_at_startup: 1
-asynchronous_insert_log:
-  create_at_startup: 1
-zookeeper_log:
-  create_at_startup: 1
-transactions_info_log:
-  create_at_startup: 1
-session_log:
-  create_at_startup: 1
-filesystem_cache_log:
-  create_at_startup: 1
-filesystem_read_prefetches_log:
-  create_at_startup: 1
diff --git a/tests/config/install.sh b/tests/config/install.sh
index 44b515965f4..503da780061 100755
--- a/tests/config/install.sh
+++ b/tests/config/install.sh
@@ -60,7 +60,6 @@ ln -sf $SRC_PATH/config.d/compressed_marks_and_index.xml $DEST_SERVER_PATH/confi
 ln -sf $SRC_PATH/config.d/disable_s3_env_credentials.xml $DEST_SERVER_PATH/config.d/
 ln -sf $SRC_PATH/config.d/enable_wait_for_shutdown_replicated_tables.xml $DEST_SERVER_PATH/config.d/
 ln -sf $SRC_PATH/config.d/backups.xml $DEST_SERVER_PATH/config.d/
-ln -sf $SRC_PATH/config.d/create_logs_at_startup.yaml $DEST_SERVER_PATH/config.d/
 
 # Not supported with fasttest.
 if [ "${DEST_SERVER_PATH}" = "/etc/clickhouse-server" ]

From 1ef301f5b1da2ee161f1194981da883d15dd1085 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 11 Aug 2023 03:11:16 +0200
Subject: [PATCH 212/249] Fix `01485_256_bit_multiply`

---
 tests/queries/0_stateless/01485_256_bit_multiply.sql | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/queries/0_stateless/01485_256_bit_multiply.sql b/tests/queries/0_stateless/01485_256_bit_multiply.sql
index f3f6cc05058..5c8c47c9127 100644
--- a/tests/queries/0_stateless/01485_256_bit_multiply.sql
+++ b/tests/queries/0_stateless/01485_256_bit_multiply.sql
@@ -1,3 +1,5 @@
+-- Tags: no-random-settings, no-asan, no-msan, no-tsan, no-ubsan, no-debug
+
 select count() from
 (
     select toInt128(number) * number x, toInt256(number) * number y from numbers_mt(100000000) where x != y

From 978ab1d07ef6eb24d4200c92ccd6f12bef46240a Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 11 Aug 2023 03:15:27 +0200
Subject: [PATCH 213/249] Remove old test

---
 ...khouse_benchmark_connection_hang.reference |  3 -
 ...55_clickhouse_benchmark_connection_hang.sh | 72 -------------------
 2 files changed, 75 deletions(-)
 delete mode 100644 tests/queries/0_stateless/01955_clickhouse_benchmark_connection_hang.reference
 delete mode 100755 tests/queries/0_stateless/01955_clickhouse_benchmark_connection_hang.sh

diff --git a/tests/queries/0_stateless/01955_clickhouse_benchmark_connection_hang.reference b/tests/queries/0_stateless/01955_clickhouse_benchmark_connection_hang.reference
deleted file mode 100644
index 9e388b62601..00000000000
--- a/tests/queries/0_stateless/01955_clickhouse_benchmark_connection_hang.reference
+++ /dev/null
@@ -1,3 +0,0 @@
-Loaded 1 queries.
-I/O error: Too many open files
-70
diff --git a/tests/queries/0_stateless/01955_clickhouse_benchmark_connection_hang.sh b/tests/queries/0_stateless/01955_clickhouse_benchmark_connection_hang.sh
deleted file mode 100755
index 7740ffcce7b..00000000000
--- a/tests/queries/0_stateless/01955_clickhouse_benchmark_connection_hang.sh
+++ /dev/null
@@ -1,72 +0,0 @@
-#!/usr/bin/env bash
-# Tags: no-ubsan
-# Tag no-ubsan: Limits RLIMIT_NOFILE, see comment in the test
-
-# shellcheck disable=SC2086
-
-CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# shellcheck source=../shell_config.sh
-. "$CURDIR"/../shell_config.sh
-
-# NOTE: Tests with limit for number of opened files cannot be run under UBsan.
-#
-# UBsan needs to create pipe each time it need to check the type:
-#
-#     pipe()
-#     __sanitizer::IsAccessibleMemoryRange(unsigned long, unsigned long)
-#     __ubsan::checkDynamicType(void*, void*, unsigned long) + 271
-#     HandleDynamicTypeCacheMiss(__ubsan::DynamicTypeCacheMissData*, unsigned long, unsigned long, __ubsan::ReportOptions) + 34
-#     __ubsan_handle_dynamic_type_cache_miss_abort + 58
-#
-# Obviously it will fail if RLIMIT_NOFILE exceeded (like in this test), and the UBsan will falsely report [1]:
-#
-#     01955_clickhouse_benchmark_connection_hang:                             [ FAIL ] 1.56 sec. - result differs with reference:
-#     --- /usr/share/clickhouse-test/queries/0_stateless/01955_clickhouse_benchmark_connection_hang.reference	2021-07-21 11:14:58.000000000 +0300
-#     +++ /tmp/clickhouse-test/0_stateless/01955_clickhouse_benchmark_connection_hang.stdout	2021-07-21 11:53:45.684050372 +0300
-#     @@ -1,3 +1,22 @@
-#      Loaded 1 queries.
-#     -I/O error: Too many open files
-#     -70
-#     +../contrib/libcxx/include/memory:3212:19: runtime error: member call on address 0x00002939d5c0 which does not point to an object of type 'std::__1::__shared_weak_count'
-#     +0x00002939d5c0: note: object has invalid vptr
-#     +<memory cannot be printed>
-#     +==558==WARNING: Can't create a socket pair to start external symbolizer (errno: 24)
-#     +==558==WARNING: Can't create a socket pair to start external symbolizer (errno: 24)
-#     +==558==WARNING: Can't create a socket pair to start external symbolizer (errno: 24)
-#     +==558==WARNING: Can't create a socket pair to start external symbolizer (errno: 24)
-#     +==558==WARNING: Can't create a socket pair to start external symbolizer (errno: 24)
-#     +==558==WARNING: Failed to use and restart external symbolizer!
-#     +    #0 0xfe86b57  (/usr/bin/clickhouse+0xfe86b57)
-#     +    #1 0xfe83fd7  (/usr/bin/clickhouse+0xfe83fd7)
-#     +    #2 0xfe89af4  (/usr/bin/clickhouse+0xfe89af4)
-#     +    #3 0xfe81fa9  (/usr/bin/clickhouse+0xfe81fa9)
-#     +    #4 0x1f377609  (/usr/bin/clickhouse+0x1f377609)
-#     +    #5 0xfe7e2a1  (/usr/bin/clickhouse+0xfe7e2a1)
-#     +    #6 0xfce1003  (/usr/bin/clickhouse+0xfce1003)
-#     +    #7 0x7f3345bd30b2  (/lib/x86_64-linux-gnu/libc.so.6+0x270b2)
-#     +    #8 0xfcbf0ed  (/usr/bin/clickhouse+0xfcbf0ed)
-#     +
-#     +SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior ../contrib/libcxx/include/memory:3212:19 in 
-#     +1
-#
-# Stacktrace from lldb:
-#
-#      thread #1, name = 'clickhouse-benc', stop reason = Dynamic type mismatch
-#    * frame #0: 0x000000000fffc070 clickhouse`__ubsan_on_report
-#      frame #1: 0x000000000fff6511 clickhouse`__ubsan::Diag::~Diag() + 209
-#      frame #2: 0x000000000fffcb11 clickhouse`HandleDynamicTypeCacheMiss(__ubsan::DynamicTypeCacheMissData*, unsigned long, unsigned long, __ubsan::ReportOptions) + 609
-#      frame #3: 0x000000000fffcf2a clickhouse`__ubsan_handle_dynamic_type_cache_miss_abort + 58
-#      frame #4: 0x00000000101a33f8 clickhouse`std::__1::shared_ptr<PoolBase<DB::Connection>::PoolEntryHelper>::~shared_ptr(this=<unavailable>) + 152 at memory:3212
-#      frame #5: 0x00000000101a267a clickhouse`PoolBase<DB::Connection>::Entry::~Entry(this=<unavailable>) + 26 at PoolBase.h:67
-#      frame #6: 0x00000000101a0878 clickhouse`DB::ConnectionPool::get(this=<unavailable>, timeouts=0x00007fffffffc278, settings=<unavailable>, force_connected=true) + 664 at ConnectionPool.h:93
-#      frame #7: 0x00000000101a6395 clickhouse`DB::Benchmark::runBenchmark(this=<unavailable>) + 981 at Benchmark.cpp:309
-#      frame #8: 0x000000001019e84a clickhouse`DB::Benchmark::main(this=0x00007fffffffd8c8, (null)=<unavailable>) + 586 at Benchmark.cpp:128
-#      frame #9: 0x000000001f5d028a clickhouse`Poco::Util::Application::run(this=0x00007fffffffd8c8) + 42 at Application.cpp:334
-#      frame #10: 0x000000001019ab42 clickhouse`mainEntryClickHouseBenchmark(argc=<unavailable>, argv=<unavailable>) + 6978 at Benchmark.cpp:655
-#      frame #11: 0x000000000fffdfc4 clickhouse`main(argc_=<unavailable>, argv_=<unavailable>) + 356 at main.cpp:366
-#      frame #12: 0x00007ffff7de6d0a libc.so.6`__libc_start_main(main=(clickhouse`main at main.cpp:339), argc=7, argv=0x00007fffffffe1e8, init=<unavailable>, fini=<unavailable>, rtld_fini=<unavailable>, stack_end=0x00007fffffffe1d8) + 234 at libc-start.c:308
-#      frame #13: 0x000000000ffdc0aa clickhouse`_start + 42
-
-# Limit number of files to 50, and we will get EMFILE for some of socket()
-prlimit --nofile=50 $CLICKHOUSE_BENCHMARK --iterations 1 --concurrency 50 --query 'select 1' 2>&1
-echo $?

From b253d78770354e9265b3b23abf34cade4ad63f66 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 11 Aug 2023 03:37:31 +0200
Subject: [PATCH 214/249] Remove flaky tests for the experimental UNDROP
 feature

---
 .../0_stateless/02681_undrop_query.reference  | 32 -------
 .../0_stateless/02681_undrop_query.sql        | 90 -------------------
 .../02681_undrop_query_uuid.reference         |  6 --
 .../0_stateless/02681_undrop_query_uuid.sh    | 19 ----
 4 files changed, 147 deletions(-)
 delete mode 100644 tests/queries/0_stateless/02681_undrop_query.reference
 delete mode 100644 tests/queries/0_stateless/02681_undrop_query.sql
 delete mode 100644 tests/queries/0_stateless/02681_undrop_query_uuid.reference
 delete mode 100755 tests/queries/0_stateless/02681_undrop_query_uuid.sh

diff --git a/tests/queries/0_stateless/02681_undrop_query.reference b/tests/queries/0_stateless/02681_undrop_query.reference
deleted file mode 100644
index 5df6c392eb9..00000000000
--- a/tests/queries/0_stateless/02681_undrop_query.reference
+++ /dev/null
@@ -1,32 +0,0 @@
-test MergeTree undrop
-02681_undrop_mergetree
-1
-2
-3
-test detach
-UPDATE num = 2 WHERE id = 1
-test MergeTree with cluster
-02681_undrop_uuid_on_cluster
-1
-2
-3
-test MergeTree without uuid on cluster
-02681_undrop_no_uuid_on_cluster
-1
-2
-3
-test ReplicatedMergeTree undrop
-02681_undrop_replicatedmergetree
-1
-2
-3
-test Log undrop
-02681_undrop_log
-1
-2
-3
-test Distributed undrop
-02681_undrop_distributed
-test MergeTree drop and undrop multiple times
-02681_undrop_multiple
-3
diff --git a/tests/queries/0_stateless/02681_undrop_query.sql b/tests/queries/0_stateless/02681_undrop_query.sql
deleted file mode 100644
index 39ca1548d53..00000000000
--- a/tests/queries/0_stateless/02681_undrop_query.sql
+++ /dev/null
@@ -1,90 +0,0 @@
--- Tags: no-ordinary-database, no-replicated-database, distributed, zookeeper
-
-set database_atomic_wait_for_drop_and_detach_synchronously = 0;
-set allow_experimental_undrop_table_query = 1;
-
-select 'test MergeTree undrop';
-drop table if exists 02681_undrop_mergetree sync;
-create table 02681_undrop_mergetree (id Int32) Engine=MergeTree() order by id;
-insert into 02681_undrop_mergetree values (1),(2),(3);
-drop table 02681_undrop_mergetree;
-select table from system.dropped_tables where table = '02681_undrop_mergetree' limit 1;
-undrop table 02681_undrop_mergetree;
-select * from 02681_undrop_mergetree order by id;
-drop table 02681_undrop_mergetree sync;
-
-select 'test detach';
-drop table if exists 02681_undrop_detach sync;
-create table 02681_undrop_detach (id Int32, num Int32) Engine=MergeTree() order by id;
-insert into 02681_undrop_detach values (1, 1);
-detach table 02681_undrop_detach;
-undrop table 02681_undrop_detach; -- { serverError 57 }
-attach table 02681_undrop_detach;
-alter table 02681_undrop_detach update num = 2 where id = 1;
-select command from system.mutations where table='02681_undrop_detach' and database=currentDatabase() limit 1;
-drop table 02681_undrop_detach sync;
-
-select 'test MergeTree with cluster';
-drop table if exists 02681_undrop_uuid_on_cluster on cluster test_shard_localhost sync format Null;
-create table 02681_undrop_uuid_on_cluster on cluster test_shard_localhost (id Int32) Engine=MergeTree() order by id format Null;
-insert into 02681_undrop_uuid_on_cluster values (1),(2),(3);
-drop table 02681_undrop_uuid_on_cluster on cluster test_shard_localhost format Null;
-select table from system.dropped_tables where table = '02681_undrop_uuid_on_cluster' limit 1;
-undrop table 02681_undrop_uuid_on_cluster on cluster test_shard_localhost format Null;
-select * from 02681_undrop_uuid_on_cluster order by id;
-drop table 02681_undrop_uuid_on_cluster sync;
-
-select 'test MergeTree without uuid on cluster';
-drop table if exists 02681_undrop_no_uuid_on_cluster on cluster test_shard_localhost sync format Null;
-create table 02681_undrop_no_uuid_on_cluster on cluster test_shard_localhost (id Int32) Engine=MergeTree() order by id format Null;
-insert into 02681_undrop_no_uuid_on_cluster values (1),(2),(3);
-drop table 02681_undrop_no_uuid_on_cluster on cluster test_shard_localhost format Null;
-select table from system.dropped_tables where table = '02681_undrop_no_uuid_on_cluster' limit 1;
-undrop table 02681_undrop_no_uuid_on_cluster on cluster test_shard_localhost format Null;
-select * from 02681_undrop_no_uuid_on_cluster order by id;
-drop table 02681_undrop_no_uuid_on_cluster on cluster test_shard_localhost sync format Null;
-
-select 'test ReplicatedMergeTree undrop';
-drop table if exists 02681_undrop_replicatedmergetree sync;
-create table 02681_undrop_replicatedmergetree (id Int32) Engine=ReplicatedMergeTree('/clickhouse/tables/{database}/02681_undrop_replicatedmergetree', 'test_undrop') order by id;
-insert into 02681_undrop_replicatedmergetree values (1),(2),(3);
-drop table 02681_undrop_replicatedmergetree;
-select table from system.dropped_tables where table = '02681_undrop_replicatedmergetree' limit 1;
-undrop table 02681_undrop_replicatedmergetree;
-select * from 02681_undrop_replicatedmergetree order by id;
-drop table 02681_undrop_replicatedmergetree sync;
-
-select 'test Log undrop';
-drop table if exists 02681_undrop_log sync;
-create table 02681_undrop_log (id Int32) Engine=Log();
-insert into 02681_undrop_log values (1),(2),(3);
-drop table 02681_undrop_log;
-select table from system.dropped_tables where table = '02681_undrop_log' limit 1;
-undrop table 02681_undrop_log;
-select * from 02681_undrop_log order by id;
-drop table 02681_undrop_log sync;
-
-select 'test Distributed undrop';
-drop table if exists 02681_undrop_distributed sync;
-create table 02681_undrop_distributed (id Int32) Engine = Distributed(test_shard_localhost, currentDatabase(), 02681_undrop, rand());
-drop table 02681_undrop_distributed;
-select table from system.dropped_tables where table = '02681_undrop_distributed' limit 1;
-undrop table 02681_undrop_distributed;
-drop table 02681_undrop_distributed sync;
-
-select 'test MergeTree drop and undrop multiple times';
-drop table if exists 02681_undrop_multiple sync;
-create table 02681_undrop_multiple (id Int32) Engine=MergeTree() order by id;
-insert into 02681_undrop_multiple values (1);
-drop table 02681_undrop_multiple;
-create table 02681_undrop_multiple (id Int32) Engine=MergeTree() order by id;
-insert into 02681_undrop_multiple values (2);
-drop table 02681_undrop_multiple;
-create table 02681_undrop_multiple (id Int32) Engine=MergeTree() order by id;
-insert into 02681_undrop_multiple values (3);
-drop table 02681_undrop_multiple;
-select table from system.dropped_tables where table = '02681_undrop_multiple' limit 1;
-undrop table 02681_undrop_multiple;
-select * from 02681_undrop_multiple order by id;
-undrop table 02681_undrop_multiple; -- { serverError 57 }
-drop table 02681_undrop_multiple sync;
diff --git a/tests/queries/0_stateless/02681_undrop_query_uuid.reference b/tests/queries/0_stateless/02681_undrop_query_uuid.reference
deleted file mode 100644
index beae016401b..00000000000
--- a/tests/queries/0_stateless/02681_undrop_query_uuid.reference
+++ /dev/null
@@ -1,6 +0,0 @@
-test MergeTree with uuid
-02681_undrop_uuid
-OK
-1
-2
-3
diff --git a/tests/queries/0_stateless/02681_undrop_query_uuid.sh b/tests/queries/0_stateless/02681_undrop_query_uuid.sh
deleted file mode 100755
index a93f30ef459..00000000000
--- a/tests/queries/0_stateless/02681_undrop_query_uuid.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env bash
-# Tags: no-ordinary-database, no-replicated-database
-
-CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# shellcheck source=../shell_config.sh
-. "$CURDIR"/../shell_config.sh
-
-echo 'test MergeTree with uuid'
-${CLICKHOUSE_CLIENT} -q "drop table if exists 02681_undrop_uuid sync;"
-uuid=$(${CLICKHOUSE_CLIENT} --query "SELECT generateUUIDv4()")
-uuid2=$(${CLICKHOUSE_CLIENT} --query "SELECT generateUUIDv4()")
-${CLICKHOUSE_CLIENT} --distributed_ddl_output_mode=none -q "create table 02681_undrop_uuid UUID '$uuid' on cluster test_shard_localhost (id Int32) Engine=MergeTree() order by id;"
-${CLICKHOUSE_CLIENT} -q "insert into 02681_undrop_uuid values (1),(2),(3);"
-${CLICKHOUSE_CLIENT} --distributed_ddl_output_mode=none -q "drop table 02681_undrop_uuid on cluster test_shard_localhost settings database_atomic_wait_for_drop_and_detach_synchronously = 0;"
-${CLICKHOUSE_CLIENT} -q "select table from system.dropped_tables where table = '02681_undrop_uuid' limit 1;"
-${CLICKHOUSE_CLIENT} -q "undrop table 02681_undrop_uuid UUID '$uuid2' settings allow_experimental_undrop_table_query = 1;" 2>&1| grep -Faq "UNKNOWN_TABLE" && echo OK
-${CLICKHOUSE_CLIENT} --distributed_ddl_output_mode=none -q "undrop table 02681_undrop_uuid UUID '$uuid' on cluster test_shard_localhost settings allow_experimental_undrop_table_query = 1;"
-${CLICKHOUSE_CLIENT} -q "select * from 02681_undrop_uuid order by id;"
-${CLICKHOUSE_CLIENT} -q "drop table 02681_undrop_uuid sync;"

From ebf34f8c176de9a25487690a926817a8ceba6679 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 11 Aug 2023 03:46:31 +0200
Subject: [PATCH 215/249] Create system logs at startup

---
 utils/export-logs-in-ci/setup.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/utils/export-logs-in-ci/setup.sh b/utils/export-logs-in-ci/setup.sh
index d7b5f1e5d37..b7b8826c498 100755
--- a/utils/export-logs-in-ci/setup.sh
+++ b/utils/export-logs-in-ci/setup.sh
@@ -14,6 +14,9 @@ EXTRA_ORDER_BY_COLUMNS=${EXTRA_ORDER_BY_COLUMNS:="check_name, "}
 
 CONNECTION_PARAMETERS=${CONNECTION_PARAMETERS:=""}
 
+# Create all configured system logs:
+clickhouse-client --query "SYSTEM FLUSH LOGS"
+
 # For each system log table:
 clickhouse-client --query "SHOW TABLES FROM system LIKE '%\\_log'" | while read table
 do

From 0a49f45f7a8e392c94abe05cc9828a6513d4338c Mon Sep 17 00:00:00 2001
From: Alexey Gerasimchuck <a.gerasimchuck@arenadata.io>
Date: Fri, 11 Aug 2023 03:42:39 +0000
Subject: [PATCH 216/249] Fixed inconsistency between login success and logout

---
 src/Interpreters/Session.cpp    | 44 +++++++++++++++++++++++----------
 src/Interpreters/Session.h      |  2 ++
 src/Interpreters/SessionLog.cpp | 14 +++++------
 src/Interpreters/SessionLog.h   |  9 ++++++-
 src/Server/HTTPHandler.cpp      |  3 +--
 5 files changed, 49 insertions(+), 23 deletions(-)

diff --git a/src/Interpreters/Session.cpp b/src/Interpreters/Session.cpp
index de2a779b740..f8bd70afdb6 100644
--- a/src/Interpreters/Session.cpp
+++ b/src/Interpreters/Session.cpp
@@ -520,6 +520,8 @@ ContextMutablePtr Session::makeSessionContext()
         {},
         session_context->getSettingsRef().max_sessions_for_user);
 
+    recordLoginSucess(session_context);
+
     return session_context;
 }
 
@@ -582,6 +584,8 @@ ContextMutablePtr Session::makeSessionContext(const String & session_name_, std:
         { session_name_ },
         max_sessions_for_user);
 
+    recordLoginSucess(session_context);
+
     return session_context;
 }
 
@@ -655,24 +659,38 @@ ContextMutablePtr Session::makeQueryContextImpl(const ClientInfo * client_info_t
     if (user_id)
         user = query_context->getUser();
 
-    if (!notified_session_log_about_login)
-    {
-        if (auto session_log = getSessionLog())
-        {
-            session_log->addLoginSuccess(
-                    auth_id,
-                    named_session ? std::optional<std::string>(named_session->key.second) : std::nullopt,
-                    *query_context,
-                    user);
-
-            notified_session_log_about_login = true;
-        }
-    }
+    /// Interserver does not create session context
+    recordLoginSucess(query_context);
 
     return query_context;
 }
 
 
+void Session::recordLoginSucess(ContextPtr login_context) const
+{
+    if (notified_session_log_about_login)
+        return;
+
+    if (!login_context)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Session or query context must be created");
+
+    if (auto session_log = getSessionLog())
+    {
+        const auto & settings   = login_context->getSettingsRef();
+        const auto access       = login_context->getAccess();
+
+        session_log->addLoginSuccess(auth_id,
+                                     named_session ? named_session->key.second : "",
+                                     settings,
+                                     access,
+                                     getClientInfo(),
+                                     user);
+    }
+
+    notified_session_log_about_login = true;
+}
+
+
 void Session::releaseSessionID()
 {
     if (!named_session)
diff --git a/src/Interpreters/Session.h b/src/Interpreters/Session.h
index 51c0e3c71fa..81ef987b428 100644
--- a/src/Interpreters/Session.h
+++ b/src/Interpreters/Session.h
@@ -97,6 +97,8 @@ public:
 private:
     std::shared_ptr<SessionLog> getSessionLog() const;
     ContextMutablePtr makeQueryContextImpl(const ClientInfo * client_info_to_copy, ClientInfo * client_info_to_move) const;
+    void recordLoginSucess(ContextPtr login_context) const;
+
 
     mutable bool notified_session_log_about_login = false;
     const UUID auth_id;
diff --git a/src/Interpreters/SessionLog.cpp b/src/Interpreters/SessionLog.cpp
index 0a8a7fc18c5..b847eaf9824 100644
--- a/src/Interpreters/SessionLog.cpp
+++ b/src/Interpreters/SessionLog.cpp
@@ -199,12 +199,13 @@ void SessionLogElement::appendToBlock(MutableColumns & columns) const
     columns[i++]->insertData(auth_failure_reason.data(), auth_failure_reason.length());
 }
 
-void SessionLog::addLoginSuccess(const UUID & auth_id, std::optional<String> session_id, const Context & login_context, const UserPtr & login_user)
+void SessionLog::addLoginSuccess(const UUID & auth_id,
+                                 const String & session_id,
+                                 const Settings & settings,
+                                 const ContextAccessPtr & access,
+                                 const ClientInfo & client_info,
+                                 const UserPtr & login_user)
 {
-    const auto access = login_context.getAccess();
-    const auto & settings = login_context.getSettingsRef();
-    const auto & client_info = login_context.getClientInfo();
-
     DB::SessionLogElement log_entry(auth_id, SESSION_LOGIN_SUCCESS);
     log_entry.client_info = client_info;
 
@@ -215,8 +216,7 @@ void SessionLog::addLoginSuccess(const UUID & auth_id, std::optional<String> ses
     }
     log_entry.external_auth_server = login_user ? login_user->auth_data.getLDAPServerName() : "";
 
-    if (session_id)
-        log_entry.session_id = *session_id;
+    log_entry.session_id = session_id;
 
     if (const auto roles_info = access->getRolesInfo())
         log_entry.roles = roles_info->getCurrentRolesNames();
diff --git a/src/Interpreters/SessionLog.h b/src/Interpreters/SessionLog.h
index 1282ac09c4d..8757bc12270 100644
--- a/src/Interpreters/SessionLog.h
+++ b/src/Interpreters/SessionLog.h
@@ -20,6 +20,7 @@ enum SessionLogElementType : int8_t
 class ContextAccess;
 struct User;
 using UserPtr = std::shared_ptr<const User>;
+using ContextAccessPtr = std::shared_ptr<const ContextAccess>;
 
 /** A struct which will be inserted as row into session_log table.
   *
@@ -72,7 +73,13 @@ class SessionLog : public SystemLog<SessionLogElement>
     using SystemLog<SessionLogElement>::SystemLog;
 
 public:
-    void addLoginSuccess(const UUID & auth_id, std::optional<String> session_id, const Context & login_context, const UserPtr & login_user);
+    void addLoginSuccess(const UUID & auth_id,
+                         const String & session_id,
+                         const Settings & settings,
+                         const ContextAccessPtr & access,
+                         const ClientInfo & client_info,
+                         const UserPtr & login_user);
+
     void addLoginFailure(const UUID & auth_id, const ClientInfo & info, const std::optional<String> & user, const Exception & reason);
     void addLogOut(const UUID & auth_id, const UserPtr & login_user, const ClientInfo & client_info);
 };
diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp
index ff5690a3b07..10d5e7a0242 100644
--- a/src/Server/HTTPHandler.cpp
+++ b/src/Server/HTTPHandler.cpp
@@ -561,8 +561,7 @@ void HTTPHandler::processQuery(
         session->makeSessionContext();
     }
 
-    auto client_info = session->getClientInfo();
-    auto context = session->makeQueryContext(std::move(client_info));
+    auto context = session->makeQueryContext();
 
     /// This parameter is used to tune the behavior of output formats (such as Native) for compatibility.
     if (params.has("client_protocol_version"))

From 5bc5649732ab3c94ae6dcfe4d2caf5f58bf51502 Mon Sep 17 00:00:00 2001
From: Alexey Gerasimchuck <a.gerasimchuck@arenadata.io>
Date: Fri, 11 Aug 2023 05:39:13 +0000
Subject: [PATCH 217/249] added an integration test for peak_memory_usage

---
 .../test_peak_memory_usage/test.py            | 133 ++++++++++++++++++
 1 file changed, 133 insertions(+)
 create mode 100644 tests/integration/test_peak_memory_usage/test.py

diff --git a/tests/integration/test_peak_memory_usage/test.py b/tests/integration/test_peak_memory_usage/test.py
new file mode 100644
index 00000000000..502415d9aae
--- /dev/null
+++ b/tests/integration/test_peak_memory_usage/test.py
@@ -0,0 +1,133 @@
+import pytest
+import tempfile
+import re
+
+from helpers.cluster import ClickHouseCluster
+from helpers.uclient import client, prompt
+
+cluster = ClickHouseCluster(__file__)
+
+shard_1 = cluster.add_instance(
+    "shard_1",
+    main_configs=["configs/remote_servers.xml"],
+    with_zookeeper=True,
+    macros={
+        "shard": "shard_1",
+    },
+)
+shard_2 = cluster.add_instance(
+    "shard_2",
+    main_configs=["configs/remote_servers.xml"],
+    with_zookeeper=True,
+    macros={
+        "shard": "shard_2",
+    },
+)
+
+
+@pytest.fixture(scope="module")
+def started_cluster():
+    try:
+        cluster.start()
+
+        shard_1.query(
+            "CREATE TABLE fixed_numbers ON CLUSTER 'cluster' ("
+            "number UInt64"
+            ") ENGINE=MergeTree()"
+            "ORDER BY number"
+        )
+
+        shard_1.query(
+            "CREATE TABLE distributed_fixed_numbers (number UInt64) ENGINE=Distributed('cluster', 'default', 'fixed_numbers')"
+        )
+
+        # Shard 1 has  singificantly less data then shard 2
+        shard_1.query(
+            "INSERT INTO fixed_numbers SELECT number FROM numbers(1999900, 2000000)"
+        )
+
+        shard_2.query(
+            "INSERT INTO fixed_numbers SELECT number FROM numbers(0, 1999900)"
+        )
+
+        yield cluster
+    finally:
+        cluster.shutdown()
+
+
+def get_memory_usage_from_client_output_and_close(client_output):
+    client_output.seek(0)
+    peek_memory_usage_str_found = False
+    for line in client_output:
+        print(f"'{line}'\n")
+        if not peek_memory_usage_str_found:
+            peek_memory_usage_str_found = "Peak memory usage" in line
+
+        if peek_memory_usage_str_found:
+            search_obj = re.search(r"([-+]?(?:\d*\.*\d+))", line)
+            if search_obj:
+                client_output.close()
+                return search_obj.group()
+
+    client_output.close()
+    return ""
+
+
+def test_clickhouse_client_max_peak_memory_usage_distributed(started_cluster):
+    client_output = tempfile.TemporaryFile(mode="w+t")
+    command_text = (
+        f"{started_cluster.get_client_cmd()} --host {shard_1.ip_address} --port 9000"
+    )
+    with client(name="client1>", log=client_output, command=command_text) as client1:
+        client1.expect(prompt)
+        client1.send(
+            "SELECT COUNT(*) FROM distributed_fixed_numbers WHERE number IN (SELECT number from numbers(1999890, 1999910))"
+        )
+        client1.expect("Peak memory usage")
+        client1.expect(prompt)
+
+    peak_memory_usage = get_memory_usage_from_client_output_and_close(client_output)
+    assert peak_memory_usage
+    assert shard_2.contains_in_log(
+        f"Peak memory usage (for query): {peak_memory_usage}"
+    )
+
+def test_clickhouse_client_max_peak_memory_usage_cluster(started_cluster):
+    client_output = tempfile.TemporaryFile(mode="w+t")
+    command_text = (
+        f"{started_cluster.get_client_cmd()} --host {shard_1.ip_address} --port 9000"
+    )
+    with client(name="client1>", log=client_output, command=command_text) as client1:
+        client1.expect(prompt)
+        client1.send(
+            "SELECT COUNT(*) FROM (SELECT number FROM numbers(1,100000) INTERSECT SELECT * FROM clusterAllReplicas(cluster, default, fixed_numbers))"
+        )
+        client1.expect("Peak memory usage")
+        client1.expect(prompt)
+
+    peak_memory_usage = get_memory_usage_from_client_output_and_close(client_output)
+    assert peak_memory_usage
+    assert shard_2.contains_in_log(
+        f"Peak memory usage (for query): {peak_memory_usage}"
+    )
+
+
+def test_clickhouse_client_max_peak_memory_single_node(started_cluster):
+    client_output = tempfile.TemporaryFile(mode="w+t")
+
+    command_text = (
+        f"{started_cluster.get_client_cmd()} --host {shard_1.ip_address} --port 9000"
+    )
+    with client(name="client1>", log=client_output, command=command_text) as client1:
+        client1.expect(prompt)
+        client1.send(
+            "SELECT COUNT(*) FROM (SELECT number FROM numbers(1,300000) INTERSECT SELECT number FROM numbers(10000,1200000))"
+        )
+        client1.expect("Peak memory usage")
+        client1.expect(prompt)
+
+    peak_memory_usage = get_memory_usage_from_client_output_and_close(client_output)
+    assert peak_memory_usage
+    assert shard_1.contains_in_log(
+        f"Peak memory usage (for query): {peak_memory_usage}"
+    )

From b7c47af8bfc4c1d488deec21bfeb95c145ecc293 Mon Sep 17 00:00:00 2001
From: Alexey Gerasimchuck <a.gerasimchuck@arenadata.io>
Date: Fri, 11 Aug 2023 06:18:20 +0000
Subject: [PATCH 218/249] added  missed files

---
 .../test_peak_memory_usage/__init__.py        |  0
 .../configs/remote_servers.xml                | 20 +++++++++++++++++++
 2 files changed, 20 insertions(+)
 create mode 100644 tests/integration/test_peak_memory_usage/__init__.py
 create mode 100644 tests/integration/test_peak_memory_usage/configs/remote_servers.xml

diff --git a/tests/integration/test_peak_memory_usage/__init__.py b/tests/integration/test_peak_memory_usage/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/integration/test_peak_memory_usage/configs/remote_servers.xml b/tests/integration/test_peak_memory_usage/configs/remote_servers.xml
new file mode 100644
index 00000000000..b137758489e
--- /dev/null
+++ b/tests/integration/test_peak_memory_usage/configs/remote_servers.xml
@@ -0,0 +1,20 @@
+<clickhouse>
+    <remote_servers>
+        <cluster>
+            <shard>
+                <weight>1</weight>
+                <replica>
+                    <host>shard_1</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+            <shard>
+                <weight>3</weight>
+                <replica>
+                    <host>shard_2</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </cluster>
+    </remote_servers>
+</clickhouse>

From 820d821f50c80992ff717c95dff6cf3460e3bd6f Mon Sep 17 00:00:00 2001
From: Alexey Gerasimchuck <a.gerasimchuck@arenadata.io>
Date: Fri, 11 Aug 2023 06:30:48 +0000
Subject: [PATCH 219/249] black run

---
 tests/integration/test_peak_memory_usage/test.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/integration/test_peak_memory_usage/test.py b/tests/integration/test_peak_memory_usage/test.py
index 502415d9aae..3994ee66a4a 100644
--- a/tests/integration/test_peak_memory_usage/test.py
+++ b/tests/integration/test_peak_memory_usage/test.py
@@ -92,6 +92,7 @@ def test_clickhouse_client_max_peak_memory_usage_distributed(started_cluster):
         f"Peak memory usage (for query): {peak_memory_usage}"
     )
 
+
 def test_clickhouse_client_max_peak_memory_usage_cluster(started_cluster):
     client_output = tempfile.TemporaryFile(mode="w+t")
     command_text = (

From a6f89c05465753379f481cfe91130a9cd87c66b0 Mon Sep 17 00:00:00 2001
From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com>
Date: Fri, 11 Aug 2023 13:24:16 +0300
Subject: [PATCH 220/249] =?UTF-8?q?Added=20suggestions=20for=20mistyped=20?=
 =?UTF-8?q?names=20for=20db=20and=20tables=20with=20different=E2=80=A6=20(?=
 =?UTF-8?q?#49801)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Added suggestions for mistyped names for db and tables with different scenarios commented

* fixed bugs

* fixed style check

* fixed errors

* fixed errors

* fixed error with exceptions

* fixed exceptions

* fixed exceptions

* fixed exceptions

* added test and fixed bugs

* fixed style check

* fixed style check

* fixed style check

* fixed check black

* Update test.py

* Fixed server crash

* Fixed server crash and style check

* Fixed style check

* Fixed style check

* Fixed style check

* Fixed bugs with drop_db

* fixed fast test

* added tests

* fixed style check

* fixed style check

* fixed bug with lock_db

* fixed bug with lock_db and fixed reviews

* fixed bug with lock_db and fixed reviews

* fixed style check

* fixed fast test

* fixed fast test

* revert to fd582a2

* revert to fd582a2

* Removed unused parameters

Co-authored-by: Yakov Olkhovskiy <99031427+yakov-olkhovskiy@users.noreply.github.com>

* Remove unused parameters

Co-authored-by: Yakov Olkhovskiy <99031427+yakov-olkhovskiy@users.noreply.github.com>

* resolved arguments issue in assertDatabaseExists

* fixing fast test

* fixed fast test

* fixed stateless test for default db

* Update src/Interpreters/DatabaseCatalog.cpp

Co-authored-by: Yakov Olkhovskiy <99031427+yakov-olkhovskiy@users.noreply.github.com>

* Update src/Interpreters/DatabaseCatalog.cpp

Co-authored-by: Yakov Olkhovskiy <99031427+yakov-olkhovskiy@users.noreply.github.com>

* Update src/Interpreters/DatabaseCatalog.cpp

Co-authored-by: Yakov Olkhovskiy <99031427+yakov-olkhovskiy@users.noreply.github.com>

* Fixing tests.

* resolved problem with mutex

* Fixed mutex in assertDatabaseExists

* changes about assertDatabaseExists

* fixed bugs with file types

* fixed string types

* fixed fast test

* fixed mutex

* fixed mutex

* fixed style check

* Update src/Interpreters/DatabaseCatalog.cpp

Co-authored-by: Yakov Olkhovskiy <99031427+yakov-olkhovskiy@users.noreply.github.com>

* Update src/Interpreters/DatabaseCatalog.cpp

Co-authored-by: Yakov Olkhovskiy <99031427+yakov-olkhovskiy@users.noreply.github.com>

* Update src/Interpreters/DatabaseCatalog.cpp

Co-authored-by: Yakov Olkhovskiy <99031427+yakov-olkhovskiy@users.noreply.github.com>

* Update src/Interpreters/DatabaseCatalog.cpp

Co-authored-by: Yakov Olkhovskiy <99031427+yakov-olkhovskiy@users.noreply.github.com>

* Update src/Interpreters/DatabaseCatalog.cpp

Co-authored-by: Yakov Olkhovskiy <99031427+yakov-olkhovskiy@users.noreply.github.com>

* Update src/Interpreters/DatabaseCatalog.cpp

Co-authored-by: Yakov Olkhovskiy <99031427+yakov-olkhovskiy@users.noreply.github.com>

* fixed build

* added -unlocked versions of functions

* Revert "fixed build"

This reverts commit 8ce961be210c3cfbca35e0d5fb9f86b2ec42ff78.

* Revert "fixed build"

This reverts commit 8ce961be210c3cfbca35e0d5fb9f86b2ec42ff78.

* changed usage of assertDatabaseExistsUnlocked()

* fixed style check

* style check

* style check

* Revert "style check"

This reverts commit 28a9ee85a0a801438f05ca8c6604d67b8c132e69.

* Merge branch 'master' into hints-for-wrong-db-or-table-name

* Changed AssertDatabaseExists and unified exception output

* resolved proposed changes and modified tests

* Revert "resolved proposed changes and modified tests"

This reverts commit d45337d65c16bd55b6bbb5c29023a1d0e5e1a38e.

* resolved requested changes

* fixed tests

* fixed tests

* fixed check black

* Update include brackets

Co-authored-by: Yakov Olkhovskiy <99031427+yakov-olkhovskiy@users.noreply.github.com>

* Update suggested changes

* Update suggested changes

* Fixed style check

* Added test to analyzer_integration_broken

* Update DatabaseCatalog.cpp

* Update test.py

* fixed test

* Revert "fixed test"

This reverts commit ca6d4c17c85b2bebba56ebc24dceac0e425afa0e.

* fixed test

* Revert "fixed test"

This reverts commit fe6d0d9c865fcfe030ea63f370d2930a3bb3b53a, reversing
changes made to 22f44967041fbd32f5f8884daa6f8736247f6fd8.

* Update test.py

* fixed black check

* Update test.py

* fixed long_log_tinylog_deadlock_race

* Update DatabaseCatalog.cpp

* Update test.py

* style

* Update DatabaseCatalog.cpp

* Fixed test

* implemented for IDatabase

* Style check

* removed const_cast

* Update DatabaseCatalog.h

* Update DatabaseCatalog.h

* Update DatabaseCatalog.cpp

* Update DatabaseCatalog.cpp

* Added db name to hints

* Update 00002_log_and_exception_messages_formatting.sql

* Update 00002_log_and_exception_messages_formatting.sql

---------

Co-authored-by: Yakov Olkhovskiy <99031427+yakov-olkhovskiy@users.noreply.github.com>
---
 src/Common/NamePrompter.h                     |   5 +
 src/Databases/IDatabase.cpp                   |  11 +-
 src/Databases/IDatabase.h                     |   1 +
 src/Interpreters/DatabaseCatalog.cpp          | 167 ++++++++++++++----
 src/Interpreters/DatabaseCatalog.h            |  30 +++-
 tests/analyzer_integration_broken_tests.txt   |   1 +
 .../test.py                                   |   2 +-
 .../test_drop_is_lock_free/test.py            |   2 +-
 tests/integration/test_drop_replica/test.py   |   4 +-
 .../test_insert_into_distributed/test.py      |   5 +-
 tests/integration/test_mysql_protocol/test.py |   6 +-
 .../test_system_flush_logs/test.py            |   2 +-
 .../test_wrong_db_or_table_name/__init__.py   |   0
 .../test_wrong_db_or_table_name/test.py       | 108 +++++++++++
 ..._log_and_exception_messages_formatting.sql |   6 +-
 .../01192_rename_database_zookeeper.sh        |   4 +-
 .../0_stateless/01600_detach_permanently.sh   |   2 +-
 .../0_stateless/01848_http_insert_segfault.sh |   2 +-
 18 files changed, 307 insertions(+), 51 deletions(-)
 create mode 100644 tests/integration/test_wrong_db_or_table_name/__init__.py
 create mode 100644 tests/integration/test_wrong_db_or_table_name/test.py

diff --git a/src/Common/NamePrompter.h b/src/Common/NamePrompter.h
index 4a087548a72..79e78529d95 100644
--- a/src/Common/NamePrompter.h
+++ b/src/Common/NamePrompter.h
@@ -106,6 +106,11 @@ public:
         return prompter.getHints(name, getAllRegisteredNames());
     }
 
+    std::vector<String> getHints(const String & name, const std::vector<String> & prompting_strings) const
+    {
+        return prompter.getHints(name, prompting_strings);
+    }
+
     void appendHintsMessage(String & error_message, const String & name) const
     {
         auto hints = getHints(name);
diff --git a/src/Databases/IDatabase.cpp b/src/Databases/IDatabase.cpp
index 9e33548b0dd..7d12ae6c588 100644
--- a/src/Databases/IDatabase.cpp
+++ b/src/Databases/IDatabase.cpp
@@ -1,7 +1,10 @@
+#include <memory>
 #include <Databases/IDatabase.h>
 #include <Storages/IStorage.h>
 #include <Parsers/ASTCreateQuery.h>
 #include <Common/quoteString.h>
+#include <Interpreters/DatabaseCatalog.h>
+#include <Common/NamePrompter.h>
 
 
 namespace DB
@@ -18,7 +21,13 @@ StoragePtr IDatabase::getTable(const String & name, ContextPtr context) const
 {
     if (auto storage = tryGetTable(name, context))
         return storage;
-    throw Exception(ErrorCodes::UNKNOWN_TABLE, "Table {}.{} doesn't exist", backQuoteIfNeed(getDatabaseName()), backQuoteIfNeed(name));
+    TableNameHints hints(this->shared_from_this(), context);
+    std::vector<String> names = hints.getHints(name);
+    if (!names.empty())
+    {
+        throw Exception(ErrorCodes::UNKNOWN_TABLE, "Table {}.{} does not exist. Maybe you meant {}?", backQuoteIfNeed(getDatabaseName()), backQuoteIfNeed(name), backQuoteIfNeed(names[0]));
+    }
+    else throw Exception(ErrorCodes::UNKNOWN_TABLE, "Table {}.{} does not exist", backQuoteIfNeed(getDatabaseName()), backQuoteIfNeed(name));
 }
 
 std::vector<std::pair<ASTPtr, StoragePtr>> IDatabase::getTablesForBackup(const FilterByNameFunction &, const ContextPtr &) const
diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h
index 9bed3c4bfc5..01d940b0429 100644
--- a/src/Databases/IDatabase.h
+++ b/src/Databases/IDatabase.h
@@ -372,6 +372,7 @@ protected:
 };
 
 using DatabasePtr = std::shared_ptr<IDatabase>;
+using ConstDatabasePtr = std::shared_ptr<const IDatabase>;
 using Databases = std::map<String, DatabasePtr>;
 
 }
diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp
index f8481e3f1d8..3baaa182d51 100644
--- a/src/Interpreters/DatabaseCatalog.cpp
+++ b/src/Interpreters/DatabaseCatalog.cpp
@@ -1,3 +1,4 @@
+#include <string>
 #include <Interpreters/DatabaseCatalog.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/loadMetadata.h>
@@ -14,6 +15,7 @@
 #include <IO/ReadHelpers.h>
 #include <Poco/DirectoryIterator.h>
 #include <Poco/Util/AbstractConfiguration.h>
+#include <Common/Exception.h>
 #include <Common/quoteString.h>
 #include <Common/atomicRename.h>
 #include <Common/CurrentMetrics.h>
@@ -23,6 +25,7 @@
 #include <Common/noexcept_scope.h>
 #include <Common/checkStackSize.h>
 
+#include "Interpreters/Context_fwd.h"
 #include "config.h"
 
 #if USE_MYSQL
@@ -35,7 +38,6 @@
 #    include <Storages/PostgreSQL/StorageMaterializedPostgreSQL.h>
 #endif
 
-
 namespace CurrentMetrics
 {
     extern const Metric TablesToDropQueueSize;
@@ -59,6 +61,29 @@ namespace ErrorCodes
     extern const int UNFINISHED;
 }
 
+class DatabaseNameHints : public IHints<1, DatabaseNameHints>
+{
+public:
+    explicit DatabaseNameHints(const DatabaseCatalog & database_catalog_)
+        : database_catalog(database_catalog_)
+    {
+    }
+    Names getAllRegisteredNames() const override
+    {
+        Names result;
+        auto databases_list = database_catalog.getDatabases();
+        for (const auto & database_name : databases_list | boost::adaptors::map_keys)
+        {
+            if (database_name == DatabaseCatalog::TEMPORARY_DATABASE)
+                continue;
+            result.emplace_back(database_name);
+        }
+        return result;
+    }
+private:
+    const DatabaseCatalog & database_catalog;
+};
+
 TemporaryTableHolder::TemporaryTableHolder(ContextPtr context_, const TemporaryTableHolder::Creator & creator, const ASTPtr & query)
     : WithContext(context_->getGlobalContext())
     , temporary_tables(DatabaseCatalog::instance().getDatabaseForTemporaryTables().get())
@@ -313,7 +338,18 @@ DatabaseAndTable DatabaseCatalog::getTableImpl(
         {
             assert(!db_and_table.first && !db_and_table.second);
             if (exception)
-                exception->emplace(Exception(ErrorCodes::UNKNOWN_TABLE, "Table {} doesn't exist", table_id.getNameForLogs()));
+            {
+                TableNameHints hints(this->tryGetDatabase(table_id.getDatabaseName()), getContext());
+                std::vector<String> names = hints.getHints(table_id.getTableName());
+                if (!names.empty())
+                {
+                    /// There is two options: first is to print just the name of the table
+                    /// and the second is to print the result in format: db_name.table_name. I'll comment out the second option below
+                    /// I also leave possibility to print several suggestions
+                    exception->emplace(Exception(ErrorCodes::UNKNOWN_TABLE, "Table {} does not exist. Maybe you meant {}?", table_id.getNameForLogs(), backQuoteIfNeed(names[0])));
+                }
+                else exception->emplace(Exception(ErrorCodes::UNKNOWN_TABLE, "Table {} does not exist", table_id.getNameForLogs()));
+            }
             return {};
         }
 
@@ -359,13 +395,26 @@ DatabaseAndTable DatabaseCatalog::getTableImpl(
 
         std::lock_guard lock{databases_mutex};
         auto it = databases.find(table_id.getDatabaseName());
-        if (databases.end() == it)
+        if (databases.end() != it)
+            database = it->second;
+    }
+
+    if (!database)
+    {
+        if (exception)
         {
-            if (exception)
-                exception->emplace(Exception(ErrorCodes::UNKNOWN_DATABASE, "Database {} doesn't exist", backQuoteIfNeed(table_id.getDatabaseName())));
-            return {};
+            DatabaseNameHints hints(*this);
+            std::vector<String> names = hints.getHints(table_id.getDatabaseName());
+            if (names.empty())
+            {
+                exception->emplace(Exception(ErrorCodes::UNKNOWN_DATABASE, "Database {} does not exist", backQuoteIfNeed(table_id.getDatabaseName())));
+            }
+            else
+            {
+                exception->emplace(Exception(ErrorCodes::UNKNOWN_DATABASE, "Database {} does not exist. Maybe you meant {}?", backQuoteIfNeed(table_id.getDatabaseName()), backQuoteIfNeed(names[0])));
+            }
         }
-        database = it->second;
+        return {};
     }
 
     StoragePtr table;
@@ -386,8 +435,18 @@ DatabaseAndTable DatabaseCatalog::getTableImpl(
     }
 
     if (!table && exception && !exception->has_value())
-        exception->emplace(Exception(ErrorCodes::UNKNOWN_TABLE, "Table {} doesn't exist", table_id.getNameForLogs()));
-
+    {
+        TableNameHints hints(this->tryGetDatabase(table_id.getDatabaseName()), getContext());
+        std::vector<String> names = hints.getHints(table_id.getTableName());
+        if (names.empty())
+        {
+            exception->emplace(Exception(ErrorCodes::UNKNOWN_TABLE, "Table {} does not exist", table_id.getNameForLogs()));
+        }
+        else
+        {
+            exception->emplace(Exception(ErrorCodes::UNKNOWN_TABLE, "Table {} does not exist. Maybe you meant {}?", table_id.getNameForLogs(), backQuoteIfNeed(names[0])));
+        }
+    }
     if (!table)
         database = nullptr;
 
@@ -438,8 +497,26 @@ bool DatabaseCatalog::isPredefinedTable(const StorageID & table_id) const
 
 void DatabaseCatalog::assertDatabaseExists(const String & database_name) const
 {
-    std::lock_guard lock{databases_mutex};
-    assertDatabaseExistsUnlocked(database_name);
+    DatabasePtr db;
+    {
+        std::lock_guard lock{databases_mutex};
+        assert(!database_name.empty());
+        if (auto it = databases.find(database_name); it != databases.end())
+            db = it->second;
+    }
+    if (!db)
+    {
+        DatabaseNameHints hints(*this);
+        std::vector<String> names = hints.getHints(database_name);
+        if (names.empty())
+        {
+            throw Exception(ErrorCodes::UNKNOWN_DATABASE, "Database {} does not exist", backQuoteIfNeed(database_name));
+        }
+        else
+        {
+            throw Exception(ErrorCodes::UNKNOWN_DATABASE, "Database {} does not exist. Maybe you meant {}?", backQuoteIfNeed(database_name), backQuoteIfNeed(names[0]));
+        }
+    }
 }
 
 void DatabaseCatalog::assertDatabaseDoesntExist(const String & database_name) const
@@ -448,19 +525,11 @@ void DatabaseCatalog::assertDatabaseDoesntExist(const String & database_name) co
     assertDatabaseDoesntExistUnlocked(database_name);
 }
 
-void DatabaseCatalog::assertDatabaseExistsUnlocked(const String & database_name) const
-{
-    assert(!database_name.empty());
-    if (databases.end() == databases.find(database_name))
-        throw Exception(ErrorCodes::UNKNOWN_DATABASE, "Database {} doesn't exist", backQuoteIfNeed(database_name));
-}
-
-
 void DatabaseCatalog::assertDatabaseDoesntExistUnlocked(const String & database_name) const
 {
     assert(!database_name.empty());
     if (databases.end() != databases.find(database_name))
-        throw Exception(ErrorCodes::DATABASE_ALREADY_EXISTS, "Database {} already exists.", backQuoteIfNeed(database_name));
+        throw Exception(ErrorCodes::DATABASE_ALREADY_EXISTS, "Database {} already exists", backQuoteIfNeed(database_name));
 }
 
 void DatabaseCatalog::attachDatabase(const String & database_name, const DatabasePtr & database)
@@ -480,18 +549,34 @@ DatabasePtr DatabaseCatalog::detachDatabase(ContextPtr local_context, const Stri
 {
     if (database_name == TEMPORARY_DATABASE)
         throw Exception(ErrorCodes::DATABASE_ACCESS_DENIED, "Cannot detach database with temporary tables.");
-
+    assert(!database_name.empty());
     DatabasePtr db;
     {
         std::lock_guard lock{databases_mutex};
-        assertDatabaseExistsUnlocked(database_name);
-        db = databases.find(database_name)->second;
-        UUID db_uuid = db->getUUID();
-        if (db_uuid != UUIDHelpers::Nil)
-            removeUUIDMapping(db_uuid);
-        databases.erase(database_name);
-    }
+        if (auto it = databases.find(database_name); it != databases.end())
+        {
+            db = it->second;
 
+            UUID db_uuid = db->getUUID();
+            if (db_uuid != UUIDHelpers::Nil)
+                removeUUIDMapping(db_uuid);
+            databases.erase(database_name);
+
+        }
+    }
+    if (!db)
+    {
+        DatabaseNameHints hints(*this);
+        std::vector<String> names = hints.getHints(database_name);
+        if (names.empty())
+        {
+            throw Exception(ErrorCodes::UNKNOWN_DATABASE, "Database {} does not exist", backQuoteIfNeed(database_name));
+        }
+        else
+        {
+            throw Exception(ErrorCodes::UNKNOWN_DATABASE, "Database {} does not exist. Maybe you meant {}?", backQuoteIfNeed(database_name), backQuoteIfNeed(names[0]));
+        }
+    }
     if (check_empty)
     {
         try
@@ -527,7 +612,6 @@ DatabasePtr DatabaseCatalog::detachDatabase(ContextPtr local_context, const Stri
         if (db_uuid != UUIDHelpers::Nil)
             removeUUIDMappingFinally(db_uuid);
     }
-
     return db;
 }
 
@@ -553,9 +637,28 @@ void DatabaseCatalog::updateDatabaseName(const String & old_name, const String &
 
 DatabasePtr DatabaseCatalog::getDatabase(const String & database_name) const
 {
-    std::lock_guard lock{databases_mutex};
-    assertDatabaseExistsUnlocked(database_name);
-    return databases.find(database_name)->second;
+    assert(!database_name.empty());
+    DatabasePtr db;
+    {
+        std::lock_guard lock{databases_mutex};
+        if (auto it = databases.find(database_name); it != databases.end())
+            db = it->second;
+    }
+
+    if (!db)
+    {
+        DatabaseNameHints hints(*this);
+        std::vector<String> names = hints.getHints(database_name);
+        if (names.empty())
+        {
+            throw Exception(ErrorCodes::UNKNOWN_DATABASE, "Database {} does not exist", backQuoteIfNeed(database_name));
+        }
+        else
+        {
+            throw Exception(ErrorCodes::UNKNOWN_DATABASE, "Database {} does not exist. Maybe you meant {}?", backQuoteIfNeed(database_name), backQuoteIfNeed(names[0]));
+        }
+    }
+    return db;
 }
 
 DatabasePtr DatabaseCatalog::tryGetDatabase(const String & database_name) const
diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h
index 805d7786569..d734e6bc149 100644
--- a/src/Interpreters/DatabaseCatalog.h
+++ b/src/Interpreters/DatabaseCatalog.h
@@ -6,7 +6,10 @@
 #include <Databases/TablesDependencyGraph.h>
 #include <Parsers/IAST_fwd.h>
 #include <Storages/IStorage_fwd.h>
+#include "Common/NamePrompter.h"
 #include <Common/SharedMutex.h>
+#include "Storages/IStorage.h"
+#include "Databases/IDatabase.h"
 
 #include <boost/noncopyable.hpp>
 #include <Poco/Logger.h>
@@ -27,6 +30,32 @@ namespace fs = std::filesystem;
 namespace DB
 {
 
+class TableNameHints : public IHints<1, TableNameHints>
+{
+public:
+    TableNameHints(ConstDatabasePtr database_, ContextPtr context_)
+        : context(context_),
+        database(database_)
+    {
+    }
+    Names getAllRegisteredNames() const override
+    {
+        Names result;
+        if (database)
+        {
+            for (auto table_it = database->getTablesIterator(context); table_it->isValid(); table_it->next())
+            {
+                const auto & storage_id = table_it->table()->getStorageID();
+                result.emplace_back(storage_id.getTableName());
+            }
+        }
+        return result;
+    }
+private:
+    ContextPtr context;
+    ConstDatabasePtr database;
+};
+
 class IDatabase;
 class Exception;
 class ColumnsDescription;
@@ -262,7 +291,6 @@ private:
     static std::unique_ptr<DatabaseCatalog> database_catalog;
 
     explicit DatabaseCatalog(ContextMutablePtr global_context_);
-    void assertDatabaseExistsUnlocked(const String & database_name) const TSA_REQUIRES(databases_mutex);
     void assertDatabaseDoesntExistUnlocked(const String & database_name) const TSA_REQUIRES(databases_mutex);
 
     void shutdownImpl();
diff --git a/tests/analyzer_integration_broken_tests.txt b/tests/analyzer_integration_broken_tests.txt
index b9cca4bf677..68822fbf311 100644
--- a/tests/analyzer_integration_broken_tests.txt
+++ b/tests/analyzer_integration_broken_tests.txt
@@ -139,6 +139,7 @@ test_quota/test.py::test_tracking_quota
 test_quota/test.py::test_users_xml_is_readonly
 test_replicating_constants/test.py::test_different_versions
 test_merge_tree_s3/test.py::test_heavy_insert_select_check_memory[node]
+test_wrong_db_or_table_name/test.py::test_wrong_table_name
 test_drop_is_lock_free/test.py::test_query_is_lock_free[detach table]
 test_odbc_interaction/test.py::test_postgres_insert
 test_zookeeper_config/test.py::test_chroot_with_different_root
diff --git a/tests/integration/test_dictionaries_update_and_reload/test.py b/tests/integration/test_dictionaries_update_and_reload/test.py
index a973b697d0d..99d08f1b6ea 100644
--- a/tests/integration/test_dictionaries_update_and_reload/test.py
+++ b/tests/integration/test_dictionaries_update_and_reload/test.py
@@ -281,7 +281,7 @@ def test_reload_after_fail_in_cache_dictionary(started_cluster):
     query_and_get_error = instance.query_and_get_error
 
     # Can't get a value from the cache dictionary because the source (table `test.xypairs`) doesn't respond.
-    expected_error = "Table test.xypairs doesn't exist"
+    expected_error = "Table test.xypairs does not exist"
     update_error = "Could not update cache dictionary cache_xypairs now"
     assert expected_error in query_and_get_error(
         "SELECT dictGetUInt64('cache_xypairs', 'y', toUInt64(1))"
diff --git a/tests/integration/test_drop_is_lock_free/test.py b/tests/integration/test_drop_is_lock_free/test.py
index 9f595800bea..61d52a1d9b1 100644
--- a/tests/integration/test_drop_is_lock_free/test.py
+++ b/tests/integration/test_drop_is_lock_free/test.py
@@ -125,7 +125,7 @@ def test_query_is_lock_free(lock_free_query, exclusive_table):
                 SELECT count() FROM {exclusive_table};
             """
         )
-        assert f"Table default.{exclusive_table} doesn't exist" in result
+        assert f"Table default.{exclusive_table} does not exist" in result
     else:
         assert 0 == int(
             node.query(
diff --git a/tests/integration/test_drop_replica/test.py b/tests/integration/test_drop_replica/test.py
index 0941e664982..e0928c6ab08 100644
--- a/tests/integration/test_drop_replica/test.py
+++ b/tests/integration/test_drop_replica/test.py
@@ -159,11 +159,11 @@ def test_drop_replica(start_cluster):
     for i in range(1, 5):
         node_1_1.query("DETACH DATABASE test{}".format(i))
 
-    assert "doesn't exist" in node_1_3.query_and_get_error(
+    assert "does not exist" in node_1_3.query_and_get_error(
         "SYSTEM DROP REPLICA 'node_1_1' FROM TABLE test.test_table"
     )
 
-    assert "doesn't exist" in node_1_3.query_and_get_error(
+    assert "does not exist" in node_1_3.query_and_get_error(
         "SYSTEM DROP REPLICA 'node_1_1' FROM DATABASE test1"
     )
 
diff --git a/tests/integration/test_insert_into_distributed/test.py b/tests/integration/test_insert_into_distributed/test.py
index 3bee2149387..09a14c11727 100644
--- a/tests/integration/test_insert_into_distributed/test.py
+++ b/tests/integration/test_insert_into_distributed/test.py
@@ -246,7 +246,7 @@ def test_inserts_local(started_cluster):
 
 def test_inserts_single_replica_local_internal_replication(started_cluster):
     with pytest.raises(
-        QueryRuntimeException, match="Table default.single_replicated doesn't exist"
+        QueryRuntimeException, match="Table default.single_replicated does not exist"
     ):
         node1.query(
             "INSERT INTO distributed_one_replica_internal_replication VALUES ('2000-01-01', 1)",
@@ -279,7 +279,8 @@ def test_inserts_single_replica_internal_replication(started_cluster):
 def test_inserts_single_replica_no_internal_replication(started_cluster):
     try:
         with pytest.raises(
-            QueryRuntimeException, match="Table default.single_replicated doesn't exist"
+            QueryRuntimeException,
+            match="Table default.single_replicated does not exist",
         ):
             node1.query(
                 "INSERT INTO distributed_one_replica_no_internal_replication VALUES ('2000-01-01', 1)",
diff --git a/tests/integration/test_mysql_protocol/test.py b/tests/integration/test_mysql_protocol/test.py
index 494af3c219d..f2bffe69495 100644
--- a/tests/integration/test_mysql_protocol/test.py
+++ b/tests/integration/test_mysql_protocol/test.py
@@ -207,7 +207,7 @@ def test_mysql_client(started_cluster):
     expected_msg = "\n".join(
         [
             "mysql: [Warning] Using a password on the command line interface can be insecure.",
-            "ERROR 81 (00000) at line 1: Code: 81. DB::Exception: Database system2 doesn't exist",
+            "ERROR 81 (00000) at line 1: Code: 81. DB::Exception: Database system2 does not exist",
         ]
     )
     assert stderr[: len(expected_msg)].decode() == expected_msg
@@ -621,7 +621,7 @@ def test_python_client(started_cluster):
         client.select_db("system2")
 
     assert exc_info.value.args[1].startswith(
-        "Code: 81. DB::Exception: Database system2 doesn't exist"
+        "Code: 81. DB::Exception: Database system2 does not exist"
     ), exc_info.value.args[1]
 
     cursor = client.cursor(pymysql.cursors.DictCursor)
@@ -646,7 +646,7 @@ def test_golang_client(started_cluster, golang_container):
     )
 
     assert code == 1
-    assert stderr.decode() == "Error 81: Database abc doesn't exist\n"
+    assert stderr.decode() == "Error 81: Database abc does not exist\n"
 
     code, (stdout, stderr) = golang_container.exec_run(
         "./main --host {host} --port {port} --user default --password 123 --database "
diff --git a/tests/integration/test_system_flush_logs/test.py b/tests/integration/test_system_flush_logs/test.py
index 5bf3eff8b2d..084d342d736 100644
--- a/tests/integration/test_system_flush_logs/test.py
+++ b/tests/integration/test_system_flush_logs/test.py
@@ -47,7 +47,7 @@ def test_system_logs(flush_logs, table, exists):
     else:
         response = node.query_and_get_error(q)
         assert (
-            "Table {} doesn't exist".format(table) in response
+            "Table {} does not exist".format(table) in response
             or "Unknown table expression identifier '{}'".format(table) in response
         )
 
diff --git a/tests/integration/test_wrong_db_or_table_name/__init__.py b/tests/integration/test_wrong_db_or_table_name/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/integration/test_wrong_db_or_table_name/test.py b/tests/integration/test_wrong_db_or_table_name/test.py
new file mode 100644
index 00000000000..68af383b6c3
--- /dev/null
+++ b/tests/integration/test_wrong_db_or_table_name/test.py
@@ -0,0 +1,108 @@
+import pytest
+from helpers.client import QueryRuntimeException
+from helpers.cluster import ClickHouseCluster
+from helpers.test_tools import TSV
+
+cluster = ClickHouseCluster(__file__)
+node = cluster.add_instance("node")
+
+
+@pytest.fixture(scope="module")
+def start():
+    try:
+        cluster.start()
+        yield cluster
+    finally:
+        cluster.shutdown()
+
+
+def test_wrong_database_name(start):
+    node.query(
+        """
+        CREATE DATABASE test;
+        CREATE TABLE test.table_test (i Int64) ENGINE=Memory;
+        INSERT INTO test.table_test SELECT 1;
+        """
+    )
+
+    with pytest.raises(
+        QueryRuntimeException,
+        match="DB::Exception: Database tes does not exist. Maybe you meant test?.",
+    ):
+        node.query("SELECT * FROM tes.table_test LIMIT 1;")
+    assert int(node.query("SELECT count() FROM test.table_test;")) == 1
+    node.query(
+        """
+        DROP TABLE test.table_test;
+        DROP DATABASE test;
+        """
+    )
+
+
+def test_drop_wrong_database_name(start):
+    node.query(
+        """
+        CREATE DATABASE test;
+        CREATE TABLE test.table_test (i Int64) ENGINE=Memory;
+        INSERT INTO test.table_test SELECT 1;
+        """
+    )
+
+    with pytest.raises(
+        QueryRuntimeException,
+        match="DB::Exception: Database tes does not exist. Maybe you meant test?.",
+    ):
+        node.query("DROP DATABASE tes;")
+    assert int(node.query("SELECT count() FROM test.table_test;")) == 1
+    node.query("DROP DATABASE test;")
+
+
+def test_wrong_table_name(start):
+    node.query(
+        """
+        CREATE DATABASE test;
+        CREATE TABLE test.table_test (i Int64) ENGINE=Memory;
+        CREATE TABLE test.table_test2 (i Int64) ENGINE=Memory;
+        INSERT INTO test.table_test SELECT 1;
+        """
+    )
+    with pytest.raises(
+        QueryRuntimeException,
+        match="DB::Exception: Table test.table_test1 does not exist. Maybe you meant table_test?.",
+    ):
+        node.query(
+            """
+            SELECT * FROM test.table_test1 LIMIT 1;
+            """
+        )
+    assert int(node.query("SELECT count() FROM test.table_test;")) == 1
+    node.query(
+        """
+            DROP TABLE test.table_test;
+            DROP TABLE test.table_test2;
+            DROP DATABASE test;
+            """
+    )
+
+
+def test_drop_wrong_table_name(start):
+    node.query(
+        """
+        CREATE DATABASE test;
+        CREATE TABLE test.table_test (i Int64) ENGINE=Memory;
+        INSERT INTO test.table_test SELECT 1;
+        """
+    )
+
+    with pytest.raises(
+        QueryRuntimeException,
+        match="DB::Exception: Table test.table_tes does not exist. Maybe you meant table_test?.",
+    ):
+        node.query("DROP TABLE test.table_tes;")
+    assert int(node.query("SELECT count() FROM test.table_test;")) == 1
+    node.query(
+        """
+        DROP TABLE test.table_test;
+        DROP DATABASE test;
+        """
+    )
diff --git a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql
index 86fe01dc0e3..a993c330d2a 100644
--- a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql
+++ b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql
@@ -32,8 +32,8 @@ create temporary table known_short_messages (s String) as select * from (select
 'brotli decode error{}', 'Invalid H3 index: {}', 'Too large node state size', 'No additional keys found.',
 'Attempt to read after EOF.', 'Replication was stopped', '{}	building file infos', 'Cannot parse uuid {}',
 'Query was cancelled', 'Cancelled merging parts', 'Cancelled mutating parts', 'Log pulling is cancelled',
-'Transaction was cancelled', 'Could not find table: {}', 'Table {} doesn''t exist',
-'Database {} doesn''t exist', 'Dictionary ({}) not found', 'Unknown table function {}',
+'Transaction was cancelled', 'Could not find table: {}', 'Table {} does not exist',
+'Database {} does not exist', 'Dictionary ({}) not found', 'Unknown table function {}',
 'Unknown format {}', 'Unknown explain kind ''{}''', 'Unknown setting {}', 'Unknown input format {}',
 'Unknown identifier: ''{}''', 'User name is empty', 'Expected function, got: {}',
 'Attempt to read after eof', 'String size is too big ({}), maximum: {}', 'API mode: {}'
@@ -48,7 +48,7 @@ select 'messages shorter than 16', max2(countDistinctOrDefault(message_format_st
 -- Unlike above, here we look at length of the formatted message, not format string. Most short format strings are fine because they end up decorated with context from outer or inner exceptions, e.g.:
 -- "Expected end of line" -> "Code: 117. DB::Exception: Expected end of line: (in file/uri /var/lib/clickhouse/user_files/data_02118): (at row 1)"
 -- But we have to cut out the boilerplate, e.g.:
--- "Code: 60. DB::Exception: Table default.a doesn't exist. (UNKNOWN_TABLE), Stack trace" -> "Table default.a doesn't exist."
+-- "Code: 60. DB::Exception: Table default.a does not exist. (UNKNOWN_TABLE), Stack trace" -> "Table default.a does not exist."
 -- This table currently doesn't have enough information to do this reliably, so we just regex search for " (ERROR_NAME_IN_CAPS)" and hope that's good enough.
 -- For the "Code: 123. DB::Exception: " part, we just subtract 26 instead of searching for it. Because sometimes it's not at the start, e.g.:
 -- "Unexpected error, will try to restart main thread: Code: 341. DB::Exception: Unexpected error: Code: 57. DB::Exception:[...]"
diff --git a/tests/queries/0_stateless/01192_rename_database_zookeeper.sh b/tests/queries/0_stateless/01192_rename_database_zookeeper.sh
index 6dd7ff3cdc8..d1a7144e886 100755
--- a/tests/queries/0_stateless/01192_rename_database_zookeeper.sh
+++ b/tests/queries/0_stateless/01192_rename_database_zookeeper.sh
@@ -24,7 +24,7 @@ $CLICKHOUSE_CLIENT --function_sleep_max_microseconds_per_block 15000000 -q "INSE
 sleep 1
 
 $CLICKHOUSE_CLIENT -q "RENAME DATABASE test_01192 TO default" 2>&1| grep -F "already exists" > /dev/null && echo "ok"
-$CLICKHOUSE_CLIENT -q "RENAME DATABASE test_01192_notexisting TO test_01192_renamed" 2>&1| grep -F "doesn't exist" > /dev/null && echo "ok"
+$CLICKHOUSE_CLIENT -q "RENAME DATABASE test_01192_notexisting TO test_01192_renamed" 2>&1| grep -F "does not exist" > /dev/null && echo "ok"
 $CLICKHOUSE_CLIENT -q "RENAME DATABASE test_01192 TO test_01192_renamed" && echo "renamed"
 wait
 
@@ -50,7 +50,7 @@ $CLICKHOUSE_CLIENT -q "RENAME TABLE test_01192.mt TO test_01192_atomic.mt, test_
 # 6. check data after RENAME
 $CLICKHOUSE_CLIENT -q "SELECT count(n), sum(n) FROM test_01192_atomic.mt"
 $CLICKHOUSE_CLIENT -q "SELECT count(n), sum(n) FROM test_01192_atomic.rmt"
-$CLICKHOUSE_CLIENT -q "SELECT count(n), sum(n) FROM test_01192_atomic.mv" 2>&1| grep -F "doesn't exist" > /dev/null && echo "ok"
+$CLICKHOUSE_CLIENT -q "SELECT count(n), sum(n) FROM test_01192_atomic.mv" 2>&1| grep -F "does not exist" > /dev/null && echo "ok"
 
 # 7. create dictionary and check it
 $CLICKHOUSE_CLIENT -q "CREATE TABLE test_01192.mt (n UInt64, _part String) ENGINE=Memory" # mock
diff --git a/tests/queries/0_stateless/01600_detach_permanently.sh b/tests/queries/0_stateless/01600_detach_permanently.sh
index c32a255448e..4b270265cdc 100755
--- a/tests/queries/0_stateless/01600_detach_permanently.sh
+++ b/tests/queries/0_stateless/01600_detach_permanently.sh
@@ -108,7 +108,7 @@ clickhouse_local "INSERT INTO db_ordinary.src SELECT * FROM numbers(10)"
 clickhouse_local "SELECT if(count() = 10, 'MV is working', 'MV failed') FROM db_ordinary.src_mv_with_inner"
 
 clickhouse_local "DETACH VIEW db_ordinary.src_mv_with_inner PERMANENTLY; INSERT INTO db_ordinary.src SELECT * FROM numbers(10)" --stacktrace
-clickhouse_local "SELECT if(count() = 10, 'MV can be detached permanently', 'MV detach failed') FROM db_ordinary.src_mv_with_inner" 2>&1 | grep -c "db_ordinary.src_mv_with_inner doesn't exist"
+clickhouse_local "SELECT if(count() = 10, 'MV can be detached permanently', 'MV detach failed') FROM db_ordinary.src_mv_with_inner" 2>&1 | grep -c "db_ordinary.src_mv_with_inner does not exist"
 ## Quite silly: ATTACH MATERIALIZED VIEW don't work with short syntax (w/o select), but i can attach it using ATTACH TABLE ...
 clickhouse_local "ATTACH TABLE db_ordinary.src_mv_with_inner"
 clickhouse_local "INSERT INTO db_ordinary.src SELECT * FROM numbers(10)"
diff --git a/tests/queries/0_stateless/01848_http_insert_segfault.sh b/tests/queries/0_stateless/01848_http_insert_segfault.sh
index 1f2e9eebcdc..6397068a77a 100755
--- a/tests/queries/0_stateless/01848_http_insert_segfault.sh
+++ b/tests/queries/0_stateless/01848_http_insert_segfault.sh
@@ -6,4 +6,4 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 
-${CLICKHOUSE_LOCAL} --query "select col1, initializeAggregation('argMaxState', col2, insertTime) as col2, now() as insertTime FROM generateRandom('col1 String, col2 Array(Float64)') LIMIT 1000000 FORMAT CSV"  | ${CLICKHOUSE_CURL} -s 'http://localhost:8123/?query=INSERT%20INTO%20non_existing_table%20SELECT%20col1%2C%20initializeAggregation(%27argMaxState%27%2C%20col2%2C%20insertTime)%20as%20col2%2C%20now()%20as%20insertTime%20FROM%20input(%27col1%20String%2C%20col2%20Array(Float64)%27)%20FORMAT%20CSV' --data-binary @- | grep -q "Table default.non_existing_table doesn't exist" && echo 'Ok.' || echo 'FAIL' ||:
+${CLICKHOUSE_LOCAL} --query "select col1, initializeAggregation('argMaxState', col2, insertTime) as col2, now() as insertTime FROM generateRandom('col1 String, col2 Array(Float64)') LIMIT 1000000 FORMAT CSV"  | ${CLICKHOUSE_CURL} -s 'http://localhost:8123/?query=INSERT%20INTO%20non_existing_table%20SELECT%20col1%2C%20initializeAggregation(%27argMaxState%27%2C%20col2%2C%20insertTime)%20as%20col2%2C%20now()%20as%20insertTime%20FROM%20input(%27col1%20String%2C%20col2%20Array(Float64)%27)%20FORMAT%20CSV' --data-binary @- | grep -q "Table default.non_existing_table does not exist" && echo 'Ok.' || echo 'FAIL' ||:

From ec50c61a8ce6cdaa8092c8cdee4c6f5e0ebbca30 Mon Sep 17 00:00:00 2001
From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com>
Date: Fri, 11 Aug 2023 12:28:51 +0200
Subject: [PATCH 221/249] Try fix test

---
 .../queries/0_stateless/01053_window_view_proc_hop_to_now.sh  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/01053_window_view_proc_hop_to_now.sh b/tests/queries/0_stateless/01053_window_view_proc_hop_to_now.sh
index fe757acfd06..8e28995980f 100755
--- a/tests/queries/0_stateless/01053_window_view_proc_hop_to_now.sh
+++ b/tests/queries/0_stateless/01053_window_view_proc_hop_to_now.sh
@@ -16,14 +16,14 @@ DROP TABLE IF EXISTS wv;
 
 CREATE TABLE dst(count UInt64) Engine=MergeTree ORDER BY tuple();
 CREATE TABLE mt(a Int32) ENGINE=MergeTree ORDER BY tuple();
-CREATE WINDOW VIEW wv TO dst AS SELECT count(a) AS count FROM mt GROUP BY hop(now('US/Samoa'), INTERVAL '5' SECOND, INTERVAL '5' SECOND, 'US/Samoa') AS wid;
+CREATE WINDOW VIEW wv TO dst AS SELECT count(a) AS count FROM mt GROUP BY hop(now('US/Samoa'), INTERVAL '10' SECOND, INTERVAL '10' SECOND, 'US/Samoa') AS wid;
 
 INSERT INTO mt VALUES (1);
 EOF
 
 for _ in {1..100}; do
 	$CLICKHOUSE_CLIENT "${opts[@]}" --query="SELECT count(*) FROM dst" | grep -q "1" && echo 'OK' && break
-	sleep .5
+	sleep .2
 done
 
 $CLICKHOUSE_CLIENT "${opts[@]}" --query="SELECT count FROM dst"

From 6a343a0b6f61efc7ac7e7e25c734dd3c72b6d34c Mon Sep 17 00:00:00 2001
From: selfuppen <selfuppen@qq.com>
Date: Fri, 11 Aug 2023 19:43:29 +0800
Subject: [PATCH 222/249] [doc/zh]: issues/53254:1. remove incorrect
 description about the  orange-marked column values;2. remove dated
 description about the default value of `optimize_use_projections` ;3. add new
 note items for projection

---
 .../sparse-primary-indexes.md                 | 29 ++++++++++---------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/docs/zh/guides/improving-query-performance/sparse-primary-indexes.md b/docs/zh/guides/improving-query-performance/sparse-primary-indexes.md
index eedc913cf82..3f42f3f8da4 100644
--- a/docs/zh/guides/improving-query-performance/sparse-primary-indexes.md
+++ b/docs/zh/guides/improving-query-performance/sparse-primary-indexes.md
@@ -346,9 +346,7 @@ UserID.bin，URL.bin，和EventTime.bin是<font face = "monospace">UserID</font>
 
 - 我们将主键列(<font face = "monospace">UserID</font>, <font face = "monospace">URL</font>)中的一些列值标记为橙色。
 
-  这些橙色标记的列值是每个颗粒中每个主键列的最小值。这里的例外是最后一个颗粒(上图中的颗粒1082)，最后一个颗粒我们标记的是最大的值。
-
-  正如我们将在下面看到的，这些橙色标记的列值将是表主索引中的条目。
+  这些橙色标记的列值是每个颗粒中第一行的主键列值。正如我们将在下面看到的，这些橙色标记的列值将是表主索引中的条目。
 
 - 我们从0开始对行进行编号，以便与ClickHouse内部行编号方案对齐，该方案也用于记录消息。
 :::
@@ -1071,13 +1069,6 @@ ClickHouse服务器日志文件中相应的跟踪日志确认了ClickHouse正在
 ## 通过projections使用联合主键索引
 
 
-<a href="https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree/#projections" target="_blank">Projections</a>目前是一个实验性的功能，因此我们需要告诉ClickHouse：
-
-```sql
-SET optimize_use_projections = 1;
-```
-
-
 在原表上创建projection：
 ```sql
 ALTER TABLE hits_UserID_URL
@@ -1096,10 +1087,12 @@ ALTER TABLE hits_UserID_URL
 
 :::note
 - 该projection正在创建一个隐藏表，该表的行顺序和主索引基于该projection的给定order BY子句
-- 我们使用MATERIALIZE关键字，以便立即用源表hits_UserID_URL的所有887万行导入隐藏表
+- `SHOW TABLES` 语句查询是不会列出这个隐藏表的
+- 我们使用`MATERIALIZE`关键字，以便立即用源表<a href="https://clickhouse.com/docs/zh/guides/best-practices#%E5%8C%85%E5%90%AB%E4%B8%BB%E9%94%AE%E7%9A%84%E8%A1%A8" target="_blank">hits_UserID_URL</a>的所有887万行导入隐藏表
 - 如果在源表hits_UserID_URL中插入了新行，那么这些行也会自动插入到隐藏表中
 - 查询总是(从语法上)针对源表hits_UserID_URL，但是如果隐藏表的行顺序和主索引允许更有效地执行查询，那么将使用该隐藏表
-- 实际上，隐式创建的隐藏表的行顺序和主索引与我们显式创建的辅助表相同：
+- 请注意，投影(projections)不会使 `ORDER BY` 查询语句的效率更高，即使 `ORDER BY` 匹配上了 projection 的 `ORDER BY` 语句(请参阅：https://github.com/ClickHouse/ClickHouse/issues/47333)
+- 实际上，隐式创建的隐藏表的行顺序和主索引与我们显式创建的<a href="https://clickhouse.com/docs/zh/guides/best-practices#%E9%80%9A%E8%BF%87%E8%BE%85%E5%8A%A9%E8%A1%A8%E4%BD%BF%E7%94%A8%E8%81%94%E5%90%88%E4%B8%BB%E9%94%AE%E7%B4%A2%E5%BC%95" target="_blank">辅助表</a>相同：
 
 <img src={require('../../../en/guides/best-practices/images/sparse-primary-indexes-12c-1.png').default} class="image"/>
 
@@ -1163,7 +1156,7 @@ ClickHouse服务器日志文件中跟踪日志确认了ClickHouse正在对索引
 ```
 
 
-## 移除无效的主键列
+## 小结
 
 
 带有联合主键(UserID, URL)的表的主索引对于加快UserID的查询过滤非常有用。但是，尽管URL列是联合主键的一部分，但该索引在加速URL查询过滤方面并没有提供显著的帮助。
@@ -1176,4 +1169,12 @@ ClickHouse服务器日志文件中跟踪日志确认了ClickHouse正在对索引
 
 但是，如果复合主键中的键列在基数上有很大的差异，那么查询按基数升序对主键列进行排序是有益的。
 
-主键键列之间的基数差越大，主键键列的顺序越重要。我们将在以后的文章中对此进行演示。请继续关注。
+主键键列之间的基数差得越大，主键中的列的顺序越重要。我们将在下一章节对此进行演示。
+
+# 高效地为键列排序
+
+TODO
+
+# 高效地识别单行
+
+TODO

From 22d1ddeb7176a653130df99d505a32b66e7a481f Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 11 Aug 2023 14:45:12 +0200
Subject: [PATCH 223/249] Fix tidy

---
 src/Interpreters/InterpreterSystemQuery.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp
index 1ea68dad043..c0341330846 100644
--- a/src/Interpreters/InterpreterSystemQuery.cpp
+++ b/src/Interpreters/InterpreterSystemQuery.cpp
@@ -552,10 +552,11 @@ BlockIO InterpreterSystemQuery::execute()
         {
             getContext()->checkAccess(AccessType::SYSTEM_FLUSH_LOGS);
 
-            std::vector<std::function<void()>> commands;
             auto logs = getContext()->getSystemLogs();
+            std::vector<std::function<void()>> commands;
+            commands.reserve(logs.size());
             for (auto * system_log : logs)
-                commands.push_back([system_log] { system_log->flush(true); });
+                commands.emplace_back([system_log] { system_log->flush(true); });
 
             executeCommandsAndThrowIfError(commands);
             break;

From e70dc7d507c1a1409fe399bbe79da811b943bafe Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Fri, 11 Aug 2023 17:36:52 +0200
Subject: [PATCH 224/249] improve `move_primary_key_columns_to_end_of_prewhere`

---
 .../MergeTree/MergeTreeWhereOptimizer.cpp     | 27 +++++++++++++--
 .../MergeTree/MergeTreeWhereOptimizer.h       |  2 +-
 ...02842_move_pk_to_end_of_prewhere.reference | 20 +++++++++++
 .../02842_move_pk_to_end_of_prewhere.sql      | 34 +++++++++++++++++++
 4 files changed, 79 insertions(+), 4 deletions(-)
 create mode 100644 tests/queries/0_stateless/02842_move_pk_to_end_of_prewhere.reference
 create mode 100644 tests/queries/0_stateless/02842_move_pk_to_end_of_prewhere.sql

diff --git a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp
index 5efb7286685..d73ee970683 100644
--- a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp
+++ b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp
@@ -226,7 +226,7 @@ static bool isConditionGood(const RPNBuilderTreeNode & condition, const NameSet
     return false;
 }
 
-void MergeTreeWhereOptimizer::analyzeImpl(Conditions & res, const RPNBuilderTreeNode & node, const WhereOptimizerContext & where_optimizer_context) const
+void MergeTreeWhereOptimizer::analyzeImpl(Conditions & res, const RPNBuilderTreeNode & node, const WhereOptimizerContext & where_optimizer_context, std::set<Int64> & pk_positions) const
 {
     auto function_node_optional = node.toFunctionNodeOrNull();
 
@@ -237,7 +237,7 @@ void MergeTreeWhereOptimizer::analyzeImpl(Conditions & res, const RPNBuilderTree
         for (size_t i = 0; i < arguments_size; ++i)
         {
             auto argument = function_node_optional->getArgumentAt(i);
-            analyzeImpl(res, argument, where_optimizer_context);
+            analyzeImpl(res, argument, where_optimizer_context, pk_positions);
         }
     }
     else
@@ -270,6 +270,7 @@ void MergeTreeWhereOptimizer::analyzeImpl(Conditions & res, const RPNBuilderTree
             cond.good = cond.viable;
             /// Find min position in PK of any column that is used in this condition.
             cond.min_position_in_primary_key = findMinPosition(cond.table_columns, primary_key_names_positions);
+            pk_positions.emplace(cond.min_position_in_primary_key);
         }
 
         res.emplace_back(std::move(cond));
@@ -281,7 +282,27 @@ MergeTreeWhereOptimizer::Conditions MergeTreeWhereOptimizer::analyze(const RPNBu
     const WhereOptimizerContext & where_optimizer_context) const
 {
     Conditions res;
-    analyzeImpl(res, node, where_optimizer_context);
+    std::set<Int64> pk_positions;
+    analyzeImpl(res, node, where_optimizer_context, pk_positions);
+    /// e.g. if primary kes are (a,b,c) but the condition is a = 1 and c = 1, we should only put (a = 1) to the last,
+    /// and treat (c = 1) as normal column.
+    if (where_optimizer_context.move_primary_key_columns_to_end_of_prewhere)
+    {
+        Int64 min_valid_pk_pos = -1;
+        for (auto pk_pos : pk_positions)
+        {
+            if (pk_pos != min_valid_pk_pos + 1)
+                break;
+            min_valid_pk_pos = pk_pos;
+        }
+        for (auto & cond : res)
+        {
+            if (cond.min_position_in_primary_key > min_valid_pk_pos)
+                cond.min_position_in_primary_key = std::numeric_limits<Int64>::max() - 1;
+        }
+        LOG_TRACE(log, "the min valid primary key position is {}", min_valid_pk_pos);
+    }
+
     return res;
 }
 
diff --git a/src/Storages/MergeTree/MergeTreeWhereOptimizer.h b/src/Storages/MergeTree/MergeTreeWhereOptimizer.h
index fb5e84b67c6..dd9dc803f35 100644
--- a/src/Storages/MergeTree/MergeTreeWhereOptimizer.h
+++ b/src/Storages/MergeTree/MergeTreeWhereOptimizer.h
@@ -108,7 +108,7 @@ private:
 
     std::optional<OptimizeResult> optimizeImpl(const RPNBuilderTreeNode & node, const WhereOptimizerContext & where_optimizer_context) const;
 
-    void analyzeImpl(Conditions & res, const RPNBuilderTreeNode & node, const WhereOptimizerContext & where_optimizer_context) const;
+    void analyzeImpl(Conditions & res, const RPNBuilderTreeNode & node, const WhereOptimizerContext & where_optimizer_context, std::set<Int64> & pk_positions) const;
 
     /// Transform conjunctions chain in WHERE expression to Conditions list.
     Conditions analyze(const RPNBuilderTreeNode & node, const WhereOptimizerContext & where_optimizer_context) const;
diff --git a/tests/queries/0_stateless/02842_move_pk_to_end_of_prewhere.reference b/tests/queries/0_stateless/02842_move_pk_to_end_of_prewhere.reference
new file mode 100644
index 00000000000..6e04d969e67
--- /dev/null
+++ b/tests/queries/0_stateless/02842_move_pk_to_end_of_prewhere.reference
@@ -0,0 +1,20 @@
+SELECT count()
+FROM t_02848_mt1
+PREWHERE notEmpty(v) AND (k = 3)
+1
+SELECT count()
+FROM t_02848_mt2
+PREWHERE (d LIKE \'%es%\') AND (c < 20) AND (b = \'3\') AND (a = 3)
+1
+SELECT count()
+FROM t_02848_mt2
+PREWHERE (d LIKE \'%es%\') AND (c < 20) AND (c > 0) AND (a = 3)
+1
+SELECT count()
+FROM t_02848_mt2
+PREWHERE (d LIKE \'%es%\') AND (b = \'3\') AND (c < 20)
+1
+SELECT count()
+FROM t_02848_mt2
+PREWHERE (d LIKE \'%es%\') AND (b = \'3\') AND (a = 3)
+1
diff --git a/tests/queries/0_stateless/02842_move_pk_to_end_of_prewhere.sql b/tests/queries/0_stateless/02842_move_pk_to_end_of_prewhere.sql
new file mode 100644
index 00000000000..bc9d7e5664e
--- /dev/null
+++ b/tests/queries/0_stateless/02842_move_pk_to_end_of_prewhere.sql
@@ -0,0 +1,34 @@
+SET optimize_move_to_prewhere = 1;
+SET enable_multiple_prewhere_read_steps = 1;
+
+DROP TABLE IF EXISTS t_02848_mt1;
+DROP TABLE IF EXISTS t_02848_mt2;
+
+CREATE TABLE t_02848_mt1 (k UInt32, v String) ENGINE = MergeTree ORDER BY k SETTINGS min_bytes_for_wide_part=0;
+
+INSERT INTO t_02848_mt1 SELECT number, toString(number) FROM numbers(100);
+
+EXPLAIN SYNTAX SELECT count() FROM t_02848_mt1 WHERE k = 3 AND notEmpty(v);
+SELECT count() FROM t_02848_mt1 WHERE k = 3 AND notEmpty(v);
+
+CREATE TABLE t_02848_mt2 (a UInt32, b String, c Int32, d String) ENGINE = MergeTree ORDER BY (a,b,c) SETTINGS min_bytes_for_wide_part=0;
+
+INSERT INTO t_02848_mt2 SELECT number, toString(number), number, 'aaaabbbbccccddddtestxxxyyy' FROM numbers(100);
+
+-- the estimated column sizes are: {a: 428, b: 318, c: 428, d: 73}
+-- it's not correct but let's fix it in the future.
+
+EXPLAIN SYNTAX SELECT count() FROM t_02848_mt2 WHERE a = 3 AND b == '3' AND c < 20 AND d like '%es%';
+SELECT count() FROM t_02848_mt2 WHERE a = 3 AND b == '3' AND c < 20 AND d like '%es%';
+
+EXPLAIN SYNTAX SELECT count() FROM t_02848_mt2 WHERE a = 3 AND c < 20 AND c > 0 AND d like '%es%';
+SELECT count() FROM t_02848_mt2 WHERE a = 3 AND c < 20 AND c > 0 AND d like '%es%';
+
+EXPLAIN SYNTAX SELECT count() FROM t_02848_mt2 WHERE  b == '3' AND c < 20 AND d like '%es%';
+SELECT count() FROM t_02848_mt2 WHERE b == '3' AND c < 20 AND d like '%es%';
+
+EXPLAIN SYNTAX SELECT count() FROM t_02848_mt2 WHERE a = 3 AND b == '3' AND d like '%es%';
+SELECT count() FROM t_02848_mt2 WHERE a = 3 AND b == '3' AND d like '%es%';
+
+DROP TABLE t_02848_mt1;
+DROP TABLE t_02848_mt2;

From 2245bde85d8696fa1926455bf42b86ab1f4e9bc2 Mon Sep 17 00:00:00 2001
From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com>
Date: Fri, 11 Aug 2023 17:53:06 +0200
Subject: [PATCH 225/249] Fix log message

---
 src/Interpreters/Cache/FileCache.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp
index c9bd39a0a2c..df75b31db11 100644
--- a/src/Interpreters/Cache/FileCache.cpp
+++ b/src/Interpreters/Cache/FileCache.cpp
@@ -1005,7 +1005,7 @@ void FileCache::loadMetadata()
                         log,
                         "Cache capacity changed (max size: {}, used: {}), "
                         "cached file `{}` does not fit in cache anymore (size: {})",
-                        main_priority->getSizeLimit(), main_priority->getSize(lock), key_directory.string(), size);
+                        main_priority->getSizeLimit(), main_priority->getSize(lock), offset_it->path(), size);
 
                     fs::remove(offset_it->path());
                 }

From 65fac88e7c6b5d2079b64d1c50baedc0566b0a25 Mon Sep 17 00:00:00 2001
From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com>
Date: Fri, 11 Aug 2023 19:34:43 +0200
Subject: [PATCH 226/249] Update FileCache.cpp

---
 src/Interpreters/Cache/FileCache.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp
index df75b31db11..662d6b3f554 100644
--- a/src/Interpreters/Cache/FileCache.cpp
+++ b/src/Interpreters/Cache/FileCache.cpp
@@ -1005,7 +1005,7 @@ void FileCache::loadMetadata()
                         log,
                         "Cache capacity changed (max size: {}, used: {}), "
                         "cached file `{}` does not fit in cache anymore (size: {})",
-                        main_priority->getSizeLimit(), main_priority->getSize(lock), offset_it->path(), size);
+                        main_priority->getSizeLimit(), main_priority->getSize(lock), offset_it->path().string(), size);
 
                     fs::remove(offset_it->path());
                 }

From ddb7fb2ce9371373d7be28d900430c63296b1f08 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 11 Aug 2023 23:56:23 +0300
Subject: [PATCH 227/249] Update tests/ci/functional_test_check.py

Co-authored-by: Mikhail f. Shiryaev <felixoid@clickhouse.com>
---
 tests/ci/functional_test_check.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py
index c84884bcfaf..378a98ddb54 100644
--- a/tests/ci/functional_test_check.py
+++ b/tests/ci/functional_test_check.py
@@ -390,8 +390,8 @@ def main():
     # Nevertheless, the credentials of the CI logs have limited scope
     # and does not provide access to sensitive info.
 
-    ci_logs_host = os.getenv("CLICKHOUSE_CI_LOGS_HOST")
-    ci_logs_password = os.getenv("CLICKHOUSE_CI_LOGS_PASSWORD")
+    ci_logs_host = os.getenv("CLICKHOUSE_CI_LOGS_HOST", "CLICKHOUSE_CI_LOGS_HOST")
+    ci_logs_password = os.getenv("CLICKHOUSE_CI_LOGS_PASSWORD", "CLICKHOUSE_CI_LOGS_PASSWORD")
     subprocess.check_call(
         f"sed -i -r -e 's!{ci_logs_host}!CLICKHOUSE_CI_LOGS_HOST!g; s!{ci_logs_password}!CLICKHOUSE_CI_LOGS_PASSWORD!g;' '{run_log_path}'",
         shell=True,

From 2b852e55b6f61ec292380f411bedd4d6527b4f0c Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Fri, 11 Aug 2023 21:13:53 +0000
Subject: [PATCH 228/249] Automatic style fix

---
 tests/ci/functional_test_check.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py
index 378a98ddb54..d06da94d0f0 100644
--- a/tests/ci/functional_test_check.py
+++ b/tests/ci/functional_test_check.py
@@ -391,7 +391,9 @@ def main():
     # and does not provide access to sensitive info.
 
     ci_logs_host = os.getenv("CLICKHOUSE_CI_LOGS_HOST", "CLICKHOUSE_CI_LOGS_HOST")
-    ci_logs_password = os.getenv("CLICKHOUSE_CI_LOGS_PASSWORD", "CLICKHOUSE_CI_LOGS_PASSWORD")
+    ci_logs_password = os.getenv(
+        "CLICKHOUSE_CI_LOGS_PASSWORD", "CLICKHOUSE_CI_LOGS_PASSWORD"
+    )
     subprocess.check_call(
         f"sed -i -r -e 's!{ci_logs_host}!CLICKHOUSE_CI_LOGS_HOST!g; s!{ci_logs_password}!CLICKHOUSE_CI_LOGS_PASSWORD!g;' '{run_log_path}'",
         shell=True,

From d25961f7f95bdfd8b26a580088c13654be64de7f Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 12 Aug 2023 00:52:09 +0200
Subject: [PATCH 229/249] Fix "Context has expired" error in dictionaries

---
 src/Functions/FunctionsExternalDictionaries.h |  2 +-
 .../02843_context_has_expired.reference       |  1 +
 .../0_stateless/02843_context_has_expired.sql | 23 +++++++++++++++++++
 3 files changed, 25 insertions(+), 1 deletion(-)
 create mode 100644 tests/queries/0_stateless/02843_context_has_expired.reference
 create mode 100644 tests/queries/0_stateless/02843_context_has_expired.sql

diff --git a/src/Functions/FunctionsExternalDictionaries.h b/src/Functions/FunctionsExternalDictionaries.h
index db6529da73c..2c824606396 100644
--- a/src/Functions/FunctionsExternalDictionaries.h
+++ b/src/Functions/FunctionsExternalDictionaries.h
@@ -65,7 +65,7 @@ namespace ErrorCodes
 class FunctionDictHelper : WithContext
 {
 public:
-    explicit FunctionDictHelper(ContextPtr context_) : WithContext(context_) {}
+    explicit FunctionDictHelper(ContextPtr context_) : WithContext(context_->getGlobalContext()) {}
 
     std::shared_ptr<const IDictionary> getDictionary(const String & dictionary_name)
     {
diff --git a/tests/queries/0_stateless/02843_context_has_expired.reference b/tests/queries/0_stateless/02843_context_has_expired.reference
new file mode 100644
index 00000000000..573541ac970
--- /dev/null
+++ b/tests/queries/0_stateless/02843_context_has_expired.reference
@@ -0,0 +1 @@
+0
diff --git a/tests/queries/0_stateless/02843_context_has_expired.sql b/tests/queries/0_stateless/02843_context_has_expired.sql
new file mode 100644
index 00000000000..ccef3458ed7
--- /dev/null
+++ b/tests/queries/0_stateless/02843_context_has_expired.sql
@@ -0,0 +1,23 @@
+DROP DICTIONARY IF EXISTS dict;
+DROP TABLE IF EXISTS source;
+
+CREATE TABLE source
+(
+  id UInt64,
+  value String
+)
+ENGINE=Memory;
+
+CREATE DICTIONARY dict
+(
+    id UInt64,
+    value String
+)
+PRIMARY KEY id
+SOURCE(CLICKHOUSE(TABLE 'source'))
+LAYOUT(DIRECT());
+
+SELECT 1 IN (SELECT dictGet('dict', 'value', materialize('1')));
+
+DROP DICTIONARY dict;
+DROP TABLE source;

From 96018399808b133f5adc6b477203d5613b2f625e Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 12 Aug 2023 01:04:08 +0200
Subject: [PATCH 230/249] Better

---
 src/Functions/FunctionsExternalDictionaries.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Functions/FunctionsExternalDictionaries.h b/src/Functions/FunctionsExternalDictionaries.h
index 2c824606396..1b4d2e95b09 100644
--- a/src/Functions/FunctionsExternalDictionaries.h
+++ b/src/Functions/FunctionsExternalDictionaries.h
@@ -65,7 +65,7 @@ namespace ErrorCodes
 class FunctionDictHelper : WithContext
 {
 public:
-    explicit FunctionDictHelper(ContextPtr context_) : WithContext(context_->getGlobalContext()) {}
+    explicit FunctionDictHelper(ContextPtr context_) : WithContext(context_->getSessionContext()) {}
 
     std::shared_ptr<const IDictionary> getDictionary(const String & dictionary_name)
     {

From 36e74b4b51682fd135906cb83e968d026d6788e1 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 12 Aug 2023 01:31:21 +0200
Subject: [PATCH 231/249] Might be better

---
 src/Functions/FunctionsExternalDictionaries.h | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/Functions/FunctionsExternalDictionaries.h b/src/Functions/FunctionsExternalDictionaries.h
index 1b4d2e95b09..1b2e2eb3bd6 100644
--- a/src/Functions/FunctionsExternalDictionaries.h
+++ b/src/Functions/FunctionsExternalDictionaries.h
@@ -62,14 +62,13 @@ namespace ErrorCodes
   */
 
 
-class FunctionDictHelper : WithContext
+class FunctionDictHelper
 {
 public:
-    explicit FunctionDictHelper(ContextPtr context_) : WithContext(context_->getSessionContext()) {}
+    explicit FunctionDictHelper(ContextPtr context_) : current_context(context_) {}
 
     std::shared_ptr<const IDictionary> getDictionary(const String & dictionary_name)
     {
-        auto current_context = getContext();
         auto dict = current_context->getExternalDictionariesLoader().getDictionary(dictionary_name, current_context);
 
         if (!access_checked)
@@ -132,10 +131,12 @@ public:
 
     DictionaryStructure getDictionaryStructure(const String & dictionary_name) const
     {
-        return getContext()->getExternalDictionariesLoader().getDictionaryStructure(dictionary_name, getContext());
+        return current_context->getExternalDictionariesLoader().getDictionaryStructure(dictionary_name, current_context);
     }
 
 private:
+    ContextPtr current_context;
+
     /// Access cannot be not granted, since in this case checkAccess() will throw and access_checked will not be updated.
     std::atomic<bool> access_checked = false;
 

From 930b3d3bffc304d3fb9ff031f078fb6ffb862840 Mon Sep 17 00:00:00 2001
From: Alexey Gerasimchuck <a.gerasimchuck@arenadata.io>
Date: Fri, 11 Aug 2023 23:48:17 +0000
Subject: [PATCH 232/249] Improved test

---
 .../test_peak_memory_usage/test.py            | 46 +++++++------------
 1 file changed, 17 insertions(+), 29 deletions(-)

diff --git a/tests/integration/test_peak_memory_usage/test.py b/tests/integration/test_peak_memory_usage/test.py
index 3994ee66a4a..a1313461482 100644
--- a/tests/integration/test_peak_memory_usage/test.py
+++ b/tests/integration/test_peak_memory_usage/test.py
@@ -37,17 +37,23 @@ def started_cluster():
             "ORDER BY number"
         )
 
+        shard_1.query(
+            "CREATE TABLE fixed_numbers_2 ON CLUSTER 'cluster' ("
+            "number UInt64"
+            ") ENGINE=Memory ()"
+        )
+
         shard_1.query(
             "CREATE TABLE distributed_fixed_numbers (number UInt64) ENGINE=Distributed('cluster', 'default', 'fixed_numbers')"
         )
+        shard_1.query("INSERT INTO fixed_numbers SELECT number FROM numbers(0, 100)")
 
-        # Shard 1 has  singificantly less data then shard 2
-        shard_1.query(
-            "INSERT INTO fixed_numbers SELECT number FROM numbers(1999900, 2000000)"
-        )
+        shard_2.query("INSERT INTO fixed_numbers SELECT number FROM numbers(100, 200)")
+
+        shard_1.query("INSERT INTO fixed_numbers_2 SELECT number FROM numbers(0, 10)")
 
         shard_2.query(
-            "INSERT INTO fixed_numbers SELECT number FROM numbers(0, 1999900)"
+            "INSERT INTO fixed_numbers_2 SELECT number FROM numbers(0, 120000)"
         )
 
         yield cluster
@@ -64,11 +70,13 @@ def get_memory_usage_from_client_output_and_close(client_output):
             peek_memory_usage_str_found = "Peak memory usage" in line
 
         if peek_memory_usage_str_found:
-            search_obj = re.search(r"([-+]?(?:\d*\.*\d+))", line)
+            search_obj = re.search(r"[+-]?[0-9]+\.[0-9]+", line)
             if search_obj:
                 client_output.close()
+                print(f"peak_memory_usage {search_obj.group()}")
                 return search_obj.group()
 
+    print(f"peak_memory_usage not found")
     client_output.close()
     return ""
 
@@ -81,29 +89,9 @@ def test_clickhouse_client_max_peak_memory_usage_distributed(started_cluster):
     with client(name="client1>", log=client_output, command=command_text) as client1:
         client1.expect(prompt)
         client1.send(
-            "SELECT COUNT(*) FROM distributed_fixed_numbers WHERE number IN (SELECT number from numbers(1999890, 1999910))"
+            "SELECT COUNT(*) FROM distributed_fixed_numbers JOIN fixed_numbers_2 ON distributed_fixed_numbers.number=fixed_numbers_2.number",
         )
-        client1.expect("Peak memory usage")
-        client1.expect(prompt)
-
-    peak_memory_usage = get_memory_usage_from_client_output_and_close(client_output)
-    assert peak_memory_usage
-    assert shard_2.contains_in_log(
-        f"Peak memory usage (for query): {peak_memory_usage}"
-    )
-
-
-def test_clickhouse_client_max_peak_memory_usage_cluster(started_cluster):
-    client_output = tempfile.TemporaryFile(mode="w+t")
-    command_text = (
-        f"{started_cluster.get_client_cmd()} --host {shard_1.ip_address} --port 9000"
-    )
-    with client(name="client1>", log=client_output, command=command_text) as client1:
-        client1.expect(prompt)
-        client1.send(
-            "SELECT COUNT(*) FROM (SELECT number FROM numbers(1,100000) INTERSECT SELECT * FROM clusterAllReplicas(cluster, default, fixed_numbers))"
-        )
-        client1.expect("Peak memory usage")
+        client1.expect("Peak memory usage", timeout=60)
         client1.expect(prompt)
 
     peak_memory_usage = get_memory_usage_from_client_output_and_close(client_output)
@@ -124,7 +112,7 @@ def test_clickhouse_client_max_peak_memory_single_node(started_cluster):
         client1.send(
             "SELECT COUNT(*) FROM (SELECT number FROM numbers(1,300000) INTERSECT SELECT number FROM numbers(10000,1200000))"
         )
-        client1.expect("Peak memory usage")
+        client1.expect("Peak memory usage", timeout=60)
         client1.expect(prompt)
 
     peak_memory_usage = get_memory_usage_from_client_output_and_close(client_output)

From aa815bfc74708bcd1c5cadc8ec2b619a814d2399 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 12 Aug 2023 01:54:11 +0200
Subject: [PATCH 233/249] Relax mergeable check

---
 tests/ci/ci_config.py | 90 -------------------------------------------
 1 file changed, 90 deletions(-)

diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py
index 979f7c35c2c..c30101d9f27 100644
--- a/tests/ci/ci_config.py
+++ b/tests/ci/ci_config.py
@@ -283,96 +283,6 @@ REQUIRED_CHECKS = [
     "Fast test",
     "Stateful tests (release)",
     "Stateless tests (release)",
-    "Stateless tests (debug) [1/5]",
-    "Stateless tests (debug) [2/5]",
-    "Stateless tests (debug) [3/5]",
-    "Stateless tests (debug) [4/5]",
-    "Stateless tests (debug) [5/5]",
-    "AST fuzzer (asan)",
-    "AST fuzzer (msan)",
-    "AST fuzzer (tsan)",
-    "AST fuzzer (ubsan)",
-    "AST fuzzer (debug)",
-    "Compatibility check (aarch64)",
-    "Compatibility check (amd64)",
-    "Install packages (amd64)",
-    "Install packages (arm64)",
-    "Integration tests (asan) [1/6]",
-    "Integration tests (asan) [2/6]",
-    "Integration tests (asan) [3/6]",
-    "Integration tests (asan) [4/6]",
-    "Integration tests (asan) [5/6]",
-    "Integration tests (asan) [6/6]",
-    "Integration tests (release) [1/4]",
-    "Integration tests (release) [2/4]",
-    "Integration tests (release) [3/4]",
-    "Integration tests (release) [4/4]",
-    "Integration tests (tsan) [1/6]",
-    "Integration tests (tsan) [2/6]",
-    "Integration tests (tsan) [3/6]",
-    "Integration tests (tsan) [4/6]",
-    "Integration tests (tsan) [5/6]",
-    "Integration tests (tsan) [6/6]",
-    "Integration tests flaky check (asan)",
-    "Stateful tests (aarch64)",
-    "Stateful tests (asan)",
-    "Stateful tests (asan, ParallelReplicas)",
-    "Stateful tests (debug)",
-    "Stateful tests (debug, ParallelReplicas)",
-    "Stateful tests (msan)",
-    "Stateful tests (msan, ParallelReplicas)",
-    "Stateful tests (release, ParallelReplicas)",
-    "Stateful tests (tsan)",
-    "Stateful tests (tsan, ParallelReplicas)",
-    "Stateful tests (ubsan)",
-    "Stateful tests (ubsan, ParallelReplicas)",
-    "Stateless tests (aarch64)",
-    "Stateless tests (asan) [1/4]",
-    "Stateless tests (asan) [2/4]",
-    "Stateless tests (asan) [3/4]",
-    "Stateless tests (asan) [4/4]",
-    "Stateless tests (debug) [1/5]",
-    "Stateless tests (debug) [2/5]",
-    "Stateless tests (debug) [3/5]",
-    "Stateless tests (debug) [4/5]",
-    "Stateless tests (debug) [5/5]",
-    "Stateless tests (debug, s3 storage) [1/6]",
-    "Stateless tests (debug, s3 storage) [2/6]",
-    "Stateless tests (debug, s3 storage) [3/6]",
-    "Stateless tests (debug, s3 storage) [4/6]",
-    "Stateless tests (debug, s3 storage) [5/6]",
-    "Stateless tests (debug, s3 storage) [6/6]",
-    "Stateless tests (msan) [1/6]",
-    "Stateless tests (msan) [2/6]",
-    "Stateless tests (msan) [3/6]",
-    "Stateless tests (msan) [4/6]",
-    "Stateless tests (msan) [5/6]",
-    "Stateless tests (msan) [6/6]",
-    "Stateless tests (release, DatabaseReplicated) [1/4]",
-    "Stateless tests (release, DatabaseReplicated) [2/4]",
-    "Stateless tests (release, DatabaseReplicated) [3/4]",
-    "Stateless tests (release, DatabaseReplicated) [4/4]",
-    "Stateless tests (release, s3 storage) [1/2]",
-    "Stateless tests (release, s3 storage) [2/2]",
-    "Stateless tests (release, wide parts enabled)",
-    "Stateless tests (tsan) [1/5]",
-    "Stateless tests (tsan) [2/5]",
-    "Stateless tests (tsan) [3/5]",
-    "Stateless tests (tsan) [4/5]",
-    "Stateless tests (tsan) [5/5]",
-    "Stateless tests (tsan, s3 storage) [1/5]",
-    "Stateless tests (tsan, s3 storage) [2/5]",
-    "Stateless tests (tsan, s3 storage) [3/5]",
-    "Stateless tests (tsan, s3 storage) [4/5]",
-    "Stateless tests (tsan, s3 storage) [5/5]",
-    "Stateless tests (ubsan) [1/2]",
-    "Stateless tests (ubsan) [2/2]",
-    "Stress test (asan)",
-    "Stress test (debug)",
-    "Stress test (msan)",
-    "Stress test (tsan)",
-    "Stress test (ubsan)",
-    "Upgrade check (asan)",
     "Style Check",
     "Unit tests (asan)",
     "Unit tests (msan)",

From 3b85c5ed0e2a3a66348936add80428e4893f1b8a Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 12 Aug 2023 02:14:58 +0200
Subject: [PATCH 234/249] Put the script into the base Docker image

---
 docker/test/base/Dockerfile                                    | 3 +++
 .../setup.sh => docker/test/base/setup_export_logs.sh          | 0
 docker/test/stateless/Dockerfile                               | 1 -
 docker/test/stateless/setup_export_logs.sh                     | 1 -
 4 files changed, 3 insertions(+), 2 deletions(-)
 rename utils/export-logs-in-ci/setup.sh => docker/test/base/setup_export_logs.sh (100%)
 delete mode 120000 docker/test/stateless/setup_export_logs.sh

diff --git a/docker/test/base/Dockerfile b/docker/test/base/Dockerfile
index 47edbbb14a5..b55baa0e0fc 100644
--- a/docker/test/base/Dockerfile
+++ b/docker/test/base/Dockerfile
@@ -35,4 +35,7 @@ ENV LC_ALL en_US.UTF-8
 ENV TZ=Europe/Amsterdam
 RUN ln -snf "/usr/share/zoneinfo/$TZ" /etc/localtime && echo "$TZ" > /etc/timezone
 
+# This script is used to setup realtime export of server logs from the CI into external ClickHouse cluster:
+COPY setup_export_logs.sh /
+
 CMD sleep 1
diff --git a/utils/export-logs-in-ci/setup.sh b/docker/test/base/setup_export_logs.sh
similarity index 100%
rename from utils/export-logs-in-ci/setup.sh
rename to docker/test/base/setup_export_logs.sh
diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile
index 48d26c84ed8..ab670209a69 100644
--- a/docker/test/stateless/Dockerfile
+++ b/docker/test/stateless/Dockerfile
@@ -87,6 +87,5 @@ RUN npm install -g azurite \
 COPY run.sh /
 COPY setup_minio.sh /
 COPY setup_hdfs_minicluster.sh /
-COPY setup_export_logs.sh /
 
 CMD ["/bin/bash", "/run.sh"]
diff --git a/docker/test/stateless/setup_export_logs.sh b/docker/test/stateless/setup_export_logs.sh
deleted file mode 120000
index b6e2f981cd4..00000000000
--- a/docker/test/stateless/setup_export_logs.sh
+++ /dev/null
@@ -1 +0,0 @@
-../../../utils/export-logs-in-ci/setup.sh
\ No newline at end of file

From 6176971b3f2eb58d85402fd8fccbdb6f1e36e293 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 12 Aug 2023 03:33:58 +0300
Subject: [PATCH 235/249] Update StoragePostgreSQL.cpp

---
 src/Storages/StoragePostgreSQL.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/Storages/StoragePostgreSQL.cpp b/src/Storages/StoragePostgreSQL.cpp
index f233d4ff213..7961c44e844 100644
--- a/src/Storages/StoragePostgreSQL.cpp
+++ b/src/Storages/StoragePostgreSQL.cpp
@@ -45,7 +45,6 @@
 
 #include <Databases/PostgreSQL/fetchPostgreSQLTableStructure.h>
 
-#include <boost/algorithm/string/replace.hpp>
 
 namespace DB
 {

From 3c2dd4e543a41cb25980740a1b629826f0970285 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 12 Aug 2023 03:40:19 +0300
Subject: [PATCH 236/249] Update MergeTreeWhereOptimizer.cpp

---
 src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp
index d73ee970683..6fbc8ffa759 100644
--- a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp
+++ b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp
@@ -284,8 +284,10 @@ MergeTreeWhereOptimizer::Conditions MergeTreeWhereOptimizer::analyze(const RPNBu
     Conditions res;
     std::set<Int64> pk_positions;
     analyzeImpl(res, node, where_optimizer_context, pk_positions);
-    /// e.g. if primary kes are (a,b,c) but the condition is a = 1 and c = 1, we should only put (a = 1) to the last,
-    /// and treat (c = 1) as normal column.
+    
+    /// E.g., if the primary key is (a, b, c) but the condition is a = 1 and c = 1,
+    /// we should only put (a = 1) to the tail of PREWHERE,
+    /// and treat (c = 1) as a normal column.
     if (where_optimizer_context.move_primary_key_columns_to_end_of_prewhere)
     {
         Int64 min_valid_pk_pos = -1;
@@ -300,7 +302,7 @@ MergeTreeWhereOptimizer::Conditions MergeTreeWhereOptimizer::analyze(const RPNBu
             if (cond.min_position_in_primary_key > min_valid_pk_pos)
                 cond.min_position_in_primary_key = std::numeric_limits<Int64>::max() - 1;
         }
-        LOG_TRACE(log, "the min valid primary key position is {}", min_valid_pk_pos);
+        LOG_TRACE(log, "The min valid primary key position for moving to the tail of PREWHERE is {}", min_valid_pk_pos);
     }
 
     return res;

From 0985a769534af8b76d41dca224a5b5f20dcd2bec Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 12 Aug 2023 02:44:13 +0200
Subject: [PATCH 237/249] Fix test

---
 tests/queries/0_stateless/01683_text_log_deadlock.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/01683_text_log_deadlock.sh b/tests/queries/0_stateless/01683_text_log_deadlock.sh
index debb7a0017f..af7f348a6a2 100755
--- a/tests/queries/0_stateless/01683_text_log_deadlock.sh
+++ b/tests/queries/0_stateless/01683_text_log_deadlock.sh
@@ -5,4 +5,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-$CLICKHOUSE_BENCHMARK --secure -i 25000 -c 32 --query 'SELECT 1' 2>&1 | grep -oF 'queries 25000'
+$CLICKHOUSE_BENCHMARK --secure -i 25000 -c 32 --query 'SELECT 1' 2>&1 | grep -oF 'queries: 25000'

From dee10b79accd575bb0b8af69cffa0ec95b16ad89 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 12 Aug 2023 02:44:20 +0200
Subject: [PATCH 238/249] Remove tests for old feature

---
 .../0_stateless/01306_benchmark_json.reference        |  1 -
 tests/queries/0_stateless/01306_benchmark_json.sh     | 11 -----------
 2 files changed, 12 deletions(-)
 delete mode 100644 tests/queries/0_stateless/01306_benchmark_json.reference
 delete mode 100755 tests/queries/0_stateless/01306_benchmark_json.sh

diff --git a/tests/queries/0_stateless/01306_benchmark_json.reference b/tests/queries/0_stateless/01306_benchmark_json.reference
deleted file mode 100644
index d00491fd7e5..00000000000
--- a/tests/queries/0_stateless/01306_benchmark_json.reference
+++ /dev/null
@@ -1 +0,0 @@
-1
diff --git a/tests/queries/0_stateless/01306_benchmark_json.sh b/tests/queries/0_stateless/01306_benchmark_json.sh
deleted file mode 100755
index ae361826250..00000000000
--- a/tests/queries/0_stateless/01306_benchmark_json.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/usr/bin/env bash
-# Tags: no-fasttest
-
-CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# shellcheck source=../shell_config.sh
-. "$CURDIR"/../shell_config.sh
-
-set -e
-
-$CLICKHOUSE_BENCHMARK --iterations 10 --json "${CLICKHOUSE_TMP}"/out.json <<< "SELECT 1" 2>/dev/null && cat "${CLICKHOUSE_TMP}"/out.json |
-    $CLICKHOUSE_LOCAL --input-format JSONAsString --structure "s String" --query "SELECT isValidJSON(s) FROM table"

From a1d3f1836e0dc02bf7d1156c694d70367a9617f3 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 12 Aug 2023 02:46:28 +0200
Subject: [PATCH 239/249] Fix shellcheck

---
 docker/test/base/setup_export_logs.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/test/base/setup_export_logs.sh b/docker/test/base/setup_export_logs.sh
index b7b8826c498..12fae855b03 100755
--- a/docker/test/base/setup_export_logs.sh
+++ b/docker/test/base/setup_export_logs.sh
@@ -18,7 +18,7 @@ CONNECTION_PARAMETERS=${CONNECTION_PARAMETERS:=""}
 clickhouse-client --query "SYSTEM FLUSH LOGS"
 
 # For each system log table:
-clickhouse-client --query "SHOW TABLES FROM system LIKE '%\\_log'" | while read table
+clickhouse-client --query "SHOW TABLES FROM system LIKE '%\\_log'" | while read -r table
 do
     # Calculate hash of its structure:
     hash=$(clickhouse-client --query "

From bcbd43ed5f372bc7b38d33a4d0472a36f0570ec9 Mon Sep 17 00:00:00 2001
From: Yakov Olkhovskiy <yakov@clickhouse.com>
Date: Sat, 12 Aug 2023 02:04:13 +0000
Subject: [PATCH 240/249] disable rust

---
 cmake/target.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/target.cmake b/cmake/target.cmake
index ffab08f1103..204a67d4357 100644
--- a/cmake/target.cmake
+++ b/cmake/target.cmake
@@ -47,7 +47,7 @@ if (CMAKE_CROSSCOMPILING)
             set (ENABLE_RUST OFF CACHE INTERNAL "")
         elseif (ARCH_S390X)
             set (ENABLE_GRPC OFF CACHE INTERNAL "")
-            set (ENABLE_SENTRY OFF CACHE INTERNAL "")
+            set (ENABLE_RUST OFF CACHE INTERNAL "")
         endif ()
     elseif (OS_FREEBSD)
         # FIXME: broken dependencies

From 13a03c046f9b14dafa4bd2cb3b5a0e5be0a6be64 Mon Sep 17 00:00:00 2001
From: Rich Raposa <richraposa@gmail.com>
Date: Fri, 11 Aug 2023 23:45:58 -0600
Subject: [PATCH 241/249] Remove duplicate section from Dictionary page

---
 docs/en/sql-reference/dictionaries/index.md | 49 ---------------------
 1 file changed, 49 deletions(-)

diff --git a/docs/en/sql-reference/dictionaries/index.md b/docs/en/sql-reference/dictionaries/index.md
index 6c3d80683db..c95ff5758f4 100644
--- a/docs/en/sql-reference/dictionaries/index.md
+++ b/docs/en/sql-reference/dictionaries/index.md
@@ -2476,52 +2476,3 @@ Dictionary updates (other than loading at first use) do not block queries. Durin
 We recommend periodically updating the dictionaries with the geobase. During an update, generate new files and write them to a separate location. When everything is ready, rename them to the files used by the server.
 
 There are also functions for working with OS identifiers and search engines, but they shouldn’t be used.
-
-## Embedded Dictionaries
-
-<SelfManaged />
-
-ClickHouse contains a built-in feature for working with a geobase.
-
-This allows you to:
-
-- Use a region’s ID to get its name in the desired language.
-- Use a region’s ID to get the ID of a city, area, federal district, country, or continent.
-- Check whether a region is part of another region.
-- Get a chain of parent regions.
-
-All the functions support “translocality,” the ability to simultaneously use different perspectives on region ownership. For more information, see the section “Functions for working with web analytics dictionaries”.
-
-The internal dictionaries are disabled in the default package.
-To enable them, uncomment the parameters `path_to_regions_hierarchy_file` and `path_to_regions_names_files` in the server configuration file.
-
-The geobase is loaded from text files.
-
-Place the `regions_hierarchy*.txt` files into the `path_to_regions_hierarchy_file` directory. This configuration parameter must contain the path to the `regions_hierarchy.txt` file (the default regional hierarchy), and the other files (`regions_hierarchy_ua.txt`) must be located in the same directory.
-
-Put the `regions_names_*.txt` files in the `path_to_regions_names_files` directory.
-
-You can also create these files yourself. The file format is as follows:
-
-`regions_hierarchy*.txt`: TabSeparated (no header), columns:
-
-- region ID (`UInt32`)
-- parent region ID (`UInt32`)
-- region type (`UInt8`): 1 - continent, 3 - country, 4 - federal district, 5 - region, 6 - city; other types do not have values
-- population (`UInt32`) — optional column
-
-`regions_names_*.txt`: TabSeparated (no header), columns:
-
-- region ID (`UInt32`)
-- region name (`String`) — Can’t contain tabs or line feeds, even escaped ones.
-
-A flat array is used for storing in RAM. For this reason, IDs shouldn’t be more than a million.
-
-Dictionaries can be updated without restarting the server. However, the set of available dictionaries is not updated.
-For updates, the file modification times are checked. If a file has changed, the dictionary is updated.
-The interval to check for changes is configured in the `builtin_dictionaries_reload_interval` parameter.
-Dictionary updates (other than loading at first use) do not block queries. During updates, queries use the old versions of dictionaries. If an error occurs during an update, the error is written to the server log, and queries continue using the old version of dictionaries.
-
-We recommend periodically updating the dictionaries with the geobase. During an update, generate new files and write them to a separate location. When everything is ready, rename them to the files used by the server.
-
-There are also functions for working with OS identifiers and search engines, but they shouldn’t be used.

From ec7301b0604349902fa941469504dddd04ff875e Mon Sep 17 00:00:00 2001
From: Amos Bird <amosbird@gmail.com>
Date: Sat, 12 Aug 2023 15:07:47 +0800
Subject: [PATCH 242/249] Fix incorrect normal projection AST format

---
 src/Parsers/ASTProjectionSelectQuery.cpp        |  4 ++--
 .../01710_normal_projection_format.reference    |  1 +
 .../01710_normal_projection_format.sql          | 17 +++++++++++++++++
 3 files changed, 20 insertions(+), 2 deletions(-)
 create mode 100644 tests/queries/0_stateless/01710_normal_projection_format.reference
 create mode 100644 tests/queries/0_stateless/01710_normal_projection_format.sql

diff --git a/src/Parsers/ASTProjectionSelectQuery.cpp b/src/Parsers/ASTProjectionSelectQuery.cpp
index 0cfdc3762a1..90d9ede7337 100644
--- a/src/Parsers/ASTProjectionSelectQuery.cpp
+++ b/src/Parsers/ASTProjectionSelectQuery.cpp
@@ -73,11 +73,11 @@ void ASTProjectionSelectQuery::formatImpl(const FormatSettings & s, FormatState
 
     if (orderBy())
     {
-        /// Let's convert the ASTFunction into ASTExpressionList, which generates consistent format
+        /// Let's convert tuple ASTFunction into ASTExpressionList, which generates consistent format
         /// between GROUP BY and ORDER BY projection definition.
         s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "ORDER BY " << (s.hilite ? hilite_none : "");
         ASTPtr order_by;
-        if (auto * func = orderBy()->as<ASTFunction>())
+        if (auto * func = orderBy()->as<ASTFunction>(); func && func->name == "tuple")
             order_by = func->arguments;
         else
         {
diff --git a/tests/queries/0_stateless/01710_normal_projection_format.reference b/tests/queries/0_stateless/01710_normal_projection_format.reference
new file mode 100644
index 00000000000..318c7253724
--- /dev/null
+++ b/tests/queries/0_stateless/01710_normal_projection_format.reference
@@ -0,0 +1 @@
+CREATE TABLE default.test\n(\n    `uuid` FixedString(16),\n    `id` Int32,\n    `ns` FixedString(16),\n    `dt` DateTime64(6),\n    PROJECTION mtlog_proj_source_reference\n    (\n        SELECT *\n        ORDER BY substring(ns, 1, 5)\n    )\n)\nENGINE = MergeTree\nORDER BY (id, dt, uuid)\nSETTINGS index_granularity = 8192
diff --git a/tests/queries/0_stateless/01710_normal_projection_format.sql b/tests/queries/0_stateless/01710_normal_projection_format.sql
new file mode 100644
index 00000000000..6210be166b3
--- /dev/null
+++ b/tests/queries/0_stateless/01710_normal_projection_format.sql
@@ -0,0 +1,17 @@
+DROP TABLE if exists test;
+
+CREATE TABLE test
+(
+    uuid FixedString(16),
+    id int,
+    ns FixedString(16),
+    dt DateTime64(6),
+)
+ENGINE = MergeTree
+ORDER BY (id, dt, uuid);
+
+ALTER TABLE test ADD PROJECTION mtlog_proj_source_reference (SELECT * ORDER BY substring(ns, 1, 5));
+
+SHOW CREATE test;
+
+drop table test;

From 0b3f9918a3b642d041471d03761eb0dcce952f53 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Sat, 12 Aug 2023 14:40:02 +0200
Subject: [PATCH 243/249] fix style

---
 src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp
index 6fbc8ffa759..466dbb45a4d 100644
--- a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp
+++ b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp
@@ -284,7 +284,7 @@ MergeTreeWhereOptimizer::Conditions MergeTreeWhereOptimizer::analyze(const RPNBu
     Conditions res;
     std::set<Int64> pk_positions;
     analyzeImpl(res, node, where_optimizer_context, pk_positions);
-    
+
     /// E.g., if the primary key is (a, b, c) but the condition is a = 1 and c = 1,
     /// we should only put (a = 1) to the tail of PREWHERE,
     /// and treat (c = 1) as a normal column.

From da4c0f3f910c638b67478af25153afb5d626a416 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 12 Aug 2023 16:33:49 +0200
Subject: [PATCH 244/249] Fix test

---
 tests/queries/0_stateless/01683_text_log_deadlock.reference | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/01683_text_log_deadlock.reference b/tests/queries/0_stateless/01683_text_log_deadlock.reference
index 76de47c4a46..4cf61460252 100644
--- a/tests/queries/0_stateless/01683_text_log_deadlock.reference
+++ b/tests/queries/0_stateless/01683_text_log_deadlock.reference
@@ -1 +1 @@
-queries 25000
+queries: 25000

From 38a391ef57cb89e2310ed9d79f72bcac538e1e00 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 12 Aug 2023 16:34:30 +0200
Subject: [PATCH 245/249] Fix tidy

---
 programs/benchmark/Benchmark.cpp | 2 --
 1 file changed, 2 deletions(-)

diff --git a/programs/benchmark/Benchmark.cpp b/programs/benchmark/Benchmark.cpp
index da4f01e4f4a..c5acd10f791 100644
--- a/programs/benchmark/Benchmark.cpp
+++ b/programs/benchmark/Benchmark.cpp
@@ -34,8 +34,6 @@
 #include <filesystem>
 
 
-namespace fs = std::filesystem;
-
 /** A tool for evaluating ClickHouse performance.
   * The tool emulates a case with fixed amount of simultaneously executing queries.
   */

From f314c29386c58ba789b769ae0b700d62c5f17fc7 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 12 Aug 2023 21:57:39 +0200
Subject: [PATCH 246/249] Export logs to CI from stateful tests

---
 docker/test/stateful/run.sh | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/docker/test/stateful/run.sh b/docker/test/stateful/run.sh
index c973b6c6ec6..300d613e3b0 100755
--- a/docker/test/stateful/run.sh
+++ b/docker/test/stateful/run.sh
@@ -20,6 +20,22 @@ ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test
 azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log &
 ./setup_minio.sh stateful
 
+# Setup a cluster for logs export to ClickHouse Cloud
+# Note: these variables are provided to the Docker run command by the Python script in tests/ci
+if [ -n "${CLICKHOUSE_CI_LOGS_HOST}" ]
+then
+    echo "
+    remote_servers:
+        system_logs_export:
+            shard:
+                replica:
+                    secure: 1
+                    user: ci
+                    host: '${CLICKHOUSE_CI_LOGS_HOST}'
+                    password: '${CLICKHOUSE_CI_LOGS_PASSWORD}'
+    " > /etc/clickhouse-server/config.d/system_logs_export.yaml
+fi
+
 function start()
 {
     if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
@@ -65,6 +81,22 @@ function start()
 }
 
 start
+
+# Initialize export of system logs to ClickHouse Cloud
+if [ -n "${CLICKHOUSE_CI_LOGS_HOST}" ]
+then
+    export EXTRA_COLUMNS_EXPRESSION="$PULL_REQUEST_NUMBER AS pull_request_number, '$COMMIT_SHA' AS commit_sha, '$CHECK_START_TIME' AS check_start_time, '$CHECK_NAME' AS check_name, '$INSTANCE_TYPE' AS instance_type"
+    # TODO: Check if the password will appear in the logs.
+    export CONNECTION_PARAMETERS="--secure --user ci --host ${CLICKHOUSE_CI_LOGS_HOST} --password ${CLICKHOUSE_CI_LOGS_PASSWORD}"
+
+    ./setup_export_logs.sh
+
+    # Unset variables after use
+    export CONNECTION_PARAMETERS=''
+    export CLICKHOUSE_CI_LOGS_HOST=''
+    export CLICKHOUSE_CI_LOGS_PASSWORD=''
+fi
+
 # shellcheck disable=SC2086 # No quotes because I want to split it into words.
 /s3downloader --url-prefix "$S3_URL" --dataset-names $DATASETS
 chmod 777 -R /var/lib/clickhouse

From 176c16fcc53fb9eca21d5fd70259f5ea87f1a04e Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 12 Aug 2023 22:30:18 +0200
Subject: [PATCH 247/249] Export logs from CI in stress tests

---
 docker/test/stress/run.sh | 31 ++++++++++++++++++++++
 tests/ci/stress_check.py  | 56 +++++++++++++++++++++++++++++++++++----
 2 files changed, 82 insertions(+), 5 deletions(-)

diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh
index a2264b8f3e6..073c94e7a9b 100644
--- a/docker/test/stress/run.sh
+++ b/docker/test/stress/run.sh
@@ -51,8 +51,39 @@ configure
 azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log &
 ./setup_minio.sh stateless # to have a proper environment
 
+# Setup a cluster for logs export to ClickHouse Cloud
+# Note: these variables are provided to the Docker run command by the Python script in tests/ci
+if [ -n "${CLICKHOUSE_CI_LOGS_HOST}" ]
+then
+    echo "
+    remote_servers:
+        system_logs_export:
+            shard:
+                replica:
+                    secure: 1
+                    user: ci
+                    host: '${CLICKHOUSE_CI_LOGS_HOST}'
+                    password: '${CLICKHOUSE_CI_LOGS_PASSWORD}'
+    " > /etc/clickhouse-server/config.d/system_logs_export.yaml
+fi
+
 start
 
+# Initialize export of system logs to ClickHouse Cloud
+if [ -n "${CLICKHOUSE_CI_LOGS_HOST}" ]
+then
+    export EXTRA_COLUMNS_EXPRESSION="$PULL_REQUEST_NUMBER AS pull_request_number, '$COMMIT_SHA' AS commit_sha, '$CHECK_START_TIME' AS check_start_time, '$CHECK_NAME' AS check_name, '$INSTANCE_TYPE' AS instance_type"
+    # TODO: Check if the password will appear in the logs.
+    export CONNECTION_PARAMETERS="--secure --user ci --host ${CLICKHOUSE_CI_LOGS_HOST} --password ${CLICKHOUSE_CI_LOGS_PASSWORD}"
+
+    ./setup_export_logs.sh
+
+    # Unset variables after use
+    export CONNECTION_PARAMETERS=''
+    export CLICKHOUSE_CI_LOGS_HOST=''
+    export CLICKHOUSE_CI_LOGS_PASSWORD=''
+fi
+
 # shellcheck disable=SC2086 # No quotes because I want to split it into words.
 /s3downloader --url-prefix "$S3_URL" --dataset-names $DATASETS
 chmod 777 -R /var/lib/clickhouse
diff --git a/tests/ci/stress_check.py b/tests/ci/stress_check.py
index 895eb318bc4..62e9b8a35d5 100644
--- a/tests/ci/stress_check.py
+++ b/tests/ci/stress_check.py
@@ -14,6 +14,7 @@ from build_download_helper import download_all_deb_packages
 from clickhouse_helper import (
     ClickHouseHelper,
     prepare_tests_results_for_clickhouse,
+    get_instance_type,
 )
 from commit_status_helper import RerunHelper, get_commit, post_commit_status
 from docker_pull_helper import get_image_with_version
@@ -28,12 +29,34 @@ from upload_result_helper import upload_results
 
 
 def get_run_command(
-    build_path, result_folder, repo_tests_path, server_log_folder, image
+    pr_info,
+    check_start_time,
+    check_name,
+    build_path,
+    result_folder,
+    repo_tests_path,
+    server_log_folder,
+    image
 ):
+    instance_type = get_instance_type()
+
+    envs = [
+        # a static link, don't use S3_URL or S3_DOWNLOAD
+        "-e S3_URL='https://s3.amazonaws.com/clickhouse-datasets'",
+        "-e CLICKHOUSE_CI_LOGS_HOST",
+        "-e CLICKHOUSE_CI_LOGS_PASSWORD",
+        f"-e PULL_REQUEST_NUMBER='{pr_info.number}'",
+        f"-e COMMIT_SHA='{pr_info.sha}'",
+        f"-e CHECK_START_TIME='{check_start_time}'",
+        f"-e CHECK_NAME='{check_name}'",
+        f"-e INSTANCE_TYPE='{instance_type}'",
+    ]
+
+    env_str = " ".join(envs)
+
     cmd = (
         "docker run --cap-add=SYS_PTRACE "
-        # a static link, don't use S3_URL or S3_DOWNLOAD
-        "-e S3_URL='https://s3.amazonaws.com/clickhouse-datasets' "
+        f"{env_str} "
         # For dmesg and sysctl
         "--privileged "
         f"--volume={build_path}:/package_folder "
@@ -149,9 +172,16 @@ def run_stress_test(docker_image_name):
     run_log_path = os.path.join(temp_path, "run.log")
 
     run_command = get_run_command(
-        packages_path, result_path, repo_tests_path, server_log_path, docker_image
+        pr_info,
+        stopwatch.start_time_str,
+        check_name,
+        packages_path,
+        result_path,
+        repo_tests_path,
+        server_log_path,
+        docker_image
     )
-    logging.info("Going to run func tests: %s", run_command)
+    logging.info("Going to run stress test: %s", run_command)
 
     with TeePopen(run_command, run_log_path, timeout=60 * 150) as process:
         retcode = process.wait()
@@ -168,6 +198,22 @@ def run_stress_test(docker_image_name):
     )
     ch_helper = ClickHouseHelper()
 
+    # Cleanup run log from the credentials of CI logs database.
+    # Note: a malicious user can still print them by splitting the value into parts.
+    # But we will be warned when a malicious user modifies CI script.
+    # Although they can also print them from inside tests.
+    # Nevertheless, the credentials of the CI logs have limited scope
+    # and does not provide access to sensitive info.
+
+    ci_logs_host = os.getenv("CLICKHOUSE_CI_LOGS_HOST", "CLICKHOUSE_CI_LOGS_HOST")
+    ci_logs_password = os.getenv(
+        "CLICKHOUSE_CI_LOGS_PASSWORD", "CLICKHOUSE_CI_LOGS_PASSWORD"
+    )
+    subprocess.check_call(
+        f"sed -i -r -e 's!{ci_logs_host}!CLICKHOUSE_CI_LOGS_HOST!g; s!{ci_logs_password}!CLICKHOUSE_CI_LOGS_PASSWORD!g;' '{run_log_path}'",
+        shell=True,
+    )
+
     report_url = upload_results(
         s3_helper,
         pr_info.number,

From e6e5a3858d79e93a30ce3985aae2995604119d2e Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Sat, 12 Aug 2023 20:43:29 +0000
Subject: [PATCH 248/249] Automatic style fix

---
 tests/ci/stress_check.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/ci/stress_check.py b/tests/ci/stress_check.py
index 62e9b8a35d5..42d372efb5d 100644
--- a/tests/ci/stress_check.py
+++ b/tests/ci/stress_check.py
@@ -36,7 +36,7 @@ def get_run_command(
     result_folder,
     repo_tests_path,
     server_log_folder,
-    image
+    image,
 ):
     instance_type = get_instance_type()
 
@@ -179,7 +179,7 @@ def run_stress_test(docker_image_name):
         result_path,
         repo_tests_path,
         server_log_path,
-        docker_image
+        docker_image,
     )
     logging.info("Going to run stress test: %s", run_command)
 

From a3ae62a1d1c8a939db5dc85fd0dbfc75c472abb1 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 13 Aug 2023 01:18:08 +0300
Subject: [PATCH 249/249] Fix YAML

---
 docker/test/stress/run.sh | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh
index 073c94e7a9b..547b3ac7204 100644
--- a/docker/test/stress/run.sh
+++ b/docker/test/stress/run.sh
@@ -56,15 +56,15 @@ azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log &
 if [ -n "${CLICKHOUSE_CI_LOGS_HOST}" ]
 then
     echo "
-    remote_servers:
-        system_logs_export:
-            shard:
-                replica:
-                    secure: 1
-                    user: ci
-                    host: '${CLICKHOUSE_CI_LOGS_HOST}'
-                    password: '${CLICKHOUSE_CI_LOGS_PASSWORD}'
-    " > /etc/clickhouse-server/config.d/system_logs_export.yaml
+remote_servers:
+    system_logs_export:
+        shard:
+            replica:
+                secure: 1
+                user: ci
+                host: '${CLICKHOUSE_CI_LOGS_HOST}'
+                password: '${CLICKHOUSE_CI_LOGS_PASSWORD}'
+" > /etc/clickhouse-server/config.d/system_logs_export.yaml
 fi
 
 start