Merge branch 'master' into analyzer-fix-test_wrong_db_or_table_name

docker/keeper/Dockerfile

@@ -34,7 +34,7 @@ RUN arch=${TARGETARCH:-amd64} \
 # lts / testing / prestable / etc
 ARG REPO_CHANNEL="stable"
 ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
-ARG VERSION="24.1.3.31"
+ARG VERSION="24.1.4.20"
 ARG PACKAGES="clickhouse-keeper"
 ARG DIRECT_DOWNLOAD_URLS=""
 

docker/server/Dockerfile.alpine

@@ -32,7 +32,7 @@ RUN arch=${TARGETARCH:-amd64} \
 # lts / testing / prestable / etc
 ARG REPO_CHANNEL="stable"
 ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
-ARG VERSION="24.1.3.31"
+ARG VERSION="24.1.4.20"
 ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
 ARG DIRECT_DOWNLOAD_URLS=""
 

docker/server/Dockerfile.ubuntu

@@ -27,7 +27,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list
 
 ARG REPO_CHANNEL="stable"
 ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main"
-ARG VERSION="24.1.3.31"
+ARG VERSION="24.1.4.20"
 ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
 
 # set non-empty deb_location_url url to create a docker image

docker/test/integration/runner/Dockerfile

@@ -62,46 +62,47 @@ RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
 # kazoo 2.10.0 is broken
 # https://s3.amazonaws.com/clickhouse-test-reports/59337/524625a1d2f4cc608a3f1059e3df2c30f353a649/integration_tests__asan__analyzer__[5_6].html
 RUN python3 -m pip install --no-cache-dir \
-    PyMySQL \
     aerospike==11.1.0 \
-    asyncio \
+    PyMySQL==1.1.0 \
+    asyncio==3.4.3 \
     avro==1.10.2 \
-    azure-storage-blob \
-    boto3 \
-    cassandra-driver \
+    azure-storage-blob==12.19.0 \
+    boto3==1.34.24 \
+    cassandra-driver==3.29.0 \
     confluent-kafka==2.3.0 \
     delta-spark==2.3.0 \
-    dict2xml \
-    dicttoxml \
+    dict2xml==1.7.4 \
+    dicttoxml==1.7.16 \
     docker==6.1.3 \
     docker-compose==1.29.2 \
-    grpcio \
-    grpcio-tools \
-    kafka-python \
+    grpcio==1.60.0 \
+    grpcio-tools==1.60.0 \
+    kafka-python==2.0.2 \
+    lz4==4.3.3 \
+    minio==7.2.3 \
+    nats-py==2.6.0 \
+    protobuf==4.25.2 \
     kazoo==2.9.0 \
-    lz4 \
-    minio \
-    nats-py \
-    protobuf \
     psycopg2-binary==2.9.6 \
-    pyhdfs \
+    pyhdfs==0.3.1 \
     pymongo==3.11.0 \
     pyspark==3.3.2 \
-    pytest \
+    pytest==7.4.4 \
     pytest-order==1.0.0 \
-    pytest-random \
-    pytest-repeat \
-    pytest-timeout \
-    pytest-xdist \
-    pytz \
+    pytest-random==0.2 \
+    pytest-repeat==0.9.3 \
+    pytest-timeout==2.2.0 \
+    pytest-xdist==3.5.0 \
+    pytest-reportlog==0.4.0 \
+    pytz==2023.3.post1 \
     pyyaml==5.3.1 \
-    redis \
-    requests-kerberos \
+    redis==5.0.1 \
+    requests-kerberos==0.14.0 \
     tzlocal==2.1 \
-    retry \
-    bs4 \
-    lxml \
-    urllib3
+    retry==0.9.2 \
+    bs4==0.0.2 \
+    lxml==5.1.0 \
+    urllib3==2.0.7
 # bs4, lxml are for cloud tests, do not delete
 
 # Hudi supports only spark 3.3.*, not 3.4

docs/changelogs/v24.1.4.20-stable.md

@@ -0,0 +1,28 @@
+---
+sidebar_position: 1
+sidebar_label: 2024
+---
+
+# 2024 Changelog
+
+### ClickHouse release v24.1.4.20-stable (f59d842b3fa) FIXME as compared to v24.1.3.31-stable (135b08cbd28)
+
+#### Improvement
+* Backported in [#59826](https://github.com/ClickHouse/ClickHouse/issues/59826): In case when `merge_max_block_size_bytes` is small enough and tables contain wide rows (strings or tuples) background merges may stuck in an endless loop. This behaviour is fixed. Follow-up for https://github.com/ClickHouse/ClickHouse/pull/59340. [#59812](https://github.com/ClickHouse/ClickHouse/pull/59812) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+
+#### Build/Testing/Packaging Improvement
+* Backported in [#59885](https://github.com/ClickHouse/ClickHouse/issues/59885): If you want to run initdb scripts every time when ClickHouse container is starting you shoud initialize environment varible CLICKHOUSE_ALWAYS_RUN_INITDB_SCRIPTS. [#59808](https://github.com/ClickHouse/ClickHouse/pull/59808) ([Alexander Nikolaev](https://github.com/AlexNik)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+
+* Fix digest calculation in Keeper [#59439](https://github.com/ClickHouse/ClickHouse/pull/59439) ([Antonio Andelic](https://github.com/antonio2368)).
+* Fix distributed table with a constant sharding key [#59606](https://github.com/ClickHouse/ClickHouse/pull/59606) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Fix query start time on non initial queries [#59662](https://github.com/ClickHouse/ClickHouse/pull/59662) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix parsing of partition expressions surrounded by parens [#59901](https://github.com/ClickHouse/ClickHouse/pull/59901) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Temporarily remove a feature that doesn't work [#59688](https://github.com/ClickHouse/ClickHouse/pull/59688) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Make ZooKeeper actually sequentialy consistent [#59735](https://github.com/ClickHouse/ClickHouse/pull/59735) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Fix special build reports in release branches [#59797](https://github.com/ClickHouse/ClickHouse/pull/59797) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+

docs/en/interfaces/formats.md

@@ -253,7 +253,7 @@ This format is also available under the name `TSVRawWithNamesAndNames`.
 
 This format allows specifying a custom format string with placeholders for values with a specified escaping rule.
 
-It uses settings `format_template_resultset`, `format_template_row`, `format_template_rows_between_delimiter` and some settings of other formats (e.g. `output_format_json_quote_64bit_integers` when using `JSON` escaping, see further)
+It uses settings `format_template_resultset`, `format_template_row` (`format_template_row_format`), `format_template_rows_between_delimiter` and some settings of other formats (e.g. `output_format_json_quote_64bit_integers` when using `JSON` escaping, see further)
 
 Setting `format_template_row` specifies the path to the file containing format strings for rows with the following syntax:
 
@@ -279,9 +279,11 @@ the values of `SearchPhrase`, `c` and `price` columns, which are escaped as `Quo
 
 `Search phrase: 'bathroom interior design', count: 2166, ad price: $3;`
 
+In cases where it is challenging or not possible to deploy format output configuration for the template format to a directory on all nodes in a cluster, or if the format is trivial then `format_template_row_format` can be used to set the template string directly in the query, rather than a path to the file which contains it.
+
 The `format_template_rows_between_delimiter` setting specifies the delimiter between rows, which is printed (or expected) after every row except the last one (`\n` by default)
 
-Setting `format_template_resultset` specifies the path to the file, which contains a format string for resultset. Format string for resultset has the same syntax as a format string for row and allows to specify a prefix, a suffix and a way to print some additional information. It contains the following placeholders instead of column names:
+Setting `format_template_resultset` specifies the path to the file, which contains a format string for resultset. Setting `format_template_resultset_format` can be used to set the template string for the result set directly in the query itself. Format string for resultset has the same syntax as a format string for row and allows to specify a prefix, a suffix and a way to print some additional information. It contains the following placeholders instead of column names:
 
 - `data` is the rows with data in `format_template_row` format, separated by `format_template_rows_between_delimiter`. This placeholder must be the first placeholder in the format string.
 - `totals` is the row with total values in `format_template_row` format (when using WITH TOTALS)

docs/en/operations/settings/settings-formats.md

@@ -1662,6 +1662,10 @@ Result:
 
 Path to file which contains format string for result set (for Template format).
 
+### format_template_resultset_format {#format_template_resultset_format}
+
+Format string for result set (for Template format)
+
 ### format_template_row {#format_template_row}
 
 Path to file which contains format string for rows (for Template format).
@@ -1670,6 +1674,10 @@ Path to file which contains format string for rows (for Template format).
 
 Delimiter between rows (for Template format).
 
+### format_template_row_format {#format_template_row_format}
+
+Format string for rows (for Template format)
+
 ## CustomSeparated format settings {custom-separated-format-settings}
 
 ### format_custom_escaping_rule {#format_custom_escaping_rule}

docs/ru/interfaces/formats.md

@@ -201,7 +201,7 @@ SELECT * FROM nestedt FORMAT TSV
 
 Этот формат позволяет указать произвольную форматную строку, в которую подставляются значения, сериализованные выбранным способом.
 
-Для этого используются настройки `format_template_resultset`, `format_template_row`, `format_template_rows_between_delimiter` и настройки экранирования других форматов (например, `output_format_json_quote_64bit_integers` при экранировании как в `JSON`, см. далее)
+Для этого используются настройки `format_template_resultset`, `format_template_row` (`format_template_row_format`), `format_template_rows_between_delimiter` и настройки экранирования других форматов (например, `output_format_json_quote_64bit_integers` при экранировании как в `JSON`, см. далее)
 
 Настройка `format_template_row` задаёт путь к файлу, содержащему форматную строку для строк таблицы, которая должна иметь вид:
 
@@ -227,9 +227,11 @@ SELECT * FROM nestedt FORMAT TSV
 
     `Search phrase: 'bathroom interior design', count: 2166, ad price: $3;`
 
+В тех случаях, когда не удобно или не возможно указать произвольную форматную строку в файле, можно использовать `format_template_row_format` указать произвольную форматную строку в запросе.  
+
 Настройка `format_template_rows_between_delimiter` задаёт разделитель между строками, который выводится (или ожмдается при вводе) после каждой строки, кроме последней. По умолчанию `\n`.
 
-Настройка `format_template_resultset` задаёт путь к файлу, содержащему форматную строку для результата. Форматная строка для результата имеет синтаксис аналогичный форматной строке для строк таблицы и позволяет указать префикс, суффикс и способ вывода дополнительной информации. Вместо имён столбцов в ней указываются следующие имена подстановок:
+Настройка `format_template_resultset` задаёт путь к файлу, содержащему форматную строку для результата. Настройка `format_template_resultset_format` используется для установки форматной строки для результата непосредственно в запросе. Форматная строка для результата имеет синтаксис аналогичный форматной строке для строк таблицы и позволяет указать префикс, суффикс и способ вывода дополнительной информации. Вместо имён столбцов в ней указываются следующие имена подстановок:
 
 -   `data` - строки с данными в формате `format_template_row`, разделённые `format_template_rows_between_delimiter`. Эта подстановка должна быть первой подстановкой в форматной строке.
 -   `totals` - строка с тотальными значениями в формате `format_template_row` (при использовании WITH TOTALS)

programs/disks/CMakeLists.txt

@@ -11,6 +11,10 @@ set (CLICKHOUSE_DISKS_SOURCES
     CommandRemove.cpp
     CommandWrite.cpp)
 
+if (CLICKHOUSE_CLOUD)
+    set (CLICKHOUSE_DISKS_SOURCES ${CLICKHOUSE_DISKS_SOURCES} CommandPackedIO.cpp)
+endif ()
+
 set (CLICKHOUSE_DISKS_LINK
     PRIVATE
         boost::program_options

programs/disks/CommandRead.cpp

@@ -61,7 +61,6 @@ public:
             auto out = disk->writeFile(relative_path_output);
             copyData(*in, *out);
             out->finalize();
-            return;
         }
         else
         {

programs/disks/DisksApp.cpp

@@ -65,6 +65,9 @@ void DisksApp::addOptions(
     positional_options_description.add("command_name", 1);
 
     supported_commands = {"list-disks", "list", "move", "remove", "link", "copy", "write", "read", "mkdir"};
+#ifdef CLICKHOUSE_CLOUD
+    supported_commands.insert("packed-io");
+#endif
 
     command_descriptions.emplace("list-disks", makeCommandListDisks());
     command_descriptions.emplace("list", makeCommandList());
@@ -75,6 +78,9 @@ void DisksApp::addOptions(
     command_descriptions.emplace("write", makeCommandWrite());
     command_descriptions.emplace("read", makeCommandRead());
     command_descriptions.emplace("mkdir", makeCommandMkDir());
+#ifdef CLICKHOUSE_CLOUD
+    command_descriptions.emplace("packed-io", makeCommandPackedIO());
+#endif
 }
 
 void DisksApp::processOptions()
@@ -89,6 +95,11 @@ void DisksApp::processOptions()
         config().setString("log-level", options["log-level"].as<String>());
 }
 
+DisksApp::~DisksApp()
+{
+    global_context->shutdown();
+}
+
 void DisksApp::init(std::vector<String> & common_arguments)
 {
     stopOptionsProcessing();
@@ -134,6 +145,7 @@ void DisksApp::parseAndCheckOptions(
         .options(options_description_)
         .positional(positional_options_description)
         .allow_unregistered();
+
     po::parsed_options parsed = parser.run();
     po::store(parsed, options);
 
@@ -199,8 +211,8 @@ int DisksApp::main(const std::vector<String> & /*args*/)
         po::parsed_options parsed = parser.run();
         po::store(parsed, options);
         po::notify(options);
-        args = po::collect_unrecognized(parsed.options, po::collect_unrecognized_mode::include_positional);
 
+        args = po::collect_unrecognized(parsed.options, po::collect_unrecognized_mode::include_positional);
         command->processOptions(config(), options);
     }
     else

programs/disks/DisksApp.h

@@ -21,6 +21,7 @@ class DisksApp : public Poco::Util::Application, public Loggers
 {
 public:
     DisksApp() = default;
+    ~DisksApp() override;
 
     void init(std::vector<String> & common_arguments);
 
@@ -52,9 +53,9 @@ protected:
     std::vector<String> command_arguments;
 
     std::unordered_set<String> supported_commands;
-
     std::unordered_map<String, CommandPtr> command_descriptions;
 
     po::variables_map options;
 };
+
 }

programs/disks/ICommand.h

@@ -63,3 +63,4 @@ DB::CommandPtr makeCommandRead();
 DB::CommandPtr makeCommandRemove();
 DB::CommandPtr makeCommandWrite();
 DB::CommandPtr makeCommandMkDir();
+DB::CommandPtr makeCommandPackedIO();

programs/keeper-client/Commands.cpp

@@ -115,7 +115,7 @@ bool CreateCommand::parse(IParser::Pos & pos, std::shared_ptr<ASTKeeperQuery> &
     else if (ParserKeyword{"PERSISTENT SEQUENTIAL"}.ignore(pos, expected))
         mode = zkutil::CreateMode::PersistentSequential;
 
-    node->args.push_back(mode);
+    node->args.push_back(std::move(mode));
 
     return true;
 }

programs/server/config.xml

@@ -937,6 +937,11 @@
     </macros>
     -->
 
+    <!--
+    <default_replica_path>/clickhouse/tables/{database}/{table}</default_replica_path>
+    <default_replica_name>{replica}</default_replica_name>
+    -->
+
     <!-- Replica group name for database Replicated.
           The cluster created by Replicated database will consist of replicas in the same group.
           DDL queries will only wail for the replicas in the same group.

src/AggregateFunctions/UniqExactSet.h

@@ -156,7 +156,6 @@ public:
     void read(ReadBuffer & in)
     {
         size_t new_size = 0;
-        auto * const position = in.position();
         readVarUInt(new_size, in);
         if (new_size > 100'000'000'000)
             throw DB::Exception(
@@ -174,8 +173,14 @@ public:
         }
         else
         {
-            in.position() = position; // Rollback position
-            asSingleLevel().read(in);
+            asSingleLevel().reserve(new_size);
+
+            for (size_t i = 0; i < new_size; ++i)
+            {
+                typename SingleLevelSet::Cell x;
+                x.read(in);
+                asSingleLevel().insert(x.getValue());
+            }
         }
     }
 

src/Analyzer/Passes/QueryAnalysisPass.cpp

@@ -2171,21 +2171,45 @@ void QueryAnalyzer::replaceNodesWithPositionalArguments(QueryTreeNodePtr & node_
             node_to_replace = &sort_node->getExpression();
 
         auto * constant_node = (*node_to_replace)->as<ConstantNode>();
-        if (!constant_node || constant_node->getValue().getType() != Field::Types::UInt64)
+
+        if (!constant_node
+            || (constant_node->getValue().getType() != Field::Types::UInt64 && constant_node->getValue().getType() != Field::Types::Int64))
             continue;
 
-        UInt64 positional_argument_number = constant_node->getValue().get<UInt64>();
-        if (positional_argument_number == 0 || positional_argument_number > projection_nodes.size())
-            throw Exception(ErrorCodes::BAD_ARGUMENTS,
+        UInt64 pos;
+        if (constant_node->getValue().getType() == Field::Types::UInt64)
+        {
+            pos = constant_node->getValue().get<UInt64>();
+        }
+        else // Int64
+        {
+            auto value = constant_node->getValue().get<Int64>();
+            if (value > 0)
+                pos = value;
+            else
+            {
+                if (static_cast<size_t>(std::abs(value)) > projection_nodes.size())
+                    throw Exception(
+                        ErrorCodes::BAD_ARGUMENTS,
+                        "Negative positional argument number {} is out of bounds. Expected in range [-{}, -1]. In scope {}",
+                        value,
+                        projection_nodes.size(),
+                        scope.scope_node->formatASTForErrorMessage());
+                pos = projection_nodes.size() + value + 1;
+            }
+        }
+
+        if (!pos || pos > projection_nodes.size())
+            throw Exception(
+                ErrorCodes::BAD_ARGUMENTS,
                 "Positional argument number {} is out of bounds. Expected in range [1, {}]. In scope {}",
-                positional_argument_number,
+                pos,
                 projection_nodes.size(),
                 scope.scope_node->formatASTForErrorMessage());
 
-        --positional_argument_number;
-        *node_to_replace = projection_nodes[positional_argument_number]->clone();
-        if (auto it = resolved_expressions.find(projection_nodes[positional_argument_number]);
-            it != resolved_expressions.end())
+        --pos;
+        *node_to_replace = projection_nodes[pos]->clone();
+        if (auto it = resolved_expressions.find(projection_nodes[pos]); it != resolved_expressions.end())
         {
             resolved_expressions[*node_to_replace] = it->second;
         }

src/Backups/DDLAdjustingForBackupVisitor.cpp

@@ -57,9 +57,9 @@ namespace
                 if (size_t uuid_pos = zookeeper_path_arg.find(table_uuid_str); uuid_pos != String::npos)
                     zookeeper_path_arg.replace(uuid_pos, table_uuid_str.size(), "{uuid}");
             }
-            const auto & config = data.global_context->getConfigRef();
-            if ((zookeeper_path_arg == StorageReplicatedMergeTree::getDefaultZooKeeperPath(config))
-                && (replica_name_arg == StorageReplicatedMergeTree::getDefaultReplicaName(config))
+            const auto & server_settings = data.global_context->getServerSettings();
+            if ((zookeeper_path_arg == server_settings.default_replica_path.value)
+                && (replica_name_arg == server_settings.default_replica_name.value)
                 && ((engine_args.size() == 2) || !engine_args[2]->as<ASTLiteral>()))
             {
                 engine_args.erase(engine_args.begin(), engine_args.begin() + 2);

src/Common/Macros.cpp

@@ -175,15 +175,6 @@ String Macros::expand(const String & s) const
     return expand(s, info);
 }
 
-String Macros::expand(const String & s, const StorageID & table_id, bool allow_uuid) const
-{
-    MacroExpansionInfo info;
-    info.table_id = table_id;
-    if (!allow_uuid)
-        info.table_id.uuid = UUIDHelpers::Nil;
-    return expand(s, info);
-}
-
 Names Macros::expand(const Names & source_names, size_t level) const
 {
     Names result_names;

src/Common/Macros.h

@@ -57,8 +57,6 @@ public:
 
     String expand(const String & s) const;
 
-    String expand(const String & s, const StorageID & table_id, bool allow_uuid) const;
-
 
     /** Apply expand for the list.
       */

src/Common/ProfileEvents.cpp

@@ -8,6 +8,7 @@
     M(Query, "Number of queries to be interpreted and potentially executed. Does not include queries that failed to parse or were rejected due to AST size limits, quota limits or limits on the number of simultaneously running queries. May include internal queries initiated by ClickHouse itself. Does not count subqueries.") \
     M(SelectQuery, "Same as Query, but only for SELECT queries.") \
     M(InsertQuery, "Same as Query, but only for INSERT queries.") \
+    M(InitialQuery, "Same as Query, but only counts initial queries (see is_initial_query).")\
     M(QueriesWithSubqueries, "Count queries with all subqueries") \
     M(SelectQueriesWithSubqueries, "Count SELECT queries with all subqueries") \
     M(InsertQueriesWithSubqueries, "Count INSERT queries with all subqueries") \

src/Core/ServerSettings.h

@@ -115,6 +115,8 @@ namespace DB
     M(Bool, storage_metadata_write_full_object_key, false, "Write disk metadata files with VERSION_FULL_OBJECT_KEY format", 0) \
     M(UInt64, max_materialized_views_count_for_table, 0, "A limit on the number of materialized views attached to a table.", 0) \
     M(UInt32, max_database_replicated_create_table_thread_pool_size, 1, "The number of threads to create tables during replica recovery in DatabaseReplicated. Zero means number of threads equal number of cores.", 0) \
+    M(String, default_replica_path, "/clickhouse/tables/{uuid}/{shard}", "The path to the table in ZooKeeper", 0) \
+    M(String, default_replica_name, "{replica}", "The replica name in ZooKeeper", 0) \
 
     /// If you add a setting which can be updated at runtime, please update 'changeable_settings' map in StorageSystemServerSettings.cpp
 

src/Core/ServerUUID.cpp

@@ -14,6 +14,11 @@ namespace ErrorCodes
 }
 
 void ServerUUID::load(const fs::path & server_uuid_file, Poco::Logger * log)
+{
+    server_uuid = loadServerUUID(server_uuid_file, log);
+}
+
+UUID loadServerUUID(const fs::path & server_uuid_file, Poco::Logger * log)
 {
     /// Write a uuid file containing a unique uuid if the file doesn't already exist during server start.
 
@@ -25,8 +30,7 @@ void ServerUUID::load(const fs::path & server_uuid_file, Poco::Logger * log)
             ReadBufferFromFile in(server_uuid_file);
             readUUIDText(uuid, in);
             assertEOF(in);
-            server_uuid = uuid;
-            return;
+            return uuid;
         }
         catch (...)
         {
@@ -44,7 +48,7 @@ void ServerUUID::load(const fs::path & server_uuid_file, Poco::Logger * log)
         out.write(uuid_str.data(), uuid_str.size());
         out.sync();
         out.finalize();
-        server_uuid = new_uuid;
+        return new_uuid;
     }
     catch (...)
     {

src/Core/ServerUUID.h

@@ -21,4 +21,6 @@ public:
     static void load(const fs::path & server_uuid_file, Poco::Logger * log);
 };
 
+UUID loadServerUUID(const fs::path & server_uuid_file, Poco::Logger * log);
+
 }

src/Core/Settings.h

@@ -774,7 +774,7 @@ class IColumn;
     M(Bool, load_marks_asynchronously, false, "Load MergeTree marks asynchronously", 0) \
     M(Bool, enable_filesystem_read_prefetches_log, false, "Log to system.filesystem prefetch_log during query. Should be used only for testing or debugging, not recommended to be turned on by default", 0) \
     M(Bool, allow_prefetched_read_pool_for_remote_filesystem, true, "Prefer prefetched threadpool if all parts are on remote filesystem", 0) \
-    M(Bool, allow_prefetched_read_pool_for_local_filesystem, false, "Prefer prefetched threadpool if all parts are on remote filesystem", 0) \
+    M(Bool, allow_prefetched_read_pool_for_local_filesystem, false, "Prefer prefetched threadpool if all parts are on local filesystem", 0) \
     \
     M(UInt64, prefetch_buffer_size, DBMS_DEFAULT_BUFFER_SIZE, "The maximum size of the prefetch buffer to read from the filesystem.", 0) \
     M(UInt64, filesystem_prefetch_step_bytes, 0, "Prefetch step in bytes. Zero means `auto` - approximately the best prefetch step will be auto deduced, but might not be 100% the best. The actual value might be different because of setting filesystem_prefetch_min_bytes_for_single_read_task", 0) \
@@ -858,7 +858,7 @@ class IColumn;
     M(UInt64, grace_hash_join_max_buckets, 1024, "Limit on the number of grace hash join buckets", 0) \
     M(Bool, optimize_distinct_in_order, true, "Enable DISTINCT optimization if some columns in DISTINCT form a prefix of sorting. For example, prefix of sorting key in merge tree or ORDER BY statement", 0) \
     M(Bool, keeper_map_strict_mode, false, "Enforce additional checks during operations on KeeperMap. E.g. throw an exception on an insert for already existing key", 0) \
-    M(UInt64, extract_kvp_max_pairs_per_row, 1000, "Max number pairs that can be produced by extractKeyValuePairs function. Used to safeguard against consuming too much memory.", 0) \
+    M(UInt64, extract_key_value_pairs_max_pairs_per_row, 1000, "Max number of pairs that can be produced by the `extractKeyValuePairs` function. Used as a safeguard against consuming too much memory.", 0) ALIAS(extract_kvp_max_pairs_per_row) \
     M(Timezone, session_timezone, "", "This setting can be removed in the future due to potential caveats. It is experimental and is not suitable for production usage. The default timezone for current session or query. The server default timezone if empty.", 0) \
     M(Bool, allow_create_index_without_type, false, "Allow CREATE INDEX query without TYPE. Query will be ignored. Made for SQL compatibility tests.", 0) \
     M(Bool, create_index_ignore_unique, false, "Ignore UNIQUE keyword in CREATE UNIQUE INDEX. Made for SQL compatibility tests.", 0) \
@@ -1093,6 +1093,8 @@ class IColumn;
     M(String, format_schema, "", "Schema identifier (used by schema-based formats)", 0) \
     M(String, format_template_resultset, "", "Path to file which contains format string for result set (for Template format)", 0) \
     M(String, format_template_row, "", "Path to file which contains format string for rows (for Template format)", 0) \
+    M(String, format_template_row_format, "", "Format string for rows (for Template format)", 0) \
+    M(String, format_template_resultset_format, "", "Format string for result set (for Template format)", 0) \
     M(String, format_template_rows_between_delimiter, "\n", "Delimiter between rows (for Template format)", 0) \
     \
     M(EscapingRule, format_custom_escaping_rule, "Escaped", "Field escaping rule (for CustomSeparated format)", 0) \

src/Core/SettingsChangesHistory.h

@@ -91,9 +91,12 @@ static std::map<ClickHouseVersion, SettingsChangesHistory::SettingsChanges> sett
               {"async_insert_busy_timeout_max_ms", 200, 200, "The minimum value of the asynchronous insert timeout in milliseconds; async_insert_busy_timeout_ms is aliased to async_insert_busy_timeout_max_ms"},
               {"async_insert_busy_timeout_increase_rate", 0.2, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout increases"},
               {"async_insert_busy_timeout_decrease_rate", 0.2, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout decreases"},
+              {"format_template_row_format", "", "", "Template row format string can be set directly in query"},
+              {"format_template_resultset_format", "", "", "Template result set format string can be set in query"},
               {"split_parts_ranges_into_intersecting_and_non_intersecting_final", true, true, "Allow to split parts ranges into intersecting and non intersecting during FINAL optimization"},
               {"split_intersecting_parts_ranges_into_layers_final", true, true, "Allow to split intersecting parts ranges into layers during FINAL optimization"},
-              {"azure_max_single_part_copy_size", 256*1024*1024, 256*1024*1024, "The maximum size of object to copy using single part copy to Azure blob storage."}}},
+              {"azure_max_single_part_copy_size", 256*1024*1024, 256*1024*1024, "The maximum size of object to copy using single part copy to Azure blob storage."},
+              {"extract_key_value_pairs_max_pairs_per_row", 0, 0, "Max number of pairs that can be produced by the `extractKeyValuePairs` function. Used as a safeguard against consuming too much memory."}}},
     {"24.1", {{"print_pretty_type_names", false, true, "Better user experience."},
               {"input_format_json_read_bools_as_strings", false, true, "Allow to read bools as strings in JSON formats by default"},
               {"output_format_arrow_use_signed_indexes_for_dictionary", false, true, "Use signed indexes type for Arrow dictionaries by default as it's recommended"},

src/DataTypes/DataTypeLowCardinalityHelpers.cpp

@@ -20,6 +20,7 @@ namespace ErrorCodes
 {
     extern const int ILLEGAL_COLUMN;
     extern const int TYPE_MISMATCH;
+    extern const int LOGICAL_ERROR;
 }
 
 DataTypePtr recursiveRemoveLowCardinality(const DataTypePtr & type)
@@ -55,62 +56,61 @@ DataTypePtr recursiveRemoveLowCardinality(const DataTypePtr & type)
 
 ColumnPtr recursiveRemoveLowCardinality(const ColumnPtr & column)
 {
-    if (!column)
-        return column;
+    ColumnPtr res = column;
 
     if (const auto * column_array = typeid_cast<const ColumnArray *>(column.get()))
     {
         const auto & data = column_array->getDataPtr();
         auto data_no_lc = recursiveRemoveLowCardinality(data);
-        if (data.get() == data_no_lc.get())
-            return column;
-
-        return ColumnArray::create(data_no_lc, column_array->getOffsetsPtr());
+        if (data.get() != data_no_lc.get())
+            res = ColumnArray::create(data_no_lc, column_array->getOffsetsPtr());
     }
-
-    if (const auto * column_const = typeid_cast<const ColumnConst *>(column.get()))
+    else if (const auto * column_const = typeid_cast<const ColumnConst *>(column.get()))
     {
         const auto & nested = column_const->getDataColumnPtr();
         auto nested_no_lc = recursiveRemoveLowCardinality(nested);
-        if (nested.get() == nested_no_lc.get())
-            return column;
-
-        return ColumnConst::create(nested_no_lc, column_const->size());
+        if (nested.get() != nested_no_lc.get())
+            res = ColumnConst::create(nested_no_lc, column_const->size());
     }
-
-    if (const auto * column_tuple = typeid_cast<const ColumnTuple *>(column.get()))
+    else if (const auto * column_tuple = typeid_cast<const ColumnTuple *>(column.get()))
     {
         auto columns = column_tuple->getColumns();
         for (auto & element : columns)
             element = recursiveRemoveLowCardinality(element);
-        return ColumnTuple::create(columns);
+        res = ColumnTuple::create(columns);
     }
-
-    if (const auto * column_map = typeid_cast<const ColumnMap *>(column.get()))
+    else if (const auto * column_map = typeid_cast<const ColumnMap *>(column.get()))
     {
         const auto & nested = column_map->getNestedColumnPtr();
         auto nested_no_lc = recursiveRemoveLowCardinality(nested);
-        if (nested.get() == nested_no_lc.get())
-            return column;
-
-        return ColumnMap::create(nested_no_lc);
+        if (nested.get() != nested_no_lc.get())
+            res = ColumnMap::create(nested_no_lc);
     }
-
     /// Special case when column is a lazy argument of short circuit function.
     /// We should call recursiveRemoveLowCardinality on the result column
     /// when function will be executed.
-    if (const auto * column_function = typeid_cast<const ColumnFunction *>(column.get()))
+    else if (const auto * column_function = typeid_cast<const ColumnFunction *>(column.get()))
     {
-        if (!column_function->isShortCircuitArgument())
-            return column;
-
-        return column_function->recursivelyConvertResultToFullColumnIfLowCardinality();
+        if (column_function->isShortCircuitArgument())
+            res = column_function->recursivelyConvertResultToFullColumnIfLowCardinality();
+    }
+    else if (const auto * column_low_cardinality = typeid_cast<const ColumnLowCardinality *>(column.get()))
+    {
+        res = column_low_cardinality->convertToFullColumn();
     }
 
-    if (const auto * column_low_cardinality = typeid_cast<const ColumnLowCardinality *>(column.get()))
-        return column_low_cardinality->convertToFullColumn();
+    if (res != column)
+    {
+        /// recursiveRemoveLowCardinality() must not change the size of a passed column!
+        if (res->size() != column->size())
+        {
+            throw Exception(ErrorCodes::LOGICAL_ERROR,
+                            "recursiveRemoveLowCardinality() somehow changed the size of column {}. Old size={}, new size={}. It's a bug",
+                            column->getName(), column->size(), res->size());
+        }
+    }
 
-    return column;
+    return res;
 }
 
 ColumnPtr recursiveLowCardinalityTypeConversion(const ColumnPtr & column, const DataTypePtr & from_type, const DataTypePtr & to_type)

src/Disks/DiskEncrypted.cpp

@@ -4,9 +4,9 @@
 #include <Disks/DiskFactory.h>
 #include <IO/FileEncryptionCommon.h>
 #include <IO/ReadBufferFromEncryptedFile.h>
+#include <IO/ReadBufferFromFileDecorator.h>
 #include <IO/ReadBufferFromString.h>
 #include <IO/WriteBufferFromEncryptedFile.h>
-#include <IO/ReadBufferFromEmptyFile.h>
 #include <boost/algorithm/hex.hpp>
 #include <Common/quoteString.h>
 #include <Common/typeid_cast.h>
@@ -374,7 +374,7 @@ std::unique_ptr<ReadBufferFromFileBase> DiskEncrypted::readFile(
     {
         /// File is empty, that's a normal case, see DiskEncrypted::truncateFile().
         /// There is no header so we just return `ReadBufferFromString("")`.
-        return std::make_unique<ReadBufferFromEmptyFile>(wrapped_path);
+        return std::make_unique<ReadBufferFromFileDecorator>(std::make_unique<ReadBufferFromString>(std::string_view{}), wrapped_path);
     }
     auto encryption_settings = current_settings.get();
     FileEncryption::Header header = readHeader(*buffer);

src/Disks/DiskEncryptedTransaction.cpp

@@ -6,6 +6,7 @@
 #include <Common/Exception.h>
 #include <boost/algorithm/hex.hpp>
 #include <IO/ReadBufferFromEncryptedFile.h>
+#include <IO/ReadBufferFromFileDecorator.h>
 #include <IO/ReadBufferFromString.h>
 #include <IO/WriteBufferFromEncryptedFile.h>
 #include <Common/quoteString.h>

src/Disks/IO/ReadBufferFromRemoteFSGather.cpp

@@ -1,6 +1,6 @@
 #include "ReadBufferFromRemoteFSGather.h"
 
-#include <IO/ReadBufferFromFileBase.h>
+#include <IO/SeekableReadBuffer.h>
 
 #include <Disks/IO/CachedOnDiskReadBufferFromFile.h>
 #include <Disks/ObjectStorages/Cached/CachedObjectStorage.h>
@@ -62,7 +62,7 @@ ReadBufferFromRemoteFSGather::ReadBufferFromRemoteFSGather(
         current_object = blobs_to_read.front();
 }
 
-std::unique_ptr<ReadBufferFromFileBase> ReadBufferFromRemoteFSGather::createImplementationBuffer(const StoredObject & object)
+SeekableReadBufferPtr ReadBufferFromRemoteFSGather::createImplementationBuffer(const StoredObject & object)
 {
     if (current_buf && !with_cache)
     {
@@ -79,7 +79,7 @@ std::unique_ptr<ReadBufferFromFileBase> ReadBufferFromRemoteFSGather::createImpl
     if (with_cache)
     {
         auto cache_key = settings.remote_fs_cache->createKeyForPath(object_path);
-        return std::make_unique<CachedOnDiskReadBufferFromFile>(
+        return std::make_shared<CachedOnDiskReadBufferFromFile>(
             object_path,
             cache_key,
             settings.remote_fs_cache,

src/Disks/IO/ReadBufferFromRemoteFSGather.h

@@ -53,7 +53,7 @@ public:
     bool isContentCached(size_t offset, size_t size) override;
 
 private:
-    std::unique_ptr<ReadBufferFromFileBase> createImplementationBuffer(const StoredObject & object);
+    SeekableReadBufferPtr createImplementationBuffer(const StoredObject & object);
 
     bool nextImpl() override;
 
@@ -80,7 +80,7 @@ private:
 
     StoredObject current_object;
     size_t current_buf_idx = 0;
-    std::unique_ptr<ReadBufferFromFileBase> current_buf;
+    SeekableReadBufferPtr current_buf;
 
     LoggerPtr log;
 };

src/Disks/IO/createReadBufferFromFileBase.cpp

@@ -39,7 +39,7 @@ std::unique_ptr<ReadBufferFromFileBase> createReadBufferFromFileBase(
     size_t alignment)
 {
     if (file_size.has_value() && !*file_size)
-        return std::make_unique<ReadBufferFromEmptyFile>(filename);
+        return std::make_unique<ReadBufferFromEmptyFile>();
 
     size_t estimated_size = 0;
     if (read_hint.has_value())

src/Disks/ObjectStorages/DiskObjectStorage.cpp

@@ -389,6 +389,7 @@ void DiskObjectStorage::shutdown()
 {
     LOG_INFO(log, "Shutting down disk {}", name);
     object_storage->shutdown();
+    metadata_storage->shutdown();
     LOG_INFO(log, "Disk {} shut down", name);
 }
 
@@ -531,7 +532,7 @@ std::unique_ptr<ReadBufferFromFileBase> DiskObjectStorage::readFile(
     const bool file_can_be_empty = !file_size.has_value() || *file_size == 0;
 
     if (storage_objects.empty() && file_can_be_empty)
-        return std::make_unique<ReadBufferFromEmptyFile>(path);
+        return std::make_unique<ReadBufferFromEmptyFile>();
 
     return object_storage->readObjects(
         storage_objects,

src/Disks/ObjectStorages/IMetadataStorage.h

@@ -210,6 +210,11 @@ public:
         throwNotImplemented();
     }
 
+    virtual void shutdown()
+    {
+        /// This method is overridden for specific metadata implementations in ClickHouse Cloud.
+    }
+
     virtual ~IMetadataStorage() = default;
 
     /// ==== More specific methods. Previous were almost general purpose. ====

src/Formats/FormatFactory.cpp

@@ -166,6 +166,8 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings)
     format_settings.template_settings.resultset_format = settings.format_template_resultset;
     format_settings.template_settings.row_between_delimiter = settings.format_template_rows_between_delimiter;
     format_settings.template_settings.row_format = settings.format_template_row;
+    format_settings.template_settings.row_format_template = settings.format_template_row_format;
+    format_settings.template_settings.resultset_format_template = settings.format_template_resultset_format;
     format_settings.tsv.crlf_end_of_line = settings.output_format_tsv_crlf_end_of_line;
     format_settings.tsv.empty_as_default = settings.input_format_tsv_empty_as_default;
     format_settings.tsv.enum_as_number = settings.input_format_tsv_enum_as_number;

src/Formats/FormatSettings.h

@@ -338,6 +338,8 @@ struct FormatSettings
         String resultset_format;
         String row_format;
         String row_between_delimiter;
+        String row_format_template;
+        String resultset_format_template;
     } template_settings;
 
     struct

src/Functions/coalesce.cpp

@@ -29,7 +29,14 @@ public:
         return std::make_shared<FunctionCoalesce>(context);
     }
 
-    explicit FunctionCoalesce(ContextPtr context_) : context(context_) {}
+    explicit FunctionCoalesce(ContextPtr context_)
+        : context(context_)
+        , is_not_null(FunctionFactory::instance().get("isNotNull", context))
+        , assume_not_null(FunctionFactory::instance().get("assumeNotNull", context))
+        , if_function(FunctionFactory::instance().get("if", context))
+        , multi_if_function(FunctionFactory::instance().get("multiIf", context))
+    {
+    }
 
     std::string getName() const override
     {
@@ -110,8 +117,6 @@ public:
                 break;
         }
 
-        auto is_not_null = FunctionFactory::instance().get("isNotNull", context);
-        auto assume_not_null = FunctionFactory::instance().get("assumeNotNull", context);
 
         ColumnsWithTypeAndName multi_if_args;
         ColumnsWithTypeAndName tmp_args(1);
@@ -146,13 +151,8 @@ public:
         /// If there was only two arguments (3 arguments passed to multiIf)
         /// use function "if" instead, because it's implemented more efficient.
         /// TODO: make "multiIf" the same efficient.
-        FunctionOverloadResolverPtr if_function;
-        if (multi_if_args.size() == 3)
-            if_function = FunctionFactory::instance().get("if", context);
-        else
-            if_function = FunctionFactory::instance().get("multiIf", context);
-
-        ColumnPtr res = if_function->build(multi_if_args)->execute(multi_if_args, result_type, input_rows_count);
+        FunctionOverloadResolverPtr if_or_multi_if = multi_if_args.size() == 3 ? if_function : multi_if_function;
+        ColumnPtr res = if_or_multi_if->build(multi_if_args)->execute(multi_if_args, result_type, input_rows_count);
 
         /// if last argument is not nullable, result should be also not nullable
         if (!multi_if_args.back().column->isNullable() && res->isNullable())
@@ -170,6 +170,10 @@ public:
 
 private:
     ContextPtr context;
+    FunctionOverloadResolverPtr is_not_null;
+    FunctionOverloadResolverPtr assume_not_null;
+    FunctionOverloadResolverPtr if_function;
+    FunctionOverloadResolverPtr multi_if_function;
 };
 
 }

src/Functions/keyvaluepair/extractKeyValuePairs.cpp

@@ -43,11 +43,11 @@ class ExtractKeyValuePairs : public IFunction
             builder.withQuotingCharacter(parsed_arguments.quoting_character.value());
         }
 
-        bool is_number_of_pairs_unlimited = context->getSettingsRef().extract_kvp_max_pairs_per_row == 0;
+        bool is_number_of_pairs_unlimited = context->getSettingsRef().extract_key_value_pairs_max_pairs_per_row == 0;
 
         if (!is_number_of_pairs_unlimited)
         {
-            builder.withMaxNumberOfPairs(context->getSettingsRef().extract_kvp_max_pairs_per_row);
+            builder.withMaxNumberOfPairs(context->getSettingsRef().extract_key_value_pairs_max_pairs_per_row);
         }
 
         return builder.build();

src/IO/Archives/LibArchiveReader.cpp

@@ -228,12 +228,7 @@ public:
 
     off_t getPosition() override
     {
-        throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "getPosition is not supported when reading from archive");
-    }
-
-    size_t getFileOffsetOfBufferEnd() const override
-    {
-        throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "getFileOffsetOfBufferEnd is not supported when reading from archive");
+        throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "getPosition not supported when reading from archive");
     }
 
     String getFileName() const override { return handle.getFileName(); }

src/IO/Archives/ZipArchiveReader.cpp

@@ -15,7 +15,6 @@ namespace ErrorCodes
     extern const int CANNOT_UNPACK_ARCHIVE;
     extern const int LOGICAL_ERROR;
     extern const int SEEK_POSITION_OUT_OF_BOUND;
-    extern const int UNSUPPORTED_METHOD;
     extern const int CANNOT_SEEK_THROUGH_FILE;
 }
 
@@ -253,11 +252,6 @@ public:
         checkResult(err);
     }
 
-    size_t getFileOffsetOfBufferEnd() const override
-    {
-        throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "getFileOffsetOfBufferEnd is not supported when reading from zip archive");
-    }
-
     off_t seek(off_t off, int whence) override
     {
         off_t current_pos = getPosition();

src/IO/BoundedReadBuffer.cpp

@@ -4,7 +4,8 @@
 namespace DB
 {
 
-BoundedReadBuffer::BoundedReadBuffer(std::unique_ptr<ReadBufferFromFileBase> impl_) : impl(std::move(impl_))
+BoundedReadBuffer::BoundedReadBuffer(std::unique_ptr<SeekableReadBuffer> impl_)
+    : ReadBufferFromFileDecorator(std::move(impl_))
 {
 }
 

src/IO/BoundedReadBuffer.h

@@ -1,5 +1,5 @@
 #pragma once
-#include <IO/ReadBufferFromFileBase.h>
+#include <IO/ReadBufferFromFileDecorator.h>
 
 
 namespace DB
@@ -7,10 +7,10 @@ namespace DB
 
 /// A buffer which allows to make an underlying buffer as right bounded,
 /// e.g. the buffer cannot return data beyond offset specified in `setReadUntilPosition`.
-class BoundedReadBuffer : public ReadBufferFromFileBase
+class BoundedReadBuffer : public ReadBufferFromFileDecorator
 {
 public:
-    explicit BoundedReadBuffer(std::unique_ptr<ReadBufferFromFileBase> impl_);
+    explicit BoundedReadBuffer(std::unique_ptr<SeekableReadBuffer> impl_);
 
     bool supportsRightBoundedReads() const override { return true; }
 
@@ -23,8 +23,6 @@ public:
     off_t seek(off_t off, int whence) override;
 
     size_t getFileOffsetOfBufferEnd() const override { return file_offset_of_buffer_end; }
-    String getFileName() const override { return impl->getFileName(); }
-    size_t getFileSize() override { return impl->getFileSize(); }
 
     /// file_offset_of_buffer_end can differ from impl's file_offset_of_buffer_end
     /// because of resizing of the tail. => Need to also override getPosition() as
@@ -32,8 +30,6 @@ public:
     off_t getPosition() override;
 
 private:
-    std::unique_ptr<ReadBufferFromFileBase> impl;
-
     std::optional<size_t> read_until_position;
     /// atomic because can be used in log or exception messages while being updated.
     std::atomic<size_t> file_offset_of_buffer_end = 0;

src/IO/LimitSeekableReadBuffer.h

@@ -18,6 +18,7 @@ public:
 
     /// Returns adjusted position, i.e. returns `3` if the position in the nested buffer is `start_offset + 3`.
     off_t getPosition() override;
+
     off_t seek(off_t off, int whence) override;
 
 private:

src/IO/MMapReadBufferFromFileDescriptor.cpp

@@ -92,11 +92,6 @@ size_t MMapReadBufferFromFileDescriptor::getFileSize()
     return getSizeFromFileDescriptor(getFD(), getFileName());
 }
 
-size_t MMapReadBufferFromFileDescriptor::getFileOffsetOfBufferEnd() const
-{
-     return mapped.getOffset() + mapped.getLength();
-}
-
 size_t MMapReadBufferFromFileDescriptor::readBigAt(char * to, size_t n, size_t offset, const std::function<bool(size_t)> &)
 {
     if (offset >= mapped.getLength())

src/IO/MMapReadBufferFromFileDescriptor.h

@@ -36,8 +36,6 @@ public:
 
     std::string getFileName() const override;
 
-    size_t getFileOffsetOfBufferEnd() const override;
-
     int getFD() const;
 
     size_t getFileSize() override;

src/IO/MMapReadBufferFromFileWithCache.cpp

@@ -76,9 +76,4 @@ off_t MMapReadBufferFromFileWithCache::seek(off_t offset, int whence)
     return new_pos;
 }
 
-size_t MMapReadBufferFromFileWithCache::getFileOffsetOfBufferEnd() const
-{
-    return mapped->getOffset() + mapped->getLength();
-}
-
 }

src/IO/MMapReadBufferFromFileWithCache.h

@@ -19,7 +19,7 @@ public:
     off_t getPosition() override;
     std::string getFileName() const override;
     off_t seek(off_t offset, int whence) override;
-    size_t getFileOffsetOfBufferEnd() const override;
+
     bool isRegularLocalFile(size_t * /* out_view_offset */) override { return true; }
 
 private:

src/IO/ReadBufferFromEmptyFile.h

@@ -14,18 +14,12 @@ namespace DB
 /// - ThreadPoolReader
 class ReadBufferFromEmptyFile : public ReadBufferFromFileBase
 {
-public:
-    explicit ReadBufferFromEmptyFile(const String & file_name_) : file_name(file_name_) {}
-
 private:
-    String file_name;
-
     bool nextImpl() override { return false; }
-    std::string getFileName() const override { return file_name; }
+    std::string getFileName() const override { return "<empty>"; }
     off_t seek(off_t /*off*/, int /*whence*/) override { return 0; }
     off_t getPosition() override { return 0; }
     size_t getFileSize() override { return 0; }
-    size_t getFileOffsetOfBufferEnd() const override { return 0; }
 };
 
 }

src/IO/ReadBufferFromEncryptedFile.cpp

@@ -101,18 +101,6 @@ bool ReadBufferFromEncryptedFile::nextImpl()
     return true;
 }
 
-size_t ReadBufferFromEncryptedFile::getFileSize()
-{
-    size_t size = in->getFileSize();
-    return size > FileEncryption::Header::kSize ? size - FileEncryption::Header::kSize : size;
-}
-
-size_t ReadBufferFromEncryptedFile::getFileOffsetOfBufferEnd() const
-{
-    size_t file_offset = in->getFileOffsetOfBufferEnd();
-    return file_offset > FileEncryption::Header::kSize ? file_offset - FileEncryption::Header::kSize : file_offset;
-}
-
 }
 
 #endif

src/IO/ReadBufferFromEncryptedFile.h

@@ -27,10 +27,10 @@ public:
     std::string getFileName() const override { return in->getFileName(); }
 
     void setReadUntilPosition(size_t position) override { in->setReadUntilPosition(position + FileEncryption::Header::kSize); }
+
     void setReadUntilEnd() override { in->setReadUntilEnd(); }
 
-    size_t getFileSize() override;
-    size_t getFileOffsetOfBufferEnd() const override;
+    size_t getFileSize() override { return in->getFileSize(); }
 
 private:
     bool nextImpl() override;

src/IO/ReadBufferFromFileBase.h

@@ -60,12 +60,6 @@ public:
     /// file offset and what getPosition() returns.
     virtual bool isRegularLocalFile(size_t * /* out_view_offset */ = nullptr) { return false; }
 
-    /// NOTE: This method should be thread-safe against seek(), since it can be
-    /// used in CachedOnDiskReadBufferFromFile from multiple threads (because
-    /// it first releases the buffer, and then do logging, and so other thread
-    /// can already call seek() which will lead to data-race).
-    virtual size_t getFileOffsetOfBufferEnd() const = 0;
-
 protected:
     std::optional<size_t> file_size;
     ProfileCallback profile_callback;

src/IO/ReadBufferFromFileDecorator.cpp

@@ -0,0 +1,60 @@
+#include <IO/ReadBufferFromFileDecorator.h>
+
+
+namespace DB
+{
+
+ReadBufferFromFileDecorator::ReadBufferFromFileDecorator(std::unique_ptr<SeekableReadBuffer> impl_)
+    : ReadBufferFromFileDecorator(std::move(impl_), "")
+{
+}
+
+
+ReadBufferFromFileDecorator::ReadBufferFromFileDecorator(std::unique_ptr<SeekableReadBuffer> impl_, const String & file_name_)
+    : impl(std::move(impl_)), file_name(file_name_)
+{
+    swap(*impl);
+}
+
+
+std::string ReadBufferFromFileDecorator::getFileName() const
+{
+    if (!file_name.empty())
+        return file_name;
+
+    return getFileNameFromReadBuffer(*impl);
+}
+
+
+off_t ReadBufferFromFileDecorator::getPosition()
+{
+    swap(*impl);
+    auto position = impl->getPosition();
+    swap(*impl);
+    return position;
+}
+
+
+off_t ReadBufferFromFileDecorator::seek(off_t off, int whence)
+{
+    swap(*impl);
+    auto result = impl->seek(off, whence);
+    swap(*impl);
+    return result;
+}
+
+
+bool ReadBufferFromFileDecorator::nextImpl()
+{
+    swap(*impl);
+    auto result = impl->next();
+    swap(*impl);
+    return result;
+}
+
+size_t ReadBufferFromFileDecorator::getFileSize()
+{
+    return getFileSizeFromReadBuffer(*impl);
+}
+
+}

src/IO/ReadBufferFromFileDecorator.h

@@ -0,0 +1,37 @@
+#pragma once
+
+#include <IO/ReadBufferFromFileBase.h>
+
+
+namespace DB
+{
+
+/// Delegates all reads to underlying buffer. Doesn't have own memory.
+class ReadBufferFromFileDecorator : public ReadBufferFromFileBase
+{
+public:
+    explicit ReadBufferFromFileDecorator(std::unique_ptr<SeekableReadBuffer> impl_);
+    ReadBufferFromFileDecorator(std::unique_ptr<SeekableReadBuffer> impl_, const String & file_name_);
+
+    std::string getFileName() const override;
+
+    off_t getPosition() override;
+
+    off_t seek(off_t off, int whence) override;
+
+    bool nextImpl() override;
+
+    bool isWithFileSize() const { return dynamic_cast<const WithFileSize *>(impl.get()) != nullptr; }
+
+    const ReadBuffer & getWrappedReadBuffer() const { return *impl; }
+
+    ReadBuffer & getWrappedReadBuffer() { return *impl; }
+
+    size_t getFileSize() override;
+
+protected:
+    std::unique_ptr<SeekableReadBuffer> impl;
+    String file_name;
+};
+
+}

src/IO/ReadBufferFromMemory.h

@@ -20,6 +20,7 @@ public:
         : SeekableReadBuffer(const_cast<char *>(str.data()), str.size(), 0) {}
 
     off_t seek(off_t off, int whence) override;
+
     off_t getPosition() override;
 };
 

src/IO/SeekableReadBuffer.h

@@ -44,6 +44,12 @@ public:
 
     virtual String getInfoForLog() { return ""; }
 
+    /// NOTE: This method should be thread-safe against seek(), since it can be
+    /// used in CachedOnDiskReadBufferFromFile from multiple threads (because
+    /// it first releases the buffer, and then do logging, and so other thread
+    /// can already call seek() which will lead to data-race).
+    virtual size_t getFileOffsetOfBufferEnd() const { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getFileOffsetOfBufferEnd() not implemented"); }
+
     /// If true, setReadUntilPosition() guarantees that eof will be reported at the given position.
     virtual bool supportsRightBoundedReads() const { return false; }
 

src/IO/WithFileSize.cpp

@@ -2,6 +2,7 @@
 #include <IO/ReadBufferFromFile.h>
 #include <IO/CompressedReadBufferWrapper.h>
 #include <IO/ParallelReadBuffer.h>
+#include <IO/ReadBufferFromFileDecorator.h>
 #include <IO/PeekableReadBuffer.h>
 
 namespace DB
@@ -16,15 +17,23 @@ template <typename T>
 static size_t getFileSize(T & in)
 {
     if (auto * with_file_size = dynamic_cast<WithFileSize *>(&in))
+    {
         return with_file_size->getFileSize();
+    }
 
     throw Exception(ErrorCodes::UNKNOWN_FILE_SIZE, "Cannot find out file size");
 }
 
 size_t getFileSizeFromReadBuffer(ReadBuffer & in)
 {
-    if (auto * compressed = dynamic_cast<CompressedReadBufferWrapper *>(&in))
+    if (auto * delegate = dynamic_cast<ReadBufferFromFileDecorator *>(&in))
+    {
+        return getFileSize(delegate->getWrappedReadBuffer());
+    }
+    else if (auto * compressed = dynamic_cast<CompressedReadBufferWrapper *>(&in))
+    {
         return getFileSize(compressed->getWrappedReadBuffer());
+    }
 
     return getFileSize(in);
 }
@@ -43,7 +52,11 @@ std::optional<size_t> tryGetFileSizeFromReadBuffer(ReadBuffer & in)
 
 bool isBufferWithFileSize(const ReadBuffer & in)
 {
-    if (const auto * compressed = dynamic_cast<const CompressedReadBufferWrapper *>(&in))
+    if (const auto * delegate = dynamic_cast<const ReadBufferFromFileDecorator *>(&in))
+    {
+        return delegate->isWithFileSize();
+    }
+    else if (const auto * compressed = dynamic_cast<const CompressedReadBufferWrapper *>(&in))
     {
         return isBufferWithFileSize(compressed->getWrappedReadBuffer());
     }
@@ -53,7 +66,11 @@ bool isBufferWithFileSize(const ReadBuffer & in)
 
 size_t getDataOffsetMaybeCompressed(const ReadBuffer & in)
 {
-    if (const auto * compressed = dynamic_cast<const CompressedReadBufferWrapper *>(&in))
+    if (const auto * delegate = dynamic_cast<const ReadBufferFromFileDecorator *>(&in))
+    {
+        return getDataOffsetMaybeCompressed(delegate->getWrappedReadBuffer());
+    }
+    else if (const auto * compressed = dynamic_cast<const CompressedReadBufferWrapper *>(&in))
     {
         return getDataOffsetMaybeCompressed(compressed->getWrappedReadBuffer());
     }

src/Interpreters/InterpreterFactory.cpp

@@ -65,6 +65,7 @@
 namespace ProfileEvents
 {
     extern const Event Query;
+    extern const Event InitialQuery;
     extern const Event QueriesWithSubqueries;
     extern const Event SelectQuery;
     extern const Event InsertQuery;
@@ -94,7 +95,8 @@ void InterpreterFactory::registerInterpreter(const std::string & name, CreatorFn
 InterpreterFactory::InterpreterPtr InterpreterFactory::get(ASTPtr & query, ContextMutablePtr context, const SelectQueryOptions & options)
 {
     ProfileEvents::increment(ProfileEvents::Query);
-
+    if (context->getClientInfo().query_kind == ClientInfo::QueryKind::INITIAL_QUERY)
+        ProfileEvents::increment(ProfileEvents::InitialQuery);
     /// SELECT and INSERT query will handle QueriesWithSubqueries on their own.
     if (!(query->as<ASTSelectQuery>() ||
         query->as<ASTSelectWithUnionQuery>() ||

src/Interpreters/TreeOptimizer.cpp

@@ -77,11 +77,10 @@ const std::unordered_set<String> possibly_injective_function_names
   */
 void appendUnusedGroupByColumn(ASTSelectQuery * select_query)
 {
-    /// You must insert a constant that is not the name of the column in the table. Such a case is rare, but it happens.
-    /// Also start unused_column integer must not intersect with ([1, source_columns.size()])
-    /// might be in positional GROUP BY.
+    /// Since ASTLiteral is different from ASTIdentifier, so we can use a special constant String Literal for this,
+    /// and do not need to worry about it conflict with the name of the column in the table.
     select_query->setExpression(ASTSelectQuery::Expression::GROUP_BY, std::make_shared<ASTExpressionList>());
-    select_query->groupBy()->children.emplace_back(std::make_shared<ASTLiteral>(static_cast<Int64>(-1)));
+    select_query->groupBy()->children.emplace_back(std::make_shared<ASTLiteral>("__unused_group_by_column"));
 }
 
 /// Eliminates injective function calls and constant expressions from group by statement.

src/Interpreters/replaceForPositionalArguments.cpp

@@ -10,7 +10,8 @@ namespace DB
 
 namespace ErrorCodes
 {
-    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+extern const int BAD_ARGUMENTS;
+extern const int ILLEGAL_TYPE_OF_ARGUMENT;
 }
 
 bool replaceForPositionalArguments(ASTPtr & argument, const ASTSelectQuery * select_query, ASTSelectQuery::Expression expression)
@@ -27,14 +28,38 @@ bool replaceForPositionalArguments(ASTPtr & argument, const ASTSelectQuery * sel
         return false;
 
     auto which = ast_literal->value.getType();
-    if (which != Field::Types::UInt64)
+    if (which != Field::Types::UInt64 && which != Field::Types::Int64)
         return false;
 
-    auto pos = ast_literal->value.get<UInt64>();
+    UInt64 pos;
+
+    if (which == Field::Types::UInt64)
+    {
+        pos = ast_literal->value.get<UInt64>();
+    }
+    else if (which == Field::Types::Int64)
+    {
+        auto value = ast_literal->value.get<Int64>();
+        if (value > 0)
+            pos = value;
+        else
+        {
+            if (static_cast<size_t>(std::abs(value)) > columns.size())
+                throw Exception(
+                    ErrorCodes::BAD_ARGUMENTS,
+                    "Negative positional argument number {} is out of bounds. Expected in range [-{}, -1]",
+                    value,
+                    columns.size());
+            pos = columns.size() + value + 1;
+        }
+    }
+    else
+    {
+        return false;
+    }
+
     if (!pos || pos > columns.size())
-        throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
-                "Positional argument out of bounds: {} (expected in range [1, {}]",
-                        pos, columns.size());
+        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Positional argument out of bounds: {} (expected in range [1, {}]", pos, columns.size());
 
     const auto & column = columns[--pos];
     if (typeid_cast<const ASTIdentifier *>(column.get()) || typeid_cast<const ASTLiteral *>(column.get()))

src/Parsers/ParserPartition.cpp

@@ -18,8 +18,6 @@ bool ParserPartition::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
     ParserKeyword s_all("ALL");
     ParserStringLiteral parser_string_literal;
     ParserSubstitution parser_substitution;
-    ParserLiteral literal_parser;
-    ParserTupleOfLiterals tuple_of_literals;
     ParserExpression parser_expr;
 
     auto partition = std::make_shared<ASTPartition>();
@@ -45,34 +43,35 @@ bool ParserPartition::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
     {
         ASTPtr value;
         std::optional<size_t> fields_count;
-        if (literal_parser.parse(pos, value, expected) || tuple_of_literals.parse(pos, value, expected))
-        {
-            auto * literal = value->as<ASTLiteral>();
-            if (literal->value.getType() == Field::Types::Tuple)
-            {
-                fields_count = literal->value.get<const Tuple &>().size();
-            }
-            else
-            {
-                fields_count = 1;
-            }
-        }
-        else if (parser_substitution.parse(pos, value, expected))
+        if (parser_substitution.parse(pos, value, expected))
         {
             /// It can be tuple substitution
             fields_count = std::nullopt;
         }
         else if (parser_expr.parse(pos, value, expected))
         {
-            const auto * tuple_ast = value->as<ASTFunction>();
-            if (tuple_ast && tuple_ast->name == "tuple")
+            if (const auto * tuple_ast = value->as<ASTFunction>(); tuple_ast)
             {
+                if (tuple_ast->name != "tuple")
+                    return false;
+
                 const auto * arguments_ast = tuple_ast->arguments->as<ASTExpressionList>();
                 if (arguments_ast)
                     fields_count = arguments_ast->children.size();
                 else
                     fields_count = 0;
             }
+            else if (const auto* literal_ast = value->as<ASTLiteral>(); literal_ast)
+            {
+                if (literal_ast->value.getType() == Field::Types::Tuple)
+                {
+                    fields_count = literal_ast->value.get<const Tuple &>().size();
+                }
+                else
+                {
+                    fields_count = 1;
+                }
+            }
             else
                 return false;
         }

src/Processors/Formats/Impl/JSONColumnsWithMetadataBlockInputFormat.cpp

@@ -60,7 +60,7 @@ void registerInputFormatJSONColumnsWithMetadata(FormatFactory & factory)
     factory.registerInputFormat(
         "JSONColumnsWithMetadata",
         [](ReadBuffer & buf,
-           const Block &sample,
+           const Block & sample,
            const RowInputFormatParams &,
            const FormatSettings & settings)
         {

src/Processors/Formats/Impl/JSONColumnsWithMetadataBlockInputFormat.h

@@ -15,7 +15,7 @@ public:
     bool checkChunkEnd() override;
 
 private:
-    const Block & header;
+    const Block header;
     const bool validate_types_from_metadata;
 };
 

src/Processors/Formats/Impl/TemplateBlockOutputFormat.cpp

@@ -11,6 +11,7 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int SYNTAX_ERROR;
+    extern const int INVALID_TEMPLATE_FORMAT;
 }
 
 TemplateBlockOutputFormat::TemplateBlockOutputFormat(const Block & header_, WriteBuffer & out_, const FormatSettings & settings_,
@@ -193,13 +194,25 @@ void registerOutputFormatTemplate(FormatFactory & factory)
             const FormatSettings & settings)
     {
         ParsedTemplateFormatString resultset_format;
+        auto idx_resultset_by_name = [&](const String & partName)
+        {
+            return static_cast<size_t>(TemplateBlockOutputFormat::stringToResultsetPart(partName));
+        };
         if (settings.template_settings.resultset_format.empty())
         {
             /// Default format string: "${data}"
-            resultset_format.delimiters.resize(2);
-            resultset_format.escaping_rules.emplace_back(ParsedTemplateFormatString::EscapingRule::None);
-            resultset_format.format_idx_to_column_idx.emplace_back(0);
-            resultset_format.column_names.emplace_back("data");
+            if (settings.template_settings.resultset_format_template.empty())
+            {
+                resultset_format.delimiters.resize(2);
+                resultset_format.escaping_rules.emplace_back(ParsedTemplateFormatString::EscapingRule::None);
+                resultset_format.format_idx_to_column_idx.emplace_back(0);
+                resultset_format.column_names.emplace_back("data");
+            }
+            else
+            {
+                resultset_format = ParsedTemplateFormatString();
+                resultset_format.parse(settings.template_settings.resultset_format_template, idx_resultset_by_name);
+            }
         }
         else
         {
@@ -207,20 +220,34 @@ void registerOutputFormatTemplate(FormatFactory & factory)
             resultset_format = ParsedTemplateFormatString(
                     FormatSchemaInfo(settings.template_settings.resultset_format, "Template", false,
                             settings.schema.is_server, settings.schema.format_schema_path),
-                    [&](const String & partName)
-                    {
-                        return static_cast<size_t>(TemplateBlockOutputFormat::stringToResultsetPart(partName));
-                    });
+                    idx_resultset_by_name);
+            if (!settings.template_settings.resultset_format_template.empty())
+            {
+                throw Exception(DB::ErrorCodes::INVALID_TEMPLATE_FORMAT, "Expected either format_template_resultset or format_template_resultset_format, but not both");
+            }
         }
 
-        ParsedTemplateFormatString row_format = ParsedTemplateFormatString(
+        ParsedTemplateFormatString row_format;
+        auto idx_row_by_name = [&](const String & colName)
+        {
+            return sample.getPositionByName(colName);
+        };
+        if (settings.template_settings.row_format.empty())
+        {
+            row_format = ParsedTemplateFormatString();
+            row_format.parse(settings.template_settings.row_format_template, idx_row_by_name);
+        }
+        else
+        {
+            row_format = ParsedTemplateFormatString(
                 FormatSchemaInfo(settings.template_settings.row_format, "Template", false,
                         settings.schema.is_server, settings.schema.format_schema_path),
-                [&](const String & colName)
-                {
-                    return sample.getPositionByName(colName);
-                });
-
+                idx_row_by_name);
+            if (!settings.template_settings.row_format_template.empty())
+            {
+                throw Exception(DB::ErrorCodes::INVALID_TEMPLATE_FORMAT, "Expected either format_template_row or format_template_row_format, but not both");
+            }
+        }
         return std::make_shared<TemplateBlockOutputFormat>(sample, buf, settings, resultset_format, row_format, settings.template_settings.row_between_delimiter);
     });
 

src/Processors/Transforms/AggregatingTransform.cpp

@@ -600,6 +600,12 @@ IProcessor::Status AggregatingTransform::prepare()
         if (is_consume_finished)
         {
             output.finish();
+            /// input.isFinished() means that merging is done. Now we can release our reference to aggregation states.
+            /// TODO: there is another case, when output port is getting closed first.
+            /// E.g. `select ... group by x limit 10`, if it was two-level aggregation and first few buckets contained already enough rows
+            /// limit will stop merging. It turned out to be not trivial to both release aggregation states and ensure that
+            /// ManyAggregatedData holds the last references to them to trigger parallel destruction in its dtor. Will work on that.
+            many_data.reset();
             return Status::Finished;
         }
         else
@@ -828,8 +834,6 @@ void AggregatingTransform::initGenerate()
 
         processors = Pipe::detachProcessors(std::move(pipe));
     }
-
-    many_data.reset();
 }
 
 }

src/Processors/Transforms/AggregatingTransform.h

@@ -71,16 +71,12 @@ struct AggregatingTransformParams
 struct ManyAggregatedData
 {
     ManyAggregatedDataVariants variants;
-    std::vector<std::unique_ptr<std::mutex>> mutexes;
     std::atomic<UInt32> num_finished = 0;
 
-    explicit ManyAggregatedData(size_t num_threads = 0) : variants(num_threads), mutexes(num_threads)
+    explicit ManyAggregatedData(size_t num_threads = 0) : variants(num_threads)
     {
         for (auto & elem : variants)
             elem = std::make_shared<AggregatedDataVariants>();
-
-        for (auto & mut : mutexes)
-            mut = std::make_unique<std::mutex>();
     }
 
     ~ManyAggregatedData()

src/QueryPipeline/RemoteQueryExecutor.cpp

@@ -249,7 +249,19 @@ RemoteQueryExecutor::~RemoteQueryExecutor()
     {
         /// Set was_cancelled, so the query won't be sent after creating connections.
         was_cancelled = true;
-        read_context->cancel();
+
+        /// Cancellation may throw (i.e. some timeout), and in case of pipeline
+        /// had not been properly created properly (EXCEPTION_BEFORE_START)
+        /// cancel will not be sent, so cancellation will be done from dtor and
+        /// will throw.
+        try
+        {
+            read_context->cancel();
+        }
+        catch (...)
+        {
+            tryLogCurrentException(log ? log : getLogger("RemoteQueryExecutor"));
+        }
     }
 
     /** If interrupted in the middle of the loop of communication with replicas, then interrupt
@@ -257,7 +269,17 @@ RemoteQueryExecutor::~RemoteQueryExecutor()
       * these connections did not remain hanging in the out-of-sync state.
       */
     if (established || (isQueryPending() && connections))
-        connections->disconnect();
+    {
+        /// May also throw (so as cancel() above)
+        try
+        {
+            connections->disconnect();
+        }
+        catch (...)
+        {
+            tryLogCurrentException(log ? log : getLogger("RemoteQueryExecutor"));
+        }
+    }
 }
 
 /** If we receive a block with slightly different column types, or with excessive columns,

src/Storages/FileLog/DirectoryWatcherBase.cpp

@@ -34,8 +34,8 @@ DirectoryWatcherBase::DirectoryWatcherBase(
     if (!std::filesystem::is_directory(path))
         throw Exception(ErrorCodes::BAD_FILE_TYPE, "Path {} is not a directory", path);
 
-    fd = inotify_init();
-    if (fd == -1)
+    inotify_fd = inotify_init();
+    if (inotify_fd == -1)
         throw ErrnoException(ErrorCodes::IO_SETUP_ERROR, "Cannot initialize inotify");
 
     watch_task = getContext()->getSchedulePool().createTask("directory_watch", [this] { watchFunc(); });
@@ -56,7 +56,7 @@ void DirectoryWatcherBase::watchFunc()
     if (eventMask() & DirectoryWatcherBase::DW_ITEM_MOVED_TO)
         mask |= IN_MOVED_TO;
 
-    int wd = inotify_add_watch(fd, path.c_str(), mask);
+    int wd = inotify_add_watch(inotify_fd, path.c_str(), mask);
     if (wd == -1)
     {
         owner.onError(Exception(ErrorCodes::IO_SETUP_ERROR, "Watch directory {} failed", path));
@@ -65,16 +65,20 @@ void DirectoryWatcherBase::watchFunc()
 
     std::string buffer;
     buffer.resize(buffer_size);
-    pollfd pfd;
-    pfd.fd = fd;
-    pfd.events = POLLIN;
+    pollfd pfds[2];
+    /// inotify descriptor
+    pfds[0].fd = inotify_fd;
+    pfds[0].events = POLLIN;
+    // notifier
+    pfds[1].fd = event_pipe.fds_rw[0];
+    pfds[1].events = POLLIN;
     while (!stopped)
     {
         const auto & settings = owner.storage.getFileLogSettings();
-        if (poll(&pfd, 1, static_cast<int>(milliseconds_to_wait)) > 0 && pfd.revents & POLLIN)
+        if (poll(pfds, 2, static_cast<int>(milliseconds_to_wait)) > 0 && pfds[0].revents & POLLIN)
         {
             milliseconds_to_wait = settings->poll_directory_watch_events_backoff_init.totalMilliseconds();
-            ssize_t n = read(fd, buffer.data(), buffer.size());
+            ssize_t n = read(inotify_fd, buffer.data(), buffer.size());
             int i = 0;
             if (n > 0)
             {
@@ -130,7 +134,7 @@ void DirectoryWatcherBase::watchFunc()
 DirectoryWatcherBase::~DirectoryWatcherBase()
 {
     stop();
-    int err = ::close(fd);
+    int err = ::close(inotify_fd);
     chassert(!err || errno == EINTR);
 }
 
@@ -143,6 +147,7 @@ void DirectoryWatcherBase::start()
 void DirectoryWatcherBase::stop()
 {
     stopped = true;
+    ::write(event_pipe.fds_rw[1], "\0", 1);
     if (watch_task)
         watch_task->deactivate();
 }

src/Storages/FileLog/DirectoryWatcherBase.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <Core/BackgroundSchedulePool.h>
+#include <Common/PipeFDs.h>
 
 #include <atomic>
 #include <memory>
@@ -85,10 +86,6 @@ public:
 
     void watchFunc();
 
-protected:
-    void start();
-    void stop();
-
 private:
     FileLogDirectoryWatcher & owner;
 
@@ -102,7 +99,11 @@ private:
     int event_mask;
     uint64_t milliseconds_to_wait;
 
-    int fd;
+    int inotify_fd;
+    PipeFDs event_pipe;
+
+    void start();
+    void stop();
 };
 
 }

src/Storages/HDFS/AsynchronousReadBufferFromHDFS.cpp

@@ -37,7 +37,7 @@ namespace ErrorCodes
 
 AsynchronousReadBufferFromHDFS::AsynchronousReadBufferFromHDFS(
     IAsynchronousReader & reader_, const ReadSettings & settings_, std::shared_ptr<ReadBufferFromHDFS> impl_)
-    : ReadBufferFromFileBase(settings_.remote_fs_buffer_size, nullptr, 0)
+    : BufferWithOwnMemory<SeekableReadBuffer>(settings_.remote_fs_buffer_size)
     , reader(reader_)
     , base_priority(settings_.priority)
     , impl(std::move(impl_))

src/Storages/HDFS/AsynchronousReadBufferFromHDFS.h

@@ -21,7 +21,7 @@ namespace DB
 
 class IAsynchronousReader;
 
-class AsynchronousReadBufferFromHDFS : public ReadBufferFromFileBase
+class AsynchronousReadBufferFromHDFS : public BufferWithOwnMemory<SeekableReadBuffer>, public WithFileName, public WithFileSize
 {
 public:
     AsynchronousReadBufferFromHDFS(

src/Storages/MergeTree/registerStorageMergeTree.cpp

@@ -194,7 +194,7 @@ static StoragePtr create(const StorageFactory::Arguments & args)
     auto add_optional_param = [&](const char * desc)
     {
         ++max_num_params;
-        needed_params += needed_params.empty() ? "\n" : ",\n[";
+        needed_params += needed_params.empty() ? "\n[" : ",\n[";
         needed_params += desc;
         needed_params += "]";
     };
@@ -404,10 +404,10 @@ static StoragePtr create(const StorageFactory::Arguments & args)
         {
             /// Try use default values if arguments are not specified.
             /// Note: {uuid} macro works for ON CLUSTER queries when database engine is Atomic.
-            const auto & config = args.getContext()->getConfigRef();
-            zookeeper_path = StorageReplicatedMergeTree::getDefaultZooKeeperPath(config);
+            const auto & server_settings = args.getContext()->getServerSettings();
+            zookeeper_path = server_settings.default_replica_path;
             /// TODO maybe use hostname if {replica} is not defined?
-            replica_name = StorageReplicatedMergeTree::getDefaultReplicaName(config);
+            replica_name = server_settings.default_replica_name;
 
             /// Modify query, so default values will be written to metadata
             assert(arg_num == 0);

src/Storages/StorageReplicatedMergeTree.cpp

@@ -545,18 +545,6 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree(
 }
 
 
-String StorageReplicatedMergeTree::getDefaultZooKeeperPath(const Poco::Util::AbstractConfiguration & config)
-{
-    return config.getString("default_replica_path", "/clickhouse/tables/{uuid}/{shard}");
-}
-
-
-String StorageReplicatedMergeTree::getDefaultReplicaName(const Poco::Util::AbstractConfiguration & config)
-{
-    return config.getString("default_replica_name", "{replica}");
-}
-
-
 bool StorageReplicatedMergeTree::checkFixedGranularityInZookeeper()
 {
     auto zookeeper = getZooKeeper();

src/Storages/StorageReplicatedMergeTree.h

@@ -143,9 +143,6 @@ public:
 
     ~StorageReplicatedMergeTree() override;
 
-    static String getDefaultZooKeeperPath(const Poco::Util::AbstractConfiguration & config);
-    static String getDefaultReplicaName(const Poco::Util::AbstractConfiguration & config);
-
     std::string getName() const override { return "Replicated" + merging_params.getModeName() + "MergeTree"; }
 
     bool supportsParallelInsert() const override { return true; }

tests/analyzer_integration_broken_tests.txt

@@ -7,13 +7,9 @@ test_mask_sensitive_info/test.py::test_encryption_functions
 test_merge_table_over_distributed/test.py::test_global_in
 test_merge_table_over_distributed/test.py::test_select_table_name_from_merge_over_distributed
 test_mutations_with_merge_tree/test.py::test_mutations_with_merge_background_task
-test_mysql_database_engine/test.py::test_mysql_ddl_for_mysql_database
 test_passing_max_partitions_to_read_remotely/test.py::test_default_database_on_cluster
 test_select_access_rights/test_main.py::test_alias_columns
 test_settings_profile/test.py::test_show_profiles
 test_shard_level_const_function/test.py::test_remote
 test_sql_user_defined_functions_on_cluster/test.py::test_sql_user_defined_functions_on_cluster
 test_storage_rabbitmq/test.py::test_rabbitmq_materialized_view
-test_user_defined_object_persistence/test.py::test_persistence
-test_zookeeper_config/test.py::test_chroot_with_same_root
-test_zookeeper_config/test.py::test_chroot_with_different_root

tests/analyzer_tech_debt.txt

@@ -8,8 +8,6 @@
 01584_distributed_buffer_cannot_find_column
 01624_soft_constraints
 01656_test_query_log_factories_info
-01739_index_hint
-02880_indexHint__partition_id
 01747_join_view_filter_dictionary
 01761_cast_to_enum_nullable
 01925_join_materialized_columns

tests/integration/test_materialized_mysql_database/materialized_with_ddl.py

@@ -3379,7 +3379,7 @@ def gtid_after_attach_test(clickhouse_node, mysql_node, replication):
         f"CREATE TABLE {db}.t(id INT PRIMARY KEY AUTO_INCREMENT, score int, create_time DATETIME DEFAULT NOW())"
     )
 
-    db_count = 6
+    db_count = 4
     for i in range(db_count):
         replication.create_db_ch(
             f"{db}{i}",
@@ -3392,7 +3392,11 @@ def gtid_after_attach_test(clickhouse_node, mysql_node, replication):
         "t\n",
     )
     for i in range(int(db_count / 2)):
-        clickhouse_node.query(f"DETACH DATABASE {db}{i}")
+        check_query(
+            clickhouse_node,
+            f"DETACH DATABASE {db}{i}",
+            "",
+        )
 
     mysql_node.query(f"USE {db}")
     rows = 10000

tests/integration/test_user_defined_object_persistence/test.py

@@ -35,9 +35,16 @@ def test_persistence():
 
     instance.restart_clickhouse()
 
-    assert "Unknown function MySum1" in instance.query_and_get_error(
-        "SELECT MySum1(1, 2)"
+    error_message = instance.query_and_get_error("SELECT MySum1(1, 2)")
+    assert (
+        "Unknown function MySum1" in error_message
+        or "Function with name 'MySum1' does not exists. In scope SELECT MySum1(1, 2)"
+        in error_message
     )
-    assert "Unknown function MySum2" in instance.query_and_get_error(
-        "SELECT MySum2(1, 2)"
+
+    error_message = instance.query_and_get_error("SELECT MySum2(1, 2)")
+    assert (
+        "Unknown function MySum2" in error_message
+        or "Function with name 'MySum2' does not exists. In scope SELECT MySum2(1, 2)"
+        in error_message
     )

tests/performance/coalesce.xml

@@ -0,0 +1,3 @@
+<test>
+    <query>select coalesce(materialize(null), -1) from numbers(1000000000) format Null settings max_block_size = 8192</query>
+</test>

tests/queries/0_stateless/00937_format_schema_rows_template.reference

@@ -0,0 +1,9 @@
+Question: 'How awesome is clickhouse?', Answer: 'unbelievably awesome!', Number of Likes: 456, Date: 2016-01-02;
+Question: 'How fast is clickhouse?', Answer: 'Lightning fast!', Number of Likes: 9876543210, Date: 2016-01-03;
+Question: 'Is it opensource?', Answer: 'of course it is!', Number of Likes: 789, Date: 2016-01-04
+
+===== Results ===== 
+Question: 'How awesome is clickhouse?', Answer: 'unbelievably awesome!', Number of Likes: 456, Date: 2016-01-02;
+Question: 'How fast is clickhouse?', Answer: 'Lightning fast!', Number of Likes: 9876543210, Date: 2016-01-03;
+Question: 'Is it opensource?', Answer: 'of course it is!', Number of Likes: 789, Date: 2016-01-04
+===================

tests/queries/0_stateless/00937_format_schema_rows_template.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC2016
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+# Test format_template_row_format setting 
+
+$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS template";
+$CLICKHOUSE_CLIENT --query="CREATE TABLE template (question String, answer String, likes UInt64, date Date) ENGINE = Memory";
+$CLICKHOUSE_CLIENT --query="INSERT INTO template VALUES
+('How awesome is clickhouse?', 'unbelievably awesome!', 456, '2016-01-02'),\
+('How fast is clickhouse?', 'Lightning fast!', 9876543210, '2016-01-03'),\
+('Is it opensource?', 'of course it is!', 789, '2016-01-04')";
+
+$CLICKHOUSE_CLIENT --query="SELECT * FROM template GROUP BY question, answer, likes, date WITH TOTALS ORDER BY date LIMIT 3 FORMAT Template SETTINGS \
+format_template_row_format = 'Question: \${question:Quoted}, Answer: \${answer:Quoted}, Number of Likes: \${likes:Raw}, Date: \${date:Raw}', \
+format_template_rows_between_delimiter = ';\n'";
+
+echo -e "\n"
+
+# Test that if both format_template_row_format setting and format_template_row are provided, error is thrown
+row_format_file="$CURDIR"/"${CLICKHOUSE_TEST_UNIQUE_NAME}"_template_output_format_row.tmp
+echo -ne 'Question: ${question:Quoted}, Answer: ${answer:Quoted}, Number of Likes: ${likes:Raw}, Date: ${date:Raw}' > $row_format_file
+$CLICKHOUSE_CLIENT --multiline --multiquery --query "SELECT * FROM template GROUP BY question, answer, likes, date WITH TOTALS ORDER BY date LIMIT 3 FORMAT Template SETTINGS \
+format_template_row = '$row_format_file', \
+format_template_row_format = 'Question: \${question:Quoted}, Answer: \${answer:Quoted}, Number of Likes: \${likes:Raw}, Date: \${date:Raw}', \
+format_template_rows_between_delimiter = ';\n'; --{clientError 474}"
+
+# Test format_template_resultset_format setting 
+
+$CLICKHOUSE_CLIENT --query="SELECT * FROM template GROUP BY question, answer, likes, date WITH TOTALS ORDER BY date LIMIT 3 FORMAT Template SETTINGS \
+format_template_row_format = 'Question: \${question:Quoted}, Answer: \${answer:Quoted}, Number of Likes: \${likes:Raw}, Date: \${date:Raw}', \
+format_template_resultset_format = '===== Results ===== \n\${data}\n===================\n', \
+format_template_rows_between_delimiter = ';\n'";
+
+# Test that if both format_template_result_format setting and format_template_resultset are provided, error is thrown
+resultset_output_file="$CURDIR"/"$CLICKHOUSE_TEST_UNIQUE_NAME"_template_output_format_resultset.tmp
+echo -ne '===== Resultset ===== \n \${data} \n ===============' > $resultset_output_file
+$CLICKHOUSE_CLIENT --multiline --multiquery --query "SELECT * FROM template GROUP BY question, answer, likes, date WITH TOTALS ORDER BY date LIMIT 3 FORMAT Template SETTINGS \
+format_template_resultset = '$resultset_output_file', \
+format_template_resultset_format = '===== Resultset ===== \n \${data} \n ===============', \
+format_template_row_format = 'Question: \${question:Quoted}, Answer: \${answer:Quoted}, Number of Likes: \${likes:Raw}, Date: \${date:Raw}', \
+format_template_rows_between_delimiter = ';\n'; --{clientError 474}"
+
+$CLICKHOUSE_CLIENT --query="DROP TABLE template";
+rm $row_format_file
+rm $resultset_output_file

tests/queries/0_stateless/01739_index_hint.reference

@@ -35,6 +35,9 @@ SELECT count() FROM XXXX WHERE indexHint(t = toDateTime(0)) SETTINGS optimize_us
 drop table XXXX;
 CREATE TABLE XXXX (p Nullable(Int64), k Decimal(76, 39)) ENGINE = MergeTree PARTITION BY toDate(p) ORDER BY k SETTINGS index_granularity = 1, allow_nullable_key = 1;
 INSERT INTO XXXX FORMAT Values ('2020-09-01 00:01:02', 1), ('2020-09-01 20:01:03', 2), ('2020-09-02 00:01:03', 3);
-SELECT count() FROM XXXX WHERE indexHint(p = 1.) SETTINGS optimize_use_implicit_projections = 1;
+SELECT count() FROM XXXX WHERE indexHint(p = 1.) SETTINGS optimize_use_implicit_projections = 1, allow_experimental_analyzer=0;
 0
+-- TODO: optimize_use_implicit_projections ignores indexHint (with analyzer) because source columns might be aliased.
+SELECT count() FROM XXXX WHERE indexHint(p = 1.) SETTINGS optimize_use_implicit_projections = 1, allow_experimental_analyzer=1;
+3
 drop table XXXX;

tests/queries/0_stateless/01739_index_hint.sql

@@ -38,6 +38,8 @@ CREATE TABLE XXXX (p Nullable(Int64), k Decimal(76, 39)) ENGINE = MergeTree PART
 
 INSERT INTO XXXX FORMAT Values ('2020-09-01 00:01:02', 1), ('2020-09-01 20:01:03', 2), ('2020-09-02 00:01:03', 3);
 
-SELECT count() FROM XXXX WHERE indexHint(p = 1.) SETTINGS optimize_use_implicit_projections = 1;
+SELECT count() FROM XXXX WHERE indexHint(p = 1.) SETTINGS optimize_use_implicit_projections = 1, allow_experimental_analyzer=0;
+-- TODO: optimize_use_implicit_projections ignores indexHint (with analyzer) because source columns might be aliased.
+SELECT count() FROM XXXX WHERE indexHint(p = 1.) SETTINGS optimize_use_implicit_projections = 1, allow_experimental_analyzer=1;
 
 drop table XXXX;

tests/queries/0_stateless/01798_having_push_down.sql

@@ -8,11 +8,12 @@ SELECT sum(c0 = 0), min(c0 + 1), sum(c0 + 2) FROM t_having
 GROUP BY c0 HAVING c0 = 0
 SETTINGS enable_optimize_predicate_expression=0;
 
+SET enable_positional_arguments=0;
+
 SELECT c0 + -1, sum(intDivOrZero(intDivOrZero(NULL, NULL), '2'), intDivOrZero(10000000000., intDivOrZero(intDivOrZero(intDivOrZero(NULL, NULL), 10), NULL))) FROM t_having GROUP BY c0 = 2, c0 = 10, intDivOrZero(intDivOrZero(intDivOrZero(NULL, NULL), NULL), NULL), c0 HAVING c0 = 2 SETTINGS enable_optimize_predicate_expression = 0;
 
 SELECT sum(c0 + 257) FROM t_having GROUP BY c0 = -9223372036854775808, NULL, -2147483649, c0 HAVING c0 = -9223372036854775808 SETTINGS enable_optimize_predicate_expression = 0;
 
-SET enable_positional_arguments=0;
 SELECT c0 + -2, c0 + -9223372036854775807, c0 = NULL FROM t_having GROUP BY c0 = 0.9998999834060669, 1023, c0 HAVING c0 = 0.9998999834060669 SETTINGS enable_optimize_predicate_expression = 0;
 
 DROP TABLE t_having;

tests/queries/0_stateless/02006_test_positional_arguments.reference

@@ -3,18 +3,50 @@ select x3, x2, x1 from test order by 1;
 1	100	100
 10	1	10
 100	10	1
+select x3, x2, x1 from test order by -3;
+1	100	100
+10	1	10
+100	10	1
 select x3, x2, x1 from test order by x3;
 1	100	100
 10	1	10
 100	10	1
+select x3, x2, x1 from test order by 3;
+100	10	1
+10	1	10
+1	100	100
+select x3, x2, x1 from test order by -1;
+100	10	1
+10	1	10
+1	100	100
+select x3, x2, x1 from test order by x1;
+100	10	1
+10	1	10
+1	100	100
 select x3, x2, x1 from test order by 1 desc;
 100	10	1
 10	1	10
 1	100	100
+select x3, x2, x1 from test order by -3 desc;
+100	10	1
+10	1	10
+1	100	100
 select x3, x2, x1 from test order by x3 desc;
 100	10	1
 10	1	10
 1	100	100
+select x3, x2, x1 from test order by 3 desc;
+1	100	100
+10	1	10
+100	10	1
+select x3, x2, x1 from test order by -1 desc;
+1	100	100
+10	1	10
+100	10	1
+select x3, x2, x1 from test order by x1 desc;
+1	100	100
+10	1	10
+100	10	1
 insert into test values (1, 10, 100), (10, 1, 10), (100, 100, 1);
 select x3, x2 from test group by x3, x2 order by x3;
 1	100
@@ -54,6 +86,20 @@ SELECT
     x1
 FROM test
 ORDER BY x3 + 1 ASC
+explain syntax select x3, x2, x1 from test order by -1;
+SELECT
+    x3,
+    x2,
+    x1
+FROM test
+ORDER BY x1 ASC
+explain syntax select x3 + 1, x2, x1 from test order by -1;
+SELECT
+    x3 + 1,
+    x2,
+    x1
+FROM test
+ORDER BY x1 ASC
 explain syntax select x3, x3 - x2, x2, x1 from test order by 2;
 SELECT
     x3,
@@ -62,6 +108,14 @@ SELECT
     x1
 FROM test
 ORDER BY x3 - x2 ASC
+explain syntax select x3, x3 - x2, x2, x1 from test order by -2;
+SELECT
+    x3,
+    x3 - x2,
+    x2,
+    x1
+FROM test
+ORDER BY x2 ASC
 explain syntax select x3, if(x3 > 10, x3, plus(x1, x2)), x1 + x2 from test order by 2;
 SELECT
     x3,
@@ -69,12 +123,28 @@ SELECT
     x1 + x2
 FROM test
 ORDER BY if(x3 > 10, x3, x1 + x2) ASC
+explain syntax select x3, if(x3 > 10, x3, plus(x1, x2)), x1 + x2 from test order by -2;
+SELECT
+    x3,
+    if(x3 > 10, x3, x1 + x2),
+    x1 + x2
+FROM test
+ORDER BY if(x3 > 10, x3, x1 + x2) ASC
 explain syntax select max(x1), x2 from test group by 2 order by 1, 2;
 SELECT
     max(x1),
     x2
 FROM test
 GROUP BY x2
+ORDER BY
+    max(x1) ASC,
+    x2 ASC
+explain syntax select max(x1), x2 from test group by -1 order by -2, -1;
+SELECT
+    max(x1),
+    x2
+FROM test
+GROUP BY x2
 ORDER BY
     max(x1) ASC,
     x2 ASC
@@ -83,16 +153,34 @@ SELECT
     1 + greatest(x1, 1),
     x2
 FROM test
+GROUP BY
+    1 + greatest(x1, 1),
+    x2
+explain syntax select 1 + greatest(x1, 1), x2 from test group by -2, -1;
+SELECT
+    1 + greatest(x1, 1),
+    x2
+FROM test
 GROUP BY
     1 + greatest(x1, 1),
     x2
 select max(x1), x2 from test group by 1, 2; -- { serverError 43, 184 }
 select 1 + max(x1), x2 from test group by 1, 2; -- { serverError 43, 184 }
+select max(x1), x2 from test group by -2, -1; -- { serverError 43, 184 }
+select 1 + max(x1), x2 from test group by -2, -1; -- { serverError 43, 184 }
 explain syntax select x1 + x3, x3 from test group by 1, 2;
 SELECT
     x1 + x3,
     x3
 FROM test
+GROUP BY
+    x1 + x3,
+    x3
+explain syntax select x1 + x3, x3 from test group by -2, -1;
+SELECT
+    x1 + x3,
+    x3
+FROM test
 GROUP BY
     x1 + x3,
     x3
@@ -102,8 +190,14 @@ select x1, x1 * 2, max(x2), max(x3) from test2 group by 2, 1, x1 order by 1, 2,
 1	2	10	100
 10	20	1	10
 100	200	100	1
+select x1, x1 * 2, max(x2), max(x3) from test2 group by 2, 1, x1 order by 1, 2, -1 desc, -2 asc;
+1	2	10	100
+10	20	1	10
+100	200	100	1
 select a, b, c, d, e, f  from (select 44 a, 88 b, 13 c, 14 d, 15 e, 16 f) t group by 1,2,3,4,5,6 order by a;
 44	88	13	14	15	16
+select a, b, c, d, e, f  from (select 44 a, 88 b, 13 c, 14 d, 15 e, 16 f) t group by 1,2,3,-3,-2,-1 order by a;
+44	88	13	14	15	16
 explain syntax select plus(1, 1) as a group by a;
 SELECT 1 + 1 AS a
 GROUP BY a

tests/queries/0_stateless/02006_test_positional_arguments.sql

@@ -9,11 +9,21 @@ insert into test values (1, 10, 100), (10, 1, 10), (100, 100, 1);
 
 -- { echo }
 select x3, x2, x1 from test order by 1;
+select x3, x2, x1 from test order by -3;
 select x3, x2, x1 from test order by x3;
 
+select x3, x2, x1 from test order by 3;
+select x3, x2, x1 from test order by -1;
+select x3, x2, x1 from test order by x1;
+
 select x3, x2, x1 from test order by 1 desc;
+select x3, x2, x1 from test order by -3 desc;
 select x3, x2, x1 from test order by x3 desc;
 
+select x3, x2, x1 from test order by 3 desc;
+select x3, x2, x1 from test order by -1 desc;
+select x3, x2, x1 from test order by x1 desc;
+
 insert into test values (1, 10, 100), (10, 1, 10), (100, 100, 1);
 select x3, x2 from test group by x3, x2 order by x3;
 select x3, x2 from test group by 1, 2 order by x3;
@@ -25,21 +35,32 @@ select x1, x2, x3 from test order by 3 limit 1 by 1;
 
 explain syntax select x3, x2, x1 from test order by 1;
 explain syntax select x3 + 1, x2, x1 from test order by 1;
+explain syntax select x3, x2, x1 from test order by -1;
+explain syntax select x3 + 1, x2, x1 from test order by -1;
 explain syntax select x3, x3 - x2, x2, x1 from test order by 2;
+explain syntax select x3, x3 - x2, x2, x1 from test order by -2;
 explain syntax select x3, if(x3 > 10, x3, plus(x1, x2)), x1 + x2 from test order by 2;
+explain syntax select x3, if(x3 > 10, x3, plus(x1, x2)), x1 + x2 from test order by -2;
 explain syntax select max(x1), x2 from test group by 2 order by 1, 2;
+explain syntax select max(x1), x2 from test group by -1 order by -2, -1;
 explain syntax select 1 + greatest(x1, 1), x2 from test group by 1, 2;
+explain syntax select 1 + greatest(x1, 1), x2 from test group by -2, -1;
 
 select max(x1), x2 from test group by 1, 2; -- { serverError 43, 184 }
 select 1 + max(x1), x2 from test group by 1, 2; -- { serverError 43, 184 }
+select max(x1), x2 from test group by -2, -1; -- { serverError 43, 184 }
+select 1 + max(x1), x2 from test group by -2, -1; -- { serverError 43, 184 }
 
 explain syntax select x1 + x3, x3 from test group by 1, 2;
+explain syntax select x1 + x3, x3 from test group by -2, -1;
 
 create table test2(x1 Int, x2 Int, x3 Int) engine=Memory;
 insert into test2 values (1, 10, 100), (10, 1, 10), (100, 100, 1);
 select x1, x1 * 2, max(x2), max(x3) from test2 group by 2, 1, x1 order by 1, 2, 4 desc, 3 asc;
+select x1, x1 * 2, max(x2), max(x3) from test2 group by 2, 1, x1 order by 1, 2, -1 desc, -2 asc;
 
 select a, b, c, d, e, f  from (select 44 a, 88 b, 13 c, 14 d, 15 e, 16 f) t group by 1,2,3,4,5,6 order by a;
+select a, b, c, d, e, f  from (select 44 a, 88 b, 13 c, 14 d, 15 e, 16 f) t group by 1,2,3,-3,-2,-1 order by a;
 
 explain syntax select plus(1, 1) as a group by a;
 select substr('aaaaaaaaaaaaaa', 8) as a  group by a order by a;

tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference

@@ -293,7 +293,7 @@ SELECT
 {'age':'31','last_key':'last_value','name':'neymar','nationality':'brazil','team':'psg'}
 -- { echoOn }
 
-SET extract_kvp_max_pairs_per_row = 2;
+SET extract_key_value_pairs_max_pairs_per_row = 2;
 -- Should be allowed because it no longer exceeds the max number of pairs
 -- expected output: {'key1':'value1','key2':'value2'}
 WITH
@@ -307,7 +307,7 @@ WITH
 SELECT
     x;
 {'key1':'value1','key2':'value2'}
-SET extract_kvp_max_pairs_per_row = 0;
+SET extract_key_value_pairs_max_pairs_per_row = 0;
 -- Should be allowed because max pairs per row is set to 0 (unlimited)
 -- expected output: {'key1':'value1','key2':'value2'}
 WITH

tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql

@@ -415,7 +415,7 @@ SELECT
     x; -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH}
 
 -- Should fail allowed because it exceeds the max number of pairs
-SET extract_kvp_max_pairs_per_row = 1;
+SET extract_key_value_pairs_max_pairs_per_row = 1;
 WITH
     extractKeyValuePairs('key1:value1,key2:value2') AS s_map,
     CAST(
@@ -429,7 +429,7 @@ SELECT
 
 -- { echoOn }
 
-SET extract_kvp_max_pairs_per_row = 2;
+SET extract_key_value_pairs_max_pairs_per_row = 2;
 -- Should be allowed because it no longer exceeds the max number of pairs
 -- expected output: {'key1':'value1','key2':'value2'}
 WITH
@@ -443,7 +443,7 @@ WITH
 SELECT
     x;
 
-SET extract_kvp_max_pairs_per_row = 0;
+SET extract_key_value_pairs_max_pairs_per_row = 0;
 -- Should be allowed because max pairs per row is set to 0 (unlimited)
 -- expected output: {'key1':'value1','key2':'value2'}
 WITH

tests/queries/0_stateless/02880_indexHint__partition_id.reference

@@ -1,9 +1,10 @@
 -- { echoOn }
 select * from data prewhere indexHint(_partition_id = '1');
 1
-select count() from data prewhere indexHint(_partition_id = '1');
+-- TODO: optimize_use_implicit_projections ignores indexHint (with analyzer) because source columns might be aliased.
+select count() from data prewhere indexHint(_partition_id = '1') settings optimize_use_implicit_projections = 0;
 1
 select * from data where indexHint(_partition_id = '1');
 1
-select count() from data where indexHint(_partition_id = '1');
+select count() from data where indexHint(_partition_id = '1') settings optimize_use_implicit_projections = 0;
 1

tests/queries/0_stateless/02880_indexHint__partition_id.sql

@@ -4,6 +4,7 @@ insert into data values (1)(2);
 
 -- { echoOn }
 select * from data prewhere indexHint(_partition_id = '1');
-select count() from data prewhere indexHint(_partition_id = '1');
+-- TODO: optimize_use_implicit_projections ignores indexHint (with analyzer) because source columns might be aliased.
+select count() from data prewhere indexHint(_partition_id = '1') settings optimize_use_implicit_projections = 0;
 select * from data where indexHint(_partition_id = '1');
-select count() from data where indexHint(_partition_id = '1');
+select count() from data where indexHint(_partition_id = '1') settings optimize_use_implicit_projections = 0;

tests/queries/0_stateless/02897_alter_partition_parameters.reference

@@ -7,3 +7,8 @@
 0
 0
 0
+0
+0
+0
+0
+0

tests/queries/0_stateless/02897_alter_partition_parameters.sql

@@ -10,6 +10,24 @@ PARTITION BY toMonday(EventDate);
 
 INSERT INTO test VALUES(toDate('2023-10-09'));
 
+ALTER TABLE test DROP PARTITION ('2023-10-09');
+
+SELECT count() FROM test;
+
+INSERT INTO test VALUES(toDate('2023-10-09'));
+
+ALTER TABLE test DROP PARTITION (('2023-10-09'));
+
+SELECT count() FROM test;
+
+INSERT INTO test VALUES(toDate('2023-10-09'));
+
+ALTER TABLE test DROP PARTITION '2023-10-09';
+
+SELECT count() FROM test;
+
+INSERT INTO test VALUES(toDate('2023-10-09'));
+
 SET param_partition='2023-10-09';
 
 ALTER TABLE test DROP PARTITION {partition:String};
@@ -51,6 +69,17 @@ ENGINE = MergeTree
 ORDER BY tuple()
 PARTITION BY (a * b, b * b);
 
+INSERT INTO test2 VALUES(1, 2);
+
+ALTER TABLE test2 DROP PARTITION tuple(2, 4);
+
+SELECT count() FROM test2;
+
+INSERT INTO test2 VALUES(1, 2);
+
+ALTER TABLE test2 DROP PARTITION (2, 4);
+
+SELECT count() FROM test2;
 
 INSERT INTO test2 VALUES(1, 2);
 

tests/queries/0_stateless/02932_group_by_null_fuzzer.sql

@@ -1,5 +1,6 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/43202
 -- Queries are generated by the fuzzer, so don't expect them to make sense
+SET enable_positional_arguments=0;
 SELECT NULL, '' FROM (SELECT toNullable(''), NULL AS key GROUP BY GROUPING SETS ((NULL))) AS s1 ALL LEFT JOIN (SELECT '' AS key, NULL AS value GROUP BY GROUPING SETS (('')) WITH TOTALS UNION ALL SELECT NULL AS key, toNullable(NULL) AS value GROUP BY '', NULL, '' WITH TOTALS) AS s2 USING (key);
 SELECT NULL GROUP BY NULL WITH TOTALS;
 SELECT 1048575, NULL, b FROM (SELECT '25.5' AS a, NULL, NULL AS b GROUP BY GROUPING SETS ((0.0001)) WITH TOTALS) AS js1 ANY RIGHT JOIN (SELECT NULL AS a, NULL AS b WHERE NULL GROUP BY NULL, -9223372036854775807 WITH CUBE WITH TOTALS UNION ALL SELECT NULL AS a, NULL AS b GROUP BY 1, '21474836.46' WITH TOTALS) AS js2 USING (a, b) ORDER BY nan DESC NULLS LAST, '9223372036854775807' DESC NULLS LAST, a ASC NULLS LAST;

tests/queries/0_stateless/02943_positional_arguments_bugs.reference

@@ -1,2 +1,12 @@
+0	0
+1	1
+2	2
+3	3
+4	4
+5	5
+6	6
+7	7
+8	8
+9	9
 45	1
 processed	99	0

tests/queries/0_stateless/02943_positional_arguments_bugs.sql

@@ -2,18 +2,21 @@
 DROP TABLE IF EXISTS t;
 CREATE TABLE t
 (
-    `n` int
+    `n` int,
+    `__unused_group_by_column` int
 )
-    ENGINE = MergeTree
-        ORDER BY n AS
-SELECT *
+ENGINE = MergeTree
+ORDER BY n AS
+SELECT number, number
 FROM numbers(10);
 
 SELECT
     sum(n),
-    1 AS x
+    __unused_group_by_column 
 FROM t
-GROUP BY x;
+GROUP BY __unused_group_by_column ORDER BY __unused_group_by_column;
+
+SELECT sum(n), 1 as x from t group by x;
 
 SELECT
     'processed' AS type,

tests/queries/0_stateless/02950_distributed_initial_query_event.reference

@@ -0,0 +1,6 @@
+Local situation
+Initial Query Difference: 1
+Query Difference: 1
+Distributed situation
+Initial Query Difference: 1
+Query Difference: 3

tests/queries/0_stateless/02950_distributed_initial_query_event.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# Tags:no-parallel,shard
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CUR_DIR"/../shell_config.sh
+# CREATE TABLE local (x UInt8) Engine=Memory;
+# CREATE TABLE distributed ON CLUSTER cluster (p Date, i Int32) ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), x)
+$CLICKHOUSE_CLIENT -n -q "
+DROP TABLE IF EXISTS local;
+DROP TABLE IF EXISTS distributed;
+CREATE TABLE local (x UInt8) Engine=Memory;
+CREATE TABLE distributed AS local ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), local, x);
+INSERT INTO distributed SELECT number FROM numbers(10);
+SYSTEM FLUSH DISTRIBUTED distributed;
+"
+echo "Local situation"
+# before SELECT * FROM local
+query_countI=$($CLICKHOUSE_CLIENT -q "SELECT value FROM system.events WHERE event = 'InitialQuery'")
+query_countQ=$($CLICKHOUSE_CLIENT -q "SELECT value FROM system.events WHERE event = 'Query'")
+
+# Execute SELECT * FROM local
+$CLICKHOUSE_CLIENT -q "SELECT * FROM local" > /dev/null
+
+# Counts after SELECT * FROM local
+After_query_countI=$($CLICKHOUSE_CLIENT -q "SELECT value FROM system.events WHERE event = 'InitialQuery'")
+After_query_countQ=$($CLICKHOUSE_CLIENT -q "SELECT value FROM system.events WHERE event = 'Query'")
+
+# Calculate the differences
+Initial_query_diff=$(($After_query_countI-$query_countI-2))
+query_diff=$(($After_query_countQ-$query_countQ-2))
+
+echo "Initial Query Difference: $Initial_query_diff"
+echo "Query Difference: $query_diff"
+echo "Distributed situation"
+
+# before SELECT * FROM distributed
+query_countI=$($CLICKHOUSE_CLIENT -q "SELECT value FROM system.events WHERE event = 'InitialQuery'")
+query_countQ=$($CLICKHOUSE_CLIENT -q "SELECT value FROM system.events WHERE event = 'Query'")
+
+# Execute SELECT * FROM distributed
+$CLICKHOUSE_CLIENT -q "SELECT * FROM distributed SETTINGS prefer_localhost_replica = 0" > /dev/null
+
+# Counts after SELECT * FROM distributed
+After_query_countI=$($CLICKHOUSE_CLIENT -q "SELECT value FROM system.events WHERE event = 'InitialQuery'")
+After_query_countQ=$($CLICKHOUSE_CLIENT -q "SELECT value FROM system.events WHERE event = 'Query'")
+
+# Calculate the differences
+Initial_query_diff=$(($After_query_countI-$query_countI-2))
+query_diff=$(($After_query_countQ-$query_countQ-2))
+
+echo "Initial Query Difference: $Initial_query_diff"
+echo "Query Difference: $query_diff"

tests/queries/0_stateless/02981_vertical_merges_memory_usage.sql

@@ -13,8 +13,8 @@ SETTINGS
     merge_max_block_size = 8192,
     merge_max_block_size_bytes = '10M';
 
-INSERT INTO t_vertical_merge_memory SELECT number, arrayMap(x -> repeat('a', 50), range(1000)) FROM numbers(30000);
-INSERT INTO t_vertical_merge_memory SELECT number, arrayMap(x -> repeat('a', 50), range(1000)) FROM numbers(30000);
+INSERT INTO t_vertical_merge_memory SELECT number, arrayMap(x -> repeat('a', 50), range(1000)) FROM numbers(3000);
+INSERT INTO t_vertical_merge_memory SELECT number, arrayMap(x -> repeat('a', 50), range(1000)) FROM numbers(3000);
 
 OPTIMIZE TABLE t_vertical_merge_memory FINAL;
 

tests/queries/0_stateless/02982_aggregation_states_destruction.reference

@@ -0,0 +1 @@
+1

tests/queries/0_stateless/02982_aggregation_states_destruction.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# Tags: no-random-settings
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+
+query_id="02982_$RANDOM"
+$CLICKHOUSE_CLIENT --query_id $query_id --log_query_threads 1 --query="select number, uniq(number) from numbers_mt(1e7) group by number limit 100 format Null;"
+
+$CLICKHOUSE_CLIENT -q "system flush logs;"
+
+$CLICKHOUSE_CLIENT -q "select count() > 1 from system.query_thread_log where query_id = '$query_id' and current_database = currentDatabase() and thread_name = 'AggregDestruct';"

tests/queries/0_stateless/02982_json_columns_with_metadata_http.reference

@@ -0,0 +1,3 @@
+1	4
+2	5
+3	6