Merge branch 'master' into in_memory_raft

base/common/DateLUTImpl.h

@@ -851,7 +851,7 @@ public:
     }
 
     /// Saturation can occur if 29 Feb is mapped to non-leap year.
-    inline time_t addYears(time_t t, Int64 delta) const
+    inline NO_SANITIZE_UNDEFINED time_t addYears(time_t t, Int64 delta) const
     {
         DayNum result_day = addYears(toDayNum(t), delta);
 

base/common/extended_types.h

@@ -104,8 +104,3 @@ template <> struct is_big_int<wUInt256> { static constexpr bool value = true; };
 template <typename T>
 inline constexpr bool is_big_int_v = is_big_int<T>::value;
 
-template <typename To, typename From>
-inline To bigint_cast(const From & x [[maybe_unused]])
-{
-    return static_cast<To>(x);
-}

contrib/cassandra

@@ -1 +1 @@
-Subproject commit 9cbc1a806df5d40fddbf84533b9873542c6513d8
+Subproject commit b446d7eb68e6962f431e2b3771313bfe9a2bbd93

docker/test/performance-comparison/perf.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import argparse
 import clickhouse_driver

docs/en/engines/table-engines/integrations/s3.md

@@ -114,6 +114,10 @@ CREATE TABLE big_table (name String, value UInt32) ENGINE = S3('https://storage.
 -   `_path` — Path to the file.
 -   `_file` — Name of the file.
 
+**See Also**
+
+-   [Virtual columns](../../../engines/table-engines/index.md#table_engines-virtual_columns)
+
 ## S3-related settings {#settings}
 
 The following settings can be set before query execution or placed into configuration file.
@@ -124,8 +128,29 @@ The following settings can be set before query execution or placed into configur
 
 Security consideration: if malicious user can specify arbitrary S3 URLs, `s3_max_redirects` must be set to zero to avoid [SSRF](https://en.wikipedia.org/wiki/Server-side_request_forgery) attacks; or alternatively, `remote_host_filter` must be specified in server configuration.
 
-**See Also**
+### Endpoint-based settings {#endpointsettings}
 
--   [Virtual columns](../../../engines/table-engines/index.md#table_engines-virtual_columns)
+The following settings can be specified in configuration file for given endpoint (which will be matched by exact prefix of a URL):
+
+-   `endpoint` — Mandatory. Specifies prefix of an endpoint.
+-   `access_key_id` and `secret_access_key` — Optional. Specifies credentials to use with given endpoint.
+-   `use_environment_credentials` — Optional, default value is `false`. If set to `true`, S3 client will try to obtain credentials from environment variables and Amazon EC2 metadata for given endpoint.
+-   `header` — Optional, can be speficied multiple times. Adds specified HTTP header to a request to given endpoint.
+
+This configuration also applies to S3 disks in `MergeTree` table engine family.
+
+Example:
+
+```
+<s3>
+    <endpoint-name>
+        <endpoint>https://storage.yandexcloud.net/my-test-bucket-768/</endpoint>
+        <!-- <access_key_id>ACCESS_KEY_ID</access_key_id> -->
+        <!-- <secret_access_key>SECRET_ACCESS_KEY</secret_access_key> -->
+        <!-- <use_environment_credentials>false</use_environment_credentials> -->
+        <!-- <header>Authorization: Bearer SOME-TOKEN</header> -->
+    </endpoint-name>
+</s3>
+```
 
 [Original article](https://clickhouse.tech/docs/en/operations/table_engines/s3/) <!--hide-->

docs/en/getting-started/tutorial.md

@@ -254,7 +254,6 @@ ENGINE = MergeTree()
 PARTITION BY toYYYYMM(EventDate)
 ORDER BY (CounterID, EventDate, intHash32(UserID))
 SAMPLE BY intHash32(UserID)
-SETTINGS index_granularity = 8192
 ```
 
 ``` sql
@@ -450,7 +449,6 @@ ENGINE = CollapsingMergeTree(Sign)
 PARTITION BY toYYYYMM(StartDate)
 ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)
 SAMPLE BY intHash32(UserID)
-SETTINGS index_granularity = 8192
 ```
 
 You can execute those queries using the interactive mode of `clickhouse-client` (just launch it in a terminal without specifying a query in advance) or try some [alternative interface](../interfaces/index.md) if you want.

docs/en/operations/settings/settings.md

@@ -2489,7 +2489,6 @@ Possible values:
 
 Default value: `0`.
 
-
 ## aggregate_functions_null_for_empty {#aggregate_functions_null_for_empty}
 
 Enables or disables rewriting all aggregate functions in a query, adding [-OrNull](../../sql-reference/aggregate-functions/combinators.md#agg-functions-combinator-ornull) suffix to them. Enable it for SQL standard compatibility.
@@ -2523,7 +2522,6 @@ With `aggregate_functions_null_for_empty = 1` the result would be:
 └───────────────┴──────────────┘
 ```
 
-
 ## union_default_mode {#union-default-mode}
 
 Sets a mode for combining `SELECT` query results. The setting is only used when shared with [UNION](../../sql-reference/statements/select/union.md) without explicitly specifying the `UNION ALL` or `UNION DISTINCT`.
@@ -2538,7 +2536,6 @@ Default value: `''`.
 
 See examples in [UNION](../../sql-reference/statements/select/union.md).
 
-
 ## data_type_default_nullable {#data_type_default_nullable}
 
 Allows data types without explicit modifiers [NULL or NOT NULL](../../sql-reference/statements/create/table.md#null-modifiers) in column definition will be [Nullable](../../sql-reference/data-types/nullable.md#data_type-nullable).
@@ -2550,7 +2547,6 @@ Possible values:
 
 Default value: `0`.
 
-
 ## execute_merges_on_single_replica_time_threshold {#execute-merges-on-single-replica-time-threshold}
 
 Enables special logic to perform merges on replicas.
@@ -2570,4 +2566,15 @@ High values for that threshold may lead to replication delays.
 
 It can be useful when merges are CPU bounded not IO bounded (performing heavy data compression, calculating aggregate functions or default expressions that require a large amount of calculations, or just very high number of tiny merges).
 
+## max_final_threads {#max-final-threads}
+
+Sets the maximum number of parallel threads for the `SELECT` query data read phase with the [FINAL](../../sql-reference/statements/select/from.md#select-from-final) modifier.
+
+Possible values:
+
+-   Positive integer.
+-   0 or 1 — Disabled. `SELECT` queries are executed in a single thread.
+
+Default value: `16`.
+
 [Original article](https://clickhouse.tech/docs/en/operations/settings/settings/) <!-- hide -->

docs/en/sql-reference/functions/math-functions.md

@@ -413,4 +413,68 @@ Result:
 
 -   [log(x)](../../sql-reference/functions/math-functions.md#logx-lnx)
 
+## sign(x) {#signx}
+
+The `sign` function can extract the sign of a real number.
+
+**Syntax**
+
+``` sql
+sign(x)
+```
+
+**Parameters**
+
+-   `x` — Values from  `-∞` to `+∞`. Support all numeric types in ClickHouse.
+
+**Returned value**
+
+- -1 for `x < 0`
+-  0 for `x = 0`
+-  1 for `x > 0`
+
+**Example**
+
+Query:
+
+``` sql
+SELECT sign(0);
+```
+
+Result:
+
+``` text
+┌─sign(0)─┐
+│       0 │
+└─────────┘
+```
+
+Query:
+
+``` sql
+SELECT sign(1);
+```
+
+Result:
+
+``` text
+┌─sign(1)─┐
+│       1 │
+└─────────┘
+```
+
+Query:
+
+``` sql
+SELECT sign(-1);
+```
+
+Result:
+
+``` text
+┌─sign(-1)─┐
+│       -1 │
+└──────────┘
+```
+
 [Original article](https://clickhouse.tech/docs/en/query_language/functions/math_functions/) <!--hide-->

docs/en/sql-reference/statements/create/table.md

@@ -45,7 +45,7 @@ Creates a table with the same structure as another table. You can specify a diff
 CREATE TABLE [IF NOT EXISTS] [db.]table_name AS table_function()
 ```
 
-Creates a table with the structure and data returned by a [table function](../../../sql-reference/table-functions/index.md#table-functions).
+Creates a table with the same result as that of the [table function](../../../sql-reference/table-functions/index.md#table-functions) specified. The created table will also work in the same way as the corresponding table function that was specified.
 
 ``` sql
 CREATE TABLE [IF NOT EXISTS] [db.]table_name ENGINE = engine AS SELECT ...

docs/en/sql-reference/statements/insert-into.md

@@ -30,7 +30,6 @@ CREATE TABLE insert_select_testtable
 )
 ENGINE = MergeTree()
 ORDER BY a
-SETTINGS index_granularity = 8192 
 ```
 
 ``` sql

docs/en/sql-reference/statements/select/from.md

@@ -25,6 +25,8 @@ It is applicable when selecting data from tables that use the [MergeTree](../../
 -   [Replicated](../../../engines/table-engines/mergetree-family/replication.md) versions of `MergeTree` engines.
 -   [View](../../../engines/table-engines/special/view.md), [Buffer](../../../engines/table-engines/special/buffer.md), [Distributed](../../../engines/table-engines/special/distributed.md), and [MaterializedView](../../../engines/table-engines/special/materializedview.md) engines that operate over other engines, provided they were created over `MergeTree`-engine tables.
 
+Now `SELECT` queries with `FINAL` are executed in parallel and slightly faster. But there are drawbacks (see below). The [max_final_threads](../../../operations/settings/settings.md#max-final-threads) setting limits the number of threads used.
+
 ### Drawbacks {#drawbacks}
 
 Queries that use `FINAL` are executed slightly slower than similar queries that don’t, because:

docs/es/getting-started/tutorial.md

@@ -256,7 +256,6 @@ ENGINE = MergeTree()
 PARTITION BY toYYYYMM(EventDate)
 ORDER BY (CounterID, EventDate, intHash32(UserID))
 SAMPLE BY intHash32(UserID)
-SETTINGS index_granularity = 8192
 ```
 
 ``` sql
@@ -452,7 +451,6 @@ ENGINE = CollapsingMergeTree(Sign)
 PARTITION BY toYYYYMM(StartDate)
 ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)
 SAMPLE BY intHash32(UserID)
-SETTINGS index_granularity = 8192
 ```
 
 Puede ejecutar esas consultas utilizando el modo interactivo de `clickhouse-client` (simplemente ejecútelo en un terminal sin especificar una consulta por adelantado) o pruebe algunos [interfaz alternativa](../interfaces/index.md) Si quieres.

docs/fr/getting-started/tutorial.md

@@ -256,7 +256,6 @@ ENGINE = MergeTree()
 PARTITION BY toYYYYMM(EventDate)
 ORDER BY (CounterID, EventDate, intHash32(UserID))
 SAMPLE BY intHash32(UserID)
-SETTINGS index_granularity = 8192
 ```
 
 ``` sql
@@ -452,7 +451,6 @@ ENGINE = CollapsingMergeTree(Sign)
 PARTITION BY toYYYYMM(StartDate)
 ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)
 SAMPLE BY intHash32(UserID)
-SETTINGS index_granularity = 8192
 ```
 
 Vous pouvez exécuter ces requêtes en utilisant le mode interactif de `clickhouse-client` (lancez - le simplement dans un terminal sans spécifier une requête à l'avance) ou essayez-en [interface de rechange](../interfaces/index.md) Si tu veux.

docs/ja/getting-started/tutorial.md

@@ -262,7 +262,6 @@ ENGINE = MergeTree()
 PARTITION BY toYYYYMM(EventDate)
 ORDER BY (CounterID, EventDate, intHash32(UserID))
 SAMPLE BY intHash32(UserID)
-SETTINGS index_granularity = 8192
 ```
 
 ``` sql
@@ -458,7 +457,6 @@ ENGINE = CollapsingMergeTree(Sign)
 PARTITION BY toYYYYMM(StartDate)
 ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)
 SAMPLE BY intHash32(UserID)
-SETTINGS index_granularity = 8192
 ```
 
 これらのクエリは、`clickhouse-client` の対話型モード(事前にクエリを指定せずにターミナルで起動するだけです)を使って実行するか、[代替インターフェイス](../interfaces/index.md) で実行できます。

docs/ru/getting-started/tutorial.md

@@ -254,7 +254,6 @@ ENGINE = MergeTree()
 PARTITION BY toYYYYMM(EventDate)
 ORDER BY (CounterID, EventDate, intHash32(UserID))
 SAMPLE BY intHash32(UserID)
-SETTINGS index_granularity = 8192
 ```
 
 ``` sql
@@ -450,7 +449,6 @@ ENGINE = CollapsingMergeTree(Sign)
 PARTITION BY toYYYYMM(StartDate)
 ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)
 SAMPLE BY intHash32(UserID)
-SETTINGS index_granularity = 8192
 ```
 
 You can execute those queries using the interactive mode of `clickhouse-client` (just launch it in a terminal without specifying a query in advance) or try some [alternative interface](../interfaces/index.md) if you want.

docs/ru/operations/settings/settings.md

@@ -2437,4 +2437,15 @@ SELECT SUM(-1), MAX(0) FROM system.one WHERE 0;
 
 Эта настройка полезна, когда скорость слияния ограничивается мощностью процессора, а не скоростью операций ввода-вывода (при выполнении "тяжелого" сжатия данных, при расчете агрегатных функций или выражений по умолчанию, требующих большого объема вычислений, или просто при большом количестве мелких слияний).
 
+## max_final_threads {#max-final-threads}
+
+Устанавливает максимальное количество параллельных потоков для фазы чтения данных запроса `SELECT` с модификатором [FINAL](../../sql-reference/statements/select/from.md#select-from-final).
+
+Возможные значения:
+
+-   Положительное целое число.
+-   0 или 1 — настройка отключена. `SELECT` запросы выполняются в один поток.
+
+Значение по умолчанию: `16`.
+
 [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/settings/) <!--hide-->

docs/ru/sql-reference/statements/insert-into.md

@@ -30,8 +30,7 @@ SHOW CREATE insert_select_testtable
     `c` Int8
 )
 ENGINE = MergeTree()
-ORDER BY a
+ORDER BY a │
-SETTINGS index_granularity = 8192 │
 └──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
 ```
 

docs/ru/sql-reference/statements/select/from.md

@@ -25,6 +25,8 @@ toc_title: FROM
 -   [Replicated](../../../engines/table-engines/mergetree-family/replication.md) варианты исполнения `MergeTree` движков.
 -   [View](../../../engines/table-engines/special/view.md), [Buffer](../../../engines/table-engines/special/buffer.md), [Distributed](../../../engines/table-engines/special/distributed.md), и [MaterializedView](../../../engines/table-engines/special/materializedview.md), которые работают поверх других движков, если они созданы для таблиц с движками семейства `MergeTree`.
 
+Теперь `SELECT` запросы с `FINAL` выполняются параллельно и, следовательно, немного быстрее. Но имеются серьезные недостатки при их использовании (смотрите ниже). Настройка [max_final_threads](../../../operations/settings/settings.md#max-final-threads) устанавливает максимальное количество потоков.
+
 ### Недостатки {#drawbacks}
 
 Запросы, которые используют `FINAL` выполняются немного медленее, чем аналогичные запросы без него, потому что:

docs/zh/getting-started/tutorial.md

@@ -254,7 +254,6 @@ ENGINE = MergeTree()
 PARTITION BY toYYYYMM(EventDate)
 ORDER BY (CounterID, EventDate, intHash32(UserID))
 SAMPLE BY intHash32(UserID)
-SETTINGS index_granularity = 8192
 ```
 
 ``` sql
@@ -450,7 +449,6 @@ ENGINE = CollapsingMergeTree(Sign)
 PARTITION BY toYYYYMM(StartDate)
 ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)
 SAMPLE BY intHash32(UserID)
-SETTINGS index_granularity = 8192
 ```
 
 您可以使用`clickhouse-client`的交互模式执行这些查询（只需在终端中启动它，而不需要提前指定查询）。或者如果你愿意，可以尝试一些[替代接口](../interfaces/index.md)。

docs/zh/sql-reference/statements/insert-into.md

@@ -25,7 +25,6 @@ CREATE TABLE insert_select_testtable
 )
 ENGINE = MergeTree()
 ORDER BY a
-SETTINGS index_granularity = 8192 
 ```
 
 ``` sql

src/Client/Connection.cpp

@@ -138,6 +138,7 @@ void Connection::connect(const ConnectionTimeouts & timeouts)
 
 void Connection::disconnect()
 {
+    maybe_compressed_out = nullptr;
     in = nullptr;
     last_input_packet_type.reset();
     out = nullptr; // can write to socket

src/Columns/ColumnDecimal.cpp

@@ -4,7 +4,6 @@
 #include <Common/assert_cast.h>
 #include <Common/WeakHash.h>
 #include <Common/HashTable/Hash.h>
-#include <Core/BigInt.h>
 
 #include <common/unaligned.h>
 #include <common/sort.h>

src/Columns/ColumnVector.cpp

@@ -36,35 +36,18 @@ namespace ErrorCodes
 
 template <typename T>
 StringRef ColumnVector<T>::serializeValueIntoArena(size_t n, Arena & arena, char const *& begin) const
-{
-    if constexpr (is_big_int_v<T>)
-    {
-        static constexpr size_t bytesize = BigInt<T>::size;
-        char * pos = arena.allocContinue(bytesize, begin);
-        return BigInt<T>::serialize(data[n], pos);
-    }
-    else
 {
     auto * pos = arena.allocContinue(sizeof(T), begin);
     unalignedStore<T>(pos, data[n]);
     return StringRef(pos, sizeof(T));
 }
-}
 
 template <typename T>
 const char * ColumnVector<T>::deserializeAndInsertFromArena(const char * pos)
-{
-    if constexpr (is_big_int_v<T>)
-    {
-        data.emplace_back(BigInt<T>::deserialize(pos));
-        return pos + BigInt<T>::size;
-    }
-    else
 {
     data.emplace_back(unalignedLoad<T>(pos));
     return pos + sizeof(T);
 }
-}
 
 template <typename T>
 void ColumnVector<T>::updateHashWithValue(size_t n, SipHash & hash) const
@@ -299,19 +282,11 @@ MutableColumnPtr ColumnVector<T>::cloneResized(size_t size) const
         new_col.data.resize(size);
 
         size_t count = std::min(this->size(), size);
-        if constexpr (is_POD)
-        {
         memcpy(new_col.data.data(), data.data(), count * sizeof(data[0]));
 
         if (size > count)
             memset(static_cast<void *>(&new_col.data[count]), static_cast<int>(ValueType()), (size - count) * sizeof(ValueType));
     }
-        else
-        {
-            for (size_t i = 0; i < count; i++)
-                new_col.data[i] = data[i];
-        }
-    }
 
     return res;
 }
@@ -348,16 +323,8 @@ void ColumnVector<T>::insertRangeFrom(const IColumn & src, size_t start, size_t
 
     size_t old_size = data.size();
     data.resize(old_size + length);
-    if constexpr (is_POD)
-    {
     memcpy(data.data() + old_size, &src_vec.data[start], length * sizeof(data[0]));
 }
-    else
-    {
-        for (size_t i = 0; i < length; i++)
-            data[old_size + i] = src_vec.data[start + i];
-    }
-}
 
 template <typename T>
 ColumnPtr ColumnVector<T>::filter(const IColumn::Filter & filt, ssize_t result_size_hint) const
@@ -372,8 +339,6 @@ ColumnPtr ColumnVector<T>::filter(const IColumn::Filter & filt, ssize_t result_s
     if (result_size_hint)
         res_data.reserve(result_size_hint > 0 ? result_size_hint : size);
 
-    if constexpr (is_POD)
-    {
     const UInt8 * filt_pos = filt.data();
     const UInt8 * filt_end = filt_pos + size;
     const T * data_pos = data.data();
@@ -421,22 +386,6 @@ ColumnPtr ColumnVector<T>::filter(const IColumn::Filter & filt, ssize_t result_s
         ++filt_pos;
         ++data_pos;
     }
-    }
-    else
-    {
-        const auto * filt_pos = filt.begin();
-        const auto * filt_end = filt.end();
-        auto data_pos = data.begin();
-
-        while (filt_pos < filt_end)
-        {
-            if (*filt_pos)
-                res_data.push_back(*data_pos);
-
-            ++filt_pos;
-            ++data_pos;
-        }
-    }
 
     return res;
 }

src/Columns/ColumnVector.h

@@ -6,7 +6,6 @@
 #include <Columns/ColumnVectorHelper.h>
 #include <common/unaligned.h>
 #include <Core/Field.h>
-#include <Core/BigInt.h>
 #include <Common/assert_cast.h>
 
 
@@ -107,10 +106,7 @@ private:
 
 public:
     using ValueType = T;
-    static constexpr bool is_POD = !is_big_int_v<T>;
+    using Container = PaddedPODArray<ValueType>;
-    using Container = std::conditional_t<is_POD,
-                                         PaddedPODArray<ValueType>,
-                                         std::vector<ValueType>>;
 
 private:
     ColumnVector() {}
@@ -136,10 +132,7 @@ public:
 
     void insertData(const char * pos, size_t) override
     {
-        if constexpr (is_POD)
         data.emplace_back(unalignedLoad<T>(pos));
-        else
-            data.emplace_back(BigInt<T>::deserialize(pos));
     }
 
     void insertDefault() override
@@ -149,18 +142,12 @@ public:
 
     void insertManyDefaults(size_t length) override
     {
-        if constexpr (is_POD)
         data.resize_fill(data.size() + length, T());
-        else
-            data.resize(data.size() + length, T());
     }
 
     void popBack(size_t n) override
     {
-        if constexpr (is_POD)
         data.resize_assume_reserved(data.size() - n);
-        else
-            data.resize(data.size() - n);
     }
 
     StringRef serializeValueIntoArena(size_t n, Arena & arena, char const *& begin) const override;
@@ -185,15 +172,11 @@ public:
 
     size_t allocatedBytes() const override
     {
-        if constexpr (is_POD)
         return data.allocated_bytes();
-        else
-            return data.capacity() * sizeof(data[0]);
     }
 
     void protect() override
     {
-        if constexpr (is_POD)
         data.protect();
     }
 

src/Common/FieldVisitors.h

@@ -235,7 +235,7 @@ public:
         else if constexpr (std::is_same_v<T, UInt128>)
             throw Exception("No conversion to old UInt128 from " + demangle(typeid(U).name()), ErrorCodes::NOT_IMPLEMENTED);
         else
-            return bigint_cast<T>(x);
+            return static_cast<T>(x);
     }
 };
 

src/Common/HashTable/Hash.h

@@ -1,7 +1,6 @@
 #pragma once
 
 #include <common/types.h>
-#include <Core/BigInt.h>
 #include <Common/UInt128.h>
 #include <common/unaligned.h>
 

src/Common/SipHash.h

@@ -18,7 +18,7 @@
 #include <string>
 #include <type_traits>
 #include <Core/Defines.h>
-#include <Core/BigInt.h>
+
 
 #define ROTL(x, b) static_cast<UInt64>(((x) << (b)) | ((x) >> (64 - (b))))
 
@@ -136,23 +136,11 @@ public:
     }
 
     template <typename T>
-    std::enable_if_t<std::has_unique_object_representations_v<T>, void> update(const T & x)
+    void update(const T & x)
     {
         update(reinterpret_cast<const char *>(&x), sizeof(x));
     }
 
-    template <typename T>
-    std::enable_if_t<(std::is_floating_point_v<T> || std::is_same_v<T, CityHash_v1_0_2::uint128>), void> update(const T & x)
-    {
-        update(reinterpret_cast<const char *>(&x), sizeof(x));
-    }
-
-    template <typename T>
-    std::enable_if_t<is_big_int_v<T> && !std::has_unique_object_representations_v<T>, void> update(const T & x)
-    {
-        update(DB::BigInt<T>::serialize(x));
-    }
-
     void update(const std::string & x)
     {
         update(x.data(), x.length());
@@ -205,27 +193,13 @@ inline UInt64 sipHash64(const char * data, const size_t size)
 }
 
 template <typename T>
-std::enable_if_t<std::has_unique_object_representations_v<T>, UInt64> sipHash64(const T & x)
+UInt64 sipHash64(const T & x)
 {
     SipHash hash;
     hash.update(x);
     return hash.get64();
 }
 
-template <typename T>
-std::enable_if_t<(std::is_floating_point_v<T> || (is_big_int_v<T> && !std::has_unique_object_representations_v<T>)), UInt64> sipHash64(const T & x)
-{
-    SipHash hash;
-    hash.update(x);
-    return hash.get64();
-}
-
-template <typename T>
-std::enable_if_t<DB::IsDecimalNumber<T>, UInt64> sipHash64(const T & x)
-{
-    return sipHash64(x.value);
-}
-
 inline UInt64 sipHash64(const std::string & s)
 {
     return sipHash64(s.data(), s.size());

src/Common/ZooKeeper/TestKeeperStorageDispatcher.cpp

@@ -0,0 +1,139 @@
+#include <Common/ZooKeeper/TestKeeperStorageDispatcher.h>
+#include <Common/setThreadName.h>
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+
+    extern const int LOGICAL_ERROR;
+    extern const int TIMEOUT_EXCEEDED;
+}
+
+}
+namespace zkutil
+{
+
+void TestKeeperStorageDispatcher::processingThread()
+{
+    setThreadName("TestKeeperSProc");
+
+    while (!shutdown)
+    {
+        RequestInfo info;
+
+        UInt64 max_wait = UInt64(operation_timeout.totalMilliseconds());
+
+        if (requests_queue.tryPop(info, max_wait))
+        {
+            if (shutdown)
+                break;
+
+            try
+            {
+                auto responses = storage.processRequest(info.request, info.session_id);
+                for (const auto & response_for_session : responses)
+                    setResponse(response_for_session.session_id, response_for_session.response);
+            }
+            catch (...)
+            {
+                tryLogCurrentException(__PRETTY_FUNCTION__);
+            }
+        }
+    }
+}
+
+void TestKeeperStorageDispatcher::setResponse(int64_t session_id, const Coordination::ZooKeeperResponsePtr & response)
+{
+    std::lock_guard lock(session_to_response_callback_mutex);
+    auto session_writer = session_to_response_callback.find(session_id);
+    if (session_writer == session_to_response_callback.end())
+        return;
+
+    session_writer->second(response);
+    /// Session closed, no more writes
+    if (response->xid != Coordination::WATCH_XID && response->getOpNum() == Coordination::OpNum::Close)
+        session_to_response_callback.erase(session_writer);
+}
+
+void TestKeeperStorageDispatcher::finalize()
+{
+    {
+        std::lock_guard lock(push_request_mutex);
+
+        if (shutdown)
+            return;
+
+        shutdown = true;
+
+        if (processing_thread.joinable())
+            processing_thread.join();
+    }
+
+    RequestInfo info;
+    TestKeeperStorage::RequestsForSessions expired_requests;
+    while (requests_queue.tryPop(info))
+        expired_requests.push_back(TestKeeperStorage::RequestForSession{info.session_id, info.request});
+
+    auto expired_responses = storage.finalize(expired_requests);
+
+    for (const auto & response_for_session : expired_responses)
+        setResponse(response_for_session.session_id, response_for_session.response);
+}
+
+void TestKeeperStorageDispatcher::putRequest(const Coordination::ZooKeeperRequestPtr & request, int64_t session_id)
+{
+
+    {
+        std::lock_guard lock(session_to_response_callback_mutex);
+        if (session_to_response_callback.count(session_id) == 0)
+            throw Exception(DB::ErrorCodes::LOGICAL_ERROR, "Unknown session id {}", session_id);
+    }
+
+    RequestInfo request_info;
+    request_info.time = clock::now();
+    request_info.request = request;
+    request_info.session_id = session_id;
+
+    std::lock_guard lock(push_request_mutex);
+    /// Put close requests without timeouts
+    if (request->getOpNum() == Coordination::OpNum::Close)
+        requests_queue.push(std::move(request_info));
+    else if (!requests_queue.tryPush(std::move(request_info), operation_timeout.totalMilliseconds()))
+        throw Exception("Cannot push request to queue within operation timeout", ErrorCodes::TIMEOUT_EXCEEDED);
+}
+
+TestKeeperStorageDispatcher::TestKeeperStorageDispatcher()
+{
+    processing_thread = ThreadFromGlobalPool([this] { processingThread(); });
+}
+
+TestKeeperStorageDispatcher::~TestKeeperStorageDispatcher()
+{
+    try
+    {
+        finalize();
+    }
+    catch (...)
+    {
+        tryLogCurrentException(__PRETTY_FUNCTION__);
+    }
+}
+
+void TestKeeperStorageDispatcher::registerSession(int64_t session_id, ZooKeeperResponseCallback callback)
+{
+    std::lock_guard lock(session_to_response_callback_mutex);
+    if (!session_to_response_callback.try_emplace(session_id, callback).second)
+        throw Exception(DB::ErrorCodes::LOGICAL_ERROR, "Session with id {} already registered in dispatcher", session_id);
+}
+
+void TestKeeperStorageDispatcher::finishSession(int64_t session_id)
+{
+    std::lock_guard lock(session_to_response_callback_mutex);
+    auto session_it = session_to_response_callback.find(session_id);
+    if (session_it != session_to_response_callback.end())
+        session_to_response_callback.erase(session_it);
+}
+
+}

src/Compression/CompressionCodecT64.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include <common/types.h>
+#include <Core/Types.h>
 #include <Compression/ICompressionCodec.h>
 
 

src/Core/AccurateComparison.h

@@ -93,7 +93,7 @@ using bool_if_gt_int_vs_uint = std::enable_if_t<is_gt_int_vs_uint<TInt, TUInt>,
 template <typename TInt, typename TUInt>
 inline bool_if_gt_int_vs_uint<TInt, TUInt> greaterOpTmpl(TInt a, TUInt b)
 {
-    return bigint_cast<TInt>(a) > bigint_cast<TInt>(b);
+    return static_cast<TInt>(a) > static_cast<TInt>(b);
 }
 
 template <typename TInt, typename TUInt>
@@ -101,19 +101,19 @@ inline bool_if_gt_int_vs_uint<TInt, TUInt> greaterOpTmpl(TUInt a, TInt b)
 {
     using CastA = std::conditional_t<is_big_int_v<TInt> && std::is_same_v<TUInt, DB::UInt128>, DB::UInt256, TInt>;
 
-    return bigint_cast<CastA>(a) > b;
+    return static_cast<CastA>(a) > b;
 }
 
 template <typename TInt, typename TUInt>
 inline bool_if_gt_int_vs_uint<TInt, TUInt> equalsOpTmpl(TInt a, TUInt b)
 {
-    return bigint_cast<TInt>(a) == bigint_cast<TInt>(b);
+    return static_cast<TInt>(a) == static_cast<TInt>(b);
 }
 
 template <typename TInt, typename TUInt>
 inline bool_if_gt_int_vs_uint<TInt, TUInt> equalsOpTmpl(TUInt a, TInt b)
 {
-    return bigint_cast<TInt>(a) == bigint_cast<TInt>(b);
+    return static_cast<TInt>(a) == static_cast<TInt>(b);
 }
 
 
@@ -196,7 +196,7 @@ inline bool_if_safe_conversion<A, B> greaterOp(A a, B b)
     using CastB = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, DB::UInt128>, A, CastB1>;
 
     if constexpr (is_big_int_v<A> || is_big_int_v<B>)
-        return bigint_cast<CastA>(a) > bigint_cast<CastB>(b);
+        return static_cast<CastA>(a) > static_cast<CastB>(b);
     else
         return a > b;
 }
@@ -306,7 +306,7 @@ inline bool_if_safe_conversion<A, B> equalsOp(A a, B b)
 {
     using LargestType = std::conditional_t<(sizeof(A) > sizeof(B)) || ((sizeof(A) == sizeof(B)) && !std::is_same_v<A, DB::UInt128>), A, B>;
 
-    return bigint_cast<LargestType>(a) == bigint_cast<LargestType>(b);
+    return static_cast<LargestType>(a) == static_cast<LargestType>(b);
 }
 
 template <>
@@ -429,7 +429,7 @@ inline bool_if_safe_conversion<A, B> notEqualsOp(A a, B b)
     using CastB = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, DB::UInt128>, A, CastB1>;
 
     if constexpr (is_big_int_v<A> || is_big_int_v<B>)
-        return bigint_cast<CastA>(a) != bigint_cast<CastB>(b);
+        return static_cast<CastA>(a) != static_cast<CastB>(b);
     else
         return a != b;
 }
@@ -451,7 +451,7 @@ inline bool_if_safe_conversion<A, B> lessOp(A a, B b)
     using CastB = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, DB::UInt128>, A, CastB1>;
 
     if constexpr (is_big_int_v<A> || is_big_int_v<B>)
-        return bigint_cast<CastA>(a) < bigint_cast<CastB>(b);
+        return static_cast<CastA>(a) < static_cast<CastB>(b);
     else
         return a < b;
 }
@@ -475,7 +475,7 @@ inline bool_if_safe_conversion<A, B> lessOrEqualsOp(A a, B b)
     using CastB = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, DB::UInt128>, A, CastB1>;
 
     if constexpr (is_big_int_v<A> || is_big_int_v<B>)
-        return bigint_cast<CastA>(a) <= bigint_cast<CastB>(b);
+        return static_cast<CastA>(a) <= static_cast<CastB>(b);
     else
         return a <= b;
 }
@@ -499,7 +499,7 @@ inline bool_if_safe_conversion<A, B> greaterOrEqualsOp(A a, B b)
     using CastB = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, DB::UInt128>, A, CastB1>;
 
     if constexpr (is_big_int_v<A> || is_big_int_v<B>)
-        return bigint_cast<CastA>(a) >= bigint_cast<CastB>(b);
+        return static_cast<CastA>(a) >= static_cast<CastB>(b);
     else
         return a >= b;
 }

src/Core/BigInt.h

@@ -1,36 +0,0 @@
-#pragma once
-
-#include <common/StringRef.h>
-#include <common/unaligned.h>
-#include <Core/Types.h>
-
-
-namespace DB
-{
-
-template <typename T>
-struct BigInt
-{
-    static_assert(sizeof(T) == 32);
-    static constexpr size_t size = 32;
-
-    static StringRef serialize(const T & x, char * pos)
-    {
-        unalignedStore<T>(pos, x);
-        return StringRef(pos, size);
-    }
-
-    static String serialize(const T & x)
-    {
-        String str(size, '\0');
-        serialize(x, str.data());
-        return str;
-    }
-
-    static T deserialize(const char * pos)
-    {
-        return unalignedLoad<T>(pos);
-    }
-};
-
-}

src/Core/DecimalComparison.h

@@ -233,9 +233,9 @@ private:
             bool overflow = false;
 
             if constexpr (sizeof(A) > sizeof(CompareInt))
-                overflow |= (bigint_cast<A>(x) != a);
+                overflow |= (static_cast<A>(x) != a);
             if constexpr (sizeof(B) > sizeof(CompareInt))
-                overflow |= (bigint_cast<B>(y) != b);
+                overflow |= (static_cast<B>(y) != b);
             if constexpr (is_unsigned_v<A>)
                 overflow |= (x < 0);
             if constexpr (is_unsigned_v<B>)

src/Core/Settings.h

@@ -326,6 +326,7 @@ class IColumn;
     M(Bool, log_profile_events, true, "Log query performance statistics into the query_log and query_thread_log.", 0) \
     M(Bool, log_query_settings, true, "Log query settings into the query_log.", 0) \
     M(Bool, log_query_threads, true, "Log query threads into system.query_thread_log table. This setting have effect only when 'log_queries' is true.", 0) \
+    M(String, log_comment, "", "Log comment into system.query_log table and server log. It can be set to arbitrary string no longer than max_query_size.", 0) \
     M(LogsLevel, send_logs_level, LogsLevel::fatal, "Send server text logs with specified minimum level to client. Valid values: 'trace', 'debug', 'information', 'warning', 'error', 'fatal', 'none'", 0) \
     M(Bool, enable_optimize_predicate_expression, 1, "If it is set to true, optimize predicates to subqueries.", 0) \
     M(Bool, enable_optimize_predicate_expression_to_final_subquery, 1, "Allow push predicate to final subquery.", 0) \
@@ -404,7 +405,7 @@ class IColumn;
     M(MySQLDataTypesSupport, mysql_datatypes_support_level, 0, "Which MySQL types should be converted to corresponding ClickHouse types (rather than being represented as String). Can be empty or any combination of 'decimal' or 'datetime64'. When empty MySQL's DECIMAL and DATETIME/TIMESTAMP with non-zero precision are seen as String on ClickHouse's side.", 0) \
     M(Bool, optimize_trivial_insert_select, true, "Optimize trivial 'INSERT INTO table SELECT ... FROM TABLES' query", 0) \
     M(Bool, allow_non_metadata_alters, true, "Allow to execute alters which affects not only tables metadata, but also data on disk", 0) \
-    M(Bool, enable_global_with_statement, false, "Propagate WITH statements to UNION queries and all subqueries", 0) \
+    M(Bool, enable_global_with_statement, true, "Propagate WITH statements to UNION queries and all subqueries", 0) \
     M(Bool, aggregate_functions_null_for_empty, false, "Rewrite all aggregate functions in a query, adding -OrNull suffix to them", 0) \
     M(Bool, flatten_nested, true, "If true, columns of type Nested will be flatten to separate array columns instead of one array of tuples", 0) \
     M(Bool, asterisk_include_materialized_columns, false, "Include MATERIALIZED columns for wildcard query", 0) \

src/Core/Types.h

@@ -158,7 +158,7 @@ struct Decimal
             return convertTo<typename U::NativeType>();
         }
         else
-            return bigint_cast<U>(value);
+            return static_cast<U>(value);
     }
 
     const Decimal<T> & operator += (const T & x) { value += x; return *this; }

src/DataTypes/IDataType.h

@@ -597,6 +597,7 @@ inline bool isEnum(const DataTypePtr & data_type) { return WhichDataType(data_ty
 inline bool isDecimal(const DataTypePtr & data_type) { return WhichDataType(data_type).isDecimal(); }
 inline bool isTuple(const DataTypePtr & data_type) { return WhichDataType(data_type).isTuple(); }
 inline bool isArray(const DataTypePtr & data_type) { return WhichDataType(data_type).isArray(); }
+inline bool isMap(const DataTypePtr & data_type) {return WhichDataType(data_type).isMap(); }
 
 template <typename T>
 inline bool isUInt8(const T & data_type)

src/DataTypes/NumberTraits.h

@@ -218,7 +218,7 @@ using ResultOfGreatest = std::conditional_t<LeastGreatestSpecialCase<A, B>,
 template <typename T>
 static inline auto littleBits(const T & x)
 {
-    return bigint_cast<UInt8>(x);
+    return static_cast<UInt8>(x);
 }
 
 }

src/DataTypes/convertMySQLDataType.cpp

@@ -45,7 +45,7 @@ DataTypePtr convertMySQLDataType(MultiEnum<MySQLDataTypesSupport> type_support,
     // 4. type_with_params(param1, param2, ...) options
     // The options can be unsigned, zerofill, or some other strings.
     auto data_type = std::string_view(mysql_data_type);
-    const auto type_end_pos = data_type.find_first_of("(\x20"); // FIXME: fix style-check script instead
+    const auto type_end_pos = data_type.find_first_of(R"(( )"); // FIXME: fix style-check script instead
     const auto type_name = data_type.substr(0, type_end_pos);
 
     DataTypePtr res;

src/Dictionaries/DictionaryStructure.cpp

@@ -344,7 +344,9 @@ std::vector<DictionaryAttribute> DictionaryStructure::getAttributes(
             }
             catch (Exception & e)
             {
-                e.addMessage("error parsing null_value");
+                String dictionary_name = config.getString(".dictionary.name", "");
+                e.addMessage("While parsing null_value for attribute with name " + name
+                    + " in dictionary " + dictionary_name);
                 throw;
             }
         }

src/Functions/DivisionUtils.h

@@ -51,9 +51,9 @@ inline auto checkedDivision(A a, B b)
     throwIfDivisionLeadsToFPE(a, b);
 
     if constexpr (is_big_int_v<A> && std::is_floating_point_v<B>)
-        return bigint_cast<B>(a) / b;
+        return static_cast<B>(a) / b;
     else if constexpr (is_big_int_v<B> && std::is_floating_point_v<A>)
-        return a / bigint_cast<A>(b);
+        return a / static_cast<A>(b);
     else if constexpr (is_big_int_v<A> && is_big_int_v<B>)
         return static_cast<A>(a / b);
     else if constexpr (!is_big_int_v<A> && is_big_int_v<B>)
@@ -84,10 +84,10 @@ struct DivideIntegralImpl
             using SignedCastA = make_signed_t<CastA>;
             using SignedCastB = std::conditional_t<sizeof(A) <= sizeof(B), make_signed_t<CastB>, SignedCastA>;
 
-            return bigint_cast<Result>(checkedDivision(bigint_cast<SignedCastA>(a), bigint_cast<SignedCastB>(b)));
+            return static_cast<Result>(checkedDivision(static_cast<SignedCastA>(a), static_cast<SignedCastB>(b)));
         }
         else
-            return bigint_cast<Result>(checkedDivision(CastA(a), CastB(b)));
+            return static_cast<Result>(checkedDivision(CastA(a), CastB(b)));
     }
 
 #if USE_EMBEDDED_COMPILER
@@ -110,7 +110,7 @@ struct ModuloImpl
         if constexpr (std::is_floating_point_v<ResultType>)
         {
             /// This computation is similar to `fmod` but the latter is not inlined and has 40 times worse performance.
-            return bigint_cast<ResultType>(a) - trunc(bigint_cast<ResultType>(a) / bigint_cast<ResultType>(b)) * bigint_cast<ResultType>(b);
+            return static_cast<ResultType>(a) - trunc(static_cast<ResultType>(a) / static_cast<ResultType>(b)) * static_cast<ResultType>(b);
         }
         else
         {
@@ -125,9 +125,9 @@ struct ModuloImpl
                 CastB int_b(b);
 
                 if constexpr (is_big_int_v<IntegerBType> && sizeof(IntegerAType) <= sizeof(IntegerBType))
-                    return bigint_cast<Result>(bigint_cast<CastB>(int_a) % int_b);
+                    return static_cast<Result>(static_cast<CastB>(int_a) % int_b);
                 else
-                    return bigint_cast<Result>(int_a % bigint_cast<CastA>(int_b));
+                    return static_cast<Result>(int_a % static_cast<CastA>(int_b));
             }
             else
                 return IntegerAType(a) % IntegerBType(b);

src/Functions/FunctionStringOrArrayToT.h

@@ -7,6 +7,7 @@
 #include <Columns/ColumnString.h>
 #include <Columns/ColumnFixedString.h>
 #include <Columns/ColumnArray.h>
+#include <Columns/ColumnMap.h>
 
 
 namespace DB
@@ -42,7 +43,7 @@ public:
     DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
     {
         if (!isStringOrFixedString(arguments[0])
-            && !isArray(arguments[0]))
+            && !isArray(arguments[0]) && !isMap(arguments[0]))
             throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
 
         return std::make_shared<DataTypeNumber<ResultType>>();
@@ -93,6 +94,16 @@ public:
 
             return col_res;
         }
+        else if (const ColumnMap * col_map = checkAndGetColumn<ColumnMap>(column.get()))
+        {
+            auto col_res = ColumnVector<ResultType>::create();
+            typename ColumnVector<ResultType>::Container & vec_res = col_res->getData();
+            vec_res.resize(col_map->size());
+            const auto & col_nested = col_map->getNestedColumn();
+
+            Impl::array(col_nested.getOffsets(), vec_res);
+            return col_res;
+        }
         else
             throw Exception("Illegal column " + arguments[0].column->getName() + " of argument of function " + getName(),
                 ErrorCodes::ILLEGAL_COLUMN);

src/Functions/FunctionUnaryArithmetic.h

@@ -79,8 +79,9 @@ struct InvalidType;
 template <template <typename> class Op, typename Name, bool is_injective>
 class FunctionUnaryArithmetic : public IFunction
 {
-    static constexpr bool allow_decimal = IsUnaryOperation<Op>::negate || IsUnaryOperation<Op>::abs;
+    static constexpr bool allow_decimal = IsUnaryOperation<Op>::negate || IsUnaryOperation<Op>::abs || IsUnaryOperation<Op>::sign;
     static constexpr bool allow_fixed_string = Op<UInt8>::allow_fixed_string;
+    static constexpr bool is_sign_function = IsUnaryOperation<Op>::sign;
 
     template <typename F>
     static bool castType(const IDataType * type, F && f)
@@ -137,7 +138,7 @@ public:
             {
                 using T0 = typename DataType::FieldType;
 
-                if constexpr (IsDataTypeDecimal<DataType>)
+                if constexpr (IsDataTypeDecimal<DataType> && !is_sign_function)
                 {
                     if constexpr (!allow_decimal)
                         return false;
@@ -182,6 +183,17 @@ public:
                 if constexpr (allow_decimal)
                 {
                     if (auto col = checkAndGetColumn<ColumnDecimal<T0>>(arguments[0].column.get()))
+                    {
+                        if constexpr (is_sign_function)
+                        {
+                            auto col_res = ColumnVector<typename Op<T0>::ResultType>::create();
+                            auto & vec_res = col_res->getData();
+                            vec_res.resize(col->getData().size());
+                            UnaryOperationImpl<T0, Op<T0>>::vector(col->getData(), vec_res);
+                            result_column = std::move(col_res);
+                            return true;
+                        }
+                        else
                         {
                             auto col_res = ColumnDecimal<typename Op<T0>::ResultType>::create(0, type.getScale());
                             auto & vec_res = col_res->getData();
@@ -192,6 +204,7 @@ public:
                         }
                     }
                 }
+            }
             else
             {
                 using T0 = typename DataType::FieldType;

src/Functions/FunctionsComparison.h

@@ -81,8 +81,8 @@ namespace ErrorCodes
 template <typename A, typename B, typename Op>
 struct NumComparisonImpl
 {
-    using ContainerA = std::conditional_t<!is_big_int_v<A>, PaddedPODArray<A>, std::vector<A>>;
+    using ContainerA = PaddedPODArray<A>;
-    using ContainerB = std::conditional_t<!is_big_int_v<B>, PaddedPODArray<B>, std::vector<B>>;
+    using ContainerB = PaddedPODArray<B>;
 
     /// If you don't specify NO_INLINE, the compiler will inline this function, but we don't need this as this function contains tight loop inside.
     static void NO_INLINE vectorVector(const ContainerA & a, const ContainerB & b, PaddedPODArray<UInt8> & c)

src/Functions/FunctionsHashing.h

@@ -806,16 +806,7 @@ private:
             size_t size = vec_from.size();
             for (size_t i = 0; i < size; ++i)
             {
-                ToType h;
+                ToType h = Impl::apply(reinterpret_cast<const char *>(&vec_from[i]), sizeof(vec_from[i]));
-                if constexpr (OverBigInt<FromType>)
-                {
-                    using NativeT = typename NativeType<FromType>::Type;
-
-                    std::string buffer = BigInt<NativeT>::serialize(vec_from[i]);
-                    h = Impl::apply(buffer.data(), buffer.size());
-                }
-                else
-                    h = Impl::apply(reinterpret_cast<const char *>(&vec_from[i]), sizeof(vec_from[i]));
 
                 if constexpr (first)
                     vec_to[i] = h;
@@ -827,16 +818,7 @@ private:
         {
             auto value = col_from_const->template getValue<FromType>();
 
-            ToType h;
+            ToType h = Impl::apply(reinterpret_cast<const char *>(&value), sizeof(value));
-            if constexpr (OverBigInt<FromType>)
-            {
-                using NativeT = typename NativeType<FromType>::Type;
-
-                std::string buffer = BigInt<NativeT>::serialize(value);
-                h = Impl::apply(buffer.data(), buffer.size());
-            }
-            else
-                h = Impl::apply(reinterpret_cast<const char *>(&value), sizeof(value));
 
             size_t size = vec_to.size();
             if constexpr (first)

src/Functions/GCDLCMImpl.h

@@ -0,0 +1,67 @@
+#pragma once
+
+#include <DataTypes/NumberTraits.h>
+#include <Common/Exception.h>
+#include <numeric>
+#include <limits>
+#include <type_traits>
+
+#if !defined(ARCADIA_BUILD)
+#    include "config_core.h"
+#endif
+
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int NOT_IMPLEMENTED;
+    extern const int DECIMAL_OVERFLOW;
+}
+
+template <class T>
+inline constexpr bool is_gcd_lcm_implemeted = !(is_big_int_v<T> || std::is_floating_point_v<T>);
+
+template <typename A, typename B, typename Impl, typename Name>
+struct GCDLCMImpl
+{
+    using ResultType = typename NumberTraits::ResultOfAdditionMultiplication<A, B>::Type;
+    static const constexpr bool allow_fixed_string = false;
+
+    template <typename Result = ResultType>
+    static inline std::enable_if_t<!is_gcd_lcm_implemeted<Result>, Result>
+    apply(A, B)
+    {
+        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "{} is not implemented for big integers and floats", Name::name);
+    }
+
+    template <typename Result = ResultType>
+    static inline std::enable_if_t<is_gcd_lcm_implemeted<Result>, Result>
+    apply(A a, B b)
+    {
+        throwIfDivisionLeadsToFPE(typename NumberTraits::ToInteger<A>::Type(a), typename NumberTraits::ToInteger<B>::Type(b));
+        throwIfDivisionLeadsToFPE(typename NumberTraits::ToInteger<B>::Type(b), typename NumberTraits::ToInteger<A>::Type(a));
+
+        using Int = typename NumberTraits::ToInteger<Result>::Type;
+
+        if constexpr (is_signed_v<Result>)
+        {
+            /// gcd() internally uses std::abs()
+            Int a_s = static_cast<Int>(a);
+            Int b_s = static_cast<Int>(b);
+            Int min = std::numeric_limits<Int>::min();
+            Int max = std::numeric_limits<Int>::max();
+            if (unlikely((a_s == min || a_s == max) || (b_s == min || b_s == max)))
+                throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "Intermediate result overflow (signed a = {}, signed b = {}, min = {}, max = {})", a_s, b_s, min, max);
+        }
+
+        return Impl::applyImpl(a, b);
+    }
+
+#if USE_EMBEDDED_COMPILER
+    static constexpr bool compilable = false; /// exceptions (and a non-trivial algorithm)
+#endif
+};
+
+}

src/Functions/GatherUtils/Algorithms.h

@@ -50,9 +50,9 @@ void writeSlice(const NumericArraySlice<T> & slice, NumericArraySink<U> & sink)
                 throw Exception("No conversion between UInt128 and " + demangle(typeid(T).name()), ErrorCodes::NOT_IMPLEMENTED);
             }
             else if constexpr (IsDecimalNumber<T>)
-                dst = bigint_cast<NativeU>(src.value);
+                dst = static_cast<NativeU>(src.value);
             else
-                dst = bigint_cast<NativeU>(src);
+                dst = static_cast<NativeU>(src);
         }
         else
             dst = static_cast<NativeU>(src);

src/Functions/IsOperation.h

@@ -20,6 +20,9 @@ template <typename, typename> struct NotEqualsOp;
 template <typename, typename> struct LessOrEqualsOp;
 template <typename, typename> struct GreaterOrEqualsOp;
 
+template <typename>
+struct SignImpl;
+
 template <template <typename, typename> typename Op1, template <typename, typename> typename Op2>
 struct IsSameOperation
 {
@@ -31,6 +34,7 @@ struct IsUnaryOperation
 {
     static constexpr bool abs = std::is_same_v<Op<Int8>, AbsImpl<Int8>>;
     static constexpr bool negate = std::is_same_v<Op<Int8>, NegateImpl<Int8>>;
+    static constexpr bool sign = std::is_same_v<Op<Int8>, SignImpl<Int8>>;
 };
 
 template <template <typename, typename> typename Op>

src/Functions/bitAnd.cpp

@@ -21,7 +21,7 @@ struct BitAndImpl
     template <typename Result = ResultType>
     static inline Result apply(A a, B b)
     {
-        return bigint_cast<Result>(a) & bigint_cast<Result>(b);
+        return static_cast<Result>(a) & static_cast<Result>(b);
     }
 
 #if USE_EMBEDDED_COMPILER

src/Functions/bitOr.cpp

@@ -20,7 +20,7 @@ struct BitOrImpl
     template <typename Result = ResultType>
     static inline Result apply(A a, B b)
     {
-        return bigint_cast<Result>(a) | bigint_cast<Result>(b);
+        return static_cast<Result>(a) | static_cast<Result>(b);
     }
 
 #if USE_EMBEDDED_COMPILER

src/Functions/bitShiftLeft.cpp

@@ -24,7 +24,7 @@ struct BitShiftLeftImpl
         if constexpr (is_big_int_v<B>)
             throw Exception("BitShiftLeft is not implemented for big integers as second argument", ErrorCodes::NOT_IMPLEMENTED);
         else if constexpr (is_big_int_v<A>)
-            return bigint_cast<Result>(a) << bigint_cast<UInt32>(b);
+            return static_cast<Result>(a) << static_cast<UInt32>(b);
         else
             return static_cast<Result>(a) << static_cast<Result>(b);
     }

src/Functions/bitShiftRight.cpp

@@ -24,7 +24,7 @@ struct BitShiftRightImpl
         if constexpr (is_big_int_v<B>)
             throw Exception("BitShiftRight is not implemented for big integers as second argument", ErrorCodes::NOT_IMPLEMENTED);
         else if constexpr (is_big_int_v<A>)
-            return bigint_cast<Result>(a) >> bigint_cast<UInt32>(b);
+            return static_cast<Result>(a) >> static_cast<UInt32>(b);
         else
             return static_cast<Result>(a) >> static_cast<Result>(b);
     }

src/Functions/bitXor.cpp

@@ -20,7 +20,7 @@ struct BitXorImpl
     template <typename Result = ResultType>
     static inline Result apply(A a, B b)
     {
-        return bigint_cast<Result>(a) ^ bigint_cast<Result>(b);
+        return static_cast<Result>(a) ^ static_cast<Result>(b);
     }
 
 #if USE_EMBEDDED_COMPILER

src/Functions/gcd.cpp

@@ -1,45 +1,28 @@
 #include <Functions/FunctionFactory.h>
 #include <Functions/FunctionBinaryArithmetic.h>
-#include <numeric>
+#include <Functions/GCDLCMImpl.h>
 
 
 namespace DB
 {
-namespace ErrorCodes
-{
-    extern const int NOT_IMPLEMENTED;
-}
 
 namespace
 {
 
+struct NameGCD { static constexpr auto name = "gcd"; };
+
 template <typename A, typename B>
-struct GCDImpl
+struct GCDImpl : public GCDLCMImpl<A, B, GCDImpl<A, B>, NameGCD>
 {
-    using ResultType = typename NumberTraits::ResultOfAdditionMultiplication<A, B>::Type;
+    using ResultType = typename GCDLCMImpl<A, B, GCDImpl, NameGCD>::ResultType;
-    static const constexpr bool allow_fixed_string = false;
 
-    template <typename Result = ResultType>
+    static ResultType applyImpl(A a, B b)
-    static inline Result apply([[maybe_unused]] A a, [[maybe_unused]] B b)
     {
-        if constexpr (is_big_int_v<A> || is_big_int_v<B> || is_big_int_v<Result>)
+        using Int = typename NumberTraits::ToInteger<ResultType>::Type;
-            throw Exception("GCD is not implemented for big integers", ErrorCodes::NOT_IMPLEMENTED);
+        return std::gcd(Int(a), Int(b));
-        else
-        {
-            throwIfDivisionLeadsToFPE(typename NumberTraits::ToInteger<A>::Type(a), typename NumberTraits::ToInteger<B>::Type(b));
-            throwIfDivisionLeadsToFPE(typename NumberTraits::ToInteger<B>::Type(b), typename NumberTraits::ToInteger<A>::Type(a));
-            return std::gcd(
-                typename NumberTraits::ToInteger<Result>::Type(a),
-                typename NumberTraits::ToInteger<Result>::Type(b));
     }
-    }
-
-#if USE_EMBEDDED_COMPILER
-    static constexpr bool compilable = false; /// exceptions (and a non-trivial algorithm)
-#endif
 };
 
-struct NameGCD { static constexpr auto name = "gcd"; };
 using FunctionGCD = BinaryArithmeticOverloadResolver<GCDImpl, NameGCD, false>;
 
 }

src/Functions/greatest.cpp

@@ -16,8 +16,8 @@ struct GreatestBaseImpl
     template <typename Result = ResultType>
     static inline Result apply(A a, B b)
     {
-        return bigint_cast<Result>(a) > bigint_cast<Result>(b) ?
+        return static_cast<Result>(a) > static_cast<Result>(b) ?
-               bigint_cast<Result>(a) : bigint_cast<Result>(b);
+               static_cast<Result>(a) : static_cast<Result>(b);
     }
 
 #if USE_EMBEDDED_COMPILER

src/Functions/if.cpp

@@ -60,7 +60,7 @@ struct NumIfImpl
         typename ColVecResult::Container & res = col_res->getData();
 
         for (size_t i = 0; i < size; ++i)
-            res[i] = cond[i] ? bigint_cast<ResultType>(a[i]) : bigint_cast<ResultType>(b[i]);
+            res[i] = cond[i] ? static_cast<ResultType>(a[i]) : static_cast<ResultType>(b[i]);
         return col_res;
     }
 
@@ -71,7 +71,7 @@ struct NumIfImpl
         typename ColVecResult::Container & res = col_res->getData();
 
         for (size_t i = 0; i < size; ++i)
-            res[i] = cond[i] ? bigint_cast<ResultType>(a[i]) : bigint_cast<ResultType>(b);
+            res[i] = cond[i] ? static_cast<ResultType>(a[i]) : static_cast<ResultType>(b);
         return col_res;
     }
 
@@ -82,7 +82,7 @@ struct NumIfImpl
         typename ColVecResult::Container & res = col_res->getData();
 
         for (size_t i = 0; i < size; ++i)
-            res[i] = cond[i] ? bigint_cast<ResultType>(a) : bigint_cast<ResultType>(b[i]);
+            res[i] = cond[i] ? static_cast<ResultType>(a) : static_cast<ResultType>(b[i]);
         return col_res;
     }
 
@@ -93,7 +93,7 @@ struct NumIfImpl
         typename ColVecResult::Container & res = col_res->getData();
 
         for (size_t i = 0; i < size; ++i)
-            res[i] = cond[i] ? bigint_cast<ResultType>(a) : bigint_cast<ResultType>(b);
+            res[i] = cond[i] ? static_cast<ResultType>(a) : static_cast<ResultType>(b);
         return col_res;
     }
 };

src/Functions/lcm.cpp

@@ -1,10 +1,6 @@
 #include <Functions/FunctionFactory.h>
 #include <Functions/FunctionBinaryArithmetic.h>
-
+#include <Functions/GCDLCMImpl.h>
-#include <numeric>
-#include <limits>
-#include <type_traits>
-
 
 namespace
 {
@@ -27,33 +23,21 @@ constexpr T abs(T value) noexcept
 
 namespace DB
 {
-namespace ErrorCodes
-{
-    extern const int NOT_IMPLEMENTED;
-}
 
 namespace
 {
 
+struct NameLCM { static constexpr auto name = "lcm"; };
+
 template <typename A, typename B>
-struct LCMImpl
+struct LCMImpl : public GCDLCMImpl<A, B, LCMImpl<A, B>, NameLCM>
 {
-    using ResultType = typename NumberTraits::ResultOfAdditionMultiplication<A, B>::Type;
+    using ResultType = typename GCDLCMImpl<A, B, LCMImpl<A, B>, NameLCM>::ResultType;
-    static const constexpr bool allow_fixed_string = false;
 
-    template <typename Result = ResultType>
+    static ResultType applyImpl(A a, B b)
-    static inline std::enable_if_t<is_big_int_v<A> || is_big_int_v<B> || is_big_int_v<Result>, Result>
-    apply([[maybe_unused]] A a, [[maybe_unused]] B b)
     {
-        throw Exception("LCM is not implemented for big integers", ErrorCodes::NOT_IMPLEMENTED);
+        using Int = typename NumberTraits::ToInteger<ResultType>::Type;
-    }
+        using Unsigned = make_unsigned_t<Int>;
-
-    template <typename Result = ResultType>
-    static inline std::enable_if_t<!is_big_int_v<A> && !is_big_int_v<B> && !is_big_int_v<Result>, Result>
-    apply([[maybe_unused]] A a, [[maybe_unused]] B b)
-    {
-        throwIfDivisionLeadsToFPE(typename NumberTraits::ToInteger<A>::Type(a), typename NumberTraits::ToInteger<B>::Type(b));
-        throwIfDivisionLeadsToFPE(typename NumberTraits::ToInteger<B>::Type(b), typename NumberTraits::ToInteger<A>::Type(a));
 
         /** It's tempting to use std::lcm function.
           * But it has undefined behaviour on overflow.
@@ -62,22 +46,14 @@ struct LCMImpl
           * (example: throw an exception or overflow in implementation specific way).
           */
 
-        using Int = typename NumberTraits::ToInteger<Result>::Type;
-        using Unsigned = make_unsigned_t<Int>;
-
         Unsigned val1 = abs<Int>(a) / std::gcd(Int(a), Int(b));
         Unsigned val2 = abs<Int>(b);
 
         /// Overflow in implementation specific way.
-        return Result(val1 * val2);
+        return ResultType(val1 * val2);
     }
-
-#if USE_EMBEDDED_COMPILER
-    static constexpr bool compilable = false; /// exceptions (and a non-trivial algorithm)
-#endif
 };
 
-struct NameLCM { static constexpr auto name = "lcm"; };
 using FunctionLCM = BinaryArithmeticOverloadResolver<LCMImpl, NameLCM, false>;
 
 }

src/Functions/least.cpp

@@ -17,7 +17,7 @@ struct LeastBaseImpl
     static inline Result apply(A a, B b)
     {
         /** gcc 4.9.2 successfully vectorizes a loop from this function. */
-        return bigint_cast<Result>(a) < bigint_cast<Result>(b) ? bigint_cast<Result>(a) : bigint_cast<Result>(b);
+        return static_cast<Result>(a) < static_cast<Result>(b) ? static_cast<Result>(a) : static_cast<Result>(b);
     }
 
 #if USE_EMBEDDED_COMPILER

src/Functions/minus.cpp

@@ -19,7 +19,7 @@ struct MinusImpl
             using CastA = std::conditional_t<std::is_floating_point_v<B>, B, A>;
             using CastB = std::conditional_t<std::is_floating_point_v<A>, A, B>;
 
-            return bigint_cast<Result>(bigint_cast<CastA>(a)) - bigint_cast<Result>(bigint_cast<CastB>(b));
+            return static_cast<Result>(static_cast<CastA>(a)) - static_cast<Result>(static_cast<CastB>(b));
         }
         else
             return static_cast<Result>(a) - b;

src/Functions/multiply.cpp

@@ -20,7 +20,7 @@ struct MultiplyImpl
             using CastA = std::conditional_t<std::is_floating_point_v<B>, B, A>;
             using CastB = std::conditional_t<std::is_floating_point_v<A>, A, B>;
 
-            return bigint_cast<Result>(bigint_cast<CastA>(a)) * bigint_cast<Result>(bigint_cast<CastB>(b));
+            return static_cast<Result>(static_cast<CastA>(a)) * static_cast<Result>(static_cast<CastB>(b));
         }
         else
             return static_cast<Result>(a) * b;

src/Functions/normalizeQuery.cpp

@@ -17,8 +17,10 @@ namespace ErrorCodes
 namespace
 {
 
+template <bool keep_names>
 struct Impl
 {
+    static constexpr auto name = keep_names ? "normalizeQueryKeepNames" : "normalizeQuery";
     static void vector(const ColumnString::Chars & data,
         const ColumnString::Offsets & offsets,
         ColumnString::Chars & res_data,
@@ -32,7 +34,7 @@ struct Impl
         for (size_t i = 0; i < size; ++i)
         {
             ColumnString::Offset curr_src_offset = offsets[i];
-            normalizeQueryToPODArray(
+            normalizeQueryToPODArray<keep_names>(
                 reinterpret_cast<const char *>(&data[prev_src_offset]),
                 reinterpret_cast<const char *>(&data[curr_src_offset - 1]),
                 res_data);
@@ -47,16 +49,12 @@ struct Impl
     }
 };
 
-struct Name
-{
-    static constexpr auto name = "normalizeQuery";
-};
-
 }
 
 void registerFunctionNormalizeQuery(FunctionFactory & factory)
 {
-    factory.registerFunction<FunctionStringToString<Impl, Name>>();
+    factory.registerFunction<FunctionStringToString<Impl<true>, Impl<true>>>();
+    factory.registerFunction<FunctionStringToString<Impl<false>, Impl<false>>>();
 }
 
 }

src/Functions/normalizedQueryHash.cpp

@@ -24,6 +24,7 @@ namespace ErrorCodes
 namespace
 {
 
+template <bool keep_names>
 struct Impl
 {
     static void vector(
@@ -38,17 +39,18 @@ struct Impl
         for (size_t i = 0; i < size; ++i)
         {
             ColumnString::Offset curr_src_offset = offsets[i];
-            res_data[i] = normalizedQueryHash(
+            res_data[i] = normalizedQueryHash<keep_names>(
                 reinterpret_cast<const char *>(&data[prev_src_offset]), reinterpret_cast<const char *>(&data[curr_src_offset - 1]));
             prev_src_offset = offsets[i];
         }
     }
 };
 
+template <bool keep_names>
 class FunctionNormalizedQueryHash : public IFunction
 {
 public:
-    static constexpr auto name = "normalizedQueryHash";
+    static constexpr auto name = keep_names ? "normalizedQueryHashKeepNames" : "normalizedQueryHash";
     static FunctionPtr create(const Context &)
     {
         return std::make_shared<FunctionNormalizedQueryHash>();
@@ -82,7 +84,7 @@ public:
             auto col_res = ColumnUInt64::create();
             typename ColumnUInt64::Container & vec_res = col_res->getData();
             vec_res.resize(col->size());
-            Impl::vector(col->getChars(), col->getOffsets(), vec_res);
+            Impl<keep_names>::vector(col->getChars(), col->getOffsets(), vec_res);
             return col_res;
         }
         else
@@ -95,7 +97,8 @@ public:
 
 void registerFunctionNormalizedQueryHash(FunctionFactory & factory)
 {
-    factory.registerFunction<FunctionNormalizedQueryHash>();
+    factory.registerFunction<FunctionNormalizedQueryHash<true>>();
+    factory.registerFunction<FunctionNormalizedQueryHash<false>>();
 }
 
 }

src/Functions/plus.cpp

@@ -21,7 +21,7 @@ struct PlusImpl
             using CastA = std::conditional_t<std::is_floating_point_v<B>, B, A>;
             using CastB = std::conditional_t<std::is_floating_point_v<A>, A, B>;
 
-            return bigint_cast<Result>(bigint_cast<CastA>(a)) + bigint_cast<Result>(bigint_cast<CastB>(b));
+            return static_cast<Result>(static_cast<CastA>(a)) + static_cast<Result>(static_cast<CastB>(b));
         }
         else
             return static_cast<Result>(a) + b;

src/Functions/registerFunctionsMath.cpp

@@ -33,6 +33,7 @@ void registerFunctionAsinh(FunctionFactory & factory);
 void registerFunctionAcosh(FunctionFactory & factory);
 void registerFunctionAtanh(FunctionFactory & factory);
 void registerFunctionPow(FunctionFactory & factory);
+void registerFunctionSign(FunctionFactory & factory);
 
 
 void registerFunctionsMath(FunctionFactory & factory)
@@ -68,6 +69,7 @@ void registerFunctionsMath(FunctionFactory & factory)
     registerFunctionAcosh(factory);
     registerFunctionAtanh(factory);
     registerFunctionPow(factory);
+    registerFunctionSign(factory);
 }
 
 }

src/Functions/sign.cpp

@@ -0,0 +1,47 @@
+#include <Functions/FunctionFactory.h>
+#include <Functions/FunctionUnaryArithmetic.h>
+#include <DataTypes/NumberTraits.h>
+#include <Common/FieldVisitors.h>
+
+namespace DB
+{
+template <typename A>
+struct SignImpl
+{
+    using ResultType = Int8;
+    static const constexpr bool allow_fixed_string = false;
+
+    static inline NO_SANITIZE_UNDEFINED ResultType apply(A a)
+    {
+        if constexpr (IsDecimalNumber<A> || std::is_floating_point_v<A>)
+            return a < A(0) ? -1 : a == A(0) ? 0 : 1;
+        else if constexpr (is_signed_v<A>)
+            return a < 0 ? -1 : a == 0 ? 0 : 1;
+        else if constexpr (is_unsigned_v<A>)
+            return a == 0 ? 0 : 1;
+    }
+
+#if USE_EMBEDDED_COMPILER
+    static constexpr bool compilable = false;
+#endif
+};
+
+struct NameSign
+{
+    static constexpr auto name = "sign";
+};
+using FunctionSign = FunctionUnaryArithmetic<SignImpl, NameSign, false>;
+
+template <>
+struct FunctionUnaryArithmeticMonotonicity<NameSign>
+{
+    static bool has() { return true; }
+    static IFunction::Monotonicity get(const Field &, const Field &) { return {true, true, false}; }
+};
+
+void registerFunctionSign(FunctionFactory & factory)
+{
+    factory.registerFunction<FunctionSign>(FunctionFactory::CaseInsensitive);
+}
+
+}

src/Functions/ya.make

@@ -428,6 +428,7 @@ SRCS(
     runningDifference.cpp
     runningDifferenceStartingWithFirstValue.cpp
     sigmoid.cpp
+    sign.cpp
     sin.cpp
     sinh.cpp
     sleep.cpp

src/IO/ReadHelpers.h

@@ -17,7 +17,6 @@
 #include <Core/Types.h>
 #include <Core/DecimalFunctions.h>
 #include <Core/UUID.h>
-#include <Core/BigInt.h>
 
 #include <Common/Exception.h>
 #include <Common/StringUtils/StringUtils.h>
@@ -121,17 +120,6 @@ inline void readFloatBinary(T & x, ReadBuffer & buf)
     readPODBinary(x, buf);
 }
 
-template <typename T>
-void readBigIntBinary(T & x, ReadBuffer & buf)
-{
-    static const constexpr size_t bytesize = BigInt<T>::size;
-    char bytes[bytesize];
-
-    buf.readStrict(bytes, bytesize);
-
-    x = BigInt<T>::deserialize(bytes);
-}
-
 inline void readStringBinary(std::string & s, ReadBuffer & buf, size_t MAX_STRING_SIZE = DEFAULT_MAX_STRING_SIZE)
 {
     size_t size = 0;
@@ -849,11 +837,11 @@ inline void readBinary(DummyUInt256 & x, ReadBuffer & buf) { readPODBinary(x, bu
 inline void readBinary(Decimal32 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
 inline void readBinary(Decimal64 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
 inline void readBinary(Decimal128 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
-inline void readBinary(Decimal256 & x, ReadBuffer & buf) { readBigIntBinary(x.value, buf); }
+inline void readBinary(Decimal256 & x, ReadBuffer & buf) { readPODBinary(x.value, buf); }
 inline void readBinary(LocalDate & x, ReadBuffer & buf) { readPODBinary(x, buf); }
 
-inline void readBinary(UInt256 & x, ReadBuffer & buf) { readBigIntBinary(x, buf); }
+inline void readBinary(UInt256 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
-inline void readBinary(Int256 & x, ReadBuffer & buf) { readBigIntBinary(x, buf); }
+inline void readBinary(Int256 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
 
 template <typename T>
 inline std::enable_if_t<is_arithmetic_v<T> && (sizeof(T) <= 8), void>

src/IO/WriteHelpers.h

@@ -16,7 +16,6 @@
 #include <Core/DecimalFunctions.h>
 #include <Core/Types.h>
 #include <Core/UUID.h>
-#include <Core/BigInt.h>
 
 #include <Common/Exception.h>
 #include <Common/StringUtils/StringUtils.h>
@@ -121,17 +120,6 @@ inline void writeStringBinary(const std::string_view & s, WriteBuffer & buf)
     writeStringBinary(StringRef{s}, buf);
 }
 
-template <typename T>
-void writeBigIntBinary(const T & x, WriteBuffer & buf)
-{
-    static const constexpr size_t bytesize = BigInt<T>::size;
-    char bytes[bytesize];
-
-    BigInt<T>::serialize(x, bytes);
-
-    buf.write(bytes, bytesize);
-}
-
 template <typename T>
 void writeVectorBinary(const std::vector<T> & v, WriteBuffer & buf)
 {
@@ -926,12 +914,12 @@ inline void writeBinary(const DummyUInt256 & x, WriteBuffer & buf) { writePODBin
 inline void writeBinary(const Decimal32 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
 inline void writeBinary(const Decimal64 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
 inline void writeBinary(const Decimal128 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
-inline void writeBinary(const Decimal256 & x, WriteBuffer & buf) { writeBigIntBinary(x.value, buf); }
+inline void writeBinary(const Decimal256 & x, WriteBuffer & buf) { writePODBinary(x.value, buf); }
 inline void writeBinary(const LocalDate & x, WriteBuffer & buf) { writePODBinary(x, buf); }
 inline void writeBinary(const LocalDateTime & x, WriteBuffer & buf) { writePODBinary(x, buf); }
 
-inline void writeBinary(const UInt256 & x, WriteBuffer & buf) { writeBigIntBinary(x, buf); }
+inline void writeBinary(const UInt256 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
-inline void writeBinary(const Int256 & x, WriteBuffer & buf) { writeBigIntBinary(x, buf); }
+inline void writeBinary(const Int256 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
 
 /// Methods for outputting the value in text form for a tab-separated format.
 template <typename T>

src/Interpreters/ApplyWithAliasVisitor.cpp

@@ -12,12 +12,10 @@ void ApplyWithAliasVisitor::visit(ASTPtr & ast, const Data & data)
         std::optional<Data> new_data;
         if (auto with = node_select->with())
         {
-            for (auto & child : with->children)
-                visit(child, data);
-
             std::set<String> current_names;
             for (auto & child : with->children)
             {
+                visit(child, new_data ? *new_data : data);
                 if (auto * ast_with_alias = dynamic_cast<ASTWithAlias *>(child.get()))
                 {
                     if (!new_data)

src/Interpreters/ApplyWithGlobalVisitor.cpp

@@ -3,17 +3,53 @@
 #include <Parsers/ASTSelectWithUnionQuery.h>
 #include <Parsers/ASTWithAlias.h>
 
-#include <map>
-
 namespace DB
 {
+
+void ApplyWithGlobalVisitor::visit(ASTSelectQuery & select, const std::map<String, ASTPtr> & exprs, const ASTPtr & with_expression_list)
+{
+    auto with = select.with();
+    if (with)
+    {
+        std::set<String> current_names;
+        for (const auto & child : with->children)
+        {
+            if (const auto * ast_with_alias = dynamic_cast<const ASTWithAlias *>(child.get()))
+                current_names.insert(ast_with_alias->alias);
+        }
+        for (const auto & with_alias : exprs)
+        {
+            if (!current_names.count(with_alias.first))
+                with->children.push_back(with_alias.second->clone());
+        }
+    }
+    else
+        select.setExpression(ASTSelectQuery::Expression::WITH, with_expression_list->clone());
+}
+
+void ApplyWithGlobalVisitor::visit(
+    ASTSelectWithUnionQuery & selects, const std::map<String, ASTPtr> & exprs, const ASTPtr & with_expression_list)
+{
+    for (auto & select : selects.list_of_selects->children)
+    {
+        if (ASTSelectWithUnionQuery * node_union = select->as<ASTSelectWithUnionQuery>())
+        {
+            visit(*node_union, exprs, with_expression_list);
+        }
+        else if (ASTSelectQuery * node_select = select->as<ASTSelectQuery>())
+        {
+            visit(*node_select, exprs, with_expression_list);
+        }
+    }
+}
+
 void ApplyWithGlobalVisitor::visit(ASTPtr & ast)
 {
     if (ASTSelectWithUnionQuery * node_union = ast->as<ASTSelectWithUnionQuery>())
     {
-        auto & first_select = node_union->list_of_selects->children[0]->as<ASTSelectQuery &>();
+        if (auto * first_select = node_union->list_of_selects->children[0]->as<ASTSelectQuery>())
-        ASTPtr with_expression_list = first_select.with();
+        {
-
+            ASTPtr with_expression_list = first_select->with();
             if (with_expression_list)
             {
                 std::map<String, ASTPtr> exprs;
@@ -24,30 +60,20 @@ void ApplyWithGlobalVisitor::visit(ASTPtr & ast)
                 }
                 for (auto it = node_union->list_of_selects->children.begin() + 1; it != node_union->list_of_selects->children.end(); ++it)
                 {
-                auto & select = (*it)->as<ASTSelectQuery &>();
+                    if (auto * union_child = (*it)->as<ASTSelectWithUnionQuery>())
-                auto with = select.with();
+                        visit(*union_child, exprs, with_expression_list);
-                if (with)
+                    else if (auto * select_child = (*it)->as<ASTSelectQuery>())
-                {
+                        visit(*select_child, exprs, with_expression_list);
-                    std::set<String> current_names;
+                }
-                    for (auto & child : with->children)
-                    {
-                        if (auto * ast_with_alias = dynamic_cast<ASTWithAlias *>(child.get()))
-                            current_names.insert(ast_with_alias->alias);
             }
-                    for (auto & with_alias : exprs)
-                    {
-                        if (!current_names.count(with_alias.first))
-                            with->children.push_back(with_alias.second->clone());
         }
     }
     else
-                    select.setExpression(ASTSelectQuery::Expression::WITH, with_expression_list->clone());
+    {
-            }
+        // Other non-SELECT queries that contains SELECT children, such as EXPLAIN or INSERT
-        }
-    }
-
         for (auto & child : ast->children)
             visit(child);
     }
+}
 
 }

src/Interpreters/ApplyWithGlobalVisitor.h

@@ -1,14 +1,23 @@
 #pragma once
 
 #include <Parsers/IAST.h>
+#include <map>
 
 namespace DB
 {
+
+class ASTSelectWithUnionQuery;
+class ASTSelectQuery;
+
 /// Pull out the WITH statement from the first child of ASTSelectWithUnion query if any.
 class ApplyWithGlobalVisitor
 {
 public:
     static void visit(ASTPtr & ast);
+
+private:
+    static void visit(ASTSelectWithUnionQuery & selects, const std::map<String, ASTPtr> & exprs, const ASTPtr & with_expression_list);
+    static void visit(ASTSelectQuery & select, const std::map<String, ASTPtr> & exprs, const ASTPtr & with_expression_list);
 };
 
 }

src/Interpreters/ApplyWithSubqueryVisitor.cpp

@@ -4,6 +4,7 @@
 #include <Interpreters/misc.h>
 #include <Parsers/ASTFunction.h>
 #include <Parsers/ASTSelectQuery.h>
+#include <Parsers/ASTSubquery.h>
 #include <Parsers/ASTTablesInSelectQuery.h>
 #include <Parsers/ASTWithElement.h>
 
@@ -55,10 +56,13 @@ void ApplyWithSubqueryVisitor::visit(ASTTableExpression & table, const Data & da
             auto subquery_it = data.subqueries.find(table_id.table_name);
             if (subquery_it != data.subqueries.end())
             {
+                auto old_alias = table.database_and_table_name->tryGetAlias();
                 table.children.clear();
                 table.database_and_table_name.reset();
                 table.subquery = subquery_it->second->clone();
-                dynamic_cast<ASTWithAlias &>(*table.subquery).alias = table_id.table_name;
+                table.subquery->as<ASTSubquery &>().cte_name = table_id.table_name;
+                if (!old_alias.empty())
+                    table.subquery->setAlias(old_alias);
                 table.children.emplace_back(table.subquery);
             }
         }
@@ -78,8 +82,11 @@ void ApplyWithSubqueryVisitor::visit(ASTFunction & func, const Data & data)
                 auto subquery_it = data.subqueries.find(table_id.table_name);
                 if (subquery_it != data.subqueries.end())
                 {
+                    auto old_alias = func.arguments->children[1]->tryGetAlias();
                     func.arguments->children[1] = subquery_it->second->clone();
-                    dynamic_cast<ASTWithAlias &>(*func.arguments->children[1]).alias = table_id.table_name;
+                    func.arguments->children[1]->as<ASTSubquery &>().cte_name = table_id.table_name;
+                    if (!old_alias.empty())
+                        func.arguments->children[1]->setAlias(old_alias);
                 }
             }
         }

src/Interpreters/DatabaseAndTableWithAlias.cpp

@@ -44,7 +44,15 @@ DatabaseAndTableWithAlias::DatabaseAndTableWithAlias(const ASTTableExpression &
     else if (table_expression.table_function)
         alias = table_expression.table_function->tryGetAlias();
     else if (table_expression.subquery)
+    {
+        const auto & cte_name = table_expression.subquery->as<const ASTSubquery &>().cte_name;
+        if (!cte_name.empty())
+        {
+            database = current_database;
+            table = cte_name;
+        }
         alias = table_expression.subquery->tryGetAlias();
+    }
     else
         throw Exception("Logical error: no known elements in ASTTableExpression", ErrorCodes::LOGICAL_ERROR);
 }

src/Interpreters/ExpressionAnalyzer.cpp

@@ -55,6 +55,7 @@
 #include <IO/WriteBufferFromString.h>
 
 #include <Processors/Executors/PullingPipelineExecutor.h>
+#include <Parsers/formatAST.h>
 
 namespace DB
 {

src/Interpreters/GetAggregatesVisitor.h

@@ -31,6 +31,12 @@ public:
     {
         if (child->as<ASTSubquery>() || child->as<ASTSelectQuery>())
             return false;
+        if (auto * select = node->as<ASTSelectQuery>())
+        {
+            // We don't analysis WITH statement because it might contain useless aggregates
+            if (child == select->with())
+                return false;
+        }
         if (auto * func = node->as<ASTFunction>())
         {
             if (isAggregateFunction(*func))

src/Interpreters/InterpreterSelectQuery.cpp

@@ -294,9 +294,13 @@ InterpreterSelectQuery::InterpreterSelectQuery(
         source_header = input_pipe->getHeader();
     }
 
+    // Only propagate WITH elements to subqueries if we're not a subquery
+    if (options.subquery_depth == 0)
+    {
         if (context->getSettingsRef().enable_global_with_statement)
             ApplyWithAliasVisitor().visit(query_ptr);
         ApplyWithSubqueryVisitor().visit(query_ptr);
+    }
 
     JoinedTables joined_tables(getSubqueryContext(*context), getSelectQuery());
 

src/Interpreters/MergeJoin.cpp

@@ -930,7 +930,7 @@ std::shared_ptr<Block> MergeJoin::loadRightBlock(size_t pos) const
     {
         auto load_func = [&]() -> std::shared_ptr<Block>
         {
-            TemporaryFileStream input(flushed_right_blocks[pos]->path(), right_sample_block);
+            TemporaryFileStream input(flushed_right_blocks[pos]->path(), materializeBlock(right_sample_block));
             return std::make_shared<Block>(input.block_in->read());
         };
 

src/Interpreters/ProcessList.cpp

@@ -60,12 +60,6 @@ static bool isUnlimitedQuery(const IAST * ast)
 }
 
 
-ProcessList::ProcessList(size_t max_size_)
-    : max_size(max_size_)
-{
-}
-
-
 ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * ast, Context & query_context)
 {
     EntryPtr res;
@@ -202,7 +196,7 @@ ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * as
             thread_group->performance_counters.setParent(&user_process_list.user_performance_counters);
             thread_group->memory_tracker.setParent(&user_process_list.user_memory_tracker);
             thread_group->query = process_it->query;
-            thread_group->normalized_query_hash = normalizedQueryHash(process_it->query);
+            thread_group->normalized_query_hash = normalizedQueryHash<false>(process_it->query);
 
             /// Set query-level memory trackers
             thread_group->memory_tracker.setOrRaiseHardLimit(settings.max_memory_usage);

src/Interpreters/ProcessList.h

@@ -297,8 +297,6 @@ protected:
     QueryStatus * tryGetProcessListElement(const String & current_query_id, const String & current_user);
 
 public:
-    ProcessList(size_t max_size_ = 0);
-
     using EntryPtr = std::shared_ptr<ProcessListEntry>;
 
     /** Register running query. Returns refcounted object, that will remove element from list in destructor.

src/Interpreters/QueryLog.cpp

@@ -92,6 +92,8 @@ Block QueryLogElement::createBlock()
 
         {std::make_shared<DataTypeUInt32>(),                                  "revision"},
 
+        {std::make_shared<DataTypeString>(),                                  "log_comment"},
+
         {std::make_shared<DataTypeArray>(std::make_shared<DataTypeUInt64>()), "thread_ids"},
         {std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "ProfileEvents.Names"},
         {std::make_shared<DataTypeArray>(std::make_shared<DataTypeUInt64>()), "ProfileEvents.Values"},
@@ -168,6 +170,8 @@ void QueryLogElement::appendToBlock(MutableColumns & columns) const
 
     columns[i++]->insert(ClickHouseRevision::getVersionRevision());
 
+    columns[i++]->insertData(log_comment.data(), log_comment.size());
+
     {
         Array threads_array;
         threads_array.reserve(thread_ids.size());

src/Interpreters/QueryLog.h

@@ -74,6 +74,8 @@ struct QueryLogElement
 
     ClientInfo client_info;
 
+    String log_comment;
+
     std::vector<UInt64> thread_ids;
     std::shared_ptr<ProfileEvents::Counters> profile_counters;
     std::shared_ptr<Settings> query_settings;

src/Interpreters/RequiredSourceColumnsVisitor.cpp

@@ -123,8 +123,12 @@ void RequiredSourceColumnsMatcher::visit(const ASTSelectQuery & select, const AS
 
     std::vector<ASTPtr *> out;
     for (const auto & node : select.children)
-        if (node != select.select())
+    {
+        // We should not go into WITH statement because all needed aliases are already expanded to
+        // the right place after normalization. And it might contain unused unknown columns.
+        if (node != select.select() && node != select.with())
             Visitor(data).visit(node);
+    }
 
     /// revisit select_expression_list (with children) when all the aliases are set
     Visitor(data).visit(select.select());

src/Interpreters/SortedBlocksWriter.cpp

@@ -3,6 +3,7 @@
 #include <DataStreams/MergingSortedBlockInputStream.h>
 #include <DataStreams/OneBlockInputStream.h>
 #include <DataStreams/TemporaryFileStream.h>
+#include <DataStreams/materializeBlock.h>
 #include <Disks/StoragePolicy.h>
 
 namespace DB
@@ -198,7 +199,7 @@ SortedBlocksWriter::SortedFiles SortedBlocksWriter::finishMerge(std::function<vo
 
 BlockInputStreamPtr SortedBlocksWriter::streamFromFile(const TmpFilePtr & file) const
 {
-    return std::make_shared<TemporaryFileLazyInputStream>(file->path(), sample_block);
+    return std::make_shared<TemporaryFileLazyInputStream>(file->path(), materializeBlock(sample_block));
 }
 
 String SortedBlocksWriter::getPath() const

src/Interpreters/executeQuery.cpp

@@ -157,11 +157,21 @@ static void logQuery(const String & query, const Context & context, bool interna
         const auto & initial_query_id = client_info.initial_query_id;
         const auto & current_user = client_info.current_user;
 
-        LOG_DEBUG(&Poco::Logger::get("executeQuery"), "(from {}{}{}, using {} parser) {}",
+        String comment = context.getSettingsRef().log_comment;
+        size_t max_query_size = context.getSettingsRef().max_query_size;
+
+        if (comment.size() > max_query_size)
+            comment.resize(max_query_size);
+
+        if (!comment.empty())
+            comment = fmt::format(" (comment: {})", comment);
+
+        LOG_DEBUG(&Poco::Logger::get("executeQuery"), "(from {}{}{}, using {} parser){} {}",
             client_info.current_address.toString(),
             (current_user != "default" ? ", user: " + current_user : ""),
             (!initial_query_id.empty() && current_query_id != initial_query_id ? ", initial_query_id: " + initial_query_id : std::string()),
             (context.getSettingsRef().use_antlr_parser ? "experimental" : "production"),
+            comment,
             joinLines(query));
 
         if (client_info.client_trace_context.trace_id)
@@ -196,13 +206,17 @@ static void setExceptionStackTrace(QueryLogElement & elem)
 /// Log exception (with query info) into text log (not into system table).
 static void logException(Context & context, QueryLogElement & elem)
 {
+    String comment;
+    if (!elem.log_comment.empty())
+        comment = fmt::format(" (comment: {})", elem.log_comment);
+
     if (elem.stack_trace.empty())
-        LOG_ERROR(&Poco::Logger::get("executeQuery"), "{} (from {}) (in query: {})",
+        LOG_ERROR(&Poco::Logger::get("executeQuery"), "{} (from {}){} (in query: {})",
-            elem.exception, context.getClientInfo().current_address.toString(), joinLines(elem.query));
+            elem.exception, context.getClientInfo().current_address.toString(), comment, joinLines(elem.query));
     else
-        LOG_ERROR(&Poco::Logger::get("executeQuery"), "{} (from {}) (in query: {})"
+        LOG_ERROR(&Poco::Logger::get("executeQuery"), "{} (from {}){} (in query: {})"
             ", Stack trace (when copying this message, always include the lines below):\n\n{}",
-            elem.exception, context.getClientInfo().current_address.toString(), joinLines(elem.query), elem.stack_trace);
+            elem.exception, context.getClientInfo().current_address.toString(), comment, joinLines(elem.query), elem.stack_trace);
 }
 
 inline UInt64 time_in_microseconds(std::chrono::time_point<std::chrono::system_clock> timepoint)
@@ -239,7 +253,7 @@ static void onExceptionBeforeStart(const String & query_for_logging, Context & c
 
     elem.current_database = context.getCurrentDatabase();
     elem.query = query_for_logging;
-    elem.normalized_query_hash = normalizedQueryHash(query_for_logging);
+    elem.normalized_query_hash = normalizedQueryHash<false>(query_for_logging);
 
     // We don't calculate query_kind, databases, tables and columns when the query isn't able to start
 
@@ -248,6 +262,10 @@ static void onExceptionBeforeStart(const String & query_for_logging, Context & c
 
     elem.client_info = context.getClientInfo();
 
+    elem.log_comment = settings.log_comment;
+    if (elem.log_comment.size() > settings.max_query_size)
+        elem.log_comment.resize(settings.max_query_size);
+
     if (settings.calculate_text_stack_trace)
         setExceptionStackTrace(elem);
     logException(context, elem);
@@ -604,7 +622,7 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
 
             elem.current_database = context.getCurrentDatabase();
             elem.query = query_for_logging;
-            elem.normalized_query_hash = normalizedQueryHash(query_for_logging);
+            elem.normalized_query_hash = normalizedQueryHash<false>(query_for_logging);
 
             elem.client_info = context.getClientInfo();
 
@@ -626,6 +644,10 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
                 if (settings.log_query_settings)
                     elem.query_settings = std::make_shared<Settings>(context.getSettingsRef());
 
+                elem.log_comment = settings.log_comment;
+                if (elem.log_comment.size() > settings.max_query_size)
+                    elem.log_comment.resize(settings.max_query_size);
+
                 if (elem.type >= settings.log_queries_min_type && !settings.log_queries_min_query_duration_ms.totalMilliseconds())
                 {
                     if (auto query_log = context.getQueryLog())

src/Interpreters/getTableExpressions.cpp

@@ -73,8 +73,12 @@ ASTPtr extractTableExpression(const ASTSelectQuery & select, size_t table_number
     return nullptr;
 }
 
-static NamesAndTypesList getColumnsFromTableExpression(const ASTTableExpression & table_expression, const Context & context,
+static NamesAndTypesList getColumnsFromTableExpression(
-                                                NamesAndTypesList & materialized, NamesAndTypesList & aliases, NamesAndTypesList & virtuals)
+    const ASTTableExpression & table_expression,
+    const Context & context,
+    NamesAndTypesList & materialized,
+    NamesAndTypesList & aliases,
+    NamesAndTypesList & virtuals)
 {
     NamesAndTypesList names_and_type_list;
     if (table_expression.subquery)

src/Parsers/ASTSubquery.cpp

@@ -1,6 +1,7 @@
 #include <Parsers/ASTSubquery.h>
 #include <IO/WriteHelpers.h>
 #include <IO/Operators.h>
+#include <Common/SipHash.h>
 
 namespace DB
 {
@@ -12,6 +13,10 @@ void ASTSubquery::appendColumnNameImpl(WriteBuffer & ostr) const
     {
         writeString(alias, ostr);
     }
+    else if (!cte_name.empty())
+    {
+        writeString(cte_name, ostr);
+    }
     else
     {
         Hash hash = getTreeHash();
@@ -24,6 +29,14 @@ void ASTSubquery::appendColumnNameImpl(WriteBuffer & ostr) const
 
 void ASTSubquery::formatImplWithoutAlias(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
 {
+    if (!cte_name.empty())
+    {
+        settings.ostr << (settings.hilite ? hilite_identifier : "");
+        settings.writeIdentifier(cte_name);
+        settings.ostr << (settings.hilite ? hilite_none : "");
+        return;
+    }
+
     std::string indent_str = settings.one_line ? "" : std::string(4u * frame.indent, ' ');
     std::string nl_or_nothing = settings.one_line ? "" : "\n";
 
@@ -35,5 +48,12 @@ void ASTSubquery::formatImplWithoutAlias(const FormatSettings & settings, Format
     settings.ostr << nl_or_nothing << indent_str << ")";
 }
 
+void ASTSubquery::updateTreeHashImpl(SipHash & hash_state) const
+{
+    if (!cte_name.empty())
+        hash_state.update(cte_name);
+    IAST::updateTreeHashImpl(hash_state);
+}
+
 }
 

src/Parsers/ASTSubquery.h

@@ -12,6 +12,10 @@ namespace DB
 class ASTSubquery : public ASTWithAlias
 {
 public:
+    // Stored the name when the subquery is defined in WITH clause. For example:
+    // WITH (SELECT 1) AS a SELECT * FROM a AS b; cte_name will be `a`.
+    std::string cte_name;
+
     /** Get the text that identifies this element. */
     String getID(char) const override { return "Subquery"; }
 
@@ -28,6 +32,8 @@ public:
         return ptr;
     }
 
+    void updateTreeHashImpl(SipHash & hash_state) const override;
+
 protected:
     void formatImplWithoutAlias(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
     void appendColumnNameImpl(WriteBuffer & ostr) const override;

src/Parsers/ASTWithElement.cpp

@@ -1,4 +1,5 @@
 #include <Parsers/ASTWithElement.h>
+#include <Parsers/ASTWithAlias.h>
 #include <IO/Operators.h>
 
 namespace DB
@@ -8,7 +9,6 @@ ASTPtr ASTWithElement::clone() const
 {
     const auto res = std::make_shared<ASTWithElement>(*this);
     res->children.clear();
-    res->name = name;
     res->subquery = subquery->clone();
     res->children.emplace_back(res->subquery);
     return res;
@@ -18,6 +18,7 @@ void ASTWithElement::formatImpl(const FormatSettings & settings, FormatState & s
 {
     settings.writeIdentifier(name);
     settings.ostr << (settings.hilite ? hilite_keyword : "") << " AS " << (settings.hilite ? hilite_none : "");
-    subquery->formatImpl(settings, state, frame);
+    dynamic_cast<const ASTWithAlias *>(subquery.get())->formatImplWithoutAlias(settings, state, frame);
 }
+
 }

src/Parsers/ParserWithElement.cpp

@@ -23,6 +23,7 @@ bool ParserWithElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
         auto with_element = std::make_shared<ASTWithElement>();
         tryGetIdentifierNameInto(name, with_element->name);
         with_element->subquery = subquery;
+        with_element->children.push_back(with_element->subquery);
         node = with_element;
     }
     else

src/Parsers/queryNormalization.h

@@ -10,6 +10,7 @@
 
 namespace DB
 {
+template <bool keep_names>
 inline UInt64 ALWAYS_INLINE normalizedQueryHash(const char * begin, const char * end)
 {
     SipHash hash;
@@ -60,6 +61,13 @@ inline UInt64 ALWAYS_INLINE normalizedQueryHash(const char * begin, const char *
             /// Differentiate identifier from function (example: SHA224(x)).
             /// By the way, there is padding in columns and pointer dereference is Ok.
             || (token.type == TokenType::BareWord && *token.end != '('))
+        {
+            /// Explicitly ask to keep identifier names
+            if constexpr (keep_names)
+            {
+                hash.update(token.begin, token.size());
+            }
+            else
             {
                 /// Identifier is complex if it contains whitespace or more than two digits
                 /// or it's at least 36 bytes long (UUID for example).
@@ -86,6 +94,7 @@ inline UInt64 ALWAYS_INLINE normalizedQueryHash(const char * begin, const char *
                     hash.update(token.begin, token.size());
                 else
                     hash.update("\x01", 1);
+            }
 
             continue;
         }
@@ -99,12 +108,14 @@ inline UInt64 ALWAYS_INLINE normalizedQueryHash(const char * begin, const char *
     return hash.get64();
 }
 
+template <bool keep_names>
 inline UInt64 ALWAYS_INLINE normalizedQueryHash(const String & query)
 {
-    return normalizedQueryHash(query.data(), query.data() + query.size());
+    return normalizedQueryHash<keep_names>(query.data(), query.data() + query.size());
 }
 
 
+template <bool keep_names>
 inline void ALWAYS_INLINE normalizeQueryToPODArray(const char * begin, const char * end, PaddedPODArray<UInt8> & res_data)
 {
     Lexer lexer(begin, end);
@@ -186,6 +197,13 @@ inline void ALWAYS_INLINE normalizeQueryToPODArray(const char * begin, const cha
             /// Differentiate identifier from function (example: SHA224(x)).
             /// By the way, there is padding in columns and pointer dereference is Ok.
             || (token.type == TokenType::BareWord && *token.end != '('))
+        {
+            /// Explicitly ask to normalize with identifier names
+            if constexpr (keep_names)
+            {
+                res_data.insert(token.begin, token.end);
+            }
+            else
             {
                 /// Identifier is complex if it contains whitespace or more than two digits
                 /// or it's at least 36 bytes long (UUID for example).
@@ -218,6 +236,7 @@ inline void ALWAYS_INLINE normalizeQueryToPODArray(const char * begin, const cha
                     res_data.push_back('?');
                     res_data.push_back('`');
                 }
+            }
 
             continue;
         }

src/Storages/StorageS3.cpp

@@ -216,7 +216,8 @@ StorageS3::StorageS3(
     storage_metadata.setConstraints(constraints_);
     setInMemoryMetadata(storage_metadata);
 
-    auto settings = context_.getStorageS3Settings().getSettings(uri.endpoint);
+    auto settings = context_.getStorageS3Settings().getSettings(uri.uri.toString());
+
     Aws::Auth::AWSCredentials credentials(access_key_id_, secret_access_key_);
     if (access_key_id_.empty())
         credentials = Aws::Auth::AWSCredentials(std::move(settings.access_key_id), std::move(settings.secret_access_key));

src/Storages/StorageS3Settings.cpp

@@ -3,6 +3,8 @@
 #include <Poco/Util/AbstractConfiguration.h>
 #include <Common/Exception.h>
 
+#include <boost/algorithm/string/predicate.hpp>
+
 
 namespace DB
 {
@@ -22,6 +24,8 @@ void StorageS3Settings::loadFromConfig(const String & config_elem, const Poco::U
     config.keys(config_elem, config_keys);
 
     for (const String & key : config_keys)
+    {
+        if (config.has(config_elem + "." + key + ".endpoint"))
         {
             auto endpoint = config.getString(config_elem + "." + key + ".endpoint");
             auto access_key_id = config.getString(config_elem + "." + key + ".access_key_id", "");
@@ -50,12 +54,21 @@ void StorageS3Settings::loadFromConfig(const String & config_elem, const Poco::U
             settings.emplace(endpoint, S3AuthSettings{std::move(access_key_id), std::move(secret_access_key), std::move(headers), use_environment_credentials});
         }
     }
+}
 
 S3AuthSettings StorageS3Settings::getSettings(const String & endpoint) const
 {
     std::lock_guard lock(mutex);
-    if (auto setting = settings.find(endpoint); setting != settings.end())
+    auto next_prefix_setting = settings.upper_bound(endpoint);
-        return setting->second;
+
+    /// Linear time algorithm may be replaced with logarithmic with prefix tree map.
+    for (auto possible_prefix_setting = next_prefix_setting; possible_prefix_setting != settings.begin();)
+    {
+        std::advance(possible_prefix_setting, -1);
+        if (boost::algorithm::starts_with(endpoint, possible_prefix_setting->first))
+            return possible_prefix_setting->second;
+    }
+
     return {};
 }
 

src/TableFunctions/TableFunctionView.cpp

@@ -38,7 +38,8 @@ ColumnsDescription TableFunctionView::getActualTableStructure(const Context & co
     return ColumnsDescription(sample.getNamesAndTypesList());
 }
 
-StoragePtr TableFunctionView::executeImpl(const ASTPtr & /*ast_function*/, const Context & context, const std::string & table_name, ColumnsDescription /*cached_columns*/) const
+StoragePtr TableFunctionView::executeImpl(
+    const ASTPtr & /*ast_function*/, const Context & context, const std::string & table_name, ColumnsDescription /*cached_columns*/) const
 {
     auto columns = getActualTableStructure(context);
     auto res = StorageView::create(StorageID(getDatabaseName(), table_name), create, columns);

tests/clickhouse-test

@@ -151,7 +151,7 @@ def run_single_test(args, ext, server_logs_level, client_options, case_file, std
     pattern = '{test} > {stdout} 2> {stderr}'
 
     if ext == '.sql':
-        pattern = "{client} --send_logs_level={logs_level} --testmode --multiquery {options} < " + pattern
+        pattern = "{client} --send_logs_level={logs_level} --testmode --multiquery {options} --log_comment='{test}' < " + pattern
 
     command = pattern.format(**params)
 
@@ -777,7 +777,7 @@ def main(args):
                 print(colored("\nFound hung queries in processlist:", args, "red", attrs=["bold"]))
                 print(processlist)
             else:
-                print(colored("Seems like server hung and cannot respond to queries", "red", attrs=["bold"]))
+                print(colored("Seems like server hung and cannot respond to queries", args, "red", attrs=["bold"]))
 
             clickhouse_tcp_port = os.getenv("CLICKHOUSE_PORT_TCP", '9000')
             server_pid = get_server_pid(clickhouse_tcp_port)

tests/integration/helpers/client.py

@@ -61,7 +61,10 @@ class QueryTimeoutExceedException(Exception):
 
 
 class QueryRuntimeException(Exception):
-    pass
+    def __init__(self, message, returncode, stderr):
+        super(QueryRuntimeException, self).__init__(message)
+        self.returncode = returncode
+        self.stderr = stderr
 
 
 class CommandRequest:
@@ -106,7 +109,7 @@ class CommandRequest:
 
         if (self.process.returncode != 0 or stderr) and not self.ignore_error:
             raise QueryRuntimeException(
-                'Client failed! Return code: {}, stderr: {}'.format(self.process.returncode, stderr))
+                'Client failed! Return code: {}, stderr: {}'.format(self.process.returncode, stderr), self.process.returncode, stderr)
 
         return stdout
 
@@ -122,7 +125,7 @@ class CommandRequest:
             raise QueryTimeoutExceedException('Client timed out!')
 
         if (self.process.returncode == 0):
-            raise QueryRuntimeException('Client expected to be failed but succeeded! stdout: {}'.format(stdout))
+            raise QueryRuntimeException('Client expected to be failed but succeeded! stdout: {}'.format(stdout), self.process.returncode, stderr)
 
         return stderr
 

tests/integration/test_storage_s3/configs/defaultS3.xml

@@ -4,5 +4,8 @@
             <endpoint>http://resolver:8080</endpoint>
             <header>Authorization: Bearer TOKEN</header>
         </s3_mock>
+        <s3_mock_restricted_directory>
+            <endpoint>http://resolver:8080/root-with-auth/restricteddirectory/</endpoint>
+        </s3_mock_restricted_directory>
     </s3>
 </yandex>

tests/integration/test_storage_s3/s3_mock/mock_s3.py

@@ -1,14 +1,14 @@
 from bottle import abort, route, run, request, response
 
 
-@route('/redirected/<_path>')
+@route('/redirected/<_path:path>')
 def infinite_redirect(_path):
     response.set_header("Location", request.url)
     response.status = 307
     return 'Redirected'
 
 
-@route('/<_bucket>/<_path>')
+@route('/<_bucket>/<_path:path>')
 def server(_bucket, _path):
     for name in request.headers:
         if name == 'Authorization' and request.headers[name] == 'Bearer TOKEN':

tests/integration/test_storage_s3/test.py

@@ -410,6 +410,20 @@ def test_custom_auth_headers(cluster):
     assert result == '1\t2\t3\n'
 
 
+def test_custom_auth_headers_exclusion(cluster):
+    table_format = "column1 UInt32, column2 UInt32, column3 UInt32"
+    filename = "test.csv"
+    get_query = f"SELECT * FROM s3('http://resolver:8080/{cluster.minio_restricted_bucket}/restricteddirectory/{filename}', 'CSV', '{table_format}')"
+
+    instance = cluster.instances["dummy"]  # type: ClickHouseInstance
+    with pytest.raises(helpers.client.QueryRuntimeException) as ei:
+        result = run_query(instance, get_query)
+        print(result)
+
+    assert ei.value.returncode == 243
+    assert '403 Forbidden' in ei.value.stderr
+
+
 def test_infinite_redirect(cluster):
     bucket = "redirected"
     table_format = "column1 UInt32, column2 UInt32, column3 UInt32"

tests/queries/0_stateless/00515_gcd_lcm.reference

@@ -21,17 +21,3 @@
 4611686011984936962
 4611686011984936962
 2147483648
-256
-11
-64
-1
-2
-1
-1
-5120
-121
-256
-1
-4
-735
-64770

tests/queries/0_stateless/00515_gcd_lcm.sql

@@ -23,19 +23,19 @@ select lcm(255, 254);
 select lcm(2147483647, 2147483646);
 select lcm(4611686011984936962, 2147483647);
 select lcm(-2147483648, 1);
--- test gcd float will cast to int
+-- test gcd float
-select gcd(1280.1, 1024.1);
+select gcd(1280.1, 1024.1); -- { serverError 48 }
-select gcd(11.1, 121.1);
+select gcd(11.1, 121.1); -- { serverError 48 }
-select gcd(-256.1, 64.1);
+select gcd(-256.1, 64.1); -- { serverError 48 }
-select gcd(1.1, 1.1);
+select gcd(1.1, 1.1); -- { serverError 48 }
-select gcd(4.1, 2.1);
+select gcd(4.1, 2.1); -- { serverError 48 }
-select gcd(15.1, 49.1);
+select gcd(15.1, 49.1); -- { serverError 48 }
-select gcd(255.1, 254.1);
+select gcd(255.1, 254.1); -- { serverError 48 }
--- test lcm float cast to int
+-- test lcm float
-select lcm(1280.1, 1024.1);
+select lcm(1280.1, 1024.1); -- { serverError 48 }
-select lcm(11.1, 121.1);
+select lcm(11.1, 121.1); -- { serverError 48 }
-select lcm(-256.1, 64.1);
+select lcm(-256.1, 64.1); -- { serverError 48 }
-select lcm(1.1, 1.1);
+select lcm(1.1, 1.1); -- { serverError 48 }
-select lcm(4.1, 2.1);
+select lcm(4.1, 2.1); -- { serverError 48 }
-select lcm(15.1, 49.1);
+select lcm(15.1, 49.1); -- { serverError 48 }
-select lcm(255.1, 254.1);
+select lcm(255.1, 254.1); -- { serverError 48 }

tests/queries/0_stateless/01435_lcm_overflow.reference

@@ -5,4 +5,3 @@
 0
 0
 0
-0

tests/queries/0_stateless/01435_lcm_overflow.sql

@@ -6,5 +6,5 @@ SELECT lcm(-15, -10);
 -- Implementation specific result on overflow:
 SELECT ignore(lcm(256, 9223372036854775807));
 SELECT ignore(lcm(256, -9223372036854775807));
-SELECT ignore(lcm(-256, 9223372036854775807));
+SELECT ignore(lcm(-256, 9223372036854775807)); -- { serverError 407 }
 SELECT ignore(lcm(-256, -9223372036854775807));