From aa092eeffb8f09d9d65294bfa2a62cacc258e562 Mon Sep 17 00:00:00 2001
From: Anton Popov <pad11rus@gmail.com>
Date: Mon, 27 Dec 2021 16:42:06 +0300
Subject: [PATCH 001/104] proper handle of 'max_rows_to_read' in case of
 reading in order of sorting key and limit

---
 .../MergeTree/MergeTreeDataSelectExecutor.cpp | 14 ++++++++++--
 .../MergeTree/MergeTreeSelectProcessor.cpp    |  5 -----
 ...5_read_in_order_max_rows_to_read.reference |  6 +++++
 .../02155_read_in_order_max_rows_to_read.sql  | 22 +++++++++++++++++++
 4 files changed, 40 insertions(+), 7 deletions(-)
 create mode 100644 tests/queries/0_stateless/02155_read_in_order_max_rows_to_read.reference
 create mode 100644 tests/queries/0_stateless/02155_read_in_order_max_rows_to_read.sql

diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
index cdedd37e14a..07ac6f5764b 100644
--- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
+++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
@@ -875,12 +875,22 @@ RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipInd
     {
         std::atomic<size_t> total_rows{0};
 
+        /// Do not check number of read rows if we have reading
+        /// in order of sorting key with limit.
+        /// In general case, when there exists WHERE clause
+        /// it's impossible to estimate number of rows precisely,
+        /// because we can stop reading at any time.
+
         SizeLimits limits;
-        if (settings.read_overflow_mode == OverflowMode::THROW && settings.max_rows_to_read)
+        if (settings.read_overflow_mode == OverflowMode::THROW
+            && settings.max_rows_to_read
+            && !query_info.input_order_info)
             limits = SizeLimits(settings.max_rows_to_read, 0, settings.read_overflow_mode);
 
         SizeLimits leaf_limits;
-        if (settings.read_overflow_mode_leaf == OverflowMode::THROW && settings.max_rows_to_read_leaf)
+        if (settings.read_overflow_mode_leaf == OverflowMode::THROW
+            && settings.max_rows_to_read_leaf
+            && !query_info.input_order_info)
             leaf_limits = SizeLimits(settings.max_rows_to_read_leaf, 0, settings.read_overflow_mode_leaf);
 
         auto mark_cache = context->getIndexMarkCache();
diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp
index 2d4d3617cee..332eb27094a 100644
--- a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp
+++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp
@@ -37,11 +37,6 @@ MergeTreeSelectProcessor::MergeTreeSelectProcessor(
     has_limit_below_one_block(has_limit_below_one_block_),
     total_rows(data_part->index_granularity.getRowsCountInRanges(all_mark_ranges))
 {
-    /// Actually it means that parallel reading from replicas enabled
-    /// and we have to collaborate with initiator.
-    /// In this case we won't set approximate rows, because it will be accounted multiple times
-    if (!extension_.has_value())
-        addTotalRowsApprox(total_rows);
     ordered_names = header_without_virtual_columns.getNames();
 }
 
diff --git a/tests/queries/0_stateless/02155_read_in_order_max_rows_to_read.reference b/tests/queries/0_stateless/02155_read_in_order_max_rows_to_read.reference
new file mode 100644
index 00000000000..b73ab43cabb
--- /dev/null
+++ b/tests/queries/0_stateless/02155_read_in_order_max_rows_to_read.reference
@@ -0,0 +1,6 @@
+10
+0
+1
+2
+3
+4
diff --git a/tests/queries/0_stateless/02155_read_in_order_max_rows_to_read.sql b/tests/queries/0_stateless/02155_read_in_order_max_rows_to_read.sql
new file mode 100644
index 00000000000..e82c78b5e42
--- /dev/null
+++ b/tests/queries/0_stateless/02155_read_in_order_max_rows_to_read.sql
@@ -0,0 +1,22 @@
+DROP TABLE IF EXISTS t_max_rows_to_read;
+
+CREATE TABLE t_max_rows_to_read (a UInt64)
+ENGINE = MergeTree ORDER BY a
+SETTINGS index_granularity = 4;
+
+INSERT INTO t_max_rows_to_read SELECT number FROM numbers(100);
+
+SET max_threads = 1;
+
+SELECT a FROM t_max_rows_to_read WHERE a = 10 SETTINGS max_rows_to_read = 4;
+
+SELECT a FROM t_max_rows_to_read ORDER BY a LIMIT 5 SETTINGS max_rows_to_read = 12;
+
+-- This should work, but actually it doesn't. Need to investigate.
+-- SELECT a FROM t_max_rows_to_read WHERE a > 10 ORDER BY a LIMIT 5 SETTINGS max_rows_to_read = 20;
+
+SELECT a FROM t_max_rows_to_read ORDER BY a LIMIT 20 FORMAT Null SETTINGS max_rows_to_read = 12; -- { serverError 158 }
+SELECT a FROM t_max_rows_to_read WHERE a > 10 ORDER BY a LIMIT 5 FORMAT Null SETTINGS max_rows_to_read = 12; -- { serverError 158 }
+SELECT a FROM t_max_rows_to_read WHERE a = 10 OR a = 20 FORMAT Null SETTINGS max_rows_to_read = 4; -- { serverError 158 }
+
+DROP TABLE t_max_rows_to_read;

From e813f6413f12887d0a7a4fec1b5455379073bbe4 Mon Sep 17 00:00:00 2001
From: Pablo Alegre <palegre@tinybird.co>
Date: Wed, 29 Dec 2021 16:56:58 +0100
Subject: [PATCH 002/104] Add groupSortedArray() function

---
 .../reference/groupsortedarray.md             |  48 ++++
 .../aggregate-functions/reference/index.md    |   1 +
 .../AggregateFunctionGroupSortedArray.cpp     | 163 +++++++++++++
 .../AggregateFunctionGroupSortedArray.h       | 229 ++++++++++++++++++
 .../AggregateFunctionGroupSortedArrayData.h   | 162 +++++++++++++
 .../registerAggregateFunctions.cpp            |   2 +
 tests/performance/group_sorted_array.xml      |  25 ++
 .../02158_groupsortedarray.reference          |  14 ++
 .../0_stateless/02158_groupsortedarray.sql    |  38 +++
 9 files changed, 682 insertions(+)
 create mode 100644 docs/en/sql-reference/aggregate-functions/reference/groupsortedarray.md
 create mode 100644 src/AggregateFunctions/AggregateFunctionGroupSortedArray.cpp
 create mode 100644 src/AggregateFunctions/AggregateFunctionGroupSortedArray.h
 create mode 100644 src/AggregateFunctions/AggregateFunctionGroupSortedArrayData.h
 create mode 100644 tests/performance/group_sorted_array.xml
 create mode 100644 tests/queries/0_stateless/02158_groupsortedarray.reference
 create mode 100644 tests/queries/0_stateless/02158_groupsortedarray.sql

diff --git a/docs/en/sql-reference/aggregate-functions/reference/groupsortedarray.md b/docs/en/sql-reference/aggregate-functions/reference/groupsortedarray.md
new file mode 100644
index 00000000000..06c004173b8
--- /dev/null
+++ b/docs/en/sql-reference/aggregate-functions/reference/groupsortedarray.md
@@ -0,0 +1,48 @@
+---
+toc_priority: 108
+---
+
+# groupSortedArray {#groupSortedArray}
+
+Returns an array with the first N items in ascending order.
+
+``` sql
+groupSortedArray(N)(column)
+```
+
+**Arguments**
+
+-   `N` – The number of elements to return.
+
+If the parameter is omitted, default value 10 is used.
+
+**Arguments**
+
+-   `x` – The value.
+-   `expr` — Optional. The field or expresion to sort by. If not set values are sorted by themselves. [Integer](../../../sql-reference/data-types/int-uint.md).
+
+**Example**
+
+Gets the first 10 numbers:
+
+``` sql
+SELECT groupSortedArray(10)(number) FROM numbers(100)
+```
+
+``` text
+┌─groupSortedArray(10)(number)─┐
+│ [0,1,2,3,4,5,6,7,8,9]        │
+└──────────────────────────────┘
+```
+
+Or the last 10:
+
+``` sql
+SELECT groupSortedArray(10)(number, -number) FROM numbers(100)
+```
+
+``` text
+┌─groupSortedArray(10)(number, negate(number))─┐
+│ [99,98,97,96,95,94,93,92,91,90]              │
+└──────────────────────────────────────────────┘
+```
\ No newline at end of file
diff --git a/docs/en/sql-reference/aggregate-functions/reference/index.md b/docs/en/sql-reference/aggregate-functions/reference/index.md
index 59befed8785..14a8ecc9dcf 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/index.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/index.md
@@ -42,6 +42,7 @@ ClickHouse-specific aggregate functions:
 -   [groupBitmapAnd](../../../sql-reference/aggregate-functions/reference/groupbitmapand.md)
 -   [groupBitmapOr](../../../sql-reference/aggregate-functions/reference/groupbitmapor.md)
 -   [groupBitmapXor](../../../sql-reference/aggregate-functions/reference/groupbitmapxor.md)
+-   [groupSortedArray](../../../sql-reference/aggregate-functions/reference/groupsortedarray.md)
 -   [sumWithOverflow](../../../sql-reference/aggregate-functions/reference/sumwithoverflow.md)
 -   [sumMap](../../../sql-reference/aggregate-functions/reference/summap.md)
 -   [minMap](../../../sql-reference/aggregate-functions/reference/minmap.md)
diff --git a/src/AggregateFunctions/AggregateFunctionGroupSortedArray.cpp b/src/AggregateFunctions/AggregateFunctionGroupSortedArray.cpp
new file mode 100644
index 00000000000..e52091fc597
--- /dev/null
+++ b/src/AggregateFunctions/AggregateFunctionGroupSortedArray.cpp
@@ -0,0 +1,163 @@
+#include <AggregateFunctions/AggregateFunctionFactory.h>
+#include <AggregateFunctions/AggregateFunctionGroupSortedArray.h>
+#include <AggregateFunctions/FactoryHelpers.h>
+#include <AggregateFunctions/Helpers.h>
+#include <DataTypes/DataTypeDate.h>
+#include <DataTypes/DataTypeDateTime.h>
+#include <DataTypes/DataTypeString.h>
+#include <Common/FieldVisitorConvertToNumber.h>
+
+
+static inline constexpr UInt64 GROUP_SORTED_ARRAY_MAX_SIZE = 0xFFFFFF;
+static inline constexpr UInt64 GROUP_SORTED_ARRAY_DEFAULT_THRESHOLD = 10;
+
+
+namespace DB
+{
+struct Settings;
+
+namespace ErrorCodes
+{
+    extern const int ARGUMENT_OUT_OF_BOUND;
+    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+    extern const int LOGICAL_ERROR;
+    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+}
+
+
+namespace
+{
+    template <typename T, bool expr_sorted, typename TIndex>
+    class AggregateFunctionGroupSortedArrayNumeric : public AggregateFunctionGroupSortedArray<false, T, expr_sorted, TIndex>
+    {
+        using AggregateFunctionGroupSortedArray<false, T, expr_sorted, TIndex>::AggregateFunctionGroupSortedArray;
+    };
+
+    template <typename T, bool expr_sorted, typename TIndex>
+    class AggregateFunctionGroupSortedArrayFieldType
+        : public AggregateFunctionGroupSortedArray<false, typename T::FieldType, expr_sorted, TIndex>
+    {
+        using AggregateFunctionGroupSortedArray<false, typename T::FieldType, expr_sorted, TIndex>::AggregateFunctionGroupSortedArray;
+        DataTypePtr getReturnType() const override { return std::make_shared<DataTypeArray>(std::make_shared<T>()); }
+    };
+
+    template <bool expr_sorted, typename TIndex>
+    static IAggregateFunction * createWithExtraTypes(const DataTypes & argument_types, UInt64 threshold, const Array & params)
+    {
+        if (argument_types.empty())
+            throw DB::Exception(ErrorCodes::LOGICAL_ERROR, "Got empty arguments list");
+
+        WhichDataType which(argument_types[0]);
+        if (which.idx == TypeIndex::Date)
+            return new AggregateFunctionGroupSortedArrayFieldType<DataTypeDate, expr_sorted, TIndex>(threshold, argument_types, params);
+        if (which.idx == TypeIndex::DateTime)
+            return new AggregateFunctionGroupSortedArrayFieldType<DataTypeDateTime, expr_sorted, TIndex>(threshold, argument_types, params);
+
+        if (argument_types[0]->isValueUnambiguouslyRepresentedInContiguousMemoryRegion())
+        {
+            return new AggregateFunctionGroupSortedArray<true, StringRef, expr_sorted, TIndex>(threshold, argument_types, params);
+        }
+        else
+        {
+            return new AggregateFunctionGroupSortedArray<false, StringRef, expr_sorted, TIndex>(threshold, argument_types, params);
+        }
+    }
+
+    template <template <typename, bool, typename> class AggregateFunctionTemplate, bool bool_param, typename TIndex, typename... TArgs>
+    static IAggregateFunction * createWithNumericType2(const IDataType & argument_type, TArgs &&... args)
+    {
+        WhichDataType which(argument_type);
+#define DISPATCH(TYPE) \
+    if (which.idx == TypeIndex::TYPE) \
+        return new AggregateFunctionTemplate<TYPE, bool_param, TIndex>(std::forward<TArgs>(args)...);
+        FOR_NUMERIC_TYPES(DISPATCH)
+#undef DISPATCH
+        if (which.idx == TypeIndex::Enum8)
+            return new AggregateFunctionTemplate<Int8, bool_param, TIndex>(std::forward<TArgs>(args)...);
+        if (which.idx == TypeIndex::Enum16)
+            return new AggregateFunctionTemplate<Int16, bool_param, TIndex>(std::forward<TArgs>(args)...);
+        return nullptr;
+    }
+
+    template <bool expr_sorted, typename TIndex>
+    AggregateFunctionPtr createAggregateFunctionGroupSortedArrayTyped(
+        const std::string & name, const DataTypes & argument_types, const Array & params, UInt64 threshold)
+    {
+        AggregateFunctionPtr res(createWithNumericType2<AggregateFunctionGroupSortedArrayNumeric, expr_sorted, TIndex>(
+            *argument_types[0], threshold, argument_types, params));
+
+        if (!res)
+            res = AggregateFunctionPtr(createWithExtraTypes<expr_sorted, TIndex>(argument_types, threshold, params));
+
+        if (!res)
+            throw Exception(
+                "Illegal type " + argument_types[0]->getName() + " of argument for aggregate function " + name,
+                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+
+        return res;
+    }
+
+
+    template <bool expr_sorted>
+    AggregateFunctionPtr createAggregateFunctionGroupSortedArray_custom(
+        const std::string & name, const DataTypes & argument_types, const Array & params, const Settings *)
+    {
+        UInt64 threshold = GROUP_SORTED_ARRAY_DEFAULT_THRESHOLD;
+
+        if constexpr (!expr_sorted)
+        {
+            assertUnary(name, argument_types);
+        }
+        else
+        {
+            assertBinary(name, argument_types);
+            if (!isInteger(argument_types[1]))
+                throw Exception(
+                    "The second argument for aggregate function 'groupSortedArray' must have integer type",
+                    ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+        }
+
+        if (params.size() == 1)
+        {
+            UInt64 k = applyVisitor(FieldVisitorConvertToNumber<UInt64>(), params[0]);
+
+            if (k > GROUP_SORTED_ARRAY_MAX_SIZE)
+                throw Exception(
+                    "Too large parameter(s) for aggregate function " + name + ". Maximum: " + toString(GROUP_SORTED_ARRAY_MAX_SIZE),
+                    ErrorCodes::ARGUMENT_OUT_OF_BOUND);
+
+            if (k == 0)
+                throw Exception("Parameter 0 is illegal for aggregate function " + name, ErrorCodes::ARGUMENT_OUT_OF_BOUND);
+
+            threshold = k;
+        }
+
+        if (expr_sorted && isUnsignedInteger(argument_types[1]))
+            return createAggregateFunctionGroupSortedArrayTyped<expr_sorted, UInt64>(name, argument_types, params, threshold);
+        else
+            return createAggregateFunctionGroupSortedArrayTyped<expr_sorted, Int64>(name, argument_types, params, threshold);
+    }
+
+    AggregateFunctionPtr createAggregateFunctionGroupSortedArray(
+        const std::string & name, const DataTypes & argument_types, const Array & params, const Settings * settings)
+    {
+        if (argument_types.size() > 2 || argument_types.size() < 1)
+        {
+            throw Exception(
+                "Aggregate function " + name + " requires one or two parameters.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
+        }
+
+        if (argument_types.size() > 1)
+            return createAggregateFunctionGroupSortedArray_custom<true>(name, argument_types, params, settings);
+        else
+            return createAggregateFunctionGroupSortedArray_custom<false>(name, argument_types, params, settings);
+    }
+}
+
+void registerAggregateFunctionGroupSortedArray(AggregateFunctionFactory & factory)
+{
+    AggregateFunctionProperties properties = {.returns_default_when_only_null = false, .is_order_dependent = true};
+    factory.registerFunction("groupSortedArray", {createAggregateFunctionGroupSortedArray, properties});
+}
+
+}
diff --git a/src/AggregateFunctions/AggregateFunctionGroupSortedArray.h b/src/AggregateFunctions/AggregateFunctionGroupSortedArray.h
new file mode 100644
index 00000000000..b912ef1a7ba
--- /dev/null
+++ b/src/AggregateFunctions/AggregateFunctionGroupSortedArray.h
@@ -0,0 +1,229 @@
+#pragma once
+
+#include <Columns/ColumnArray.h>
+#include <DataTypes/DataTypeArray.h>
+
+#include <AggregateFunctions/AggregateFunctionGroupSortedArrayData.h>
+#include <AggregateFunctions/IAggregateFunction.h>
+
+namespace DB
+{
+template <typename T, bool is_plain_column>
+inline T readItem(const IColumn * column, Arena * arena, size_t row)
+{
+    if constexpr (std::is_same_v<T, StringRef>)
+    {
+        if constexpr (is_plain_column)
+        {
+            StringRef str = column->getDataAt(row);
+            auto ptr = arena->alloc(str.size);
+            std::copy(str.data, str.data + str.size, ptr);
+            return StringRef(ptr, str.size);
+        }
+        else
+        {
+            const char * begin = nullptr;
+            return column->serializeValueIntoArena(row, *arena, begin);
+        }
+    }
+    else
+    {
+        return column->getUInt(row);
+    }
+}
+
+template <typename T>
+void getFirstNElements(const T * data, int num_elements, int threshold, size_t * results)
+{
+    for (int i = 0; i < threshold; i++)
+    {
+        results[i] = 0;
+    }
+
+    threshold = std::min(num_elements, threshold);
+    int current_max = 0;
+    int cur;
+    int z;
+    for (int i = 0; i < num_elements; i++)
+    {
+        //Starting from the highest values and we look for the immediately lower than the given one
+        for (cur = current_max; cur > 0 && (data[i] < data[results[cur - 1]]); cur--)
+            ;
+
+        if (cur < threshold)
+        {
+            //Move all the higher values 1 position to the right
+            for (z = current_max - 1; z >= cur; z--)
+                results[z] = results[z - 1];
+
+            if (current_max < threshold)
+                ++current_max;
+
+            //insert element into the given position
+            results[cur] = i;
+        }
+    }
+}
+
+template <bool is_plain_column, typename T, bool expr_sorted, typename TIndex>
+class AggregateFunctionGroupSortedArray : public IAggregateFunctionDataHelper<
+                                              AggregateFunctionGroupSortedArrayData<T, expr_sorted, TIndex>,
+                                              AggregateFunctionGroupSortedArray<is_plain_column, T, expr_sorted, TIndex>>
+{
+protected:
+    using State = AggregateFunctionGroupSortedArrayData<T, expr_sorted, TIndex>;
+    using Base
+        = IAggregateFunctionDataHelper<AggregateFunctionGroupSortedArrayData<T, expr_sorted, TIndex>, AggregateFunctionGroupSortedArray>;
+
+    UInt64 threshold;
+    DataTypePtr & input_data_type;
+    mutable std::mutex mutex;
+
+    static void deserializeAndInsert(StringRef str, IColumn & data_to);
+
+public:
+    AggregateFunctionGroupSortedArray(UInt64 threshold_, const DataTypes & argument_types_, const Array & params)
+        : IAggregateFunctionDataHelper<AggregateFunctionGroupSortedArrayData<T, expr_sorted, TIndex>, AggregateFunctionGroupSortedArray>(
+            argument_types_, params)
+        , threshold(threshold_)
+        , input_data_type(this->argument_types[0])
+    {
+    }
+
+    void create(AggregateDataPtr place) const override
+    {
+        Base::create(place);
+        this->data(place).threshold = threshold;
+    }
+
+    String getName() const override { return "groupSortedArray"; }
+
+    DataTypePtr getReturnType() const override { return std::make_shared<DataTypeArray>(input_data_type); }
+
+    bool allocatesMemoryInArena() const override
+    {
+        if constexpr (std::is_same_v<T, StringRef>)
+            return true;
+        else
+            return false;
+    }
+
+    void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena * arena) const override
+    {
+        State & data = this->data(place);
+        if constexpr (expr_sorted)
+        {
+            data.add(readItem<T, is_plain_column>(columns[0], arena, row_num), readItem<TIndex, false>(columns[1], arena, row_num));
+        }
+        else
+        {
+            data.add(readItem<T, is_plain_column>(columns[0], arena, row_num));
+        }
+    }
+
+    void addBatchSinglePlace(
+        size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena, ssize_t if_argument_pos) const override
+    {
+        State & data = this->data(place);
+        if constexpr (expr_sorted)
+        {
+            StringRef ref = columns[1]->getRawData();
+            TIndex values[batch_size];
+            memcpy(values, ref.data, batch_size * sizeof(TIndex));
+            size_t num_results = std::min(this->threshold, batch_size);
+            size_t * bestRows = new size_t[batch_size];
+
+            //First store the first n elements with the column number
+            if (if_argument_pos >= 0)
+            {
+                TIndex * value_w = values;
+
+                const auto & flags = assert_cast<const ColumnUInt8 &>(*columns[if_argument_pos]).getData();
+                for (size_t i = 0; i < batch_size; ++i)
+                {
+                    if (flags[i])
+                        *(value_w++) = values[i];
+                }
+
+                batch_size = value_w - values;
+            }
+
+            num_results = std::min(this->threshold, batch_size);
+            getFirstNElements(values, batch_size, num_results, bestRows);
+            for (size_t i = 0; i < num_results; i++)
+            {
+                auto row = bestRows[i];
+                data.add(readItem<T, is_plain_column>(columns[0], arena, row), values[row]);
+            }
+            delete[] bestRows;
+        }
+        else
+        {
+            if (if_argument_pos >= 0)
+            {
+                const auto & flags = assert_cast<const ColumnUInt8 &>(*columns[if_argument_pos]).getData();
+                for (size_t i = 0; i < batch_size; ++i)
+                {
+                    if (flags[i])
+                    {
+                        data.add(readItem<T, is_plain_column>(columns[0], arena, i));
+                    }
+                }
+            }
+            else
+            {
+                for (size_t i = 0; i < batch_size; ++i)
+                {
+                    data.add(readItem<T, is_plain_column>(columns[0], arena, i));
+                }
+            }
+        }
+    }
+
+    void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena *) const override
+    {
+        this->data(place).merge(this->data(rhs));
+    }
+
+    void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional<size_t> /* version */) const override
+    {
+        this->data(place).serialize(buf);
+    }
+
+    void
+    deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, std::optional<size_t> /* version  */, Arena * arena) const override
+    {
+        this->data(place).deserialize(buf, arena);
+    }
+
+    void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena * /*arena*/) const override
+    {
+        ColumnArray & arr_to = assert_cast<ColumnArray &>(to);
+        ColumnArray::Offsets & offsets_to = arr_to.getOffsets();
+
+        auto & values = this->data(place).values;
+        offsets_to.push_back(offsets_to.back() + values.size());
+
+        IColumn & data_to = arr_to.getData();
+        for (auto value : values)
+        {
+            if constexpr (std::is_same_v<T, StringRef>)
+            {
+                auto str = State::itemValue(value);
+                if constexpr (is_plain_column)
+                {
+                    data_to.insertData(str.data, str.size);
+                }
+                else
+                {
+                    data_to.deserializeAndInsertFromArena(str.data);
+                }
+            }
+            else
+            {
+                data_to.insert(State::itemValue(value));
+            }
+        }
+    }
+};
+}
diff --git a/src/AggregateFunctions/AggregateFunctionGroupSortedArrayData.h b/src/AggregateFunctions/AggregateFunctionGroupSortedArrayData.h
new file mode 100644
index 00000000000..922fda14fa5
--- /dev/null
+++ b/src/AggregateFunctions/AggregateFunctionGroupSortedArrayData.h
@@ -0,0 +1,162 @@
+#pragma once
+
+#include <IO/ReadBuffer.h>
+#include <IO/ReadHelpers.h>
+#include <IO/VarInt.h>
+#include <IO/WriteBuffer.h>
+#include <IO/WriteHelpers.h>
+
+
+static inline constexpr UInt64 GROUP_SORTED_DEFAULT_THRESHOLD = 0xFFFFFF;
+
+namespace DB
+{
+template <typename T>
+static void writeOneItem(WriteBuffer & buf, T item)
+{
+    if constexpr (std::numeric_limits<T>::is_signed)
+    {
+        writeVarInt(item, buf);
+    }
+    else
+    {
+        writeVarUInt(item, buf);
+    }
+}
+
+static void writeOneItem(WriteBuffer & buf, const StringRef & item)
+{
+    writeBinary(item, buf);
+}
+
+template <typename T>
+static void readOneItem(ReadBuffer & buf, Arena * /*arena*/, T & item)
+{
+    if constexpr (std::numeric_limits<T>::is_signed)
+    {
+        DB::Int64 val;
+        readVarT(val, buf);
+        item = val;
+    }
+    else
+    {
+        DB::UInt64 val;
+        readVarT(val, buf);
+        item = val;
+    }
+}
+
+static void readOneItem(ReadBuffer & buf, Arena * arena, StringRef & item)
+{
+    item = readStringBinaryInto(*arena, buf);
+}
+
+template <typename Storage>
+struct AggregateFunctionGroupSortedArrayDataBase
+{
+    typedef typename Storage::value_type ValueType;
+    AggregateFunctionGroupSortedArrayDataBase(UInt64 threshold_ = GROUP_SORTED_DEFAULT_THRESHOLD) : threshold(threshold_) { }
+
+    virtual ~AggregateFunctionGroupSortedArrayDataBase() { }
+    inline void narrowDown()
+    {
+        while (values.size() > threshold)
+            values.erase(--values.end());
+    }
+
+    void merge(const AggregateFunctionGroupSortedArrayDataBase & other)
+    {
+        values.insert(other.values.begin(), other.values.end());
+        narrowDown();
+    }
+
+    void serialize(WriteBuffer & buf) const
+    {
+        writeOneItem(buf, UInt64(values.size()));
+        for (auto value : values)
+        {
+            serializeItem(buf, value);
+        }
+    }
+
+    virtual void serializeItem(WriteBuffer & buf, ValueType & val) const = 0;
+    virtual ValueType deserializeItem(ReadBuffer & buf, Arena * arena) const = 0;
+
+    void deserialize(ReadBuffer & buf, Arena * arena)
+    {
+        values.clear();
+        UInt64 length;
+        readOneItem(buf, nullptr, length);
+
+        while (length--)
+        {
+            values.insert(deserializeItem(buf, arena));
+        }
+
+        narrowDown();
+    }
+
+    UInt64 threshold;
+    Storage values;
+};
+
+template <typename T, bool expr_sorted, typename TIndex>
+struct AggregateFunctionGroupSortedArrayData
+{
+};
+
+template <typename T, typename TIndex>
+struct AggregateFunctionGroupSortedArrayData<T, true, TIndex> : public AggregateFunctionGroupSortedArrayDataBase<std::multimap<TIndex, T>>
+{
+    using Base = AggregateFunctionGroupSortedArrayDataBase<std::multimap<TIndex, T>>;
+    using Base::Base;
+
+    void add(T item, TIndex weight)
+    {
+        Base::values.insert({weight, item});
+        Base::narrowDown();
+    }
+
+    void serializeItem(WriteBuffer & buf, typename Base::ValueType & value) const override
+    {
+        writeOneItem(buf, value.first);
+        writeOneItem(buf, value.second);
+    }
+
+    virtual typename Base::ValueType deserializeItem(ReadBuffer & buf, Arena * arena) const override
+    {
+        Int64 first;
+        T second;
+        readOneItem(buf, arena, first);
+        readOneItem(buf, arena, second);
+
+        return {first, second};
+    }
+
+    static T itemValue(typename Base::ValueType & value) { return value.second; }
+};
+
+template <typename T, typename TIndex>
+struct AggregateFunctionGroupSortedArrayData<T, false, TIndex> : public AggregateFunctionGroupSortedArrayDataBase<std::multiset<T>>
+{
+    using Base = AggregateFunctionGroupSortedArrayDataBase<std::multiset<T>>;
+    using Base::Base;
+
+    void add(T item)
+    {
+        Base::values.insert(item);
+        Base::narrowDown();
+    }
+
+    void serializeItem(WriteBuffer & buf, typename Base::ValueType & value) const override { writeOneItem(buf, value); }
+
+    typename Base::ValueType deserializeItem(ReadBuffer & buf, Arena * arena) const override
+    {
+        T value;
+        readOneItem(buf, arena, value);
+        return value;
+    }
+
+    static T itemValue(typename Base::ValueType & value) { return value; }
+};
+}
diff --git a/src/AggregateFunctions/registerAggregateFunctions.cpp b/src/AggregateFunctions/registerAggregateFunctions.cpp
index 351adac31bb..c991407822c 100644
--- a/src/AggregateFunctions/registerAggregateFunctions.cpp
+++ b/src/AggregateFunctions/registerAggregateFunctions.cpp
@@ -59,6 +59,7 @@ void registerAggregateFunctionNothing(AggregateFunctionFactory &);
 void registerAggregateFunctionExponentialMovingAverage(AggregateFunctionFactory &);
 void registerAggregateFunctionSparkbar(AggregateFunctionFactory &);
 void registerAggregateFunctionIntervalLengthSum(AggregateFunctionFactory &);
+void registerAggregateFunctionGroupSortedArray(AggregateFunctionFactory & factory);
 
 class AggregateFunctionCombinatorFactory;
 void registerAggregateFunctionCombinatorIf(AggregateFunctionCombinatorFactory &);
@@ -130,6 +131,7 @@ void registerAggregateFunctions()
         registerAggregateFunctionIntervalLengthSum(factory);
         registerAggregateFunctionExponentialMovingAverage(factory);
         registerAggregateFunctionSparkbar(factory);
+        registerAggregateFunctionGroupSortedArray(factory);
 
         registerWindowFunctions(factory);
     }
diff --git a/tests/performance/group_sorted_array.xml b/tests/performance/group_sorted_array.xml
new file mode 100644
index 00000000000..16b64859c26
--- /dev/null
+++ b/tests/performance/group_sorted_array.xml
@@ -0,0 +1,25 @@
+<test>
+    <settings>
+        <max_threads>10</max_threads>
+    </settings>
+    <substitutions>
+        <substitution>
+            <name>items</name>
+            <values>
+                <value>1000</value>
+                <value>100000</value>
+                <value>10000000</value>
+            </values>
+        </substitution>
+    </substitutions>
+
+    <create_query>CREATE TABLE test ( `id` UInt64, `value` UInt64, `text` String ) ENGINE = Memory</create_query>
+    <fill_query>INSERT INTO test SELECT number as id, rand64() as value, toString(number) as text FROM numbers({items})</fill_query>
+    <query>SELECT groupSortedArray(10)(id, value) FROM test</query>
+    <query>SELECT groupSortedArray(10)(id, -value) FROM test</query>
+    <query>SELECT groupSortedArray(10)(text, value) FROM test</query>
+    <query>SELECT groupSortedArray(10)((id, text), value) FROM test</query>
+    <query>SELECT groupSortedArray(10)(value) FROM test</query>
+    <query>SELECT groupSortedArray(10)(text) FROM test</query>
+    <drop_query>DROP TABLE IF EXISTS test</drop_query>
+</test>
diff --git a/tests/queries/0_stateless/02158_groupsortedarray.reference b/tests/queries/0_stateless/02158_groupsortedarray.reference
new file mode 100644
index 00000000000..3970554484b
--- /dev/null
+++ b/tests/queries/0_stateless/02158_groupsortedarray.reference
@@ -0,0 +1,14 @@
+[0,1,2,3,4]
+[0,1,2,3,4,5,6,7,8,9]
+[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99]
+[999,998,997,996,995,994,993,992,991,990,989,988,987,986,985,984,983,982,981,980,979,978,977,976,975,974,973,972,971,970,969,968,967,966,965,964,963,962,961,960,959,958,957,956,955,954,953,952,951,950,949,948,947,946,945,944,943,942,941,940,939,938,937,936,935,934,933,932,931,930,929,928,927,926,925,924,923,922,921,920,919,918,917,916,915,914,913,912,911,910,909,908,907,906,905,904,903,902,901,900]
+['0','1','2','3','4']
+['0','1','2','3','4']
+['9','8','7','6','5']
+[0,1,2,3,4,5,6,7,8,9]
+[0,1,2,3,4,5,6,7,8,9]
+[(0,'0'),(1,'1'),(2,'2'),(3,'3'),(4,'4')]
+pablo	[1,2]
+luis	[1,3]
+pablo	[1,2]
+luis	[1,3]
diff --git a/tests/queries/0_stateless/02158_groupsortedarray.sql b/tests/queries/0_stateless/02158_groupsortedarray.sql
new file mode 100644
index 00000000000..d7ed1175cdc
--- /dev/null
+++ b/tests/queries/0_stateless/02158_groupsortedarray.sql
@@ -0,0 +1,38 @@
+
+SELECT groupSortedArray(5)(number) from numbers(100);
+
+SELECT groupSortedArray(number, number) from numbers(100);
+
+SELECT groupSortedArray(100)(number, number) from numbers(1000);
+
+SELECT groupSortedArray(100)(number, -number) from numbers(1000);
+
+SELECT groupSortedArray(5)(str, number) FROM (SELECT toString(number) as str, number FROM numbers(10));
+
+SELECT groupSortedArray(5)(text) FROM (select toString(number) as text from numbers(10));
+
+SELECT groupSortedArray(5)(text, -number) FROM (select toString(number) as text, number from numbers(10));
+
+SELECT groupSortedArray(10)(number, number) from numbers(100); -- { serverError 42 }
+
+SELECT groupSortedArray(10)(number) from numbers(100); -- { serverError 42 }
+
+SELECT groupSortedArray(5)(number, text) FROM (select toString(number) as text, number from numbers(10)); -- { serverError 43 }
+
+SELECT groupSortedArray(5)((number,text)) from (SELECT toString(number) as text, number FROM numbers(100));
+
+DROP TABLE IF EXISTS test;
+DROP VIEW IF EXISTS mv_test;
+CREATE TABLE test (`n` String, `h` Int64) ENGINE = MergeTree ORDER BY n;
+CREATE MATERIALIZED VIEW mv_test (`n` String, `h` AggregateFunction(groupSortedArray(2), Int64, Int64)) ENGINE = AggregatingMergeTree ORDER BY n AS SELECT n, groupSortedArrayState(2)(h, h) as h FROM test GROUP BY n;
+INSERT INTO test VALUES ('pablo',1)('pablo', 2)('luis', 1)('luis', 3)('pablo', 5)('pablo',4)('pablo', 5)('luis', 6)('luis', 7)('pablo', 8)('pablo',9)('pablo',10)('luis',11)('luis',12)('pablo',13);
+SELECT n, groupSortedArrayMerge(2)(h) from mv_test GROUP BY n;
+
+DROP TABLE IF EXISTS test;
+DROP VIEW IF EXISTS mv_test;
+CREATE TABLE test (`n` String, `h` Int64) ENGINE = MergeTree ORDER BY n;
+CREATE MATERIALIZED VIEW mv_test (`n` String, `h` AggregateFunction(groupSortedArray(2), Int64)) ENGINE = AggregatingMergeTree ORDER BY n AS SELECT n, groupSortedArrayState(2)(h) as h FROM test GROUP BY n;
+INSERT INTO test VALUES ('pablo',1)('pablo', 2)('luis', 1)('luis', 3)('pablo', 5)('pablo',4)('pablo', 5)('luis', 6)('luis', 7)('pablo', 8)('pablo',9)('pablo',10)('luis',11)('luis',12)('pablo',13);
+SELECT n, groupSortedArrayMerge(2)(h) from mv_test GROUP BY n;
+DROP TABLE test;
+DROP VIEW mv_test;
\ No newline at end of file

From 7f553d55ae57a2dcd1dda497736afce1612bdf99 Mon Sep 17 00:00:00 2001
From: Pablo Alegre <palegre@tinybird.co>
Date: Fri, 4 Feb 2022 15:53:27 +0100
Subject: [PATCH 003/104] fixup! Add groupSortedArray() function

- Fix memory access
- Support any type as sorting parameter
- Fix tests
- Rewrite/simplify function addBatchSinglePlace
---
 .../reference/groupsortedarray.md             |   4 +-
 .../AggregateFunctionGroupSortedArray.cpp     | 106 ++++++------
 .../AggregateFunctionGroupSortedArray.h       | 157 ++++++++++--------
 .../AggregateFunctionGroupSortedArrayData.h   |   2 +-
 .../02158_groupsortedarray.reference          |   7 +-
 .../0_stateless/02158_groupsortedarray.sql    |  17 +-
 6 files changed, 162 insertions(+), 131 deletions(-)

diff --git a/docs/en/sql-reference/aggregate-functions/reference/groupsortedarray.md b/docs/en/sql-reference/aggregate-functions/reference/groupsortedarray.md
index 06c004173b8..28ad7db3bc6 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/groupsortedarray.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/groupsortedarray.md
@@ -18,8 +18,8 @@ If the parameter is omitted, default value 10 is used.
 
 **Arguments**
 
--   `x` – The value.
--   `expr` — Optional. The field or expresion to sort by. If not set values are sorted by themselves. [Integer](../../../sql-reference/data-types/int-uint.md).
+-   `column` – The value.
+-   `expr` — Optional. The field or expresion to sort by. If not set values are sorted by themselves.
 
 **Example**
 
diff --git a/src/AggregateFunctions/AggregateFunctionGroupSortedArray.cpp b/src/AggregateFunctions/AggregateFunctionGroupSortedArray.cpp
index e52091fc597..d47aab2b5be 100644
--- a/src/AggregateFunctions/AggregateFunctionGroupSortedArray.cpp
+++ b/src/AggregateFunctions/AggregateFunctionGroupSortedArray.cpp
@@ -27,21 +27,22 @@ namespace ErrorCodes
 
 namespace
 {
-    template <typename T, bool expr_sorted, typename TIndex>
-    class AggregateFunctionGroupSortedArrayNumeric : public AggregateFunctionGroupSortedArray<false, T, expr_sorted, TIndex>
+    template <typename T, bool expr_sorted, typename TColumnB, bool is_plain_b>
+    class AggregateFunctionGroupSortedArrayNumeric : public AggregateFunctionGroupSortedArray<T, false, expr_sorted, TColumnB, is_plain_b>
     {
-        using AggregateFunctionGroupSortedArray<false, T, expr_sorted, TIndex>::AggregateFunctionGroupSortedArray;
+        using AggregateFunctionGroupSortedArray<T, false, expr_sorted, TColumnB, is_plain_b>::AggregateFunctionGroupSortedArray;
     };
 
-    template <typename T, bool expr_sorted, typename TIndex>
+    template <typename T, bool expr_sorted, typename TColumnB, bool is_plain_b>
     class AggregateFunctionGroupSortedArrayFieldType
-        : public AggregateFunctionGroupSortedArray<false, typename T::FieldType, expr_sorted, TIndex>
+        : public AggregateFunctionGroupSortedArray<typename T::FieldType, false, expr_sorted, TColumnB, is_plain_b>
     {
-        using AggregateFunctionGroupSortedArray<false, typename T::FieldType, expr_sorted, TIndex>::AggregateFunctionGroupSortedArray;
+        using AggregateFunctionGroupSortedArray<typename T::FieldType, false, expr_sorted, TColumnB, is_plain_b>::
+            AggregateFunctionGroupSortedArray;
         DataTypePtr getReturnType() const override { return std::make_shared<DataTypeArray>(std::make_shared<T>()); }
     };
 
-    template <bool expr_sorted, typename TIndex>
+    template <bool expr_sorted, typename TColumnB, bool is_plain_b>
     static IAggregateFunction * createWithExtraTypes(const DataTypes & argument_types, UInt64 threshold, const Array & params)
     {
         if (argument_types.empty())
@@ -49,45 +50,56 @@ namespace
 
         WhichDataType which(argument_types[0]);
         if (which.idx == TypeIndex::Date)
-            return new AggregateFunctionGroupSortedArrayFieldType<DataTypeDate, expr_sorted, TIndex>(threshold, argument_types, params);
+            return new AggregateFunctionGroupSortedArrayFieldType<DataTypeDate, expr_sorted, TColumnB, is_plain_b>(
+                threshold, argument_types, params);
         if (which.idx == TypeIndex::DateTime)
-            return new AggregateFunctionGroupSortedArrayFieldType<DataTypeDateTime, expr_sorted, TIndex>(threshold, argument_types, params);
+            return new AggregateFunctionGroupSortedArrayFieldType<DataTypeDateTime, expr_sorted, TColumnB, is_plain_b>(
+                threshold, argument_types, params);
 
         if (argument_types[0]->isValueUnambiguouslyRepresentedInContiguousMemoryRegion())
         {
-            return new AggregateFunctionGroupSortedArray<true, StringRef, expr_sorted, TIndex>(threshold, argument_types, params);
+            return new AggregateFunctionGroupSortedArray<StringRef, true, expr_sorted, TColumnB, is_plain_b>(
+                threshold, argument_types, params);
         }
         else
+
         {
-            return new AggregateFunctionGroupSortedArray<false, StringRef, expr_sorted, TIndex>(threshold, argument_types, params);
+            return new AggregateFunctionGroupSortedArray<StringRef, false, expr_sorted, TColumnB, is_plain_b>(
+                threshold, argument_types, params);
         }
     }
 
-    template <template <typename, bool, typename> class AggregateFunctionTemplate, bool bool_param, typename TIndex, typename... TArgs>
+    template <
+        template <typename, bool, typename, bool>
+        class AggregateFunctionTemplate,
+        bool bool_param,
+        typename TColumnB,
+        bool is_plain_b,
+        typename... TArgs>
     static IAggregateFunction * createWithNumericType2(const IDataType & argument_type, TArgs &&... args)
     {
         WhichDataType which(argument_type);
 #define DISPATCH(TYPE) \
     if (which.idx == TypeIndex::TYPE) \
-        return new AggregateFunctionTemplate<TYPE, bool_param, TIndex>(std::forward<TArgs>(args)...);
+        return new AggregateFunctionTemplate<TYPE, bool_param, TColumnB, is_plain_b>(std::forward<TArgs>(args)...);
         FOR_NUMERIC_TYPES(DISPATCH)
 #undef DISPATCH
         if (which.idx == TypeIndex::Enum8)
-            return new AggregateFunctionTemplate<Int8, bool_param, TIndex>(std::forward<TArgs>(args)...);
+            return new AggregateFunctionTemplate<Int8, bool_param, TColumnB, is_plain_b>(std::forward<TArgs>(args)...);
         if (which.idx == TypeIndex::Enum16)
-            return new AggregateFunctionTemplate<Int16, bool_param, TIndex>(std::forward<TArgs>(args)...);
+            return new AggregateFunctionTemplate<Int16, bool_param, TColumnB, is_plain_b>(std::forward<TArgs>(args)...);
         return nullptr;
     }
 
-    template <bool expr_sorted, typename TIndex>
+    template <bool expr_sorted = false, typename TColumnB = UInt64, bool is_plain_b = false>
     AggregateFunctionPtr createAggregateFunctionGroupSortedArrayTyped(
         const std::string & name, const DataTypes & argument_types, const Array & params, UInt64 threshold)
     {
-        AggregateFunctionPtr res(createWithNumericType2<AggregateFunctionGroupSortedArrayNumeric, expr_sorted, TIndex>(
+        AggregateFunctionPtr res(createWithNumericType2<AggregateFunctionGroupSortedArrayNumeric, expr_sorted, TColumnB, is_plain_b>(
             *argument_types[0], threshold, argument_types, params));
 
         if (!res)
-            res = AggregateFunctionPtr(createWithExtraTypes<expr_sorted, TIndex>(argument_types, threshold, params));
+            res = AggregateFunctionPtr(createWithExtraTypes<expr_sorted, TColumnB, is_plain_b>(argument_types, threshold, params));
 
         if (!res)
             throw Exception(
@@ -98,25 +110,11 @@ namespace
     }
 
 
-    template <bool expr_sorted>
-    AggregateFunctionPtr createAggregateFunctionGroupSortedArray_custom(
+    AggregateFunctionPtr createAggregateFunctionGroupSortedArray(
         const std::string & name, const DataTypes & argument_types, const Array & params, const Settings *)
     {
         UInt64 threshold = GROUP_SORTED_ARRAY_DEFAULT_THRESHOLD;
 
-        if constexpr (!expr_sorted)
-        {
-            assertUnary(name, argument_types);
-        }
-        else
-        {
-            assertBinary(name, argument_types);
-            if (!isInteger(argument_types[1]))
-                throw Exception(
-                    "The second argument for aggregate function 'groupSortedArray' must have integer type",
-                    ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
-        }
-
         if (params.size() == 1)
         {
             UInt64 k = applyVisitor(FieldVisitorConvertToNumber<UInt64>(), params[0]);
@@ -131,26 +129,39 @@ namespace
 
             threshold = k;
         }
+        else if (params.size() != 0)
+        {
+            throw Exception("Aggregate function " + name + " only supports 1 parameter.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
+        }
 
-        if (expr_sorted && isUnsignedInteger(argument_types[1]))
-            return createAggregateFunctionGroupSortedArrayTyped<expr_sorted, UInt64>(name, argument_types, params, threshold);
+        if (argument_types.size() == 2)
+        {
+            if (isUnsignedInteger(argument_types[1]))
+            {
+                return createAggregateFunctionGroupSortedArrayTyped<true, UInt64>(name, argument_types, params, threshold);
+            }
+            else if (isInteger(argument_types[1]))
+            {
+                return createAggregateFunctionGroupSortedArrayTyped<true, Int64>(name, argument_types, params, threshold);
+            }
+            else if (argument_types[1]->isValueUnambiguouslyRepresentedInContiguousMemoryRegion())
+            {
+                return createAggregateFunctionGroupSortedArrayTyped<true, StringRef, true>(name, argument_types, params, threshold);
+            }
+            else
+            {
+                return createAggregateFunctionGroupSortedArrayTyped<true, StringRef, false>(name, argument_types, params, threshold);
+            }
+        }
+        else if (argument_types.size() == 1)
+        {
+            return createAggregateFunctionGroupSortedArrayTyped<>(name, argument_types, params, threshold);
+        }
         else
-            return createAggregateFunctionGroupSortedArrayTyped<expr_sorted, Int64>(name, argument_types, params, threshold);
-    }
-
-    AggregateFunctionPtr createAggregateFunctionGroupSortedArray(
-        const std::string & name, const DataTypes & argument_types, const Array & params, const Settings * settings)
-    {
-        if (argument_types.size() > 2 || argument_types.size() < 1)
         {
             throw Exception(
                 "Aggregate function " + name + " requires one or two parameters.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
         }
-
-        if (argument_types.size() > 1)
-            return createAggregateFunctionGroupSortedArray_custom<true>(name, argument_types, params, settings);
-        else
-            return createAggregateFunctionGroupSortedArray_custom<false>(name, argument_types, params, settings);
     }
 }
 
@@ -159,5 +170,4 @@ void registerAggregateFunctionGroupSortedArray(AggregateFunctionFactory & factor
     AggregateFunctionProperties properties = {.returns_default_when_only_null = false, .is_order_dependent = true};
     factory.registerFunction("groupSortedArray", {createAggregateFunctionGroupSortedArray, properties});
 }
-
 }
diff --git a/src/AggregateFunctions/AggregateFunctionGroupSortedArray.h b/src/AggregateFunctions/AggregateFunctionGroupSortedArray.h
index b912ef1a7ba..5f9a67a21ae 100644
--- a/src/AggregateFunctions/AggregateFunctionGroupSortedArray.h
+++ b/src/AggregateFunctions/AggregateFunctionGroupSortedArray.h
@@ -8,12 +8,12 @@
 
 namespace DB
 {
-template <typename T, bool is_plain_column>
-inline T readItem(const IColumn * column, Arena * arena, size_t row)
+template <typename TColumn, bool is_plain>
+inline TColumn readItem(const IColumn * column, Arena * arena, size_t row)
 {
-    if constexpr (std::is_same_v<T, StringRef>)
+    if constexpr (std::is_same_v<TColumn, StringRef>)
     {
-        if constexpr (is_plain_column)
+        if constexpr (is_plain)
         {
             StringRef str = column->getDataAt(row);
             auto ptr = arena->alloc(str.size);
@@ -28,12 +28,15 @@ inline T readItem(const IColumn * column, Arena * arena, size_t row)
     }
     else
     {
-        return column->getUInt(row);
+        if constexpr (std::is_same_v<TColumn, UInt64>)
+            return column->getUInt(row);
+        else
+            return column->getInt(row);
     }
 }
 
-template <typename T>
-void getFirstNElements(const T * data, int num_elements, int threshold, size_t * results)
+template <typename TColumn>
+size_t getFirstNElements(const TColumn * data, int num_elements, int threshold, size_t * results, const UInt8 * filter = nullptr)
 {
     for (int i = 0; i < threshold; i++)
     {
@@ -46,14 +49,20 @@ void getFirstNElements(const T * data, int num_elements, int threshold, size_t *
     int z;
     for (int i = 0; i < num_elements; i++)
     {
+        if (filter && (filter[i] == 0))
+            continue;
+
         //Starting from the highest values and we look for the immediately lower than the given one
-        for (cur = current_max; cur > 0 && (data[i] < data[results[cur - 1]]); cur--)
-            ;
+        for (cur = current_max; cur > 0; cur--)
+        {
+            if (!(data[i] < data[results[cur - 1]]))
+                break;
+        }
 
         if (cur < threshold)
         {
             //Move all the higher values 1 position to the right
-            for (z = current_max - 1; z >= cur; z--)
+            for (z = current_max - 1; z > cur; z--)
                 results[z] = results[z - 1];
 
             if (current_max < threshold)
@@ -63,17 +72,20 @@ void getFirstNElements(const T * data, int num_elements, int threshold, size_t *
             results[cur] = i;
         }
     }
+
+    return current_max;
 }
 
-template <bool is_plain_column, typename T, bool expr_sorted, typename TIndex>
+template <typename TColumnA, bool is_plain_a, bool use_column_b, typename TColumnB, bool is_plain_b>
 class AggregateFunctionGroupSortedArray : public IAggregateFunctionDataHelper<
-                                              AggregateFunctionGroupSortedArrayData<T, expr_sorted, TIndex>,
-                                              AggregateFunctionGroupSortedArray<is_plain_column, T, expr_sorted, TIndex>>
+                                              AggregateFunctionGroupSortedArrayData<TColumnA, use_column_b, TColumnB>,
+                                              AggregateFunctionGroupSortedArray<TColumnA, is_plain_a, use_column_b, TColumnB, is_plain_b>>
 {
 protected:
-    using State = AggregateFunctionGroupSortedArrayData<T, expr_sorted, TIndex>;
-    using Base
-        = IAggregateFunctionDataHelper<AggregateFunctionGroupSortedArrayData<T, expr_sorted, TIndex>, AggregateFunctionGroupSortedArray>;
+    using State = AggregateFunctionGroupSortedArrayData<TColumnA, use_column_b, TColumnB>;
+    using Base = IAggregateFunctionDataHelper<
+        AggregateFunctionGroupSortedArrayData<TColumnA, use_column_b, TColumnB>,
+        AggregateFunctionGroupSortedArray>;
 
     UInt64 threshold;
     DataTypePtr & input_data_type;
@@ -83,8 +95,9 @@ protected:
 
 public:
     AggregateFunctionGroupSortedArray(UInt64 threshold_, const DataTypes & argument_types_, const Array & params)
-        : IAggregateFunctionDataHelper<AggregateFunctionGroupSortedArrayData<T, expr_sorted, TIndex>, AggregateFunctionGroupSortedArray>(
-            argument_types_, params)
+        : IAggregateFunctionDataHelper<
+            AggregateFunctionGroupSortedArrayData<TColumnA, use_column_b, TColumnB>,
+            AggregateFunctionGroupSortedArray>(argument_types_, params)
         , threshold(threshold_)
         , input_data_type(this->argument_types[0])
     {
@@ -102,7 +115,7 @@ public:
 
     bool allocatesMemoryInArena() const override
     {
-        if constexpr (std::is_same_v<T, StringRef>)
+        if constexpr (std::is_same_v<TColumnA, StringRef>)
             return true;
         else
             return false;
@@ -111,13 +124,51 @@ public:
     void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena * arena) const override
     {
         State & data = this->data(place);
-        if constexpr (expr_sorted)
+        if constexpr (use_column_b)
         {
-            data.add(readItem<T, is_plain_column>(columns[0], arena, row_num), readItem<TIndex, false>(columns[1], arena, row_num));
+            data.add(
+                readItem<TColumnA, is_plain_a>(columns[0], arena, row_num), readItem<TColumnB, is_plain_b>(columns[1], arena, row_num));
         }
         else
         {
-            data.add(readItem<T, is_plain_column>(columns[0], arena, row_num));
+            data.add(readItem<TColumnA, is_plain_a>(columns[0], arena, row_num));
+        }
+    }
+
+    template <typename TColumn, bool is_plain, typename TFunc>
+    void
+    forFirstRows(size_t batch_size, const IColumn ** columns, size_t data_column, Arena * arena, ssize_t if_argument_pos, TFunc func) const
+    {
+        const TColumn * values = nullptr;
+        std::unique_ptr<std::vector<TColumn>> values_vector;
+        std::vector<size_t> best_rows(threshold);
+
+        if constexpr (std::is_same_v<TColumn, StringRef>)
+        {
+            values_vector.reset(new std::vector<TColumn>(batch_size));
+            for (size_t i = 0; i < batch_size; i++)
+                (*values_vector)[i] = readItem<TColumn, is_plain>(columns[data_column], arena, i);
+            values = (*values_vector).data();
+        }
+        else
+        {
+            StringRef ref = columns[data_column]->getRawData();
+            values = reinterpret_cast<const TColumn *>(ref.data);
+        }
+
+        const UInt8 * filter = nullptr;
+        StringRef refFilter;
+
+        if (if_argument_pos >= 0)
+        {
+            refFilter = columns[if_argument_pos]->getRawData();
+            filter = reinterpret_cast<const UInt8 *>(refFilter.data);
+        }
+
+        size_t num_elements = getFirstNElements(values, batch_size, threshold, best_rows.data(), filter);
+        for (size_t i = 0; i < num_elements; i++)
+        {
+            func(best_rows[i], values);
         }
     }
 
@@ -125,58 +176,22 @@ public:
         size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena, ssize_t if_argument_pos) const override
     {
         State & data = this->data(place);
-        if constexpr (expr_sorted)
+
+        if constexpr (use_column_b)
         {
-            StringRef ref = columns[1]->getRawData();
-            TIndex values[batch_size];
-            memcpy(values, ref.data, batch_size * sizeof(TIndex));
-            size_t num_results = std::min(this->threshold, batch_size);
-            size_t * bestRows = new size_t[batch_size];
-
-            //First store the first n elements with the column number
-            if (if_argument_pos >= 0)
-            {
-                TIndex * value_w = values;
-
-                const auto & flags = assert_cast<const ColumnUInt8 &>(*columns[if_argument_pos]).getData();
-                for (size_t i = 0; i < batch_size; ++i)
+            forFirstRows<TColumnB, is_plain_b>(
+                batch_size, columns, 1, arena, if_argument_pos, [columns, &arena, &data](size_t row, const TColumnB * values) 
                 {
-                    if (flags[i])
-                        *(value_w++) = values[i];
-                }
-
-                batch_size = value_w - values;
-            }
-
-            num_results = std::min(this->threshold, batch_size);
-            getFirstNElements(values, batch_size, num_results, bestRows);
-            for (size_t i = 0; i < num_results; i++)
-            {
-                auto row = bestRows[i];
-                data.add(readItem<T, is_plain_column>(columns[0], arena, row), values[row]);
-            }
-            delete[] bestRows;
+                    data.add(readItem<TColumnA, is_plain_a>(columns[0], arena, row), values[row]);
+                });
         }
         else
         {
-            if (if_argument_pos >= 0)
-            {
-                const auto & flags = assert_cast<const ColumnUInt8 &>(*columns[if_argument_pos]).getData();
-                for (size_t i = 0; i < batch_size; ++i)
-                {
-                    if (flags[i])
-                    {
-                        data.add(readItem<T, is_plain_column>(columns[0], arena, i));
-                    }
-                }
-            }
-            else
-            {
-                for (size_t i = 0; i < batch_size; ++i)
-                {
-                    data.add(readItem<T, is_plain_column>(columns[0], arena, i));
-                }
-            }
+            forFirstRows<TColumnA, is_plain_a>(
+                batch_size, columns, 0, arena, if_argument_pos, [&data](size_t row, const TColumnA * values) 
+                { 
+                    data.add(values[row]); 
+                });
         }
     }
 
@@ -207,10 +222,10 @@ public:
         IColumn & data_to = arr_to.getData();
         for (auto value : values)
         {
-            if constexpr (std::is_same_v<T, StringRef>)
+            if constexpr (std::is_same_v<TColumnA, StringRef>)
             {
                 auto str = State::itemValue(value);
-                if constexpr (is_plain_column)
+                if constexpr (is_plain_a)
                 {
                     data_to.insertData(str.data, str.size);
                 }
diff --git a/src/AggregateFunctions/AggregateFunctionGroupSortedArrayData.h b/src/AggregateFunctions/AggregateFunctionGroupSortedArrayData.h
index 922fda14fa5..ca2d216ba8e 100644
--- a/src/AggregateFunctions/AggregateFunctionGroupSortedArrayData.h
+++ b/src/AggregateFunctions/AggregateFunctionGroupSortedArrayData.h
@@ -125,7 +125,7 @@ struct AggregateFunctionGroupSortedArrayData<T, true, TIndex> : public Aggregate
 
     virtual typename Base::ValueType deserializeItem(ReadBuffer & buf, Arena * arena) const override
     {
-        Int64 first;
+        TIndex first;
         T second;
         readOneItem(buf, arena, first);
         readOneItem(buf, arena, second);
diff --git a/tests/queries/0_stateless/02158_groupsortedarray.reference b/tests/queries/0_stateless/02158_groupsortedarray.reference
index 3970554484b..b4fec227621 100644
--- a/tests/queries/0_stateless/02158_groupsortedarray.reference
+++ b/tests/queries/0_stateless/02158_groupsortedarray.reference
@@ -5,10 +5,13 @@
 ['0','1','2','3','4']
 ['0','1','2','3','4']
 ['9','8','7','6','5']
-[0,1,2,3,4,5,6,7,8,9]
-[0,1,2,3,4,5,6,7,8,9]
 [(0,'0'),(1,'1'),(2,'2'),(3,'3'),(4,'4')]
+['0','1','10','11','12']
+['0','1','2','3','4','5','6','7','8','9','10','11','12','13','14','15','16','17','18','19','20','21','22','23','24','25','26','27','28','29','30','31','32','33','34','35','36','37','38','39','40','41','42','43','44','45','46','47','48','49']
 pablo	[1,2]
 luis	[1,3]
 pablo	[1,2]
 luis	[1,3]
+[4,5,6,7,8]
+[10,11,12,13,14]
+['10','11','12','13','14']
diff --git a/tests/queries/0_stateless/02158_groupsortedarray.sql b/tests/queries/0_stateless/02158_groupsortedarray.sql
index d7ed1175cdc..e60741d1332 100644
--- a/tests/queries/0_stateless/02158_groupsortedarray.sql
+++ b/tests/queries/0_stateless/02158_groupsortedarray.sql
@@ -13,14 +13,13 @@ SELECT groupSortedArray(5)(text) FROM (select toString(number) as text from numb
 
 SELECT groupSortedArray(5)(text, -number) FROM (select toString(number) as text, number from numbers(10));
 
-SELECT groupSortedArray(10)(number, number) from numbers(100); -- { serverError 42 }
-
-SELECT groupSortedArray(10)(number) from numbers(100); -- { serverError 42 }
-
-SELECT groupSortedArray(5)(number, text) FROM (select toString(number) as text, number from numbers(10)); -- { serverError 43 }
-
 SELECT groupSortedArray(5)((number,text)) from (SELECT toString(number) as text, number FROM numbers(100));
 
+SELECT groupSortedArray(5)(text,text) from (SELECT toString(number) as text FROM numbers(100));
+
+SELECT groupSortedArray(50)(text,(number,text)) from (SELECT toString(number) as text, number FROM numbers(100));
+
+
 DROP TABLE IF EXISTS test;
 DROP VIEW IF EXISTS mv_test;
 CREATE TABLE test (`n` String, `h` Int64) ENGINE = MergeTree ORDER BY n;
@@ -35,4 +34,8 @@ CREATE MATERIALIZED VIEW mv_test (`n` String, `h` AggregateFunction(groupSortedA
 INSERT INTO test VALUES ('pablo',1)('pablo', 2)('luis', 1)('luis', 3)('pablo', 5)('pablo',4)('pablo', 5)('luis', 6)('luis', 7)('pablo', 8)('pablo',9)('pablo',10)('luis',11)('luis',12)('pablo',13);
 SELECT n, groupSortedArrayMerge(2)(h) from mv_test GROUP BY n;
 DROP TABLE test;
-DROP VIEW mv_test;
\ No newline at end of file
+DROP VIEW mv_test;
+
+SELECT groupSortedArrayIf(5)(number, number, number>3) from numbers(100);
+SELECT groupSortedArrayIf(5)(number, toString(number), number>3) from numbers(100);
+SELECT groupSortedArrayIf(5)(toString(number), number>3) from numbers(100);

From 9c38b1a0310db616caf342b9595441b2f0081525 Mon Sep 17 00:00:00 2001
From: Pablo Alegre <palegre@tinybird.co>
Date: Mon, 7 Feb 2022 14:25:23 +0100
Subject: [PATCH 004/104] fixup! Add groupSortedArray() function

---
 .../AggregateFunctionGroupSortedArray.cpp     | 98 +++++++------------
 .../AggregateFunctionGroupSortedArray.h       |  8 +-
 2 files changed, 37 insertions(+), 69 deletions(-)

diff --git a/src/AggregateFunctions/AggregateFunctionGroupSortedArray.cpp b/src/AggregateFunctions/AggregateFunctionGroupSortedArray.cpp
index d47aab2b5be..ef52428aecd 100644
--- a/src/AggregateFunctions/AggregateFunctionGroupSortedArray.cpp
+++ b/src/AggregateFunctions/AggregateFunctionGroupSortedArray.cpp
@@ -42,73 +42,35 @@ namespace
         DataTypePtr getReturnType() const override { return std::make_shared<DataTypeArray>(std::make_shared<T>()); }
     };
 
-    template <bool expr_sorted, typename TColumnB, bool is_plain_b>
-    static IAggregateFunction * createWithExtraTypes(const DataTypes & argument_types, UInt64 threshold, const Array & params)
+    template <bool expr_sorted = false, typename TColumnB = UInt64, bool is_plain_b = false>
+    AggregateFunctionPtr
+    createAggregateFunctionGroupSortedArrayTyped(const DataTypes & argument_types, const Array & params, UInt64 threshold)
     {
-        if (argument_types.empty())
-            throw DB::Exception(ErrorCodes::LOGICAL_ERROR, "Got empty arguments list");
-
+#define DISPATCH(A, CLS, B) \
+    if (which.idx == TypeIndex::A) \
+        return AggregateFunctionPtr(new CLS<B, expr_sorted, TColumnB, is_plain_b>(threshold, argument_types, params));
+#define DISPATCH_NUMERIC(A) DISPATCH(A, AggregateFunctionGroupSortedArrayNumeric, A)
         WhichDataType which(argument_types[0]);
-        if (which.idx == TypeIndex::Date)
-            return new AggregateFunctionGroupSortedArrayFieldType<DataTypeDate, expr_sorted, TColumnB, is_plain_b>(
-                threshold, argument_types, params);
-        if (which.idx == TypeIndex::DateTime)
-            return new AggregateFunctionGroupSortedArrayFieldType<DataTypeDateTime, expr_sorted, TColumnB, is_plain_b>(
-                threshold, argument_types, params);
+        FOR_NUMERIC_TYPES(DISPATCH_NUMERIC)
+        DISPATCH(Enum8, AggregateFunctionGroupSortedArrayNumeric, Int8)
+        DISPATCH(Enum16, AggregateFunctionGroupSortedArrayNumeric, Int16)
+        DISPATCH(Date, AggregateFunctionGroupSortedArrayFieldType, DataTypeDate)
+        DISPATCH(DateTime, AggregateFunctionGroupSortedArrayFieldType, DataTypeDateTime)
+#undef DISPATCH
+#undef DISPATCH_NUMERIC
 
         if (argument_types[0]->isValueUnambiguouslyRepresentedInContiguousMemoryRegion())
         {
-            return new AggregateFunctionGroupSortedArray<StringRef, true, expr_sorted, TColumnB, is_plain_b>(
-                threshold, argument_types, params);
+            return AggregateFunctionPtr(new AggregateFunctionGroupSortedArray<StringRef, true, expr_sorted, TColumnB, is_plain_b>(
+                threshold, argument_types, params));
         }
         else
-
         {
-            return new AggregateFunctionGroupSortedArray<StringRef, false, expr_sorted, TColumnB, is_plain_b>(
-                threshold, argument_types, params);
+            return AggregateFunctionPtr(new AggregateFunctionGroupSortedArray<StringRef, false, expr_sorted, TColumnB, is_plain_b>(
+                threshold, argument_types, params));
         }
     }
 
-    template <
-        template <typename, bool, typename, bool>
-        class AggregateFunctionTemplate,
-        bool bool_param,
-        typename TColumnB,
-        bool is_plain_b,
-        typename... TArgs>
-    static IAggregateFunction * createWithNumericType2(const IDataType & argument_type, TArgs &&... args)
-    {
-        WhichDataType which(argument_type);
-#define DISPATCH(TYPE) \
-    if (which.idx == TypeIndex::TYPE) \
-        return new AggregateFunctionTemplate<TYPE, bool_param, TColumnB, is_plain_b>(std::forward<TArgs>(args)...);
-        FOR_NUMERIC_TYPES(DISPATCH)
-#undef DISPATCH
-        if (which.idx == TypeIndex::Enum8)
-            return new AggregateFunctionTemplate<Int8, bool_param, TColumnB, is_plain_b>(std::forward<TArgs>(args)...);
-        if (which.idx == TypeIndex::Enum16)
-            return new AggregateFunctionTemplate<Int16, bool_param, TColumnB, is_plain_b>(std::forward<TArgs>(args)...);
-        return nullptr;
-    }
-
-    template <bool expr_sorted = false, typename TColumnB = UInt64, bool is_plain_b = false>
-    AggregateFunctionPtr createAggregateFunctionGroupSortedArrayTyped(
-        const std::string & name, const DataTypes & argument_types, const Array & params, UInt64 threshold)
-    {
-        AggregateFunctionPtr res(createWithNumericType2<AggregateFunctionGroupSortedArrayNumeric, expr_sorted, TColumnB, is_plain_b>(
-            *argument_types[0], threshold, argument_types, params));
-
-        if (!res)
-            res = AggregateFunctionPtr(createWithExtraTypes<expr_sorted, TColumnB, is_plain_b>(argument_types, threshold, params));
-
-        if (!res)
-            throw Exception(
-                "Illegal type " + argument_types[0]->getName() + " of argument for aggregate function " + name,
-                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
-
-        return res;
-    }
-
 
     AggregateFunctionPtr createAggregateFunctionGroupSortedArray(
         const std::string & name, const DataTypes & argument_types, const Array & params, const Settings *)
@@ -136,26 +98,32 @@ namespace
 
         if (argument_types.size() == 2)
         {
-            if (isUnsignedInteger(argument_types[1]))
+            if (isNumber(argument_types[1]))
             {
-                return createAggregateFunctionGroupSortedArrayTyped<true, UInt64>(name, argument_types, params, threshold);
-            }
-            else if (isInteger(argument_types[1]))
-            {
-                return createAggregateFunctionGroupSortedArrayTyped<true, Int64>(name, argument_types, params, threshold);
+#define DISPATCH2(A, B) \
+    if (which.idx == TypeIndex::A) \
+        return createAggregateFunctionGroupSortedArrayTyped<true, B>(argument_types, params, threshold);
+#define DISPATCH(A) DISPATCH2(A, A)
+                WhichDataType which(argument_types[1]);
+                FOR_NUMERIC_TYPES(DISPATCH)
+                DISPATCH2(Enum8, Int8)
+                DISPATCH2(Enum16, Int16)
+#undef DISPATCH
+#undef DISPATCH2
+                throw Exception("Invalid parameter type.", ErrorCodes::BAD_ARGUMENTS);
             }
             else if (argument_types[1]->isValueUnambiguouslyRepresentedInContiguousMemoryRegion())
             {
-                return createAggregateFunctionGroupSortedArrayTyped<true, StringRef, true>(name, argument_types, params, threshold);
+                return createAggregateFunctionGroupSortedArrayTyped<true, StringRef, true>(argument_types, params, threshold);
             }
             else
             {
-                return createAggregateFunctionGroupSortedArrayTyped<true, StringRef, false>(name, argument_types, params, threshold);
+                return createAggregateFunctionGroupSortedArrayTyped<true, StringRef, false>(argument_types, params, threshold);
             }
         }
         else if (argument_types.size() == 1)
         {
-            return createAggregateFunctionGroupSortedArrayTyped<>(name, argument_types, params, threshold);
+            return createAggregateFunctionGroupSortedArrayTyped<>(argument_types, params, threshold);
         }
         else
         {
diff --git a/src/AggregateFunctions/AggregateFunctionGroupSortedArray.h b/src/AggregateFunctions/AggregateFunctionGroupSortedArray.h
index 5f9a67a21ae..79eeb5202f9 100644
--- a/src/AggregateFunctions/AggregateFunctionGroupSortedArray.h
+++ b/src/AggregateFunctions/AggregateFunctionGroupSortedArray.h
@@ -180,7 +180,7 @@ public:
         if constexpr (use_column_b)
         {
             forFirstRows<TColumnB, is_plain_b>(
-                batch_size, columns, 1, arena, if_argument_pos, [columns, &arena, &data](size_t row, const TColumnB * values) 
+                batch_size, columns, 1, arena, if_argument_pos, [columns, &arena, &data](size_t row, const TColumnB * values)
                 {
                     data.add(readItem<TColumnA, is_plain_a>(columns[0], arena, row), values[row]);
                 });
@@ -188,9 +188,9 @@ public:
         else
         {
             forFirstRows<TColumnA, is_plain_a>(
-                batch_size, columns, 0, arena, if_argument_pos, [&data](size_t row, const TColumnA * values) 
-                { 
-                    data.add(values[row]); 
+                batch_size, columns, 0, arena, if_argument_pos, [&data](size_t row, const TColumnA * values)
+                {
+                    data.add(values[row]);
                 });
         }
     }

From de7bbc720c2cb6bd935ca60056f8267356757854 Mon Sep 17 00:00:00 2001
From: Pablo Alegre <palegre@tinybird.co>
Date: Thu, 10 Feb 2022 15:56:02 +0100
Subject: [PATCH 005/104] fixup! Add groupSortedArray() function

fixup! Add groupSortedArray() function

fixup! Add groupSortedArray() function
---
 .../AggregateFunctionGroupSortedArray.cpp                | 9 ++++-----
 .../AggregateFunctionGroupSortedArrayData.h              | 2 +-
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/src/AggregateFunctions/AggregateFunctionGroupSortedArray.cpp b/src/AggregateFunctions/AggregateFunctionGroupSortedArray.cpp
index ef52428aecd..f33d5bcc22e 100644
--- a/src/AggregateFunctions/AggregateFunctionGroupSortedArray.cpp
+++ b/src/AggregateFunctions/AggregateFunctionGroupSortedArray.cpp
@@ -20,7 +20,6 @@ namespace ErrorCodes
 {
     extern const int ARGUMENT_OUT_OF_BOUND;
     extern const int ILLEGAL_TYPE_OF_ARGUMENT;
-    extern const int LOGICAL_ERROR;
     extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
 }
 
@@ -46,9 +45,9 @@ namespace
     AggregateFunctionPtr
     createAggregateFunctionGroupSortedArrayTyped(const DataTypes & argument_types, const Array & params, UInt64 threshold)
     {
-#define DISPATCH(A, CLS, B) \
+#define DISPATCH(A, C, B) \
     if (which.idx == TypeIndex::A) \
-        return AggregateFunctionPtr(new CLS<B, expr_sorted, TColumnB, is_plain_b>(threshold, argument_types, params));
+        return AggregateFunctionPtr(new C<B, expr_sorted, TColumnB, is_plain_b>(threshold, argument_types, params));
 #define DISPATCH_NUMERIC(A) DISPATCH(A, AggregateFunctionGroupSortedArrayNumeric, A)
         WhichDataType which(argument_types[0]);
         FOR_NUMERIC_TYPES(DISPATCH_NUMERIC)
@@ -91,7 +90,7 @@ namespace
 
             threshold = k;
         }
-        else if (params.size() != 0)
+        else if (!params.empty())
         {
             throw Exception("Aggregate function " + name + " only supports 1 parameter.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
         }
@@ -110,7 +109,7 @@ namespace
                 DISPATCH2(Enum16, Int16)
 #undef DISPATCH
 #undef DISPATCH2
-                throw Exception("Invalid parameter type.", ErrorCodes::BAD_ARGUMENTS);
+                throw Exception("Invalid parameter type.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
             }
             else if (argument_types[1]->isValueUnambiguouslyRepresentedInContiguousMemoryRegion())
             {
diff --git a/src/AggregateFunctions/AggregateFunctionGroupSortedArrayData.h b/src/AggregateFunctions/AggregateFunctionGroupSortedArrayData.h
index ca2d216ba8e..05682567024 100644
--- a/src/AggregateFunctions/AggregateFunctionGroupSortedArrayData.h
+++ b/src/AggregateFunctions/AggregateFunctionGroupSortedArrayData.h
@@ -66,7 +66,7 @@ struct AggregateFunctionGroupSortedArrayDataBase
 
     void merge(const AggregateFunctionGroupSortedArrayDataBase & other)
     {
-        values.insert(other.values.begin(), other.values.end());
+        values.merge(Storage(other.values));
         narrowDown();
     }
 

From 1e4b504ae2093d81908952e91f4b059a88f0ebc9 Mon Sep 17 00:00:00 2001
From: Pablo Alegre <palegre@tinybird.co>
Date: Tue, 8 Feb 2022 23:08:44 +0100
Subject: [PATCH 006/104] fixup! Add groupSortedArray() function

---
 ...roupsortedarray.md => grouparraysorted.md} | 12 ++---
 .../aggregate-functions/reference/index.md    |  2 +-
 .../reference/grouparraysorted.md             |  0
 ... => AggregateFunctionGroupArraySorted.cpp} | 44 +++++++++----------
 ....h => AggregateFunctionGroupArraySorted.h} | 22 +++++-----
 ...> AggregateFunctionGroupArraySortedData.h} | 18 ++++----
 .../registerAggregateFunctions.cpp            |  4 +-
 ...orted_array.xml => group_array_sorted.xml} | 12 ++---
 ...rence => 02158_grouparraysorted.reference} |  0
 .../0_stateless/02158_grouparraysorted.sql    | 41 +++++++++++++++++
 .../0_stateless/02158_groupsortedarray.sql    | 41 -----------------
 11 files changed, 98 insertions(+), 98 deletions(-)
 rename docs/en/sql-reference/aggregate-functions/reference/{groupsortedarray.md => grouparraysorted.md} (74%)
 create mode 100644 docs/ja/sql-reference/aggregate-functions/reference/grouparraysorted.md
 rename src/AggregateFunctions/{AggregateFunctionGroupSortedArray.cpp => AggregateFunctionGroupArraySorted.cpp} (73%)
 rename src/AggregateFunctions/{AggregateFunctionGroupSortedArray.h => AggregateFunctionGroupArraySorted.h} (91%)
 rename src/AggregateFunctions/{AggregateFunctionGroupSortedArrayData.h => AggregateFunctionGroupArraySortedData.h} (83%)
 rename tests/performance/{group_sorted_array.xml => group_array_sorted.xml} (64%)
 rename tests/queries/0_stateless/{02158_groupsortedarray.reference => 02158_grouparraysorted.reference} (100%)
 create mode 100644 tests/queries/0_stateless/02158_grouparraysorted.sql
 delete mode 100644 tests/queries/0_stateless/02158_groupsortedarray.sql

diff --git a/docs/en/sql-reference/aggregate-functions/reference/groupsortedarray.md b/docs/en/sql-reference/aggregate-functions/reference/grouparraysorted.md
similarity index 74%
rename from docs/en/sql-reference/aggregate-functions/reference/groupsortedarray.md
rename to docs/en/sql-reference/aggregate-functions/reference/grouparraysorted.md
index 28ad7db3bc6..0237885bcb6 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/groupsortedarray.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/grouparraysorted.md
@@ -2,12 +2,12 @@
 toc_priority: 108
 ---
 
-# groupSortedArray {#groupSortedArray}
+# groupArraySorted {#groupArraySorted}
 
 Returns an array with the first N items in ascending order.
 
 ``` sql
-groupSortedArray(N)(column)
+groupArraySorted(N)(column)
 ```
 
 **Arguments**
@@ -26,11 +26,11 @@ If the parameter is omitted, default value 10 is used.
 Gets the first 10 numbers:
 
 ``` sql
-SELECT groupSortedArray(10)(number) FROM numbers(100)
+SELECT groupArraySorted(10)(number) FROM numbers(100)
 ```
 
 ``` text
-┌─groupSortedArray(10)(number)─┐
+┌─groupArraySorted(10)(number)─┐
 │ [0,1,2,3,4,5,6,7,8,9]        │
 └──────────────────────────────┘
 ```
@@ -38,11 +38,11 @@ SELECT groupSortedArray(10)(number) FROM numbers(100)
 Or the last 10:
 
 ``` sql
-SELECT groupSortedArray(10)(number, -number) FROM numbers(100)
+SELECT groupArraySorted(10)(number, -number) FROM numbers(100)
 ```
 
 ``` text
-┌─groupSortedArray(10)(number, negate(number))─┐
+┌─groupArraySorted(10)(number, negate(number))─┐
 │ [99,98,97,96,95,94,93,92,91,90]              │
 └──────────────────────────────────────────────┘
 ```
\ No newline at end of file
diff --git a/docs/en/sql-reference/aggregate-functions/reference/index.md b/docs/en/sql-reference/aggregate-functions/reference/index.md
index 14a8ecc9dcf..2a8a2843510 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/index.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/index.md
@@ -35,6 +35,7 @@ ClickHouse-specific aggregate functions:
 -   [groupArrayInsertAt](../../../sql-reference/aggregate-functions/reference/grouparrayinsertat.md)
 -   [groupArrayMovingAvg](../../../sql-reference/aggregate-functions/reference/grouparraymovingavg.md)
 -   [groupArrayMovingSum](../../../sql-reference/aggregate-functions/reference/grouparraymovingsum.md)
+-   [groupArraySorted](../../../sql-reference/aggregate-functions/reference/grouparraysorted.md)
 -   [groupBitAnd](../../../sql-reference/aggregate-functions/reference/groupbitand.md)
 -   [groupBitOr](../../../sql-reference/aggregate-functions/reference/groupbitor.md)
 -   [groupBitXor](../../../sql-reference/aggregate-functions/reference/groupbitxor.md)
@@ -42,7 +43,6 @@ ClickHouse-specific aggregate functions:
 -   [groupBitmapAnd](../../../sql-reference/aggregate-functions/reference/groupbitmapand.md)
 -   [groupBitmapOr](../../../sql-reference/aggregate-functions/reference/groupbitmapor.md)
 -   [groupBitmapXor](../../../sql-reference/aggregate-functions/reference/groupbitmapxor.md)
--   [groupSortedArray](../../../sql-reference/aggregate-functions/reference/groupsortedarray.md)
 -   [sumWithOverflow](../../../sql-reference/aggregate-functions/reference/sumwithoverflow.md)
 -   [sumMap](../../../sql-reference/aggregate-functions/reference/summap.md)
 -   [minMap](../../../sql-reference/aggregate-functions/reference/minmap.md)
diff --git a/docs/ja/sql-reference/aggregate-functions/reference/grouparraysorted.md b/docs/ja/sql-reference/aggregate-functions/reference/grouparraysorted.md
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/src/AggregateFunctions/AggregateFunctionGroupSortedArray.cpp b/src/AggregateFunctions/AggregateFunctionGroupArraySorted.cpp
similarity index 73%
rename from src/AggregateFunctions/AggregateFunctionGroupSortedArray.cpp
rename to src/AggregateFunctions/AggregateFunctionGroupArraySorted.cpp
index f33d5bcc22e..82e8af4ac32 100644
--- a/src/AggregateFunctions/AggregateFunctionGroupSortedArray.cpp
+++ b/src/AggregateFunctions/AggregateFunctionGroupArraySorted.cpp
@@ -1,5 +1,5 @@
 #include <AggregateFunctions/AggregateFunctionFactory.h>
-#include <AggregateFunctions/AggregateFunctionGroupSortedArray.h>
+#include <AggregateFunctions/AggregateFunctionGroupArraySorted.h>
 #include <AggregateFunctions/FactoryHelpers.h>
 #include <AggregateFunctions/Helpers.h>
 #include <DataTypes/DataTypeDate.h>
@@ -27,51 +27,51 @@ namespace ErrorCodes
 namespace
 {
     template <typename T, bool expr_sorted, typename TColumnB, bool is_plain_b>
-    class AggregateFunctionGroupSortedArrayNumeric : public AggregateFunctionGroupSortedArray<T, false, expr_sorted, TColumnB, is_plain_b>
+    class AggregateFunctionGroupArraySortedNumeric : public AggregateFunctionGroupArraySorted<T, false, expr_sorted, TColumnB, is_plain_b>
     {
-        using AggregateFunctionGroupSortedArray<T, false, expr_sorted, TColumnB, is_plain_b>::AggregateFunctionGroupSortedArray;
+        using AggregateFunctionGroupArraySorted<T, false, expr_sorted, TColumnB, is_plain_b>::AggregateFunctionGroupArraySorted;
     };
 
     template <typename T, bool expr_sorted, typename TColumnB, bool is_plain_b>
-    class AggregateFunctionGroupSortedArrayFieldType
-        : public AggregateFunctionGroupSortedArray<typename T::FieldType, false, expr_sorted, TColumnB, is_plain_b>
+    class AggregateFunctionGroupArraySortedFieldType
+        : public AggregateFunctionGroupArraySorted<typename T::FieldType, false, expr_sorted, TColumnB, is_plain_b>
     {
-        using AggregateFunctionGroupSortedArray<typename T::FieldType, false, expr_sorted, TColumnB, is_plain_b>::
-            AggregateFunctionGroupSortedArray;
+        using AggregateFunctionGroupArraySorted<typename T::FieldType, false, expr_sorted, TColumnB, is_plain_b>::
+            AggregateFunctionGroupArraySorted;
         DataTypePtr getReturnType() const override { return std::make_shared<DataTypeArray>(std::make_shared<T>()); }
     };
 
     template <bool expr_sorted = false, typename TColumnB = UInt64, bool is_plain_b = false>
     AggregateFunctionPtr
-    createAggregateFunctionGroupSortedArrayTyped(const DataTypes & argument_types, const Array & params, UInt64 threshold)
+    createAggregateFunctionGroupArraySortedTyped(const DataTypes & argument_types, const Array & params, UInt64 threshold)
     {
 #define DISPATCH(A, C, B) \
     if (which.idx == TypeIndex::A) \
         return AggregateFunctionPtr(new C<B, expr_sorted, TColumnB, is_plain_b>(threshold, argument_types, params));
-#define DISPATCH_NUMERIC(A) DISPATCH(A, AggregateFunctionGroupSortedArrayNumeric, A)
+#define DISPATCH_NUMERIC(A) DISPATCH(A, AggregateFunctionGroupArraySortedNumeric, A)
         WhichDataType which(argument_types[0]);
         FOR_NUMERIC_TYPES(DISPATCH_NUMERIC)
-        DISPATCH(Enum8, AggregateFunctionGroupSortedArrayNumeric, Int8)
-        DISPATCH(Enum16, AggregateFunctionGroupSortedArrayNumeric, Int16)
-        DISPATCH(Date, AggregateFunctionGroupSortedArrayFieldType, DataTypeDate)
-        DISPATCH(DateTime, AggregateFunctionGroupSortedArrayFieldType, DataTypeDateTime)
+        DISPATCH(Enum8, AggregateFunctionGroupArraySortedNumeric, Int8)
+        DISPATCH(Enum16, AggregateFunctionGroupArraySortedNumeric, Int16)
+        DISPATCH(Date, AggregateFunctionGroupArraySortedFieldType, DataTypeDate)
+        DISPATCH(DateTime, AggregateFunctionGroupArraySortedFieldType, DataTypeDateTime)
 #undef DISPATCH
 #undef DISPATCH_NUMERIC
 
         if (argument_types[0]->isValueUnambiguouslyRepresentedInContiguousMemoryRegion())
         {
-            return AggregateFunctionPtr(new AggregateFunctionGroupSortedArray<StringRef, true, expr_sorted, TColumnB, is_plain_b>(
+            return AggregateFunctionPtr(new AggregateFunctionGroupArraySorted<StringRef, true, expr_sorted, TColumnB, is_plain_b>(
                 threshold, argument_types, params));
         }
         else
         {
-            return AggregateFunctionPtr(new AggregateFunctionGroupSortedArray<StringRef, false, expr_sorted, TColumnB, is_plain_b>(
+            return AggregateFunctionPtr(new AggregateFunctionGroupArraySorted<StringRef, false, expr_sorted, TColumnB, is_plain_b>(
                 threshold, argument_types, params));
         }
     }
 
 
-    AggregateFunctionPtr createAggregateFunctionGroupSortedArray(
+    AggregateFunctionPtr createAggregateFunctionGroupArraySorted(
         const std::string & name, const DataTypes & argument_types, const Array & params, const Settings *)
     {
         UInt64 threshold = GROUP_SORTED_ARRAY_DEFAULT_THRESHOLD;
@@ -101,7 +101,7 @@ namespace
             {
 #define DISPATCH2(A, B) \
     if (which.idx == TypeIndex::A) \
-        return createAggregateFunctionGroupSortedArrayTyped<true, B>(argument_types, params, threshold);
+        return createAggregateFunctionGroupArraySortedTyped<true, B>(argument_types, params, threshold);
 #define DISPATCH(A) DISPATCH2(A, A)
                 WhichDataType which(argument_types[1]);
                 FOR_NUMERIC_TYPES(DISPATCH)
@@ -113,16 +113,16 @@ namespace
             }
             else if (argument_types[1]->isValueUnambiguouslyRepresentedInContiguousMemoryRegion())
             {
-                return createAggregateFunctionGroupSortedArrayTyped<true, StringRef, true>(argument_types, params, threshold);
+                return createAggregateFunctionGroupArraySortedTyped<true, StringRef, true>(argument_types, params, threshold);
             }
             else
             {
-                return createAggregateFunctionGroupSortedArrayTyped<true, StringRef, false>(argument_types, params, threshold);
+                return createAggregateFunctionGroupArraySortedTyped<true, StringRef, false>(argument_types, params, threshold);
             }
         }
         else if (argument_types.size() == 1)
         {
-            return createAggregateFunctionGroupSortedArrayTyped<>(argument_types, params, threshold);
+            return createAggregateFunctionGroupArraySortedTyped<>(argument_types, params, threshold);
         }
         else
         {
@@ -132,9 +132,9 @@ namespace
     }
 }
 
-void registerAggregateFunctionGroupSortedArray(AggregateFunctionFactory & factory)
+void registerAggregateFunctionGroupArraySorted(AggregateFunctionFactory & factory)
 {
     AggregateFunctionProperties properties = {.returns_default_when_only_null = false, .is_order_dependent = true};
-    factory.registerFunction("groupSortedArray", {createAggregateFunctionGroupSortedArray, properties});
+    factory.registerFunction("groupArraySorted", {createAggregateFunctionGroupArraySorted, properties});
 }
 }
diff --git a/src/AggregateFunctions/AggregateFunctionGroupSortedArray.h b/src/AggregateFunctions/AggregateFunctionGroupArraySorted.h
similarity index 91%
rename from src/AggregateFunctions/AggregateFunctionGroupSortedArray.h
rename to src/AggregateFunctions/AggregateFunctionGroupArraySorted.h
index 79eeb5202f9..4459fbb3cdc 100644
--- a/src/AggregateFunctions/AggregateFunctionGroupSortedArray.h
+++ b/src/AggregateFunctions/AggregateFunctionGroupArraySorted.h
@@ -3,7 +3,7 @@
 #include <Columns/ColumnArray.h>
 #include <DataTypes/DataTypeArray.h>
 
-#include <AggregateFunctions/AggregateFunctionGroupSortedArrayData.h>
+#include <AggregateFunctions/AggregateFunctionGroupArraySortedData.h>
 #include <AggregateFunctions/IAggregateFunction.h>
 
 namespace DB
@@ -77,15 +77,15 @@ size_t getFirstNElements(const TColumn * data, int num_elements, int threshold,
 }
 
 template <typename TColumnA, bool is_plain_a, bool use_column_b, typename TColumnB, bool is_plain_b>
-class AggregateFunctionGroupSortedArray : public IAggregateFunctionDataHelper<
-                                              AggregateFunctionGroupSortedArrayData<TColumnA, use_column_b, TColumnB>,
-                                              AggregateFunctionGroupSortedArray<TColumnA, is_plain_a, use_column_b, TColumnB, is_plain_b>>
+class AggregateFunctionGroupArraySorted : public IAggregateFunctionDataHelper<
+                                              AggregateFunctionGroupArraySortedData<TColumnA, use_column_b, TColumnB>,
+                                              AggregateFunctionGroupArraySorted<TColumnA, is_plain_a, use_column_b, TColumnB, is_plain_b>>
 {
 protected:
-    using State = AggregateFunctionGroupSortedArrayData<TColumnA, use_column_b, TColumnB>;
+    using State = AggregateFunctionGroupArraySortedData<TColumnA, use_column_b, TColumnB>;
     using Base = IAggregateFunctionDataHelper<
-        AggregateFunctionGroupSortedArrayData<TColumnA, use_column_b, TColumnB>,
-        AggregateFunctionGroupSortedArray>;
+        AggregateFunctionGroupArraySortedData<TColumnA, use_column_b, TColumnB>,
+        AggregateFunctionGroupArraySorted>;
 
     UInt64 threshold;
     DataTypePtr & input_data_type;
@@ -94,10 +94,10 @@ protected:
     static void deserializeAndInsert(StringRef str, IColumn & data_to);
 
 public:
-    AggregateFunctionGroupSortedArray(UInt64 threshold_, const DataTypes & argument_types_, const Array & params)
+    AggregateFunctionGroupArraySorted(UInt64 threshold_, const DataTypes & argument_types_, const Array & params)
         : IAggregateFunctionDataHelper<
-            AggregateFunctionGroupSortedArrayData<TColumnA, use_column_b, TColumnB>,
-            AggregateFunctionGroupSortedArray>(argument_types_, params)
+            AggregateFunctionGroupArraySortedData<TColumnA, use_column_b, TColumnB>,
+            AggregateFunctionGroupArraySorted>(argument_types_, params)
         , threshold(threshold_)
         , input_data_type(this->argument_types[0])
     {
@@ -109,7 +109,7 @@ public:
         this->data(place).threshold = threshold;
     }
 
-    String getName() const override { return "groupSortedArray"; }
+    String getName() const override { return "groupArraySorted"; }
 
     DataTypePtr getReturnType() const override { return std::make_shared<DataTypeArray>(input_data_type); }
 
diff --git a/src/AggregateFunctions/AggregateFunctionGroupSortedArrayData.h b/src/AggregateFunctions/AggregateFunctionGroupArraySortedData.h
similarity index 83%
rename from src/AggregateFunctions/AggregateFunctionGroupSortedArrayData.h
rename to src/AggregateFunctions/AggregateFunctionGroupArraySortedData.h
index 05682567024..c3f60a8dbb9 100644
--- a/src/AggregateFunctions/AggregateFunctionGroupSortedArrayData.h
+++ b/src/AggregateFunctions/AggregateFunctionGroupArraySortedData.h
@@ -52,19 +52,19 @@ static void readOneItem(ReadBuffer & buf, Arena * arena, StringRef & item)
 }
 
 template <typename Storage>
-struct AggregateFunctionGroupSortedArrayDataBase
+struct AggregateFunctionGroupArraySortedDataBase
 {
     typedef typename Storage::value_type ValueType;
-    AggregateFunctionGroupSortedArrayDataBase(UInt64 threshold_ = GROUP_SORTED_DEFAULT_THRESHOLD) : threshold(threshold_) { }
+    AggregateFunctionGroupArraySortedDataBase(UInt64 threshold_ = GROUP_SORTED_DEFAULT_THRESHOLD) : threshold(threshold_) { }
 
-    virtual ~AggregateFunctionGroupSortedArrayDataBase() { }
+    virtual ~AggregateFunctionGroupArraySortedDataBase() { }
     inline void narrowDown()
     {
         while (values.size() > threshold)
             values.erase(--values.end());
     }
 
-    void merge(const AggregateFunctionGroupSortedArrayDataBase & other)
+    void merge(const AggregateFunctionGroupArraySortedDataBase & other)
     {
         values.merge(Storage(other.values));
         narrowDown();
@@ -101,14 +101,14 @@ struct AggregateFunctionGroupSortedArrayDataBase
 };
 
 template <typename T, bool expr_sorted, typename TIndex>
-struct AggregateFunctionGroupSortedArrayData
+struct AggregateFunctionGroupArraySortedData
 {
 };
 
 template <typename T, typename TIndex>
-struct AggregateFunctionGroupSortedArrayData<T, true, TIndex> : public AggregateFunctionGroupSortedArrayDataBase<std::multimap<TIndex, T>>
+struct AggregateFunctionGroupArraySortedData<T, true, TIndex> : public AggregateFunctionGroupArraySortedDataBase<std::multimap<TIndex, T>>
 {
-    using Base = AggregateFunctionGroupSortedArrayDataBase<std::multimap<TIndex, T>>;
+    using Base = AggregateFunctionGroupArraySortedDataBase<std::multimap<TIndex, T>>;
     using Base::Base;
 
     void add(T item, TIndex weight)
@@ -137,9 +137,9 @@ struct AggregateFunctionGroupSortedArrayData<T, true, TIndex> : public Aggregate
 };
 
 template <typename T, typename TIndex>
-struct AggregateFunctionGroupSortedArrayData<T, false, TIndex> : public AggregateFunctionGroupSortedArrayDataBase<std::multiset<T>>
+struct AggregateFunctionGroupArraySortedData<T, false, TIndex> : public AggregateFunctionGroupArraySortedDataBase<std::multiset<T>>
 {
-    using Base = AggregateFunctionGroupSortedArrayDataBase<std::multiset<T>>;
+    using Base = AggregateFunctionGroupArraySortedDataBase<std::multiset<T>>;
     using Base::Base;
 
     void add(T item)
diff --git a/src/AggregateFunctions/registerAggregateFunctions.cpp b/src/AggregateFunctions/registerAggregateFunctions.cpp
index c991407822c..034f36340e2 100644
--- a/src/AggregateFunctions/registerAggregateFunctions.cpp
+++ b/src/AggregateFunctions/registerAggregateFunctions.cpp
@@ -59,7 +59,7 @@ void registerAggregateFunctionNothing(AggregateFunctionFactory &);
 void registerAggregateFunctionExponentialMovingAverage(AggregateFunctionFactory &);
 void registerAggregateFunctionSparkbar(AggregateFunctionFactory &);
 void registerAggregateFunctionIntervalLengthSum(AggregateFunctionFactory &);
-void registerAggregateFunctionGroupSortedArray(AggregateFunctionFactory & factory);
+void registerAggregateFunctionGroupArraySorted(AggregateFunctionFactory & factory);
 
 class AggregateFunctionCombinatorFactory;
 void registerAggregateFunctionCombinatorIf(AggregateFunctionCombinatorFactory &);
@@ -131,7 +131,7 @@ void registerAggregateFunctions()
         registerAggregateFunctionIntervalLengthSum(factory);
         registerAggregateFunctionExponentialMovingAverage(factory);
         registerAggregateFunctionSparkbar(factory);
-        registerAggregateFunctionGroupSortedArray(factory);
+        registerAggregateFunctionGroupArraySorted(factory);
 
         registerWindowFunctions(factory);
     }
diff --git a/tests/performance/group_sorted_array.xml b/tests/performance/group_array_sorted.xml
similarity index 64%
rename from tests/performance/group_sorted_array.xml
rename to tests/performance/group_array_sorted.xml
index 16b64859c26..2ed11f43b47 100644
--- a/tests/performance/group_sorted_array.xml
+++ b/tests/performance/group_array_sorted.xml
@@ -15,11 +15,11 @@
 
     <create_query>CREATE TABLE test ( `id` UInt64, `value` UInt64, `text` String ) ENGINE = Memory</create_query>
     <fill_query>INSERT INTO test SELECT number as id, rand64() as value, toString(number) as text FROM numbers({items})</fill_query>
-    <query>SELECT groupSortedArray(10)(id, value) FROM test</query>
-    <query>SELECT groupSortedArray(10)(id, -value) FROM test</query>
-    <query>SELECT groupSortedArray(10)(text, value) FROM test</query>
-    <query>SELECT groupSortedArray(10)((id, text), value) FROM test</query>
-    <query>SELECT groupSortedArray(10)(value) FROM test</query>
-    <query>SELECT groupSortedArray(10)(text) FROM test</query>
+    <query>SELECT groupArraySorted(10)(id, value) FROM test</query>
+    <query>SELECT groupArraySorted(10)(id, -value) FROM test</query>
+    <query>SELECT groupArraySorted(10)(text, value) FROM test</query>
+    <query>SELECT groupArraySorted(10)((id, text), value) FROM test</query>
+    <query>SELECT groupArraySorted(10)(value) FROM test</query>
+    <query>SELECT groupArraySorted(10)(text) FROM test</query>
     <drop_query>DROP TABLE IF EXISTS test</drop_query>
 </test>
diff --git a/tests/queries/0_stateless/02158_groupsortedarray.reference b/tests/queries/0_stateless/02158_grouparraysorted.reference
similarity index 100%
rename from tests/queries/0_stateless/02158_groupsortedarray.reference
rename to tests/queries/0_stateless/02158_grouparraysorted.reference
diff --git a/tests/queries/0_stateless/02158_grouparraysorted.sql b/tests/queries/0_stateless/02158_grouparraysorted.sql
new file mode 100644
index 00000000000..4ceb8374a6d
--- /dev/null
+++ b/tests/queries/0_stateless/02158_grouparraysorted.sql
@@ -0,0 +1,41 @@
+
+SELECT groupArraySorted(5)(number) from numbers(100);
+
+SELECT groupArraySorted(number, number) from numbers(100);
+
+SELECT groupArraySorted(100)(number, number) from numbers(1000);
+
+SELECT groupArraySorted(100)(number, -number) from numbers(1000);
+
+SELECT groupArraySorted(5)(str, number) FROM (SELECT toString(number) as str, number FROM numbers(10));
+
+SELECT groupArraySorted(5)(text) FROM (select toString(number) as text from numbers(10));
+
+SELECT groupArraySorted(5)(text, -number) FROM (select toString(number) as text, number from numbers(10));
+
+SELECT groupArraySorted(5)((number,text)) from (SELECT toString(number) as text, number FROM numbers(100));
+
+SELECT groupArraySorted(5)(text,text) from (SELECT toString(number) as text FROM numbers(100));
+
+SELECT groupArraySorted(50)(text,(number,text)) from (SELECT toString(number) as text, number FROM numbers(100));
+
+
+DROP TABLE IF EXISTS test;
+DROP VIEW IF EXISTS mv_test;
+CREATE TABLE test (`n` String, `h` Int64) ENGINE = MergeTree ORDER BY n;
+CREATE MATERIALIZED VIEW mv_test (`n` String, `h` AggregateFunction(groupArraySorted(2), Int64, Int64)) ENGINE = AggregatingMergeTree ORDER BY n AS SELECT n, groupArraySortedState(2)(h, h) as h FROM test GROUP BY n;
+INSERT INTO test VALUES ('pablo',1)('pablo', 2)('luis', 1)('luis', 3)('pablo', 5)('pablo',4)('pablo', 5)('luis', 6)('luis', 7)('pablo', 8)('pablo',9)('pablo',10)('luis',11)('luis',12)('pablo',13);
+SELECT n, groupArraySortedMerge(2)(h) from mv_test GROUP BY n;
+
+DROP TABLE IF EXISTS test;
+DROP VIEW IF EXISTS mv_test;
+CREATE TABLE test (`n` String, `h` Int64) ENGINE = MergeTree ORDER BY n;
+CREATE MATERIALIZED VIEW mv_test (`n` String, `h` AggregateFunction(groupArraySorted(2), Int64)) ENGINE = AggregatingMergeTree ORDER BY n AS SELECT n, groupArraySortedState(2)(h) as h FROM test GROUP BY n;
+INSERT INTO test VALUES ('pablo',1)('pablo', 2)('luis', 1)('luis', 3)('pablo', 5)('pablo',4)('pablo', 5)('luis', 6)('luis', 7)('pablo', 8)('pablo',9)('pablo',10)('luis',11)('luis',12)('pablo',13);
+SELECT n, groupArraySortedMerge(2)(h) from mv_test GROUP BY n;
+DROP TABLE test;
+DROP VIEW mv_test;
+
+SELECT groupArraySortedIf(5)(number, number, number>3) from numbers(100);
+SELECT groupArraySortedIf(5)(number, toString(number), number>3) from numbers(100);
+SELECT groupArraySortedIf(5)(toString(number), number>3) from numbers(100);
diff --git a/tests/queries/0_stateless/02158_groupsortedarray.sql b/tests/queries/0_stateless/02158_groupsortedarray.sql
deleted file mode 100644
index e60741d1332..00000000000
--- a/tests/queries/0_stateless/02158_groupsortedarray.sql
+++ /dev/null
@@ -1,41 +0,0 @@
-
-SELECT groupSortedArray(5)(number) from numbers(100);
-
-SELECT groupSortedArray(number, number) from numbers(100);
-
-SELECT groupSortedArray(100)(number, number) from numbers(1000);
-
-SELECT groupSortedArray(100)(number, -number) from numbers(1000);
-
-SELECT groupSortedArray(5)(str, number) FROM (SELECT toString(number) as str, number FROM numbers(10));
-
-SELECT groupSortedArray(5)(text) FROM (select toString(number) as text from numbers(10));
-
-SELECT groupSortedArray(5)(text, -number) FROM (select toString(number) as text, number from numbers(10));
-
-SELECT groupSortedArray(5)((number,text)) from (SELECT toString(number) as text, number FROM numbers(100));
-
-SELECT groupSortedArray(5)(text,text) from (SELECT toString(number) as text FROM numbers(100));
-
-SELECT groupSortedArray(50)(text,(number,text)) from (SELECT toString(number) as text, number FROM numbers(100));
-
-
-DROP TABLE IF EXISTS test;
-DROP VIEW IF EXISTS mv_test;
-CREATE TABLE test (`n` String, `h` Int64) ENGINE = MergeTree ORDER BY n;
-CREATE MATERIALIZED VIEW mv_test (`n` String, `h` AggregateFunction(groupSortedArray(2), Int64, Int64)) ENGINE = AggregatingMergeTree ORDER BY n AS SELECT n, groupSortedArrayState(2)(h, h) as h FROM test GROUP BY n;
-INSERT INTO test VALUES ('pablo',1)('pablo', 2)('luis', 1)('luis', 3)('pablo', 5)('pablo',4)('pablo', 5)('luis', 6)('luis', 7)('pablo', 8)('pablo',9)('pablo',10)('luis',11)('luis',12)('pablo',13);
-SELECT n, groupSortedArrayMerge(2)(h) from mv_test GROUP BY n;
-
-DROP TABLE IF EXISTS test;
-DROP VIEW IF EXISTS mv_test;
-CREATE TABLE test (`n` String, `h` Int64) ENGINE = MergeTree ORDER BY n;
-CREATE MATERIALIZED VIEW mv_test (`n` String, `h` AggregateFunction(groupSortedArray(2), Int64)) ENGINE = AggregatingMergeTree ORDER BY n AS SELECT n, groupSortedArrayState(2)(h) as h FROM test GROUP BY n;
-INSERT INTO test VALUES ('pablo',1)('pablo', 2)('luis', 1)('luis', 3)('pablo', 5)('pablo',4)('pablo', 5)('luis', 6)('luis', 7)('pablo', 8)('pablo',9)('pablo',10)('luis',11)('luis',12)('pablo',13);
-SELECT n, groupSortedArrayMerge(2)(h) from mv_test GROUP BY n;
-DROP TABLE test;
-DROP VIEW mv_test;
-
-SELECT groupSortedArrayIf(5)(number, number, number>3) from numbers(100);
-SELECT groupSortedArrayIf(5)(number, toString(number), number>3) from numbers(100);
-SELECT groupSortedArrayIf(5)(toString(number), number>3) from numbers(100);

From c65eaa5c096bb3870653df7f3997644165b56329 Mon Sep 17 00:00:00 2001
From: Pablo Alegre <palegre@tinybird.co>
Date: Fri, 11 Feb 2022 10:28:35 +0100
Subject: [PATCH 007/104] fixup! Evolve to TinySum

---
 .../AggregateFunctionGroupArraySorted.cpp                | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/AggregateFunctions/AggregateFunctionGroupArraySorted.cpp b/src/AggregateFunctions/AggregateFunctionGroupArraySorted.cpp
index 82e8af4ac32..50d5f075322 100644
--- a/src/AggregateFunctions/AggregateFunctionGroupArraySorted.cpp
+++ b/src/AggregateFunctions/AggregateFunctionGroupArraySorted.cpp
@@ -41,13 +41,20 @@ namespace
         DataTypePtr getReturnType() const override { return std::make_shared<DataTypeArray>(std::make_shared<T>()); }
     };
 
+    template <template <typename, bool, typename, bool> class AggregateFunctionTemplate, typename TColumnA, bool expr_sorted, typename TColumnB, bool is_plain_b, typename... TArgs>
+    AggregateFunctionPtr
+    createAggregateFunctionGroupArraySortedTypedFinal(TArgs && ... args)
+    {
+        return AggregateFunctionPtr(new AggregateFunctionTemplate<TColumnA, expr_sorted, TColumnB, is_plain_b>(std::forward<TArgs>(args)...));
+    }
+
     template <bool expr_sorted = false, typename TColumnB = UInt64, bool is_plain_b = false>
     AggregateFunctionPtr
     createAggregateFunctionGroupArraySortedTyped(const DataTypes & argument_types, const Array & params, UInt64 threshold)
     {
 #define DISPATCH(A, C, B) \
     if (which.idx == TypeIndex::A) \
-        return AggregateFunctionPtr(new C<B, expr_sorted, TColumnB, is_plain_b>(threshold, argument_types, params));
+        return createAggregateFunctionGroupArraySortedTypedFinal<C, B, expr_sorted, TColumnB, is_plain_b>(threshold, argument_types, params);
 #define DISPATCH_NUMERIC(A) DISPATCH(A, AggregateFunctionGroupArraySortedNumeric, A)
         WhichDataType which(argument_types[0]);
         FOR_NUMERIC_TYPES(DISPATCH_NUMERIC)

From 3e537a711871ada666fbb212ece596fb6e4e16ef Mon Sep 17 00:00:00 2001
From: Pablo Alegre <palegre@tinybird.co>
Date: Fri, 11 Feb 2022 18:33:01 +0100
Subject: [PATCH 008/104] fixup! Evolve to TinySum

---
 src/AggregateFunctions/AggregateFunctionGroupArraySorted.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/AggregateFunctions/AggregateFunctionGroupArraySorted.h b/src/AggregateFunctions/AggregateFunctionGroupArraySorted.h
index 4459fbb3cdc..89bcb221e10 100644
--- a/src/AggregateFunctions/AggregateFunctionGroupArraySorted.h
+++ b/src/AggregateFunctions/AggregateFunctionGroupArraySorted.h
@@ -55,7 +55,7 @@ size_t getFirstNElements(const TColumn * data, int num_elements, int threshold,
         //Starting from the highest values and we look for the immediately lower than the given one
         for (cur = current_max; cur > 0; cur--)
         {
-            if (!(data[i] < data[results[cur - 1]]))
+            if (data[i] > data[results[cur - 1]])
                 break;
         }
 

From 2b0b03276be1b5b763f11e9197096ab026caa773 Mon Sep 17 00:00:00 2001
From: Sergei Trifonov <sergei@clickhouse.com>
Date: Sun, 13 Feb 2022 19:58:38 +0300
Subject: [PATCH 009/104] add simple sanity checks for server startup

---
 programs/server/Server.cpp       | 88 +++++++++++++++++++++++++++++++-
 src/Common/filesystemHelpers.cpp | 62 ++++++++++++++++++++++
 src/Common/filesystemHelpers.h   | 21 ++++++++
 3 files changed, 169 insertions(+), 2 deletions(-)

diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp
index 9fa36e3de89..94c9b686bed 100644
--- a/programs/server/Server.cpp
+++ b/programs/server/Server.cpp
@@ -77,6 +77,7 @@
 #include <Common/SensitiveDataMasker.h>
 #include <Common/ThreadFuzzer.h>
 #include <Common/getHashOfLoadedBinary.h>
+#include <Common/filesystemHelpers.h>
 #include <Common/Elf.h>
 #include <Server/MySQLHandlerFactory.h>
 #include <Server/PostgreSQLHandlerFactory.h>
@@ -516,6 +517,58 @@ void checkForUsersNotInMainConfig(
 #endif
 }
 
+static String readString(const String& path)
+{
+    ReadBufferFromFile in(path);
+    String contents;
+    readStringUntilEOF(contents, in);
+    return contents;
+}
+
+static int readNumber(const String& path)
+{
+    ReadBufferFromFile in(path);
+    int result;
+    readText(result, in);
+    return result;
+}
+
+static void sanityChecks(Server* server)
+{
+    std::string data_path = getCanonicalPath(server->config().getString("path", DBMS_DEFAULT_PATH));
+    std::string logs_path = server->config().getString("logger.log", "");
+
+#if defined(OS_LINUX)
+    if (readString("/sys/devices/system/clocksource/clocksource0/current_clocksource").find("tsc") == std::string::npos)
+        server->context()->addWarningMessage("Linux is not using fast TSC clock source. Performance could be degraded.");
+
+    if (readNumber("/proc/sys/vm/overcommit_memory") == 2)
+        server->context()->addWarningMessage("Linux memory overcommit is disabled.");
+
+    if (readString("/sys/kernel/mm/transparent_hugepage/enabled").find("[always]") != std::string::npos)
+        server->context()->addWarningMessage("Linux transparent hugepage are set to \"always\".");
+
+    if (readNumber("/proc/sys/kernel/pid_max") < 30000)
+        server->context()->addWarningMessage("Linux max PID is too low.");
+
+    if (readNumber("/proc/sys/kernel/threads-max") < 30000)
+        server->context()->addWarningMessage("Linux threads max count is too low.");
+
+    std::string devId = getBlockDeviceId(data_path);
+    if (getBlockDeviceType(devId) == BlockDeviceType::ROT && getBlockDeviceReadAheadBytes(devId) == 0)
+        server->context()->addWarningMessage("Rotational disk with disabled readahead is in use. Performance could be degraded.");
+#endif
+    if (sysconf(_SC_AVPHYS_PAGES) * sysconf(_SC_PAGE_SIZE) < (2l << 30))
+        server->context()->addWarningMessage("Available memory at server startup is too low (2GiB).");
+
+    if (!enoughSpaceInDirectory(data_path, 1ull << 30))
+        server->context()->addWarningMessage("Available disk space at server startup is too low (1GiB).");
+
+    if (!logs_path.empty()) {
+        if (!enoughSpaceInDirectory(logs_path, 1ull << 30))
+            server->context()->addWarningMessage("Available disk space at server startup is too low (1GiB).");
+    }
+}
 
 int Server::main(const std::vector<std::string> & /*args*/)
 {
@@ -550,13 +603,14 @@ int Server::main(const std::vector<std::string> & /*args*/)
     global_context->addWarningMessage("Server was built in debug mode. It will work slowly.");
 #endif
 
-if (ThreadFuzzer::instance().isEffective())
-    global_context->addWarningMessage("ThreadFuzzer is enabled. Application will run slowly and unstable.");
+    if (ThreadFuzzer::instance().isEffective())
+        global_context->addWarningMessage("ThreadFuzzer is enabled. Application will run slowly and unstable.");
 
 #if defined(SANITIZER)
     global_context->addWarningMessage("Server was built with sanitizer. It will work slowly.");
 #endif
 
+    sanityChecks(this);
 
     // Initialize global thread pool. Do it before we fetch configs from zookeeper
     // nodes (`from_zk`), because ZooKeeper interface uses the pool. We will
@@ -774,6 +828,36 @@ if (ThreadFuzzer::instance().isEffective())
         }
     }
 
+    /// Try to increase limit on number of threads.
+    {
+        rlimit rlim;
+        if (getrlimit(RLIMIT_NPROC, &rlim))
+            throw Poco::Exception("Cannot getrlimit");
+
+        if (rlim.rlim_cur == rlim.rlim_max)
+        {
+            LOG_DEBUG(log, "rlimit on number of threads is {}", rlim.rlim_cur);
+        }
+        else
+        {
+            rlim_t old = rlim.rlim_cur;
+            rlim.rlim_cur = rlim.rlim_max;
+            int rc = setrlimit(RLIMIT_NPROC, &rlim);
+            if (rc != 0) {
+                LOG_WARNING(log, "Cannot set max number of threads to {}. error: {}", rlim.rlim_cur, strerror(errno));
+                rlim.rlim_cur = old;
+            }
+            else
+            {
+                LOG_DEBUG(log, "Set max number of threads to {} (was {}).", rlim.rlim_cur, old);
+            }
+        }
+
+        if (rlim.rlim_cur < 30000) {
+            global_context->addWarningMessage("Maximum number of threads is lower than 30000. There could be problems with handling a lot of simultaneous queries.");
+        }
+    }
+
     static ServerErrorHandler error_handler;
     Poco::ErrorHandler::set(&error_handler);
 
diff --git a/src/Common/filesystemHelpers.cpp b/src/Common/filesystemHelpers.cpp
index f9fe8c97a14..30b14003126 100644
--- a/src/Common/filesystemHelpers.cpp
+++ b/src/Common/filesystemHelpers.cpp
@@ -4,6 +4,8 @@
 #if defined(__linux__)
 #    include <cstdio>
 #    include <mntent.h>
+#    include <sys/stat.h>
+#    include <sys/sysmacros.h>
 #endif
 #include <cerrno>
 #include <Poco/Version.h>
@@ -13,6 +15,9 @@
 #include <unistd.h>
 #include <sys/types.h>
 #include <utime.h>
+#include <IO/ReadBufferFromFile.h>
+#include <IO/Operators.h>
+#include <IO/WriteBufferFromString.h>
 
 namespace fs = std::filesystem;
 
@@ -24,6 +29,7 @@ namespace ErrorCodes
     extern const int LOGICAL_ERROR;
     extern const int SYSTEM_ERROR;
     extern const int NOT_IMPLEMENTED;
+    extern const int CANNOT_STAT;
     extern const int CANNOT_STATVFS;
     extern const int PATH_ACCESS_DENIED;
     extern const int CANNOT_CREATE_FILE;
@@ -57,6 +63,62 @@ std::unique_ptr<TemporaryFile> createTemporaryFile(const std::string & path)
     return std::make_unique<TemporaryFile>(path);
 }
 
+#if !defined(__linux__)
+[[noreturn]]
+#endif
+String getBlockDeviceId(const String & path)
+{
+#if defined(__linux__)
+    struct stat sb;
+    if (lstat(path.c_str(), &sb))
+        throwFromErrnoWithPath("Cannot lstat " + path, path, ErrorCodes::CANNOT_STAT);
+    WriteBufferFromOwnString ss;
+    ss << major(sb.st_dev) << ":" << minor(sb.st_dev);
+    return ss.str();
+#else
+    throw DB::Exception("The function getDeviceId is supported on Linux only", ErrorCodes::NOT_IMPLEMENTED);
+#endif
+}
+
+#if !defined(__linux__)
+[[noreturn]]
+#endif
+BlockDeviceType getBlockDeviceType(const String & deviceId)
+{
+#if defined(__linux__)
+    try {
+        ReadBufferFromFile in("/sys/dev/block/" + deviceId + "/queue/rotational");
+        int rotational;
+        readText(rotational, in);
+        return rotational ? BlockDeviceType::ROT : BlockDeviceType::NONROT;
+    } catch (...) {
+        return BlockDeviceType::UNKNOWN;
+    }
+#else
+    throw DB::Exception("The function getDeviceType is supported on Linux only", ErrorCodes::NOT_IMPLEMENTED);
+#endif
+}
+
+#if !defined(__linux__)
+[[noreturn]]
+#endif
+UInt64 getBlockDeviceReadAheadBytes(const String & deviceId)
+{
+#if defined(__linux__)
+    try {
+        ReadBufferFromFile in("/sys/dev/block/" + deviceId + "/queue/read_ahead_kb");
+        int read_ahead_kb;
+        readText(read_ahead_kb, in);
+        return read_ahead_kb * 1024;
+    } catch (...) {
+        return static_cast<UInt64>(-1);
+    }
+#else
+    throw DB::Exception("The function getDeviceType is supported on Linux only", ErrorCodes::NOT_IMPLEMENTED);
+#endif
+}
+
+/// Returns name of filesystem mounted to mount_point
 std::filesystem::path getMountPoint(std::filesystem::path absolute_path)
 {
     if (absolute_path.is_relative())
diff --git a/src/Common/filesystemHelpers.h b/src/Common/filesystemHelpers.h
index fc3a4f15573..e82ec396472 100644
--- a/src/Common/filesystemHelpers.h
+++ b/src/Common/filesystemHelpers.h
@@ -18,6 +18,27 @@ using TemporaryFile = Poco::TemporaryFile;
 bool enoughSpaceInDirectory(const std::string & path, size_t data_size);
 std::unique_ptr<TemporaryFile> createTemporaryFile(const std::string & path);
 
+#if !defined(__linux__)
+[[noreturn]]
+#endif
+String getBlockDeviceId(const String & path);
+
+enum class BlockDeviceType {
+    UNKNOWN = 0,
+    NONROT = 1,
+    ROT = 2
+};
+
+#if !defined(__linux__)
+[[noreturn]]
+#endif
+BlockDeviceType getBlockDeviceType(const String & deviceId);
+
+#if !defined(__linux__)
+[[noreturn]]
+#endif
+UInt64 getBlockDeviceReadAheadBytes(const String & deviceId);
+
 /// Returns mount point of filesystem where absolute_path (must exist) is located
 std::filesystem::path getMountPoint(std::filesystem::path absolute_path);
 

From 3d66acc56b1569f67a4baa15f19d6f1d228bc585 Mon Sep 17 00:00:00 2001
From: Sergei Trifonov <sergei@clickhouse.com>
Date: Mon, 14 Feb 2022 17:16:05 +0300
Subject: [PATCH 010/104] Apply suggestions from code review

Co-authored-by: alesapin <alesapin@clickhouse.com>
---
 programs/server/Server.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp
index 94c9b686bed..e29035ea602 100644
--- a/programs/server/Server.cpp
+++ b/programs/server/Server.cpp
@@ -517,7 +517,7 @@ void checkForUsersNotInMainConfig(
 #endif
 }
 
-static String readString(const String& path)
+static String readString(const String & path)
 {
     ReadBufferFromFile in(path);
     String contents;
@@ -525,7 +525,7 @@ static String readString(const String& path)
     return contents;
 }
 
-static int readNumber(const String& path)
+static int readNumber(const String & path)
 {
     ReadBufferFromFile in(path);
     int result;

From a819cb7afe8fd84cdd5a1464a4a62a9d6aa8d636 Mon Sep 17 00:00:00 2001
From: Sergei Trifonov <sergei@clickhouse.com>
Date: Tue, 15 Feb 2022 12:56:21 +0300
Subject: [PATCH 011/104] avoid unhandled exceptions on sanity checks at server
 startup

---
 programs/server/Server.cpp     | 52 +++++++++++++++++++++++++++-------
 src/Common/filesystemHelpers.h |  9 ++++--
 2 files changed, 47 insertions(+), 14 deletions(-)

diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp
index e29035ea602..92826cb2906 100644
--- a/programs/server/Server.cpp
+++ b/programs/server/Server.cpp
@@ -539,24 +539,54 @@ static void sanityChecks(Server* server)
     std::string logs_path = server->config().getString("logger.log", "");
 
 #if defined(OS_LINUX)
-    if (readString("/sys/devices/system/clocksource/clocksource0/current_clocksource").find("tsc") == std::string::npos)
-        server->context()->addWarningMessage("Linux is not using fast TSC clock source. Performance could be degraded.");
+    try
+    {
+        if (readString("/sys/devices/system/clocksource/clocksource0/current_clocksource").find("tsc") == std::string::npos)
+            server->context()->addWarningMessage("Linux is not using fast TSC clock source. Performance can be degraded.");
+    }
+    catch (...)
+    {
+    }
 
-    if (readNumber("/proc/sys/vm/overcommit_memory") == 2)
-        server->context()->addWarningMessage("Linux memory overcommit is disabled.");
+    try
+    {
+        if (readNumber("/proc/sys/vm/overcommit_memory") == 2)
+            server->context()->addWarningMessage("Linux memory overcommit is disabled.");
+    }
+    catch (...)
+    {
+    }
 
-    if (readString("/sys/kernel/mm/transparent_hugepage/enabled").find("[always]") != std::string::npos)
-        server->context()->addWarningMessage("Linux transparent hugepage are set to \"always\".");
+    try
+    {
+        if (readString("/sys/kernel/mm/transparent_hugepage/enabled").find("[always]") != std::string::npos)
+            server->context()->addWarningMessage("Linux transparent hugepage are set to \"always\".");
+    }
+    catch (...)
+    {
+    }
 
-    if (readNumber("/proc/sys/kernel/pid_max") < 30000)
-        server->context()->addWarningMessage("Linux max PID is too low.");
+    try
+    {
+        if (readNumber("/proc/sys/kernel/pid_max") < 30000)
+            server->context()->addWarningMessage("Linux max PID is too low.");
+    }
+    catch (...)
+    {
+    }
 
-    if (readNumber("/proc/sys/kernel/threads-max") < 30000)
-        server->context()->addWarningMessage("Linux threads max count is too low.");
+    try
+    {
+        if (readNumber("/proc/sys/kernel/threads-max") < 30000)
+            server->context()->addWarningMessage("Linux threads max count is too low.");
+    }
+    catch (...)
+    {
+    }
 
     std::string devId = getBlockDeviceId(data_path);
     if (getBlockDeviceType(devId) == BlockDeviceType::ROT && getBlockDeviceReadAheadBytes(devId) == 0)
-        server->context()->addWarningMessage("Rotational disk with disabled readahead is in use. Performance could be degraded.");
+        server->context()->addWarningMessage("Rotational disk with disabled readahead is in use. Performance can be degraded.");
 #endif
     if (sysconf(_SC_AVPHYS_PAGES) * sysconf(_SC_PAGE_SIZE) < (2l << 30))
         server->context()->addWarningMessage("Available memory at server startup is too low (2GiB).");
diff --git a/src/Common/filesystemHelpers.h b/src/Common/filesystemHelpers.h
index e82ec396472..bbdc36819b9 100644
--- a/src/Common/filesystemHelpers.h
+++ b/src/Common/filesystemHelpers.h
@@ -18,22 +18,25 @@ using TemporaryFile = Poco::TemporaryFile;
 bool enoughSpaceInDirectory(const std::string & path, size_t data_size);
 std::unique_ptr<TemporaryFile> createTemporaryFile(const std::string & path);
 
+// Determine what block device is responsible for specified path
 #if !defined(__linux__)
 [[noreturn]]
 #endif
 String getBlockDeviceId(const String & path);
 
 enum class BlockDeviceType {
-    UNKNOWN = 0,
-    NONROT = 1,
-    ROT = 2
+    UNKNOWN = 0, // we were unable to determine device type
+    NONROT = 1, // not a rotational device (SSD, NVME, etc)
+    ROT = 2 // rotational device (HDD)
 };
 
+// Try to determine block device type
 #if !defined(__linux__)
 [[noreturn]]
 #endif
 BlockDeviceType getBlockDeviceType(const String & deviceId);
 
+// Get size of read-ahead in bytes for specified block device
 #if !defined(__linux__)
 [[noreturn]]
 #endif

From 7ea3c003d54a28b2530a6f4f466de7a6f1aebbc2 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Tue, 15 Feb 2022 10:39:45 +0000
Subject: [PATCH 012/104] fix style

---
 programs/server/Server.cpp       | 13 ++++++++-----
 src/Common/filesystemHelpers.cpp | 20 +++++++++++++-------
 src/Common/filesystemHelpers.h   |  6 +++---
 3 files changed, 24 insertions(+), 15 deletions(-)

diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp
index 92826cb2906..378a76a48f6 100644
--- a/programs/server/Server.cpp
+++ b/programs/server/Server.cpp
@@ -584,8 +584,8 @@ static void sanityChecks(Server* server)
     {
     }
 
-    std::string devId = getBlockDeviceId(data_path);
-    if (getBlockDeviceType(devId) == BlockDeviceType::ROT && getBlockDeviceReadAheadBytes(devId) == 0)
+    std::string dev_id = getBlockDeviceId(data_path);
+    if (getBlockDeviceType(dev_id) == BlockDeviceType::ROT && getBlockDeviceReadAheadBytes(dev_id) == 0)
         server->context()->addWarningMessage("Rotational disk with disabled readahead is in use. Performance can be degraded.");
 #endif
     if (sysconf(_SC_AVPHYS_PAGES) * sysconf(_SC_PAGE_SIZE) < (2l << 30))
@@ -594,7 +594,8 @@ static void sanityChecks(Server* server)
     if (!enoughSpaceInDirectory(data_path, 1ull << 30))
         server->context()->addWarningMessage("Available disk space at server startup is too low (1GiB).");
 
-    if (!logs_path.empty()) {
+    if (!logs_path.empty())
+    {
         if (!enoughSpaceInDirectory(logs_path, 1ull << 30))
             server->context()->addWarningMessage("Available disk space at server startup is too low (1GiB).");
     }
@@ -873,7 +874,8 @@ int Server::main(const std::vector<std::string> & /*args*/)
             rlim_t old = rlim.rlim_cur;
             rlim.rlim_cur = rlim.rlim_max;
             int rc = setrlimit(RLIMIT_NPROC, &rlim);
-            if (rc != 0) {
+            if (rc != 0)
+            {
                 LOG_WARNING(log, "Cannot set max number of threads to {}. error: {}", rlim.rlim_cur, strerror(errno));
                 rlim.rlim_cur = old;
             }
@@ -883,7 +885,8 @@ int Server::main(const std::vector<std::string> & /*args*/)
             }
         }
 
-        if (rlim.rlim_cur < 30000) {
+        if (rlim.rlim_cur < 30000)
+        {
             global_context->addWarningMessage("Maximum number of threads is lower than 30000. There could be problems with handling a lot of simultaneous queries.");
         }
     }
diff --git a/src/Common/filesystemHelpers.cpp b/src/Common/filesystemHelpers.cpp
index 30b14003126..8bdd10e3cb1 100644
--- a/src/Common/filesystemHelpers.cpp
+++ b/src/Common/filesystemHelpers.cpp
@@ -66,7 +66,7 @@ std::unique_ptr<TemporaryFile> createTemporaryFile(const std::string & path)
 #if !defined(__linux__)
 [[noreturn]]
 #endif
-String getBlockDeviceId(const String & path)
+String getBlockDeviceId([[maybe_unused]] const String & path)
 {
 #if defined(__linux__)
     struct stat sb;
@@ -83,15 +83,18 @@ String getBlockDeviceId(const String & path)
 #if !defined(__linux__)
 [[noreturn]]
 #endif
-BlockDeviceType getBlockDeviceType(const String & deviceId)
+BlockDeviceType getBlockDeviceType([[maybe_unused]] const String & deviceId)
 {
 #if defined(__linux__)
-    try {
+    try
+    {
         ReadBufferFromFile in("/sys/dev/block/" + deviceId + "/queue/rotational");
         int rotational;
         readText(rotational, in);
         return rotational ? BlockDeviceType::ROT : BlockDeviceType::NONROT;
-    } catch (...) {
+    }
+    catch (...)
+    {
         return BlockDeviceType::UNKNOWN;
     }
 #else
@@ -102,15 +105,18 @@ BlockDeviceType getBlockDeviceType(const String & deviceId)
 #if !defined(__linux__)
 [[noreturn]]
 #endif
-UInt64 getBlockDeviceReadAheadBytes(const String & deviceId)
+UInt64 getBlockDeviceReadAheadBytes([[maybe_unused]] const String & deviceId)
 {
 #if defined(__linux__)
-    try {
+    try
+    {
         ReadBufferFromFile in("/sys/dev/block/" + deviceId + "/queue/read_ahead_kb");
         int read_ahead_kb;
         readText(read_ahead_kb, in);
         return read_ahead_kb * 1024;
-    } catch (...) {
+    }
+    catch (...)
+    {
         return static_cast<UInt64>(-1);
     }
 #else
diff --git a/src/Common/filesystemHelpers.h b/src/Common/filesystemHelpers.h
index bbdc36819b9..05d1fe9192e 100644
--- a/src/Common/filesystemHelpers.h
+++ b/src/Common/filesystemHelpers.h
@@ -22,7 +22,7 @@ std::unique_ptr<TemporaryFile> createTemporaryFile(const std::string & path);
 #if !defined(__linux__)
 [[noreturn]]
 #endif
-String getBlockDeviceId(const String & path);
+String getBlockDeviceId([[maybe_unused]] const String & path);
 
 enum class BlockDeviceType {
     UNKNOWN = 0, // we were unable to determine device type
@@ -34,13 +34,13 @@ enum class BlockDeviceType {
 #if !defined(__linux__)
 [[noreturn]]
 #endif
-BlockDeviceType getBlockDeviceType(const String & deviceId);
+BlockDeviceType getBlockDeviceType([[maybe_unused]] const String & deviceId);
 
 // Get size of read-ahead in bytes for specified block device
 #if !defined(__linux__)
 [[noreturn]]
 #endif
-UInt64 getBlockDeviceReadAheadBytes(const String & deviceId);
+UInt64 getBlockDeviceReadAheadBytes([[maybe_unused]] const String & deviceId);
 
 /// Returns mount point of filesystem where absolute_path (must exist) is located
 std::filesystem::path getMountPoint(std::filesystem::path absolute_path);

From 0f66743104ae8c6bf9e777939ad41af89f9cfafd Mon Sep 17 00:00:00 2001
From: Sergei Trifonov <sergei@clickhouse.com>
Date: Tue, 15 Feb 2022 13:59:00 +0300
Subject: [PATCH 013/104] support determine available memory for FreeBSD

---
 base/base/CMakeLists.txt               |  1 +
 base/base/getAvailableMemoryAmount.cpp | 41 ++++++++++++++++++++++++++
 base/base/getAvailableMemoryAmount.h   | 12 ++++++++
 programs/server/Server.cpp             | 11 +++++--
 4 files changed, 63 insertions(+), 2 deletions(-)
 create mode 100644 base/base/getAvailableMemoryAmount.cpp
 create mode 100644 base/base/getAvailableMemoryAmount.h

diff --git a/base/base/CMakeLists.txt b/base/base/CMakeLists.txt
index 3e6f174c6dc..de7ace7c178 100644
--- a/base/base/CMakeLists.txt
+++ b/base/base/CMakeLists.txt
@@ -2,6 +2,7 @@ set (SRCS
     argsToConfig.cpp
     coverage.cpp
     demangle.cpp
+    getAvailableMemoryAmount.cpp
     getFQDNOrHostName.cpp
     getMemoryAmount.cpp
     getPageSize.cpp
diff --git a/base/base/getAvailableMemoryAmount.cpp b/base/base/getAvailableMemoryAmount.cpp
new file mode 100644
index 00000000000..a7fd970c216
--- /dev/null
+++ b/base/base/getAvailableMemoryAmount.cpp
@@ -0,0 +1,41 @@
+#include <stdexcept>
+#include <fstream>
+#include <base/getAvailableMemoryAmount.h>
+#include <base/getPageSize.h>
+
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/param.h>
+#if defined(BSD)
+#include <sys/sysctl.h>
+#include <sys/vmmeter.h>
+#endif
+
+
+uint64_t getAvailableMemoryAmountOrZero()
+{
+    int64_t page_size = getPageSize();
+    if (page_size <= 0)
+        return 0;
+
+#if defined(__FreeBSD__)
+    struct vmtotal vmt;
+    size_t vmt_size = sizeof(vmt);
+    if (sysctlbyname("vm.vmtotal", &vmt, &vmt_size, NULL, 0) < 0)
+       return 0;
+    uint64_t available_pages = vmt.t_avm;
+#else
+    uint64_t available_pages = sysconf(_SC_AVPHYS_PAGES);
+#endif
+
+    return page_size * available_pages;
+}
+
+
+uint64_t getAvailableMemoryAmount()
+{
+    auto res = getAvailableMemoryAmountOrZero();
+    if (!res)
+        throw std::runtime_error("Cannot determine available memory amount");
+    return res;
+}
diff --git a/base/base/getAvailableMemoryAmount.h b/base/base/getAvailableMemoryAmount.h
new file mode 100644
index 00000000000..44612945016
--- /dev/null
+++ b/base/base/getAvailableMemoryAmount.h
@@ -0,0 +1,12 @@
+#pragma once
+
+#include <cstdint>
+
+/** Returns the size of currently available physical memory (RAM) in bytes.
+  * Returns 0 on unsupported platform or if it cannot determine the size of physical memory.
+  */
+uint64_t getAvailableMemoryAmountOrZero();
+
+/** Throws exception if it cannot determine the size of physical memory.
+  */
+uint64_t getAvailableMemoryAmount();
diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp
index 378a76a48f6..82522068d85 100644
--- a/programs/server/Server.cpp
+++ b/programs/server/Server.cpp
@@ -20,6 +20,7 @@
 #include <base/phdr_cache.h>
 #include <base/ErrorHandlers.h>
 #include <base/getMemoryAmount.h>
+#include <base/getAvailableMemoryAmount.h>
 #include <base/errnoToString.h>
 #include <base/coverage.h>
 #include <Common/ClickHouseRevision.h>
@@ -588,8 +589,14 @@ static void sanityChecks(Server* server)
     if (getBlockDeviceType(dev_id) == BlockDeviceType::ROT && getBlockDeviceReadAheadBytes(dev_id) == 0)
         server->context()->addWarningMessage("Rotational disk with disabled readahead is in use. Performance can be degraded.");
 #endif
-    if (sysconf(_SC_AVPHYS_PAGES) * sysconf(_SC_PAGE_SIZE) < (2l << 30))
-        server->context()->addWarningMessage("Available memory at server startup is too low (2GiB).");
+    try
+    {
+        if (getAvailableMemoryAmount() < (2l << 30))
+            server->context()->addWarningMessage("Available memory at server startup is too low (2GiB).");
+    }
+    catch (...)
+    {
+    }
 
     if (!enoughSpaceInDirectory(data_path, 1ull << 30))
         server->context()->addWarningMessage("Available disk space at server startup is too low (1GiB).");

From 9466aafb3c3c41dfaa85a11721cccc7752a407bb Mon Sep 17 00:00:00 2001
From: Pablo Alegre <palegre@tinybird.co>
Date: Mon, 14 Feb 2022 20:23:08 +0100
Subject: [PATCH 014/104] fixup! Add groupSortedArray() function

---
 .../AggregateFunctionGroupArraySorted.h       | 74 ++++++++++++++++++-
 tests/performance/group_array_sorted.xml      |  6 +-
 2 files changed, 74 insertions(+), 6 deletions(-)

diff --git a/src/AggregateFunctions/AggregateFunctionGroupArraySorted.h b/src/AggregateFunctions/AggregateFunctionGroupArraySorted.h
index 89bcb221e10..7f8a591e154 100644
--- a/src/AggregateFunctions/AggregateFunctionGroupArraySorted.h
+++ b/src/AggregateFunctions/AggregateFunctionGroupArraySorted.h
@@ -35,8 +35,9 @@ inline TColumn readItem(const IColumn * column, Arena * arena, size_t row)
     }
 }
 
-template <typename TColumn>
-size_t getFirstNElements(const TColumn * data, int num_elements, int threshold, size_t * results, const UInt8 * filter = nullptr)
+template <typename TColumn, typename TFilter = void>
+size_t
+getFirstNElements_low_threshold(const TColumn * data, int num_elements, int threshold, size_t * results, const TFilter * filter = nullptr)
 {
     for (int i = 0; i < threshold; i++)
     {
@@ -49,8 +50,11 @@ size_t getFirstNElements(const TColumn * data, int num_elements, int threshold,
     int z;
     for (int i = 0; i < num_elements; i++)
     {
-        if (filter && (filter[i] == 0))
-            continue;
+        if constexpr (!std::is_same_v<TFilter, void>)
+        {
+            if (filter[i] == 0)
+                continue;
+        }
 
         //Starting from the highest values and we look for the immediately lower than the given one
         for (cur = current_max; cur > 0; cur--)
@@ -76,6 +80,68 @@ size_t getFirstNElements(const TColumn * data, int num_elements, int threshold,
     return current_max;
 }
 
+template <typename T>
+struct SortableItem
+{
+    T a;
+    size_t b;
+    bool operator<(const SortableItem & other) const { return (this->a < other.a); }
+};
+
+template <typename TColumn, typename TFilter = void>
+size_t getFirstNElements_high_threshold(
+    const TColumn * data, size_t num_elements, size_t threshold, size_t * results, const TFilter * filter = nullptr)
+{
+    std::vector<SortableItem<TColumn>> dataIndexed(num_elements);
+    size_t num_elements_filtered = 0;
+
+    for (size_t i = 0; i < num_elements; i++)
+    {
+        if constexpr (!std::is_same_v<TFilter, void>)
+        {
+            if (filter[i] == 0)
+                continue;
+        }
+
+        dataIndexed.data()[num_elements_filtered].a = data[i];
+        dataIndexed.data()[num_elements_filtered].b = i;
+        num_elements_filtered++;
+    }
+
+    threshold = std::min(num_elements_filtered, threshold);
+
+    std::nth_element(dataIndexed.data(), dataIndexed.data() + threshold, dataIndexed.data() + num_elements_filtered);
+    std::sort(dataIndexed.data(), dataIndexed.data() + threshold);
+
+    for (size_t i = 0; i < threshold; i++)
+    {
+        results[i] = dataIndexed[i].b;
+    }
+
+    return threshold;
+}
+
+static const size_t THRESHOLD_MAX_CUSTOM_FUNCTION = 1000;
+
+template <typename TColumn>
+size_t getFirstNElements(const TColumn * data, size_t num_elements, size_t threshold, size_t * results, const UInt8 * filter = nullptr)
+{
+    if (threshold < THRESHOLD_MAX_CUSTOM_FUNCTION)
+    {
+        if (filter != nullptr)
+            return getFirstNElements_low_threshold(data, num_elements, threshold, results, filter);
+        else
+            return getFirstNElements_low_threshold(data, num_elements, threshold, results);
+    }
+    else
+    {
+        if (filter != nullptr)
+            return getFirstNElements_high_threshold(data, num_elements, threshold, results, filter);
+        else
+            return getFirstNElements_high_threshold(data, num_elements, threshold, results);
+    }
+}
+
 template <typename TColumnA, bool is_plain_a, bool use_column_b, typename TColumnB, bool is_plain_b>
 class AggregateFunctionGroupArraySorted : public IAggregateFunctionDataHelper<
                                               AggregateFunctionGroupArraySortedData<TColumnA, use_column_b, TColumnB>,
diff --git a/tests/performance/group_array_sorted.xml b/tests/performance/group_array_sorted.xml
index 2ed11f43b47..7d26459b4b1 100644
--- a/tests/performance/group_array_sorted.xml
+++ b/tests/performance/group_array_sorted.xml
@@ -16,10 +16,12 @@
     <create_query>CREATE TABLE test ( `id` UInt64, `value` UInt64, `text` String ) ENGINE = Memory</create_query>
     <fill_query>INSERT INTO test SELECT number as id, rand64() as value, toString(number) as text FROM numbers({items})</fill_query>
     <query>SELECT groupArraySorted(10)(id, value) FROM test</query>
-    <query>SELECT groupArraySorted(10)(id, -value) FROM test</query>
     <query>SELECT groupArraySorted(10)(text, value) FROM test</query>
     <query>SELECT groupArraySorted(10)((id, text), value) FROM test</query>
-    <query>SELECT groupArraySorted(10)(value) FROM test</query>
     <query>SELECT groupArraySorted(10)(text) FROM test</query>
+    <query>SELECT groupArraySorted(10000)(id, value) FROM test</query>
+    <query>SELECT groupArraySorted(10000)(text, value) FROM test</query>
+    <query>SELECT groupArraySorted(10000)((id, text), value) FROM test</query>
+    <query>SELECT groupArraySorted(10000)(text) FROM test</query>
     <drop_query>DROP TABLE IF EXISTS test</drop_query>
 </test>

From 74f99f736543a9d590027ebb2dc9a6e7f4eb7375 Mon Sep 17 00:00:00 2001
From: Pablo Alegre <palegre@tinybird.co>
Date: Thu, 24 Feb 2022 16:16:56 +0100
Subject: [PATCH 015/104] fixup! fixup! Add groupSortedArray() function

---
 src/AggregateFunctions/AggregateFunctionGroupArraySorted.h | 6 +++---
 tests/queries/0_stateless/02158_grouparraysorted.reference | 1 +
 tests/queries/0_stateless/02158_grouparraysorted.sql       | 2 ++
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/AggregateFunctions/AggregateFunctionGroupArraySorted.h b/src/AggregateFunctions/AggregateFunctionGroupArraySorted.h
index 7f8a591e154..4be8fe138a9 100644
--- a/src/AggregateFunctions/AggregateFunctionGroupArraySorted.h
+++ b/src/AggregateFunctions/AggregateFunctionGroupArraySorted.h
@@ -66,7 +66,7 @@ getFirstNElements_low_threshold(const TColumn * data, int num_elements, int thre
         if (cur < threshold)
         {
             //Move all the higher values 1 position to the right
-            for (z = current_max - 1; z > cur; z--)
+            for (z = std::min(threshold - 1, current_max); z > cur; z--)
                 results[z] = results[z - 1];
 
             if (current_max < threshold)
@@ -218,8 +218,8 @@ public:
         }
         else
         {
-            StringRef ref = columns[data_column]->getRawData();
-            values = reinterpret_cast<const TColumn *>(ref.data);
+            const auto & column = assert_cast<const ColumnVector<TColumn> &>(*columns[data_column]);
+            values = column.getData().data();
         }
 
         const UInt8 * filter = nullptr;
diff --git a/tests/queries/0_stateless/02158_grouparraysorted.reference b/tests/queries/0_stateless/02158_grouparraysorted.reference
index b4fec227621..2d93478c9cb 100644
--- a/tests/queries/0_stateless/02158_grouparraysorted.reference
+++ b/tests/queries/0_stateless/02158_grouparraysorted.reference
@@ -8,6 +8,7 @@
 [(0,'0'),(1,'1'),(2,'2'),(3,'3'),(4,'4')]
 ['0','1','10','11','12']
 ['0','1','2','3','4','5','6','7','8','9','10','11','12','13','14','15','16','17','18','19','20','21','22','23','24','25','26','27','28','29','30','31','32','33','34','35','36','37','38','39','40','41','42','43','44','45','46','47','48','49']
+[0,0,1,1,2,2,3,3,4,4]
 pablo	[1,2]
 luis	[1,3]
 pablo	[1,2]
diff --git a/tests/queries/0_stateless/02158_grouparraysorted.sql b/tests/queries/0_stateless/02158_grouparraysorted.sql
index 4ceb8374a6d..00cb32142ca 100644
--- a/tests/queries/0_stateless/02158_grouparraysorted.sql
+++ b/tests/queries/0_stateless/02158_grouparraysorted.sql
@@ -19,6 +19,8 @@ SELECT groupArraySorted(5)(text,text) from (SELECT toString(number) as text FROM
 
 SELECT groupArraySorted(50)(text,(number,text)) from (SELECT toString(number) as text, number FROM numbers(100));
 
+SELECT groupArraySorted(10)(toInt64(number/2)) FROM numbers(100);
+
 
 DROP TABLE IF EXISTS test;
 DROP VIEW IF EXISTS mv_test;

From 04e6e6ab5e4818f8774ecf3878ca2205d8e1aae6 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Mon, 21 Mar 2022 14:52:26 +0000
Subject: [PATCH 016/104] Add ParallelReadBuffer for S3

---
 src/Common/RangeGenerator.h      | 46 ++++++++++++++++++++
 src/IO/ReadBufferFromS3.h        | 73 ++++++++++++++++++++++++++++++++
 src/IO/ReadWriteBufferFromHTTP.h | 38 +----------------
 src/IO/S3Common.cpp              | 23 ++++++++++
 src/IO/S3Common.h                |  2 +
 src/Storages/StorageS3.cpp       |  8 +++-
 6 files changed, 152 insertions(+), 38 deletions(-)
 create mode 100644 src/Common/RangeGenerator.h

diff --git a/src/Common/RangeGenerator.h b/src/Common/RangeGenerator.h
new file mode 100644
index 00000000000..8d3b88936fc
--- /dev/null
+++ b/src/Common/RangeGenerator.h
@@ -0,0 +1,46 @@
+#pragma once
+
+#include <optional>
+#include <cmath>
+
+namespace DB
+{
+
+class RangeGenerator
+{
+public:
+    explicit RangeGenerator(size_t total_size_, size_t range_step_, size_t range_start = 0)
+        : from(range_start), range_step(range_step_), total_size(total_size_)
+    {
+    }
+
+    size_t totalRanges() const { return static_cast<size_t>(round(static_cast<float>(total_size - from) / range_step)); }
+
+    using Range = std::pair<size_t, size_t>;
+
+    // return upper exclusive range of values, i.e. [from_range, to_range>
+    std::optional<Range> nextRange()
+    {
+        if (from >= total_size)
+        {
+            return std::nullopt;
+        }
+
+        auto to = from + range_step;
+        if (to >= total_size)
+        {
+            to = total_size;
+        }
+
+        Range range{from, to};
+        from = to;
+        return std::move(range);
+    }
+
+private:
+    size_t from;
+    size_t range_step;
+    size_t total_size;
+};
+
+}
diff --git a/src/IO/ReadBufferFromS3.h b/src/IO/ReadBufferFromS3.h
index 157b6d46b6d..6668f570cb6 100644
--- a/src/IO/ReadBufferFromS3.h
+++ b/src/IO/ReadBufferFromS3.h
@@ -1,5 +1,6 @@
 #pragma once
 
+#include <Common/RangeGenerator.h>
 #include <Common/config.h>
 
 #if USE_AWS_S3
@@ -10,6 +11,7 @@
 #include <IO/ReadBuffer.h>
 #include <IO/ReadSettings.h>
 #include <IO/SeekableReadBuffer.h>
+#include <IO/ParallelReadBuffer.h>
 
 #include <aws/s3/model/GetObjectResult.h>
 
@@ -62,6 +64,12 @@ public:
 
     size_t getFileOffsetOfBufferEnd() const override { return offset; }
 
+    void setRange(off_t begin, off_t end)
+    {
+        offset = begin;
+        read_until_position = end;
+    }
+
 private:
     std::unique_ptr<ReadBuffer> initialize();
 
@@ -76,6 +84,71 @@ private:
     bool restricted_seek;
 };
 
+/// Creates separate ReadBufferFromS3 for sequence of ranges of particular object
+class ReadBufferS3Factory : public ParallelReadBuffer::ReadBufferFactory
+{
+public:
+    explicit ReadBufferS3Factory(
+        std::shared_ptr<Aws::S3::S3Client> client_ptr_,
+        const String & bucket_,
+        const String & key_,
+        size_t range_step_,
+        size_t object_size_,
+        UInt64 s3_max_single_read_retries_,
+        const ReadSettings &read_settings)
+        : client_ptr(client_ptr_)
+        , bucket(bucket_)
+        , key(key_)
+        , settings(read_settings)
+        , range_generator(object_size_, range_step_)
+        , range_step(range_step_)
+        , object_size(object_size_)
+        , s3_max_single_read_retries(s3_max_single_read_retries_)
+    {
+        assert(range_step > 0);
+        assert(range_step < object_size);
+    }
+
+    size_t totalRanges() const
+    {
+        return static_cast<size_t>(round(static_cast<float>(object_size) / range_step));
+    }
+
+    SeekableReadBufferPtr getReader() override
+    {
+        const auto next_range = range_generator.nextRange();
+        if (!next_range)
+        {
+            return nullptr;
+        }
+
+        auto reader = std::make_shared<ReadBufferFromS3>(
+                client_ptr, bucket, key, s3_max_single_read_retries, settings);
+        reader->setRange(next_range->first, next_range->second - 1);
+        return reader;
+    }
+
+    off_t seek(off_t off, [[maybe_unused]] int whence) override
+    {
+        range_generator = RangeGenerator{object_size, range_step, static_cast<size_t>(off)};
+        return off;
+    }
+
+    std::optional<size_t> getTotalSize() override { return object_size; }
+
+private:
+    std::shared_ptr<Aws::S3::S3Client> client_ptr;
+    const String bucket;
+    const String key;
+    ReadSettings settings;
+
+    RangeGenerator range_generator;
+    size_t range_step;
+    size_t object_size;
+
+    UInt64 s3_max_single_read_retries;
+};
+
 }
 
 #endif
diff --git a/src/IO/ReadWriteBufferFromHTTP.h b/src/IO/ReadWriteBufferFromHTTP.h
index 061dd772212..f0c10759142 100644
--- a/src/IO/ReadWriteBufferFromHTTP.h
+++ b/src/IO/ReadWriteBufferFromHTTP.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <functional>
+#include <Common/RangeGenerator.h>
 #include <IO/ConnectionTimeouts.h>
 #include <IO/HTTPCommon.h>
 #include <IO/ParallelReadBuffer.h>
@@ -635,43 +636,6 @@ public:
     void buildNewSession(const Poco::URI & uri) override { session = makeHTTPSession(uri, timeouts); }
 };
 
-class RangeGenerator
-{
-public:
-    explicit RangeGenerator(size_t total_size_, size_t range_step_, size_t range_start = 0)
-        : from(range_start), range_step(range_step_), total_size(total_size_)
-    {
-    }
-
-    size_t totalRanges() const { return static_cast<size_t>(round(static_cast<float>(total_size - from) / range_step)); }
-
-    using Range = std::pair<size_t, size_t>;
-
-    // return upper exclusive range of values, i.e. [from_range, to_range>
-    std::optional<Range> nextRange()
-    {
-        if (from >= total_size)
-        {
-            return std::nullopt;
-        }
-
-        auto to = from + range_step;
-        if (to >= total_size)
-        {
-            to = total_size;
-        }
-
-        Range range{from, to};
-        from = to;
-        return std::move(range);
-    }
-
-private:
-    size_t from;
-    size_t range_step;
-    size_t total_size;
-};
-
 class ReadWriteBufferFromHTTP : public detail::ReadWriteBufferFromHTTPBase<std::shared_ptr<UpdatableSession>>
 {
     using Parent = detail::ReadWriteBufferFromHTTPBase<std::shared_ptr<UpdatableSession>>;
diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp
index e63e6fde1f4..971ac0d8a73 100644
--- a/src/IO/S3Common.cpp
+++ b/src/IO/S3Common.cpp
@@ -33,6 +33,8 @@
 #    include <base/logger_useful.h>
 
 #    include <fstream>
+#include <aws/s3/S3Client.h>  // Y_IGNORE
+#include <aws/s3/model/HeadObjectRequest.h>  // Y_IGNORE
 
 namespace
 {
@@ -682,6 +684,7 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int BAD_ARGUMENTS;
+    extern const int S3_ERROR;
 }
 
 namespace S3
@@ -839,6 +842,26 @@ namespace S3
             throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bucket name length is out of bounds in virtual hosted style S3 URI:     {}{}",
                             quoteString(bucket), !uri.empty() ? " (" + uri.toString() + ")" : "");
     }
+
+    size_t getObjectSize(std::shared_ptr<Aws::S3::S3Client> client_ptr, const String & bucket, const String & key, bool throw_on_error)
+    {
+        Aws::S3::Model::HeadObjectRequest req;
+        req.SetBucket(bucket);
+        req.SetKey(key);
+
+        Aws::S3::Model::HeadObjectOutcome outcome = client_ptr->HeadObject(req);
+
+        if (outcome.IsSuccess())
+        {
+            auto read_result = outcome.GetResultWithOwnership();
+            return static_cast<size_t>(read_result.GetContentLength());
+        }
+        else if (throw_on_error)
+        {
+            throw DB::Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR);
+        }
+        return 0;
+    }
 }
 
 }
diff --git a/src/IO/S3Common.h b/src/IO/S3Common.h
index 97cb4f74f90..c33ce427e66 100644
--- a/src/IO/S3Common.h
+++ b/src/IO/S3Common.h
@@ -75,6 +75,8 @@ struct URI
     static void validateBucket(const String & bucket, const Poco::URI & uri);
 };
 
+size_t getObjectSize(std::shared_ptr<Aws::S3::S3Client> client_ptr, const String & bucket, const String & key, bool throw_on_error = true);
+
 }
 
 #endif
diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp
index f319bd1097b..6f5228c2c63 100644
--- a/src/Storages/StorageS3.cpp
+++ b/src/Storages/StorageS3.cpp
@@ -1,4 +1,6 @@
 #include <Common/config.h>
+#include "IO/ParallelReadBuffer.h"
+#include "IO/IOThreadPool.h"
 #include "Parsers/ASTCreateQuery.h"
 
 #if USE_AWS_S3
@@ -273,9 +275,13 @@ bool StorageS3Source::initialize()
 
     file_path = fs::path(bucket) / current_key;
 
+    size_t object_size = DB::S3::getObjectSize(client, bucket, current_key, false);
+    auto factory = std::make_unique<ReadBufferS3Factory>(client, bucket, current_key, 10 * 1024 * 1024, object_size, max_single_read_retries, getContext()->getReadSettings());
+
     read_buf = wrapReadBufferWithCompressionMethod(
-        std::make_unique<ReadBufferFromS3>(client, bucket, current_key, max_single_read_retries, getContext()->getReadSettings()),
+        std::make_unique<ParallelReadBuffer>(std::move(factory), &IOThreadPool::get(), 4),
         chooseCompressionMethod(current_key, compression_hint));
+
     auto input_format = getContext()->getInputFormat(format, *read_buf, sample_block, max_block_size, format_settings);
     QueryPipelineBuilder builder;
     builder.init(Pipe(input_format));

From 6785ad165a72e372f155a10e71a872e37bdfc2c1 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Tue, 22 Mar 2022 14:06:30 +0000
Subject: [PATCH 017/104] Fix issue for mutliple download threads

---
 src/IO/ParallelReadBuffer.cpp |  2 +-
 src/IO/ParallelReadBuffer.h   |  4 ++--
 src/IO/ReadBufferFromS3.h     |  4 ++--
 src/Storages/StorageS3.cpp    | 20 +++++++++++++++-----
 4 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/src/IO/ParallelReadBuffer.cpp b/src/IO/ParallelReadBuffer.cpp
index 7fa10c160ad..3ee62148c23 100644
--- a/src/IO/ParallelReadBuffer.cpp
+++ b/src/IO/ParallelReadBuffer.cpp
@@ -237,7 +237,7 @@ void ParallelReadBuffer::readerThreadFunction(ReadWorkerPtr read_worker)
         while (!emergency_stop && !read_worker->cancel)
         {
             if (!read_worker->reader->next())
-                throw Exception("Failed to read all the data from the reader", ErrorCodes::LOGICAL_ERROR);
+                throw Exception(ErrorCodes::LOGICAL_ERROR, "Failed to read all the data from the reader, missing {} bytes", read_worker->bytes_left);
 
             if (emergency_stop || read_worker->cancel)
                 break;
diff --git a/src/IO/ParallelReadBuffer.h b/src/IO/ParallelReadBuffer.h
index 7b364205e8e..7a3f0cb3773 100644
--- a/src/IO/ParallelReadBuffer.h
+++ b/src/IO/ParallelReadBuffer.h
@@ -82,8 +82,8 @@ public:
         std::unique_ptr<ReadBufferFactory> reader_factory_,
         ThreadPool * pool,
         size_t max_working_readers,
-        WorkerSetup worker_setup = {},
-        WorkerCleanup worker_cleanup = {});
+        WorkerSetup worker_setup = [](ThreadStatus &){},
+        WorkerCleanup worker_cleanup = [](ThreadStatus &){});
 
     ~ParallelReadBuffer() override { finishAndWait(); }
 
diff --git a/src/IO/ReadBufferFromS3.h b/src/IO/ReadBufferFromS3.h
index 6668f570cb6..e3708c4b748 100644
--- a/src/IO/ReadBufferFromS3.h
+++ b/src/IO/ReadBufferFromS3.h
@@ -60,7 +60,7 @@ public:
 
     void setReadUntilPosition(size_t position) override;
 
-    Range getRemainingReadRange() const override { return Range{ .left = static_cast<size_t>(offset), .right = read_until_position }; }
+    Range getRemainingReadRange() const override { return Range{ .left = static_cast<size_t>(offset), .right = read_until_position - 1}; }
 
     size_t getFileOffsetOfBufferEnd() const override { return offset; }
 
@@ -124,7 +124,7 @@ public:
 
         auto reader = std::make_shared<ReadBufferFromS3>(
                 client_ptr, bucket, key, s3_max_single_read_retries, settings);
-        reader->setRange(next_range->first, next_range->second - 1);
+        reader->setRange(next_range->first, next_range->second);
         return reader;
     }
 
diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp
index 6f5228c2c63..9dc15e54ecc 100644
--- a/src/Storages/StorageS3.cpp
+++ b/src/Storages/StorageS3.cpp
@@ -275,12 +275,22 @@ bool StorageS3Source::initialize()
 
     file_path = fs::path(bucket) / current_key;
 
-    size_t object_size = DB::S3::getObjectSize(client, bucket, current_key, false);
-    auto factory = std::make_unique<ReadBufferS3Factory>(client, bucket, current_key, 10 * 1024 * 1024, object_size, max_single_read_retries, getContext()->getReadSettings());
+    const auto max_download_threads = getContext()->getSettings().max_download_threads;
+    if (max_download_threads == 1)
+    {
+        read_buf = wrapReadBufferWithCompressionMethod(
+         std::make_unique<ReadBufferFromS3>(client, bucket, current_key, max_single_read_retries, getContext()->getReadSettings()),
+            chooseCompressionMethod(current_key, compression_hint));
+    }
+    else
+    {
+        size_t object_size = DB::S3::getObjectSize(client, bucket, current_key, false);
+        auto factory = std::make_unique<ReadBufferS3Factory>(client, bucket, current_key, 10 * 1024 * 1024, object_size, max_single_read_retries, getContext()->getReadSettings());
 
-    read_buf = wrapReadBufferWithCompressionMethod(
-        std::make_unique<ParallelReadBuffer>(std::move(factory), &IOThreadPool::get(), 4),
-        chooseCompressionMethod(current_key, compression_hint));
+        read_buf = wrapReadBufferWithCompressionMethod(
+         std::make_unique<ParallelReadBuffer>(std::move(factory), &IOThreadPool::get(), getContext()->getSettings().max_download_threads),
+            chooseCompressionMethod(current_key, compression_hint));
+    }
 
     auto input_format = getContext()->getInputFormat(format, *read_buf, sample_block, max_block_size, format_settings);
     QueryPipelineBuilder builder;

From f693eba568415e66d856da7c45df824d97949361 Mon Sep 17 00:00:00 2001
From: Anton Popov <ap@clickhouse.com>
Date: Tue, 22 Mar 2022 14:30:40 +0000
Subject: [PATCH 018/104] fix tests with approx rows

---
 src/Processors/QueryPlan/ReadFromMergeTree.cpp      | 9 ++++++---
 src/Storages/MergeTree/MergeTreeIOSettings.h        | 2 ++
 src/Storages/MergeTree/MergeTreeSelectProcessor.cpp | 8 ++++++++
 3 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp
index 1bfc1ec7306..cf7b2dcc855 100644
--- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp
+++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp
@@ -45,7 +45,8 @@ namespace ErrorCodes
     extern const int LOGICAL_ERROR;
 }
 
-static MergeTreeReaderSettings getMergeTreeReaderSettings(const ContextPtr & context)
+static MergeTreeReaderSettings getMergeTreeReaderSettings(
+    const ContextPtr & context, const SelectQueryInfo & query_info)
 {
     const auto & settings = context->getSettingsRef();
     return
@@ -53,6 +54,7 @@ static MergeTreeReaderSettings getMergeTreeReaderSettings(const ContextPtr & con
         .read_settings = context->getReadSettings(),
         .save_marks_in_cache = true,
         .checksum_on_read = settings.checksum_on_read,
+        .read_in_order = query_info.input_order_info != nullptr,
     };
 }
 
@@ -82,7 +84,7 @@ ReadFromMergeTree::ReadFromMergeTree(
         getPrewhereInfo(query_info_),
         data_.getPartitionValueType(),
         virt_column_names_)})
-    , reader_settings(getMergeTreeReaderSettings(context_))
+    , reader_settings(getMergeTreeReaderSettings(context_, query_info_))
     , prepared_parts(std::move(parts_))
     , real_column_names(std::move(real_column_names_))
     , virt_column_names(std::move(virt_column_names_))
@@ -203,6 +205,7 @@ ProcessorPtr ReadFromMergeTree::createSource(
             .colums_to_read = required_columns
         };
     }
+
     return std::make_shared<TSource>(
             data, storage_snapshot, part.data_part, max_block_size, preferred_block_size_bytes,
             preferred_max_column_in_block_size_bytes, required_columns, part.ranges, use_uncompressed_cache, prewhere_info,
@@ -918,7 +921,7 @@ MergeTreeDataSelectAnalysisResultPtr ReadFromMergeTree::selectRangesToRead(
             total_marks_pk += part->index_granularity.getMarksCountWithoutFinal();
         parts_before_pk = parts.size();
 
-        auto reader_settings = getMergeTreeReaderSettings(context);
+        auto reader_settings = getMergeTreeReaderSettings(context, query_info);
 
         bool use_skip_indexes = settings.use_skip_indexes;
         if (select.final() && !settings.use_skip_indexes_if_final)
diff --git a/src/Storages/MergeTree/MergeTreeIOSettings.h b/src/Storages/MergeTree/MergeTreeIOSettings.h
index aaa8fae7dba..4fe424c98ff 100644
--- a/src/Storages/MergeTree/MergeTreeIOSettings.h
+++ b/src/Storages/MergeTree/MergeTreeIOSettings.h
@@ -20,6 +20,8 @@ struct MergeTreeReaderSettings
     bool save_marks_in_cache = false;
     /// Validate checksums on reading (should be always enabled in production).
     bool checksum_on_read = true;
+    /// True if we read in order of sorting key.
+    bool read_in_order = false;
 };
 
 struct MergeTreeWriterSettings
diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp
index 90e272e329e..f6cbf54b752 100644
--- a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp
+++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp
@@ -37,6 +37,14 @@ MergeTreeSelectProcessor::MergeTreeSelectProcessor(
     has_limit_below_one_block(has_limit_below_one_block_),
     total_rows(data_part->index_granularity.getRowsCountInRanges(all_mark_ranges))
 {
+    /// Actually it means that parallel reading from replicas enabled
+    /// and we have to collaborate with initiator.
+    /// In this case we won't set approximate rows, because it will be accounted multiple times.
+    /// Also do not count amount of read rows if we read in order of sorting key,
+    /// because we don't know actual amount of read rows in case when limit is set.
+    if (!extension_.has_value() && !reader_settings.read_in_order)
+        addTotalRowsApprox(total_rows);
+
     ordered_names = header_without_virtual_columns.getNames();
 }
 

From f32ef2a556f4b6a667ec2a0ff3d3553703046dc0 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Wed, 23 Mar 2022 08:15:18 +0000
Subject: [PATCH 019/104] Small polishing for S3 reader

---
 src/IO/ReadBufferFromS3.cpp |  4 +++-
 src/IO/ReadBufferFromS3.h   | 48 +++++++++++++++++--------------------
 2 files changed, 25 insertions(+), 27 deletions(-)

diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp
index 93bbe02c9cd..05e1276bc63 100644
--- a/src/IO/ReadBufferFromS3.cpp
+++ b/src/IO/ReadBufferFromS3.cpp
@@ -42,6 +42,7 @@ ReadBufferFromS3::ReadBufferFromS3(
     UInt64 max_single_read_retries_,
     const ReadSettings & settings_,
     bool use_external_buffer_,
+    size_t offset_,
     size_t read_until_position_,
     bool restricted_seek_)
     : SeekableReadBufferWithSize(nullptr, 0)
@@ -49,9 +50,10 @@ ReadBufferFromS3::ReadBufferFromS3(
     , bucket(bucket_)
     , key(key_)
     , max_single_read_retries(max_single_read_retries_)
+    , offset(offset_)
+    , read_until_position(read_until_position_)
     , read_settings(settings_)
     , use_external_buffer(use_external_buffer_)
-    , read_until_position(read_until_position_)
     , restricted_seek(restricted_seek_)
 {
 }
diff --git a/src/IO/ReadBufferFromS3.h b/src/IO/ReadBufferFromS3.h
index e3708c4b748..8d71a03fb87 100644
--- a/src/IO/ReadBufferFromS3.h
+++ b/src/IO/ReadBufferFromS3.h
@@ -5,15 +5,15 @@
 
 #if USE_AWS_S3
 
-#include <memory>
+#    include <memory>
 
-#include <IO/HTTPCommon.h>
-#include <IO/ReadBuffer.h>
-#include <IO/ReadSettings.h>
-#include <IO/SeekableReadBuffer.h>
-#include <IO/ParallelReadBuffer.h>
+#    include <IO/HTTPCommon.h>
+#    include <IO/ParallelReadBuffer.h>
+#    include <IO/ReadBuffer.h>
+#    include <IO/ReadSettings.h>
+#    include <IO/SeekableReadBuffer.h>
 
-#include <aws/s3/model/GetObjectResult.h>
+#    include <aws/s3/model/GetObjectResult.h>
 
 namespace Aws::S3
 {
@@ -32,7 +32,9 @@ private:
     String bucket;
     String key;
     UInt64 max_single_read_retries;
+
     off_t offset = 0;
+    off_t read_until_position = 0;
 
     Aws::S3::Model::GetObjectResult read_result;
     std::unique_ptr<ReadBuffer> impl;
@@ -47,6 +49,7 @@ public:
         UInt64 max_single_read_retries_,
         const ReadSettings & settings_,
         bool use_external_buffer = false,
+        size_t offset_ = 0,
         size_t read_until_position_ = 0,
         bool restricted_seek_ = false);
 
@@ -60,16 +63,10 @@ public:
 
     void setReadUntilPosition(size_t position) override;
 
-    Range getRemainingReadRange() const override { return Range{ .left = static_cast<size_t>(offset), .right = read_until_position - 1}; }
+    Range getRemainingReadRange() const override { return Range{.left = static_cast<size_t>(offset), .right = read_until_position - 1}; }
 
     size_t getFileOffsetOfBufferEnd() const override { return offset; }
 
-    void setRange(off_t begin, off_t end)
-    {
-        offset = begin;
-        read_until_position = end;
-    }
-
 private:
     std::unique_ptr<ReadBuffer> initialize();
 
@@ -77,8 +74,6 @@ private:
 
     bool use_external_buffer;
 
-    off_t read_until_position = 0;
-
     /// There is different seek policy for disk seek and for non-disk seek
     /// (non-disk seek is applied for seekable input formats: orc, arrow, parquet).
     bool restricted_seek;
@@ -95,11 +90,11 @@ public:
         size_t range_step_,
         size_t object_size_,
         UInt64 s3_max_single_read_retries_,
-        const ReadSettings &read_settings)
+        const ReadSettings & read_settings_)
         : client_ptr(client_ptr_)
         , bucket(bucket_)
         , key(key_)
-        , settings(read_settings)
+        , read_settings(read_settings_)
         , range_generator(object_size_, range_step_)
         , range_step(range_step_)
         , object_size(object_size_)
@@ -109,11 +104,6 @@ public:
         assert(range_step < object_size);
     }
 
-    size_t totalRanges() const
-    {
-        return static_cast<size_t>(round(static_cast<float>(object_size) / range_step));
-    }
-
     SeekableReadBufferPtr getReader() override
     {
         const auto next_range = range_generator.nextRange();
@@ -123,8 +113,14 @@ public:
         }
 
         auto reader = std::make_shared<ReadBufferFromS3>(
-                client_ptr, bucket, key, s3_max_single_read_retries, settings);
-        reader->setRange(next_range->first, next_range->second);
+            client_ptr,
+            bucket,
+            key,
+            s3_max_single_read_retries,
+            read_settings,
+            false /*use_external_buffer*/,
+            next_range->first,
+            next_range->second + 1);
         return reader;
     }
 
@@ -140,7 +136,7 @@ private:
     std::shared_ptr<Aws::S3::S3Client> client_ptr;
     const String bucket;
     const String key;
-    ReadSettings settings;
+    ReadSettings read_settings;
 
     RangeGenerator range_generator;
     size_t range_step;

From 131b3a091c60a13ed7cd7ddfe090e5dda204b6ad Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Wed, 23 Mar 2022 08:40:00 +0000
Subject: [PATCH 020/104] Refactor StorageS3

---
 src/IO/ReadBufferFromS3.h  |  2 +-
 src/Storages/StorageS3.cpp | 70 ++++++++++++++++++++++++++------------
 src/Storages/StorageS3.h   |  6 +++-
 3 files changed, 54 insertions(+), 24 deletions(-)

diff --git a/src/IO/ReadBufferFromS3.h b/src/IO/ReadBufferFromS3.h
index 8d71a03fb87..d984fbebe5f 100644
--- a/src/IO/ReadBufferFromS3.h
+++ b/src/IO/ReadBufferFromS3.h
@@ -120,7 +120,7 @@ public:
             read_settings,
             false /*use_external_buffer*/,
             next_range->first,
-            next_range->second + 1);
+            next_range->second);
         return reader;
     }
 
diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp
index 9dc15e54ecc..6966d356070 100644
--- a/src/Storages/StorageS3.cpp
+++ b/src/Storages/StorageS3.cpp
@@ -238,7 +238,8 @@ StorageS3Source::StorageS3Source(
     String compression_hint_,
     const std::shared_ptr<Aws::S3::S3Client> & client_,
     const String & bucket_,
-    std::shared_ptr<IteratorWrapper> file_iterator_)
+    std::shared_ptr<IteratorWrapper> file_iterator_,
+    const size_t download_thread_num_)
     : SourceWithProgress(getHeader(sample_block_, need_path, need_file))
     , WithContext(context_)
     , name(std::move(name_))
@@ -254,6 +255,7 @@ StorageS3Source::StorageS3Source(
     , with_file_column(need_file)
     , with_path_column(need_path)
     , file_iterator(file_iterator_)
+    , download_thread_num(download_thread_num_)
 {
     initialize();
 }
@@ -275,22 +277,7 @@ bool StorageS3Source::initialize()
 
     file_path = fs::path(bucket) / current_key;
 
-    const auto max_download_threads = getContext()->getSettings().max_download_threads;
-    if (max_download_threads == 1)
-    {
-        read_buf = wrapReadBufferWithCompressionMethod(
-         std::make_unique<ReadBufferFromS3>(client, bucket, current_key, max_single_read_retries, getContext()->getReadSettings()),
-            chooseCompressionMethod(current_key, compression_hint));
-    }
-    else
-    {
-        size_t object_size = DB::S3::getObjectSize(client, bucket, current_key, false);
-        auto factory = std::make_unique<ReadBufferS3Factory>(client, bucket, current_key, 10 * 1024 * 1024, object_size, max_single_read_retries, getContext()->getReadSettings());
-
-        read_buf = wrapReadBufferWithCompressionMethod(
-         std::make_unique<ParallelReadBuffer>(std::move(factory), &IOThreadPool::get(), getContext()->getSettings().max_download_threads),
-            chooseCompressionMethod(current_key, compression_hint));
-    }
+    read_buf = wrapReadBufferWithCompressionMethod(createS3ReadBuffer(current_key), chooseCompressionMethod(current_key, compression_hint));
 
     auto input_format = getContext()->getInputFormat(format, *read_buf, sample_block, max_block_size, format_settings);
     QueryPipelineBuilder builder;
@@ -298,10 +285,9 @@ bool StorageS3Source::initialize()
 
     if (columns_desc.hasDefaults())
     {
-        builder.addSimpleTransform([&](const Block & header)
-        {
-            return std::make_shared<AddingDefaultsTransform>(header, columns_desc, *input_format, getContext());
-        });
+        builder.addSimpleTransform(
+            [&](const Block & header)
+            { return std::make_shared<AddingDefaultsTransform>(header, columns_desc, *input_format, getContext()); });
     }
 
     pipeline = std::make_unique<QueryPipeline>(QueryPipelineBuilder::getPipeline(std::move(builder)));
@@ -311,6 +297,43 @@ bool StorageS3Source::initialize()
     return true;
 }
 
+std::unique_ptr<ReadBuffer> StorageS3Source::createS3ReadBuffer(const String & key)
+{
+    const size_t object_size = DB::S3::getObjectSize(client, bucket, key, false);
+
+    auto download_buffer_size = getContext()->getSettings().max_download_buffer_size;
+    const bool use_parallel_download = download_buffer_size > 0 && download_thread_num > 1;
+    const bool object_too_small = object_size < download_thread_num * download_buffer_size;
+    if (!use_parallel_download || object_too_small)
+    {
+        LOG_TRACE(&Poco::Logger::get("StorageS3Source"), "Downloading object of size {} from S3 in single thread", object_size);
+        return std::make_unique<ReadBufferFromS3>(client, bucket, key, max_single_read_retries, getContext()->getReadSettings());
+    }
+
+    assert(object_size > 0);
+
+    if (download_buffer_size < DBMS_DEFAULT_BUFFER_SIZE)
+    {
+        LOG_WARNING(
+            &Poco::Logger::get("StorageS3Source"),
+            "Downloading buffer {} bytes too small, set at least {} bytes",
+            download_buffer_size,
+            DBMS_DEFAULT_BUFFER_SIZE);
+        download_buffer_size = DBMS_DEFAULT_BUFFER_SIZE;
+    }
+
+    auto factory = std::make_unique<ReadBufferS3Factory>(
+        client, bucket, key, download_buffer_size, object_size, max_single_read_retries, getContext()->getReadSettings());
+    LOG_TRACE(
+        &Poco::Logger::get("StorageS3Source"),
+        "Downloading from S3 in {} threads. Object size: {}, Range size: {}.",
+        download_thread_num,
+        object_size,
+        download_buffer_size);
+
+    return std::make_unique<ParallelReadBuffer>(std::move(factory), &IOThreadPool::get(), download_thread_num);
+}
+
 String StorageS3Source::getName() const
 {
     return name;
@@ -674,6 +697,8 @@ Pipe StorageS3::read(
         block_for_format = storage_snapshot->metadata->getSampleBlock();
     }
 
+    const size_t max_download_threads = local_context->getSettingsRef().max_download_threads;
+    const size_t download_threads_per_stream = num_streams >= max_download_threads ? 1 : (max_download_threads / num_streams);
     for (size_t i = 0; i < num_streams; ++i)
     {
         pipes.emplace_back(std::make_shared<StorageS3Source>(
@@ -690,7 +715,8 @@ Pipe StorageS3::read(
             compression_method,
             client_auth.client,
             client_auth.uri.bucket,
-            iterator_wrapper));
+            iterator_wrapper,
+            download_threads_per_stream));
     }
     auto pipe = Pipe::unitePipes(std::move(pipes));
 
diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h
index 300b7becb93..6c88f5bba56 100644
--- a/src/Storages/StorageS3.h
+++ b/src/Storages/StorageS3.h
@@ -74,7 +74,8 @@ public:
         String compression_hint_,
         const std::shared_ptr<Aws::S3::S3Client> & client_,
         const String & bucket,
-        std::shared_ptr<IteratorWrapper> file_iterator_);
+        std::shared_ptr<IteratorWrapper> file_iterator_,
+        size_t download_thread_num);
 
     String getName() const override;
 
@@ -105,9 +106,12 @@ private:
     bool with_file_column = false;
     bool with_path_column = false;
     std::shared_ptr<IteratorWrapper> file_iterator;
+    size_t download_thread_num = 1;
 
     /// Recreate ReadBuffer and BlockInputStream for each file.
     bool initialize();
+
+    std::unique_ptr<ReadBuffer> createS3ReadBuffer(const String & key);
 };
 
 /**

From 93ad209910cb00c492bd727de0e5eed3baf2e55d Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Wed, 23 Mar 2022 11:52:31 +0000
Subject: [PATCH 021/104] Attach memory tracker

---
 src/Storages/StorageS3.cpp | 32 ++++++++++++++++++++++++++++----
 src/Storages/StorageS3.h   |  1 -
 2 files changed, 28 insertions(+), 5 deletions(-)

diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp
index 6966d356070..35a8405eeb2 100644
--- a/src/Storages/StorageS3.cpp
+++ b/src/Storages/StorageS3.cpp
@@ -293,7 +293,6 @@ bool StorageS3Source::initialize()
     pipeline = std::make_unique<QueryPipeline>(QueryPipelineBuilder::getPipeline(std::move(builder)));
     reader = std::make_unique<PullingPipelineExecutor>(*pipeline);
 
-    initialized = false;
     return true;
 }
 
@@ -331,7 +330,32 @@ std::unique_ptr<ReadBuffer> StorageS3Source::createS3ReadBuffer(const String & k
         object_size,
         download_buffer_size);
 
-    return std::make_unique<ParallelReadBuffer>(std::move(factory), &IOThreadPool::get(), download_thread_num);
+    ThreadGroupStatusPtr running_group = CurrentThread::isInitialized() && CurrentThread::get().getThreadGroup()
+        ? CurrentThread::get().getThreadGroup()
+        : MainThreadStatus::getInstance().getThreadGroup();
+
+    ContextPtr query_context
+        = CurrentThread::isInitialized() ? CurrentThread::get().getQueryContext() : nullptr;
+
+    auto worker_cleanup = [has_running_group = running_group == nullptr](ThreadStatus & thread_status)
+    {
+        if (has_running_group)
+            thread_status.detachQuery(false);
+    };
+
+    auto worker_setup = [query_context = std::move(query_context),
+                         running_group = std::move(running_group)](ThreadStatus & thread_status)
+    {
+        /// Save query context if any, because cache implementation needs it.
+        if (query_context)
+            thread_status.attachQueryContext(query_context);
+
+        /// To be able to pass ProfileEvents.
+        if (running_group)
+            thread_status.attachQuery(running_group);
+    };
+
+    return std::make_unique<ParallelReadBuffer>(std::move(factory), &IOThreadPool::get(), download_thread_num, worker_setup, worker_cleanup);
 }
 
 String StorageS3Source::getName() const
@@ -698,7 +722,7 @@ Pipe StorageS3::read(
     }
 
     const size_t max_download_threads = local_context->getSettingsRef().max_download_threads;
-    const size_t download_threads_per_stream = num_streams >= max_download_threads ? 1 : (max_download_threads / num_streams);
+    // const size_t download_threads_per_stream = num_streams >= max_download_threads ? 1 : (max_download_threads / num_streams);
     for (size_t i = 0; i < num_streams; ++i)
     {
         pipes.emplace_back(std::make_shared<StorageS3Source>(
@@ -716,7 +740,7 @@ Pipe StorageS3::read(
             client_auth.client,
             client_auth.uri.bucket,
             iterator_wrapper,
-            download_threads_per_stream));
+            max_download_threads));
     }
     auto pipe = Pipe::unitePipes(std::move(pipes));
 
diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h
index 6c88f5bba56..6362e777df6 100644
--- a/src/Storages/StorageS3.h
+++ b/src/Storages/StorageS3.h
@@ -102,7 +102,6 @@ private:
     std::unique_ptr<PullingPipelineExecutor> reader;
     /// onCancel and generate can be called concurrently
     std::mutex reader_mutex;
-    bool initialized = false;
     bool with_file_column = false;
     bool with_path_column = false;
     std::shared_ptr<IteratorWrapper> file_iterator;

From 1a5f5c32bb2e2a96d22c0ef8132d29fc21db5aba Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Wed, 23 Mar 2022 14:36:25 +0000
Subject: [PATCH 022/104] Remove commented out code

---
 src/Storages/StorageS3.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp
index 35a8405eeb2..b80b9ead53f 100644
--- a/src/Storages/StorageS3.cpp
+++ b/src/Storages/StorageS3.cpp
@@ -722,7 +722,6 @@ Pipe StorageS3::read(
     }
 
     const size_t max_download_threads = local_context->getSettingsRef().max_download_threads;
-    // const size_t download_threads_per_stream = num_streams >= max_download_threads ? 1 : (max_download_threads / num_streams);
     for (size_t i = 0; i < num_streams; ++i)
     {
         pipes.emplace_back(std::make_shared<StorageS3Source>(

From c7cf14e7436b0d968f3542bebee53cb418bdb34e Mon Sep 17 00:00:00 2001
From: Sergei Trifonov <sergei@clickhouse.com>
Date: Wed, 23 Mar 2022 19:15:01 +0100
Subject: [PATCH 023/104] fix style and darwin build

---
 base/base/getAvailableMemoryAmount.cpp | 27 ++++++++++++++------------
 src/Common/filesystemHelpers.h         |  3 ++-
 2 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/base/base/getAvailableMemoryAmount.cpp b/base/base/getAvailableMemoryAmount.cpp
index a7fd970c216..a7b12ef4699 100644
--- a/base/base/getAvailableMemoryAmount.cpp
+++ b/base/base/getAvailableMemoryAmount.cpp
@@ -14,21 +14,24 @@
 
 uint64_t getAvailableMemoryAmountOrZero()
 {
-    int64_t page_size = getPageSize();
-    if (page_size <= 0)
-        return 0;
-
-#if defined(__FreeBSD__)
+#if defined(_SC_AVPHYS_PAGES) // linux
+    return getPageSize() * sysconf(_SC_AVPHYS_PAGES);
+#elif defined(__FreeBSD__)
     struct vmtotal vmt;
     size_t vmt_size = sizeof(vmt);
-    if (sysctlbyname("vm.vmtotal", &vmt, &vmt_size, NULL, 0) < 0)
-       return 0;
-    uint64_t available_pages = vmt.t_avm;
-#else
-    uint64_t available_pages = sysconf(_SC_AVPHYS_PAGES);
+    if (sysctlbyname("vm.vmtotal", &vmt, &vmt_size, NULL, 0) == 0)
+        return getPageSize() * vmt.t_avm;
+    else
+        return 0;
+#else // darwin
+    unsigned int usermem;
+    size_t len = sizeof(usermem);
+    static int mib[2] = { CTL_HW, HW_USERMEM };
+    if (sysctl(mib, 2, &usermem, &len, NULL, 0) == 0 && len == sizeof(usermem))
+        return usermem;
+    else
+        return 0;
 #endif
-
-    return page_size * available_pages;
 }
 
 
diff --git a/src/Common/filesystemHelpers.h b/src/Common/filesystemHelpers.h
index 05d1fe9192e..a0c761a516f 100644
--- a/src/Common/filesystemHelpers.h
+++ b/src/Common/filesystemHelpers.h
@@ -24,7 +24,8 @@ std::unique_ptr<TemporaryFile> createTemporaryFile(const std::string & path);
 #endif
 String getBlockDeviceId([[maybe_unused]] const String & path);
 
-enum class BlockDeviceType {
+enum class BlockDeviceType
+{
     UNKNOWN = 0, // we were unable to determine device type
     NONROT = 1, // not a rotational device (SSD, NVME, etc)
     ROT = 2 // rotational device (HDD)

From e0d7b6dc3ed239ef4075add09090751d45941a4e Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Thu, 24 Mar 2022 09:30:06 +0000
Subject: [PATCH 024/104] Add tests for S3 multithreaded download

---
 tests/integration/test_storage_s3/test.py | 34 ++++++++++++++++++++++-
 1 file changed, 33 insertions(+), 1 deletion(-)

diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py
index dd29d0a5d6a..7364af4d2dd 100644
--- a/tests/integration/test_storage_s3/test.py
+++ b/tests/integration/test_storage_s3/test.py
@@ -535,8 +535,9 @@ def test_multipart_put(started_cluster, maybe_auth, positive):
 
     one_line_length = 6  # 3 digits, 2 commas, 1 line separator.
 
+    total_rows = csv_size_bytes // one_line_length
     # Generate data having size more than one part
-    int_data = [[1, 2, 3] for i in range(csv_size_bytes // one_line_length)]
+    int_data = [[1, 2, 3] for i in range(total_rows)]
     csv_data = "".join(["{},{},{}\n".format(x, y, z) for x, y, z in int_data])
 
     assert len(csv_data) > min_part_size_bytes
@@ -573,6 +574,37 @@ def test_multipart_put(started_cluster, maybe_auth, positive):
 
         assert csv_data == get_s3_file_content(started_cluster, bucket, filename)
 
+    # select uploaded data from many threads
+    select_query = (
+        "select sum(column1), sum(column2), sum(column3) "
+        "from s3('http://{host}:{port}/{bucket}/{filename}', {auth}'CSV', '{table_format}')".format(
+            host=started_cluster.minio_redirect_host,
+            port=started_cluster.minio_redirect_port,
+            bucket=bucket,
+            filename=filename,
+            auth=maybe_auth,
+            table_format=table_format,
+        )
+    )
+    try:
+        select_result = run_query(
+            instance,
+            select_query,
+            settings={
+                "max_download_threads": random.randint(4, 16),
+                "max_download_buffer_size": 1024 * 1024,
+            },
+        )
+    except helpers.client.QueryRuntimeException:
+        if positive:
+            raise
+    else:
+        assert positive
+        assert (
+            select_result
+            == "\t".join(map(str, [total_rows, total_rows * 2, total_rows * 3])) + "\n"
+        )
+
 
 def test_remote_host_filter(started_cluster):
     instance = started_cluster.instances["restricted_dummy"]

From 62a495a5fc64c24b83982a362c8e72b965f2fb09 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Thu, 24 Mar 2022 09:38:11 +0000
Subject: [PATCH 025/104] Small refactor

---
 src/IO/ReadBufferFromS3.cpp               | 30 ++++++++++++++++
 src/IO/ReadBufferFromS3.h                 | 42 ++++++-----------------
 src/IO/S3Common.cpp                       |  3 +-
 src/Storages/StorageS3.cpp                |  3 +-
 tests/integration/test_storage_s3/test.py |  2 +-
 5 files changed, 44 insertions(+), 36 deletions(-)

diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp
index 05e1276bc63..c78f41a0f9d 100644
--- a/src/IO/ReadBufferFromS3.cpp
+++ b/src/IO/ReadBufferFromS3.cpp
@@ -274,6 +274,36 @@ std::unique_ptr<ReadBuffer> ReadBufferFromS3::initialize()
         throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR);
 }
 
+SeekableReadBufferPtr ReadBufferS3Factory::getReader()
+{
+    const auto next_range = range_generator.nextRange();
+    if (!next_range)
+    {
+        return nullptr;
+    }
+
+    auto reader = std::make_shared<ReadBufferFromS3>(
+        client_ptr,
+        bucket,
+        key,
+        s3_max_single_read_retries,
+        read_settings,
+        false /*use_external_buffer*/,
+        next_range->first,
+        next_range->second);
+    return reader;
+}
+
+off_t ReadBufferS3Factory::seek(off_t off, [[maybe_unused]] int whence)
+{
+    range_generator = RangeGenerator{object_size, range_step, static_cast<size_t>(off)};
+    return off;
+}
+
+std::optional<size_t> ReadBufferS3Factory::getTotalSize()
+{
+    return object_size;
+}
 }
 
 #endif
diff --git a/src/IO/ReadBufferFromS3.h b/src/IO/ReadBufferFromS3.h
index d984fbebe5f..dbba751b321 100644
--- a/src/IO/ReadBufferFromS3.h
+++ b/src/IO/ReadBufferFromS3.h
@@ -5,15 +5,15 @@
 
 #if USE_AWS_S3
 
-#    include <memory>
+#include <memory>
 
-#    include <IO/HTTPCommon.h>
-#    include <IO/ParallelReadBuffer.h>
-#    include <IO/ReadBuffer.h>
-#    include <IO/ReadSettings.h>
-#    include <IO/SeekableReadBuffer.h>
+#include <IO/HTTPCommon.h>
+#include <IO/ParallelReadBuffer.h>
+#include <IO/ReadBuffer.h>
+#include <IO/ReadSettings.h>
+#include <IO/SeekableReadBuffer.h>
 
-#    include <aws/s3/model/GetObjectResult.h>
+#include <aws/s3/model/GetObjectResult.h>
 
 namespace Aws::S3
 {
@@ -104,33 +104,11 @@ public:
         assert(range_step < object_size);
     }
 
-    SeekableReadBufferPtr getReader() override
-    {
-        const auto next_range = range_generator.nextRange();
-        if (!next_range)
-        {
-            return nullptr;
-        }
+    SeekableReadBufferPtr getReader() override;
 
-        auto reader = std::make_shared<ReadBufferFromS3>(
-            client_ptr,
-            bucket,
-            key,
-            s3_max_single_read_retries,
-            read_settings,
-            false /*use_external_buffer*/,
-            next_range->first,
-            next_range->second);
-        return reader;
-    }
+    off_t seek(off_t off, [[maybe_unused]] int whence) override;
 
-    off_t seek(off_t off, [[maybe_unused]] int whence) override
-    {
-        range_generator = RangeGenerator{object_size, range_step, static_cast<size_t>(off)};
-        return off;
-    }
-
-    std::optional<size_t> getTotalSize() override { return object_size; }
+    std::optional<size_t> getTotalSize() override;
 
 private:
     std::shared_ptr<Aws::S3::S3Client> client_ptr;
diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp
index 971ac0d8a73..e706f0a75c1 100644
--- a/src/IO/S3Common.cpp
+++ b/src/IO/S3Common.cpp
@@ -24,6 +24,7 @@
 #    include <aws/core/utils/UUID.h>
 #    include <aws/core/http/HttpClientFactory.h>
 #    include <aws/s3/S3Client.h>
+#    include <aws/s3/model/HeadObjectRequest.h>  // Y_IGNORE
 
 #    include <IO/S3/PocoHTTPClientFactory.h>
 #    include <IO/S3/PocoHTTPClient.h>
@@ -33,8 +34,6 @@
 #    include <base/logger_useful.h>
 
 #    include <fstream>
-#include <aws/s3/S3Client.h>  // Y_IGNORE
-#include <aws/s3/model/HeadObjectRequest.h>  // Y_IGNORE
 
 namespace
 {
diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp
index b80b9ead53f..5625ff16a10 100644
--- a/src/Storages/StorageS3.cpp
+++ b/src/Storages/StorageS3.cpp
@@ -355,7 +355,8 @@ std::unique_ptr<ReadBuffer> StorageS3Source::createS3ReadBuffer(const String & k
             thread_status.attachQuery(running_group);
     };
 
-    return std::make_unique<ParallelReadBuffer>(std::move(factory), &IOThreadPool::get(), download_thread_num, worker_setup, worker_cleanup);
+    return std::make_unique<ParallelReadBuffer>(
+        std::move(factory), &IOThreadPool::get(), download_thread_num, std::move(worker_setup), std::move(worker_cleanup));
 }
 
 String StorageS3Source::getName() const
diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py
index 7364af4d2dd..7fc59d9d9ef 100644
--- a/tests/integration/test_storage_s3/test.py
+++ b/tests/integration/test_storage_s3/test.py
@@ -517,7 +517,7 @@ def test_put_get_with_globs(started_cluster):
         # ("'minio','minio123',",True), Redirect with credentials not working with nginx.
     ],
 )
-def test_multipart_put(started_cluster, maybe_auth, positive):
+def test_multipart(started_cluster, maybe_auth, positive):
     # type: (ClickHouseCluster) -> None
 
     bucket = (

From 6eccd09e2b3c1d193c80e1dfa1f4f3761bb48546 Mon Sep 17 00:00:00 2001
From: taiyang-li <654010905@qq.com>
Date: Thu, 24 Mar 2022 18:04:29 +0800
Subject: [PATCH 026/104] deduce hdfs config path

---
 src/Storages/HDFS/HDFSCommon.cpp | 16 ++++++++++++----
 src/Storages/HDFS/HDFSCommon.h   |  2 +-
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/src/Storages/HDFS/HDFSCommon.cpp b/src/Storages/HDFS/HDFSCommon.cpp
index 85eb16d7de2..b9a8e90f15e 100644
--- a/src/Storages/HDFS/HDFSCommon.cpp
+++ b/src/Storages/HDFS/HDFSCommon.cpp
@@ -2,6 +2,7 @@
 #include <Poco/URI.h>
 #include <boost/algorithm/string/replace.hpp>
 #include <re2/re2.h>
+#include <filesystem>
 
 #if USE_HDFS
 #include <Common/ShellCommand.h>
@@ -25,14 +26,14 @@ const String HDFSBuilderWrapper::CONFIG_PREFIX = "hdfs";
 const String HDFS_URL_REGEXP = "^hdfs://[^/]*/.*";
 
 void HDFSBuilderWrapper::loadFromConfig(const Poco::Util::AbstractConfiguration & config,
-    const String & config_path, bool isUser)
+    const String & prefix, bool isUser)
 {
     Poco::Util::AbstractConfiguration::Keys keys;
 
-    config.keys(config_path, keys);
+    config.keys(prefix, keys);
     for (const auto & key : keys)
     {
-        const String key_path = config_path + "." + key;
+        const String key_path = prefix + "." + key;
 
         String key_name;
         if (key == "hadoop_kerberos_keytab")
@@ -122,9 +123,16 @@ HDFSBuilderWrapper createHDFSBuilder(const String & uri_str, const Poco::Util::A
         throw Exception("Illegal HDFS URI: " + uri.toString(), ErrorCodes::BAD_ARGUMENTS);
 
     // Shall set env LIBHDFS3_CONF *before* HDFSBuilderWrapper construction.
-    const String & libhdfs3_conf = config.getString(HDFSBuilderWrapper::CONFIG_PREFIX + ".libhdfs3_conf", "");
+    String libhdfs3_conf = config.getString(HDFSBuilderWrapper::CONFIG_PREFIX + ".libhdfs3_conf", "");
     if (!libhdfs3_conf.empty())
     {
+        const String config_path = config.getString("config-file", "config.xml");
+        const auto config_dir = std::filesystem::path{config_path}.remove_filename();
+        if (std::filesystem::path{libhdfs3_conf}.is_relative() && std::filesystem::exists(config_dir / libhdfs3_conf))
+            libhdfs3_conf = std::filesystem::absolute(config_dir / libhdfs3_conf);
+        std::cout << "config_path:" << config_path << std::endl;
+        std::cout << "config_dir:" << config_dir << std::endl;
+        std::cout << "libhdfs3_conf:" << libhdfs3_conf << std::endl;
         setenv("LIBHDFS3_CONF", libhdfs3_conf.c_str(), 1);
     }
     HDFSBuilderWrapper builder;
diff --git a/src/Storages/HDFS/HDFSCommon.h b/src/Storages/HDFS/HDFSCommon.h
index 31a8df74652..0523849abe5 100644
--- a/src/Storages/HDFS/HDFSCommon.h
+++ b/src/Storages/HDFS/HDFSCommon.h
@@ -67,7 +67,7 @@ public:
     hdfsBuilder * get() { return hdfs_builder; }
 
 private:
-    void loadFromConfig(const Poco::Util::AbstractConfiguration & config, const String & config_path, bool isUser = false);
+    void loadFromConfig(const Poco::Util::AbstractConfiguration & config, const String & prefix, bool isUser = false);
 
     String getKinitCmd();
 

From d3fbd9b2ad0846728cf6a9eefad9614992c0680d Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Thu, 24 Mar 2022 11:25:28 +0100
Subject: [PATCH 027/104] Fix style

---
 programs/server/Server.cpp       | 2 +-
 src/Common/filesystemHelpers.cpp | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp
index eaa03c0d08a..686ea72f811 100644
--- a/programs/server/Server.cpp
+++ b/programs/server/Server.cpp
@@ -535,7 +535,7 @@ static int readNumber(const String & path)
     return result;
 }
 
-static void sanityChecks(Server* server)
+static void sanityChecks(Server * server)
 {
     std::string data_path = getCanonicalPath(server->config().getString("path", DBMS_DEFAULT_PATH));
     std::string logs_path = server->config().getString("logger.log", "");
diff --git a/src/Common/filesystemHelpers.cpp b/src/Common/filesystemHelpers.cpp
index 8bdd10e3cb1..6c5813ab29c 100644
--- a/src/Common/filesystemHelpers.cpp
+++ b/src/Common/filesystemHelpers.cpp
@@ -83,12 +83,12 @@ String getBlockDeviceId([[maybe_unused]] const String & path)
 #if !defined(__linux__)
 [[noreturn]]
 #endif
-BlockDeviceType getBlockDeviceType([[maybe_unused]] const String & deviceId)
+BlockDeviceType getBlockDeviceType([[maybe_unused]] const String & device_id)
 {
 #if defined(__linux__)
     try
     {
-        ReadBufferFromFile in("/sys/dev/block/" + deviceId + "/queue/rotational");
+        ReadBufferFromFile in("/sys/dev/block/" + device_id + "/queue/rotational");
         int rotational;
         readText(rotational, in);
         return rotational ? BlockDeviceType::ROT : BlockDeviceType::NONROT;
@@ -105,12 +105,12 @@ BlockDeviceType getBlockDeviceType([[maybe_unused]] const String & deviceId)
 #if !defined(__linux__)
 [[noreturn]]
 #endif
-UInt64 getBlockDeviceReadAheadBytes([[maybe_unused]] const String & deviceId)
+UInt64 getBlockDeviceReadAheadBytes([[maybe_unused]] const String & device_id)
 {
 #if defined(__linux__)
     try
     {
-        ReadBufferFromFile in("/sys/dev/block/" + deviceId + "/queue/read_ahead_kb");
+        ReadBufferFromFile in("/sys/dev/block/" + device_id + "/queue/read_ahead_kb");
         int read_ahead_kb;
         readText(read_ahead_kb, in);
         return read_ahead_kb * 1024;

From c0eb9f73a90299965357ff6dfe36ed589d424555 Mon Sep 17 00:00:00 2001
From: taiyang-li <654010905@qq.com>
Date: Thu, 24 Mar 2022 21:17:43 +0800
Subject: [PATCH 028/104] remote std::cout

---
 src/Storages/HDFS/HDFSCommon.cpp | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/Storages/HDFS/HDFSCommon.cpp b/src/Storages/HDFS/HDFSCommon.cpp
index b9a8e90f15e..da8b7c8d63a 100644
--- a/src/Storages/HDFS/HDFSCommon.cpp
+++ b/src/Storages/HDFS/HDFSCommon.cpp
@@ -130,9 +130,7 @@ HDFSBuilderWrapper createHDFSBuilder(const String & uri_str, const Poco::Util::A
         const auto config_dir = std::filesystem::path{config_path}.remove_filename();
         if (std::filesystem::path{libhdfs3_conf}.is_relative() && std::filesystem::exists(config_dir / libhdfs3_conf))
             libhdfs3_conf = std::filesystem::absolute(config_dir / libhdfs3_conf);
-        std::cout << "config_path:" << config_path << std::endl;
-        std::cout << "config_dir:" << config_dir << std::endl;
-        std::cout << "libhdfs3_conf:" << libhdfs3_conf << std::endl;
+
         setenv("LIBHDFS3_CONF", libhdfs3_conf.c_str(), 1);
     }
     HDFSBuilderWrapper builder;

From 041e34807673812419b2aab8ae8fa5009d3f69e1 Mon Sep 17 00:00:00 2001
From: Sergei Trifonov <sergei@clickhouse.com>
Date: Thu, 24 Mar 2022 15:30:10 +0100
Subject: [PATCH 029/104] cosmetic bugfix

---
 src/Common/filesystemHelpers.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Common/filesystemHelpers.h b/src/Common/filesystemHelpers.h
index a0c761a516f..27041a2f0a8 100644
--- a/src/Common/filesystemHelpers.h
+++ b/src/Common/filesystemHelpers.h
@@ -35,13 +35,13 @@ enum class BlockDeviceType
 #if !defined(__linux__)
 [[noreturn]]
 #endif
-BlockDeviceType getBlockDeviceType([[maybe_unused]] const String & deviceId);
+BlockDeviceType getBlockDeviceType([[maybe_unused]] const String & device_id);
 
 // Get size of read-ahead in bytes for specified block device
 #if !defined(__linux__)
 [[noreturn]]
 #endif
-UInt64 getBlockDeviceReadAheadBytes([[maybe_unused]] const String & deviceId);
+UInt64 getBlockDeviceReadAheadBytes([[maybe_unused]] const String & device_id);
 
 /// Returns mount point of filesystem where absolute_path (must exist) is located
 std::filesystem::path getMountPoint(std::filesystem::path absolute_path);

From 85f2ed12e113eafe5a1b8d2e489a7920b31a5456 Mon Sep 17 00:00:00 2001
From: Sergei Trifonov <sergei@clickhouse.com>
Date: Fri, 25 Mar 2022 12:52:50 +0100
Subject: [PATCH 030/104] fix darwin build

---
 base/base/getAvailableMemoryAmount.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/base/base/getAvailableMemoryAmount.cpp b/base/base/getAvailableMemoryAmount.cpp
index a7b12ef4699..d2f794e8952 100644
--- a/base/base/getAvailableMemoryAmount.cpp
+++ b/base/base/getAvailableMemoryAmount.cpp
@@ -27,7 +27,7 @@ uint64_t getAvailableMemoryAmountOrZero()
     unsigned int usermem;
     size_t len = sizeof(usermem);
     static int mib[2] = { CTL_HW, HW_USERMEM };
-    if (sysctl(mib, 2, &usermem, &len, NULL, 0) == 0 && len == sizeof(usermem))
+    if (sysctl(mib, 2, &usermem, &len, nullptr, 0) == 0 && len == sizeof(usermem))
         return usermem;
     else
         return 0;

From fbadd8fc0e78eed814b104fc375bdce3b4bf7ac3 Mon Sep 17 00:00:00 2001
From: taiyang-li <654010905@qq.com>
Date: Fri, 25 Mar 2022 22:28:47 +0800
Subject: [PATCH 031/104] only deduce absolute path of hdfs config when it's
 relative path do not exists

---
 src/Storages/HDFS/HDFSCommon.cpp | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/Storages/HDFS/HDFSCommon.cpp b/src/Storages/HDFS/HDFSCommon.cpp
index da8b7c8d63a..b186808a2db 100644
--- a/src/Storages/HDFS/HDFSCommon.cpp
+++ b/src/Storages/HDFS/HDFSCommon.cpp
@@ -126,10 +126,13 @@ HDFSBuilderWrapper createHDFSBuilder(const String & uri_str, const Poco::Util::A
     String libhdfs3_conf = config.getString(HDFSBuilderWrapper::CONFIG_PREFIX + ".libhdfs3_conf", "");
     if (!libhdfs3_conf.empty())
     {
-        const String config_path = config.getString("config-file", "config.xml");
-        const auto config_dir = std::filesystem::path{config_path}.remove_filename();
-        if (std::filesystem::path{libhdfs3_conf}.is_relative() && std::filesystem::exists(config_dir / libhdfs3_conf))
-            libhdfs3_conf = std::filesystem::absolute(config_dir / libhdfs3_conf);
+        if (std::filesystem::path{libhdfs3_conf}.is_relative() && !std::filesystem::exists(libhdfs3_conf))
+        {
+            const String config_path = config.getString("config-file", "config.xml");
+            const auto config_dir = std::filesystem::path{config_path}.remove_filename();
+            if (std::filesystem::exists(config_dir / libhdfs3_conf))
+                libhdfs3_conf = std::filesystem::absolute(config_dir / libhdfs3_conf);
+        }
 
         setenv("LIBHDFS3_CONF", libhdfs3_conf.c_str(), 1);
     }

From 58a78fba12da9e298611a2ecdd1558eff31bfe53 Mon Sep 17 00:00:00 2001
From: Amos Bird <amosbird@gmail.com>
Date: Mon, 28 Mar 2022 12:58:22 +0800
Subject: [PATCH 032/104] formatString bug fix and refactoring

---
 src/Common/format.h                           | 176 +++++++++++++++++
 src/Functions/concat.cpp                      |  24 ++-
 src/Functions/formatString.cpp                |  24 ++-
 src/Functions/formatString.h                  | 180 +-----------------
 ...245_format_string_stack_overflow.reference |   1 +
 .../02245_format_string_stack_overflow.sql    |   1 +
 6 files changed, 209 insertions(+), 197 deletions(-)
 create mode 100644 src/Common/format.h
 create mode 100644 tests/queries/0_stateless/02245_format_string_stack_overflow.reference
 create mode 100644 tests/queries/0_stateless/02245_format_string_stack_overflow.sql

diff --git a/src/Common/format.h b/src/Common/format.h
new file mode 100644
index 00000000000..234ff90061c
--- /dev/null
+++ b/src/Common/format.h
@@ -0,0 +1,176 @@
+#include <base/types.h>
+#include <Common/Exception.h>
+#include <Common/PODArray.h>
+#include <Common/StringUtils/StringUtils.h>
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int BAD_ARGUMENTS;
+}
+
+namespace Format
+{
+    using IndexPositions = PODArrayWithStackMemory<UInt64, 64>;
+
+    static inline void parseNumber(const String & description, UInt64 l, UInt64 r, UInt64 & res, UInt64 argument_number)
+    {
+        res = 0;
+        for (UInt64 pos = l; pos < r; ++pos)
+        {
+            if (!isNumericASCII(description[pos]))
+                throw Exception("Not a number in curly braces at position " + std::to_string(pos), ErrorCodes::BAD_ARGUMENTS);
+            res = res * 10 + description[pos] - '0';
+            if (res >= argument_number)
+                throw Exception(
+                    "Too big number for arguments, must be at most " + std::to_string(argument_number - 1), ErrorCodes::BAD_ARGUMENTS);
+        }
+    }
+
+    static inline void init(
+        const String & pattern,
+        size_t argument_number,
+        const std::vector<std::optional<String>> & constant_strings,
+        IndexPositions & index_positions,
+        std::vector<String> & substrings)
+    {
+        /// Is current position after open curly brace.
+        bool is_open_curly = false;
+        /// The position of last open token.
+        size_t last_open = -1;
+
+        /// Is formatting in a plain {} token.
+        std::optional<bool> is_plain_numbering;
+        UInt64 index_if_plain = 0;
+
+        /// Left position of adding substrings, just to the closed brace position or the start of the string.
+        /// Invariant --- the start of substring is in this position.
+        size_t start_pos = 0;
+
+        /// A flag to decide whether we should glue the constant strings.
+        bool glue_to_next = false;
+
+        /// Handling double braces (escaping).
+        auto double_brace_removal = [](String & str)
+        {
+            size_t i = 0;
+            bool should_delete = true;
+            str.erase(
+                std::remove_if(
+                    str.begin(),
+                    str.end(),
+                    [&i, &should_delete, &str](char)
+                    {
+                        bool is_double_brace = (str[i] == '{' && str[i + 1] == '{') || (str[i] == '}' && str[i + 1] == '}');
+                        ++i;
+                        if (is_double_brace && should_delete)
+                        {
+                            should_delete = false;
+                            return true;
+                        }
+                        should_delete = true;
+                        return false;
+                    }),
+                str.end());
+        };
+
+        index_positions.emplace_back();
+
+        for (size_t i = 0; i < pattern.size(); ++i)
+        {
+            if (pattern[i] == '{')
+            {
+                /// Escaping handling
+                /// It is safe to access because of null termination
+                if (pattern[i + 1] == '{')
+                {
+                    ++i;
+                    continue;
+                }
+
+                if (is_open_curly)
+                    throw Exception("Two open curly braces without close one at position " + std::to_string(i), ErrorCodes::BAD_ARGUMENTS);
+
+                String to_add = String(pattern.data() + start_pos, i - start_pos);
+                double_brace_removal(to_add);
+                if (!glue_to_next)
+                    substrings.emplace_back(to_add);
+                else
+                    substrings.back() += to_add;
+
+                glue_to_next = false;
+
+                is_open_curly = true;
+                last_open = i + 1;
+            }
+            else if (pattern[i] == '}')
+            {
+                if (pattern[i + 1] == '}')
+                {
+                    ++i;
+                    continue;
+                }
+
+                if (!is_open_curly)
+                    throw Exception("Closed curly brace without open one at position " + std::to_string(i), ErrorCodes::BAD_ARGUMENTS);
+
+                is_open_curly = false;
+
+                if (last_open == i)
+                {
+                    if (is_plain_numbering && !*is_plain_numbering)
+                        throw Exception(
+                            "Cannot switch from automatic field numbering to manual field specification", ErrorCodes::BAD_ARGUMENTS);
+                    is_plain_numbering = true;
+                    if (index_if_plain >= argument_number)
+                        throw Exception("Argument is too big for formatting", ErrorCodes::BAD_ARGUMENTS);
+                    index_positions.back() = index_if_plain++;
+                }
+                else
+                {
+                    if (is_plain_numbering && *is_plain_numbering)
+                        throw Exception(
+                            "Cannot switch from automatic field numbering to manual field specification", ErrorCodes::BAD_ARGUMENTS);
+                    is_plain_numbering = false;
+
+                    UInt64 arg;
+                    parseNumber(pattern, last_open, i, arg, argument_number);
+
+                    if (arg >= argument_number)
+                        throw Exception(
+                            "Argument is too big for formatting. Note that indexing starts from zero", ErrorCodes::BAD_ARGUMENTS);
+
+                    index_positions.back() = arg;
+                }
+
+                if (!constant_strings.empty() && constant_strings[index_positions.back()])
+                {
+                    /// The next string should be glued to last `A {} C`.format('B') -> `A B C`.
+                    glue_to_next = true;
+                    substrings.back() += *constant_strings[index_positions.back()];
+                }
+                else
+                    index_positions.emplace_back(); /// Otherwise we commit arg number and proceed.
+
+                start_pos = i + 1;
+            }
+        }
+
+        if (is_open_curly)
+            throw Exception("Last open curly brace is not closed", ErrorCodes::BAD_ARGUMENTS);
+
+        String to_add = String(pattern.data() + start_pos, pattern.size() - start_pos);
+        double_brace_removal(to_add);
+
+        if (!glue_to_next)
+            substrings.emplace_back(to_add);
+        else
+            substrings.back() += to_add;
+
+        index_positions.pop_back();
+    }
+}
+
+}
diff --git a/src/Functions/concat.cpp b/src/Functions/concat.cpp
index cd83223de3e..e11071265ce 100644
--- a/src/Functions/concat.cpp
+++ b/src/Functions/concat.cpp
@@ -52,23 +52,21 @@ public:
     {
         if (arguments.size() < 2)
             throw Exception(
-                "Number of arguments for function " + getName() + " doesn't match: passed " + toString(arguments.size())
-                    + ", should be at least 2.",
-                ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
-
-        if (arguments.size() > FormatImpl::argument_threshold)
-            throw Exception(
-                "Number of arguments for function " + getName() + " doesn't match: passed " + toString(arguments.size())
-                    + ", should be at most " + std::to_string(FormatImpl::argument_threshold),
-                ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
+                ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
+                "Number of arguments for function {} doesn't match: passed {}, should be at least 2",
+                getName(),
+                arguments.size());
 
         for (const auto arg_idx : collections::range(0, arguments.size()))
         {
             const auto * arg = arguments[arg_idx].get();
             if (!isStringOrFixedString(arg))
-                throw Exception{"Illegal type " + arg->getName() + " of argument " + std::to_string(arg_idx + 1) + " of function "
-                                    + getName(),
-                                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
+                throw Exception(
+                    ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+                    "Illegal type {} of argument {} of function {}",
+                    arg->getName(),
+                    arg_idx + 1,
+                    getName());
         }
 
         return std::make_shared<DataTypeString>();
@@ -125,7 +123,7 @@ private:
         std::vector<const ColumnString::Chars *> data(num_arguments);
         std::vector<const ColumnString::Offsets *> offsets(num_arguments);
         std::vector<size_t> fixed_string_sizes(num_arguments);
-        std::vector<String> constant_strings(num_arguments);
+        std::vector<std::optional<String>> constant_strings(num_arguments);
         bool has_column_string = false;
         bool has_column_fixed_string = false;
         for (size_t i = 0; i < num_arguments; ++i)
diff --git a/src/Functions/formatString.cpp b/src/Functions/formatString.cpp
index e138c072639..ee90a082e5b 100644
--- a/src/Functions/formatString.cpp
+++ b/src/Functions/formatString.cpp
@@ -45,25 +45,23 @@ public:
 
     DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
     {
-        if (arguments.empty())
+        if (arguments.size() < 2)
             throw Exception(
-                "Number of arguments for function " + getName() + " doesn't match: passed " + toString(arguments.size())
-                    + ", should be at least 1",
-                ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
-
-        if (arguments.size() > FormatImpl::argument_threshold)
-            throw Exception(
-                "Number of arguments for function " + getName() + " doesn't match: passed " + toString(arguments.size())
-                    + ", should be at most " + std::to_string(FormatImpl::argument_threshold),
-                ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
+                ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
+                "Number of arguments for function {} doesn't match: passed {}, should be at least 2",
+                getName(),
+                arguments.size());
 
         for (const auto arg_idx : collections::range(0, arguments.size()))
         {
             const auto * arg = arguments[arg_idx].get();
             if (!isStringOrFixedString(arg))
                 throw Exception(
-                    "Illegal type " + arg->getName() + " of argument " + std::to_string(arg_idx + 1) + " of function " + getName(),
-                    ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+                    ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+                    "Illegal type {} of argument {} of function {}",
+                    arg->getName(),
+                    arg_idx + 1,
+                    getName());
         }
 
         return std::make_shared<DataTypeString>();
@@ -84,7 +82,7 @@ public:
         std::vector<const ColumnString::Chars *> data(arguments.size() - 1);
         std::vector<const ColumnString::Offsets *> offsets(arguments.size() - 1);
         std::vector<size_t> fixed_string_sizes(arguments.size() - 1);
-        std::vector<String> constant_strings(arguments.size() - 1);
+        std::vector<std::optional<String>> constant_strings(arguments.size() - 1);
 
         bool has_column_string = false;
         bool has_column_fixed_string = false;
diff --git a/src/Functions/formatString.h b/src/Functions/formatString.h
index 419ecf1c773..3c5057ecb97 100644
--- a/src/Functions/formatString.h
+++ b/src/Functions/formatString.h
@@ -4,8 +4,10 @@
 #include <base/types.h>
 #include <Common/Exception.h>
 #include <Common/StringUtils/StringUtils.h>
+#include <Common/format.h>
 #include <Common/memcpySmall.h>
 
+
 #include <algorithm>
 #include <optional>
 #include <string>
@@ -22,8 +24,6 @@ namespace ErrorCodes
 
 struct FormatImpl
 {
-    static constexpr size_t small_argument_threshold = 1024;
-    static constexpr size_t argument_threshold = std::numeric_limits<UInt32>::max();
     static constexpr size_t right_padding = 15;
 
     template <typename... Args>
@@ -39,165 +39,10 @@ struct FormatImpl
             format<false, false>(std::forward<Args>(args)...);
     }
 
-    static void parseNumber(const String & description, UInt64 l, UInt64 r, UInt64 & res)
-    {
-        res = 0;
-        for (UInt64 pos = l; pos < r; ++pos)
-        {
-            if (!isNumericASCII(description[pos]))
-                throw Exception("Not a number in curly braces at position " + std::to_string(pos), ErrorCodes::BAD_ARGUMENTS);
-            res = res * 10 + description[pos] - '0';
-            if (res >= argument_threshold)
-                throw Exception(
-                    "Too big number for arguments, must be at most " + std::to_string(argument_threshold), ErrorCodes::BAD_ARGUMENTS);
-        }
-    }
-
-    static inline void init(
-        const String & pattern,
-        const std::vector<const ColumnString::Chars *> & data,
-        size_t argument_number,
-        const std::vector<String> & constant_strings,
-        UInt64 * index_positions_ptr,
-        std::vector<String> & substrings)
-    {
-        /// Is current position after open curly brace.
-        bool is_open_curly = false;
-        /// The position of last open token.
-        size_t last_open = -1;
-
-        /// Is formatting in a plain {} token.
-        std::optional<bool> is_plain_numbering;
-        UInt64 index_if_plain = 0;
-
-        /// Left position of adding substrings, just to the closed brace position or the start of the string.
-        /// Invariant --- the start of substring is in this position.
-        size_t start_pos = 0;
-
-        /// A flag to decide whether we should glue the constant strings.
-        bool glue_to_next = false;
-
-        /// Handling double braces (escaping).
-        auto double_brace_removal = [](String & str)
-        {
-            size_t i = 0;
-            bool should_delete = true;
-            str.erase(
-                std::remove_if(
-                    str.begin(),
-                    str.end(),
-                    [&i, &should_delete, &str](char)
-                    {
-                        bool is_double_brace = (str[i] == '{' && str[i + 1] == '{') || (str[i] == '}' && str[i + 1] == '}');
-                        ++i;
-                        if (is_double_brace && should_delete)
-                        {
-                            should_delete = false;
-                            return true;
-                        }
-                        should_delete = true;
-                        return false;
-                    }),
-                str.end());
-        };
-
-        for (size_t i = 0; i < pattern.size(); ++i)
-        {
-            if (pattern[i] == '{')
-            {
-                /// Escaping handling
-                /// It is safe to access because of null termination
-                if (pattern[i + 1] == '{')
-                {
-                    ++i;
-                    continue;
-                }
-
-                if (is_open_curly)
-                    throw Exception("Two open curly braces without close one at position " + std::to_string(i), ErrorCodes::BAD_ARGUMENTS);
-
-                String to_add = String(pattern.data() + start_pos, i - start_pos);
-                double_brace_removal(to_add);
-                if (!glue_to_next)
-                    substrings.emplace_back(to_add);
-                else
-                    substrings.back() += to_add;
-
-                glue_to_next = false;
-
-                is_open_curly = true;
-                last_open = i + 1;
-            }
-            else if (pattern[i] == '}')
-            {
-                if (pattern[i + 1] == '}')
-                {
-                    ++i;
-                    continue;
-                }
-
-                if (!is_open_curly)
-                    throw Exception("Closed curly brace without open one at position " + std::to_string(i), ErrorCodes::BAD_ARGUMENTS);
-
-                is_open_curly = false;
-
-                if (last_open == i)
-                {
-                    if (is_plain_numbering && !*is_plain_numbering)
-                        throw Exception(
-                            "Cannot switch from automatic field numbering to manual field specification", ErrorCodes::BAD_ARGUMENTS);
-                    is_plain_numbering = true;
-                    if (index_if_plain >= argument_number)
-                        throw Exception("Argument is too big for formatting", ErrorCodes::BAD_ARGUMENTS);
-                    *index_positions_ptr = index_if_plain++;
-                }
-                else
-                {
-                    if (is_plain_numbering && *is_plain_numbering)
-                        throw Exception(
-                            "Cannot switch from automatic field numbering to manual field specification", ErrorCodes::BAD_ARGUMENTS);
-                    is_plain_numbering = false;
-
-                    UInt64 arg;
-                    parseNumber(pattern, last_open, i, arg);
-
-                    if (arg >= argument_number)
-                        throw Exception(
-                            "Argument is too big for formatting. Note that indexing starts from zero", ErrorCodes::BAD_ARGUMENTS);
-
-                    *index_positions_ptr = arg;
-                }
-
-                /// Constant string.
-                if (!data[*index_positions_ptr])
-                {
-                    /// The next string should be glued to last `A {} C`.format('B') -> `A B C`.
-                    glue_to_next = true;
-                    substrings.back() += constant_strings[*index_positions_ptr];
-                }
-                else
-                    ++index_positions_ptr; /// Otherwise we commit arg number and proceed.
-
-                start_pos = i + 1;
-            }
-        }
-
-        if (is_open_curly)
-            throw Exception("Last open curly brace is not closed", ErrorCodes::BAD_ARGUMENTS);
-
-        String to_add = String(pattern.data() + start_pos, pattern.size() - start_pos);
-        double_brace_removal(to_add);
-
-        if (!glue_to_next)
-            substrings.emplace_back(to_add);
-        else
-            substrings.back() += to_add;
-    }
-
     /// data for ColumnString and ColumnFixed. Nullptr means no data, it is const string.
     /// offsets for ColumnString, nullptr is an indicator that there is a fixed string rather than ColumnString.
     /// fixed_string_N for savings N to fixed strings.
-    /// constant_strings for constant strings. If data[i] is nullptr, than it is constant string.
+    /// constant_strings for constant strings. If data[i] is nullptr, it is constant string.
     /// res_data is result_data, res_offsets is offset result.
     /// input_rows_count is the number of rows processed.
     /// Precondition: data.size() == offsets.size() == fixed_string_N.size() == constant_strings.size().
@@ -207,29 +52,22 @@ struct FormatImpl
         const std::vector<const ColumnString::Chars *> & data,
         const std::vector<const ColumnString::Offsets *> & offsets,
         [[maybe_unused]] /* Because sometimes !has_column_fixed_string */ const std::vector<size_t> & fixed_string_N,
-        const std::vector<String> & constant_strings,
+        const std::vector<std::optional<String>> & constant_strings,
         ColumnString::Chars & res_data,
         ColumnString::Offsets & res_offsets,
         size_t input_rows_count)
     {
         const size_t argument_number = offsets.size();
 
-        UInt64 small_index_positions_buffer[small_argument_threshold];
-        /// The subsequent indexes of strings we should use. e.g `Hello world {1} {3} {1} {0}` this array will be filled with [1, 3, 1, 0, ... (garbage)] but without constant string indices.
-        UInt64 * index_positions = small_index_positions_buffer;
-
-        std::unique_ptr<UInt64[]> big_index_positions_buffer;
-        if (argument_number > small_argument_threshold)
-        {
-            big_index_positions_buffer.reset(new UInt64[argument_number]);
-            index_positions = big_index_positions_buffer.get();
-        }
+        /// The subsequent indexes of strings we should use. e.g `Hello world {1} {3} {1} {0}` this
+        /// array will be filled with [1, 3, 1, 0] but without constant string indices.
+        Format::IndexPositions index_positions;
 
         /// Vector of substrings of pattern that will be copied to the answer, not string view because of escaping and iterators invalidation.
         /// These are exactly what is between {} tokens, for `Hello {} world {}` we will have [`Hello `, ` world `, ``].
         std::vector<String> substrings;
 
-        init(pattern, data, argument_number, constant_strings, index_positions, substrings);
+        Format::init(pattern, argument_number, constant_strings, index_positions, substrings);
 
         UInt64 final_size = 0;
 
@@ -271,7 +109,7 @@ struct FormatImpl
                 for (size_t j = 1; j < substrings.size(); ++j)
                 {
                     UInt64 arg = index_positions[j - 1];
-                    auto offset_ptr = offsets[arg];
+                    const auto * offset_ptr = offsets[arg];
                     UInt64 arg_offset = 0;
                     UInt64 size = 0;
 
diff --git a/tests/queries/0_stateless/02245_format_string_stack_overflow.reference b/tests/queries/0_stateless/02245_format_string_stack_overflow.reference
new file mode 100644
index 00000000000..6e163a64914
--- /dev/null
+++ b/tests/queries/0_stateless/02245_format_string_stack_overflow.reference
@@ -0,0 +1 @@
+00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
diff --git a/tests/queries/0_stateless/02245_format_string_stack_overflow.sql b/tests/queries/0_stateless/02245_format_string_stack_overflow.sql
new file mode 100644
index 00000000000..1ee3606d3a6
--- /dev/null
+++ b/tests/queries/0_stateless/02245_format_string_stack_overflow.sql
@@ -0,0 +1 @@
+select format('{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}', toString(number)) str from numbers(1);

From d9d826c813cafced31842be0ac3397b1c6292f60 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Mon, 28 Mar 2022 08:19:23 +0000
Subject: [PATCH 033/104] Address PR review

---
 src/Common/RangeGenerator.h |  2 +-
 src/IO/ReadBufferFromS3.cpp | 19 +++++++++++++------
 src/IO/ReadBufferFromS3.h   |  2 +-
 src/Storages/StorageS3.cpp  | 17 ++++-------------
 src/Storages/StorageS3.h    |  2 ++
 5 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/src/Common/RangeGenerator.h b/src/Common/RangeGenerator.h
index 8d3b88936fc..8f097087c8c 100644
--- a/src/Common/RangeGenerator.h
+++ b/src/Common/RangeGenerator.h
@@ -34,7 +34,7 @@ public:
 
         Range range{from, to};
         from = to;
-        return std::move(range);
+        return range;
     }
 
 private:
diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp
index c78f41a0f9d..6616d92b492 100644
--- a/src/IO/ReadBufferFromS3.cpp
+++ b/src/IO/ReadBufferFromS3.cpp
@@ -1,4 +1,5 @@
 #include <Common/config.h>
+#include "IO/S3Common.h"
 
 #if USE_AWS_S3
 
@@ -212,13 +213,14 @@ std::optional<size_t> ReadBufferFromS3::getTotalSize()
     if (file_size)
         return file_size;
 
-    Aws::S3::Model::HeadObjectRequest request;
-    request.SetBucket(bucket);
-    request.SetKey(key);
+    auto object_size = S3::getObjectSize(client_ptr, bucket, key, false);
 
-    auto outcome = client_ptr->HeadObject(request);
-    auto head_result = outcome.GetResultWithOwnership();
-    file_size = head_result.GetContentLength();
+    if (!object_size)
+    {
+        return std::nullopt;
+    }
+
+    file_size = object_size;
     return file_size;
 }
 
@@ -236,6 +238,11 @@ void ReadBufferFromS3::setReadUntilPosition(size_t position)
     }
 }
 
+SeekableReadBuffer::Range ReadBufferFromS3::getRemainingReadRange() const
+{
+    return Range{.left = static_cast<size_t>(offset), .right = read_until_position ? std::optional{read_until_position - 1} : std::nullopt};
+}
+
 std::unique_ptr<ReadBuffer> ReadBufferFromS3::initialize()
 {
     Aws::S3::Model::GetObjectRequest req;
diff --git a/src/IO/ReadBufferFromS3.h b/src/IO/ReadBufferFromS3.h
index dbba751b321..5c9d709d58e 100644
--- a/src/IO/ReadBufferFromS3.h
+++ b/src/IO/ReadBufferFromS3.h
@@ -63,7 +63,7 @@ public:
 
     void setReadUntilPosition(size_t position) override;
 
-    Range getRemainingReadRange() const override { return Range{.left = static_cast<size_t>(offset), .right = read_until_position - 1}; }
+    Range getRemainingReadRange() const override;
 
     size_t getFileOffsetOfBufferEnd() const override { return offset; }
 
diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp
index b40e352e1fb..411f78e44d6 100644
--- a/src/Storages/StorageS3.cpp
+++ b/src/Storages/StorageS3.cpp
@@ -307,7 +307,7 @@ std::unique_ptr<ReadBuffer> StorageS3Source::createS3ReadBuffer(const String & k
     const bool object_too_small = object_size < download_thread_num * download_buffer_size;
     if (!use_parallel_download || object_too_small)
     {
-        LOG_TRACE(&Poco::Logger::get("StorageS3Source"), "Downloading object of size {} from S3 in single thread", object_size);
+        LOG_TRACE(log, "Downloading object of size {} from S3 in single thread", object_size);
         return std::make_unique<ReadBufferFromS3>(client, bucket, key, max_single_read_retries, getContext()->getReadSettings());
     }
 
@@ -315,29 +315,20 @@ std::unique_ptr<ReadBuffer> StorageS3Source::createS3ReadBuffer(const String & k
 
     if (download_buffer_size < DBMS_DEFAULT_BUFFER_SIZE)
     {
-        LOG_WARNING(
-            &Poco::Logger::get("StorageS3Source"),
-            "Downloading buffer {} bytes too small, set at least {} bytes",
-            download_buffer_size,
-            DBMS_DEFAULT_BUFFER_SIZE);
+        LOG_WARNING(log, "Downloading buffer {} bytes too small, set at least {} bytes", download_buffer_size, DBMS_DEFAULT_BUFFER_SIZE);
         download_buffer_size = DBMS_DEFAULT_BUFFER_SIZE;
     }
 
     auto factory = std::make_unique<ReadBufferS3Factory>(
         client, bucket, key, download_buffer_size, object_size, max_single_read_retries, getContext()->getReadSettings());
     LOG_TRACE(
-        &Poco::Logger::get("StorageS3Source"),
-        "Downloading from S3 in {} threads. Object size: {}, Range size: {}.",
-        download_thread_num,
-        object_size,
-        download_buffer_size);
+        log, "Downloading from S3 in {} threads. Object size: {}, Range size: {}.", download_thread_num, object_size, download_buffer_size);
 
     ThreadGroupStatusPtr running_group = CurrentThread::isInitialized() && CurrentThread::get().getThreadGroup()
         ? CurrentThread::get().getThreadGroup()
         : MainThreadStatus::getInstance().getThreadGroup();
 
-    ContextPtr query_context
-        = CurrentThread::isInitialized() ? CurrentThread::get().getQueryContext() : nullptr;
+    ContextPtr query_context = CurrentThread::isInitialized() ? CurrentThread::get().getQueryContext() : nullptr;
 
     auto worker_cleanup = [has_running_group = running_group == nullptr](ThreadStatus & thread_status)
     {
diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h
index 6362e777df6..e67223190ba 100644
--- a/src/Storages/StorageS3.h
+++ b/src/Storages/StorageS3.h
@@ -107,6 +107,8 @@ private:
     std::shared_ptr<IteratorWrapper> file_iterator;
     size_t download_thread_num = 1;
 
+    Poco::Logger * log = &Poco::Logger::get("StorageS3Source");
+
     /// Recreate ReadBuffer and BlockInputStream for each file.
     bool initialize();
 

From 6003a49a877908ac1d9f4651b7d507e6055e9a9b Mon Sep 17 00:00:00 2001
From: Amos Bird <amosbird@gmail.com>
Date: Mon, 28 Mar 2022 17:48:47 +0800
Subject: [PATCH 034/104] Fix style

---
 src/Common/format.h          | 2 ++
 src/Functions/formatString.h | 4 ----
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/Common/format.h b/src/Common/format.h
index 234ff90061c..a9382f247ab 100644
--- a/src/Common/format.h
+++ b/src/Common/format.h
@@ -1,3 +1,5 @@
+#pragma once
+
 #include <base/types.h>
 #include <Common/Exception.h>
 #include <Common/PODArray.h>
diff --git a/src/Functions/formatString.h b/src/Functions/formatString.h
index 3c5057ecb97..44fbdac9378 100644
--- a/src/Functions/formatString.h
+++ b/src/Functions/formatString.h
@@ -17,10 +17,6 @@
 
 namespace DB
 {
-namespace ErrorCodes
-{
-    extern const int BAD_ARGUMENTS;
-}
 
 struct FormatImpl
 {

From 065bc13aad41c6a564e5803d0408a91f640fec43 Mon Sep 17 00:00:00 2001
From: pdai <pdai@users.noreply.github.com>
Date: Mon, 28 Mar 2022 20:58:18 +0800
Subject: [PATCH 035/104] fix typo

---
 .../mergetree-family/versionedcollapsingmergetree.md            | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/zh/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md b/docs/zh/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md
index b81d2206bf4..f5f2c428ea7 100644
--- a/docs/zh/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md
+++ b/docs/zh/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md
@@ -8,7 +8,7 @@ toc_title: "版本折叠MergeTree"
 这个引擎:
 
 -   允许快速写入不断变化的对象状态。
--   删除后台中的旧对象状态。 这显着降低了存储体积。
+-   删除后台中的旧对象状态。 这显著降低了存储体积。
 
 请参阅部分 [崩溃](#table_engines_versionedcollapsingmergetree) 有关详细信息。
 

From 8143526cd276e6c9ba3d85b5dbe37352deb93a7b Mon Sep 17 00:00:00 2001
From: Anton Kozlov <anton@cloudflare.com>
Date: Mon, 28 Mar 2022 16:11:20 +0100
Subject: [PATCH 036/104] [test] Adds clean-up for functional test 02167

Running this test twice now doesn't result in a failure due to the fact that the temporary file is being cleaned up
---
 .../queries/0_stateless/02167_format_from_file_extension.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tests/queries/0_stateless/02167_format_from_file_extension.sh b/tests/queries/0_stateless/02167_format_from_file_extension.sh
index a5902935450..3d9e0a080cc 100755
--- a/tests/queries/0_stateless/02167_format_from_file_extension.sh
+++ b/tests/queries/0_stateless/02167_format_from_file_extension.sh
@@ -5,6 +5,12 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
+function cleanup()
+{
+    user_files_path=$(clickhouse-client --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}')
+    rm $user_files_path/test_02167.*
+}
+trap cleanup EXIT
 
 for format in TSV TabSeparated TSVWithNames TSVWithNamesAndTypes CSV Parquet ORC Arrow JSONEachRow JSONCompactEachRow CustomSeparatedWithNamesAndTypes
 do

From 0e4af89f69e1340d27846f24faf14268ae0c4a10 Mon Sep 17 00:00:00 2001
From: Anton Popov <ap@clickhouse.com>
Date: Mon, 28 Mar 2022 17:21:47 +0000
Subject: [PATCH 037/104] fix reading from type object

---
 src/Storages/MergeTree/IMergeTreeDataPart.cpp |  5 +-
 src/Storages/MergeTree/IMergeTreeDataPart.h   |  2 +-
 .../MergeTree/MergeTreeBlockReadUtils.cpp     | 39 ++++++------
 .../MergeTree/MergeTreeBlockReadUtils.h       |  2 +-
 .../MergeTree/MergeTreeSequentialSource.cpp   |  2 +-
 src/Storages/StorageSnapshot.cpp              | 62 ++++++++++---------
 src/Storages/StorageSnapshot.h                |  4 ++
 .../01825_type_json_missed_values.reference   |  2 +
 .../01825_type_json_missed_values.sql         | 19 ++++++
 9 files changed, 82 insertions(+), 55 deletions(-)
 create mode 100644 tests/queries/0_stateless/01825_type_json_missed_values.reference
 create mode 100644 tests/queries/0_stateless/01825_type_json_missed_values.sql

diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp
index 9028023dc80..6e1d73da427 100644
--- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp
+++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp
@@ -558,9 +558,10 @@ size_t IMergeTreeDataPart::getFileSizeOrZero(const String & file_name) const
     return checksum->second.file_size;
 }
 
-String IMergeTreeDataPart::getColumnNameWithMinimumCompressedSize(const StorageMetadataPtr & metadata_snapshot) const
+String IMergeTreeDataPart::getColumnNameWithMinimumCompressedSize(const StorageSnapshotPtr & storage_snapshot) const
 {
-    const auto & storage_columns = metadata_snapshot->getColumns().getAllPhysical();
+    auto options = GetColumnsOptions(GetColumnsOptions::AllPhysical).withExtendedObjects().withSubcolumns();
+    auto storage_columns = storage_snapshot->getColumns(options);
     MergeTreeData::AlterConversions alter_conversions;
     if (!parent_part)
         alter_conversions = storage.getAlterConversionsForPart(shared_from_this());
diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h
index 85088b10f70..0dfac49bf30 100644
--- a/src/Storages/MergeTree/IMergeTreeDataPart.h
+++ b/src/Storages/MergeTree/IMergeTreeDataPart.h
@@ -164,7 +164,7 @@ public:
 
     /// Returns the name of a column with minimum compressed size (as returned by getColumnSize()).
     /// If no checksums are present returns the name of the first physically existing column.
-    String getColumnNameWithMinimumCompressedSize(const StorageMetadataPtr & metadata_snapshot) const;
+    String getColumnNameWithMinimumCompressedSize(const StorageSnapshotPtr & storage_snapshot) const;
 
     bool contains(const IMergeTreeDataPart & other) const { return info.contains(other.info); }
 
diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp
index 6e72b843f10..97516ea06c6 100644
--- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp
+++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp
@@ -24,7 +24,7 @@ namespace
 /// least one existing (physical) column in part.
 bool injectRequiredColumnsRecursively(
     const String & column_name,
-    const ColumnsDescription & storage_columns,
+    const StorageSnapshotPtr & storage_snapshot,
     const MergeTreeData::AlterConversions & alter_conversions,
     const MergeTreeData::DataPartPtr & part,
     Names & columns,
@@ -36,7 +36,8 @@ bool injectRequiredColumnsRecursively(
     /// stages.
     checkStackSize();
 
-    auto column_in_storage = storage_columns.tryGetColumnOrSubcolumn(GetColumnsOptions::AllPhysical, column_name);
+    auto options = GetColumnsOptions(GetColumnsOptions::AllPhysical).withSubcolumns().withExtendedObjects();
+    auto column_in_storage = storage_snapshot->tryGetColumn(options, column_name);
     if (column_in_storage)
     {
         auto column_name_in_part = column_in_storage->getNameInStorage();
@@ -63,7 +64,8 @@ bool injectRequiredColumnsRecursively(
 
     /// Column doesn't have default value and don't exist in part
     /// don't need to add to required set.
-    const auto column_default = storage_columns.getDefault(column_name);
+    auto metadata_snapshot = storage_snapshot->getMetadataForQuery();
+    const auto column_default = metadata_snapshot->getColumns().getDefault(column_name);
     if (!column_default)
         return false;
 
@@ -73,39 +75,36 @@ bool injectRequiredColumnsRecursively(
 
     bool result = false;
     for (const auto & identifier : identifiers)
-        result |= injectRequiredColumnsRecursively(identifier, storage_columns, alter_conversions, part, columns, required_columns, injected_columns);
+        result |= injectRequiredColumnsRecursively(identifier, storage_snapshot, alter_conversions, part, columns, required_columns, injected_columns);
 
     return result;
 }
 
 }
 
-NameSet injectRequiredColumns(const MergeTreeData & storage, const StorageMetadataPtr & metadata_snapshot, const MergeTreeData::DataPartPtr & part, Names & columns)
+NameSet injectRequiredColumns(
+    const MergeTreeData & storage,
+    const StorageSnapshotPtr & storage_snapshot,
+    const MergeTreeData::DataPartPtr & part,
+    Names & columns)
 {
     NameSet required_columns{std::begin(columns), std::end(columns)};
     NameSet injected_columns;
 
     bool have_at_least_one_physical_column = false;
-
-    const auto & storage_columns = metadata_snapshot->getColumns();
     MergeTreeData::AlterConversions alter_conversions;
     if (!part->isProjectionPart())
         alter_conversions = storage.getAlterConversionsForPart(part);
+
     for (size_t i = 0; i < columns.size(); ++i)
     {
-        auto name_in_storage = Nested::extractTableName(columns[i]);
-        if (storage_columns.has(name_in_storage) && isObject(storage_columns.get(name_in_storage).type))
-        {
-            have_at_least_one_physical_column = true;
-            continue;
-        }
-
         /// We are going to fetch only physical columns
-        if (!storage_columns.hasColumnOrSubcolumn(GetColumnsOptions::AllPhysical, columns[i]))
-            throw Exception("There is no physical column or subcolumn " + columns[i] + " in table.", ErrorCodes::NO_SUCH_COLUMN_IN_TABLE);
+        auto options = GetColumnsOptions(GetColumnsOptions::AllPhysical).withSubcolumns().withExtendedObjects();
+        if (!storage_snapshot->tryGetColumn(options, columns[i]))
+            throw Exception(ErrorCodes::NO_SUCH_COLUMN_IN_TABLE, "There is no physical column or subcolumn {} in table", columns[i]);
 
         have_at_least_one_physical_column |= injectRequiredColumnsRecursively(
-            columns[i], storage_columns, alter_conversions,
+            columns[i], storage_snapshot, alter_conversions,
             part, columns, required_columns, injected_columns);
     }
 
@@ -115,7 +114,7 @@ NameSet injectRequiredColumns(const MergeTreeData & storage, const StorageMetada
         */
     if (!have_at_least_one_physical_column)
     {
-        const auto minimum_size_column_name = part->getColumnNameWithMinimumCompressedSize(metadata_snapshot);
+        const auto minimum_size_column_name = part->getColumnNameWithMinimumCompressedSize(storage_snapshot);
         columns.push_back(minimum_size_column_name);
         /// correctly report added column
         injected_columns.insert(columns.back());
@@ -271,7 +270,7 @@ MergeTreeReadTaskColumns getReadTaskColumns(
     Names pre_column_names;
 
     /// inject columns required for defaults evaluation
-    bool should_reorder = !injectRequiredColumns(storage, storage_snapshot->getMetadataForQuery(), data_part, column_names).empty();
+    bool should_reorder = !injectRequiredColumns(storage, storage_snapshot, data_part, column_names).empty();
 
     if (prewhere_info)
     {
@@ -296,7 +295,7 @@ MergeTreeReadTaskColumns getReadTaskColumns(
         if (pre_column_names.empty())
             pre_column_names.push_back(column_names[0]);
 
-        const auto injected_pre_columns = injectRequiredColumns(storage, storage_snapshot->getMetadataForQuery(), data_part, pre_column_names);
+        const auto injected_pre_columns = injectRequiredColumns(storage, storage_snapshot, data_part, pre_column_names);
         if (!injected_pre_columns.empty())
             should_reorder = true;
 
diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.h b/src/Storages/MergeTree/MergeTreeBlockReadUtils.h
index 2373881f954..b4293b4ce3d 100644
--- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.h
+++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.h
@@ -22,7 +22,7 @@ using MergeTreeBlockSizePredictorPtr = std::shared_ptr<MergeTreeBlockSizePredict
   * so that you can calculate the DEFAULT expression for these columns.
   * Adds them to the `columns`.
   */
-NameSet injectRequiredColumns(const MergeTreeData & storage, const StorageMetadataPtr & metadata_snapshot, const MergeTreeData::DataPartPtr & part, Names & columns);
+NameSet injectRequiredColumns(const MergeTreeData & storage, const StorageSnapshotPtr & storage_snapshot, const MergeTreeData::DataPartPtr & part, Names & columns);
 
 
 /// A batch of work for MergeTreeThreadSelectBlockInputStream
diff --git a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp
index 5dbc59ba2d5..c5a3b7935d9 100644
--- a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp
+++ b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp
@@ -41,7 +41,7 @@ MergeTreeSequentialSource::MergeTreeSequentialSource(
     addTotalRowsApprox(data_part->rows_count);
 
     /// Add columns because we don't want to read empty blocks
-    injectRequiredColumns(storage, storage_snapshot->metadata, data_part, columns_to_read);
+    injectRequiredColumns(storage, storage_snapshot, data_part, columns_to_read);
     NamesAndTypesList columns_for_reader;
     if (take_column_types_from_storage)
     {
diff --git a/src/Storages/StorageSnapshot.cpp b/src/Storages/StorageSnapshot.cpp
index e214afc6a90..8a82c5387c5 100644
--- a/src/Storages/StorageSnapshot.cpp
+++ b/src/Storages/StorageSnapshot.cpp
@@ -51,40 +51,42 @@ NamesAndTypesList StorageSnapshot::getColumns(const GetColumnsOptions & options)
 NamesAndTypesList StorageSnapshot::getColumnsByNames(const GetColumnsOptions & options, const Names & names) const
 {
     NamesAndTypesList res;
-    const auto & columns = getMetadataForQuery()->getColumns();
     for (const auto & name : names)
+        res.push_back(getColumn(options, name));
+    return res;
+}
+
+std::optional<NameAndTypePair> StorageSnapshot::tryGetColumn(const GetColumnsOptions & options, const String & column_name) const
+{
+    const auto & columns = getMetadataForQuery()->getColumns();
+    auto column = columns.tryGetColumn(options, column_name);
+    if (column && (!isObject(column->type) || !options.with_extended_objects))
+        return column;
+
+    if (options.with_extended_objects)
     {
-        auto column = columns.tryGetColumn(options, name);
-        if (column && !isObject(column->type))
-        {
-            res.emplace_back(std::move(*column));
-            continue;
-        }
-
-        if (options.with_extended_objects)
-        {
-            auto object_column = object_columns.tryGetColumn(options, name);
-            if (object_column)
-            {
-                res.emplace_back(std::move(*object_column));
-                continue;
-            }
-        }
-
-        if (options.with_virtuals)
-        {
-            auto it = virtual_columns.find(name);
-            if (it != virtual_columns.end())
-            {
-                res.emplace_back(name, it->second);
-                continue;
-            }
-        }
-
-        throw Exception(ErrorCodes::NO_SUCH_COLUMN_IN_TABLE, "There is no column {} in table", name);
+        auto object_column = object_columns.tryGetColumn(options, column_name);
+        if (object_column)
+            return object_column;
     }
 
-    return res;
+    if (options.with_virtuals)
+    {
+        auto it = virtual_columns.find(column_name);
+        if (it != virtual_columns.end())
+            return NameAndTypePair(column_name, it->second);
+    }
+
+    return {};
+}
+
+NameAndTypePair StorageSnapshot::getColumn(const GetColumnsOptions & options, const String & column_name) const
+{
+    auto column = tryGetColumn(options, column_name);
+    if (!column)
+        throw Exception(ErrorCodes::NO_SUCH_COLUMN_IN_TABLE, "There is no column {} in table", column_name);
+
+    return *column;
 }
 
 Block StorageSnapshot::getSampleBlockForColumns(const Names & column_names) const
diff --git a/src/Storages/StorageSnapshot.h b/src/Storages/StorageSnapshot.h
index 46244827f6c..909f4fd5cab 100644
--- a/src/Storages/StorageSnapshot.h
+++ b/src/Storages/StorageSnapshot.h
@@ -61,6 +61,10 @@ struct StorageSnapshot
     /// Get columns with types according to options only for requested names.
     NamesAndTypesList getColumnsByNames(const GetColumnsOptions & options, const Names & names) const;
 
+    /// Get column with type according to options for requested name.
+    std::optional<NameAndTypePair> tryGetColumn(const GetColumnsOptions & options, const String & column_name) const;
+    NameAndTypePair getColumn(const GetColumnsOptions & options, const String & column_name) const;
+
     /// Block with ordinary + materialized + aliases + virtuals + subcolumns.
     Block getSampleBlockForColumns(const Names & column_names) const;
 
diff --git a/tests/queries/0_stateless/01825_type_json_missed_values.reference b/tests/queries/0_stateless/01825_type_json_missed_values.reference
new file mode 100644
index 00000000000..b480493995b
--- /dev/null
+++ b/tests/queries/0_stateless/01825_type_json_missed_values.reference
@@ -0,0 +1,2 @@
+Tuple(foo Int8, k1 Int8, k2 Int8)
+1
diff --git a/tests/queries/0_stateless/01825_type_json_missed_values.sql b/tests/queries/0_stateless/01825_type_json_missed_values.sql
new file mode 100644
index 00000000000..2420ab7cf34
--- /dev/null
+++ b/tests/queries/0_stateless/01825_type_json_missed_values.sql
@@ -0,0 +1,19 @@
+-- Tags: no-fasttest
+
+DROP TABLE IF EXISTS t_json;
+
+SET allow_experimental_object_type = 1;
+
+CREATE TABLE t_json(id UInt64, obj JSON)
+ENGINE = MergeTree ORDER BY id
+SETTINGS min_bytes_for_wide_part = 0;
+
+SYSTEM STOP MERGES t_json;
+
+INSERT INTO t_json SELECT number, '{"k1": 1, "k2": 2}' FROM numbers(1000000);
+INSERT INTO t_json VALUES (1000001, '{"foo": 1}');
+
+SELECT toTypeName(obj) FROM t_json LIMIT 1;
+SELECT count() FROM t_json WHERE obj.foo != 0;
+
+DROP TABLE IF EXISTS t_json;

From 24c0cf86d44682ea0d8fb967cb17837191381d82 Mon Sep 17 00:00:00 2001
From: Anton Popov <ap@clickhouse.com>
Date: Mon, 28 Mar 2022 19:25:51 +0000
Subject: [PATCH 038/104] add function 'flattenTuple'

---
 src/Functions/flattenTuple.cpp                | 68 +++++++++++++++++++
 .../registerFunctionsMiscellaneous.cpp        |  2 +
 .../0_stateless/02246_flatten_tuple.reference |  4 ++
 .../0_stateless/02246_flatten_tuple.sql       | 22 ++++++
 4 files changed, 96 insertions(+)
 create mode 100644 src/Functions/flattenTuple.cpp
 create mode 100644 tests/queries/0_stateless/02246_flatten_tuple.reference
 create mode 100644 tests/queries/0_stateless/02246_flatten_tuple.sql

diff --git a/src/Functions/flattenTuple.cpp b/src/Functions/flattenTuple.cpp
new file mode 100644
index 00000000000..f5d5b4cb773
--- /dev/null
+++ b/src/Functions/flattenTuple.cpp
@@ -0,0 +1,68 @@
+#include <Functions/IFunction.h>
+#include <Functions/FunctionFactory.h>
+#include <Functions/FunctionHelpers.h>
+#include <DataTypes/DataTypeTuple.h>
+#include <DataTypes/ObjectUtils.h>
+#include <Columns/ColumnTuple.h>
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int ILLEGAL_COLUMN;
+    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+}
+
+namespace
+{
+
+class FunctionFlattenTuple : public IFunction
+{
+public:
+    static constexpr auto name = "flattenTuple";
+    static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionFlattenTuple>(); }
+
+    String getName() const override { return name; }
+    size_t getNumberOfArguments() const override { return 1; }
+    bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override { return true; }
+    bool useDefaultImplementationForConstants() const override { return true; }
+
+    DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
+    {
+        const auto & type = arguments[0];
+        const auto * type_tuple = checkAndGetDataType<DataTypeTuple>(type.get());
+        if (!type_tuple || !type_tuple->haveExplicitNames())
+            throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+                "Argument for function '{}' must be Named Tuple. Got '{}'",
+                getName(), type->getName());
+
+        auto [paths, types] = flattenTuple(type);
+        Names names;
+        names.reserve(paths.size());
+        for (const auto & path : paths)
+            names.push_back(path.getPath());
+
+        return std::make_shared<DataTypeTuple>(types, names);
+    }
+
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    {
+        auto column = arguments.at(0).column;
+        if (!checkAndGetColumn<ColumnTuple>(column.get()))
+            throw Exception(ErrorCodes::ILLEGAL_COLUMN,
+                "Illegal column {} of first argument of function {}. Expected ColumnTuple",
+                column->getName(), getName());
+
+        return flattenTuple(column);
+    }
+};
+
+}
+
+void registerFunctionFlattenTuple(FunctionFactory & factory)
+{
+    factory.registerFunction<FunctionFlattenTuple>();
+}
+
+}
diff --git a/src/Functions/registerFunctionsMiscellaneous.cpp b/src/Functions/registerFunctionsMiscellaneous.cpp
index 76d61ce509a..40a8409cd15 100644
--- a/src/Functions/registerFunctionsMiscellaneous.cpp
+++ b/src/Functions/registerFunctionsMiscellaneous.cpp
@@ -80,6 +80,7 @@ void registerFunctionInitialQueryID(FunctionFactory & factory);
 void registerFunctionServerUUID(FunctionFactory &);
 void registerFunctionZooKeeperSessionUptime(FunctionFactory &);
 void registerFunctionGetOSKernelVersion(FunctionFactory &);
+void registerFunctionFlattenTuple(FunctionFactory &);
 
 #if USE_ICU
 void registerFunctionConvertCharset(FunctionFactory &);
@@ -166,6 +167,7 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory)
     registerFunctionServerUUID(factory);
     registerFunctionZooKeeperSessionUptime(factory);
     registerFunctionGetOSKernelVersion(factory);
+    registerFunctionFlattenTuple(factory);
 
 #if USE_ICU
     registerFunctionConvertCharset(factory);
diff --git a/tests/queries/0_stateless/02246_flatten_tuple.reference b/tests/queries/0_stateless/02246_flatten_tuple.reference
new file mode 100644
index 00000000000..0320150025d
--- /dev/null
+++ b/tests/queries/0_stateless/02246_flatten_tuple.reference
@@ -0,0 +1,4 @@
+([1,2],['a','b'],3,'c',4)	Tuple(`t1.a` Array(UInt32), `t1.s` Array(String), b UInt32, `t2.k` String, `t2.v` UInt32)
+Tuple(id Int8, obj Tuple(k1 Int8, k2 Tuple(k3 String, k4 Nested(k5 Int8, k6 Int8)), some Int8), s String)	Tuple(id Int8, `obj.k1` Int8, `obj.k2.k3` String, `obj.k2.k4.k5` Array(Int8), `obj.k2.k4.k6` Array(Int8), `obj.some` Int8, s String)
+1	1	2	[3,4]	[0,0]	0	foo
+2	0	str	[0]	[55]	42	bar
diff --git a/tests/queries/0_stateless/02246_flatten_tuple.sql b/tests/queries/0_stateless/02246_flatten_tuple.sql
new file mode 100644
index 00000000000..e73f382b261
--- /dev/null
+++ b/tests/queries/0_stateless/02246_flatten_tuple.sql
@@ -0,0 +1,22 @@
+DROP TABLE IF EXISTS t_flatten_tuple;
+DROP TABLE IF EXISTS t_flatten_object;
+
+SET flatten_nested = 0;
+
+CREATE TABLE t_flatten_tuple(t Tuple(t1 Nested(a UInt32, s String), b UInt32, t2 Tuple(k String, v UInt32))) ENGINE = Memory;
+
+INSERT INTO t_flatten_tuple VALUES (([(1, 'a'), (2, 'b')], 3, ('c', 4)));
+
+SELECT flattenTuple(t) AS ft, toTypeName(ft) FROM t_flatten_tuple;
+
+SET allow_experimental_object_type = 1;
+CREATE TABLE t_flatten_object(data JSON) ENGINE = Memory;
+
+INSERT INTO t_flatten_object VALUES ('{"id": 1, "obj": {"k1": 1, "k2": {"k3": 2, "k4": [{"k5": 3}, {"k5": 4}]}}, "s": "foo"}');
+INSERT INTO t_flatten_object VALUES ('{"id": 2, "obj": {"k2": {"k3": "str", "k4": [{"k6": 55}]}, "some": 42}, "s": "bar"}');
+
+SELECT toTypeName(data), toTypeName(flattenTuple(data)) FROM t_flatten_object LIMIT 1;
+SELECT untuple(flattenTuple(data)) FROM t_flatten_object;
+
+DROP TABLE IF EXISTS t_flatten_tuple;
+DROP TABLE IF EXISTS t_flatten_object;

From 36ea9ef11fce7a8d8672d376606ccbe6f6547831 Mon Sep 17 00:00:00 2001
From: avogar <pav.cruglov@yandex.ru>
Date: Mon, 28 Mar 2022 19:52:10 +0000
Subject: [PATCH 039/104] Fix creating materialized view with subquery after
 server restart

---
 src/Databases/DatabaseOnDisk.cpp              |  4 ++++
 .../__init__.py                               |  0
 .../test.py                                   | 23 +++++++++++++++++++
 3 files changed, 27 insertions(+)
 create mode 100755 tests/integration/test_materialized_view_restart_server/__init__.py
 create mode 100755 tests/integration/test_materialized_view_restart_server/test.py

diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp
index 9f56b5f7676..f12009cef1a 100644
--- a/src/Databases/DatabaseOnDisk.cpp
+++ b/src/Databases/DatabaseOnDisk.cpp
@@ -6,6 +6,7 @@
 #include <IO/WriteHelpers.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/InterpreterCreateQuery.h>
+#include <Interpreters/ApplyWithSubqueryVisitor.h>
 #include <Parsers/ASTCreateQuery.h>
 #include <Parsers/ASTFunction.h>
 #include <Parsers/ParserCreateQuery.h>
@@ -55,6 +56,9 @@ std::pair<String, StoragePtr> createTableFromAST(
     ast_create_query.attach = true;
     ast_create_query.setDatabase(database_name);
 
+    if (ast_create_query.select && ast_create_query.isView())
+        ApplyWithSubqueryVisitor().visit(*ast_create_query.select);
+
     if (ast_create_query.as_table_function)
     {
         const auto & factory = TableFunctionFactory::instance();
diff --git a/tests/integration/test_materialized_view_restart_server/__init__.py b/tests/integration/test_materialized_view_restart_server/__init__.py
new file mode 100755
index 00000000000..e69de29bb2d
diff --git a/tests/integration/test_materialized_view_restart_server/test.py b/tests/integration/test_materialized_view_restart_server/test.py
new file mode 100755
index 00000000000..828d8c279bf
--- /dev/null
+++ b/tests/integration/test_materialized_view_restart_server/test.py
@@ -0,0 +1,23 @@
+import pytest
+from helpers.cluster import ClickHouseCluster
+
+cluster = ClickHouseCluster(__file__)
+node = cluster.add_instance("node", stay_alive=True)
+
+
+@pytest.fixture(scope="module")
+def start_cluster():
+    try:
+        cluster.start()
+        yield cluster
+    finally:
+        cluster.shutdown()
+
+
+def test_materialized_view_with_subquery(start_cluster):
+    node.query("create table test (x UInt32) engine=TineLog()")
+    node.query("create materialized view mv engine = TinyLog() as with subquery as (select * from test) select * from subquery")
+    node.restart_clickhouse(kill=True)
+    node.query("insert into test select 1")
+    result = node.query("select * from mv")
+    assert int(result) == 1

From c066593595bd70a16cc0c84bd88778da1d801167 Mon Sep 17 00:00:00 2001
From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com>
Date: Mon, 28 Mar 2022 22:02:28 +0200
Subject: [PATCH 040/104] Update test.py

---
 .../integration/test_materialized_view_restart_server/test.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/integration/test_materialized_view_restart_server/test.py b/tests/integration/test_materialized_view_restart_server/test.py
index 828d8c279bf..4ede9f0f062 100755
--- a/tests/integration/test_materialized_view_restart_server/test.py
+++ b/tests/integration/test_materialized_view_restart_server/test.py
@@ -16,7 +16,9 @@ def start_cluster():
 
 def test_materialized_view_with_subquery(start_cluster):
     node.query("create table test (x UInt32) engine=TineLog()")
-    node.query("create materialized view mv engine = TinyLog() as with subquery as (select * from test) select * from subquery")
+    node.query(
+        "create materialized view mv engine = TinyLog() as with subquery as (select * from test) select * from subquery"
+    )
     node.restart_clickhouse(kill=True)
     node.query("insert into test select 1")
     result = node.query("select * from mv")

From cd3cc60d5dbdf05bcf585bac7b2bd4c71d1177ed Mon Sep 17 00:00:00 2001
From: Anton Popov <ap@clickhouse.com>
Date: Mon, 28 Mar 2022 21:13:20 +0000
Subject: [PATCH 041/104] fix fasttest

---
 tests/queries/0_stateless/02246_flatten_tuple.sql | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/queries/0_stateless/02246_flatten_tuple.sql b/tests/queries/0_stateless/02246_flatten_tuple.sql
index e73f382b261..927e3b9ec80 100644
--- a/tests/queries/0_stateless/02246_flatten_tuple.sql
+++ b/tests/queries/0_stateless/02246_flatten_tuple.sql
@@ -1,3 +1,5 @@
+-- Tags: no-fasttest
+
 DROP TABLE IF EXISTS t_flatten_tuple;
 DROP TABLE IF EXISTS t_flatten_object;
 

From 98fe1ca1ecefed61b3ea4b4416aef4988546ab2c Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Tue, 29 Mar 2022 12:02:21 +0200
Subject: [PATCH 042/104] Fix

---
 programs/local/LocalServer.cpp                |  9 +++----
 ...6_clickhouse_local_drop_database.reference |  0
 .../02246_clickhouse_local_drop_database.sh   | 24 +++++++++++++++++++
 3 files changed, 29 insertions(+), 4 deletions(-)
 create mode 100644 tests/queries/0_stateless/02246_clickhouse_local_drop_database.reference
 create mode 100755 tests/queries/0_stateless/02246_clickhouse_local_drop_database.sh

diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp
index 26d42a11315..bb6684ca137 100644
--- a/programs/local/LocalServer.cpp
+++ b/programs/local/LocalServer.cpp
@@ -184,6 +184,11 @@ void LocalServer::tryInitPath()
     if (path.back() != '/')
         path += '/';
 
+    fs::create_directories(fs::path(path) / "user_defined/");
+    fs::create_directories(fs::path(path) / "data/");
+    fs::create_directories(fs::path(path) / "metadata/");
+    fs::create_directories(fs::path(path) / "metadata_dropped/");
+
     global_context->setPath(path);
 
     global_context->setTemporaryStorage(path + "tmp");
@@ -565,7 +570,6 @@ void LocalServer::processConfig()
         /// Lock path directory before read
         status.emplace(fs::path(path) / "status", StatusFile::write_full_info);
 
-        fs::create_directories(fs::path(path) / "user_defined/");
         LOG_DEBUG(log, "Loading user defined objects from {}", path);
         Poco::File(path + "user_defined/").createDirectories();
         UserDefinedSQLObjectsLoader::instance().loadObjects(global_context);
@@ -573,9 +577,6 @@ void LocalServer::processConfig()
         LOG_DEBUG(log, "Loaded user defined objects.");
 
         LOG_DEBUG(log, "Loading metadata from {}", path);
-        fs::create_directories(fs::path(path) / "data/");
-        fs::create_directories(fs::path(path) / "metadata/");
-
         loadMetadataSystem(global_context);
         attachSystemTablesLocal(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::SYSTEM_DATABASE));
         attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA));
diff --git a/tests/queries/0_stateless/02246_clickhouse_local_drop_database.reference b/tests/queries/0_stateless/02246_clickhouse_local_drop_database.reference
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/queries/0_stateless/02246_clickhouse_local_drop_database.sh b/tests/queries/0_stateless/02246_clickhouse_local_drop_database.sh
new file mode 100755
index 00000000000..00f3904192f
--- /dev/null
+++ b/tests/queries/0_stateless/02246_clickhouse_local_drop_database.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+dir=${CLICKHOUSE_TEST_UNIQUE_NAME}
+[[ -d $dir ]] && rm -r $dir
+mkdir $dir
+$CLICKHOUSE_LOCAL --multiline --multiquery --path $dir -q """
+DROP DATABASE IF EXISTS test;
+CREATE DATABASE IF NOT EXISTS test;
+USE test;
+CREATE TABLE test (id Int32) ENGINE=MergeTree() ORDER BY id;
+DROP DATABASE test;
+"""
+
+$CLICKHOUSE_LOCAL --multiline --multiquery -q """
+DROP DATABASE IF EXISTS test;
+CREATE DATABASE IF NOT EXISTS test;
+USE test;
+CREATE TABLE test (id Int32) ENGINE=MergeTree() ORDER BY id;
+DROP DATABASE test;
+"""

From f0b4a9fd15642240619dddfd4ebd5419b8b54e79 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Tue, 29 Mar 2022 10:56:49 +0000
Subject: [PATCH 043/104] Fix construction of ReadBufferFromS3

---
 src/Disks/IO/ReadBufferFromRemoteFSGather.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp
index 8f91804bbbe..0d50b24f7a5 100644
--- a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp
+++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp
@@ -44,7 +44,7 @@ SeekableReadBufferPtr ReadBufferFromS3Gather::createImplementationBuffer(const S
     {
         return std::make_unique<ReadBufferFromS3>(
             client_ptr, bucket, fs::path(metadata.remote_fs_root_path) / path, max_single_read_retries,
-            settings, /* use_external_buffer */true, read_until_position, /* restricted_seek */true);
+            settings, /* use_external_buffer */true, /* offset */ 0, read_until_position, /* restricted_seek */true);
     };
 
     if (with_cache)

From a2aab8ffacbfa74b9d86bdb938eb04b356152f89 Mon Sep 17 00:00:00 2001
From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com>
Date: Tue, 29 Mar 2022 14:07:19 +0200
Subject: [PATCH 044/104] Fix test

---
 tests/integration/test_materialized_view_restart_server/test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/test_materialized_view_restart_server/test.py b/tests/integration/test_materialized_view_restart_server/test.py
index 4ede9f0f062..bfadec5136b 100755
--- a/tests/integration/test_materialized_view_restart_server/test.py
+++ b/tests/integration/test_materialized_view_restart_server/test.py
@@ -15,7 +15,7 @@ def start_cluster():
 
 
 def test_materialized_view_with_subquery(start_cluster):
-    node.query("create table test (x UInt32) engine=TineLog()")
+    node.query("create table test (x UInt32) engine=TinyLog()")
     node.query(
         "create materialized view mv engine = TinyLog() as with subquery as (select * from test) select * from subquery"
     )

From c59926dcc7e3bb52be7b9b86ead120ae0d8b640d Mon Sep 17 00:00:00 2001
From: Anton Popov <ap@clickhouse.com>
Date: Tue, 29 Mar 2022 12:13:02 +0000
Subject: [PATCH 045/104] fix tests

---
 tests/queries/0_stateless/02246_flatten_tuple.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02246_flatten_tuple.sql b/tests/queries/0_stateless/02246_flatten_tuple.sql
index 927e3b9ec80..139f7a621ef 100644
--- a/tests/queries/0_stateless/02246_flatten_tuple.sql
+++ b/tests/queries/0_stateless/02246_flatten_tuple.sql
@@ -18,7 +18,7 @@ INSERT INTO t_flatten_object VALUES ('{"id": 1, "obj": {"k1": 1, "k2": {"k3": 2,
 INSERT INTO t_flatten_object VALUES ('{"id": 2, "obj": {"k2": {"k3": "str", "k4": [{"k6": 55}]}, "some": 42}, "s": "bar"}');
 
 SELECT toTypeName(data), toTypeName(flattenTuple(data)) FROM t_flatten_object LIMIT 1;
-SELECT untuple(flattenTuple(data)) FROM t_flatten_object;
+SELECT untuple(flattenTuple(data)) FROM t_flatten_object ORDER BY data.id;
 
 DROP TABLE IF EXISTS t_flatten_tuple;
 DROP TABLE IF EXISTS t_flatten_object;

From 4db85e280969a5814290bf81196167114c01d347 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Tue, 29 Mar 2022 14:20:46 +0200
Subject: [PATCH 046/104] Fix build

---
 programs/server/Server.cpp | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp
index 1f1e5749e1f..a9db14665eb 100644
--- a/programs/server/Server.cpp
+++ b/programs/server/Server.cpp
@@ -507,18 +507,10 @@ void checkForUsersNotInMainConfig(
     }
 }
 
-[[noreturn]] void forceShutdown()
-{
-#if defined(THREAD_SANITIZER) && defined(OS_LINUX)
-    /// Thread sanitizer tries to do something on exit that we don't need if we want to exit immediately,
-    /// while connection handling threads are still run.
-    (void)syscall(SYS_exit_group, 0);
-    __builtin_unreachable();
-#else
-    _exit(0);
-#endif
 }
 
+/// Unused in other builds
+#if defined(OS_LINUX)
 static String readString(const String & path)
 {
     ReadBufferFromFile in(path);
@@ -535,6 +527,8 @@ static int readNumber(const String & path)
     return result;
 }
 
+#endif
+
 static void sanityChecks(Server * server)
 {
     std::string data_path = getCanonicalPath(server->config().getString("path", DBMS_DEFAULT_PATH));

From ebd72e433d83dba85b3563af95b6aef015f8a4b8 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Tue, 29 Mar 2022 14:41:47 +0200
Subject: [PATCH 047/104] Fail checks in case of errors

---
 tests/ci/build_report_check.py           | 3 +++
 tests/ci/compatibility_check.py          | 3 +++
 tests/ci/docker_images_check.py          | 3 +++
 tests/ci/docs_check.py                   | 3 +++
 tests/ci/fast_test_check.py              | 2 +-
 tests/ci/integration_test_check.py       | 3 +++
 tests/ci/performance_comparison_check.py | 3 +++
 tests/ci/split_build_smoke_check.py      | 3 +++
 tests/ci/stress_check.py                 | 3 +++
 tests/ci/style_check.py                  | 3 +++
 tests/ci/unit_tests_check.py             | 3 +++
 11 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/tests/ci/build_report_check.py b/tests/ci/build_report_check.py
index 592e905bcb5..50315ad3d78 100644
--- a/tests/ci/build_report_check.py
+++ b/tests/ci/build_report_check.py
@@ -244,3 +244,6 @@ if __name__ == "__main__":
         state=summary_status,
         target_url=url,
     )
+
+    if summary_status == "error":
+        sys.exit(1)
diff --git a/tests/ci/compatibility_check.py b/tests/ci/compatibility_check.py
index d546fabf231..1f408ba0445 100644
--- a/tests/ci/compatibility_check.py
+++ b/tests/ci/compatibility_check.py
@@ -198,3 +198,6 @@ if __name__ == "__main__":
         CHECK_NAME,
     )
     ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events)
+
+    if state == "error":
+        sys.exit(1)
diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py
index 3d0cc468aec..ef6d369bf4f 100644
--- a/tests/ci/docker_images_check.py
+++ b/tests/ci/docker_images_check.py
@@ -7,6 +7,7 @@ import platform
 import shutil
 import subprocess
 import time
+import sys
 from typing import Dict, List, Optional, Set, Tuple, Union
 
 from github import Github
@@ -461,6 +462,8 @@ def main():
     ch_helper = ClickHouseHelper()
     ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events)
 
+    if status == "error":
+        sys.exit(1)
 
 if __name__ == "__main__":
     main()
diff --git a/tests/ci/docs_check.py b/tests/ci/docs_check.py
index 58678b160a4..6e0c7e5d5fb 100644
--- a/tests/ci/docs_check.py
+++ b/tests/ci/docs_check.py
@@ -115,3 +115,6 @@ if __name__ == "__main__":
         NAME,
     )
     ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events)
+
+    if status == "error":
+        sys.exit(1)
diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py
index 64e04594786..40fa5bff091 100644
--- a/tests/ci/fast_test_check.py
+++ b/tests/ci/fast_test_check.py
@@ -208,7 +208,7 @@ if __name__ == "__main__":
 
     # Refuse other checks to run if fast test failed
     if state != "success":
-        if "force-tests" in pr_info.labels:
+        if "force-tests" in pr_info.labels and state != "error":
             print("'force-tests' enabled, will report success")
         else:
             sys.exit(1)
diff --git a/tests/ci/integration_test_check.py b/tests/ci/integration_test_check.py
index 30009414d6e..7481beb2329 100644
--- a/tests/ci/integration_test_check.py
+++ b/tests/ci/integration_test_check.py
@@ -280,3 +280,6 @@ if __name__ == "__main__":
         check_name_with_group,
     )
     ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events)
+
+    if state == "error":
+        sys.exit(1)
diff --git a/tests/ci/performance_comparison_check.py b/tests/ci/performance_comparison_check.py
index c6ce86b2ce1..2e8a0de7100 100644
--- a/tests/ci/performance_comparison_check.py
+++ b/tests/ci/performance_comparison_check.py
@@ -217,3 +217,6 @@ if __name__ == "__main__":
     post_commit_status(
         gh, pr_info.sha, check_name_with_group, message, status, report_url
     )
+
+    if status == "error":
+        sys.exit(1)
diff --git a/tests/ci/split_build_smoke_check.py b/tests/ci/split_build_smoke_check.py
index 41ba6c2fedb..6629c2b3f71 100644
--- a/tests/ci/split_build_smoke_check.py
+++ b/tests/ci/split_build_smoke_check.py
@@ -148,3 +148,6 @@ if __name__ == "__main__":
         CHECK_NAME,
     )
     ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events)
+
+    if state == "error":
+        sys.exit(1)
diff --git a/tests/ci/stress_check.py b/tests/ci/stress_check.py
index 32c181140e2..904ac38bffe 100644
--- a/tests/ci/stress_check.py
+++ b/tests/ci/stress_check.py
@@ -177,3 +177,6 @@ if __name__ == "__main__":
         check_name,
     )
     ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events)
+
+    if state == "error":
+        sys.exit(1)
diff --git a/tests/ci/style_check.py b/tests/ci/style_check.py
index 1b3037217c8..811e6d1d746 100644
--- a/tests/ci/style_check.py
+++ b/tests/ci/style_check.py
@@ -118,3 +118,6 @@ if __name__ == "__main__":
         NAME,
     )
     ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events)
+
+    if state == "error":
+        sys.exit(1)
diff --git a/tests/ci/unit_tests_check.py b/tests/ci/unit_tests_check.py
index 84c4faa822d..f86560562b9 100644
--- a/tests/ci/unit_tests_check.py
+++ b/tests/ci/unit_tests_check.py
@@ -174,3 +174,6 @@ if __name__ == "__main__":
         check_name,
     )
     ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events)
+
+    if state == "error":
+        sys.exit(1)

From 0cc9c8212410e8bad45e5124f7beb6e98e17c8d9 Mon Sep 17 00:00:00 2001
From: Dmitry Novik <n0vik@clickhouse.com>
Date: Tue, 29 Mar 2022 13:09:37 +0000
Subject: [PATCH 048/104] Try to fix test_global_overcommit_tracker flakyness

---
 tests/integration/test_global_overcommit_tracker/test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/test_global_overcommit_tracker/test.py b/tests/integration/test_global_overcommit_tracker/test.py
index c2f3a22915f..cacc447be1a 100644
--- a/tests/integration/test_global_overcommit_tracker/test.py
+++ b/tests/integration/test_global_overcommit_tracker/test.py
@@ -30,7 +30,7 @@ def test_overcommited_is_killed():
 
     responses_A = list()
     responses_B = list()
-    for _ in range(100):
+    for _ in range(500):
         responses_A.append(node.get_query_request(TEST_QUERY_A, user="A"))
         responses_B.append(node.get_query_request(TEST_QUERY_B, user="B"))
 

From 9990abb76afad4ac7c65895976d777ded0ca9d1c Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Mon, 28 Mar 2022 09:48:17 +0000
Subject: [PATCH 049/104] Use compile-time check for Exception messages, fix
 wrong messages

---
 src/Common/Exception.h                        | 22 +++++++++----------
 src/Databases/DatabaseReplicatedWorker.cpp    |  8 +++++--
 src/Dictionaries/XDBCDictionarySource.cpp     |  2 +-
 src/Disks/IO/CachedReadBufferFromRemoteFS.cpp |  9 ++++++--
 src/Formats/FormatSchemaInfo.cpp              |  9 +++++---
 src/Functions/FunctionsStringArray.h          |  2 +-
 src/Functions/IFunction.cpp                   |  9 +++++---
 src/Functions/castOrDefault.cpp               |  2 +-
 src/Functions/dateName.cpp                    |  2 +-
 src/Interpreters/CatBoostModel.cpp            |  1 +
 src/Interpreters/DatabaseCatalog.cpp          |  8 +++----
 src/Interpreters/executeDDLQueryOnCluster.cpp | 19 ++++++++++------
 .../Formats/Impl/MsgPackRowInputFormat.cpp    |  4 ++--
 13 files changed, 59 insertions(+), 38 deletions(-)

diff --git a/src/Common/Exception.h b/src/Common/Exception.h
index 0bf89e7a447..c9bb8cc9f12 100644
--- a/src/Common/Exception.h
+++ b/src/Common/Exception.h
@@ -35,10 +35,10 @@ public:
     {}
 
     // Format message with fmt::format, like the logging functions.
-    template <typename ...Args>
-    Exception(int code, const std::string & fmt, Args&&... args)
-        : Exception(fmt::format(fmt::runtime(fmt), std::forward<Args>(args)...), code)
-    {}
+    template <typename... Args>
+    Exception(int code, fmt::format_string<Args...> fmt, Args &&... args) : Exception(fmt::format(fmt, std::forward<Args>(args)...), code)
+    {
+    }
 
     struct CreateFromPocoTag {};
     struct CreateFromSTDTag {};
@@ -52,10 +52,10 @@ public:
     const char * what() const throw() override { return message().data(); }
 
     /// Add something to the existing message.
-    template <typename ...Args>
-    void addMessage(const std::string& format, Args&&... args)
+    template <typename... Args>
+    void addMessage(fmt::format_string<Args...> format, Args &&... args)
     {
-        extendedMessage(fmt::format(fmt::runtime(format), std::forward<Args>(args)...));
+        extendedMessage(fmt::format(format, std::forward<Args>(args)...));
     }
 
     void addMessage(const std::string& message)
@@ -117,10 +117,10 @@ public:
     ParsingException(int code, const std::string & message);
 
     // Format message with fmt::format, like the logging functions.
-    template <typename ...Args>
-    ParsingException(int code, const std::string & fmt, Args&&... args)
-        : Exception(fmt::format(fmt::runtime(fmt), std::forward<Args>(args)...), code)
-    {}
+    template <typename... Args>
+    ParsingException(int code, fmt::format_string<Args...> fmt, Args &&... args) : Exception(code, fmt, std::forward<Args>(args)...)
+    {
+    }
 
 
     std::string displayText() const
diff --git a/src/Databases/DatabaseReplicatedWorker.cpp b/src/Databases/DatabaseReplicatedWorker.cpp
index 365a5d02816..b45cfb16362 100644
--- a/src/Databases/DatabaseReplicatedWorker.cpp
+++ b/src/Databases/DatabaseReplicatedWorker.cpp
@@ -179,8 +179,12 @@ String DatabaseReplicatedDDLWorker::tryEnqueueAndExecuteEntry(DDLLogEntry & entr
 
     if (!task->was_executed)
     {
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "Entry {} was executed, but was not committed: code {}: {}",
-                        task->execution_status.code, task->execution_status.message);
+        throw Exception(
+            ErrorCodes::LOGICAL_ERROR,
+            "Entry {} was executed, but was not committed: code {}: {}",
+            task->entry_name,
+            task->execution_status.code,
+            task->execution_status.message);
     }
 
     try_node->setAlreadyRemoved();
diff --git a/src/Dictionaries/XDBCDictionarySource.cpp b/src/Dictionaries/XDBCDictionarySource.cpp
index f08abcdc516..764a7072ca0 100644
--- a/src/Dictionaries/XDBCDictionarySource.cpp
+++ b/src/Dictionaries/XDBCDictionarySource.cpp
@@ -50,7 +50,7 @@ namespace
         {
             if (!qualified_name.database.empty())
                 throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
-                    "Dictionary source of type {} specifies a schema but schema is not supported by {}-driver",
+                    "Dictionary source specifies a schema but schema is not supported by {}-driver",
                     bridge_.getName());
         }
 
diff --git a/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp b/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp
index 5cab2cb2995..4b614cd10e0 100644
--- a/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp
+++ b/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp
@@ -392,8 +392,13 @@ void CachedReadBufferFromRemoteFS::predownload(FileSegmentPtr & file_segment)
                 if (bytes_to_predownload)
                     throw Exception(
                         ErrorCodes::LOGICAL_ERROR,
-                        "Failed to predownload remaining {} bytes. Current file segment: {}, current download offset: {}, expected: {}, eof: {}",
-                        file_segment->range().toString(), file_segment->getDownloadOffset(), file_offset_of_buffer_end, implementation_buffer->eof());
+                        "Failed to predownload remaining {} bytes. Current file segment: {}, current download offset: {}, expected: {}, "
+                        "eof: {}",
+                        bytes_to_predownload,
+                        file_segment->range().toString(),
+                        file_segment->getDownloadOffset(),
+                        file_offset_of_buffer_end,
+                        implementation_buffer->eof());
 
                 auto result = implementation_buffer->hasPendingData();
 
diff --git a/src/Formats/FormatSchemaInfo.cpp b/src/Formats/FormatSchemaInfo.cpp
index 24c8dfc14f2..c1157be6e7a 100644
--- a/src/Formats/FormatSchemaInfo.cpp
+++ b/src/Formats/FormatSchemaInfo.cpp
@@ -85,9 +85,12 @@ FormatSchemaInfo::FormatSchemaInfo(const String & format_schema, const String &
     else if (path.has_parent_path() && !fs::weakly_canonical(default_schema_directory_path / path).string().starts_with(fs::weakly_canonical(default_schema_directory_path).string()))
     {
         if (is_server)
-            throw Exception(ErrorCodes::BAD_ARGUMENTS,
-                            "Path in the 'format_schema' setting shouldn't go outside the 'format_schema_path' directory: {} ({} not in {})",
-                            path.string());
+            throw Exception(
+                ErrorCodes::BAD_ARGUMENTS,
+                "Path in the 'format_schema' setting shouldn't go outside the 'format_schema_path' directory: {} ({} not in {})",
+                default_schema_directory(),
+                path.string(),
+                default_schema_directory());
         path = default_schema_directory_path / path;
         schema_path = path.filename();
         schema_directory = path.parent_path() / "";
diff --git a/src/Functions/FunctionsStringArray.h b/src/Functions/FunctionsStringArray.h
index a1256598f1b..6b3adf46ff5 100644
--- a/src/Functions/FunctionsStringArray.h
+++ b/src/Functions/FunctionsStringArray.h
@@ -259,7 +259,7 @@ public:
             throw Exception(
                 ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
                 "Function '{}' needs at least 2 arguments, at most 3 arguments; passed {}.",
-                arguments.size());
+                name, arguments.size());
 
         if (!isString(arguments[0]))
             throw Exception("Illegal type " + arguments[0]->getName() + " of first argument of function " + getName() + ". Must be String.",
diff --git a/src/Functions/IFunction.cpp b/src/Functions/IFunction.cpp
index cfb4e12a025..19638c78daf 100644
--- a/src/Functions/IFunction.cpp
+++ b/src/Functions/IFunction.cpp
@@ -181,9 +181,12 @@ ColumnPtr IExecutableFunction::defaultImplementationForNulls(
         // Default implementation for nulls returns null result for null arguments,
         // so the result type must be nullable.
         if (!result_type->isNullable())
-            throw Exception(ErrorCodes::LOGICAL_ERROR,
-                            "Function {} with Null argument and default implementation for Nulls "
-                            "is expected to return Nullable result, got {}", result_type->getName());
+            throw Exception(
+                ErrorCodes::LOGICAL_ERROR,
+                "Function {} with Null argument and default implementation for Nulls "
+                "is expected to return Nullable result, got {}",
+                getName(),
+                result_type->getName());
 
         return result_type->createColumnConstWithDefaultValue(input_rows_count);
     }
diff --git a/src/Functions/castOrDefault.cpp b/src/Functions/castOrDefault.cpp
index 628ac57f34d..f7b93ec2e83 100644
--- a/src/Functions/castOrDefault.cpp
+++ b/src/Functions/castOrDefault.cpp
@@ -231,7 +231,7 @@ private:
             {
                 throw Exception(ErrorCodes::BAD_ARGUMENTS,
                     "Function {} decimal scale should have native UInt type. Actual {}",
-                    scale_argument.type->getName());
+                    getName(), scale_argument.type->getName());
             }
 
             scale = arguments[additional_argument_index].column->getUInt(0);
diff --git a/src/Functions/dateName.cpp b/src/Functions/dateName.cpp
index eef9bc3955b..91ea8995777 100644
--- a/src/Functions/dateName.cpp
+++ b/src/Functions/dateName.cpp
@@ -112,7 +112,7 @@ public:
             || (res = executeType<DataTypeDateTime64>(arguments, result_type))))
             throw Exception(
                 ErrorCodes::ILLEGAL_COLUMN,
-                "Illegal column {} of function {], must be Date or DateTime.",
+                "Illegal column {} of function {}, must be Date or DateTime.",
                 arguments[1].column->getName(),
                 getName());
 
diff --git a/src/Interpreters/CatBoostModel.cpp b/src/Interpreters/CatBoostModel.cpp
index cffaa81c4f0..d5803ed9e36 100644
--- a/src/Interpreters/CatBoostModel.cpp
+++ b/src/Interpreters/CatBoostModel.cpp
@@ -169,6 +169,7 @@ public:
         if (columns.size() != float_features_count + cat_features_count)
             throw Exception(ErrorCodes::BAD_ARGUMENTS,
                 "Number of columns is different with number of features: columns size {} float features size {} + cat features size {}",
+                columns.size(),
                 float_features_count,
                 cat_features_count);
 
diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp
index 360a5d430e0..2f51d942403 100644
--- a/src/Interpreters/DatabaseCatalog.cpp
+++ b/src/Interpreters/DatabaseCatalog.cpp
@@ -233,7 +233,7 @@ DatabaseAndTable DatabaseCatalog::getTableImpl(
         {
             assert(!db_and_table.first && !db_and_table.second);
             if (exception)
-                exception->emplace(ErrorCodes::UNKNOWN_TABLE, "Table {} doesn't exist", table_id.getNameForLogs());
+                exception->emplace(fmt::format("Table {} doesn't exist", table_id.getNameForLogs()), ErrorCodes::UNKNOWN_TABLE);
             return {};
         }
 
@@ -263,7 +263,7 @@ DatabaseAndTable DatabaseCatalog::getTableImpl(
         /// If table_id has no UUID, then the name of database was specified by user and table_id was not resolved through context.
         /// Do not allow access to TEMPORARY_DATABASE because it contains all temporary tables of all contexts and users.
         if (exception)
-            exception->emplace(ErrorCodes::DATABASE_ACCESS_DENIED, "Direct access to `{}` database is not allowed", String(TEMPORARY_DATABASE));
+            exception->emplace(fmt::format("Direct access to `{}` database is not allowed", TEMPORARY_DATABASE), ErrorCodes::DATABASE_ACCESS_DENIED);
         return {};
     }
 
@@ -274,7 +274,7 @@ DatabaseAndTable DatabaseCatalog::getTableImpl(
         if (databases.end() == it)
         {
             if (exception)
-                exception->emplace(ErrorCodes::UNKNOWN_DATABASE, "Database {} doesn't exist", backQuoteIfNeed(table_id.getDatabaseName()));
+                exception->emplace(fmt::format("Database {} doesn't exist", backQuoteIfNeed(table_id.getDatabaseName())), ErrorCodes::UNKNOWN_DATABASE);
             return {};
         }
         database = it->second;
@@ -282,7 +282,7 @@ DatabaseAndTable DatabaseCatalog::getTableImpl(
 
     auto table = database->tryGetTable(table_id.table_name, context_);
     if (!table && exception)
-            exception->emplace(ErrorCodes::UNKNOWN_TABLE, "Table {} doesn't exist", table_id.getNameForLogs());
+            exception->emplace(fmt::format("Table {} doesn't exist", table_id.getNameForLogs()), ErrorCodes::UNKNOWN_TABLE);
     if (!table)
         database = nullptr;
 
diff --git a/src/Interpreters/executeDDLQueryOnCluster.cpp b/src/Interpreters/executeDDLQueryOnCluster.cpp
index ce00676b2ed..f0279bafca2 100644
--- a/src/Interpreters/executeDDLQueryOnCluster.cpp
+++ b/src/Interpreters/executeDDLQueryOnCluster.cpp
@@ -320,12 +320,13 @@ Chunk DDLQueryStatusSource::generate()
             if (throw_on_timeout)
             {
                 if (!first_exception)
-                    first_exception = std::make_unique<Exception>(ErrorCodes::TIMEOUT_EXCEEDED, msg_format,
-                        node_path, timeout_seconds, num_unfinished_hosts, num_active_hosts);
+                    first_exception = std::make_unique<Exception>(
+                        fmt::format(msg_format, node_path, timeout_seconds, num_unfinished_hosts, num_active_hosts),
+                        ErrorCodes::TIMEOUT_EXCEEDED);
                 return {};
             }
 
-            LOG_INFO(log, fmt::runtime(msg_format), node_path, timeout_seconds, num_unfinished_hosts, num_active_hosts);
+            LOG_INFO(log, msg_format, node_path, timeout_seconds, num_unfinished_hosts, num_active_hosts);
 
             NameSet unfinished_hosts = waiting_hosts;
             for (const auto & host_id : finished_hosts)
@@ -358,9 +359,12 @@ Chunk DDLQueryStatusSource::generate()
             /// Paradoxically, this exception will be throw even in case of "never_throw" mode.
 
             if (!first_exception)
-                first_exception = std::make_unique<Exception>(ErrorCodes::UNFINISHED,
-                    "Cannot provide query execution status. The query's node {} has been deleted by the cleaner"
-                    " since it was finished (or its lifetime is expired)", node_path);
+                first_exception = std::make_unique<Exception>(
+                    fmt::format(
+                        "Cannot provide query execution status. The query's node {} has been deleted by the cleaner"
+                        " since it was finished (or its lifetime is expired)",
+                        node_path),
+                    ErrorCodes::UNFINISHED);
             return {};
         }
 
@@ -386,7 +390,8 @@ Chunk DDLQueryStatusSource::generate()
             if (status.code != 0 && !first_exception
                 && context->getSettingsRef().distributed_ddl_output_mode != DistributedDDLOutputMode::NEVER_THROW)
             {
-                first_exception = std::make_unique<Exception>(status.code, "There was an error on [{}:{}]: {}", host, port, status.message);
+                first_exception = std::make_unique<Exception>(
+                    fmt::format("There was an error on [{}:{}]: {}", host, port, status.message), status.code);
             }
 
             ++num_hosts_finished;
diff --git a/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp b/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp
index 607e6f36767..722cedbab30 100644
--- a/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp
+++ b/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp
@@ -359,7 +359,7 @@ bool MsgPackVisitor::visit_ext(const char * value, uint32_t size)
         return true;
     }
 
-    throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unsupported MsgPack extension type: {%x}", type);
+    throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unsupported MsgPack extension type: {:x}", type);
 }
 
 void MsgPackVisitor::parse_error(size_t, size_t) // NOLINT
@@ -498,7 +498,7 @@ DataTypePtr MsgPackSchemaReader::getDataType(const msgpack::object & object)
             msgpack::object_ext object_ext = object.via.ext;
             if (object_ext.type() == int8_t(MsgPackExtensionTypes::UUIDType))
                 return std::make_shared<DataTypeUUID>();
-            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Msgpack extension type {%x} is not supported", object_ext.type());
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Msgpack extension type {:x} is not supported", object_ext.type());
         }
     }
     __builtin_unreachable();

From ef484547bf9927c222c588113487a1534e61e39a Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Tue, 29 Mar 2022 10:14:38 +0000
Subject: [PATCH 050/104] Fix GCC build

---
 src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp
index a72d7b12a4f..ebfeaed8346 100644
--- a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp
+++ b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp
@@ -1,10 +1,13 @@
+// Needs to go first because its partial specialization of fmt::formatter
+// should be defined before any instantiation
+#include <fmt/ostream.h>
+
 #include <Storages/Kafka/ReadBufferFromKafkaConsumer.h>
 
 #include <base/logger_useful.h>
 
 #include <cppkafka/cppkafka.h>
 #include <boost/algorithm/string/join.hpp>
-#include <fmt/ostream.h>
 #include <algorithm>
 
 namespace DB

From c37945b2c81fd3f042331faf693d5df20b813cc1 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Tue, 29 Mar 2022 15:24:29 +0200
Subject: [PATCH 051/104] run stateful tests in parallel

---
 docker/test/stateful/run.sh               | 2 +-
 tests/queries/1_stateful/00140_rename.sql | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/test/stateful/run.sh b/docker/test/stateful/run.sh
index 77dc61e6cd0..861e17848a4 100755
--- a/docker/test/stateful/run.sh
+++ b/docker/test/stateful/run.sh
@@ -115,7 +115,7 @@ function run_tests()
     fi
 
     set +e
-    clickhouse-test --testname --shard --zookeeper --check-zookeeper-session --no-stateless --hung-check --print-time \
+    clickhouse-test -j 2 --testname --shard --zookeeper --check-zookeeper-session --no-stateless --hung-check --print-time \
         --skip 00168_parallel_processing_on_replicas "${ADDITIONAL_OPTIONS[@]}" \
         "$SKIP_TESTS_OPTION" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
 
diff --git a/tests/queries/1_stateful/00140_rename.sql b/tests/queries/1_stateful/00140_rename.sql
index 9702d00a4d5..88671d51027 100644
--- a/tests/queries/1_stateful/00140_rename.sql
+++ b/tests/queries/1_stateful/00140_rename.sql
@@ -1,4 +1,4 @@
--- Tags: no-replicated-database
+-- Tags: no-replicated-database, no-parallel
 -- Tag no-replicated-database: Does not support renaming of multiple tables in single query
 
 RENAME TABLE test.hits TO test.visits_tmp, test.visits TO test.hits, test.visits_tmp TO test.visits;

From c9feb7f7515cf2711877fa40844a1b55b4e03267 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Tue, 29 Mar 2022 13:40:14 +0000
Subject: [PATCH 052/104] Format test file

---
 tests/integration/test_allowed_url_from_config/test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/test_allowed_url_from_config/test.py b/tests/integration/test_allowed_url_from_config/test.py
index 13f3929902d..01a2a500ebf 100644
--- a/tests/integration/test_allowed_url_from_config/test.py
+++ b/tests/integration/test_allowed_url_from_config/test.py
@@ -280,4 +280,4 @@ def test_HDFS(start_cluster):
 
 def test_schema_inference(start_cluster):
     error = node7.query_and_get_error("desc url('http://test.com`, 'TSVRaw'')")
-    assert(error.find('ReadWriteBufferFromHTTPBase') == -1)
+    assert error.find("ReadWriteBufferFromHTTPBase") == -1

From d0f01516db0c2d403f9d45898251bd417e78ad54 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Tue, 29 Mar 2022 15:48:57 +0200
Subject: [PATCH 053/104] Resurrect automatic labelling

---
 .github/workflows/master.yml         |  10 -
 tests/ci/cherry_pick_utils/parser.py |   8 -
 tests/ci/commit_status_helper.py     |   5 +
 tests/ci/run_check.py                |  34 +-
 utils/github/__init__.py             |   1 -
 utils/github/backport.py             | 185 ----------
 utils/github/cherrypick.py           | 323 ------------------
 utils/github/local.py                | 108 ------
 utils/github/parser.py               |  64 ----
 utils/github/query.py                | 492 ---------------------------
 10 files changed, 31 insertions(+), 1199 deletions(-)
 delete mode 100644 utils/github/__init__.py
 delete mode 100644 utils/github/backport.py
 delete mode 100644 utils/github/cherrypick.py
 delete mode 100644 utils/github/local.py
 delete mode 100644 utils/github/parser.py
 delete mode 100644 utils/github/query.py

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 73cffc1d5d3..cfa95b84ee5 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -149,7 +149,6 @@ jobs:
           sudo rm -fr "$TEMP_PATH"
   SplitBuildSmokeTest:
     needs: [BuilderDebSplitted]
-    if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
     runs-on: [self-hosted, style-checker]
     steps:
       - name: Set envs
@@ -316,7 +315,6 @@ jobs:
           sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
   BuilderBinRelease:
     needs: [DockerHubPush]
-    if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
     runs-on: [self-hosted, builder]
     steps:
       - name: Set envs
@@ -362,7 +360,6 @@ jobs:
           sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
   BuilderBinGCC:
     needs: [DockerHubPush]
-    if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
     runs-on: [self-hosted, builder]
     steps:
       - name: Set envs
@@ -636,7 +633,6 @@ jobs:
 ##########################################################################################
   BuilderDebSplitted:
     needs: [DockerHubPush]
-    if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
     runs-on: [self-hosted, builder]
     steps:
       - name: Set envs
@@ -682,7 +678,6 @@ jobs:
           sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
   BuilderBinTidy:
     needs: [DockerHubPush]
-    if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
     runs-on: [self-hosted, builder]
     steps:
       - name: Set envs
@@ -728,7 +723,6 @@ jobs:
           sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
   BuilderBinDarwin:
     needs: [DockerHubPush]
-    if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
     runs-on: [self-hosted, builder]
     steps:
       - name: Set envs
@@ -774,7 +768,6 @@ jobs:
           sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
   BuilderBinAarch64:
     needs: [DockerHubPush]
-    if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
     runs-on: [self-hosted, builder]
     steps:
       - name: Set envs
@@ -820,7 +813,6 @@ jobs:
           sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
   BuilderBinFreeBSD:
     needs: [DockerHubPush]
-    if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
     runs-on: [self-hosted, builder]
     steps:
       - name: Set envs
@@ -866,7 +858,6 @@ jobs:
           sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
   BuilderBinDarwinAarch64:
     needs: [DockerHubPush]
-    if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
     runs-on: [self-hosted, builder]
     steps:
       - name: Set envs
@@ -912,7 +903,6 @@ jobs:
           sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
   BuilderBinPPC64:
     needs: [DockerHubPush]
-    if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
     runs-on: [self-hosted, builder]
     steps:
       - name: Set envs
diff --git a/tests/ci/cherry_pick_utils/parser.py b/tests/ci/cherry_pick_utils/parser.py
index d8348e6d964..29c05e5328f 100644
--- a/tests/ci/cherry_pick_utils/parser.py
+++ b/tests/ci/cherry_pick_utils/parser.py
@@ -20,8 +20,6 @@ class Description:
 
     def __init__(self, pull_request):
         self.label_name = str()
-        self.legal = False
-
         self._parse(pull_request["bodyText"])
 
     def _parse(self, text):
@@ -39,12 +37,6 @@ class Description:
                 category = stripped
                 next_category = False
 
-            if (
-                stripped
-                == "I hereby agree to the terms of the CLA available at: https://yandex.ru/legal/cla/?lang=en"
-            ):
-                self.legal = True
-
             category_headers = (
                 "Category (leave one):",
                 "Changelog category (leave one):",
diff --git a/tests/ci/commit_status_helper.py b/tests/ci/commit_status_helper.py
index e379c9a2254..53dfb145b44 100644
--- a/tests/ci/commit_status_helper.py
+++ b/tests/ci/commit_status_helper.py
@@ -59,3 +59,8 @@ def post_commit_status_to_file(file_path, description, state, report_url):
     with open(file_path, "w", encoding="utf-8") as f:
         out = csv.writer(f, delimiter="\t")
         out.writerow([state, report_url, description])
+
+def post_label(gh, pr_info, label_name):
+    repo = gh.get_repo(GITHUB_REPOSITORY)
+    pull_request = repo.get_pull(pr_info.number)
+    pull_request.add_to_labels(label_name)
diff --git a/tests/ci/run_check.py b/tests/ci/run_check.py
index 9c7ba13f8e4..74e60582e8d 100644
--- a/tests/ci/run_check.py
+++ b/tests/ci/run_check.py
@@ -8,7 +8,7 @@ from github import Github
 from env_helper import GITHUB_RUN_URL, GITHUB_REPOSITORY, GITHUB_SERVER_URL
 from pr_info import PRInfo
 from get_robot_token import get_best_robot_token
-from commit_status_helper import get_commit
+from commit_status_helper import get_commit, post_label
 
 NAME = "Run Check (actions)"
 
@@ -81,6 +81,21 @@ TRUSTED_CONTRIBUTORS = {
     ]
 }
 
+MAP_CATEGORY_TO_LABEL = {
+    "New Feature": "pr-feature",
+    "Bug Fix": "pr-bugfix",
+    "Bug Fix (user-visible misbehaviour in official stable or prestable release)": "pr-bugfix",
+    "Improvement": "pr-improvement",
+    "Performance Improvement": "pr-performance",
+    "Backward Incompatible Change": "pr-backward-incompatible",
+    "Build/Testing/Packaging Improvement": "pr-build",
+    "Not for changelog (changelog entry is not required)": "pr-not-for-changelog",
+    "Not for changelog": "pr-not-for-changelog",
+    "Documentation (changelog entry is not required)": "pr-documentation",
+    "Documentation": "pr-documentation",
+    # 'Other': doesn't match anything
+}
+
 
 def pr_is_by_trusted_user(pr_user_login, pr_user_orgs):
     if pr_user_login.lower() in TRUSTED_CONTRIBUTORS:
@@ -168,7 +183,7 @@ def check_pr_description(pr_info):
                     + second_category
                     + "'"
                 )
-                return result_status[:140]
+                return result_status[:140], category
 
         elif re.match(
             r"(?i)^[>*_ ]*(short\s*description|change\s*log\s*entry)", lines[i]
@@ -190,19 +205,19 @@ def check_pr_description(pr_info):
             i += 1
 
     if not category:
-        return "Changelog category is empty"
+        return "Changelog category is empty", category
 
     # Filter out the PR categories that are not for changelog.
     if re.match(
         r"(?i)doc|((non|in|not|un)[-\s]*significant)|(not[ ]*for[ ]*changelog)",
         category,
     ):
-        return ""
+        return "", category
 
     if not entry:
-        return f"Changelog entry required for category '{category}'"
+        return f"Changelog entry required for category '{category}'", category
 
-    return ""
+    return "", category
 
 
 if __name__ == "__main__":
@@ -213,7 +228,10 @@ if __name__ == "__main__":
     gh = Github(get_best_robot_token())
     commit = get_commit(gh, pr_info.sha)
 
-    description_report = check_pr_description(pr_info)[:139]
+    description_report, category = check_pr_description(pr_info)
+    if category in MAP_CATEGORY_TO_LABEL:
+        post_label(gh, pr_info, MAP_CATEGORY_TO_LABEL[category])
+
     if description_report:
         print("::notice ::Cannot run, description does not match the template")
         logging.info(
@@ -225,7 +243,7 @@ if __name__ == "__main__":
         )
         commit.create_status(
             context=NAME,
-            description=description_report,
+            description=description_report[:139],
             state="failure",
             target_url=url,
         )
diff --git a/utils/github/__init__.py b/utils/github/__init__.py
deleted file mode 100644
index 40a96afc6ff..00000000000
--- a/utils/github/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-# -*- coding: utf-8 -*-
diff --git a/utils/github/backport.py b/utils/github/backport.py
deleted file mode 100644
index 615c0d19ffa..00000000000
--- a/utils/github/backport.py
+++ /dev/null
@@ -1,185 +0,0 @@
-# -*- coding: utf-8 -*-
-
-try:
-    from clickhouse.utils.github.cherrypick import CherryPick
-    from clickhouse.utils.github.query import Query as RemoteRepo
-    from clickhouse.utils.github.local import Repository as LocalRepo
-except:
-    from .cherrypick import CherryPick
-    from .query import Query as RemoteRepo
-    from .local import Repository as LocalRepo
-
-import argparse
-import logging
-import re
-import sys
-
-
-class Backport:
-    def __init__(self, token, owner, name, team):
-        self._gh = RemoteRepo(
-            token, owner=owner, name=name, team=team, max_page_size=30, min_page_size=7
-        )
-        self._token = token
-        self.default_branch_name = self._gh.default_branch
-        self.ssh_url = self._gh.ssh_url
-
-    def getPullRequests(self, from_commit):
-        return self._gh.get_pull_requests(from_commit)
-
-    def getBranchesWithRelease(self):
-        branches = set()
-        for pull_request in self._gh.find_pull_requests("release"):
-            branches.add(pull_request["headRefName"])
-        return branches
-
-    def execute(self, repo, upstream, until_commit, run_cherrypick):
-        repo = LocalRepo(repo, upstream, self.default_branch_name)
-        all_branches = repo.get_release_branches()  # [(branch_name, base_commit)]
-
-        release_branches = self.getBranchesWithRelease()
-
-        branches = []
-        # iterate over all branches to preserve their precedence.
-        for branch in all_branches:
-            if branch[0] in release_branches:
-                branches.append(branch)
-
-        if not branches:
-            logging.info("No release branches found!")
-            return
-
-        for branch in branches:
-            logging.info("Found release branch: %s", branch[0])
-
-        if not until_commit:
-            until_commit = branches[0][1]
-        pull_requests = self.getPullRequests(until_commit)
-
-        backport_map = {}
-
-        RE_MUST_BACKPORT = re.compile(r"^v(\d+\.\d+)-must-backport$")
-        RE_NO_BACKPORT = re.compile(r"^v(\d+\.\d+)-no-backport$")
-        RE_BACKPORTED = re.compile(r"^v(\d+\.\d+)-backported$")
-
-        # pull-requests are sorted by ancestry from the most recent.
-        for pr in pull_requests:
-            while repo.comparator(branches[-1][1]) >= repo.comparator(
-                pr["mergeCommit"]["oid"]
-            ):
-                logging.info(
-                    "PR #{} is already inside {}. Dropping this branch for further PRs".format(
-                        pr["number"], branches[-1][0]
-                    )
-                )
-                branches.pop()
-
-            logging.info("Processing PR #{}".format(pr["number"]))
-
-            assert len(branches)
-
-            branch_set = set([branch[0] for branch in branches])
-
-            # First pass. Find all must-backports
-            for label in pr["labels"]["nodes"]:
-                if label["name"] == "pr-must-backport":
-                    backport_map[pr["number"]] = branch_set.copy()
-                    continue
-                matched = RE_MUST_BACKPORT.match(label["name"])
-                if matched:
-                    if pr["number"] not in backport_map:
-                        backport_map[pr["number"]] = set()
-                    backport_map[pr["number"]].add(matched.group(1))
-
-            # Second pass. Find all no-backports
-            for label in pr["labels"]["nodes"]:
-                if label["name"] == "pr-no-backport" and pr["number"] in backport_map:
-                    del backport_map[pr["number"]]
-                    break
-                matched_no_backport = RE_NO_BACKPORT.match(label["name"])
-                matched_backported = RE_BACKPORTED.match(label["name"])
-                if (
-                    matched_no_backport
-                    and pr["number"] in backport_map
-                    and matched_no_backport.group(1) in backport_map[pr["number"]]
-                ):
-                    backport_map[pr["number"]].remove(matched_no_backport.group(1))
-                    logging.info(
-                        "\tskipping %s because of forced no-backport",
-                        matched_no_backport.group(1),
-                    )
-                elif (
-                    matched_backported
-                    and pr["number"] in backport_map
-                    and matched_backported.group(1) in backport_map[pr["number"]]
-                ):
-                    backport_map[pr["number"]].remove(matched_backported.group(1))
-                    logging.info(
-                        "\tskipping %s because it's already backported manually",
-                        matched_backported.group(1),
-                    )
-
-        for pr, branches in list(backport_map.items()):
-            logging.info("PR #%s needs to be backported to:", pr)
-            for branch in branches:
-                logging.info(
-                    "\t%s, and the status is: %s",
-                    branch,
-                    run_cherrypick(self._token, pr, branch),
-                )
-
-        # print API costs
-        logging.info("\nGitHub API total costs per query:")
-        for name, value in list(self._gh.api_costs.items()):
-            logging.info("%s : %s", name, value)
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "--token", type=str, required=True, help="token for Github access"
-    )
-    parser.add_argument(
-        "--repo",
-        type=str,
-        required=True,
-        help="path to full repository",
-        metavar="PATH",
-    )
-    parser.add_argument(
-        "--til", type=str, help="check PRs from HEAD til this commit", metavar="COMMIT"
-    )
-    parser.add_argument(
-        "--dry-run",
-        action="store_true",
-        help="do not create or merge any PRs",
-        default=False,
-    )
-    parser.add_argument(
-        "--verbose",
-        "-v",
-        action="store_true",
-        help="more verbose output",
-        default=False,
-    )
-    parser.add_argument(
-        "--upstream",
-        "-u",
-        type=str,
-        help="remote name of upstream in repository",
-        default="origin",
-    )
-    args = parser.parse_args()
-
-    if args.verbose:
-        logging.basicConfig(
-            format="%(message)s", stream=sys.stdout, level=logging.DEBUG
-        )
-    else:
-        logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.INFO)
-
-    cherrypick_run = lambda token, pr, branch: CherryPick(
-        token, "ClickHouse", "ClickHouse", "core", pr, branch
-    ).execute(args.repo, args.dry_run)
-    bp = Backport(args.token, "ClickHouse", "ClickHouse", "core")
-    bp.execute(args.repo, args.upstream, args.til, cherrypick_run)
diff --git a/utils/github/cherrypick.py b/utils/github/cherrypick.py
deleted file mode 100644
index c6469fa62a9..00000000000
--- a/utils/github/cherrypick.py
+++ /dev/null
@@ -1,323 +0,0 @@
-# -*- coding: utf-8 -*-
-
-"""
-Backports changes from PR to release branch.
-Requires multiple separate runs as part of the implementation.
-
-First run should do the following:
-1. Merge release branch with a first parent of merge-commit of PR (using 'ours' strategy). (branch: backport/{branch}/{pr})
-2. Create temporary branch over merge-commit to use it for PR creation. (branch: cherrypick/{merge_commit})
-3. Create PR from temporary branch to backport branch (emulating cherry-pick).
-
-Second run checks PR from previous run to be merged or at least being mergeable. If it's not merged then try to merge it.
-
-Third run creates PR from backport branch (with merged previous PR) to release branch.
-"""
-
-try:
-    from clickhouse.utils.github.query import Query as RemoteRepo
-except:
-    from .query import Query as RemoteRepo
-
-import argparse
-from enum import Enum
-import logging
-import os
-import subprocess
-import sys
-
-
-class CherryPick:
-    class Status(Enum):
-        DISCARDED = "discarded"
-        NOT_INITIATED = "not started"
-        FIRST_MERGEABLE = "waiting for 1st stage"
-        FIRST_CONFLICTS = "conflicts on 1st stage"
-        SECOND_MERGEABLE = "waiting for 2nd stage"
-        SECOND_CONFLICTS = "conflicts on 2nd stage"
-        MERGED = "backported"
-
-    def _run(self, args):
-        out = subprocess.check_output(args).rstrip()
-        logging.debug(out)
-        return out
-
-    def __init__(self, token, owner, name, team, pr_number, target_branch):
-        self._gh = RemoteRepo(token, owner=owner, name=name, team=team)
-        self._pr = self._gh.get_pull_request(pr_number)
-
-        self.ssh_url = self._gh.ssh_url
-
-        # TODO: check if pull-request is merged.
-
-        self.merge_commit_oid = self._pr["mergeCommit"]["oid"]
-
-        self.target_branch = target_branch
-        self.backport_branch = "backport/{branch}/{pr}".format(
-            branch=target_branch, pr=pr_number
-        )
-        self.cherrypick_branch = "cherrypick/{branch}/{oid}".format(
-            branch=target_branch, oid=self.merge_commit_oid
-        )
-
-    def getCherryPickPullRequest(self):
-        return self._gh.find_pull_request(
-            base=self.backport_branch, head=self.cherrypick_branch
-        )
-
-    def createCherryPickPullRequest(self, repo_path):
-        DESCRIPTION = (
-            "This pull-request is a first step of an automated backporting.\n"
-            "It contains changes like after calling a local command `git cherry-pick`.\n"
-            "If you intend to continue backporting this changes, then resolve all conflicts if any.\n"
-            "Otherwise, if you do not want to backport them, then just close this pull-request.\n"
-            "\n"
-            "The check results does not matter at this step - you can safely ignore them.\n"
-            "Also this pull-request will be merged automatically as it reaches the mergeable state, but you always can merge it manually.\n"
-        )
-
-        # FIXME: replace with something better than os.system()
-        git_prefix = [
-            "git",
-            "-C",
-            repo_path,
-            "-c",
-            "user.email=robot-clickhouse@yandex-team.ru",
-            "-c",
-            "user.name=robot-clickhouse",
-        ]
-        base_commit_oid = self._pr["mergeCommit"]["parents"]["nodes"][0]["oid"]
-
-        # Create separate branch for backporting, and make it look like real cherry-pick.
-        self._run(git_prefix + ["checkout", "-f", self.target_branch])
-        self._run(git_prefix + ["checkout", "-B", self.backport_branch])
-        self._run(git_prefix + ["merge", "-s", "ours", "--no-edit", base_commit_oid])
-
-        # Create secondary branch to allow pull request with cherry-picked commit.
-        self._run(
-            git_prefix + ["branch", "-f", self.cherrypick_branch, self.merge_commit_oid]
-        )
-
-        self._run(
-            git_prefix
-            + [
-                "push",
-                "-f",
-                "origin",
-                "{branch}:{branch}".format(branch=self.backport_branch),
-            ]
-        )
-        self._run(
-            git_prefix
-            + [
-                "push",
-                "-f",
-                "origin",
-                "{branch}:{branch}".format(branch=self.cherrypick_branch),
-            ]
-        )
-
-        # Create pull-request like a local cherry-pick
-        pr = self._gh.create_pull_request(
-            source=self.cherrypick_branch,
-            target=self.backport_branch,
-            title="Cherry pick #{number} to {target}: {title}".format(
-                number=self._pr["number"],
-                target=self.target_branch,
-                title=self._pr["title"].replace('"', '\\"'),
-            ),
-            description="Original pull-request #{}\n\n{}".format(
-                self._pr["number"], DESCRIPTION
-            ),
-        )
-
-        # FIXME: use `team` to leave a single eligible assignee.
-        self._gh.add_assignee(pr, self._pr["author"])
-        self._gh.add_assignee(pr, self._pr["mergedBy"])
-
-        self._gh.set_label(pr, "do not test")
-        self._gh.set_label(pr, "pr-cherrypick")
-
-        return pr
-
-    def mergeCherryPickPullRequest(self, cherrypick_pr):
-        return self._gh.merge_pull_request(cherrypick_pr["id"])
-
-    def getBackportPullRequest(self):
-        return self._gh.find_pull_request(
-            base=self.target_branch, head=self.backport_branch
-        )
-
-    def createBackportPullRequest(self, cherrypick_pr, repo_path):
-        DESCRIPTION = (
-            "This pull-request is a last step of an automated backporting.\n"
-            "Treat it as a standard pull-request: look at the checks and resolve conflicts.\n"
-            "Merge it only if you intend to backport changes to the target branch, otherwise just close it.\n"
-        )
-
-        git_prefix = [
-            "git",
-            "-C",
-            repo_path,
-            "-c",
-            "user.email=robot-clickhouse@clickhouse.com",
-            "-c",
-            "user.name=robot-clickhouse",
-        ]
-
-        pr_title = "Backport #{number} to {target}: {title}".format(
-            number=self._pr["number"],
-            target=self.target_branch,
-            title=self._pr["title"].replace('"', '\\"'),
-        )
-
-        self._run(git_prefix + ["checkout", "-f", self.backport_branch])
-        self._run(git_prefix + ["pull", "--ff-only", "origin", self.backport_branch])
-        self._run(
-            git_prefix
-            + [
-                "reset",
-                "--soft",
-                self._run(
-                    git_prefix
-                    + [
-                        "merge-base",
-                        "origin/" + self.target_branch,
-                        self.backport_branch,
-                    ]
-                ),
-            ]
-        )
-        self._run(git_prefix + ["commit", "-a", "--allow-empty", "-m", pr_title])
-        self._run(
-            git_prefix
-            + [
-                "push",
-                "-f",
-                "origin",
-                "{branch}:{branch}".format(branch=self.backport_branch),
-            ]
-        )
-
-        pr = self._gh.create_pull_request(
-            source=self.backport_branch,
-            target=self.target_branch,
-            title=pr_title,
-            description="Original pull-request #{}\nCherry-pick pull-request #{}\n\n{}".format(
-                self._pr["number"], cherrypick_pr["number"], DESCRIPTION
-            ),
-        )
-
-        # FIXME: use `team` to leave a single eligible assignee.
-        self._gh.add_assignee(pr, self._pr["author"])
-        self._gh.add_assignee(pr, self._pr["mergedBy"])
-
-        self._gh.set_label(pr, "pr-backport")
-
-        return pr
-
-    def execute(self, repo_path, dry_run=False):
-        pr1 = self.getCherryPickPullRequest()
-        if not pr1:
-            if not dry_run:
-                pr1 = self.createCherryPickPullRequest(repo_path)
-                logging.debug(
-                    "Created PR with cherry-pick of %s to %s: %s",
-                    self._pr["number"],
-                    self.target_branch,
-                    pr1["url"],
-                )
-            else:
-                return CherryPick.Status.NOT_INITIATED
-        else:
-            logging.debug(
-                "Found PR with cherry-pick of %s to %s: %s",
-                self._pr["number"],
-                self.target_branch,
-                pr1["url"],
-            )
-
-        if not pr1["merged"] and pr1["mergeable"] == "MERGEABLE" and not pr1["closed"]:
-            if not dry_run:
-                pr1 = self.mergeCherryPickPullRequest(pr1)
-                logging.debug(
-                    "Merged PR with cherry-pick of %s to %s: %s",
-                    self._pr["number"],
-                    self.target_branch,
-                    pr1["url"],
-                )
-
-        if not pr1["merged"]:
-            logging.debug(
-                "Waiting for PR with cherry-pick of %s to %s: %s",
-                self._pr["number"],
-                self.target_branch,
-                pr1["url"],
-            )
-
-            if pr1["closed"]:
-                return CherryPick.Status.DISCARDED
-            elif pr1["mergeable"] == "CONFLICTING":
-                return CherryPick.Status.FIRST_CONFLICTS
-            else:
-                return CherryPick.Status.FIRST_MERGEABLE
-
-        pr2 = self.getBackportPullRequest()
-        if not pr2:
-            if not dry_run:
-                pr2 = self.createBackportPullRequest(pr1, repo_path)
-                logging.debug(
-                    "Created PR with backport of %s to %s: %s",
-                    self._pr["number"],
-                    self.target_branch,
-                    pr2["url"],
-                )
-            else:
-                return CherryPick.Status.FIRST_MERGEABLE
-        else:
-            logging.debug(
-                "Found PR with backport of %s to %s: %s",
-                self._pr["number"],
-                self.target_branch,
-                pr2["url"],
-            )
-
-        if pr2["merged"]:
-            return CherryPick.Status.MERGED
-        elif pr2["closed"]:
-            return CherryPick.Status.DISCARDED
-        elif pr2["mergeable"] == "CONFLICTING":
-            return CherryPick.Status.SECOND_CONFLICTS
-        else:
-            return CherryPick.Status.SECOND_MERGEABLE
-
-
-if __name__ == "__main__":
-    logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG)
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "--token", "-t", type=str, required=True, help="token for Github access"
-    )
-    parser.add_argument("--pr", type=str, required=True, help="PR# to cherry-pick")
-    parser.add_argument(
-        "--branch",
-        "-b",
-        type=str,
-        required=True,
-        help="target branch name for cherry-pick",
-    )
-    parser.add_argument(
-        "--repo",
-        "-r",
-        type=str,
-        required=True,
-        help="path to full repository",
-        metavar="PATH",
-    )
-    args = parser.parse_args()
-
-    cp = CherryPick(
-        args.token, "ClickHouse", "ClickHouse", "core", args.pr, args.branch
-    )
-    cp.execute(args.repo)
diff --git a/utils/github/local.py b/utils/github/local.py
deleted file mode 100644
index 571c9102ba0..00000000000
--- a/utils/github/local.py
+++ /dev/null
@@ -1,108 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import functools
-import logging
-import os
-import re
-
-
-class RepositoryBase:
-    def __init__(self, repo_path):
-        import git
-
-        self._repo = git.Repo(repo_path, search_parent_directories=(not repo_path))
-
-        # comparator of commits
-        def cmp(x, y):
-            if str(x) == str(y):
-                return 0
-            if self._repo.is_ancestor(x, y):
-                return -1
-            else:
-                return 1
-
-        self.comparator = functools.cmp_to_key(cmp)
-
-    def get_head_commit(self):
-        return self._repo.commit(self._default)
-
-    def iterate(self, begin, end):
-        rev_range = "{}...{}".format(begin, end)
-        for commit in self._repo.iter_commits(rev_range, first_parent=True):
-            yield commit
-
-
-class Repository(RepositoryBase):
-    def __init__(self, repo_path, remote_name, default_branch_name):
-        super(Repository, self).__init__(repo_path)
-        self._remote = self._repo.remotes[remote_name]
-        self._remote.fetch()
-        self._default = self._remote.refs[default_branch_name]
-
-    def get_release_branches(self):
-        """
-        Returns sorted list of tuples:
-         * remote branch (git.refs.remote.RemoteReference),
-         * base commit (git.Commit),
-         * head (git.Commit)).
-        List is sorted by commits in ascending order.
-        """
-        release_branches = []
-
-        RE_RELEASE_BRANCH_REF = re.compile(r"^refs/remotes/.+/\d+\.\d+$")
-
-        for branch in [
-            r for r in self._remote.refs if RE_RELEASE_BRANCH_REF.match(r.path)
-        ]:
-            base = self._repo.merge_base(self._default, self._repo.commit(branch))
-            if not base:
-                logging.info(
-                    "Branch %s is not based on branch %s. Ignoring.",
-                    branch.path,
-                    self._default,
-                )
-            elif len(base) > 1:
-                logging.info(
-                    "Branch %s has more than one base commit. Ignoring.", branch.path
-                )
-            else:
-                release_branches.append((os.path.basename(branch.name), base[0]))
-
-        return sorted(release_branches, key=lambda x: self.comparator(x[1]))
-
-
-class BareRepository(RepositoryBase):
-    def __init__(self, repo_path, default_branch_name):
-        super(BareRepository, self).__init__(repo_path)
-        self._default = self._repo.branches[default_branch_name]
-
-    def get_release_branches(self):
-        """
-        Returns sorted list of tuples:
-         * branch (git.refs.head?),
-         * base commit (git.Commit),
-         * head (git.Commit)).
-        List is sorted by commits in ascending order.
-        """
-        release_branches = []
-
-        RE_RELEASE_BRANCH_REF = re.compile(r"^refs/heads/\d+\.\d+$")
-
-        for branch in [
-            r for r in self._repo.branches if RE_RELEASE_BRANCH_REF.match(r.path)
-        ]:
-            base = self._repo.merge_base(self._default, self._repo.commit(branch))
-            if not base:
-                logging.info(
-                    "Branch %s is not based on branch %s. Ignoring.",
-                    branch.path,
-                    self._default,
-                )
-            elif len(base) > 1:
-                logging.info(
-                    "Branch %s has more than one base commit. Ignoring.", branch.path
-                )
-            else:
-                release_branches.append((os.path.basename(branch.name), base[0]))
-
-        return sorted(release_branches, key=lambda x: self.comparator(x[1]))
diff --git a/utils/github/parser.py b/utils/github/parser.py
deleted file mode 100644
index d8348e6d964..00000000000
--- a/utils/github/parser.py
+++ /dev/null
@@ -1,64 +0,0 @@
-# -*- coding: utf-8 -*-
-
-
-class Description:
-    """Parsed description representation"""
-
-    MAP_CATEGORY_TO_LABEL = {
-        "New Feature": "pr-feature",
-        "Bug Fix": "pr-bugfix",
-        "Improvement": "pr-improvement",
-        "Performance Improvement": "pr-performance",
-        # 'Backward Incompatible Change': doesn't match anything
-        "Build/Testing/Packaging Improvement": "pr-build",
-        "Non-significant (changelog entry is not needed)": "pr-non-significant",
-        "Non-significant (changelog entry is not required)": "pr-non-significant",
-        "Non-significant": "pr-non-significant",
-        "Documentation (changelog entry is not required)": "pr-documentation",
-        # 'Other': doesn't match anything
-    }
-
-    def __init__(self, pull_request):
-        self.label_name = str()
-        self.legal = False
-
-        self._parse(pull_request["bodyText"])
-
-    def _parse(self, text):
-        lines = text.splitlines()
-        next_category = False
-        category = str()
-
-        for line in lines:
-            stripped = line.strip()
-
-            if not stripped:
-                continue
-
-            if next_category:
-                category = stripped
-                next_category = False
-
-            if (
-                stripped
-                == "I hereby agree to the terms of the CLA available at: https://yandex.ru/legal/cla/?lang=en"
-            ):
-                self.legal = True
-
-            category_headers = (
-                "Category (leave one):",
-                "Changelog category (leave one):",
-                "Changelog category:",
-                "Category:",
-            )
-
-            if stripped in category_headers:
-                next_category = True
-
-        if category in Description.MAP_CATEGORY_TO_LABEL:
-            self.label_name = Description.MAP_CATEGORY_TO_LABEL[category]
-        else:
-            if not category:
-                print("Cannot find category in pr description")
-            else:
-                print(("Unknown category: " + category))
diff --git a/utils/github/query.py b/utils/github/query.py
deleted file mode 100644
index 7afbc57781c..00000000000
--- a/utils/github/query.py
+++ /dev/null
@@ -1,492 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import requests
-
-
-class Query:
-    """
-    Implements queries to the Github API using GraphQL
-    """
-
-    _PULL_REQUEST = """
-        author {{
-            ... on User {{
-                id
-                login
-            }}
-        }}
-
-        baseRepository {{
-            nameWithOwner
-        }}
-
-        mergeCommit {{
-            oid
-            parents(first: {min_page_size}) {{
-                totalCount
-                nodes {{
-                    oid
-                }}
-            }}
-        }}
-
-        mergedBy {{
-            ... on User {{
-                id
-                login
-            }}
-        }}
-
-        baseRefName
-        closed
-        headRefName
-        id
-        mergeable
-        merged
-        number
-        title
-        url
-    """
-
-    def __init__(self, token, owner, name, team, max_page_size=100, min_page_size=10):
-        self._PULL_REQUEST = Query._PULL_REQUEST.format(min_page_size=min_page_size)
-
-        self._token = token
-        self._owner = owner
-        self._name = name
-        self._team = team
-
-        self._max_page_size = max_page_size
-        self._min_page_size = min_page_size
-
-        self.api_costs = {}
-
-        repo = self.get_repository()
-        self._id = repo["id"]
-        self.ssh_url = repo["sshUrl"]
-        self.default_branch = repo["defaultBranchRef"]["name"]
-
-        self.members = set(self.get_members())
-
-    def get_repository(self):
-        _QUERY = """
-            repository(owner: "{owner}" name: "{name}") {{
-                defaultBranchRef {{
-                    name
-                }}
-                id
-                sshUrl
-            }}
-        """
-
-        query = _QUERY.format(owner=self._owner, name=self._name)
-        return self._run(query)["repository"]
-
-    def get_members(self):
-        """Get all team members for organization
-
-        Returns:
-            members: a map of members' logins to ids
-        """
-
-        _QUERY = """
-            organization(login: "{organization}") {{
-                team(slug: "{team}") {{
-                    members(first: {max_page_size} {next}) {{
-                        pageInfo {{
-                            hasNextPage
-                            endCursor
-                        }}
-                        nodes {{
-                            id
-                            login
-                        }}
-                    }}
-                }}
-            }}
-        """
-
-        members = {}
-        not_end = True
-        query = _QUERY.format(
-            organization=self._owner,
-            team=self._team,
-            max_page_size=self._max_page_size,
-            next="",
-        )
-
-        while not_end:
-            result = self._run(query)["organization"]["team"]
-            if result is None:
-                break
-            result = result["members"]
-            not_end = result["pageInfo"]["hasNextPage"]
-            query = _QUERY.format(
-                organization=self._owner,
-                team=self._team,
-                max_page_size=self._max_page_size,
-                next='after: "{}"'.format(result["pageInfo"]["endCursor"]),
-            )
-
-            members += dict([(node["login"], node["id"]) for node in result["nodes"]])
-
-        return members
-
-    def get_pull_request(self, number):
-        _QUERY = """
-            repository(owner: "{owner}" name: "{name}") {{
-                pullRequest(number: {number}) {{
-                    {pull_request_data}
-                }}
-            }}
-        """
-
-        query = _QUERY.format(
-            owner=self._owner,
-            name=self._name,
-            number=number,
-            pull_request_data=self._PULL_REQUEST,
-            min_page_size=self._min_page_size,
-        )
-        return self._run(query)["repository"]["pullRequest"]
-
-    def find_pull_request(self, base, head):
-        _QUERY = """
-            repository(owner: "{owner}" name: "{name}") {{
-                pullRequests(first: {min_page_size} baseRefName: "{base}" headRefName: "{head}") {{
-                    nodes {{
-                        {pull_request_data}
-                    }}
-                    totalCount
-                }}
-            }}
-        """
-
-        query = _QUERY.format(
-            owner=self._owner,
-            name=self._name,
-            base=base,
-            head=head,
-            pull_request_data=self._PULL_REQUEST,
-            min_page_size=self._min_page_size,
-        )
-        result = self._run(query)["repository"]["pullRequests"]
-        if result["totalCount"] > 0:
-            return result["nodes"][0]
-        else:
-            return {}
-
-    def find_pull_requests(self, label_name):
-        """
-        Get all pull-requests filtered by label name
-        """
-        _QUERY = """
-            repository(owner: "{owner}" name: "{name}") {{
-                pullRequests(first: {min_page_size} labels: "{label_name}" states: OPEN) {{
-                    nodes {{
-                        {pull_request_data}
-                    }}
-                }}
-            }}
-        """
-
-        query = _QUERY.format(
-            owner=self._owner,
-            name=self._name,
-            label_name=label_name,
-            pull_request_data=self._PULL_REQUEST,
-            min_page_size=self._min_page_size,
-        )
-        return self._run(query)["repository"]["pullRequests"]["nodes"]
-
-    def get_pull_requests(self, before_commit):
-        """
-        Get all merged pull-requests from the HEAD of default branch to the last commit (excluding)
-        """
-
-        _QUERY = """
-            repository(owner: "{owner}" name: "{name}") {{
-                defaultBranchRef {{
-                    target {{
-                        ... on Commit {{
-                            history(first: {max_page_size} {next}) {{
-                                pageInfo {{
-                                    hasNextPage
-                                    endCursor
-                                }}
-                                nodes {{
-                                    oid
-                                    associatedPullRequests(first: {min_page_size}) {{
-                                        totalCount
-                                        nodes {{
-                                            ... on PullRequest {{
-                                                {pull_request_data}
-
-                                                labels(first: {min_page_size}) {{
-                                                    totalCount
-                                                    pageInfo {{
-                                                        hasNextPage
-                                                        endCursor
-                                                    }}
-                                                    nodes {{
-                                                        name
-                                                        color
-                                                    }}
-                                                }}
-                                            }}
-                                        }}
-                                    }}
-                                }}
-                            }}
-                        }}
-                    }}
-                }}
-            }}
-        """
-
-        pull_requests = []
-        not_end = True
-        query = _QUERY.format(
-            owner=self._owner,
-            name=self._name,
-            max_page_size=self._max_page_size,
-            min_page_size=self._min_page_size,
-            pull_request_data=self._PULL_REQUEST,
-            next="",
-        )
-
-        while not_end:
-            result = self._run(query)["repository"]["defaultBranchRef"]["target"][
-                "history"
-            ]
-            not_end = result["pageInfo"]["hasNextPage"]
-            query = _QUERY.format(
-                owner=self._owner,
-                name=self._name,
-                max_page_size=self._max_page_size,
-                min_page_size=self._min_page_size,
-                pull_request_data=self._PULL_REQUEST,
-                next='after: "{}"'.format(result["pageInfo"]["endCursor"]),
-            )
-
-            for commit in result["nodes"]:
-                # FIXME: maybe include `before_commit`?
-                if str(commit["oid"]) == str(before_commit):
-                    not_end = False
-                    break
-
-                # TODO: fetch all pull-requests that were merged in a single commit.
-                assert (
-                    commit["associatedPullRequests"]["totalCount"]
-                    <= self._min_page_size
-                )
-
-                for pull_request in commit["associatedPullRequests"]["nodes"]:
-                    if (
-                        pull_request["baseRepository"]["nameWithOwner"]
-                        == "{}/{}".format(self._owner, self._name)
-                        and pull_request["baseRefName"] == self.default_branch
-                        and pull_request["mergeCommit"]["oid"] == commit["oid"]
-                    ):
-                        pull_requests.append(pull_request)
-
-        return pull_requests
-
-    def create_pull_request(
-        self, source, target, title, description="", draft=False, can_modify=True
-    ):
-        _QUERY = """
-            createPullRequest(input: {{
-                baseRefName: "{target}",
-                headRefName: "{source}",
-                repositoryId: "{id}",
-                title: "{title}",
-                body: "{body}",
-                draft: {draft},
-                maintainerCanModify: {modify}
-            }}) {{
-                pullRequest {{
-                    {pull_request_data}
-                }}
-            }}
-        """
-
-        query = _QUERY.format(
-            target=target,
-            source=source,
-            id=self._id,
-            title=title,
-            body=description,
-            draft="true" if draft else "false",
-            modify="true" if can_modify else "false",
-            pull_request_data=self._PULL_REQUEST,
-        )
-        return self._run(query, is_mutation=True)["createPullRequest"]["pullRequest"]
-
-    def merge_pull_request(self, id):
-        _QUERY = """
-            mergePullRequest(input: {{
-                pullRequestId: "{id}"
-            }}) {{
-                pullRequest {{
-                    {pull_request_data}
-                }}
-            }}
-        """
-
-        query = _QUERY.format(id=id, pull_request_data=self._PULL_REQUEST)
-        return self._run(query, is_mutation=True)["mergePullRequest"]["pullRequest"]
-
-    # FIXME: figure out how to add more assignees at once
-    def add_assignee(self, pr, assignee):
-        _QUERY = """
-            addAssigneesToAssignable(input: {{
-                assignableId: "{id1}",
-                assigneeIds: "{id2}"
-            }}) {{
-                clientMutationId
-            }}
-        """
-
-        query = _QUERY.format(id1=pr["id"], id2=assignee["id"])
-        self._run(query, is_mutation=True)
-
-    def set_label(self, pull_request, label_name):
-        """
-        Set label by name to the pull request
-
-        Args:
-            pull_request: JSON object returned by `get_pull_requests()`
-            label_name (string): label name
-        """
-
-        _GET_LABEL = """
-            repository(owner: "{owner}" name: "{name}") {{
-                labels(first: {max_page_size} {next} query: "{label_name}") {{
-                    pageInfo {{
-                        hasNextPage
-                        endCursor
-                    }}
-                    nodes {{
-                        id
-                        name
-                        color
-                    }}
-                }}
-            }}
-        """
-
-        _SET_LABEL = """
-            addLabelsToLabelable(input: {{
-                labelableId: "{pr_id}",
-                labelIds: "{label_id}"
-            }}) {{
-                clientMutationId
-            }}
-        """
-
-        labels = []
-        not_end = True
-        query = _GET_LABEL.format(
-            owner=self._owner,
-            name=self._name,
-            label_name=label_name,
-            max_page_size=self._max_page_size,
-            next="",
-        )
-
-        while not_end:
-            result = self._run(query)["repository"]["labels"]
-            not_end = result["pageInfo"]["hasNextPage"]
-            query = _GET_LABEL.format(
-                owner=self._owner,
-                name=self._name,
-                label_name=label_name,
-                max_page_size=self._max_page_size,
-                next='after: "{}"'.format(result["pageInfo"]["endCursor"]),
-            )
-
-            labels += [label for label in result["nodes"]]
-
-        if not labels:
-            return
-
-        query = _SET_LABEL.format(pr_id=pull_request["id"], label_id=labels[0]["id"])
-        self._run(query, is_mutation=True)
-
-    def _run(self, query, is_mutation=False):
-        from requests.adapters import HTTPAdapter
-        from urllib3.util.retry import Retry
-
-        def requests_retry_session(
-            retries=3,
-            backoff_factor=0.3,
-            status_forcelist=(500, 502, 504),
-            session=None,
-        ):
-            session = session or requests.Session()
-            retry = Retry(
-                total=retries,
-                read=retries,
-                connect=retries,
-                backoff_factor=backoff_factor,
-                status_forcelist=status_forcelist,
-            )
-            adapter = HTTPAdapter(max_retries=retry)
-            session.mount("http://", adapter)
-            session.mount("https://", adapter)
-            return session
-
-        headers = {"Authorization": "bearer {}".format(self._token)}
-        if is_mutation:
-            query = """
-            mutation {{
-                {query}
-            }}
-            """.format(
-                query=query
-            )
-        else:
-            query = """
-            query {{
-                {query}
-                rateLimit {{
-                    cost
-                    remaining
-                }}
-            }}
-            """.format(
-                query=query
-            )
-
-        while True:
-            request = requests_retry_session().post(
-                "https://api.github.com/graphql", json={"query": query}, headers=headers
-            )
-            if request.status_code == 200:
-                result = request.json()
-                if "errors" in result:
-                    raise Exception(
-                        "Errors occurred: {}\nOriginal query: {}".format(
-                            result["errors"], query
-                        )
-                    )
-
-                if not is_mutation:
-                    import inspect
-
-                    caller = inspect.getouterframes(inspect.currentframe(), 2)[1][3]
-                    if caller not in list(self.api_costs.keys()):
-                        self.api_costs[caller] = 0
-                    self.api_costs[caller] += result["data"]["rateLimit"]["cost"]
-
-                return result["data"]
-            else:
-                import json
-
-                raise Exception(
-                    "Query failed with code {code}:\n{json}".format(
-                        code=request.status_code,
-                        json=json.dumps(request.json(), indent=4),
-                    )
-                )

From bf591b971a6d9dbd1b63357ac353066d6348fc34 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Tue, 29 Mar 2022 15:53:08 +0200
Subject: [PATCH 054/104] Moar categories

---
 tests/ci/run_check.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/ci/run_check.py b/tests/ci/run_check.py
index 74e60582e8d..608a708b04b 100644
--- a/tests/ci/run_check.py
+++ b/tests/ci/run_check.py
@@ -89,6 +89,10 @@ MAP_CATEGORY_TO_LABEL = {
     "Performance Improvement": "pr-performance",
     "Backward Incompatible Change": "pr-backward-incompatible",
     "Build/Testing/Packaging Improvement": "pr-build",
+    "Build Improvement": "pr-build",
+    "Build/Testing Improvement": "pr-build",
+    "Build": "pr-build",
+    "Packaging Improvement": "pr-build",
     "Not for changelog (changelog entry is not required)": "pr-not-for-changelog",
     "Not for changelog": "pr-not-for-changelog",
     "Documentation (changelog entry is not required)": "pr-documentation",

From 693b47cdc9c39ee4d51202592cabc915ed9b0962 Mon Sep 17 00:00:00 2001
From: tavplubix <tavplubix@gmail.com>
Date: Tue, 29 Mar 2022 17:51:56 +0300
Subject: [PATCH 055/104] Revert "Fix enable LLVM for JIT compilation in CMake"

---
 contrib/llvm-cmake/CMakeLists.txt | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/contrib/llvm-cmake/CMakeLists.txt b/contrib/llvm-cmake/CMakeLists.txt
index 87c8a65510f..6ff07f0e016 100644
--- a/contrib/llvm-cmake/CMakeLists.txt
+++ b/contrib/llvm-cmake/CMakeLists.txt
@@ -1,9 +1,12 @@
-if (APPLE OR NOT ARCH_AMD64 OR SANITIZE STREQUAL "undefined")
-   set (ENABLE_EMBEDDED_COMPILER_DEFAULT OFF)
+# During cross-compilation in our CI we have to use llvm-tblgen and other building tools
+# tools to be build for host architecture and everything else for target architecture (e.g. AArch64)
+# Possible workaround is to use llvm-tblgen from some package...
+# But lets just enable LLVM for native builds
+if (CMAKE_CROSSCOMPILING OR SANITIZE STREQUAL "undefined")
+    set (ENABLE_EMBEDDED_COMPILER_DEFAULT OFF)
 else()
-   set (ENABLE_EMBEDDED_COMPILER_DEFAULT ON)
+    set (ENABLE_EMBEDDED_COMPILER_DEFAULT ON)
 endif()
-
 option (ENABLE_EMBEDDED_COMPILER "Enable support for 'compile_expressions' option for query execution" ${ENABLE_EMBEDDED_COMPILER_DEFAULT})
 
 if (NOT ENABLE_EMBEDDED_COMPILER)

From a2e950bd3be3a13fe10726cbac5c4628ffa12f9e Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Tue, 29 Mar 2022 17:34:50 +0200
Subject: [PATCH 056/104] Fix

---
 src/Parsers/ExpressionListParsers.cpp         | 22 ++++++++++---------
 src/Parsers/ExpressionListParsers.h           | 11 +++++-----
 .../02007_test_any_all_operators.reference    |  4 ++++
 .../02007_test_any_all_operators.sql          |  2 ++
 4 files changed, 23 insertions(+), 16 deletions(-)

diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp
index 13af308736b..e78f0e2bb56 100644
--- a/src/Parsers/ExpressionListParsers.cpp
+++ b/src/Parsers/ExpressionListParsers.cpp
@@ -326,17 +326,19 @@ bool ParserLeftAssociativeBinaryOperatorList::parseImpl(Pos & pos, ASTPtr & node
             /// function arguments
             auto exp_list = std::make_shared<ASTExpressionList>();
 
-            ASTPtr elem;
             SubqueryFunctionType subquery_function_type = SubqueryFunctionType::NONE;
-            if (allow_any_all_operators && ParserKeyword("ANY").ignore(pos, expected))
-                subquery_function_type = SubqueryFunctionType::ANY;
-            else if (allow_any_all_operators && ParserKeyword("ALL").ignore(pos, expected))
-                subquery_function_type = SubqueryFunctionType::ALL;
-            else if (!(remaining_elem_parser ? remaining_elem_parser : first_elem_parser)->parse(pos, elem, expected))
-                return false;
+            if (ParserKeyword("ANY").ignore(pos, expected))
+                 subquery_function_type = SubqueryFunctionType::ANY;
+            else if (ParserKeyword("ALL").ignore(pos, expected))
+                 subquery_function_type = SubqueryFunctionType::ALL;
 
-            if (subquery_function_type != SubqueryFunctionType::NONE && !ParserSubquery().parse(pos, elem, expected))
-                return false;
+            ASTPtr elem;
+            if (!ParserSubquery().parse(pos, elem, expected))
+            {
+                subquery_function_type = SubqueryFunctionType::NONE;
+                if (!(remaining_elem_parser ? remaining_elem_parser : first_elem_parser)->parse(pos, elem, expected))
+                    return false;
+            }
 
             /// the first argument of the function is the previous element, the second is the next one
             function->name = it[1];
@@ -346,7 +348,7 @@ bool ParserLeftAssociativeBinaryOperatorList::parseImpl(Pos & pos, ASTPtr & node
             exp_list->children.push_back(node);
             exp_list->children.push_back(elem);
 
-            if (allow_any_all_operators && subquery_function_type != SubqueryFunctionType::NONE && !modifyAST(function, subquery_function_type))
+            if (subquery_function_type != SubqueryFunctionType::NONE && !modifyAST(function, subquery_function_type))
                 return false;
 
             /** special exception for the access operator to the element of the array `x[y]`, which
diff --git a/src/Parsers/ExpressionListParsers.h b/src/Parsers/ExpressionListParsers.h
index 86d0fd0f861..4cd5e5114a1 100644
--- a/src/Parsers/ExpressionListParsers.h
+++ b/src/Parsers/ExpressionListParsers.h
@@ -121,8 +121,6 @@ private:
     Operators_t overlapping_operators_to_skip = { (const char *[]){ nullptr } };
     ParserPtr first_elem_parser;
     ParserPtr remaining_elem_parser;
-    /// =, !=, <, > ALL (subquery) / ANY (subquery)
-    bool allow_any_all_operators = false;
 
 public:
     /** `operators_` - allowed operators and their corresponding functions
@@ -132,10 +130,11 @@ public:
     {
     }
 
-    ParserLeftAssociativeBinaryOperatorList(Operators_t operators_,
-            Operators_t overlapping_operators_to_skip_, ParserPtr && first_elem_parser_, bool allow_any_all_operators_ = false)
+    ParserLeftAssociativeBinaryOperatorList(
+        Operators_t operators_, Operators_t overlapping_operators_to_skip_,
+        ParserPtr && first_elem_parser_)
         : operators(operators_), overlapping_operators_to_skip(overlapping_operators_to_skip_),
-          first_elem_parser(std::move(first_elem_parser_)), allow_any_all_operators(allow_any_all_operators_)
+          first_elem_parser(std::move(first_elem_parser_))
     {
     }
 
@@ -352,7 +351,7 @@ private:
     static const char * operators[];
     static const char * overlapping_operators_to_skip[];
     ParserLeftAssociativeBinaryOperatorList operator_parser {operators,
-        overlapping_operators_to_skip, std::make_unique<ParserBetweenExpression>(), true};
+    overlapping_operators_to_skip, std::make_unique<ParserBetweenExpression>()};
 
 protected:
     const char * getName() const  override{ return "comparison expression"; }
diff --git a/tests/queries/0_stateless/02007_test_any_all_operators.reference b/tests/queries/0_stateless/02007_test_any_all_operators.reference
index a232320d15c..69436560ff1 100644
--- a/tests/queries/0_stateless/02007_test_any_all_operators.reference
+++ b/tests/queries/0_stateless/02007_test_any_all_operators.reference
@@ -49,3 +49,7 @@ select 11 > all (select 11 from numbers(10));
 0
 select 11 >= all (select 11 from numbers(10));
 1
+select sum(number) = any(number) from numbers(1) group by number;
+1
+select 1 == any (1);
+1
diff --git a/tests/queries/0_stateless/02007_test_any_all_operators.sql b/tests/queries/0_stateless/02007_test_any_all_operators.sql
index 10d7325afca..dd539bcbc5c 100644
--- a/tests/queries/0_stateless/02007_test_any_all_operators.sql
+++ b/tests/queries/0_stateless/02007_test_any_all_operators.sql
@@ -24,3 +24,5 @@ select 11 <= all (select number from numbers(11));
 select 11 < all (select 11 from numbers(10));
 select 11 > all (select 11 from numbers(10));
 select 11 >= all (select 11 from numbers(10));
+select sum(number) = any(number) from numbers(1) group by number;
+select 1 == any (1);

From c6c0c91b39ad30e4d243f0a14125ab534040d1d2 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Tue, 29 Mar 2022 18:20:20 +0200
Subject: [PATCH 057/104] Followup

---
 programs/server/Server.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp
index a9db14665eb..fca5b39d2da 100644
--- a/programs/server/Server.cpp
+++ b/programs/server/Server.cpp
@@ -46,6 +46,7 @@
 #include <Core/ServerUUID.h>
 #include <IO/HTTPCommon.h>
 #include <IO/ReadHelpers.h>
+#include <IO/ReadBufferFromFile.h>
 #include <IO/IOThreadPool.h>
 #include <IO/UseSSL.h>
 #include <Interpreters/AsynchronousMetrics.h>
@@ -507,8 +508,6 @@ void checkForUsersNotInMainConfig(
     }
 }
 
-}
-
 /// Unused in other builds
 #if defined(OS_LINUX)
 static String readString(const String & path)

From 27390799c186c0f2b69d882a983d4f36bf9945a5 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Tue, 29 Mar 2022 18:23:18 +0200
Subject: [PATCH 058/104] Fix black

---
 tests/ci/docker_images_check.py                        | 1 +
 tests/integration/test_allowed_url_from_config/test.py | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py
index ef6d369bf4f..4277ef1e15f 100644
--- a/tests/ci/docker_images_check.py
+++ b/tests/ci/docker_images_check.py
@@ -465,5 +465,6 @@ def main():
     if status == "error":
         sys.exit(1)
 
+
 if __name__ == "__main__":
     main()
diff --git a/tests/integration/test_allowed_url_from_config/test.py b/tests/integration/test_allowed_url_from_config/test.py
index 13f3929902d..01a2a500ebf 100644
--- a/tests/integration/test_allowed_url_from_config/test.py
+++ b/tests/integration/test_allowed_url_from_config/test.py
@@ -280,4 +280,4 @@ def test_HDFS(start_cluster):
 
 def test_schema_inference(start_cluster):
     error = node7.query_and_get_error("desc url('http://test.com`, 'TSVRaw'')")
-    assert(error.find('ReadWriteBufferFromHTTPBase') == -1)
+    assert error.find("ReadWriteBufferFromHTTPBase") == -1

From 7da5d4f1458be99dab47d5228e5ace33a2e0af27 Mon Sep 17 00:00:00 2001
From: Anton Kozlov <anton@cloudflare.com>
Date: Tue, 29 Mar 2022 17:27:12 +0100
Subject: [PATCH 059/104] Updated a comment that clarifies the expected error

---
 tests/queries/0_stateless/02167_format_from_file_extension.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02167_format_from_file_extension.sh b/tests/queries/0_stateless/02167_format_from_file_extension.sh
index 3d9e0a080cc..4b88788d9e0 100755
--- a/tests/queries/0_stateless/02167_format_from_file_extension.sh
+++ b/tests/queries/0_stateless/02167_format_from_file_extension.sh
@@ -7,7 +7,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 function cleanup()
 {
-    user_files_path=$(clickhouse-client --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}')
+    # this command expects an error message like 'Code: 107. DB::Exception: Received <...> nonexist.txt doesn't exist. (FILE_DOESNT_EXIST)'
+    user_files_path=$($CLICKHOUSE_CLIENT --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep FILE_DOESNT_EXIST | awk '{gsub("/nonexist.txt","",$9); print $9}')
     rm $user_files_path/test_02167.*
 }
 trap cleanup EXIT

From d59941e4f6ccd623b1f938460e6a712dd69a73d0 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Tue, 29 Mar 2022 19:15:25 +0200
Subject: [PATCH 060/104] Style

---
 tests/ci/commit_status_helper.py                       | 1 +
 tests/integration/test_allowed_url_from_config/test.py | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/ci/commit_status_helper.py b/tests/ci/commit_status_helper.py
index 53dfb145b44..4c79d5ace07 100644
--- a/tests/ci/commit_status_helper.py
+++ b/tests/ci/commit_status_helper.py
@@ -60,6 +60,7 @@ def post_commit_status_to_file(file_path, description, state, report_url):
         out = csv.writer(f, delimiter="\t")
         out.writerow([state, report_url, description])
 
+
 def post_label(gh, pr_info, label_name):
     repo = gh.get_repo(GITHUB_REPOSITORY)
     pull_request = repo.get_pull(pr_info.number)
diff --git a/tests/integration/test_allowed_url_from_config/test.py b/tests/integration/test_allowed_url_from_config/test.py
index 13f3929902d..01a2a500ebf 100644
--- a/tests/integration/test_allowed_url_from_config/test.py
+++ b/tests/integration/test_allowed_url_from_config/test.py
@@ -280,4 +280,4 @@ def test_HDFS(start_cluster):
 
 def test_schema_inference(start_cluster):
     error = node7.query_and_get_error("desc url('http://test.com`, 'TSVRaw'')")
-    assert(error.find('ReadWriteBufferFromHTTPBase') == -1)
+    assert error.find("ReadWriteBufferFromHTTPBase") == -1

From 9220bedb7dc57a1f0ee2be1999610fad75ea674a Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Tue, 29 Mar 2022 19:28:18 +0200
Subject: [PATCH 061/104] Add submodule changed

---
 tests/ci/commit_status_helper.py |  5 +++--
 tests/ci/pr_info.py              |  9 +++++++++
 tests/ci/run_check.py            | 12 +++++++++---
 3 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/tests/ci/commit_status_helper.py b/tests/ci/commit_status_helper.py
index 4c79d5ace07..b4a3e49372f 100644
--- a/tests/ci/commit_status_helper.py
+++ b/tests/ci/commit_status_helper.py
@@ -61,7 +61,8 @@ def post_commit_status_to_file(file_path, description, state, report_url):
         out.writerow([state, report_url, description])
 
 
-def post_label(gh, pr_info, label_name):
+def post_labels(gh, pr_info, labels_names):
     repo = gh.get_repo(GITHUB_REPOSITORY)
     pull_request = repo.get_pull(pr_info.number)
-    pull_request.add_to_labels(label_name)
+    for label in labels_names:
+        pull_request.add_to_labels(label)
diff --git a/tests/ci/pr_info.py b/tests/ci/pr_info.py
index ee4399792ae..0de4ec89124 100644
--- a/tests/ci/pr_info.py
+++ b/tests/ci/pr_info.py
@@ -236,6 +236,15 @@ class PRInfo:
                 return True
         return False
 
+    def has_changes_in_submodules(self):
+        if self.changed_files is None or not self.changed_files:
+            return True
+
+        for f in self.changed_files:
+            if "contrib" in f:
+                return True
+        return False
+
     def can_skip_builds_and_use_version_from_master(self):
         # TODO: See a broken loop
         if "force tests" in self.labels:
diff --git a/tests/ci/run_check.py b/tests/ci/run_check.py
index 608a708b04b..f0100082c71 100644
--- a/tests/ci/run_check.py
+++ b/tests/ci/run_check.py
@@ -8,7 +8,7 @@ from github import Github
 from env_helper import GITHUB_RUN_URL, GITHUB_REPOSITORY, GITHUB_SERVER_URL
 from pr_info import PRInfo
 from get_robot_token import get_best_robot_token
-from commit_status_helper import get_commit, post_label
+from commit_status_helper import get_commit, post_labels
 
 NAME = "Run Check (actions)"
 
@@ -22,6 +22,7 @@ OK_SKIP_LABELS = {"release", "pr-backport", "pr-cherrypick"}
 CAN_BE_TESTED_LABEL = "can be tested"
 DO_NOT_TEST_LABEL = "do not test"
 FORCE_TESTS_LABEL = "force tests"
+SUBMODULE_CHANGED_LABEL = "submodule changed"
 
 # Individual trusted contirbutors who are not in any trusted organization.
 # Can be changed in runtime: we will append users that we learned to be in
@@ -227,15 +228,20 @@ def check_pr_description(pr_info):
 if __name__ == "__main__":
     logging.basicConfig(level=logging.INFO)
 
-    pr_info = PRInfo(need_orgs=True, pr_event_from_api=True)
+    pr_info = PRInfo(need_orgs=True, pr_event_from_api=True, need_changed_files=True)
     can_run, description, labels_state = should_run_checks_for_pr(pr_info)
     gh = Github(get_best_robot_token())
     commit = get_commit(gh, pr_info.sha)
 
     description_report, category = check_pr_description(pr_info)
+    pr_labels = []
     if category in MAP_CATEGORY_TO_LABEL:
-        post_label(gh, pr_info, MAP_CATEGORY_TO_LABEL[category])
+        pr_labels.append(MAP_CATEGORY_TO_LABEL[category])
 
+    if pr_info.has_changes_in_submodules():
+        pr_labels.append(SUBMODULE_CHANGED_LABEL)
+
+    post_labels(gh, pr_info, pr_labels)
     if description_report:
         print("::notice ::Cannot run, description does not match the template")
         logging.info(

From 999558f00fa01605c42eacd2f85243cf74291e4c Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Tue, 29 Mar 2022 19:28:45 +0200
Subject: [PATCH 062/104] Try fake update submodule

---
 contrib/NuRaft | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/NuRaft b/contrib/NuRaft
index 1707a7572aa..7c6c6961347 160000
--- a/contrib/NuRaft
+++ b/contrib/NuRaft
@@ -1 +1 @@
-Subproject commit 1707a7572aa66ec5d0a2dbe2bf5effa3352e6b2d
+Subproject commit 7c6c696134706eeef274787c9282e40416dc03e8

From 1a8b2c9637c53bef291c0ab33866f84ef437f2e4 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Tue, 29 Mar 2022 19:50:06 +0200
Subject: [PATCH 063/104] Remove submodule changed

---
 tests/ci/commit_status_helper.py |  7 +++++++
 tests/ci/run_check.py            | 12 +++++++++---
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/tests/ci/commit_status_helper.py b/tests/ci/commit_status_helper.py
index b4a3e49372f..a53ce6715d5 100644
--- a/tests/ci/commit_status_helper.py
+++ b/tests/ci/commit_status_helper.py
@@ -61,6 +61,13 @@ def post_commit_status_to_file(file_path, description, state, report_url):
         out.writerow([state, report_url, description])
 
 
+def remove_labels(gh, pr_info, labels_names):
+    repo = gh.get_repo(GITHUB_REPOSITORY)
+    pull_request = repo.get_pull(pr_info.number)
+    for label in labels_names:
+        pull_request.remove_from_labels(label)
+
+
 def post_labels(gh, pr_info, labels_names):
     repo = gh.get_repo(GITHUB_REPOSITORY)
     pull_request = repo.get_pull(pr_info.number)
diff --git a/tests/ci/run_check.py b/tests/ci/run_check.py
index f0100082c71..0138808c6ae 100644
--- a/tests/ci/run_check.py
+++ b/tests/ci/run_check.py
@@ -8,7 +8,7 @@ from github import Github
 from env_helper import GITHUB_RUN_URL, GITHUB_REPOSITORY, GITHUB_SERVER_URL
 from pr_info import PRInfo
 from get_robot_token import get_best_robot_token
-from commit_status_helper import get_commit, post_labels
+from commit_status_helper import get_commit, post_labels, remove_labels
 
 NAME = "Run Check (actions)"
 
@@ -235,13 +235,19 @@ if __name__ == "__main__":
 
     description_report, category = check_pr_description(pr_info)
     pr_labels = []
-    if category in MAP_CATEGORY_TO_LABEL:
+    if (
+        category in MAP_CATEGORY_TO_LABEL
+        and MAP_CATEGORY_TO_LABEL[category] not in pr_info.labels
+    ):
         pr_labels.append(MAP_CATEGORY_TO_LABEL[category])
 
     if pr_info.has_changes_in_submodules():
         pr_labels.append(SUBMODULE_CHANGED_LABEL)
+    elif SUBMODULE_CHANGED_LABEL in pr_info.labels:
+        remove_labels(gh, pr_info, [SUBMODULE_CHANGED_LABEL])
 
-    post_labels(gh, pr_info, pr_labels)
+    if pr_labels:
+        post_labels(gh, pr_info, pr_labels)
     if description_report:
         print("::notice ::Cannot run, description does not match the template")
         logging.info(

From 2e4ab305dd2a6951ad045b6cbdb920730710ddff Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Tue, 29 Mar 2022 19:50:13 +0200
Subject: [PATCH 064/104] Revert "Try fake update submodule"

This reverts commit 999558f00fa01605c42eacd2f85243cf74291e4c.
---
 contrib/NuRaft | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/NuRaft b/contrib/NuRaft
index 7c6c6961347..1707a7572aa 160000
--- a/contrib/NuRaft
+++ b/contrib/NuRaft
@@ -1 +1 @@
-Subproject commit 7c6c696134706eeef274787c9282e40416dc03e8
+Subproject commit 1707a7572aa66ec5d0a2dbe2bf5effa3352e6b2d

From 74cafe154a4e707d7f21ed7c39aa8c0e04e08927 Mon Sep 17 00:00:00 2001
From: Anton Popov <ap@clickhouse.com>
Date: Tue, 29 Mar 2022 19:58:59 +0200
Subject: [PATCH 065/104] fix flaky test 02246_async_insert_quota

---
 tests/queries/0_stateless/02246_async_insert_quota.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02246_async_insert_quota.sh b/tests/queries/0_stateless/02246_async_insert_quota.sh
index 9fc4df4660c..4da93f94f19 100755
--- a/tests/queries/0_stateless/02246_async_insert_quota.sh
+++ b/tests/queries/0_stateless/02246_async_insert_quota.sh
@@ -16,7 +16,7 @@ ${CLICKHOUSE_CLIENT} -q "CREATE ROLE r02246"
 ${CLICKHOUSE_CLIENT} -q "CREATE USER u02246"
 ${CLICKHOUSE_CLIENT} -q "GRANT INSERT ON async_inserts_02246 TO r02246"
 ${CLICKHOUSE_CLIENT} -q "GRANT r02246 to u02246"
-${CLICKHOUSE_CLIENT} -q "CREATE QUOTA q02246 FOR INTERVAL 1 HOUR MAX QUERY INSERTS = 2 TO r02246"
+${CLICKHOUSE_CLIENT} -q "CREATE QUOTA q02246 FOR INTERVAL 100 YEAR MAX QUERY INSERTS = 2 TO r02246"
 
 ${CLICKHOUSE_CLIENT} --user u02246 --async_insert 1 -q "INSERT INTO async_inserts_02246 VALUES (1, 'a')"
 ${CLICKHOUSE_CLIENT} --user u02246 --async_insert 1 -q "INSERT INTO async_inserts_02246 VALUES (2, 'b')"

From 476c7f9d7e10e882a085a2b2a08cec43ed951718 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Tue, 29 Mar 2022 18:32:28 +0000
Subject: [PATCH 066/104] Update ci checks server.

---
 tests/ci/clickhouse_helper.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/tests/ci/clickhouse_helper.py b/tests/ci/clickhouse_helper.py
index 7ccbcb4a47e..3731037bf4c 100644
--- a/tests/ci/clickhouse_helper.py
+++ b/tests/ci/clickhouse_helper.py
@@ -10,13 +10,13 @@ from get_robot_token import get_parameter_from_ssm
 class ClickHouseHelper:
     def __init__(self, url=None):
         if url is None:
-            self.url = get_parameter_from_ssm("clickhouse-test-stat-url2")
-            self.auth = {
-                "X-ClickHouse-User": get_parameter_from_ssm(
-                    "clickhouse-test-stat-login2"
-                ),
-                "X-ClickHouse-Key": "",
-            }
+            self.url = get_parameter_from_ssm("clickhouse-test-stat-url")
+
+        self.url = url
+        self.auth = {
+            'X-ClickHouse-User': user if user is not None else get_parameter_from_ssm("clickhouse-test-stat-login"),
+            'X-ClickHouse-Key': password if password is not None else get_parameter_from_ssm("clickhouse-test-stat-password")
+        }
 
     @staticmethod
     def _insert_json_str_info_impl(url, auth, db, table, json_str):

From 3995a52da61cb727f9604b5d0114adc500d4bced Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Tue, 29 Mar 2022 18:36:36 +0000
Subject: [PATCH 067/104] Fix check.

---
 tests/ci/clickhouse_helper.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/ci/clickhouse_helper.py b/tests/ci/clickhouse_helper.py
index 3731037bf4c..ea99b97c7d8 100644
--- a/tests/ci/clickhouse_helper.py
+++ b/tests/ci/clickhouse_helper.py
@@ -8,7 +8,7 @@ from get_robot_token import get_parameter_from_ssm
 
 
 class ClickHouseHelper:
-    def __init__(self, url=None):
+    def __init__(self, url=None, user=None, password=None):
         if url is None:
             self.url = get_parameter_from_ssm("clickhouse-test-stat-url")
 

From bb82f77477038fa9623f40c21d81ed9899eff429 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Tue, 29 Mar 2022 18:46:48 +0000
Subject: [PATCH 068/104] Fix check.

---
 tests/ci/clickhouse_helper.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/ci/clickhouse_helper.py b/tests/ci/clickhouse_helper.py
index ea99b97c7d8..093d5541362 100644
--- a/tests/ci/clickhouse_helper.py
+++ b/tests/ci/clickhouse_helper.py
@@ -8,14 +8,14 @@ from get_robot_token import get_parameter_from_ssm
 
 
 class ClickHouseHelper:
-    def __init__(self, url=None, user=None, password=None):
+    def __init__(self, url=None):
         if url is None:
-            self.url = get_parameter_from_ssm("clickhouse-test-stat-url")
+            url = get_parameter_from_ssm("clickhouse-test-stat-url")
 
         self.url = url
         self.auth = {
-            'X-ClickHouse-User': user if user is not None else get_parameter_from_ssm("clickhouse-test-stat-login"),
-            'X-ClickHouse-Key': password if password is not None else get_parameter_from_ssm("clickhouse-test-stat-password")
+            'X-ClickHouse-User': get_parameter_from_ssm("clickhouse-test-stat-login"),
+            'X-ClickHouse-Key': get_parameter_from_ssm("clickhouse-test-stat-password")
         }
 
     @staticmethod

From 081073763748106499d3279894f99705ab332700 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Tue, 29 Mar 2022 18:58:16 +0000
Subject: [PATCH 069/104] Fix style

---
 tests/ci/clickhouse_helper.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/ci/clickhouse_helper.py b/tests/ci/clickhouse_helper.py
index 093d5541362..6627a3974d1 100644
--- a/tests/ci/clickhouse_helper.py
+++ b/tests/ci/clickhouse_helper.py
@@ -14,8 +14,8 @@ class ClickHouseHelper:
 
         self.url = url
         self.auth = {
-            'X-ClickHouse-User': get_parameter_from_ssm("clickhouse-test-stat-login"),
-            'X-ClickHouse-Key': get_parameter_from_ssm("clickhouse-test-stat-password")
+            "X-ClickHouse-User": get_parameter_from_ssm("clickhouse-test-stat-login"),
+            "X-ClickHouse-Key": get_parameter_from_ssm("clickhouse-test-stat-password")
         }
 
     @staticmethod

From 3849e63ab186d86a29ed0c77e1524800ff87bafd Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Tue, 29 Mar 2022 19:06:50 +0000
Subject: [PATCH 070/104] Change database

---
 docker/test/performance-comparison/compare.sh | 2 +-
 tests/ci/clickhouse_helper.py                 | 2 +-
 tests/ci/compatibility_check.py               | 2 +-
 tests/ci/docker_images_check.py               | 2 +-
 tests/ci/docker_manifests_merge.py            | 2 +-
 tests/ci/docs_check.py                        | 2 +-
 tests/ci/fast_test_check.py                   | 2 +-
 tests/ci/functional_test_check.py             | 2 +-
 tests/ci/integration_test_check.py            | 2 +-
 tests/ci/keeper_jepsen_check.py               | 2 +-
 tests/ci/split_build_smoke_check.py           | 2 +-
 tests/ci/stress_check.py                      | 2 +-
 tests/ci/style_check.py                       | 2 +-
 tests/ci/unit_tests_check.py                  | 2 +-
 14 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh
index 54f71ce05bb..cdfa080a475 100755
--- a/docker/test/performance-comparison/compare.sh
+++ b/docker/test/performance-comparison/compare.sh
@@ -1378,7 +1378,7 @@ $REF_SHA	$SHA_TO_TEST	$(numactl --hardware | sed -n 's/^available:[[:space:]]\+/
 EOF
 
     # Also insert some data about the check into the CI checks table.
-    "${client[@]}" --query "INSERT INTO "'"'"gh-data"'"'".checks FORMAT TSVWithNamesAndTypes" \
+    "${client[@]}" --query "INSERT INTO "'"'"default"'"'".checks FORMAT TSVWithNamesAndTypes" \
         < ci-checks.tsv
 
     set -x
diff --git a/tests/ci/clickhouse_helper.py b/tests/ci/clickhouse_helper.py
index 6627a3974d1..218aaca8b91 100644
--- a/tests/ci/clickhouse_helper.py
+++ b/tests/ci/clickhouse_helper.py
@@ -179,7 +179,7 @@ def mark_flaky_tests(clickhouse_helper, check_name, test_results):
             check_name=check_name
         )
 
-        tests_data = clickhouse_helper.select_json_each_row("gh-data", query)
+        tests_data = clickhouse_helper.select_json_each_row("default", query)
         master_failed_tests = {row["test_name"] for row in tests_data}
         logging.info("Found flaky tests: %s", ", ".join(master_failed_tests))
 
diff --git a/tests/ci/compatibility_check.py b/tests/ci/compatibility_check.py
index d546fabf231..a6846257cce 100644
--- a/tests/ci/compatibility_check.py
+++ b/tests/ci/compatibility_check.py
@@ -197,4 +197,4 @@ if __name__ == "__main__":
         report_url,
         CHECK_NAME,
     )
-    ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events)
+    ch_helper.insert_events_into(db="default", table="checks", events=prepared_events)
diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py
index 3d0cc468aec..44c4623b469 100644
--- a/tests/ci/docker_images_check.py
+++ b/tests/ci/docker_images_check.py
@@ -459,7 +459,7 @@ def main():
         NAME,
     )
     ch_helper = ClickHouseHelper()
-    ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events)
+    ch_helper.insert_events_into(db="default", table="checks", events=prepared_events)
 
 
 if __name__ == "__main__":
diff --git a/tests/ci/docker_manifests_merge.py b/tests/ci/docker_manifests_merge.py
index 8bd50819877..9371440346e 100644
--- a/tests/ci/docker_manifests_merge.py
+++ b/tests/ci/docker_manifests_merge.py
@@ -234,7 +234,7 @@ def main():
         NAME,
     )
     ch_helper = ClickHouseHelper()
-    ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events)
+    ch_helper.insert_events_into(db="default", table="checks", events=prepared_events)
 
 
 if __name__ == "__main__":
diff --git a/tests/ci/docs_check.py b/tests/ci/docs_check.py
index 58678b160a4..a7de66859e2 100644
--- a/tests/ci/docs_check.py
+++ b/tests/ci/docs_check.py
@@ -114,4 +114,4 @@ if __name__ == "__main__":
         report_url,
         NAME,
     )
-    ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events)
+    ch_helper.insert_events_into(db="default", table="checks", events=prepared_events)
diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py
index 64e04594786..8c17c15b463 100644
--- a/tests/ci/fast_test_check.py
+++ b/tests/ci/fast_test_check.py
@@ -204,7 +204,7 @@ if __name__ == "__main__":
         report_url,
         NAME,
     )
-    ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events)
+    ch_helper.insert_events_into(db="default", table="checks", events=prepared_events)
 
     # Refuse other checks to run if fast test failed
     if state != "success":
diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py
index 52ec5a0f8e9..c5a44ba66d2 100644
--- a/tests/ci/functional_test_check.py
+++ b/tests/ci/functional_test_check.py
@@ -356,7 +356,7 @@ if __name__ == "__main__":
         report_url,
         check_name_with_group,
     )
-    ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events)
+    ch_helper.insert_events_into(db="default", table="checks", events=prepared_events)
 
     if state != "success":
         if "force-tests" in pr_info.labels:
diff --git a/tests/ci/integration_test_check.py b/tests/ci/integration_test_check.py
index 30009414d6e..636fbe664ce 100644
--- a/tests/ci/integration_test_check.py
+++ b/tests/ci/integration_test_check.py
@@ -279,4 +279,4 @@ if __name__ == "__main__":
         report_url,
         check_name_with_group,
     )
-    ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events)
+    ch_helper.insert_events_into(db="default", table="checks", events=prepared_events)
diff --git a/tests/ci/keeper_jepsen_check.py b/tests/ci/keeper_jepsen_check.py
index 24d720e67ab..14c31927b75 100644
--- a/tests/ci/keeper_jepsen_check.py
+++ b/tests/ci/keeper_jepsen_check.py
@@ -271,5 +271,5 @@ if __name__ == "__main__":
         report_url,
         CHECK_NAME,
     )
-    ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events)
+    ch_helper.insert_events_into(db="default", table="checks", events=prepared_events)
     clear_autoscaling_group()
diff --git a/tests/ci/split_build_smoke_check.py b/tests/ci/split_build_smoke_check.py
index 41ba6c2fedb..f281bdc55a8 100644
--- a/tests/ci/split_build_smoke_check.py
+++ b/tests/ci/split_build_smoke_check.py
@@ -147,4 +147,4 @@ if __name__ == "__main__":
         report_url,
         CHECK_NAME,
     )
-    ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events)
+    ch_helper.insert_events_into(db="default", table="checks", events=prepared_events)
diff --git a/tests/ci/stress_check.py b/tests/ci/stress_check.py
index 32c181140e2..4ecdd249351 100644
--- a/tests/ci/stress_check.py
+++ b/tests/ci/stress_check.py
@@ -176,4 +176,4 @@ if __name__ == "__main__":
         report_url,
         check_name,
     )
-    ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events)
+    ch_helper.insert_events_into(db="default", table="checks", events=prepared_events)
diff --git a/tests/ci/style_check.py b/tests/ci/style_check.py
index 1b3037217c8..4c210f9beab 100644
--- a/tests/ci/style_check.py
+++ b/tests/ci/style_check.py
@@ -117,4 +117,4 @@ if __name__ == "__main__":
         report_url,
         NAME,
     )
-    ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events)
+    ch_helper.insert_events_into(db="default", table="checks", events=prepared_events)
diff --git a/tests/ci/unit_tests_check.py b/tests/ci/unit_tests_check.py
index 84c4faa822d..6db160d9a54 100644
--- a/tests/ci/unit_tests_check.py
+++ b/tests/ci/unit_tests_check.py
@@ -173,4 +173,4 @@ if __name__ == "__main__":
         report_url,
         check_name,
     )
-    ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events)
+    ch_helper.insert_events_into(db="default", table="checks", events=prepared_events)

From 6191eaa9839e367473a933860a6cd74693585de0 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Tue, 29 Mar 2022 23:12:06 +0200
Subject: [PATCH 071/104] fix some flaky tests

---
 tests/clickhouse-test                                           | 1 +
 .../00385_storage_file_and_clickhouse-local_app_long.sh         | 2 +-
 tests/queries/1_stateful/00152_insert_different_granularity.sql | 2 +-
 tests/queries/1_stateful/00170_s3_cache.sql                     | 1 +
 4 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index 863e061085a..b00f22574b8 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -52,6 +52,7 @@ MESSAGES_TO_RETRY = [
     "DB::Exception: New table appeared in database being dropped or detached. Try again",
     "is already started to be removing by another replica right now",
     "DB::Exception: Cannot enqueue query",
+    "environment: line 1: wait_for: No record of process", # Something weird from bash internals, let's just retry
     "is executing longer than distributed_ddl_task_timeout" # FIXME
 ]
 
diff --git a/tests/queries/0_stateless/00385_storage_file_and_clickhouse-local_app_long.sh b/tests/queries/0_stateless/00385_storage_file_and_clickhouse-local_app_long.sh
index a78cdd445cb..b6cc270994f 100755
--- a/tests/queries/0_stateless/00385_storage_file_and_clickhouse-local_app_long.sh
+++ b/tests/queries/0_stateless/00385_storage_file_and_clickhouse-local_app_long.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long
+# Tags: long, no-random-settings
 
 set -e
 
diff --git a/tests/queries/1_stateful/00152_insert_different_granularity.sql b/tests/queries/1_stateful/00152_insert_different_granularity.sql
index da99173ed96..6415cdad8a5 100644
--- a/tests/queries/1_stateful/00152_insert_different_granularity.sql
+++ b/tests/queries/1_stateful/00152_insert_different_granularity.sql
@@ -1,4 +1,4 @@
--- Tags: no-tsan, no-replicated-database
+-- Tags: no-tsan, no-replicated-database, no-parallel
 -- Tag no-replicated-database: Fails due to additional replicas or shards
 
 DROP TABLE IF EXISTS fixed_granularity_table;
diff --git a/tests/queries/1_stateful/00170_s3_cache.sql b/tests/queries/1_stateful/00170_s3_cache.sql
index af3fd402596..5d616916048 100644
--- a/tests/queries/1_stateful/00170_s3_cache.sql
+++ b/tests/queries/1_stateful/00170_s3_cache.sql
@@ -1,3 +1,4 @@
+-- Tags: no-parallel
 -- { echo }
 SET max_memory_usage='20G';
 SELECT count() FROM test.hits_s3;

From 16bb4c4ad06178149b7398ec36cbd5791ac017c0 Mon Sep 17 00:00:00 2001
From: taiyang-li <654010905@qq.com>
Date: Wed, 30 Mar 2022 15:33:59 +0800
Subject: [PATCH 072/104] respect remote_url_allow_hosts for hive

---
 src/Storages/Hive/StorageHive.cpp | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/Storages/Hive/StorageHive.cpp b/src/Storages/Hive/StorageHive.cpp
index b95f38d4886..c62ba491040 100644
--- a/src/Storages/Hive/StorageHive.cpp
+++ b/src/Storages/Hive/StorageHive.cpp
@@ -289,6 +289,7 @@ StorageHive::StorageHive(
     , partition_by_ast(partition_by_ast_)
     , storage_settings(std::move(storage_settings_))
 {
+    /// Check hive metastore url.
     getContext()->getRemoteHostFilter().checkURL(Poco::URI(hive_metastore_url));
 
     StorageInMemoryMetadata storage_metadata;
@@ -304,11 +305,13 @@ void StorageHive::lazyInitialize()
     if (has_initialized)
         return;
 
-
     auto hive_metastore_client = HiveMetastoreClientFactory::instance().getOrCreate(hive_metastore_url, getContext());
     auto hive_table_metadata = hive_metastore_client->getHiveTable(hive_database, hive_table);
 
     hdfs_namenode_url = getNameNodeUrl(hive_table_metadata->sd.location);
+    /// Check HDFS namenode url.
+    getContext()->getRemoteHostFilter().checkURL(Poco::URI(hdfs_namenode_url));
+
     table_schema = hive_table_metadata->sd.cols;
 
     FileFormat hdfs_file_format = IHiveFile::toFileFormat(hive_table_metadata->sd.inputFormat);
@@ -319,10 +322,8 @@ void StorageHive::lazyInitialize()
             format_name = "HiveText";
             break;
         case FileFormat::RC_FILE:
-            /// TODO to be implemented
             throw Exception("Unsopported hive format rc_file", ErrorCodes::NOT_IMPLEMENTED);
         case FileFormat::SEQUENCE_FILE:
-            /// TODO to be implemented
             throw Exception("Unsopported hive format sequence_file", ErrorCodes::NOT_IMPLEMENTED);
         case FileFormat::AVRO:
             format_name = "Avro";

From f2779eac3e68fbe3704ce2480be9a749b258ba7e Mon Sep 17 00:00:00 2001
From: taiyang-li <654010905@qq.com>
Date: Wed, 30 Mar 2022 15:52:34 +0800
Subject: [PATCH 073/104] fix black_style

---
 tests/ci/clickhouse_helper.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/ci/clickhouse_helper.py b/tests/ci/clickhouse_helper.py
index 218aaca8b91..d52b6262a78 100644
--- a/tests/ci/clickhouse_helper.py
+++ b/tests/ci/clickhouse_helper.py
@@ -15,7 +15,7 @@ class ClickHouseHelper:
         self.url = url
         self.auth = {
             "X-ClickHouse-User": get_parameter_from_ssm("clickhouse-test-stat-login"),
-            "X-ClickHouse-Key": get_parameter_from_ssm("clickhouse-test-stat-password")
+            "X-ClickHouse-Key": get_parameter_from_ssm("clickhouse-test-stat-password"),
         }
 
     @staticmethod

From 7d140c9a59c1760d7f89d6b1cbc3ef149f3324e5 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Tue, 29 Mar 2022 22:30:50 +0300
Subject: [PATCH 074/104] clickhouse-test: increase timeout for obtaining
 processlist

Sometimes 30 seconds is not enough on CI [1]:

    2022.03.29 04:46:09.805406 [ 3885 ] {ae06ad04-f9fa-44d4-adeb-9e91aaf9bfa9} <Debug> executeQuery: (from [::1]:33506)  SELECT * FROM system.processes WHERE query NOT LIKE '%system.processes%' AND Settings['log_comment'] = '02016_order_by_with_fill_monotonic_functions_removal.sql' AND current_database = 'test_7fqp6w'
    2022.03.29 04:46:09.866100 [ 3885 ] {ae06ad04-f9fa-44d4-adeb-9e91aaf9bfa9} <Trace> ContextAccess (default): Access granted: SELECT(is_initial_query, user, query_id, address, port, initial_user, initial_query_id, initial_address, initial_port, interface, os_user, client_hostname, client_name, client_revision, client_version_major, client_version_minor, client_version_patch, http_method, http_user_agent, http_referer, forwarded_for, quota_key, distributed_depth, elapsed, is_cancelled, read_rows, read_bytes, total_rows_approx, written_rows, written_bytes, memory_usage, peak_memory_usage, query, thread_ids, ProfileEvents, Settings, current_database) ON system.processes
    2022.03.29 04:46:12.787395 [ 3885 ] {ae06ad04-f9fa-44d4-adeb-9e91aaf9bfa9} <Trace> InterpreterSelectQuery: FetchColumns -> Complete
    2022.03.29 04:46:19.749163 [ 3885 ] {ae06ad04-f9fa-44d4-adeb-9e91aaf9bfa9} <Test> ParallelFormattingOutputFormat: Parallel formatting is being used
    2022.03.29 04:46:37.923282 [ 3885 ] {ae06ad04-f9fa-44d4-adeb-9e91aaf9bfa9} <Information> executeQuery: Read 15 rows, 44.20 KiB in 28.117172383 sec., 0 rows/sec., 1.57 KiB/sec.
    2022.03.29 04:46:40.020586 [ 3885 ] {ae06ad04-f9fa-44d4-adeb-9e91aaf9bfa9} <Debug> DynamicQueryHandler: Done processing query
    2022.03.29 04:46:40.033535 [ 3885 ] {ae06ad04-f9fa-44d4-adeb-9e91aaf9bfa9} <Debug> MemoryTracker: Peak memory usage (for query): 4.00 MiB.

  [1]: https://s3.amazonaws.com/clickhouse-test-reports/32928/ddd5bebe555ce8feebcdd339e47fc45184c20dd1/stateless_tests__thread__actions__[1/3].html

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 tests/clickhouse-test | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index 863e061085a..5f9ab01cdcc 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -116,7 +116,7 @@ def clickhouse_execute_http(base_args, query, timeout=30, settings=None, default
 def clickhouse_execute(base_args, query, timeout=30, settings=None):
     return clickhouse_execute_http(base_args, query, timeout, settings).strip()
 
-def clickhouse_execute_json(base_args, query, timeout=30, settings=None):
+def clickhouse_execute_json(base_args, query, timeout=60, settings=None):
     data = clickhouse_execute_http(base_args, query, timeout, settings, 'JSONEachRow')
     if not data:
         return None

From b264bdcf5e65874e41457ff94d8d6fe853560c2a Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 20 Mar 2022 21:40:58 +0300
Subject: [PATCH 075/104] tests: wait for left queries in 00417_kill_query

CI: https://s3.amazonaws.com/clickhouse-test-reports/0/c244ee7cbb61fea384679e18a577ff579060288b/stateless_tests__release__s3_storage__actions_/runlog.log
Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 tests/queries/0_stateless/00417_kill_query.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/queries/0_stateless/00417_kill_query.sh b/tests/queries/0_stateless/00417_kill_query.sh
index 28f03fc9c37..e40eb5c3f80 100755
--- a/tests/queries/0_stateless/00417_kill_query.sh
+++ b/tests/queries/0_stateless/00417_kill_query.sh
@@ -21,3 +21,5 @@ $CLICKHOUSE_CLIENT -q "KILL QUERY WHERE 0 ASYNC"
 $CLICKHOUSE_CLIENT -q "KILL QUERY WHERE 0 FORMAT TabSeparated"
 $CLICKHOUSE_CLIENT -q "KILL QUERY WHERE 0 SYNC FORMAT TabSeparated"
 $CLICKHOUSE_CLIENT -q "KILL QUERY WHERE 1 TEST" &>/dev/null
+
+clickhouse_test_wait_queries 60

From 2a099d90fe5dcdd788123ab9e2e604d2dcb9d7ba Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Wed, 30 Mar 2022 10:45:13 +0300
Subject: [PATCH 076/104] tests: make
 02152_http_external_tables_memory_tracking more stable

After settings randomization, it can not reach max_untracked_memory,
i.e. due to max_block_size=67990 for example (like in [1]).

  [1]: https://s3.amazonaws.com/clickhouse-test-reports/35683/c609426d091ea8b9f2d37f8d7fcbf9e759cc94e5/stateless_tests__release__actions_.html

Fix this, by settings max_untracked_memory to 0, to track any
allocations.

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 .../0_stateless/02152_http_external_tables_memory_tracking.sh   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02152_http_external_tables_memory_tracking.sh b/tests/queries/0_stateless/02152_http_external_tables_memory_tracking.sh
index 44de0e15370..4132ac91ae4 100755
--- a/tests/queries/0_stateless/02152_http_external_tables_memory_tracking.sh
+++ b/tests/queries/0_stateless/02152_http_external_tables_memory_tracking.sh
@@ -23,7 +23,7 @@ function run_and_check()
     echo "Checking $*"
 
     # Run query with external table (implicit StorageMemory user)
-    $CLICKHOUSE_CURL -sS -F "s=@$tmp_file;" "$CLICKHOUSE_URL&s_structure=key+Int&query=SELECT+count()+FROM+s&memory_profiler_sample_probability=1&query_id=$query_id&$*" -o /dev/null
+    $CLICKHOUSE_CURL -sS -F "s=@$tmp_file;" "$CLICKHOUSE_URL&s_structure=key+Int&query=SELECT+count()+FROM+s&memory_profiler_sample_probability=1&max_untracked_memory=0&query_id=$query_id&$*" -o /dev/null
 
     ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" --data-binary @- <<<'SYSTEM FLUSH LOGS'
 

From 1173eab15edfa01aaba661135394d8132908b0a3 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Wed, 30 Mar 2022 08:15:20 +0000
Subject: [PATCH 077/104] Use same IOThreadPool for URL and S3

---
 src/IO/ParallelReadBuffer.cpp                 | 61 +++++++++----------
 src/IO/ParallelReadBuffer.h                   | 21 +++----
 src/Interpreters/threadPoolCallbackRunner.cpp | 48 +++++++++------
 src/Interpreters/threadPoolCallbackRunner.h   |  2 +-
 src/Storages/StorageS3.cpp                    | 55 +++++------------
 src/Storages/StorageURL.cpp                   | 30 +--------
 6 files changed, 82 insertions(+), 135 deletions(-)

diff --git a/src/IO/ParallelReadBuffer.cpp b/src/IO/ParallelReadBuffer.cpp
index 3ee62148c23..3a62be1511b 100644
--- a/src/IO/ParallelReadBuffer.cpp
+++ b/src/IO/ParallelReadBuffer.cpp
@@ -13,23 +13,39 @@ namespace ErrorCodes
 
 }
 
-ParallelReadBuffer::ParallelReadBuffer(
-    std::unique_ptr<ReadBufferFactory> reader_factory_,
-    ThreadPool * pool_,
-    size_t max_working_readers_,
-    WorkerSetup worker_setup_,
-    WorkerCleanup worker_cleanup_)
+ParallelReadBuffer::ParallelReadBuffer(std::unique_ptr<ReadBufferFactory> reader_factory_, ThreadPool * pool, size_t max_working_readers_)
     : SeekableReadBufferWithSize(nullptr, 0)
-    , pool(pool_)
     , max_working_readers(max_working_readers_)
+    , schedule(threadPoolCallbackRunner(*pool, getWorkerSetup(), getWorkerCleanup()))
     , reader_factory(std::move(reader_factory_))
-    , worker_setup(std::move(worker_setup_))
-    , worker_cleanup(std::move(worker_cleanup_))
 {
     std::unique_lock<std::mutex> lock{mutex};
     addReaders(lock);
 }
 
+
+ParallelReadBuffer::WorkerSetup ParallelReadBuffer::getWorkerSetup()
+{
+    return [this]
+    {
+        std::lock_guard lock{mutex};
+        ++active_working_reader;
+    };
+}
+
+ParallelReadBuffer::WorkerCleanup ParallelReadBuffer::getWorkerCleanup()
+{
+    return [this]
+    {
+        std::lock_guard lock{mutex};
+        --active_working_reader;
+        if (active_working_reader == 0)
+        {
+            readers_done.notify_all();
+        }
+    };
+}
+
 bool ParallelReadBuffer::addReaderToPool(std::unique_lock<std::mutex> & /*buffer_lock*/)
 {
     auto reader = reader_factory->getReader();
@@ -40,30 +56,8 @@ bool ParallelReadBuffer::addReaderToPool(std::unique_lock<std::mutex> & /*buffer
 
     auto worker = read_workers.emplace_back(std::make_shared<ReadWorker>(std::move(reader)));
 
-    pool->scheduleOrThrow(
-        [&, this, worker = std::move(worker)]() mutable
-        {
-            ThreadStatus thread_status;
+    schedule([this, worker = std::move(worker)]() mutable { readerThreadFunction(std::move(worker)); });
 
-            {
-                std::lock_guard lock{mutex};
-                ++active_working_reader;
-            }
-
-            SCOPE_EXIT({
-                worker_cleanup(thread_status);
-
-                std::lock_guard lock{mutex};
-                --active_working_reader;
-                if (active_working_reader == 0)
-                {
-                    readers_done.notify_all();
-                }
-            });
-            worker_setup(thread_status);
-
-            readerThreadFunction(std::move(worker));
-        });
     return true;
 }
 
@@ -237,7 +231,8 @@ void ParallelReadBuffer::readerThreadFunction(ReadWorkerPtr read_worker)
         while (!emergency_stop && !read_worker->cancel)
         {
             if (!read_worker->reader->next())
-                throw Exception(ErrorCodes::LOGICAL_ERROR, "Failed to read all the data from the reader, missing {} bytes", read_worker->bytes_left);
+                throw Exception(
+                    ErrorCodes::LOGICAL_ERROR, "Failed to read all the data from the reader, missing {} bytes", read_worker->bytes_left);
 
             if (emergency_stop || read_worker->cancel)
                 break;
diff --git a/src/IO/ParallelReadBuffer.h b/src/IO/ParallelReadBuffer.h
index 7a3f0cb3773..3759064c2a8 100644
--- a/src/IO/ParallelReadBuffer.h
+++ b/src/IO/ParallelReadBuffer.h
@@ -3,6 +3,7 @@
 #include <IO/BufferWithOwnMemory.h>
 #include <IO/ReadBuffer.h>
 #include <IO/SeekableReadBuffer.h>
+#include <Interpreters/threadPoolCallbackRunner.h>
 #include <Common/ArenaWithFreeLists.h>
 #include <Common/ThreadPool.h>
 
@@ -76,14 +77,7 @@ public:
         virtual std::optional<size_t> getTotalSize() = 0;
     };
 
-    using WorkerSetup = std::function<void(ThreadStatus &)>;
-    using WorkerCleanup = std::function<void(ThreadStatus &)>;
-    explicit ParallelReadBuffer(
-        std::unique_ptr<ReadBufferFactory> reader_factory_,
-        ThreadPool * pool,
-        size_t max_working_readers,
-        WorkerSetup worker_setup = [](ThreadStatus &){},
-        WorkerCleanup worker_cleanup = [](ThreadStatus &){});
+    explicit ParallelReadBuffer(std::unique_ptr<ReadBufferFactory> reader_factory_, ThreadPool * pool, size_t max_working_readers);
 
     ~ParallelReadBuffer() override { finishAndWait(); }
 
@@ -136,20 +130,23 @@ private:
     void onBackgroundException();
     void finishAndWait();
 
+    using WorkerSetup = std::function<void()>;
+    WorkerSetup getWorkerSetup();
+    using WorkerCleanup = std::function<void()>;
+    WorkerCleanup getWorkerCleanup();
+
     SynchronizedArenaWithFreeLists arena;
 
     Segment current_segment;
 
-    ThreadPool * pool;
     size_t max_working_readers;
     size_t active_working_reader{0};
     // Triggered when all reader workers are done
     std::condition_variable readers_done;
 
-    std::unique_ptr<ReadBufferFactory> reader_factory;
+    CallbackRunner schedule;
 
-    WorkerSetup worker_setup;
-    WorkerCleanup worker_cleanup;
+    std::unique_ptr<ReadBufferFactory> reader_factory;
 
     /**
      * FIFO queue of readers.
diff --git a/src/Interpreters/threadPoolCallbackRunner.cpp b/src/Interpreters/threadPoolCallbackRunner.cpp
index c580d352c49..58ce3789bc0 100644
--- a/src/Interpreters/threadPoolCallbackRunner.cpp
+++ b/src/Interpreters/threadPoolCallbackRunner.cpp
@@ -7,32 +7,40 @@
 namespace DB
 {
 
-CallbackRunner threadPoolCallbackRunner(ThreadPool & pool)
+CallbackRunner threadPoolCallbackRunner(ThreadPool & pool, std::function<void()> setup, std::function<void()> cleanup)
 {
-    return [pool = &pool, thread_group = CurrentThread::getGroup()](auto callback)
+    return [pool = &pool, thread_group = CurrentThread::getGroup(), setup = std::move(setup), cleanup = std::move(cleanup)](
+               auto callback) mutable
     {
-        pool->scheduleOrThrow([callback = std::move(callback), thread_group]()
-        {
-            if (thread_group)
-                CurrentThread::attachTo(thread_group);
+        pool->scheduleOrThrow(
+            [&, callback = std::move(callback), thread_group, setup = std::move(setup), cleanup = std::move(cleanup)]()
+            {
+                if (setup)
+                    setup();
 
-            SCOPE_EXIT_SAFE({
                 if (thread_group)
-                    CurrentThread::detachQueryIfNotDetached();
+                    CurrentThread::attachTo(thread_group);
 
-                /// After we detached from the thread_group, parent for memory_tracker inside ThreadStatus will be reset to it's parent.
-                /// Typically, it may be changes from Process to User.
-                /// Usually it could be ok, because thread pool task is executed before user-level memory tracker is destroyed.
-                /// However, thread could stay alive inside the thread pool, and it's ThreadStatus as well.
-                /// When, finally, we destroy the thread (and the ThreadStatus),
-                /// it can use memory tracker in the ~ThreadStatus in order to alloc/free untracked_memory,
-                /// and by this time user-level memory tracker may be already destroyed.
-                ///
-                /// As a work-around, reset memory tracker to total, which is always alive.
-                CurrentThread::get().memory_tracker.setParent(&total_memory_tracker);
+                SCOPE_EXIT_SAFE({
+                    if (thread_group)
+                        CurrentThread::detachQueryIfNotDetached();
+
+                    /// After we detached from the thread_group, parent for memory_tracker inside ThreadStatus will be reset to it's parent.
+                    /// Typically, it may be changes from Process to User.
+                    /// Usually it could be ok, because thread pool task is executed before user-level memory tracker is destroyed.
+                    /// However, thread could stay alive inside the thread pool, and it's ThreadStatus as well.
+                    /// When, finally, we destroy the thread (and the ThreadStatus),
+                    /// it can use memory tracker in the ~ThreadStatus in order to alloc/free untracked_memory,
+                    /// and by this time user-level memory tracker may be already destroyed.
+                    ///
+                    /// As a work-around, reset memory tracker to total, which is always alive.
+                    CurrentThread::get().memory_tracker.setParent(&total_memory_tracker);
+
+                    if (cleanup)
+                        cleanup();
+                });
+                callback();
             });
-            callback();
-        });
     };
 }
 
diff --git a/src/Interpreters/threadPoolCallbackRunner.h b/src/Interpreters/threadPoolCallbackRunner.h
index 59d06f2f1bc..b76485a1c7c 100644
--- a/src/Interpreters/threadPoolCallbackRunner.h
+++ b/src/Interpreters/threadPoolCallbackRunner.h
@@ -10,6 +10,6 @@ namespace DB
 using CallbackRunner = std::function<void(std::function<void()>)>;
 
 /// Creates CallbackRunner that runs every callback with 'pool->scheduleOrThrow()'
-CallbackRunner threadPoolCallbackRunner(ThreadPool & pool);
+CallbackRunner threadPoolCallbackRunner(ThreadPool & pool, std::function<void()> setup = {}, std::function<void()> cleanup = {});
 
 }
diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp
index 411f78e44d6..cb3bc84c587 100644
--- a/src/Storages/StorageS3.cpp
+++ b/src/Storages/StorageS3.cpp
@@ -324,32 +324,7 @@ std::unique_ptr<ReadBuffer> StorageS3Source::createS3ReadBuffer(const String & k
     LOG_TRACE(
         log, "Downloading from S3 in {} threads. Object size: {}, Range size: {}.", download_thread_num, object_size, download_buffer_size);
 
-    ThreadGroupStatusPtr running_group = CurrentThread::isInitialized() && CurrentThread::get().getThreadGroup()
-        ? CurrentThread::get().getThreadGroup()
-        : MainThreadStatus::getInstance().getThreadGroup();
-
-    ContextPtr query_context = CurrentThread::isInitialized() ? CurrentThread::get().getQueryContext() : nullptr;
-
-    auto worker_cleanup = [has_running_group = running_group == nullptr](ThreadStatus & thread_status)
-    {
-        if (has_running_group)
-            thread_status.detachQuery(false);
-    };
-
-    auto worker_setup = [query_context = std::move(query_context),
-                         running_group = std::move(running_group)](ThreadStatus & thread_status)
-    {
-        /// Save query context if any, because cache implementation needs it.
-        if (query_context)
-            thread_status.attachQueryContext(query_context);
-
-        /// To be able to pass ProfileEvents.
-        if (running_group)
-            thread_status.attachQuery(running_group);
-    };
-
-    return std::make_unique<ParallelReadBuffer>(
-        std::move(factory), &IOThreadPool::get(), download_thread_num, std::move(worker_setup), std::move(worker_cleanup));
+    return std::make_unique<ParallelReadBuffer>(std::move(factory), &IOThreadPool::get(), download_thread_num);
 }
 
 String StorageS3Source::getName() const
@@ -431,16 +406,6 @@ static bool checkIfObjectExists(const std::shared_ptr<Aws::S3::S3Client> & clien
     return false;
 }
 
-// TODO: common thread pool for IO must be used instead after PR #35150
-static ThreadPool & getThreadPoolStorageS3()
-{
-    constexpr size_t pool_size = 100;
-    constexpr size_t queue_size = 1000000;
-    static ThreadPool pool(pool_size, pool_size, queue_size);
-    return pool;
-}
-
-
 class StorageS3Sink : public SinkToStorage
 {
 public:
@@ -463,10 +428,20 @@ public:
     {
         write_buf = wrapWriteBufferWithCompressionMethod(
             std::make_unique<WriteBufferFromS3>(
-                client, bucket, key, min_upload_part_size,
-                upload_part_size_multiply_factor, upload_part_size_multiply_parts_count_threshold,
-                max_single_part_upload_size, std::nullopt, DBMS_DEFAULT_BUFFER_SIZE, threadPoolCallbackRunner(getThreadPoolStorageS3())), compression_method, 3);
-        writer = FormatFactory::instance().getOutputFormatParallelIfPossible(format, *write_buf, sample_block, context, {}, format_settings);
+                client,
+                bucket,
+                key,
+                min_upload_part_size,
+                upload_part_size_multiply_factor,
+                upload_part_size_multiply_parts_count_threshold,
+                max_single_part_upload_size,
+                std::nullopt,
+                DBMS_DEFAULT_BUFFER_SIZE,
+                threadPoolCallbackRunner(IOThreadPool::get())),
+            compression_method,
+            3);
+        writer
+            = FormatFactory::instance().getOutputFormatParallelIfPossible(format, *write_buf, sample_block, context, {}, format_settings);
     }
 
     String getName() const override { return "StorageS3Sink"; }
diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp
index c30e1bdbd10..eba0848c3b6 100644
--- a/src/Storages/StorageURL.cpp
+++ b/src/Storages/StorageURL.cpp
@@ -346,39 +346,11 @@ namespace
                                     /* use_external_buffer */ false,
                                     /* skip_url_not_found_error */ skip_url_not_found_error);
 
-                                ThreadGroupStatusPtr running_group = CurrentThread::isInitialized() && CurrentThread::get().getThreadGroup()
-                                    ? CurrentThread::get().getThreadGroup()
-                                    : MainThreadStatus::getInstance().getThreadGroup();
-
-                                ContextPtr query_context
-                                    = CurrentThread::isInitialized() ? CurrentThread::get().getQueryContext() : nullptr;
-
-                                auto worker_cleanup = [has_running_group = running_group == nullptr](ThreadStatus & thread_status)
-                                {
-                                    if (has_running_group)
-                                        thread_status.detachQuery(false);
-                                };
-
-                                auto worker_setup = [query_context = std::move(query_context),
-                                                     running_group = std::move(running_group)](ThreadStatus & thread_status)
-                                {
-                                    /// Save query context if any, because cache implementation needs it.
-                                    if (query_context)
-                                        thread_status.attachQueryContext(query_context);
-
-                                    /// To be able to pass ProfileEvents.
-                                    if (running_group)
-                                        thread_status.attachQuery(running_group);
-                                };
-
-
                                 return wrapReadBufferWithCompressionMethod(
                                     std::make_unique<ParallelReadBuffer>(
                                         std::move(read_buffer_factory),
                                         &IOThreadPool::get(),
-                                        download_threads,
-                                        std::move(worker_setup),
-                                        std::move(worker_cleanup)),
+                                        download_threads),
                                     chooseCompressionMethod(request_uri.getPath(), compression_method));
                             }
                         }

From f04756079eb5f125cc60296226a9ccfbe47f16e4 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Wed, 30 Mar 2022 08:35:30 +0000
Subject: [PATCH 078/104] Fix black style

---
 tests/ci/clickhouse_helper.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/ci/clickhouse_helper.py b/tests/ci/clickhouse_helper.py
index 218aaca8b91..d52b6262a78 100644
--- a/tests/ci/clickhouse_helper.py
+++ b/tests/ci/clickhouse_helper.py
@@ -15,7 +15,7 @@ class ClickHouseHelper:
         self.url = url
         self.auth = {
             "X-ClickHouse-User": get_parameter_from_ssm("clickhouse-test-stat-login"),
-            "X-ClickHouse-Key": get_parameter_from_ssm("clickhouse-test-stat-password")
+            "X-ClickHouse-Key": get_parameter_from_ssm("clickhouse-test-stat-password"),
         }
 
     @staticmethod

From 68ec0d92c0c5446d7a32156ae2940e78bedb582f Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Wed, 30 Mar 2022 10:58:34 +0200
Subject: [PATCH 079/104] Remove if category changed

---
 tests/ci/run_check.py | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/tests/ci/run_check.py b/tests/ci/run_check.py
index 0138808c6ae..5f3792f3432 100644
--- a/tests/ci/run_check.py
+++ b/tests/ci/run_check.py
@@ -234,20 +234,29 @@ if __name__ == "__main__":
     commit = get_commit(gh, pr_info.sha)
 
     description_report, category = check_pr_description(pr_info)
-    pr_labels = []
+    pr_labels_to_add = []
+    pr_labels_to_remove = []
     if (
         category in MAP_CATEGORY_TO_LABEL
         and MAP_CATEGORY_TO_LABEL[category] not in pr_info.labels
     ):
-        pr_labels.append(MAP_CATEGORY_TO_LABEL[category])
+        pr_labels_to_add.append(MAP_CATEGORY_TO_LABEL[category])
+
+    for label in pr_info.labels:
+        if label in MAP_CATEGORY_TO_LABEL and label not in pr_labels_to_add:
+            pr_labels_to_remove.append(label)
 
     if pr_info.has_changes_in_submodules():
-        pr_labels.append(SUBMODULE_CHANGED_LABEL)
+        pr_labels_to_add.append(SUBMODULE_CHANGED_LABEL)
     elif SUBMODULE_CHANGED_LABEL in pr_info.labels:
-        remove_labels(gh, pr_info, [SUBMODULE_CHANGED_LABEL])
+        pr_labels_to_remove.append(SUBMODULE_CHANGED_LABEL)
+
+    if pr_labels_to_add:
+        post_labels(gh, pr_info, pr_labels_to_add)
+
+    if pr_labels_to_remove:
+        remove_labels(gh, pr_info, pr_labels_to_remove)
 
-    if pr_labels:
-        post_labels(gh, pr_info, pr_labels)
     if description_report:
         print("::notice ::Cannot run, description does not match the template")
         logging.info(

From de2c6684a34f66af7a1659533ee271bdf058bd6d Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Wed, 30 Mar 2022 09:00:06 +0000
Subject: [PATCH 080/104] Remove setup and cleanup

---
 src/IO/ParallelReadBuffer.cpp                 | 39 +++++++------------
 src/IO/ParallelReadBuffer.h                   |  5 ---
 src/Interpreters/threadPoolCallbackRunner.cpp | 13 ++-----
 src/Interpreters/threadPoolCallbackRunner.h   |  2 +-
 4 files changed, 19 insertions(+), 40 deletions(-)

diff --git a/src/IO/ParallelReadBuffer.cpp b/src/IO/ParallelReadBuffer.cpp
index 3a62be1511b..256abd06086 100644
--- a/src/IO/ParallelReadBuffer.cpp
+++ b/src/IO/ParallelReadBuffer.cpp
@@ -16,36 +16,13 @@ namespace ErrorCodes
 ParallelReadBuffer::ParallelReadBuffer(std::unique_ptr<ReadBufferFactory> reader_factory_, ThreadPool * pool, size_t max_working_readers_)
     : SeekableReadBufferWithSize(nullptr, 0)
     , max_working_readers(max_working_readers_)
-    , schedule(threadPoolCallbackRunner(*pool, getWorkerSetup(), getWorkerCleanup()))
+    , schedule(threadPoolCallbackRunner(*pool))
     , reader_factory(std::move(reader_factory_))
 {
     std::unique_lock<std::mutex> lock{mutex};
     addReaders(lock);
 }
 
-
-ParallelReadBuffer::WorkerSetup ParallelReadBuffer::getWorkerSetup()
-{
-    return [this]
-    {
-        std::lock_guard lock{mutex};
-        ++active_working_reader;
-    };
-}
-
-ParallelReadBuffer::WorkerCleanup ParallelReadBuffer::getWorkerCleanup()
-{
-    return [this]
-    {
-        std::lock_guard lock{mutex};
-        --active_working_reader;
-        if (active_working_reader == 0)
-        {
-            readers_done.notify_all();
-        }
-    };
-}
-
 bool ParallelReadBuffer::addReaderToPool(std::unique_lock<std::mutex> & /*buffer_lock*/)
 {
     auto reader = reader_factory->getReader();
@@ -226,6 +203,20 @@ bool ParallelReadBuffer::nextImpl()
 
 void ParallelReadBuffer::readerThreadFunction(ReadWorkerPtr read_worker)
 {
+    {
+        std::lock_guard lock{mutex};
+        ++active_working_reader;
+    }
+
+    SCOPE_EXIT({
+        std::lock_guard lock{mutex};
+        --active_working_reader;
+        if (active_working_reader == 0)
+        {
+            readers_done.notify_all();
+        }
+    });
+
     try
     {
         while (!emergency_stop && !read_worker->cancel)
diff --git a/src/IO/ParallelReadBuffer.h b/src/IO/ParallelReadBuffer.h
index 3759064c2a8..a1db7181294 100644
--- a/src/IO/ParallelReadBuffer.h
+++ b/src/IO/ParallelReadBuffer.h
@@ -130,11 +130,6 @@ private:
     void onBackgroundException();
     void finishAndWait();
 
-    using WorkerSetup = std::function<void()>;
-    WorkerSetup getWorkerSetup();
-    using WorkerCleanup = std::function<void()>;
-    WorkerCleanup getWorkerCleanup();
-
     SynchronizedArenaWithFreeLists arena;
 
     Segment current_segment;
diff --git a/src/Interpreters/threadPoolCallbackRunner.cpp b/src/Interpreters/threadPoolCallbackRunner.cpp
index 58ce3789bc0..288079e49d2 100644
--- a/src/Interpreters/threadPoolCallbackRunner.cpp
+++ b/src/Interpreters/threadPoolCallbackRunner.cpp
@@ -7,17 +7,13 @@
 namespace DB
 {
 
-CallbackRunner threadPoolCallbackRunner(ThreadPool & pool, std::function<void()> setup, std::function<void()> cleanup)
+CallbackRunner threadPoolCallbackRunner(ThreadPool & pool)
 {
-    return [pool = &pool, thread_group = CurrentThread::getGroup(), setup = std::move(setup), cleanup = std::move(cleanup)](
-               auto callback) mutable
+    return [pool = &pool, thread_group = CurrentThread::getGroup()](auto callback) mutable
     {
         pool->scheduleOrThrow(
-            [&, callback = std::move(callback), thread_group, setup = std::move(setup), cleanup = std::move(cleanup)]()
+            [&, callback = std::move(callback), thread_group]()
             {
-                if (setup)
-                    setup();
-
                 if (thread_group)
                     CurrentThread::attachTo(thread_group);
 
@@ -35,9 +31,6 @@ CallbackRunner threadPoolCallbackRunner(ThreadPool & pool, std::function<void()>
                     ///
                     /// As a work-around, reset memory tracker to total, which is always alive.
                     CurrentThread::get().memory_tracker.setParent(&total_memory_tracker);
-
-                    if (cleanup)
-                        cleanup();
                 });
                 callback();
             });
diff --git a/src/Interpreters/threadPoolCallbackRunner.h b/src/Interpreters/threadPoolCallbackRunner.h
index b76485a1c7c..59d06f2f1bc 100644
--- a/src/Interpreters/threadPoolCallbackRunner.h
+++ b/src/Interpreters/threadPoolCallbackRunner.h
@@ -10,6 +10,6 @@ namespace DB
 using CallbackRunner = std::function<void(std::function<void()>)>;
 
 /// Creates CallbackRunner that runs every callback with 'pool->scheduleOrThrow()'
-CallbackRunner threadPoolCallbackRunner(ThreadPool & pool, std::function<void()> setup = {}, std::function<void()> cleanup = {});
+CallbackRunner threadPoolCallbackRunner(ThreadPool & pool);
 
 }

From bcf64a73d1c272dca22a1ca910df3153a6a66784 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Wed, 30 Mar 2022 11:00:45 +0200
Subject: [PATCH 081/104] Followup

---
 tests/ci/run_check.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/ci/run_check.py b/tests/ci/run_check.py
index 5f3792f3432..8f28287e88b 100644
--- a/tests/ci/run_check.py
+++ b/tests/ci/run_check.py
@@ -243,7 +243,7 @@ if __name__ == "__main__":
         pr_labels_to_add.append(MAP_CATEGORY_TO_LABEL[category])
 
     for label in pr_info.labels:
-        if label in MAP_CATEGORY_TO_LABEL and label not in pr_labels_to_add:
+        if label in MAP_CATEGORY_TO_LABEL.values() and label not in pr_labels_to_add:
             pr_labels_to_remove.append(label)
 
     if pr_info.has_changes_in_submodules():

From 2a1b331c0de65c680285e777e9a90ad49f1d86f2 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Wed, 30 Mar 2022 11:15:54 +0200
Subject: [PATCH 082/104] Fix build check

---
 tests/ci/build_report_check.py | 6 +++++-
 tests/ci/docs_check.py         | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/tests/ci/build_report_check.py b/tests/ci/build_report_check.py
index 50315ad3d78..4e0859ef865 100644
--- a/tests/ci/build_report_check.py
+++ b/tests/ci/build_report_check.py
@@ -233,7 +233,11 @@ if __name__ == "__main__":
     if ok_builds == 0 or some_builds_are_missing:
         summary_status = "error"
 
-    description = f"{ok_builds}/{total_builds} builds are OK"
+    addition = ""
+    if some_builds_are_missing:
+        addition = "(some builds are missing)"
+
+    description = f"{ok_builds}/{total_builds} builds are OK {addition}"
 
     print("::notice ::Report url: {}".format(url))
 
diff --git a/tests/ci/docs_check.py b/tests/ci/docs_check.py
index 6dde0dfaea4..803fb3cff7d 100644
--- a/tests/ci/docs_check.py
+++ b/tests/ci/docs_check.py
@@ -114,7 +114,7 @@ if __name__ == "__main__":
         report_url,
         NAME,
     )
-    ch_helper.insert_events_into(db="default", table="checks", events=prepared_events)
 
+    ch_helper.insert_events_into(db="default", table="checks", events=prepared_events)
     if status == "error":
         sys.exit(1)

From c510ede8dd5b9e739be4de6b6c57aa8bbddfa22b Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Wed, 30 Mar 2022 11:19:11 +0200
Subject: [PATCH 083/104] Fixup

---
 tests/ci/run_check.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tests/ci/run_check.py b/tests/ci/run_check.py
index 8f28287e88b..93dc77124c2 100644
--- a/tests/ci/run_check.py
+++ b/tests/ci/run_check.py
@@ -243,7 +243,11 @@ if __name__ == "__main__":
         pr_labels_to_add.append(MAP_CATEGORY_TO_LABEL[category])
 
     for label in pr_info.labels:
-        if label in MAP_CATEGORY_TO_LABEL.values() and label not in pr_labels_to_add:
+        if (
+            label in MAP_CATEGORY_TO_LABEL.values()
+            and category in MAP_CATEGORY_TO_LABEL
+            and label != MAP_CATEGORY_TO_LABEL[category]
+        ):
             pr_labels_to_remove.append(label)
 
     if pr_info.has_changes_in_submodules():

From b19ecf60ad9bd518de109dcd536b45ac716faaaf Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Wed, 30 Mar 2022 11:30:21 +0200
Subject: [PATCH 084/104] Fix style

---
 tests/ci/clickhouse_helper.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/ci/clickhouse_helper.py b/tests/ci/clickhouse_helper.py
index 218aaca8b91..d52b6262a78 100644
--- a/tests/ci/clickhouse_helper.py
+++ b/tests/ci/clickhouse_helper.py
@@ -15,7 +15,7 @@ class ClickHouseHelper:
         self.url = url
         self.auth = {
             "X-ClickHouse-User": get_parameter_from_ssm("clickhouse-test-stat-login"),
-            "X-ClickHouse-Key": get_parameter_from_ssm("clickhouse-test-stat-password")
+            "X-ClickHouse-Key": get_parameter_from_ssm("clickhouse-test-stat-password"),
         }
 
     @staticmethod

From 531942ce5ce1a22424b44f589c8aabc1ca7cdc88 Mon Sep 17 00:00:00 2001
From: tavplubix <tavplubix@clickhouse.com>
Date: Wed, 30 Mar 2022 12:43:34 +0300
Subject: [PATCH 085/104] Update 00170_s3_cache.sql

---
 tests/queries/1_stateful/00170_s3_cache.sql | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/queries/1_stateful/00170_s3_cache.sql b/tests/queries/1_stateful/00170_s3_cache.sql
index 5d616916048..d4837a1a800 100644
--- a/tests/queries/1_stateful/00170_s3_cache.sql
+++ b/tests/queries/1_stateful/00170_s3_cache.sql
@@ -1,5 +1,7 @@
 -- Tags: no-parallel
+
 -- { echo }
+
 SET max_memory_usage='20G';
 SELECT count() FROM test.hits_s3;
 SELECT count() FROM test.hits_s3 WHERE AdvEngineID != 0;

From d3697625a93a32f34b6955e1763e081909f71584 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Wed, 30 Mar 2022 10:55:49 +0000
Subject: [PATCH 086/104] Fix stylecheck

---
 tests/ci/clickhouse_helper.py     | 2 +-
 utils/tests-visualizer/index.html | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/ci/clickhouse_helper.py b/tests/ci/clickhouse_helper.py
index 218aaca8b91..d52b6262a78 100644
--- a/tests/ci/clickhouse_helper.py
+++ b/tests/ci/clickhouse_helper.py
@@ -15,7 +15,7 @@ class ClickHouseHelper:
         self.url = url
         self.auth = {
             "X-ClickHouse-User": get_parameter_from_ssm("clickhouse-test-stat-login"),
-            "X-ClickHouse-Key": get_parameter_from_ssm("clickhouse-test-stat-password")
+            "X-ClickHouse-Key": get_parameter_from_ssm("clickhouse-test-stat-password"),
         }
 
     @staticmethod
diff --git a/utils/tests-visualizer/index.html b/utils/tests-visualizer/index.html
index 13f8daaa151..00076f683fa 100644
--- a/utils/tests-visualizer/index.html
+++ b/utils/tests-visualizer/index.html
@@ -84,14 +84,14 @@ let render_data_query = `
     SELECT groupArray([d, n, fail]) FROM
     (
         SELECT n, check_start_time::Date - start_date AS d, max(test_status LIKE 'F%' OR test_status LIKE 'E%') AS fail
-        FROM "gh-data".checks
+        FROM "default".checks
 
         INNER JOIN
         (
             SELECT test_name, toUInt16(rowNumberInAllBlocks()) AS n FROM
             (
                 SELECT DISTINCT test_name
-                FROM "gh-data".checks
+                FROM "default".checks
                 WHERE match(test_name, '^\\d+_') AND check_name ILIKE '%stateless%' AND check_start_time > now() - INTERVAL 1 DAY
                 ORDER BY test_name
             )
@@ -112,7 +112,7 @@ let test_names_query = `
     SELECT test_name, toUInt16(rowNumberInAllBlocks()) AS n FROM
     (
         SELECT DISTINCT test_name
-        FROM "gh-data".checks
+        FROM "default".checks
         WHERE match(test_name, '^\\d+_') AND check_name ILIKE '%stateless%' AND check_start_time > now() - INTERVAL 1 DAY
         ORDER BY test_name
     ) FORMAT JSONCompact`;

From af4bfec051605ffdc43451f2ca4256a57f5346ec Mon Sep 17 00:00:00 2001
From: avogar <pav.cruglov@yandex.ru>
Date: Wed, 30 Mar 2022 11:19:03 +0000
Subject: [PATCH 087/104] Fix bug in conversion from custom types to string

---
 src/Functions/FunctionsConversion.h                         | 2 +-
 .../02248_nullable_custom_types_to_string.reference         | 6 ++++++
 .../0_stateless/02248_nullable_custom_types_to_string.sql   | 6 ++++++
 3 files changed, 13 insertions(+), 1 deletion(-)
 create mode 100644 tests/queries/0_stateless/02248_nullable_custom_types_to_string.reference
 create mode 100644 tests/queries/0_stateless/02248_nullable_custom_types_to_string.sql

diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h
index edddb200648..16fca18c824 100644
--- a/src/Functions/FunctionsConversion.h
+++ b/src/Functions/FunctionsConversion.h
@@ -887,7 +887,7 @@ struct ConvertImplGenericToString
         const IColumn & col_from = *col_with_type_and_name.column;
 
         size_t size = col_from.size();
-        auto col_to = result_type->createColumn();
+        auto col_to = removeNullable(result_type)->createColumn();
 
         {
             ColumnStringHelpers::WriteHelper write_helper(
diff --git a/tests/queries/0_stateless/02248_nullable_custom_types_to_string.reference b/tests/queries/0_stateless/02248_nullable_custom_types_to_string.reference
new file mode 100644
index 00000000000..46232c6fce2
--- /dev/null
+++ b/tests/queries/0_stateless/02248_nullable_custom_types_to_string.reference
@@ -0,0 +1,6 @@
+true
+\N
+0.0.0.0
+\N
+::ffff:127.0.0.1
+\N
diff --git a/tests/queries/0_stateless/02248_nullable_custom_types_to_string.sql b/tests/queries/0_stateless/02248_nullable_custom_types_to_string.sql
new file mode 100644
index 00000000000..313f703fd03
--- /dev/null
+++ b/tests/queries/0_stateless/02248_nullable_custom_types_to_string.sql
@@ -0,0 +1,6 @@
+select toString(toNullable(true));
+select toString(CAST(NULL, 'Nullable(Bool)'));
+select toString(toNullable(toIPv4('0.0.0.0')));
+select toString(CAST(NULL, 'Nullable(IPv4)'));
+select toString(toNullable(toIPv6('::ffff:127.0.0.1')));
+select toString(CAST(NULL, 'Nullable(IPv6)'));

From a3ad99be039eb4d95b883e50daaaa7e1543af7da Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Wed, 30 Mar 2022 10:49:37 +0000
Subject: [PATCH 088/104] Create CallbackRunner in Storage

---
 src/IO/ParallelReadBuffer.cpp | 4 ++--
 src/IO/ParallelReadBuffer.h   | 2 +-
 src/Storages/StorageS3.cpp    | 2 +-
 src/Storages/StorageURL.cpp   | 3 ++-
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/IO/ParallelReadBuffer.cpp b/src/IO/ParallelReadBuffer.cpp
index 256abd06086..f036d6a08c8 100644
--- a/src/IO/ParallelReadBuffer.cpp
+++ b/src/IO/ParallelReadBuffer.cpp
@@ -13,10 +13,10 @@ namespace ErrorCodes
 
 }
 
-ParallelReadBuffer::ParallelReadBuffer(std::unique_ptr<ReadBufferFactory> reader_factory_, ThreadPool * pool, size_t max_working_readers_)
+ParallelReadBuffer::ParallelReadBuffer(std::unique_ptr<ReadBufferFactory> reader_factory_, CallbackRunner schedule_, size_t max_working_readers_)
     : SeekableReadBufferWithSize(nullptr, 0)
     , max_working_readers(max_working_readers_)
-    , schedule(threadPoolCallbackRunner(*pool))
+    , schedule(std::move(schedule_))
     , reader_factory(std::move(reader_factory_))
 {
     std::unique_lock<std::mutex> lock{mutex};
diff --git a/src/IO/ParallelReadBuffer.h b/src/IO/ParallelReadBuffer.h
index a1db7181294..eda88969afa 100644
--- a/src/IO/ParallelReadBuffer.h
+++ b/src/IO/ParallelReadBuffer.h
@@ -77,7 +77,7 @@ public:
         virtual std::optional<size_t> getTotalSize() = 0;
     };
 
-    explicit ParallelReadBuffer(std::unique_ptr<ReadBufferFactory> reader_factory_, ThreadPool * pool, size_t max_working_readers);
+    explicit ParallelReadBuffer(std::unique_ptr<ReadBufferFactory> reader_factory_, CallbackRunner schedule_, size_t max_working_readers);
 
     ~ParallelReadBuffer() override { finishAndWait(); }
 
diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp
index cb3bc84c587..396adb801f5 100644
--- a/src/Storages/StorageS3.cpp
+++ b/src/Storages/StorageS3.cpp
@@ -324,7 +324,7 @@ std::unique_ptr<ReadBuffer> StorageS3Source::createS3ReadBuffer(const String & k
     LOG_TRACE(
         log, "Downloading from S3 in {} threads. Object size: {}, Range size: {}.", download_thread_num, object_size, download_buffer_size);
 
-    return std::make_unique<ParallelReadBuffer>(std::move(factory), &IOThreadPool::get(), download_thread_num);
+    return std::make_unique<ParallelReadBuffer>(std::move(factory), threadPoolCallbackRunner(IOThreadPool::get()), download_thread_num);
 }
 
 String StorageS3Source::getName() const
diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp
index eba0848c3b6..a435ab1a654 100644
--- a/src/Storages/StorageURL.cpp
+++ b/src/Storages/StorageURL.cpp
@@ -2,6 +2,7 @@
 
 #include <Interpreters/Context.h>
 #include <Interpreters/evaluateConstantExpression.h>
+#include <Interpreters/threadPoolCallbackRunner.h>
 #include <Parsers/ASTCreateQuery.h>
 #include <Parsers/ASTInsertQuery.h>
 #include <Parsers/ASTLiteral.h>
@@ -349,7 +350,7 @@ namespace
                                 return wrapReadBufferWithCompressionMethod(
                                     std::make_unique<ParallelReadBuffer>(
                                         std::move(read_buffer_factory),
-                                        &IOThreadPool::get(),
+                                        threadPoolCallbackRunner(IOThreadPool::get()),
                                         download_threads),
                                     chooseCompressionMethod(request_uri.getPath(), compression_method));
                             }

From cf8332e109cb884ef20fda9c99fb1fc383053df1 Mon Sep 17 00:00:00 2001
From: tavplubix <tavplubix@clickhouse.com>
Date: Wed, 30 Mar 2022 14:38:21 +0300
Subject: [PATCH 089/104] Update 00170_s3_cache.reference

---
 tests/queries/1_stateful/00170_s3_cache.reference | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/queries/1_stateful/00170_s3_cache.reference b/tests/queries/1_stateful/00170_s3_cache.reference
index 9c9c3bc537f..aeb7022b3f7 100644
--- a/tests/queries/1_stateful/00170_s3_cache.reference
+++ b/tests/queries/1_stateful/00170_s3_cache.reference
@@ -1,4 +1,5 @@
 -- { echo }
+
 SET max_memory_usage='20G';
 SELECT count() FROM test.hits_s3;
 8873898

From 6e1036de50663eb6954d9784c85e44e5b2d1aafb Mon Sep 17 00:00:00 2001
From: Alexander Gololobov <440544+davenger@users.noreply.github.com>
Date: Wed, 30 Mar 2022 16:12:13 +0200
Subject: [PATCH 090/104] Fix user file path extraction

It turned out that $CLICHOUSE_CLIENT could produce more verbose output that was not properly parsed. I made a change that should fix this parsing.
---
 tests/queries/0_stateless/02167_format_from_file_extension.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02167_format_from_file_extension.sh b/tests/queries/0_stateless/02167_format_from_file_extension.sh
index 4b88788d9e0..050ec2b7332 100755
--- a/tests/queries/0_stateless/02167_format_from_file_extension.sh
+++ b/tests/queries/0_stateless/02167_format_from_file_extension.sh
@@ -8,7 +8,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 function cleanup()
 {
     # this command expects an error message like 'Code: 107. DB::Exception: Received <...> nonexist.txt doesn't exist. (FILE_DOESNT_EXIST)'
-    user_files_path=$($CLICKHOUSE_CLIENT --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep FILE_DOESNT_EXIST | awk '{gsub("/nonexist.txt","",$9); print $9}')
+    user_files_path=$($CLICKHOUSE_CLIENT --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep -E '^Code: 107.*FILE_DOESNT_EXIST' | head -1 | awk '{gsub("/nonexist.txt","",$9); print $9}')
+    echo "Removing $user_files_path/test_02167.*"
     rm $user_files_path/test_02167.*
 }
 trap cleanup EXIT

From 8980994faacbc806a9ed1bd3a76c7321132fd270 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Wed, 30 Mar 2022 16:29:13 +0200
Subject: [PATCH 091/104] Resurrect build hash

---
 tests/ci/build_check.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py
index 24af9e5194c..eb2aa616383 100644
--- a/tests/ci/build_check.py
+++ b/tests/ci/build_check.py
@@ -54,6 +54,7 @@ def get_packager_cmd(
     build_version: str,
     image_version: str,
     ccache_path: str,
+    official: bool,
 ) -> str:
     package_type = build_config["package_type"]
     comp = build_config["compiler"]
@@ -83,6 +84,9 @@ def get_packager_cmd(
     if _can_export_binaries(build_config):
         cmd += " --with-binaries=tests"
 
+    if official:
+        cmd += " --official"
+
     return cmd
 
 
@@ -254,9 +258,11 @@ def main():
 
     logging.info("Got version from repo %s", version.string)
 
+    official_flag = pr_info.number == 0
     version_type = "testing"
     if "release" in pr_info.labels or "release-lts" in pr_info.labels:
         version_type = "stable"
+        official_flag = True
 
     update_version_local(REPO_COPY, version, version_type)
 
@@ -290,7 +296,9 @@ def main():
         version.string,
         image_version,
         ccache_path,
+        official=official_flag
     )
+
     logging.info("Going to run packager with %s", packager_cmd)
 
     build_clickhouse_log = os.path.join(TEMP_PATH, "build_log")

From c34d5125864a1e33ab846bcf41e986357dc01a3a Mon Sep 17 00:00:00 2001
From: tavplubix <tavplubix@clickhouse.com>
Date: Wed, 30 Mar 2022 18:36:37 +0300
Subject: [PATCH 092/104] Update
 00484_preferred_max_column_in_block_size_bytes.sql

---
 .../00484_preferred_max_column_in_block_size_bytes.sql          | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/queries/0_stateless/00484_preferred_max_column_in_block_size_bytes.sql b/tests/queries/0_stateless/00484_preferred_max_column_in_block_size_bytes.sql
index dc021ad52db..470bca70e06 100644
--- a/tests/queries/0_stateless/00484_preferred_max_column_in_block_size_bytes.sql
+++ b/tests/queries/0_stateless/00484_preferred_max_column_in_block_size_bytes.sql
@@ -1,3 +1,5 @@
+-- Tags: no-random-settings
+
 drop table if exists tab_00484;
 create table tab_00484 (date Date, x UInt64, s FixedString(128)) engine = MergeTree PARTITION BY date ORDER BY (date, x) SETTINGS min_bytes_for_wide_part = 0;
 insert into tab_00484 select today(), number, toFixedString('', 128) from system.numbers limit 8192;

From fcd4e0be654159e34f61d7c4f95f123ba9b6c67c Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Wed, 30 Mar 2022 19:15:27 +0200
Subject: [PATCH 093/104] Black

---
 tests/ci/build_check.py       | 2 +-
 tests/ci/clickhouse_helper.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py
index eb2aa616383..2a079a60367 100644
--- a/tests/ci/build_check.py
+++ b/tests/ci/build_check.py
@@ -296,7 +296,7 @@ def main():
         version.string,
         image_version,
         ccache_path,
-        official=official_flag
+        official=official_flag,
     )
 
     logging.info("Going to run packager with %s", packager_cmd)
diff --git a/tests/ci/clickhouse_helper.py b/tests/ci/clickhouse_helper.py
index 218aaca8b91..d52b6262a78 100644
--- a/tests/ci/clickhouse_helper.py
+++ b/tests/ci/clickhouse_helper.py
@@ -15,7 +15,7 @@ class ClickHouseHelper:
         self.url = url
         self.auth = {
             "X-ClickHouse-User": get_parameter_from_ssm("clickhouse-test-stat-login"),
-            "X-ClickHouse-Key": get_parameter_from_ssm("clickhouse-test-stat-password")
+            "X-ClickHouse-Key": get_parameter_from_ssm("clickhouse-test-stat-password"),
         }
 
     @staticmethod

From 9820230e957784b3b170a2019f101d9f328127ea Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Wed, 30 Mar 2022 20:27:31 +0300
Subject: [PATCH 094/104] clickhouse-keeper: correctly handle closed client
 connection

This will avoid noisy message like:

    DB::Exception: Cannot read all data. Bytes read: 0. Bytes expected: 4.

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Server/KeeperTCPHandler.cpp | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/Server/KeeperTCPHandler.cpp b/src/Server/KeeperTCPHandler.cpp
index 655d17e61fa..15438522369 100644
--- a/src/Server/KeeperTCPHandler.cpp
+++ b/src/Server/KeeperTCPHandler.cpp
@@ -415,6 +415,12 @@ void KeeperTCPHandler::runImpl()
             log_long_operation("Polling socket");
             if (result.has_requests && !close_received)
             {
+                if (in->eof())
+                {
+                    LOG_DEBUG(log, "Client closed connection, session id #{}", session_id);
+                    break;
+                }
+
                 auto [received_op, received_xid] = receiveRequest();
                 packageReceived();
                 log_long_operation("Receiving request");

From 8f7329d7dd45ba3daab820fa96c07496ab33df34 Mon Sep 17 00:00:00 2001
From: Alexander Gololobov <440544+davenger@users.noreply.github.com>
Date: Wed, 30 Mar 2022 20:10:25 +0200
Subject: [PATCH 095/104] Removed debug print

---
 tests/queries/0_stateless/02167_format_from_file_extension.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/queries/0_stateless/02167_format_from_file_extension.sh b/tests/queries/0_stateless/02167_format_from_file_extension.sh
index 050ec2b7332..14985233524 100755
--- a/tests/queries/0_stateless/02167_format_from_file_extension.sh
+++ b/tests/queries/0_stateless/02167_format_from_file_extension.sh
@@ -9,7 +9,6 @@ function cleanup()
 {
     # this command expects an error message like 'Code: 107. DB::Exception: Received <...> nonexist.txt doesn't exist. (FILE_DOESNT_EXIST)'
     user_files_path=$($CLICKHOUSE_CLIENT --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep -E '^Code: 107.*FILE_DOESNT_EXIST' | head -1 | awk '{gsub("/nonexist.txt","",$9); print $9}')
-    echo "Removing $user_files_path/test_02167.*"
     rm $user_files_path/test_02167.*
 }
 trap cleanup EXIT

From ac4897f4f9bf70049439bd8255a9f539487aac46 Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Wed, 30 Mar 2022 20:16:41 +0200
Subject: [PATCH 096/104] Fix

---
 src/Parsers/ExpressionListParsers.cpp | 26 +++++++++++++++-----------
 src/Parsers/ExpressionListParsers.h   | 11 ++++++-----
 2 files changed, 21 insertions(+), 16 deletions(-)

diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp
index e78f0e2bb56..10cbd95ec59 100644
--- a/src/Parsers/ExpressionListParsers.cpp
+++ b/src/Parsers/ExpressionListParsers.cpp
@@ -326,20 +326,24 @@ bool ParserLeftAssociativeBinaryOperatorList::parseImpl(Pos & pos, ASTPtr & node
             /// function arguments
             auto exp_list = std::make_shared<ASTExpressionList>();
 
-            SubqueryFunctionType subquery_function_type = SubqueryFunctionType::NONE;
-            if (ParserKeyword("ANY").ignore(pos, expected))
-                 subquery_function_type = SubqueryFunctionType::ANY;
-            else if (ParserKeyword("ALL").ignore(pos, expected))
-                 subquery_function_type = SubqueryFunctionType::ALL;
-
             ASTPtr elem;
-            if (!ParserSubquery().parse(pos, elem, expected))
+            SubqueryFunctionType subquery_function_type = SubqueryFunctionType::NONE;
+
+            if (comparison_expression)
             {
-                subquery_function_type = SubqueryFunctionType::NONE;
-                if (!(remaining_elem_parser ? remaining_elem_parser : first_elem_parser)->parse(pos, elem, expected))
-                    return false;
+                if (ParserKeyword("ANY").ignore(pos, expected))
+                    subquery_function_type = SubqueryFunctionType::ANY;
+                else if (ParserKeyword("ALL").ignore(pos, expected))
+                    subquery_function_type = SubqueryFunctionType::ALL;
             }
 
+            if (subquery_function_type != SubqueryFunctionType::NONE && !ParserSubquery().parse(pos, elem, expected))
+                subquery_function_type = SubqueryFunctionType::NONE;
+
+            if (subquery_function_type == SubqueryFunctionType::NONE
+                && !(remaining_elem_parser ? remaining_elem_parser : first_elem_parser)->parse(pos, elem, expected))
+                return false;
+
             /// the first argument of the function is the previous element, the second is the next one
             function->name = it[1];
             function->arguments = exp_list;
@@ -348,7 +352,7 @@ bool ParserLeftAssociativeBinaryOperatorList::parseImpl(Pos & pos, ASTPtr & node
             exp_list->children.push_back(node);
             exp_list->children.push_back(elem);
 
-            if (subquery_function_type != SubqueryFunctionType::NONE && !modifyAST(function, subquery_function_type))
+            if (comparison_expression && subquery_function_type != SubqueryFunctionType::NONE && !modifyAST(function, subquery_function_type))
                 return false;
 
             /** special exception for the access operator to the element of the array `x[y]`, which
diff --git a/src/Parsers/ExpressionListParsers.h b/src/Parsers/ExpressionListParsers.h
index 4cd5e5114a1..18d25bcb6d9 100644
--- a/src/Parsers/ExpressionListParsers.h
+++ b/src/Parsers/ExpressionListParsers.h
@@ -121,6 +121,8 @@ private:
     Operators_t overlapping_operators_to_skip = { (const char *[]){ nullptr } };
     ParserPtr first_elem_parser;
     ParserPtr remaining_elem_parser;
+    /// =, !=, <, > ALL (subquery) / ANY (subquery)
+    bool comparison_expression = false;
 
 public:
     /** `operators_` - allowed operators and their corresponding functions
@@ -130,11 +132,10 @@ public:
     {
     }
 
-    ParserLeftAssociativeBinaryOperatorList(
-        Operators_t operators_, Operators_t overlapping_operators_to_skip_,
-        ParserPtr && first_elem_parser_)
+    ParserLeftAssociativeBinaryOperatorList(Operators_t operators_,
+            Operators_t overlapping_operators_to_skip_, ParserPtr && first_elem_parser_, bool comparison_expression_ = false)
         : operators(operators_), overlapping_operators_to_skip(overlapping_operators_to_skip_),
-          first_elem_parser(std::move(first_elem_parser_))
+          first_elem_parser(std::move(first_elem_parser_)), comparison_expression(comparison_expression_)
     {
     }
 
@@ -351,7 +352,7 @@ private:
     static const char * operators[];
     static const char * overlapping_operators_to_skip[];
     ParserLeftAssociativeBinaryOperatorList operator_parser {operators,
-    overlapping_operators_to_skip, std::make_unique<ParserBetweenExpression>()};
+        overlapping_operators_to_skip, std::make_unique<ParserBetweenExpression>(), true};
 
 protected:
     const char * getName() const  override{ return "comparison expression"; }

From 38873d726d22b15d4c5a284410956927b2e2f95f Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Wed, 30 Mar 2022 21:30:12 +0300
Subject: [PATCH 097/104] clickhouse-keeper: fix implementation of server with
 poll()

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Server/KeeperTCPHandler.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Server/KeeperTCPHandler.cpp b/src/Server/KeeperTCPHandler.cpp
index 15438522369..b0e2bc85245 100644
--- a/src/Server/KeeperTCPHandler.cpp
+++ b/src/Server/KeeperTCPHandler.cpp
@@ -171,7 +171,7 @@ struct SocketInterruptablePollWrapper
 
             if (rc >= 1 && poll_buf[0].revents & POLLIN)
                 socket_ready = true;
-            if (rc >= 1 && poll_buf[1].revents & POLLIN)
+            if (rc >= 2 && poll_buf[1].revents & POLLIN)
                 fd_ready = true;
 #endif
         }

From 8b84829e93d2bb2b984ffc57df3dd8dc6e272cff Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Wed, 30 Mar 2022 20:49:39 +0300
Subject: [PATCH 098/104] ci: replace directory system log tables artifacts
 with tsv

Changelog:
- remove query_thread_log (query_log should be enough)
- for s3 storage there /var/lib/clickhouse/data/system/*_log cannot be
  used, replace them with plain TSVWithNamesAndTypes, also note, that
  after schema inference it is even pretty easy to use
- support replicated database env correctly

Co-authored-by: tavplubix <tavplubix@clickhouse.com>
Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 docker/test/stateless/run.sh | 29 +++++++++++++++++------------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh
index 5fd78502337..f8b73791388 100755
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@@ -131,8 +131,23 @@ clickhouse-client -q "system flush logs" ||:
 
 grep -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server.log ||:
 pigz < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.gz &
-clickhouse-client -q "select * from system.query_log format TSVWithNamesAndTypes" | pigz > /test_output/query-log.tsv.gz &
-clickhouse-client -q "select * from system.query_thread_log format TSVWithNamesAndTypes" | pigz > /test_output/query-thread-log.tsv.gz &
+
+# Compress tables.
+#
+# NOTE:
+# - that due to tests with s3 storage we cannot use /var/lib/clickhouse/data
+#   directly
+# - even though ci auto-compress some files (but not *.tsv) it does this only
+#   for files >64MB, we want this files to be compressed explicitly
+for table in query_log zookeeper_log trace_log
+do
+    clickhouse-client -q "select * from system.$table format TSVWithNamesAndTypes" | pigz > /test_output/$table.tsv.gz &
+    if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
+        clickhouse-client --port 19000 -q "select * from system.$table format TSVWithNamesAndTypes" | pigz > /test_output/$table.1.tsv.gz &
+        clickhouse-client --port 29000 -q "select * from system.$table format TSVWithNamesAndTypes" | pigz > /test_output/$table.2.tsv.gz &
+    fi
+done
+wait ||:
 
 # Also export trace log in flamegraph-friendly format.
 for trace_type in CPU Memory Real
@@ -161,14 +176,6 @@ fi
 
 tar -chf /test_output/coordination.tar /var/lib/clickhouse/coordination ||:
 
-# Replace the engine with Ordinary to avoid extra symlinks stuff in artifacts.
-# (so that clickhouse-local --path can read it w/o extra care).
-sed -i -e "s/ATTACH DATABASE _ UUID '[^']*'/ATTACH DATABASE system/" -e "s/Atomic/Ordinary/" /var/lib/clickhouse/metadata/system.sql
-for table in text_log query_log zookeeper_log trace_log; do
-    sed -i "s/ATTACH TABLE _ UUID '[^']*'/ATTACH TABLE $table/" /var/lib/clickhouse/metadata/system/${table}.sql
-    tar -chf /test_output/${table}_dump.tar /var/lib/clickhouse/metadata/system.sql /var/lib/clickhouse/metadata/system/${table}.sql /var/lib/clickhouse/data/system/${table} ||:
-done
-
 if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
     grep -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server1.log ||:
     grep -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server2.log ||:
@@ -179,8 +186,6 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]
     rm /var/log/clickhouse-server/clickhouse-server2.log
     mv /var/log/clickhouse-server/stderr1.log /test_output/ ||:
     mv /var/log/clickhouse-server/stderr2.log /test_output/ ||:
-    tar -chf /test_output/zookeeper_log_dump1.tar /var/lib/clickhouse1/data/system/zookeeper_log ||:
-    tar -chf /test_output/zookeeper_log_dump2.tar /var/lib/clickhouse2/data/system/zookeeper_log ||:
     tar -chf /test_output/coordination1.tar /var/lib/clickhouse1/coordination ||:
     tar -chf /test_output/coordination2.tar /var/lib/clickhouse2/coordination ||:
 fi

From 30f2a942c5a41459b893f1bee971f216ac75847f Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Wed, 30 Mar 2022 22:47:51 +0200
Subject: [PATCH 099/104] Predict size of hash table for GROUP BY (#33439)

* use AggregationMethod ctor with reserve

* add new settings

* add HashTablesStatistics

* support queries with limit

* support distributed and with external aggregation

* add new profile events

* add some tests

* add perf test

* export cache stats through AsynchronousMetrics

* rm redundant trace

* fix style

* fix 02122_parallel_formatting test

* review fixes

* fix 02122_parallel_formatting test

* apply also to two-level HTs

* try simpler strategy

* increase max_size_to_preallocate_for_aggregation for experiment

* fixes

* Revert "increase max_size_to_preallocate_for_aggregation for experiment"

This reverts commit 6cf6f75704d8363602cfa4df87d0f2edf14cb9c8.

* fix test

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
---
 src/Common/HashTable/FixedHashTable.h         |   6 +
 src/Common/HashTable/TwoLevelHashTable.h      |   6 +
 src/Common/ProfileEvents.cpp                  |   3 +
 src/Core/Settings.h                           |   4 +
 src/Interpreters/Aggregator.cpp               | 302 +++++++++++++++++-
 src/Interpreters/Aggregator.h                 | 105 ++++--
 src/Interpreters/AsynchronousMetrics.cpp      |  10 +
 src/Interpreters/InterpreterSelectQuery.cpp   |  10 +-
 tests/performance/hash_table_sizes_stats.xml  |  29 ++
 .../0_stateless/02122_parallel_formatting.sh  |  24 +-
 .../02151_hash_table_sizes_stats.reference    |  21 ++
 .../02151_hash_table_sizes_stats.sh           |  90 ++++++
 .../02151_hash_table_sizes_stats.testcases    | 195 +++++++++++
 ...sh_table_sizes_stats_distributed.reference |  33 ++
 ...2151_hash_table_sizes_stats_distributed.sh |  97 ++++++
 15 files changed, 875 insertions(+), 60 deletions(-)
 create mode 100644 tests/performance/hash_table_sizes_stats.xml
 create mode 100644 tests/queries/0_stateless/02151_hash_table_sizes_stats.reference
 create mode 100755 tests/queries/0_stateless/02151_hash_table_sizes_stats.sh
 create mode 100644 tests/queries/0_stateless/02151_hash_table_sizes_stats.testcases
 create mode 100644 tests/queries/0_stateless/02151_hash_table_sizes_stats_distributed.reference
 create mode 100755 tests/queries/0_stateless/02151_hash_table_sizes_stats_distributed.sh

diff --git a/src/Common/HashTable/FixedHashTable.h b/src/Common/HashTable/FixedHashTable.h
index 6ca8b6c4b17..d96e4d7342f 100644
--- a/src/Common/HashTable/FixedHashTable.h
+++ b/src/Common/HashTable/FixedHashTable.h
@@ -67,6 +67,9 @@ struct FixedHashTableCalculatedSize
 {
     size_t getSize(const Cell * buf, const typename Cell::State & state, size_t num_cells) const
     {
+        if (!buf)
+            return 0;
+
         size_t res = 0;
         for (const Cell * end = buf + num_cells; buf != end; ++buf)
             if (!buf->isZero(state))
@@ -76,6 +79,9 @@ struct FixedHashTableCalculatedSize
 
     bool isEmpty(const Cell * buf, const typename Cell::State & state, size_t num_cells) const
     {
+        if (!buf)
+            return true;
+
         for (const Cell * end = buf + num_cells; buf != end; ++buf)
             if (!buf->isZero(state))
                 return false;
diff --git a/src/Common/HashTable/TwoLevelHashTable.h b/src/Common/HashTable/TwoLevelHashTable.h
index 290ce054b5e..35c224c53f8 100644
--- a/src/Common/HashTable/TwoLevelHashTable.h
+++ b/src/Common/HashTable/TwoLevelHashTable.h
@@ -94,6 +94,12 @@ public:
 
     TwoLevelHashTable() = default;
 
+    explicit TwoLevelHashTable(size_t size_hint)
+    {
+        for (auto & impl : impls)
+            impl.reserve(size_hint / NUM_BUCKETS);
+    }
+
     /// Copy the data from another (normal) hash table. It should have the same hash function.
     template <typename Source>
     explicit TwoLevelHashTable(const Source & src)
diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp
index bff7b9589aa..074ec02394b 100644
--- a/src/Common/ProfileEvents.cpp
+++ b/src/Common/ProfileEvents.cpp
@@ -285,6 +285,9 @@
     \
     M(MainConfigLoads, "Number of times the main configuration was reloaded.") \
     \
+    M(AggregationPreallocatedElementsInHashTables, "How many elements were preallocated in hash tables for aggregation.") \
+    M(AggregationHashTablesInitializedAsTwoLevel, "How many hash tables were inited as two-level for aggregation.") \
+    \
     M(MergeTreeMetadataCacheGet, "Number of rocksdb reads(used for merge tree metadata cache)") \
     M(MergeTreeMetadataCachePut, "Number of rocksdb puts(used for merge tree metadata cache)") \
     M(MergeTreeMetadataCacheDelete, "Number of rocksdb deletes(used for merge tree metadata cache)") \
diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index f81b61ea648..1089a506181 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -500,6 +500,10 @@ class IColumn;
     M(Bool, optimize_rewrite_sum_if_to_count_if, true, "Rewrite sumIf() and sum(if()) function countIf() function when logically equivalent", 0) \
     M(UInt64, insert_shard_id, 0, "If non zero, when insert into a distributed table, the data will be inserted into the shard `insert_shard_id` synchronously. Possible values range from 1 to `shards_number` of corresponding distributed table", 0) \
     \
+    M(Bool, collect_hash_table_stats_during_aggregation, true, "Enable collecting hash table statistics to optimize memory allocation", 0) \
+    M(UInt64, max_entries_for_hash_table_stats, 10'000, "How many entries hash table statistics collected during aggregation is allowed to have", 0) \
+    M(UInt64, max_size_to_preallocate_for_aggregation, 10'000'000, "For how many elements it is allowed to preallocate space in all hash tables in total before aggregation", 0) \
+    \
     /** Experimental feature for moving data between shards. */ \
     \
     M(Bool, allow_experimental_query_deduplication, false, "Experimental data deduplication for SELECT queries based on part UUIDs", 0) \
diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp
index a2f24a79e40..55f9fe602f4 100644
--- a/src/Interpreters/Aggregator.cpp
+++ b/src/Interpreters/Aggregator.cpp
@@ -1,4 +1,6 @@
+#include <algorithm>
 #include <future>
+#include <numeric>
 #include <Poco/Util/Application.h>
 
 #include <base/sort.h>
@@ -15,6 +17,7 @@
 #include <IO/WriteBufferFromFile.h>
 #include <Compression/CompressedWriteBuffer.h>
 #include <Interpreters/Aggregator.h>
+#include <Common/LRUCache.h>
 #include <Common/MemoryTracker.h>
 #include <Common/CurrentThread.h>
 #include <Common/typeid_cast.h>
@@ -27,12 +30,236 @@
 #include <Interpreters/JIT/CompiledExpressionCache.h>
 #include <Core/ProtocolDefines.h>
 
+#include <Parsers/ASTSelectQuery.h>
 
 namespace ProfileEvents
 {
-    extern const Event ExternalAggregationWritePart;
-    extern const Event ExternalAggregationCompressedBytes;
-    extern const Event ExternalAggregationUncompressedBytes;
+extern const Event ExternalAggregationWritePart;
+extern const Event ExternalAggregationCompressedBytes;
+extern const Event ExternalAggregationUncompressedBytes;
+extern const Event AggregationPreallocatedElementsInHashTables;
+extern const Event AggregationHashTablesInitializedAsTwoLevel;
+}
+
+namespace
+{
+/** Collects observed HashMap-s sizes to avoid redundant intermediate resizes.
+  */
+class HashTablesStatistics
+{
+public:
+    struct Entry
+    {
+        size_t sum_of_sizes; // used to determine if it's better to convert aggregation to two-level from the beginning
+        size_t median_size; // roughly the size we're going to preallocate on each thread
+    };
+
+    using Cache = DB::LRUCache<UInt64, Entry>;
+    using CachePtr = std::shared_ptr<Cache>;
+    using Params = DB::Aggregator::Params::StatsCollectingParams;
+
+    /// Collection and use of the statistics should be enabled.
+    std::optional<Entry> getSizeHint(const Params & params)
+    {
+        if (!params.isCollectionAndUseEnabled())
+            throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Collection and use of the statistics should be enabled.");
+
+        std::lock_guard lock(mutex);
+        const auto cache = getHashTableStatsCache(params, lock);
+        if (const auto hint = cache->get(params.key))
+        {
+            LOG_DEBUG(
+                &Poco::Logger::get("Aggregator"),
+                "An entry for key={} found in cache: sum_of_sizes={}, median_size={}",
+                params.key,
+                hint->sum_of_sizes,
+                hint->median_size);
+            return *hint;
+        }
+        return std::nullopt;
+    }
+
+    /// Collection and use of the statistics should be enabled.
+    void update(size_t sum_of_sizes, size_t median_size, const Params & params)
+    {
+        if (!params.isCollectionAndUseEnabled())
+            throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Collection and use of the statistics should be enabled.");
+
+        std::lock_guard lock(mutex);
+        const auto cache = getHashTableStatsCache(params, lock);
+        const auto hint = cache->get(params.key);
+        // We'll maintain the maximum among all the observed values until the next prediction turns out to be too wrong.
+        if (!hint || sum_of_sizes < hint->sum_of_sizes / 2 || hint->sum_of_sizes < sum_of_sizes || median_size < hint->median_size / 2
+            || hint->median_size < median_size)
+        {
+            LOG_DEBUG(
+                &Poco::Logger::get("Aggregator"),
+                "Statistics updated for key={}: new sum_of_sizes={}, median_size={}",
+                params.key,
+                sum_of_sizes,
+                median_size);
+            cache->set(params.key, std::make_shared<Entry>(Entry{.sum_of_sizes = sum_of_sizes, .median_size = median_size}));
+        }
+    }
+
+    std::optional<DB::HashTablesCacheStatistics> getCacheStats() const
+    {
+        std::lock_guard lock(mutex);
+        if (hash_table_stats)
+        {
+            size_t hits = 0, misses = 0;
+            hash_table_stats->getStats(hits, misses);
+            return DB::HashTablesCacheStatistics{.entries = hash_table_stats->count(), .hits = hits, .misses = misses};
+        }
+        return std::nullopt;
+    }
+
+    static size_t calculateCacheKey(const DB::ASTPtr & select_query)
+    {
+        if (!select_query)
+            throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Query ptr cannot be null");
+
+        const auto & select = select_query->as<DB::ASTSelectQuery &>();
+
+        // It may happen in some corner cases like `select 1 as num group by num`.
+        if (!select.tables())
+            return 0;
+
+        SipHash hash;
+        hash.update(select.tables()->getTreeHash());
+        if (const auto where = select.where())
+            hash.update(where->getTreeHash());
+        if (const auto group_by = select.groupBy())
+            hash.update(group_by->getTreeHash());
+        return hash.get64();
+    }
+
+private:
+    CachePtr getHashTableStatsCache(const Params & params, const std::lock_guard<std::mutex> &)
+    {
+        if (!hash_table_stats || hash_table_stats->maxSize() != params.max_entries_for_hash_table_stats)
+            hash_table_stats = std::make_shared<Cache>(params.max_entries_for_hash_table_stats);
+        return hash_table_stats;
+    }
+
+    mutable std::mutex mutex;
+    CachePtr hash_table_stats;
+};
+
+HashTablesStatistics & getHashTablesStatistics()
+{
+    static HashTablesStatistics hash_tables_stats;
+    return hash_tables_stats;
+}
+
+bool worthConvertToTwoLevel(
+    size_t group_by_two_level_threshold, size_t result_size, size_t group_by_two_level_threshold_bytes, auto result_size_bytes)
+{
+    // params.group_by_two_level_threshold will be equal to 0 if we have only one thread to execute aggregation (refer to AggregatingStep::transformPipeline).
+    return (group_by_two_level_threshold && result_size >= group_by_two_level_threshold)
+        || (group_by_two_level_threshold_bytes && result_size_bytes >= static_cast<Int64>(group_by_two_level_threshold_bytes));
+}
+
+DB::AggregatedDataVariants::Type convertToTwoLevelTypeIfPossible(DB::AggregatedDataVariants::Type type)
+{
+    using Type = DB::AggregatedDataVariants::Type;
+    switch (type)
+    {
+#define M(NAME) \
+    case Type::NAME: \
+        return Type::NAME##_two_level;
+        APPLY_FOR_VARIANTS_CONVERTIBLE_TO_TWO_LEVEL(M)
+#undef M
+        default:
+            return type;
+    }
+    __builtin_unreachable();
+}
+
+void initDataVariantsWithSizeHint(
+    DB::AggregatedDataVariants & result, DB::AggregatedDataVariants::Type method_chosen, const DB::Aggregator::Params & params)
+{
+    const auto & stats_collecting_params = params.stats_collecting_params;
+    if (stats_collecting_params.isCollectionAndUseEnabled())
+    {
+        if (auto hint = getHashTablesStatistics().getSizeHint(stats_collecting_params))
+        {
+            const auto max_threads = params.group_by_two_level_threshold != 0 ? std::max(params.max_threads, 1ul) : 1;
+            const auto lower_limit = hint->sum_of_sizes / max_threads;
+            const auto upper_limit = stats_collecting_params.max_size_to_preallocate_for_aggregation / max_threads;
+            const auto adjusted = std::min(std::max(lower_limit, hint->median_size), upper_limit);
+            if (worthConvertToTwoLevel(
+                    params.group_by_two_level_threshold,
+                    hint->sum_of_sizes,
+                    /*group_by_two_level_threshold_bytes*/ 0,
+                    /*result_size_bytes*/ 0))
+                method_chosen = convertToTwoLevelTypeIfPossible(method_chosen);
+            result.init(method_chosen, adjusted);
+            ProfileEvents::increment(ProfileEvents::AggregationHashTablesInitializedAsTwoLevel, result.isTwoLevel());
+            return;
+        }
+    }
+    result.init(method_chosen);
+}
+
+/// Collection and use of the statistics should be enabled.
+void updateStatistics(const DB::ManyAggregatedDataVariants & data_variants, const DB::Aggregator::Params::StatsCollectingParams & params)
+{
+    if (!params.isCollectionAndUseEnabled())
+        throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Collection and use of the statistics should be enabled.");
+
+    std::vector<size_t> sizes(data_variants.size());
+    for (size_t i = 0; i < data_variants.size(); ++i)
+        sizes[i] = data_variants[i]->size();
+    const auto median_size = sizes.begin() + sizes.size() / 2; // not precisely though...
+    std::nth_element(sizes.begin(), median_size, sizes.end());
+    const auto sum_of_sizes = std::accumulate(sizes.begin(), sizes.end(), 0ull);
+    getHashTablesStatistics().update(sum_of_sizes, *median_size, params);
+}
+
+// The std::is_constructible trait isn't suitable here because some classes have template constructors with semantics different from providing size hints.
+// Also string hash table variants are not supported due to the fact that both local perf tests and tests in CI showed slowdowns for them.
+template <typename...>
+struct HasConstructorOfNumberOfElements : std::false_type
+{
+};
+
+template <typename... Ts>
+struct HasConstructorOfNumberOfElements<HashMapTable<Ts...>> : std::true_type
+{
+};
+
+template <typename Key, typename Cell, typename Hash, typename Grower, typename Allocator, template <typename...> typename ImplTable>
+struct HasConstructorOfNumberOfElements<TwoLevelHashMapTable<Key, Cell, Hash, Grower, Allocator, ImplTable>> : std::true_type
+{
+};
+
+template <typename... Ts>
+struct HasConstructorOfNumberOfElements<HashTable<Ts...>> : std::true_type
+{
+};
+
+template <typename... Ts>
+struct HasConstructorOfNumberOfElements<TwoLevelHashTable<Ts...>> : std::true_type
+{
+};
+
+template <template <typename> typename Method, typename Base>
+struct HasConstructorOfNumberOfElements<Method<Base>> : HasConstructorOfNumberOfElements<Base>
+{
+};
+
+template <typename Method>
+auto constructWithReserveIfPossible(size_t size_hint)
+{
+    if constexpr (HasConstructorOfNumberOfElements<typename Method::Data>::value)
+    {
+        ProfileEvents::increment(ProfileEvents::AggregationPreallocatedElementsInHashTables, size_hint);
+        return std::make_unique<Method>(size_hint);
+    }
+    else
+        return std::make_unique<Method>();
+}
 }
 
 namespace DB
@@ -64,6 +291,10 @@ AggregatedDataVariants::~AggregatedDataVariants()
     }
 }
 
+std::optional<HashTablesCacheStatistics> getHashTablesCacheStatistics()
+{
+    return getHashTablesStatistics().getCacheStats();
+}
 
 void AggregatedDataVariants::convertToTwoLevel()
 {
@@ -88,6 +319,47 @@ void AggregatedDataVariants::convertToTwoLevel()
     }
 }
 
+void AggregatedDataVariants::init(Type type_, std::optional<size_t> size_hint)
+{
+    switch (type_)
+    {
+        case Type::EMPTY:
+            break;
+        case Type::without_key:
+            break;
+
+#define M(NAME, IS_TWO_LEVEL) \
+    case Type::NAME: \
+        if (size_hint) \
+            (NAME) = constructWithReserveIfPossible<decltype(NAME)::element_type>(*size_hint); \
+        else \
+            (NAME) = std::make_unique<decltype(NAME)::element_type>(); \
+        break;
+            APPLY_FOR_AGGREGATED_VARIANTS(M)
+#undef M
+    }
+
+    type = type_;
+}
+
+Aggregator::Params::StatsCollectingParams::StatsCollectingParams() = default;
+
+Aggregator::Params::StatsCollectingParams::StatsCollectingParams(
+    const ASTPtr & select_query_,
+    bool collect_hash_table_stats_during_aggregation_,
+    size_t max_entries_for_hash_table_stats_,
+    size_t max_size_to_preallocate_for_aggregation_)
+    : key(collect_hash_table_stats_during_aggregation_ ? HashTablesStatistics::calculateCacheKey(select_query_) : 0)
+    , max_entries_for_hash_table_stats(max_entries_for_hash_table_stats_)
+    , max_size_to_preallocate_for_aggregation(max_size_to_preallocate_for_aggregation_)
+{
+}
+
+bool Aggregator::Params::StatsCollectingParams::isCollectionAndUseEnabled() const
+{
+    return key != 0;
+}
+
 Block Aggregator::getHeader(bool final) const
 {
     return params.getHeader(final);
@@ -237,8 +509,7 @@ public:
 
 #endif
 
-Aggregator::Aggregator(const Params & params_)
-    : params(params_)
+Aggregator::Aggregator(const Params & params_) : params(params_)
 {
     /// Use query-level memory tracker
     if (auto * memory_tracker_child = CurrentThread::getMemoryTracker())
@@ -292,7 +563,6 @@ Aggregator::Aggregator(const Params & params_)
 #if USE_EMBEDDED_COMPILER
     compileAggregateFunctionsIfNeeded();
 #endif
-
 }
 
 #if USE_EMBEDDED_COMPILER
@@ -958,7 +1228,7 @@ bool Aggregator::executeOnBlock(Columns columns, UInt64 num_rows, AggregatedData
     /// How to perform the aggregation?
     if (result.empty())
     {
-        result.init(method_chosen);
+        initDataVariantsWithSizeHint(result, method_chosen, params);
         result.keys_size = params.keys_size;
         result.key_sizes = key_sizes;
         LOG_TRACE(log, "Aggregation method: {}", result.getMethodName());
@@ -1038,9 +1308,8 @@ bool Aggregator::executeOnBlock(Columns columns, UInt64 num_rows, AggregatedData
     /// Here all the results in the sum are taken into account, from different threads.
     auto result_size_bytes = current_memory_usage - memory_usage_before_aggregation;
 
-    bool worth_convert_to_two_level
-        = (params.group_by_two_level_threshold && result_size >= params.group_by_two_level_threshold)
-        || (params.group_by_two_level_threshold_bytes && result_size_bytes >= static_cast<Int64>(params.group_by_two_level_threshold_bytes));
+    bool worth_convert_to_two_level = worthConvertToTwoLevel(
+        params.group_by_two_level_threshold, result_size, params.group_by_two_level_threshold_bytes, result_size_bytes);
 
     /** Converting to a two-level data structure.
       * It allows you to make, in the subsequent, an effective merge - either economical from memory or parallel.
@@ -1327,10 +1596,7 @@ void Aggregator::convertToBlockImpl(
 
 
 template <typename Mapped>
-inline void Aggregator::insertAggregatesIntoColumns(
-    Mapped & mapped,
-    MutableColumns & final_aggregate_columns,
-    Arena * arena) const
+inline void Aggregator::insertAggregatesIntoColumns(Mapped & mapped, MutableColumns & final_aggregate_columns, Arena * arena) const
 {
     /** Final values of aggregate functions are inserted to columns.
       * Then states of aggregate functions, that are not longer needed, are destroyed.
@@ -2179,6 +2445,9 @@ ManyAggregatedDataVariants Aggregator::prepareVariantsToMerge(ManyAggregatedData
 
     LOG_TRACE(log, "Merging aggregated data");
 
+    if (params.stats_collecting_params.isCollectionAndUseEnabled())
+        updateStatistics(data_variants, params.stats_collecting_params);
+
     ManyAggregatedDataVariants non_empty_data;
     non_empty_data.reserve(data_variants.size());
     for (auto & data : data_variants)
@@ -2388,9 +2657,8 @@ bool Aggregator::mergeOnBlock(Block block, AggregatedDataVariants & result, bool
     /// Here all the results in the sum are taken into account, from different threads.
     auto result_size_bytes = current_memory_usage - memory_usage_before_aggregation;
 
-    bool worth_convert_to_two_level
-        = (params.group_by_two_level_threshold && result_size >= params.group_by_two_level_threshold)
-        || (params.group_by_two_level_threshold_bytes && result_size_bytes >= static_cast<Int64>(params.group_by_two_level_threshold_bytes));
+    bool worth_convert_to_two_level = worthConvertToTwoLevel(
+        params.group_by_two_level_threshold, result_size, params.group_by_two_level_threshold_bytes, result_size_bytes);
 
     /** Converting to a two-level data structure.
       * It allows you to make, in the subsequent, an effective merge - either economical from memory or parallel.
diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h
index f03bf45fbc6..49094cff7b9 100644
--- a/src/Interpreters/Aggregator.h
+++ b/src/Interpreters/Aggregator.h
@@ -34,6 +34,7 @@
 #include <Columns/ColumnNullable.h>
 #include <Columns/ColumnLowCardinality.h>
 
+#include <Parsers/IAST_fwd.h>
 
 namespace DB
 {
@@ -129,6 +130,7 @@ private:
 template <typename Base>
 struct AggregationDataWithNullKeyTwoLevel : public Base
 {
+    using Base::Base;
     using Base::impls;
 
     AggregationDataWithNullKeyTwoLevel() = default;
@@ -183,6 +185,8 @@ struct AggregationMethodOneNumber
 
     AggregationMethodOneNumber() = default;
 
+    explicit AggregationMethodOneNumber(size_t size_hint) : data(size_hint) { }
+
     template <typename Other>
     explicit AggregationMethodOneNumber(const Other & other) : data(other.data)
     {
@@ -225,6 +229,8 @@ struct AggregationMethodString
     {
     }
 
+    explicit AggregationMethodString(size_t size_hint) : data(size_hint) { }
+
     using State = ColumnsHashing::HashMethodString<typename Data::value_type, Mapped>;
 
     static const bool low_cardinality_optimization = false;
@@ -250,6 +256,8 @@ struct AggregationMethodStringNoCache
 
     AggregationMethodStringNoCache() = default;
 
+    explicit AggregationMethodStringNoCache(size_t size_hint) : data(size_hint) { }
+
     template <typename Other>
     explicit AggregationMethodStringNoCache(const Other & other) : data(other.data)
     {
@@ -280,6 +288,8 @@ struct AggregationMethodFixedString
 
     AggregationMethodFixedString() = default;
 
+    explicit AggregationMethodFixedString(size_t size_hint) : data(size_hint) { }
+
     template <typename Other>
     explicit AggregationMethodFixedString(const Other & other) : data(other.data)
     {
@@ -309,6 +319,8 @@ struct AggregationMethodFixedStringNoCache
 
     AggregationMethodFixedStringNoCache() = default;
 
+    explicit AggregationMethodFixedStringNoCache(size_t size_hint) : data(size_hint) { }
+
     template <typename Other>
     explicit AggregationMethodFixedStringNoCache(const Other & other) : data(other.data)
     {
@@ -382,6 +394,8 @@ struct AggregationMethodKeysFixed
 
     AggregationMethodKeysFixed() = default;
 
+    explicit AggregationMethodKeysFixed(size_t size_hint) : data(size_hint) { }
+
     template <typename Other>
     explicit AggregationMethodKeysFixed(const Other & other) : data(other.data)
     {
@@ -473,6 +487,8 @@ struct AggregationMethodSerialized
 
     AggregationMethodSerialized() = default;
 
+    explicit AggregationMethodSerialized(size_t size_hint) : data(size_hint) { }
+
     template <typename Other>
     explicit AggregationMethodSerialized(const Other & other) : data(other.data)
     {
@@ -652,21 +668,7 @@ struct AggregatedDataVariants : private boost::noncopyable
 
     ~AggregatedDataVariants();
 
-    void init(Type type_)
-    {
-        switch (type_)
-        {
-            case Type::EMPTY:       break;
-            case Type::without_key: break;
-
-        #define M(NAME, IS_TWO_LEVEL) \
-            case Type::NAME: (NAME) = std::make_unique<decltype(NAME)::element_type>(); break;
-            APPLY_FOR_AGGREGATED_VARIANTS(M)
-        #undef M
-        }
-
-        type = type_;
-    }
+    void init(Type type_, std::optional<size_t> size_hint = std::nullopt);
 
     /// Number of rows (different keys).
     size_t size() const
@@ -929,29 +931,61 @@ public:
         bool compile_aggregate_expressions;
         size_t min_count_to_compile_aggregate_expression;
 
+        struct StatsCollectingParams
+        {
+            StatsCollectingParams();
+
+            StatsCollectingParams(
+                const ASTPtr & select_query_,
+                bool collect_hash_table_stats_during_aggregation_,
+                size_t max_entries_for_hash_table_stats_,
+                size_t max_size_to_preallocate_for_aggregation_);
+
+            bool isCollectionAndUseEnabled() const;
+
+            const UInt64 key = 0;
+            const size_t max_entries_for_hash_table_stats = 0;
+            const size_t max_size_to_preallocate_for_aggregation = 0;
+        };
+        StatsCollectingParams stats_collecting_params;
+
         Params(
             const Block & src_header_,
-            const ColumnNumbers & keys_, const AggregateDescriptions & aggregates_,
-            bool overflow_row_, size_t max_rows_to_group_by_, OverflowMode group_by_overflow_mode_,
-            size_t group_by_two_level_threshold_, size_t group_by_two_level_threshold_bytes_,
+            const ColumnNumbers & keys_,
+            const AggregateDescriptions & aggregates_,
+            bool overflow_row_,
+            size_t max_rows_to_group_by_,
+            OverflowMode group_by_overflow_mode_,
+            size_t group_by_two_level_threshold_,
+            size_t group_by_two_level_threshold_bytes_,
             size_t max_bytes_before_external_group_by_,
             bool empty_result_for_aggregation_by_empty_set_,
-            VolumePtr tmp_volume_, size_t max_threads_,
+            VolumePtr tmp_volume_,
+            size_t max_threads_,
             size_t min_free_disk_space_,
             bool compile_aggregate_expressions_,
             size_t min_count_to_compile_aggregate_expression_,
-            const Block & intermediate_header_ = {})
-            : src_header(src_header_),
-            intermediate_header(intermediate_header_),
-            keys(keys_), aggregates(aggregates_), keys_size(keys.size()), aggregates_size(aggregates.size()),
-            overflow_row(overflow_row_), max_rows_to_group_by(max_rows_to_group_by_), group_by_overflow_mode(group_by_overflow_mode_),
-            group_by_two_level_threshold(group_by_two_level_threshold_), group_by_two_level_threshold_bytes(group_by_two_level_threshold_bytes_),
-            max_bytes_before_external_group_by(max_bytes_before_external_group_by_),
-            empty_result_for_aggregation_by_empty_set(empty_result_for_aggregation_by_empty_set_),
-            tmp_volume(tmp_volume_), max_threads(max_threads_),
-            min_free_disk_space(min_free_disk_space_),
-            compile_aggregate_expressions(compile_aggregate_expressions_),
-            min_count_to_compile_aggregate_expression(min_count_to_compile_aggregate_expression_)
+            const Block & intermediate_header_ = {},
+            const StatsCollectingParams & stats_collecting_params_ = {})
+            : src_header(src_header_)
+            , intermediate_header(intermediate_header_)
+            , keys(keys_)
+            , aggregates(aggregates_)
+            , keys_size(keys.size())
+            , aggregates_size(aggregates.size())
+            , overflow_row(overflow_row_)
+            , max_rows_to_group_by(max_rows_to_group_by_)
+            , group_by_overflow_mode(group_by_overflow_mode_)
+            , group_by_two_level_threshold(group_by_two_level_threshold_)
+            , group_by_two_level_threshold_bytes(group_by_two_level_threshold_bytes_)
+            , max_bytes_before_external_group_by(max_bytes_before_external_group_by_)
+            , empty_result_for_aggregation_by_empty_set(empty_result_for_aggregation_by_empty_set_)
+            , tmp_volume(tmp_volume_)
+            , max_threads(max_threads_)
+            , min_free_disk_space(min_free_disk_space_)
+            , compile_aggregate_expressions(compile_aggregate_expressions_)
+            , min_count_to_compile_aggregate_expression(min_count_to_compile_aggregate_expression_)
+            , stats_collecting_params(stats_collecting_params_)
         {
         }
 
@@ -1350,4 +1384,13 @@ APPLY_FOR_AGGREGATED_VARIANTS(M)
 
 #undef M
 
+
+struct HashTablesCacheStatistics
+{
+    size_t entries = 0;
+    size_t hits = 0;
+    size_t misses = 0;
+};
+
+std::optional<HashTablesCacheStatistics> getHashTablesCacheStatistics();
 }
diff --git a/src/Interpreters/AsynchronousMetrics.cpp b/src/Interpreters/AsynchronousMetrics.cpp
index 0a4e4ec7ecf..7d2e06a4ed9 100644
--- a/src/Interpreters/AsynchronousMetrics.cpp
+++ b/src/Interpreters/AsynchronousMetrics.cpp
@@ -1,3 +1,4 @@
+#include <Interpreters/Aggregator.h>
 #include <Interpreters/AsynchronousMetrics.h>
 #include <Interpreters/AsynchronousMetricLog.h>
 #include <Interpreters/JIT/CompiledExpressionCache.h>
@@ -630,6 +631,15 @@ void AsynchronousMetrics::update(std::chrono::system_clock::time_point update_ti
 
     new_values["Uptime"] = getContext()->getUptimeSeconds();
 
+    {
+        if (const auto stats = getHashTablesCacheStatistics())
+        {
+            new_values["HashTableStatsCacheEntries"] = stats->entries;
+            new_values["HashTableStatsCacheHits"] = stats->hits;
+            new_values["HashTableStatsCacheMisses"] = stats->misses;
+        }
+    }
+
     /// Process process memory usage according to OS
 #if defined(OS_LINUX) || defined(OS_FREEBSD)
     {
diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp
index 27ed8438fc8..dda2e3f2142 100644
--- a/src/Interpreters/InterpreterSelectQuery.cpp
+++ b/src/Interpreters/InterpreterSelectQuery.cpp
@@ -2082,6 +2082,12 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac
 
     const Settings & settings = context->getSettingsRef();
 
+    const auto stats_collecting_params = Aggregator::Params::StatsCollectingParams(
+        query_ptr,
+        settings.collect_hash_table_stats_during_aggregation,
+        settings.max_entries_for_hash_table_stats,
+        settings.max_size_to_preallocate_for_aggregation);
+
     Aggregator::Params params(
         header_before_aggregation,
         keys,
@@ -2099,7 +2105,9 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac
         settings.max_threads,
         settings.min_free_disk_space_for_temporary_data,
         settings.compile_aggregate_expressions,
-        settings.min_count_to_compile_aggregate_expression);
+        settings.min_count_to_compile_aggregate_expression,
+        Block{},
+        stats_collecting_params);
 
     SortDescription group_by_sort_description;
 
diff --git a/tests/performance/hash_table_sizes_stats.xml b/tests/performance/hash_table_sizes_stats.xml
new file mode 100644
index 00000000000..5bdd83126b8
--- /dev/null
+++ b/tests/performance/hash_table_sizes_stats.xml
@@ -0,0 +1,29 @@
+<test>
+    <preconditions>
+        <table_exists>hits_10m_single</table_exists>
+        <table_exists>hits_100m_single</table_exists>
+    </preconditions>
+
+    <settings>
+        <max_size_to_preallocate_for_aggregation>1000000000</max_size_to_preallocate_for_aggregation>
+    </settings>
+
+    <query>SELECT number FROM numbers(5000000) GROUP BY number FORMAT Null</query>
+    <query>SELECT number FROM numbers(10000000) GROUP BY number FORMAT Null</query>
+    <query short="1">SELECT number FROM numbers_mt(500000) GROUP BY number FORMAT Null</query>
+    <query short="1">SELECT number FROM numbers_mt(1000000) GROUP BY number FORMAT Null</query>
+    <query>SELECT number FROM numbers_mt(10000000) GROUP BY number FORMAT Null</query>
+    <query>SELECT number FROM numbers_mt(50000000) GROUP BY number FORMAT Null</query>
+    <query>WITH number % 524289 AS k, toUInt64(k) AS k1, k1 + 1 AS k2 SELECT k1, k2, count() FROM numbers(100000000) GROUP BY k1, k2 FORMAT Null</query>
+    <query>SELECT number FROM numbers_mt(10000000) GROUP BY number FORMAT Null SETTINGS group_by_two_level_threshold = 1e12, group_by_two_level_threshold_bytes = 1e12</query>
+    <query>SELECT number FROM numbers_mt(50000000) GROUP BY number FORMAT Null SETTINGS group_by_two_level_threshold = 1e12, group_by_two_level_threshold_bytes = 1e12</query>
+
+    <query>SELECT WatchID FROM hits_10m_single GROUP BY WatchID FORMAT Null</query>
+    <query>SELECT WatchID FROM hits_100m_single GROUP BY WatchID FORMAT Null</query>
+    <query>SELECT ClientIP AS x, x - 1, x - 2, x - 3, count() AS c FROM hits_10m_single GROUP BY x, x - 1, x - 2, x - 3 ORDER BY c DESC LIMIT 10</query>
+    <query>SELECT ClientIP AS x, x - 1, x - 2, x - 3, count() AS c FROM hits_100m_single GROUP BY x, x - 1, x - 2, x - 3 ORDER BY c DESC LIMIT 10</query>
+    <query>SELECT WatchID, ClientIP, count() AS c, sum(Refresh), avg(ResolutionWidth) FROM hits_10m_single WHERE SearchPhrase != '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10</query>
+    <query>SELECT WatchID, ClientIP, count() AS c, sum(Refresh), avg(ResolutionWidth) FROM hits_100m_single WHERE SearchPhrase != '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10</query>
+    <query>SELECT min(MobilePhoneModel) FROM hits_10m_single WHERE MobilePhoneModel != '' GROUP BY intHash32(UserID) % 1000000 FORMAT Null</query>
+    <query>SELECT min(MobilePhoneModel) FROM hits_100m_single WHERE MobilePhoneModel != '' GROUP BY intHash32(UserID) % 1000000 FORMAT Null</query>
+</test>
diff --git a/tests/queries/0_stateless/02122_parallel_formatting.sh b/tests/queries/0_stateless/02122_parallel_formatting.sh
index 97f7a3903e7..4257b20f11f 100755
--- a/tests/queries/0_stateless/02122_parallel_formatting.sh
+++ b/tests/queries/0_stateless/02122_parallel_formatting.sh
@@ -18,8 +18,8 @@ for format in ${formats}; do
     diff $non_parallel_file $parallel_file
 
     echo $format-2
-    $CLICKHOUSE_CLIENT -q "select number, number + 1, concat('string: ', toString(number)) from numbers(200000) group by number with totals limit 190000 format $format" --extremes=1 --output_format_parallel_formatting=0 --output_format_pretty_max_rows=1000000 | grep -a -v "elapsed" > $non_parallel_file
-    $CLICKHOUSE_CLIENT -q "select number, number + 1, concat('string: ', toString(number)) from numbers(200000) group by number with totals limit 190000 format $format" --extremes=1 --output_format_parallel_formatting=1 --output_format_pretty_max_rows=1000000 | grep -a -v "elapsed" > $parallel_file
+    $CLICKHOUSE_CLIENT -q "select number, number + 1, concat('string: ', toString(number)) from numbers(200000) group by number with totals order by number limit 190000 format $format" --extremes=1 --output_format_parallel_formatting=0 --output_format_pretty_max_rows=1000000 | grep -a -v "elapsed" > $non_parallel_file
+    $CLICKHOUSE_CLIENT -q "select number, number + 1, concat('string: ', toString(number)) from numbers(200000) group by number with totals order by number limit 190000 format $format" --extremes=1 --output_format_parallel_formatting=1 --output_format_pretty_max_rows=1000000 | grep -a -v "elapsed" > $parallel_file
 
     diff $non_parallel_file $parallel_file
 done
@@ -33,15 +33,17 @@ $CLICKHOUSE_CLIENT -q "select number, number + 1, concat('string: ', toString(nu
 diff $non_parallel_file $parallel_file
 
 echo "CustomSeparated-2"
-$CLICKHOUSE_CLIENT -q "select number, number + 1, concat('string: ', toString(number)) from numbers(200000) group by number with totals limit 190000 format CustomSeparated $CUSTOM_SETTINGS" --output_format_parallel_formatting=0 --extremes=1 > $non_parallel_file
-$CLICKHOUSE_CLIENT -q "select number, number + 1, concat('string: ', toString(number)) from numbers(200000) group by number with totals limit 190000 format CustomSeparated $CUSTOM_SETTINGS" --output_format_parallel_formatting=1 --extremes=1 > $parallel_file
+$CLICKHOUSE_CLIENT -q "select number, number + 1, concat('string: ', toString(number)) from numbers(200000) group by number with totals order by number limit 190000 format CustomSeparated $CUSTOM_SETTINGS" --output_format_parallel_formatting=0 --extremes=1 > $non_parallel_file
+$CLICKHOUSE_CLIENT -q "select number, number + 1, concat('string: ', toString(number)) from numbers(200000) group by number with totals order by number limit 190000 format CustomSeparated $CUSTOM_SETTINGS" --output_format_parallel_formatting=1 --extremes=1 > $parallel_file
 
 diff $non_parallel_file $parallel_file
 
-echo -ne '{prefix} \n${data}\n $$ suffix $$\n' > "$CUR_DIR"/02122_template_format_resultset.tmp
-echo -ne 'x:${x:Quoted}, y:${y:Quoted}, s:${s:Quoted}' > "$CUR_DIR"/02122_template_format_row.tmp
+resultset_path=$CUR_DIR/$CLICKHOUSE_TEST_UNIQUE_NAME"_template_format_resultset.tmp"
+echo -ne '{prefix} \n${data}\n $$ suffix $$\n' > $resultset_path
+row_path=$CUR_DIR/$CLICKHOUSE_TEST_UNIQUE_NAME"_template_format_row.tmp"
+echo -ne 'x:${x:Quoted}, y:${y:Quoted}, s:${s:Quoted}' > $row_path
 
-TEMPLATE_SETTINGS="SETTINGS format_template_resultset = '$CUR_DIR/02122_template_format_resultset.tmp', format_template_row = '$CUR_DIR/02122_template_format_row.tmp', format_template_rows_between_delimiter = ';\n'"
+TEMPLATE_SETTINGS="SETTINGS format_template_resultset = '$resultset_path', format_template_row = '$row_path', format_template_rows_between_delimiter = ';\n'"
 
 echo "Template-1"
 $CLICKHOUSE_CLIENT -q "select number as x, number + 1 as y, concat('string: ', toString(number)) as s from numbers(200000) format Template $TEMPLATE_SETTINGS" --output_format_parallel_formatting=0 > $non_parallel_file
@@ -49,14 +51,14 @@ $CLICKHOUSE_CLIENT -q "select number as x, number + 1 as y, concat('string: ', t
 
 diff $non_parallel_file $parallel_file
 
-echo -ne '{prefix} \n${data}\n $$ suffix $$\n${totals}\n${min}\n${max}\n${rows:Quoted}\n${rows_before_limit:Quoted}\n${rows_read:Quoted}\n${bytes_read:Quoted}\n' > "$CUR_DIR"/02122_template_format_resultset.tmp
+echo -ne '{prefix} \n${data}\n $$ suffix $$\n${totals}\n${min}\n${max}\n${rows:Quoted}\n${rows_before_limit:Quoted}\n${rows_read:Quoted}\n${bytes_read:Quoted}\n' > $resultset_path
 
 echo "Template-2"
-$CLICKHOUSE_CLIENT -q "select number as x, number + 1 as y, concat('string: ', toString(number)) as s from numbers(200000) group by number with totals limit 190000 format Template $TEMPLATE_SETTINGS" --output_format_parallel_formatting=0 --extremes=1 > $non_parallel_file
-$CLICKHOUSE_CLIENT -q "select number as x, number + 1 as y, concat('string: ', toString(number)) as s from numbers(200000) group by number with totals limit 190000 format Template $TEMPLATE_SETTINGS" --output_format_parallel_formatting=1 --extremes=1 > $parallel_file
+$CLICKHOUSE_CLIENT -q "select number as x, number + 1 as y, concat('string: ', toString(number)) as s from numbers(200000) group by number with totals order by number limit 190000 format Template $TEMPLATE_SETTINGS" --output_format_parallel_formatting=0 --extremes=1 > $non_parallel_file
+$CLICKHOUSE_CLIENT -q "select number as x, number + 1 as y, concat('string: ', toString(number)) as s from numbers(200000) group by number with totals order by number limit 190000 format Template $TEMPLATE_SETTINGS" --output_format_parallel_formatting=1 --extremes=1 > $parallel_file
 
 diff $non_parallel_file $parallel_file
 
 rm $non_parallel_file $parallel_file
-rm "$CUR_DIR"/02122_template_format_resultset.tmp "$CUR_DIR"/02122_template_format_row.tmp
+rm $resultset_path $row_path
 
diff --git a/tests/queries/0_stateless/02151_hash_table_sizes_stats.reference b/tests/queries/0_stateless/02151_hash_table_sizes_stats.reference
new file mode 100644
index 00000000000..712e2b058a4
--- /dev/null
+++ b/tests/queries/0_stateless/02151_hash_table_sizes_stats.reference
@@ -0,0 +1,21 @@
+1
+--
+1
+--
+1
+--
+1
+--
+1
+1
+--
+1
+--
+1
+1
+--
+1
+--
+1
+1
+--
diff --git a/tests/queries/0_stateless/02151_hash_table_sizes_stats.sh b/tests/queries/0_stateless/02151_hash_table_sizes_stats.sh
new file mode 100755
index 00000000000..9a15111051a
--- /dev/null
+++ b/tests/queries/0_stateless/02151_hash_table_sizes_stats.sh
@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+# Tags: long
+
+# shellcheck disable=SC2154
+
+unset CLICKHOUSE_LOG_COMMENT
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+
+# tests rely on that all the rows are unique and max_threads divides table_size
+table_size=10000
+max_threads=5
+
+
+prepare_table() {
+  table_name="t_hash_table_sizes_stats_$RANDOM$RANDOM"
+  $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS $table_name;"
+  if [ -z "$1" ]; then
+    $CLICKHOUSE_CLIENT -q "CREATE TABLE $table_name(number UInt64) Engine=MergeTree() ORDER BY tuple();"
+  else
+    $CLICKHOUSE_CLIENT -q "CREATE TABLE $table_name(number UInt64) Engine=MergeTree() ORDER BY $1;"
+  fi
+  $CLICKHOUSE_CLIENT -q "SYSTEM STOP MERGES $table_name;"
+  for ((i = 1; i <= max_threads; i++)); do
+    cnt=$((table_size / max_threads))
+    from=$(((i - 1) * cnt))
+    $CLICKHOUSE_CLIENT -q "INSERT INTO $table_name SELECT * FROM numbers($from, $cnt);"
+  done
+}
+
+prepare_table_with_sorting_key() {
+  prepare_table "$1"
+}
+
+run_query() {
+  query_id="${CLICKHOUSE_DATABASE}_hash_table_sizes_stats_$RANDOM$RANDOM"
+  $CLICKHOUSE_CLIENT --query_id="$query_id" --multiquery -q "
+    SET max_block_size = $((table_size / 10));
+    SET merge_tree_min_rows_for_concurrent_read = 1;
+    SET max_untracked_memory = 0;
+    SET max_size_to_preallocate_for_aggregation = 1e12;
+    $query"
+}
+
+check_preallocated_elements() {
+  $CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS"
+  # rows may be distributed in any way including "everything goes to the one particular thread"
+  min=$1
+  if [ -z "$2" ]; then
+    max=$1
+  else
+    max=$2
+  fi
+  $CLICKHOUSE_CLIENT --param_query_id="$query_id" -q "
+    SELECT COUNT(*)
+      FROM system.query_log
+     WHERE event_date >= yesterday() AND query_id = {query_id:String} AND current_database = currentDatabase()
+           AND ProfileEvents['AggregationPreallocatedElementsInHashTables'] BETWEEN $min AND $max"
+}
+
+check_convertion_to_two_level() {
+  $CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS"
+  # rows may be distributed in any way including "everything goes to the one particular thread"
+  $CLICKHOUSE_CLIENT --param_query_id="$query_id" -q "
+    SELECT SUM(ProfileEvents['AggregationHashTablesInitializedAsTwoLevel']) BETWEEN 1 AND $max_threads
+      FROM system.query_log
+     WHERE event_date >= yesterday() AND query_id = {query_id:String} AND current_database = currentDatabase()"
+}
+
+print_border() {
+  echo "--"
+}
+
+
+# shellcheck source=./02151_hash_table_sizes_stats.testcases
+source "$CURDIR"/02151_hash_table_sizes_stats.testcases
+
+
+test_one_thread_simple_group_by
+test_one_thread_simple_group_by_with_limit
+test_one_thread_simple_group_by_with_join_and_subquery
+test_several_threads_simple_group_by_with_limit_single_level_ht
+test_several_threads_simple_group_by_with_limit_two_level_ht
+test_several_threads_simple_group_by_with_limit_and_rollup_single_level_ht
+test_several_threads_simple_group_by_with_limit_and_rollup_two_level_ht
+test_several_threads_simple_group_by_with_limit_and_cube_single_level_ht
+test_several_threads_simple_group_by_with_limit_and_cube_two_level_ht
diff --git a/tests/queries/0_stateless/02151_hash_table_sizes_stats.testcases b/tests/queries/0_stateless/02151_hash_table_sizes_stats.testcases
new file mode 100644
index 00000000000..00054b89cc7
--- /dev/null
+++ b/tests/queries/0_stateless/02151_hash_table_sizes_stats.testcases
@@ -0,0 +1,195 @@
+test_one_thread_simple_group_by() {
+  expected_size_hint=$table_size
+  prepare_table
+
+  query="
+  -- size_hint = $expected_size_hint --
+    SELECT number
+      FROM $table_name
+  GROUP BY number
+  SETTINGS max_threads = 1
+    FORMAT Null;"
+
+  run_query
+  run_query
+  check_preallocated_elements $expected_size_hint
+  print_border
+}
+
+test_one_thread_simple_group_by_with_limit() {
+  expected_size_hint=$table_size
+  prepare_table
+
+  query="
+  -- size_hint = $expected_size_hint despite the presence of limit --
+    SELECT number
+      FROM $table_name
+  GROUP BY number
+     LIMIT 5
+  SETTINGS max_threads = 1
+    FORMAT Null;"
+
+  run_query
+  run_query
+  check_preallocated_elements $expected_size_hint
+  print_border
+}
+
+test_one_thread_simple_group_by_with_join_and_subquery() {
+  expected_size_hint=$((table_size + table_size / 2))
+  prepare_table
+
+  query="
+  -- expected two size_hints for different keys: for the inner ($table_size) and the outer aggregation ($((table_size / 2)))
+    SELECT number
+      FROM $table_name AS t1
+      JOIN
+      (
+        SELECT number
+          FROM $table_name AS t2
+      GROUP BY number
+         LIMIT $((table_size / 2))
+      ) AS t3 USING(number)
+  GROUP BY number
+  SETTINGS max_threads = 1,
+           distributed_product_mode = 'local'
+    FORMAT Null;"
+
+  run_query
+  run_query
+  check_preallocated_elements $expected_size_hint
+  print_border
+}
+
+test_several_threads_simple_group_by_with_limit_single_level_ht() {
+  expected_size_hint=$table_size
+  prepare_table
+
+  query="
+  -- size_hint = $expected_size_hint despite the presence of limit --
+    SELECT number
+      FROM $table_name
+  GROUP BY number
+     LIMIT 5
+  SETTINGS max_threads = $max_threads,
+           group_by_two_level_threshold = $((expected_size_hint + 1)),
+           group_by_two_level_threshold_bytes = $((table_size * 1000))
+    FORMAT Null;"
+
+  run_query
+  run_query
+  check_preallocated_elements $((expected_size_hint / max_threads)) $((expected_size_hint * max_threads))
+  print_border
+}
+
+test_several_threads_simple_group_by_with_limit_two_level_ht() {
+  expected_size_hint=$table_size
+  prepare_table
+
+  query="
+  -- size_hint = $expected_size_hint despite the presence of limit --
+    SELECT number
+      FROM $table_name
+  GROUP BY number
+     LIMIT 5
+  SETTINGS max_threads = $max_threads,
+           group_by_two_level_threshold = $expected_size_hint,
+           group_by_two_level_threshold_bytes = $((table_size * 1000))
+    FORMAT Null;"
+
+  run_query
+  run_query
+  check_convertion_to_two_level
+  check_preallocated_elements $((expected_size_hint / max_threads)) $((expected_size_hint * max_threads))
+  print_border
+}
+
+test_several_threads_simple_group_by_with_limit_and_rollup_single_level_ht() {
+  expected_size_hint=$table_size
+  prepare_table
+
+  query="
+  -- size_hint = $expected_size_hint despite the presence of limit --
+       SELECT number
+         FROM $table_name
+     GROUP BY number
+  WITH ROLLUP
+        LIMIT 5
+     SETTINGS max_threads = $max_threads,
+              group_by_two_level_threshold = $((expected_size_hint + 1)),
+              group_by_two_level_threshold_bytes = $((table_size * 1000))
+       FORMAT Null;"
+
+  run_query
+  run_query
+  check_preallocated_elements $((expected_size_hint / max_threads)) $((expected_size_hint * max_threads))
+  print_border
+}
+
+test_several_threads_simple_group_by_with_limit_and_rollup_two_level_ht() {
+  expected_size_hint=$table_size
+  prepare_table
+
+  query="
+  -- size_hint = $expected_size_hint despite the presence of limit --
+       SELECT number
+         FROM $table_name
+     GROUP BY number
+  WITH ROLLUP
+        LIMIT 5
+     SETTINGS max_threads = $max_threads,
+              group_by_two_level_threshold = $expected_size_hint,
+              group_by_two_level_threshold_bytes = $((table_size * 1000))
+       FORMAT Null;"
+
+  run_query
+  run_query
+  check_convertion_to_two_level
+  check_preallocated_elements $((expected_size_hint / max_threads)) $((expected_size_hint * max_threads))
+  print_border
+}
+
+test_several_threads_simple_group_by_with_limit_and_cube_single_level_ht() {
+  expected_size_hint=$table_size
+  prepare_table
+
+  query="
+  -- size_hint = $expected_size_hint despite the presence of limit --
+     SELECT number
+       FROM $table_name
+   GROUP BY number
+  WITH CUBE
+      LIMIT 5
+   SETTINGS max_threads = $max_threads,
+            group_by_two_level_threshold = $((expected_size_hint + 1)),
+            group_by_two_level_threshold_bytes = $((table_size * 1000))
+     FORMAT Null;"
+
+  run_query
+  run_query
+  check_preallocated_elements $((expected_size_hint / max_threads)) $((expected_size_hint * max_threads))
+  print_border
+}
+
+test_several_threads_simple_group_by_with_limit_and_cube_two_level_ht() {
+  expected_size_hint=$table_size
+  prepare_table
+
+  query="
+  -- size_hint = $expected_size_hint despite the presence of limit --
+     SELECT number
+       FROM $table_name
+   GROUP BY number
+  WITH CUBE
+      LIMIT 5
+   SETTINGS max_threads = $max_threads,
+            group_by_two_level_threshold = $expected_size_hint,
+            group_by_two_level_threshold_bytes = $((table_size * 1000))
+     FORMAT Null;"
+
+  run_query
+  run_query
+  check_convertion_to_two_level
+  check_preallocated_elements $((expected_size_hint / max_threads)) $((expected_size_hint * max_threads))
+  print_border
+}
diff --git a/tests/queries/0_stateless/02151_hash_table_sizes_stats_distributed.reference b/tests/queries/0_stateless/02151_hash_table_sizes_stats_distributed.reference
new file mode 100644
index 00000000000..0d10114f4ff
--- /dev/null
+++ b/tests/queries/0_stateless/02151_hash_table_sizes_stats_distributed.reference
@@ -0,0 +1,33 @@
+1
+1
+--
+1
+1
+--
+1
+1
+--
+1
+1
+--
+1
+1
+1
+1
+--
+1
+1
+--
+1
+1
+1
+1
+--
+1
+1
+--
+1
+1
+1
+1
+--
diff --git a/tests/queries/0_stateless/02151_hash_table_sizes_stats_distributed.sh b/tests/queries/0_stateless/02151_hash_table_sizes_stats_distributed.sh
new file mode 100755
index 00000000000..4ae61609d00
--- /dev/null
+++ b/tests/queries/0_stateless/02151_hash_table_sizes_stats_distributed.sh
@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+# Tags: long, distributed
+
+# These tests don't use `current_database = currentDatabase()` condition, because database name isn't propagated during remote queries.
+
+# shellcheck disable=SC2154
+
+unset CLICKHOUSE_LOG_COMMENT
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+
+# tests rely on that all the rows are unique and max_threads divides table_size
+table_size=10000
+max_threads=5
+
+
+prepare_table() {
+  table_name="t_hash_table_sizes_stats_$RANDOM$RANDOM"
+  $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS $table_name;"
+  if [ -z "$1" ]; then
+    $CLICKHOUSE_CLIENT -q "CREATE TABLE $table_name(number UInt64) Engine=MergeTree() ORDER BY tuple();"
+  else
+    $CLICKHOUSE_CLIENT -q "CREATE TABLE $table_name(number UInt64) Engine=MergeTree() ORDER BY $1;"
+  fi
+  $CLICKHOUSE_CLIENT -q "SYSTEM STOP MERGES $table_name;"
+  for ((i = 1; i <= max_threads; i++)); do
+    cnt=$((table_size / max_threads))
+    from=$(((i - 1) * cnt))
+    $CLICKHOUSE_CLIENT -q "INSERT INTO $table_name SELECT * FROM numbers($from, $cnt);"
+  done
+  $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS ${table_name}_d;"
+  $CLICKHOUSE_CLIENT -q "CREATE TABLE ${table_name}_d AS $table_name ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), $table_name);"
+  table_name="${table_name}_d"
+}
+
+prepare_table_with_sorting_key() {
+  prepare_table "$1"
+}
+
+run_query() {
+  query_id="${CLICKHOUSE_DATABASE}_hash_table_sizes_stats_$RANDOM$RANDOM"
+  $CLICKHOUSE_CLIENT --query_id="$query_id" --multiquery -q "
+    SET max_block_size = $((table_size / 10));
+    SET merge_tree_min_rows_for_concurrent_read = 1;
+    SET max_untracked_memory = 0;
+    SET prefer_localhost_replica = 1;
+    $query"
+}
+
+check_preallocated_elements() {
+  $CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS"
+  # rows may be distributed in any way including "everything goes to the one particular thread"
+  min=$1
+  if [ -z "$2" ]; then
+    max=$1
+  else
+    max=$2
+  fi
+  $CLICKHOUSE_CLIENT --param_query_id="$query_id" -q "
+    SELECT COUNT(*)
+      FROM system.query_log
+     WHERE event_date >= yesterday() AND (query_id = {query_id:String} OR initial_query_id = {query_id:String})
+           AND ProfileEvents['AggregationPreallocatedElementsInHashTables'] BETWEEN $min AND $max
+  GROUP BY query_id"
+}
+
+check_convertion_to_two_level() {
+  $CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS"
+  # rows may be distributed in any way including "everything goes to the one particular thread"
+  $CLICKHOUSE_CLIENT --param_query_id="$query_id" -q "
+    SELECT SUM(ProfileEvents['AggregationHashTablesInitializedAsTwoLevel']) BETWEEN 1 AND $max_threads
+      FROM system.query_log
+     WHERE event_date >= yesterday() AND (query_id = {query_id:String} OR initial_query_id = {query_id:String})
+  GROUP BY query_id"
+}
+
+print_border() {
+  echo "--"
+}
+
+
+# shellcheck source=./02151_hash_table_sizes_stats.testcases
+source "$CURDIR"/02151_hash_table_sizes_stats.testcases
+
+
+test_one_thread_simple_group_by
+test_one_thread_simple_group_by_with_limit
+test_one_thread_simple_group_by_with_join_and_subquery
+test_several_threads_simple_group_by_with_limit_single_level_ht
+test_several_threads_simple_group_by_with_limit_two_level_ht
+test_several_threads_simple_group_by_with_limit_and_rollup_single_level_ht
+test_several_threads_simple_group_by_with_limit_and_rollup_two_level_ht
+test_several_threads_simple_group_by_with_limit_and_cube_single_level_ht
+test_several_threads_simple_group_by_with_limit_and_cube_two_level_ht

From 10db099d2fa5e0bc2c2c2b5514edc40b1a83c05e Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Wed, 30 Mar 2022 23:27:46 +0200
Subject: [PATCH 100/104] One more try to resurrect build hash

---
 CMakeLists.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 5157f0f9903..a9ce64b87ba 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -261,8 +261,8 @@ endif ()
 # Add a section with the hash of the compiled machine code for integrity checks.
 # Only for official builds, because adding a section can be time consuming (rewrite of several GB).
 # And cross compiled binaries are not supported (since you cannot execute clickhouse hash-binary)
-if (OBJCOPY_PATH AND CLICKHOUSE_OFFICIAL_BUILD AND (NOT CMAKE_TOOLCHAIN_FILE))
-    set (USE_BINARY_HASH 1)
+if (OBJCOPY_PATH AND CLICKHOUSE_OFFICIAL_BUILD AND (NOT CMAKE_TOOLCHAIN_FILE OR CMAKE_TOOLCHAIN_FILE MATCHES "linux/toolchain-x86_64.cmake$"))
+    set (USE_BINARY_HASH 1 CACHE STRING "Calculate binary hash and store it in the separate section")
 endif ()
 
 # Allows to build stripped binary in a separate directory

From ae3bcbb99b0a3e75a2bd6229fcf7fc9a802d1880 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Thu, 31 Mar 2022 00:22:36 +0200
Subject: [PATCH 101/104] Fix logs space check

---
 programs/server/Server.cpp | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp
index fca5b39d2da..c12abda9594 100644
--- a/programs/server/Server.cpp
+++ b/programs/server/Server.cpp
@@ -583,23 +583,24 @@ static void sanityChecks(Server * server)
     if (getBlockDeviceType(dev_id) == BlockDeviceType::ROT && getBlockDeviceReadAheadBytes(dev_id) == 0)
         server->context()->addWarningMessage("Rotational disk with disabled readahead is in use. Performance can be degraded.");
 #endif
+
     try
     {
         if (getAvailableMemoryAmount() < (2l << 30))
             server->context()->addWarningMessage("Available memory at server startup is too low (2GiB).");
+
+        if (!enoughSpaceInDirectory(data_path, 1ull << 30))
+            server->context()->addWarningMessage("Available disk space at server startup is too low (1GiB).");
+
+        if (!logs_path.empty())
+        {
+            if (!enoughSpaceInDirectory(fs::path(logs_path).parent_path(), 1ull << 30))
+                server->context()->addWarningMessage("Available disk space at server startup is too low (1GiB).");
+        }
     }
     catch (...)
     {
     }
-
-    if (!enoughSpaceInDirectory(data_path, 1ull << 30))
-        server->context()->addWarningMessage("Available disk space at server startup is too low (1GiB).");
-
-    if (!logs_path.empty())
-    {
-        if (!enoughSpaceInDirectory(logs_path, 1ull << 30))
-            server->context()->addWarningMessage("Available disk space at server startup is too low (1GiB).");
-    }
 }
 
 int Server::main(const std::vector<std::string> & /*args*/)

From ff0e46c394a79884260525956fa8b9b35d49dde4 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Thu, 31 Mar 2022 09:28:50 +0300
Subject: [PATCH 102/104] clickhouse-keeper: finish session on client EOF
 (fixes heap-use-after-free)

CI founds [1]:

    WARNING: ThreadSanitizer: heap-use-after-free (pid=1)
      Read of size 8 at 0x7b48001e0088 by thread T7 (mutexes: write M2588):
        ...
        1 DB::KeeperTCPHandler::runImpl()::$_0::operator()() const build_docker/../src/Server/KeeperTCPHandler.cpp:384:14 (clickhouse+0x194f974f)
        ...
        8 DB::KeeperDispatcher::setResponse() build_docker/../src/Coordination/KeeperDispatcher.cpp:222:9 (clickhouse+0x1999de82)
        9 DB::KeeperDispatcher::responseThread() build_docker/../src/Coordination/KeeperDispatcher.cpp:161:18 (clickhouse+0x1999d7a7)
        10 DB::KeeperDispatcher::initialize()::$_1::operator()() const build_docker/../src/Coordination/KeeperDispatcher.cpp:273:54 (clickhouse+0x199a55d1)

      Previous write of size 8 at 0x7b48001e0088 by thread T3:
        0 operator delete(void*, unsigned long) <null> (clickhouse+0xa0dad50)
        1 DB::KeeperTCPHandler::~KeeperTCPHandler() build_docker/../src/Server/KeeperTCPHandler.cpp:648:1 (clickhouse+0x194f8ee6)

  [1]: https://s3.amazonaws.com/clickhouse-test-reports/35772/38873d726d22b15d4c5a284410956927b2e2f95f/integration_tests__thread__actions__[4/4].html

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Server/KeeperTCPHandler.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Server/KeeperTCPHandler.cpp b/src/Server/KeeperTCPHandler.cpp
index b0e2bc85245..d6eca9bec2f 100644
--- a/src/Server/KeeperTCPHandler.cpp
+++ b/src/Server/KeeperTCPHandler.cpp
@@ -418,6 +418,7 @@ void KeeperTCPHandler::runImpl()
                 if (in->eof())
                 {
                     LOG_DEBUG(log, "Client closed connection, session id #{}", session_id);
+                    keeper_dispatcher->finishSession(session_id);
                     break;
                 }
 

From f6bfdcc0c91ddc6e7280f0bbf2efb0f3a6a4d5a0 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <mikhaylovnikitka@gmail.com>
Date: Thu, 31 Mar 2022 12:47:36 +0200
Subject: [PATCH 103/104] Added support for schema inference for `hdfsCluster`
 (#35602)

---
 docker/test/stateless/Dockerfile              | 10 +++-
 docker/test/stateless/run.sh                  |  1 +
 .../test/stateless/setup_hdfs_minicluster.sh  | 20 +++++++
 src/Storages/HDFS/StorageHDFS.cpp             |  5 +-
 src/Storages/HDFS/StorageHDFSCluster.cpp      | 26 +++++++--
 src/Storages/HDFS/StorageHDFSCluster.h        |  1 +
 .../TableFunctionHDFSCluster.cpp              | 53 ++++++++++++-------
 src/TableFunctions/TableFunctionHDFSCluster.h | 16 ++----
 .../0_stateless/02244_hdfs_cluster.reference  | 48 +++++++++++++++++
 .../0_stateless/02244_hdfs_cluster.sql        | 26 +++++++++
 10 files changed, 168 insertions(+), 38 deletions(-)
 create mode 100755 docker/test/stateless/setup_hdfs_minicluster.sh
 create mode 100644 tests/queries/0_stateless/02244_hdfs_cluster.reference
 create mode 100644 tests/queries/0_stateless/02244_hdfs_cluster.sql

diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile
index 68c08c23b3f..c1c50fe6b4e 100644
--- a/docker/test/stateless/Dockerfile
+++ b/docker/test/stateless/Dockerfile
@@ -32,7 +32,9 @@ RUN apt-get update -y \
             mysql-client=8.0* \
             postgresql-client \
             sqlite3 \
-            awscli
+            awscli \
+            openjdk-11-jre-headless
+
 
 RUN pip3 install numpy scipy pandas Jinja2
 
@@ -58,10 +60,16 @@ RUN arch=${TARGETARCH:-amd64} \
     && wget "https://dl.min.io/client/mc/release/linux-${arch}/mc" \
     && chmod +x ./mc
 
+
+RUN wget 'https://dlcdn.apache.org/hadoop/common/hadoop-3.2.2/hadoop-3.2.2.tar.gz' \
+    && tar -xvf hadoop-3.2.2.tar.gz \
+    && rm -rf hadoop-3.2.2.tar.gz
+
 ENV MINIO_ROOT_USER="clickhouse"
 ENV MINIO_ROOT_PASSWORD="clickhouse"
 ENV EXPORT_S3_STORAGE_POLICIES=1
 
 COPY run.sh /
 COPY setup_minio.sh /
+COPY setup_hdfs_minicluster.sh /
 CMD ["/bin/bash", "/run.sh"]
diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh
index f8b73791388..e9752061dca 100755
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@@ -19,6 +19,7 @@ ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test
 /usr/share/clickhouse-test/config/install.sh
 
 ./setup_minio.sh
+./setup_hdfs_minicluster.sh
 
 # For flaky check we also enable thread fuzzer
 if [ "$NUM_TRIES" -gt "1" ]; then
diff --git a/docker/test/stateless/setup_hdfs_minicluster.sh b/docker/test/stateless/setup_hdfs_minicluster.sh
new file mode 100755
index 00000000000..9754364321f
--- /dev/null
+++ b/docker/test/stateless/setup_hdfs_minicluster.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -e -x -a -u
+
+ls -lha
+
+cd hadoop-3.2.2
+
+export JAVA_HOME=/usr
+mkdir -p target/test/data
+chown clickhouse ./target/test/data
+sudo -E -u clickhouse bin/mapred minicluster -format -nomr -nnport 12222 &
+
+while ! nc -z localhost 12222; do
+  sleep 1
+done
+
+lsof -i :12222
+
+sleep 5
diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp
index 74f6937dbae..b681a31c7c8 100644
--- a/src/Storages/HDFS/StorageHDFS.cpp
+++ b/src/Storages/HDFS/StorageHDFS.cpp
@@ -15,6 +15,7 @@
 #include <Processors/Transforms/AddingDefaultsTransform.h>
 
 #include <IO/WriteHelpers.h>
+#include <IO/CompressionMethod.h>
 
 #include <Interpreters/evaluateConstantExpression.h>
 #include <Interpreters/ExpressionAnalyzer.h>
@@ -247,7 +248,7 @@ public:
     {
         auto path_and_uri = getPathFromUriAndUriWithoutPath(uris_[0]);
         HDFSBuilderWrapper builder = createHDFSBuilder(path_and_uri.second + "/", context->getGlobalContext()->getConfigRef());
-        HDFSFSPtr fs = createHDFSFS(builder.get());
+        auto fs = createHDFSFS(builder.get());
         for (const auto & uri : uris_)
         {
             path_and_uri = getPathFromUriAndUriWithoutPath(uri);
@@ -656,7 +657,7 @@ void StorageHDFS::truncate(const ASTPtr & /* query */, const StorageMetadataPtr
     const String url = uris[0].substr(0, begin_of_path);
 
     HDFSBuilderWrapper builder = createHDFSBuilder(url + "/", local_context->getGlobalContext()->getConfigRef());
-    HDFSFSPtr fs = createHDFSFS(builder.get());
+    auto fs = createHDFSFS(builder.get());
 
     for (const auto & uri : uris)
     {
diff --git a/src/Storages/HDFS/StorageHDFSCluster.cpp b/src/Storages/HDFS/StorageHDFSCluster.cpp
index b039caa4330..c11ebe11183 100644
--- a/src/Storages/HDFS/StorageHDFSCluster.cpp
+++ b/src/Storages/HDFS/StorageHDFSCluster.cpp
@@ -2,6 +2,8 @@
 
 #if USE_HDFS
 
+#include <Storages/HDFS/StorageHDFSCluster.h>
+
 #include <Client/Connection.h>
 #include <Core/QueryProcessingStage.h>
 #include <DataTypes/DataTypeString.h>
@@ -10,16 +12,20 @@
 #include <Interpreters/SelectQueryOptions.h>
 #include <Interpreters/InterpreterSelectQuery.h>
 #include <Interpreters/getTableExpressions.h>
-#include <Processors/Transforms/AddingDefaultsTransform.h>
+
 #include <QueryPipeline/narrowBlockInputStreams.h>
 #include <QueryPipeline/Pipe.h>
-#include <Processors/Sources/RemoteSource.h>
 #include <QueryPipeline/RemoteQueryExecutor.h>
+
+#include <Processors/Transforms/AddingDefaultsTransform.h>
+
+#include <Processors/Sources/RemoteSource.h>
 #include <Parsers/queryToString.h>
 #include <Parsers/ASTTablesInSelectQuery.h>
+
 #include <Storages/IStorage.h>
 #include <Storages/SelectQueryInfo.h>
-#include <Storages/HDFS/StorageHDFSCluster.h>
+#include <Storages/HDFS/HDFSCommon.h>
 
 #include <memory>
 
@@ -28,6 +34,7 @@ namespace DB
 {
 
 StorageHDFSCluster::StorageHDFSCluster(
+    ContextPtr context_,
     String cluster_name_,
     const String & uri_,
     const StorageID & table_id_,
@@ -41,8 +48,19 @@ StorageHDFSCluster::StorageHDFSCluster(
     , format_name(format_name_)
     , compression_method(compression_method_)
 {
+    context_->getRemoteHostFilter().checkURL(Poco::URI(uri_));
+    checkHDFSURL(uri_);
+
     StorageInMemoryMetadata storage_metadata;
-    storage_metadata.setColumns(columns_);
+
+    if (columns_.empty())
+    {
+        auto columns = StorageHDFS::getTableStructureFromData(format_name, uri_, compression_method, context_);
+        storage_metadata.setColumns(columns);
+    }
+    else
+        storage_metadata.setColumns(columns_);
+
     storage_metadata.setConstraints(constraints_);
     setInMemoryMetadata(storage_metadata);
 }
diff --git a/src/Storages/HDFS/StorageHDFSCluster.h b/src/Storages/HDFS/StorageHDFSCluster.h
index 953311de056..677a00f661c 100644
--- a/src/Storages/HDFS/StorageHDFSCluster.h
+++ b/src/Storages/HDFS/StorageHDFSCluster.h
@@ -34,6 +34,7 @@ public:
 
 protected:
     StorageHDFSCluster(
+        ContextPtr context_,
         String cluster_name_,
         const String & uri_,
         const StorageID & table_id_,
diff --git a/src/TableFunctions/TableFunctionHDFSCluster.cpp b/src/TableFunctions/TableFunctionHDFSCluster.cpp
index ca1ac6a11cd..f1d3567efa7 100644
--- a/src/TableFunctions/TableFunctionHDFSCluster.cpp
+++ b/src/TableFunctions/TableFunctionHDFSCluster.cpp
@@ -31,13 +31,15 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+    extern const int BAD_GET;
 }
 
 
 void TableFunctionHDFSCluster::parseArguments(const ASTPtr & ast_function, ContextPtr context)
 {
+    auto ast_copy = ast_function->clone();
     /// Parse args
-    ASTs & args_func = ast_function->children;
+    ASTs & args_func = ast_copy->children;
 
     if (args_func.size() != 1)
         throw Exception("Table function '" + getName() + "' must have arguments.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
@@ -46,41 +48,50 @@ void TableFunctionHDFSCluster::parseArguments(const ASTPtr & ast_function, Conte
 
     const auto message = fmt::format(
         "The signature of table function {} shall be the following:\n" \
-        " - cluster, uri, format, structure",
+        " - cluster, uri\n",\
+        " - cluster, format\n",\
+        " - cluster, uri, format, structure\n",\
         " - cluster, uri, format, structure, compression_method",
         getName());
 
-    if (args.size() < 4 || args.size() > 5)
+    if (args.size() < 2 || args.size() > 5)
         throw Exception(message, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
 
     for (auto & arg : args)
         arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context);
 
-    /// This arguments are always the first
+    /// This argument is always the first
     cluster_name = args[0]->as<ASTLiteral &>().value.safeGet<String>();
-    uri = args[1]->as<ASTLiteral &>().value.safeGet<String>();
-    format = args[2]->as<ASTLiteral &>().value.safeGet<String>();
-    structure = args[3]->as<ASTLiteral &>().value.safeGet<String>();
-    if (args.size() >= 5)
-        compression_method = args[4]->as<ASTLiteral &>().value.safeGet<String>();
+
+    if (!context->tryGetCluster(cluster_name))
+        throw Exception(ErrorCodes::BAD_GET, "Requested cluster '{}' not found", cluster_name);
+
+     /// Just cut the first arg (cluster_name) and try to parse other table function arguments as is
+    args.erase(args.begin());
+
+    ITableFunctionFileLike::parseArguments(ast_copy, context);
 }
 
 
 ColumnsDescription TableFunctionHDFSCluster::getActualTableStructure(ContextPtr context) const
 {
+    if (structure == "auto")
+        return StorageHDFS::getTableStructureFromData(format, filename, compression_method, context);
+
     return parseColumnsListFromString(structure, context);
 }
 
-StoragePtr TableFunctionHDFSCluster::executeImpl(
-    const ASTPtr & /*function*/, ContextPtr context,
-    const std::string & table_name, ColumnsDescription /*cached_columns*/) const
+
+StoragePtr TableFunctionHDFSCluster::getStorage(
+    const String & /*source*/, const String & /*format_*/, const ColumnsDescription &, ContextPtr context,
+    const std::string & table_name, const String & /*compression_method_*/) const
 {
     StoragePtr storage;
     if (context->getClientInfo().query_kind == ClientInfo::QueryKind::SECONDARY_QUERY)
     {
         /// On worker node this uri won't contains globs
         storage = StorageHDFS::create(
-            uri,
+            filename,
             StorageID(getDatabaseName(), table_name),
             format,
             getActualTableStructure(context),
@@ -94,17 +105,19 @@ StoragePtr TableFunctionHDFSCluster::executeImpl(
     else
     {
         storage = StorageHDFSCluster::create(
-            cluster_name, uri, StorageID(getDatabaseName(), table_name),
-            format, getActualTableStructure(context), ConstraintsDescription{},
-            compression_method);
+            context,
+            cluster_name,
+            filename,
+            StorageID(getDatabaseName(), table_name),
+            format,
+            getActualTableStructure(context),
+            ConstraintsDescription{},
+            compression_method
+        );
     }
-
-    storage->startup();
-
     return storage;
 }
 
-
 void registerTableFunctionHDFSCluster(TableFunctionFactory & factory)
 {
     factory.registerFunction<TableFunctionHDFSCluster>();
diff --git a/src/TableFunctions/TableFunctionHDFSCluster.h b/src/TableFunctions/TableFunctionHDFSCluster.h
index 58d1c3d9b05..0560d103303 100644
--- a/src/TableFunctions/TableFunctionHDFSCluster.h
+++ b/src/TableFunctions/TableFunctionHDFSCluster.h
@@ -4,7 +4,7 @@
 
 #if USE_HDFS
 
-#include <TableFunctions/ITableFunction.h>
+#include <TableFunctions/ITableFunctionFileLike.h>
 
 
 namespace DB
@@ -20,7 +20,7 @@ class Context;
  * On worker node it asks initiator about next task to process, processes it.
  * This is repeated until the tasks are finished.
  */
-class TableFunctionHDFSCluster : public ITableFunction
+class TableFunctionHDFSCluster : public ITableFunctionFileLike
 {
 public:
     static constexpr auto name = "hdfsCluster";
@@ -31,11 +31,9 @@ public:
     bool hasStaticStructure() const override { return true; }
 
 protected:
-    StoragePtr executeImpl(
-        const ASTPtr & ast_function,
-        ContextPtr context,
-        const std::string & table_name,
-        ColumnsDescription cached_columns) const override;
+    StoragePtr getStorage(
+        const String & source, const String & format_, const ColumnsDescription & columns, ContextPtr global_context,
+        const std::string & table_name, const String & compression_method_) const override;
 
     const char * getStorageTypeName() const override { return "HDFSCluster"; }
 
@@ -43,10 +41,6 @@ protected:
     void parseArguments(const ASTPtr &, ContextPtr) override;
 
     String cluster_name;
-    String uri;
-    String format;
-    String structure;
-    String compression_method = "auto";
 };
 
 }
diff --git a/tests/queries/0_stateless/02244_hdfs_cluster.reference b/tests/queries/0_stateless/02244_hdfs_cluster.reference
new file mode 100644
index 00000000000..390627138ef
--- /dev/null
+++ b/tests/queries/0_stateless/02244_hdfs_cluster.reference
@@ -0,0 +1,48 @@
+1	2	3
+4	5	6
+7	8	9
+1	2	3
+4	5	6
+7	8	9
+1	2	3
+4	5	6
+7	8	9
+1	2	3
+4	5	6
+7	8	9
+1	2	3
+4	5	6
+7	8	9
+1	2	3
+4	5	6
+7	8	9
+1	2	3
+4	5	6
+7	8	9
+1	2	3
+4	5	6
+7	8	9
+c1	Nullable(String)					
+c2	Nullable(String)					
+c3	Nullable(String)					
+c1	Nullable(String)					
+c2	Nullable(String)					
+c3	Nullable(String)					
+c1	UInt32					
+c2	UInt32					
+c3	UInt32					
+c1	UInt32					
+c2	UInt32					
+c3	UInt32					
+c1	Nullable(String)					
+c2	Nullable(String)					
+c3	Nullable(String)					
+c1	Nullable(String)					
+c2	Nullable(String)					
+c3	Nullable(String)					
+c1	UInt32					
+c2	UInt32					
+c3	UInt32					
+c1	UInt32					
+c2	UInt32					
+c3	UInt32					
diff --git a/tests/queries/0_stateless/02244_hdfs_cluster.sql b/tests/queries/0_stateless/02244_hdfs_cluster.sql
new file mode 100644
index 00000000000..ffd4a35a506
--- /dev/null
+++ b/tests/queries/0_stateless/02244_hdfs_cluster.sql
@@ -0,0 +1,26 @@
+-- Tags: no-fasttest, no-parallel, no-cpu-aarch64
+-- Tag no-fasttest: Depends on Java
+
+insert into table function hdfs('hdfs://localhost:12222/test_1.tsv', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32') select 1, 2, 3 settings hdfs_truncate_on_insert=1;
+insert into table function hdfs('hdfs://localhost:12222/test_2.tsv', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32') select 4, 5, 6 settings hdfs_truncate_on_insert=1;
+insert into table function hdfs('hdfs://localhost:12222/test_3.tsv', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32') select 7, 8, 9 settings hdfs_truncate_on_insert=1;
+
+select * from hdfs('hdfs://localhost:12222/test_{1,2,3}.tsv') order by c1, c2, c3;
+select * from hdfs('hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV') order by c1, c2, c3;
+select * from hdfs('hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV', 'c1 UInt32, c2 UInt32, c3 UInt32') order by c1, c2, c3;
+select * from hdfs('hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV', 'c1 UInt32, c2 UInt32, c3 UInt32', 'auto') order by c1, c2, c3;
+
+select * from hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_{1,2,3}.tsv') order by c1, c2, c3;
+select * from hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV') order by c1, c2, c3;
+select * from hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV', 'c1 UInt32, c2 UInt32, c3 UInt32') order by c1, c2, c3;
+select * from hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV', 'c1 UInt32, c2 UInt32, c3 UInt32', 'auto') order by c1, c2, c3;
+
+desc hdfs('hdfs://localhost:12222/test_{1,2,3}.tsv');
+desc hdfs('hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV');
+desc hdfs('hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV', 'c1 UInt32, c2 UInt32, c3 UInt32');
+desc hdfs('hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV', 'c1 UInt32, c2 UInt32, c3 UInt32', 'auto');
+
+desc hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_{1,2,3}.tsv');
+desc hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV');
+desc hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV', 'c1 UInt32, c2 UInt32, c3 UInt32');
+desc hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV', 'c1 UInt32, c2 UInt32, c3 UInt32', 'auto');

From 5f27f690f471c1c54a472d9b37402d08f858d1b0 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <mikhaylovnikitka@gmail.com>
Date: Thu, 31 Mar 2022 14:19:56 +0200
Subject: [PATCH 104/104] Revert "Added support for schema inference for
 `hdfsCluster` (#35602)"

This reverts commit f6bfdcc0c91ddc6e7280f0bbf2efb0f3a6a4d5a0.
---
 docker/test/stateless/Dockerfile              | 10 +---
 docker/test/stateless/run.sh                  |  1 -
 .../test/stateless/setup_hdfs_minicluster.sh  | 20 -------
 src/Storages/HDFS/StorageHDFS.cpp             |  5 +-
 src/Storages/HDFS/StorageHDFSCluster.cpp      | 26 ++-------
 src/Storages/HDFS/StorageHDFSCluster.h        |  1 -
 .../TableFunctionHDFSCluster.cpp              | 53 +++++++------------
 src/TableFunctions/TableFunctionHDFSCluster.h | 16 ++++--
 .../0_stateless/02244_hdfs_cluster.reference  | 48 -----------------
 .../0_stateless/02244_hdfs_cluster.sql        | 26 ---------
 10 files changed, 38 insertions(+), 168 deletions(-)
 delete mode 100755 docker/test/stateless/setup_hdfs_minicluster.sh
 delete mode 100644 tests/queries/0_stateless/02244_hdfs_cluster.reference
 delete mode 100644 tests/queries/0_stateless/02244_hdfs_cluster.sql

diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile
index c1c50fe6b4e..68c08c23b3f 100644
--- a/docker/test/stateless/Dockerfile
+++ b/docker/test/stateless/Dockerfile
@@ -32,9 +32,7 @@ RUN apt-get update -y \
             mysql-client=8.0* \
             postgresql-client \
             sqlite3 \
-            awscli \
-            openjdk-11-jre-headless
-
+            awscli
 
 RUN pip3 install numpy scipy pandas Jinja2
 
@@ -60,16 +58,10 @@ RUN arch=${TARGETARCH:-amd64} \
     && wget "https://dl.min.io/client/mc/release/linux-${arch}/mc" \
     && chmod +x ./mc
 
-
-RUN wget 'https://dlcdn.apache.org/hadoop/common/hadoop-3.2.2/hadoop-3.2.2.tar.gz' \
-    && tar -xvf hadoop-3.2.2.tar.gz \
-    && rm -rf hadoop-3.2.2.tar.gz
-
 ENV MINIO_ROOT_USER="clickhouse"
 ENV MINIO_ROOT_PASSWORD="clickhouse"
 ENV EXPORT_S3_STORAGE_POLICIES=1
 
 COPY run.sh /
 COPY setup_minio.sh /
-COPY setup_hdfs_minicluster.sh /
 CMD ["/bin/bash", "/run.sh"]
diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh
index e9752061dca..f8b73791388 100755
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@@ -19,7 +19,6 @@ ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test
 /usr/share/clickhouse-test/config/install.sh
 
 ./setup_minio.sh
-./setup_hdfs_minicluster.sh
 
 # For flaky check we also enable thread fuzzer
 if [ "$NUM_TRIES" -gt "1" ]; then
diff --git a/docker/test/stateless/setup_hdfs_minicluster.sh b/docker/test/stateless/setup_hdfs_minicluster.sh
deleted file mode 100755
index 9754364321f..00000000000
--- a/docker/test/stateless/setup_hdfs_minicluster.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/bash
-
-set -e -x -a -u
-
-ls -lha
-
-cd hadoop-3.2.2
-
-export JAVA_HOME=/usr
-mkdir -p target/test/data
-chown clickhouse ./target/test/data
-sudo -E -u clickhouse bin/mapred minicluster -format -nomr -nnport 12222 &
-
-while ! nc -z localhost 12222; do
-  sleep 1
-done
-
-lsof -i :12222
-
-sleep 5
diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp
index b681a31c7c8..74f6937dbae 100644
--- a/src/Storages/HDFS/StorageHDFS.cpp
+++ b/src/Storages/HDFS/StorageHDFS.cpp
@@ -15,7 +15,6 @@
 #include <Processors/Transforms/AddingDefaultsTransform.h>
 
 #include <IO/WriteHelpers.h>
-#include <IO/CompressionMethod.h>
 
 #include <Interpreters/evaluateConstantExpression.h>
 #include <Interpreters/ExpressionAnalyzer.h>
@@ -248,7 +247,7 @@ public:
     {
         auto path_and_uri = getPathFromUriAndUriWithoutPath(uris_[0]);
         HDFSBuilderWrapper builder = createHDFSBuilder(path_and_uri.second + "/", context->getGlobalContext()->getConfigRef());
-        auto fs = createHDFSFS(builder.get());
+        HDFSFSPtr fs = createHDFSFS(builder.get());
         for (const auto & uri : uris_)
         {
             path_and_uri = getPathFromUriAndUriWithoutPath(uri);
@@ -657,7 +656,7 @@ void StorageHDFS::truncate(const ASTPtr & /* query */, const StorageMetadataPtr
     const String url = uris[0].substr(0, begin_of_path);
 
     HDFSBuilderWrapper builder = createHDFSBuilder(url + "/", local_context->getGlobalContext()->getConfigRef());
-    auto fs = createHDFSFS(builder.get());
+    HDFSFSPtr fs = createHDFSFS(builder.get());
 
     for (const auto & uri : uris)
     {
diff --git a/src/Storages/HDFS/StorageHDFSCluster.cpp b/src/Storages/HDFS/StorageHDFSCluster.cpp
index c11ebe11183..b039caa4330 100644
--- a/src/Storages/HDFS/StorageHDFSCluster.cpp
+++ b/src/Storages/HDFS/StorageHDFSCluster.cpp
@@ -2,8 +2,6 @@
 
 #if USE_HDFS
 
-#include <Storages/HDFS/StorageHDFSCluster.h>
-
 #include <Client/Connection.h>
 #include <Core/QueryProcessingStage.h>
 #include <DataTypes/DataTypeString.h>
@@ -12,20 +10,16 @@
 #include <Interpreters/SelectQueryOptions.h>
 #include <Interpreters/InterpreterSelectQuery.h>
 #include <Interpreters/getTableExpressions.h>
-
+#include <Processors/Transforms/AddingDefaultsTransform.h>
 #include <QueryPipeline/narrowBlockInputStreams.h>
 #include <QueryPipeline/Pipe.h>
-#include <QueryPipeline/RemoteQueryExecutor.h>
-
-#include <Processors/Transforms/AddingDefaultsTransform.h>
-
 #include <Processors/Sources/RemoteSource.h>
+#include <QueryPipeline/RemoteQueryExecutor.h>
 #include <Parsers/queryToString.h>
 #include <Parsers/ASTTablesInSelectQuery.h>
-
 #include <Storages/IStorage.h>
 #include <Storages/SelectQueryInfo.h>
-#include <Storages/HDFS/HDFSCommon.h>
+#include <Storages/HDFS/StorageHDFSCluster.h>
 
 #include <memory>
 
@@ -34,7 +28,6 @@ namespace DB
 {
 
 StorageHDFSCluster::StorageHDFSCluster(
-    ContextPtr context_,
     String cluster_name_,
     const String & uri_,
     const StorageID & table_id_,
@@ -48,19 +41,8 @@ StorageHDFSCluster::StorageHDFSCluster(
     , format_name(format_name_)
     , compression_method(compression_method_)
 {
-    context_->getRemoteHostFilter().checkURL(Poco::URI(uri_));
-    checkHDFSURL(uri_);
-
     StorageInMemoryMetadata storage_metadata;
-
-    if (columns_.empty())
-    {
-        auto columns = StorageHDFS::getTableStructureFromData(format_name, uri_, compression_method, context_);
-        storage_metadata.setColumns(columns);
-    }
-    else
-        storage_metadata.setColumns(columns_);
-
+    storage_metadata.setColumns(columns_);
     storage_metadata.setConstraints(constraints_);
     setInMemoryMetadata(storage_metadata);
 }
diff --git a/src/Storages/HDFS/StorageHDFSCluster.h b/src/Storages/HDFS/StorageHDFSCluster.h
index 677a00f661c..953311de056 100644
--- a/src/Storages/HDFS/StorageHDFSCluster.h
+++ b/src/Storages/HDFS/StorageHDFSCluster.h
@@ -34,7 +34,6 @@ public:
 
 protected:
     StorageHDFSCluster(
-        ContextPtr context_,
         String cluster_name_,
         const String & uri_,
         const StorageID & table_id_,
diff --git a/src/TableFunctions/TableFunctionHDFSCluster.cpp b/src/TableFunctions/TableFunctionHDFSCluster.cpp
index f1d3567efa7..ca1ac6a11cd 100644
--- a/src/TableFunctions/TableFunctionHDFSCluster.cpp
+++ b/src/TableFunctions/TableFunctionHDFSCluster.cpp
@@ -31,15 +31,13 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
-    extern const int BAD_GET;
 }
 
 
 void TableFunctionHDFSCluster::parseArguments(const ASTPtr & ast_function, ContextPtr context)
 {
-    auto ast_copy = ast_function->clone();
     /// Parse args
-    ASTs & args_func = ast_copy->children;
+    ASTs & args_func = ast_function->children;
 
     if (args_func.size() != 1)
         throw Exception("Table function '" + getName() + "' must have arguments.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
@@ -48,50 +46,41 @@ void TableFunctionHDFSCluster::parseArguments(const ASTPtr & ast_function, Conte
 
     const auto message = fmt::format(
         "The signature of table function {} shall be the following:\n" \
-        " - cluster, uri\n",\
-        " - cluster, format\n",\
-        " - cluster, uri, format, structure\n",\
+        " - cluster, uri, format, structure",
         " - cluster, uri, format, structure, compression_method",
         getName());
 
-    if (args.size() < 2 || args.size() > 5)
+    if (args.size() < 4 || args.size() > 5)
         throw Exception(message, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
 
     for (auto & arg : args)
         arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context);
 
-    /// This argument is always the first
+    /// This arguments are always the first
     cluster_name = args[0]->as<ASTLiteral &>().value.safeGet<String>();
-
-    if (!context->tryGetCluster(cluster_name))
-        throw Exception(ErrorCodes::BAD_GET, "Requested cluster '{}' not found", cluster_name);
-
-     /// Just cut the first arg (cluster_name) and try to parse other table function arguments as is
-    args.erase(args.begin());
-
-    ITableFunctionFileLike::parseArguments(ast_copy, context);
+    uri = args[1]->as<ASTLiteral &>().value.safeGet<String>();
+    format = args[2]->as<ASTLiteral &>().value.safeGet<String>();
+    structure = args[3]->as<ASTLiteral &>().value.safeGet<String>();
+    if (args.size() >= 5)
+        compression_method = args[4]->as<ASTLiteral &>().value.safeGet<String>();
 }
 
 
 ColumnsDescription TableFunctionHDFSCluster::getActualTableStructure(ContextPtr context) const
 {
-    if (structure == "auto")
-        return StorageHDFS::getTableStructureFromData(format, filename, compression_method, context);
-
     return parseColumnsListFromString(structure, context);
 }
 
-
-StoragePtr TableFunctionHDFSCluster::getStorage(
-    const String & /*source*/, const String & /*format_*/, const ColumnsDescription &, ContextPtr context,
-    const std::string & table_name, const String & /*compression_method_*/) const
+StoragePtr TableFunctionHDFSCluster::executeImpl(
+    const ASTPtr & /*function*/, ContextPtr context,
+    const std::string & table_name, ColumnsDescription /*cached_columns*/) const
 {
     StoragePtr storage;
     if (context->getClientInfo().query_kind == ClientInfo::QueryKind::SECONDARY_QUERY)
     {
         /// On worker node this uri won't contains globs
         storage = StorageHDFS::create(
-            filename,
+            uri,
             StorageID(getDatabaseName(), table_name),
             format,
             getActualTableStructure(context),
@@ -105,19 +94,17 @@ StoragePtr TableFunctionHDFSCluster::getStorage(
     else
     {
         storage = StorageHDFSCluster::create(
-            context,
-            cluster_name,
-            filename,
-            StorageID(getDatabaseName(), table_name),
-            format,
-            getActualTableStructure(context),
-            ConstraintsDescription{},
-            compression_method
-        );
+            cluster_name, uri, StorageID(getDatabaseName(), table_name),
+            format, getActualTableStructure(context), ConstraintsDescription{},
+            compression_method);
     }
+
+    storage->startup();
+
     return storage;
 }
 
+
 void registerTableFunctionHDFSCluster(TableFunctionFactory & factory)
 {
     factory.registerFunction<TableFunctionHDFSCluster>();
diff --git a/src/TableFunctions/TableFunctionHDFSCluster.h b/src/TableFunctions/TableFunctionHDFSCluster.h
index 0560d103303..58d1c3d9b05 100644
--- a/src/TableFunctions/TableFunctionHDFSCluster.h
+++ b/src/TableFunctions/TableFunctionHDFSCluster.h
@@ -4,7 +4,7 @@
 
 #if USE_HDFS
 
-#include <TableFunctions/ITableFunctionFileLike.h>
+#include <TableFunctions/ITableFunction.h>
 
 
 namespace DB
@@ -20,7 +20,7 @@ class Context;
  * On worker node it asks initiator about next task to process, processes it.
  * This is repeated until the tasks are finished.
  */
-class TableFunctionHDFSCluster : public ITableFunctionFileLike
+class TableFunctionHDFSCluster : public ITableFunction
 {
 public:
     static constexpr auto name = "hdfsCluster";
@@ -31,9 +31,11 @@ public:
     bool hasStaticStructure() const override { return true; }
 
 protected:
-    StoragePtr getStorage(
-        const String & source, const String & format_, const ColumnsDescription & columns, ContextPtr global_context,
-        const std::string & table_name, const String & compression_method_) const override;
+    StoragePtr executeImpl(
+        const ASTPtr & ast_function,
+        ContextPtr context,
+        const std::string & table_name,
+        ColumnsDescription cached_columns) const override;
 
     const char * getStorageTypeName() const override { return "HDFSCluster"; }
 
@@ -41,6 +43,10 @@ protected:
     void parseArguments(const ASTPtr &, ContextPtr) override;
 
     String cluster_name;
+    String uri;
+    String format;
+    String structure;
+    String compression_method = "auto";
 };
 
 }
diff --git a/tests/queries/0_stateless/02244_hdfs_cluster.reference b/tests/queries/0_stateless/02244_hdfs_cluster.reference
deleted file mode 100644
index 390627138ef..00000000000
--- a/tests/queries/0_stateless/02244_hdfs_cluster.reference
+++ /dev/null
@@ -1,48 +0,0 @@
-1	2	3
-4	5	6
-7	8	9
-1	2	3
-4	5	6
-7	8	9
-1	2	3
-4	5	6
-7	8	9
-1	2	3
-4	5	6
-7	8	9
-1	2	3
-4	5	6
-7	8	9
-1	2	3
-4	5	6
-7	8	9
-1	2	3
-4	5	6
-7	8	9
-1	2	3
-4	5	6
-7	8	9
-c1	Nullable(String)					
-c2	Nullable(String)					
-c3	Nullable(String)					
-c1	Nullable(String)					
-c2	Nullable(String)					
-c3	Nullable(String)					
-c1	UInt32					
-c2	UInt32					
-c3	UInt32					
-c1	UInt32					
-c2	UInt32					
-c3	UInt32					
-c1	Nullable(String)					
-c2	Nullable(String)					
-c3	Nullable(String)					
-c1	Nullable(String)					
-c2	Nullable(String)					
-c3	Nullable(String)					
-c1	UInt32					
-c2	UInt32					
-c3	UInt32					
-c1	UInt32					
-c2	UInt32					
-c3	UInt32					
diff --git a/tests/queries/0_stateless/02244_hdfs_cluster.sql b/tests/queries/0_stateless/02244_hdfs_cluster.sql
deleted file mode 100644
index ffd4a35a506..00000000000
--- a/tests/queries/0_stateless/02244_hdfs_cluster.sql
+++ /dev/null
@@ -1,26 +0,0 @@
--- Tags: no-fasttest, no-parallel, no-cpu-aarch64
--- Tag no-fasttest: Depends on Java
-
-insert into table function hdfs('hdfs://localhost:12222/test_1.tsv', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32') select 1, 2, 3 settings hdfs_truncate_on_insert=1;
-insert into table function hdfs('hdfs://localhost:12222/test_2.tsv', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32') select 4, 5, 6 settings hdfs_truncate_on_insert=1;
-insert into table function hdfs('hdfs://localhost:12222/test_3.tsv', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32') select 7, 8, 9 settings hdfs_truncate_on_insert=1;
-
-select * from hdfs('hdfs://localhost:12222/test_{1,2,3}.tsv') order by c1, c2, c3;
-select * from hdfs('hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV') order by c1, c2, c3;
-select * from hdfs('hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV', 'c1 UInt32, c2 UInt32, c3 UInt32') order by c1, c2, c3;
-select * from hdfs('hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV', 'c1 UInt32, c2 UInt32, c3 UInt32', 'auto') order by c1, c2, c3;
-
-select * from hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_{1,2,3}.tsv') order by c1, c2, c3;
-select * from hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV') order by c1, c2, c3;
-select * from hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV', 'c1 UInt32, c2 UInt32, c3 UInt32') order by c1, c2, c3;
-select * from hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV', 'c1 UInt32, c2 UInt32, c3 UInt32', 'auto') order by c1, c2, c3;
-
-desc hdfs('hdfs://localhost:12222/test_{1,2,3}.tsv');
-desc hdfs('hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV');
-desc hdfs('hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV', 'c1 UInt32, c2 UInt32, c3 UInt32');
-desc hdfs('hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV', 'c1 UInt32, c2 UInt32, c3 UInt32', 'auto');
-
-desc hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_{1,2,3}.tsv');
-desc hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV');
-desc hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV', 'c1 UInt32, c2 UInt32, c3 UInt32');
-desc hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV', 'c1 UInt32, c2 UInt32, c3 UInt32', 'auto');