From 578fd9e60b4ad851dc358a71c951ca3e7a964266 Mon Sep 17 00:00:00 2001
From: Bharat Nallan Chakravarthy <bharatnc@gmail.com>
Date: Mon, 6 Jun 2022 15:00:37 +0530
Subject: [PATCH 001/158] implement quantileApproximateWeighted

---
 .../AggregateFunctionQuantile.cpp             |  15 +-
 .../AggregateFunctionQuantile.h               |   4 +
 .../QuantileApproximateWeighted.h             | 256 ++++++++++++++++++
 .../GatherFunctionQuantileVisitor.cpp         |   2 +
 4 files changed, 275 insertions(+), 2 deletions(-)
 create mode 100644 src/AggregateFunctions/QuantileApproximateWeighted.h

diff --git a/src/AggregateFunctions/AggregateFunctionQuantile.cpp b/src/AggregateFunctions/AggregateFunctionQuantile.cpp
index 6783a55418a..ebd6f44df17 100644
--- a/src/AggregateFunctions/AggregateFunctionQuantile.cpp
+++ b/src/AggregateFunctions/AggregateFunctionQuantile.cpp
@@ -41,6 +41,9 @@ template <typename Value, bool _> using FuncQuantilesExactInclusive = AggregateF
 template <typename Value, bool _> using FuncQuantileExactWeighted = AggregateFunctionQuantile<Value, QuantileExactWeighted<Value>, NameQuantileExactWeighted, true, void, false>;
 template <typename Value, bool _> using FuncQuantilesExactWeighted = AggregateFunctionQuantile<Value, QuantileExactWeighted<Value>, NameQuantilesExactWeighted, true, void, true>;
 
+template <typename Value, bool _> using FuncQuantileApproximateWeighted = AggregateFunctionQuantile<Value, QuantileApproximateWeighted<Value>, NameQuantileApproximateWeighted, true, void, false>;
+template <typename Value, bool _> using FuncQuantilesApproximateWeighted = AggregateFunctionQuantile<Value, QuantileApproximateWeighted<Value>, NameQuantilesApproximateWeighted, true, void, true>;
+
 template <typename Value, bool _> using FuncQuantileTiming = AggregateFunctionQuantile<Value, QuantileTiming<Value>, NameQuantileTiming, false, Float32, false>;
 template <typename Value, bool _> using FuncQuantilesTiming = AggregateFunctionQuantile<Value, QuantileTiming<Value>, NameQuantilesTiming, false, Float32, true>;
 
@@ -71,7 +74,9 @@ constexpr bool supportDecimal()
         std::is_same_v<Function<Float32, false>, FuncQuantilesExactLow<Float32, false>> ||
         std::is_same_v<Function<Float32, false>, FuncQuantilesExactHigh<Float32, false>> ||
         std::is_same_v<Function<Float32, false>, FuncQuantileExactWeighted<Float32, false>> ||
-        std::is_same_v<Function<Float32, false>, FuncQuantilesExactWeighted<Float32, false>>;
+        std::is_same_v<Function<Float32, false>, FuncQuantilesExactWeighted<Float32, false>> ||
+        std::is_same_v<Function<Float32, false>, FuncQuantileApproximateWeighted<Float32, false>> ||
+        std::is_same_v<Function<Float32, false>, FuncQuantilesApproximateWeighted<Float32, false>>;
 }
 
 template <template <typename, bool> class Function>
@@ -82,7 +87,9 @@ constexpr bool supportBigInt()
         std::is_same_v<Function<Float32, false>, FuncQuantileExact<Float32, false>> ||
         std::is_same_v<Function<Float32, false>, FuncQuantilesExact<Float32, false>> ||
         std::is_same_v<Function<Float32, false>, FuncQuantileExactWeighted<Float32, false>> ||
-        std::is_same_v<Function<Float32, false>, FuncQuantilesExactWeighted<Float32, false>>;
+        std::is_same_v<Function<Float32, false>, FuncQuantilesExactWeighted<Float32, false>> ||
+        std::is_same_v<Function<Float32, false>, FuncQuantilesApproximateWeighted<Float32, false>> ||
+        std::is_same_v<Function<Float32, false>, FuncQuantilesApproximateWeighted<Float32, false>>;
 }
 
 template <template <typename, bool> class Function>
@@ -154,6 +161,9 @@ void registerAggregateFunctionsQuantile(AggregateFunctionFactory & factory)
     factory.registerFunction(NameQuantileExactWeighted::name, createAggregateFunctionQuantile<FuncQuantileExactWeighted>);
     factory.registerFunction(NameQuantilesExactWeighted::name, { createAggregateFunctionQuantile<FuncQuantilesExactWeighted>, properties });
 
+    factory.registerFunction(NameQuantileApproximateWeighted::name, createAggregateFunctionQuantile<FuncQuantileApproximateWeighted>);
+    factory.registerFunction(NameQuantilesApproximateWeighted::name, { createAggregateFunctionQuantile<FuncQuantilesApproximateWeighted>, properties });
+
     factory.registerFunction(NameQuantileTiming::name, createAggregateFunctionQuantile<FuncQuantileTiming>);
     factory.registerFunction(NameQuantilesTiming::name, { createAggregateFunctionQuantile<FuncQuantilesTiming>, properties });
 
@@ -179,6 +189,7 @@ void registerAggregateFunctionsQuantile(AggregateFunctionFactory & factory)
     factory.registerAlias("medianExactLow", NameQuantileExactLow::name);
     factory.registerAlias("medianExactHigh", NameQuantileExactHigh::name);
     factory.registerAlias("medianExactWeighted", NameQuantileExactWeighted::name);
+    factory.registerAlias("medianApproximateWeighted", NameQuantileApproximateWeighted::name);
     factory.registerAlias("medianTiming", NameQuantileTiming::name);
     factory.registerAlias("medianTimingWeighted", NameQuantileTimingWeighted::name);
     factory.registerAlias("medianTDigest", NameQuantileTDigest::name);
diff --git a/src/AggregateFunctions/AggregateFunctionQuantile.h b/src/AggregateFunctions/AggregateFunctionQuantile.h
index bb2a68da4e6..9c172595a3d 100644
--- a/src/AggregateFunctions/AggregateFunctionQuantile.h
+++ b/src/AggregateFunctions/AggregateFunctionQuantile.h
@@ -7,6 +7,7 @@
 #include <AggregateFunctions/QuantileReservoirSamplerDeterministic.h>
 #include <AggregateFunctions/QuantileExact.h>
 #include <AggregateFunctions/QuantileExactWeighted.h>
+#include <AggregateFunctions/QuantileApproximateWeighted.h>
 #include <AggregateFunctions/QuantileTiming.h>
 #include <AggregateFunctions/QuantileTDigest.h>
 #include <AggregateFunctions/QuantileBFloat16Histogram.h>
@@ -225,6 +226,9 @@ struct NameQuantilesExactInclusive { static constexpr auto name = "quantilesExac
 struct NameQuantileExactWeighted { static constexpr auto name = "quantileExactWeighted"; };
 struct NameQuantilesExactWeighted { static constexpr auto name = "quantilesExactWeighted"; };
 
+struct NameQuantileApproximateWeighted { static constexpr auto name = "quantileApproximateWeighted"; };
+struct NameQuantilesApproximateWeighted { static constexpr auto name = "quantilesApproximateWeighted"; };
+
 struct NameQuantileTiming { static constexpr auto name = "quantileTiming"; };
 struct NameQuantileTimingWeighted { static constexpr auto name = "quantileTimingWeighted"; };
 struct NameQuantilesTiming { static constexpr auto name = "quantilesTiming"; };
diff --git a/src/AggregateFunctions/QuantileApproximateWeighted.h b/src/AggregateFunctions/QuantileApproximateWeighted.h
new file mode 100644
index 00000000000..3cb1c6a1e29
--- /dev/null
+++ b/src/AggregateFunctions/QuantileApproximateWeighted.h
@@ -0,0 +1,256 @@
+#pragma once
+
+#include <base/sort.h>
+
+#include <Common/HashTable/HashMap.h>
+#include <Common/NaNUtils.h>
+
+
+namespace DB
+{
+struct Settings;
+
+namespace ErrorCodes
+{
+    extern const int NOT_IMPLEMENTED;
+}
+
+/** Calculates quantile by counting number of occurrences for each value in a hash map.
+  *
+  * It uses O(distinct(N)) memory. Can be naturally applied for values with weight.
+  * In case of many identical values, it can be more efficient than QuantileExact even when weight is not used.
+  */
+template <typename Value>
+struct QuantileApproximateWeighted
+{
+    struct Int128Hash
+    {
+        size_t operator()(Int128 x) const
+        {
+            return CityHash_v1_0_2::Hash128to64({x >> 64, x & 0xffffffffffffffffll});
+        }
+    };
+
+    using Weight = UInt64;
+    using UnderlyingType = NativeType<Value>;
+    using Hasher = std::conditional_t<std::is_same_v<Value, Decimal128>, Int128Hash, HashCRC32<UnderlyingType>>;
+
+    /// When creating, the hash table must be small.
+    using Map = HashMapWithStackMemory<UnderlyingType, Weight, Hasher, 4>;
+
+    Map map;
+
+    void add(const Value & x)
+    {
+        /// We must skip NaNs as they are not compatible with comparison sorting.
+        if (!isNaN(x))
+            ++map[x];
+    }
+
+    void add(const Value & x, Weight weight)
+    {
+        if (!isNaN(x))
+            map[x] += weight;
+    }
+
+    void merge(const QuantileApproximateWeighted & rhs)
+    {
+        for (const auto & pair : rhs.map)
+            map[pair.getKey()] += pair.getMapped();
+    }
+
+    void serialize(WriteBuffer & buf) const
+    {
+        map.write(buf);
+    }
+
+    void deserialize(ReadBuffer & buf)
+    {
+        typename Map::Reader reader(buf);
+        while (reader.next())
+        {
+            const auto & pair = reader.get();
+            map[pair.first] = pair.second;
+        }
+    }
+
+    /// Get the value of the `level` quantile. The level must be between 0 and 1.
+    Value get(Float64 level) const
+    {
+        size_t size = map.size();
+
+        if (0 == size)
+            return std::numeric_limits<Value>::quiet_NaN();
+
+        /// Copy the data to a temporary array to get the element you need in order.
+        using Pair = typename std::pair<UnderlyingType, Float64>;
+        std::vector<Pair> value_weight_pairs;
+
+        /// Note: weight provided must be a 64-bit integer
+        /// Float64 is used as accumulator here to get approximate results.
+        /// But weight used in the internal array is stored as Float64 as we
+        /// do some quantile estimation operation which involves division and
+        /// require Float64 level of precision.
+
+        Float64 sum_weight = 0;
+        for (const auto & pair : map)
+        {
+            sum_weight += pair.getMapped();
+            auto value = pair.getKey();
+            auto weight = pair.getMapped();
+            value_weight_pairs.push_back(std::make_pair(value, weight));
+        }
+
+        ::sort(value_weight_pairs.begin(), value_weight_pairs.end(), [](const Pair & a, const Pair & b) { return a.first < b.first; });
+
+        Float64 accumulated = 0;
+
+        std::vector<Float64> cum_sum_array;
+
+        bool first = true;
+        for(size_t idx = 0 ; idx < size ; ++idx)
+        {
+            accumulated += value_weight_pairs[idx].second;
+
+            if (first)
+            {
+                cum_sum_array.push_back(value_weight_pairs[idx].second);
+                first = false;
+            }
+            else
+            {
+                cum_sum_array.push_back(accumulated);
+            }
+        }
+
+        /// weighted_quantile = cum_sum_arr - (0.5 * sample_weights)
+        for (size_t idx = 0; idx < size; ++idx)
+            value_weight_pairs[idx].second = (cum_sum_array[idx] - 0.5 * value_weight_pairs[idx].second) / sum_weight;
+
+        /// linear interpolation
+        UnderlyingType g;
+
+        size_t idx = 0;
+        if (level >= value_weight_pairs[size - 2].second)
+        {
+            idx = size - 2;
+        }
+        else
+        {
+            while (level > value_weight_pairs[idx + 1].second)
+                idx++;
+        }
+
+        Float64 xl = value_weight_pairs[idx].second, xr = value_weight_pairs[idx + 1].second;
+        UnderlyingType yl = value_weight_pairs[idx].first, yr = value_weight_pairs[idx + 1].first;
+
+        if (level < xl)
+            yr = yl;
+        if (level > xr)
+            yl = yr;
+
+        auto dydx = (yr - yl) / (xr - xl);
+        g = yl + dydx * (level - xl);
+
+        return g;
+    }
+
+
+    /// Get the `size` values of `levels` quantiles. Write `size` results starting with `result` address.
+    /// indices - an array of index levels such that the corresponding elements will go in ascending order.
+    void getMany(const Float64 * levels, const size_t * indices, size_t num_levels, Value * result) const
+    {
+        size_t size = map.size();
+
+        if (0 == size)
+        {
+            for (size_t i = 0; i < num_levels; ++i)
+                result[i] = Value();
+            return;
+        }
+
+        /// Copy the data to a temporary array to get the element you need in order.
+        using Pair = typename std::pair<UnderlyingType, Float64>;
+        std::vector<Pair> value_weight_pairs;
+
+        Float64 sum_weight = 0;
+        for (const auto & pair : map)
+        {
+            sum_weight += pair.getMapped();
+            auto value = pair.getKey();
+            auto weight = pair.getMapped();
+            value_weight_pairs.push_back(std::make_pair(value, weight));
+        }
+
+        ::sort(value_weight_pairs.begin(), value_weight_pairs.end(), [](const Pair & a, const Pair & b) { return a.first < b.first; });
+
+        Float64 accumulated = 0;
+        std::vector<Float64> cum_sum_array;
+
+        bool first = true;
+        for (size_t idx = 0 ; idx < size ; ++idx)
+        {
+            accumulated += value_weight_pairs[idx].second;
+
+            if (first)
+            {
+                cum_sum_array.emplace_back(value_weight_pairs[idx].second);
+                first = false;
+            }
+            else
+            {
+                cum_sum_array.emplace_back(accumulated);
+            }
+        }
+
+        /// weighted_quantile = cum_sum_arr - (0.5 * sample_weights)
+        for (size_t idx = 0; idx < size; ++idx)
+            value_weight_pairs[idx].second = (cum_sum_array[idx] - 0.5 * value_weight_pairs[idx].second) / sum_weight;
+
+        size_t level_index = 0;
+
+        while (level_index < num_levels)
+        {
+            /// linear interpolation for every level
+            UnderlyingType g;
+            auto level = levels[indices[level_index]];
+            size_t idx = 0;
+            if (level >= value_weight_pairs[size - 2].second)
+            {
+                idx = size - 2;
+            }
+            else
+            {
+                while (level > value_weight_pairs[idx + 1].second)
+                    idx++;
+            }
+
+            Float64 xl = value_weight_pairs[idx].second, xr = value_weight_pairs[idx + 1].second;
+            UnderlyingType yl = value_weight_pairs[idx].first, yr = value_weight_pairs[idx + 1].first;
+
+            if (level < xl)
+                yr = yl;
+            if (level > xr)
+                yl = yr;
+
+            auto dydx = (yr - yl) / (xr - xl);
+            g = yl + dydx * (level - xl);
+
+            result[indices[level_index]] = g;
+            ++level_index;
+        }
+    }
+
+    /// The same, but in the case of an empty state, NaN is returned.
+    Float64 getFloat(Float64) const
+    {
+        throw Exception("Method getFloat is not implemented for QuantileApproximateWeighted", ErrorCodes::NOT_IMPLEMENTED);
+    }
+
+    void getManyFloat(const Float64 *, const size_t *, size_t, Float64 *) const
+    {
+        throw Exception("Method getManyFloat is not implemented for QuantileExact", ErrorCodes::NOT_IMPLEMENTED);
+    }
+};
+
+}
diff --git a/src/Interpreters/GatherFunctionQuantileVisitor.cpp b/src/Interpreters/GatherFunctionQuantileVisitor.cpp
index 93b35a3416a..13abc954d78 100644
--- a/src/Interpreters/GatherFunctionQuantileVisitor.cpp
+++ b/src/Interpreters/GatherFunctionQuantileVisitor.cpp
@@ -24,6 +24,7 @@ static const std::unordered_map<String, String> quantile_fuse_name_mapping = {
     {NameQuantileExactInclusive::name, NameQuantilesExactInclusive::name},
     {NameQuantileExactLow::name, NameQuantilesExactLow::name},
     {NameQuantileExactWeighted::name, NameQuantilesExactWeighted::name},
+    {NameQuantileApproximateWeighted::name, NameQuantilesApproximateWeighted::name},
     {NameQuantileTDigest::name, NameQuantilesTDigest::name},
     {NameQuantileTDigestWeighted::name, NameQuantilesTDigestWeighted::name},
     {NameQuantileTiming::name, NameQuantilesTiming::name},
@@ -55,6 +56,7 @@ void GatherFunctionQuantileData::FuseQuantileAggregatesData::addFuncNode(ASTPtr
 
     bool need_two_args = func->name == NameQuantileDeterministic::name
                          || func->name == NameQuantileExactWeighted::name
+                         || func->name == NameQuantileApproximateWeighted::name
                          || func->name == NameQuantileTimingWeighted::name
                          || func->name == NameQuantileTDigestWeighted::name
                          || func->name == NameQuantileBFloat16Weighted::name;

From b9706fbf6dcbcf39b1d5b7fd772d70b093fd27d9 Mon Sep 17 00:00:00 2001
From: Bharat Nallan Chakravarthy <bharatnc@gmail.com>
Date: Tue, 21 Jun 2022 07:24:56 +0530
Subject: [PATCH 002/158] add a test for comparision

---
 ...19_quantile_approximate_weighted.reference | 24 +++++++++++
 .../02319_quantile_approximate_weighted.sql   | 41 +++++++++++++++++++
 2 files changed, 65 insertions(+)
 create mode 100644 tests/queries/0_stateless/02319_quantile_approximate_weighted.reference
 create mode 100644 tests/queries/0_stateless/02319_quantile_approximate_weighted.sql

diff --git a/tests/queries/0_stateless/02319_quantile_approximate_weighted.reference b/tests/queries/0_stateless/02319_quantile_approximate_weighted.reference
new file mode 100644
index 00000000000..af5ebdcbf35
--- /dev/null
+++ b/tests/queries/0_stateless/02319_quantile_approximate_weighted.reference
@@ -0,0 +1,24 @@
+quantileExactWeighted
+0	0	0	Decimal(38, 8)
+-26	-8.66666666	-5.2	Decimal(38, 8)
+0	0	0
+10	3.33333333	2
+20	6.66666666	4
+30	10	6
+40	13.33333333	8
+50	16.66666666	10
+[-50,-40,-30,-20,-10,0,10,20,30,40,50]
+[-16.66666666,-13.33333333,-10,-6.66666666,-3.33333333,0,3.33333333,6.66666666,10,13.33333333,16.66666666]
+[-10,-8,-6,-4,-2,0,2,4,6,8,10]
+quantileApproximateWeighted
+0	0	0	Decimal(38, 8)
+-25.5	-8.49999999	-5.1	Decimal(38, 8)
+0	0	0
+9.7	3.23333333	1.94
+19.9	6.63333332	3.98
+30.1	10.03333333	6.02
+40.3	13.43333332	8.06
+50	16.66666666	10
+[-50,-40.4,-30.3,-20.2,-10.1,0,10.1,20.2,30.3,40.4,50]
+[-16.66666666,-13.46666666,-10.09999999,-6.73333332,-3.36666666,0,3.36666666,6.73333332,10.09999999,13.46666666,16.66666666]
+[-10,-8.08,-6.06,-4.04,-2.02,0,2.02,4.04,6.06,8.08,10]
diff --git a/tests/queries/0_stateless/02319_quantile_approximate_weighted.sql b/tests/queries/0_stateless/02319_quantile_approximate_weighted.sql
new file mode 100644
index 00000000000..d79d8e62c35
--- /dev/null
+++ b/tests/queries/0_stateless/02319_quantile_approximate_weighted.sql
@@ -0,0 +1,41 @@
+DROP TABLE IF EXISTS decimal;
+
+CREATE TABLE decimal
+(
+    a Decimal32(4),
+    b Decimal64(8),
+    c Decimal128(8)
+) ENGINE = Memory;
+
+INSERT INTO decimal (a, b, c)
+SELECT toDecimal32(number - 50, 4), toDecimal64(number - 50, 8) / 3, toDecimal128(number - 50, 8) / 5
+FROM system.numbers LIMIT 101;
+
+SELECT 'quantileExactWeighted';
+SELECT medianExactWeighted(a, 1), medianExactWeighted(b, 2), medianExactWeighted(c, 3) as x, toTypeName(x) FROM decimal;
+SELECT quantileExactWeighted(a, 1), quantileExactWeighted(b, 2), quantileExactWeighted(c, 3) as x, toTypeName(x) FROM decimal WHERE a < 0;
+SELECT quantileExactWeighted(0.0)(a, 1), quantileExactWeighted(0.0)(b, 2), quantileExactWeighted(0.0)(c, 3) FROM decimal WHERE a >= 0;
+SELECT quantileExactWeighted(0.2)(a, 1), quantileExactWeighted(0.2)(b, 2), quantileExactWeighted(0.2)(c, 3) FROM decimal WHERE a >= 0;
+SELECT quantileExactWeighted(0.4)(a, 1), quantileExactWeighted(0.4)(b, 2), quantileExactWeighted(0.4)(c, 3) FROM decimal WHERE a >= 0;
+SELECT quantileExactWeighted(0.6)(a, 1), quantileExactWeighted(0.6)(b, 2), quantileExactWeighted(0.6)(c, 3) FROM decimal WHERE a >= 0;
+SELECT quantileExactWeighted(0.8)(a, 1), quantileExactWeighted(0.8)(b, 2), quantileExactWeighted(0.8)(c, 3) FROM decimal WHERE a >= 0;
+SELECT quantileExactWeighted(1.0)(a, 1), quantileExactWeighted(1.0)(b, 2), quantileExactWeighted(1.0)(c, 3) FROM decimal WHERE a >= 0;
+SELECT quantilesExactWeighted(0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0)(a, 1) FROM decimal;
+SELECT quantilesExactWeighted(0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0)(b, 2) FROM decimal;
+SELECT quantilesExactWeighted(0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0)(c, 3) FROM decimal;
+
+
+SELECT 'quantileApproximateWeighted';
+SELECT medianApproximateWeighted(a, 1), medianApproximateWeighted(b, 2), medianApproximateWeighted(c, 3) as x, toTypeName(x) FROM decimal;
+SELECT quantileApproximateWeighted(a, 1), quantileApproximateWeighted(b, 2), quantileApproximateWeighted(c, 3) as x, toTypeName(x) FROM decimal WHERE a < 0;
+SELECT quantileApproximateWeighted(0.0)(a, 1), quantileApproximateWeighted(0.0)(b, 2), quantileApproximateWeighted(0.0)(c, 3) FROM decimal WHERE a >= 0;
+SELECT quantileApproximateWeighted(0.2)(a, 1), quantileApproximateWeighted(0.2)(b, 2), quantileApproximateWeighted(0.2)(c, 3) FROM decimal WHERE a >= 0;
+SELECT quantileApproximateWeighted(0.4)(a, 1), quantileApproximateWeighted(0.4)(b, 2), quantileApproximateWeighted(0.4)(c, 3) FROM decimal WHERE a >= 0;
+SELECT quantileApproximateWeighted(0.6)(a, 1), quantileApproximateWeighted(0.6)(b, 2), quantileApproximateWeighted(0.6)(c, 3) FROM decimal WHERE a >= 0;
+SELECT quantileApproximateWeighted(0.8)(a, 1), quantileApproximateWeighted(0.8)(b, 2), quantileApproximateWeighted(0.8)(c, 3) FROM decimal WHERE a >= 0;
+SELECT quantileApproximateWeighted(1.0)(a, 1), quantileApproximateWeighted(1.0)(b, 2), quantileApproximateWeighted(1.0)(c, 3) FROM decimal WHERE a >= 0;
+SELECT quantilesApproximateWeighted(0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0)(a, 1) FROM decimal;
+SELECT quantilesApproximateWeighted(0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0)(b, 2) FROM decimal;
+SELECT quantilesApproximateWeighted(0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0)(c, 3) FROM decimal;
+
+DROP TABLE IF EXISTS decimal;

From ff5a30a1f8c05fa0b5800f2b5ede47361293417e Mon Sep 17 00:00:00 2001
From: Bharat Nallan Chakravarthy <bharatnc@gmail.com>
Date: Tue, 21 Jun 2022 17:51:17 +0530
Subject: [PATCH 003/158] fix tests

---
 .../QuantileApproximateWeighted.h             | 72 ++++++++++++-------
 .../00315_quantile_off_by_one.reference       |  1 +
 .../0_stateless/00315_quantile_off_by_one.sql |  1 +
 .../00753_quantile_format.reference           |  2 +
 .../0_stateless/00753_quantile_format.sql     |  3 +
 5 files changed, 55 insertions(+), 24 deletions(-)

diff --git a/src/AggregateFunctions/QuantileApproximateWeighted.h b/src/AggregateFunctions/QuantileApproximateWeighted.h
index 3cb1c6a1e29..359169dfc82 100644
--- a/src/AggregateFunctions/QuantileApproximateWeighted.h
+++ b/src/AggregateFunctions/QuantileApproximateWeighted.h
@@ -108,7 +108,7 @@ struct QuantileApproximateWeighted
         std::vector<Float64> cum_sum_array;
 
         bool first = true;
-        for(size_t idx = 0 ; idx < size ; ++idx)
+        for (size_t idx = 0; idx < size; ++idx)
         {
             accumulated += value_weight_pairs[idx].second;
 
@@ -131,25 +131,36 @@ struct QuantileApproximateWeighted
         UnderlyingType g;
 
         size_t idx = 0;
-        if (level >= value_weight_pairs[size - 2].second)
+        if (size >= 2)
         {
-            idx = size - 2;
-        }
-        else
-        {
-            while (level > value_weight_pairs[idx + 1].second)
-                idx++;
+            if (level >= value_weight_pairs[size - 2].second)
+            {
+                idx = size - 2;
+            }
+            else
+            {
+                while (level > value_weight_pairs[idx + 1].second)
+                    idx++;
+            }
         }
 
-        Float64 xl = value_weight_pairs[idx].second, xr = value_weight_pairs[idx + 1].second;
-        UnderlyingType yl = value_weight_pairs[idx].first, yr = value_weight_pairs[idx + 1].first;
+        size_t l = idx;
+        size_t u = idx + 1 < size ? idx + 1 : idx;
+
+        Float64 xl = value_weight_pairs[l].second, xr = value_weight_pairs[u].second;
+        UnderlyingType yl = value_weight_pairs[l].first, yr = value_weight_pairs[u].first;
 
         if (level < xl)
             yr = yl;
         if (level > xr)
             yl = yr;
 
-        auto dydx = (yr - yl) / (xr - xl);
+        auto dy = yr - yl;
+        auto dx = xr - xl;
+
+        dx = dx == 0 ? 1 : dx;
+        auto dydx = dy / dx;
+
         g = yl + dydx * (level - xl);
 
         return g;
@@ -188,7 +199,7 @@ struct QuantileApproximateWeighted
         std::vector<Float64> cum_sum_array;
 
         bool first = true;
-        for (size_t idx = 0 ; idx < size ; ++idx)
+        for (size_t idx = 0; idx < size; ++idx)
         {
             accumulated += value_weight_pairs[idx].second;
 
@@ -205,7 +216,7 @@ struct QuantileApproximateWeighted
 
         /// weighted_quantile = cum_sum_arr - (0.5 * sample_weights)
         for (size_t idx = 0; idx < size; ++idx)
-            value_weight_pairs[idx].second = (cum_sum_array[idx] - 0.5 * value_weight_pairs[idx].second) / sum_weight;
+            value_weight_pairs[idx].second = sum_weight == 0 ? 0 : (cum_sum_array[idx] - 0.5 * value_weight_pairs[idx].second) / sum_weight;
 
         size_t level_index = 0;
 
@@ -214,26 +225,39 @@ struct QuantileApproximateWeighted
             /// linear interpolation for every level
             UnderlyingType g;
             auto level = levels[indices[level_index]];
+
             size_t idx = 0;
-            if (level >= value_weight_pairs[size - 2].second)
+            if (size >= 2)
             {
-                idx = size - 2;
-            }
-            else
-            {
-                while (level > value_weight_pairs[idx + 1].second)
-                    idx++;
+                if (level >= value_weight_pairs[size - 2].second)
+                {
+                    idx = size - 2;
+                }
+                else
+                {
+                    while (level > value_weight_pairs[idx + 1].second)
+                        idx++;
+                }
             }
 
-            Float64 xl = value_weight_pairs[idx].second, xr = value_weight_pairs[idx + 1].second;
-            UnderlyingType yl = value_weight_pairs[idx].first, yr = value_weight_pairs[idx + 1].first;
+            size_t l = idx;
+            size_t u = idx + 1 < size ? idx + 1 : idx;
+
+            Float64 xl = value_weight_pairs[l].second, xr = value_weight_pairs[u].second;
+            UnderlyingType yl = value_weight_pairs[l].first, yr = value_weight_pairs[u].first;
 
             if (level < xl)
                 yr = yl;
             if (level > xr)
                 yl = yr;
 
-            auto dydx = (yr - yl) / (xr - xl);
+
+            auto dy = yr - yl;
+            auto dx = xr - xl;
+
+            dx = dx == 0 ? 1 : dx;
+            auto dydx = dy / dx;
+
             g = yl + dydx * (level - xl);
 
             result[indices[level_index]] = g;
@@ -249,7 +273,7 @@ struct QuantileApproximateWeighted
 
     void getManyFloat(const Float64 *, const size_t *, size_t, Float64 *) const
     {
-        throw Exception("Method getManyFloat is not implemented for QuantileExact", ErrorCodes::NOT_IMPLEMENTED);
+        throw Exception("Method getManyFloat is not implemented for QuantileApproximateWeighted", ErrorCodes::NOT_IMPLEMENTED);
     }
 };
 
diff --git a/tests/queries/0_stateless/00315_quantile_off_by_one.reference b/tests/queries/0_stateless/00315_quantile_off_by_one.reference
index 97ccf188bf4..9b9ab8b3532 100644
--- a/tests/queries/0_stateless/00315_quantile_off_by_one.reference
+++ b/tests/queries/0_stateless/00315_quantile_off_by_one.reference
@@ -1,3 +1,4 @@
 10	[1,1,1,1,10,10,10,10,100,100,100]
+10	[1,1,2,4,7,10,35,61,87,100,100]
 100	100
 61	61
diff --git a/tests/queries/0_stateless/00315_quantile_off_by_one.sql b/tests/queries/0_stateless/00315_quantile_off_by_one.sql
index a1c162eaefb..468069e5c52 100644
--- a/tests/queries/0_stateless/00315_quantile_off_by_one.sql
+++ b/tests/queries/0_stateless/00315_quantile_off_by_one.sql
@@ -1,3 +1,4 @@
 SELECT quantileExactWeighted(0.5)(x, 1) AS q5, quantilesExactWeighted(0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1)(x, 1) AS qs FROM (SELECT arrayJoin([1, 1, 1, 10, 10, 10, 10, 100, 100, 100]) AS x);
+SELECT quantileApproximateWeighted(0.5)(x, 1) AS q5, quantilesApproximateWeighted(0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1)(x, 1) AS qs FROM (SELECT arrayJoin([1, 1, 1, 10, 10, 10, 10, 100, 100, 100]) AS x);
 SELECT quantileExact(0)(x), quantileTiming(0)(x) FROM (SELECT number + 100 AS x FROM system.numbers LIMIT 10000);
 SELECT quantileExact(x), quantileTiming(x) FROM (SELECT number % 123 AS x FROM system.numbers LIMIT 10000);
diff --git a/tests/queries/0_stateless/00753_quantile_format.reference b/tests/queries/0_stateless/00753_quantile_format.reference
index cd662caf5c7..2b267b640a0 100644
--- a/tests/queries/0_stateless/00753_quantile_format.reference
+++ b/tests/queries/0_stateless/00753_quantile_format.reference
@@ -6,6 +6,8 @@
 ['2016-06-15 23:00:00']
 2016-06-15 23:00:00
 ['2016-06-15 23:00:00']
+2016-06-15 23:00:00
+['2016-06-15 23:00:00']
 30000
 [30000]
 30000
diff --git a/tests/queries/0_stateless/00753_quantile_format.sql b/tests/queries/0_stateless/00753_quantile_format.sql
index 8caa3f31368..84a6b18e248 100644
--- a/tests/queries/0_stateless/00753_quantile_format.sql
+++ b/tests/queries/0_stateless/00753_quantile_format.sql
@@ -15,6 +15,9 @@ SELECT quantilesExact(0.2)(d) FROM datetime;
 SELECT quantileExactWeighted(0.2)(d, 1) FROM datetime;
 SELECT quantilesExactWeighted(0.2)(d, 1) FROM datetime;
 
+SELECT quantileApproximateWeighted(0.2)(d, 1) FROM datetime;
+SELECT quantilesApproximateWeighted(0.2)(d, 1) FROM datetime;
+
 SELECT quantileTiming(0.2)(d) FROM datetime;
 SELECT quantilesTiming(0.2)(d) FROM datetime;
 

From 5c878d0ef922a9874c584e82641fc34b45a94090 Mon Sep 17 00:00:00 2001
From: Bharat Nallan Chakravarthy <bharatnc@gmail.com>
Date: Tue, 21 Jun 2022 19:23:16 +0530
Subject: [PATCH 004/158] add comments

---
 .../QuantileApproximateWeighted.h             | 40 ++++++++++++-------
 1 file changed, 26 insertions(+), 14 deletions(-)

diff --git a/src/AggregateFunctions/QuantileApproximateWeighted.h b/src/AggregateFunctions/QuantileApproximateWeighted.h
index 359169dfc82..fa95ba93b4f 100644
--- a/src/AggregateFunctions/QuantileApproximateWeighted.h
+++ b/src/AggregateFunctions/QuantileApproximateWeighted.h
@@ -15,10 +15,11 @@ namespace ErrorCodes
     extern const int NOT_IMPLEMENTED;
 }
 
-/** Calculates quantile by counting number of occurrences for each value in a hash map.
+/** Approximates Quantile by:
+  * - sorting input values and weights
+  * - building a cumulative distribution based on weights
+  * - performing linear interpolation between the weights and values
   *
-  * It uses O(distinct(N)) memory. Can be naturally applied for values with weight.
-  * In case of many identical values, it can be more efficient than QuantileExact even when weight is not used.
   */
 template <typename Value>
 struct QuantileApproximateWeighted
@@ -82,7 +83,8 @@ struct QuantileApproximateWeighted
         if (0 == size)
             return std::numeric_limits<Value>::quiet_NaN();
 
-        /// Copy the data to a temporary array to get the element you need in order.
+        /// Maintain a vector of pair of values and weights for easier sorting and for building
+        /// a cumulative distribution using the provided weights.
         using Pair = typename std::pair<UnderlyingType, Float64>;
         std::vector<Pair> value_weight_pairs;
 
@@ -105,7 +107,9 @@ struct QuantileApproximateWeighted
 
         Float64 accumulated = 0;
 
-        std::vector<Float64> cum_sum_array;
+        /// vector for populating and storing the cumulative sum using the provided weights.
+        /// example: [0,1,2,3,4,5] -> [0,1,3,6,10,15]
+        std::vector<Float64> weights_cum_sum;
 
         bool first = true;
         for (size_t idx = 0; idx < size; ++idx)
@@ -114,23 +118,28 @@ struct QuantileApproximateWeighted
 
             if (first)
             {
-                cum_sum_array.push_back(value_weight_pairs[idx].second);
+                weights_cum_sum.push_back(value_weight_pairs[idx].second);
                 first = false;
             }
             else
             {
-                cum_sum_array.push_back(accumulated);
+                weights_cum_sum.push_back(accumulated);
             }
         }
 
-        /// weighted_quantile = cum_sum_arr - (0.5 * sample_weights)
-        for (size_t idx = 0; idx < size; ++idx)
-            value_weight_pairs[idx].second = (cum_sum_array[idx] - 0.5 * value_weight_pairs[idx].second) / sum_weight;
+        /// The following estimation of quantile is general and the idea is:
+        /// https://en.wikipedia.org/wiki/Percentile#The_weighted_percentile_method
 
-        /// linear interpolation
+        /// calculates a simple cumulative distribution based on weights
+        /// Note: if sum_weight is 0 then subsequent division is inf, so set the value of the exp to 0.
+        for (size_t idx = 0; idx < size; ++idx)
+            value_weight_pairs[idx].second = (weights_cum_sum[idx] - 0.5 * value_weight_pairs[idx].second) / sum_weight;
+
+        /// perform linear interpolation
         UnderlyingType g;
 
         size_t idx = 0;
+
         if (size >= 2)
         {
             if (level >= value_weight_pairs[size - 2].second)
@@ -180,7 +189,6 @@ struct QuantileApproximateWeighted
             return;
         }
 
-        /// Copy the data to a temporary array to get the element you need in order.
         using Pair = typename std::pair<UnderlyingType, Float64>;
         std::vector<Pair> value_weight_pairs;
 
@@ -214,7 +222,11 @@ struct QuantileApproximateWeighted
             }
         }
 
-        /// weighted_quantile = cum_sum_arr - (0.5 * sample_weights)
+        /// The following estimation of quantile is general and the idea is:
+        /// https://en.wikipedia.org/wiki/Percentile#The_weighted_percentile_method
+
+        /// calculates a simple cumulative distribution based on weights
+        /// Note: if sum_weight is 0 then subsequent division is inf, so set the value of the exp to 0.
         for (size_t idx = 0; idx < size; ++idx)
             value_weight_pairs[idx].second = sum_weight == 0 ? 0 : (cum_sum_array[idx] - 0.5 * value_weight_pairs[idx].second) / sum_weight;
 
@@ -222,7 +234,7 @@ struct QuantileApproximateWeighted
 
         while (level_index < num_levels)
         {
-            /// linear interpolation for every level
+            /// perform linear interpolation for every level
             UnderlyingType g;
             auto level = levels[indices[level_index]];
 

From 4b727e812c06e5bd03fc940e2957ec1cb24e9627 Mon Sep 17 00:00:00 2001
From: Bharat Nallan Chakravarthy <bharatnc@gmail.com>
Date: Wed, 22 Jun 2022 08:17:22 +0530
Subject: [PATCH 005/158] fix tests

---
 .../QuantileApproximateWeighted.h                | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/src/AggregateFunctions/QuantileApproximateWeighted.h b/src/AggregateFunctions/QuantileApproximateWeighted.h
index fa95ba93b4f..411a24bb5a9 100644
--- a/src/AggregateFunctions/QuantileApproximateWeighted.h
+++ b/src/AggregateFunctions/QuantileApproximateWeighted.h
@@ -133,7 +133,9 @@ struct QuantileApproximateWeighted
         /// calculates a simple cumulative distribution based on weights
         /// Note: if sum_weight is 0 then subsequent division is inf, so set the value of the exp to 0.
         for (size_t idx = 0; idx < size; ++idx)
-            value_weight_pairs[idx].second = (weights_cum_sum[idx] - 0.5 * value_weight_pairs[idx].second) / sum_weight;
+            value_weight_pairs[idx].second
+                = sum_weight == 0 ? 0 : (weights_cum_sum[idx] - 0.5 * value_weight_pairs[idx].second) / sum_weight;
+
 
         /// perform linear interpolation
         UnderlyingType g;
@@ -204,7 +206,10 @@ struct QuantileApproximateWeighted
         ::sort(value_weight_pairs.begin(), value_weight_pairs.end(), [](const Pair & a, const Pair & b) { return a.first < b.first; });
 
         Float64 accumulated = 0;
-        std::vector<Float64> cum_sum_array;
+
+        /// vector for populating and storing the cumulative sum using the provided weights.
+        /// example: [0,1,2,3,4,5] -> [0,1,3,6,10,15]
+        std::vector<Float64> weights_cum_sum;
 
         bool first = true;
         for (size_t idx = 0; idx < size; ++idx)
@@ -213,12 +218,12 @@ struct QuantileApproximateWeighted
 
             if (first)
             {
-                cum_sum_array.emplace_back(value_weight_pairs[idx].second);
+                weights_cum_sum.emplace_back(value_weight_pairs[idx].second);
                 first = false;
             }
             else
             {
-                cum_sum_array.emplace_back(accumulated);
+                weights_cum_sum.emplace_back(accumulated);
             }
         }
 
@@ -228,7 +233,8 @@ struct QuantileApproximateWeighted
         /// calculates a simple cumulative distribution based on weights
         /// Note: if sum_weight is 0 then subsequent division is inf, so set the value of the exp to 0.
         for (size_t idx = 0; idx < size; ++idx)
-            value_weight_pairs[idx].second = sum_weight == 0 ? 0 : (cum_sum_array[idx] - 0.5 * value_weight_pairs[idx].second) / sum_weight;
+            value_weight_pairs[idx].second
+                = sum_weight == 0 ? 0 : (weights_cum_sum[idx] - 0.5 * value_weight_pairs[idx].second) / sum_weight;
 
         size_t level_index = 0;
 

From 4e7f8b093527fca38175e8247a9ca09aae784ef2 Mon Sep 17 00:00:00 2001
From: Bharat Nallan Chakravarthy <bharatnc@gmail.com>
Date: Thu, 23 Jun 2022 10:33:24 +0530
Subject: [PATCH 006/158] try fix fuzzer error

---
 src/AggregateFunctions/QuantileApproximateWeighted.h   |  8 ++++----
 .../0_stateless/00315_quantile_off_by_one.reference    |  2 +-
 .../02319_quantile_approximate_weighted.reference      | 10 +++++-----
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/AggregateFunctions/QuantileApproximateWeighted.h b/src/AggregateFunctions/QuantileApproximateWeighted.h
index 411a24bb5a9..e602e990d5a 100644
--- a/src/AggregateFunctions/QuantileApproximateWeighted.h
+++ b/src/AggregateFunctions/QuantileApproximateWeighted.h
@@ -166,11 +166,11 @@ struct QuantileApproximateWeighted
         if (level > xr)
             yl = yr;
 
-        auto dy = yr - yl;
+        UnderlyingType dy = yr - yl;
         auto dx = xr - xl;
 
         dx = dx == 0 ? 1 : dx;
-        auto dydx = dy / dx;
+        UnderlyingType dydx = dy / dx;
 
         g = yl + dydx * (level - xl);
 
@@ -270,11 +270,11 @@ struct QuantileApproximateWeighted
                 yl = yr;
 
 
-            auto dy = yr - yl;
+            UnderlyingType dy = yr - yl;
             auto dx = xr - xl;
 
             dx = dx == 0 ? 1 : dx;
-            auto dydx = dy / dx;
+            UnderlyingType dydx = dy / dx;
 
             g = yl + dydx * (level - xl);
 
diff --git a/tests/queries/0_stateless/00315_quantile_off_by_one.reference b/tests/queries/0_stateless/00315_quantile_off_by_one.reference
index 9b9ab8b3532..4b97df2d13d 100644
--- a/tests/queries/0_stateless/00315_quantile_off_by_one.reference
+++ b/tests/queries/0_stateless/00315_quantile_off_by_one.reference
@@ -1,4 +1,4 @@
 10	[1,1,1,1,10,10,10,10,100,100,100]
-10	[1,1,2,4,7,10,35,61,87,100,100]
+10	[1,1,2,4,7,10,10,10,10,100,100]
 100	100
 61	61
diff --git a/tests/queries/0_stateless/02319_quantile_approximate_weighted.reference b/tests/queries/0_stateless/02319_quantile_approximate_weighted.reference
index af5ebdcbf35..6874db9ca74 100644
--- a/tests/queries/0_stateless/02319_quantile_approximate_weighted.reference
+++ b/tests/queries/0_stateless/02319_quantile_approximate_weighted.reference
@@ -15,10 +15,10 @@ quantileApproximateWeighted
 -25.5	-8.49999999	-5.1	Decimal(38, 8)
 0	0	0
 9.7	3.23333333	1.94
-19.9	6.63333332	3.98
-30.1	10.03333333	6.02
-40.3	13.43333332	8.06
+19.8999	6.63333332	3.97999999
+30.0999	10.03333333	6.01999999
+40.2999	13.43333332	8.05999999
 50	16.66666666	10
-[-50,-40.4,-30.3,-20.2,-10.1,0,10.1,20.2,30.3,40.4,50]
+[-50,-40.4,-30.3,-20.2,-10.1,0,10.0999,20.1999,30.2999,40.3999,50]
 [-16.66666666,-13.46666666,-10.09999999,-6.73333332,-3.36666666,0,3.36666666,6.73333332,10.09999999,13.46666666,16.66666666]
-[-10,-8.08,-6.06,-4.04,-2.02,0,2.02,4.04,6.06,8.08,10]
+[-10,-8.08,-6.06,-4.04,-2.02,0,2.01999999,4.03999999,6.05999999,8.07999999,10]

From 33704888e57c93288d2270869b7d951dce594038 Mon Sep 17 00:00:00 2001
From: Bharat Nallan Chakravarthy <bharatnc@gmail.com>
Date: Thu, 23 Jun 2022 20:21:50 +0530
Subject: [PATCH 007/158] try add some numeric limits checks

---
 .../QuantileApproximateWeighted.h             | 53 ++++++++++++++++---
 .../00315_quantile_off_by_one.reference       |  2 +-
 ...19_quantile_approximate_weighted.reference | 10 ++--
 3 files changed, 51 insertions(+), 14 deletions(-)

diff --git a/src/AggregateFunctions/QuantileApproximateWeighted.h b/src/AggregateFunctions/QuantileApproximateWeighted.h
index e602e990d5a..7f6139e5de7 100644
--- a/src/AggregateFunctions/QuantileApproximateWeighted.h
+++ b/src/AggregateFunctions/QuantileApproximateWeighted.h
@@ -166,13 +166,32 @@ struct QuantileApproximateWeighted
         if (level > xr)
             yl = yr;
 
-        UnderlyingType dy = yr - yl;
-        auto dx = xr - xl;
+        UnderlyingType dy;
+
+        if (yr - yl > std::numeric_limits<UnderlyingType>::max())
+            dy = std::numeric_limits<UnderlyingType>::max();
+        else if (yr - yl < std::numeric_limits<UnderlyingType>::min())
+            dy = std::numeric_limits<UnderlyingType>::min();
+        else
+            dy = yr - yl;
+
+        Float64 dx;
+
+        if (xr - xl > std::numeric_limits<Float64>::max())
+            dx = std::numeric_limits<Float64>::max();
+        else if (xr - xl < std::numeric_limits<Float64>::min())
+            dx = std::numeric_limits<Float64>::min();
+        else
+            dx = xr - xl;
 
         dx = dx == 0 ? 1 : dx;
-        UnderlyingType dydx = dy / dx;
 
-        g = yl + dydx * (level - xl);
+        if ((yl + (dy / dx) * (level - xl)) > std::numeric_limits<UnderlyingType>::max())
+            g = std::numeric_limits<UnderlyingType>::max();
+        else if (yl + (dy / dx) * (level - xl) < std::numeric_limits<UnderlyingType>::min())
+            g = std::numeric_limits<UnderlyingType>::min();
+        else
+            g = yl + (dy / dx) * (level - xl);
 
         return g;
     }
@@ -269,14 +288,32 @@ struct QuantileApproximateWeighted
             if (level > xr)
                 yl = yr;
 
+            UnderlyingType dy;
 
-            UnderlyingType dy = yr - yl;
-            auto dx = xr - xl;
+            if (yr - yl > std::numeric_limits<UnderlyingType>::max())
+                dy = std::numeric_limits<UnderlyingType>::max();
+            else if (yr - yl < std::numeric_limits<UnderlyingType>::min())
+                dy = std::numeric_limits<UnderlyingType>::min();
+            else
+                dy = yr - yl;
+
+            Float64 dx;
+
+            if (xr - xl > std::numeric_limits<Float64>::max())
+                dx = std::numeric_limits<Float64>::max();
+            else if (xr - xl < std::numeric_limits<Float64>::min())
+                dx = std::numeric_limits<Float64>::min();
+            else
+                dx = xr - xl;
 
             dx = dx == 0 ? 1 : dx;
-            UnderlyingType dydx = dy / dx;
 
-            g = yl + dydx * (level - xl);
+            if ((yl + (dy / dx) * (level - xl)) > std::numeric_limits<UnderlyingType>::max())
+                g = std::numeric_limits<UnderlyingType>::max();
+            else if ((yl + (dy / dx) * (level - xl)) < std::numeric_limits<UnderlyingType>::min())
+                g = std::numeric_limits<UnderlyingType>::min();
+            else
+                g = yl + (dy / dx) * (level - xl);
 
             result[indices[level_index]] = g;
             ++level_index;
diff --git a/tests/queries/0_stateless/00315_quantile_off_by_one.reference b/tests/queries/0_stateless/00315_quantile_off_by_one.reference
index 4b97df2d13d..9b9ab8b3532 100644
--- a/tests/queries/0_stateless/00315_quantile_off_by_one.reference
+++ b/tests/queries/0_stateless/00315_quantile_off_by_one.reference
@@ -1,4 +1,4 @@
 10	[1,1,1,1,10,10,10,10,100,100,100]
-10	[1,1,2,4,7,10,10,10,10,100,100]
+10	[1,1,2,4,7,10,35,61,87,100,100]
 100	100
 61	61
diff --git a/tests/queries/0_stateless/02319_quantile_approximate_weighted.reference b/tests/queries/0_stateless/02319_quantile_approximate_weighted.reference
index 6874db9ca74..af5ebdcbf35 100644
--- a/tests/queries/0_stateless/02319_quantile_approximate_weighted.reference
+++ b/tests/queries/0_stateless/02319_quantile_approximate_weighted.reference
@@ -15,10 +15,10 @@ quantileApproximateWeighted
 -25.5	-8.49999999	-5.1	Decimal(38, 8)
 0	0	0
 9.7	3.23333333	1.94
-19.8999	6.63333332	3.97999999
-30.0999	10.03333333	6.01999999
-40.2999	13.43333332	8.05999999
+19.9	6.63333332	3.98
+30.1	10.03333333	6.02
+40.3	13.43333332	8.06
 50	16.66666666	10
-[-50,-40.4,-30.3,-20.2,-10.1,0,10.0999,20.1999,30.2999,40.3999,50]
+[-50,-40.4,-30.3,-20.2,-10.1,0,10.1,20.2,30.3,40.4,50]
 [-16.66666666,-13.46666666,-10.09999999,-6.73333332,-3.36666666,0,3.36666666,6.73333332,10.09999999,13.46666666,16.66666666]
-[-10,-8.08,-6.06,-4.04,-2.02,0,2.01999999,4.03999999,6.05999999,8.07999999,10]
+[-10,-8.08,-6.06,-4.04,-2.02,0,2.02,4.04,6.06,8.08,10]

From d0bfb4e1f55d0409063f4e21b84a7cdf154ddae9 Mon Sep 17 00:00:00 2001
From: Bharat Nallan Chakravarthy <bharatnc@gmail.com>
Date: Thu, 23 Jun 2022 21:39:00 +0530
Subject: [PATCH 008/158] handle nan

---
 src/AggregateFunctions/QuantileApproximateWeighted.h | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/AggregateFunctions/QuantileApproximateWeighted.h b/src/AggregateFunctions/QuantileApproximateWeighted.h
index 7f6139e5de7..5a661d12111 100644
--- a/src/AggregateFunctions/QuantileApproximateWeighted.h
+++ b/src/AggregateFunctions/QuantileApproximateWeighted.h
@@ -190,6 +190,8 @@ struct QuantileApproximateWeighted
             g = std::numeric_limits<UnderlyingType>::max();
         else if (yl + (dy / dx) * (level - xl) < std::numeric_limits<UnderlyingType>::min())
             g = std::numeric_limits<UnderlyingType>::min();
+        else if (isNaN(yl + (dy / dx) * (level - xl)))
+            g = std::numeric_limits<UnderlyingType>::quiet_NaN();
         else
             g = yl + (dy / dx) * (level - xl);
 
@@ -312,6 +314,8 @@ struct QuantileApproximateWeighted
                 g = std::numeric_limits<UnderlyingType>::max();
             else if ((yl + (dy / dx) * (level - xl)) < std::numeric_limits<UnderlyingType>::min())
                 g = std::numeric_limits<UnderlyingType>::min();
+            else if (isNaN(yl + (dy / dx) * (level - xl)))
+                g = std::numeric_limits<UnderlyingType>::quiet_NaN();
             else
                 g = yl + (dy / dx) * (level - xl);
 

From 054ce6f712da04324dadbef5b9be8d2ad34e92e6 Mon Sep 17 00:00:00 2001
From: Bharat Nallan Chakravarthy <bharatnc@gmail.com>
Date: Sat, 9 Jul 2022 14:02:39 +0530
Subject: [PATCH 009/158] rename func to quantileInterpolatedWeighted

---
 .../AggregateFunctionQuantile.cpp             | 18 ++++----
 .../AggregateFunctionQuantile.h               | 14 +++----
 ...ghted.h => QuantileInterpolatedWeighted.h} |  8 ++--
 .../GatherFunctionQuantileVisitor.cpp         |  4 +-
 .../0_stateless/00315_quantile_off_by_one.sql |  2 +-
 .../0_stateless/00753_quantile_format.sql     |  4 +-
 .../02319_quantile_approximate_weighted.sql   | 41 -------------------
 ..._quantile_interpolated_weighted.reference} | 14 +------
 .../02319_quantile_interpolated_weighted.sql  | 27 ++++++++++++
 9 files changed, 53 insertions(+), 79 deletions(-)
 rename src/AggregateFunctions/{QuantileApproximateWeighted.h => QuantileInterpolatedWeighted.h} (98%)
 delete mode 100644 tests/queries/0_stateless/02319_quantile_approximate_weighted.sql
 rename tests/queries/0_stateless/{02319_quantile_approximate_weighted.reference => 02319_quantile_interpolated_weighted.reference} (52%)
 create mode 100644 tests/queries/0_stateless/02319_quantile_interpolated_weighted.sql

diff --git a/src/AggregateFunctions/AggregateFunctionQuantile.cpp b/src/AggregateFunctions/AggregateFunctionQuantile.cpp
index ebd6f44df17..120fe3b6a59 100644
--- a/src/AggregateFunctions/AggregateFunctionQuantile.cpp
+++ b/src/AggregateFunctions/AggregateFunctionQuantile.cpp
@@ -41,8 +41,8 @@ template <typename Value, bool _> using FuncQuantilesExactInclusive = AggregateF
 template <typename Value, bool _> using FuncQuantileExactWeighted = AggregateFunctionQuantile<Value, QuantileExactWeighted<Value>, NameQuantileExactWeighted, true, void, false>;
 template <typename Value, bool _> using FuncQuantilesExactWeighted = AggregateFunctionQuantile<Value, QuantileExactWeighted<Value>, NameQuantilesExactWeighted, true, void, true>;
 
-template <typename Value, bool _> using FuncQuantileApproximateWeighted = AggregateFunctionQuantile<Value, QuantileApproximateWeighted<Value>, NameQuantileApproximateWeighted, true, void, false>;
-template <typename Value, bool _> using FuncQuantilesApproximateWeighted = AggregateFunctionQuantile<Value, QuantileApproximateWeighted<Value>, NameQuantilesApproximateWeighted, true, void, true>;
+template <typename Value, bool _> using FuncQuantileInterpolatedWeighted = AggregateFunctionQuantile<Value, QuantileInterpolatedWeighted<Value>, NameQuantileInterpolatedWeighted, true, void, false>;
+template <typename Value, bool _> using FuncQuantilesInterpolatedWeighted = AggregateFunctionQuantile<Value, QuantileInterpolatedWeighted<Value>, NameQuantilesInterpolatedWeighted, true, void, true>;
 
 template <typename Value, bool _> using FuncQuantileTiming = AggregateFunctionQuantile<Value, QuantileTiming<Value>, NameQuantileTiming, false, Float32, false>;
 template <typename Value, bool _> using FuncQuantilesTiming = AggregateFunctionQuantile<Value, QuantileTiming<Value>, NameQuantilesTiming, false, Float32, true>;
@@ -75,8 +75,8 @@ constexpr bool supportDecimal()
         std::is_same_v<Function<Float32, false>, FuncQuantilesExactHigh<Float32, false>> ||
         std::is_same_v<Function<Float32, false>, FuncQuantileExactWeighted<Float32, false>> ||
         std::is_same_v<Function<Float32, false>, FuncQuantilesExactWeighted<Float32, false>> ||
-        std::is_same_v<Function<Float32, false>, FuncQuantileApproximateWeighted<Float32, false>> ||
-        std::is_same_v<Function<Float32, false>, FuncQuantilesApproximateWeighted<Float32, false>>;
+        std::is_same_v<Function<Float32, false>, FuncQuantileInterpolatedWeighted<Float32, false>> ||
+        std::is_same_v<Function<Float32, false>, FuncQuantilesInterpolatedWeighted<Float32, false>>;
 }
 
 template <template <typename, bool> class Function>
@@ -88,8 +88,8 @@ constexpr bool supportBigInt()
         std::is_same_v<Function<Float32, false>, FuncQuantilesExact<Float32, false>> ||
         std::is_same_v<Function<Float32, false>, FuncQuantileExactWeighted<Float32, false>> ||
         std::is_same_v<Function<Float32, false>, FuncQuantilesExactWeighted<Float32, false>> ||
-        std::is_same_v<Function<Float32, false>, FuncQuantilesApproximateWeighted<Float32, false>> ||
-        std::is_same_v<Function<Float32, false>, FuncQuantilesApproximateWeighted<Float32, false>>;
+        std::is_same_v<Function<Float32, false>, FuncQuantilesInterpolatedWeighted<Float32, false>> ||
+        std::is_same_v<Function<Float32, false>, FuncQuantilesInterpolatedWeighted<Float32, false>>;
 }
 
 template <template <typename, bool> class Function>
@@ -161,8 +161,8 @@ void registerAggregateFunctionsQuantile(AggregateFunctionFactory & factory)
     factory.registerFunction(NameQuantileExactWeighted::name, createAggregateFunctionQuantile<FuncQuantileExactWeighted>);
     factory.registerFunction(NameQuantilesExactWeighted::name, { createAggregateFunctionQuantile<FuncQuantilesExactWeighted>, properties });
 
-    factory.registerFunction(NameQuantileApproximateWeighted::name, createAggregateFunctionQuantile<FuncQuantileApproximateWeighted>);
-    factory.registerFunction(NameQuantilesApproximateWeighted::name, { createAggregateFunctionQuantile<FuncQuantilesApproximateWeighted>, properties });
+    factory.registerFunction(NameQuantileInterpolatedWeighted::name, createAggregateFunctionQuantile<FuncQuantileInterpolatedWeighted>);
+    factory.registerFunction(NameQuantilesInterpolatedWeighted::name, { createAggregateFunctionQuantile<FuncQuantilesInterpolatedWeighted>, properties });
 
     factory.registerFunction(NameQuantileTiming::name, createAggregateFunctionQuantile<FuncQuantileTiming>);
     factory.registerFunction(NameQuantilesTiming::name, { createAggregateFunctionQuantile<FuncQuantilesTiming>, properties });
@@ -189,7 +189,7 @@ void registerAggregateFunctionsQuantile(AggregateFunctionFactory & factory)
     factory.registerAlias("medianExactLow", NameQuantileExactLow::name);
     factory.registerAlias("medianExactHigh", NameQuantileExactHigh::name);
     factory.registerAlias("medianExactWeighted", NameQuantileExactWeighted::name);
-    factory.registerAlias("medianApproximateWeighted", NameQuantileApproximateWeighted::name);
+    factory.registerAlias("medianInterpolatedWeighted", NameQuantileInterpolatedWeighted::name);
     factory.registerAlias("medianTiming", NameQuantileTiming::name);
     factory.registerAlias("medianTimingWeighted", NameQuantileTimingWeighted::name);
     factory.registerAlias("medianTDigest", NameQuantileTDigest::name);
diff --git a/src/AggregateFunctions/AggregateFunctionQuantile.h b/src/AggregateFunctions/AggregateFunctionQuantile.h
index 9c172595a3d..3adf32b94fd 100644
--- a/src/AggregateFunctions/AggregateFunctionQuantile.h
+++ b/src/AggregateFunctions/AggregateFunctionQuantile.h
@@ -3,14 +3,14 @@
 #include <AggregateFunctions/FactoryHelpers.h>
 
 /// These must be exposed in header for the purpose of dynamic compilation.
-#include <AggregateFunctions/QuantileReservoirSampler.h>
-#include <AggregateFunctions/QuantileReservoirSamplerDeterministic.h>
+#include <AggregateFunctions/QuantileBFloat16Histogram.h>
 #include <AggregateFunctions/QuantileExact.h>
 #include <AggregateFunctions/QuantileExactWeighted.h>
-#include <AggregateFunctions/QuantileApproximateWeighted.h>
-#include <AggregateFunctions/QuantileTiming.h>
+#include <AggregateFunctions/QuantileInterpolatedWeighted.h>
+#include <AggregateFunctions/QuantileReservoirSampler.h>
+#include <AggregateFunctions/QuantileReservoirSamplerDeterministic.h>
 #include <AggregateFunctions/QuantileTDigest.h>
-#include <AggregateFunctions/QuantileBFloat16Histogram.h>
+#include <AggregateFunctions/QuantileTiming.h>
 
 #include <AggregateFunctions/IAggregateFunction.h>
 #include <AggregateFunctions/QuantilesCommon.h>
@@ -226,8 +226,8 @@ struct NameQuantilesExactInclusive { static constexpr auto name = "quantilesExac
 struct NameQuantileExactWeighted { static constexpr auto name = "quantileExactWeighted"; };
 struct NameQuantilesExactWeighted { static constexpr auto name = "quantilesExactWeighted"; };
 
-struct NameQuantileApproximateWeighted { static constexpr auto name = "quantileApproximateWeighted"; };
-struct NameQuantilesApproximateWeighted { static constexpr auto name = "quantilesApproximateWeighted"; };
+struct NameQuantileInterpolatedWeighted { static constexpr auto name = "quantileInterpolatedWeighted"; };
+struct NameQuantilesInterpolatedWeighted { static constexpr auto name = "quantilesInterpolatedWeighted"; };
 
 struct NameQuantileTiming { static constexpr auto name = "quantileTiming"; };
 struct NameQuantileTimingWeighted { static constexpr auto name = "quantileTimingWeighted"; };
diff --git a/src/AggregateFunctions/QuantileApproximateWeighted.h b/src/AggregateFunctions/QuantileInterpolatedWeighted.h
similarity index 98%
rename from src/AggregateFunctions/QuantileApproximateWeighted.h
rename to src/AggregateFunctions/QuantileInterpolatedWeighted.h
index 5a661d12111..a50bada8635 100644
--- a/src/AggregateFunctions/QuantileApproximateWeighted.h
+++ b/src/AggregateFunctions/QuantileInterpolatedWeighted.h
@@ -22,7 +22,7 @@ namespace ErrorCodes
   *
   */
 template <typename Value>
-struct QuantileApproximateWeighted
+struct QuantileInterpolatedWeighted
 {
     struct Int128Hash
     {
@@ -54,7 +54,7 @@ struct QuantileApproximateWeighted
             map[x] += weight;
     }
 
-    void merge(const QuantileApproximateWeighted & rhs)
+    void merge(const QuantileInterpolatedWeighted & rhs)
     {
         for (const auto & pair : rhs.map)
             map[pair.getKey()] += pair.getMapped();
@@ -327,12 +327,12 @@ struct QuantileApproximateWeighted
     /// The same, but in the case of an empty state, NaN is returned.
     Float64 getFloat(Float64) const
     {
-        throw Exception("Method getFloat is not implemented for QuantileApproximateWeighted", ErrorCodes::NOT_IMPLEMENTED);
+        throw Exception("Method getFloat is not implemented for QuantileInterpolatedWeighted", ErrorCodes::NOT_IMPLEMENTED);
     }
 
     void getManyFloat(const Float64 *, const size_t *, size_t, Float64 *) const
     {
-        throw Exception("Method getManyFloat is not implemented for QuantileApproximateWeighted", ErrorCodes::NOT_IMPLEMENTED);
+        throw Exception("Method getManyFloat is not implemented for QuantileInterpolatedWeighted", ErrorCodes::NOT_IMPLEMENTED);
     }
 };
 
diff --git a/src/Interpreters/GatherFunctionQuantileVisitor.cpp b/src/Interpreters/GatherFunctionQuantileVisitor.cpp
index 13abc954d78..aff1cccdca7 100644
--- a/src/Interpreters/GatherFunctionQuantileVisitor.cpp
+++ b/src/Interpreters/GatherFunctionQuantileVisitor.cpp
@@ -24,7 +24,7 @@ static const std::unordered_map<String, String> quantile_fuse_name_mapping = {
     {NameQuantileExactInclusive::name, NameQuantilesExactInclusive::name},
     {NameQuantileExactLow::name, NameQuantilesExactLow::name},
     {NameQuantileExactWeighted::name, NameQuantilesExactWeighted::name},
-    {NameQuantileApproximateWeighted::name, NameQuantilesApproximateWeighted::name},
+    {NameQuantileInterpolatedWeighted::name, NameQuantilesInterpolatedWeighted::name},
     {NameQuantileTDigest::name, NameQuantilesTDigest::name},
     {NameQuantileTDigestWeighted::name, NameQuantilesTDigestWeighted::name},
     {NameQuantileTiming::name, NameQuantilesTiming::name},
@@ -56,7 +56,7 @@ void GatherFunctionQuantileData::FuseQuantileAggregatesData::addFuncNode(ASTPtr
 
     bool need_two_args = func->name == NameQuantileDeterministic::name
                          || func->name == NameQuantileExactWeighted::name
-                         || func->name == NameQuantileApproximateWeighted::name
+                         || func->name == NameQuantileInterpolatedWeighted::name
                          || func->name == NameQuantileTimingWeighted::name
                          || func->name == NameQuantileTDigestWeighted::name
                          || func->name == NameQuantileBFloat16Weighted::name;
diff --git a/tests/queries/0_stateless/00315_quantile_off_by_one.sql b/tests/queries/0_stateless/00315_quantile_off_by_one.sql
index 468069e5c52..50383183d5e 100644
--- a/tests/queries/0_stateless/00315_quantile_off_by_one.sql
+++ b/tests/queries/0_stateless/00315_quantile_off_by_one.sql
@@ -1,4 +1,4 @@
 SELECT quantileExactWeighted(0.5)(x, 1) AS q5, quantilesExactWeighted(0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1)(x, 1) AS qs FROM (SELECT arrayJoin([1, 1, 1, 10, 10, 10, 10, 100, 100, 100]) AS x);
-SELECT quantileApproximateWeighted(0.5)(x, 1) AS q5, quantilesApproximateWeighted(0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1)(x, 1) AS qs FROM (SELECT arrayJoin([1, 1, 1, 10, 10, 10, 10, 100, 100, 100]) AS x);
+SELECT quantileInterpolatedWeighted(0.5)(x, 1) AS q5, quantilesInterpolatedWeighted(0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1)(x, 1) AS qs FROM (SELECT arrayJoin([1, 1, 1, 10, 10, 10, 10, 100, 100, 100]) AS x);
 SELECT quantileExact(0)(x), quantileTiming(0)(x) FROM (SELECT number + 100 AS x FROM system.numbers LIMIT 10000);
 SELECT quantileExact(x), quantileTiming(x) FROM (SELECT number % 123 AS x FROM system.numbers LIMIT 10000);
diff --git a/tests/queries/0_stateless/00753_quantile_format.sql b/tests/queries/0_stateless/00753_quantile_format.sql
index 84a6b18e248..fc3358ae271 100644
--- a/tests/queries/0_stateless/00753_quantile_format.sql
+++ b/tests/queries/0_stateless/00753_quantile_format.sql
@@ -15,8 +15,8 @@ SELECT quantilesExact(0.2)(d) FROM datetime;
 SELECT quantileExactWeighted(0.2)(d, 1) FROM datetime;
 SELECT quantilesExactWeighted(0.2)(d, 1) FROM datetime;
 
-SELECT quantileApproximateWeighted(0.2)(d, 1) FROM datetime;
-SELECT quantilesApproximateWeighted(0.2)(d, 1) FROM datetime;
+SELECT quantileInterpolatedWeighted(0.2)(d, 1) FROM datetime;
+SELECT quantilesInterpolatedWeighted(0.2)(d, 1) FROM datetime;
 
 SELECT quantileTiming(0.2)(d) FROM datetime;
 SELECT quantilesTiming(0.2)(d) FROM datetime;
diff --git a/tests/queries/0_stateless/02319_quantile_approximate_weighted.sql b/tests/queries/0_stateless/02319_quantile_approximate_weighted.sql
deleted file mode 100644
index d79d8e62c35..00000000000
--- a/tests/queries/0_stateless/02319_quantile_approximate_weighted.sql
+++ /dev/null
@@ -1,41 +0,0 @@
-DROP TABLE IF EXISTS decimal;
-
-CREATE TABLE decimal
-(
-    a Decimal32(4),
-    b Decimal64(8),
-    c Decimal128(8)
-) ENGINE = Memory;
-
-INSERT INTO decimal (a, b, c)
-SELECT toDecimal32(number - 50, 4), toDecimal64(number - 50, 8) / 3, toDecimal128(number - 50, 8) / 5
-FROM system.numbers LIMIT 101;
-
-SELECT 'quantileExactWeighted';
-SELECT medianExactWeighted(a, 1), medianExactWeighted(b, 2), medianExactWeighted(c, 3) as x, toTypeName(x) FROM decimal;
-SELECT quantileExactWeighted(a, 1), quantileExactWeighted(b, 2), quantileExactWeighted(c, 3) as x, toTypeName(x) FROM decimal WHERE a < 0;
-SELECT quantileExactWeighted(0.0)(a, 1), quantileExactWeighted(0.0)(b, 2), quantileExactWeighted(0.0)(c, 3) FROM decimal WHERE a >= 0;
-SELECT quantileExactWeighted(0.2)(a, 1), quantileExactWeighted(0.2)(b, 2), quantileExactWeighted(0.2)(c, 3) FROM decimal WHERE a >= 0;
-SELECT quantileExactWeighted(0.4)(a, 1), quantileExactWeighted(0.4)(b, 2), quantileExactWeighted(0.4)(c, 3) FROM decimal WHERE a >= 0;
-SELECT quantileExactWeighted(0.6)(a, 1), quantileExactWeighted(0.6)(b, 2), quantileExactWeighted(0.6)(c, 3) FROM decimal WHERE a >= 0;
-SELECT quantileExactWeighted(0.8)(a, 1), quantileExactWeighted(0.8)(b, 2), quantileExactWeighted(0.8)(c, 3) FROM decimal WHERE a >= 0;
-SELECT quantileExactWeighted(1.0)(a, 1), quantileExactWeighted(1.0)(b, 2), quantileExactWeighted(1.0)(c, 3) FROM decimal WHERE a >= 0;
-SELECT quantilesExactWeighted(0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0)(a, 1) FROM decimal;
-SELECT quantilesExactWeighted(0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0)(b, 2) FROM decimal;
-SELECT quantilesExactWeighted(0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0)(c, 3) FROM decimal;
-
-
-SELECT 'quantileApproximateWeighted';
-SELECT medianApproximateWeighted(a, 1), medianApproximateWeighted(b, 2), medianApproximateWeighted(c, 3) as x, toTypeName(x) FROM decimal;
-SELECT quantileApproximateWeighted(a, 1), quantileApproximateWeighted(b, 2), quantileApproximateWeighted(c, 3) as x, toTypeName(x) FROM decimal WHERE a < 0;
-SELECT quantileApproximateWeighted(0.0)(a, 1), quantileApproximateWeighted(0.0)(b, 2), quantileApproximateWeighted(0.0)(c, 3) FROM decimal WHERE a >= 0;
-SELECT quantileApproximateWeighted(0.2)(a, 1), quantileApproximateWeighted(0.2)(b, 2), quantileApproximateWeighted(0.2)(c, 3) FROM decimal WHERE a >= 0;
-SELECT quantileApproximateWeighted(0.4)(a, 1), quantileApproximateWeighted(0.4)(b, 2), quantileApproximateWeighted(0.4)(c, 3) FROM decimal WHERE a >= 0;
-SELECT quantileApproximateWeighted(0.6)(a, 1), quantileApproximateWeighted(0.6)(b, 2), quantileApproximateWeighted(0.6)(c, 3) FROM decimal WHERE a >= 0;
-SELECT quantileApproximateWeighted(0.8)(a, 1), quantileApproximateWeighted(0.8)(b, 2), quantileApproximateWeighted(0.8)(c, 3) FROM decimal WHERE a >= 0;
-SELECT quantileApproximateWeighted(1.0)(a, 1), quantileApproximateWeighted(1.0)(b, 2), quantileApproximateWeighted(1.0)(c, 3) FROM decimal WHERE a >= 0;
-SELECT quantilesApproximateWeighted(0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0)(a, 1) FROM decimal;
-SELECT quantilesApproximateWeighted(0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0)(b, 2) FROM decimal;
-SELECT quantilesApproximateWeighted(0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0)(c, 3) FROM decimal;
-
-DROP TABLE IF EXISTS decimal;
diff --git a/tests/queries/0_stateless/02319_quantile_approximate_weighted.reference b/tests/queries/0_stateless/02319_quantile_interpolated_weighted.reference
similarity index 52%
rename from tests/queries/0_stateless/02319_quantile_approximate_weighted.reference
rename to tests/queries/0_stateless/02319_quantile_interpolated_weighted.reference
index af5ebdcbf35..88919ca8aad 100644
--- a/tests/queries/0_stateless/02319_quantile_approximate_weighted.reference
+++ b/tests/queries/0_stateless/02319_quantile_interpolated_weighted.reference
@@ -1,16 +1,4 @@
-quantileExactWeighted
-0	0	0	Decimal(38, 8)
--26	-8.66666666	-5.2	Decimal(38, 8)
-0	0	0
-10	3.33333333	2
-20	6.66666666	4
-30	10	6
-40	13.33333333	8
-50	16.66666666	10
-[-50,-40,-30,-20,-10,0,10,20,30,40,50]
-[-16.66666666,-13.33333333,-10,-6.66666666,-3.33333333,0,3.33333333,6.66666666,10,13.33333333,16.66666666]
-[-10,-8,-6,-4,-2,0,2,4,6,8,10]
-quantileApproximateWeighted
+quantileInterpolatedWeighted
 0	0	0	Decimal(38, 8)
 -25.5	-8.49999999	-5.1	Decimal(38, 8)
 0	0	0
diff --git a/tests/queries/0_stateless/02319_quantile_interpolated_weighted.sql b/tests/queries/0_stateless/02319_quantile_interpolated_weighted.sql
new file mode 100644
index 00000000000..e2da1de9bbf
--- /dev/null
+++ b/tests/queries/0_stateless/02319_quantile_interpolated_weighted.sql
@@ -0,0 +1,27 @@
+DROP TABLE IF EXISTS decimal;
+
+CREATE TABLE decimal
+(
+    a Decimal32(4),
+    b Decimal64(8),
+    c Decimal128(8)
+) ENGINE = Memory;
+
+INSERT INTO decimal (a, b, c)
+SELECT toDecimal32(number - 50, 4), toDecimal64(number - 50, 8) / 3, toDecimal128(number - 50, 8) / 5
+FROM system.numbers LIMIT 101;
+
+SELECT 'quantileInterpolatedWeighted';
+SELECT medianInterpolatedWeighted(a, 1), medianInterpolatedWeighted(b, 2), medianInterpolatedWeighted(c, 3) as x, toTypeName(x) FROM decimal;
+SELECT quantileInterpolatedWeighted(a, 1), quantileInterpolatedWeighted(b, 2), quantileInterpolatedWeighted(c, 3) as x, toTypeName(x) FROM decimal WHERE a < 0;
+SELECT quantileInterpolatedWeighted(0.0)(a, 1), quantileInterpolatedWeighted(0.0)(b, 2), quantileInterpolatedWeighted(0.0)(c, 3) FROM decimal WHERE a >= 0;
+SELECT quantileInterpolatedWeighted(0.2)(a, 1), quantileInterpolatedWeighted(0.2)(b, 2), quantileInterpolatedWeighted(0.2)(c, 3) FROM decimal WHERE a >= 0;
+SELECT quantileInterpolatedWeighted(0.4)(a, 1), quantileInterpolatedWeighted(0.4)(b, 2), quantileInterpolatedWeighted(0.4)(c, 3) FROM decimal WHERE a >= 0;
+SELECT quantileInterpolatedWeighted(0.6)(a, 1), quantileInterpolatedWeighted(0.6)(b, 2), quantileInterpolatedWeighted(0.6)(c, 3) FROM decimal WHERE a >= 0;
+SELECT quantileInterpolatedWeighted(0.8)(a, 1), quantileInterpolatedWeighted(0.8)(b, 2), quantileInterpolatedWeighted(0.8)(c, 3) FROM decimal WHERE a >= 0;
+SELECT quantileInterpolatedWeighted(1.0)(a, 1), quantileInterpolatedWeighted(1.0)(b, 2), quantileInterpolatedWeighted(1.0)(c, 3) FROM decimal WHERE a >= 0;
+SELECT quantilesInterpolatedWeighted(0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0)(a, 1) FROM decimal;
+SELECT quantilesInterpolatedWeighted(0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0)(b, 2) FROM decimal;
+SELECT quantilesInterpolatedWeighted(0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0)(c, 3) FROM decimal;
+
+DROP TABLE IF EXISTS decimal;

From 46129a9bcd4782d612364a8add9304e5712437c9 Mon Sep 17 00:00:00 2001
From: bharatnc <bharatnc@gmail.com>
Date: Wed, 13 Jul 2022 17:52:50 -0700
Subject: [PATCH 010/158]  try simplifying limit checks

---
 .../QuantileInterpolatedWeighted.h            | 79 ++++---------------
 1 file changed, 16 insertions(+), 63 deletions(-)

diff --git a/src/AggregateFunctions/QuantileInterpolatedWeighted.h b/src/AggregateFunctions/QuantileInterpolatedWeighted.h
index a50bada8635..f2e00dac973 100644
--- a/src/AggregateFunctions/QuantileInterpolatedWeighted.h
+++ b/src/AggregateFunctions/QuantileInterpolatedWeighted.h
@@ -138,8 +138,6 @@ struct QuantileInterpolatedWeighted
 
 
         /// perform linear interpolation
-        UnderlyingType g;
-
         size_t idx = 0;
 
         if (size >= 2)
@@ -166,36 +164,7 @@ struct QuantileInterpolatedWeighted
         if (level > xr)
             yl = yr;
 
-        UnderlyingType dy;
-
-        if (yr - yl > std::numeric_limits<UnderlyingType>::max())
-            dy = std::numeric_limits<UnderlyingType>::max();
-        else if (yr - yl < std::numeric_limits<UnderlyingType>::min())
-            dy = std::numeric_limits<UnderlyingType>::min();
-        else
-            dy = yr - yl;
-
-        Float64 dx;
-
-        if (xr - xl > std::numeric_limits<Float64>::max())
-            dx = std::numeric_limits<Float64>::max();
-        else if (xr - xl < std::numeric_limits<Float64>::min())
-            dx = std::numeric_limits<Float64>::min();
-        else
-            dx = xr - xl;
-
-        dx = dx == 0 ? 1 : dx;
-
-        if ((yl + (dy / dx) * (level - xl)) > std::numeric_limits<UnderlyingType>::max())
-            g = std::numeric_limits<UnderlyingType>::max();
-        else if (yl + (dy / dx) * (level - xl) < std::numeric_limits<UnderlyingType>::min())
-            g = std::numeric_limits<UnderlyingType>::min();
-        else if (isNaN(yl + (dy / dx) * (level - xl)))
-            g = std::numeric_limits<UnderlyingType>::quiet_NaN();
-        else
-            g = yl + (dy / dx) * (level - xl);
-
-        return g;
+        return interpolate(level, xl, xr, yl, yr);
     }
 
 
@@ -262,7 +231,6 @@ struct QuantileInterpolatedWeighted
         while (level_index < num_levels)
         {
             /// perform linear interpolation for every level
-            UnderlyingType g;
             auto level = levels[indices[level_index]];
 
             size_t idx = 0;
@@ -290,40 +258,25 @@ struct QuantileInterpolatedWeighted
             if (level > xr)
                 yl = yr;
 
-            UnderlyingType dy;
-
-            if (yr - yl > std::numeric_limits<UnderlyingType>::max())
-                dy = std::numeric_limits<UnderlyingType>::max();
-            else if (yr - yl < std::numeric_limits<UnderlyingType>::min())
-                dy = std::numeric_limits<UnderlyingType>::min();
-            else
-                dy = yr - yl;
-
-            Float64 dx;
-
-            if (xr - xl > std::numeric_limits<Float64>::max())
-                dx = std::numeric_limits<Float64>::max();
-            else if (xr - xl < std::numeric_limits<Float64>::min())
-                dx = std::numeric_limits<Float64>::min();
-            else
-                dx = xr - xl;
-
-            dx = dx == 0 ? 1 : dx;
-
-            if ((yl + (dy / dx) * (level - xl)) > std::numeric_limits<UnderlyingType>::max())
-                g = std::numeric_limits<UnderlyingType>::max();
-            else if ((yl + (dy / dx) * (level - xl)) < std::numeric_limits<UnderlyingType>::min())
-                g = std::numeric_limits<UnderlyingType>::min();
-            else if (isNaN(yl + (dy / dx) * (level - xl)))
-                g = std::numeric_limits<UnderlyingType>::quiet_NaN();
-            else
-                g = yl + (dy / dx) * (level - xl);
-
-            result[indices[level_index]] = g;
+            result[indices[level_index]] = interpolate(level, xl, xr, yl, yr);
             ++level_index;
         }
     }
 
+    /// This ignores overflows that may arise during add, sub and mul operations and doesn't aim to provide exact results.
+    /// Since `the quantileInterpolatedWeighted` function itself relies mainly on approximation, this is fine.
+    UnderlyingType interpolate(Float64 level, Float64 xl, Float64 xr, UnderlyingType yl, UnderlyingType yr) const
+    {
+        UnderlyingType dy = common::subIgnoreOverflow(yr, yl);
+        Float64 dx = common::subIgnoreOverflow(xr, xl);
+        dx = dx == 0 ? 1 : dx; /// to handle NaN behavior that might arise during integer division below.
+
+        /// yl + (dy / dx) * (level - xl)
+        UnderlyingType g = common::addIgnoreOverflow(yl, common::mulIgnoreOverflow((dy / dx), common::subIgnoreOverflow(level, xl)));
+
+        return g;
+    }
+
     /// The same, but in the case of an empty state, NaN is returned.
     Float64 getFloat(Float64) const
     {

From a48f181d21b4bc834e608c4a9bcfbd660f3ce183 Mon Sep 17 00:00:00 2001
From: Bharat Nallan <bharatnc@gmail.com>
Date: Thu, 28 Jul 2022 08:55:57 -0700
Subject: [PATCH 011/158] Update
 src/AggregateFunctions/AggregateFunctionQuantile.cpp

Co-authored-by: Nikolay Degterinsky <43110995+evillique@users.noreply.github.com>
---
 src/AggregateFunctions/AggregateFunctionQuantile.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/AggregateFunctions/AggregateFunctionQuantile.cpp b/src/AggregateFunctions/AggregateFunctionQuantile.cpp
index 120fe3b6a59..96104ef68e8 100644
--- a/src/AggregateFunctions/AggregateFunctionQuantile.cpp
+++ b/src/AggregateFunctions/AggregateFunctionQuantile.cpp
@@ -88,7 +88,7 @@ constexpr bool supportBigInt()
         std::is_same_v<Function<Float32, false>, FuncQuantilesExact<Float32, false>> ||
         std::is_same_v<Function<Float32, false>, FuncQuantileExactWeighted<Float32, false>> ||
         std::is_same_v<Function<Float32, false>, FuncQuantilesExactWeighted<Float32, false>> ||
-        std::is_same_v<Function<Float32, false>, FuncQuantilesInterpolatedWeighted<Float32, false>> ||
+        std::is_same_v<Function<Float32, false>, FuncQuantileInterpolatedWeighted<Float32, false>> ||
         std::is_same_v<Function<Float32, false>, FuncQuantilesInterpolatedWeighted<Float32, false>>;
 }
 

From 86397a8f5d3d24e852bad6a861cd54a19f126807 Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Mon, 12 Sep 2022 18:26:43 +0200
Subject: [PATCH 012/158] Improve

---
 .../IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp | 11 +++++++++++
 src/Disks/IO/ReadBufferFromRemoteFSGather.cpp         |  2 +-
 src/Disks/ObjectStorages/DiskObjectStorage.cpp        |  7 ++++++-
 3 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp
index cbfcbdf7b88..0e50b064471 100644
--- a/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp
+++ b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp
@@ -164,6 +164,8 @@ bool AsynchronousReadIndirectBufferFromRemoteFS::nextImpl()
     if (!hasPendingDataToRead())
         return false;
 
+    assert(file_offset_of_buffer_end <= impl->getFileSize());
+
     Stopwatch watch;
     CurrentMetrics::Increment metric_increment{CurrentMetrics::AsynchronousReadWait};
 
@@ -292,6 +294,15 @@ off_t AsynchronousReadIndirectBufferFromRemoteFS::seek(off_t offset, int whence)
     /// First reset the buffer so the next read will fetch new data to the buffer.
     resetWorkingBuffer();
 
+    if (read_until_position && new_pos > *read_until_position)
+    {
+        ProfileEvents::increment(ProfileEvents::RemoteFSSeeksWithReset);
+        impl->reset();
+
+        file_offset_of_buffer_end = new_pos = *read_until_position; /// read_until_position is a non-included boundary.
+        return new_pos;
+    }
+
     /**
     * Lazy ignore. Save number of bytes to ignore and ignore it either for prefetch buffer or current buffer.
     * Note: we read in range [file_offset_of_buffer_end, read_until_position).
diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp
index 01d4154199a..712a9a7c3b1 100644
--- a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp
+++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp
@@ -256,7 +256,7 @@ size_t ReadBufferFromRemoteFSGather::getFileSize() const
 String ReadBufferFromRemoteFSGather::getInfoForLog()
 {
     if (!current_buf)
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot get info: buffer not initialized");
+        return "";
 
     return current_buf->getInfoForLog();
 }
diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.cpp b/src/Disks/ObjectStorages/DiskObjectStorage.cpp
index db8f90e777d..24c001b3046 100644
--- a/src/Disks/ObjectStorages/DiskObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/DiskObjectStorage.cpp
@@ -520,11 +520,16 @@ std::unique_ptr<ReadBufferFromFileBase> DiskObjectStorage::readFile(
     std::optional<size_t> read_hint,
     std::optional<size_t> file_size) const
 {
-    return object_storage->readObjects(
+    auto result = object_storage->readObjects(
         metadata_storage->getStorageObjects(path),
         object_storage->getAdjustedSettingsFromMetadataFile(settings, path),
         read_hint,
         file_size);
+
+    if (read_hint)
+        result->setReadUntilPosition(*read_hint);
+
+    return result;
 }
 
 std::unique_ptr<WriteBufferFromFileBase> DiskObjectStorage::writeFile(

From 81f369ac2b056557c5fa709d5c3b1cb546baa859 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Fri, 7 Oct 2022 19:53:54 +0200
Subject: [PATCH 013/158] Add loging during server startup

---
 src/Storages/MergeTree/MergeTreeData.cpp | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 4a7d2b2dd63..83bedb34344 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -954,6 +954,8 @@ void MergeTreeData::loadDataPartsFromDisk(
     /// Parallel loading of data parts.
     pool.setMaxThreads(std::min(static_cast<size_t>(settings->max_part_loading_threads), num_parts));
     size_t num_threads = pool.getMaxThreads();
+    LOG_DEBUG(log, "Going to use {} threads to load parts", num_threads);
+
     std::vector<size_t> parts_per_thread(num_threads, num_parts / num_threads);
     for (size_t i = 0ul; i < num_parts % num_threads; ++i)
         ++parts_per_thread[i];
@@ -1016,6 +1018,8 @@ void MergeTreeData::loadDataPartsFromDisk(
         auto part_opt = MergeTreePartInfo::tryParsePartName(part_name, format_version);
         if (!part_opt)
             return;
+
+        LOG_TRACE(log, "Loading part {} from disk {}", part_name, part_disk_ptr->getName());
         const auto & part_info = *part_opt;
         auto single_disk_volume = std::make_shared<SingleDiskVolume>("volume_" + part_name, part_disk_ptr, 0);
         auto data_part_storage = std::make_shared<DataPartStorageOnDisk>(single_disk_volume, relative_data_path, part_name);
@@ -1119,6 +1123,7 @@ void MergeTreeData::loadDataPartsFromDisk(
         }
 
         addPartContributionToDataVolume(part);
+        LOG_TRACE(log, "Finished part {} load on disk {}", part_name, part_disk_ptr->getName());
     };
 
     std::mutex part_select_mutex;
@@ -1311,8 +1316,10 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
 
     size_t num_parts = 0;
     std::queue<std::vector<std::pair<String, DiskPtr>>> parts_queue;
-    for (auto & [_, disk_parts] : disk_part_map)
+    for (auto & [disk_name, disk_parts] : disk_part_map)
     {
+        LOG_INFO(log, "Found {} parts for disk '{}' to load", disk_name, disk_parts.size());
+
         if (disk_parts.empty())
             continue;
         num_parts += disk_parts.size();

From 9f1104c2ad01b99455bd2270aeddd2dc2fea8dfe Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Sat, 8 Oct 2022 01:39:29 +0000
Subject: [PATCH 014/158] do not load inactive parts

---
 src/Storages/MergeTree/MergeTreeData.cpp      | 146 +++++++++++++----
 src/Storages/MergeTree/MergeTreeData.h        |  31 +++-
 .../test_merge_tree_load_parts/__init__.py    |   0
 .../test_merge_tree_load_parts/test.py        | 147 ++++++++++++++++++
 4 files changed, 297 insertions(+), 27 deletions(-)
 create mode 100644 tests/integration/test_merge_tree_load_parts/__init__.py
 create mode 100644 tests/integration/test_merge_tree_load_parts/test.py

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 83bedb34344..afe32833cd1 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -942,12 +942,77 @@ Int64 MergeTreeData::getMaxBlockNumber() const
     return max_block_num;
 }
 
+void MergeTreeData::PartLoadingTree::add(const MergeTreePartInfo & info, const DiskPtr & disk)
+{
+    auto & current_ptr = root_by_partition[info.partition_id];
+    if (!current_ptr)
+        current_ptr = std::make_shared<Node>(MergeTreePartInfo{}, disk);
+
+    auto * current = current_ptr.get();
+    while (true)
+    {
+        auto it = current->children.lower_bound(info);
+        if (it != current->children.begin())
+        {
+            auto prev = std::prev(it);
+            const auto & prev_info = prev->first;
+
+            if (prev_info.contains(info))
+            {
+                current = prev->second.get();
+                continue;
+            }
+            else if (!prev_info.isDisjoint(info))
+            {
+                throw Exception(ErrorCodes::LOGICAL_ERROR,
+                    "Part {} intersects previous part {}. It is a bug!",
+                    info.getPartName(), prev_info.getPartName());
+            }
+        }
+
+        if (it != current->children.end())
+        {
+            const auto & next_info = it->first;
+
+            if (next_info.contains(info))
+            {
+                current = it->second.get();
+                continue;
+            }
+            else if (!next_info.isDisjoint(info))
+            {
+                throw Exception(ErrorCodes::LOGICAL_ERROR,
+                    "Part {} intersects next part {}. It is a bug!",
+                    info.getPartName(), next_info.getPartName());
+            }
+        }
+
+        current->children.emplace(info, std::make_shared<Node>(info, disk));
+        break;
+    }
+}
+
+MergeTreeData::PartLoadingTree
+MergeTreeData::PartLoadingTree::build(std::vector<PartInfoWithDisk> nodes)
+{
+    std::sort(nodes.begin(), nodes.end(),
+        [](const auto & lhs, const auto & rhs)
+        {
+            return lhs.first.level > rhs.first.level;
+        });
+
+    PartLoadingTree tree;
+    for (const auto & [info, disk] : nodes)
+        tree.add(info, disk);
+    return tree;
+}
+
 void MergeTreeData::loadDataPartsFromDisk(
     DataPartsVector & broken_parts_to_detach,
     DataPartsVector & duplicate_parts_to_remove,
     ThreadPool & pool,
     size_t num_parts,
-    std::queue<std::vector<std::pair<String, DiskPtr>>> & parts_queue,
+    std::queue<PartLoadingTreeNodes> & parts_queue,
     bool skip_sanity_checks,
     const MergeTreeSettingsPtr & settings)
 {
@@ -962,9 +1027,10 @@ void MergeTreeData::loadDataPartsFromDisk(
 
     /// Prepare data parts for parallel loading. Threads will focus on given disk first, then steal
     /// others' tasks when finish current disk part loading process.
-    std::vector<std::vector<std::pair<String, DiskPtr>>> threads_parts(num_threads);
+    std::vector<PartLoadingTreeNodes> threads_parts(num_threads);
     std::set<size_t> remaining_thread_parts;
     std::queue<size_t> threads_queue;
+
     for (size_t i = 0; i < num_threads; ++i)
     {
         remaining_thread_parts.insert(i);
@@ -977,11 +1043,12 @@ void MergeTreeData::loadDataPartsFromDisk(
         size_t i = threads_queue.front();
         auto & need_parts = parts_per_thread[i];
         assert(need_parts > 0);
+
         auto & thread_parts = threads_parts[i];
         auto & current_parts = parts_queue.front();
         assert(!current_parts.empty());
-        auto parts_to_grab = std::min(need_parts, current_parts.size());
 
+        auto parts_to_grab = std::min(need_parts, current_parts.size());
         thread_parts.insert(thread_parts.end(), current_parts.end() - parts_to_grab, current_parts.end());
         current_parts.resize(current_parts.size() - parts_to_grab);
         need_parts -= parts_to_grab;
@@ -993,13 +1060,16 @@ void MergeTreeData::loadDataPartsFromDisk(
         /// If current disk still has some parts, push it to the tail.
         if (!current_parts.empty())
             parts_queue.push(std::move(current_parts));
+
         parts_queue.pop();
 
         /// If current thread still want some parts, push it to the tail.
         if (need_parts > 0)
             threads_queue.push(i);
+
         threads_queue.pop();
     }
+
     assert(threads_queue.empty());
     assert(std::all_of(threads_parts.begin(), threads_parts.end(), [](const std::vector<std::pair<String, DiskPtr>> & parts)
     {
@@ -1011,16 +1081,13 @@ void MergeTreeData::loadDataPartsFromDisk(
     std::atomic<bool> has_adaptive_parts = false;
     std::atomic<bool> has_non_adaptive_parts = false;
     std::atomic<bool> has_lightweight_in_parts = false;
+    std::mutex loading_mutex;
 
-    std::mutex mutex;
-    auto load_part = [&](const String & part_name, const DiskPtr & part_disk_ptr)
+    auto load_part = [&](const MergeTreePartInfo & part_info, const DiskPtr & part_disk_ptr)
     {
-        auto part_opt = MergeTreePartInfo::tryParsePartName(part_name, format_version);
-        if (!part_opt)
-            return;
-
+        auto part_name = part_info.getPartName();
         LOG_TRACE(log, "Loading part {} from disk {}", part_name, part_disk_ptr->getName());
-        const auto & part_info = *part_opt;
+
         auto single_disk_volume = std::make_shared<SingleDiskVolume>("volume_" + part_name, part_disk_ptr, 0);
         auto data_part_storage = std::make_shared<DataPartStorageOnDisk>(single_disk_volume, relative_data_path, part_name);
         auto part = createPart(part_name, part_info, data_part_storage);
@@ -1028,6 +1095,7 @@ void MergeTreeData::loadDataPartsFromDisk(
 
         String part_path = fs::path(relative_data_path) / part_name;
         String marker_path = fs::path(part_path) / IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME;
+
         if (part_disk_ptr->exists(marker_path))
         {
             /// NOTE: getBytesOnDisk() cannot be used here, since it maybe zero of checksums.txt will not exist
@@ -1036,11 +1104,13 @@ void MergeTreeData::loadDataPartsFromDisk(
                 "Detaching stale part {}{} (size: {}), which should have been deleted after a move. "
                 "That can only happen after unclean restart of ClickHouse after move of a part having an operation blocking that stale copy of part.",
                 getFullPathOnDisk(part_disk_ptr), part_name, formatReadableSizeWithBinarySuffix(size_of_part));
-            std::lock_guard loading_lock(mutex);
+
+            std::lock_guard loading_lock(loading_mutex);
+
             broken_parts_to_detach.push_back(part);
             ++suspicious_broken_parts;
             suspicious_broken_parts_bytes += size_of_part;
-            return;
+            return false;
         }
 
         try
@@ -1082,19 +1152,21 @@ void MergeTreeData::loadDataPartsFromDisk(
             if (size_of_part.has_value())
                 part_size_str = formatReadableSizeWithBinarySuffix(*size_of_part);
 
-
             LOG_ERROR(log,
                 "Detaching broken part {}{} (size: {}). "
                 "If it happened after update, it is likely because of backward incompatibility. "
                 "You need to resolve this manually",
                 getFullPathOnDisk(part_disk_ptr), part_name, part_size_str);
-            std::lock_guard loading_lock(mutex);
+
+            std::lock_guard loading_lock(loading_mutex);
+
             broken_parts_to_detach.push_back(part);
             ++suspicious_broken_parts;
             if (size_of_part.has_value())
                 suspicious_broken_parts_bytes += *size_of_part;
-            return;
+            return false;
         }
+
         if (!part->index_granularity_info.mark_type.adaptive)
             has_non_adaptive_parts.store(true, std::memory_order_relaxed);
         else
@@ -1108,8 +1180,9 @@ void MergeTreeData::loadDataPartsFromDisk(
         /// Assume that all parts are Active, covered parts will be detected and marked as Outdated later
         part->setState(DataPartState::Active);
 
-        std::lock_guard loading_lock(mutex);
+        std::lock_guard loading_lock(loading_mutex);
         auto [it, inserted] = data_parts_indexes.insert(part);
+
         /// Remove duplicate parts with the same checksum.
         if (!inserted)
         {
@@ -1124,6 +1197,7 @@ void MergeTreeData::loadDataPartsFromDisk(
 
         addPartContributionToDataVolume(part);
         LOG_TRACE(log, "Finished part {} load on disk {}", part_name, part_disk_ptr->getName());
+        return true;
     };
 
     std::mutex part_select_mutex;
@@ -1135,15 +1209,16 @@ void MergeTreeData::loadDataPartsFromDisk(
             {
                 while (true)
                 {
-                    std::pair<String, DiskPtr> thread_part;
+                    PartLoadingTree::NodePtr thread_part;
+                    size_t thread_idx = thread;
+
                     {
-                        const std::lock_guard lock{part_select_mutex};
+                        std::lock_guard lock{part_select_mutex};
 
                         if (remaining_thread_parts.empty())
                             return;
 
                         /// Steal task if nothing to do
-                        auto thread_idx = thread;
                         if (threads_parts[thread].empty())
                         {
                             // Try random steal tasks from the next thread
@@ -1152,13 +1227,23 @@ void MergeTreeData::loadDataPartsFromDisk(
                             std::advance(it, distribution(thread_local_rng));
                             thread_idx = *it;
                         }
+
                         auto & thread_parts = threads_parts[thread_idx];
                         thread_part = thread_parts.back();
                         thread_parts.pop_back();
                         if (thread_parts.empty())
                             remaining_thread_parts.erase(thread_idx);
                     }
-                    load_part(thread_part.first, thread_part.second);
+
+                    bool part_is_ok = load_part(thread_part->info, thread_part->disk);
+
+                    if (!part_is_ok)
+                    {
+                        std::lock_guard lock{part_select_mutex};
+                        for (const auto & [_, node] : thread_part->children)
+                            threads_parts[thread].push_back(node);
+                        remaining_thread_parts.insert(thread);
+                    }
                 }
             });
         }
@@ -1287,17 +1372,14 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
         }
     }
 
-    /// Collect part names by disk.
-    std::map<String, std::vector<std::pair<String, DiskPtr>>> disk_part_map;
     ThreadPool pool(disks.size());
+    std::vector<PartLoadingTree::PartInfoWithDisk> parts_with_disks;
 
     for (const auto & disk_ptr : disks)
     {
         if (disk_ptr->isBroken())
             continue;
 
-        auto & disk_parts = disk_part_map[disk_ptr->getName()];
-
         pool.scheduleOrThrowOnError([&, disk_ptr]()
         {
             for (auto it = disk_ptr->iterateDirectory(relative_data_path); it->isValid(); it->next())
@@ -1308,20 +1390,32 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
                     continue;
 
                 if (!startsWith(it->name(), MergeTreeWriteAheadLog::WAL_FILE_NAME))
-                    disk_parts.emplace_back(std::make_pair(it->name(), disk_ptr));
+                    if (auto part_info = MergeTreePartInfo::tryParsePartName(it->name(), format_version))
+                        parts_with_disks.emplace_back(*part_info, disk_ptr);
             }
         });
     }
+
     pool.wait();
 
+    auto loading_tree = PartLoadingTree::build(std::move(parts_with_disks));
+    /// Collect parts by disks' names.
+    std::map<String, PartLoadingTreeNodes> disk_part_map;
+
+    for (const auto & elem : loading_tree.getRoots())
+        for (const auto & [_, node] : elem.second->children)
+            disk_part_map[node->disk->getName()].emplace_back(node);
+
     size_t num_parts = 0;
-    std::queue<std::vector<std::pair<String, DiskPtr>>> parts_queue;
+    std::queue<PartLoadingTreeNodes> parts_queue;
+
     for (auto & [disk_name, disk_parts] : disk_part_map)
     {
         LOG_INFO(log, "Found {} parts for disk '{}' to load", disk_name, disk_parts.size());
 
         if (disk_parts.empty())
             continue;
+
         num_parts += disk_parts.size();
         parts_queue.push(std::move(disk_parts));
     }
diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h
index c3a70a9893b..54a8e4fffe8 100644
--- a/src/Storages/MergeTree/MergeTreeData.h
+++ b/src/Storages/MergeTree/MergeTreeData.h
@@ -1329,12 +1329,41 @@ private:
     /// Returns default settings for storage with possible changes from global config.
     virtual std::unique_ptr<MergeTreeSettings> getDefaultSettings() const = 0;
 
+    class PartLoadingTree
+    {
+    public:
+        struct Node
+        {
+            Node(const MergeTreePartInfo & info_, const DiskPtr & disk_)
+                :  info(info_), disk(disk_)
+            {
+            }
+
+            MergeTreePartInfo info;
+            DiskPtr disk;
+
+            std::map<MergeTreePartInfo, std::shared_ptr<Node>> children;
+        };
+
+        using NodePtr = std::shared_ptr<Node>;
+        using PartInfoWithDisk = std::pair<MergeTreePartInfo, DiskPtr>;
+
+        static PartLoadingTree build(std::vector<PartInfoWithDisk> nodes);
+        void add(const MergeTreePartInfo & info, const DiskPtr & disk);
+        const std::unordered_map<String, NodePtr> & getRoots() const { return root_by_partition; }
+
+    private:
+        std::unordered_map<String, NodePtr> root_by_partition;
+    };
+
+    using PartLoadingTreeNodes = std::vector<PartLoadingTree::NodePtr>;
+
     void loadDataPartsFromDisk(
         DataPartsVector & broken_parts_to_detach,
         DataPartsVector & duplicate_parts_to_remove,
         ThreadPool & pool,
         size_t num_parts,
-        std::queue<std::vector<std::pair<String, DiskPtr>>> & parts_queue,
+        std::queue<PartLoadingTreeNodes> & parts_queue,
         bool skip_sanity_checks,
         const MergeTreeSettingsPtr & settings);
 
diff --git a/tests/integration/test_merge_tree_load_parts/__init__.py b/tests/integration/test_merge_tree_load_parts/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/integration/test_merge_tree_load_parts/test.py b/tests/integration/test_merge_tree_load_parts/test.py
new file mode 100644
index 00000000000..cb0d902f677
--- /dev/null
+++ b/tests/integration/test_merge_tree_load_parts/test.py
@@ -0,0 +1,147 @@
+import pytest
+import helpers.client
+import helpers.cluster
+from helpers.corrupt_part_data_on_disk import corrupt_part_data_on_disk
+
+
+cluster = helpers.cluster.ClickHouseCluster(__file__)
+node1 = cluster.add_instance("node1", with_zookeeper=True, stay_alive=True)
+node2 = cluster.add_instance("node2", with_zookeeper=True, stay_alive=True)
+
+
+@pytest.fixture(scope="module")
+def started_cluster():
+    try:
+        cluster.start()
+        yield cluster
+
+    finally:
+        cluster.shutdown()
+
+
+def test_merge_tree_load_parts(started_cluster):
+    node1.query(
+        """
+        CREATE TABLE mt_load_parts (pk UInt32, id UInt32, s String)
+        ENGINE = MergeTree ORDER BY id PARTITION BY pk"""
+    )
+
+    node1.query("SYSTEM STOP MERGES mt_load_parts")
+
+    for i in range(20):
+        node1.query(
+            f"INSERT INTO mt_load_parts VALUES (44, {i}, randomPrintableASCII(10))"
+        )
+
+    node1.restart_clickhouse(kill=True)
+    for i in range(1, 21):
+        assert node1.contains_in_log(f"Loading part 44_{i}_{i}_0")
+
+    node1.query("OPTIMIZE TABLE mt_load_parts FINAL")
+    node1.restart_clickhouse(kill=True)
+
+    assert node1.contains_in_log("Loading part 44_1_20")
+    for i in range(1, 21):
+        assert not node1.contains_in_log(f"Loading part 44_{i}_{i}_0")
+
+    assert node1.query("SELECT count() FROM mt_load_parts") == "20\n"
+    assert (
+        node1.query(
+            "SELECT count() FROM system.parts WHERE table = 'mt_load_parts' AND active"
+        )
+        == "1\n"
+    )
+
+
+def test_merge_tree_load_parts_corrupted(started_cluster):
+    for i, node in enumerate([node1, node2]):
+        node.query(
+            f"""
+            CREATE TABLE mt_load_parts_2 (pk UInt32, id UInt32, s String)
+            ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/mt_load_parts_2', '{i}') ORDER BY id PARTITION BY pk"""
+        )
+
+    """min-max blocks in created parts: 1_1_0, 2_2_0, 1_2_1, 3_3_0, 1_3_2"""
+    for partition in [111, 222, 333]:
+        node1.query(
+            f"INSERT INTO mt_load_parts_2 VALUES ({partition}, 0, randomPrintableASCII(10))"
+        )
+        node1.query(
+            f"INSERT INTO mt_load_parts_2 VALUES ({partition}, 1, randomPrintableASCII(10))"
+        )
+
+        node1.query(f"OPTIMIZE TABLE mt_load_parts_2 PARTITION {partition} FINAL")
+
+        node1.query(
+            f"INSERT INTO mt_load_parts_2 VALUES ({partition}, 2, randomPrintableASCII(10))"
+        )
+
+        node1.query(f"OPTIMIZE TABLE mt_load_parts_2 PARTITION {partition} FINAL")
+
+    node2.query("SYSTEM SYNC REPLICA mt_load_parts_2", timeout=30)
+
+    def get_part_name(node, partition, min_block, max_block):
+        return node.query(
+            f"""
+            SELECT name FROM system.parts
+            WHERE table = 'mt_load_parts_2'
+            AND partition = '{partition}'
+            AND min_block_number = {min_block}
+            AND max_block_number = {max_block}"""
+        ).strip()
+
+    corrupt_part_data_on_disk(node1, "mt_load_parts_2", get_part_name(node1, 111, 0, 2))
+    corrupt_part_data_on_disk(node1, "mt_load_parts_2", get_part_name(node1, 222, 0, 2))
+    corrupt_part_data_on_disk(node1, "mt_load_parts_2", get_part_name(node1, 222, 0, 1))
+    corrupt_part_data_on_disk(node1, "mt_load_parts_2", get_part_name(node1, 333, 0, 1))
+    corrupt_part_data_on_disk(node1, "mt_load_parts_2", get_part_name(node1, 333, 2, 2))
+
+    node1.restart_clickhouse(kill=True)
+
+    def check_parts_loading(node, partition, loaded, failed, skipped):
+        for (min_block, max_block) in loaded:
+            part_name = f"{partition}_{min_block}_{max_block}"
+            assert node.contains_in_log(f"Loading part {part_name}")
+            assert node.contains_in_log(f"Finished part {part_name}")
+
+        for (min_block, max_block) in failed:
+            part_name = f"{partition}_{min_block}_{max_block}"
+            assert node.contains_in_log(f"Loading part {part_name}")
+            assert not node.contains_in_log(f"Finished part {part_name}")
+
+        for (min_block, max_block) in skipped:
+            part_name = f"{partition}_{min_block}_{max_block}"
+            assert not node.contains_in_log(f"Loading part {part_name}")
+            assert not node.contains_in_log(f"Finished part {part_name}")
+
+    check_parts_loading(
+        node1, 111, loaded=[(0, 1), (2, 2)], failed=[(0, 2)], skipped=[(0, 0), (1, 1)]
+    )
+    check_parts_loading(
+        node1, 222, loaded=[(0, 0), (1, 1), (2, 2)], failed=[(0, 2), (0, 1)], skipped=[]
+    )
+    check_parts_loading(
+        node1, 333, loaded=[(0, 2)], failed=[], skipped=[(0, 0), (1, 1), (2, 2), (0, 1)]
+    )
+
+    node1.query("SYSTEM SYNC REPLICA mt_load_parts_2", timeout=30)
+    node1.query("OPTIMIZE TABLE mt_load_parts_2 FINAL")
+    node1.query("SYSTEM SYNC REPLICA mt_load_parts_2", timeout=30)
+
+    assert (
+        node1.query(
+            """
+            SELECT pk, count() FROM mt_load_parts_2
+            GROUP BY pk ORDER BY pk"""
+        )
+        == "111\t3\n222\t3\n333\t3\n"
+    )
+    assert (
+        node1.query(
+            """
+            SELECT partition, count()
+            FROM system.parts WHERE table = 'mt_load_parts_2' AND active
+            GROUP BY partition ORDER BY partition"""
+        )
+        == "111\t1\n222\t1\n333\t1\n"
+    )

From 286da7ee19a9a05e0956fa580bf072284ebcfd4f Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Sun, 9 Oct 2022 15:39:20 +0000
Subject: [PATCH 015/158] fix loading of parts with old syntax

---
 src/Storages/MergeTree/MergeTreeData.cpp | 38 ++++++++++++------------
 src/Storages/MergeTree/MergeTreeData.h   | 15 +++++-----
 2 files changed, 27 insertions(+), 26 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index afe32833cd1..e7bdb8b5b3a 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -942,11 +942,11 @@ Int64 MergeTreeData::getMaxBlockNumber() const
     return max_block_num;
 }
 
-void MergeTreeData::PartLoadingTree::add(const MergeTreePartInfo & info, const DiskPtr & disk)
+void MergeTreeData::PartLoadingTree::add(const MergeTreePartInfo & info, const String & name, const DiskPtr & disk)
 {
     auto & current_ptr = root_by_partition[info.partition_id];
     if (!current_ptr)
-        current_ptr = std::make_shared<Node>(MergeTreePartInfo{}, disk);
+        current_ptr = std::make_shared<Node>(MergeTreePartInfo{}, "", disk);
 
     auto * current = current_ptr.get();
     while (true)
@@ -987,23 +987,24 @@ void MergeTreeData::PartLoadingTree::add(const MergeTreePartInfo & info, const D
             }
         }
 
-        current->children.emplace(info, std::make_shared<Node>(info, disk));
+        current->children.emplace(info, std::make_shared<Node>(info, name, disk));
         break;
     }
 }
 
 MergeTreeData::PartLoadingTree
-MergeTreeData::PartLoadingTree::build(std::vector<PartInfoWithDisk> nodes)
+MergeTreeData::PartLoadingTree::build(std::vector<PartLoadingInfo> nodes)
 {
-    std::sort(nodes.begin(), nodes.end(),
-        [](const auto & lhs, const auto & rhs)
-        {
-            return lhs.first.level > rhs.first.level;
-        });
+    std::sort(nodes.begin(), nodes.end(), [](const auto & lhs, const auto & rhs)
+    {
+        const auto & lhs_info = std::get<0>(lhs);
+        const auto & rhs_info = std::get<0>(rhs);
+        return std::tie(lhs_info.level, lhs_info.mutation) > std::tie(rhs_info.level, rhs_info.mutation);
+    });
 
     PartLoadingTree tree;
-    for (const auto & [info, disk] : nodes)
-        tree.add(info, disk);
+    for (const auto & [info, name, disk] : nodes)
+        tree.add(info, name, disk);
     return tree;
 }
 
@@ -1083,9 +1084,8 @@ void MergeTreeData::loadDataPartsFromDisk(
     std::atomic<bool> has_lightweight_in_parts = false;
     std::mutex loading_mutex;
 
-    auto load_part = [&](const MergeTreePartInfo & part_info, const DiskPtr & part_disk_ptr)
+    auto load_part = [&](const MergeTreePartInfo & part_info, const String & part_name, const DiskPtr & part_disk_ptr)
     {
-        auto part_name = part_info.getPartName();
         LOG_TRACE(log, "Loading part {} from disk {}", part_name, part_disk_ptr->getName());
 
         auto single_disk_volume = std::make_shared<SingleDiskVolume>("volume_" + part_name, part_disk_ptr, 0);
@@ -1235,7 +1235,7 @@ void MergeTreeData::loadDataPartsFromDisk(
                             remaining_thread_parts.erase(thread_idx);
                     }
 
-                    bool part_is_ok = load_part(thread_part->info, thread_part->disk);
+                    bool part_is_ok = load_part(thread_part->info, thread_part->name, thread_part->disk);
 
                     if (!part_is_ok)
                     {
@@ -1373,7 +1373,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
     }
 
     ThreadPool pool(disks.size());
-    std::vector<PartLoadingTree::PartInfoWithDisk> parts_with_disks;
+    std::vector<PartLoadingTree::PartLoadingInfo> parts_with_disks;
 
     for (const auto & disk_ptr : disks)
     {
@@ -1386,12 +1386,12 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
             {
                 /// Skip temporary directories, file 'format_version.txt' and directory 'detached'.
                 if (startsWith(it->name(), "tmp") || it->name() == MergeTreeData::FORMAT_VERSION_FILE_NAME
-                    || it->name() == MergeTreeData::DETACHED_DIR_NAME)
+                    || it->name() == MergeTreeData::DETACHED_DIR_NAME
+                    || startsWith(it->name(), MergeTreeWriteAheadLog::WAL_FILE_NAME))
                     continue;
 
-                if (!startsWith(it->name(), MergeTreeWriteAheadLog::WAL_FILE_NAME))
-                    if (auto part_info = MergeTreePartInfo::tryParsePartName(it->name(), format_version))
-                        parts_with_disks.emplace_back(*part_info, disk_ptr);
+                if (auto part_info = MergeTreePartInfo::tryParsePartName(it->name(), format_version))
+                    parts_with_disks.emplace_back(*part_info, it->name(), disk_ptr);
             }
         });
     }
diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h
index 54a8e4fffe8..2ca379df3f4 100644
--- a/src/Storages/MergeTree/MergeTreeData.h
+++ b/src/Storages/MergeTree/MergeTreeData.h
@@ -1334,22 +1334,23 @@ private:
     public:
         struct Node
         {
-            Node(const MergeTreePartInfo & info_, const DiskPtr & disk_)
-                :  info(info_), disk(disk_)
+            Node(const MergeTreePartInfo & info_, const String & name_, const DiskPtr & disk_)
+                : info(info_), name(name_), disk(disk_)
             {
             }
 
-            MergeTreePartInfo info;
-            DiskPtr disk;
+            const MergeTreePartInfo info;
+            const String name;
+            const DiskPtr disk;
 
             std::map<MergeTreePartInfo, std::shared_ptr<Node>> children;
         };
 
         using NodePtr = std::shared_ptr<Node>;
-        using PartInfoWithDisk = std::pair<MergeTreePartInfo, DiskPtr>;
+        using PartLoadingInfo = std::tuple<MergeTreePartInfo, String, DiskPtr>;
 
-        static PartLoadingTree build(std::vector<PartInfoWithDisk> nodes);
-        void add(const MergeTreePartInfo & info, const DiskPtr & disk);
+        static PartLoadingTree build(std::vector<PartLoadingInfo> nodes);
+        void add(const MergeTreePartInfo & info, const String & name, const DiskPtr & disk);
         const std::unordered_map<String, NodePtr> & getRoots() const { return root_by_partition; }
 
     private:

From 95fdb0a877c4b9d9308ceea6c264e36a1c616a92 Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Fri, 14 Oct 2022 14:52:57 +0000
Subject: [PATCH 016/158] remove outdated parts asynchronously

---
 src/Storages/MergeTree/IMergeTreeDataPart.cpp |   1 -
 src/Storages/MergeTree/MergeTreeData.cpp      | 700 +++++++++---------
 src/Storages/MergeTree/MergeTreeData.h        |  93 ++-
 .../ReplicatedMergeTreeRestartingThread.cpp   |   3 +
 src/Storages/StorageMergeTree.cpp             |   3 +-
 .../configs/fast_background_pool.xml          |   9 +
 .../test_merge_tree_load_parts/test.py        |  73 +-
 7 files changed, 497 insertions(+), 385 deletions(-)
 create mode 100644 tests/integration/test_merge_tree_load_parts/configs/fast_background_pool.xml

diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp
index 46323f12305..d8dda924b15 100644
--- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp
+++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp
@@ -1460,7 +1460,6 @@ void IMergeTreeDataPart::remove() const
         return CanRemoveDescription{.can_remove_anything = can_remove, .files_not_to_remove = files_not_to_remove };
     };
 
-
     if (!isStoredOnDisk())
         return;
 
diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 8c787e7c45a..1916db740be 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -992,6 +992,22 @@ void MergeTreeData::PartLoadingTree::add(const MergeTreePartInfo & info, const S
     }
 }
 
+template <typename Func>
+void MergeTreeData::PartLoadingTree::traverse(bool recursive, Func && func)
+{
+    std::function<void(const NodePtr &)> traverse_impl = [&](const auto & node)
+    {
+        func(node);
+        if (recursive)
+            for (const auto & [_, child] : node->children)
+                traverse_impl(child);
+    };
+
+    for (const auto & elem : root_by_partition)
+        for (const auto & [_, node] : elem.second->children)
+            traverse_impl(node);
+}
+
 MergeTreeData::PartLoadingTree
 MergeTreeData::PartLoadingTree::build(std::vector<PartLoadingInfo> nodes)
 {
@@ -1008,13 +1024,215 @@ MergeTreeData::PartLoadingTree::build(std::vector<PartLoadingInfo> nodes)
     return tree;
 }
 
-void MergeTreeData::loadDataPartsFromDisk(
-    DataPartsVector & broken_parts_to_detach,
-    DataPartsVector & duplicate_parts_to_remove,
+static std::optional<size_t> calculatePartSizeSafe(
+    const MergeTreeData::DataPartPtr & part, Poco::Logger * log)
+{
+    try
+    {
+        return part->data_part_storage->calculateTotalSizeOnDisk();
+    }
+    catch (...)
+    {
+        tryLogCurrentException(log, fmt::format("while calculating part size {} on path {}",
+            part->name, part->data_part_storage->getRelativePath()));
+        return {};
+    }
+}
+
+static void preparePartForRemoval(const MergeTreeMutableDataPartPtr & part)
+{
+    part->remove_time.store(part->modification_time, std::memory_order_relaxed);
+    auto creation_csn = part->version.creation_csn.load(std::memory_order_relaxed);
+    if (creation_csn != Tx::RolledBackCSN && creation_csn != Tx::PrehistoricCSN && !part->version.isRemovalTIDLocked())
+    {
+        /// It's possible that covering part was created without transaction,
+        /// but if covered part was created with transaction (i.e. creation_tid is not prehistoric),
+        /// then it must have removal tid in metadata file.
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Data part {} is Outdated and has creation TID {} and CSN {}, "
+                        "but does not have removal tid. It's a bug or a result of manual intervention.",
+                        part->name, part->version.creation_tid, creation_csn);
+    }
+
+    /// Explicitly set removal_tid_lock for parts w/o transaction (i.e. w/o txn_version.txt)
+    /// to avoid keeping part forever (see VersionMetadata::canBeRemoved())
+    if (!part->version.isRemovalTIDLocked())
+    {
+        TransactionInfoContext transaction_context{part->storage.getStorageID(), part->name};
+        part->version.lockRemovalTID(Tx::PrehistoricTID, transaction_context);
+    }
+}
+
+MergeTreeData::LoadPartResult MergeTreeData::loadDataPart(
+    const MergeTreePartInfo & part_info,
+    const String & part_name,
+    const DiskPtr & part_disk_ptr,
+    MergeTreeDataPartState to_state,
+    std::mutex & part_loading_mutex)
+{
+    LOG_TRACE(log, "Loading {} part {} from disk {}", magic_enum::enum_name(to_state), part_name, part_disk_ptr->getName());
+
+    LoadPartResult res;
+    auto single_disk_volume = std::make_shared<SingleDiskVolume>("volume_" + part_name, part_disk_ptr, 0);
+    auto data_part_storage = std::make_shared<DataPartStorageOnDisk>(single_disk_volume, relative_data_path, part_name);
+
+    res.part = createPart(part_name, part_info, data_part_storage);
+
+    String part_path = fs::path(relative_data_path) / part_name;
+    String marker_path = fs::path(part_path) / IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME;
+
+    if (part_disk_ptr->exists(marker_path))
+    {
+        /// NOTE: getBytesOnDisk() cannot be used here, since it maybe zero of checksums.txt will not exist.
+        res.size_of_part = calculatePartSizeSafe(res.part, log);
+        res.is_broken = true;
+
+        auto part_size_str = res.size_of_part ? formatReadableSizeWithBinarySuffix(*res.size_of_part) : "failed to calculate size";
+
+        LOG_WARNING(log,
+            "Detaching stale part {} (size: {}), which should have been deleted after a move. "
+            "That can only happen after unclean restart of ClickHouse after move of a part having an operation blocking that stale copy of part.",
+            res.part->data_part_storage->getFullPath(), part_size_str);
+
+        return res;
+    }
+
+    try
+    {
+        res.part->loadColumnsChecksumsIndexes(require_part_metadata, true);
+    }
+    catch (const Exception & e)
+    {
+        /// Don't count the part as broken if there is not enough memory to load it.
+        /// In fact, there can be many similar situations.
+        /// But it is OK, because there is a safety guard against deleting too many parts.
+        if (isNotEnoughMemoryErrorCode(e.code()))
+            throw;
+
+        res.is_broken = true;
+        tryLogCurrentException(log, fmt::format("while loading part {} on path {}", res.part->name, part_path));
+    }
+    catch (...)
+    {
+        res.is_broken = true;
+        tryLogCurrentException(log, fmt::format("while loading part {} on path {}", res.part->name, part_path));
+    }
+
+    /// Ignore broken parts that can appear as a result of hard server restart.
+    if (res.is_broken)
+    {
+        res.size_of_part = calculatePartSizeSafe(res.part, log);
+        auto part_size_str = res.size_of_part ? formatReadableSizeWithBinarySuffix(*res.size_of_part) : "failed to calculate size";
+
+        LOG_ERROR(log,
+            "Detaching broken part {}{} (size: {}). "
+            "If it happened after update, it is likely because of backward incompatibility. "
+            "You need to resolve this manually",
+            getFullPathOnDisk(part_disk_ptr), part_name, part_size_str);
+
+        return res;
+    }
+
+    res.part->modification_time = part_disk_ptr->getLastModified(fs::path(relative_data_path) / part_name).epochTime();
+    res.part->loadVersionMetadata();
+
+    if (res.part->wasInvolvedInTransaction())
+    {
+        /// Check if CSNs were written after committing transaction, update and write if needed.
+        bool version_updated = false;
+        auto & version = res.part->version;
+        chassert(!version.creation_tid.isEmpty());
+
+        if (!res.part->version.creation_csn)
+        {
+            auto min = TransactionLog::getCSN(res.part->version.creation_tid);
+            if (!min)
+            {
+                /// Transaction that created this part was not committed. Remove part.
+                TransactionLog::assertTIDIsNotOutdated(res.part->version.creation_tid);
+                min = Tx::RolledBackCSN;
+            }
+
+            LOG_TRACE(log, "Will fix version metadata of {} after unclean restart: part has creation_tid={}, setting creation_csn={}",
+                        res.part->name, res.part->version.creation_tid, min);
+
+            version.creation_csn = min;
+            version_updated = true;
+        }
+
+        if (!version.removal_tid.isEmpty() && !version.removal_csn)
+        {
+            auto max = TransactionLog::getCSN(version.removal_tid);
+            if (max)
+            {
+                LOG_TRACE(log, "Will fix version metadata of {} after unclean restart: part has removal_tid={}, setting removal_csn={}",
+                            res.part->name, version.removal_tid, max);
+                version.removal_csn = max;
+            }
+            else
+            {
+                TransactionLog::assertTIDIsNotOutdated(version.removal_tid);
+                /// Transaction that tried to remove this part was not committed. Clear removal_tid.
+                LOG_TRACE(log, "Will fix version metadata of {} after unclean restart: clearing removal_tid={}",
+                            res.part->name, version.removal_tid);
+                version.unlockRemovalTID(version.removal_tid, TransactionInfoContext{getStorageID(), res.part->name});
+            }
+
+            version_updated = true;
+        }
+
+        /// Sanity checks
+        bool csn_order = !version.removal_csn || version.creation_csn <= version.removal_csn || version.removal_csn == Tx::PrehistoricCSN;
+        bool min_start_csn_order = version.creation_tid.start_csn <= version.creation_csn;
+        bool max_start_csn_order = version.removal_tid.start_csn <= version.removal_csn;
+        bool creation_csn_known = version.creation_csn;
+        if (!csn_order || !min_start_csn_order || !max_start_csn_order || !creation_csn_known)
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Part {} has invalid version metadata: {}", res.part->name, version.toString());
+
+        if (version_updated)
+            res.part->storeVersionMetadata(/* force */ true);
+
+        /// Deactivate part if creation was not committed or if removal was.
+        if (version.creation_csn == Tx::RolledBackCSN || version.removal_csn)
+        {
+            preparePartForRemoval(res.part);
+            to_state = DataPartState::Outdated;
+        }
+    }
+
+    res.part->setState(to_state);
+
+    DataPartIteratorByInfo it;
+    bool inserted;
+
+    {
+        std::lock_guard lock(part_loading_mutex);
+        std::tie(it, inserted) = data_parts_indexes.insert(res.part);
+    }
+
+    /// Remove duplicate parts with the same checksum.
+    if (!inserted)
+    {
+        if ((*it)->checksums.getTotalChecksumHex() == res.part->checksums.getTotalChecksumHex())
+        {
+            LOG_ERROR(log, "Remove duplicate part {}", data_part_storage->getFullPath());
+            res.part->is_duplicate = true;
+            return res;
+        }
+        else
+            throw Exception("Part " + res.part->name + " already exists but with different checksums", ErrorCodes::DUPLICATE_DATA_PART);
+    }
+
+    if (to_state == DataPartState::Active)
+        addPartContributionToDataVolume(res.part);
+
+    LOG_TRACE(log, "Finished loading {} part {} on disk {}", magic_enum::enum_name(to_state), part_name, part_disk_ptr->getName());
+    return res;
+};
+
+std::vector<MergeTreeData::LoadPartResult> MergeTreeData::loadDataPartsFromDisk(
     ThreadPool & pool,
     size_t num_parts,
     std::queue<PartLoadingTreeNodes> & parts_queue,
-    bool skip_sanity_checks,
     const MergeTreeSettingsPtr & settings)
 {
     /// Parallel loading of data parts.
@@ -1038,6 +1256,7 @@ void MergeTreeData::loadDataPartsFromDisk(
         threads_queue.push(i);
     }
 
+    std::vector<LoadPartResult> loaded_parts;
     while (!parts_queue.empty())
     {
         assert(!threads_queue.empty());
@@ -1077,132 +1296,11 @@ void MergeTreeData::loadDataPartsFromDisk(
         return !parts.empty();
     }));
 
-    size_t suspicious_broken_parts = 0;
-    size_t suspicious_broken_parts_bytes = 0;
-    std::atomic<bool> has_adaptive_parts = false;
-    std::atomic<bool> has_non_adaptive_parts = false;
-    std::atomic<bool> has_lightweight_in_parts = false;
-    std::mutex loading_mutex;
-
-    auto load_part = [&](const MergeTreePartInfo & part_info, const String & part_name, const DiskPtr & part_disk_ptr)
-    {
-        LOG_TRACE(log, "Loading part {} from disk {}", part_name, part_disk_ptr->getName());
-
-        auto single_disk_volume = std::make_shared<SingleDiskVolume>("volume_" + part_name, part_disk_ptr, 0);
-        auto data_part_storage = std::make_shared<DataPartStorageOnDisk>(single_disk_volume, relative_data_path, part_name);
-        auto part = createPart(part_name, part_info, data_part_storage);
-        bool broken = false;
-
-        String part_path = fs::path(relative_data_path) / part_name;
-        String marker_path = fs::path(part_path) / IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME;
-
-        if (part_disk_ptr->exists(marker_path))
-        {
-            /// NOTE: getBytesOnDisk() cannot be used here, since it maybe zero of checksums.txt will not exist
-            size_t size_of_part = data_part_storage->calculateTotalSizeOnDisk();
-            LOG_WARNING(log,
-                "Detaching stale part {}{} (size: {}), which should have been deleted after a move. "
-                "That can only happen after unclean restart of ClickHouse after move of a part having an operation blocking that stale copy of part.",
-                getFullPathOnDisk(part_disk_ptr), part_name, formatReadableSizeWithBinarySuffix(size_of_part));
-
-            std::lock_guard loading_lock(loading_mutex);
-
-            broken_parts_to_detach.push_back(part);
-            ++suspicious_broken_parts;
-            suspicious_broken_parts_bytes += size_of_part;
-            return false;
-        }
-
-        try
-        {
-            part->loadColumnsChecksumsIndexes(require_part_metadata, true);
-        }
-        catch (const Exception & e)
-        {
-            /// Don't count the part as broken if there is not enough memory to load it.
-            /// In fact, there can be many similar situations.
-            /// But it is OK, because there is a safety guard against deleting too many parts.
-            if (isNotEnoughMemoryErrorCode(e.code()))
-                throw;
-
-            broken = true;
-            tryLogCurrentException(log, fmt::format("while loading part {} on path {}", part->name, part_path));
-        }
-        catch (...)
-        {
-            broken = true;
-            tryLogCurrentException(log, fmt::format("while loading part {} on path {}", part->name, part_path));
-        }
-
-        /// Ignore broken parts that can appear as a result of hard server restart.
-        if (broken)
-        {
-            std::optional<size_t> size_of_part;
-            try
-            {
-                /// NOTE: getBytesOnDisk() cannot be used here, since it maybe zero of checksums.txt will not exist
-                size_of_part = data_part_storage->calculateTotalSizeOnDisk();
-            }
-            catch (...)
-            {
-                tryLogCurrentException(log, fmt::format("while calculating part size {} on path {}", part->name, part_path));
-            }
-
-            std::string part_size_str = "failed to calculate size";
-            if (size_of_part.has_value())
-                part_size_str = formatReadableSizeWithBinarySuffix(*size_of_part);
-
-            LOG_ERROR(log,
-                "Detaching broken part {}{} (size: {}). "
-                "If it happened after update, it is likely because of backward incompatibility. "
-                "You need to resolve this manually",
-                getFullPathOnDisk(part_disk_ptr), part_name, part_size_str);
-
-            std::lock_guard loading_lock(loading_mutex);
-
-            broken_parts_to_detach.push_back(part);
-            ++suspicious_broken_parts;
-            if (size_of_part.has_value())
-                suspicious_broken_parts_bytes += *size_of_part;
-            return false;
-        }
-
-        if (!part->index_granularity_info.mark_type.adaptive)
-            has_non_adaptive_parts.store(true, std::memory_order_relaxed);
-        else
-            has_adaptive_parts.store(true, std::memory_order_relaxed);
-
-        /// Check if there is lightweight delete in part
-        if (part->hasLightweightDelete())
-            has_lightweight_in_parts.store(true, std::memory_order_relaxed);
-
-        part->modification_time = part_disk_ptr->getLastModified(fs::path(relative_data_path) / part_name).epochTime();
-        /// Assume that all parts are Active, covered parts will be detected and marked as Outdated later
-        part->setState(DataPartState::Active);
-
-        std::lock_guard loading_lock(loading_mutex);
-        auto [it, inserted] = data_parts_indexes.insert(part);
-
-        /// Remove duplicate parts with the same checksum.
-        if (!inserted)
-        {
-            if ((*it)->checksums.getTotalChecksumHex() == part->checksums.getTotalChecksumHex())
-            {
-                LOG_ERROR(log, "Remove duplicate part {}", data_part_storage->getFullPath());
-                duplicate_parts_to_remove.push_back(part);
-            }
-            else
-                throw Exception("Part " + part->name + " already exists but with different checksums", ErrorCodes::DUPLICATE_DATA_PART);
-        }
-
-        addPartContributionToDataVolume(part);
-        LOG_TRACE(log, "Finished part {} load on disk {}", part_name, part_disk_ptr->getName());
-        return true;
-    };
-
-    std::mutex part_select_mutex;
     try
     {
+        std::mutex part_select_mutex;
+        std::mutex part_loading_mutex;
+
         for (size_t thread = 0; thread < num_threads; ++thread)
         {
             pool.scheduleOrThrowOnError([&, thread]
@@ -1235,15 +1333,19 @@ void MergeTreeData::loadDataPartsFromDisk(
                             remaining_thread_parts.erase(thread_idx);
                     }
 
-                    bool part_is_ok = load_part(thread_part->info, thread_part->name, thread_part->disk);
+                    thread_part->is_loaded = true;
+                    auto res = loadDataPart(thread_part->info, thread_part->name, thread_part->disk, DataPartState::Active, part_loading_mutex);
+                    bool is_active_part = res.part->getState() == DataPartState::Active && !res.part->is_duplicate;
 
-                    if (!part_is_ok)
+                    if (!is_active_part && !thread_part->children.empty())
                     {
                         std::lock_guard lock{part_select_mutex};
                         for (const auto & [_, node] : thread_part->children)
                             threads_parts[thread].push_back(node);
                         remaining_thread_parts.insert(thread);
                     }
+
+                    loaded_parts.push_back(std::move(res));
                 }
             });
         }
@@ -1256,53 +1358,37 @@ void MergeTreeData::loadDataPartsFromDisk(
     }
 
     pool.wait();
-
-    if (has_non_adaptive_parts && has_adaptive_parts && !settings->enable_mixed_granularity_parts)
-        throw Exception(
-            "Table contains parts with adaptive and non adaptive marks, but `setting enable_mixed_granularity_parts` is disabled",
-            ErrorCodes::LOGICAL_ERROR);
-
-    has_non_adaptive_index_granularity_parts = has_non_adaptive_parts;
-
-    if (has_lightweight_in_parts)
-        has_lightweight_delete_parts.store(true);
-
-    if (suspicious_broken_parts > settings->max_suspicious_broken_parts && !skip_sanity_checks)
-        throw Exception(ErrorCodes::TOO_MANY_UNEXPECTED_DATA_PARTS,
-            "Suspiciously many ({} parts, {} in total) broken parts to remove while maximum allowed broken parts count is {}. You can change the maximum value "
-                        "with merge tree setting 'max_suspicious_broken_parts' in <merge_tree> configuration section or in table settings in .sql file "
-                        "(don't forget to return setting back to default value)",
-            suspicious_broken_parts, formatReadableSizeWithBinarySuffix(suspicious_broken_parts_bytes), settings->max_suspicious_broken_parts);
-
-    if (suspicious_broken_parts_bytes > settings->max_suspicious_broken_parts_bytes && !skip_sanity_checks)
-        throw Exception(ErrorCodes::TOO_MANY_UNEXPECTED_DATA_PARTS,
-            "Suspiciously big size ({} parts, {} in total) of all broken parts to remove while maximum allowed broken parts size is {}. "
-            "You can change the maximum value with merge tree setting 'max_suspicious_broken_parts_bytes' in <merge_tree> configuration "
-            "section or in table settings in .sql file (don't forget to return setting back to default value)",
-            suspicious_broken_parts, formatReadableSizeWithBinarySuffix(suspicious_broken_parts_bytes),
-            formatReadableSizeWithBinarySuffix(settings->max_suspicious_broken_parts_bytes));
+    return loaded_parts;
 }
 
 
-void MergeTreeData::loadDataPartsFromWAL(
-    DataPartsVector & /* broken_parts_to_detach */,
-    DataPartsVector & duplicate_parts_to_remove,
-    MutableDataPartsVector & parts_from_wal)
+void MergeTreeData::loadDataPartsFromWAL(MutableDataPartsVector & parts_from_wal)
 {
+    std::sort(parts_from_wal.begin(), parts_from_wal.end(), [](const auto & lhs, const auto & rhs)
+    {
+        return std::tie(lhs->info.level, lhs->info.mutation) > std::tie(rhs->info.level, rhs->info.mutation);
+    });
+
     for (auto & part : parts_from_wal)
     {
         part->modification_time = time(nullptr);
         /// Assume that all parts are Active, covered parts will be detected and marked as Outdated later
         part->setState(DataPartState::Active);
 
+        auto lo = data_parts_by_state_and_info.lower_bound(DataPartStateAndInfo{DataPartState::Active, part->info});
+
+        if (lo != data_parts_by_state_and_info.begin() && (*std::prev(lo))->info.contains(part->info))
+            continue;
+
+        if (lo != data_parts_by_state_and_info.end() && (*lo)->info.contains(part->info))
+            continue;
+
         auto [it, inserted] = data_parts_indexes.insert(part);
+
         if (!inserted)
         {
             if ((*it)->checksums.getTotalChecksumHex() == part->checksums.getTotalChecksumHex())
-            {
                 LOG_ERROR(log, "Remove duplicate part {}", part->data_part_storage->getFullPath());
-                duplicate_parts_to_remove.push_back(part);
-            }
             else
                 throw Exception("Part " + part->name + " already exists but with different checksums", ErrorCodes::DUPLICATE_DATA_PART);
         }
@@ -1402,9 +1488,10 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
     /// Collect parts by disks' names.
     std::map<String, PartLoadingTreeNodes> disk_part_map;
 
-    for (const auto & elem : loading_tree.getRoots())
-        for (const auto & [_, node] : elem.second->children)
-            disk_part_map[node->disk->getName()].emplace_back(node);
+    loading_tree.traverse(/*recursive=*/ false, [&](const auto & node)
+    {
+        disk_part_map[node->disk->getName()].emplace_back(node);
+    });
 
     size_t num_parts = 0;
     std::queue<PartLoadingTreeNodes> parts_queue;
@@ -1426,9 +1513,40 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
     DataPartsVector broken_parts_to_detach;
     DataPartsVector duplicate_parts_to_remove;
 
+    size_t suspicious_broken_parts = 0;
+    size_t suspicious_broken_parts_bytes = 0;
+    bool have_adaptive_parts = false;
+    bool have_non_adaptive_parts = false;
+    bool have_lightweight_in_parts = false;
+    bool have_parts_with_version_metadata = false;
+
     if (num_parts > 0)
-        loadDataPartsFromDisk(
-            broken_parts_to_detach, duplicate_parts_to_remove, pool, num_parts, parts_queue, skip_sanity_checks, settings);
+    {
+        auto loaded_parts = loadDataPartsFromDisk(pool, num_parts, parts_queue, settings);
+
+        for (const auto & res : loaded_parts)
+        {
+            if (res.is_broken)
+            {
+                broken_parts_to_detach.push_back(res.part);
+                ++suspicious_broken_parts;
+                if (res.size_of_part)
+                    suspicious_broken_parts_bytes += *res.size_of_part;
+            }
+            else if (res.part->is_duplicate)
+            {
+                res.part->remove();
+            }
+            else
+            {
+                bool is_adaptive = res.part->index_granularity_info.mark_type.adaptive;
+                have_adaptive_parts |= is_adaptive;
+                have_non_adaptive_parts |= !is_adaptive;
+                have_lightweight_in_parts |= res.part->hasLightweightDelete();
+                have_parts_with_version_metadata |= res.part->wasInvolvedInTransaction();
+            }
+        }
+    }
 
     if (settings->in_memory_parts_enable_wal)
     {
@@ -1478,8 +1596,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
             parts_from_wal.insert(
                 parts_from_wal.end(), std::make_move_iterator(disk_wal_parts.begin()), std::make_move_iterator(disk_wal_parts.end()));
 
-        loadDataPartsFromWAL(broken_parts_to_detach, duplicate_parts_to_remove, parts_from_wal);
-
+        loadDataPartsFromWAL(parts_from_wal);
         num_parts += parts_from_wal.size();
     }
 
@@ -1490,6 +1607,30 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
         return;
     }
 
+    if (have_non_adaptive_parts && have_adaptive_parts && !settings->enable_mixed_granularity_parts)
+        throw Exception(
+            "Table contains parts with adaptive and non adaptive marks, but `setting enable_mixed_granularity_parts` is disabled",
+            ErrorCodes::LOGICAL_ERROR);
+
+    has_non_adaptive_index_granularity_parts = have_non_adaptive_parts;
+    has_lightweight_delete_parts = have_lightweight_in_parts;
+    transactions_enabled = have_parts_with_version_metadata;
+
+    if (suspicious_broken_parts > settings->max_suspicious_broken_parts && !skip_sanity_checks)
+        throw Exception(ErrorCodes::TOO_MANY_UNEXPECTED_DATA_PARTS,
+            "Suspiciously many ({} parts, {} in total) broken parts to remove while maximum allowed broken parts count is {}. You can change the maximum value "
+                        "with merge tree setting 'max_suspicious_broken_parts' in <merge_tree> configuration section or in table settings in .sql file "
+                        "(don't forget to return setting back to default value)",
+            suspicious_broken_parts, formatReadableSizeWithBinarySuffix(suspicious_broken_parts_bytes), settings->max_suspicious_broken_parts);
+
+    if (suspicious_broken_parts_bytes > settings->max_suspicious_broken_parts_bytes && !skip_sanity_checks)
+        throw Exception(ErrorCodes::TOO_MANY_UNEXPECTED_DATA_PARTS,
+            "Suspiciously big size ({} parts, {} in total) of all broken parts to remove while maximum allowed broken parts size is {}. "
+            "You can change the maximum value with merge tree setting 'max_suspicious_broken_parts_bytes' in <merge_tree> configuration "
+            "section or in table settings in .sql file (don't forget to return setting back to default value)",
+            suspicious_broken_parts, formatReadableSizeWithBinarySuffix(suspicious_broken_parts_bytes),
+            formatReadableSizeWithBinarySuffix(settings->max_suspicious_broken_parts_bytes));
+
     for (auto & part : broken_parts_to_detach)
     {
         auto builder = part->data_part_storage->getBuilder();
@@ -1497,178 +1638,69 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
         builder->commit();
     }
 
-    for (auto & part : duplicate_parts_to_remove)
-        part->remove();
-
-    auto deactivate_part = [&] (DataPartIteratorByStateAndInfo it)
-    {
-        const DataPartPtr & part = *it;
-
-        part->remove_time.store(part->modification_time, std::memory_order_relaxed);
-        auto creation_csn = part->version.creation_csn.load(std::memory_order_relaxed);
-        if (creation_csn != Tx::RolledBackCSN && creation_csn != Tx::PrehistoricCSN && !part->version.isRemovalTIDLocked())
-        {
-            /// It's possible that covering part was created without transaction,
-            /// but if covered part was created with transaction (i.e. creation_tid is not prehistoric),
-            /// then it must have removal tid in metadata file.
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Data part {} is Outdated and has creation TID {} and CSN {}, "
-                            "but does not have removal tid. It's a bug or a result of manual intervention.",
-                            part->name, part->version.creation_tid, creation_csn);
-        }
-        modifyPartState(it, DataPartState::Outdated);
-        removePartContributionToDataVolume(part);
-
-        /// Explicitly set removal_tid_lock for parts w/o transaction (i.e. w/o txn_version.txt)
-        /// to avoid keeping part forever (see VersionMetadata::canBeRemoved())
-        if (!part->version.isRemovalTIDLocked())
-        {
-            TransactionInfoContext transaction_context{getStorageID(), part->name};
-            part->version.lockRemovalTID(Tx::PrehistoricTID, transaction_context);
-        }
-    };
-
-    /// All parts are in "Active" state after loading
-    assert(std::find_if(data_parts_by_state_and_info.begin(), data_parts_by_state_and_info.end(),
-    [](const auto & part)
-    {
-        return part->getState() != DataPartState::Active;
-    }) == data_parts_by_state_and_info.end());
-
-    bool have_parts_with_version_metadata = false;
-    auto iter = data_parts_by_state_and_info.begin();
-    while (iter != data_parts_by_state_and_info.end() && (*iter)->getState() == DataPartState::Active)
-    {
-        const DataPartPtr & part = *iter;
-        part->loadVersionMetadata();
-        VersionMetadata & version = part->version;
-        if (part->wasInvolvedInTransaction())
-        {
-            have_parts_with_version_metadata = true;
-        }
-        else
-        {
-            ++iter;
-            continue;
-        }
-
-        /// Check if CSNs were written after committing transaction, update and write if needed.
-        bool version_updated = false;
-        chassert(!version.creation_tid.isEmpty());
-        if (!part->version.creation_csn)
-        {
-            auto min = TransactionLog::getCSN(version.creation_tid);
-            if (!min)
-            {
-                /// Transaction that created this part was not committed. Remove part.
-                TransactionLog::assertTIDIsNotOutdated(version.creation_tid);
-                min = Tx::RolledBackCSN;
-            }
-            LOG_TRACE(log, "Will fix version metadata of {} after unclean restart: part has creation_tid={}, setting creation_csn={}",
-                      part->name, version.creation_tid, min);
-            version.creation_csn = min;
-            version_updated = true;
-        }
-        if (!version.removal_tid.isEmpty() && !part->version.removal_csn)
-        {
-            auto max = TransactionLog::getCSN(version.removal_tid);
-            if (max)
-            {
-                LOG_TRACE(log, "Will fix version metadata of {} after unclean restart: part has removal_tid={}, setting removal_csn={}",
-                          part->name, version.removal_tid, max);
-                version.removal_csn = max;
-            }
-            else
-            {
-                TransactionLog::assertTIDIsNotOutdated(version.removal_tid);
-                /// Transaction that tried to remove this part was not committed. Clear removal_tid.
-                LOG_TRACE(log, "Will fix version metadata of {} after unclean restart: clearing removal_tid={}",
-                          part->name, version.removal_tid);
-                version.unlockRemovalTID(version.removal_tid, TransactionInfoContext{getStorageID(), part->name});
-            }
-            version_updated = true;
-        }
-
-        /// Sanity checks
-        bool csn_order = !version.removal_csn || version.creation_csn <= version.removal_csn || version.removal_csn == Tx::PrehistoricCSN;
-        bool min_start_csn_order = version.creation_tid.start_csn <= version.creation_csn;
-        bool max_start_csn_order = version.removal_tid.start_csn <= version.removal_csn;
-        bool creation_csn_known = version.creation_csn;
-        if (!csn_order || !min_start_csn_order || !max_start_csn_order || !creation_csn_known)
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Part {} has invalid version metadata: {}", part->name, version.toString());
-
-        if (version_updated)
-            part->storeVersionMetadata(/* force */ true);
-
-        /// Deactivate part if creation was not committed or if removal was.
-        if (version.creation_csn == Tx::RolledBackCSN || version.removal_csn)
-        {
-            auto next_it = std::next(iter);
-            deactivate_part(iter);
-            iter = next_it;
-        }
-        else
-        {
-            ++iter;
-        }
-    }
-
-    if (have_parts_with_version_metadata)
-        transactions_enabled.store(true);
-
-    /// Delete from the set of current parts those parts that are covered by another part (those parts that
-    /// were merged), but that for some reason are still not deleted from the filesystem.
-    /// Deletion of files will be performed later in the clearOldParts() method.
-
-    auto active_parts_range = getDataPartsStateRange(DataPartState::Active);
-    auto prev_it = active_parts_range.begin();
-    auto end_it = active_parts_range.end();
-
-    bool less_than_two_active_parts = prev_it == end_it || std::next(prev_it) == end_it;
-
-    if (!less_than_two_active_parts)
-    {
-        (*prev_it)->assertState({DataPartState::Active});
-        auto curr_it = std::next(prev_it);
-
-        while (curr_it != data_parts_by_state_and_info.end() && (*curr_it)->getState() == DataPartState::Active)
-        {
-            (*curr_it)->assertState({DataPartState::Active});
-
-            /// Don't consider data parts belonging to different partitions.
-            if ((*curr_it)->info.partition_id != (*prev_it)->info.partition_id)
-            {
-                ++prev_it;
-                ++curr_it;
-                continue;
-            }
-
-            if ((*curr_it)->contains(**prev_it))
-            {
-                deactivate_part(prev_it);
-                prev_it = curr_it;
-                ++curr_it;
-            }
-            else if ((*prev_it)->contains(**curr_it))
-            {
-                auto next = std::next(curr_it);
-                deactivate_part(curr_it);
-                curr_it = next;
-            }
-            else
-            {
-                ++prev_it;
-                ++curr_it;
-            }
-        }
-    }
-
     resetObjectColumnsFromActiveParts(part_lock);
     calculateColumnAndSecondaryIndexSizesImpl();
 
+    PartLoadingTreeNodes unloaded_parts;
+    loading_tree.traverse(/*recursive=*/ true, [&](const auto & node)
+    {
+        if (!node->is_loaded)
+            unloaded_parts.push_back(node);
+    });
+
+    {
+        std::lock_guard lock(unloaded_outdated_parts_mutex);
+        unloaded_outdated_parts = std::move(unloaded_parts);
+    }
+
     LOG_DEBUG(log, "Loaded data parts ({} items)", data_parts_indexes.size());
     data_parts_loading_finished = true;
 }
 
+void MergeTreeData::loadOutdatedDataParts(PartLoadingTreeNodes parts_to_load)
+{
+    LOG_DEBUG(log, "Loading {} outdated data parts", parts_to_load.size());
+    if (parts_to_load.empty())
+        return;
+
+    for (const auto & part : parts_to_load)
+    {
+        auto res = loadDataPart(part->info, part->name, part->disk, MergeTreeDataPartState::Outdated, data_parts_mutex);
+
+        if (res.is_broken)
+        {
+            auto builder = res.part->data_part_storage->getBuilder();
+            res.part->renameToDetached("broken-on-start", builder); /// detached parts must not have '_' in prefixes
+            builder->commit();
+        }
+        else if (res.part->is_duplicate)
+        {
+            res.part->remove();
+        }
+        else
+        {
+            preparePartForRemoval(res.part);
+        }
+    }
+
+    LOG_DEBUG(log, "Loaded {} outdated data parts", parts_to_load.size());
+}
+
+void MergeTreeData::scheduleOutdatedDataPartsLoadingJob(BackgroundJobsAssignee & assignee)
+{
+    std::lock_guard lock(unloaded_outdated_parts_mutex);
+    if (unloaded_outdated_parts.empty())
+        return;
+
+    auto share_lock = lockForShare(RWLockImpl::NO_QUERY, getSettings()->lock_acquire_timeout_for_background_operations);
+    assignee.scheduleCommonTask(std::make_shared<ExecutableLambdaAdapter>(
+        [this, share_lock, parts_to_load = std::move(unloaded_outdated_parts)]
+        {
+            loadOutdatedDataParts(std::move(parts_to_load));
+            return false;
+        }, common_assignee_trigger, getStorageID()), /*need_trigger=*/ false);
+}
+
 /// Is the part directory old.
 /// True if its modification time and the modification time of all files inside it is less then threshold.
 /// (Only files on the first level of nesting are considered).
diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h
index 5dbeaa0e958..69b41610643 100644
--- a/src/Storages/MergeTree/MergeTreeData.h
+++ b/src/Storages/MergeTree/MergeTreeData.h
@@ -1021,7 +1021,6 @@ protected:
     /// under lockForShare if rename is possible.
     String relative_data_path;
 
-
     /// Current column sizes in compressed and uncompressed form.
     ColumnSizeByName column_sizes;
 
@@ -1260,6 +1259,53 @@ protected:
     void resetObjectColumnsFromActiveParts(const DataPartsLock & lock);
     void updateObjectColumns(const DataPartPtr & part, const DataPartsLock & lock);
 
+    class PartLoadingTree
+    {
+    public:
+        struct Node
+        {
+            Node(const MergeTreePartInfo & info_, const String & name_, const DiskPtr & disk_)
+                : info(info_), name(name_), disk(disk_)
+            {
+            }
+
+            const MergeTreePartInfo info;
+            const String name;
+            const DiskPtr disk;
+
+            bool is_loaded = false;
+            std::map<MergeTreePartInfo, std::shared_ptr<Node>> children;
+        };
+
+        using NodePtr = std::shared_ptr<Node>;
+        using PartLoadingInfo = std::tuple<MergeTreePartInfo, String, DiskPtr>;
+
+        static PartLoadingTree build(std::vector<PartLoadingInfo> nodes);
+
+        template <typename Func>
+        void traverse(bool recursive, Func && func);
+
+    private:
+        void add(const MergeTreePartInfo & info, const String & name, const DiskPtr & disk);
+
+        std::unordered_map<String, NodePtr> root_by_partition;
+    };
+
+    using PartLoadingTreeNodes = std::vector<PartLoadingTree::NodePtr>;
+
+    struct LoadPartResult
+    {
+        bool is_broken = false;
+        std::optional<size_t> size_of_part;
+        MutableDataPartPtr part;
+    };
+
+    mutable std::mutex unloaded_outdated_parts_mutex;
+    PartLoadingTreeNodes unloaded_outdated_parts TSA_GUARDED_BY(unloaded_outdated_parts_mutex);
+
+    void loadOutdatedDataParts(PartLoadingTreeNodes parts_to_load);
+    void scheduleOutdatedDataPartsLoadingJob(BackgroundJobsAssignee & assignee);
+
     static void incrementInsertedPartsProfileEvent(MergeTreeDataPartType type);
     static void incrementMergedPartsProfileEvent(MergeTreeDataPartType type);
 
@@ -1333,49 +1379,20 @@ private:
     /// Returns default settings for storage with possible changes from global config.
     virtual std::unique_ptr<MergeTreeSettings> getDefaultSettings() const = 0;
 
-    class PartLoadingTree
-    {
-    public:
-        struct Node
-        {
-            Node(const MergeTreePartInfo & info_, const String & name_, const DiskPtr & disk_)
-                : info(info_), name(name_), disk(disk_)
-            {
-            }
+    LoadPartResult loadDataPart(
+        const MergeTreePartInfo & part_info,
+        const String & part_name,
+        const DiskPtr & part_disk_ptr,
+        MergeTreeDataPartState to_state,
+        std::mutex & part_loading_mutex);
 
-            const MergeTreePartInfo info;
-            const String name;
-            const DiskPtr disk;
-
-            std::map<MergeTreePartInfo, std::shared_ptr<Node>> children;
-        };
-
-        using NodePtr = std::shared_ptr<Node>;
-        using PartLoadingInfo = std::tuple<MergeTreePartInfo, String, DiskPtr>;
-
-        static PartLoadingTree build(std::vector<PartLoadingInfo> nodes);
-        void add(const MergeTreePartInfo & info, const String & name, const DiskPtr & disk);
-        const std::unordered_map<String, NodePtr> & getRoots() const { return root_by_partition; }
-
-    private:
-        std::unordered_map<String, NodePtr> root_by_partition;
-    };
-
-    using PartLoadingTreeNodes = std::vector<PartLoadingTree::NodePtr>;
-
-    void loadDataPartsFromDisk(
-        DataPartsVector & broken_parts_to_detach,
-        DataPartsVector & duplicate_parts_to_remove,
+    std::vector<LoadPartResult> loadDataPartsFromDisk(
         ThreadPool & pool,
         size_t num_parts,
         std::queue<PartLoadingTreeNodes> & parts_queue,
-        bool skip_sanity_checks,
         const MergeTreeSettingsPtr & settings);
 
-    void loadDataPartsFromWAL(
-        DataPartsVector & broken_parts_to_detach,
-        DataPartsVector & duplicate_parts_to_remove,
-        MutableDataPartsVector & parts_from_wal);
+    void loadDataPartsFromWAL(MutableDataPartsVector & parts_from_wal);
 
     /// Create zero-copy exclusive lock for part and disk. Useful for coordination of
     /// distributed operations which can lead to data duplication. Implemented only in ReplicatedMergeTree.
diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp
index e2b23d75746..c68adfbf3c6 100644
--- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp
+++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp
@@ -158,6 +158,9 @@ bool ReplicatedMergeTreeRestartingThread::runImpl()
     storage.cleanup_thread.start();
     storage.part_check_thread.start();
 
+    if (first_time)
+        storage.scheduleOutdatedDataPartsLoadingJob(storage.background_operations_assignee);
+
     return true;
 }
 
diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp
index aea853b6c39..7cf6bf04a16 100644
--- a/src/Storages/StorageMergeTree.cpp
+++ b/src/Storages/StorageMergeTree.cpp
@@ -110,7 +110,6 @@ StorageMergeTree::StorageMergeTree(
     increment.set(getMaxBlockNumber());
 
     loadMutations();
-
     loadDeduplicationLog();
 }
 
@@ -136,6 +135,7 @@ void StorageMergeTree::startup()
     try
     {
         background_operations_assignee.start();
+        scheduleOutdatedDataPartsLoadingJob(background_operations_assignee);
         startBackgroundMovesIfNeeded();
     }
     catch (...)
@@ -1216,6 +1216,7 @@ bool StorageMergeTree::scheduleDataProcessingJob(BackgroundJobsAssignee & assign
         scheduled = true;
     }
 
+
     return scheduled;
 }
 
diff --git a/tests/integration/test_merge_tree_load_parts/configs/fast_background_pool.xml b/tests/integration/test_merge_tree_load_parts/configs/fast_background_pool.xml
new file mode 100644
index 00000000000..74038c5472f
--- /dev/null
+++ b/tests/integration/test_merge_tree_load_parts/configs/fast_background_pool.xml
@@ -0,0 +1,9 @@
+<clickhouse>
+    <background_processing_pool_thread_sleep_seconds>1</background_processing_pool_thread_sleep_seconds>
+    <background_processing_pool_thread_sleep_seconds_random_part>0</background_processing_pool_thread_sleep_seconds_random_part>
+    <background_processing_pool_thread_sleep_seconds_if_nothing_to_do>0.0</background_processing_pool_thread_sleep_seconds_if_nothing_to_do>
+    <background_processing_pool_task_sleep_seconds_when_no_work_min>0</background_processing_pool_task_sleep_seconds_when_no_work_min>
+    <background_processing_pool_task_sleep_seconds_when_no_work_max>1</background_processing_pool_task_sleep_seconds_when_no_work_max>
+    <background_processing_pool_task_sleep_seconds_when_no_work_multiplier>1</background_processing_pool_task_sleep_seconds_when_no_work_multiplier>
+    <background_processing_pool_task_sleep_seconds_when_no_work_random_part>0</background_processing_pool_task_sleep_seconds_when_no_work_random_part>
+</clickhouse>
diff --git a/tests/integration/test_merge_tree_load_parts/test.py b/tests/integration/test_merge_tree_load_parts/test.py
index cb0d902f677..4b5dec68c6e 100644
--- a/tests/integration/test_merge_tree_load_parts/test.py
+++ b/tests/integration/test_merge_tree_load_parts/test.py
@@ -1,12 +1,23 @@
 import pytest
 import helpers.client
 import helpers.cluster
+import time
 from helpers.corrupt_part_data_on_disk import corrupt_part_data_on_disk
 
 
 cluster = helpers.cluster.ClickHouseCluster(__file__)
-node1 = cluster.add_instance("node1", with_zookeeper=True, stay_alive=True)
-node2 = cluster.add_instance("node2", with_zookeeper=True, stay_alive=True)
+node1 = cluster.add_instance(
+    "node1",
+    main_configs=["configs/fast_background_pool.xml"],
+    with_zookeeper=True,
+    stay_alive=True,
+)
+node2 = cluster.add_instance(
+    "node2",
+    main_configs=["configs/fast_background_pool.xml"],
+    with_zookeeper=True,
+    stay_alive=True,
+)
 
 
 @pytest.fixture(scope="module")
@@ -35,14 +46,14 @@ def test_merge_tree_load_parts(started_cluster):
 
     node1.restart_clickhouse(kill=True)
     for i in range(1, 21):
-        assert node1.contains_in_log(f"Loading part 44_{i}_{i}_0")
+        assert node1.contains_in_log(f"Loading Active part 44_{i}_{i}_0")
 
     node1.query("OPTIMIZE TABLE mt_load_parts FINAL")
     node1.restart_clickhouse(kill=True)
 
-    assert node1.contains_in_log("Loading part 44_1_20")
+    assert node1.contains_in_log("Loading Active part 44_1_20")
     for i in range(1, 21):
-        assert not node1.contains_in_log(f"Loading part 44_{i}_{i}_0")
+        assert not node1.contains_in_log(f"Loading Active part 44_{i}_{i}_0")
 
     assert node1.query("SELECT count() FROM mt_load_parts") == "20\n"
     assert (
@@ -52,6 +63,45 @@ def test_merge_tree_load_parts(started_cluster):
         == "1\n"
     )
 
+    MAX_RETRY = 20
+
+    all_outdated_loaded = False
+    for _ in range(MAX_RETRY):
+        all_outdated_loaded = all(
+            [
+                node1.contains_in_log(f"Loading Outdated part 44_{i}_{i}_0")
+                for i in range(1, 21)
+            ]
+        )
+        if all_outdated_loaded:
+            break
+        time.sleep(2)
+
+    assert all_outdated_loaded
+
+    node1.query("ALTER TABLE mt_load_parts MODIFY SETTING old_parts_lifetime = 1")
+    node1.query("DETACH TABLE mt_load_parts")
+    node1.query("ATTACH TABLE mt_load_parts")
+
+    table_path = node1.query(
+        "SELECT data_paths[1] FROM system.tables WHERE table = 'mt_load_parts'"
+    ).strip()
+
+    part_dirs_ok = False
+    for _ in range(MAX_RETRY):
+        part_dirs = node1.exec_in_container(
+            ["bash", "-c", f"ls {table_path}"], user="root"
+        )
+        part_dirs = list(
+            set(part_dirs.strip().split("\n")) - {"detached", "format_version.txt"}
+        )
+        part_dirs_ok = len(part_dirs) == 1 and part_dirs[0].startswith("44_1_20")
+        if part_dirs_ok:
+            break
+        time.sleep(2)
+
+    assert part_dirs_ok
+
 
 def test_merge_tree_load_parts_corrupted(started_cluster):
     for i, node in enumerate([node1, node2]):
@@ -66,6 +116,7 @@ def test_merge_tree_load_parts_corrupted(started_cluster):
         node1.query(
             f"INSERT INTO mt_load_parts_2 VALUES ({partition}, 0, randomPrintableASCII(10))"
         )
+
         node1.query(
             f"INSERT INTO mt_load_parts_2 VALUES ({partition}, 1, randomPrintableASCII(10))"
         )
@@ -101,18 +152,18 @@ def test_merge_tree_load_parts_corrupted(started_cluster):
     def check_parts_loading(node, partition, loaded, failed, skipped):
         for (min_block, max_block) in loaded:
             part_name = f"{partition}_{min_block}_{max_block}"
-            assert node.contains_in_log(f"Loading part {part_name}")
-            assert node.contains_in_log(f"Finished part {part_name}")
+            assert node.contains_in_log(f"Loading Active part {part_name}")
+            assert node.contains_in_log(f"Finished loading Active part {part_name}")
 
         for (min_block, max_block) in failed:
             part_name = f"{partition}_{min_block}_{max_block}"
-            assert node.contains_in_log(f"Loading part {part_name}")
-            assert not node.contains_in_log(f"Finished part {part_name}")
+            assert node.contains_in_log(f"Loading Active part {part_name}")
+            assert not node.contains_in_log(f"Finished loading Active part {part_name}")
 
         for (min_block, max_block) in skipped:
             part_name = f"{partition}_{min_block}_{max_block}"
-            assert not node.contains_in_log(f"Loading part {part_name}")
-            assert not node.contains_in_log(f"Finished part {part_name}")
+            assert not node.contains_in_log(f"Loading Active part {part_name}")
+            assert not node.contains_in_log(f"Finished loading Active part {part_name}")
 
     check_parts_loading(
         node1, 111, loaded=[(0, 1), (2, 2)], failed=[(0, 2)], skipped=[(0, 0), (1, 1)]

From ce1b5eb1600bcdcf1f1a76d3a6e2b5c76bcc5e00 Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Fri, 14 Oct 2022 22:01:02 +0000
Subject: [PATCH 017/158] fix race while loading parts

---
 src/Storages/MergeTree/MergeTreeData.cpp | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 1916db740be..91ca2997214 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -1256,7 +1256,6 @@ std::vector<MergeTreeData::LoadPartResult> MergeTreeData::loadDataPartsFromDisk(
         threads_queue.push(i);
     }
 
-    std::vector<LoadPartResult> loaded_parts;
     while (!parts_queue.empty())
     {
         assert(!threads_queue.empty());
@@ -1296,6 +1295,9 @@ std::vector<MergeTreeData::LoadPartResult> MergeTreeData::loadDataPartsFromDisk(
         return !parts.empty();
     }));
 
+    std::mutex loaded_parts_mutex;
+    std::vector<LoadPartResult> loaded_parts;
+
     try
     {
         std::mutex part_select_mutex;
@@ -1335,8 +1337,8 @@ std::vector<MergeTreeData::LoadPartResult> MergeTreeData::loadDataPartsFromDisk(
 
                     thread_part->is_loaded = true;
                     auto res = loadDataPart(thread_part->info, thread_part->name, thread_part->disk, DataPartState::Active, part_loading_mutex);
-                    bool is_active_part = res.part->getState() == DataPartState::Active && !res.part->is_duplicate;
 
+                    bool is_active_part = res.part->getState() == DataPartState::Active && !res.part->is_duplicate;
                     if (!is_active_part && !thread_part->children.empty())
                     {
                         std::lock_guard lock{part_select_mutex};
@@ -1345,7 +1347,10 @@ std::vector<MergeTreeData::LoadPartResult> MergeTreeData::loadDataPartsFromDisk(
                         remaining_thread_parts.insert(thread);
                     }
 
-                    loaded_parts.push_back(std::move(res));
+                    {
+                        std::lock_guard lock(loaded_parts_mutex);
+                        loaded_parts.push_back(std::move(res));
+                    }
                 }
             });
         }

From 88d815c63ee80ba6599dd69d7f2a5088b35f51d5 Mon Sep 17 00:00:00 2001
From: Nikolay Degterinsky <evillique@gmail.com>
Date: Wed, 2 Nov 2022 06:11:37 +0000
Subject: [PATCH 018/158] Improve Asterisk and ColumnMatcher parser

---
 src/Analyzer/MatcherNode.cpp                  |  45 ++++++--
 src/Analyzer/QueryTreeBuilder.cpp             |  27 ++---
 .../DatabaseAndTableWithAlias.cpp             |  26 ++++-
 src/Interpreters/DatabaseAndTableWithAlias.h  |   2 +
 .../JoinToSubqueryTransformVisitor.cpp        |  32 +++---
 src/Interpreters/JoinedTables.cpp             |   9 +-
 .../TranslateQualifiedNamesVisitor.cpp        |  27 ++---
 src/Parsers/ASTAsterisk.cpp                   |  15 ++-
 src/Parsers/ASTAsterisk.h                     |   2 +
 src/Parsers/ASTColumnsMatcher.cpp             |  94 ++++++++++++----
 src/Parsers/ASTColumnsMatcher.h               |  11 +-
 src/Parsers/ASTQualifiedAsterisk.cpp          |   6 +-
 src/Parsers/ASTQualifiedAsterisk.h            |  10 +-
 src/Parsers/ExpressionElementParsers.cpp      | 104 ++++++++++--------
 src/Parsers/ExpressionListParsers.cpp         |  69 ++++++++----
 src/Storages/WindowView/StorageWindowView.cpp |   5 +-
 16 files changed, 328 insertions(+), 156 deletions(-)

diff --git a/src/Analyzer/MatcherNode.cpp b/src/Analyzer/MatcherNode.cpp
index 9d822771087..f0b44b47b44 100644
--- a/src/Analyzer/MatcherNode.cpp
+++ b/src/Analyzer/MatcherNode.cpp
@@ -207,18 +207,32 @@ ASTPtr MatcherNode::toASTImpl() const
 {
     ASTPtr result;
 
+    auto transformers = std::make_shared<ASTExpressionList>();
+
+    for (const auto & child : children)
+        transformers->children.push_back(child->toAST());
+
     if (matcher_type == MatcherNodeType::ASTERISK)
     {
         if (qualified_identifier.empty())
         {
-            result = std::make_shared<ASTAsterisk>();
+            auto asterisk = std::make_shared<ASTAsterisk>();
+
+            asterisk->transformers = std::move(transformers);
+            asterisk->children.push_back(asterisk->transformers);
+
+            result = asterisk;
         }
         else
         {
             auto qualified_asterisk = std::make_shared<ASTQualifiedAsterisk>();
 
             auto identifier_parts = qualified_identifier.getParts();
-            qualified_asterisk->children.push_back(std::make_shared<ASTIdentifier>(std::move(identifier_parts)));
+            qualified_asterisk->qualifier = std::make_shared<ASTIdentifier>(std::move(identifier_parts));
+            qualified_asterisk->transformers = std::move(transformers);
+
+            qualified_asterisk->children.push_back(qualified_asterisk->qualifier);
+            qualified_asterisk->children.push_back(qualified_asterisk->transformers);
 
             result = qualified_asterisk;
         }
@@ -229,6 +243,10 @@ ASTPtr MatcherNode::toASTImpl() const
         {
             auto regexp_matcher = std::make_shared<ASTColumnsRegexpMatcher>();
             regexp_matcher->setPattern(columns_matcher->pattern());
+
+            regexp_matcher->transformers = std::move(transformers);
+            regexp_matcher->children.push_back(regexp_matcher->transformers);
+
             result = regexp_matcher;
         }
         else
@@ -237,7 +255,11 @@ ASTPtr MatcherNode::toASTImpl() const
             regexp_matcher->setPattern(columns_matcher->pattern());
 
             auto identifier_parts = qualified_identifier.getParts();
-            regexp_matcher->children.push_back(std::make_shared<ASTIdentifier>(std::move(identifier_parts)));
+            regexp_matcher->qualifier = std::make_shared<ASTIdentifier>(std::move(identifier_parts));
+            regexp_matcher->transformers = std::move(transformers);
+
+            regexp_matcher->children.push_back(regexp_matcher->qualifier);
+            regexp_matcher->children.push_back(regexp_matcher->transformers);
 
             result = regexp_matcher;
         }
@@ -257,23 +279,30 @@ ASTPtr MatcherNode::toASTImpl() const
         {
             auto columns_list_matcher = std::make_shared<ASTColumnsListMatcher>();
             columns_list_matcher->column_list = std::move(column_list);
+            columns_list_matcher->transformers = std::move(transformers);
+
+            columns_list_matcher->children.push_back(columns_list_matcher->column_list);
+            columns_list_matcher->children.push_back(columns_list_matcher->transformers);
+
             result = columns_list_matcher;
         }
         else
         {
             auto columns_list_matcher = std::make_shared<ASTQualifiedColumnsListMatcher>();
-            columns_list_matcher->column_list = std::move(column_list);
 
             auto identifier_parts = qualified_identifier.getParts();
-            columns_list_matcher->children.push_back(std::make_shared<ASTIdentifier>(std::move(identifier_parts)));
+            columns_list_matcher->qualifier = std::make_shared<ASTIdentifier>(std::move(identifier_parts));
+            columns_list_matcher->column_list = std::move(column_list);
+            columns_list_matcher->transformers = std::move(transformers);
+
+            columns_list_matcher->children.push_back(columns_list_matcher->qualifier);
+            columns_list_matcher->children.push_back(columns_list_matcher->column_list);
+            columns_list_matcher->children.push_back(columns_list_matcher->transformers);
 
             result = columns_list_matcher;
         }
     }
 
-    for (const auto & child : children)
-        result->children.push_back(child->toAST());
-
     return result;
 }
 
diff --git a/src/Analyzer/QueryTreeBuilder.cpp b/src/Analyzer/QueryTreeBuilder.cpp
index 890aa2b01a2..6271b749181 100644
--- a/src/Analyzer/QueryTreeBuilder.cpp
+++ b/src/Analyzer/QueryTreeBuilder.cpp
@@ -99,7 +99,7 @@ private:
 
     QueryTreeNodePtr buildJoinTree(const ASTPtr & tables_in_select_query) const;
 
-    ColumnTransformersNodes buildColumnTransformers(const ASTPtr & matcher_expression, size_t start_child_index) const;
+    ColumnTransformersNodes buildColumnTransformers(const ASTPtr & matcher_expression) const;
 
     ASTPtr query;
     ContextPtr context;
@@ -413,13 +413,13 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression) co
     }
     else if (const auto * asterisk = expression->as<ASTAsterisk>())
     {
-        auto column_transformers = buildColumnTransformers(expression, 0 /*start_child_index*/);
+        auto column_transformers = buildColumnTransformers(asterisk->transformers);
         result = std::make_shared<MatcherNode>(std::move(column_transformers));
     }
     else if (const auto * qualified_asterisk = expression->as<ASTQualifiedAsterisk>())
     {
-        auto & qualified_identifier = qualified_asterisk->children.at(0)->as<ASTTableIdentifier &>();
-        auto column_transformers = buildColumnTransformers(expression, 1 /*start_child_index*/);
+        auto & qualified_identifier = qualified_asterisk->qualifier->as<ASTIdentifier &>();
+        auto column_transformers = buildColumnTransformers(qualified_asterisk->transformers);
         result = std::make_shared<MatcherNode>(Identifier(qualified_identifier.name_parts), std::move(column_transformers));
     }
     else if (const auto * ast_literal = expression->as<ASTLiteral>())
@@ -517,7 +517,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression) co
     }
     else if (const auto * columns_regexp_matcher = expression->as<ASTColumnsRegexpMatcher>())
     {
-        auto column_transformers = buildColumnTransformers(expression, 0 /*start_child_index*/);
+        auto column_transformers = buildColumnTransformers(columns_regexp_matcher->transformers);
         result = std::make_shared<MatcherNode>(columns_regexp_matcher->getMatcher(), std::move(column_transformers));
     }
     else if (const auto * columns_list_matcher = expression->as<ASTColumnsListMatcher>())
@@ -531,18 +531,18 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression) co
             column_list_identifiers.emplace_back(Identifier{column_list_identifier.name_parts});
         }
 
-        auto column_transformers = buildColumnTransformers(expression, 0 /*start_child_index*/);
+        auto column_transformers = buildColumnTransformers(columns_list_matcher->transformers);
         result = std::make_shared<MatcherNode>(std::move(column_list_identifiers), std::move(column_transformers));
     }
     else if (const auto * qualified_columns_regexp_matcher = expression->as<ASTQualifiedColumnsRegexpMatcher>())
     {
-        auto & qualified_identifier = qualified_columns_regexp_matcher->children.at(0)->as<ASTTableIdentifier &>();
-        auto column_transformers = buildColumnTransformers(expression, 1 /*start_child_index*/);
+        auto & qualified_identifier = qualified_columns_regexp_matcher->qualifier->as<ASTIdentifier &>();
+        auto column_transformers = buildColumnTransformers(qualified_columns_regexp_matcher->transformers);
         result = std::make_shared<MatcherNode>(Identifier(qualified_identifier.name_parts), qualified_columns_regexp_matcher->getMatcher(), std::move(column_transformers));
     }
     else if (const auto * qualified_columns_list_matcher = expression->as<ASTQualifiedColumnsListMatcher>())
     {
-        auto & qualified_identifier = qualified_columns_list_matcher->children.at(0)->as<ASTTableIdentifier &>();
+        auto & qualified_identifier = qualified_columns_list_matcher->qualifier->as<ASTIdentifier &>();
 
         Identifiers column_list_identifiers;
         column_list_identifiers.reserve(qualified_columns_list_matcher->column_list->children.size());
@@ -553,7 +553,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression) co
             column_list_identifiers.emplace_back(Identifier{column_list_identifier.name_parts});
         }
 
-        auto column_transformers = buildColumnTransformers(expression, 1 /*start_child_index*/);
+        auto column_transformers = buildColumnTransformers(qualified_columns_list_matcher->transformers);
         result = std::make_shared<MatcherNode>(Identifier(qualified_identifier.name_parts), std::move(column_list_identifiers), std::move(column_transformers));
     }
     else
@@ -811,15 +811,12 @@ QueryTreeNodePtr QueryTreeBuilder::buildJoinTree(const ASTPtr & tables_in_select
 }
 
 
-ColumnTransformersNodes QueryTreeBuilder::buildColumnTransformers(const ASTPtr & matcher_expression, size_t start_child_index) const
+ColumnTransformersNodes QueryTreeBuilder::buildColumnTransformers(const ASTPtr & matcher_expression) const
 {
     ColumnTransformersNodes column_transformers;
-    size_t children_size = matcher_expression->children.size();
 
-    for (; start_child_index < children_size; ++start_child_index)
+    for (const auto & child : matcher_expression->children)
     {
-        const auto & child = matcher_expression->children[start_child_index];
-
         if (auto * apply_transformer = child->as<ASTColumnsApplyTransformer>())
         {
             if (apply_transformer->lambda)
diff --git a/src/Interpreters/DatabaseAndTableWithAlias.cpp b/src/Interpreters/DatabaseAndTableWithAlias.cpp
index 70825ea8292..7fb581c1b4d 100644
--- a/src/Interpreters/DatabaseAndTableWithAlias.cpp
+++ b/src/Interpreters/DatabaseAndTableWithAlias.cpp
@@ -28,13 +28,29 @@ DatabaseAndTableWithAlias::DatabaseAndTableWithAlias(const ASTTableIdentifier &
         database = current_database;
 }
 
+DatabaseAndTableWithAlias::DatabaseAndTableWithAlias(const ASTIdentifier & identifier, const String & current_database)
+{
+    alias = identifier.tryGetAlias();
+
+    if (identifier.name_parts.size() == 2)
+        std::tie(database, table) = std::tie(identifier.name_parts[0], identifier.name_parts[1]);
+    else if (identifier.name_parts.size() == 1)
+        table = identifier.name_parts[0];
+    else
+        throw Exception("Logical error: invalid identifier", ErrorCodes::LOGICAL_ERROR);
+
+    if (database.empty())
+        database = current_database;
+}
+
 DatabaseAndTableWithAlias::DatabaseAndTableWithAlias(const ASTPtr & node, const String & current_database)
 {
-    const auto * identifier = node->as<ASTTableIdentifier>();
-    if (!identifier)
-        throw Exception("Logical error: table identifier expected", ErrorCodes::LOGICAL_ERROR);
-
-    *this = DatabaseAndTableWithAlias(*identifier, current_database);
+    if (const auto * table_identifier = node->as<ASTTableIdentifier>())
+        *this = DatabaseAndTableWithAlias(*table_identifier, current_database);
+    else if (const auto * identifier = node->as<ASTIdentifier>())
+        *this = DatabaseAndTableWithAlias(*identifier, current_database);
+    else
+        throw Exception("Logical error: identifier or table identifier expected", ErrorCodes::LOGICAL_ERROR);
 }
 
 DatabaseAndTableWithAlias::DatabaseAndTableWithAlias(const ASTTableExpression & table_expression, const String & current_database)
diff --git a/src/Interpreters/DatabaseAndTableWithAlias.h b/src/Interpreters/DatabaseAndTableWithAlias.h
index 237c56d3ce3..58327ff1d81 100644
--- a/src/Interpreters/DatabaseAndTableWithAlias.h
+++ b/src/Interpreters/DatabaseAndTableWithAlias.h
@@ -14,6 +14,7 @@ namespace DB
 {
 
 class ASTSelectQuery;
+class ASTIdentifier;
 class ASTTableIdentifier;
 struct ASTTableExpression;
 
@@ -28,6 +29,7 @@ struct DatabaseAndTableWithAlias
 
     DatabaseAndTableWithAlias() = default;
     explicit DatabaseAndTableWithAlias(const ASTPtr & identifier_node, const String & current_database = "");
+    explicit DatabaseAndTableWithAlias(const ASTIdentifier & identifier, const String & current_database = "");
     explicit DatabaseAndTableWithAlias(const ASTTableIdentifier & identifier, const String & current_database = "");
     explicit DatabaseAndTableWithAlias(const ASTTableExpression & table_expression, const String & current_database = "");
 
diff --git a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp
index 8e515caace4..3f15447fec1 100644
--- a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp
+++ b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp
@@ -49,7 +49,10 @@ ASTPtr makeSubqueryTemplate()
 ASTPtr makeSubqueryQualifiedAsterisk()
 {
     auto asterisk = std::make_shared<ASTQualifiedAsterisk>();
-    asterisk->children.emplace_back(std::make_shared<ASTTableIdentifier>("--.s"));
+    asterisk->qualifier = std::make_shared<ASTIdentifier>("--.s");
+    asterisk->transformers = std::make_shared<ASTExpressionList>();
+    asterisk->children.push_back(asterisk->qualifier);
+    asterisk->children.push_back(asterisk->transformers);
     return asterisk;
 }
 
@@ -139,24 +142,21 @@ private:
                 for (auto & table_name : data.tables_order)
                     data.addTableColumns(table_name, columns);
 
-                for (const auto & transformer : asterisk->children)
+                for (const auto & transformer : asterisk->transformers->children)
                     IASTColumnsTransformer::transform(transformer, columns);
             }
             else if (const auto * qualified_asterisk = child->as<ASTQualifiedAsterisk>())
             {
                 has_asterisks = true;
 
-                if (child->children.size() != 1)
-                    throw Exception("Logical error: qualified asterisk must have exactly one child", ErrorCodes::LOGICAL_ERROR);
-                auto & identifier = child->children[0]->as<ASTTableIdentifier &>();
+                if (!qualified_asterisk->qualifier)
+                    throw Exception("Logical error: qualified asterisk must have a qualifier", ErrorCodes::LOGICAL_ERROR);
+                auto & identifier = qualified_asterisk->qualifier->as<ASTIdentifier &>();
 
                 data.addTableColumns(identifier.name(), columns);
 
-                // QualifiedAsterisk's transformers start to appear at child 1
-                for (auto it = qualified_asterisk->children.begin() + 1; it != qualified_asterisk->children.end(); ++it)
-                {
-                    IASTColumnsTransformer::transform(*it, columns);
-                }
+                for (const auto & transformer : qualified_asterisk->transformers->children)
+                    IASTColumnsTransformer::transform(transformer, columns);
             }
             else if (const auto * columns_list_matcher = child->as<ASTColumnsListMatcher>())
             {
@@ -165,7 +165,7 @@ private:
                 for (const auto & ident : columns_list_matcher->column_list->children)
                     columns.emplace_back(ident->clone());
 
-                for (const auto & transformer : columns_list_matcher->children)
+                for (const auto & transformer : columns_list_matcher->transformers->children)
                     IASTColumnsTransformer::transform(transformer, columns);
             }
             else if (const auto * columns_regexp_matcher = child->as<ASTColumnsRegexpMatcher>())
@@ -178,7 +178,7 @@ private:
                         columns,
                         [&](const String & column_name) { return columns_regexp_matcher->isColumnMatching(column_name); });
 
-                for (const auto & transformer : columns_regexp_matcher->children)
+                for (const auto & transformer : columns_regexp_matcher->transformers->children)
                     IASTColumnsTransformer::transform(transformer, columns);
             }
             else
@@ -409,7 +409,13 @@ private:
         if (!data.done)
         {
             if (data.expression_list->children.empty())
-                data.expression_list->children.emplace_back(std::make_shared<ASTAsterisk>());
+            {
+                auto asterisk = std::make_shared<ASTAsterisk>();
+                asterisk->transformers = std::make_shared<ASTExpressionList>();
+                asterisk->children.push_back(asterisk->transformers);
+                data.expression_list->children.push_back(std::move(asterisk));
+            }
+
             select.setExpression(ASTSelectQuery::Expression::SELECT, std::move(data.expression_list));
         }
         data.done = true;
diff --git a/src/Interpreters/JoinedTables.cpp b/src/Interpreters/JoinedTables.cpp
index f4a98ada199..fa2a4b870fc 100644
--- a/src/Interpreters/JoinedTables.cpp
+++ b/src/Interpreters/JoinedTables.cpp
@@ -96,7 +96,12 @@ void replaceJoinedTable(const ASTSelectQuery & select_query)
 
             auto new_select = addASTChildren<ASTSelectQuery>(*list_of_selects);
             new_select->setExpression(ASTSelectQuery::Expression::SELECT, std::make_shared<ASTExpressionList>());
-            addASTChildren<ASTAsterisk>(*new_select->select());
+
+            auto asterisk = std::make_shared<ASTAsterisk>();
+            asterisk->transformers = std::make_shared<ASTExpressionList>();
+            asterisk->children.push_back(asterisk->transformers);
+            new_select->select()->children.push_back(std::move(asterisk));
+
             new_select->setExpression(ASTSelectQuery::Expression::TABLES, std::make_shared<ASTTablesInSelectQuery>());
 
             auto tables_elem = addASTChildren<ASTTablesInSelectQueryElement>(*new_select->tables());
@@ -154,7 +159,7 @@ private:
 
     static void visit(const ASTQualifiedAsterisk & node, const ASTPtr &, Data & data)
     {
-        auto & identifier = node.children[0]->as<ASTTableIdentifier &>();
+        auto & identifier = node.qualifier->as<ASTIdentifier &>();
         bool rewritten = false;
         for (const auto & table : data)
         {
diff --git a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp
index bc862ed7b38..bcd2c6f44d1 100644
--- a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp
+++ b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp
@@ -155,21 +155,19 @@ void TranslateQualifiedNamesMatcher::visit(ASTFunction & node, const ASTPtr &, D
         func_arguments->children.clear();
 }
 
-void TranslateQualifiedNamesMatcher::visit(const ASTQualifiedAsterisk &, const ASTPtr & ast, Data & data)
+void TranslateQualifiedNamesMatcher::visit(const ASTQualifiedAsterisk & node, const ASTPtr &, Data & data)
 {
-    if (ast->children.empty())
-        throw Exception("Logical error: qualified asterisk must have children", ErrorCodes::LOGICAL_ERROR);
-
-    auto & ident = ast->children[0];
+    if (!node.qualifier)
+        throw Exception("Logical error: qualified asterisk must have a qualifier", ErrorCodes::LOGICAL_ERROR);
 
     /// @note it could contain table alias as table name.
-    DatabaseAndTableWithAlias db_and_table(ident);
+    DatabaseAndTableWithAlias db_and_table(node.qualifier);
 
     for (const auto & known_table : data.tables)
         if (db_and_table.satisfies(known_table.table, true))
             return;
 
-    throw Exception("Unknown qualified identifier: " + ident->getAliasOrColumnName(), ErrorCodes::UNKNOWN_IDENTIFIER);
+    throw Exception("Unknown qualified identifier: " + node.qualifier->getAliasOrColumnName(), ErrorCodes::UNKNOWN_IDENTIFIER);
 }
 
 void TranslateQualifiedNamesMatcher::visit(ASTTableJoin & join, const ASTPtr & , Data & data)
@@ -255,7 +253,7 @@ void TranslateQualifiedNamesMatcher::visit(ASTExpressionList & node, const ASTPt
                 first_table = false;
             }
 
-            for (const auto & transformer : asterisk->children)
+            for (const auto & transformer : asterisk->transformers->children)
                 IASTColumnsTransformer::transform(transformer, columns);
         }
         else if (auto * asterisk_column_list = child->as<ASTColumnsListMatcher>())
@@ -263,7 +261,7 @@ void TranslateQualifiedNamesMatcher::visit(ASTExpressionList & node, const ASTPt
             for (const auto & ident : asterisk_column_list->column_list->children)
                 columns.emplace_back(ident->clone());
 
-            for (const auto & transformer : asterisk_column_list->children)
+            for (const auto & transformer : asterisk_column_list->transformers->children)
                 IASTColumnsTransformer::transform(transformer, columns);
         }
         else if (const auto * asterisk_regexp_pattern = child->as<ASTColumnsRegexpMatcher>())
@@ -281,12 +279,12 @@ void TranslateQualifiedNamesMatcher::visit(ASTExpressionList & node, const ASTPt
                 first_table = false;
             }
 
-            for (const auto & transformer : asterisk_regexp_pattern->children)
+            for (const auto & transformer : asterisk_regexp_pattern->transformers->children)
                 IASTColumnsTransformer::transform(transformer, columns);
         }
         else if (const auto * qualified_asterisk = child->as<ASTQualifiedAsterisk>())
         {
-            DatabaseAndTableWithAlias ident_db_and_name(qualified_asterisk->children[0]);
+            DatabaseAndTableWithAlias ident_db_and_name(qualified_asterisk->qualifier);
 
             for (const auto & table : tables_with_columns)
             {
@@ -298,11 +296,8 @@ void TranslateQualifiedNamesMatcher::visit(ASTExpressionList & node, const ASTPt
                 }
             }
 
-            // QualifiedAsterisk's transformers start to appear at child 1
-            for (auto it = qualified_asterisk->children.begin() + 1; it != qualified_asterisk->children.end(); ++it)
-            {
-                IASTColumnsTransformer::transform(*it, columns);
-            }
+            for (const auto & transformer : qualified_asterisk->transformers->children)
+                IASTColumnsTransformer::transform(transformer, columns);
         }
         else
             columns.emplace_back(child);
diff --git a/src/Parsers/ASTAsterisk.cpp b/src/Parsers/ASTAsterisk.cpp
index e2f45d04fa4..c3c3b5b32e4 100644
--- a/src/Parsers/ASTAsterisk.cpp
+++ b/src/Parsers/ASTAsterisk.cpp
@@ -8,7 +8,12 @@ namespace DB
 ASTPtr ASTAsterisk::clone() const
 {
     auto clone = std::make_shared<ASTAsterisk>(*this);
-    clone->cloneChildren();
+
+    if (expression) { clone->expression = expression->clone(); clone->children.push_back(clone->expression); }
+
+    clone->transformers = transformers->clone();
+    clone->children.push_back(clone->transformers);
+
     return clone;
 }
 
@@ -16,10 +21,16 @@ void ASTAsterisk::appendColumnName(WriteBuffer & ostr) const { ostr.write('*');
 
 void ASTAsterisk::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
 {
+    if (expression)
+    {
+        expression->formatImpl(settings, state, frame);
+        settings.ostr << ".";
+    }
+
     settings.ostr << "*";
 
     /// Format column transformers
-    for (const auto & child : children)
+    for (const auto & child : transformers->children)
     {
         settings.ostr << ' ';
         child->formatImpl(settings, state, frame);
diff --git a/src/Parsers/ASTAsterisk.h b/src/Parsers/ASTAsterisk.h
index 027758ba48c..840b7996536 100644
--- a/src/Parsers/ASTAsterisk.h
+++ b/src/Parsers/ASTAsterisk.h
@@ -16,6 +16,8 @@ public:
     ASTPtr clone() const override;
     void appendColumnName(WriteBuffer & ostr) const override;
 
+    ASTPtr expression;
+    ASTPtr transformers;
 protected:
     void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override;
 };
diff --git a/src/Parsers/ASTColumnsMatcher.cpp b/src/Parsers/ASTColumnsMatcher.cpp
index 0fc6847de68..0be43dd199d 100644
--- a/src/Parsers/ASTColumnsMatcher.cpp
+++ b/src/Parsers/ASTColumnsMatcher.cpp
@@ -18,12 +18,22 @@ namespace ErrorCodes
 ASTPtr ASTColumnsRegexpMatcher::clone() const
 {
     auto clone = std::make_shared<ASTColumnsRegexpMatcher>(*this);
-    clone->cloneChildren();
+
+    if (expression) { clone->expression = expression->clone(); clone->children.push_back(clone->expression); }
+
+    clone->transformers = transformers->clone();
+    clone->children.push_back(clone->transformers);
+
     return clone;
 }
 
 void ASTColumnsRegexpMatcher::appendColumnName(WriteBuffer & ostr) const
 {
+    if (expression)
+    {
+        expression->appendColumnName(ostr);
+        writeCString(".", ostr);
+    }
     writeCString("COLUMNS(", ostr);
     writeQuotedString(original_pattern, ostr);
     writeChar(')', ostr);
@@ -38,12 +48,20 @@ void ASTColumnsRegexpMatcher::updateTreeHashImpl(SipHash & hash_state) const
 
 void ASTColumnsRegexpMatcher::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
 {
-    settings.ostr << (settings.hilite ? hilite_keyword : "") << "COLUMNS" << (settings.hilite ? hilite_none : "") << "(";
+    settings.ostr << (settings.hilite ? hilite_keyword : "");
+
+    if (expression)
+    {
+        expression->formatImpl(settings, state, frame);
+        settings.ostr << ".";
+    }
+
+    settings.ostr << "COLUMNS" << (settings.hilite ? hilite_none : "") << "(";
     settings.ostr << quoteString(original_pattern);
     settings.ostr << ")";
 
     /// Format column transformers
-    for (const auto & child : children)
+    for (const auto & child : transformers->children)
     {
         settings.ostr << ' ';
         child->formatImpl(settings, state, frame);
@@ -60,6 +78,11 @@ void ASTColumnsRegexpMatcher::setPattern(String pattern)
             DB::ErrorCodes::CANNOT_COMPILE_REGEXP);
 }
 
+const String & ASTColumnsRegexpMatcher::getPattern() const
+{
+    return original_pattern;
+}
+
 const std::shared_ptr<re2::RE2> & ASTColumnsRegexpMatcher::getMatcher() const
 {
     return column_matcher;
@@ -73,8 +96,15 @@ bool ASTColumnsRegexpMatcher::isColumnMatching(const String & column_name) const
 ASTPtr ASTColumnsListMatcher::clone() const
 {
     auto clone = std::make_shared<ASTColumnsListMatcher>(*this);
+
+    if (expression) { clone->expression = expression->clone(); clone->children.push_back(clone->expression); }
+
     clone->column_list = column_list->clone();
-    clone->cloneChildren();
+    clone->transformers = transformers->clone();
+
+    clone->children.push_back(clone->column_list);
+    clone->children.push_back(clone->transformers);
+
     return clone;
 }
 
@@ -86,6 +116,11 @@ void ASTColumnsListMatcher::updateTreeHashImpl(SipHash & hash_state) const
 
 void ASTColumnsListMatcher::appendColumnName(WriteBuffer & ostr) const
 {
+    if (expression)
+    {
+        expression->appendColumnName(ostr);
+        writeCString(".", ostr);
+    }
     writeCString("COLUMNS(", ostr);
     for (auto it = column_list->children.begin(); it != column_list->children.end(); ++it)
     {
@@ -99,7 +134,15 @@ void ASTColumnsListMatcher::appendColumnName(WriteBuffer & ostr) const
 
 void ASTColumnsListMatcher::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
 {
-    settings.ostr << (settings.hilite ? hilite_keyword : "") << "COLUMNS" << (settings.hilite ? hilite_none : "") << "(";
+    settings.ostr << (settings.hilite ? hilite_keyword : "");
+
+    if (expression)
+    {
+        expression->formatImpl(settings, state, frame);
+        settings.ostr << ".";
+    }
+
+    settings.ostr << "COLUMNS" << (settings.hilite ? hilite_none : "") << "(";
 
     for (ASTs::const_iterator it = column_list->children.begin(); it != column_list->children.end(); ++it)
     {
@@ -112,7 +155,7 @@ void ASTColumnsListMatcher::formatImpl(const FormatSettings & settings, FormatSt
     settings.ostr << ")";
 
     /// Format column transformers
-    for (const auto & child : children)
+    for (const auto & child : transformers->children)
     {
         settings.ostr << ' ';
         child->formatImpl(settings, state, frame);
@@ -122,22 +165,31 @@ void ASTColumnsListMatcher::formatImpl(const FormatSettings & settings, FormatSt
 ASTPtr ASTQualifiedColumnsRegexpMatcher::clone() const
 {
     auto clone = std::make_shared<ASTQualifiedColumnsRegexpMatcher>(*this);
-    clone->cloneChildren();
+
+    clone->qualifier = qualifier->clone();
+    clone->transformers = transformers->clone();
+
+    clone->children.push_back(clone->qualifier);
+    clone->children.push_back(clone->transformers);
+
     return clone;
 }
 
 void ASTQualifiedColumnsRegexpMatcher::appendColumnName(WriteBuffer & ostr) const
 {
-    const auto & qualifier = children.at(0);
     qualifier->appendColumnName(ostr);
     writeCString(".COLUMNS(", ostr);
     writeQuotedString(original_pattern, ostr);
     writeChar(')', ostr);
 }
 
-void ASTQualifiedColumnsRegexpMatcher::setPattern(String pattern)
+void ASTQualifiedColumnsRegexpMatcher::setPattern(String pattern, bool set_matcher)
 {
     original_pattern = std::move(pattern);
+
+    if (!set_matcher)
+        return;
+
     column_matcher = std::make_shared<RE2>(original_pattern, RE2::Quiet);
     if (!column_matcher->ok())
         throw DB::Exception(
@@ -166,7 +218,6 @@ void ASTQualifiedColumnsRegexpMatcher::formatImpl(const FormatSettings & setting
 {
     settings.ostr << (settings.hilite ? hilite_keyword : "");
 
-    const auto & qualifier = children.at(0);
     qualifier->formatImpl(settings, state, frame);
 
     settings.ostr << ".COLUMNS" << (settings.hilite ? hilite_none : "") << "(";
@@ -174,11 +225,8 @@ void ASTQualifiedColumnsRegexpMatcher::formatImpl(const FormatSettings & setting
     settings.ostr << ")";
 
     /// Format column transformers
-    size_t children_size = children.size();
-
-    for (size_t i = 1; i < children_size; ++i)
+    for (const auto & child : transformers->children)
     {
-        const auto & child = children[i];
         settings.ostr << ' ';
         child->formatImpl(settings, state, frame);
     }
@@ -187,14 +235,20 @@ void ASTQualifiedColumnsRegexpMatcher::formatImpl(const FormatSettings & setting
 ASTPtr ASTQualifiedColumnsListMatcher::clone() const
 {
     auto clone = std::make_shared<ASTQualifiedColumnsListMatcher>(*this);
+
+    clone->qualifier = qualifier->clone();
     clone->column_list = column_list->clone();
-    clone->cloneChildren();
+    clone->transformers = transformers->clone();
+
+    clone->children.push_back(clone->qualifier);
+    clone->children.push_back(clone->column_list);
+    clone->children.push_back(clone->transformers);
+
     return clone;
 }
 
 void ASTQualifiedColumnsListMatcher::appendColumnName(WriteBuffer & ostr) const
 {
-    const auto & qualifier = children.at(0);
     qualifier->appendColumnName(ostr);
     writeCString(".COLUMNS(", ostr);
 
@@ -217,10 +271,7 @@ void ASTQualifiedColumnsListMatcher::updateTreeHashImpl(SipHash & hash_state) co
 void ASTQualifiedColumnsListMatcher::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
 {
     settings.ostr << (settings.hilite ? hilite_keyword : "");
-
-    const auto & qualifier = children.at(0);
     qualifier->formatImpl(settings, state, frame);
-
     settings.ostr << ".COLUMNS" << (settings.hilite ? hilite_none : "") << "(";
 
     for (ASTs::const_iterator it = column_list->children.begin(); it != column_list->children.end(); ++it)
@@ -233,11 +284,8 @@ void ASTQualifiedColumnsListMatcher::formatImpl(const FormatSettings & settings,
     settings.ostr << ")";
 
     /// Format column transformers
-    size_t children_size = children.size();
-
-    for (size_t i = 1; i < children_size; ++i)
+    for (const auto & child : transformers->children)
     {
-        const auto & child = children[i];
         settings.ostr << ' ';
         child->formatImpl(settings, state, frame);
     }
diff --git a/src/Parsers/ASTColumnsMatcher.h b/src/Parsers/ASTColumnsMatcher.h
index 7ce246608b9..d590292e7f9 100644
--- a/src/Parsers/ASTColumnsMatcher.h
+++ b/src/Parsers/ASTColumnsMatcher.h
@@ -24,10 +24,13 @@ public:
 
     void appendColumnName(WriteBuffer & ostr) const override;
     void setPattern(String pattern);
+    const String & getPattern() const;
     const std::shared_ptr<re2::RE2> & getMatcher() const;
     bool isColumnMatching(const String & column_name) const;
     void updateTreeHashImpl(SipHash & hash_state) const override;
 
+    ASTPtr expression;
+    ASTPtr transformers;
 protected:
     void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override;
 
@@ -45,7 +48,9 @@ public:
     void appendColumnName(WriteBuffer & ostr) const override;
     void updateTreeHashImpl(SipHash & hash_state) const override;
 
+    ASTPtr expression;
     ASTPtr column_list;
+    ASTPtr transformers;
 protected:
     void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override;
 };
@@ -59,10 +64,12 @@ public:
 
     void appendColumnName(WriteBuffer & ostr) const override;
     const std::shared_ptr<re2::RE2> & getMatcher() const;
-    void setPattern(String pattern);
+    void setPattern(String pattern, bool set_matcher = true);
     void setMatcher(std::shared_ptr<re2::RE2> matcher);
     void updateTreeHashImpl(SipHash & hash_state) const override;
 
+    ASTPtr qualifier;
+    ASTPtr transformers;
 protected:
     void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override;
 
@@ -80,7 +87,9 @@ public:
     void appendColumnName(WriteBuffer & ostr) const override;
     void updateTreeHashImpl(SipHash & hash_state) const override;
 
+    ASTPtr qualifier;
     ASTPtr column_list;
+    ASTPtr transformers;
 protected:
     void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override;
 };
diff --git a/src/Parsers/ASTQualifiedAsterisk.cpp b/src/Parsers/ASTQualifiedAsterisk.cpp
index b755e4eb98c..c14d2259465 100644
--- a/src/Parsers/ASTQualifiedAsterisk.cpp
+++ b/src/Parsers/ASTQualifiedAsterisk.cpp
@@ -7,22 +7,20 @@ namespace DB
 
 void ASTQualifiedAsterisk::appendColumnName(WriteBuffer & ostr) const
 {
-    const auto & qualifier = children.at(0);
     qualifier->appendColumnName(ostr);
     writeCString(".*", ostr);
 }
 
 void ASTQualifiedAsterisk::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
 {
-    const auto & qualifier = children.at(0);
     qualifier->formatImpl(settings, state, frame);
     settings.ostr << ".*";
 
     /// Format column transformers
-    for (ASTs::const_iterator it = children.begin() + 1; it != children.end(); ++it)
+    for (const auto & child : transformers->children)
     {
         settings.ostr << ' ';
-        (*it)->formatImpl(settings, state, frame);
+        child->formatImpl(settings, state, frame);
     }
 }
 
diff --git a/src/Parsers/ASTQualifiedAsterisk.h b/src/Parsers/ASTQualifiedAsterisk.h
index 1b644532f53..a7016b900e9 100644
--- a/src/Parsers/ASTQualifiedAsterisk.h
+++ b/src/Parsers/ASTQualifiedAsterisk.h
@@ -17,11 +17,19 @@ public:
     ASTPtr clone() const override
     {
         auto clone = std::make_shared<ASTQualifiedAsterisk>(*this);
-        clone->cloneChildren();
+
+        clone->qualifier = qualifier->clone();
+        clone->transformers = transformers->clone();
+
+        clone->children.push_back(clone->qualifier);
+        clone->children.push_back(clone->transformers);
+
         return clone;
     }
     void appendColumnName(WriteBuffer & ostr) const override;
 
+    ASTPtr qualifier;
+    ASTPtr transformers;
 protected:
     void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
 };
diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp
index c4e07ea2e15..46550691ce4 100644
--- a/src/Parsers/ExpressionElementParsers.cpp
+++ b/src/Parsers/ExpressionElementParsers.cpp
@@ -80,7 +80,12 @@ static ASTPtr buildSelectFromTableFunction(const std::shared_ptr<ASTFunction> &
     {
         auto select_ast = std::make_shared<ASTSelectQuery>();
         select_ast->setExpression(ASTSelectQuery::Expression::SELECT, std::make_shared<ASTExpressionList>());
-        select_ast->select()->children.push_back(std::make_shared<ASTAsterisk>());
+
+        auto asterisk = std::make_shared<ASTAsterisk>();
+        asterisk->transformers = std::make_shared<ASTExpressionList>();
+        asterisk->children.push_back(asterisk->transformers);
+
+        select_ast->select()->children.push_back(std::move(asterisk));
 
         auto list_of_selects = std::make_shared<ASTExpressionList>();
         list_of_selects->children.push_back(select_ast);
@@ -1402,13 +1407,18 @@ bool ParserAsterisk::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
     {
         ++pos;
         auto asterisk = std::make_shared<ASTAsterisk>();
+        auto transformers = std::make_shared<ASTExpressionList>();
         ParserColumnsTransformers transformers_p(allowed_transformers);
         ASTPtr transformer;
         while (transformers_p.parse(pos, transformer, expected))
         {
-            asterisk->children.push_back(transformer);
+            transformers->children.push_back(transformer);
         }
-        node = asterisk;
+
+        asterisk->transformers = std::move(transformers);
+        asterisk->children.push_back(asterisk->transformers);
+
+        node = std::move(asterisk);
         return true;
     }
     return false;
@@ -1417,7 +1427,7 @@ bool ParserAsterisk::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
 
 bool ParserQualifiedAsterisk::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
 {
-    if (!ParserCompoundIdentifier(true, true).parse(pos, node, expected))
+    if (!ParserCompoundIdentifier(false, true).parse(pos, node, expected))
         return false;
 
     if (pos->type != TokenType::Dot)
@@ -1429,13 +1439,19 @@ bool ParserQualifiedAsterisk::parseImpl(Pos & pos, ASTPtr & node, Expected & exp
     ++pos;
 
     auto res = std::make_shared<ASTQualifiedAsterisk>();
-    res->children.push_back(node);
+    auto transformers = std::make_shared<ASTExpressionList>();
     ParserColumnsTransformers transformers_p;
     ASTPtr transformer;
     while (transformers_p.parse(pos, transformer, expected))
     {
-        res->children.push_back(transformer);
+        transformers->children.push_back(transformer);
     }
+
+    res->qualifier = std::move(node);
+    res->transformers = std::move(transformers);
+    res->children.push_back(res->qualifier);
+    res->children.push_back(res->transformers);
+
     node = std::move(res);
     return true;
 }
@@ -1459,28 +1475,37 @@ static bool parseColumnsMatcherBody(IParser::Pos & pos, ASTPtr & node, Expected
         return false;
     ++pos;
 
+    auto transformers = std::make_shared<ASTExpressionList>();
+    ParserColumnsTransformers transformers_p(allowed_transformers);
+    ASTPtr transformer;
+    while (transformers_p.parse(pos, transformer, expected))
+    {
+        transformers->children.push_back(transformer);
+    }
+
     ASTPtr res;
     if (column_list)
     {
         auto list_matcher = std::make_shared<ASTColumnsListMatcher>();
-        list_matcher->column_list = column_list;
-        res = list_matcher;
+
+        list_matcher->column_list = std::move(column_list);
+        list_matcher->transformers = std::move(transformers);
+        list_matcher->children.push_back(list_matcher->column_list);
+        list_matcher->children.push_back(list_matcher->transformers);
+
+        node = std::move(list_matcher);
     }
     else
     {
         auto regexp_matcher = std::make_shared<ASTColumnsRegexpMatcher>();
         regexp_matcher->setPattern(regex_node->as<ASTLiteral &>().value.get<String>());
-        res = regexp_matcher;
+
+        regexp_matcher->transformers = std::move(transformers);
+        regexp_matcher->children.push_back(regexp_matcher->transformers);
+
+        node = std::move(regexp_matcher);
     }
 
-    ParserColumnsTransformers transformers_p(allowed_transformers);
-    ASTPtr transformer;
-    while (transformers_p.parse(pos, transformer, expected))
-    {
-        res->children.push_back(transformer);
-    }
-
-    node = std::move(res);
     return true;
 }
 
@@ -1496,29 +1521,19 @@ bool ParserColumnsMatcher::parseImpl(Pos & pos, ASTPtr & node, Expected & expect
 
 bool ParserQualifiedColumnsMatcher::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
 {
-    if (!ParserCompoundIdentifier(true, true).parse(pos, node, expected))
+    if (!ParserCompoundIdentifier(false, true).parse(pos, node, expected))
         return false;
 
     auto identifier_node = node;
-    const auto & identifier_node_typed = identifier_node->as<ASTTableIdentifier &>();
+    auto & identifier_node_typed = identifier_node->as<ASTIdentifier &>();
+    auto & name_parts = identifier_node_typed.name_parts;
 
     /// ParserCompoundIdentifier parse identifier.COLUMNS
-    if (identifier_node_typed.name_parts.size() == 1 || identifier_node_typed.name_parts.back() != "COLUMNS")
+    if (name_parts.size() == 1 || name_parts.back() != "COLUMNS")
         return false;
 
-    /// TODO: ASTTableIdentifier can contain only 2 parts
-
-    if (identifier_node_typed.name_parts.size() == 2)
-    {
-        auto table_name = identifier_node_typed.name_parts[0];
-        identifier_node = std::make_shared<ASTTableIdentifier>(table_name);
-    }
-    else
-    {
-        throw Exception(ErrorCodes::LOGICAL_ERROR,
-            "Expected identifier to contain no more than 2 parts. Actual {}",
-            identifier_node_typed.full_name);
-    }
+    name_parts.pop_back();
+    identifier_node = std::make_shared<ASTIdentifier>(std::move(name_parts), false, std::move(node->children));
 
     if (!parseColumnsMatcherBody(pos, node, expected, allowed_transformers))
         return false;
@@ -1526,28 +1541,29 @@ bool ParserQualifiedColumnsMatcher::parseImpl(Pos & pos, ASTPtr & node, Expected
     if (auto * columns_list_matcher = node->as<ASTColumnsListMatcher>())
     {
         auto result = std::make_shared<ASTQualifiedColumnsListMatcher>();
+        result->qualifier = std::move(identifier_node);
         result->column_list = std::move(columns_list_matcher->column_list);
+        result->transformers = std::move(columns_list_matcher->transformers);
 
-        result->children.reserve(columns_list_matcher->children.size() + 1);
-        result->children.push_back(std::move(identifier_node));
+        result->children.push_back(result->qualifier);
+        result->children.push_back(result->column_list);
+        result->children.push_back(result->transformers);
 
-        for (auto && child : columns_list_matcher->children)
-            result->children.push_back(std::move(child));
-
-        node = result;
+        node = std::move(result);
     }
     else if (auto * column_regexp_matcher = node->as<ASTColumnsRegexpMatcher>())
     {
         auto result = std::make_shared<ASTQualifiedColumnsRegexpMatcher>();
+        result->setPattern(column_regexp_matcher->getPattern(), false);
         result->setMatcher(column_regexp_matcher->getMatcher());
 
-        result->children.reserve(column_regexp_matcher->children.size() + 1);
-        result->children.push_back(std::move(identifier_node));
+        result->qualifier = std::move(identifier_node);
+        result->transformers = std::move(column_regexp_matcher->transformers);
 
-        for (auto && child : column_regexp_matcher->children)
-            result->children.push_back(std::move(child));
+        result->children.push_back(result->qualifier);
+        result->children.push_back(result->transformers);
 
-        node = result;
+        node = std::move(result);
     }
     else
     {
diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp
index 4af4dabb12e..24f24c2b28b 100644
--- a/src/Parsers/ExpressionListParsers.cpp
+++ b/src/Parsers/ExpressionListParsers.cpp
@@ -4,6 +4,7 @@
 #include <Parsers/ParserSetQuery.h>
 
 #include <Parsers/ASTAsterisk.h>
+#include <Parsers/ASTColumnsMatcher.h>
 #include <Parsers/ASTExpressionList.h>
 #include <Parsers/ASTFunction.h>
 #include <Parsers/ASTFunctionWithKeyValueArguments.h>
@@ -193,7 +194,11 @@ static bool modifyAST(ASTPtr ast, SubqueryFunctionType type)
         return false;
 
     /// subquery --> (SELECT aggregate_function(*) FROM subquery)
-    auto aggregate_function = makeASTFunction(aggregate_function_name, std::make_shared<ASTAsterisk>());
+    auto asterisk = std::make_shared<ASTAsterisk>();
+    asterisk->transformers = std::make_shared<ASTExpressionList>();
+    asterisk->children.push_back(asterisk->transformers);
+
+    auto aggregate_function = makeASTFunction(aggregate_function_name, std::move(asterisk));
     auto subquery_node = function->children[0]->children[1];
 
     auto table_expression = std::make_shared<ASTTableExpression>();
@@ -2110,7 +2115,7 @@ struct ParserExpressionImpl
     using Layers = std::vector<std::unique_ptr<Layer>>;
 
     Action tryParseOperand(Layers & layers, IParser::Pos & pos, Expected & expected);
-    static Action tryParseOperator(Layers & layers, IParser::Pos & pos, Expected & expected);
+    Action tryParseOperator(Layers & layers, IParser::Pos & pos, Expected & expected);
 };
 
 
@@ -2434,8 +2439,6 @@ Action ParserExpressionImpl::tryParseOperand(Layers & layers, IParser::Pos & pos
 
 Action ParserExpressionImpl::tryParseOperator(Layers & layers, IParser::Pos & pos, Expected & expected)
 {
-    ASTPtr tmp;
-
     /// ParserExpression can be called in this part of the query:
     ///  ALTER TABLE partition_all2 CLEAR INDEX [ p ] IN PARTITION ALL
     ///
@@ -2455,17 +2458,17 @@ Action ParserExpressionImpl::tryParseOperator(Layers & layers, IParser::Pos & po
 
     if (cur_op == operators_table.end())
     {
+        ASTPtr alias;
         ParserAlias alias_parser(layers.back()->allow_alias_without_as_keyword);
-        auto old_pos = pos;
+
         if (layers.back()->allow_alias &&
             !layers.back()->parsed_alias &&
-            alias_parser.parse(pos, tmp, expected) &&
-            layers.back()->insertAlias(tmp))
+            alias_parser.parse(pos, alias, expected) &&
+            layers.back()->insertAlias(alias))
         {
             layers.back()->parsed_alias = true;
             return Action::OPERATOR;
         }
-        pos = old_pos;
         return Action::NONE;
     }
 
@@ -2529,33 +2532,57 @@ Action ParserExpressionImpl::tryParseOperator(Layers & layers, IParser::Pos & po
         layers.back()->pushOperand(function);
     }
 
+    /// Dot (TupleElement operator) can be a beginning of a .* or .COLUMNS expressions
+    if (op.type == OperatorType::TupleElement)
+    {
+        ASTPtr tmp;
+        if (asterisk_parser.parse(pos, tmp, expected) ||
+            columns_matcher_parser.parse(pos, tmp, expected))
+        {
+            if (auto * asterisk = tmp->as<ASTAsterisk>())
+            {
+                if (!layers.back()->popOperand(asterisk->expression))
+                    return Action::NONE;
+            }
+            else if (auto * columns_list_matcher = tmp->as<ASTColumnsListMatcher>())
+            {
+                if (!layers.back()->popOperand(columns_list_matcher->expression))
+                    return Action::NONE;
+            }
+            else if (auto * columns_regexp_matcher = tmp->as<ASTColumnsRegexpMatcher>())
+            {
+                if (!layers.back()->popOperand(columns_regexp_matcher->expression))
+                    return Action::NONE;
+            }
+
+            layers.back()->pushOperand(std::move(tmp));
+            return Action::OPERATOR;
+        }
+    }
+
     layers.back()->pushOperator(op);
 
-    if (op.type == OperatorType::ArrayElement)
-        layers.push_back(std::make_unique<ArrayElementLayer>());
-
-
-    Action next = Action::OPERAND;
-
     /// isNull & isNotNull are postfix unary operators
     if (op.type == OperatorType::IsNull)
-        next = Action::OPERATOR;
-
-    if (op.type == OperatorType::StartBetween || op.type == OperatorType::StartNotBetween)
-        layers.back()->between_counter++;
+        return Action::OPERATOR;
 
     if (op.type == OperatorType::Cast)
     {
-        next = Action::OPERATOR;
-
         ASTPtr type_ast;
         if (!ParserDataType().parse(pos, type_ast, expected))
             return Action::NONE;
 
         layers.back()->pushOperand(std::make_shared<ASTLiteral>(queryToString(type_ast)));
+        return Action::OPERATOR;
     }
 
-    return next;
+    if (op.type == OperatorType::ArrayElement)
+        layers.push_back(std::make_unique<ArrayElementLayer>());
+
+    if (op.type == OperatorType::StartBetween || op.type == OperatorType::StartNotBetween)
+        layers.back()->between_counter++;
+
+    return Action::OPERAND;
 }
 
 }
diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp
index c0bc5ad8da9..64604d80a8b 100644
--- a/src/Storages/WindowView/StorageWindowView.cpp
+++ b/src/Storages/WindowView/StorageWindowView.cpp
@@ -327,7 +327,10 @@ namespace
     {
         auto fetch_query = std::make_shared<ASTSelectQuery>();
         auto select = std::make_shared<ASTExpressionList>();
-        select->children.push_back(std::make_shared<ASTAsterisk>());
+        auto asterisk = std::make_shared<ASTAsterisk>();
+        asterisk->transformers = std::make_shared<ASTExpressionList>();
+        asterisk->children.push_back(asterisk->transformers);
+        select->children.push_back(asterisk);
         fetch_query->setExpression(ASTSelectQuery::Expression::SELECT, select);
         fetch_query->setExpression(ASTSelectQuery::Expression::TABLES, std::make_shared<ASTTablesInSelectQuery>());
         auto tables_elem = std::make_shared<ASTTablesInSelectQueryElement>();

From d96ed50610d81c5158eae6886671b9fab71bc0f1 Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Wed, 16 Nov 2022 13:21:07 +0000
Subject: [PATCH 019/158] fix race

---
 src/Storages/MergeTree/MergeTreeData.cpp | 84 +++++++++++++-----------
 src/Storages/MergeTree/MergeTreeData.h   |  4 +-
 2 files changed, 47 insertions(+), 41 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 9198f180818..7c4757d1224 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -1009,7 +1009,7 @@ void MergeTreeData::PartLoadingTree::traverse(bool recursive, Func && func)
 }
 
 MergeTreeData::PartLoadingTree
-MergeTreeData::PartLoadingTree::build(std::vector<PartLoadingInfo> nodes)
+MergeTreeData::PartLoadingTree::build(PartLoadingInfos nodes)
 {
     std::sort(nodes.begin(), nodes.end(), [](const auto & lhs, const auto & rhs)
     {
@@ -1029,12 +1029,12 @@ static std::optional<size_t> calculatePartSizeSafe(
 {
     try
     {
-        return part->data_part_storage->calculateTotalSizeOnDisk();
+        return part->getDataPartStorage().calculateTotalSizeOnDisk();
     }
     catch (...)
     {
         tryLogCurrentException(log, fmt::format("while calculating part size {} on path {}",
-            part->name, part->data_part_storage->getRelativePath()));
+            part->name, part->getDataPartStorage().getRelativePath()));
         return {};
     }
 }
@@ -1091,7 +1091,7 @@ MergeTreeData::LoadPartResult MergeTreeData::loadDataPart(
         LOG_WARNING(log,
             "Detaching stale part {} (size: {}), which should have been deleted after a move. "
             "That can only happen after unclean restart of ClickHouse after move of a part having an operation blocking that stale copy of part.",
-            res.part->data_part_storage->getFullPath(), part_size_str);
+            res.part->getDataPartStorage().getFullPath(), part_size_str);
 
         return res;
     }
@@ -1380,9 +1380,6 @@ void MergeTreeData::loadDataPartsFromWAL(MutableDataPartsVector & parts_from_wal
     for (auto & part : parts_from_wal)
     {
         part->modification_time = time(nullptr);
-        /// Assume that all parts are Active, covered parts will be detected and marked as Outdated later
-        part->setState(DataPartState::Active);
-
         auto lo = data_parts_by_state_and_info.lower_bound(DataPartStateAndInfo{DataPartState::Active, part->info});
 
         if (lo != data_parts_by_state_and_info.begin() && (*std::prev(lo))->info.contains(part->info))
@@ -1391,6 +1388,7 @@ void MergeTreeData::loadDataPartsFromWAL(MutableDataPartsVector & parts_from_wal
         if (lo != data_parts_by_state_and_info.end() && (*lo)->info.contains(part->info))
             continue;
 
+        part->setState(DataPartState::Active);
         auto [it, inserted] = data_parts_indexes.insert(part);
 
         if (!inserted)
@@ -1400,8 +1398,10 @@ void MergeTreeData::loadDataPartsFromWAL(MutableDataPartsVector & parts_from_wal
             else
                 throw Exception("Part " + part->name + " already exists but with different checksums", ErrorCodes::DUPLICATE_DATA_PART);
         }
-
-        addPartContributionToDataVolume(part);
+        else
+        {
+            addPartContributionToDataVolume(part);
+        }
     }
 }
 
@@ -1467,13 +1467,16 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
     }
 
     ThreadPool pool(disks.size());
-    std::vector<PartLoadingTree::PartLoadingInfo> parts_with_disks;
+    std::vector<PartLoadingTree::PartLoadingInfos> parts_to_load_by_disk(disks.size());
 
-    for (const auto & disk_ptr : disks)
+    for (size_t i = 0; i < disks.size(); ++i)
     {
+        const auto & disk_ptr = disks[i];
         if (disk_ptr->isBroken())
             continue;
 
+        auto & disk_parts = parts_to_load_by_disk[i];
+
         pool.scheduleOrThrowOnError([&, disk_ptr]()
         {
             for (auto it = disk_ptr->iterateDirectory(relative_data_path); it->isValid(); it->next())
@@ -1485,14 +1488,18 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
                     continue;
 
                 if (auto part_info = MergeTreePartInfo::tryParsePartName(it->name(), format_version))
-                    parts_with_disks.emplace_back(*part_info, it->name(), disk_ptr);
+                    disk_parts.emplace_back(*part_info, it->name(), disk_ptr);
             }
         });
     }
 
     pool.wait();
 
-    auto loading_tree = PartLoadingTree::build(std::move(parts_with_disks));
+    PartLoadingTree::PartLoadingInfos parts_to_load;
+    for (auto & disk_parts : parts_to_load_by_disk)
+        std::move(disk_parts.begin(), disk_parts.end(), std::back_inserter(parts_to_load));
+
+    auto loading_tree = PartLoadingTree::build(std::move(parts_to_load));
     /// Collect parts by disks' names.
     std::map<String, PartLoadingTreeNodes> disk_part_map;
 
@@ -1558,15 +1565,17 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
 
     if (settings->in_memory_parts_enable_wal)
     {
-        std::map<String, MutableDataPartsVector> disk_wal_part_map;
-
+        pool.setMaxThreads(disks.size());
+        std::vector<MutableDataPartsVector> disks_wal_parts(disks.size());
         std::mutex wal_init_lock;
-        for (const auto & disk_ptr : disks)
+
+        for (size_t i = 0; i < disks.size(); ++i)
         {
+            const auto & disk_ptr = disks[i];
             if (disk_ptr->isBroken())
                 continue;
 
-            auto & disk_wal_parts = disk_wal_part_map[disk_ptr->getName()];
+            auto & disk_wal_parts = disks_wal_parts[i];
 
             pool.scheduleOrThrowOnError([&, disk_ptr]()
             {
@@ -1600,9 +1609,8 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
         pool.wait();
 
         MutableDataPartsVector parts_from_wal;
-        for (auto & [_, disk_wal_parts] : disk_wal_part_map)
-            parts_from_wal.insert(
-                parts_from_wal.end(), std::make_move_iterator(disk_wal_parts.begin()), std::make_move_iterator(disk_wal_parts.end()));
+        for (auto & disk_wal_parts : disks_wal_parts)
+            std::move(disk_wal_parts.begin(), disk_wal_parts.end(), std::back_inserter(parts_from_wal));
 
         loadDataPartsFromWAL(parts_from_wal);
         num_parts += parts_from_wal.size();
@@ -1672,19 +1680,11 @@ void MergeTreeData::loadOutdatedDataParts(PartLoadingTreeNodes parts_to_load)
         auto res = loadDataPart(part->info, part->name, part->disk, MergeTreeDataPartState::Outdated, data_parts_mutex);
 
         if (res.is_broken)
-        {
-            auto builder = res.part->data_part_storage->getBuilder();
-            res.part->renameToDetached("broken-on-start", builder); /// detached parts must not have '_' in prefixes
-            builder->commit();
-        }
+            res.part->renameToDetached("broken-on-start"); /// detached parts must not have '_' in prefixes
         else if (res.part->is_duplicate)
-        {
             res.part->remove();
-        }
         else
-        {
             preparePartForRemoval(res.part);
-        }
     }
 
     LOG_DEBUG(log, "Loaded {} outdated data parts", parts_to_load.size());
@@ -1692,13 +1692,19 @@ void MergeTreeData::loadOutdatedDataParts(PartLoadingTreeNodes parts_to_load)
 
 void MergeTreeData::scheduleOutdatedDataPartsLoadingJob(BackgroundJobsAssignee & assignee)
 {
-    std::lock_guard lock(unloaded_outdated_parts_mutex);
-    if (unloaded_outdated_parts.empty())
+    PartLoadingTreeNodes parts_to_load;
+
+    {
+        std::lock_guard lock(unloaded_outdated_parts_mutex);
+        parts_to_load = unloaded_outdated_parts;
+    }
+
+    if (parts_to_load.empty())
         return;
 
     auto share_lock = lockForShare(RWLockImpl::NO_QUERY, getSettings()->lock_acquire_timeout_for_background_operations);
     assignee.scheduleCommonTask(std::make_shared<ExecutableLambdaAdapter>(
-        [this, share_lock, parts_to_load = std::move(unloaded_outdated_parts)]
+        [this, share_lock, parts_to_load = std::move(parts_to_load)]
         {
             loadOutdatedDataParts(std::move(parts_to_load));
             return false;
@@ -1819,12 +1825,12 @@ scope_guard MergeTreeData::getTemporaryPartDirectoryHolder(const String & part_d
     return [this, part_dir_name]() { temporary_parts.remove(part_dir_name); };
 }
 
-MergeTreeData::MutableDataPartPtr MergeTreeData::preparePartForRemoval(const DataPartPtr & part)
+MergeTreeData::MutableDataPartPtr MergeTreeData::asMutableDeletingPart(const DataPartPtr & part)
 {
     auto state = part->getState();
     if (state != DataPartState::Deleting && state != DataPartState::DeleteOnDestroy)
         throw Exception(ErrorCodes::LOGICAL_ERROR,
-            "Cannot remove part {}, because it has state: {}", part->name, magic_enum::enum_name(part->getState()));
+            "Cannot remove part {}, because it has state: {}", part->name, magic_enum::enum_name(state));
 
     return std::const_pointer_cast<IMergeTreeDataPart>(part);
 }
@@ -2087,7 +2093,7 @@ void MergeTreeData::clearPartsFromFilesystemImpl(const DataPartsVector & parts_t
                 if (thread_group)
                     CurrentThread::attachToIfDetached(thread_group);
 
-                preparePartForRemoval(part)->remove();
+                asMutableDeletingPart(part)->remove();
                 if (part_names_succeed)
                 {
                     std::lock_guard lock(part_names_mutex);
@@ -2103,7 +2109,7 @@ void MergeTreeData::clearPartsFromFilesystemImpl(const DataPartsVector & parts_t
         LOG_DEBUG(log, "Removing {} parts from filesystem: {}", parts_to_remove.size(), fmt::join(parts_to_remove, ", "));
         for (const DataPartPtr & part : parts_to_remove)
         {
-            preparePartForRemoval(part)->remove();
+            asMutableDeletingPart(part)->remove();
             if (part_names_succeed)
                 part_names_succeed->insert(part->name);
         }
@@ -3427,7 +3433,7 @@ void MergeTreeData::forcefullyMovePartToDetachedAndRemoveFromMemory(const MergeT
     }
 
     modifyPartState(it_part, DataPartState::Deleting);
-    preparePartForRemoval(part)->renameToDetached(prefix);
+    asMutableDeletingPart(part)->renameToDetached(prefix);
     data_parts_indexes.erase(it_part);
 
     if (restore_covered && part->info.level == 0)
@@ -3581,7 +3587,7 @@ void MergeTreeData::tryRemovePartImmediately(DataPartPtr && part)
 
     try
     {
-        preparePartForRemoval(part_to_delete)->remove();
+        asMutableDeletingPart(part_to_delete)->remove();
     }
     catch (...)
     {
@@ -3813,7 +3819,7 @@ void MergeTreeData::swapActivePart(MergeTreeData::DataPartPtr part_copy)
             /// All other locks are taken in StorageReplicatedMergeTree
             lockSharedData(*part_copy);
 
-            preparePartForRemoval(original_active_part)->writeDeleteOnDestroyMarker();
+            asMutableDeletingPart(original_active_part)->writeDeleteOnDestroyMarker();
             return;
         }
     }
diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h
index c18f7d5a5d6..5bab007a71f 100644
--- a/src/Storages/MergeTree/MergeTreeData.h
+++ b/src/Storages/MergeTree/MergeTreeData.h
@@ -1301,6 +1301,7 @@ protected:
 
         using NodePtr = std::shared_ptr<Node>;
         using PartLoadingInfo = std::tuple<MergeTreePartInfo, String, DiskPtr>;
+        using PartLoadingInfos = std::vector<PartLoadingInfo>;
 
         static PartLoadingTree build(std::vector<PartLoadingInfo> nodes);
 
@@ -1413,7 +1414,6 @@ private:
         const MergeTreeSettingsPtr & settings);
 
     void loadDataPartsFromWAL(MutableDataPartsVector & parts_from_wal);
-
     /// Create zero-copy exclusive lock for part and disk. Useful for coordination of
     /// distributed operations which can lead to data duplication. Implemented only in ReplicatedMergeTree.
     virtual std::optional<ZeroCopyLock> tryCreateZeroCopyExclusiveLock(const String &, const DiskPtr &) { return std::nullopt; }
@@ -1423,7 +1423,7 @@ private:
     /// Otherwise, in non-parallel case will break and return.
     void clearPartsFromFilesystemImpl(const DataPartsVector & parts, NameSet * part_names_succeed);
 
-    static MutableDataPartPtr preparePartForRemoval(const DataPartPtr & part);
+    static MutableDataPartPtr asMutableDeletingPart(const DataPartPtr & part);
 
     TemporaryParts temporary_parts;
 };

From 92ac9385b993fa932aabd1fae75b863c05132a3a Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Mon, 21 Nov 2022 16:09:51 +0000
Subject: [PATCH 020/158] wait loading of outdated parts for partition commands

---
 src/Storages/MergeTree/MergeTreeData.cpp      | 78 +++++++++++-----
 src/Storages/MergeTree/MergeTreeData.h        | 11 ++-
 .../ReplicatedMergeTreeRestartingThread.cpp   |  3 -
 src/Storages/StorageMergeTree.cpp             |  8 +-
 src/Storages/StorageReplicatedMergeTree.cpp   | 10 +-
 ...482_load_parts_parition_commands.reference | 58 ++++++++++++
 .../02482_load_parts_parition_commands.sh     | 91 +++++++++++++++++++
 .../02482_load_parts_refcounts.reference      |  1 +
 .../0_stateless/02482_load_parts_refcounts.sh | 54 +++++++++++
 9 files changed, 279 insertions(+), 35 deletions(-)
 create mode 100644 tests/queries/0_stateless/02482_load_parts_parition_commands.reference
 create mode 100755 tests/queries/0_stateless/02482_load_parts_parition_commands.sh
 create mode 100644 tests/queries/0_stateless/02482_load_parts_refcounts.reference
 create mode 100755 tests/queries/0_stateless/02482_load_parts_refcounts.sh

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 7c4757d1224..30f62adaae5 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -1290,19 +1290,18 @@ std::vector<MergeTreeData::LoadPartResult> MergeTreeData::loadDataPartsFromDisk(
     }
 
     assert(threads_queue.empty());
-    assert(std::all_of(threads_parts.begin(), threads_parts.end(), [](const std::vector<std::pair<String, DiskPtr>> & parts)
+    assert(std::all_of(threads_parts.begin(), threads_parts.end(), [](const auto & parts)
     {
         return !parts.empty();
     }));
 
-    std::mutex loaded_parts_mutex;
+    std::mutex part_select_mutex;
+    std::mutex part_loading_mutex;
+
     std::vector<LoadPartResult> loaded_parts;
 
     try
     {
-        std::mutex part_select_mutex;
-        std::mutex part_loading_mutex;
-
         for (size_t thread = 0; thread < num_threads; ++thread)
         {
             pool.scheduleOrThrowOnError([&, thread, thread_group = CurrentThread::getGroup()]
@@ -1338,8 +1337,8 @@ std::vector<MergeTreeData::LoadPartResult> MergeTreeData::loadDataPartsFromDisk(
                             remaining_thread_parts.erase(thread_idx);
                     }
 
-                    thread_part->is_loaded = true;
                     auto res = loadDataPart(thread_part->info, thread_part->name, thread_part->disk, DataPartState::Active, part_loading_mutex);
+                    thread_part->is_loaded = true;
 
                     bool is_active_part = res.part->getState() == DataPartState::Active && !res.part->is_duplicate;
                     if (!is_active_part && !thread_part->children.empty())
@@ -1351,7 +1350,7 @@ std::vector<MergeTreeData::LoadPartResult> MergeTreeData::loadDataPartsFromDisk(
                     }
 
                     {
-                        std::lock_guard lock(loaded_parts_mutex);
+                        std::lock_guard lock(part_loading_mutex);
                         loaded_parts.push_back(std::move(res));
                     }
                 }
@@ -1620,6 +1619,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
     {
         resetObjectColumnsFromActiveParts(part_lock);
         LOG_DEBUG(log, "There are no data parts");
+        are_outdated_data_parts_loaded = true;
         return;
     }
 
@@ -1660,10 +1660,21 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
             unloaded_parts.push_back(node);
     });
 
+    if (!unloaded_parts.empty())
     {
-        std::lock_guard lock(unloaded_outdated_parts_mutex);
-        unloaded_outdated_parts = std::move(unloaded_parts);
+        LOG_DEBUG(log, "Found {} outdated data parts. They will be loaded asynchronously", unloaded_parts.size());
+
+        outdated_data_parts_loading_task = getContext()->getSchedulePool().createTask(
+            "MergeTreeData::loadOutdatedDataParts",
+            [this, unloaded_parts]() mutable
+            {
+                loadOutdatedDataParts(std::move(unloaded_parts));
+            });
+
+        outdated_data_parts_loading_task->deactivate();
     }
+    else
+        are_outdated_data_parts_loaded = true;
 
     LOG_DEBUG(log, "Loaded data parts ({} items)", data_parts_indexes.size());
     data_parts_loading_finished = true;
@@ -1671,10 +1682,11 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
 
 void MergeTreeData::loadOutdatedDataParts(PartLoadingTreeNodes parts_to_load)
 {
-    LOG_DEBUG(log, "Loading {} outdated data parts", parts_to_load.size());
     if (parts_to_load.empty())
         return;
 
+    LOG_DEBUG(log, "Loading {} outdated data parts", parts_to_load.size());
+
     for (const auto & part : parts_to_load)
     {
         auto res = loadDataPart(part->info, part->name, part->disk, MergeTreeDataPartState::Outdated, data_parts_mutex);
@@ -1688,27 +1700,25 @@ void MergeTreeData::loadOutdatedDataParts(PartLoadingTreeNodes parts_to_load)
     }
 
     LOG_DEBUG(log, "Loaded {} outdated data parts", parts_to_load.size());
+
+    are_outdated_data_parts_loaded.store(true);
+    are_outdated_data_parts_loaded.notify_all();
 }
 
-void MergeTreeData::scheduleOutdatedDataPartsLoadingJob(BackgroundJobsAssignee & assignee)
+void MergeTreeData::waitForOutdatedPartsToBeLoaded() const
 {
-    PartLoadingTreeNodes parts_to_load;
-
+    if (!are_outdated_data_parts_loaded)
     {
-        std::lock_guard lock(unloaded_outdated_parts_mutex);
-        parts_to_load = unloaded_outdated_parts;
+        LOG_TRACE(log, "Will wait for outdated data parts to be loaded");
+        are_outdated_data_parts_loaded.wait(false);
+        LOG_TRACE(log, "Finished waiting for outdated data parts to be loaded");
     }
+}
 
-    if (parts_to_load.empty())
-        return;
-
-    auto share_lock = lockForShare(RWLockImpl::NO_QUERY, getSettings()->lock_acquire_timeout_for_background_operations);
-    assignee.scheduleCommonTask(std::make_shared<ExecutableLambdaAdapter>(
-        [this, share_lock, parts_to_load = std::move(parts_to_load)]
-        {
-            loadOutdatedDataParts(std::move(parts_to_load));
-            return false;
-        }, common_assignee_trigger, getStorageID()), /*need_trigger=*/ false);
+void MergeTreeData::startOutdatedDataPartsLoadingTask()
+{
+    if (outdated_data_parts_loading_task)
+        outdated_data_parts_loading_task->activateAndSchedule();
 }
 
 /// Is the part directory old.
@@ -4208,6 +4218,8 @@ Pipe MergeTreeData::alterPartition(
     const PartitionCommands & commands,
     ContextPtr query_context)
 {
+    waitForOutdatedPartsToBeLoaded();
+
     PartitionCommandsResultInfo result;
     for (const PartitionCommand & command : commands)
     {
@@ -4253,6 +4265,14 @@ Pipe MergeTreeData::alterPartition(
                     {
                         String dest_database = query_context->resolveDatabase(command.to_database);
                         auto dest_storage = DatabaseCatalog::instance().getTable({dest_database, command.to_table}, query_context);
+
+                        auto * dest_storage_merge_tree = dynamic_cast<MergeTreeData *>(dest_storage.get());
+                        if (!dest_storage_merge_tree)
+                            throw Exception(ErrorCodes::NOT_IMPLEMENTED,
+                                "Cannot move partition from table {} to table {} with storage {}",
+                                getStorageID().getNameForLogs(), dest_storage->getStorageID().getNameForLogs(), dest_storage->getName());
+
+                        dest_storage_merge_tree->waitForOutdatedPartsToBeLoaded();
                         movePartitionToTable(dest_storage, command.partition, query_context);
                     }
                     break;
@@ -4276,6 +4296,14 @@ Pipe MergeTreeData::alterPartition(
                     checkPartitionCanBeDropped(command.partition, query_context);
                 String from_database = query_context->resolveDatabase(command.from_database);
                 auto from_storage = DatabaseCatalog::instance().getTable({from_database, command.from_table}, query_context);
+
+                auto * from_storage_merge_tree = dynamic_cast<MergeTreeData *>(from_storage.get());
+                if (!from_storage_merge_tree)
+                    throw Exception(ErrorCodes::NOT_IMPLEMENTED,
+                        "Cannot replace partition from table {} with storage {} to table {}",
+                        from_storage->getStorageID().getNameForLogs(), from_storage->getName(), getStorageID().getNameForLogs());
+
+                from_storage_merge_tree->waitForOutdatedPartsToBeLoaded();
                 replacePartitionFrom(from_storage, command.partition, command.replace, query_context);
             }
             break;
diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h
index 5bab007a71f..d9c6f403e33 100644
--- a/src/Storages/MergeTree/MergeTreeData.h
+++ b/src/Storages/MergeTree/MergeTreeData.h
@@ -1303,14 +1303,13 @@ protected:
         using PartLoadingInfo = std::tuple<MergeTreePartInfo, String, DiskPtr>;
         using PartLoadingInfos = std::vector<PartLoadingInfo>;
 
-        static PartLoadingTree build(std::vector<PartLoadingInfo> nodes);
+        static PartLoadingTree build(PartLoadingInfos nodes);
 
         template <typename Func>
         void traverse(bool recursive, Func && func);
 
     private:
         void add(const MergeTreePartInfo & info, const String & name, const DiskPtr & disk);
-
         std::unordered_map<String, NodePtr> root_by_partition;
     };
 
@@ -1323,11 +1322,12 @@ protected:
         MutableDataPartPtr part;
     };
 
-    mutable std::mutex unloaded_outdated_parts_mutex;
-    PartLoadingTreeNodes unloaded_outdated_parts TSA_GUARDED_BY(unloaded_outdated_parts_mutex);
+    BackgroundSchedulePool::TaskHolder outdated_data_parts_loading_task;
+    std::atomic_bool are_outdated_data_parts_loaded = false;
 
     void loadOutdatedDataParts(PartLoadingTreeNodes parts_to_load);
-    void scheduleOutdatedDataPartsLoadingJob(BackgroundJobsAssignee & assignee);
+    void waitForOutdatedPartsToBeLoaded() const;
+    void startOutdatedDataPartsLoadingTask();
 
     static void incrementInsertedPartsProfileEvent(MergeTreeDataPartType type);
     static void incrementMergedPartsProfileEvent(MergeTreeDataPartType type);
@@ -1414,6 +1414,7 @@ private:
         const MergeTreeSettingsPtr & settings);
 
     void loadDataPartsFromWAL(MutableDataPartsVector & parts_from_wal);
+
     /// Create zero-copy exclusive lock for part and disk. Useful for coordination of
     /// distributed operations which can lead to data duplication. Implemented only in ReplicatedMergeTree.
     virtual std::optional<ZeroCopyLock> tryCreateZeroCopyExclusiveLock(const String &, const DiskPtr &) { return std::nullopt; }
diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp
index 7ecd49ef5c3..f94f4d4747c 100644
--- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp
+++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp
@@ -151,9 +151,6 @@ bool ReplicatedMergeTreeRestartingThread::runImpl()
     storage.cleanup_thread.start();
     storage.part_check_thread.start();
 
-    if (first_time)
-        storage.scheduleOutdatedDataPartsLoadingJob(storage.background_operations_assignee);
-
     LOG_DEBUG(log, "Table started successfully");
     return true;
 }
diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp
index cf56997195c..d3c21a015eb 100644
--- a/src/Storages/StorageMergeTree.cpp
+++ b/src/Storages/StorageMergeTree.cpp
@@ -136,8 +136,8 @@ void StorageMergeTree::startup()
     try
     {
         background_operations_assignee.start();
-        scheduleOutdatedDataPartsLoadingJob(background_operations_assignee);
         startBackgroundMovesIfNeeded();
+        startOutdatedDataPartsLoadingTask();
     }
     catch (...)
     {
@@ -177,6 +177,10 @@ void StorageMergeTree::shutdown()
         mutation_wait_event.notify_all();
     }
 
+    waitForOutdatedPartsToBeLoaded();
+    if (outdated_data_parts_loading_task)
+        outdated_data_parts_loading_task->deactivate();
+
     merger_mutator.merges_blocker.cancelForever();
     parts_mover.moves_blocker.cancelForever();
 
@@ -281,6 +285,8 @@ void StorageMergeTree::drop()
 
 void StorageMergeTree::truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr local_context, TableExclusiveLockHolder &)
 {
+    waitForOutdatedPartsToBeLoaded();
+
     {
         /// Asks to complete merges and does not allow them to start.
         /// This protects against "revival" of data for a removed partition after completion of merge.
diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index 8b4788c8d55..4380d038709 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -869,6 +869,8 @@ zkutil::ZooKeeperPtr StorageReplicatedMergeTree::getZooKeeperIfTableShutDown() c
 
 void StorageReplicatedMergeTree::drop()
 {
+    waitForOutdatedPartsToBeLoaded();
+
     /// There is also the case when user has configured ClickHouse to wrong ZooKeeper cluster
     /// or metadata of staled replica were removed manually,
     /// in this case, has_metadata_in_zookeeper = false, and we also permit to drop the table.
@@ -4224,6 +4226,7 @@ MutableDataPartStoragePtr StorageReplicatedMergeTree::fetchExistsPart(
 
 void StorageReplicatedMergeTree::startup()
 {
+    startOutdatedDataPartsLoadingTask();
     if (attach_thread)
     {
         attach_thread->start();
@@ -4303,6 +4306,10 @@ void StorageReplicatedMergeTree::shutdown()
 
     session_expired_callback_handler.reset();
 
+    waitForOutdatedPartsToBeLoaded();
+    if (outdated_data_parts_loading_task)
+        outdated_data_parts_loading_task->deactivate();
+
     /// Cancel fetches, merges and mutations to force the queue_task to finish ASAP.
     fetcher.blocker.cancelForever();
     merger_mutator.merges_blocker.cancelForever();
@@ -5071,7 +5078,6 @@ void StorageReplicatedMergeTree::restoreMetadataInZooKeeper()
     if (!is_readonly)
         throw Exception(ErrorCodes::BAD_ARGUMENTS, "Replica must be readonly");
 
-
     if (getZooKeeper()->exists(replica_path))
         throw Exception(ErrorCodes::BAD_ARGUMENTS,
                         "Replica path is present at {} - nothing to restore. "
@@ -5088,6 +5094,7 @@ void StorageReplicatedMergeTree::restoreMetadataInZooKeeper()
 
     auto metadata_snapshot = getInMemoryMetadataPtr();
 
+    waitForOutdatedPartsToBeLoaded();
     const DataPartsVector all_parts = getAllDataPartsVector();
     Strings active_parts_names;
 
@@ -5202,6 +5209,7 @@ void StorageReplicatedMergeTree::truncate(
     if (!is_leader)
         throw Exception("TRUNCATE cannot be done on this replica because it is not a leader", ErrorCodes::NOT_A_LEADER);
 
+    waitForOutdatedPartsToBeLoaded();
     zkutil::ZooKeeperPtr zookeeper = getZooKeeperAndAssertNotReadonly();
     dropAllPartitionsImpl(zookeeper, /* detach */ false, query_context);
 }
diff --git a/tests/queries/0_stateless/02482_load_parts_parition_commands.reference b/tests/queries/0_stateless/02482_load_parts_parition_commands.reference
new file mode 100644
index 00000000000..da28188439d
--- /dev/null
+++ b/tests/queries/0_stateless/02482_load_parts_parition_commands.reference
@@ -0,0 +1,58 @@
+Engine: MergeTree, Command: ALTER TABLE load_parts_drop_partition_src DROP PARTITION 1
+100
+1
+1
+1
+0
+0
+Engine: MergeTree, Command: ALTER TABLE load_parts_drop_partition_src DETACH PARTITION 1
+100
+1
+1
+1
+0
+0
+Engine: MergeTree, Command: ALTER TABLE load_parts_drop_partition_src MOVE PARTITION 1 TO TABLE load_parts_drop_partition_dst
+100
+100
+1
+1
+1
+0
+0
+Engine: MergeTree, Command: TRUNCATE load_parts_drop_partition_src
+100
+1
+1
+1
+0
+0
+Engine: ReplicatedMergeTree('/test/02482_load_parts_partitions/{database}/{table}', '1'), Command: ALTER TABLE load_parts_drop_partition_src DROP PARTITION 1
+100
+1
+1
+1
+0
+0
+Engine: ReplicatedMergeTree('/test/02482_load_parts_partitions/{database}/{table}', '1'), Command: ALTER TABLE load_parts_drop_partition_src DETACH PARTITION 1
+100
+1
+1
+1
+0
+0
+Engine: ReplicatedMergeTree('/test/02482_load_parts_partitions/{database}/{table}', '1'), Command: ALTER TABLE load_parts_drop_partition_src MOVE PARTITION 1 TO TABLE load_parts_drop_partition_dst
+100
+100
+1
+1
+1
+0
+0
+Engine: ReplicatedMergeTree('/test/02482_load_parts_partitions/{database}/{table}', '1'), Command: TRUNCATE load_parts_drop_partition_src
+100
+1
+1
+1
+0
+0
diff --git a/tests/queries/0_stateless/02482_load_parts_parition_commands.sh b/tests/queries/0_stateless/02482_load_parts_parition_commands.sh
new file mode 100755
index 00000000000..6d1baec7580
--- /dev/null
+++ b/tests/queries/0_stateless/02482_load_parts_parition_commands.sh
@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+# Tags: zookeeper
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+function query_with_retry
+{
+    retry=0
+    until [ $retry -ge 5 ]
+    do
+        result=$($CLICKHOUSE_CLIENT $2 --query="$1" 2>&1)
+        if [ "$?" == 0 ]; then
+            echo -n "$result"
+            return
+        else
+            retry=$(($retry + 1))
+            sleep 3
+        fi
+    done
+    echo "Query '$1' failed with '$result'"
+}
+
+function test_case
+{
+    engine="$1"
+    partition_command="$2"
+    tables=$(echo "$partition_command" | grep -E "load_parts_drop_partition_[a-z]*" -o)
+
+    echo "Engine: $engine, Command: $partition_command"
+
+    for table in $tables; do
+        $CLICKHOUSE_CLIENT -n --query "
+            DROP TABLE IF EXISTS $table SYNC;
+
+            CREATE TABLE $table(id UInt64, val UInt64)
+            ENGINE = $engine
+            ORDER BY id PARTITION BY id;
+
+            SYSTEM STOP MERGES $table;
+        "
+
+        $CLICKHOUSE_CLIENT --max_block_size=1 --max_insert_block_size=1 --min_insert_block_size_rows=1 --min_insert_block_size_bytes=1 -n --query "
+            INSERT INTO $table SELECT 1, number FROM numbers(100);
+
+            SELECT count() FROM system.parts WHERE database = '$CLICKHOUSE_DATABASE' AND table = '$table' AND active;
+
+            SYSTEM START MERGES $table;
+        "
+
+        query_with_retry "OPTIMIZE TABLE $table PARTITION 1 FINAL SETTINGS optimize_throw_if_noop = 1"
+    done
+
+    $CLICKHOUSE_CLIENT -n --query "
+        SELECT count() FROM system.parts WHERE database = '$CLICKHOUSE_DATABASE' AND table = 'load_parts_drop_partition_src' AND active;
+        SELECT count() > 0 FROM system.parts WHERE database = '$CLICKHOUSE_DATABASE' AND table = 'load_parts_drop_partition_src' AND NOT active;
+
+        DETACH TABLE load_parts_drop_partition_src;
+        ATTACH TABLE load_parts_drop_partition_src;
+
+        SELECT count() FROM system.parts WHERE database = '$CLICKHOUSE_DATABASE' AND table = 'load_parts_drop_partition_src' AND active;
+    "
+
+    $CLICKHOUSE_CLIENT --query "$partition_command"
+
+    $CLICKHOUSE_CLIENT -n --query "
+        SELECT count() FROM system.parts WHERE database = '$CLICKHOUSE_DATABASE' AND table = 'load_parts_drop_partition_src' AND active;
+
+        DETACH TABLE load_parts_drop_partition_src;
+        ATTACH TABLE load_parts_drop_partition_src;
+
+        SELECT count() FROM system.parts WHERE database = '$CLICKHOUSE_DATABASE' AND table = 'load_parts_drop_partition_src';
+    "
+
+    for table in $tables; do
+        $CLICKHOUSE_CLIENT --query "DROP TABLE $table SYNC"
+    done
+}
+
+declare -a engines=(
+    "MergeTree"
+    "ReplicatedMergeTree('/test/02482_load_parts_partitions/{database}/{table}', '1')"
+)
+
+for engine in "${engines[@]}"; do
+    test_case "$engine" "ALTER TABLE load_parts_drop_partition_src DROP PARTITION 1"
+    test_case "$engine" "ALTER TABLE load_parts_drop_partition_src DETACH PARTITION 1"
+    test_case "$engine" "ALTER TABLE load_parts_drop_partition_src MOVE PARTITION 1 TO TABLE load_parts_drop_partition_dst"
+    test_case "$engine" "TRUNCATE load_parts_drop_partition_src"
+done
diff --git a/tests/queries/0_stateless/02482_load_parts_refcounts.reference b/tests/queries/0_stateless/02482_load_parts_refcounts.reference
new file mode 100644
index 00000000000..d00491fd7e5
--- /dev/null
+++ b/tests/queries/0_stateless/02482_load_parts_refcounts.reference
@@ -0,0 +1 @@
+1
diff --git a/tests/queries/0_stateless/02482_load_parts_refcounts.sh b/tests/queries/0_stateless/02482_load_parts_refcounts.sh
new file mode 100755
index 00000000000..27549499a45
--- /dev/null
+++ b/tests/queries/0_stateless/02482_load_parts_refcounts.sh
@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+# Tags: zookeeper
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+function query_with_retry
+{
+    retry=0
+    until [ $retry -ge 5 ]
+    do
+        result=$($CLICKHOUSE_CLIENT $2 --query="$1" 2>&1)
+        if [ "$?" == 0 ]; then
+            echo -n "$result"
+            return
+        else
+            retry=$(($retry + 1))
+            sleep 3
+        fi
+    done
+    echo "Query '$1' failed with '$result'"
+}
+
+$CLICKHOUSE_CLIENT -n --query "
+    DROP TABLE IF EXISTS load_parts_refcounts SYNC;
+
+    CREATE TABLE load_parts_refcounts (id UInt32)
+    ENGINE = ReplicatedMergeTree('/test/02482_load_parts_refcounts/{database}/{table}', '1')
+    ORDER BY id;
+
+    SYSTEM STOP MERGES load_parts_refcounts;
+
+    INSERT INTO load_parts_refcounts VALUES (1);
+    INSERT INTO load_parts_refcounts VALUES (2);
+    INSERT INTO load_parts_refcounts VALUES (3);
+
+    SYSTEM START MERGES load_parts_refcounts;
+"
+
+query_with_retry "OPTIMIZE TABLE load_parts_refcounts FINAL SETTINGS optimize_throw_if_noop = 1"
+
+$CLICKHOUSE_CLIENT --query "DETACH TABLE load_parts_refcounts"
+$CLICKHOUSE_CLIENT --query "ATTACH TABLE load_parts_refcounts"
+
+query_with_retry "
+    SELECT throwIf(count() == 0) FROM system.parts
+    WHERE database = '$CLICKHOUSE_DATABASE' AND table = 'load_parts_refcounts' AND NOT active FORMAT Null"
+
+$CLICKHOUSE_CLIENT --query "
+    SELECT DISTINCT refcount FROM system.parts
+    WHERE database = '$CLICKHOUSE_DATABASE' AND table = 'load_parts_refcounts' AND NOT active"
+
+$CLICKHOUSE_CLIENT --query "DROP TABLE load_parts_refcounts SYNC"

From e3db29aebff14f6d7732060fabc52aa6023cf632 Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Tue, 22 Nov 2022 14:47:24 +0000
Subject: [PATCH 021/158] better waiting for outdated data parts

---
 src/Storages/MergeTree/MergeTreeData.cpp    | 88 ++++++++++++++++-----
 src/Storages/MergeTree/MergeTreeData.h      |  8 +-
 src/Storages/StorageMergeTree.cpp           |  5 +-
 src/Storages/StorageReplicatedMergeTree.cpp |  5 +-
 4 files changed, 80 insertions(+), 26 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 30f62adaae5..f87a73eed05 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -167,6 +167,7 @@ namespace ErrorCodes
     extern const int INCORRECT_QUERY;
     extern const int CANNOT_RESTORE_TABLE;
     extern const int ZERO_COPY_REPLICATION_ERROR;
+    extern const int NOT_INITIALIZED;
 }
 
 
@@ -1619,7 +1620,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
     {
         resetObjectColumnsFromActiveParts(part_lock);
         LOG_DEBUG(log, "There are no data parts");
-        are_outdated_data_parts_loaded = true;
+        outdated_data_parts_loading_finished = true;
         return;
     }
 
@@ -1670,11 +1671,9 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
             {
                 loadOutdatedDataParts(std::move(unloaded_parts));
             });
-
-        outdated_data_parts_loading_task->deactivate();
     }
     else
-        are_outdated_data_parts_loaded = true;
+        outdated_data_parts_loading_finished = true;
 
     LOG_DEBUG(log, "Loaded data parts ({} items)", data_parts_indexes.size());
     data_parts_loading_finished = true;
@@ -1689,30 +1688,66 @@ void MergeTreeData::loadOutdatedDataParts(PartLoadingTreeNodes parts_to_load)
 
     for (const auto & part : parts_to_load)
     {
-        auto res = loadDataPart(part->info, part->name, part->disk, MergeTreeDataPartState::Outdated, data_parts_mutex);
+        if (outdated_data_parts_loading_canceled)
+        {
+            LOG_DEBUG(log, "Stopped loading outdated parts because task was canceled");
+            return;
+        }
 
-        if (res.is_broken)
-            res.part->renameToDetached("broken-on-start"); /// detached parts must not have '_' in prefixes
-        else if (res.part->is_duplicate)
-            res.part->remove();
-        else
-            preparePartForRemoval(res.part);
+        std::string_view what;
+
+        try
+        {
+            what = "load";
+            auto res = loadDataPart(part->info, part->name, part->disk, MergeTreeDataPartState::Outdated, data_parts_mutex);
+
+            if (res.is_broken)
+            {
+                what = "rename to detached";
+                res.part->renameToDetached("broken-on-start"); /// detached parts must not have '_' in prefixes
+            }
+            else if (res.part->is_duplicate)
+            {
+                what = "remove";
+                res.part->remove();
+            }
+            else
+            {
+                what = "prepare for removal";
+                preparePartForRemoval(res.part);
+            }
+        }
+        catch (...)
+        {
+            LOG_ERROR(log, "Failed to {} outdated data part {}. You need to resolve it manually. Exception: {}",
+                what, part->name, getCurrentExceptionMessage(false));
+            continue;
+        }
     }
 
     LOG_DEBUG(log, "Loaded {} outdated data parts", parts_to_load.size());
 
-    are_outdated_data_parts_loaded.store(true);
-    are_outdated_data_parts_loaded.notify_all();
+    {
+        std::lock_guard lock(outdated_data_parts_mutex);
+        outdated_data_parts_loading_finished = true;
+    }
+
+    outdated_data_parts_cv.notify_all();
 }
 
 void MergeTreeData::waitForOutdatedPartsToBeLoaded() const
 {
-    if (!are_outdated_data_parts_loaded)
-    {
-        LOG_TRACE(log, "Will wait for outdated data parts to be loaded");
-        are_outdated_data_parts_loaded.wait(false);
-        LOG_TRACE(log, "Finished waiting for outdated data parts to be loaded");
-    }
+    if (outdated_data_parts_loading_finished)
+        return;
+
+    LOG_TRACE(log, "Will wait for outdated data parts to be loaded");
+
+    std::unique_lock lock(outdated_data_parts_mutex);
+    outdated_data_parts_cv.wait(lock, [this] { return outdated_data_parts_loading_finished || outdated_data_parts_loading_canceled; });
+    if (outdated_data_parts_loading_canceled)
+        throw Exception(ErrorCodes::NOT_INITIALIZED, "Loading of outdated data parts was canceled");
+
+    LOG_TRACE(log, "Finished waiting for outdated data parts to be loaded");
 }
 
 void MergeTreeData::startOutdatedDataPartsLoadingTask()
@@ -1721,6 +1756,20 @@ void MergeTreeData::startOutdatedDataPartsLoadingTask()
         outdated_data_parts_loading_task->activateAndSchedule();
 }
 
+void MergeTreeData::stopOutdatedDataPartsLoadingTask()
+{
+    if (!outdated_data_parts_loading_task)
+        return;
+
+    {
+        std::lock_guard lock(outdated_data_parts_mutex);
+        outdated_data_parts_loading_canceled = true;
+    }
+
+    outdated_data_parts_loading_task->deactivate();
+    outdated_data_parts_cv.notify_all();
+}
+
 /// Is the part directory old.
 /// True if its modification time and the modification time of all files inside it is less then threshold.
 /// (Only files on the first level of nesting are considered).
@@ -4294,6 +4343,7 @@ Pipe MergeTreeData::alterPartition(
             {
                 if (command.replace)
                     checkPartitionCanBeDropped(command.partition, query_context);
+
                 String from_database = query_context->resolveDatabase(command.from_database);
                 auto from_storage = DatabaseCatalog::instance().getTable({from_database, command.from_table}, query_context);
 
diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h
index d9c6f403e33..83c504cce73 100644
--- a/src/Storages/MergeTree/MergeTreeData.h
+++ b/src/Storages/MergeTree/MergeTreeData.h
@@ -1323,11 +1323,17 @@ protected:
     };
 
     BackgroundSchedulePool::TaskHolder outdated_data_parts_loading_task;
-    std::atomic_bool are_outdated_data_parts_loaded = false;
+
+    mutable std::mutex outdated_data_parts_mutex;
+    mutable std::condition_variable outdated_data_parts_cv;
+
+    std::atomic_bool outdated_data_parts_loading_finished = false;
+    std::atomic_bool outdated_data_parts_loading_canceled = false;
 
     void loadOutdatedDataParts(PartLoadingTreeNodes parts_to_load);
     void waitForOutdatedPartsToBeLoaded() const;
     void startOutdatedDataPartsLoadingTask();
+    void stopOutdatedDataPartsLoadingTask();
 
     static void incrementInsertedPartsProfileEvent(MergeTreeDataPartType type);
     static void incrementMergedPartsProfileEvent(MergeTreeDataPartType type);
diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp
index d3c21a015eb..31deda9d8de 100644
--- a/src/Storages/StorageMergeTree.cpp
+++ b/src/Storages/StorageMergeTree.cpp
@@ -177,9 +177,10 @@ void StorageMergeTree::shutdown()
         mutation_wait_event.notify_all();
     }
 
+    /// We need to for loading of oudated data parts
+    /// for correct execution of 'clearOldPartsFromFilesystem'
     waitForOutdatedPartsToBeLoaded();
-    if (outdated_data_parts_loading_task)
-        outdated_data_parts_loading_task->deactivate();
+    stopOutdatedDataPartsLoadingTask();
 
     merger_mutator.merges_blocker.cancelForever();
     parts_mover.moves_blocker.cancelForever();
diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index 4380d038709..88b9338f645 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -4305,10 +4305,7 @@ void StorageReplicatedMergeTree::shutdown()
         return;
 
     session_expired_callback_handler.reset();
-
-    waitForOutdatedPartsToBeLoaded();
-    if (outdated_data_parts_loading_task)
-        outdated_data_parts_loading_task->deactivate();
+    stopOutdatedDataPartsLoadingTask();
 
     /// Cancel fetches, merges and mutations to force the queue_task to finish ASAP.
     fetcher.blocker.cancelForever();

From 75c807609b0424b81fc8f7c33500ca48242fb56c Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Tue, 22 Nov 2022 16:09:23 +0000
Subject: [PATCH 022/158] add some comments

---
 src/Storages/MergeTree/MergeTreeData.cpp |  9 +++++++--
 src/Storages/MergeTree/MergeTreeData.h   | 17 +++++++++++++++++
 src/Storages/StorageMergeTree.cpp        |  2 +-
 3 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index f87a73eed05..441064c1aea 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -967,7 +967,7 @@ void MergeTreeData::PartLoadingTree::add(const MergeTreePartInfo & info, const S
             {
                 throw Exception(ErrorCodes::LOGICAL_ERROR,
                     "Part {} intersects previous part {}. It is a bug!",
-                    info.getPartName(), prev_info.getPartName());
+                    name, prev->second->name);
             }
         }
 
@@ -984,7 +984,7 @@ void MergeTreeData::PartLoadingTree::add(const MergeTreePartInfo & info, const S
             {
                 throw Exception(ErrorCodes::LOGICAL_ERROR,
                     "Part {} intersects next part {}. It is a bug!",
-                    info.getPartName(), next_info.getPartName());
+                    name, it->second->name);
             }
         }
 
@@ -1338,10 +1338,14 @@ std::vector<MergeTreeData::LoadPartResult> MergeTreeData::loadDataPartsFromDisk(
                             remaining_thread_parts.erase(thread_idx);
                     }
 
+                    /// Pass a separate mutex to guard the set of parts, because this lambda
+                    /// is called concurrently but with already locked @data_parts_mutex.
                     auto res = loadDataPart(thread_part->info, thread_part->name, thread_part->disk, DataPartState::Active, part_loading_mutex);
                     thread_part->is_loaded = true;
 
                     bool is_active_part = res.part->getState() == DataPartState::Active && !res.part->is_duplicate;
+                    /// If part is broken or duplicate or should be removed according to transaction
+                    /// and it has any covered parts then try to load them to replace this part.
                     if (!is_active_part && !thread_part->children.empty())
                     {
                         std::lock_guard lock{part_select_mutex};
@@ -1503,6 +1507,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
     /// Collect parts by disks' names.
     std::map<String, PartLoadingTreeNodes> disk_part_map;
 
+    /// Collect only "the most covering" parts from the top level of the tree.
     loading_tree.traverse(/*recursive=*/ false, [&](const auto & node)
     {
         disk_part_map[node->disk->getName()].emplace_back(node);
diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h
index 83c504cce73..d7f08436c19 100644
--- a/src/Storages/MergeTree/MergeTreeData.h
+++ b/src/Storages/MergeTree/MergeTreeData.h
@@ -1281,6 +1281,18 @@ protected:
     void resetObjectColumnsFromActiveParts(const DataPartsLock & lock);
     void updateObjectColumns(const DataPartPtr & part, const DataPartsLock & lock);
 
+    /** A structure that explicitly represents a "merge tree" of parts
+     *  which is implicitly presented by min-max block numbers and levels of parts.
+     *  The children of node are parts which are covered by parent part.
+     *  This tree provides the order of loading of parts.
+     *
+     *  We start to traverse tree from the top level and load parts
+     *  corresposponded to nodes. If part is loaded successfully then
+     *  we stop traversal at this node. Otherwise part is broken and we
+     *  traverse its children and try to load covered parts which will
+     *  replace broken covering part. Unloaded nodes represent outdated parts
+     *  nd they are pushed to background task and loaded asynchronoulsy.
+     */
     class PartLoadingTree
     {
     public:
@@ -1303,12 +1315,17 @@ protected:
         using PartLoadingInfo = std::tuple<MergeTreePartInfo, String, DiskPtr>;
         using PartLoadingInfos = std::vector<PartLoadingInfo>;
 
+        /// Builds a tree from the list of part infos.
         static PartLoadingTree build(PartLoadingInfos nodes);
 
+        /// Traverses a tree and call @func on each node.
+        /// If recursive is false traverses only the top level.
         template <typename Func>
         void traverse(bool recursive, Func && func);
 
     private:
+        /// NOTE: Parts should be added in descending order of their levels
+        /// because rearranging tree to the new root is not supported.
         void add(const MergeTreePartInfo & info, const String & name, const DiskPtr & disk);
         std::unordered_map<String, NodePtr> root_by_partition;
     };
diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp
index 31deda9d8de..dfdee24134b 100644
--- a/src/Storages/StorageMergeTree.cpp
+++ b/src/Storages/StorageMergeTree.cpp
@@ -177,7 +177,7 @@ void StorageMergeTree::shutdown()
         mutation_wait_event.notify_all();
     }
 
-    /// We need to for loading of oudated data parts
+    /// We need to wait for loading of oudated data parts
     /// for correct execution of 'clearOldPartsFromFilesystem'
     waitForOutdatedPartsToBeLoaded();
     stopOutdatedDataPartsLoadingTask();

From 16a3eb9bfcd66fc44a1077635e44b2868d6bdd37 Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Tue, 22 Nov 2022 17:09:11 +0000
Subject: [PATCH 023/158] better loading of outdated parts

---
 src/Storages/MergeTree/MergeTreeData.cpp | 46 ++++++++++--------------
 1 file changed, 18 insertions(+), 28 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 441064c1aea..cb72ada2b30 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -1685,6 +1685,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
 }
 
 void MergeTreeData::loadOutdatedDataParts(PartLoadingTreeNodes parts_to_load)
+try
 {
     if (parts_to_load.empty())
         return;
@@ -1699,35 +1700,14 @@ void MergeTreeData::loadOutdatedDataParts(PartLoadingTreeNodes parts_to_load)
             return;
         }
 
-        std::string_view what;
+        auto res = loadDataPart(part->info, part->name, part->disk, MergeTreeDataPartState::Outdated, data_parts_mutex);
 
-        try
-        {
-            what = "load";
-            auto res = loadDataPart(part->info, part->name, part->disk, MergeTreeDataPartState::Outdated, data_parts_mutex);
-
-            if (res.is_broken)
-            {
-                what = "rename to detached";
-                res.part->renameToDetached("broken-on-start"); /// detached parts must not have '_' in prefixes
-            }
-            else if (res.part->is_duplicate)
-            {
-                what = "remove";
-                res.part->remove();
-            }
-            else
-            {
-                what = "prepare for removal";
-                preparePartForRemoval(res.part);
-            }
-        }
-        catch (...)
-        {
-            LOG_ERROR(log, "Failed to {} outdated data part {}. You need to resolve it manually. Exception: {}",
-                what, part->name, getCurrentExceptionMessage(false));
-            continue;
-        }
+        if (res.is_broken)
+            res.part->renameToDetached("broken-on-start"); /// detached parts must not have '_' in prefixes
+        else if (res.part->is_duplicate)
+            res.part->remove();
+        else
+            preparePartForRemoval(res.part);
     }
 
     LOG_DEBUG(log, "Loaded {} outdated data parts", parts_to_load.size());
@@ -1739,6 +1719,13 @@ void MergeTreeData::loadOutdatedDataParts(PartLoadingTreeNodes parts_to_load)
 
     outdated_data_parts_cv.notify_all();
 }
+catch (...)
+{
+    LOG_ERROR(log, "Loading of outdated parts failed. "
+        "Will terminate to avoid undefined behaviour due to inconsistent set of parts. "
+        "Exception: ", getCurrentExceptionMessage(true));
+    std::terminate();
+}
 
 void MergeTreeData::waitForOutdatedPartsToBeLoaded() const
 {
@@ -4272,6 +4259,9 @@ Pipe MergeTreeData::alterPartition(
     const PartitionCommands & commands,
     ContextPtr query_context)
 {
+    /// Wait for loading of outdated parts
+    /// because partition commands (DROP, MOVE, etc.)
+    /// must be applied to all parts on disk.
     waitForOutdatedPartsToBeLoaded();
 
     PartitionCommandsResultInfo result;

From 44b4345996aa9180b394511128aba3f173684b82 Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Wed, 23 Nov 2022 04:21:27 +0000
Subject: [PATCH 024/158] fix wait in drop

---
 src/Storages/StorageReplicatedMergeTree.cpp | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index 88b9338f645..414236240c8 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -869,8 +869,6 @@ zkutil::ZooKeeperPtr StorageReplicatedMergeTree::getZooKeeperIfTableShutDown() c
 
 void StorageReplicatedMergeTree::drop()
 {
-    waitForOutdatedPartsToBeLoaded();
-
     /// There is also the case when user has configured ClickHouse to wrong ZooKeeper cluster
     /// or metadata of staled replica were removed manually,
     /// in this case, has_metadata_in_zookeeper = false, and we also permit to drop the table.

From f4d91cd7223a448c47d71094ca2fa6c7fb897226 Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Thu, 24 Nov 2022 01:34:07 +0000
Subject: [PATCH 025/158] fix drop with zero-copy

---
 src/Storages/StorageReplicatedMergeTree.cpp | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index 414236240c8..8d075667933 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -887,6 +887,12 @@ void StorageReplicatedMergeTree::drop()
         dropReplica(zookeeper, zookeeper_path, replica_name, log, getSettings());
     }
 
+    /// Wait for loading of all outdated parts because
+    /// in case of zero copy recursive removal of
+    /// directory is not supported.
+    if (canUseZeroCopyReplication())
+        waitForOutdatedPartsToBeLoaded();
+
     dropAllData();
 }
 

From b05a20da1ff302bd9948bd4333efaf526212fdda Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Thu, 24 Nov 2022 14:27:45 +0000
Subject: [PATCH 026/158] fix drop in case of zero-copy

---
 src/Storages/StorageReplicatedMergeTree.cpp   | 13 +++++-----
 ...482_load_parts_parition_commands.reference | 24 +++++++++++--------
 .../02482_load_parts_parition_commands.sh     | 12 ++++++----
 3 files changed, 29 insertions(+), 20 deletions(-)

diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index 8d075667933..dfa181d4d12 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -887,12 +887,6 @@ void StorageReplicatedMergeTree::drop()
         dropReplica(zookeeper, zookeeper_path, replica_name, log, getSettings());
     }
 
-    /// Wait for loading of all outdated parts because
-    /// in case of zero copy recursive removal of
-    /// directory is not supported.
-    if (canUseZeroCopyReplication())
-        waitForOutdatedPartsToBeLoaded();
-
     dropAllData();
 }
 
@@ -4309,6 +4303,13 @@ void StorageReplicatedMergeTree::shutdown()
         return;
 
     session_expired_callback_handler.reset();
+
+    /// Wait for loading of all outdated parts because
+    /// in case of zero copy recursive removal of directory
+    /// is not supported and table cannot be dropped.
+    if (canUseZeroCopyReplication())
+        waitForOutdatedPartsToBeLoaded();
+
     stopOutdatedDataPartsLoadingTask();
 
     /// Cancel fetches, merges and mutations to force the queue_task to finish ASAP.
diff --git a/tests/queries/0_stateless/02482_load_parts_parition_commands.reference b/tests/queries/0_stateless/02482_load_parts_parition_commands.reference
index da28188439d..bf234f57595 100644
--- a/tests/queries/0_stateless/02482_load_parts_parition_commands.reference
+++ b/tests/queries/0_stateless/02482_load_parts_parition_commands.reference
@@ -1,56 +1,60 @@
 Engine: MergeTree, Command: ALTER TABLE load_parts_drop_partition_src DROP PARTITION 1
-100
+500
 1
 1
 1
 0
 0
 Engine: MergeTree, Command: ALTER TABLE load_parts_drop_partition_src DETACH PARTITION 1
-100
+500
 1
 1
 1
 0
 0
 Engine: MergeTree, Command: ALTER TABLE load_parts_drop_partition_src MOVE PARTITION 1 TO TABLE load_parts_drop_partition_dst
-100
-100
+500
+500
 1
 1
 1
 0
 0
 Engine: MergeTree, Command: TRUNCATE load_parts_drop_partition_src
-100
+500
 1
 1
 1
 0
 0
 Engine: ReplicatedMergeTree('/test/02482_load_parts_partitions/{database}/{table}', '1'), Command: ALTER TABLE load_parts_drop_partition_src DROP PARTITION 1
-100
+500
+1
 1
 1
 1
 0
 0
 Engine: ReplicatedMergeTree('/test/02482_load_parts_partitions/{database}/{table}', '1'), Command: ALTER TABLE load_parts_drop_partition_src DETACH PARTITION 1
-100
+500
+1
 1
 1
 1
 0
 0
 Engine: ReplicatedMergeTree('/test/02482_load_parts_partitions/{database}/{table}', '1'), Command: ALTER TABLE load_parts_drop_partition_src MOVE PARTITION 1 TO TABLE load_parts_drop_partition_dst
-100
-100
+500
+500
+1
 1
 1
 1
 0
 0
 Engine: ReplicatedMergeTree('/test/02482_load_parts_partitions/{database}/{table}', '1'), Command: TRUNCATE load_parts_drop_partition_src
-100
+500
+1
 1
 1
 1
diff --git a/tests/queries/0_stateless/02482_load_parts_parition_commands.sh b/tests/queries/0_stateless/02482_load_parts_parition_commands.sh
index 6d1baec7580..768e5ddb290 100755
--- a/tests/queries/0_stateless/02482_load_parts_parition_commands.sh
+++ b/tests/queries/0_stateless/02482_load_parts_parition_commands.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: zookeeper
+# Tags: zookeeper, long
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
@@ -42,7 +42,7 @@ function test_case
         "
 
         $CLICKHOUSE_CLIENT --max_block_size=1 --max_insert_block_size=1 --min_insert_block_size_rows=1 --min_insert_block_size_bytes=1 -n --query "
-            INSERT INTO $table SELECT 1, number FROM numbers(100);
+            INSERT INTO $table SELECT 1, number FROM numbers(500);
 
             SELECT count() FROM system.parts WHERE database = '$CLICKHOUSE_DATABASE' AND table = '$table' AND active;
 
@@ -62,15 +62,19 @@ function test_case
         SELECT count() FROM system.parts WHERE database = '$CLICKHOUSE_DATABASE' AND table = 'load_parts_drop_partition_src' AND active;
     "
 
+    if [[ "$engine" != "MergeTree" ]]; then
+        $CLICKHOUSE_CLIENT --query "SELECT count() > 0 FROM system.parts WHERE database = '$CLICKHOUSE_DATABASE' AND table = 'load_parts_drop_partition_src' AND NOT active"
+    fi
+
     $CLICKHOUSE_CLIENT --query "$partition_command"
 
     $CLICKHOUSE_CLIENT -n --query "
-        SELECT count() FROM system.parts WHERE database = '$CLICKHOUSE_DATABASE' AND table = 'load_parts_drop_partition_src' AND active;
+        SELECT count() FROM system.parts WHERE database = '$CLICKHOUSE_DATABASE' AND table = 'load_parts_drop_partition_src' AND level = 0;
 
         DETACH TABLE load_parts_drop_partition_src;
         ATTACH TABLE load_parts_drop_partition_src;
 
-        SELECT count() FROM system.parts WHERE database = '$CLICKHOUSE_DATABASE' AND table = 'load_parts_drop_partition_src';
+        SELECT count() FROM system.parts WHERE database = '$CLICKHOUSE_DATABASE' AND table = 'load_parts_drop_partition_src' AND level = 0;
     "
 
     for table in $tables; do

From 25197586a0bdb101ed2c68837e0abc814416ff7e Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Fri, 25 Nov 2022 01:14:23 +0000
Subject: [PATCH 027/158] remove test

---
 src/Storages/MergeTree/MergeTreeData.cpp      |  4 +
 ...482_load_parts_parition_commands.reference | 62 ------------
 .../02482_load_parts_parition_commands.sh     | 95 -------------------
 3 files changed, 4 insertions(+), 157 deletions(-)
 delete mode 100644 tests/queries/0_stateless/02482_load_parts_parition_commands.reference
 delete mode 100755 tests/queries/0_stateless/02482_load_parts_parition_commands.sh

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index be631aa0c41..2a8fd3df57a 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -1731,6 +1731,10 @@ catch (...)
 
 void MergeTreeData::waitForOutdatedPartsToBeLoaded() const
 {
+    /// Background tasks are not run if storage is static.
+    if (isStaticStorage())
+        return;
+
     if (outdated_data_parts_loading_finished)
         return;
 
diff --git a/tests/queries/0_stateless/02482_load_parts_parition_commands.reference b/tests/queries/0_stateless/02482_load_parts_parition_commands.reference
deleted file mode 100644
index bf234f57595..00000000000
--- a/tests/queries/0_stateless/02482_load_parts_parition_commands.reference
+++ /dev/null
@@ -1,62 +0,0 @@
-Engine: MergeTree, Command: ALTER TABLE load_parts_drop_partition_src DROP PARTITION 1
-500
-1
-1
-1
-0
-0
-Engine: MergeTree, Command: ALTER TABLE load_parts_drop_partition_src DETACH PARTITION 1
-500
-1
-1
-1
-0
-0
-Engine: MergeTree, Command: ALTER TABLE load_parts_drop_partition_src MOVE PARTITION 1 TO TABLE load_parts_drop_partition_dst
-500
-500
-1
-1
-1
-0
-0
-Engine: MergeTree, Command: TRUNCATE load_parts_drop_partition_src
-500
-1
-1
-1
-0
-0
-Engine: ReplicatedMergeTree('/test/02482_load_parts_partitions/{database}/{table}', '1'), Command: ALTER TABLE load_parts_drop_partition_src DROP PARTITION 1
-500
-1
-1
-1
-1
-0
-0
-Engine: ReplicatedMergeTree('/test/02482_load_parts_partitions/{database}/{table}', '1'), Command: ALTER TABLE load_parts_drop_partition_src DETACH PARTITION 1
-500
-1
-1
-1
-1
-0
-0
-Engine: ReplicatedMergeTree('/test/02482_load_parts_partitions/{database}/{table}', '1'), Command: ALTER TABLE load_parts_drop_partition_src MOVE PARTITION 1 TO TABLE load_parts_drop_partition_dst
-500
-500
-1
-1
-1
-1
-0
-0
-Engine: ReplicatedMergeTree('/test/02482_load_parts_partitions/{database}/{table}', '1'), Command: TRUNCATE load_parts_drop_partition_src
-500
-1
-1
-1
-1
-0
-0
diff --git a/tests/queries/0_stateless/02482_load_parts_parition_commands.sh b/tests/queries/0_stateless/02482_load_parts_parition_commands.sh
deleted file mode 100755
index 768e5ddb290..00000000000
--- a/tests/queries/0_stateless/02482_load_parts_parition_commands.sh
+++ /dev/null
@@ -1,95 +0,0 @@
-#!/usr/bin/env bash
-# Tags: zookeeper, long
-
-CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# shellcheck source=../shell_config.sh
-. "$CURDIR"/../shell_config.sh
-
-function query_with_retry
-{
-    retry=0
-    until [ $retry -ge 5 ]
-    do
-        result=$($CLICKHOUSE_CLIENT $2 --query="$1" 2>&1)
-        if [ "$?" == 0 ]; then
-            echo -n "$result"
-            return
-        else
-            retry=$(($retry + 1))
-            sleep 3
-        fi
-    done
-    echo "Query '$1' failed with '$result'"
-}
-
-function test_case
-{
-    engine="$1"
-    partition_command="$2"
-    tables=$(echo "$partition_command" | grep -E "load_parts_drop_partition_[a-z]*" -o)
-
-    echo "Engine: $engine, Command: $partition_command"
-
-    for table in $tables; do
-        $CLICKHOUSE_CLIENT -n --query "
-            DROP TABLE IF EXISTS $table SYNC;
-
-            CREATE TABLE $table(id UInt64, val UInt64)
-            ENGINE = $engine
-            ORDER BY id PARTITION BY id;
-
-            SYSTEM STOP MERGES $table;
-        "
-
-        $CLICKHOUSE_CLIENT --max_block_size=1 --max_insert_block_size=1 --min_insert_block_size_rows=1 --min_insert_block_size_bytes=1 -n --query "
-            INSERT INTO $table SELECT 1, number FROM numbers(500);
-
-            SELECT count() FROM system.parts WHERE database = '$CLICKHOUSE_DATABASE' AND table = '$table' AND active;
-
-            SYSTEM START MERGES $table;
-        "
-
-        query_with_retry "OPTIMIZE TABLE $table PARTITION 1 FINAL SETTINGS optimize_throw_if_noop = 1"
-    done
-
-    $CLICKHOUSE_CLIENT -n --query "
-        SELECT count() FROM system.parts WHERE database = '$CLICKHOUSE_DATABASE' AND table = 'load_parts_drop_partition_src' AND active;
-        SELECT count() > 0 FROM system.parts WHERE database = '$CLICKHOUSE_DATABASE' AND table = 'load_parts_drop_partition_src' AND NOT active;
-
-        DETACH TABLE load_parts_drop_partition_src;
-        ATTACH TABLE load_parts_drop_partition_src;
-
-        SELECT count() FROM system.parts WHERE database = '$CLICKHOUSE_DATABASE' AND table = 'load_parts_drop_partition_src' AND active;
-    "
-
-    if [[ "$engine" != "MergeTree" ]]; then
-        $CLICKHOUSE_CLIENT --query "SELECT count() > 0 FROM system.parts WHERE database = '$CLICKHOUSE_DATABASE' AND table = 'load_parts_drop_partition_src' AND NOT active"
-    fi
-
-    $CLICKHOUSE_CLIENT --query "$partition_command"
-
-    $CLICKHOUSE_CLIENT -n --query "
-        SELECT count() FROM system.parts WHERE database = '$CLICKHOUSE_DATABASE' AND table = 'load_parts_drop_partition_src' AND level = 0;
-
-        DETACH TABLE load_parts_drop_partition_src;
-        ATTACH TABLE load_parts_drop_partition_src;
-
-        SELECT count() FROM system.parts WHERE database = '$CLICKHOUSE_DATABASE' AND table = 'load_parts_drop_partition_src' AND level = 0;
-    "
-
-    for table in $tables; do
-        $CLICKHOUSE_CLIENT --query "DROP TABLE $table SYNC"
-    done
-}
-
-declare -a engines=(
-    "MergeTree"
-    "ReplicatedMergeTree('/test/02482_load_parts_partitions/{database}/{table}', '1')"
-)
-
-for engine in "${engines[@]}"; do
-    test_case "$engine" "ALTER TABLE load_parts_drop_partition_src DROP PARTITION 1"
-    test_case "$engine" "ALTER TABLE load_parts_drop_partition_src DETACH PARTITION 1"
-    test_case "$engine" "ALTER TABLE load_parts_drop_partition_src MOVE PARTITION 1 TO TABLE load_parts_drop_partition_dst"
-    test_case "$engine" "TRUNCATE load_parts_drop_partition_src"
-done

From 1a090e99014032d363a7a3b7aef4a5b6fa0cc872 Mon Sep 17 00:00:00 2001
From: Nikolay Degterinsky <evillique@gmail.com>
Date: Wed, 30 Nov 2022 02:14:04 +0000
Subject: [PATCH 028/158] Better implementation

---
 src/Analyzer/MatcherNode.cpp                  | 61 +++++++++++++------
 src/Analyzer/QueryTreeBuilder.cpp             |  3 +
 .../JoinToSubqueryTransformVisitor.cpp        | 37 ++++++-----
 src/Interpreters/JoinedTables.cpp             |  7 +--
 .../TranslateQualifiedNamesVisitor.cpp        | 28 ++++++---
 src/Parsers/ASTAsterisk.cpp                   | 21 ++++---
 src/Parsers/ASTColumnsMatcher.cpp             | 53 +++++-----------
 src/Parsers/ASTColumnsMatcher.h               |  2 -
 src/Parsers/ASTColumnsTransformers.cpp        |  9 +++
 src/Parsers/ASTColumnsTransformers.h          | 17 ++++++
 src/Parsers/ASTQualifiedAsterisk.cpp          |  6 +-
 src/Parsers/ASTQualifiedAsterisk.h            |  5 +-
 src/Parsers/ExpressionElementParsers.cpp      | 60 +++++++++++-------
 src/Parsers/ExpressionListParsers.cpp         |  6 +-
 src/Storages/WindowView/StorageWindowView.cpp |  5 +-
 15 files changed, 184 insertions(+), 136 deletions(-)

diff --git a/src/Analyzer/MatcherNode.cpp b/src/Analyzer/MatcherNode.cpp
index f0b44b47b44..fc74b4ff67e 100644
--- a/src/Analyzer/MatcherNode.cpp
+++ b/src/Analyzer/MatcherNode.cpp
@@ -11,6 +11,7 @@
 #include <Parsers/ASTQualifiedAsterisk.h>
 #include <Parsers/ASTColumnsMatcher.h>
 #include <Parsers/ASTExpressionList.h>
+#include <Parsers/ASTColumnsTransformers.h>
 
 namespace DB
 {
@@ -206,11 +207,15 @@ QueryTreeNodePtr MatcherNode::cloneImpl() const
 ASTPtr MatcherNode::toASTImpl() const
 {
     ASTPtr result;
+    ASTPtr transformers;
 
-    auto transformers = std::make_shared<ASTExpressionList>();
+    if (!children.empty())
+    {
+        transformers = std::make_shared<ASTColumnsTransformerList>();
 
-    for (const auto & child : children)
-        transformers->children.push_back(child->toAST());
+        for (const auto & child : children)
+            transformers->children.push_back(child->toAST());
+    }
 
     if (matcher_type == MatcherNodeType::ASTERISK)
     {
@@ -218,8 +223,11 @@ ASTPtr MatcherNode::toASTImpl() const
         {
             auto asterisk = std::make_shared<ASTAsterisk>();
 
-            asterisk->transformers = std::move(transformers);
-            asterisk->children.push_back(asterisk->transformers);
+            if (transformers)
+            {
+                asterisk->transformers = std::move(transformers);
+                asterisk->children.push_back(asterisk->transformers);
+            }
 
             result = asterisk;
         }
@@ -229,10 +237,13 @@ ASTPtr MatcherNode::toASTImpl() const
 
             auto identifier_parts = qualified_identifier.getParts();
             qualified_asterisk->qualifier = std::make_shared<ASTIdentifier>(std::move(identifier_parts));
-            qualified_asterisk->transformers = std::move(transformers);
-
             qualified_asterisk->children.push_back(qualified_asterisk->qualifier);
-            qualified_asterisk->children.push_back(qualified_asterisk->transformers);
+
+            if (transformers)
+            {
+                qualified_asterisk->transformers = std::move(transformers);
+                qualified_asterisk->children.push_back(qualified_asterisk->transformers);
+            }
 
             result = qualified_asterisk;
         }
@@ -244,8 +255,11 @@ ASTPtr MatcherNode::toASTImpl() const
             auto regexp_matcher = std::make_shared<ASTColumnsRegexpMatcher>();
             regexp_matcher->setPattern(columns_matcher->pattern());
 
-            regexp_matcher->transformers = std::move(transformers);
-            regexp_matcher->children.push_back(regexp_matcher->transformers);
+            if (transformers)
+            {
+                regexp_matcher->transformers = std::move(transformers);
+                regexp_matcher->children.push_back(regexp_matcher->transformers);
+            }
 
             result = regexp_matcher;
         }
@@ -256,10 +270,13 @@ ASTPtr MatcherNode::toASTImpl() const
 
             auto identifier_parts = qualified_identifier.getParts();
             regexp_matcher->qualifier = std::make_shared<ASTIdentifier>(std::move(identifier_parts));
-            regexp_matcher->transformers = std::move(transformers);
-
             regexp_matcher->children.push_back(regexp_matcher->qualifier);
-            regexp_matcher->children.push_back(regexp_matcher->transformers);
+
+            if (transformers)
+            {
+                regexp_matcher->transformers = std::move(transformers);
+                regexp_matcher->children.push_back(regexp_matcher->transformers);
+            }
 
             result = regexp_matcher;
         }
@@ -279,10 +296,13 @@ ASTPtr MatcherNode::toASTImpl() const
         {
             auto columns_list_matcher = std::make_shared<ASTColumnsListMatcher>();
             columns_list_matcher->column_list = std::move(column_list);
-            columns_list_matcher->transformers = std::move(transformers);
-
             columns_list_matcher->children.push_back(columns_list_matcher->column_list);
-            columns_list_matcher->children.push_back(columns_list_matcher->transformers);
+
+            if (transformers)
+            {
+                columns_list_matcher->transformers = std::move(transformers);
+                columns_list_matcher->children.push_back(columns_list_matcher->transformers);
+            }
 
             result = columns_list_matcher;
         }
@@ -293,11 +313,14 @@ ASTPtr MatcherNode::toASTImpl() const
             auto identifier_parts = qualified_identifier.getParts();
             columns_list_matcher->qualifier = std::make_shared<ASTIdentifier>(std::move(identifier_parts));
             columns_list_matcher->column_list = std::move(column_list);
-            columns_list_matcher->transformers = std::move(transformers);
-
             columns_list_matcher->children.push_back(columns_list_matcher->qualifier);
             columns_list_matcher->children.push_back(columns_list_matcher->column_list);
-            columns_list_matcher->children.push_back(columns_list_matcher->transformers);
+
+            if (transformers)
+            {
+                columns_list_matcher->transformers = std::move(transformers);
+                columns_list_matcher->children.push_back(columns_list_matcher->transformers);
+            }
 
             result = columns_list_matcher;
         }
diff --git a/src/Analyzer/QueryTreeBuilder.cpp b/src/Analyzer/QueryTreeBuilder.cpp
index d640d93c5e3..375e4debed4 100644
--- a/src/Analyzer/QueryTreeBuilder.cpp
+++ b/src/Analyzer/QueryTreeBuilder.cpp
@@ -809,6 +809,9 @@ ColumnTransformersNodes QueryTreeBuilder::buildColumnTransformers(const ASTPtr &
 {
     ColumnTransformersNodes column_transformers;
 
+    if (!matcher_expression)
+        return column_transformers;
+
     for (const auto & child : matcher_expression->children)
     {
         if (auto * apply_transformer = child->as<ASTColumnsApplyTransformer>())
diff --git a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp
index 11662d764f0..6a9f9cae798 100644
--- a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp
+++ b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp
@@ -50,9 +50,7 @@ ASTPtr makeSubqueryQualifiedAsterisk()
 {
     auto asterisk = std::make_shared<ASTQualifiedAsterisk>();
     asterisk->qualifier = std::make_shared<ASTIdentifier>("--.s");
-    asterisk->transformers = std::make_shared<ASTExpressionList>();
     asterisk->children.push_back(asterisk->qualifier);
-    asterisk->children.push_back(asterisk->transformers);
     return asterisk;
 }
 
@@ -142,8 +140,11 @@ private:
                 for (auto & table_name : data.tables_order)
                     data.addTableColumns(table_name, columns);
 
-                for (const auto & transformer : asterisk->transformers->children)
-                    IASTColumnsTransformer::transform(transformer, columns);
+                if (asterisk->transformers)
+                {
+                    for (const auto & transformer : asterisk->transformers->children)
+                        IASTColumnsTransformer::transform(transformer, columns);
+                }
             }
             else if (const auto * qualified_asterisk = child->as<ASTQualifiedAsterisk>())
             {
@@ -155,8 +156,11 @@ private:
 
                 data.addTableColumns(identifier.name(), columns);
 
-                for (const auto & transformer : qualified_asterisk->transformers->children)
-                    IASTColumnsTransformer::transform(transformer, columns);
+                if (qualified_asterisk->transformers)
+                {
+                    for (const auto & transformer : qualified_asterisk->transformers->children)
+                        IASTColumnsTransformer::transform(transformer, columns);
+                }
             }
             else if (const auto * columns_list_matcher = child->as<ASTColumnsListMatcher>())
             {
@@ -165,8 +169,11 @@ private:
                 for (const auto & ident : columns_list_matcher->column_list->children)
                     columns.emplace_back(ident->clone());
 
-                for (const auto & transformer : columns_list_matcher->transformers->children)
-                    IASTColumnsTransformer::transform(transformer, columns);
+                if (columns_list_matcher->transformers)
+                {
+                    for (const auto & transformer : columns_list_matcher->transformers->children)
+                        IASTColumnsTransformer::transform(transformer, columns);
+                }
             }
             else if (const auto * columns_regexp_matcher = child->as<ASTColumnsRegexpMatcher>())
             {
@@ -178,8 +185,11 @@ private:
                         columns,
                         [&](const String & column_name) { return columns_regexp_matcher->isColumnMatching(column_name); });
 
-                for (const auto & transformer : columns_regexp_matcher->transformers->children)
-                    IASTColumnsTransformer::transform(transformer, columns);
+                if (columns_regexp_matcher->transformers)
+                {
+                    for (const auto & transformer : columns_regexp_matcher->transformers->children)
+                        IASTColumnsTransformer::transform(transformer, columns);
+                }
             }
             else
                 data.new_select_expression_list->children.push_back(child);
@@ -409,12 +419,7 @@ private:
         if (!data.done)
         {
             if (data.expression_list->children.empty())
-            {
-                auto asterisk = std::make_shared<ASTAsterisk>();
-                asterisk->transformers = std::make_shared<ASTExpressionList>();
-                asterisk->children.push_back(asterisk->transformers);
-                data.expression_list->children.push_back(std::move(asterisk));
-            }
+                data.expression_list->children.emplace_back(std::make_shared<ASTAsterisk>());
 
             select.setExpression(ASTSelectQuery::Expression::SELECT, std::move(data.expression_list));
         }
diff --git a/src/Interpreters/JoinedTables.cpp b/src/Interpreters/JoinedTables.cpp
index f41481270d4..1d8676cfc57 100644
--- a/src/Interpreters/JoinedTables.cpp
+++ b/src/Interpreters/JoinedTables.cpp
@@ -96,12 +96,7 @@ void replaceJoinedTable(const ASTSelectQuery & select_query)
 
             auto new_select = addASTChildren<ASTSelectQuery>(*list_of_selects);
             new_select->setExpression(ASTSelectQuery::Expression::SELECT, std::make_shared<ASTExpressionList>());
-
-            auto asterisk = std::make_shared<ASTAsterisk>();
-            asterisk->transformers = std::make_shared<ASTExpressionList>();
-            asterisk->children.push_back(asterisk->transformers);
-            new_select->select()->children.push_back(std::move(asterisk));
-
+            addASTChildren<ASTAsterisk>(*new_select->select());
             new_select->setExpression(ASTSelectQuery::Expression::TABLES, std::make_shared<ASTTablesInSelectQuery>());
 
             auto tables_elem = addASTChildren<ASTTablesInSelectQueryElement>(*new_select->tables());
diff --git a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp
index bcd2c6f44d1..af8d637c6e1 100644
--- a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp
+++ b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp
@@ -253,16 +253,22 @@ void TranslateQualifiedNamesMatcher::visit(ASTExpressionList & node, const ASTPt
                 first_table = false;
             }
 
-            for (const auto & transformer : asterisk->transformers->children)
-                IASTColumnsTransformer::transform(transformer, columns);
+            if (asterisk->transformers)
+            {
+                for (const auto & transformer : asterisk->transformers->children)
+                    IASTColumnsTransformer::transform(transformer, columns);
+            }
         }
         else if (auto * asterisk_column_list = child->as<ASTColumnsListMatcher>())
         {
             for (const auto & ident : asterisk_column_list->column_list->children)
                 columns.emplace_back(ident->clone());
 
-            for (const auto & transformer : asterisk_column_list->transformers->children)
-                IASTColumnsTransformer::transform(transformer, columns);
+            if (asterisk_column_list->transformers)
+            {
+                for (const auto & transformer : asterisk_column_list->transformers->children)
+                    IASTColumnsTransformer::transform(transformer, columns);
+            }
         }
         else if (const auto * asterisk_regexp_pattern = child->as<ASTColumnsRegexpMatcher>())
         {
@@ -279,8 +285,11 @@ void TranslateQualifiedNamesMatcher::visit(ASTExpressionList & node, const ASTPt
                 first_table = false;
             }
 
-            for (const auto & transformer : asterisk_regexp_pattern->transformers->children)
-                IASTColumnsTransformer::transform(transformer, columns);
+            if (asterisk_regexp_pattern->transformers)
+            {
+                for (const auto & transformer : asterisk_regexp_pattern->transformers->children)
+                    IASTColumnsTransformer::transform(transformer, columns);
+            }
         }
         else if (const auto * qualified_asterisk = child->as<ASTQualifiedAsterisk>())
         {
@@ -296,8 +305,11 @@ void TranslateQualifiedNamesMatcher::visit(ASTExpressionList & node, const ASTPt
                 }
             }
 
-            for (const auto & transformer : qualified_asterisk->transformers->children)
-                IASTColumnsTransformer::transform(transformer, columns);
+            if (qualified_asterisk->transformers)
+            {
+                for (const auto & transformer : qualified_asterisk->transformers->children)
+                    IASTColumnsTransformer::transform(transformer, columns);
+            }
         }
         else
             columns.emplace_back(child);
diff --git a/src/Parsers/ASTAsterisk.cpp b/src/Parsers/ASTAsterisk.cpp
index c3c3b5b32e4..1ffbb85da7c 100644
--- a/src/Parsers/ASTAsterisk.cpp
+++ b/src/Parsers/ASTAsterisk.cpp
@@ -10,14 +10,21 @@ ASTPtr ASTAsterisk::clone() const
     auto clone = std::make_shared<ASTAsterisk>(*this);
 
     if (expression) { clone->expression = expression->clone(); clone->children.push_back(clone->expression); }
-
-    clone->transformers = transformers->clone();
-    clone->children.push_back(clone->transformers);
+    if (transformers) { clone->transformers = transformers->clone(); clone->children.push_back(clone->transformers); }
 
     return clone;
 }
 
-void ASTAsterisk::appendColumnName(WriteBuffer & ostr) const { ostr.write('*'); }
+void ASTAsterisk::appendColumnName(WriteBuffer & ostr) const
+{
+    if (expression)
+    {
+        expression->appendColumnName(ostr);
+        writeCString(".", ostr);
+    }
+
+    ostr.write('*');
+}
 
 void ASTAsterisk::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
 {
@@ -29,11 +36,9 @@ void ASTAsterisk::formatImpl(const FormatSettings & settings, FormatState & stat
 
     settings.ostr << "*";
 
-    /// Format column transformers
-    for (const auto & child : transformers->children)
+    if (transformers)
     {
-        settings.ostr << ' ';
-        child->formatImpl(settings, state, frame);
+        transformers->formatImpl(settings, state, frame);
     }
 }
 
diff --git a/src/Parsers/ASTColumnsMatcher.cpp b/src/Parsers/ASTColumnsMatcher.cpp
index 0be43dd199d..72f5661f020 100644
--- a/src/Parsers/ASTColumnsMatcher.cpp
+++ b/src/Parsers/ASTColumnsMatcher.cpp
@@ -20,9 +20,7 @@ ASTPtr ASTColumnsRegexpMatcher::clone() const
     auto clone = std::make_shared<ASTColumnsRegexpMatcher>(*this);
 
     if (expression) { clone->expression = expression->clone(); clone->children.push_back(clone->expression); }
-
-    clone->transformers = transformers->clone();
-    clone->children.push_back(clone->transformers);
+    if (transformers) { clone->transformers = transformers->clone(); clone->children.push_back(clone->transformers); }
 
     return clone;
 }
@@ -60,11 +58,9 @@ void ASTColumnsRegexpMatcher::formatImpl(const FormatSettings & settings, Format
     settings.ostr << quoteString(original_pattern);
     settings.ostr << ")";
 
-    /// Format column transformers
-    for (const auto & child : transformers->children)
+    if (transformers)
     {
-        settings.ostr << ' ';
-        child->formatImpl(settings, state, frame);
+        transformers->formatImpl(settings, state, frame);
     }
 }
 
@@ -98,22 +94,14 @@ ASTPtr ASTColumnsListMatcher::clone() const
     auto clone = std::make_shared<ASTColumnsListMatcher>(*this);
 
     if (expression) { clone->expression = expression->clone(); clone->children.push_back(clone->expression); }
+    if (transformers) { clone->transformers = transformers->clone(); clone->children.push_back(clone->transformers); }
 
     clone->column_list = column_list->clone();
-    clone->transformers = transformers->clone();
-
     clone->children.push_back(clone->column_list);
-    clone->children.push_back(clone->transformers);
 
     return clone;
 }
 
-void ASTColumnsListMatcher::updateTreeHashImpl(SipHash & hash_state) const
-{
-    column_list->updateTreeHash(hash_state);
-    IAST::updateTreeHashImpl(hash_state);
-}
-
 void ASTColumnsListMatcher::appendColumnName(WriteBuffer & ostr) const
 {
     if (expression)
@@ -154,11 +142,9 @@ void ASTColumnsListMatcher::formatImpl(const FormatSettings & settings, FormatSt
     }
     settings.ostr << ")";
 
-    /// Format column transformers
-    for (const auto & child : transformers->children)
+    if (transformers)
     {
-        settings.ostr << ' ';
-        child->formatImpl(settings, state, frame);
+        transformers->formatImpl(settings, state, frame);
     }
 }
 
@@ -166,11 +152,10 @@ ASTPtr ASTQualifiedColumnsRegexpMatcher::clone() const
 {
     auto clone = std::make_shared<ASTQualifiedColumnsRegexpMatcher>(*this);
 
-    clone->qualifier = qualifier->clone();
-    clone->transformers = transformers->clone();
+    if (transformers) { clone->transformers = transformers->clone(); clone->children.push_back(clone->transformers); }
 
+    clone->qualifier = qualifier->clone();
     clone->children.push_back(clone->qualifier);
-    clone->children.push_back(clone->transformers);
 
     return clone;
 }
@@ -224,11 +209,9 @@ void ASTQualifiedColumnsRegexpMatcher::formatImpl(const FormatSettings & setting
     settings.ostr << quoteString(original_pattern);
     settings.ostr << ")";
 
-    /// Format column transformers
-    for (const auto & child : transformers->children)
+    if (transformers)
     {
-        settings.ostr << ' ';
-        child->formatImpl(settings, state, frame);
+        transformers->formatImpl(settings, state, frame);
     }
 }
 
@@ -236,13 +219,13 @@ ASTPtr ASTQualifiedColumnsListMatcher::clone() const
 {
     auto clone = std::make_shared<ASTQualifiedColumnsListMatcher>(*this);
 
+    if (transformers) { clone->transformers = transformers->clone(); clone->children.push_back(clone->transformers); }
+
     clone->qualifier = qualifier->clone();
     clone->column_list = column_list->clone();
-    clone->transformers = transformers->clone();
 
     clone->children.push_back(clone->qualifier);
     clone->children.push_back(clone->column_list);
-    clone->children.push_back(clone->transformers);
 
     return clone;
 }
@@ -262,12 +245,6 @@ void ASTQualifiedColumnsListMatcher::appendColumnName(WriteBuffer & ostr) const
     writeChar(')', ostr);
 }
 
-void ASTQualifiedColumnsListMatcher::updateTreeHashImpl(SipHash & hash_state) const
-{
-    column_list->updateTreeHash(hash_state);
-    IAST::updateTreeHashImpl(hash_state);
-}
-
 void ASTQualifiedColumnsListMatcher::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
 {
     settings.ostr << (settings.hilite ? hilite_keyword : "");
@@ -283,11 +260,9 @@ void ASTQualifiedColumnsListMatcher::formatImpl(const FormatSettings & settings,
     }
     settings.ostr << ")";
 
-    /// Format column transformers
-    for (const auto & child : transformers->children)
+    if (transformers)
     {
-        settings.ostr << ' ';
-        child->formatImpl(settings, state, frame);
+        transformers->formatImpl(settings, state, frame);
     }
 }
 
diff --git a/src/Parsers/ASTColumnsMatcher.h b/src/Parsers/ASTColumnsMatcher.h
index d590292e7f9..f31a8bd9a22 100644
--- a/src/Parsers/ASTColumnsMatcher.h
+++ b/src/Parsers/ASTColumnsMatcher.h
@@ -46,7 +46,6 @@ public:
     String getID(char) const override { return "ColumnsListMatcher"; }
     ASTPtr clone() const override;
     void appendColumnName(WriteBuffer & ostr) const override;
-    void updateTreeHashImpl(SipHash & hash_state) const override;
 
     ASTPtr expression;
     ASTPtr column_list;
@@ -85,7 +84,6 @@ public:
     String getID(char) const override { return "QualifiedColumnsListMatcher"; }
     ASTPtr clone() const override;
     void appendColumnName(WriteBuffer & ostr) const override;
-    void updateTreeHashImpl(SipHash & hash_state) const override;
 
     ASTPtr qualifier;
     ASTPtr column_list;
diff --git a/src/Parsers/ASTColumnsTransformers.cpp b/src/Parsers/ASTColumnsTransformers.cpp
index 118c22b463f..3024f84a5f4 100644
--- a/src/Parsers/ASTColumnsTransformers.cpp
+++ b/src/Parsers/ASTColumnsTransformers.cpp
@@ -19,6 +19,15 @@ namespace ErrorCodes
     extern const int CANNOT_COMPILE_REGEXP;
 }
 
+void ASTColumnsTransformerList::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
+{
+    for (const auto & child : children)
+    {
+        settings.ostr << ' ';
+        child->formatImpl(settings, state, frame);
+    }
+}
+
 void IASTColumnsTransformer::transform(const ASTPtr & transformer, ASTs & nodes)
 {
     if (const auto * apply = transformer->as<ASTColumnsApplyTransformer>())
diff --git a/src/Parsers/ASTColumnsTransformers.h b/src/Parsers/ASTColumnsTransformers.h
index 5179726e8cb..f67993724c1 100644
--- a/src/Parsers/ASTColumnsTransformers.h
+++ b/src/Parsers/ASTColumnsTransformers.h
@@ -9,6 +9,23 @@ namespace re2
 
 namespace DB
 {
+
+/// A list of column transformers
+class ASTColumnsTransformerList : public IAST
+{
+public:
+    String getID(char) const override { return "ColumnsTransformerList"; }
+    ASTPtr clone() const override
+    {
+        auto clone = std::make_shared<ASTColumnsTransformerList>(*this);
+        clone->cloneChildren();
+        return clone;
+    }
+
+protected:
+    void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
+};
+
 class IASTColumnsTransformer : public IAST
 {
 public:
diff --git a/src/Parsers/ASTQualifiedAsterisk.cpp b/src/Parsers/ASTQualifiedAsterisk.cpp
index c14d2259465..2dcf481adb7 100644
--- a/src/Parsers/ASTQualifiedAsterisk.cpp
+++ b/src/Parsers/ASTQualifiedAsterisk.cpp
@@ -16,11 +16,9 @@ void ASTQualifiedAsterisk::formatImpl(const FormatSettings & settings, FormatSta
     qualifier->formatImpl(settings, state, frame);
     settings.ostr << ".*";
 
-    /// Format column transformers
-    for (const auto & child : transformers->children)
+    if (transformers)
     {
-        settings.ostr << ' ';
-        child->formatImpl(settings, state, frame);
+        transformers->formatImpl(settings, state, frame);
     }
 }
 
diff --git a/src/Parsers/ASTQualifiedAsterisk.h b/src/Parsers/ASTQualifiedAsterisk.h
index a7016b900e9..e67b4cd82dd 100644
--- a/src/Parsers/ASTQualifiedAsterisk.h
+++ b/src/Parsers/ASTQualifiedAsterisk.h
@@ -18,11 +18,10 @@ public:
     {
         auto clone = std::make_shared<ASTQualifiedAsterisk>(*this);
 
-        clone->qualifier = qualifier->clone();
-        clone->transformers = transformers->clone();
+        if (transformers) { clone->transformers = transformers->clone(); clone->children.push_back(clone->transformers); }
 
+        clone->qualifier = qualifier->clone();
         clone->children.push_back(clone->qualifier);
-        clone->children.push_back(clone->transformers);
 
         return clone;
     }
diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp
index 5a057b435ff..07f2acb1e86 100644
--- a/src/Parsers/ExpressionElementParsers.cpp
+++ b/src/Parsers/ExpressionElementParsers.cpp
@@ -80,12 +80,7 @@ static ASTPtr buildSelectFromTableFunction(const std::shared_ptr<ASTFunction> &
     {
         auto select_ast = std::make_shared<ASTSelectQuery>();
         select_ast->setExpression(ASTSelectQuery::Expression::SELECT, std::make_shared<ASTExpressionList>());
-
-        auto asterisk = std::make_shared<ASTAsterisk>();
-        asterisk->transformers = std::make_shared<ASTExpressionList>();
-        asterisk->children.push_back(asterisk->transformers);
-
-        select_ast->select()->children.push_back(std::move(asterisk));
+        select_ast->select()->children.push_back(std::make_shared<ASTAsterisk>());
 
         auto list_of_selects = std::make_shared<ASTExpressionList>();
         list_of_selects->children.push_back(select_ast);
@@ -1553,7 +1548,7 @@ bool ParserAsterisk::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
     {
         ++pos;
         auto asterisk = std::make_shared<ASTAsterisk>();
-        auto transformers = std::make_shared<ASTExpressionList>();
+        auto transformers = std::make_shared<ASTColumnsTransformerList>();
         ParserColumnsTransformers transformers_p(allowed_transformers);
         ASTPtr transformer;
         while (transformers_p.parse(pos, transformer, expected))
@@ -1561,8 +1556,11 @@ bool ParserAsterisk::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
             transformers->children.push_back(transformer);
         }
 
-        asterisk->transformers = std::move(transformers);
-        asterisk->children.push_back(asterisk->transformers);
+        if (!transformers->children.empty())
+        {
+            asterisk->transformers = std::move(transformers);
+            asterisk->children.push_back(asterisk->transformers);
+        }
 
         node = std::move(asterisk);
         return true;
@@ -1585,7 +1583,7 @@ bool ParserQualifiedAsterisk::parseImpl(Pos & pos, ASTPtr & node, Expected & exp
     ++pos;
 
     auto res = std::make_shared<ASTQualifiedAsterisk>();
-    auto transformers = std::make_shared<ASTExpressionList>();
+    auto transformers = std::make_shared<ASTColumnsTransformerList>();
     ParserColumnsTransformers transformers_p;
     ASTPtr transformer;
     while (transformers_p.parse(pos, transformer, expected))
@@ -1594,9 +1592,13 @@ bool ParserQualifiedAsterisk::parseImpl(Pos & pos, ASTPtr & node, Expected & exp
     }
 
     res->qualifier = std::move(node);
-    res->transformers = std::move(transformers);
     res->children.push_back(res->qualifier);
-    res->children.push_back(res->transformers);
+
+    if (!transformers->children.empty())
+    {
+        res->transformers = std::move(transformers);
+        res->children.push_back(res->transformers);
+    }
 
     node = std::move(res);
     return true;
@@ -1621,7 +1623,7 @@ static bool parseColumnsMatcherBody(IParser::Pos & pos, ASTPtr & node, Expected
         return false;
     ++pos;
 
-    auto transformers = std::make_shared<ASTExpressionList>();
+    auto transformers = std::make_shared<ASTColumnsTransformerList>();
     ParserColumnsTransformers transformers_p(allowed_transformers);
     ASTPtr transformer;
     while (transformers_p.parse(pos, transformer, expected))
@@ -1635,9 +1637,13 @@ static bool parseColumnsMatcherBody(IParser::Pos & pos, ASTPtr & node, Expected
         auto list_matcher = std::make_shared<ASTColumnsListMatcher>();
 
         list_matcher->column_list = std::move(column_list);
-        list_matcher->transformers = std::move(transformers);
         list_matcher->children.push_back(list_matcher->column_list);
-        list_matcher->children.push_back(list_matcher->transformers);
+
+        if (!transformers->children.empty())
+        {
+            list_matcher->transformers = std::move(transformers);
+            list_matcher->children.push_back(list_matcher->transformers);
+        }
 
         node = std::move(list_matcher);
     }
@@ -1646,8 +1652,11 @@ static bool parseColumnsMatcherBody(IParser::Pos & pos, ASTPtr & node, Expected
         auto regexp_matcher = std::make_shared<ASTColumnsRegexpMatcher>();
         regexp_matcher->setPattern(regex_node->as<ASTLiteral &>().value.get<String>());
 
-        regexp_matcher->transformers = std::move(transformers);
-        regexp_matcher->children.push_back(regexp_matcher->transformers);
+        if (!transformers->children.empty())
+        {
+            regexp_matcher->transformers = std::move(transformers);
+            regexp_matcher->children.push_back(regexp_matcher->transformers);
+        }
 
         node = std::move(regexp_matcher);
     }
@@ -1689,11 +1698,15 @@ bool ParserQualifiedColumnsMatcher::parseImpl(Pos & pos, ASTPtr & node, Expected
         auto result = std::make_shared<ASTQualifiedColumnsListMatcher>();
         result->qualifier = std::move(identifier_node);
         result->column_list = std::move(columns_list_matcher->column_list);
-        result->transformers = std::move(columns_list_matcher->transformers);
 
         result->children.push_back(result->qualifier);
         result->children.push_back(result->column_list);
-        result->children.push_back(result->transformers);
+
+        if (columns_list_matcher->transformers)
+        {
+            result->transformers = std::move(columns_list_matcher->transformers);
+            result->children.push_back(result->transformers);
+        }
 
         node = std::move(result);
     }
@@ -1704,10 +1717,13 @@ bool ParserQualifiedColumnsMatcher::parseImpl(Pos & pos, ASTPtr & node, Expected
         result->setMatcher(column_regexp_matcher->getMatcher());
 
         result->qualifier = std::move(identifier_node);
-        result->transformers = std::move(column_regexp_matcher->transformers);
-
         result->children.push_back(result->qualifier);
-        result->children.push_back(result->transformers);
+
+        if (column_regexp_matcher->transformers)
+        {
+            result->transformers = std::move(column_regexp_matcher->transformers);
+            result->children.push_back(result->transformers);
+        }
 
         node = std::move(result);
     }
diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp
index 651b1bf2478..e0f5d0a9f57 100644
--- a/src/Parsers/ExpressionListParsers.cpp
+++ b/src/Parsers/ExpressionListParsers.cpp
@@ -194,11 +194,7 @@ static bool modifyAST(ASTPtr ast, SubqueryFunctionType type)
         return false;
 
     /// subquery --> (SELECT aggregate_function(*) FROM subquery)
-    auto asterisk = std::make_shared<ASTAsterisk>();
-    asterisk->transformers = std::make_shared<ASTExpressionList>();
-    asterisk->children.push_back(asterisk->transformers);
-
-    auto aggregate_function = makeASTFunction(aggregate_function_name, std::move(asterisk));
+    auto aggregate_function = makeASTFunction(aggregate_function_name, std::make_shared<ASTAsterisk>());
     auto subquery_node = function->children[0]->children[1];
 
     auto table_expression = std::make_shared<ASTTableExpression>();
diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp
index 64604d80a8b..c0bc5ad8da9 100644
--- a/src/Storages/WindowView/StorageWindowView.cpp
+++ b/src/Storages/WindowView/StorageWindowView.cpp
@@ -327,10 +327,7 @@ namespace
     {
         auto fetch_query = std::make_shared<ASTSelectQuery>();
         auto select = std::make_shared<ASTExpressionList>();
-        auto asterisk = std::make_shared<ASTAsterisk>();
-        asterisk->transformers = std::make_shared<ASTExpressionList>();
-        asterisk->children.push_back(asterisk->transformers);
-        select->children.push_back(asterisk);
+        select->children.push_back(std::make_shared<ASTAsterisk>());
         fetch_query->setExpression(ASTSelectQuery::Expression::SELECT, select);
         fetch_query->setExpression(ASTSelectQuery::Expression::TABLES, std::make_shared<ASTTablesInSelectQuery>());
         auto tables_elem = std::make_shared<ASTTablesInSelectQueryElement>();

From a2fee110b89a94c2c8ea13e2148f4a24eabcb68f Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Mon, 12 Dec 2022 14:33:42 +0000
Subject: [PATCH 029/158] add query for waiting of loading parts and fix
 test_transactions

---
 src/Access/Common/AccessType.h                |  1 +
 src/Interpreters/InterpreterSystemQuery.cpp   | 29 ++++++++++++-
 src/Interpreters/InterpreterSystemQuery.h     |  3 +-
 src/Interpreters/MergeTreeTransaction.cpp     |  1 -
 src/Parsers/ASTSystemQuery.cpp                |  1 +
 src/Parsers/ASTSystemQuery.h                  |  1 +
 src/Parsers/ParserSystemQuery.cpp             |  1 +
 src/Storages/MergeTree/MergeTreeData.h        |  3 +-
 .../test_merge_tree_load_parts/test.py        | 42 ++++++-------------
 tests/integration/test_transactions/test.py   |  6 ++-
 10 files changed, 52 insertions(+), 36 deletions(-)

diff --git a/src/Access/Common/AccessType.h b/src/Access/Common/AccessType.h
index ed87b13f01a..e9c7b19af07 100644
--- a/src/Access/Common/AccessType.h
+++ b/src/Access/Common/AccessType.h
@@ -164,6 +164,7 @@ enum class AccessType
     M(SYSTEM_SYNC_REPLICA, "SYNC REPLICA", TABLE, SYSTEM) \
     M(SYSTEM_RESTART_REPLICA, "RESTART REPLICA", TABLE, SYSTEM) \
     M(SYSTEM_RESTORE_REPLICA, "RESTORE REPLICA", TABLE, SYSTEM) \
+    M(SYSTEM_WAIT_LOADING_PARTS, "WAIT LOADING PARTS", TABLE, SYSTEM) \
     M(SYSTEM_SYNC_DATABASE_REPLICA, "SYNC DATABASE REPLICA", DATABASE, SYSTEM) \
     M(SYSTEM_SYNC_TRANSACTION_LOG, "SYNC TRANSACTION LOG", GLOBAL, SYSTEM) \
     M(SYSTEM_FLUSH_DISTRIBUTED, "FLUSH DISTRIBUTED", TABLE, SYSTEM_FLUSH) \
diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp
index d05fd70e074..3ae9b16be89 100644
--- a/src/Interpreters/InterpreterSystemQuery.cpp
+++ b/src/Interpreters/InterpreterSystemQuery.cpp
@@ -484,7 +484,7 @@ BlockIO InterpreterSystemQuery::execute()
             dropReplica(query);
             break;
         case Type::SYNC_REPLICA:
-            syncReplica(query);
+            syncReplica();
             break;
         case Type::SYNC_DATABASE_REPLICA:
             syncReplicatedDatabase(query);
@@ -504,6 +504,9 @@ BlockIO InterpreterSystemQuery::execute()
         case Type::RESTORE_REPLICA:
             restoreReplica();
             break;
+        case Type::WAIT_LOADING_PARTS:
+            waitLoadingParts();
+            break;
         case Type::RESTART_DISK:
             restartDisk(query.disk);
             break;
@@ -781,7 +784,7 @@ bool InterpreterSystemQuery::dropReplicaImpl(ASTSystemQuery & query, const Stora
     return true;
 }
 
-void InterpreterSystemQuery::syncReplica(ASTSystemQuery &)
+void InterpreterSystemQuery::syncReplica()
 {
     getContext()->checkAccess(AccessType::SYSTEM_SYNC_REPLICA, table_id);
     StoragePtr table = DatabaseCatalog::instance().getTable(table_id, getContext());
@@ -801,6 +804,23 @@ void InterpreterSystemQuery::syncReplica(ASTSystemQuery &)
         throw Exception(ErrorCodes::BAD_ARGUMENTS, table_is_not_replicated.data(), table_id.getNameForLogs());
 }
 
+void InterpreterSystemQuery::waitLoadingParts()
+{
+    getContext()->checkAccess(AccessType::SYSTEM_WAIT_LOADING_PARTS, table_id);
+    StoragePtr table = DatabaseCatalog::instance().getTable(table_id, getContext());
+
+    if (auto * merge_tree = dynamic_cast<MergeTreeData *>(table.get()))
+    {
+        LOG_TRACE(log, "Waiting for loading of parts of table {}", table_id.getFullTableName());
+        merge_tree->waitForOutdatedPartsToBeLoaded();
+        LOG_TRACE(log, "Finished waiting for loading of parts of table {}", table_id.getFullTableName());
+    }
+    else
+    {
+        throw Exception(ErrorCodes::BAD_ARGUMENTS,
+            "Command WAIT LOADING PARTS is supported only for MergeTree table, but got: {}", table->getName());
+    }
+}
 
 void InterpreterSystemQuery::syncReplicatedDatabase(ASTSystemQuery & query)
 {
@@ -1005,6 +1025,11 @@ AccessRightsElements InterpreterSystemQuery::getRequiredAccessForDDLOnCluster()
             required_access.emplace_back(AccessType::SYSTEM_RESTART_REPLICA);
             break;
         }
+        case Type::WAIT_LOADING_PARTS:
+        {
+            required_access.emplace_back(AccessType::SYSTEM_WAIT_LOADING_PARTS, query.getDatabase(), query.getTable());
+            break;
+        }
         case Type::SYNC_DATABASE_REPLICA:
         {
             required_access.emplace_back(AccessType::SYSTEM_SYNC_DATABASE_REPLICA, query.getDatabase());
diff --git a/src/Interpreters/InterpreterSystemQuery.h b/src/Interpreters/InterpreterSystemQuery.h
index af8734e8237..585a4ea80b3 100644
--- a/src/Interpreters/InterpreterSystemQuery.h
+++ b/src/Interpreters/InterpreterSystemQuery.h
@@ -56,7 +56,8 @@ private:
 
     void restartReplica(const StorageID & replica, ContextMutablePtr system_context);
     void restartReplicas(ContextMutablePtr system_context);
-    void syncReplica(ASTSystemQuery & query);
+    void syncReplica();
+    void waitLoadingParts();
 
     void syncReplicatedDatabase(ASTSystemQuery & query);
 
diff --git a/src/Interpreters/MergeTreeTransaction.cpp b/src/Interpreters/MergeTreeTransaction.cpp
index f438194b87b..f16ece46530 100644
--- a/src/Interpreters/MergeTreeTransaction.cpp
+++ b/src/Interpreters/MergeTreeTransaction.cpp
@@ -303,7 +303,6 @@ bool MergeTreeTransaction::rollback() noexcept
         part->version.unlockRemovalTID(tid, TransactionInfoContext{part->storage.getStorageID(), part->name});
     }
 
-
     assert([&]()
     {
         std::lock_guard lock{mutex};
diff --git a/src/Parsers/ASTSystemQuery.cpp b/src/Parsers/ASTSystemQuery.cpp
index ab5137d0960..7564362269a 100644
--- a/src/Parsers/ASTSystemQuery.cpp
+++ b/src/Parsers/ASTSystemQuery.cpp
@@ -166,6 +166,7 @@ void ASTSystemQuery::formatImpl(const FormatSettings & settings, FormatState &,
     else if (  type == Type::RESTART_REPLICA
             || type == Type::RESTORE_REPLICA
             || type == Type::SYNC_REPLICA
+            || type == Type::WAIT_LOADING_PARTS
             || type == Type::FLUSH_DISTRIBUTED
             || type == Type::RELOAD_DICTIONARY
             || type == Type::RELOAD_MODEL
diff --git a/src/Parsers/ASTSystemQuery.h b/src/Parsers/ASTSystemQuery.h
index 2498dfdc12b..9d589ab1aef 100644
--- a/src/Parsers/ASTSystemQuery.h
+++ b/src/Parsers/ASTSystemQuery.h
@@ -35,6 +35,7 @@ public:
         RESTART_REPLICAS,
         RESTART_REPLICA,
         RESTORE_REPLICA,
+        WAIT_LOADING_PARTS,
         DROP_REPLICA,
         SYNC_REPLICA,
         SYNC_DATABASE_REPLICA,
diff --git a/src/Parsers/ParserSystemQuery.cpp b/src/Parsers/ParserSystemQuery.cpp
index 26ba9290d04..c1958f2748c 100644
--- a/src/Parsers/ParserSystemQuery.cpp
+++ b/src/Parsers/ParserSystemQuery.cpp
@@ -238,6 +238,7 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected &
 
         case Type::RESTART_REPLICA:
         case Type::SYNC_REPLICA:
+        case Type::WAIT_LOADING_PARTS:
         {
             parseQueryWithOnCluster(res, pos, expected);
             if (!parseDatabaseAndTableAsAST(pos, expected, res->database, res->table))
diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h
index b00885376dd..38f17d1571a 100644
--- a/src/Storages/MergeTree/MergeTreeData.h
+++ b/src/Storages/MergeTree/MergeTreeData.h
@@ -1038,6 +1038,8 @@ public:
     /// Returns an object that protects temporary directory from cleanup
     scope_guard getTemporaryPartDirectoryHolder(const String & part_dir_name) const;
 
+    void waitForOutdatedPartsToBeLoaded() const;
+
 protected:
     friend class IMergeTreeDataPart;
     friend class MergeTreeDataMergerMutator;
@@ -1380,7 +1382,6 @@ protected:
     std::atomic_bool outdated_data_parts_loading_canceled = false;
 
     void loadOutdatedDataParts(PartLoadingTreeNodes parts_to_load);
-    void waitForOutdatedPartsToBeLoaded() const;
     void startOutdatedDataPartsLoadingTask();
     void stopOutdatedDataPartsLoadingTask();
 
diff --git a/tests/integration/test_merge_tree_load_parts/test.py b/tests/integration/test_merge_tree_load_parts/test.py
index 4b5dec68c6e..e890b559c5e 100644
--- a/tests/integration/test_merge_tree_load_parts/test.py
+++ b/tests/integration/test_merge_tree_load_parts/test.py
@@ -51,11 +51,15 @@ def test_merge_tree_load_parts(started_cluster):
     node1.query("OPTIMIZE TABLE mt_load_parts FINAL")
     node1.restart_clickhouse(kill=True)
 
+    node1.query("SYSTEM WAIT LOADING PARTS mt_load_parts")
+
     assert node1.contains_in_log("Loading Active part 44_1_20")
     for i in range(1, 21):
         assert not node1.contains_in_log(f"Loading Active part 44_{i}_{i}_0")
+        assert node1.contains_in_log(f"Loading Outdated part 44_{i}_{i}_0")
 
     assert node1.query("SELECT count() FROM mt_load_parts") == "20\n"
+
     assert (
         node1.query(
             "SELECT count() FROM system.parts WHERE table = 'mt_load_parts' AND active"
@@ -63,44 +67,23 @@ def test_merge_tree_load_parts(started_cluster):
         == "1\n"
     )
 
-    MAX_RETRY = 20
-
-    all_outdated_loaded = False
-    for _ in range(MAX_RETRY):
-        all_outdated_loaded = all(
-            [
-                node1.contains_in_log(f"Loading Outdated part 44_{i}_{i}_0")
-                for i in range(1, 21)
-            ]
-        )
-        if all_outdated_loaded:
-            break
-        time.sleep(2)
-
-    assert all_outdated_loaded
-
     node1.query("ALTER TABLE mt_load_parts MODIFY SETTING old_parts_lifetime = 1")
     node1.query("DETACH TABLE mt_load_parts")
     node1.query("ATTACH TABLE mt_load_parts")
 
+    node1.query("SYSTEM WAIT LOADING PARTS mt_load_parts")
+
     table_path = node1.query(
         "SELECT data_paths[1] FROM system.tables WHERE table = 'mt_load_parts'"
     ).strip()
 
-    part_dirs_ok = False
-    for _ in range(MAX_RETRY):
-        part_dirs = node1.exec_in_container(
-            ["bash", "-c", f"ls {table_path}"], user="root"
-        )
-        part_dirs = list(
-            set(part_dirs.strip().split("\n")) - {"detached", "format_version.txt"}
-        )
-        part_dirs_ok = len(part_dirs) == 1 and part_dirs[0].startswith("44_1_20")
-        if part_dirs_ok:
-            break
-        time.sleep(2)
+    part_dirs = node1.exec_in_container(["bash", "-c", f"ls {table_path}"], user="root")
 
-    assert part_dirs_ok
+    part_dirs = list(
+        set(part_dirs.strip().split("\n")) - {"detached", "format_version.txt"}
+    )
+
+    assert len(part_dirs) == 1 and part_dirs[0].startswith("44_1_20")
 
 
 def test_merge_tree_load_parts_corrupted(started_cluster):
@@ -148,6 +131,7 @@ def test_merge_tree_load_parts_corrupted(started_cluster):
     corrupt_part_data_on_disk(node1, "mt_load_parts_2", get_part_name(node1, 333, 2, 2))
 
     node1.restart_clickhouse(kill=True)
+    node1.query("SYSTEM WAIT LOADING PARTS mt_load_parts_2")
 
     def check_parts_loading(node, partition, loaded, failed, skipped):
         for (min_block, max_block) in loaded:
diff --git a/tests/integration/test_transactions/test.py b/tests/integration/test_transactions/test.py
index 7902d168707..a12d30915dd 100644
--- a/tests/integration/test_transactions/test.py
+++ b/tests/integration/test_transactions/test.py
@@ -26,7 +26,7 @@ def tx(session, query):
 
 def test_rollback_unfinished_on_restart1(start_cluster):
     node.query(
-        "create table mt (n int, m int) engine=MergeTree order by n partition by n % 2"
+        "create table mt (n int, m int) engine=MergeTree order by n partition by n % 2 settings remove_empty_parts = 0"
     )
     node.query("insert into mt values (1, 10), (2, 20)")
     tid0 = "(1,1,'00000000-0000-0000-0000-000000000000')"
@@ -82,6 +82,7 @@ def test_rollback_unfinished_on_restart1(start_cluster):
     ).strip()
 
     node.restart_clickhouse(kill=True)
+    node.query("SYSTEM WAIT LOADING PARTS mt")
 
     assert (
         node.query("select *, _part from mt order by n")
@@ -116,7 +117,7 @@ def test_rollback_unfinished_on_restart1(start_cluster):
 
 def test_rollback_unfinished_on_restart2(start_cluster):
     node.query(
-        "create table mt2 (n int, m int) engine=MergeTree order by n partition by n % 2"
+        "create table mt2 (n int, m int) engine=MergeTree order by n partition by n % 2 settings remove_empty_parts = 0"
     )
     node.query("insert into mt2 values (1, 10), (2, 20)")
     tid0 = "(1,1,'00000000-0000-0000-0000-000000000000')"
@@ -171,6 +172,7 @@ def test_rollback_unfinished_on_restart2(start_cluster):
     ).strip()
 
     node.restart_clickhouse(kill=True)
+    node.query("SYSTEM WAIT LOADING PARTS mt2")
 
     assert (
         node.query("select *, _part from mt2 order by n")

From 53a4473b066262ee90d7d17c8e182af3558f3b80 Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Mon, 12 Dec 2022 18:10:53 +0000
Subject: [PATCH 030/158] fix tests

---
 tests/queries/0_stateless/01271_show_privileges.reference   | 1 +
 .../0_stateless/02117_show_create_table_system.reference    | 6 +++---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference
index f2c3e8eda9d..f088cfaf00c 100644
--- a/tests/queries/0_stateless/01271_show_privileges.reference
+++ b/tests/queries/0_stateless/01271_show_privileges.reference
@@ -121,6 +121,7 @@ SYSTEM DROP REPLICA	['DROP REPLICA']	TABLE	SYSTEM
 SYSTEM SYNC REPLICA	['SYNC REPLICA']	TABLE	SYSTEM
 SYSTEM RESTART REPLICA	['RESTART REPLICA']	TABLE	SYSTEM
 SYSTEM RESTORE REPLICA	['RESTORE REPLICA']	TABLE	SYSTEM
+SYSTEM WAIT LOADING PARTS	['WAIT LOADING PARTS']	TABLE	SYSTEM
 SYSTEM SYNC DATABASE REPLICA	['SYNC DATABASE REPLICA']	DATABASE	SYSTEM
 SYSTEM SYNC TRANSACTION LOG	['SYNC TRANSACTION LOG']	GLOBAL	SYSTEM
 SYSTEM FLUSH DISTRIBUTED	['FLUSH DISTRIBUTED']	TABLE	SYSTEM FLUSH
diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference
index 81536c8544f..bbc28dbb6db 100644
--- a/tests/queries/0_stateless/02117_show_create_table_system.reference
+++ b/tests/queries/0_stateless/02117_show_create_table_system.reference
@@ -285,7 +285,7 @@ CREATE TABLE system.grants
 (
     `user_name` Nullable(String),
     `role_name` Nullable(String),
-    `access_type` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'ACCESS MANAGEMENT' = 92, 'SYSTEM SHUTDOWN' = 93, 'SYSTEM DROP DNS CACHE' = 94, 'SYSTEM DROP MARK CACHE' = 95, 'SYSTEM DROP UNCOMPRESSED CACHE' = 96, 'SYSTEM DROP MMAP CACHE' = 97, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 98, 'SYSTEM DROP FILESYSTEM CACHE' = 99, 'SYSTEM DROP SCHEMA CACHE' = 100, 'SYSTEM DROP CACHE' = 101, 'SYSTEM RELOAD CONFIG' = 102, 'SYSTEM RELOAD USERS' = 103, 'SYSTEM RELOAD SYMBOLS' = 104, 'SYSTEM RELOAD DICTIONARY' = 105, 'SYSTEM RELOAD MODEL' = 106, 'SYSTEM RELOAD FUNCTION' = 107, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 108, 'SYSTEM RELOAD' = 109, 'SYSTEM RESTART DISK' = 110, 'SYSTEM MERGES' = 111, 'SYSTEM TTL MERGES' = 112, 'SYSTEM FETCHES' = 113, 'SYSTEM MOVES' = 114, 'SYSTEM DISTRIBUTED SENDS' = 115, 'SYSTEM REPLICATED SENDS' = 116, 'SYSTEM SENDS' = 117, 'SYSTEM REPLICATION QUEUES' = 118, 'SYSTEM DROP REPLICA' = 119, 'SYSTEM SYNC REPLICA' = 120, 'SYSTEM RESTART REPLICA' = 121, 'SYSTEM RESTORE REPLICA' = 122, 'SYSTEM SYNC DATABASE REPLICA' = 123, 'SYSTEM SYNC TRANSACTION LOG' = 124, 'SYSTEM FLUSH DISTRIBUTED' = 125, 'SYSTEM FLUSH LOGS' = 126, 'SYSTEM FLUSH' = 127, 'SYSTEM THREAD FUZZER' = 128, 'SYSTEM UNFREEZE' = 129, 'SYSTEM' = 130, 'dictGet' = 131, 'addressToLine' = 132, 'addressToLineWithInlines' = 133, 'addressToSymbol' = 134, 'demangle' = 135, 'INTROSPECTION' = 136, 'FILE' = 137, 'URL' = 138, 'REMOTE' = 139, 'MONGO' = 140, 'MEILISEARCH' = 141, 'MYSQL' = 142, 'POSTGRES' = 143, 'SQLITE' = 144, 'ODBC' = 145, 'JDBC' = 146, 'HDFS' = 147, 'S3' = 148, 'HIVE' = 149, 'SOURCES' = 150, 'CLUSTER' = 151, 'ALL' = 152, 'NONE' = 153),
+    `access_type` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'ACCESS MANAGEMENT' = 92, 'SYSTEM SHUTDOWN' = 93, 'SYSTEM DROP DNS CACHE' = 94, 'SYSTEM DROP MARK CACHE' = 95, 'SYSTEM DROP UNCOMPRESSED CACHE' = 96, 'SYSTEM DROP MMAP CACHE' = 97, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 98, 'SYSTEM DROP FILESYSTEM CACHE' = 99, 'SYSTEM DROP SCHEMA CACHE' = 100, 'SYSTEM DROP CACHE' = 101, 'SYSTEM RELOAD CONFIG' = 102, 'SYSTEM RELOAD USERS' = 103, 'SYSTEM RELOAD SYMBOLS' = 104, 'SYSTEM RELOAD DICTIONARY' = 105, 'SYSTEM RELOAD MODEL' = 106, 'SYSTEM RELOAD FUNCTION' = 107, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 108, 'SYSTEM RELOAD' = 109, 'SYSTEM RESTART DISK' = 110, 'SYSTEM MERGES' = 111, 'SYSTEM TTL MERGES' = 112, 'SYSTEM FETCHES' = 113, 'SYSTEM MOVES' = 114, 'SYSTEM DISTRIBUTED SENDS' = 115, 'SYSTEM REPLICATED SENDS' = 116, 'SYSTEM SENDS' = 117, 'SYSTEM REPLICATION QUEUES' = 118, 'SYSTEM DROP REPLICA' = 119, 'SYSTEM SYNC REPLICA' = 120, 'SYSTEM RESTART REPLICA' = 121, 'SYSTEM RESTORE REPLICA' = 122, 'SYSTEM WAIT LOADING PARTS' = 123, 'SYSTEM SYNC DATABASE REPLICA' = 124, 'SYSTEM SYNC TRANSACTION LOG' = 125, 'SYSTEM FLUSH DISTRIBUTED' = 126, 'SYSTEM FLUSH LOGS' = 127, 'SYSTEM FLUSH' = 128, 'SYSTEM THREAD FUZZER' = 129, 'SYSTEM UNFREEZE' = 130, 'SYSTEM' = 131, 'dictGet' = 132, 'addressToLine' = 133, 'addressToLineWithInlines' = 134, 'addressToSymbol' = 135, 'demangle' = 136, 'INTROSPECTION' = 137, 'FILE' = 138, 'URL' = 139, 'REMOTE' = 140, 'MONGO' = 141, 'MEILISEARCH' = 142, 'MYSQL' = 143, 'POSTGRES' = 144, 'SQLITE' = 145, 'ODBC' = 146, 'JDBC' = 147, 'HDFS' = 148, 'S3' = 149, 'HIVE' = 150, 'SOURCES' = 151, 'CLUSTER' = 152, 'ALL' = 153, 'NONE' = 154),
     `database` Nullable(String),
     `table` Nullable(String),
     `column` Nullable(String),
@@ -564,10 +564,10 @@ ENGINE = SystemPartsColumns
 COMMENT 'SYSTEM TABLE is built on the fly.'
 CREATE TABLE system.privileges
 (
-    `privilege` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'ACCESS MANAGEMENT' = 92, 'SYSTEM SHUTDOWN' = 93, 'SYSTEM DROP DNS CACHE' = 94, 'SYSTEM DROP MARK CACHE' = 95, 'SYSTEM DROP UNCOMPRESSED CACHE' = 96, 'SYSTEM DROP MMAP CACHE' = 97, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 98, 'SYSTEM DROP FILESYSTEM CACHE' = 99, 'SYSTEM DROP SCHEMA CACHE' = 100, 'SYSTEM DROP CACHE' = 101, 'SYSTEM RELOAD CONFIG' = 102, 'SYSTEM RELOAD USERS' = 103, 'SYSTEM RELOAD SYMBOLS' = 104, 'SYSTEM RELOAD DICTIONARY' = 105, 'SYSTEM RELOAD MODEL' = 106, 'SYSTEM RELOAD FUNCTION' = 107, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 108, 'SYSTEM RELOAD' = 109, 'SYSTEM RESTART DISK' = 110, 'SYSTEM MERGES' = 111, 'SYSTEM TTL MERGES' = 112, 'SYSTEM FETCHES' = 113, 'SYSTEM MOVES' = 114, 'SYSTEM DISTRIBUTED SENDS' = 115, 'SYSTEM REPLICATED SENDS' = 116, 'SYSTEM SENDS' = 117, 'SYSTEM REPLICATION QUEUES' = 118, 'SYSTEM DROP REPLICA' = 119, 'SYSTEM SYNC REPLICA' = 120, 'SYSTEM RESTART REPLICA' = 121, 'SYSTEM RESTORE REPLICA' = 122, 'SYSTEM SYNC DATABASE REPLICA' = 123, 'SYSTEM SYNC TRANSACTION LOG' = 124, 'SYSTEM FLUSH DISTRIBUTED' = 125, 'SYSTEM FLUSH LOGS' = 126, 'SYSTEM FLUSH' = 127, 'SYSTEM THREAD FUZZER' = 128, 'SYSTEM UNFREEZE' = 129, 'SYSTEM' = 130, 'dictGet' = 131, 'addressToLine' = 132, 'addressToLineWithInlines' = 133, 'addressToSymbol' = 134, 'demangle' = 135, 'INTROSPECTION' = 136, 'FILE' = 137, 'URL' = 138, 'REMOTE' = 139, 'MONGO' = 140, 'MEILISEARCH' = 141, 'MYSQL' = 142, 'POSTGRES' = 143, 'SQLITE' = 144, 'ODBC' = 145, 'JDBC' = 146, 'HDFS' = 147, 'S3' = 148, 'HIVE' = 149, 'SOURCES' = 150, 'CLUSTER' = 151, 'ALL' = 152, 'NONE' = 153),
+    `privilege` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'ACCESS MANAGEMENT' = 92, 'SYSTEM SHUTDOWN' = 93, 'SYSTEM DROP DNS CACHE' = 94, 'SYSTEM DROP MARK CACHE' = 95, 'SYSTEM DROP UNCOMPRESSED CACHE' = 96, 'SYSTEM DROP MMAP CACHE' = 97, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 98, 'SYSTEM DROP FILESYSTEM CACHE' = 99, 'SYSTEM DROP SCHEMA CACHE' = 100, 'SYSTEM DROP CACHE' = 101, 'SYSTEM RELOAD CONFIG' = 102, 'SYSTEM RELOAD USERS' = 103, 'SYSTEM RELOAD SYMBOLS' = 104, 'SYSTEM RELOAD DICTIONARY' = 105, 'SYSTEM RELOAD MODEL' = 106, 'SYSTEM RELOAD FUNCTION' = 107, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 108, 'SYSTEM RELOAD' = 109, 'SYSTEM RESTART DISK' = 110, 'SYSTEM MERGES' = 111, 'SYSTEM TTL MERGES' = 112, 'SYSTEM FETCHES' = 113, 'SYSTEM MOVES' = 114, 'SYSTEM DISTRIBUTED SENDS' = 115, 'SYSTEM REPLICATED SENDS' = 116, 'SYSTEM SENDS' = 117, 'SYSTEM REPLICATION QUEUES' = 118, 'SYSTEM DROP REPLICA' = 119, 'SYSTEM SYNC REPLICA' = 120, 'SYSTEM RESTART REPLICA' = 121, 'SYSTEM RESTORE REPLICA' = 122, 'SYSTEM WAIT LOADING PARTS' = 123, 'SYSTEM SYNC DATABASE REPLICA' = 124, 'SYSTEM SYNC TRANSACTION LOG' = 125, 'SYSTEM FLUSH DISTRIBUTED' = 126, 'SYSTEM FLUSH LOGS' = 127, 'SYSTEM FLUSH' = 128, 'SYSTEM THREAD FUZZER' = 129, 'SYSTEM UNFREEZE' = 130, 'SYSTEM' = 131, 'dictGet' = 132, 'addressToLine' = 133, 'addressToLineWithInlines' = 134, 'addressToSymbol' = 135, 'demangle' = 136, 'INTROSPECTION' = 137, 'FILE' = 138, 'URL' = 139, 'REMOTE' = 140, 'MONGO' = 141, 'MEILISEARCH' = 142, 'MYSQL' = 143, 'POSTGRES' = 144, 'SQLITE' = 145, 'ODBC' = 146, 'JDBC' = 147, 'HDFS' = 148, 'S3' = 149, 'HIVE' = 150, 'SOURCES' = 151, 'CLUSTER' = 152, 'ALL' = 153, 'NONE' = 154),
     `aliases` Array(String),
     `level` Nullable(Enum8('GLOBAL' = 0, 'DATABASE' = 1, 'TABLE' = 2, 'DICTIONARY' = 3, 'VIEW' = 4, 'COLUMN' = 5)),
-    `parent_group` Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'ACCESS MANAGEMENT' = 92, 'SYSTEM SHUTDOWN' = 93, 'SYSTEM DROP DNS CACHE' = 94, 'SYSTEM DROP MARK CACHE' = 95, 'SYSTEM DROP UNCOMPRESSED CACHE' = 96, 'SYSTEM DROP MMAP CACHE' = 97, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 98, 'SYSTEM DROP FILESYSTEM CACHE' = 99, 'SYSTEM DROP SCHEMA CACHE' = 100, 'SYSTEM DROP CACHE' = 101, 'SYSTEM RELOAD CONFIG' = 102, 'SYSTEM RELOAD USERS' = 103, 'SYSTEM RELOAD SYMBOLS' = 104, 'SYSTEM RELOAD DICTIONARY' = 105, 'SYSTEM RELOAD MODEL' = 106, 'SYSTEM RELOAD FUNCTION' = 107, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 108, 'SYSTEM RELOAD' = 109, 'SYSTEM RESTART DISK' = 110, 'SYSTEM MERGES' = 111, 'SYSTEM TTL MERGES' = 112, 'SYSTEM FETCHES' = 113, 'SYSTEM MOVES' = 114, 'SYSTEM DISTRIBUTED SENDS' = 115, 'SYSTEM REPLICATED SENDS' = 116, 'SYSTEM SENDS' = 117, 'SYSTEM REPLICATION QUEUES' = 118, 'SYSTEM DROP REPLICA' = 119, 'SYSTEM SYNC REPLICA' = 120, 'SYSTEM RESTART REPLICA' = 121, 'SYSTEM RESTORE REPLICA' = 122, 'SYSTEM SYNC DATABASE REPLICA' = 123, 'SYSTEM SYNC TRANSACTION LOG' = 124, 'SYSTEM FLUSH DISTRIBUTED' = 125, 'SYSTEM FLUSH LOGS' = 126, 'SYSTEM FLUSH' = 127, 'SYSTEM THREAD FUZZER' = 128, 'SYSTEM UNFREEZE' = 129, 'SYSTEM' = 130, 'dictGet' = 131, 'addressToLine' = 132, 'addressToLineWithInlines' = 133, 'addressToSymbol' = 134, 'demangle' = 135, 'INTROSPECTION' = 136, 'FILE' = 137, 'URL' = 138, 'REMOTE' = 139, 'MONGO' = 140, 'MEILISEARCH' = 141, 'MYSQL' = 142, 'POSTGRES' = 143, 'SQLITE' = 144, 'ODBC' = 145, 'JDBC' = 146, 'HDFS' = 147, 'S3' = 148, 'HIVE' = 149, 'SOURCES' = 150, 'CLUSTER' = 151, 'ALL' = 152, 'NONE' = 153))
+    `parent_group` Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'ACCESS MANAGEMENT' = 92, 'SYSTEM SHUTDOWN' = 93, 'SYSTEM DROP DNS CACHE' = 94, 'SYSTEM DROP MARK CACHE' = 95, 'SYSTEM DROP UNCOMPRESSED CACHE' = 96, 'SYSTEM DROP MMAP CACHE' = 97, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 98, 'SYSTEM DROP FILESYSTEM CACHE' = 99, 'SYSTEM DROP SCHEMA CACHE' = 100, 'SYSTEM DROP CACHE' = 101, 'SYSTEM RELOAD CONFIG' = 102, 'SYSTEM RELOAD USERS' = 103, 'SYSTEM RELOAD SYMBOLS' = 104, 'SYSTEM RELOAD DICTIONARY' = 105, 'SYSTEM RELOAD MODEL' = 106, 'SYSTEM RELOAD FUNCTION' = 107, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 108, 'SYSTEM RELOAD' = 109, 'SYSTEM RESTART DISK' = 110, 'SYSTEM MERGES' = 111, 'SYSTEM TTL MERGES' = 112, 'SYSTEM FETCHES' = 113, 'SYSTEM MOVES' = 114, 'SYSTEM DISTRIBUTED SENDS' = 115, 'SYSTEM REPLICATED SENDS' = 116, 'SYSTEM SENDS' = 117, 'SYSTEM REPLICATION QUEUES' = 118, 'SYSTEM DROP REPLICA' = 119, 'SYSTEM SYNC REPLICA' = 120, 'SYSTEM RESTART REPLICA' = 121, 'SYSTEM RESTORE REPLICA' = 122, 'SYSTEM WAIT LOADING PARTS' = 123, 'SYSTEM SYNC DATABASE REPLICA' = 124, 'SYSTEM SYNC TRANSACTION LOG' = 125, 'SYSTEM FLUSH DISTRIBUTED' = 126, 'SYSTEM FLUSH LOGS' = 127, 'SYSTEM FLUSH' = 128, 'SYSTEM THREAD FUZZER' = 129, 'SYSTEM UNFREEZE' = 130, 'SYSTEM' = 131, 'dictGet' = 132, 'addressToLine' = 133, 'addressToLineWithInlines' = 134, 'addressToSymbol' = 135, 'demangle' = 136, 'INTROSPECTION' = 137, 'FILE' = 138, 'URL' = 139, 'REMOTE' = 140, 'MONGO' = 141, 'MEILISEARCH' = 142, 'MYSQL' = 143, 'POSTGRES' = 144, 'SQLITE' = 145, 'ODBC' = 146, 'JDBC' = 147, 'HDFS' = 148, 'S3' = 149, 'HIVE' = 150, 'SOURCES' = 151, 'CLUSTER' = 152, 'ALL' = 153, 'NONE' = 154))
 )
 ENGINE = SystemPrivileges
 COMMENT 'SYSTEM TABLE is built on the fly.'

From 3008e701ea3b55370448d1531a2b0eefeea27515 Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Wed, 14 Dec 2022 01:36:03 +0000
Subject: [PATCH 031/158] fix tests

---
 src/Access/tests/gtest_access_rights_ops.cpp     |  2 +-
 tests/integration/test_grant_and_revoke/test.py  |  2 +-
 .../test_merge_tree_load_parts/test.py           | 16 +++++++++++++++-
 3 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/src/Access/tests/gtest_access_rights_ops.cpp b/src/Access/tests/gtest_access_rights_ops.cpp
index 02aafb7415b..e21ebda2a31 100644
--- a/src/Access/tests/gtest_access_rights_ops.cpp
+++ b/src/Access/tests/gtest_access_rights_ops.cpp
@@ -53,7 +53,7 @@ TEST(AccessRights, Union)
               "SHOW ROW POLICIES, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, "
               "SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, "
               "SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, "
-              "SYSTEM RESTORE REPLICA, SYSTEM SYNC DATABASE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON db1.*");
+              "SYSTEM RESTORE REPLICA, SYSTEM WAIT LOADING PARTS, SYSTEM SYNC DATABASE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON db1.*");
 }
 
 
diff --git a/tests/integration/test_grant_and_revoke/test.py b/tests/integration/test_grant_and_revoke/test.py
index d0a2f2ab933..b7cb9281ba6 100644
--- a/tests/integration/test_grant_and_revoke/test.py
+++ b/tests/integration/test_grant_and_revoke/test.py
@@ -181,7 +181,7 @@ def test_grant_all_on_table():
         == "GRANT SHOW TABLES, SHOW COLUMNS, SHOW DICTIONARIES, SELECT, INSERT, ALTER TABLE, ALTER VIEW, CREATE TABLE, CREATE VIEW, CREATE DICTIONARY, "
         "DROP TABLE, DROP VIEW, DROP DICTIONARY, TRUNCATE, OPTIMIZE, BACKUP, CREATE ROW POLICY, ALTER ROW POLICY, DROP ROW POLICY, SHOW ROW POLICIES, "
         "SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, "
-        "SYSTEM RESTART REPLICA, SYSTEM RESTORE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON test.table TO B\n"
+        "SYSTEM RESTART REPLICA, SYSTEM RESTORE REPLICA, SYSTEM WAIT LOADING PARTS, SYSTEM FLUSH DISTRIBUTED, dictGet ON test.table TO B\n"
     )
     instance.query("REVOKE ALL ON test.table FROM B", user="A")
     assert instance.query("SHOW GRANTS FOR B") == ""
diff --git a/tests/integration/test_merge_tree_load_parts/test.py b/tests/integration/test_merge_tree_load_parts/test.py
index e890b559c5e..777b6f14fc6 100644
--- a/tests/integration/test_merge_tree_load_parts/test.py
+++ b/tests/integration/test_merge_tree_load_parts/test.py
@@ -83,7 +83,21 @@ def test_merge_tree_load_parts(started_cluster):
         set(part_dirs.strip().split("\n")) - {"detached", "format_version.txt"}
     )
 
-    assert len(part_dirs) == 1 and part_dirs[0].startswith("44_1_20")
+    MAX_RETRY = 10
+    part_dirs_ok = False
+    for _ in range(MAX_RETRY):
+        part_dirs = node1.exec_in_container(
+            ["bash", "-c", f"ls {table_path}"], user="root"
+        )
+        part_dirs = list(
+            set(part_dirs.strip().split("\n")) - {"detached", "format_version.txt"}
+        )
+        part_dirs_ok = len(part_dirs) == 1 and part_dirs[0].startswith("44_1_20")
+        if part_dirs_ok:
+            break
+        time.sleep(2)
+
+    assert part_dirs_ok
 
 
 def test_merge_tree_load_parts_corrupted(started_cluster):

From aa03712b6d5b6371d64b2a95f32d8f5a7a39fd83 Mon Sep 17 00:00:00 2001
From: Bharat Nallan Chakravarthy <bharatnc@gmail.com>
Date: Wed, 14 Dec 2022 23:49:51 -0800
Subject: [PATCH 032/158] fix build failure

---
 ...teFunctionQuantileInterpolatedWeighted.cpp |  1 -
 .../QuantileInterpolatedWeighted.h            | 46 ++++++++++++-------
 2 files changed, 29 insertions(+), 18 deletions(-)

diff --git a/src/AggregateFunctions/AggregateFunctionQuantileInterpolatedWeighted.cpp b/src/AggregateFunctions/AggregateFunctionQuantileInterpolatedWeighted.cpp
index c4636182262..68b42376df7 100644
--- a/src/AggregateFunctions/AggregateFunctionQuantileInterpolatedWeighted.cpp
+++ b/src/AggregateFunctions/AggregateFunctionQuantileInterpolatedWeighted.cpp
@@ -53,7 +53,6 @@ namespace
         throw Exception("Illegal type " + argument_type->getName() + " of argument for aggregate function " + name,
                         ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
     }
-
 }
 
 void registerAggregateFunctionsQuantileInterpolatedWeighted(AggregateFunctionFactory & factory)
diff --git a/src/AggregateFunctions/QuantileInterpolatedWeighted.h b/src/AggregateFunctions/QuantileInterpolatedWeighted.h
index f2e00dac973..cd6ae0473e3 100644
--- a/src/AggregateFunctions/QuantileInterpolatedWeighted.h
+++ b/src/AggregateFunctions/QuantileInterpolatedWeighted.h
@@ -75,8 +75,31 @@ struct QuantileInterpolatedWeighted
         }
     }
 
-    /// Get the value of the `level` quantile. The level must be between 0 and 1.
     Value get(Float64 level) const
+    {
+        return getImpl<Value>(level);
+    }
+
+    void getMany(const Float64 * levels, const size_t * indices, size_t size, Value * result) const
+    {
+        getManyImpl<Value>(levels, indices, size, result);
+    }
+
+    /// The same, but in the case of an empty state, NaN is returned.
+    Float64 getFloat(Float64) const
+    {
+        throw Exception("Method getFloat is not implemented for QuantileInterpolatedWeighted", ErrorCodes::NOT_IMPLEMENTED);
+    }
+
+    void getManyFloat(const Float64 *, const size_t *, size_t, Float64 *) const
+    {
+        throw Exception("Method getManyFloat is not implemented for QuantileInterpolatedWeighted", ErrorCodes::NOT_IMPLEMENTED);
+    }
+
+private:
+    /// Get the value of the `level` quantile. The level must be between 0 and 1.
+    template <typename T>
+    T getImpl(Float64 level) const
     {
         size_t size = map.size();
 
@@ -164,13 +187,13 @@ struct QuantileInterpolatedWeighted
         if (level > xr)
             yl = yr;
 
-        return interpolate(level, xl, xr, yl, yr);
+        return static_cast<T>(interpolate(level, xl, xr, yl, yr));
     }
 
-
     /// Get the `size` values of `levels` quantiles. Write `size` results starting with `result` address.
     /// indices - an array of index levels such that the corresponding elements will go in ascending order.
-    void getMany(const Float64 * levels, const size_t * indices, size_t num_levels, Value * result) const
+    template <typename T>
+    void getManyImpl(const Float64 * levels, const size_t * indices, size_t num_levels, Value * result) const
     {
         size_t size = map.size();
 
@@ -258,7 +281,7 @@ struct QuantileInterpolatedWeighted
             if (level > xr)
                 yl = yr;
 
-            result[indices[level_index]] = interpolate(level, xl, xr, yl, yr);
+            result[indices[level_index]] = static_cast<T>(interpolate(level, xl, xr, yl, yr));
             ++level_index;
         }
     }
@@ -272,21 +295,10 @@ struct QuantileInterpolatedWeighted
         dx = dx == 0 ? 1 : dx; /// to handle NaN behavior that might arise during integer division below.
 
         /// yl + (dy / dx) * (level - xl)
-        UnderlyingType g = common::addIgnoreOverflow(yl, common::mulIgnoreOverflow((dy / dx), common::subIgnoreOverflow(level, xl)));
+        UnderlyingType g = static_cast<UnderlyingType>(common::addIgnoreOverflow(yl, common::mulIgnoreOverflow((dy / dx), common::subIgnoreOverflow(level, xl))));
 
         return g;
     }
-
-    /// The same, but in the case of an empty state, NaN is returned.
-    Float64 getFloat(Float64) const
-    {
-        throw Exception("Method getFloat is not implemented for QuantileInterpolatedWeighted", ErrorCodes::NOT_IMPLEMENTED);
-    }
-
-    void getManyFloat(const Float64 *, const size_t *, size_t, Float64 *) const
-    {
-        throw Exception("Method getManyFloat is not implemented for QuantileInterpolatedWeighted", ErrorCodes::NOT_IMPLEMENTED);
-    }
 };
 
 }

From 633f2bc9d8b821153a1bacbbed98bf89ed0868ad Mon Sep 17 00:00:00 2001
From: Bharat Nallan Chakravarthy <bharatnc@gmail.com>
Date: Thu, 15 Dec 2022 20:56:49 -0800
Subject: [PATCH 033/158] add docs for quantileInterpolatedWeighted

---
 .../reference/quantileinterpolatedweighted.md | 68 +++++++++++++++++++
 1 file changed, 68 insertions(+)
 create mode 100644 docs/en/sql-reference/aggregate-functions/reference/quantileinterpolatedweighted.md

diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantileinterpolatedweighted.md b/docs/en/sql-reference/aggregate-functions/reference/quantileinterpolatedweighted.md
new file mode 100644
index 00000000000..07fcd187217
--- /dev/null
+++ b/docs/en/sql-reference/aggregate-functions/reference/quantileinterpolatedweighted.md
@@ -0,0 +1,68 @@
+---
+slug: /en/sql-reference/aggregate-functions/reference/quantileInterpolatedWeighted
+sidebar_position: 203
+---
+
+# quantileInterpolatedWeighted
+
+Computes [quantile](https://en.wikipedia.org/wiki/Quantile) of a numeric data sequence using linear interpolation, taking into account the weight of each element.
+
+To get the interpolated value, all the passed values are combined into an array, which are then sorted by their corresponding weights. Quantile interpolation is then performed using the [weighted percentile method](https://en.wikipedia.org/wiki/Percentile#The_weighted_percentile_method) by building a cumulative distribution based on weights and then a linear interpolation is performed using the weights and the values to compute the quantiles.
+
+When using multiple `quantile*` functions with different levels in a query, the internal states are not combined (that is, the query works less efficiently than it could). In this case, use the [quantiles](../../../sql-reference/aggregate-functions/reference/quantiles.md#quantiles) function.
+
+**Syntax**
+
+``` sql
+quantileInterpolatedWeighted(level)(expr, weight)
+```
+
+Alias: `medianInterpolatedWeighted`.
+
+**Arguments**
+
+-   `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` value in the range of `[0.01, 0.99]`. Default value: 0.5. At `level=0.5` the function calculates [median](https://en.wikipedia.org/wiki/Median).
+-   `expr` — Expression over the column values resulting in numeric [data types](../../../sql-reference/data-types/index.md#data_types), [Date](../../../sql-reference/data-types/date.md) or [DateTime](../../../sql-reference/data-types/datetime.md).
+-   `weight` — Column with weights of sequence members. Weight is a number of value occurrences.
+
+**Returned value**
+
+-   Quantile of the specified level.
+
+Type:
+
+-   [Float64](../../../sql-reference/data-types/float.md) for numeric data type input.
+-   [Date](../../../sql-reference/data-types/date.md) if input values have the `Date` type.
+-   [DateTime](../../../sql-reference/data-types/datetime.md) if input values have the `DateTime` type.
+
+**Example**
+
+Input table:
+
+``` text
+┌─n─┬─val─�
+│ 0 │   3 │
+│ 1 │   2 │
+│ 2 │   1 │
+│ 5 │   4 │
+└───┴─────┘
+```
+
+Query:
+
+``` sql
+SELECT quantileInterpolatedWeighted(n, val) FROM t
+```
+
+Result:
+
+``` text
+┌─quantileInterpolatedWeighted(n, val)─�
+│                                    1 │
+└──────────────────────────────────────┘
+```
+
+**See Also**
+
+-   [median](../../../sql-reference/aggregate-functions/reference/median.md#median)
+-   [quantiles](../../../sql-reference/aggregate-functions/reference/quantiles.md#quantiles)

From 1544e412f571120eef59d8bc138ba1cc0f999e55 Mon Sep 17 00:00:00 2001
From: wineternity <dawnyym@163.com>
Date: Wed, 21 Dec 2022 19:09:22 +0800
Subject: [PATCH 034/158] [DOC] Add support for signed arguments in range()
 #43333

---
 .../functions/array-functions.md              | 21 +++++++++----------
 .../functions/array-functions.md              | 21 +++++++++----------
 2 files changed, 20 insertions(+), 22 deletions(-)

diff --git a/docs/en/sql-reference/functions/array-functions.md b/docs/en/sql-reference/functions/array-functions.md
index c044b972754..9d2f89c1837 100644
--- a/docs/en/sql-reference/functions/array-functions.md
+++ b/docs/en/sql-reference/functions/array-functions.md
@@ -121,7 +121,7 @@ Accepts an empty array and returns a one-element array that is equal to the defa
 
 ## range(end), range(\[start, \] end \[, step\])
 
-Returns an array of `UInt` numbers from `start` to `end - 1` by `step`.
+Returns an array of numbers from `start` to `end - 1` by `step`. The supported types are [UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64](../data-types/int-uint.md).
 
 **Syntax**
 ``` sql
@@ -130,31 +130,30 @@ range([start, ] end [, step])
 
 **Arguments**
 
--   `start` — The first element of the array. Optional, required if `step` is used. Default value: 0. [UInt](../data-types/int-uint.md)
--   `end` — The number before which the array is constructed. Required. [UInt](../data-types/int-uint.md)
--   `step` — Determines the incremental step between each element in the array. Optional. Default value: 1. [UInt](../data-types/int-uint.md)
+-   `start` — The first element of the array. Optional, required if `step` is used. Default value: 0.
+-   `end` — The number before which the array is constructed. Required.
+-   `step` — Determines the incremental step between each element in the array. Optional. Default value: 1.
 
 **Returned value**
 
--   Array of `UInt` numbers from `start` to `end - 1` by `step`.
+-   Array of numbers from `start` to `end - 1` by `step`.
 
 **Implementation details**
 
--   All arguments must be positive values: `start`, `end`, `step` are `UInt` data types, as well as elements of the returned array.
+-   All arguments `start`, `end`, `step` must be below data types: `UInt8`, `UInt16`, `UInt32`, `UInt64`,`Int8`, `Int16`, `Int32`, `Int64`, as well as elements of the returned array, which's type is a super type of all arguments's.
 -   An exception is thrown if query results in arrays with a total length of more than number of elements specified by the [function_range_max_elements_in_block](../../operations/settings/settings.md#settings-function_range_max_elements_in_block) setting.
 
-
 **Examples**
 
 Query:
 ``` sql
-SELECT range(5), range(1, 5), range(1, 5, 2);
+SELECT range(5), range(1, 5), range(1, 5, 2), range(-1, 5, 2);
 ```
 Result:
 ```txt
-┌─range(5)────┬─range(1, 5)─┬─range(1, 5, 2)─�
-│ [0,1,2,3,4] │ [1,2,3,4]   │ [1,3]          │
-└─────────────┴─────────────┴────────────────┘
+┌─range(5)────┬─range(1, 5)─┬─range(1, 5, 2)─┬─range(-1, 5, 2)─�
+│ [0,1,2,3,4] │ [1,2,3,4]   │ [1,3]          │ [-1,1,3]        │
+└─────────────┴─────────────┴────────────────┴─────────────────┘
 ```
 
 ## array(x1, …), operator \[x1, …\]
diff --git a/docs/zh/sql-reference/functions/array-functions.md b/docs/zh/sql-reference/functions/array-functions.md
index eccf5d375ec..afb21da169c 100644
--- a/docs/zh/sql-reference/functions/array-functions.md
+++ b/docs/zh/sql-reference/functions/array-functions.md
@@ -117,7 +117,7 @@ SELECT notEmpty([1,2]);
 
 ## range(end), range(\[start, \] end \[, step\]) {#range}
 
-返回一个以`step`作为增�步长的从`start`到`end - 1`的`UInt`类型数字数组。
+返回一个以`step`作为增�步长的从`start`到`end - 1`的整形数字数组， 支�类型包括[`UInt8`, `UInt16`, `UInt32`, `UInt64`, `Int8`, `Int16`, `Int32`, `Int64`](../data-types/int-uint.md)。
 
 **语法**
 ``` sql
@@ -126,31 +126,30 @@ range([start, ] end [, step])
 
 **�数**
 
--   `start` — 数组的第一个元素。�选项，如果设置了`step`时�样需�`start`，默认值为：0，类型为[UInt](../data-types/int-uint.md)。
--   `end` — 计数到`end`结�，但�包括`end`，必填项，类型为[UInt](../data-types/int-uint.md)。
--   `step` — 确定数组中�个元素之间的增�步长。�选项，默认值为：1，类型为[UInt](../data-types/int-uint.md)。
+-   `start` — 数组的第一个元素。�选项，如果设置了`step`时�样需�`start`，默认值为：0。
+-   `end` — 计数到`end`结�，但�包括`end`，必填项。
+-   `step` — 确定数组中�个元素之间的增�步长。�选项，默认值为：1。
 
 **返回值**
 
--   以`step`作为增�步长的从`start`到`end - 1`的`UInt`类型数字数组。
+-   以`step`作为增�步长的从`start`到`end - 1`的数字数组。
 
 **注�事项**
 
--   所有�数必须是正值：`start`�`end`�`step`，类型�为`UInt`，结果数组的元素与此相�。
+-   所有�数`start`�`end`�`step`必须属于以下几�类型之一：[`UInt8`, `UInt16`, `UInt32`, `UInt64`, `Int8`, `Int16`, `Int32`, `Int64`](../data-types/int-uint.md)。结果数组的元素数�类型为所有入�类型的最�超类，也必须属于以上几�类型之一。
 -   如果查询结果的数组总长度超过[function_range_max_elements_in_block](../../operations/settings/settings.md#settings-function_range_max_elements_in_block)指定的元素数，将会抛出异常。
 
-
 **示例**
 
 查询语�:
 ``` sql
-SELECT range(5), range(1, 5), range(1, 5, 2);
+SELECT range(5), range(1, 5), range(1, 5, 2), range(-1, 5, 2);
 ```
 结果:
 ```txt
-┌─range(5)────┬─range(1, 5)─┬─range(1, 5, 2)─�
-│ [0,1,2,3,4] │ [1,2,3,4]   │ [1,3]          │
-└─────────────┴─────────────┴────────────────┘
+┌─range(5)────┬─range(1, 5)─┬─range(1, 5, 2)─┬─range(-1, 5, 2)─�
+│ [0,1,2,3,4] │ [1,2,3,4]   │ [1,3]          │ [-1,1,3]        │
+└─────────────┴─────────────┴────────────────┴─────────────────┘
 ```
 
 ## array(x1, …), operator \[x1, …\] {#arrayx1-operator-x1}

From 93509395fb70ee149a9f745e46869f1dd96b5186 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Fri, 23 Dec 2022 15:06:30 +0100
Subject: [PATCH 035/158] add a column with a message pattern to text_log

---
 contrib/poco                    |  2 +-
 src/Common/logger_useful.h      | 30 +++++++++++++++++++++++++++++-
 src/Interpreters/TextLog.cpp    |  6 +++++-
 src/Interpreters/TextLog.h      |  2 ++
 src/Loggers/OwnSplitChannel.cpp |  2 ++
 5 files changed, 39 insertions(+), 3 deletions(-)

diff --git a/contrib/poco b/contrib/poco
index 79923422618..0ab9bba7cca 160000
--- a/contrib/poco
+++ b/contrib/poco
@@ -1 +1 @@
-Subproject commit 799234226187c0ae0b8c90f23465b25ed7956e56
+Subproject commit 0ab9bba7ccad3c8dacce04a35cb3b78218547ab4
diff --git a/src/Common/logger_useful.h b/src/Common/logger_useful.h
index 1e84efd8085..4f1209a9dc2 100644
--- a/src/Common/logger_useful.h
+++ b/src/Common/logger_useful.h
@@ -17,8 +17,36 @@ namespace
 
     [[maybe_unused]] const ::Poco::Logger * getLogger(const ::Poco::Logger * logger) { return logger; };
     [[maybe_unused]] const ::Poco::Logger * getLogger(const std::atomic<::Poco::Logger *> & logger) { return logger.load(); };
+
+    template<typename T> struct is_fmt_runtime : std::false_type {};
+    template<typename T> struct is_fmt_runtime<fmt::basic_runtime<T>> : std::true_type {};
+
+    /// Usually we use LOG_*(...) macros with either string literals or fmt::runtime(whatever) as a format string.
+    /// This function is useful to get a string_view to a static format string passed to LOG_* macro.
+    template <typename T> constexpr std::string_view tryGetStaticFormatString(T && x)
+    {
+        if constexpr (is_fmt_runtime<T>::value)
+        {
+            /// It definitely was fmt::runtime(something).
+            /// We are not sure about a lifetime of the string, so return empty view.
+            /// Also it can be arbitrary string, not a formatting pattern.
+            /// So returning empty pattern will not pollute the set of patterns.
+            return std::string_view();
+        }
+        else
+        {
+            /// Most likely it was a string literal.
+            /// Unfortunately, there's no good way to check if something is a string literal.
+            /// But fmtlib requires a format string to be compile-time constant unless fmt::runtime is used.
+            static_assert(std::is_nothrow_convertible<T, const char * const>::value);
+            static_assert(!std::is_pointer<T>::value);
+            return std::string_view(x);
+        }
+    }
 }
 
+#define FIRST_ARG(X, ...) X
+
 /// Logs a message to a specified logger with that level.
 /// If more than one argument is provided,
 ///  the first argument is interpreted as template with {}-substitutions
@@ -40,7 +68,7 @@ namespace
             file_function += "; ";                                                \
             file_function += __PRETTY_FUNCTION__;                                 \
             Poco::Message poco_message(_logger->name(), formatted_message,        \
-                                 (PRIORITY), file_function.c_str(), __LINE__);    \
+                (PRIORITY), file_function.c_str(), __LINE__, tryGetStaticFormatString(FIRST_ARG(__VA_ARGS__)));    \
             _channel->log(poco_message);                                          \
         }                                                                         \
     }                                                                             \
diff --git a/src/Interpreters/TextLog.cpp b/src/Interpreters/TextLog.cpp
index 6d490e3e95f..45d5a7b2344 100644
--- a/src/Interpreters/TextLog.cpp
+++ b/src/Interpreters/TextLog.cpp
@@ -49,7 +49,9 @@ NamesAndTypesList TextLogElement::getNamesAndTypes()
         {"revision", std::make_shared<DataTypeUInt32>()},
 
         {"source_file", std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>())},
-        {"source_line", std::make_shared<DataTypeUInt64>()}
+        {"source_line", std::make_shared<DataTypeUInt64>()},
+
+        {"message_format_string", std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>())},
     };
 }
 
@@ -74,6 +76,8 @@ void TextLogElement::appendToBlock(MutableColumns & columns) const
 
     columns[i++]->insert(source_file);
     columns[i++]->insert(source_line);
+
+    columns[i++]->insert(message_format_string);
 }
 
 TextLog::TextLog(ContextPtr context_, const String & database_name_,
diff --git a/src/Interpreters/TextLog.h b/src/Interpreters/TextLog.h
index 243e001fc52..6efc1c906d4 100644
--- a/src/Interpreters/TextLog.h
+++ b/src/Interpreters/TextLog.h
@@ -28,6 +28,8 @@ struct TextLogElement
     String source_file;
     UInt64 source_line{};
 
+    std::string_view message_format_string;
+
     static std::string name() { return "TextLog"; }
     static NamesAndTypesList getNamesAndTypes();
     static NamesAndAliases getNamesAndAliases() { return {}; }
diff --git a/src/Loggers/OwnSplitChannel.cpp b/src/Loggers/OwnSplitChannel.cpp
index 35a6d4ad86a..7974d5212e1 100644
--- a/src/Loggers/OwnSplitChannel.cpp
+++ b/src/Loggers/OwnSplitChannel.cpp
@@ -133,6 +133,8 @@ void OwnSplitChannel::logSplit(const Poco::Message & msg)
             elem.source_file = msg.getSourceFile();
 
         elem.source_line = msg.getSourceLine();
+        elem.message_format_string = msg.getFormatString();
+
         std::shared_ptr<TextLog> text_log_locked{};
         {
             std::lock_guard<std::mutex> lock(text_log_mutex);

From f2b177880f3f4dcc7cddb38274324c0babecfdb9 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Fri, 23 Dec 2022 15:48:26 +0100
Subject: [PATCH 036/158] print stats in tests

---
 docker/test/stateless/run.sh |  1 +
 docker/test/stress/stress    |  1 +
 tests/clickhouse-test        | 59 ++++++++++++++++++++++++++++++++++++
 3 files changed, 61 insertions(+)

diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh
index 8151ec9d5e4..31f8e77b836 100755
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@@ -128,6 +128,7 @@ function run_tests()
 
     if [[ "${HIGH_LEVEL_COVERAGE}" = "YES" ]]; then
         ADDITIONAL_OPTIONS+=('--report-coverage')
+        ADDITIONAL_OPTIONS+=('--report-logs-stats')
     fi
 
     set +e
diff --git a/docker/test/stress/stress b/docker/test/stress/stress
index a0ec86f7fbe..b8c3e7d53e3 100755
--- a/docker/test/stress/stress
+++ b/docker/test/stress/stress
@@ -292,6 +292,7 @@ if __name__ == "__main__":
                 "--database=system",
                 "--hung-check",
                 "--stress",
+                "--report-logs-stats",
                 "00001_select_1",
             ]
         )
diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index 6e912c7ca10..eb871b0bea2 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -1792,6 +1792,56 @@ def reportCoverage(args):
         """
     )
 
+def reportLogPatterns(args):
+    query = """
+        WITH
+            120 AS mins,
+            (
+                SELECT (count(), sum(length(message)))
+                FROM system.text_log
+                WHERE (now() - toIntervalMinute(mins)) < event_time
+            ) AS total
+        SELECT
+            count() AS count,
+            round(count / (total.1), 3) AS `count_%`,
+            formatReadableSize(sum(length(message))) AS size,
+            round(sum(length(message)) / (total.2), 3) AS `size_%`,
+            countDistinct(logger_name) AS uniq_loggers,
+            countDistinct(thread_id) AS uniq_threads,
+            groupArrayDistinct(toString(level)) AS levels,
+            round(sum(query_id = '') / count, 3) AS `background_%`,
+            message_format_string
+        FROM system.text_log
+        WHERE (now() - toIntervalMinute(mins)) < event_time
+        GROUP BY message_format_string
+        ORDER BY count DESC
+        LIMIT 100
+        FORMAT TSVWithNamesAndTypes
+    """
+    value = clickhouse_execute(args, query).decode()
+    print(f"\nTop patterns of log messages:\n")
+    print(value)
+    print("\n")
+
+    query = """
+        WITH
+            120 AS mins
+        SELECT
+            count() AS count,
+            substr(replaceRegexpAll(message, '[^A-Za-z]+', ''), 1, 32) AS pattern,
+            substr(any(message), 1, 256) as runtime_message
+        FROM system.text_log
+        WHERE (now() - toIntervalMinute(mins)) < event_time AND message_format_string = ''
+        GROUP BY pattern
+        ORDER BY count DESC
+        LIMIT 50
+        FORMAT TSVWithNamesAndTypes
+    """
+    value = clickhouse_execute(args, query).decode()
+    print(f"\nTop runtime messages:\n")
+    print(value)
+    print("\n")
+
 
 def main(args):
     global server_died
@@ -1927,6 +1977,9 @@ def main(args):
     else:
         print("All tests have finished.")
 
+    if args.report_logs_stats:
+        reportLogPatterns(args)
+
     if args.report_coverage and not reportCoverage(args):
         exit_code.value = 1
 
@@ -2216,6 +2269,12 @@ if __name__ == "__main__":
         default=False,
         help="Check what high-level server components were covered by tests",
     )
+    parser.add_argument(
+        "--report-logs-stats",
+        action="store_true",
+        default=False,
+        help="Report statistics about log messages",
+    )
     args = parser.parse_args()
 
     if args.queries and not os.path.isdir(args.queries):

From ca989e92127557dec5b71f2fbf3ced721cb3dde5 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Fri, 23 Dec 2022 19:40:29 +0100
Subject: [PATCH 037/158] less runtime format strings

---
 src/Common/logger_useful.h                    |  33 +++++-
 src/Processors/Executors/PipelineExecutor.cpp |   2 +-
 src/Storages/MergeTree/DataPartsExchange.cpp  |   2 +-
 .../MergeTree/MergeFromLogEntryTask.cpp       |   7 +-
 src/Storages/MergeTree/MergeTreeData.cpp      |   4 +-
 .../ReplicatedMergeTreePartCheckThread.cpp    |  12 +-
 .../MergeTree/ReplicatedMergeTreeQueue.cpp    | 109 ++++++++----------
 src/Storages/StorageMergeTree.cpp             |   8 +-
 src/Storages/StorageReplicatedMergeTree.cpp   |   6 +-
 src/Storages/StorageURL.cpp                   |   8 +-
 10 files changed, 109 insertions(+), 82 deletions(-)

diff --git a/src/Common/logger_useful.h b/src/Common/logger_useful.h
index 4f1209a9dc2..ce2e2e5514a 100644
--- a/src/Common/logger_useful.h
+++ b/src/Common/logger_useful.h
@@ -45,7 +45,7 @@ namespace
     }
 }
 
-#define FIRST_ARG(X, ...) X
+#define LOG_IMPL_FIRST_ARG(X, ...) X
 
 /// Logs a message to a specified logger with that level.
 /// If more than one argument is provided,
@@ -68,7 +68,27 @@ namespace
             file_function += "; ";                                                \
             file_function += __PRETTY_FUNCTION__;                                 \
             Poco::Message poco_message(_logger->name(), formatted_message,        \
-                (PRIORITY), file_function.c_str(), __LINE__, tryGetStaticFormatString(FIRST_ARG(__VA_ARGS__)));    \
+                (PRIORITY), file_function.c_str(), __LINE__, tryGetStaticFormatString(LOG_IMPL_FIRST_ARG(__VA_ARGS__)));    \
+            _channel->log(poco_message);                                          \
+        }                                                                         \
+    }                                                                             \
+} while (false)
+
+#define LOG_IMPL_PREFORMATTED(logger, priority, PRIORITY, FORMAT_STRING, MESSAGE) do                              \
+{                                                                                 \
+    auto _logger = ::getLogger(logger);                                           \
+    const bool _is_clients_log = (DB::CurrentThread::getGroup() != nullptr) &&    \
+        (DB::CurrentThread::getGroup()->client_logs_level >= (priority));         \
+    if (_logger->is((PRIORITY)) || _is_clients_log)                               \
+    {                                                                             \
+        if (auto _channel = _logger->getChannel())                                \
+        {                                                                         \
+            std::string file_function;                                            \
+            file_function += __FILE__;                                            \
+            file_function += "; ";                                                \
+            file_function += __PRETTY_FUNCTION__;                                 \
+            Poco::Message poco_message(_logger->name(), MESSAGE,        \
+                (PRIORITY), file_function.c_str(), __LINE__, tryGetStaticFormatString(FORMAT_STRING));    \
             _channel->log(poco_message);                                          \
         }                                                                         \
     }                                                                             \
@@ -82,3 +102,12 @@ namespace
 #define LOG_WARNING(logger, ...) LOG_IMPL(logger, DB::LogsLevel::warning, Poco::Message::PRIO_WARNING, __VA_ARGS__)
 #define LOG_ERROR(logger, ...)   LOG_IMPL(logger, DB::LogsLevel::error, Poco::Message::PRIO_ERROR, __VA_ARGS__)
 #define LOG_FATAL(logger, ...)   LOG_IMPL(logger, DB::LogsLevel::error, Poco::Message::PRIO_FATAL, __VA_ARGS__)
+
+
+#define LOG_TEST_PREFORMATTED(logger, FORMAT_STRING, MESSAGE)    LOG_IMPL_PREFORMATTED(logger, DB::LogsLevel::test, Poco::Message::PRIO_TEST, FORMAT_STRING, MESSAGE)
+#define LOG_TRACE_PREFORMATTED(logger, FORMAT_STRING, MESSAGE)   LOG_IMPL_PREFORMATTED(logger, DB::LogsLevel::trace, Poco::Message::PRIO_TRACE, FORMAT_STRING, MESSAGE)
+#define LOG_DEBUG_PREFORMATTED(logger, FORMAT_STRING, MESSAGE)   LOG_IMPL_PREFORMATTED(logger, DB::LogsLevel::debug, Poco::Message::PRIO_DEBUG, FORMAT_STRING, MESSAGE)
+#define LOG_INFO_PREFORMATTED(logger, FORMAT_STRING, MESSAGE)    LOG_IMPL_PREFORMATTED(logger, DB::LogsLevel::information, Poco::Message::PRIO_INFORMATION, FORMAT_STRING, MESSAGE)
+#define LOG_WARNING_PREFORMATTED(logger, FORMAT_STRING, MESSAGE) LOG_IMPL_PREFORMATTED(logger, DB::LogsLevel::warning, Poco::Message::PRIO_WARNING, FORMAT_STRING, MESSAGE)
+#define LOG_ERROR_PREFORMATTED(logger, FORMAT_STRING, MESSAGE)   LOG_IMPL_PREFORMATTED(logger, DB::LogsLevel::error, Poco::Message::PRIO_ERROR, FORMAT_STRING, MESSAGE)
+#define LOG_FATAL_PREFORMATTED(logger, FORMAT_STRING, MESSAGE)   LOG_IMPL_PREFORMATTED(logger, DB::LogsLevel::error, Poco::Message::PRIO_FATAL, FORMAT_STRING, MESSAGE)
diff --git a/src/Processors/Executors/PipelineExecutor.cpp b/src/Processors/Executors/PipelineExecutor.cpp
index a9083d8c4a8..ad504e65f94 100644
--- a/src/Processors/Executors/PipelineExecutor.cpp
+++ b/src/Processors/Executors/PipelineExecutor.cpp
@@ -196,7 +196,7 @@ void PipelineExecutor::executeSingleThread(size_t thread_num)
 
 #ifndef NDEBUG
     auto & context = tasks.getThreadContext(thread_num);
-    LOG_TRACE(log,
+    LOG_TEST(log,
               "Thread finished. Total time: {} sec. Execution time: {} sec. Processing time: {} sec. Wait time: {} sec.",
               context.total_time_ns / 1e9,
               context.execution_time_ns / 1e9,
diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp
index 66f91aa6cd2..1f5c688cf53 100644
--- a/src/Storages/MergeTree/DataPartsExchange.cpp
+++ b/src/Storages/MergeTree/DataPartsExchange.cpp
@@ -651,7 +651,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchSelectedPart(
             }
 #endif
 
-            LOG_WARNING(log, fmt::runtime(e.message() + " Will retry fetching part without zero-copy."));
+            LOG_WARNING(log, "Will retry fetching part without zero-copy: {}", e.message());
 
             /// It's important to release session from HTTP pool. Otherwise it's possible to get deadlock
             /// on http pool.
diff --git a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp
index d5627774052..5054b7f727c 100644
--- a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp
+++ b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp
@@ -109,10 +109,11 @@ ReplicatedMergeMutateTaskBase::PrepareResult MergeFromLogEntryTask::prepare()
             /// 2. We have some larger merged part which covers new_part_name (and therefore it covers source_part_name too)
             /// 3. We have two intersecting parts, both cover source_part_name. It's logical error.
             /// TODO Why 1 and 2 can happen? Do we need more assertions here or somewhere else?
-            constexpr const char * message = "Part {} is covered by {} but should be merged into {}. This shouldn't happen often.";
-            LOG_WARNING(log, fmt::runtime(message), source_part_name, source_part_or_covering->name, entry.new_part_name);
+            constexpr auto fmt_string = "Part {} is covered by {} but should be merged into {}. This shouldn't happen often.";
+            String message = fmt::format(fmt_string, source_part_name, source_part_or_covering->name, entry.new_part_name);
+            LOG_WARNING_PREFORMATTED(log, fmt_string, message);
             if (!source_part_or_covering->info.contains(MergeTreePartInfo::fromPartName(entry.new_part_name, storage.format_version)))
-                throw Exception(ErrorCodes::LOGICAL_ERROR, message, source_part_name, source_part_or_covering->name, entry.new_part_name);
+                throw Exception(ErrorCodes::LOGICAL_ERROR, message);
 
             return PrepareResult{
                 .prepared_successfully = false,
diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index e6b792e8be1..e6520348583 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -3523,7 +3523,9 @@ void MergeTreeData::forcefullyMovePartToDetachedAndRemoveFromMemory(const MergeT
 
         if (error)
         {
-            LOG_ERROR(log, "The set of parts restored in place of {} looks incomplete. There might or might not be a data loss.{}", part->name, (error_parts.empty() ? "" : " Suspicious parts: " + error_parts));
+            LOG_WARNING(log, "The set of parts restored in place of {} looks incomplete. "
+                             "SELECT queries may observe gaps in data until this replica is synchronized with other replicas.{}",
+                        part->name, (error_parts.empty() ? "" : " Suspicious parts: " + error_parts));
         }
     }
 
diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp
index 7b29472e389..3b1ede41e25 100644
--- a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp
+++ b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp
@@ -368,9 +368,9 @@ CheckResult ReplicatedMergeTreePartCheckThread::checkPart(const String & part_na
                     throw;
 
                 tryLogCurrentException(log, __PRETTY_FUNCTION__);
-
-                String message = "Part " + part_name + " looks broken. Removing it and will try to fetch.";
-                LOG_ERROR(log, fmt::runtime(message));
+                constexpr auto fmt_string = "Part {} looks broken. Removing it and will try to fetch.";
+                String message = fmt::format(fmt_string, part_name);
+                LOG_ERROR(log, fmt_string, part_name);
 
                 /// Delete part locally.
                 storage.outdateBrokenPartAndCloneToDetached(part, "broken");
@@ -387,9 +387,9 @@ CheckResult ReplicatedMergeTreePartCheckThread::checkPart(const String & part_na
             /// Probably, someone just wrote down the part, and has not yet added to ZK.
             /// Therefore, delete only if the part is old (not very reliable).
             ProfileEvents::increment(ProfileEvents::ReplicatedPartChecksFailed);
-
-            String message = "Unexpected part " + part_name + " in filesystem. Removing.";
-            LOG_ERROR(log, fmt::runtime(message));
+            constexpr auto fmt_string = "Unexpected part {} in filesystem. Removing.";
+            String message = fmt::format(fmt_string, part_name);
+            LOG_ERROR(log, fmt_string, part_name);
             storage.outdateBrokenPartAndCloneToDetached(part, "unexpected");
             return {part_name, false, message};
         }
diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp
index 9ff022d5d57..0dda3d426d2 100644
--- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp
+++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp
@@ -1191,12 +1191,13 @@ bool ReplicatedMergeTreeQueue::isCoveredByFuturePartsImpl(const LogEntry & entry
     if (entry_for_same_part_it != future_parts.end())
     {
         const LogEntry & another_entry = *entry_for_same_part_it->second;
+        constexpr auto fmt_string = "Not executing log entry {} of type {} for part {} "
+                                    "because another log entry {} of type {} for the same part ({}) is being processed.";
         out_reason = fmt::format(
-            "Not executing log entry {} of type {} for part {} "
-            "because another log entry {} of type {} for the same part ({}) is being processed.",
+            fmt_string,
             entry.znode_name, entry.type, entry.new_part_name,
             another_entry.znode_name, another_entry.type, another_entry.new_part_name);
-        LOG_INFO(log, fmt::runtime(out_reason));
+        LOG_INFO_PREFORMATTED(log, fmt_string, out_reason);
         return true;
 
         /** When the corresponding action is completed, then `isNotCoveredByFuturePart` next time, will succeed,
@@ -1238,11 +1239,11 @@ bool ReplicatedMergeTreeQueue::isCoveredByFuturePartsImpl(const LogEntry & entry
         {
             if (entry.znode_name < future_part_elem.second->znode_name)
             {
+                constexpr auto fmt_string = "Not executing log entry {} for part {} "
+                                            "because it is not disjoint with part {} that is currently executing and another entry {} is newer.";
                 out_reason = fmt::format(
-                    "Not executing log entry {} for part {} "
-                    "because it is not disjoint with part {} that is currently executing and another entry {} is newer.",
-                    entry.znode_name, new_part_name, future_part_elem.first, future_part_elem.second->znode_name);
-                LOG_TRACE(log, fmt::runtime(out_reason));
+                    fmt_string, entry.znode_name, new_part_name, future_part_elem.first, future_part_elem.second->znode_name);
+                LOG_TRACE_PREFORMATTED(log, fmt_string, out_reason);
                 return true;
             }
 
@@ -1250,11 +1251,10 @@ bool ReplicatedMergeTreeQueue::isCoveredByFuturePartsImpl(const LogEntry & entry
             continue;
         }
 
-        out_reason = fmt::format(
-            "Not executing log entry {} for part {} "
-            "because it is not disjoint with part {} that is currently executing.",
-            entry.znode_name, new_part_name, future_part_elem.first);
-        LOG_TRACE(log, fmt::runtime(out_reason));
+        constexpr auto fmt_string = "Not executing log entry {} for part {} "
+                                    "because it is not disjoint with part {} that is currently executing.";
+        out_reason = fmt::format(fmt_string, entry.znode_name, new_part_name, future_part_elem.first);
+        LOG_TEST_PREFORMATTED(log, fmt_string, out_reason);
         return true;
     }
 
@@ -1337,11 +1337,10 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry(
         {
             if (future_parts.contains(name))
             {
-                out_postpone_reason = fmt::format(
-                    "Not executing log entry {} of type {} for part {} "
-                    "because part {} is not ready yet (log entry for that part is being processed).",
-                    entry.znode_name, entry.typeToString(), entry.new_part_name, name);
-                LOG_TRACE(log, fmt::runtime(out_postpone_reason));
+                constexpr auto fmt_string = "Not executing log entry {} of type {} for part {} "
+                      "because part {} is not ready yet (log entry for that part is being processed).";
+                out_postpone_reason = fmt::format(fmt_string, entry.znode_name, entry.typeToString(), entry.new_part_name, name);
+                LOG_TRACE_PREFORMATTED(log, fmt_string, out_postpone_reason);
                 return false;
             }
 
@@ -1357,10 +1356,9 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry(
 
         if (merger_mutator.merges_blocker.isCancelled())
         {
-            out_postpone_reason = fmt::format(
-                "Not executing log entry {} of type {} for part {} because merges and mutations are cancelled now.",
-                entry.znode_name, entry.typeToString(), entry.new_part_name);
-            LOG_DEBUG(log, fmt::runtime(out_postpone_reason));
+            constexpr auto fmt_string = "Not executing log entry {} of type {} for part {} because merges and mutations are cancelled now.";
+            out_postpone_reason = fmt::format(fmt_string, entry.znode_name, entry.typeToString(), entry.new_part_name);
+            LOG_DEBUG_PREFORMATTED(log, fmt_string, out_postpone_reason);
             return false;
         }
 
@@ -1375,8 +1373,8 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry(
 
             if (!disks.empty() && only_s3_storage && storage.checkZeroCopyLockExists(entry.new_part_name, disks[0]))
             {
-                out_postpone_reason =  "Not executing merge/mutation for the part " + entry.new_part_name
-                    +  ", waiting other replica to execute it and will fetch after.";
+                constexpr auto fmt_string = "Not executing merge/mutation for the part {}, waiting other replica to execute it and will fetch after.";
+                out_postpone_reason = fmt::format(fmt_string, entry.new_part_name);
                 return false;
             }
         }
@@ -1387,9 +1385,8 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry(
 
             if (replica_to_execute_merge && !merge_strategy_picker.isMergeFinishedByReplica(replica_to_execute_merge.value(), entry))
             {
-                String reason = "Not executing merge for the part " + entry.new_part_name
-                    +  ", waiting for " + replica_to_execute_merge.value() + " to execute merge.";
-                out_postpone_reason = reason;
+                constexpr auto fmt_string = "Not executing merge for the part {}, waiting for {} to execute merge.";
+                out_postpone_reason = fmt::format(fmt_string, entry.new_part_name, replica_to_execute_merge.value());
                 return false;
             }
         }
@@ -1411,20 +1408,18 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry(
             {
                 if (merger_mutator.ttl_merges_blocker.isCancelled())
                 {
-                    out_postpone_reason = fmt::format(
-                        "Not executing log entry {} for part {} because merges with TTL are cancelled now.",
-                        entry.znode_name, entry.new_part_name);
-                    LOG_DEBUG(log, fmt::runtime(out_postpone_reason));
+                    constexpr auto fmt_string = "Not executing log entry {} for part {} because merges with TTL are cancelled now.";
+                    out_postpone_reason = fmt::format(fmt_string, entry.znode_name, entry.new_part_name);
+                    LOG_DEBUG_PREFORMATTED(log, fmt_string, out_postpone_reason);
                     return false;
                 }
                 size_t total_merges_with_ttl = data.getTotalMergesWithTTLInMergeList();
                 if (total_merges_with_ttl >= data_settings->max_number_of_merges_with_ttl_in_pool)
                 {
-                    out_postpone_reason = fmt::format(
-                        "Not executing log entry {} for part {} because {} merges with TTL already executing, maximum {}.",
-                        entry.znode_name, entry.new_part_name, total_merges_with_ttl,
-                        data_settings->max_number_of_merges_with_ttl_in_pool);
-                    LOG_DEBUG(log, fmt::runtime(out_postpone_reason));
+                    constexpr auto fmt_string = "Not executing log entry {} for part {} because {} merges with TTL already executing, maximum {}.";
+                    out_postpone_reason = fmt::format(fmt_string, entry.znode_name, entry.new_part_name, total_merges_with_ttl,
+                                                      data_settings->max_number_of_merges_with_ttl_in_pool);
+                    LOG_DEBUG_PREFORMATTED(log, fmt_string, out_postpone_reason);
                     return false;
                 }
             }
@@ -1432,12 +1427,12 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry(
 
         if (!ignore_max_size && sum_parts_size_in_bytes > max_source_parts_size)
         {
-            out_postpone_reason = fmt::format("Not executing log entry {} of type {} for part {}"
-                " because source parts size ({}) is greater than the current maximum ({}).",
-                entry.znode_name, entry.typeToString(), entry.new_part_name,
+            constexpr auto fmt_string = "Not executing log entry {} of type {} for part {}"
+                                        " because source parts size ({}) is greater than the current maximum ({}).";
+            out_postpone_reason = fmt::format(fmt_string, entry.znode_name, entry.typeToString(), entry.new_part_name,
                 ReadableSize(sum_parts_size_in_bytes), ReadableSize(max_source_parts_size));
 
-            LOG_DEBUG(log, fmt::runtime(out_postpone_reason));
+            LOG_DEBUG_PREFORMATTED(log, fmt_string, out_postpone_reason);
 
             return false;
         }
@@ -1450,10 +1445,9 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry(
         if (!alter_sequence.canExecuteMetaAlter(entry.alter_version, state_lock))
         {
             int head_alter = alter_sequence.getHeadAlterVersion(state_lock);
-            out_postpone_reason = fmt::format(
-                "Cannot execute alter metadata {} with version {} because another alter {} must be executed before",
-                entry.znode_name, entry.alter_version, head_alter);
-            LOG_TRACE(log, fmt::runtime(out_postpone_reason));
+            constexpr auto fmt_string = "Cannot execute alter metadata {} with version {} because another alter {} must be executed before";
+            out_postpone_reason = fmt::format(fmt_string, entry.znode_name, entry.alter_version, head_alter);
+            LOG_TRACE_PREFORMATTED(log, fmt_string, out_postpone_reason);
             return false;
         }
     }
@@ -1466,17 +1460,15 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry(
             int head_alter = alter_sequence.getHeadAlterVersion(state_lock);
             if (head_alter == entry.alter_version)
             {
-                out_postpone_reason = fmt::format(
-                    "Cannot execute alter data {} with version {} because metadata still not altered",
-                    entry.znode_name, entry.alter_version);
-                LOG_TRACE(log, fmt::runtime(out_postpone_reason));
+                constexpr auto fmt_string = "Cannot execute alter data {} with version {} because metadata still not altered";
+                out_postpone_reason = fmt::format(fmt_string, entry.znode_name, entry.alter_version);
+                LOG_TRACE_PREFORMATTED(log, fmt_string, out_postpone_reason);
             }
             else
             {
-                out_postpone_reason = fmt::format(
-                    "Cannot execute alter data {} with version {} because another alter {} must be executed before",
-                    entry.znode_name, entry.alter_version, head_alter);
-                LOG_TRACE(log, fmt::runtime(out_postpone_reason));
+                constexpr auto fmt_string = "Cannot execute alter data {} with version {} because another alter {} must be executed before";
+                out_postpone_reason = fmt::format(fmt_string, entry.znode_name, entry.alter_version, head_alter);
+                LOG_TRACE_PREFORMATTED(log, fmt_string, out_postpone_reason);
             }
 
             return false;
@@ -1498,14 +1490,15 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry(
             {
                 if (drop_range_info.isDisjoint(info))
                     continue;
+                constexpr auto fmt_string = "Not executing log entry {} of type {} for part {} "
+                    "because another DROP_RANGE or REPLACE_RANGE entry with not disjoint range {} is currently executing.";
                 out_postpone_reason = fmt::format(
-                    "Not executing log entry {} of type {} for part {} "
-                    "because another DROP_RANGE or REPLACE_RANGE entry with not disjoint range {} is currently executing.",
+                    fmt_string,
                     entry.znode_name,
                     entry.typeToString(),
                     entry.new_part_name,
                     info.getPartName());
-                LOG_TRACE(log, fmt::runtime(out_postpone_reason));
+                LOG_TRACE_PREFORMATTED(log, fmt_string, out_postpone_reason);
                 return false;
             }
         }
@@ -1531,11 +1524,11 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry(
                     auto new_part_info = MergeTreePartInfo::fromPartName(new_part_name, format_version);
                     if (!new_part_info.isDisjoint(drop_part_info))
                     {
+                        constexpr auto fmt_string = "Not executing log entry {} of type {} for part {} "
+                            "because it probably depends on {} (REPLACE_RANGE).";
                         out_postpone_reason = fmt::format(
-                            "Not executing log entry {} of type {} for part {} "
-                            "because it probably depends on {} (REPLACE_RANGE).",
-                            entry.znode_name, entry.typeToString(), entry.new_part_name, replace_entry->znode_name);
-                        LOG_TRACE(log, fmt::runtime(out_postpone_reason));
+                            fmt_string, entry.znode_name, entry.typeToString(), entry.new_part_name, replace_entry->znode_name);
+                        LOG_TRACE_PREFORMATTED(log, fmt_string, out_postpone_reason);
                         return false;
                     }
                 }
diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp
index f935fe2854e..5d79624e6f9 100644
--- a/src/Storages/StorageMergeTree.cpp
+++ b/src/Storages/StorageMergeTree.cpp
@@ -1299,10 +1299,10 @@ bool StorageMergeTree::optimize(
                     &disable_reason,
                     local_context->getSettingsRef().optimize_skip_merged_partitions))
             {
-                constexpr const char * message = "Cannot OPTIMIZE table: {}";
+                constexpr auto message = "Cannot OPTIMIZE table: {}";
                 if (disable_reason.empty())
                     disable_reason = "unknown reason";
-                LOG_INFO(log, fmt::runtime(message), disable_reason);
+                LOG_INFO(log, message, disable_reason);
 
                 if (local_context->getSettingsRef().optimize_throw_if_noop)
                     throw Exception(ErrorCodes::CANNOT_ASSIGN_OPTIMIZE, message, disable_reason);
@@ -1326,10 +1326,10 @@ bool StorageMergeTree::optimize(
                 &disable_reason,
                 local_context->getSettingsRef().optimize_skip_merged_partitions))
         {
-            constexpr const char * message = "Cannot OPTIMIZE table: {}";
+            constexpr auto message = "Cannot OPTIMIZE table: {}";
             if (disable_reason.empty())
                 disable_reason = "unknown reason";
-            LOG_INFO(log, fmt::runtime(message), disable_reason);
+            LOG_INFO(log, message, disable_reason);
 
             if (local_context->getSettingsRef().optimize_throw_if_noop)
                 throw Exception(ErrorCodes::CANNOT_ASSIGN_OPTIMIZE, message, disable_reason);
diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index 99ceb1d90ae..4e341561fc8 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -1257,12 +1257,12 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks)
     const auto storage_settings_ptr = getSettings();
     bool insane = uncovered_unexpected_parts_rows > total_rows_on_filesystem * storage_settings_ptr->replicated_max_ratio_of_wrong_parts;
 
-    constexpr const char * sanity_report_fmt = "The local set of parts of table {} doesn't look like the set of parts in ZooKeeper: "
+    constexpr auto sanity_report_fmt = "The local set of parts of table {} doesn't look like the set of parts in ZooKeeper: "
                                                "{} rows of {} total rows in filesystem are suspicious. "
                                                "There are {} uncovered unexpected parts with {} rows ({} of them is not just-written with {} rows), "
                                                "{} missing parts (with {} blocks), {} covered unexpected parts (with {} rows).";
 
-    constexpr const char * sanity_report_debug_fmt = "Uncovered unexpected parts: {}. Missing parts: {}. Covered unexpected parts: {}. Expected parts: {}.";
+    constexpr auto sanity_report_debug_fmt = "Uncovered unexpected parts: {}. Missing parts: {}. Covered unexpected parts: {}. Expected parts: {}.";
 
     if (insane && !skip_sanity_checks)
     {
@@ -1278,7 +1278,7 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks)
     {
         LOG_DEBUG(log, sanity_report_debug_fmt, fmt::join(uncovered_unexpected_parts, ", "), fmt::join(parts_to_fetch, ", "),
                   fmt::join(covered_unexpected_parts, ", "), fmt::join(expected_parts, ", "));
-        LOG_WARNING(log, fmt::runtime(sanity_report_fmt), getStorageID().getNameForLogs(),
+        LOG_WARNING(log, sanity_report_fmt, getStorageID().getNameForLogs(),
                     formatReadableQuantity(uncovered_unexpected_parts_rows), formatReadableQuantity(total_rows_on_filesystem),
                     uncovered_unexpected_parts.size(), uncovered_unexpected_parts_rows, unexpected_parts_nonnew, unexpected_parts_nonnew_rows,
                     parts_to_fetch.size(), parts_to_fetch_blocks, covered_unexpected_parts.size(), unexpected_parts_rows - uncovered_unexpected_parts_rows);
diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp
index 0f01dc4288c..efa979e624d 100644
--- a/src/Storages/StorageURL.cpp
+++ b/src/Storages/StorageURL.cpp
@@ -318,9 +318,11 @@ namespace
                             // to check if Range header is supported, we need to send a request with it set
                             const bool supports_ranges = (res.has("Accept-Ranges") && res.get("Accept-Ranges") == "bytes")
                                 || (res.has("Content-Range") && res.get("Content-Range").starts_with("bytes"));
-                            LOG_TRACE(
-                                &Poco::Logger::get("StorageURLSource"),
-                                fmt::runtime(supports_ranges ? "HTTP Range is supported" : "HTTP Range is not supported"));
+
+                            if (supports_ranges)
+                                LOG_TRACE(&Poco::Logger::get("StorageURLSource"), "HTTP Range is supported");
+                            else
+                                LOG_TRACE(&Poco::Logger::get("StorageURLSource"), "HTTP Range is not supported");
 
 
                             if (supports_ranges && res.getStatus() == Poco::Net::HTTPResponse::HTTP_PARTIAL_CONTENT

From 4d25c9b2a316e8dae140a5408e0e38fb890075fa Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Fri, 23 Dec 2022 22:40:19 +0300
Subject: [PATCH 038/158] Update clickhouse-test

---
 tests/clickhouse-test | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index eb871b0bea2..53d17bff190 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -1819,7 +1819,7 @@ def reportLogPatterns(args):
         FORMAT TSVWithNamesAndTypes
     """
     value = clickhouse_execute(args, query).decode()
-    print(f"\nTop patterns of log messages:\n")
+    print("\nTop patterns of log messages:\n")
     print(value)
     print("\n")
 
@@ -1838,7 +1838,7 @@ def reportLogPatterns(args):
         FORMAT TSVWithNamesAndTypes
     """
     value = clickhouse_execute(args, query).decode()
-    print(f"\nTop runtime messages:\n")
+    print("\nTop runtime messages:\n")
     print(value)
     print("\n")
 

From 9619f4a5f5449d0b054246864d7cd7cc4f5f1e24 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Fri, 23 Dec 2022 20:57:19 +0100
Subject: [PATCH 039/158] fix too aggressive evaluation of args in default
 column expr

---
 src/Databases/DDLDependencyVisitor.cpp        |  3 +-
 src/Databases/DDLLoadingDependencyVisitor.cpp | 29 +++++++++++----
 src/Databases/DDLLoadingDependencyVisitor.h   |  3 +-
 .../NormalizeAndEvaluateConstantsVisitor.cpp  |  2 +-
 ...3_default_expression_operator_in.reference |  2 +
 .../02433_default_expression_operator_in.sql  | 37 +++++++++++++++++++
 6 files changed, 66 insertions(+), 10 deletions(-)
 create mode 100644 tests/queries/0_stateless/02433_default_expression_operator_in.reference
 create mode 100644 tests/queries/0_stateless/02433_default_expression_operator_in.sql

diff --git a/src/Databases/DDLDependencyVisitor.cpp b/src/Databases/DDLDependencyVisitor.cpp
index f0137e5bd60..ffe84f6fb77 100644
--- a/src/Databases/DDLDependencyVisitor.cpp
+++ b/src/Databases/DDLDependencyVisitor.cpp
@@ -2,6 +2,7 @@
 #include <Dictionaries/getDictionaryConfigurationFromAST.h>
 #include <Interpreters/Cluster.h>
 #include <Interpreters/Context.h>
+#include <Interpreters/misc.h>
 #include <Interpreters/InDepthNodeVisitor.h>
 #include <Interpreters/evaluateConstantExpression.h>
 #include <Interpreters/getClusterName.h>
@@ -175,7 +176,7 @@ namespace
         /// Finds dependencies of a function.
         void visitFunction(const ASTFunction & function)
         {
-            if (function.name == "joinGet" || function.name == "dictHas" || function.name == "dictIsIn" || function.name.starts_with("dictGet"))
+            if (functionIsJoinGet(function.name) || functionIsDictGet(function.name))
             {
                 /// dictGet('dict_name', attr_names, id_expr)
                 /// dictHas('dict_name', id_expr)
diff --git a/src/Databases/DDLLoadingDependencyVisitor.cpp b/src/Databases/DDLLoadingDependencyVisitor.cpp
index 8536d1c890d..5a9a7995bae 100644
--- a/src/Databases/DDLLoadingDependencyVisitor.cpp
+++ b/src/Databases/DDLLoadingDependencyVisitor.cpp
@@ -1,6 +1,7 @@
 #include <Databases/DDLLoadingDependencyVisitor.h>
 #include <Dictionaries/getDictionaryConfigurationFromAST.h>
 #include <Interpreters/Context.h>
+#include <Interpreters/misc.h>
 #include <Parsers/ASTCreateQuery.h>
 #include <Parsers/ASTFunction.h>
 #include <Parsers/ASTIdentifier.h>
@@ -52,23 +53,37 @@ bool DDLMatcherBase::needChildVisit(const ASTPtr & node, const ASTPtr & child)
     return true;
 }
 
-ssize_t DDLMatcherBase::getPositionOfTableNameArgument(const ASTFunction & function)
+ssize_t DDLMatcherBase::getPositionOfTableNameArgumentToEvaluate(const ASTFunction & function)
 {
-    if (function.name == "joinGet" ||
-        function.name == "dictHas" ||
-        function.name == "dictIsIn" ||
-        function.name.starts_with("dictGet"))
+    if (functionIsJoinGet(function.name) || functionIsDictGet(function.name))
         return 0;
 
-    if (Poco::toLower(function.name) == "in")
+    return -1;
+}
+
+ssize_t DDLMatcherBase::getPositionOfTableNameArgumentToVisit(const ASTFunction & function)
+{
+    ssize_t maybe_res = getPositionOfTableNameArgumentToEvaluate(function);
+    if (0 <= maybe_res)
+        return maybe_res;
+
+    if (functionIsInOrGlobalInOperator(function.name))
+    {
+        if (function.children.size() < 2)
+            return -1;
+
+        if (function.children[1]->as<ASTFunction>())
+            return -1;
+
         return 1;
+    }
 
     return -1;
 }
 
 void DDLLoadingDependencyVisitor::visit(const ASTFunction & function, Data & data)
 {
-    ssize_t table_name_arg_idx = getPositionOfTableNameArgument(function);
+    ssize_t table_name_arg_idx = getPositionOfTableNameArgumentToVisit(function);
     if (table_name_arg_idx < 0)
         return;
     extractTableNameFromArgument(function, data, table_name_arg_idx);
diff --git a/src/Databases/DDLLoadingDependencyVisitor.h b/src/Databases/DDLLoadingDependencyVisitor.h
index f987e885266..f173517f852 100644
--- a/src/Databases/DDLLoadingDependencyVisitor.h
+++ b/src/Databases/DDLLoadingDependencyVisitor.h
@@ -23,7 +23,8 @@ class DDLMatcherBase
 {
 public:
     static bool needChildVisit(const ASTPtr & node, const ASTPtr & child);
-    static ssize_t getPositionOfTableNameArgument(const ASTFunction & function);
+    static ssize_t getPositionOfTableNameArgumentToVisit(const ASTFunction & function);
+    static ssize_t getPositionOfTableNameArgumentToEvaluate(const ASTFunction & function);
 };
 
 /// Visits ASTCreateQuery and extracts the names of all tables which should be loaded before a specified table.
diff --git a/src/Databases/NormalizeAndEvaluateConstantsVisitor.cpp b/src/Databases/NormalizeAndEvaluateConstantsVisitor.cpp
index d9e494e7c9a..c4d1e8bda8c 100644
--- a/src/Databases/NormalizeAndEvaluateConstantsVisitor.cpp
+++ b/src/Databases/NormalizeAndEvaluateConstantsVisitor.cpp
@@ -23,7 +23,7 @@ void NormalizeAndEvaluateConstants::visit(const ASTFunction & function, Data & d
 {
     /// Replace expressions like "dictGet(currentDatabase() || '.dict', 'value', toUInt32(1))"
     /// with "dictGet('db_name.dict', 'value', toUInt32(1))"
-    ssize_t table_name_arg_idx = getPositionOfTableNameArgument(function);
+    ssize_t table_name_arg_idx = getPositionOfTableNameArgumentToEvaluate(function);
     if (table_name_arg_idx < 0)
         return;
 
diff --git a/tests/queries/0_stateless/02433_default_expression_operator_in.reference b/tests/queries/0_stateless/02433_default_expression_operator_in.reference
new file mode 100644
index 00000000000..0039a1e75da
--- /dev/null
+++ b/tests/queries/0_stateless/02433_default_expression_operator_in.reference
@@ -0,0 +1,2 @@
+CREATE TABLE default.dep\n(\n    `id` Int32,\n    `country` LowCardinality(String),\n    `purchase_location` UInt16 MATERIALIZED if(id IN joinGet(\'default.id_join\', \'location\', \'CLICK\'), 123, 456)\n)\nENGINE = ReplicatedMergeTree(\'/test/02433/default/dep\', \'1\')\nORDER BY tuple()\nSETTINGS index_granularity = 8192
+CREATE TABLE default.dep2\n(\n    `id` Int32,\n    `country` LowCardinality(String),\n    `purchase_location` UInt16 MATERIALIZED if(id IN joinGet(\'default.id_join\', \'location\', \'CLICK\'), 123, 456)\n)\nENGINE = ReplicatedMergeTree(\'/test/02433/default/dep\', \'2\')\nORDER BY tuple()\nSETTINGS index_granularity = 8192
diff --git a/tests/queries/0_stateless/02433_default_expression_operator_in.sql b/tests/queries/0_stateless/02433_default_expression_operator_in.sql
new file mode 100644
index 00000000000..e009bc5cd36
--- /dev/null
+++ b/tests/queries/0_stateless/02433_default_expression_operator_in.sql
@@ -0,0 +1,37 @@
+DROP TABLE IF EXISTS dep;
+DROP TABLE IF EXISTS dep2;
+DROP TABLE IF EXISTS id_join;
+
+CREATE TABLE id_join (`country` String, `location` Array(Int32)) ENGINE = Join(ANY, LEFT, country);
+
+INSERT INTO id_join values ('CLICK', [1234]);
+
+CREATE TABLE dep
+(
+    `id` Int32,
+    `country` LowCardinality(String),
+    `purchase_location` UInt16 MATERIALIZED if(id IN joinGet(concat(currentDatabase(), '.id_join'), 'location', 'CLICK'), 123, 456)
+)
+ENGINE = ReplicatedMergeTree('/test/02433/{database}/dep', '1') ORDER BY tuple();
+
+SHOW CREATE TABLE dep;
+
+TRUNCATE TABLE id_join;
+
+CREATE TABLE dep2
+(
+    `id` Int32,
+    `country` LowCardinality(String),
+    `purchase_location` UInt16 MATERIALIZED if(id IN joinGet(concat(currentDatabase(), '.id_join'), 'location', 'CLICK'), 123, 456)
+)
+ENGINE = ReplicatedMergeTree('/test/02433/{database}/dep', '2') ORDER BY tuple();
+
+SHOW CREATE TABLE dep2;
+
+-- Ensure that a table name cannot be passed to IN as string literal
+create table test (n int, m default n in 'default.table_name') engine=Memory; -- { serverError TYPE_MISMATCH }
+create table test (n int, m default in(n, 'default.table_name')) engine=Memory; -- { serverError TYPE_MISMATCH }
+
+DROP TABLE dep;
+DROP TABLE dep2;
+DROP TABLE id_join;

From cda37e73bac1a3c36d2d5b2972ed66262f1be459 Mon Sep 17 00:00:00 2001
From: Bharat Nallan Chakravarthy <bharatnc@gmail.com>
Date: Mon, 26 Dec 2022 09:05:21 -0800
Subject: [PATCH 040/158] fix test failure

---
 src/AggregateFunctions/QuantileInterpolatedWeighted.h | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/src/AggregateFunctions/QuantileInterpolatedWeighted.h b/src/AggregateFunctions/QuantileInterpolatedWeighted.h
index cd6ae0473e3..6e61cafb21a 100644
--- a/src/AggregateFunctions/QuantileInterpolatedWeighted.h
+++ b/src/AggregateFunctions/QuantileInterpolatedWeighted.h
@@ -286,18 +286,16 @@ private:
         }
     }
 
-    /// This ignores overflows that may arise during add, sub and mul operations and doesn't aim to provide exact results.
-    /// Since `the quantileInterpolatedWeighted` function itself relies mainly on approximation, this is fine.
-    UnderlyingType interpolate(Float64 level, Float64 xl, Float64 xr, UnderlyingType yl, UnderlyingType yr) const
+    /// This ignores overflows or NaN's that might arise during add, sub and mul operations and doesn't aim to provide exact
+    /// results since `the quantileInterpolatedWeighted` function itself relies mainly on approximation.
+    UnderlyingType NO_SANITIZE_UNDEFINED interpolate(Float64 level, Float64 xl, Float64 xr, UnderlyingType yl, UnderlyingType yr) const
     {
         UnderlyingType dy = common::subIgnoreOverflow(yr, yl);
         Float64 dx = common::subIgnoreOverflow(xr, xl);
         dx = dx == 0 ? 1 : dx; /// to handle NaN behavior that might arise during integer division below.
 
         /// yl + (dy / dx) * (level - xl)
-        UnderlyingType g = static_cast<UnderlyingType>(common::addIgnoreOverflow(yl, common::mulIgnoreOverflow((dy / dx), common::subIgnoreOverflow(level, xl))));
-
-        return g;
+        return static_cast<UnderlyingType>(yl + (dy/dx) * (level - xl));
     }
 };
 

From 2adf0e9db3149d6c790fc22c0ecdd33a4142a246 Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Wed, 28 Dec 2022 16:07:14 +0000
Subject: [PATCH 041/158] do not wait loading parts at shutdown

---
 src/Storages/MergeTree/MergeTreeData.cpp    | 69 +++++++++++----------
 src/Storages/MergeTree/MergeTreeData.h      | 24 +++++--
 src/Storages/StorageMergeTree.cpp           |  5 --
 src/Storages/StorageReplicatedMergeTree.cpp | 22 ++++---
 4 files changed, 69 insertions(+), 51 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 8f16abdc7a9..c537c89b16e 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -1034,9 +1034,7 @@ MergeTreeData::PartLoadingTree::build(PartLoadingInfos nodes)
 {
     std::sort(nodes.begin(), nodes.end(), [](const auto & lhs, const auto & rhs)
     {
-        const auto & lhs_info = std::get<0>(lhs);
-        const auto & rhs_info = std::get<0>(rhs);
-        return std::tie(lhs_info.level, lhs_info.mutation) > std::tie(rhs_info.level, rhs_info.mutation);
+        return std::tie(lhs.info.level, lhs.info.mutation) > std::tie(rhs.info.level, rhs.info.mutation);
     });
 
     PartLoadingTree tree;
@@ -1367,7 +1365,7 @@ std::vector<MergeTreeData::LoadPartResult> MergeTreeData::loadDataPartsFromDisk(
                     auto res = loadDataPart(thread_part->info, thread_part->name, thread_part->disk, DataPartState::Active, part_loading_mutex);
                     thread_part->is_loaded = true;
 
-                    bool is_active_part = res.part->getState() == DataPartState::Active && !res.part->is_duplicate;
+                    bool is_active_part = res.part->getState() == DataPartState::Active;
                     /// If part is broken or duplicate or should be removed according to transaction
                     /// and it has any covered parts then try to load them to replace this part.
                     if (!is_active_part && !thread_part->children.empty())
@@ -1564,6 +1562,8 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
     bool have_lightweight_in_parts = false;
     bool have_parts_with_version_metadata = false;
 
+    bool is_static_storage = isStaticStorage();
+
     if (num_parts > 0)
     {
         auto loaded_parts = loadDataPartsFromDisk(pool, num_parts, parts_queue, settings);
@@ -1593,8 +1593,6 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
         }
     }
 
-    bool is_static_storage = isStaticStorage();
-
     if (settings->in_memory_parts_enable_wal)
     {
         pool.setMaxThreads(disks.size());
@@ -1652,7 +1650,6 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
     {
         resetObjectColumnsFromActiveParts(part_lock);
         LOG_DEBUG(log, "There are no data parts");
-        outdated_data_parts_loading_finished = true;
         return;
     }
 
@@ -1697,38 +1694,50 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
     if (!unloaded_parts.empty())
     {
         LOG_DEBUG(log, "Found {} outdated data parts. They will be loaded asynchronously", unloaded_parts.size());
+        outdated_unloaded_data_parts = std::move(unloaded_parts);
 
         outdated_data_parts_loading_task = getContext()->getSchedulePool().createTask(
             "MergeTreeData::loadOutdatedDataParts",
-            [this, unloaded_parts]() mutable
-            {
-                loadOutdatedDataParts(std::move(unloaded_parts));
-            });
+            [this] { loadOutdatedDataParts(/*is_async=*/ true); });
     }
-    else
-        outdated_data_parts_loading_finished = true;
 
     LOG_DEBUG(log, "Loaded data parts ({} items)", data_parts_indexes.size());
     data_parts_loading_finished = true;
 }
 
-void MergeTreeData::loadOutdatedDataParts(PartLoadingTreeNodes parts_to_load)
+void MergeTreeData::loadOutdatedDataParts(bool is_async)
 try
 {
-    if (parts_to_load.empty())
-        return;
-
-    LOG_DEBUG(log, "Loading {} outdated data parts", parts_to_load.size());
-
-    for (const auto & part : parts_to_load)
     {
-        if (outdated_data_parts_loading_canceled)
+        std::lock_guard lock(outdated_data_parts_mutex);
+        LOG_DEBUG(log, "Loading {} outdated data parts", outdated_unloaded_data_parts.size());
+    }
+
+    size_t num_loaded_parts = 0;
+    while (true)
+    {
+        PartLoadingTree::NodePtr part;
+
         {
-            LOG_DEBUG(log, "Stopped loading outdated parts because task was canceled");
-            return;
+            std::lock_guard lock(outdated_data_parts_mutex);
+
+            if (is_async && outdated_data_parts_loading_canceled)
+            {
+                LOG_DEBUG(log,
+                    "Stopped loading outdated parts because task was canceled. "
+                    "Loaded {} parts, {} left unloaded", num_loaded_parts, outdated_unloaded_data_parts.size());
+                return;
+            }
+
+            if (outdated_unloaded_data_parts.empty())
+                break;
+
+            part = std::move(outdated_unloaded_data_parts.back());
+            outdated_unloaded_data_parts.pop_back();
         }
 
         auto res = loadDataPart(part->info, part->name, part->disk, MergeTreeDataPartState::Outdated, data_parts_mutex);
+        ++num_loaded_parts;
 
         if (res.is_broken)
             res.part->renameToDetached("broken-on-start"); /// detached parts must not have '_' in prefixes
@@ -1738,13 +1747,7 @@ try
             preparePartForRemoval(res.part);
     }
 
-    LOG_DEBUG(log, "Loaded {} outdated data parts", parts_to_load.size());
-
-    {
-        std::lock_guard lock(outdated_data_parts_mutex);
-        outdated_data_parts_loading_finished = true;
-    }
-
+    LOG_DEBUG(log, "Loaded {} outdated data parts", num_loaded_parts);
     outdated_data_parts_cv.notify_all();
 }
 catch (...)
@@ -1761,13 +1764,13 @@ void MergeTreeData::waitForOutdatedPartsToBeLoaded() const
     if (isStaticStorage())
         return;
 
-    if (outdated_data_parts_loading_finished)
+    std::unique_lock lock(outdated_data_parts_mutex);
+    if (outdated_unloaded_data_parts.empty())
         return;
 
     LOG_TRACE(log, "Will wait for outdated data parts to be loaded");
 
-    std::unique_lock lock(outdated_data_parts_mutex);
-    outdated_data_parts_cv.wait(lock, [this] { return outdated_data_parts_loading_finished || outdated_data_parts_loading_canceled; });
+    outdated_data_parts_cv.wait(lock, [this] { return outdated_unloaded_data_parts.empty() || outdated_data_parts_loading_canceled; });
     if (outdated_data_parts_loading_canceled)
         throw Exception(ErrorCodes::NOT_INITIALIZED, "Loading of outdated data parts was canceled");
 
diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h
index f9a46e98b9c..f3e8cc66c58 100644
--- a/src/Storages/MergeTree/MergeTreeData.h
+++ b/src/Storages/MergeTree/MergeTreeData.h
@@ -1359,8 +1359,21 @@ protected:
             std::map<MergeTreePartInfo, std::shared_ptr<Node>> children;
         };
 
+        struct PartLoadingInfo
+        {
+            PartLoadingInfo(const MergeTreePartInfo & info_, const String & name_, const DiskPtr & disk_)
+                : info(info_), name(name_), disk(disk_)
+            {
+            }
+
+            /// Store name explicitly because it cannot be easily
+            /// retrived from info in tables with old syntax.
+            MergeTreePartInfo info;
+            String name;
+            DiskPtr disk;
+        };
+
         using NodePtr = std::shared_ptr<Node>;
-        using PartLoadingInfo = std::tuple<MergeTreePartInfo, String, DiskPtr>;
         using PartLoadingInfos = std::vector<PartLoadingInfo>;
 
         /// Builds a tree from the list of part infos.
@@ -1387,15 +1400,14 @@ protected:
         MutableDataPartPtr part;
     };
 
-    BackgroundSchedulePool::TaskHolder outdated_data_parts_loading_task;
-
     mutable std::mutex outdated_data_parts_mutex;
     mutable std::condition_variable outdated_data_parts_cv;
 
-    std::atomic_bool outdated_data_parts_loading_finished = false;
-    std::atomic_bool outdated_data_parts_loading_canceled = false;
+    BackgroundSchedulePool::TaskHolder outdated_data_parts_loading_task;
+    PartLoadingTreeNodes outdated_unloaded_data_parts;
+    bool outdated_data_parts_loading_canceled = false;
 
-    void loadOutdatedDataParts(PartLoadingTreeNodes parts_to_load);
+    void loadOutdatedDataParts(bool is_async);
     void startOutdatedDataPartsLoadingTask();
     void stopOutdatedDataPartsLoadingTask();
 
diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp
index 735c9e8a828..0b907e25da1 100644
--- a/src/Storages/StorageMergeTree.cpp
+++ b/src/Storages/StorageMergeTree.cpp
@@ -177,11 +177,6 @@ void StorageMergeTree::shutdown()
         mutation_wait_event.notify_all();
     }
 
-    /// We need to wait for loading of oudated data parts
-    /// for correct execution of 'clearOldPartsFromFilesystem'
-    waitForOutdatedPartsToBeLoaded();
-    stopOutdatedDataPartsLoadingTask();
-
     merger_mutator.merges_blocker.cancelForever();
     parts_mover.moves_blocker.cancelForever();
 
diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index 39d661dc39c..8ab2eeb0a0c 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -902,6 +902,21 @@ void StorageReplicatedMergeTree::drop()
         dropReplica(zookeeper, zookeeper_path, replica_name, log, getSettings());
     }
 
+    /// Wait for loading of all outdated parts because
+    /// in case of zero copy recursive removal of directory
+    /// is not supported and table cannot be dropped.
+    if (canUseZeroCopyReplication())
+    {
+        /// Load remaining parts synchronously because task
+        /// for loading is already cancelled in shutdown().
+        std::lock_guard lock(outdated_data_parts_mutex);
+        if (!outdated_unloaded_data_parts.empty())
+        {
+            assert(outdated_data_parts_loading_canceled);
+            loadOutdatedDataParts(/*is_async=*/ false);
+        }
+    }
+
     dropAllData();
 }
 
@@ -4318,13 +4333,6 @@ void StorageReplicatedMergeTree::shutdown()
         return;
 
     session_expired_callback_handler.reset();
-
-    /// Wait for loading of all outdated parts because
-    /// in case of zero copy recursive removal of directory
-    /// is not supported and table cannot be dropped.
-    if (canUseZeroCopyReplication())
-        waitForOutdatedPartsToBeLoaded();
-
     stopOutdatedDataPartsLoadingTask();
 
     /// Cancel fetches, merges and mutations to force the queue_task to finish ASAP.

From 66111ef241f88a17290a160aeb89480816197f5d Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Wed, 28 Dec 2022 17:25:36 +0100
Subject: [PATCH 042/158] deduplicate async inserts in the same block earlier

---
 .../MergeTree/ReplicatedMergeTreeSink.cpp     | 207 ++++++++++++------
 .../MergeTree/tests/gtest_async_inserts.cpp   |  18 ++
 2 files changed, 156 insertions(+), 69 deletions(-)

diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
index 3da71c61482..1b31743c0e9 100644
--- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
+++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
@@ -2,9 +2,16 @@
 #include <Storages/MergeTree/ReplicatedMergeTreeQuorumEntry.h>
 #include <Storages/MergeTree/ReplicatedMergeTreeSink.h>
 #include <Interpreters/PartLog.h>
+#include "Common/Exception.h"
 #include <Common/SipHash.h>
 #include <Common/ZooKeeper/KeeperException.h>
 #include <Common/ThreadFuzzer.h>
+#include "Columns/IColumn.h"
+#include "Core/ColumnWithTypeAndName.h"
+#include "Core/ColumnsWithTypeAndName.h"
+#include "DataTypes/Serializations/ISerialization.h"
+#include "Processors/Chunk.h"
+#include "Storages/MergeTree/MergeTreeDataWriter.h"
 #include <DataTypes/ObjectUtils.h>
 #include <Core/Block.h>
 #include <IO/Operators.h>
@@ -41,15 +48,17 @@ struct ReplicatedMergeTreeSinkImpl<async_insert>::DelayedChunk
 {
     struct Partition
     {
+        Poco::Logger * log;
         MergeTreeDataWriter::TemporaryPart temp_part;
         UInt64 elapsed_ns;
         BlockIDsType block_id;
         BlockWithPartition block_with_partition;
-        std::unordered_map<String, size_t> block_id_to_offset_idx;
+        std::unordered_map<String, std::vector<size_t>> block_id_to_offset_idx;
 
         Partition() = default;
-        Partition(MergeTreeDataWriter::TemporaryPart && temp_part_, UInt64 elapsed_ns_, BlockIDsType && block_id_, BlockWithPartition && block_)
-            : temp_part(std::move(temp_part_)),
+        Partition(Poco::Logger * log_, MergeTreeDataWriter::TemporaryPart && temp_part_, UInt64 elapsed_ns_, BlockIDsType && block_id_, BlockWithPartition && block_)
+            : log(log_),
+              temp_part(std::move(temp_part_)),
               elapsed_ns(elapsed_ns_),
               block_id(std::move(block_id_)),
               block_with_partition(std::move(block_))
@@ -64,11 +73,101 @@ struct ReplicatedMergeTreeSinkImpl<async_insert>::DelayedChunk
                 block_id_to_offset_idx.clear();
                 for (size_t i = 0; i < block_id.size(); ++i)
                 {
-                    block_id_to_offset_idx[block_id[i]] = i;
+                    block_id_to_offset_idx[block_id[i]].push_back(i);
                 }
             }
         }
 
+        bool checkSelfDeduplicate()
+        {
+            if constexpr (async_insert)
+            {
+                std::vector<String> dup_block_ids;
+                for (const auto & [hash_id, offset_idxes] : block_id_to_offset_idx)
+                {
+                    if (offset_idxes.size() > 1)
+                        dup_block_ids.push_back(hash_id);
+                }
+                if (dup_block_ids.empty())
+                    return false;
+
+                rewriteBlock(dup_block_ids, true);
+                return true;
+            }
+            return false;
+        }
+
+        /// remove the conflict parts of block for rewriting again.
+        void rewriteBlock(const std::vector<String> & block_paths, bool self_dedup)
+        {
+            if constexpr (async_insert)
+            {
+                std::vector<size_t> offset_idx;
+                for (const auto & raw_path : block_paths)
+                {
+                    std::filesystem::path p(raw_path);
+                    String conflict_block_id = p.filename();
+                    auto it = block_id_to_offset_idx.find(conflict_block_id);
+                    if (it == block_id_to_offset_idx.end())
+                        throw Exception("Unknown conflict path " + conflict_block_id, ErrorCodes::LOGICAL_ERROR);
+                    /// in case of self dedup, we should remain one insert
+                    offset_idx.insert(std::end(offset_idx), std::begin(it->second) + self_dedup, std::end(it->second));
+                }
+                std::sort(offset_idx.begin(), offset_idx.end());
+
+                auto & offsets = block_with_partition.offsets->offsets;
+                size_t idx = 0, remove_count = 0;
+                auto it = offset_idx.begin();
+                std::vector<size_t> new_offsets;
+                std::vector<String> new_block_ids;
+
+                /// construct filter
+                size_t rows = block_with_partition.block.rows();
+                auto filter_col = ColumnUInt8::create(rows, 1u);
+                ColumnUInt8::Container & vec = filter_col->getData();
+                UInt8 * pos = vec.data();
+                for (auto & offset : offsets)
+                {
+                    if (it != offset_idx.end() && *it == idx)
+                    {
+                        size_t start_pos = idx > 0 ? offsets[idx - 1] : 0;
+                        size_t end_pos = offset;
+                        remove_count += end_pos - start_pos;
+                        while (start_pos < end_pos)
+                        {
+                            *(pos + start_pos) = 0;
+                            start_pos ++;
+                        }
+                        it++;
+                    }
+                    else
+                    {
+                        new_offsets.push_back(offset - remove_count);
+                        new_block_ids.push_back(block_id[idx]);
+                    }
+                    idx++;
+                }
+
+                LOG_TRACE(log, "New block IDs: {}, new offsets: {}, size: {}", toString(new_block_ids), toString(new_offsets), new_offsets.size());
+
+                offsets = std::move(new_offsets);
+                block_id = std::move(new_block_ids);
+                auto cols = block_with_partition.block.getColumns();
+                for (auto & col : cols)
+                {
+                    col = col->filter(vec, rows - remove_count);
+                }
+                block_with_partition.block.setColumns(cols);
+
+                LOG_TRACE(log, "New block rows {}", block_with_partition.block.rows());
+
+                initBlockIDMap();
+            }
+            else
+            {
+                throw Exception(ErrorCodes::LOGICAL_ERROR, "sync insert should not call rewriteBlock");
+            }
+        }
     };
 
     DelayedChunk() = default;
@@ -79,6 +178,31 @@ struct ReplicatedMergeTreeSinkImpl<async_insert>::DelayedChunk
     std::vector<Partition> partitions;
 };
 
+std::vector<Int64> testSelfDeduplicate(std::vector<Int64> data, std::vector<size_t> offsets, std::vector<String> hashes)
+{
+    MutableColumnPtr column = DataTypeInt64().createColumn();
+    auto offset_ptr = std::make_shared<ChunkOffsets>(offsets);
+    for (auto datum : data)
+    {
+        column->insert(datum);
+    }
+    Block block({ColumnWithTypeAndName(std::move(column), DataTypePtr(new DataTypeInt64()), "a")});
+
+    BlockWithPartition block1(std::move(block), Row(), offset_ptr);
+    ReplicatedMergeTreeSinkImpl<true>::DelayedChunk::Partition part(
+        &Poco::Logger::get("testSelfDeduplicate"), MergeTreeDataWriter::TemporaryPart(), 0, std::move(hashes), std::move(block1));
+
+    part.checkSelfDeduplicate();
+
+    ColumnPtr col = part.block_with_partition.block.getColumns()[0];
+    std::vector<Int64> result;
+    for (size_t i = 0; i < col->size(); i++)
+    {
+        result.push_back(col->getInt(i));
+    }
+    return result;
+}
+
 namespace
 {
     /// Convert block id vector to string. Output at most 50 ids.
@@ -90,70 +214,6 @@ namespace
         return fmt::format("({})", fmt::join(vec.begin(), vec.begin() + size, ","));
     }
 
-    /// remove the conflict parts of block for rewriting again.
-    void rewriteBlock(Poco::Logger * log, typename ReplicatedMergeTreeSinkImpl<true>::DelayedChunk::Partition & partition, const std::vector<String> & block_paths)
-    {
-        std::vector<size_t> offset_idx;
-        for (const auto & raw_path : block_paths)
-        {
-            std::filesystem::path p(raw_path);
-            String conflict_block_id = p.filename();
-            auto it = partition.block_id_to_offset_idx.find(conflict_block_id);
-            if (it == partition.block_id_to_offset_idx.end())
-                throw Exception("Unknown conflict path " + conflict_block_id, ErrorCodes::LOGICAL_ERROR);
-            offset_idx.push_back(it->second);
-        }
-        std::sort(offset_idx.begin(), offset_idx.end());
-
-        auto & offsets = partition.block_with_partition.offsets->offsets;
-        size_t idx = 0, remove_count = 0;
-        auto it = offset_idx.begin();
-        std::vector<size_t> new_offsets;
-        std::vector<String> new_block_ids;
-
-        /// construct filter
-        size_t rows = partition.block_with_partition.block.rows();
-        auto filter_col = ColumnUInt8::create(rows, 1u);
-        ColumnUInt8::Container & vec = filter_col->getData();
-        UInt8 * pos = vec.data();
-        for (auto & offset : offsets)
-        {
-            if (it != offset_idx.end() && *it == idx)
-            {
-                size_t start_pos = idx > 0 ? offsets[idx - 1] : 0;
-                size_t end_pos = offset;
-                remove_count += end_pos - start_pos;
-                while (start_pos < end_pos)
-                {
-                    *(pos + start_pos) = 0;
-                    start_pos ++;
-                }
-                it++;
-            }
-            else
-            {
-                new_offsets.push_back(offset - remove_count);
-                new_block_ids.push_back(partition.block_id[idx]);
-            }
-            idx++;
-        }
-
-        LOG_TRACE(log, "New block IDs: {}, new offsets: {}, size: {}", toString(new_block_ids), toString(new_offsets), new_offsets.size());
-
-        offsets = std::move(new_offsets);
-        partition.block_id = std::move(new_block_ids);
-        auto cols = partition.block_with_partition.block.getColumns();
-        for (auto & col : cols)
-        {
-            col = col->filter(vec, rows - remove_count);
-        }
-        partition.block_with_partition.block.setColumns(cols);
-
-        LOG_TRACE(log, "New block rows {}", partition.block_with_partition.block.rows());
-
-        partition.initBlockIDMap();
-    }
-
     std::vector<String> getHashesForBlocks(BlockWithPartition & block, String partition_id)
     {
         size_t start = 0;
@@ -416,6 +476,7 @@ void ReplicatedMergeTreeSinkImpl<async_insert>::consume(Chunk chunk)
         }
 
         partitions.emplace_back(DelayedPartition(
+            log,
             std::move(temp_part),
             elapsed_ns,
             std::move(block_id),
@@ -479,6 +540,14 @@ void ReplicatedMergeTreeSinkImpl<true>::finishDelayedChunk(const ZooKeeperWithFa
     for (auto & partition: delayed_chunk->partitions)
     {
         int retry_times = 0;
+        /// users may have lots of same inserts. It will be helpful to deduplicate in advance.
+        if (partition.checkSelfDeduplicate())
+        {
+            LOG_TRACE(log, "found duplicated inserts in the block");
+            partition.block_with_partition.partition = std::move(partition.temp_part.part->partition.value);
+            partition.temp_part = storage.writer.writeTempPart(partition.block_with_partition, metadata_snapshot, context);
+        }
+
         while (true)
         {
             partition.temp_part.finalize();
@@ -488,7 +557,7 @@ void ReplicatedMergeTreeSinkImpl<true>::finishDelayedChunk(const ZooKeeperWithFa
             ++retry_times;
             LOG_DEBUG(log, "Found duplicate block IDs: {}, retry times {}", toString(conflict_block_ids), retry_times);
             /// partition clean conflict
-            rewriteBlock(log, partition, conflict_block_ids);
+            partition.rewriteBlock(conflict_block_ids, false);
             if (partition.block_id.empty())
                 break;
             partition.block_with_partition.partition = std::move(partition.temp_part.part->partition.value);
diff --git a/src/Storages/MergeTree/tests/gtest_async_inserts.cpp b/src/Storages/MergeTree/tests/gtest_async_inserts.cpp
index 348da5f2ccd..f67c2f7fb0f 100644
--- a/src/Storages/MergeTree/tests/gtest_async_inserts.cpp
+++ b/src/Storages/MergeTree/tests/gtest_async_inserts.cpp
@@ -1,3 +1,4 @@
+#include "Storages/MergeTree/ReplicatedMergeTreeSink.h"
 #include "config.h"
 
 #include <gtest/gtest.h>
@@ -42,4 +43,21 @@ TEST(AsyncInsertsTest, testScatterOffsetsBySelector)
     test_impl({3,6,10}, {1,1,1,2,2,2,0,0,0,0}, 3, {{4},{3},{3}});
 }
 
+std::vector<Int64> testSelfDeduplicate(std::vector<Int64> data, std::vector<size_t> offsets, std::vector<String> hashes);
+
+TEST(AsyncInsertsTest, testSelfDeduplicate)
+{
+    auto test_impl = [](std::vector<Int64> data, std::vector<size_t> offsets, std::vector<String> hashes, std::vector<Int64> answer)
+    {
+        auto result = testSelfDeduplicate(data, offsets, hashes);
+        ASSERT_EQ(answer.size(), result.size());
+        for (size_t i = 0; i < result.size(); i++)
+            ASSERT_EQ(answer[i], result[i]);
+    };
+    test_impl({1,2,3,1,2,3,4,5,6,1,2,3},{3,6,9,12},{"a","a","b","a"},{1,2,3,4,5,6});
+    test_impl({1,2,3,1,2,3,1,2,3,1,2,3},{2,3,5,6,8,9,11,12},{"a","b","a","b","a","b","a","b"},{1,2,3});
+    test_impl({1,2,3,1,2,4,1,2,5,1,2},{2,3,5,6,8,9,11},{"a","b","a","c","a","d","a"},{1,2,3,4,5});
+    test_impl({1,2,1,2,1,2,1,2,1,2},{2,4,6,8,10},{"a","a","a","a","a"},{1,2});
+}
+
 }

From 17358f5e98934d253d32f9e352c65c6bef513dfc Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Wed, 28 Dec 2022 17:29:07 +0100
Subject: [PATCH 043/158] clean headers

---
 src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
index 1b31743c0e9..4aca646fd83 100644
--- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
+++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
@@ -2,16 +2,9 @@
 #include <Storages/MergeTree/ReplicatedMergeTreeQuorumEntry.h>
 #include <Storages/MergeTree/ReplicatedMergeTreeSink.h>
 #include <Interpreters/PartLog.h>
-#include "Common/Exception.h"
 #include <Common/SipHash.h>
 #include <Common/ZooKeeper/KeeperException.h>
 #include <Common/ThreadFuzzer.h>
-#include "Columns/IColumn.h"
-#include "Core/ColumnWithTypeAndName.h"
-#include "Core/ColumnsWithTypeAndName.h"
-#include "DataTypes/Serializations/ISerialization.h"
-#include "Processors/Chunk.h"
-#include "Storages/MergeTree/MergeTreeDataWriter.h"
 #include <DataTypes/ObjectUtils.h>
 #include <Core/Block.h>
 #include <IO/Operators.h>

From 8d2788445befc456c39cc13aac1c4250be5a014d Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Wed, 28 Dec 2022 19:29:56 +0000
Subject: [PATCH 044/158] fix style check

---
 src/Storages/MergeTree/MergeTreeData.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h
index f3e8cc66c58..ddf1e523eca 100644
--- a/src/Storages/MergeTree/MergeTreeData.h
+++ b/src/Storages/MergeTree/MergeTreeData.h
@@ -1367,7 +1367,7 @@ protected:
             }
 
             /// Store name explicitly because it cannot be easily
-            /// retrived from info in tables with old syntax.
+            /// retrieved from info in tables with old syntax.
             MergeTreePartInfo info;
             String name;
             DiskPtr disk;

From 6ae0ffe8d2da413c4ab7e71de3690c14bbe87d17 Mon Sep 17 00:00:00 2001
From: Dmitry Novik <n0vik@clickhouse.com>
Date: Fri, 4 Nov 2022 17:12:14 +0000
Subject: [PATCH 045/158] Implement optimize_redundant_functions_in_order_by

---
 ...ptimizeRedundantFunctionsInOrderByPass.cpp | 121 ++++++++++++++++++
 .../OptimizeRedundantFunctionsInOrderByPass.h |  23 ++++
 src/Analyzer/QueryTreePassManager.cpp         |   5 +-
 3 files changed, 148 insertions(+), 1 deletion(-)
 create mode 100644 src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp
 create mode 100644 src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.h

diff --git a/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp b/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp
new file mode 100644
index 00000000000..1359bd2616d
--- /dev/null
+++ b/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp
@@ -0,0 +1,121 @@
+#include <Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.h>
+#include <Analyzer/ColumnNode.h>
+#include <Analyzer/FunctionNode.h>
+#include <Analyzer/InDepthQueryTreeVisitor.h>
+#include <Analyzer/QueryNode.h>
+#include <Analyzer/SortNode.h>
+#include <Functions/IFunction.h>
+
+namespace DB
+{
+
+namespace
+{
+
+class OptimizeRedundantFunctionsInOrderByVisitor : public InDepthQueryTreeVisitor<OptimizeRedundantFunctionsInOrderByVisitor>
+{
+
+    struct RedundancyVerdict
+    {
+        bool redundant = true;
+        bool done = false;
+    };
+
+    static constexpr RedundancyVerdict makeNonRedundant() noexcept { return { .redundant = false, .done = true }; }
+
+    std::unordered_set<String> existing_keys;
+
+    RedundancyVerdict isRedundantExpression(FunctionNode * function)
+    {
+        if (function->getArguments().getNodes().empty())
+            return makeNonRedundant();
+
+        if (function->getFunction()->isDeterministicInScopeOfQuery())
+            return makeNonRedundant();
+
+        // TODO: handle constants here
+        for (auto & arg : function->getArguments().getNodes())
+        {
+            switch (arg->getNodeType())
+            {
+                case QueryTreeNodeType::FUNCTION:
+                {
+                    auto subresult = isRedundantExpression(arg->as<FunctionNode>());
+                    if (subresult.done)
+                        return subresult;
+                    break;
+                }
+                case QueryTreeNodeType::COLUMN:
+                {
+                    auto * column = arg->as<ColumnNode>();
+                    if (!existing_keys.contains(column->getColumnName()))
+                        return makeNonRedundant();
+                    break;
+                }
+                default:
+                    return makeNonRedundant();
+            }
+        }
+
+        return {};
+    }
+
+public:
+    bool needChildVisit(QueryTreeNodePtr & node, QueryTreeNodePtr & /*parent*/)
+    {
+        if (node->as<FunctionNode>())
+            return false;
+        return true;
+    }
+
+    void visitImpl(QueryTreeNodePtr & node)
+    {
+        auto * query = node->as<QueryNode>();
+        if (!query)
+            return;
+
+        if (!query->hasOrderBy())
+            return;
+
+        auto & order_by = query->getOrderBy();
+        for (auto & elem : order_by.getNodes())
+        {
+            auto * order_by_elem = elem->as<SortNode>();
+            if (order_by_elem->withFill())
+                return;
+        }
+
+        QueryTreeNodes new_order_by;
+        new_order_by.reserve(order_by.getNodes().size());
+
+        for (auto & elem : order_by.getNodes())
+        {
+            auto * order_by_elem = elem->as<SortNode>();
+            if (auto * expr = order_by_elem->getExpression()->as<FunctionNode>())
+            {
+                if (isRedundantExpression(expr).redundant)
+                    continue;
+            }
+            else
+            {
+                auto * column = elem->as<ColumnNode>();
+                existing_keys.insert(column->getColumnName());
+            }
+
+            new_order_by.push_back(elem);
+        }
+        existing_keys.clear();
+
+        if (new_order_by.size() < order_by.getNodes().size())
+            order_by.getNodes() = std::move(new_order_by);
+    }
+};
+
+}
+
+void OptimizeRedundantFunctionsInOrderByPass::run(QueryTreeNodePtr query_tree_node, ContextPtr /*context*/)
+{
+    OptimizeRedundantFunctionsInOrderByVisitor().visit(query_tree_node);
+}
+
+}
diff --git a/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.h b/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.h
new file mode 100644
index 00000000000..609a6360d27
--- /dev/null
+++ b/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.h
@@ -0,0 +1,23 @@
+#pragma once
+
+#include <Analyzer/IQueryTreePass.h>
+
+namespace DB
+{
+
+/** If ORDER BY has argument x followed by f(x) transforms it to ORDER BY x.
+  * Optimize ORDER BY x, y, f(x), g(x, y), f(h(x)), t(f(x), g(x)) into ORDER BY x, y
+  * in case if f(), g(), h(), t() are deterministic (in scope of query).
+  * Don't optimize ORDER BY f(x), g(x), x even if f(x) is bijection for x or g(x).
+  */
+class OptimizeRedundantFunctionsInOrderByPass final : public IQueryTreePass
+{
+public:
+    String getName() override { return "OptimizeRedundantFunctionsInOrderBy"; }
+
+    String getDescription() override { return "If ORDER BY has argument x followed by f(x) transforms it to ORDER BY x."; }
+
+    void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override;
+};
+
+}
diff --git a/src/Analyzer/QueryTreePassManager.cpp b/src/Analyzer/QueryTreePassManager.cpp
index 06a1fec4698..dd14fc269f9 100644
--- a/src/Analyzer/QueryTreePassManager.cpp
+++ b/src/Analyzer/QueryTreePassManager.cpp
@@ -15,6 +15,7 @@
 #include <Analyzer/Passes/OrderByLimitByDuplicateEliminationPass.h>
 #include <Analyzer/Passes/FuseFunctionsPass.h>
 #include <Analyzer/Passes/IfTransformStringsToEnumPass.h>
+#include <Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.h>
 
 #include <IO/WriteHelpers.h>
 #include <IO/Operators.h>
@@ -91,7 +92,6 @@ public:
   * TODO: Support setting optimize_move_functions_out_of_any.
   * TODO: Support setting optimize_aggregators_of_group_by_keys.
   * TODO: Support setting optimize_duplicate_order_by_and_distinct.
-  * TODO: Support setting optimize_redundant_functions_in_order_by.
   * TODO: Support setting optimize_monotonous_functions_in_order_by.
   * TODO: Support settings.optimize_or_like_chain.
   * TODO: Add optimizations based on function semantics. Example: SELECT * FROM test_table WHERE id != id. (id is not nullable column).
@@ -203,6 +203,9 @@ void addQueryTreePasses(QueryTreePassManager & manager)
     if (settings.optimize_if_chain_to_multiif)
         manager.addPass(std::make_unique<IfChainToMultiIfPass>());
 
+    if (settings.optimize_redundant_functions_in_order_by)
+        manager.addPass(std::make_unique<OptimizeRedundantFunctionsInOrderByPass>());
+
     manager.addPass(std::make_unique<OrderByTupleEliminationPass>());
     manager.addPass(std::make_unique<OrderByLimitByDuplicateEliminationPass>());
 

From 6e8191367c7d21a7f439d6be416173829a5bae29 Mon Sep 17 00:00:00 2001
From: Dmitry Novik <n0vik@clickhouse.com>
Date: Fri, 4 Nov 2022 19:04:16 +0000
Subject: [PATCH 046/158] Fixup

---
 .../Passes/OptimizeRedundantFunctionsInOrderByPass.cpp         | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp b/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp
index 1359bd2616d..0337b5239d7 100644
--- a/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp
+++ b/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp
@@ -96,9 +96,8 @@ public:
                 if (isRedundantExpression(expr).redundant)
                     continue;
             }
-            else
+            else if (auto * column = elem->as<ColumnNode>())
             {
-                auto * column = elem->as<ColumnNode>();
                 existing_keys.insert(column->getColumnName());
             }
 

From 0ecf6164ac77992fe83b434397076bd3858ad1df Mon Sep 17 00:00:00 2001
From: Dmitry Novik <n0vik@clickhouse.com>
Date: Mon, 7 Nov 2022 15:00:18 +0000
Subject: [PATCH 047/158] Add tests

---
 ...ptimizeRedundantFunctionsInOrderByPass.cpp |   8 +-
 ..._redundant_functions_in_order_by.reference | 191 ++++++++++++++++++
 .../01323_redundant_functions_in_order_by.sql |  12 ++
 3 files changed, 207 insertions(+), 4 deletions(-)

diff --git a/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp b/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp
index 0337b5239d7..140129a9fb2 100644
--- a/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp
+++ b/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp
@@ -30,7 +30,7 @@ class OptimizeRedundantFunctionsInOrderByVisitor : public InDepthQueryTreeVisito
         if (function->getArguments().getNodes().empty())
             return makeNonRedundant();
 
-        if (function->getFunction()->isDeterministicInScopeOfQuery())
+        if (!function->getFunction()->isDeterministicInScopeOfQuery())
             return makeNonRedundant();
 
         // TODO: handle constants here
@@ -90,13 +90,13 @@ public:
 
         for (auto & elem : order_by.getNodes())
         {
-            auto * order_by_elem = elem->as<SortNode>();
-            if (auto * expr = order_by_elem->getExpression()->as<FunctionNode>())
+            auto & order_by_expr = elem->as<SortNode>()->getExpression();
+            if (auto * expr = order_by_expr->as<FunctionNode>())
             {
                 if (isRedundantExpression(expr).redundant)
                     continue;
             }
-            else if (auto * column = elem->as<ColumnNode>())
+            else if (auto * column = order_by_expr->as<ColumnNode>())
             {
                 existing_keys.insert(column->getColumnName());
             }
diff --git a/tests/queries/0_stateless/01323_redundant_functions_in_order_by.reference b/tests/queries/0_stateless/01323_redundant_functions_in_order_by.reference
index b32ad433730..88703af7def 100644
--- a/tests/queries/0_stateless/01323_redundant_functions_in_order_by.reference
+++ b/tests/queries/0_stateless/01323_redundant_functions_in_order_by.reference
@@ -1,6 +1,15 @@
 [0,1,2]
 [0,1,2]
 [0,1,2]
+[0,1,2]
+[0,1,2]
+[0,1,2]
+0	0	0	0
+0	1	1	1
+2	2	2	2
+3	3	3	3
+4	0		0
+5	0		0
 0	0	0	0
 0	1	1	1
 2	2	2	2
@@ -15,6 +24,14 @@
 1	1
 2	2
 3	3
+0	0
+1	1
+2	2
+3	3
+0	0
+1	1
+2	2
+3	3
 SELECT groupArray(x)
 FROM
 (
@@ -22,6 +39,32 @@ FROM
     FROM numbers(3)
     ORDER BY x ASC
 )
+QUERY id: 0
+  PROJECTION COLUMNS
+    groupArray(x) Array(UInt64)
+  PROJECTION
+    LIST id: 1, nodes: 1
+      FUNCTION id: 2, function_name: groupArray, function_type: aggregate, result_type: Array(UInt64)
+        ARGUMENTS
+          LIST id: 3, nodes: 1
+            COLUMN id: 4, column_name: x, result_type: UInt64, source_id: 5
+  JOIN TREE
+    QUERY id: 5, is_subquery: 1
+      PROJECTION COLUMNS
+        x UInt64
+      PROJECTION
+        LIST id: 6, nodes: 1
+          COLUMN id: 7, column_name: number, result_type: UInt64, source_id: 8
+      JOIN TREE
+        TABLE_FUNCTION id: 8, table_function_name: numbers
+          ARGUMENTS
+            LIST id: 9, nodes: 1
+              CONSTANT id: 10, constant_value: UInt64_3, constant_value_type: UInt8
+      ORDER BY
+        LIST id: 11, nodes: 1
+          SORT id: 12, sort_direction: ASCENDING, with_fill: 0
+            EXPRESSION
+              COLUMN id: 7, column_name: number, result_type: UInt64, source_id: 8
 SELECT groupArray(x)
 FROM
 (
@@ -29,6 +72,32 @@ FROM
     FROM numbers(3)
     ORDER BY x ASC
 )
+QUERY id: 0
+  PROJECTION COLUMNS
+    groupArray(x) Array(UInt64)
+  PROJECTION
+    LIST id: 1, nodes: 1
+      FUNCTION id: 2, function_name: groupArray, function_type: aggregate, result_type: Array(UInt64)
+        ARGUMENTS
+          LIST id: 3, nodes: 1
+            COLUMN id: 4, column_name: x, result_type: UInt64, source_id: 5
+  JOIN TREE
+    QUERY id: 5, is_subquery: 1
+      PROJECTION COLUMNS
+        x UInt64
+      PROJECTION
+        LIST id: 6, nodes: 1
+          COLUMN id: 7, column_name: number, result_type: UInt64, source_id: 8
+      JOIN TREE
+        TABLE_FUNCTION id: 8, table_function_name: numbers
+          ARGUMENTS
+            LIST id: 9, nodes: 1
+              CONSTANT id: 10, constant_value: UInt64_3, constant_value_type: UInt8
+      ORDER BY
+        LIST id: 11, nodes: 1
+          SORT id: 12, sort_direction: ASCENDING, with_fill: 0
+            EXPRESSION
+              COLUMN id: 7, column_name: number, result_type: UInt64, source_id: 8
 SELECT groupArray(x)
 FROM
 (
@@ -38,6 +107,38 @@ FROM
         exp(x) ASC,
         x ASC
 )
+QUERY id: 0
+  PROJECTION COLUMNS
+    groupArray(x) Array(UInt64)
+  PROJECTION
+    LIST id: 1, nodes: 1
+      FUNCTION id: 2, function_name: groupArray, function_type: aggregate, result_type: Array(UInt64)
+        ARGUMENTS
+          LIST id: 3, nodes: 1
+            COLUMN id: 4, column_name: x, result_type: UInt64, source_id: 5
+  JOIN TREE
+    QUERY id: 5, is_subquery: 1
+      PROJECTION COLUMNS
+        x UInt64
+      PROJECTION
+        LIST id: 6, nodes: 1
+          COLUMN id: 7, column_name: number, result_type: UInt64, source_id: 8
+      JOIN TREE
+        TABLE_FUNCTION id: 8, table_function_name: numbers
+          ARGUMENTS
+            LIST id: 9, nodes: 1
+              CONSTANT id: 10, constant_value: UInt64_3, constant_value_type: UInt8
+      ORDER BY
+        LIST id: 11, nodes: 2
+          SORT id: 12, sort_direction: ASCENDING, with_fill: 0
+            EXPRESSION
+              FUNCTION id: 13, function_name: exp, function_type: ordinary, result_type: Float64
+                ARGUMENTS
+                  LIST id: 14, nodes: 1
+                    COLUMN id: 7, column_name: number, result_type: UInt64, source_id: 8
+          SORT id: 15, sort_direction: ASCENDING, with_fill: 0
+            EXPRESSION
+              COLUMN id: 7, column_name: number, result_type: UInt64, source_id: 8
 SELECT
     key,
     a,
@@ -52,6 +153,53 @@ ALL FULL OUTER JOIN test AS t USING (key)
 ORDER BY
     key ASC,
     t.key ASC
+QUERY id: 0
+  PROJECTION COLUMNS
+    key UInt64
+    a UInt8
+    b String
+    c Float64
+  PROJECTION
+    LIST id: 1, nodes: 4
+      COLUMN id: 2, column_name: key, result_type: UInt64, source_id: 3
+      COLUMN id: 4, column_name: a, result_type: UInt8, source_id: 5
+      COLUMN id: 6, column_name: b, result_type: String, source_id: 5
+      COLUMN id: 7, column_name: c, result_type: Float64, source_id: 5
+  JOIN TREE
+    JOIN id: 8, strictness: ALL, kind: FULL
+      LEFT TABLE EXPRESSION
+        QUERY id: 3, alias: s, is_subquery: 1
+          PROJECTION COLUMNS
+            key UInt64
+          PROJECTION
+            LIST id: 9, nodes: 1
+              FUNCTION id: 10, function_name: plus, function_type: ordinary, result_type: UInt64
+                ARGUMENTS
+                  LIST id: 11, nodes: 2
+                    COLUMN id: 12, column_name: number, result_type: UInt64, source_id: 13
+                    CONSTANT id: 14, constant_value: UInt64_2, constant_value_type: UInt8
+          JOIN TREE
+            TABLE_FUNCTION id: 13, table_function_name: numbers
+              ARGUMENTS
+                LIST id: 15, nodes: 1
+                  CONSTANT id: 16, constant_value: UInt64_4, constant_value_type: UInt8
+      RIGHT TABLE EXPRESSION
+        TABLE id: 5, alias: t, table_name: default.test
+      JOIN EXPRESSION
+        LIST id: 17, nodes: 1
+          COLUMN id: 18, column_name: key, result_type: UInt64, source_id: 8
+            EXPRESSION
+              LIST id: 19, nodes: 2
+                COLUMN id: 20, column_name: key, result_type: UInt64, source_id: 3
+                COLUMN id: 21, column_name: key, result_type: UInt64, source_id: 5
+  ORDER BY
+    LIST id: 22, nodes: 2
+      SORT id: 23, sort_direction: ASCENDING, with_fill: 0
+        EXPRESSION
+          COLUMN id: 24, column_name: key, result_type: UInt64, source_id: 3
+      SORT id: 25, sort_direction: ASCENDING, with_fill: 0
+        EXPRESSION
+          COLUMN id: 26, column_name: key, result_type: UInt64, source_id: 5
 SELECT
     key,
     a
@@ -59,6 +207,24 @@ FROM test
 ORDER BY
     key ASC,
     a ASC
+QUERY id: 0
+  PROJECTION COLUMNS
+    key UInt64
+    a UInt8
+  PROJECTION
+    LIST id: 1, nodes: 2
+      COLUMN id: 2, column_name: key, result_type: UInt64, source_id: 3
+      COLUMN id: 4, column_name: a, result_type: UInt8, source_id: 3
+  JOIN TREE
+    TABLE id: 3, table_name: default.test
+  ORDER BY
+    LIST id: 5, nodes: 2
+      SORT id: 6, sort_direction: ASCENDING, with_fill: 0
+        EXPRESSION
+          COLUMN id: 2, column_name: key, result_type: UInt64, source_id: 3
+      SORT id: 7, sort_direction: ASCENDING, with_fill: 0
+        EXPRESSION
+          COLUMN id: 4, column_name: a, result_type: UInt8, source_id: 3
 SELECT
     key,
     a
@@ -66,6 +232,31 @@ FROM test
 ORDER BY
     key ASC,
     exp(key + a) ASC
+QUERY id: 0
+  PROJECTION COLUMNS
+    key UInt64
+    a UInt8
+  PROJECTION
+    LIST id: 1, nodes: 2
+      COLUMN id: 2, column_name: key, result_type: UInt64, source_id: 3
+      COLUMN id: 4, column_name: a, result_type: UInt8, source_id: 3
+  JOIN TREE
+    TABLE id: 3, table_name: default.test
+  ORDER BY
+    LIST id: 5, nodes: 2
+      SORT id: 6, sort_direction: ASCENDING, with_fill: 0
+        EXPRESSION
+          COLUMN id: 2, column_name: key, result_type: UInt64, source_id: 3
+      SORT id: 7, sort_direction: ASCENDING, with_fill: 0
+        EXPRESSION
+          FUNCTION id: 8, function_name: exp, function_type: ordinary, result_type: Float64
+            ARGUMENTS
+              LIST id: 9, nodes: 1
+                FUNCTION id: 10, function_name: plus, function_type: ordinary, result_type: UInt64
+                  ARGUMENTS
+                    LIST id: 11, nodes: 2
+                      COLUMN id: 2, column_name: key, result_type: UInt64, source_id: 3
+                      COLUMN id: 4, column_name: a, result_type: UInt8, source_id: 3
 [0,1,2]
 [0,1,2]
 [0,1,2]
diff --git a/tests/queries/0_stateless/01323_redundant_functions_in_order_by.sql b/tests/queries/0_stateless/01323_redundant_functions_in_order_by.sql
index c810567f73a..9e87b5e1da4 100644
--- a/tests/queries/0_stateless/01323_redundant_functions_in_order_by.sql
+++ b/tests/queries/0_stateless/01323_redundant_functions_in_order_by.sql
@@ -6,17 +6,29 @@ INSERT INTO test SELECT number, number, toString(number), number from numbers(4)
 set optimize_redundant_functions_in_order_by = 1;
 
 SELECT groupArray(x) from (SELECT number as x FROM numbers(3) ORDER BY x, exp(x));
+SELECT groupArray(x) from (SELECT number as x FROM numbers(3) ORDER BY x, exp(x)) SETTINGS allow_experimental_analyzer=1;
 SELECT groupArray(x) from (SELECT number as x FROM numbers(3) ORDER BY x, exp(exp(x)));
+SELECT groupArray(x) from (SELECT number as x FROM numbers(3) ORDER BY x, exp(exp(x))) SETTINGS allow_experimental_analyzer=1;
 SELECT groupArray(x) from (SELECT number as x FROM numbers(3) ORDER BY exp(x), x);
+SELECT groupArray(x) from (SELECT number as x FROM numbers(3) ORDER BY exp(x), x) SETTINGS allow_experimental_analyzer=1;
 SELECT * FROM (SELECT number + 2 AS key FROM numbers(4)) s FULL JOIN test t USING(key) ORDER BY s.key, t.key;
+SELECT * FROM (SELECT number + 2 AS key FROM numbers(4)) s FULL JOIN test t USING(key) ORDER BY s.key, t.key SETTINGS allow_experimental_analyzer=1;
 SELECT key, a FROM test ORDER BY key, a, exp(key + a);
+SELECT key, a FROM test ORDER BY key, a, exp(key + a) SETTINGS allow_experimental_analyzer=1;
 SELECT key, a FROM test ORDER BY key, exp(key + a);
+SELECT key, a FROM test ORDER BY key, exp(key + a) SETTINGS allow_experimental_analyzer=1;
 EXPLAIN SYNTAX SELECT groupArray(x) from (SELECT number as x FROM numbers(3) ORDER BY x, exp(x));
+EXPLAIN QUERY TREE run_passes=1 SELECT groupArray(x) from (SELECT number as x FROM numbers(3) ORDER BY x, exp(x));
 EXPLAIN SYNTAX SELECT groupArray(x) from (SELECT number as x FROM numbers(3) ORDER BY x, exp(exp(x)));
+EXPLAIN QUERY TREE run_passes=1 SELECT groupArray(x) from (SELECT number as x FROM numbers(3) ORDER BY x, exp(exp(x)));
 EXPLAIN SYNTAX SELECT groupArray(x) from (SELECT number as x FROM numbers(3) ORDER BY exp(x), x);
+EXPLAIN QUERY TREE run_passes=1 SELECT groupArray(x) from (SELECT number as x FROM numbers(3) ORDER BY exp(x), x);
 EXPLAIN SYNTAX SELECT * FROM (SELECT number + 2 AS key FROM numbers(4)) s FULL JOIN test t USING(key) ORDER BY s.key, t.key;
+EXPLAIN QUERY TREE run_passes=1 SELECT * FROM (SELECT number + 2 AS key FROM numbers(4)) s FULL JOIN test t USING(key) ORDER BY s.key, t.key;
 EXPLAIN SYNTAX SELECT key, a FROM test ORDER BY key, a, exp(key + a);
+EXPLAIN QUERY TREE run_passes=1 SELECT key, a FROM test ORDER BY key, a, exp(key + a);
 EXPLAIN SYNTAX SELECT key, a FROM test ORDER BY key, exp(key + a);
+EXPLAIN QUERY TREE run_passes=1 SELECT key, a FROM test ORDER BY key, exp(key + a);
 
 set optimize_redundant_functions_in_order_by = 0;
 

From 0a42d698aca0ac9b7e9ae97a2074d164a4af1c22 Mon Sep 17 00:00:00 2001
From: Dmitry Novik <n0vik@clickhouse.com>
Date: Mon, 2 Jan 2023 18:49:29 +0000
Subject: [PATCH 048/158] Fixes after review

---
 ...ptimizeRedundantFunctionsInOrderByPass.cpp | 17 ++++-----
 ..._redundant_functions_in_order_by.reference | 36 +++++++++++++++----
 .../01323_redundant_functions_in_order_by.sql |  1 +
 3 files changed, 39 insertions(+), 15 deletions(-)

diff --git a/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp b/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp
index 140129a9fb2..aaa777de13c 100644
--- a/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp
+++ b/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp
@@ -1,3 +1,4 @@
+#include <string_view>
 #include <Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.h>
 #include <Analyzer/ColumnNode.h>
 #include <Analyzer/FunctionNode.h>
@@ -23,14 +24,14 @@ class OptimizeRedundantFunctionsInOrderByVisitor : public InDepthQueryTreeVisito
 
     static constexpr RedundancyVerdict makeNonRedundant() noexcept { return { .redundant = false, .done = true }; }
 
-    std::unordered_set<String> existing_keys;
+    std::unordered_set<std::string_view> existing_keys;
 
     RedundancyVerdict isRedundantExpression(FunctionNode * function)
     {
         if (function->getArguments().getNodes().empty())
             return makeNonRedundant();
-
-        if (!function->getFunction()->isDeterministicInScopeOfQuery())
+        const auto & function_base = function->getFunction();
+        if (!function_base || !function_base->isDeterministicInScopeOfQuery())
             return makeNonRedundant();
 
         // TODO: handle constants here
@@ -85,8 +86,8 @@ public:
                 return;
         }
 
-        QueryTreeNodes new_order_by;
-        new_order_by.reserve(order_by.getNodes().size());
+        QueryTreeNodes new_order_by_nodes;
+        new_order_by_nodes.reserve(order_by.getNodes().size());
 
         for (auto & elem : order_by.getNodes())
         {
@@ -101,12 +102,12 @@ public:
                 existing_keys.insert(column->getColumnName());
             }
 
-            new_order_by.push_back(elem);
+            new_order_by_nodes.push_back(elem);
         }
         existing_keys.clear();
 
-        if (new_order_by.size() < order_by.getNodes().size())
-            order_by.getNodes() = std::move(new_order_by);
+        if (new_order_by_nodes.size() < order_by.getNodes().size())
+            order_by.getNodes() = std::move(new_order_by_nodes);
     }
 };
 
diff --git a/tests/queries/0_stateless/01323_redundant_functions_in_order_by.reference b/tests/queries/0_stateless/01323_redundant_functions_in_order_by.reference
index 88703af7def..ae160ed35d6 100644
--- a/tests/queries/0_stateless/01323_redundant_functions_in_order_by.reference
+++ b/tests/queries/0_stateless/01323_redundant_functions_in_order_by.reference
@@ -190,16 +190,16 @@ QUERY id: 0
           COLUMN id: 18, column_name: key, result_type: UInt64, source_id: 8
             EXPRESSION
               LIST id: 19, nodes: 2
-                COLUMN id: 20, column_name: key, result_type: UInt64, source_id: 3
-                COLUMN id: 21, column_name: key, result_type: UInt64, source_id: 5
+                COLUMN id: 2, column_name: key, result_type: UInt64, source_id: 3
+                COLUMN id: 20, column_name: key, result_type: UInt64, source_id: 5
   ORDER BY
-    LIST id: 22, nodes: 2
-      SORT id: 23, sort_direction: ASCENDING, with_fill: 0
+    LIST id: 21, nodes: 2
+      SORT id: 22, sort_direction: ASCENDING, with_fill: 0
         EXPRESSION
-          COLUMN id: 24, column_name: key, result_type: UInt64, source_id: 3
-      SORT id: 25, sort_direction: ASCENDING, with_fill: 0
+          COLUMN id: 23, column_name: key, result_type: UInt64, source_id: 3
+      SORT id: 24, sort_direction: ASCENDING, with_fill: 0
         EXPRESSION
-          COLUMN id: 26, column_name: key, result_type: UInt64, source_id: 5
+          COLUMN id: 25, column_name: key, result_type: UInt64, source_id: 5
 SELECT
     key,
     a
@@ -257,6 +257,28 @@ QUERY id: 0
                     LIST id: 11, nodes: 2
                       COLUMN id: 2, column_name: key, result_type: UInt64, source_id: 3
                       COLUMN id: 4, column_name: a, result_type: UInt8, source_id: 3
+QUERY id: 0
+  PROJECTION COLUMNS
+    key UInt64
+  PROJECTION
+    LIST id: 1, nodes: 1
+      COLUMN id: 2, column_name: key, result_type: UInt64, source_id: 3
+  JOIN TREE
+    TABLE id: 3, table_name: default.test
+  GROUP BY
+    LIST id: 4, nodes: 1
+      COLUMN id: 2, column_name: key, result_type: UInt64, source_id: 3
+  ORDER BY
+    LIST id: 5, nodes: 2
+      SORT id: 6, sort_direction: ASCENDING, with_fill: 0
+        EXPRESSION
+          FUNCTION id: 7, function_name: avg, function_type: aggregate, result_type: Float64
+            ARGUMENTS
+              LIST id: 8, nodes: 1
+                COLUMN id: 9, column_name: a, result_type: UInt8, source_id: 3
+      SORT id: 10, sort_direction: ASCENDING, with_fill: 0
+        EXPRESSION
+          COLUMN id: 2, column_name: key, result_type: UInt64, source_id: 3
 [0,1,2]
 [0,1,2]
 [0,1,2]
diff --git a/tests/queries/0_stateless/01323_redundant_functions_in_order_by.sql b/tests/queries/0_stateless/01323_redundant_functions_in_order_by.sql
index 9e87b5e1da4..3573773b76c 100644
--- a/tests/queries/0_stateless/01323_redundant_functions_in_order_by.sql
+++ b/tests/queries/0_stateless/01323_redundant_functions_in_order_by.sql
@@ -29,6 +29,7 @@ EXPLAIN SYNTAX SELECT key, a FROM test ORDER BY key, a, exp(key + a);
 EXPLAIN QUERY TREE run_passes=1 SELECT key, a FROM test ORDER BY key, a, exp(key + a);
 EXPLAIN SYNTAX SELECT key, a FROM test ORDER BY key, exp(key + a);
 EXPLAIN QUERY TREE run_passes=1 SELECT key, a FROM test ORDER BY key, exp(key + a);
+EXPLAIN QUERY TREE run_passes=1 SELECT key FROM test GROUP BY key ORDER BY avg(a), key;
 
 set optimize_redundant_functions_in_order_by = 0;
 

From 09d9ac6c8ef253d12c34e393970f20a8205eef88 Mon Sep 17 00:00:00 2001
From: Dmitry Novik <n0vik@clickhouse.com>
Date: Mon, 2 Jan 2023 19:09:45 +0000
Subject: [PATCH 049/158] Simplify code

---
 ...ptimizeRedundantFunctionsInOrderByPass.cpp | 28 ++++++-------------
 1 file changed, 9 insertions(+), 19 deletions(-)

diff --git a/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp b/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp
index aaa777de13c..29724de0f20 100644
--- a/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp
+++ b/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp
@@ -15,24 +15,15 @@ namespace
 
 class OptimizeRedundantFunctionsInOrderByVisitor : public InDepthQueryTreeVisitor<OptimizeRedundantFunctionsInOrderByVisitor>
 {
-
-    struct RedundancyVerdict
-    {
-        bool redundant = true;
-        bool done = false;
-    };
-
-    static constexpr RedundancyVerdict makeNonRedundant() noexcept { return { .redundant = false, .done = true }; }
-
     std::unordered_set<std::string_view> existing_keys;
 
-    RedundancyVerdict isRedundantExpression(FunctionNode * function)
+    bool isRedundantExpression(FunctionNode * function)
     {
         if (function->getArguments().getNodes().empty())
-            return makeNonRedundant();
+            return false;
         const auto & function_base = function->getFunction();
         if (!function_base || !function_base->isDeterministicInScopeOfQuery())
-            return makeNonRedundant();
+            return false;
 
         // TODO: handle constants here
         for (auto & arg : function->getArguments().getNodes())
@@ -41,24 +32,23 @@ class OptimizeRedundantFunctionsInOrderByVisitor : public InDepthQueryTreeVisito
             {
                 case QueryTreeNodeType::FUNCTION:
                 {
-                    auto subresult = isRedundantExpression(arg->as<FunctionNode>());
-                    if (subresult.done)
-                        return subresult;
+                    if (!isRedundantExpression(arg->as<FunctionNode>()))
+                        return false;
                     break;
                 }
                 case QueryTreeNodeType::COLUMN:
                 {
                     auto * column = arg->as<ColumnNode>();
                     if (!existing_keys.contains(column->getColumnName()))
-                        return makeNonRedundant();
+                        return false;
                     break;
                 }
                 default:
-                    return makeNonRedundant();
+                    return false;
             }
         }
 
-        return {};
+        return true;
     }
 
 public:
@@ -94,7 +84,7 @@ public:
             auto & order_by_expr = elem->as<SortNode>()->getExpression();
             if (auto * expr = order_by_expr->as<FunctionNode>())
             {
-                if (isRedundantExpression(expr).redundant)
+                if (isRedundantExpression(expr))
                     continue;
             }
             else if (auto * column = order_by_expr->as<ColumnNode>())

From 11fa29d243fd45e0944b7e06b4c7f53a76238a07 Mon Sep 17 00:00:00 2001
From: Dmitry Novik <n0vik@clickhouse.com>
Date: Tue, 3 Jan 2023 17:59:04 +0000
Subject: [PATCH 050/158] Get rid of recursion

---
 ...ptimizeRedundantFunctionsInOrderByPass.cpp | 101 ++++++++++--------
 1 file changed, 58 insertions(+), 43 deletions(-)

diff --git a/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp b/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp
index 29724de0f20..8136052cbd5 100644
--- a/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp
+++ b/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp
@@ -15,42 +15,6 @@ namespace
 
 class OptimizeRedundantFunctionsInOrderByVisitor : public InDepthQueryTreeVisitor<OptimizeRedundantFunctionsInOrderByVisitor>
 {
-    std::unordered_set<std::string_view> existing_keys;
-
-    bool isRedundantExpression(FunctionNode * function)
-    {
-        if (function->getArguments().getNodes().empty())
-            return false;
-        const auto & function_base = function->getFunction();
-        if (!function_base || !function_base->isDeterministicInScopeOfQuery())
-            return false;
-
-        // TODO: handle constants here
-        for (auto & arg : function->getArguments().getNodes())
-        {
-            switch (arg->getNodeType())
-            {
-                case QueryTreeNodeType::FUNCTION:
-                {
-                    if (!isRedundantExpression(arg->as<FunctionNode>()))
-                        return false;
-                    break;
-                }
-                case QueryTreeNodeType::COLUMN:
-                {
-                    auto * column = arg->as<ColumnNode>();
-                    if (!existing_keys.contains(column->getColumnName()))
-                        return false;
-                    break;
-                }
-                default:
-                    return false;
-            }
-        }
-
-        return true;
-    }
-
 public:
     bool needChildVisit(QueryTreeNodePtr & node, QueryTreeNodePtr & /*parent*/)
     {
@@ -82,14 +46,22 @@ public:
         for (auto & elem : order_by.getNodes())
         {
             auto & order_by_expr = elem->as<SortNode>()->getExpression();
-            if (auto * expr = order_by_expr->as<FunctionNode>())
+            switch (order_by_expr->getNodeType())
             {
-                if (isRedundantExpression(expr))
-                    continue;
-            }
-            else if (auto * column = order_by_expr->as<ColumnNode>())
-            {
-                existing_keys.insert(column->getColumnName());
+                case QueryTreeNodeType::FUNCTION:
+                {
+                    if (isRedundantExpression(order_by_expr))
+                        continue;
+                    break;
+                }
+                case QueryTreeNodeType::COLUMN:
+                {
+                    auto * column = order_by_expr->as<ColumnNode>();
+                    existing_keys.insert(column->getColumnName());
+                    break;
+                }
+                default:
+                    break;
             }
 
             new_order_by_nodes.push_back(elem);
@@ -99,6 +71,49 @@ public:
         if (new_order_by_nodes.size() < order_by.getNodes().size())
             order_by.getNodes() = std::move(new_order_by_nodes);
     }
+
+private:
+    std::unordered_set<std::string_view> existing_keys;
+
+    bool isRedundantExpression(QueryTreeNodePtr function)
+    {
+        QueryTreeNodes nodes_to_process{ function };
+        while (!nodes_to_process.empty())
+        {
+            auto node = nodes_to_process.back();
+            nodes_to_process.pop_back();
+
+            // TODO: handle constants here
+            switch (node->getNodeType())
+            {
+                case QueryTreeNodeType::FUNCTION:
+                {
+                    auto * function_node = node->as<FunctionNode>();
+                    const auto & function_arguments = function_node->getArguments().getNodes();
+                    if (function_arguments.empty())
+                        return false;
+                    const auto & function_base = function_node->getFunction();
+                    if (!function_base || !function_base->isDeterministicInScopeOfQuery())
+                        return false;
+
+                    // Process arguments in order
+                    for (auto it = function_arguments.rbegin(); it != function_arguments.rend(); ++it)
+                        nodes_to_process.push_back(*it);
+                    break;
+                }
+                case QueryTreeNodeType::COLUMN:
+                {
+                    auto * column = node->as<ColumnNode>();
+                    if (!existing_keys.contains(column->getColumnName()))
+                        return false;
+                    break;
+                }
+                default:
+                    return false;
+            }
+        }
+        return true;
+    }
 };
 
 }

From 15f20cb4e87531e364bc4cd5e34f745fd5387245 Mon Sep 17 00:00:00 2001
From: Dmitry Novik <n0vik@clickhouse.com>
Date: Tue, 3 Jan 2023 18:13:31 +0000
Subject: [PATCH 051/158] Fix column comparison

---
 ...ptimizeRedundantFunctionsInOrderByPass.cpp | 10 +++----
 ..._redundant_functions_in_order_by.reference | 28 +++++++++++++++++++
 .../01323_redundant_functions_in_order_by.sql |  9 ++++++
 3 files changed, 41 insertions(+), 6 deletions(-)

diff --git a/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp b/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp
index 8136052cbd5..105fc0ef00a 100644
--- a/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp
+++ b/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp
@@ -1,7 +1,7 @@
-#include <string_view>
 #include <Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.h>
 #include <Analyzer/ColumnNode.h>
 #include <Analyzer/FunctionNode.h>
+#include <Analyzer/HashUtils.h>
 #include <Analyzer/InDepthQueryTreeVisitor.h>
 #include <Analyzer/QueryNode.h>
 #include <Analyzer/SortNode.h>
@@ -56,8 +56,7 @@ public:
                 }
                 case QueryTreeNodeType::COLUMN:
                 {
-                    auto * column = order_by_expr->as<ColumnNode>();
-                    existing_keys.insert(column->getColumnName());
+                    existing_keys.insert(order_by_expr);
                     break;
                 }
                 default:
@@ -73,7 +72,7 @@ public:
     }
 
 private:
-    std::unordered_set<std::string_view> existing_keys;
+    QueryTreeNodePtrWithHashSet existing_keys;
 
     bool isRedundantExpression(QueryTreeNodePtr function)
     {
@@ -103,8 +102,7 @@ private:
                 }
                 case QueryTreeNodeType::COLUMN:
                 {
-                    auto * column = node->as<ColumnNode>();
-                    if (!existing_keys.contains(column->getColumnName()))
+                    if (!existing_keys.contains(node))
                         return false;
                     break;
                 }
diff --git a/tests/queries/0_stateless/01323_redundant_functions_in_order_by.reference b/tests/queries/0_stateless/01323_redundant_functions_in_order_by.reference
index ae160ed35d6..c69f8bb2c46 100644
--- a/tests/queries/0_stateless/01323_redundant_functions_in_order_by.reference
+++ b/tests/queries/0_stateless/01323_redundant_functions_in_order_by.reference
@@ -279,6 +279,34 @@ QUERY id: 0
       SORT id: 10, sort_direction: ASCENDING, with_fill: 0
         EXPRESSION
           COLUMN id: 2, column_name: key, result_type: UInt64, source_id: 3
+QUERY id: 0
+  PROJECTION COLUMNS
+    t1.id UInt64
+    t2.id UInt64
+  PROJECTION
+    LIST id: 1, nodes: 2
+      COLUMN id: 2, column_name: id, result_type: UInt64, source_id: 3
+      COLUMN id: 4, column_name: id, result_type: UInt64, source_id: 5
+  JOIN TREE
+    JOIN id: 6, strictness: ALL, kind: INNER
+      LEFT TABLE EXPRESSION
+        TABLE id: 3, table_name: default.t1
+      RIGHT TABLE EXPRESSION
+        TABLE id: 5, table_name: default.t2
+      JOIN EXPRESSION
+        FUNCTION id: 7, function_name: equals, function_type: ordinary, result_type: UInt8
+          ARGUMENTS
+            LIST id: 8, nodes: 2
+              COLUMN id: 9, column_name: id, result_type: UInt64, source_id: 3
+              COLUMN id: 10, column_name: id, result_type: UInt64, source_id: 5
+  ORDER BY
+    LIST id: 11, nodes: 2
+      SORT id: 12, sort_direction: ASCENDING, with_fill: 0
+        EXPRESSION
+          COLUMN id: 13, column_name: id, result_type: UInt64, source_id: 3
+      SORT id: 14, sort_direction: ASCENDING, with_fill: 0
+        EXPRESSION
+          COLUMN id: 15, column_name: id, result_type: UInt64, source_id: 5
 [0,1,2]
 [0,1,2]
 [0,1,2]
diff --git a/tests/queries/0_stateless/01323_redundant_functions_in_order_by.sql b/tests/queries/0_stateless/01323_redundant_functions_in_order_by.sql
index 3573773b76c..5cdc4164d56 100644
--- a/tests/queries/0_stateless/01323_redundant_functions_in_order_by.sql
+++ b/tests/queries/0_stateless/01323_redundant_functions_in_order_by.sql
@@ -31,6 +31,13 @@ EXPLAIN SYNTAX SELECT key, a FROM test ORDER BY key, exp(key + a);
 EXPLAIN QUERY TREE run_passes=1 SELECT key, a FROM test ORDER BY key, exp(key + a);
 EXPLAIN QUERY TREE run_passes=1 SELECT key FROM test GROUP BY key ORDER BY avg(a), key;
 
+DROP TABLE IF EXISTS t1;
+DROP TABLE IF EXISTS t2;
+CREATE TABLE t1 (id UInt64) ENGINE = MergeTree() ORDER BY id;
+CREATE TABLE t2 (id UInt64) ENGINE = MergeTree() ORDER BY id;
+
+EXPLAIN QUERY TREE run_passes=1 SELECT * FROM t1 INNER JOIN t2 ON t1.id = t2.id ORDER BY t1.id, t2.id;
+
 set optimize_redundant_functions_in_order_by = 0;
 
 SELECT groupArray(x) from (SELECT number as x FROM numbers(3) ORDER BY x, exp(x));
@@ -46,4 +53,6 @@ EXPLAIN SYNTAX SELECT * FROM (SELECT number + 2 AS key FROM numbers(4)) s FULL J
 EXPLAIN SYNTAX SELECT key, a FROM test ORDER BY key, a, exp(key + a);
 EXPLAIN SYNTAX SELECT key, a FROM test ORDER BY key, exp(key + a);
 
+DROP TABLE t1;
+DROP TABLE t2;
 DROP TABLE test;

From 1dd32c8336fbcc0b41b4256d308c809dc8bcb008 Mon Sep 17 00:00:00 2001
From: Dmitry Novik <n0vik@clickhouse.com>
Date: Tue, 3 Jan 2023 19:39:46 +0000
Subject: [PATCH 052/158] Validate function arguments in query tree

---
 src/Analyzer/FunctionNode.cpp         | 10 ++++++++++
 src/Analyzer/FunctionNode.h           |  1 +
 src/Analyzer/QueryTreePassManager.cpp | 23 +++++++++++++++++++++++
 3 files changed, 34 insertions(+)

diff --git a/src/Analyzer/FunctionNode.cpp b/src/Analyzer/FunctionNode.cpp
index 1b32cd5436d..dbd8b688869 100644
--- a/src/Analyzer/FunctionNode.cpp
+++ b/src/Analyzer/FunctionNode.cpp
@@ -2,6 +2,7 @@
 
 #include <Common/SipHash.h>
 #include <Common/FieldVisitorToString.h>
+#include <DataTypes/IDataType.h>
 #include <Analyzer/ConstantNode.h>
 
 #include <IO/WriteBufferFromString.h>
@@ -31,6 +32,15 @@ FunctionNode::FunctionNode(String function_name_)
     children[arguments_child_index] = std::make_shared<ListNode>();
 }
 
+const DataTypes & FunctionNode::getExpectedArgumentTypes() const
+{
+    if (!function)
+        throw Exception(ErrorCodes::LOGICAL_ERROR,
+        "Trying to get expected argument types of function {} when IResolvedFunction is not set",
+        function_name);
+    return function->getArgumentTypes();
+}
+
 ColumnsWithTypeAndName FunctionNode::getArgumentTypes() const
 {
     ColumnsWithTypeAndName argument_types;
diff --git a/src/Analyzer/FunctionNode.h b/src/Analyzer/FunctionNode.h
index 501d439e55e..852dd1046a7 100644
--- a/src/Analyzer/FunctionNode.h
+++ b/src/Analyzer/FunctionNode.h
@@ -116,6 +116,7 @@ public:
         return children[arguments_child_index];
     }
 
+    const DataTypes & getExpectedArgumentTypes() const;
     ColumnsWithTypeAndName getArgumentTypes() const;
 
     /// Returns true if function node has window, false otherwise
diff --git a/src/Analyzer/QueryTreePassManager.cpp b/src/Analyzer/QueryTreePassManager.cpp
index 06a1fec4698..99a695b0769 100644
--- a/src/Analyzer/QueryTreePassManager.cpp
+++ b/src/Analyzer/QueryTreePassManager.cpp
@@ -60,6 +60,29 @@ class ValidationChecker : public InDepthQueryTreeVisitor<ValidationChecker>
             throw Exception(ErrorCodes::LOGICAL_ERROR,
             "Function {} is not resolved after running {} pass",
             function->dumpTree(), pass_name);
+
+        const auto & expected_arg_types = function->getExpectedArgumentTypes();
+        auto actual_arg_columns = function->getArgumentTypes();
+
+        if (expected_arg_types.size() != actual_arg_columns.size())
+            throw Exception(ErrorCodes::LOGICAL_ERROR,
+            "Function {} expects {} arguments but has {} after running {} pass",
+            function->toAST()->formatForErrorMessage(),
+            expected_arg_types.size(),
+            actual_arg_columns.size(),
+            pass_name);
+
+        for (size_t i = 0; i < expected_arg_types.size(); ++i)
+        {
+            if (!expected_arg_types[i]->equals(*actual_arg_columns[i].type))
+                throw Exception(ErrorCodes::LOGICAL_ERROR,
+                "Function {} expects {} argument to have {} type but receives {} after running {} pass",
+                function->toAST()->formatForErrorMessage(),
+                i,
+                expected_arg_types[i]->getName(),
+                actual_arg_columns[i].type->getName(),
+                pass_name);
+        }
     }
 
 public:

From 712de132d9a9927e8e93e079e81acccbb441cc6c Mon Sep 17 00:00:00 2001
From: Dmitry Novik <n0vik@clickhouse.com>
Date: Wed, 4 Jan 2023 16:05:05 +0000
Subject: [PATCH 053/158] Fix special build

---
 src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp b/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp
index 105fc0ef00a..8c9db191bbd 100644
--- a/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp
+++ b/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp
@@ -16,7 +16,7 @@ namespace
 class OptimizeRedundantFunctionsInOrderByVisitor : public InDepthQueryTreeVisitor<OptimizeRedundantFunctionsInOrderByVisitor>
 {
 public:
-    bool needChildVisit(QueryTreeNodePtr & node, QueryTreeNodePtr & /*parent*/)
+    static bool needChildVisit(QueryTreeNodePtr & node, QueryTreeNodePtr & /*parent*/)
     {
         if (node->as<FunctionNode>())
             return false;

From f5b44cbe0d11d884da40a82761ff31021eaf909a Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Wed, 4 Jan 2023 19:33:34 +0100
Subject: [PATCH 054/158] Optimize and fix metrics for Distributed async INSERT

In #43406 metrics was broken for a clean start, since they where not
initialized from disk, but metrics for broken files was never
initialized from disk.

Fix this and rework how DirectoryMonitor works with file system:
- do not iterate over directory before each send, do this only once on
  init, after the map of files will be updated from the INSERT
- call fs::create_directories() from the ctor for "broken" folder to
  avoid excessive calls
- cache "broken" paths

This patch also fixes possible issue when current_batch can be processed
multiple times (second time will be an exception), since if there is
existing current_batch.txt after processing it you should remove it
instantly.

Plus this patch implicitly fixes issues with logging, that logs
incorrect number of files in case of error (see #44907 for details).

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Storages/Distributed/DirectoryMonitor.cpp | 291 +++++++++---------
 src/Storages/Distributed/DirectoryMonitor.h   |  27 +-
 src/Storages/Distributed/DistributedSink.cpp  |  18 +-
 src/Storages/StorageDistributed.cpp           |  10 +-
 ..._INSERT_block_structure_mismatch.reference |   4 +-
 5 files changed, 186 insertions(+), 164 deletions(-)

diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp
index 39e91e19014..04fac6c3402 100644
--- a/src/Storages/Distributed/DirectoryMonitor.cpp
+++ b/src/Storages/Distributed/DirectoryMonitor.cpp
@@ -60,6 +60,7 @@ namespace ErrorCodes
     extern const int TOO_MANY_PARTITIONS;
     extern const int DISTRIBUTED_TOO_MANY_PENDING_BYTES;
     extern const int ARGUMENT_OUT_OF_BOUND;
+    extern const int LOGICAL_ERROR;
 }
 
 
@@ -365,18 +366,22 @@ StorageDistributedDirectoryMonitor::StorageDistributedDirectoryMonitor(
     const std::string & relative_path_,
     ConnectionPoolPtr pool_,
     ActionBlocker & monitor_blocker_,
-    BackgroundSchedulePool & bg_pool)
+    BackgroundSchedulePool & bg_pool,
+    bool initialize_from_disk)
     : storage(storage_)
     , pool(std::move(pool_))
     , disk(disk_)
     , relative_path(relative_path_)
     , path(fs::path(disk->getPath()) / relative_path / "")
+    , broken_relative_path(fs::path(relative_path) / "broken")
+    , broken_path(fs::path(path) / "broken" / "")
     , should_batch_inserts(storage.getDistributedSettingsRef().monitor_batch_inserts)
     , split_batch_on_failure(storage.getDistributedSettingsRef().monitor_split_batch_on_failure)
     , dir_fsync(storage.getDistributedSettingsRef().fsync_directories)
     , min_batched_block_size_rows(storage.getContext()->getSettingsRef().min_insert_block_size_rows)
     , min_batched_block_size_bytes(storage.getContext()->getSettingsRef().min_insert_block_size_bytes)
     , current_batch_file_path(path + "current_batch.txt")
+    , pending_files(std::numeric_limits<size_t>::max())
     , default_sleep_time(storage.getDistributedSettingsRef().monitor_sleep_time_ms.totalMilliseconds())
     , sleep_time(default_sleep_time)
     , max_sleep_time(storage.getDistributedSettingsRef().monitor_max_sleep_time_ms.totalMilliseconds())
@@ -385,6 +390,11 @@ StorageDistributedDirectoryMonitor::StorageDistributedDirectoryMonitor(
     , metric_pending_files(CurrentMetrics::DistributedFilesToInsert, 0)
     , metric_broken_files(CurrentMetrics::BrokenDistributedFilesToInsert, 0)
 {
+    fs::create_directory(broken_path);
+
+    if (initialize_from_disk)
+        initializeFilesFromDisk();
+
     task_handle = bg_pool.createTask(getLoggerName() + "/Bg", [this]{ run(); });
     task_handle->activateAndSchedule();
 }
@@ -392,35 +402,29 @@ StorageDistributedDirectoryMonitor::StorageDistributedDirectoryMonitor(
 
 StorageDistributedDirectoryMonitor::~StorageDistributedDirectoryMonitor()
 {
-    if (!quit)
+    if (!pending_files.isFinished())
     {
-        quit = true;
+        pending_files.clearAndFinish();
         task_handle->deactivate();
     }
 }
 
 void StorageDistributedDirectoryMonitor::flushAllData()
 {
-    if (quit)
+    if (pending_files.isFinished())
         return;
 
     std::lock_guard lock{mutex};
-
-    const auto & files = getFiles();
-    if (!files.empty())
-    {
-        processFiles(files);
-
-        /// Update counters.
-        getFiles();
-    }
+    if (!hasPendingFiles())
+        return;
+    processFiles();
 }
 
 void StorageDistributedDirectoryMonitor::shutdownAndDropAllData()
 {
-    if (!quit)
+    if (!pending_files.isFinished())
     {
-        quit = true;
+        pending_files.clearAndFinish();
         task_handle->deactivate();
     }
 
@@ -434,19 +438,21 @@ void StorageDistributedDirectoryMonitor::run()
     std::lock_guard lock{mutex};
 
     bool do_sleep = false;
-    while (!quit)
+    while (!pending_files.isFinished())
     {
         do_sleep = true;
 
-        const auto & files = getFiles();
-        if (files.empty())
+        if (!hasPendingFiles())
             break;
 
         if (!monitor_blocker.isCancelled())
         {
             try
             {
-                do_sleep = !processFiles(files);
+                processFiles();
+                /// No errors while processing existing files.
+                /// Let's see maybe there are more files to process.
+                do_sleep = false;
 
                 std::lock_guard status_lock(status_mutex);
                 status.last_exception = std::exception_ptr{};
@@ -470,9 +476,7 @@ void StorageDistributedDirectoryMonitor::run()
             }
         }
         else
-        {
             LOG_DEBUG(log, "Skipping send data over distributed table.");
-        }
 
         const auto now = std::chrono::system_clock::now();
         if (now - last_decrease_time > decrease_error_count_period)
@@ -487,10 +491,7 @@ void StorageDistributedDirectoryMonitor::run()
             break;
     }
 
-    /// Update counters.
-    getFiles();
-
-    if (!quit && do_sleep)
+    if (!pending_files.isFinished() && do_sleep)
         task_handle->scheduleAfter(sleep_time.count());
 }
 
@@ -568,41 +569,83 @@ ConnectionPoolPtr StorageDistributedDirectoryMonitor::createPool(const std::stri
         settings.distributed_replica_error_cap);
 }
 
-
-std::map<UInt64, std::string> StorageDistributedDirectoryMonitor::getFiles()
+bool StorageDistributedDirectoryMonitor::hasPendingFiles() const
 {
-    std::map<UInt64, std::string> files;
+    return fs::exists(current_batch_file_path) || !current_batch_file.empty() || !pending_files.empty();
+}
+
+void StorageDistributedDirectoryMonitor::initializeFilesFromDisk()
+{
+    /// NOTE: This method does not requires to hold status_mutex, hence, no TSA
+    /// annotations in the header file.
 
     fs::directory_iterator end;
-    for (fs::directory_iterator it{path}; it != end; ++it)
+
+    /// Initialize pending files
     {
-        const auto & file_path_str = it->path();
-        if (!it->is_directory() && startsWith(fs::path(file_path_str).extension(), ".bin"))
+        size_t bytes_count = 0;
+
+        for (fs::directory_iterator it{path}; it != end; ++it)
         {
-            files[parse<UInt64>(fs::path(file_path_str).stem())] = file_path_str;
+            const auto & file_path = it->path();
+            const auto & base_name = file_path.stem().string();
+            if (!it->is_directory() && startsWith(fs::path(file_path).extension(), ".bin") && parse<UInt64>(base_name))
+            {
+                const std::string & file_path_str = file_path.string();
+                if (!pending_files.push(file_path_str))
+                    throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot add pending file");
+                bytes_count += fs::file_size(file_path);
+            }
+            else if (base_name != "tmp" && base_name != "broken")
+            {
+                /// It is OK to log current_batch.txt here too (useful for debugging).
+                LOG_WARNING(log, "Unexpected file {} in {}", file_path.string(), path);
+            }
         }
+
+        LOG_TRACE(log, "Files set to {}", pending_files.size());
+        LOG_TRACE(log, "Bytes set to {}", bytes_count);
+
+        metric_pending_files.changeTo(pending_files.size());
+        status.files_count = pending_files.size();
+        status.bytes_count = bytes_count;
     }
 
-    return files;
+    /// Initialize broken files
+    {
+        size_t broken_bytes_count = 0;
+        size_t broken_files = 0;
+
+        for (fs::directory_iterator it{broken_path}; it != end; ++it)
+        {
+            const auto & file_path = it->path();
+            if (!it->is_directory() && startsWith(fs::path(file_path).extension(), ".bin") && parse<UInt64>(file_path.stem()))
+                broken_bytes_count += fs::file_size(file_path);
+            else
+                LOG_WARNING(log, "Unexpected file {} in {}", file_path.string(), broken_path);
+        }
+
+        LOG_TRACE(log, "Broken files set to {}", broken_files);
+        LOG_TRACE(log, "Broken bytes set to {}", broken_bytes_count);
+
+        metric_broken_files.changeTo(broken_files);
+        status.broken_files_count = broken_files;
+        status.broken_bytes_count = broken_bytes_count;
+    }
 }
-bool StorageDistributedDirectoryMonitor::processFiles(const std::map<UInt64, std::string> & files)
+void StorageDistributedDirectoryMonitor::processFiles()
 {
     if (should_batch_inserts)
-    {
-        processFilesWithBatching(files);
-    }
+        processFilesWithBatching();
     else
     {
-        for (const auto & file : files)
-        {
-            if (quit)
-                return true;
+        /// Process unprocessed file.
+        if (!current_batch_file.empty())
+            processFile(current_batch_file);
 
-            processFile(file.second);
-        }
+        while (pending_files.tryPop(current_batch_file))
+            processFile(current_batch_file);
     }
-
-    return true;
 }
 
 void StorageDistributedDirectoryMonitor::processFile(const std::string & file_path)
@@ -649,7 +692,11 @@ void StorageDistributedDirectoryMonitor::processFile(const std::string & file_pa
             thread_trace_context->root_span.addAttribute(std::current_exception());
 
         e.addMessage(fmt::format("While sending {}", file_path));
-        maybeMarkAsBroken(file_path, e);
+        if (isFileBrokenErrorCode(e.code(), e.isRemoteException()))
+        {
+            markAsBroken(file_path);
+            current_batch_file.clear();
+        }
         throw;
     }
     catch (...)
@@ -662,6 +709,7 @@ void StorageDistributedDirectoryMonitor::processFile(const std::string & file_pa
 
     auto dir_sync_guard = getDirectorySyncGuard(dir_fsync, disk, relative_path);
     markAsSend(file_path);
+    current_batch_file.clear();
     LOG_TRACE(log, "Finished processing `{}` (took {} ms)", file_path, watch.elapsedMilliseconds());
 }
 
@@ -701,23 +749,19 @@ struct StorageDistributedDirectoryMonitor::BatchHeader
 
 struct StorageDistributedDirectoryMonitor::Batch
 {
-    std::vector<UInt64> file_indices;
     size_t total_rows = 0;
     size_t total_bytes = 0;
     bool recovered = false;
 
     StorageDistributedDirectoryMonitor & parent;
-    const std::map<UInt64, String> & file_index_to_path;
+    std::vector<std::string> files;
 
     bool split_batch_on_failure = true;
     bool fsync = false;
     bool dir_fsync = false;
 
-    Batch(
-        StorageDistributedDirectoryMonitor & parent_,
-        const std::map<UInt64, String> & file_index_to_path_)
+    explicit Batch(StorageDistributedDirectoryMonitor & parent_)
         : parent(parent_)
-        , file_index_to_path(file_index_to_path_)
         , split_batch_on_failure(parent.split_batch_on_failure)
         , fsync(parent.storage.getDistributedSettingsRef().fsync_after_insert)
         , dir_fsync(parent.dir_fsync)
@@ -732,7 +776,7 @@ struct StorageDistributedDirectoryMonitor::Batch
 
     void send()
     {
-        if (file_indices.empty())
+        if (files.empty())
             return;
 
         CurrentMetrics::Increment metric_increment{CurrentMetrics::DistributedSend};
@@ -775,7 +819,7 @@ struct StorageDistributedDirectoryMonitor::Batch
             }
             catch (const Exception & e)
             {
-                if (split_batch_on_failure && file_indices.size() > 1 && isSplittableErrorCode(e.code(), e.isRemoteException()))
+                if (split_batch_on_failure && files.size() > 1 && isSplittableErrorCode(e.code(), e.isRemoteException()))
                 {
                     tryLogCurrentException(parent.log, "Trying to split batch due to");
                     sendSeparateFiles();
@@ -795,44 +839,28 @@ struct StorageDistributedDirectoryMonitor::Batch
             }
             else
             {
-                std::vector<std::string> files;
-                for (const auto && file_info : file_index_to_path | boost::adaptors::indexed())
-                {
-                    if (file_info.index() > 8)
-                    {
-                        files.push_back("...");
-                        break;
-                    }
-
-                    files.push_back(file_info.value().second);
-                }
-                e.addMessage(fmt::format("While sending batch, nums: {}, files: {}", file_index_to_path.size(), fmt::join(files, "\n")));
-
+                e.addMessage(fmt::format("While sending a batch of {} files, files: {}", files.size(), fmt::join(files, "\n")));
                 throw;
             }
         }
 
         if (!batch_broken)
         {
-            LOG_TRACE(parent.log, "Sent a batch of {} files (took {} ms).", file_indices.size(), watch.elapsedMilliseconds());
+            LOG_TRACE(parent.log, "Sent a batch of {} files (took {} ms).", files.size(), watch.elapsedMilliseconds());
 
             auto dir_sync_guard = getDirectorySyncGuard(dir_fsync, parent.disk, parent.relative_path);
-            for (UInt64 file_index : file_indices)
-                parent.markAsSend(file_index_to_path.at(file_index));
+            for (const auto & file : files)
+                parent.markAsSend(file);
         }
         else if (!batch_marked_as_broken)
         {
-            LOG_ERROR(parent.log, "Marking a batch of {} files as broken.", file_indices.size());
+            LOG_ERROR(parent.log, "Marking a batch of {} files as broken, files: {}", files.size(), fmt::join(files, "\n"));
 
-            for (UInt64 file_idx : file_indices)
-            {
-                auto file_path = file_index_to_path.find(file_idx);
-                if (file_path != file_index_to_path.end())
-                    parent.markAsBroken(file_path->second);
-            }
+            for (const auto & file : files)
+                parent.markAsBroken(file);
         }
 
-        file_indices.clear();
+        files.clear();
         total_rows = 0;
         total_bytes = 0;
         recovered = false;
@@ -842,8 +870,11 @@ struct StorageDistributedDirectoryMonitor::Batch
 
     void writeText(WriteBuffer & out)
     {
-        for (UInt64 file_idx : file_indices)
-            out << file_idx << '\n';
+        for (const auto & file : files)
+        {
+            UInt64 file_index = parse<UInt64>(fs::path(file).stem());
+            out << file_index << '\n';
+        }
     }
 
     void readText(ReadBuffer & in)
@@ -852,8 +883,9 @@ struct StorageDistributedDirectoryMonitor::Batch
         {
             UInt64 idx;
             in >> idx >> "\n";
-            file_indices.push_back(idx);
+            files.push_back(fmt::format("{}/{}.bin", parent.path, idx));
         }
+
         recovered = true;
     }
 
@@ -865,14 +897,9 @@ private:
 
         IConnectionPool::Entry connection;
 
-        for (UInt64 file_idx : file_indices)
+        for (const auto & file : files)
         {
-            auto file_path = file_index_to_path.find(file_idx);
-            if (file_path == file_index_to_path.end())
-                throw Exception(ErrorCodes::DISTRIBUTED_BROKEN_BATCH_INFO,
-                    "Failed to send batch: file with index {} is absent", file_idx);
-
-            ReadBufferFromFile in(file_path->second);
+            ReadBufferFromFile in(file);
             const auto & distributed_header = readDistributedHeader(in, parent.log);
 
             OpenTelemetry::TracingContextHolder thread_trace_context(__PRETTY_FUNCTION__,
@@ -886,7 +913,7 @@ private:
                 compression_expected = connection->getCompression() == Protocol::Compression::Enable;
 
                 LOG_DEBUG(parent.log, "Sending a batch of {} files to {} ({} rows, {} bytes).",
-                    file_indices.size(),
+                    files.size(),
                     connection->getDescription(),
                     formatReadableQuantity(total_rows),
                     formatReadableSizeWithBinarySuffix(total_bytes));
@@ -907,19 +934,11 @@ private:
     {
         size_t broken_files = 0;
 
-        for (UInt64 file_idx : file_indices)
+        for (const auto & file : files)
         {
-            auto file_path = file_index_to_path.find(file_idx);
-            if (file_path == file_index_to_path.end())
-            {
-                LOG_ERROR(parent.log, "Failed to send one file from batch: file with index {} is absent", file_idx);
-                ++broken_files;
-                continue;
-            }
-
             try
             {
-                ReadBufferFromFile in(file_path->second);
+                ReadBufferFromFile in(file);
                 const auto & distributed_header = readDistributedHeader(in, parent.log);
 
                 // this function is called in a separated thread, so we set up the trace context from the file
@@ -941,9 +960,11 @@ private:
             }
             catch (Exception & e)
             {
-                e.addMessage(fmt::format("While sending {}", file_path->second));
-                parent.maybeMarkAsBroken(file_path->second, e);
-                ++broken_files;
+                if (isFileBrokenErrorCode(e.code(), e.isRemoteException()))
+                {
+                    parent.markAsBroken(file);
+                    ++broken_files;
+                }
             }
         }
 
@@ -1023,13 +1044,18 @@ std::shared_ptr<ISource> StorageDistributedDirectoryMonitor::createSourceFromFil
     return std::make_shared<DirectoryMonitorSource>(file_name);
 }
 
-bool StorageDistributedDirectoryMonitor::addAndSchedule(size_t file_size, size_t ms)
+bool StorageDistributedDirectoryMonitor::addAndSchedule(const std::string & file_path, size_t file_size, size_t ms)
 {
-    if (quit)
+    /// NOTE: It is better not to throw in this case, since the file is already
+    /// on disk (see DistributedSink), and it will be processed next time.
+    if (pending_files.isFinished())
         return false;
 
+    if (!pending_files.push(file_path))
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot add pending file");
+
     {
-        std::lock_guard status_lock(status_mutex);
+        std::lock_guard lock(status_mutex);
         metric_pending_files.add();
         status.bytes_count += file_size;
         ++status.files_count;
@@ -1045,33 +1071,25 @@ StorageDistributedDirectoryMonitor::Status StorageDistributedDirectoryMonitor::g
     return current_status;
 }
 
-void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map<UInt64, std::string> & files)
+void StorageDistributedDirectoryMonitor::processFilesWithBatching()
 {
-    std::unordered_set<UInt64> file_indices_to_skip;
-
+    /// Possibly, we failed to send a batch on the previous iteration. Try to send exactly the same batch.
     if (fs::exists(current_batch_file_path))
     {
-        /// Possibly, we failed to send a batch on the previous iteration. Try to send exactly the same batch.
-        Batch batch(*this, files);
+        Batch batch(*this);
         ReadBufferFromFile in{current_batch_file_path};
         batch.readText(in);
-        file_indices_to_skip.insert(batch.file_indices.begin(), batch.file_indices.end());
         batch.send();
+
+        auto dir_sync_guard = getDirectorySyncGuard(dir_fsync, disk, relative_path);
+        fs::remove(current_batch_file_path);
     }
 
     std::unordered_map<BatchHeader, Batch, BatchHeader::Hash> header_to_batch;
 
-    for (const auto & file : files)
+    std::string file_path;
+    while (pending_files.tryPop(file_path))
     {
-        if (quit)
-            return;
-
-        UInt64 file_idx = file.first;
-        const String & file_path = file.second;
-
-        if (file_indices_to_skip.contains(file_idx))
-            continue;
-
         size_t total_rows = 0;
         size_t total_bytes = 0;
         Block header;
@@ -1110,8 +1128,9 @@ void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map
         }
         catch (const Exception & e)
         {
-            if (maybeMarkAsBroken(file_path, e))
+            if (isFileBrokenErrorCode(e.code(), e.isRemoteException()))
             {
+                markAsBroken(file_path);
                 tryLogCurrentException(log, "File is marked broken due to");
                 continue;
             }
@@ -1125,9 +1144,9 @@ void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map
             std::move(distributed_header.client_info),
             std::move(header)
         );
-        Batch & batch = header_to_batch.try_emplace(batch_header, *this, files).first->second;
+        Batch & batch = header_to_batch.try_emplace(batch_header, *this).first->second;
 
-        batch.file_indices.push_back(file_idx);
+        batch.files.push_back(file_path);
         batch.total_rows += total_rows;
         batch.total_bytes += total_bytes;
 
@@ -1155,16 +1174,10 @@ void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map
 
 void StorageDistributedDirectoryMonitor::markAsBroken(const std::string & file_path)
 {
-    const auto last_path_separator_pos = file_path.rfind('/');
-    const auto & base_path = file_path.substr(0, last_path_separator_pos + 1);
-    const auto & file_name = file_path.substr(last_path_separator_pos + 1);
-    const String & broken_path = fs::path(base_path) / "broken/";
-    const String & broken_file_path = fs::path(broken_path) / file_name;
-
-    fs::create_directory(broken_path);
+    const String & broken_file_path = fs::path(broken_path) / fs::path(file_path).filename();
 
     auto dir_sync_guard = getDirectorySyncGuard(dir_fsync, disk, relative_path);
-    auto broken_dir_sync_guard = getDirectorySyncGuard(dir_fsync, disk, fs::path(relative_path) / "broken/");
+    auto broken_dir_sync_guard = getDirectorySyncGuard(dir_fsync, disk, broken_relative_path);
 
     {
         std::lock_guard status_lock(status_mutex);
@@ -1198,21 +1211,9 @@ void StorageDistributedDirectoryMonitor::markAsSend(const std::string & file_pat
     fs::remove(file_path);
 }
 
-bool StorageDistributedDirectoryMonitor::maybeMarkAsBroken(const std::string & file_path, const Exception & e)
-{
-    /// Mark file as broken if necessary.
-    if (isFileBrokenErrorCode(e.code(), e.isRemoteException()))
-    {
-        markAsBroken(file_path);
-        return true;
-    }
-    else
-        return false;
-}
-
 std::string StorageDistributedDirectoryMonitor::getLoggerName() const
 {
-    return storage.getStorageID().getFullTableName() + ".DirectoryMonitor";
+    return storage.getStorageID().getFullTableName() + ".DirectoryMonitor." + disk->getName();
 }
 
 void StorageDistributedDirectoryMonitor::updatePath(const std::string & new_relative_path)
diff --git a/src/Storages/Distributed/DirectoryMonitor.h b/src/Storages/Distributed/DirectoryMonitor.h
index 7015fca0311..99e949ddcff 100644
--- a/src/Storages/Distributed/DirectoryMonitor.h
+++ b/src/Storages/Distributed/DirectoryMonitor.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <Core/BackgroundSchedulePool.h>
+#include <Common/ConcurrentBoundedQueue.h>
 #include <Client/ConnectionPool.h>
 
 #include <atomic>
@@ -38,7 +39,8 @@ public:
         const std::string & relative_path_,
         ConnectionPoolPtr pool_,
         ActionBlocker & monitor_blocker_,
-        BackgroundSchedulePool & bg_pool);
+        BackgroundSchedulePool & bg_pool,
+        bool initialize_from_disk);
 
     ~StorageDistributedDirectoryMonitor();
 
@@ -53,7 +55,7 @@ public:
     static std::shared_ptr<ISource> createSourceFromFile(const String & file_name);
 
     /// For scheduling via DistributedSink.
-    bool addAndSchedule(size_t file_size, size_t ms);
+    bool addAndSchedule(const std::string & file_path, size_t file_size, size_t ms);
 
     struct InternalStatus
     {
@@ -78,14 +80,15 @@ public:
 private:
     void run();
 
-    std::map<UInt64, std::string> getFiles();
-    bool processFiles(const std::map<UInt64, std::string> & files);
+    bool hasPendingFiles() const;
+
+    void initializeFilesFromDisk();
+    void processFiles();
     void processFile(const std::string & file_path);
-    void processFilesWithBatching(const std::map<UInt64, std::string> & files);
+    void processFilesWithBatching();
 
     void markAsBroken(const std::string & file_path);
     void markAsSend(const std::string & file_path);
-    bool maybeMarkAsBroken(const std::string & file_path, const Exception & e);
 
     std::string getLoggerName() const;
 
@@ -95,25 +98,33 @@ private:
     DiskPtr disk;
     std::string relative_path;
     std::string path;
+    std::string broken_relative_path;
+    std::string broken_path;
 
     const bool should_batch_inserts = false;
     const bool split_batch_on_failure = true;
     const bool dir_fsync = false;
     const size_t min_batched_block_size_rows = 0;
     const size_t min_batched_block_size_bytes = 0;
-    String current_batch_file_path;
+
+    /// This is pending data (due to some error) for should_batch_inserts==true
+    std::string current_batch_file_path;
+    /// This is pending data (due to some error) for should_batch_inserts==false
+    std::string current_batch_file;
 
     struct BatchHeader;
     struct Batch;
 
     std::mutex status_mutex;
+
     InternalStatus status;
 
+    ConcurrentBoundedQueue<std::string> pending_files;
+
     const std::chrono::milliseconds default_sleep_time;
     std::chrono::milliseconds sleep_time;
     const std::chrono::milliseconds max_sleep_time;
     std::chrono::time_point<std::chrono::system_clock> last_decrease_time {std::chrono::system_clock::now()};
-    std::atomic<bool> quit {false};
     std::mutex mutex;
     Poco::Logger * log;
     ActionBlocker & monitor_blocker;
diff --git a/src/Storages/Distributed/DistributedSink.cpp b/src/Storages/Distributed/DistributedSink.cpp
index 38ff06f4744..8cee3e9ee91 100644
--- a/src/Storages/Distributed/DistributedSink.cpp
+++ b/src/Storages/Distributed/DistributedSink.cpp
@@ -724,6 +724,9 @@ void DistributedSink::writeToShard(const Cluster::ShardInfo & shard_info, const
         return guard;
     };
 
+    std::vector<std::string> bin_files;
+    bin_files.reserve(dir_names.size());
+
     auto it = dir_names.begin();
     /// on first iteration write block to a temporary directory for subsequent
     /// hardlinking to ensure the inode is not freed until we're done
@@ -802,8 +805,8 @@ void DistributedSink::writeToShard(const Cluster::ShardInfo & shard_info, const
         }
 
         // Create hardlink here to reuse increment number
-        const std::string block_file_path(fs::path(path) / file_name);
-        createHardLink(first_file_tmp_path, block_file_path);
+        bin_files.push_back(fs::path(path) / file_name);
+        createHardLink(first_file_tmp_path, bin_files.back());
         auto dir_sync_guard = make_directory_sync_guard(*it);
     }
     ++it;
@@ -814,8 +817,8 @@ void DistributedSink::writeToShard(const Cluster::ShardInfo & shard_info, const
         const std::string path(fs::path(disk_path) / (data_path + *it));
         fs::create_directory(path);
 
-        const std::string block_file_path(fs::path(path) / (toString(storage.file_names_increment.get()) + ".bin"));
-        createHardLink(first_file_tmp_path, block_file_path);
+        bin_files.push_back(fs::path(path) / (toString(storage.file_names_increment.get()) + ".bin"));
+        createHardLink(first_file_tmp_path, bin_files.back());
         auto dir_sync_guard = make_directory_sync_guard(*it);
     }
 
@@ -826,10 +829,13 @@ void DistributedSink::writeToShard(const Cluster::ShardInfo & shard_info, const
 
     /// Notify
     auto sleep_ms = context->getSettingsRef().distributed_directory_monitor_sleep_time_ms;
-    for (const auto & dir_name : dir_names)
+    for (size_t i = 0; i < dir_names.size(); ++i)
     {
+        const auto & dir_name = dir_names[i];
+        const auto & bin_file = bin_files[i];
+
         auto & directory_monitor = storage.requireDirectoryMonitor(disk, dir_name, /* startup= */ false);
-        directory_monitor.addAndSchedule(file_size, sleep_ms.totalMilliseconds());
+        directory_monitor.addAndSchedule(bin_file, file_size, sleep_ms.totalMilliseconds());
     }
 }
 
diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp
index 51cca15e5ab..eccc3383602 100644
--- a/src/Storages/StorageDistributed.cpp
+++ b/src/Storages/StorageDistributed.cpp
@@ -1043,12 +1043,15 @@ void StorageDistributed::createDirectoryMonitors(const DiskPtr & disk)
         const auto & dir_path = it->path();
         if (std::filesystem::is_directory(dir_path))
         {
+            /// Created by DistributedSink
             const auto & tmp_path = dir_path / "tmp";
-
-            /// "tmp" created by DistributedSink
             if (std::filesystem::is_directory(tmp_path) && std::filesystem::is_empty(tmp_path))
                 std::filesystem::remove(tmp_path);
 
+            const auto & broken_path = dir_path / "broken";
+            if (std::filesystem::is_directory(broken_path) && std::filesystem::is_empty(broken_path))
+                std::filesystem::remove(broken_path);
+
             if (std::filesystem::is_empty(dir_path))
             {
                 LOG_DEBUG(log, "Removing {} (used for async INSERT into Distributed)", dir_path.string());
@@ -1077,7 +1080,8 @@ StorageDistributedDirectoryMonitor& StorageDistributed::requireDirectoryMonitor(
             *this, disk, relative_data_path + name,
             data.connection_pool,
             monitors_blocker,
-            getContext()->getDistributedSchedulePool());
+            getContext()->getDistributedSchedulePool(),
+            /* initialize_from_disk= */ startup);
         return data;
     };
 
diff --git a/tests/queries/0_stateless/01791_dist_INSERT_block_structure_mismatch.reference b/tests/queries/0_stateless/01791_dist_INSERT_block_structure_mismatch.reference
index f3be69d3279..b0d8284faa5 100644
--- a/tests/queries/0_stateless/01791_dist_INSERT_block_structure_mismatch.reference
+++ b/tests/queries/0_stateless/01791_dist_INSERT_block_structure_mismatch.reference
@@ -1,7 +1,7 @@
 <Warning> DistributedSink: Structure does not match (remote: n Int8 Int8(size = 0), local: n UInt64 UInt64(size = 1)), implicit conversion will be done.
 <Warning> DistributedSink: Structure does not match (remote: n Int8 Int8(size = 0), local: n UInt64 UInt64(size = 1)), implicit conversion will be done.
-<Warning> default.dist_01683.DirectoryMonitor: Structure does not match (remote: n Int8 Int8(size = 0), local: n UInt64 UInt64(size = 0)), implicit conversion will be done
-<Warning> default.dist_01683.DirectoryMonitor: Structure does not match (remote: n Int8 Int8(size = 0), local: n UInt64 UInt64(size = 0)), implicit conversion will be done
+<Warning> default.dist_01683.DirectoryMonitor.default: Structure does not match (remote: n Int8 Int8(size = 0), local: n UInt64 UInt64(size = 0)), implicit conversion will be done
+<Warning> default.dist_01683.DirectoryMonitor.default: Structure does not match (remote: n Int8 Int8(size = 0), local: n UInt64 UInt64(size = 0)), implicit conversion will be done
 1
 1
 2

From 750ce1eb48b6fdd8ee93b4be7b85b95f7e042076 Mon Sep 17 00:00:00 2001
From: Bharat Nallan Chakravarthy <bharatnc@gmail.com>
Date: Thu, 5 Jan 2023 22:47:58 -0800
Subject: [PATCH 055/158] review fixes first round

---
 .../QuantileInterpolatedWeighted.h            | 42 ++++++-------------
 1 file changed, 12 insertions(+), 30 deletions(-)

diff --git a/src/AggregateFunctions/QuantileInterpolatedWeighted.h b/src/AggregateFunctions/QuantileInterpolatedWeighted.h
index 6e61cafb21a..a551cf58c72 100644
--- a/src/AggregateFunctions/QuantileInterpolatedWeighted.h
+++ b/src/AggregateFunctions/QuantileInterpolatedWeighted.h
@@ -97,6 +97,8 @@ struct QuantileInterpolatedWeighted
     }
 
 private:
+    using Pair = typename std::pair<UnderlyingType, Float64>;
+
     /// Get the value of the `level` quantile. The level must be between 0 and 1.
     template <typename T>
     T getImpl(Float64 level) const
@@ -108,8 +110,8 @@ private:
 
         /// Maintain a vector of pair of values and weights for easier sorting and for building
         /// a cumulative distribution using the provided weights.
-        using Pair = typename std::pair<UnderlyingType, Float64>;
         std::vector<Pair> value_weight_pairs;
+        value_weight_pairs.reserve(size);
 
         /// Note: weight provided must be a 64-bit integer
         /// Float64 is used as accumulator here to get approximate results.
@@ -123,7 +125,7 @@ private:
             sum_weight += pair.getMapped();
             auto value = pair.getKey();
             auto weight = pair.getMapped();
-            value_weight_pairs.push_back(std::make_pair(value, weight));
+            value_weight_pairs.push_back({value, weight});
         }
 
         ::sort(value_weight_pairs.begin(), value_weight_pairs.end(), [](const Pair & a, const Pair & b) { return a.first < b.first; });
@@ -133,21 +135,12 @@ private:
         /// vector for populating and storing the cumulative sum using the provided weights.
         /// example: [0,1,2,3,4,5] -> [0,1,3,6,10,15]
         std::vector<Float64> weights_cum_sum;
+        weights_cum_sum.reserve(size);
 
-        bool first = true;
         for (size_t idx = 0; idx < size; ++idx)
         {
             accumulated += value_weight_pairs[idx].second;
-
-            if (first)
-            {
-                weights_cum_sum.push_back(value_weight_pairs[idx].second);
-                first = false;
-            }
-            else
-            {
-                weights_cum_sum.push_back(accumulated);
-            }
+            weights_cum_sum.push_back(accumulated);
         }
 
         /// The following estimation of quantile is general and the idea is:
@@ -204,8 +197,8 @@ private:
             return;
         }
 
-        using Pair = typename std::pair<UnderlyingType, Float64>;
         std::vector<Pair> value_weight_pairs;
+        value_weight_pairs.reserve(size);
 
         Float64 sum_weight = 0;
         for (const auto & pair : map)
@@ -213,7 +206,7 @@ private:
             sum_weight += pair.getMapped();
             auto value = pair.getKey();
             auto weight = pair.getMapped();
-            value_weight_pairs.push_back(std::make_pair(value, weight));
+            value_weight_pairs.push_back({value, weight});
         }
 
         ::sort(value_weight_pairs.begin(), value_weight_pairs.end(), [](const Pair & a, const Pair & b) { return a.first < b.first; });
@@ -223,23 +216,15 @@ private:
         /// vector for populating and storing the cumulative sum using the provided weights.
         /// example: [0,1,2,3,4,5] -> [0,1,3,6,10,15]
         std::vector<Float64> weights_cum_sum;
+        weights_cum_sum.reserve(size);
 
-        bool first = true;
         for (size_t idx = 0; idx < size; ++idx)
         {
             accumulated += value_weight_pairs[idx].second;
-
-            if (first)
-            {
-                weights_cum_sum.emplace_back(value_weight_pairs[idx].second);
-                first = false;
-            }
-            else
-            {
-                weights_cum_sum.emplace_back(accumulated);
-            }
+            weights_cum_sum.emplace_back(accumulated);
         }
 
+
         /// The following estimation of quantile is general and the idea is:
         /// https://en.wikipedia.org/wiki/Percentile#The_weighted_percentile_method
 
@@ -249,9 +234,7 @@ private:
             value_weight_pairs[idx].second
                 = sum_weight == 0 ? 0 : (weights_cum_sum[idx] - 0.5 * value_weight_pairs[idx].second) / sum_weight;
 
-        size_t level_index = 0;
-
-        while (level_index < num_levels)
+        for (size_t level_index = 0; level_index < num_levels; ++level_index)
         {
             /// perform linear interpolation for every level
             auto level = levels[indices[level_index]];
@@ -282,7 +265,6 @@ private:
                 yl = yr;
 
             result[indices[level_index]] = static_cast<T>(interpolate(level, xl, xr, yl, yr));
-            ++level_index;
         }
     }
 

From fcdbc8123df9bd5ca18499a9f3839585c6ab741b Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Fri, 6 Jan 2023 11:59:54 +0000
Subject: [PATCH 056/158] better loading of outdated parts

---
 src/Storages/MergeTree/MergeTreeData.cpp    | 15 ++++++++++++---
 src/Storages/StorageMergeTree.cpp           |  2 ++
 src/Storages/StorageReplicatedMergeTree.cpp | 10 ++++++++++
 3 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 61b8f398309..8da30c12ee6 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -1086,7 +1086,7 @@ MergeTreeData::LoadPartResult MergeTreeData::loadDataPart(
 
     if (part_disk_ptr->exists(marker_path))
     {
-        /// NOTE: getBytesOnDisk() cannot be used here, since it maybe zero of checksums.txt will not exist.
+        /// NOTE: getBytesOnDisk() cannot be used here, since it may be zero if checksums.txt does not exist.
         res.size_of_part = calculatePartSizeSafe(res.part, log);
         res.is_broken = true;
 
@@ -1695,7 +1695,9 @@ try
 {
     {
         std::lock_guard lock(outdated_data_parts_mutex);
-        LOG_DEBUG(log, "Loading {} outdated data parts", outdated_unloaded_data_parts.size());
+        LOG_DEBUG(log, "Loading {} outdated data parts {}",
+            outdated_unloaded_data_parts.size(),
+            is_async ? "asynchronously" : "synchronously");
     }
 
     size_t num_loaded_parts = 0;
@@ -1709,7 +1711,7 @@ try
             if (is_async && outdated_data_parts_loading_canceled)
             {
                 LOG_DEBUG(log,
-                    "Stopped loading outdated parts because task was canceled. "
+                    "Stopped loading outdated data parts because task was canceled. "
                     "Loaded {} parts, {} left unloaded", num_loaded_parts, outdated_unloaded_data_parts.size());
                 return;
             }
@@ -3586,6 +3588,13 @@ DataPartsVector MergeTreeData::grabActivePartsToRemoveForDropRange(
 MergeTreeData::PartsToRemoveFromZooKeeper MergeTreeData::removePartsInRangeFromWorkingSetAndGetPartsToRemoveFromZooKeeper(
         MergeTreeTransaction * txn, const MergeTreePartInfo & drop_range, DataPartsLock & lock)
 {
+#ifndef NDEBUG
+    {
+        /// All parts (including outdated) must be loaded at this moment.
+        std::lock_guard lock(outdated_data_parts_mutex);
+        assert(outdated_unloaded_data_parts.empty());
+    }
+#endif
 
     auto parts_to_remove = grabActivePartsToRemoveForDropRange(txn, drop_range, lock);
 
diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp
index 0b907e25da1..4cd6e3c6f0b 100644
--- a/src/Storages/StorageMergeTree.cpp
+++ b/src/Storages/StorageMergeTree.cpp
@@ -171,6 +171,8 @@ void StorageMergeTree::shutdown()
     if (shutdown_called.exchange(true))
         return;
 
+    stopOutdatedDataPartsLoadingTask();
+
     /// Unlock all waiting mutations
     {
         std::lock_guard lock(mutation_wait_mutex);
diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index af4f4cc1cb5..137ce34e865 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -1873,6 +1873,11 @@ MutableDataPartStoragePtr StorageReplicatedMergeTree::executeFetchShared(
 void StorageReplicatedMergeTree::executeDropRange(const LogEntry & entry)
 {
     LOG_TRACE(log, "Executing DROP_RANGE {}", entry.new_part_name);
+
+    /// Wait for loading of outdated parts because DROP_RANGE
+    /// command must be applied to all parts on disk.
+    waitForOutdatedPartsToBeLoaded();
+
     auto drop_range_info = MergeTreePartInfo::fromPartName(entry.new_part_name, format_version);
     getContext()->getMergeList().cancelInPartition(getStorageID(), drop_range_info.partition_id, drop_range_info.max_block);
     queue.removePartProducingOpsInRange(getZooKeeper(), drop_range_info, entry, /* fetch_entry_znode= */ {});
@@ -1937,6 +1942,11 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry)
     LOG_DEBUG(log, "Executing log entry {} to replace parts range {} with {} parts from {}.{}",
               entry.znode_name, entry_replace.drop_range_part_name, entry_replace.new_part_names.size(),
               entry_replace.from_database, entry_replace.from_table);
+
+    /// Wait for loading of outdated parts because REPLACE_RANGE
+    /// command must be applied to all parts on disk.
+    waitForOutdatedPartsToBeLoaded();
+
     auto metadata_snapshot = getInMemoryMetadataPtr();
     auto storage_settings_ptr = getSettings();
 

From 3211f63911f5266817dc061c71f1cf8b9db33032 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 7 Jan 2023 01:18:17 +0100
Subject: [PATCH 057/158] Add dmesg.log to Fuzzer

---
 docker/test/fuzzer/run-fuzzer.sh | 1 +
 tests/ci/ast_fuzzer_check.py     | 1 +
 2 files changed, 2 insertions(+)

diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh
index 164f2e28d76..7e25ae4cf0f 100755
--- a/docker/test/fuzzer/run-fuzzer.sh
+++ b/docker/test/fuzzer/run-fuzzer.sh
@@ -349,6 +349,7 @@ if [ -f core.gz ]; then
 fi
 
 grep --text -F '<Fatal>' server.log > fatal.log ||:
+dmesg -T > dmesg.log ||:
 
 pigz server.log
 
diff --git a/tests/ci/ast_fuzzer_check.py b/tests/ci/ast_fuzzer_check.py
index 096edeed149..ab715c72d02 100644
--- a/tests/ci/ast_fuzzer_check.py
+++ b/tests/ci/ast_fuzzer_check.py
@@ -119,6 +119,7 @@ if __name__ == "__main__":
         "fuzzer.log": os.path.join(workspace_path, "fuzzer.log"),
         "report.html": os.path.join(workspace_path, "report.html"),
         "core.gz": os.path.join(workspace_path, "core.gz"),
+        "dmesg.log": os.path.join(workspace_path, "dmesg.log"),
     }
 
     s3_helper = S3Helper()

From 1e5d6e44a5ca3388555ea64689360eed26c979c0 Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Sun, 8 Jan 2023 23:06:35 +0000
Subject: [PATCH 058/158] fix deadlock at shutdown

---
 src/Storages/MergeTree/MergeTreeData.cpp    | 14 +++++++++++---
 src/Storages/StorageReplicatedMergeTree.cpp |  7 +------
 2 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 8da30c12ee6..9bffa9d1322 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -1679,7 +1679,11 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
     if (!unloaded_parts.empty())
     {
         LOG_DEBUG(log, "Found {} outdated data parts. They will be loaded asynchronously", unloaded_parts.size());
-        outdated_unloaded_data_parts = std::move(unloaded_parts);
+
+        {
+            std::lock_guard lock(outdated_data_parts_mutex);
+            outdated_unloaded_data_parts = std::move(unloaded_parts);
+        }
 
         outdated_data_parts_loading_task = getContext()->getSchedulePool().createTask(
             "MergeTreeData::loadOutdatedDataParts",
@@ -1695,6 +1699,9 @@ try
 {
     {
         std::lock_guard lock(outdated_data_parts_mutex);
+        if (outdated_unloaded_data_parts.empty())
+            return;
+
         LOG_DEBUG(log, "Loading {} outdated data parts {}",
             outdated_unloaded_data_parts.size(),
             is_async ? "asynchronously" : "synchronously");
@@ -1734,7 +1741,8 @@ try
             preparePartForRemoval(res.part);
     }
 
-    LOG_DEBUG(log, "Loaded {} outdated data parts", num_loaded_parts);
+    LOG_DEBUG(log, "Loaded {} outdated data parts {}",
+        num_loaded_parts, is_async ? "asynchronously" : "synchronously");
     outdated_data_parts_cv.notify_all();
 }
 catch (...)
@@ -3591,7 +3599,7 @@ MergeTreeData::PartsToRemoveFromZooKeeper MergeTreeData::removePartsInRangeFromW
 #ifndef NDEBUG
     {
         /// All parts (including outdated) must be loaded at this moment.
-        std::lock_guard lock(outdated_data_parts_mutex);
+        std::lock_guard outdated_parts_lock(outdated_data_parts_mutex);
         assert(outdated_unloaded_data_parts.empty());
     }
 #endif
diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index 137ce34e865..7cdf40af6d5 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -910,12 +910,7 @@ void StorageReplicatedMergeTree::drop()
     {
         /// Load remaining parts synchronously because task
         /// for loading is already cancelled in shutdown().
-        std::lock_guard lock(outdated_data_parts_mutex);
-        if (!outdated_unloaded_data_parts.empty())
-        {
-            assert(outdated_data_parts_loading_canceled);
-            loadOutdatedDataParts(/*is_async=*/ false);
-        }
+        loadOutdatedDataParts(/*is_async=*/ false);
     }
 
     dropAllData();

From b9bf92fa5d096ca06146952de2af27fdf615cc1b Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Tue, 10 Jan 2023 11:55:08 +0000
Subject: [PATCH 059/158] check for network errors while loading parts

---
 src/Storages/MergeTree/MergeTreeData.cpp | 38 ++++++++++++++++++++----
 src/Storages/MergeTree/MergeTreeData.h   |  4 +--
 2 files changed, 34 insertions(+), 8 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 9bffa9d1322..2470b45bed7 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -18,6 +18,7 @@
 #include <Functions/IFunction.h>
 #include <IO/Operators.h>
 #include <IO/WriteBufferFromString.h>
+#include <IO/S3Common.h>
 #include <Interpreters/Aggregator.h>
 #include <Interpreters/ExpressionAnalyzer.h>
 #include <Interpreters/PartLog.h>
@@ -157,6 +158,8 @@ namespace ErrorCodes
     extern const int ZERO_COPY_REPLICATION_ERROR;
     extern const int NOT_INITIALIZED;
     extern const int SERIALIZATION_ERROR;
+    extern const int NETWORK_ERROR;
+    extern const int SOCKET_TIMEOUT;
 }
 
 
@@ -1066,6 +1069,25 @@ static void preparePartForRemoval(const MergeTreeMutableDataPartPtr & part)
     }
 }
 
+static bool isRetryableException(const Exception & e)
+{
+    if (isNotEnoughMemoryErrorCode(e.code()))
+        return true;
+
+    if (e.code() == ErrorCodes::NETWORK_ERROR || e.code() == ErrorCodes::SOCKET_TIMEOUT)
+        return true;
+
+#if USE_AWS_S3
+    const auto * s3_exception = dynamic_cast<const S3Exception *>(&e);
+    if (s3_exception && s3_exception->isRetryableError())
+        return true;
+#endif
+
+    /// In fact, there can be other similar situations.
+    /// But it is OK, because there is a safety guard against deleting too many parts.
+    return false;
+}
+
 MergeTreeData::LoadPartResult MergeTreeData::loadDataPart(
     const MergeTreePartInfo & part_info,
     const String & part_name,
@@ -1106,10 +1128,9 @@ MergeTreeData::LoadPartResult MergeTreeData::loadDataPart(
     }
     catch (const Exception & e)
     {
-        /// Don't count the part as broken if there is not enough memory to load it.
-        /// In fact, there can be many similar situations.
-        /// But it is OK, because there is a safety guard against deleting too many parts.
-        if (isNotEnoughMemoryErrorCode(e.code()))
+        /// Don't count the part as broken if there was a retryalbe error
+        /// during loading, such as "not enough memory" or network error.
+        if (isRetryableException(e))
             throw;
 
         res.is_broken = true;
@@ -1753,7 +1774,8 @@ catch (...)
     std::terminate();
 }
 
-void MergeTreeData::waitForOutdatedPartsToBeLoaded() const
+/// No TSA because of std::unique_lock and std::condition_variable.
+void MergeTreeData::waitForOutdatedPartsToBeLoaded() const TSA_NO_THREAD_SAFETY_ANALYSIS
 {
     /// Background tasks are not run if storage is static.
     if (isStaticStorage())
@@ -1765,7 +1787,11 @@ void MergeTreeData::waitForOutdatedPartsToBeLoaded() const
 
     LOG_TRACE(log, "Will wait for outdated data parts to be loaded");
 
-    outdated_data_parts_cv.wait(lock, [this] { return outdated_unloaded_data_parts.empty() || outdated_data_parts_loading_canceled; });
+    outdated_data_parts_cv.wait(lock, [this]() TSA_NO_THREAD_SAFETY_ANALYSIS
+    {
+        return outdated_unloaded_data_parts.empty() || outdated_data_parts_loading_canceled;
+    });
+
     if (outdated_data_parts_loading_canceled)
         throw Exception(ErrorCodes::NOT_INITIALIZED, "Loading of outdated data parts was canceled");
 
diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h
index ddf1e523eca..09a757c4419 100644
--- a/src/Storages/MergeTree/MergeTreeData.h
+++ b/src/Storages/MergeTree/MergeTreeData.h
@@ -1404,8 +1404,8 @@ protected:
     mutable std::condition_variable outdated_data_parts_cv;
 
     BackgroundSchedulePool::TaskHolder outdated_data_parts_loading_task;
-    PartLoadingTreeNodes outdated_unloaded_data_parts;
-    bool outdated_data_parts_loading_canceled = false;
+    PartLoadingTreeNodes outdated_unloaded_data_parts TSA_GUARDED_BY(outdated_data_parts_mutex);
+    bool outdated_data_parts_loading_canceled TSA_GUARDED_BY(outdated_data_parts_mutex) = false;
 
     void loadOutdatedDataParts(bool is_async);
     void startOutdatedDataPartsLoadingTask();

From c0b12d51bdbf4ecd31b24de83107dfc8417ffcd0 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Tue, 27 Dec 2022 15:28:31 +0000
Subject: [PATCH 060/158] Function viewExplain accept SELECT and settings

---
 src/Parsers/ASTExplainQuery.h                 | 31 ++++++++++
 src/Parsers/ExpressionElementParsers.cpp      | 38 ++++++++++++-
 src/TableFunctions/TableFunctionExplain.cpp   | 57 ++++++++++++++-----
 src/TableFunctions/TableFunctionExplain.h     |  2 +-
 .../02421_explain_subquery.reference          |  6 ++
 .../0_stateless/02421_explain_subquery.sql    | 26 +++++++++
 6 files changed, 144 insertions(+), 16 deletions(-)

diff --git a/src/Parsers/ASTExplainQuery.h b/src/Parsers/ASTExplainQuery.h
index 156ffdeacb9..2b991d415a9 100644
--- a/src/Parsers/ASTExplainQuery.h
+++ b/src/Parsers/ASTExplainQuery.h
@@ -6,6 +6,10 @@
 namespace DB
 {
 
+namespace ErrorCodes
+{
+    extern const int LOGICAL_ERROR;
+}
 
 /// AST, EXPLAIN or other query with meaning of explanation query instead of execution
 class ASTExplainQuery : public ASTQueryWithOutput
@@ -104,6 +108,11 @@ private:
     ASTPtr table_function;
     ASTPtr table_override;
 
+    /// To convert `SELECT * FROM (EXPLAIN PIPELINE header = 1 SELECT 123)` to `SELECT * FROM viewExplain("EXPLAIN PIPELINE", "header = 1", SELECT 123)`
+    /// access to methods toString/fromStringToKind is required, but we don't want to expose them to the public.
+    friend class ParserSubquery;
+    friend class TableFunctionExplain;
+
     static String toString(ExplainKind kind)
     {
         switch (kind)
@@ -120,6 +129,28 @@ private:
 
         UNREACHABLE();
     }
+
+    static ExplainKind fromStringToKind(const String & str)
+    {
+        if (str == "EXPLAIN AST")
+            return ParsedAST;
+        if (str == "EXPLAIN SYNTAX")
+            return AnalyzedSyntax;
+        if (str == "EXPLAIN QUERY TREE")
+            return QueryTree;
+        if (str == "EXPLAIN" || str == "EXPLAIN PLAN")
+            return QueryPlan;
+        if (str == "EXPLAIN PIPELINE")
+            return QueryPipeline;
+        if (str == "EXPLAIN ESTIMATE")
+            return QueryEstimates;
+        if (str == "EXPLAIN TABLE OVERRIDE")
+            return TableOverride;
+        if (str == "EXPLAIN CURRENT TRANSACTION")
+            return CurrentTransaction;
+
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown explain kind '{}'", str);
+    }
 };
 
 }
diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp
index 5951128c285..1273595c8d3 100644
--- a/src/Parsers/ExpressionElementParsers.cpp
+++ b/src/Parsers/ExpressionElementParsers.cpp
@@ -28,6 +28,8 @@
 #include <Parsers/ASTWindowDefinition.h>
 #include <Parsers/ASTAssignment.h>
 #include <Parsers/ASTColumnsMatcher.h>
+#include <Parsers/ASTExplainQuery.h>
+#include <Parsers/ASTSetQuery.h>
 #include <Parsers/ASTSelectQuery.h>
 #include <Parsers/ASTTablesInSelectQuery.h>
 #include <Parsers/ExpressionListParsers.h>
@@ -116,8 +118,40 @@ bool ParserSubquery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
     }
     else if (ASTPtr explain_node; explain.parse(pos, explain_node, expected))
     {
-        /// Replace SELECT * FROM (EXPLAIN SELECT ...) with SELECT * FROM viewExplain(EXPLAIN SELECT ...)
-        result_node = buildSelectFromTableFunction(makeASTFunction("viewExplain", explain_node));
+        const auto & explain_query = explain_node->as<const ASTExplainQuery &>();
+
+        if (explain_query.getTableFunction() || explain_query.getTableOverride())
+            throw Exception("EXPLAIN in a subquery cannot have a table function or table override", ErrorCodes::BAD_ARGUMENTS);
+
+        /// Replace `SELECT * FROM (EXPLAIN <kind> <explain_settings> SELECT ...)`
+        /// with `SELECT * FROM viewExplain("<kind>", "<explain_settings>", SELECT ...)`
+
+        String kind_str = ASTExplainQuery::toString(explain_query.getKind());
+
+        String settings_str;
+        if (ASTPtr settings_ast = explain_query.getSettings())
+        {
+            if (!settings_ast->as<ASTSetQuery>())
+                throw Exception("EXPLAIN settings must be a SET query", ErrorCodes::BAD_ARGUMENTS);
+            settings_str = queryToString(settings_ast);
+        }
+
+        const ASTPtr & explained_ast = explain_query.getExplainedQuery();
+        if (explained_ast)
+        {
+            auto view_explain = makeASTFunction("viewExplain",
+                std::make_shared<ASTLiteral>(kind_str),
+                std::make_shared<ASTLiteral>(settings_str),
+                explained_ast);
+            result_node = buildSelectFromTableFunction(std::move(view_explain));
+        }
+        else
+        {
+            auto view_explain = makeASTFunction("viewExplain",
+                std::make_shared<ASTLiteral>(kind_str),
+                std::make_shared<ASTLiteral>(settings_str));
+            result_node = buildSelectFromTableFunction(std::move(view_explain));
+        }
     }
     else
     {
diff --git a/src/TableFunctions/TableFunctionExplain.cpp b/src/TableFunctions/TableFunctionExplain.cpp
index 02b9308ed22..ee859935c34 100644
--- a/src/TableFunctions/TableFunctionExplain.cpp
+++ b/src/TableFunctions/TableFunctionExplain.cpp
@@ -1,6 +1,9 @@
 #include <Interpreters/InterpreterSelectWithUnionQuery.h>
 #include <Parsers/ASTFunction.h>
 #include <Parsers/ASTSelectWithUnionQuery.h>
+#include <Parsers/ASTSetQuery.h>
+#include <Parsers/ParserSetQuery.h>
+#include <Parsers/parseQuery.h>
 #include <Parsers/queryToString.h>
 #include <Storages/StorageValues.h>
 #include <TableFunctions/ITableFunction.h>
@@ -20,21 +23,49 @@ namespace ErrorCodes
 void TableFunctionExplain::parseArguments(const ASTPtr & ast_function, ContextPtr /*context*/)
 {
     const auto * function = ast_function->as<ASTFunction>();
-    if (function && function->arguments && function->arguments->children.size() == 1)
-    {
-        const auto & query_arg = function->arguments->children[0];
-
-        if (!query_arg->as<ASTExplainQuery>())
-            throw Exception(ErrorCodes::BAD_ARGUMENTS,
-                "Table function '{}' requires a explain query argument, got '{}'",
-                getName(), queryToString(query_arg));
-
-        query = query_arg;
-    }
-    else
-    {
+    if (!function || !function->arguments)
         throw Exception(ErrorCodes::BAD_ARGUMENTS,
             "Table function '{}' cannot be called directly, use `SELECT * FROM (EXPLAIN ...)` syntax", getName());
+
+    size_t num_args = function->arguments->children.size();
+    if (num_args != 2 && num_args != 3)
+        throw Exception(ErrorCodes::BAD_ARGUMENTS,
+            "Table function '{}' requires 2 or 3 arguments, got {}", getName(), num_args);
+
+    const auto & kind_arg = function->arguments->children[0];
+    const auto * kind_literal = kind_arg->as<ASTLiteral>();
+    if (!kind_literal || kind_literal->value.getType() != Field::Types::String)
+        throw Exception(ErrorCodes::BAD_ARGUMENTS,
+            "Table function '{}' requires a String argument for EXPLAIN kind, got '{}'",
+            getName(), queryToString(kind_arg));
+
+    ASTExplainQuery::ExplainKind kind = ASTExplainQuery::fromStringToKind(kind_literal->value.get<String>());
+    query = std::make_shared<ASTExplainQuery>(kind);
+
+    const auto * settings_arg = function->arguments->children[1]->as<ASTLiteral>();
+    if (!settings_arg || settings_arg->value.getType() != Field::Types::String)
+        throw Exception(ErrorCodes::BAD_ARGUMENTS,
+            "Table function '{}' requires a serialized string settings argument, got '{}'",
+            getName(), queryToString(function->arguments->children[1]));
+
+    ParserSetQuery settings_parser(true);
+    const auto & settings_str = settings_arg->value.get<String>();
+    if (!settings_str.empty())
+    {
+        constexpr UInt64 max_size = 4096;
+        constexpr UInt64 max_depth = 16;
+        ASTPtr settings_ast = parseQuery(settings_parser, settings_str, max_size, max_depth);
+        query->setSettings(std::move(settings_ast));
+    }
+
+    if (function->arguments->children.size() > 2)
+    {
+        const auto & query_arg = function->arguments->children[2];
+        if (!query_arg->as<ASTSelectWithUnionQuery>())
+            throw Exception(ErrorCodes::BAD_ARGUMENTS,
+                "Table function '{}' requires a EXPLAIN SELECT query argument, got EXPLAIN '{}'",
+                getName(), queryToString(query_arg));
+        query->setExplainedQuery(query_arg);
     }
 }
 
diff --git a/src/TableFunctions/TableFunctionExplain.h b/src/TableFunctions/TableFunctionExplain.h
index 9d6dde4760c..4bd771acbfd 100644
--- a/src/TableFunctions/TableFunctionExplain.h
+++ b/src/TableFunctions/TableFunctionExplain.h
@@ -24,7 +24,7 @@ private:
 
     InterpreterExplainQuery getInterpreter(ContextPtr context) const;
 
-    ASTPtr query = nullptr;
+    std::shared_ptr<ASTExplainQuery> query = nullptr;
 };
 
 
diff --git a/tests/queries/0_stateless/02421_explain_subquery.reference b/tests/queries/0_stateless/02421_explain_subquery.reference
index c18b4e9b082..23751ef6c1f 100644
--- a/tests/queries/0_stateless/02421_explain_subquery.reference
+++ b/tests/queries/0_stateless/02421_explain_subquery.reference
@@ -6,3 +6,9 @@
 1
 1
 1
+1
+1
+1
+1
+1
+1
diff --git a/tests/queries/0_stateless/02421_explain_subquery.sql b/tests/queries/0_stateless/02421_explain_subquery.sql
index af80e51bca3..b99fb1d21ed 100644
--- a/tests/queries/0_stateless/02421_explain_subquery.sql
+++ b/tests/queries/0_stateless/02421_explain_subquery.sql
@@ -1,3 +1,5 @@
+SET allow_experimental_analyzer = 0;
+
 SELECT count() > 3 FROM (EXPLAIN PIPELINE header = 1 SELECT * FROM system.numbers ORDER BY number DESC) WHERE explain LIKE '%Header: number UInt64%';
 SELECT count() > 0 FROM (EXPLAIN PLAN SELECT * FROM system.numbers ORDER BY number DESC) WHERE explain ILIKE '%Sort%';
 SELECT count() > 0 FROM (EXPLAIN SELECT * FROM system.numbers ORDER BY number DESC) WHERE explain ILIKE '%Sort%';
@@ -21,3 +23,27 @@ SELECT rows > 1000 FROM (EXPLAIN ESTIMATE SELECT sum(a) FROM t1);
 SELECT count() == 1 FROM (EXPLAIN ESTIMATE SELECT sum(a) FROM t1);
 
 DROP TABLE IF EXISTS t1;
+
+SET allow_experimental_analyzer = 1;
+
+SELECT count() > 3 FROM (EXPLAIN PIPELINE header = 1 SELECT * FROM system.numbers ORDER BY number DESC) WHERE explain LIKE '%Header: system.numbers.number__ UInt64%';
+SELECT count() > 0 FROM (EXPLAIN PLAN SELECT * FROM system.numbers ORDER BY number DESC) WHERE explain ILIKE '%Sort%';
+SELECT count() > 0 FROM (EXPLAIN SELECT * FROM system.numbers ORDER BY number DESC) WHERE explain ILIKE '%Sort%';
+SELECT count() > 0 FROM (EXPLAIN CURRENT TRANSACTION);
+SELECT count() == 1 FROM (EXPLAIN SYNTAX SELECT number FROM system.numbers ORDER BY number DESC) WHERE explain ILIKE 'SELECT%';
+
+-- Note: here we have `Identifier number` instead of `Asterisk` because it's resolved in analyzer.
+-- Without subsquery it still would be `Asterisk`, because in that case analyzer it's not used by InterpreterExplainQuery.
+SELECT trim(explain) == 'Identifier number' FROM (EXPLAIN AST SELECT * FROM system.numbers LIMIT 10) WHERE explain LIKE '%Identifier number%';
+
+SELECT * FROM (
+    EXPLAIN AST SELECT * FROM (
+        EXPLAIN PLAN SELECT * FROM (
+            EXPLAIN SYNTAX SELECT trim(explain) == 'Asterisk' FROM (
+                EXPLAIN AST SELECT * FROM system.numbers LIMIT 10
+            ) WHERE explain LIKE '%Asterisk%'
+        )
+    )
+) FORMAT Null;
+
+-- EXPLAIN ESTIMATE is not supported in experimental analyzer

From 705c1ed232f99047e9d1e1f2739118c3cfd0f0a9 Mon Sep 17 00:00:00 2001
From: Vladimir C <vdimir@clickhouse.com>
Date: Thu, 29 Dec 2022 11:19:11 +0100
Subject: [PATCH 061/158] Fix clang-tidy

---
 src/Parsers/ExpressionElementParsers.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp
index 1273595c8d3..b102d5c4a14 100644
--- a/src/Parsers/ExpressionElementParsers.cpp
+++ b/src/Parsers/ExpressionElementParsers.cpp
@@ -143,14 +143,14 @@ bool ParserSubquery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
                 std::make_shared<ASTLiteral>(kind_str),
                 std::make_shared<ASTLiteral>(settings_str),
                 explained_ast);
-            result_node = buildSelectFromTableFunction(std::move(view_explain));
+            result_node = buildSelectFromTableFunction(view_explain);
         }
         else
         {
             auto view_explain = makeASTFunction("viewExplain",
                 std::make_shared<ASTLiteral>(kind_str),
                 std::make_shared<ASTLiteral>(settings_str));
-            result_node = buildSelectFromTableFunction(std::move(view_explain));
+            result_node = buildSelectFromTableFunction(view_explain);
         }
     }
     else

From e18a95719da242c58b6f0a9a0484191dcf4b831f Mon Sep 17 00:00:00 2001
From: Bharat Nallan Chakravarthy <bharatnc@gmail.com>
Date: Mon, 9 Jan 2023 21:31:51 -0800
Subject: [PATCH 062/158] review fix

---
 .../reference/quantiles.md                    |  2 +-
 .../QuantileInterpolatedWeighted.h            | 25 ++++++++++---------
 2 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantiles.md b/docs/en/sql-reference/aggregate-functions/reference/quantiles.md
index 5c9120fb8f4..57151915336 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/quantiles.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/quantiles.md
@@ -9,7 +9,7 @@ sidebar_position: 201
 
 Syntax: `quantiles(level1, level2, …)(x)`
 
-All the quantile functions also have corresponding quantiles functions: `quantiles`, `quantilesDeterministic`, `quantilesTiming`, `quantilesTimingWeighted`, `quantilesExact`, `quantilesExactWeighted`, `quantilesTDigest`, `quantilesBFloat16`. These functions calculate all the quantiles of the listed levels in one pass, and return an array of the resulting values.
+All the quantile functions also have corresponding quantiles functions: `quantiles`, `quantilesDeterministic`, `quantilesTiming`, `quantilesTimingWeighted`, `quantilesExact`, `quantilesExactWeighted`, `quantileInterpolatedWeighted`, `quantilesTDigest`, `quantilesBFloat16`. These functions calculate all the quantiles of the listed levels in one pass, and return an array of the resulting values.
 
 ## quantilesExactExclusive
 
diff --git a/src/AggregateFunctions/QuantileInterpolatedWeighted.h b/src/AggregateFunctions/QuantileInterpolatedWeighted.h
index a551cf58c72..4baf1b721ce 100644
--- a/src/AggregateFunctions/QuantileInterpolatedWeighted.h
+++ b/src/AggregateFunctions/QuantileInterpolatedWeighted.h
@@ -147,11 +147,11 @@ private:
         /// https://en.wikipedia.org/wiki/Percentile#The_weighted_percentile_method
 
         /// calculates a simple cumulative distribution based on weights
-        /// Note: if sum_weight is 0 then subsequent division is inf, so set the value of the exp to 0.
-        for (size_t idx = 0; idx < size; ++idx)
-            value_weight_pairs[idx].second
-                = sum_weight == 0 ? 0 : (weights_cum_sum[idx] - 0.5 * value_weight_pairs[idx].second) / sum_weight;
-
+        if (sum_weight != 0)
+        {
+            for (size_t idx = 0; idx < size; ++idx)
+                value_weight_pairs[idx].second = (weights_cum_sum[idx] - 0.5 * value_weight_pairs[idx].second) / sum_weight;
+        }
 
         /// perform linear interpolation
         size_t idx = 0;
@@ -229,10 +229,11 @@ private:
         /// https://en.wikipedia.org/wiki/Percentile#The_weighted_percentile_method
 
         /// calculates a simple cumulative distribution based on weights
-        /// Note: if sum_weight is 0 then subsequent division is inf, so set the value of the exp to 0.
-        for (size_t idx = 0; idx < size; ++idx)
-            value_weight_pairs[idx].second
-                = sum_weight == 0 ? 0 : (weights_cum_sum[idx] - 0.5 * value_weight_pairs[idx].second) / sum_weight;
+        if (sum_weight != 0)
+        {
+            for (size_t idx = 0; idx < size; ++idx)
+                value_weight_pairs[idx].second = (weights_cum_sum[idx] - 0.5 * value_weight_pairs[idx].second) / sum_weight;
+        }
 
         for (size_t level_index = 0; level_index < num_levels; ++level_index)
         {
@@ -272,12 +273,12 @@ private:
     /// results since `the quantileInterpolatedWeighted` function itself relies mainly on approximation.
     UnderlyingType NO_SANITIZE_UNDEFINED interpolate(Float64 level, Float64 xl, Float64 xr, UnderlyingType yl, UnderlyingType yr) const
     {
-        UnderlyingType dy = common::subIgnoreOverflow(yr, yl);
-        Float64 dx = common::subIgnoreOverflow(xr, xl);
+        UnderlyingType dy = yr - yl;
+        Float64 dx = xr - xl;
         dx = dx == 0 ? 1 : dx; /// to handle NaN behavior that might arise during integer division below.
 
         /// yl + (dy / dx) * (level - xl)
-        return static_cast<UnderlyingType>(yl + (dy/dx) * (level - xl));
+        return static_cast<UnderlyingType>(yl + (dy / dx) * (level - xl));
     }
 };
 

From 24d3db1295535e26f91480bd868cf263330b736c Mon Sep 17 00:00:00 2001
From: Bharat Nallan Chakravarthy <bharatnc@gmail.com>
Date: Tue, 10 Jan 2023 20:43:43 -0800
Subject: [PATCH 063/158] try binary search when selecting idx

---
 .../QuantileInterpolatedWeighted.h            | 33 ++++++++++++++++---
 1 file changed, 28 insertions(+), 5 deletions(-)

diff --git a/src/AggregateFunctions/QuantileInterpolatedWeighted.h b/src/AggregateFunctions/QuantileInterpolatedWeighted.h
index 4baf1b721ce..95daeed2e57 100644
--- a/src/AggregateFunctions/QuantileInterpolatedWeighted.h
+++ b/src/AggregateFunctions/QuantileInterpolatedWeighted.h
@@ -155,7 +155,6 @@ private:
 
         /// perform linear interpolation
         size_t idx = 0;
-
         if (size >= 2)
         {
             if (level >= value_weight_pairs[size - 2].second)
@@ -164,8 +163,20 @@ private:
             }
             else
             {
-                while (level > value_weight_pairs[idx + 1].second)
-                    idx++;
+                size_t start = 0, end = size - 1;
+                while (start <= end)
+                {
+                    size_t mid = start + (end - start) / 2;
+                    if (mid > size)
+                        break;
+                    if (level > value_weight_pairs[mid + 1].second)
+                        start = mid + 1;
+                    else
+                    {
+                        idx = mid;
+                        end = mid - 1;
+                    }
+                }
             }
         }
 
@@ -249,8 +260,20 @@ private:
                 }
                 else
                 {
-                    while (level > value_weight_pairs[idx + 1].second)
-                        idx++;
+                    size_t start = 0, end = size - 1;
+                    while (start <= end)
+                    {
+                        size_t mid = start + (end - start) / 2;
+                        if (mid > size)
+                            break;
+                        if (level > value_weight_pairs[mid + 1].second)
+                            start = mid + 1;
+                        else
+                        {
+                            idx = mid;
+                            end = mid - 1;
+                        }
+                    }
                 }
             }
 

From babef5d99098c3eafc937d37026e1fdd5f2b4e36 Mon Sep 17 00:00:00 2001
From: simpleton <63032616+rgzntrade@users.noreply.github.com>
Date: Wed, 11 Jan 2023 16:08:22 +0800
Subject: [PATCH 064/158] fix: Solve the unique invalid problem of negative
 interval

---
 src/AggregateFunctions/AggregateFunctionHistogram.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/AggregateFunctions/AggregateFunctionHistogram.h b/src/AggregateFunctions/AggregateFunctionHistogram.h
index c559b3f115f..ac81f7466fa 100644
--- a/src/AggregateFunctions/AggregateFunctionHistogram.h
+++ b/src/AggregateFunctions/AggregateFunctionHistogram.h
@@ -207,7 +207,7 @@ private:
         {
             // Fuse points if their text representations differ only in last digit
             auto min_diff = 10 * (points[left].mean + points[right].mean) * std::numeric_limits<Mean>::epsilon();
-            if (points[left].mean + min_diff >= points[right].mean)
+            if (points[left].mean + std::fabs(min_diff) >= points[right].mean)
             {
                 points[left] = points[left] + points[right];
             }

From 85734bb321db4561e0b4e871535954588bb96c27 Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Wed, 11 Jan 2023 08:32:33 +0000
Subject: [PATCH 065/158] fix build

---
 src/Storages/MergeTree/MergeTreeData.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 1f1f487210c..aa735b42ba8 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -2203,7 +2203,7 @@ void MergeTreeData::clearPartsFromFilesystemImpl(const DataPartsVector & parts_t
             log, "Removing {} parts from filesystem (serially): Parts: [{}]", parts_to_remove.size(), fmt::join(parts_to_remove, ", "));
         for (const DataPartPtr & part : parts_to_remove)
         {
-            preparePartForRemoval(part)->remove();
+            asMutableDeletingPart(part)->remove();
             if (part_names_succeed)
                 part_names_succeed->insert(part->name);
         }

From 82271d6c4bff3b6bcdf09638227a012213d1cbba Mon Sep 17 00:00:00 2001
From: Maksim Kita <kitaetoya@gmail.com>
Date: Wed, 7 Dec 2022 16:05:06 +0100
Subject: [PATCH 066/158] Analyzer SumIfToCountIfPass crash fix

---
 src/Analyzer/Passes/SumIfToCountIfPass.cpp                  | 6 +++---
 .../02497_analyzer_sum_if_count_if_pass_crash_fix.reference | 1 +
 .../02497_analyzer_sum_if_count_if_pass_crash_fix.sql       | 4 ++++
 3 files changed, 8 insertions(+), 3 deletions(-)
 create mode 100644 tests/queries/0_stateless/02497_analyzer_sum_if_count_if_pass_crash_fix.reference
 create mode 100644 tests/queries/0_stateless/02497_analyzer_sum_if_count_if_pass_crash_fix.sql

diff --git a/src/Analyzer/Passes/SumIfToCountIfPass.cpp b/src/Analyzer/Passes/SumIfToCountIfPass.cpp
index 879eb4d4a8d..27717fccd78 100644
--- a/src/Analyzer/Passes/SumIfToCountIfPass.cpp
+++ b/src/Analyzer/Passes/SumIfToCountIfPass.cpp
@@ -77,7 +77,7 @@ public:
         if (!nested_function || nested_function->getFunctionName() != "if")
             return;
 
-        auto & nested_if_function_arguments_nodes = nested_function->getArguments().getNodes();
+        const auto & nested_if_function_arguments_nodes = nested_function->getArguments().getNodes();
         if (nested_if_function_arguments_nodes.size() != 3)
             return;
 
@@ -101,7 +101,7 @@ public:
         /// Rewrite `sum(if(cond, 1, 0))` into `countIf(cond)`.
         if (if_true_condition_value == 1 && if_false_condition_value == 0)
         {
-            function_node_arguments_nodes[0] = std::move(nested_if_function_arguments_nodes[0]);
+            function_node_arguments_nodes[0] = nested_if_function_arguments_nodes[0];
             function_node_arguments_nodes.resize(1);
 
             resolveAsCountIfAggregateFunction(*function_node, function_node_arguments_nodes[0]->getResultType());
@@ -120,7 +120,7 @@ public:
             auto not_function = std::make_shared<FunctionNode>("not");
 
             auto & not_function_arguments = not_function->getArguments().getNodes();
-            not_function_arguments.push_back(std::move(nested_if_function_arguments_nodes[0]));
+            not_function_arguments.push_back(nested_if_function_arguments_nodes[0]);
 
             not_function->resolveAsFunction(FunctionFactory::instance().get("not", context)->build(not_function->getArgumentColumns()));
 
diff --git a/tests/queries/0_stateless/02497_analyzer_sum_if_count_if_pass_crash_fix.reference b/tests/queries/0_stateless/02497_analyzer_sum_if_count_if_pass_crash_fix.reference
new file mode 100644
index 00000000000..cf534567c6f
--- /dev/null
+++ b/tests/queries/0_stateless/02497_analyzer_sum_if_count_if_pass_crash_fix.reference
@@ -0,0 +1 @@
+50	50	50	1	0
diff --git a/tests/queries/0_stateless/02497_analyzer_sum_if_count_if_pass_crash_fix.sql b/tests/queries/0_stateless/02497_analyzer_sum_if_count_if_pass_crash_fix.sql
new file mode 100644
index 00000000000..51522565014
--- /dev/null
+++ b/tests/queries/0_stateless/02497_analyzer_sum_if_count_if_pass_crash_fix.sql
@@ -0,0 +1,4 @@
+SET allow_experimental_analyzer = 1;
+SET optimize_rewrite_sum_if_to_count_if = 1;
+
+SELECT sum(if((number % 2) = 0 AS cond_expr, 1 AS one_expr, 0 AS zero_expr) AS if_expr), sum(cond_expr), sum(if_expr), one_expr, zero_expr FROM numbers(100);

From eac7a07f3f8c0076a119c27ef2301326024e964a Mon Sep 17 00:00:00 2001
From: Maksim Kita <kitaetoya@gmail.com>
Date: Wed, 7 Dec 2022 16:15:51 +0100
Subject: [PATCH 067/158] Analyzer AggregateFunctionsArithmeticOperationsPass
 fix

---
 ...egateFunctionsArithmericOperationsPass.cpp | 26 +++++++++++--------
 ...s_arithmetic_operations_pass_fix.reference |  1 +
 ...nctions_arithmetic_operations_pass_fix.sql | 14 ++++++++++
 3 files changed, 30 insertions(+), 11 deletions(-)
 create mode 100644 tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.reference
 create mode 100644 tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.sql

diff --git a/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.cpp b/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.cpp
index 01072e0b3fc..f1566f9639b 100644
--- a/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.cpp
+++ b/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.cpp
@@ -73,7 +73,7 @@ public:
         if (!inner_function_node)
             return;
 
-        auto & inner_function_arguments_nodes = inner_function_node->getArguments().getNodes();
+        const auto & inner_function_arguments_nodes = inner_function_node->getArguments().getNodes();
         if (inner_function_arguments_nodes.size() != 2)
             return;
 
@@ -121,11 +121,13 @@ public:
             }
             resolveAggregateFunctionNode(*aggregate_function_node, inner_function_arguments_nodes[1], lower_function_name);
 
-            auto inner_function = aggregate_function_arguments_nodes[0];
-            auto inner_function_right_argument = std::move(inner_function_arguments_nodes[1]);
-            aggregate_function_arguments_nodes = {inner_function_right_argument};
-            inner_function_arguments_nodes[1] = node;
-            node = std::move(inner_function);
+            auto inner_function_clone = inner_function_node->clone();
+            auto & inner_function_clone_arguments = inner_function_clone->as<FunctionNode &>().getArguments();
+            auto & inner_function_clone_arguments_nodes = inner_function_clone_arguments.getNodes();
+            auto inner_function_clone_right_argument = inner_function_clone_arguments_nodes[1];
+            aggregate_function_arguments_nodes = {inner_function_clone_right_argument};
+            inner_function_clone_arguments_nodes[1] = node;
+            node = std::move(inner_function_clone);
         }
         else if (right_argument_constant_node)
         {
@@ -138,11 +140,13 @@ public:
             }
             resolveAggregateFunctionNode(*aggregate_function_node, inner_function_arguments_nodes[0], function_name_if_constant_is_negative);
 
-            auto inner_function = aggregate_function_arguments_nodes[0];
-            auto inner_function_left_argument = std::move(inner_function_arguments_nodes[0]);
-            aggregate_function_arguments_nodes = {inner_function_left_argument};
-            inner_function_arguments_nodes[0] = node;
-            node = std::move(inner_function);
+            auto inner_function_clone = inner_function_node->clone();
+            auto & inner_function_clone_arguments = inner_function_clone->as<FunctionNode &>().getArguments();
+            auto & inner_function_clone_arguments_nodes = inner_function_clone_arguments.getNodes();
+            auto inner_function_clone_left_argument = inner_function_clone_arguments_nodes[0];
+            aggregate_function_arguments_nodes = {inner_function_clone_left_argument};
+            inner_function_clone_arguments_nodes[0] = node;
+            node = std::move(inner_function_clone);
         }
     }
 
diff --git a/tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.reference b/tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.reference
new file mode 100644
index 00000000000..4f9430ef608
--- /dev/null
+++ b/tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.reference
@@ -0,0 +1 @@
+4	2
diff --git a/tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.sql b/tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.sql
new file mode 100644
index 00000000000..e3e508e17be
--- /dev/null
+++ b/tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.sql
@@ -0,0 +1,14 @@
+SET allow_experimental_analyzer = 1;
+SET optimize_arithmetic_operations_in_aggregate_functions = 1;
+
+DROP TABLE IF EXISTS test_table;
+CREATE TABLE test_table
+(
+    id UInt64,
+    value UInt64
+) ENGINE=MergeTree ORDER BY id;
+
+INSERT INTO test_table VALUES (1, 1);
+INSERT INTO test_table VALUES (1, 1);
+
+SELECT sum((2 * id) as func), func FROM test_table GROUP BY id;

From f1947b94def69a99960a13becf10a269f0798327 Mon Sep 17 00:00:00 2001
From: Maksim Kita <kitaetoya@gmail.com>
Date: Wed, 11 Jan 2023 11:10:06 +0100
Subject: [PATCH 068/158] Fixed tests

---
 .../Passes/AggregateFunctionsArithmericOperationsPass.cpp   | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.cpp b/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.cpp
index f1566f9639b..33ecf549363 100644
--- a/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.cpp
+++ b/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.cpp
@@ -119,13 +119,13 @@ public:
             {
                 lower_function_name = function_name_if_constant_is_negative;
             }
-            resolveAggregateFunctionNode(*aggregate_function_node, inner_function_arguments_nodes[1], lower_function_name);
 
             auto inner_function_clone = inner_function_node->clone();
             auto & inner_function_clone_arguments = inner_function_clone->as<FunctionNode &>().getArguments();
             auto & inner_function_clone_arguments_nodes = inner_function_clone_arguments.getNodes();
             auto inner_function_clone_right_argument = inner_function_clone_arguments_nodes[1];
             aggregate_function_arguments_nodes = {inner_function_clone_right_argument};
+            resolveAggregateFunctionNode(*aggregate_function_node, inner_function_clone_right_argument, lower_function_name);
             inner_function_clone_arguments_nodes[1] = node;
             node = std::move(inner_function_clone);
         }
@@ -138,20 +138,20 @@ public:
             {
                 lower_function_name = function_name_if_constant_is_negative;
             }
-            resolveAggregateFunctionNode(*aggregate_function_node, inner_function_arguments_nodes[0], function_name_if_constant_is_negative);
 
             auto inner_function_clone = inner_function_node->clone();
             auto & inner_function_clone_arguments = inner_function_clone->as<FunctionNode &>().getArguments();
             auto & inner_function_clone_arguments_nodes = inner_function_clone_arguments.getNodes();
             auto inner_function_clone_left_argument = inner_function_clone_arguments_nodes[0];
             aggregate_function_arguments_nodes = {inner_function_clone_left_argument};
+            resolveAggregateFunctionNode(*aggregate_function_node, inner_function_clone_left_argument, function_name_if_constant_is_negative);
             inner_function_clone_arguments_nodes[0] = node;
             node = std::move(inner_function_clone);
         }
     }
 
 private:
-    static inline void resolveAggregateFunctionNode(FunctionNode & function_node, QueryTreeNodePtr & argument, const String & aggregate_function_name)
+    static inline void resolveAggregateFunctionNode(FunctionNode & function_node, const QueryTreeNodePtr & argument, const String & aggregate_function_name)
     {
         auto function_aggregate_function = function_node.getAggregateFunction();
 

From 8b65ed200c44b01172b633af7414cfe20e604531 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Tue, 10 Jan 2023 12:08:57 +0000
Subject: [PATCH 069/158] Review fixes

---
 src/Parsers/ASTExplainQuery.h                 | 85 +++++++++----------
 src/TableFunctions/TableFunctionExplain.cpp   | 14 +--
 src/TableFunctions/TableFunctionExplain.h     |  2 +-
 .../0_stateless/02421_explain_subquery.sql    | 26 +++++-
 4 files changed, 74 insertions(+), 53 deletions(-)

diff --git a/src/Parsers/ASTExplainQuery.h b/src/Parsers/ASTExplainQuery.h
index 2b991d415a9..cb8b3199c81 100644
--- a/src/Parsers/ASTExplainQuery.h
+++ b/src/Parsers/ASTExplainQuery.h
@@ -8,7 +8,7 @@ namespace DB
 
 namespace ErrorCodes
 {
-    extern const int LOGICAL_ERROR;
+    extern const int BAD_ARGUMENTS;
 }
 
 /// AST, EXPLAIN or other query with meaning of explanation query instead of execution
@@ -27,6 +27,45 @@ public:
         CurrentTransaction, /// 'EXPLAIN CURRENT TRANSACTION'
     };
 
+    static String toString(ExplainKind kind)
+    {
+        switch (kind)
+        {
+            case ParsedAST: return "EXPLAIN AST";
+            case AnalyzedSyntax: return "EXPLAIN SYNTAX";
+            case QueryTree: return "EXPLAIN QUERY TREE";
+            case QueryPlan: return "EXPLAIN";
+            case QueryPipeline: return "EXPLAIN PIPELINE";
+            case QueryEstimates: return "EXPLAIN ESTIMATE";
+            case TableOverride: return "EXPLAIN TABLE OVERRIDE";
+            case CurrentTransaction: return "EXPLAIN CURRENT TRANSACTION";
+        }
+
+        UNREACHABLE();
+    }
+
+    static ExplainKind fromString(const String & str)
+    {
+        if (str == "EXPLAIN AST")
+            return ParsedAST;
+        if (str == "EXPLAIN SYNTAX")
+            return AnalyzedSyntax;
+        if (str == "EXPLAIN QUERY TREE")
+            return QueryTree;
+        if (str == "EXPLAIN" || str == "EXPLAIN PLAN")
+            return QueryPlan;
+        if (str == "EXPLAIN PIPELINE")
+            return QueryPipeline;
+        if (str == "EXPLAIN ESTIMATE")
+            return QueryEstimates;
+        if (str == "EXPLAIN TABLE OVERRIDE")
+            return TableOverride;
+        if (str == "EXPLAIN CURRENT TRANSACTION")
+            return CurrentTransaction;
+
+        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown explain kind '{}'", str);
+    }
+
     explicit ASTExplainQuery(ExplainKind kind_) : kind(kind_) {}
 
     String getID(char delim) const override { return "Explain" + (delim + toString(kind)); }
@@ -107,50 +146,6 @@ private:
     /// Used by EXPLAIN TABLE OVERRIDE
     ASTPtr table_function;
     ASTPtr table_override;
-
-    /// To convert `SELECT * FROM (EXPLAIN PIPELINE header = 1 SELECT 123)` to `SELECT * FROM viewExplain("EXPLAIN PIPELINE", "header = 1", SELECT 123)`
-    /// access to methods toString/fromStringToKind is required, but we don't want to expose them to the public.
-    friend class ParserSubquery;
-    friend class TableFunctionExplain;
-
-    static String toString(ExplainKind kind)
-    {
-        switch (kind)
-        {
-            case ParsedAST: return "EXPLAIN AST";
-            case AnalyzedSyntax: return "EXPLAIN SYNTAX";
-            case QueryTree: return "EXPLAIN QUERY TREE";
-            case QueryPlan: return "EXPLAIN";
-            case QueryPipeline: return "EXPLAIN PIPELINE";
-            case QueryEstimates: return "EXPLAIN ESTIMATE";
-            case TableOverride: return "EXPLAIN TABLE OVERRIDE";
-            case CurrentTransaction: return "EXPLAIN CURRENT TRANSACTION";
-        }
-
-        UNREACHABLE();
-    }
-
-    static ExplainKind fromStringToKind(const String & str)
-    {
-        if (str == "EXPLAIN AST")
-            return ParsedAST;
-        if (str == "EXPLAIN SYNTAX")
-            return AnalyzedSyntax;
-        if (str == "EXPLAIN QUERY TREE")
-            return QueryTree;
-        if (str == "EXPLAIN" || str == "EXPLAIN PLAN")
-            return QueryPlan;
-        if (str == "EXPLAIN PIPELINE")
-            return QueryPipeline;
-        if (str == "EXPLAIN ESTIMATE")
-            return QueryEstimates;
-        if (str == "EXPLAIN TABLE OVERRIDE")
-            return TableOverride;
-        if (str == "EXPLAIN CURRENT TRANSACTION")
-            return CurrentTransaction;
-
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown explain kind '{}'", str);
-    }
 };
 
 }
diff --git a/src/TableFunctions/TableFunctionExplain.cpp b/src/TableFunctions/TableFunctionExplain.cpp
index ee859935c34..88b14531d9b 100644
--- a/src/TableFunctions/TableFunctionExplain.cpp
+++ b/src/TableFunctions/TableFunctionExplain.cpp
@@ -39,8 +39,8 @@ void TableFunctionExplain::parseArguments(const ASTPtr & ast_function, ContextPt
             "Table function '{}' requires a String argument for EXPLAIN kind, got '{}'",
             getName(), queryToString(kind_arg));
 
-    ASTExplainQuery::ExplainKind kind = ASTExplainQuery::fromStringToKind(kind_literal->value.get<String>());
-    query = std::make_shared<ASTExplainQuery>(kind);
+    ASTExplainQuery::ExplainKind kind = ASTExplainQuery::fromString(kind_literal->value.get<String>());
+    auto explain_query = std::make_shared<ASTExplainQuery>(kind);
 
     const auto * settings_arg = function->arguments->children[1]->as<ASTLiteral>();
     if (!settings_arg || settings_arg->value.getType() != Field::Types::String)
@@ -48,14 +48,16 @@ void TableFunctionExplain::parseArguments(const ASTPtr & ast_function, ContextPt
             "Table function '{}' requires a serialized string settings argument, got '{}'",
             getName(), queryToString(function->arguments->children[1]));
 
-    ParserSetQuery settings_parser(true);
     const auto & settings_str = settings_arg->value.get<String>();
     if (!settings_str.empty())
     {
         constexpr UInt64 max_size = 4096;
         constexpr UInt64 max_depth = 16;
+
+        /// parse_only_internals_ = true - we don't want to parse `SET` keyword
+        ParserSetQuery settings_parser(/* parse_only_internals_ = */ true);
         ASTPtr settings_ast = parseQuery(settings_parser, settings_str, max_size, max_depth);
-        query->setSettings(std::move(settings_ast));
+        explain_query->setSettings(std::move(settings_ast));
     }
 
     if (function->arguments->children.size() > 2)
@@ -65,8 +67,10 @@ void TableFunctionExplain::parseArguments(const ASTPtr & ast_function, ContextPt
             throw Exception(ErrorCodes::BAD_ARGUMENTS,
                 "Table function '{}' requires a EXPLAIN SELECT query argument, got EXPLAIN '{}'",
                 getName(), queryToString(query_arg));
-        query->setExplainedQuery(query_arg);
+        explain_query->setExplainedQuery(query_arg);
     }
+
+    query = std::move(explain_query);
 }
 
 ColumnsDescription TableFunctionExplain::getActualTableStructure(ContextPtr context) const
diff --git a/src/TableFunctions/TableFunctionExplain.h b/src/TableFunctions/TableFunctionExplain.h
index 4bd771acbfd..9d6dde4760c 100644
--- a/src/TableFunctions/TableFunctionExplain.h
+++ b/src/TableFunctions/TableFunctionExplain.h
@@ -24,7 +24,7 @@ private:
 
     InterpreterExplainQuery getInterpreter(ContextPtr context) const;
 
-    std::shared_ptr<ASTExplainQuery> query = nullptr;
+    ASTPtr query = nullptr;
 };
 
 
diff --git a/tests/queries/0_stateless/02421_explain_subquery.sql b/tests/queries/0_stateless/02421_explain_subquery.sql
index b99fb1d21ed..72c3a3bfedd 100644
--- a/tests/queries/0_stateless/02421_explain_subquery.sql
+++ b/tests/queries/0_stateless/02421_explain_subquery.sql
@@ -32,8 +32,30 @@ SELECT count() > 0 FROM (EXPLAIN SELECT * FROM system.numbers ORDER BY number DE
 SELECT count() > 0 FROM (EXPLAIN CURRENT TRANSACTION);
 SELECT count() == 1 FROM (EXPLAIN SYNTAX SELECT number FROM system.numbers ORDER BY number DESC) WHERE explain ILIKE 'SELECT%';
 
--- Note: here we have `Identifier number` instead of `Asterisk` because it's resolved in analyzer.
--- Without subsquery it still would be `Asterisk`, because in that case analyzer it's not used by InterpreterExplainQuery.
+-- We have `Identifier number` instead of `Asterisk` because query argument of `viewExplain` table function was analyzed.
+-- Compare:
+-- :) EXPLAIN AST SELECT *;
+-- ┌─explain───────────────────────────�
+-- │ SelectWithUnionQuery (children 1) │
+-- │  ExpressionList (children 1)      │
+-- │   SelectQuery (children 1)        │
+-- │    ExpressionList (children 1)    │
+-- │     Asterisk                      │
+-- └───────────────────────────────────┘
+-- :) SELECT * FROM (EXPLAIN AST SELECT *);
+-- ┌─explain─────────────────────────────────────�
+-- │ SelectWithUnionQuery (children 1)           │
+-- │  ExpressionList (children 1)                │
+-- │   SelectQuery (children 2)                  │
+-- │    ExpressionList (children 1)              │
+-- │     Identifier dummy                        │
+-- │    TablesInSelectQuery (children 1)         │
+-- │     TablesInSelectQueryElement (children 1) │
+-- │      TableExpression (children 1)           │
+-- │       TableIdentifier system.one            │
+-- └─────────────────────────────────────────────┘
+-- TODO: argument of `viewExplain` (and subquery in `EXAPLAN ...`) should not be analyzed.
+-- See _Support query tree in table functions_ in https://github.com/ClickHouse/ClickHouse/issues/42648
 SELECT trim(explain) == 'Identifier number' FROM (EXPLAIN AST SELECT * FROM system.numbers LIMIT 10) WHERE explain LIKE '%Identifier number%';
 
 SELECT * FROM (

From d17890369e125f39176ec531d9c4d33fe70c51c6 Mon Sep 17 00:00:00 2001
From: Dmitry Novik <n0vik@clickhouse.com>
Date: Wed, 11 Jan 2023 19:09:58 +0000
Subject: [PATCH 070/158] Skip lambdas during validation

---
 src/Analyzer/QueryTreePassManager.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/Analyzer/QueryTreePassManager.cpp b/src/Analyzer/QueryTreePassManager.cpp
index 99a695b0769..4ea0d257509 100644
--- a/src/Analyzer/QueryTreePassManager.cpp
+++ b/src/Analyzer/QueryTreePassManager.cpp
@@ -24,6 +24,7 @@
 #include <Analyzer/FunctionNode.h>
 #include <Analyzer/InDepthQueryTreeVisitor.h>
 #include <Common/Exception.h>
+#include <DataTypes/IDataType.h>
 
 namespace DB
 {
@@ -74,6 +75,9 @@ class ValidationChecker : public InDepthQueryTreeVisitor<ValidationChecker>
 
         for (size_t i = 0; i < expected_arg_types.size(); ++i)
         {
+            // Skip lambdas
+            if(WhichDataType(expected_arg_types[i]).isFunction())
+                continue;
             if (!expected_arg_types[i]->equals(*actual_arg_columns[i].type))
                 throw Exception(ErrorCodes::LOGICAL_ERROR,
                 "Function {} expects {} argument to have {} type but receives {} after running {} pass",

From 7112d4ec83a116a166e74a72ca04706a41c49c90 Mon Sep 17 00:00:00 2001
From: Dmitry Novik <n0vik@clickhouse.com>
Date: Wed, 11 Jan 2023 19:21:07 +0000
Subject: [PATCH 071/158] Improve exception message

---
 src/Analyzer/FunctionNode.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Analyzer/FunctionNode.cpp b/src/Analyzer/FunctionNode.cpp
index d2d340d3730..7961bfbae31 100644
--- a/src/Analyzer/FunctionNode.cpp
+++ b/src/Analyzer/FunctionNode.cpp
@@ -36,7 +36,7 @@ const DataTypes & FunctionNode::getArgumentTypes() const
 {
     if (!function)
         throw Exception(ErrorCodes::LOGICAL_ERROR,
-        "Trying to get expected argument types of function {} when IResolvedFunction is not set",
+        "Function {} is not resolved",
         function_name);
     return function->getArgumentTypes();
 }

From f2cba45f9c43e030853bd655985e532a6bcfdaad Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Thu, 8 Sep 2022 17:16:47 +0200
Subject: [PATCH 072/158] Add a script to merge a PR

---
 tests/ci/merge_pr.py | 197 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 197 insertions(+)
 create mode 100644 tests/ci/merge_pr.py

diff --git a/tests/ci/merge_pr.py b/tests/ci/merge_pr.py
new file mode 100644
index 00000000000..b92ce576d28
--- /dev/null
+++ b/tests/ci/merge_pr.py
@@ -0,0 +1,197 @@
+#!/usr/bin/env python
+
+"""Script to check if PR is mergeable and merge it"""
+
+import argparse
+import logging
+
+from datetime import datetime
+from os import getenv
+from typing import Dict, List
+
+from github.PullRequestReview import PullRequestReview
+
+from commit_status_helper import get_commit_filtered_statuses
+from get_robot_token import get_best_robot_token
+from github_helper import GitHub, NamedUser, PullRequest
+from pr_info import PRInfo
+
+
+# The team name for accepted approvals
+TEAM_NAME = getenv("GITHUB_TEAM_NAME", "core")
+
+
+class Reviews:
+    STATES = ["CHANGES_REQUESTED", "APPROVED"]
+
+    def __init__(self, pr: PullRequest):
+        """The reviews are proceed in the next logic:
+        - if review for an author does not exist, set it
+        - the review status can be changed from CHANGES_REQUESTED and APPROVED
+            only to either one
+        """
+        logging.info("Checking the PR for approvals")
+        self.pr = pr
+        self.reviews = pr.get_reviews()
+        # the reviews are ordered by time
+        self._review_per_user = {}  # type: Dict[NamedUser, PullRequestReview]
+        self.approved_at = datetime.fromtimestamp(0)
+        for r in self.reviews:
+            user = r.user
+            if self._review_per_user.get(user):
+                if r.state in self.STATES:
+                    self._review_per_user[user] = r
+                    if r.state == "APPROVED":
+                        self.approved_at = max(r.submitted_at, self.approved_at)
+                continue
+            self._review_per_user[user] = r
+
+    def is_approved(self, team: List[NamedUser]) -> bool:
+        """Checks if the PR is approved, and no changes made after the last approval"""
+        if not self.reviews:
+            logging.info("There aren't reviews for PR #%s", self.pr.number)
+            return False
+
+        # We consider reviews only from the given list of users
+        statuses = {
+            r.state
+            for user, r in self._review_per_user.items()
+            if r.state == "CHANGES_REQUESTED"
+            or (r.state == "APPROVED" and user in team)
+        }
+
+        if "CHANGES_REQUESTED" in statuses:
+            logging.info(
+                "The following users requested changes for the PR: %s",
+                ", ".join(
+                    user.login
+                    for user, r in self._review_per_user.items()
+                    if r.state == "CHANGES_REQUESTED"
+                ),
+            )
+            return False
+
+        if "APPROVED" in statuses:
+            logging.info(
+                "The following users approved the PR: %s",
+                ", ".join(
+                    user.login
+                    for user, r in self._review_per_user.items()
+                    if r.state == "APPROVED"
+                ),
+            )
+            # The only reliable place to get the 100% accurate last_modified
+            # info is when the commit was pushed to GitHub. The info is
+            # available as a header 'last-modified' of /{org}/{repo}/commits/{sha}.
+            # Unfortunately, it's formatted as 'Wed, 04 Jan 2023 11:05:13 GMT'
+
+            commit = self.pr.head.repo.get_commit(self.pr.head.sha)
+            if commit.stats.last_modified is None:
+                logging.warning(
+                    "Unable to get info about the commit %s", self.pr.head.sha
+                )
+                return False
+
+            last_changed = datetime.strptime(
+                commit.stats.last_modified, "%a, %d %b %Y %H:%M:%S GMT"
+            )
+            if self.approved_at < last_changed:
+                logging.info(
+                    "There are changes after approve at %s",
+                    self.approved_at.isoformat(),
+                )
+                return False
+            return True
+
+        logging.info("The PR #%s is not approved", self.pr.number)
+        return False
+
+
+def parse_args() -> argparse.Namespace:
+    pr_info = PRInfo()
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="Script to merge the given PR. Additional checks for approved "
+        "status and green commit statuses could be done",
+    )
+    parser.add_argument(
+        "--check-approved",
+        action="store_true",
+        help="if set, checks that the PR is approved and no changes required",
+    )
+    parser.add_argument("--check-green", default=True, help=argparse.SUPPRESS)
+    parser.add_argument(
+        "--no-check-green",
+        dest="check_green",
+        action="store_false",
+        default=argparse.SUPPRESS,
+        help="(dangerous) if set, skip check commit to having all green statuses",
+    )
+    parser.add_argument(
+        "--repo",
+        default=pr_info.repo_full_name,
+        help="PR number to check",
+    )
+    parser.add_argument(
+        "--pr",
+        type=int,
+        default=pr_info.number,
+        help="PR number to check",
+    )
+    parser.add_argument(
+        "--token",
+        type=str,
+        default="",
+        help="a token to use for GitHub API requests, will be received from SSM "
+        "if empty",
+    )
+    args = parser.parse_args()
+    args.pr_info = pr_info
+    return args
+
+
+def main():
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
+    args = parse_args()
+    logging.info("Going to process PR #%s in repo %s", args.pr, args.repo)
+    token = args.token or get_best_robot_token()
+    gh = GitHub(token, per_page=100)
+    repo = gh.get_repo(args.repo)
+    pr = repo.get_pull(args.pr)
+    if pr.is_merged():
+        logging.info("The PR #%s is already merged", pr.number)
+        return
+
+    if pr.draft:
+        logging.info("The PR #%s is a draft, stopping", pr.number)
+        return
+
+    if args.check_green:
+        logging.info("Checking that all PR's statuses are green")
+        commit = repo.get_commit(pr.head.sha)
+        failed_statuses = [
+            status.context
+            for status in get_commit_filtered_statuses(commit)
+            if status.state != "success"
+        ]
+        if failed_statuses:
+            logging.warning(
+                "Some statuses aren't success:\n  %s", ",\n  ".join(failed_statuses)
+            )
+            return
+
+    if args.check_approved:
+        reviews = Reviews(pr)
+        org = gh.get_organization(repo.organization.login)
+        team = org.get_team_by_slug(TEAM_NAME)
+        members = list(team.get_members())
+        if not reviews.is_approved(members):
+            logging.warning("We don't merge the PR")
+            return
+
+    logging.info("Merging the PR")
+    pr.merge()
+
+
+if __name__ == "__main__":
+    main()

From 570588a06f3f9303eb3414cc28fb9e9deb50ec26 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Thu, 8 Sep 2022 17:20:58 +0200
Subject: [PATCH 073/158] Add automerge for green backport PRs

---
 .github/workflows/backport_branches.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/backport_branches.yml b/.github/workflows/backport_branches.yml
index cbd3bd7bec4..c52a58eac8a 100644
--- a/.github/workflows/backport_branches.yml
+++ b/.github/workflows/backport_branches.yml
@@ -683,3 +683,4 @@ jobs:
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 finish_check.py
+          python3 merge_pr.py

From e2719de82078e0aa9c64c5563a7abba3f5be9163 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Thu, 8 Sep 2022 17:21:23 +0200
Subject: [PATCH 074/158] Add automerge for green approved PRs

---
 .github/workflows/docs_check.yml   | 1 +
 .github/workflows/pull_request.yml | 1 +
 2 files changed, 2 insertions(+)

diff --git a/.github/workflows/docs_check.yml b/.github/workflows/docs_check.yml
index a513eb9216d..d69020d810e 100644
--- a/.github/workflows/docs_check.yml
+++ b/.github/workflows/docs_check.yml
@@ -169,3 +169,4 @@ jobs:
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 finish_check.py
+          python3 merge_pr.py --check-approved
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index aecf3799a5d..c677ec4bf5c 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -4388,3 +4388,4 @@ jobs:
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 finish_check.py
+          python3 merge_pr.py --check-approved

From 2b9c76875b351acc8fb41f46905d0db633418301 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Thu, 5 Jan 2023 01:27:45 +0100
Subject: [PATCH 075/158] Consider WIP in a title as blocker for automerge

---
 tests/ci/merge_pr.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/ci/merge_pr.py b/tests/ci/merge_pr.py
index b92ce576d28..f037df847f7 100644
--- a/tests/ci/merge_pr.py
+++ b/tests/ci/merge_pr.py
@@ -162,8 +162,9 @@ def main():
         logging.info("The PR #%s is already merged", pr.number)
         return
 
-    if pr.draft:
-        logging.info("The PR #%s is a draft, stopping", pr.number)
+    not_ready_to_merge = pr.draft or "WIP" in pr.title
+    if not_ready_to_merge:
+        logging.info("The PR #%s is not ready for merge, stopping", pr.number)
         return
 
     if args.check_green:

From 98427319e03a5ee73a25447e56c9f6acb2ab14a5 Mon Sep 17 00:00:00 2001
From: Dmitry Novik <n0vik@clickhouse.com>
Date: Wed, 11 Jan 2023 19:54:12 +0000
Subject: [PATCH 076/158] Fix style check

---
 src/Analyzer/QueryTreePassManager.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/Analyzer/QueryTreePassManager.cpp b/src/Analyzer/QueryTreePassManager.cpp
index f4941dca080..ecd3a748da3 100644
--- a/src/Analyzer/QueryTreePassManager.cpp
+++ b/src/Analyzer/QueryTreePassManager.cpp
@@ -76,7 +76,8 @@ class ValidationChecker : public InDepthQueryTreeVisitor<ValidationChecker>
         for (size_t i = 0; i < expected_arg_types.size(); ++i)
         {
             // Skip lambdas
-            if(WhichDataType(expected_arg_types[i]).isFunction())
+            WhichDataType which_type(expected_arg_types[i]);
+            if(which_type.isFunction())
                 continue;
             if (!expected_arg_types[i]->equals(*actual_arg_columns[i].type))
                 throw Exception(ErrorCodes::LOGICAL_ERROR,

From ba3b828ae737105b36a5d1167faa1f51ebfb301a Mon Sep 17 00:00:00 2001
From: Dmitry Novik <n0vik@clickhouse.com>
Date: Wed, 11 Jan 2023 21:07:26 +0000
Subject: [PATCH 077/158] Fix style

---
 src/Analyzer/QueryTreePassManager.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Analyzer/QueryTreePassManager.cpp b/src/Analyzer/QueryTreePassManager.cpp
index ecd3a748da3..6aa9365316b 100644
--- a/src/Analyzer/QueryTreePassManager.cpp
+++ b/src/Analyzer/QueryTreePassManager.cpp
@@ -76,9 +76,9 @@ class ValidationChecker : public InDepthQueryTreeVisitor<ValidationChecker>
         for (size_t i = 0; i < expected_arg_types.size(); ++i)
         {
             // Skip lambdas
-            WhichDataType which_type(expected_arg_types[i]);
-            if(which_type.isFunction())
+            if (WhichDataType(expected_arg_types[i]).isFunction())
                 continue;
+
             if (!expected_arg_types[i]->equals(*actual_arg_columns[i].type))
                 throw Exception(ErrorCodes::LOGICAL_ERROR,
                 "Function {} expects {} argument to have {} type but receives {} after running {} pass",

From e3a1af654293bc564aa8307ba878124de46cb1f3 Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Wed, 11 Jan 2023 15:51:01 +0100
Subject: [PATCH 078/158] Fix flacky test

---
 tests/integration/test_multiple_disks/test.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tests/integration/test_multiple_disks/test.py b/tests/integration/test_multiple_disks/test.py
index 83817ba7a3d..b041959ac21 100644
--- a/tests/integration/test_multiple_disks/test.py
+++ b/tests/integration/test_multiple_disks/test.py
@@ -1625,6 +1625,7 @@ def test_rename(start_cluster):
         """
         )
 
+        # jbod1 disk is 40mb
         for _ in range(5):
             data = []
             for i in range(10):
@@ -1635,7 +1636,13 @@ def test_rename(start_cluster):
                 )
             )
 
-        disks = get_used_disks_for_table(node1, "renaming_table")
+        # data is moved in the background, so check with retries
+        num_try = 0
+        while get_used_disks_for_table(node1, "renaming_table") == 1:
+            time.sleep(1)
+            num_try += 1
+            if num_try == 20:
+                break
         assert len(disks) > 1
         assert node1.query("SELECT COUNT() FROM default.renaming_table") == "50\n"
 

From 2e44ad9d0f5d015aabe71ce8ef181e4e793cc043 Mon Sep 17 00:00:00 2001
From: rfraposa <richraposa@gmail.com>
Date: Wed, 11 Jan 2023 17:10:51 -0700
Subject: [PATCH 079/158] Add maxIntersections to docs

---
 .../aggregate-functions/reference/index.md    |  4 +-
 .../reference/maxintersections.md             | 64 +++++++++++++++++++
 .../reference/maxintersectionsposition.md     | 64 +++++++++++++++++++
 3 files changed, 131 insertions(+), 1 deletion(-)
 create mode 100644 docs/en/sql-reference/aggregate-functions/reference/maxintersections.md
 create mode 100644 docs/en/sql-reference/aggregate-functions/reference/maxintersectionsposition.md

diff --git a/docs/en/sql-reference/aggregate-functions/reference/index.md b/docs/en/sql-reference/aggregate-functions/reference/index.md
index 575141766dd..5d9423e0a55 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/index.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/index.md
@@ -77,4 +77,6 @@ ClickHouse-specific aggregate functions:
 -   [contingency](./contingency.md)
 -   [cramersV](./cramersv.md)
 -   [cramersVBiasCorrected](./cramersvbiascorrected.md)
--   [theilsU](./theilsu.md)
\ No newline at end of file
+-   [theilsU](./theilsu.md)
+-   [maxIntersections](./maxintersections.md)
+-   [maxIntersectionsPosition](./maxintersectionsposition.md)
\ No newline at end of file
diff --git a/docs/en/sql-reference/aggregate-functions/reference/maxintersections.md b/docs/en/sql-reference/aggregate-functions/reference/maxintersections.md
new file mode 100644
index 00000000000..a6776df303e
--- /dev/null
+++ b/docs/en/sql-reference/aggregate-functions/reference/maxintersections.md
@@ -0,0 +1,64 @@
+---
+slug: /en/sql-reference/aggregate-functions/reference/maxintersections
+sidebar_position: 360
+title: maxIntersections
+---
+
+# maxIntersections
+
+Aggregate function that calculates the maximum number of times that a group of intervals intersects each other (if all the intervals intersect at least once).
+
+The syntax is:
+
+```sql
+maxIntersections(start_column, end_column)
+```
+
+**Arguments**
+
+- `start_column` – the numeric column that represents the start of each interval. If `start_column` is `NULL` or 0 then the interval will be skipped.
+
+- `end_column` - the numeric column that represents the end of each interval. If `end_column` is `NULL` or 0 then the interval will be skipped.
+
+**Returned value**
+
+Returns the maximum number of intersected intervals.
+
+**Example**
+
+```sql
+CREATE TABLE my_events (
+    start UInt32,
+    end UInt32
+)
+Engine = MergeTree
+ORDER BY tuple();
+
+INSERT INTO my_events VALUES
+   (1, 3),
+   (1, 6),
+   (2, 5),
+   (3, 7);
+```
+
+The intervals look like the following:
+
+```response
+1 - 3
+1 - - - - 6
+  2 - - 5
+    3 - - - 7
+```
+
+Notice that three of these intervals have the value 4 in common, and that is the most frequent number of intersections:
+
+```sql
+SELECT maxIntersections(start, end) FROM my_events;
+```
+
+Response:
+```response
+3
+```
+
+If you have multiple occurrences of the maximum interval, you can use the [`maxIntersectionsPosition` function](./maxintersectionsposition.md) to locate the numer and location of those occurrences.
\ No newline at end of file
diff --git a/docs/en/sql-reference/aggregate-functions/reference/maxintersectionsposition.md b/docs/en/sql-reference/aggregate-functions/reference/maxintersectionsposition.md
new file mode 100644
index 00000000000..7dd63f09316
--- /dev/null
+++ b/docs/en/sql-reference/aggregate-functions/reference/maxintersectionsposition.md
@@ -0,0 +1,64 @@
+---
+slug: /en/sql-reference/aggregate-functions/reference/maxintersectionsposition
+sidebar_position: 361
+title: maxIntersectionsPosition
+---
+
+# maxIntersectionsPosition
+
+Aggregate function that calculates the positions of the occurrences of the [`maxIntersections` function](./maxintersections.md).
+
+The syntax is:
+
+```sql
+maxIntersectionsPosition(start_column, end_column)
+```
+
+**Arguments**
+
+- `start_column` – the numeric column that represents the start of each interval. If `start_column` is `NULL` or 0 then the interval will be skipped.
+
+- `end_column` - the numeric column that represents the end of each interval. If `end_column` is `NULL` or 0 then the interval will be skipped.
+
+**Returned value**
+
+Returns the start positions of the maximum number of intersected intervals.
+
+**Example**
+
+```sql
+CREATE TABLE my_events (
+    start UInt32,
+    end UInt32
+)
+Engine = MergeTree
+ORDER BY tuple();
+
+INSERT INTO my_events VALUES
+   (1, 3),
+   (1, 6),
+   (2, 5),
+   (3, 7);
+```
+
+The intervals look like the following:
+
+```response
+1 - 3
+1 - - - - 6
+  2 - - 5
+    3 - - - 7
+```
+
+Notice that three of these intervals have the value 4 in common, and that starts with the 2nd interval:
+
+```sql
+SELECT maxIntersectionsPosition(start, end) FROM my_events;
+```
+
+Response:
+```response
+2
+```
+
+In other words, the `(1,6)` row is the start of the 3 intervals that intersect, and 3 is the maximum number of intervals that intersect.
\ No newline at end of file

From 7a651d749ce50315dfeffc5536c557f9b5c3b1d2 Mon Sep 17 00:00:00 2001
From: Dan Roscigno <dan@roscigno.com>
Date: Wed, 11 Jan 2023 19:20:37 -0500
Subject: [PATCH 080/158] Update
 docs/en/sql-reference/aggregate-functions/reference/maxintersections.md

---
 .../aggregate-functions/reference/maxintersections.md           | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/sql-reference/aggregate-functions/reference/maxintersections.md b/docs/en/sql-reference/aggregate-functions/reference/maxintersections.md
index a6776df303e..f33b51f0374 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/maxintersections.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/maxintersections.md
@@ -61,4 +61,4 @@ Response:
 3
 ```
 
-If you have multiple occurrences of the maximum interval, you can use the [`maxIntersectionsPosition` function](./maxintersectionsposition.md) to locate the numer and location of those occurrences.
\ No newline at end of file
+If you have multiple occurrences of the maximum interval, you can use the [`maxIntersectionsPosition` function](./maxintersectionsposition.md) to locate the number and location of those occurrences.
\ No newline at end of file

From 4ffaa8736fa31384ee55a528845ff7a4834411c4 Mon Sep 17 00:00:00 2001
From: simpleton <63032616+rgzntrade@users.noreply.github.com>
Date: Thu, 12 Jan 2023 12:10:48 +0800
Subject: [PATCH 081/158] test: Add the histogram input to repeat the negative
 test

---
 tests/queries/0_stateless/00647_histogram_negative           | 4 ++++
 tests/queries/0_stateless/00647_histogram_negative.reference | 1 +
 2 files changed, 5 insertions(+)
 create mode 100644 tests/queries/0_stateless/00647_histogram_negative
 create mode 100644 tests/queries/0_stateless/00647_histogram_negative.reference

diff --git a/tests/queries/0_stateless/00647_histogram_negative b/tests/queries/0_stateless/00647_histogram_negative
new file mode 100644
index 00000000000..58753acc60b
--- /dev/null
+++ b/tests/queries/0_stateless/00647_histogram_negative
@@ -0,0 +1,4 @@
+create table histogram(num Int64) engine=Tinylog;
+insert into histogram values(-1);
+insert into histogram values(-1);
+select histogram(2)(num) from histogram;
\ No newline at end of file
diff --git a/tests/queries/0_stateless/00647_histogram_negative.reference b/tests/queries/0_stateless/00647_histogram_negative.reference
new file mode 100644
index 00000000000..e7757607a37
--- /dev/null
+++ b/tests/queries/0_stateless/00647_histogram_negative.reference
@@ -0,0 +1 @@
+[(-1,-1,2)]
\ No newline at end of file

From c75c6c229842a0e2162452585f7d7f72250b0d42 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 12 Jan 2023 07:16:10 +0300
Subject: [PATCH 082/158] Update run-fuzzer.sh

---
 docker/test/fuzzer/run-fuzzer.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh
index a158a72bc06..a143c238e3d 100755
--- a/docker/test/fuzzer/run-fuzzer.sh
+++ b/docker/test/fuzzer/run-fuzzer.sh
@@ -397,6 +397,7 @@ p.links a { padding: 5px; margin: 3px; background: #FFF; line-height: 2; white-s
   <a href="fuzzer.log">fuzzer.log</a>
   <a href="server.log.gz">server.log.gz</a>
   <a href="main.log">main.log</a>
+  <a href="dmesg.log">dmesg.log</a>
   ${CORE_LINK}
 </p>
 <table>

From 2f0eabab5441aaec64780c7bd6c70e0843ca4ca0 Mon Sep 17 00:00:00 2001
From: Maksim Kita <kitaetoya@gmail.com>
Date: Thu, 12 Jan 2023 11:30:55 +0100
Subject: [PATCH 083/158] Fixed tests

---
 src/Analyzer/Passes/SumIfToCountIfPass.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Analyzer/Passes/SumIfToCountIfPass.cpp b/src/Analyzer/Passes/SumIfToCountIfPass.cpp
index 27717fccd78..1faf79e87f9 100644
--- a/src/Analyzer/Passes/SumIfToCountIfPass.cpp
+++ b/src/Analyzer/Passes/SumIfToCountIfPass.cpp
@@ -81,7 +81,7 @@ public:
         if (nested_if_function_arguments_nodes.size() != 3)
             return;
 
-        auto & cond_argument = nested_if_function_arguments_nodes[0];
+        const auto & cond_argument = nested_if_function_arguments_nodes[0];
         const auto * if_true_condition_constant_node = nested_if_function_arguments_nodes[1]->as<ConstantNode>();
         const auto * if_false_condition_constant_node = nested_if_function_arguments_nodes[2]->as<ConstantNode>();
 

From 56f8899df89cc186225a475d433305d935252e16 Mon Sep 17 00:00:00 2001
From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com>
Date: Thu, 12 Jan 2023 12:06:31 +0100
Subject: [PATCH 084/158] Update tests/integration/test_multiple_disks/test.py

Co-authored-by: Ilya Yatsishin <2159081+qoega@users.noreply.github.com>
---
 tests/integration/test_multiple_disks/test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/test_multiple_disks/test.py b/tests/integration/test_multiple_disks/test.py
index b041959ac21..77cb9464bcb 100644
--- a/tests/integration/test_multiple_disks/test.py
+++ b/tests/integration/test_multiple_disks/test.py
@@ -1643,7 +1643,7 @@ def test_rename(start_cluster):
             num_try += 1
             if num_try == 20:
                 break
-        assert len(disks) > 1
+        assert len(get_used_disks_for_table(node1, "renaming_table")) > 1
         assert node1.query("SELECT COUNT() FROM default.renaming_table") == "50\n"
 
         node1.query("RENAME TABLE default.renaming_table TO default.renaming_table1")

From ebc88743fdc5555c8f0786fd977603516bc21bed Mon Sep 17 00:00:00 2001
From: Vladimir C <vdimir@clickhouse.com>
Date: Thu, 12 Jan 2023 12:13:29 +0100
Subject: [PATCH 085/158] Add file extension to 00647_histogram_negative

---
 tests/queries/0_stateless/00647_histogram_negative.reference    | 2 +-
 .../{00647_histogram_negative => 00647_histogram_negative.sql}  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename tests/queries/0_stateless/{00647_histogram_negative => 00647_histogram_negative.sql} (72%)

diff --git a/tests/queries/0_stateless/00647_histogram_negative.reference b/tests/queries/0_stateless/00647_histogram_negative.reference
index e7757607a37..c4d6fd93024 100644
--- a/tests/queries/0_stateless/00647_histogram_negative.reference
+++ b/tests/queries/0_stateless/00647_histogram_negative.reference
@@ -1 +1 @@
-[(-1,-1,2)]
\ No newline at end of file
+[(-1,-1,2)]
diff --git a/tests/queries/0_stateless/00647_histogram_negative b/tests/queries/0_stateless/00647_histogram_negative.sql
similarity index 72%
rename from tests/queries/0_stateless/00647_histogram_negative
rename to tests/queries/0_stateless/00647_histogram_negative.sql
index 58753acc60b..5f1b30d1503 100644
--- a/tests/queries/0_stateless/00647_histogram_negative
+++ b/tests/queries/0_stateless/00647_histogram_negative.sql
@@ -1,4 +1,4 @@
 create table histogram(num Int64) engine=Tinylog;
 insert into histogram values(-1);
 insert into histogram values(-1);
-select histogram(2)(num) from histogram;
\ No newline at end of file
+select histogram(2)(num) from histogram;

From 811f9b1e3af8f612e9556d36e61412b876ef9515 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Wed, 11 Jan 2023 13:57:09 +0100
Subject: [PATCH 086/158] Reduce the number of API requests by monkey-patching

---
 tests/ci/merge_pr.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/tests/ci/merge_pr.py b/tests/ci/merge_pr.py
index f037df847f7..946eb77745a 100644
--- a/tests/ci/merge_pr.py
+++ b/tests/ci/merge_pr.py
@@ -157,6 +157,13 @@ def main():
     token = args.token or get_best_robot_token()
     gh = GitHub(token, per_page=100)
     repo = gh.get_repo(args.repo)
+    # An ugly and not nice fix to patch the wrong organization URL,
+    # see https://github.com/PyGithub/PyGithub/issues/2395#issuecomment-1378629710
+    # pylint: disable=protected-access
+    repo.organization._url.value = repo.organization.url.replace(  # type: ignore
+        "/users/", "/orgs/", 1
+    )
+    # pylint: enable=protected-access
     pr = repo.get_pull(args.pr)
     if pr.is_merged():
         logging.info("The PR #%s is already merged", pr.number)
@@ -183,8 +190,7 @@ def main():
 
     if args.check_approved:
         reviews = Reviews(pr)
-        org = gh.get_organization(repo.organization.login)
-        team = org.get_team_by_slug(TEAM_NAME)
+        team = repo.organization.get_team_by_slug(TEAM_NAME)
         members = list(team.get_members())
         if not reviews.is_approved(members):
             logging.warning("We don't merge the PR")

From fc438cd740720bf4a8de8e3603676c52f953161a Mon Sep 17 00:00:00 2001
From: Vladimir C <vdimir@clickhouse.com>
Date: Thu, 12 Jan 2023 13:59:59 +0100
Subject: [PATCH 087/158] Fix typo

---
 tests/queries/0_stateless/00647_histogram_negative.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/00647_histogram_negative.sql b/tests/queries/0_stateless/00647_histogram_negative.sql
index 5f1b30d1503..91b3fb49529 100644
--- a/tests/queries/0_stateless/00647_histogram_negative.sql
+++ b/tests/queries/0_stateless/00647_histogram_negative.sql
@@ -1,4 +1,4 @@
-create table histogram(num Int64) engine=Tinylog;
+create table histogram(num Int64) engine=TinyLog;
 insert into histogram values(-1);
 insert into histogram values(-1);
 select histogram(2)(num) from histogram;

From 69a11574d2606c7b07b51c40f44ed2c265e129eb Mon Sep 17 00:00:00 2001
From: rfraposa <richraposa@gmail.com>
Date: Thu, 12 Jan 2023 08:30:54 -0700
Subject: [PATCH 088/158] Update maxintersections.md

---
 .../aggregate-functions/reference/maxintersections.md           | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/sql-reference/aggregate-functions/reference/maxintersections.md b/docs/en/sql-reference/aggregate-functions/reference/maxintersections.md
index f33b51f0374..6c30110970a 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/maxintersections.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/maxintersections.md
@@ -50,7 +50,7 @@ The intervals look like the following:
     3 - - - 7
 ```
 
-Notice that three of these intervals have the value 4 in common, and that is the most frequent number of intersections:
+Three of these intervals have the value 4 in common, and that is the most frequent number of intersections. (The intervals `(1,3)` and `(3,7)` share an endpoint but are not considered intersecting by the `maxIntersections` function.)
 
 ```sql
 SELECT maxIntersections(start, end) FROM my_events;

From 759a4c0940e99020f340d9a72c21dd803a5eda81 Mon Sep 17 00:00:00 2001
From: Rich Raposa <richraposa@gmail.com>
Date: Thu, 12 Jan 2023 08:53:22 -0700
Subject: [PATCH 089/158] Update
 docs/en/sql-reference/aggregate-functions/reference/maxintersections.md

Co-authored-by: Dan Roscigno <dan@roscigno.com>
---
 .../aggregate-functions/reference/maxintersections.md           | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/sql-reference/aggregate-functions/reference/maxintersections.md b/docs/en/sql-reference/aggregate-functions/reference/maxintersections.md
index 6c30110970a..29864394720 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/maxintersections.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/maxintersections.md
@@ -50,7 +50,7 @@ The intervals look like the following:
     3 - - - 7
 ```
 
-Three of these intervals have the value 4 in common, and that is the most frequent number of intersections. (The intervals `(1,3)` and `(3,7)` share an endpoint but are not considered intersecting by the `maxIntersections` function.)
+Three of these intervals have a common value (the value is `4`, but the value that is common is not important, we are measuring the count of the intersections) . The intervals `(1,3)` and `(3,7)` share an endpoint but are not considered intersecting by the `maxIntersections` function.
 
 ```sql
 SELECT maxIntersections(start, end) FROM my_events;

From 8c94ed95972acc1f4ba25c3f52a0c661278c6ced Mon Sep 17 00:00:00 2001
From: Dan Roscigno <dan@roscigno.com>
Date: Thu, 12 Jan 2023 11:01:03 -0500
Subject: [PATCH 090/158] Update
 docs/en/sql-reference/aggregate-functions/reference/maxintersections.md

---
 .../aggregate-functions/reference/maxintersections.md           | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/sql-reference/aggregate-functions/reference/maxintersections.md b/docs/en/sql-reference/aggregate-functions/reference/maxintersections.md
index 29864394720..db99b900a3e 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/maxintersections.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/maxintersections.md
@@ -50,7 +50,7 @@ The intervals look like the following:
     3 - - - 7
 ```
 
-Three of these intervals have a common value (the value is `4`, but the value that is common is not important, we are measuring the count of the intersections) . The intervals `(1,3)` and `(3,7)` share an endpoint but are not considered intersecting by the `maxIntersections` function.
+Three of these intervals have a common value (the value is `4`, but the value that is common is not important, we are measuring the count of the intersections). The intervals `(1,3)` and `(3,7)` share an endpoint but are not considered intersecting by the `maxIntersections` function.
 
 ```sql
 SELECT maxIntersections(start, end) FROM my_events;

From 87b934c4725dc69345bf4981483e2e08f70ba76d Mon Sep 17 00:00:00 2001
From: avogar <pav.cruglov@yandex.ru>
Date: Thu, 12 Jan 2023 16:36:44 +0000
Subject: [PATCH 091/158] Insert default values in case of missing tuple
 elements in JSONEachRow

---
 docs/en/interfaces/formats.md                 |  4 +-
 docs/en/operations/settings/settings.md       | 15 ++++++-
 src/Core/Settings.h                           |  2 +
 src/Core/SettingsChangesHistory.h             |  3 +-
 .../Serializations/SerializationTuple.cpp     | 39 ++++++++++++++++---
 src/Formats/FormatFactory.cpp                 |  4 +-
 src/Formats/FormatSettings.h                  |  4 +-
 ...son_missing_named_tuple_elements.reference |  6 +++
 ...02532_json_missing_named_tuple_elements.sh | 19 +++++++++
 9 files changed, 85 insertions(+), 11 deletions(-)
 create mode 100644 tests/queries/0_stateless/02532_json_missing_named_tuple_elements.reference
 create mode 100755 tests/queries/0_stateless/02532_json_missing_named_tuple_elements.sh

diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md
index e28c486afca..d384ed639eb 100644
--- a/docs/en/interfaces/formats.md
+++ b/docs/en/interfaces/formats.md
@@ -1203,12 +1203,14 @@ SELECT * FROM json_each_row_nested
 - [input_format_json_read_bools_as_numbers](/docs/en/operations/settings/settings.md/#input_format_json_read_bools_as_numbers) - allow to parse bools as numbers in JSON input formats. Default value - `true`.
 - [input_format_json_read_numbers_as_strings](/docs/en/operations/settings/settings.md/#input_format_json_read_numbers_as_strings) - allow to parse numbers as strings in JSON input formats. Default value - `false`.
 - [input_format_json_read_objects_as_strings](/docs/en/operations/settings/settings.md/#input_format_json_read_objects_as_strings) - allow to parse JSON objects as strings in JSON input formats. Default value - `false`.
+- [input_format_json_named_tuples_as_objects](/docs/en/operations/settings/settings.md/#input_format_json_named_tuples_as_objects) - parse named tuple columns as JSON objects. Default value - `true`.
+- [input_format_json_defaults_for_missing_elements_in_named_tuple](/docs/en/operations/settings/settings.md/#input_format_json_defaults_for_missing_elements_in_named_tuple) - insert default values for missing elements in JSON object while parsing named tuple. Default value - `true`.
 - [output_format_json_quote_64bit_integers](/docs/en/operations/settings/settings.md/#output_format_json_quote_64bit_integers) - controls quoting of 64-bit integers in JSON output format. Default value - `true`.
 - [output_format_json_quote_64bit_floats](/docs/en/operations/settings/settings.md/#output_format_json_quote_64bit_floats) - controls quoting of 64-bit floats in JSON output format. Default value - `false`.
 - [output_format_json_quote_denormals](/docs/en/operations/settings/settings.md/#output_format_json_quote_denormals) - enables '+nan', '-nan', '+inf', '-inf' outputs in JSON output format. Default value - `false`.
 - [output_format_json_quote_decimals](/docs/en/operations/settings/settings.md/#output_format_json_quote_decimals) - controls quoting of decimals in JSON output format. Default value - `false`.
 - [output_format_json_escape_forward_slashes](/docs/en/operations/settings/settings.md/#output_format_json_escape_forward_slashes) - controls escaping forward slashes for string outputs in JSON output format. Default value - `true`.
-- [output_format_json_named_tuples_as_objects](/docs/en/operations/settings/settings.md/#output_format_json_named_tuples_as_objects) - serialize named tuple columns as JSON objects. Default value - `false`.
+- [output_format_json_named_tuples_as_objects](/docs/en/operations/settings/settings.md/#output_format_json_named_tuples_as_objects) - serialize named tuple columns as JSON objects. Default value - `true`.
 - [output_format_json_array_of_rows](/docs/en/operations/settings/settings.md/#output_format_json_array_of_rows) - output a JSON array of all rows in JSONEachRow(Compact) format. Default value - `false`.
 - [output_format_json_validate_utf8](/docs/en/operations/settings/settings.md/#output_format_json_validate_utf8) - enables validation of UTF-8 sequences in JSON output formats (note that it doesn't impact formats JSON/JSONCompact/JSONColumnsWithMetadata, they always validate utf8). Default value - `false`.
 
diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index 4ffe2bbc7c4..53cc9f1e349 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -4122,7 +4122,20 @@ Enabled by default.
 
 Serialize named tuple columns as JSON objects.
 
-Disabled by default.
+Enabled by default.
+
+### input_format_json_named_tuples_as_objects {#input_format_json_named_tuples_as_objects}
+
+Parse named tuple columns as JSON objects.
+
+Enabled by default.
+
+### input_format_json_defaults_for_missing_elements_in_named_tuple {#input_format_json_defaults_for_missing_elements_in_named_tuple}
+
+Insert default values for missing elements in JSON object while parsing named tuple.
+This setting works only when setting `input_format_json_named_tuples_as_objects` is enabled.
+
+Enabled by default.
 
 ### output_format_json_array_of_rows {#output_format_json_array_of_rows}
 
diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index b8d46244b6c..f58bd7ebafb 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -773,6 +773,8 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
     M(Bool, input_format_json_validate_types_from_metadata, true, "For JSON/JSONCompact/JSONColumnsWithMetadata input formats this controls whether format parser should check if data types from input metadata match data types of the corresponding columns from the table", 0) \
     M(Bool, input_format_json_read_numbers_as_strings, false, "Allow to parse numbers as strings in JSON input formats", 0) \
     M(Bool, input_format_json_read_objects_as_strings, true, "Allow to parse JSON objects as strings in JSON input formats", 0) \
+    M(Bool, input_format_json_named_tuples_as_objects, true, "Deserialize named tuple columns as JSON objects", 0) \
+    M(Bool, input_format_json_defaults_for_missing_elements_in_named_tuple, true, "Insert default value in named tuple element if it's missing in json object", 0) \
     M(Bool, input_format_try_infer_integers, true, "Try to infer integers instead of floats while schema inference in text formats", 0) \
     M(Bool, input_format_try_infer_dates, true, "Try to infer dates from string fields while schema inference in text formats", 0) \
     M(Bool, input_format_try_infer_datetimes, true, "Try to infer datetimes from string fields while schema inference in text formats", 0) \
diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h
index d67f1b94d5d..534fcd42037 100644
--- a/src/Core/SettingsChangesHistory.h
+++ b/src/Core/SettingsChangesHistory.h
@@ -80,7 +80,8 @@ namespace SettingsChangesHistory
 /// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972)
 static std::map<ClickHouseVersion, SettingsChangesHistory::SettingsChanges> settings_changes_history =
 {
-    {"23.1", {{"input_format_json_read_objects_as_strings", 0, 1, "Enable reading nested json objects as strings while object type is experimental"}}},
+    {"23.1", {{"input_format_json_read_objects_as_strings", 0, 1, "Enable reading nested json objects as strings while object type is experimental"},
+              {"input_format_json_defaults_for_missing_elements_in_named_tuple", false, true, "Allow missing elements in JSON objects while reading named tuples by default"}}},
     {"22.12", {{"max_size_to_preallocate_for_aggregation", 10'000'000, 100'000'000, "This optimizes performance"},
                {"query_plan_aggregation_in_order", 0, 1, "Enable some refactoring around query plan"},
                {"format_binary_max_string_size", 0, 1_GiB, "Prevent allocating large amount of memory"}}},
diff --git a/src/DataTypes/Serializations/SerializationTuple.cpp b/src/DataTypes/Serializations/SerializationTuple.cpp
index 0ed2b034985..50d956584b9 100644
--- a/src/DataTypes/Serializations/SerializationTuple.cpp
+++ b/src/DataTypes/Serializations/SerializationTuple.cpp
@@ -16,6 +16,7 @@ namespace ErrorCodes
 {
     extern const int SIZES_OF_COLUMNS_IN_TUPLE_DOESNT_MATCH;
     extern const int NOT_FOUND_COLUMN_IN_BLOCK;
+    extern const int INCORRECT_DATA;
 }
 
 
@@ -154,7 +155,7 @@ void SerializationTuple::deserializeText(IColumn & column, ReadBuffer & istr, co
 
 void SerializationTuple::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
-    if (settings.json.named_tuples_as_objects
+    if (settings.json.write_named_tuples_as_objects
         && have_explicit_names)
     {
         writeChar('{', ostr);
@@ -185,7 +186,7 @@ void SerializationTuple::serializeTextJSON(const IColumn & column, size_t row_nu
 
 void SerializationTuple::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
-    if (settings.json.named_tuples_as_objects
+    if (settings.json.read_named_tuples_as_objects
         && have_explicit_names)
     {
         skipWhitespaceIfAny(istr);
@@ -194,12 +195,15 @@ void SerializationTuple::deserializeTextJSON(IColumn & column, ReadBuffer & istr
 
         addElementSafe(elems.size(), column, [&]
         {
-            // Require all elements but in arbitrary order.
-            for (size_t i = 0; i < elems.size(); ++i)
+            std::vector<UInt8> seen_elements(elems.size(), 0);
+            size_t i = 0;
+            while (!istr.eof() && *istr.position() != '}')
             {
+                if (i == elems.size())
+                    throw Exception(ErrorCodes::INCORRECT_DATA, "Unexpected number of elements in named tuple. Expected no more than {}", elems.size());
+
                 if (i > 0)
                 {
-                    skipWhitespaceIfAny(istr);
                     assertChar(',', istr);
                     skipWhitespaceIfAny(istr);
                 }
@@ -211,12 +215,35 @@ void SerializationTuple::deserializeTextJSON(IColumn & column, ReadBuffer & istr
                 skipWhitespaceIfAny(istr);
 
                 const size_t element_pos = getPositionByName(name);
+                seen_elements[element_pos] = 1;
                 auto & element_column = extractElementColumn(column, element_pos);
                 elems[element_pos]->deserializeTextJSON(element_column, istr, settings);
+
+                skipWhitespaceIfAny(istr);
+                ++i;
             }
 
-            skipWhitespaceIfAny(istr);
             assertChar('}', istr);
+
+            /// Check if we have missing elements.
+            if (i != elems.size())
+            {
+                for (size_t element_pos = 0; element_pos != seen_elements.size(); ++element_pos)
+                {
+                    if (seen_elements[element_pos])
+                        continue;
+
+                    if (!settings.json.defaults_for_missing_elements_in_named_tuple)
+                        throw Exception(
+                            ErrorCodes::INCORRECT_DATA,
+                            "JSON object doesn't contain tuple element {}. If you want to insert defaults in case of missing elements, "
+                            "enable setting input_format_json_defaults_for_missing_elements_in_named_tuple",
+                            elems[element_pos]->getElementName());
+
+                    auto & element_column = extractElementColumn(column, element_pos);
+                    element_column.insertDefault();
+                }
+            }
         });
     }
     else
diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp
index dc2f4ffcf55..ed2464f98e8 100644
--- a/src/Formats/FormatFactory.cpp
+++ b/src/Formats/FormatFactory.cpp
@@ -90,7 +90,9 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings)
     format_settings.input_allow_errors_ratio = settings.input_format_allow_errors_ratio;
     format_settings.json.array_of_rows = settings.output_format_json_array_of_rows;
     format_settings.json.escape_forward_slashes = settings.output_format_json_escape_forward_slashes;
-    format_settings.json.named_tuples_as_objects = settings.output_format_json_named_tuples_as_objects;
+    format_settings.json.write_named_tuples_as_objects = settings.output_format_json_named_tuples_as_objects;
+    format_settings.json.read_named_tuples_as_objects = settings.input_format_json_named_tuples_as_objects;
+    format_settings.json.defaults_for_missing_elements_in_named_tuple = settings.input_format_json_defaults_for_missing_elements_in_named_tuple;
     format_settings.json.quote_64bit_integers = settings.output_format_json_quote_64bit_integers;
     format_settings.json.quote_64bit_floats = settings.output_format_json_quote_64bit_floats;
     format_settings.json.quote_denormals = settings.output_format_json_quote_denormals;
diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h
index dcdd44edfeb..9d8680a009d 100644
--- a/src/Formats/FormatSettings.h
+++ b/src/Formats/FormatSettings.h
@@ -153,7 +153,9 @@ struct FormatSettings
         bool quote_denormals = true;
         bool quote_decimals = false;
         bool escape_forward_slashes = true;
-        bool named_tuples_as_objects = false;
+        bool read_named_tuples_as_objects = false;
+        bool write_named_tuples_as_objects = false;
+        bool defaults_for_missing_elements_in_named_tuple = false;
         bool serialize_as_strings = false;
         bool read_bools_as_numbers = true;
         bool read_numbers_as_strings = true;
diff --git a/tests/queries/0_stateless/02532_json_missing_named_tuple_elements.reference b/tests/queries/0_stateless/02532_json_missing_named_tuple_elements.reference
new file mode 100644
index 00000000000..d51dab695f2
--- /dev/null
+++ b/tests/queries/0_stateless/02532_json_missing_named_tuple_elements.reference
@@ -0,0 +1,6 @@
+(1,2,NULL)
+(1,2,NULL)
+(NULL,NULL,NULL)
+1
+1
+1
diff --git a/tests/queries/0_stateless/02532_json_missing_named_tuple_elements.sh b/tests/queries/0_stateless/02532_json_missing_named_tuple_elements.sh
new file mode 100755
index 00000000000..97847b08203
--- /dev/null
+++ b/tests/queries/0_stateless/02532_json_missing_named_tuple_elements.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# Tags: long
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CUR_DIR"/../shell_config.sh
+
+echo '{"t" : {"a" : 1, "b" : 2}}' | $CLICKHOUSE_LOCAL --input-format=NDJSON --structure='t Tuple(a Nullable(UInt32), b Nullable(UInt32), c Nullable(UInt32))' -q "select * from table"
+
+echo '{"t" : { "a" : 1 , "b" : 2 } }' | $CLICKHOUSE_LOCAL --input-format=NDJSON --structure='t Tuple(a Nullable(UInt32), b Nullable(UInt32), c Nullable(UInt32))' -q "select * from table"
+
+echo '{"t" : {}}' | $CLICKHOUSE_LOCAL --input-format=NDJSON --structure='t Tuple(a Nullable(UInt32), b Nullable(UInt32), c Nullable(UInt32))' -q "select * from table"
+
+echo '{"t" : {"a" : 1, "b" : 2}}' | $CLICKHOUSE_LOCAL --input-format=NDJSON --structure='t Tuple(a Nullable(UInt32), b Nullable(UInt32), c Nullable(UInt32))' -q "select * from table" --input_format_json_defaults_for_missing_elements_in_named_tuple=0 2>&1 | grep -F "INCORRECT_DATA" -c
+
+echo '{"t" : {"a" : 1, "d" : 2}}' | $CLICKHOUSE_LOCAL --input-format=NDJSON --structure='t Tuple(a Nullable(UInt32), b Nullable(UInt32), c Nullable(UInt32))' -q "select * from table" 2>&1 | grep -F "NOT_FOUND_COLUMN_IN_BLOCK" -c
+
+echo '{"t" : {"a" : 1, "b" : 2, "c" : 3, "d" : 4}}' | $CLICKHOUSE_LOCAL --input-format=NDJSON --structure='t Tuple(a Nullable(UInt32), b Nullable(UInt32), c Nullable(UInt32))' -q "select * from table" 2>&1 | grep -F "INCORRECT_DATA" -c
+

From 5d6ae2c3d554b78b2575a6cb4b7d7af70bb0c760 Mon Sep 17 00:00:00 2001
From: avogar <pav.cruglov@yandex.ru>
Date: Thu, 12 Jan 2023 18:33:47 +0000
Subject: [PATCH 092/158] Use structure from insertion table in generateRandom
 func

---
 src/TableFunctions/TableFunctionGenerateRandom.cpp |  6 +++++-
 src/TableFunctions/TableFunctionGenerateRandom.h   | 10 +++++++---
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/src/TableFunctions/TableFunctionGenerateRandom.cpp b/src/TableFunctions/TableFunctionGenerateRandom.cpp
index 1ddbb48962d..c545a7c364b 100644
--- a/src/TableFunctions/TableFunctionGenerateRandom.cpp
+++ b/src/TableFunctions/TableFunctionGenerateRandom.cpp
@@ -82,7 +82,11 @@ ColumnsDescription TableFunctionGenerateRandom::getActualTableStructure(ContextP
 
 StoragePtr TableFunctionGenerateRandom::executeImpl(const ASTPtr & /*ast_function*/, ContextPtr context, const std::string & table_name, ColumnsDescription /*cached_columns*/) const
 {
-    auto columns = getActualTableStructure(context);
+    ColumnsDescription columns;
+    if (structure != "auto")
+        columns = parseColumnsListFromString(structure, context);
+    else if (!structure_hint.empty())
+        columns = structure_hint;
     auto res = std::make_shared<StorageGenerateRandom>(
         StorageID(getDatabaseName(), table_name), columns, String{}, max_array_length, max_string_length, random_seed);
     res->startup();
diff --git a/src/TableFunctions/TableFunctionGenerateRandom.h b/src/TableFunctions/TableFunctionGenerateRandom.h
index bcad11156be..bdbc73e12b9 100644
--- a/src/TableFunctions/TableFunctionGenerateRandom.h
+++ b/src/TableFunctions/TableFunctionGenerateRandom.h
@@ -13,7 +13,11 @@ class TableFunctionGenerateRandom : public ITableFunction
 public:
     static constexpr auto name = "generateRandom";
     std::string getName() const override { return name; }
-    bool hasStaticStructure() const override { return true; }
+    bool hasStaticStructure() const override { return structure != "auto"; }
+
+    bool needStructureHint() const override { return structure == "auto"; }
+    void setStructureHint(const ColumnsDescription & structure_hint_) override { structure_hint = structure_hint_; }
+
 private:
     StoragePtr executeImpl(const ASTPtr & ast_function, ContextPtr context, const std::string & table_name, ColumnsDescription cached_columns) const override;
     const char * getStorageTypeName() const override { return "GenerateRandom"; }
@@ -21,11 +25,11 @@ private:
     ColumnsDescription getActualTableStructure(ContextPtr context) const override;
     void parseArguments(const ASTPtr & ast_function, ContextPtr context) override;
 
-    String structure;
+    String structure = "auto";
     UInt64 max_string_length = 10;
     UInt64 max_array_length = 10;
     std::optional<UInt64> random_seed;
-
+    ColumnsDescription structure_hint;
 };
 
 

From 82ff1fd3438451591900d22eac5a8df94e4b7e51 Mon Sep 17 00:00:00 2001
From: avogar <pav.cruglov@yandex.ru>
Date: Thu, 12 Jan 2023 22:29:23 +0000
Subject: [PATCH 093/158] Add tests and docs

---
 docs/en/sql-reference/table-functions/generate.md   | 13 +++++++++++++
 src/TableFunctions/TableFunctionGenerateRandom.cpp  |  4 +---
 src/TableFunctions/TableFunctionGenerateRandom.h    |  2 +-
 ...02533_generate_random_schema_inference.reference |  1 +
 .../02533_generate_random_schema_inference.sql      |  6 ++++++
 5 files changed, 22 insertions(+), 4 deletions(-)
 create mode 100644 tests/queries/0_stateless/02533_generate_random_schema_inference.reference
 create mode 100644 tests/queries/0_stateless/02533_generate_random_schema_inference.sql

diff --git a/docs/en/sql-reference/table-functions/generate.md b/docs/en/sql-reference/table-functions/generate.md
index dd56b47cd3a..380c8364090 100644
--- a/docs/en/sql-reference/table-functions/generate.md
+++ b/docs/en/sql-reference/table-functions/generate.md
@@ -39,3 +39,16 @@ SELECT * FROM generateRandom('a Array(Int8), d Decimal32(4), c Tuple(DateTime64(
 │ [68]     │  -67417.0770 │ ('2080-03-12 14:17:31.269','110425e5-413f-10a6-05ba-fa6b3e929f15') │
 └──────────┴──────────────┴────────────────────────────────────────────────────────────────────┘
 ```
+
+```sql
+CREATE TABLE random (a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)) engine=Memory;
+INSERT INTO random SELECT * FROM generateRandom() LIMIT 2;
+SELECT * FROM random;
+```
+
+```text
+┌─a────────────────────────────┬────────────d─┬─c──────────────────────────────────────────────────────────────────�
+│ []                           │   68091.8197 │ ('2037-10-02 12:44:23.368','039ecab7-81c2-45ee-208c-844e5c6c5652') │
+│ [8,-83,0,-22,65,9,-30,28,64] │ -186233.4909 │ ('2062-01-11 00:06:04.124','69563ea1-5ad1-f870-16d8-67061da0df25') │
+└──────────────────────────────┴──────────────┴────────────────────────────────────────────────────────────────────┘
+```
\ No newline at end of file
diff --git a/src/TableFunctions/TableFunctionGenerateRandom.cpp b/src/TableFunctions/TableFunctionGenerateRandom.cpp
index c545a7c364b..4b28bdf3f0f 100644
--- a/src/TableFunctions/TableFunctionGenerateRandom.cpp
+++ b/src/TableFunctions/TableFunctionGenerateRandom.cpp
@@ -37,9 +37,7 @@ void TableFunctionGenerateRandom::parseArguments(const ASTPtr & ast_function, Co
     ASTs & args = args_func.at(0)->children;
 
     if (args.empty())
-        throw Exception("Table function '" + getName() + "' requires at least one argument: "
-                        " structure, [random_seed, max_string_length, max_array_length].",
-                        ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
+        return;
 
     if (args.size() > 4)
         throw Exception("Table function '" + getName() + "' requires at most four arguments: "
diff --git a/src/TableFunctions/TableFunctionGenerateRandom.h b/src/TableFunctions/TableFunctionGenerateRandom.h
index bdbc73e12b9..584d65311f4 100644
--- a/src/TableFunctions/TableFunctionGenerateRandom.h
+++ b/src/TableFunctions/TableFunctionGenerateRandom.h
@@ -5,7 +5,7 @@
 namespace DB
 {
 
-/* generateRandom(structure, [max_array_length, max_string_length, random_seed])
+/* generateRandom([structure, max_array_length, max_string_length, random_seed])
  * - creates a temporary storage that generates columns with random data
  */
 class TableFunctionGenerateRandom : public ITableFunction
diff --git a/tests/queries/0_stateless/02533_generate_random_schema_inference.reference b/tests/queries/0_stateless/02533_generate_random_schema_inference.reference
new file mode 100644
index 00000000000..f599e28b8ab
--- /dev/null
+++ b/tests/queries/0_stateless/02533_generate_random_schema_inference.reference
@@ -0,0 +1 @@
+10
diff --git a/tests/queries/0_stateless/02533_generate_random_schema_inference.sql b/tests/queries/0_stateless/02533_generate_random_schema_inference.sql
new file mode 100644
index 00000000000..9ca43537439
--- /dev/null
+++ b/tests/queries/0_stateless/02533_generate_random_schema_inference.sql
@@ -0,0 +1,6 @@
+drop table if exists test;
+create table test (x UInt32, y String) engine=Memory;
+insert into test select * from generateRandom() limit 10;
+select count() from test;
+drop table test;
+

From 7410ac2b2919a032dcc9afd7a5af51ff60c5f9a9 Mon Sep 17 00:00:00 2001
From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com>
Date: Fri, 13 Jan 2023 00:43:07 +0100
Subject: [PATCH 094/158] Update 01256_negative_generate_random.sql

---
 tests/queries/0_stateless/01256_negative_generate_random.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/01256_negative_generate_random.sql b/tests/queries/0_stateless/01256_negative_generate_random.sql
index 11c169edd75..dc2b3d77ecc 100644
--- a/tests/queries/0_stateless/01256_negative_generate_random.sql
+++ b/tests/queries/0_stateless/01256_negative_generate_random.sql
@@ -1,5 +1,5 @@
 SELECT * FROM generateRandom('i8', 1, 10, 10); -- { serverError 62 }
 SELECT * FROM generateRandom; -- { serverError 60 }
-SELECT * FROM generateRandom(); -- { serverError 42 }
+SELECT * FROM generateRandom(); -- { serverError 90 }
 SELECT * FROM generateRandom('i8 UInt8', 1, 10, 10, 10, 10); -- { serverError 42 }
 SELECT * FROM generateRandom('', 1, 10, 10); -- { serverError 62 }

From 45df745011b8ae70255fe7083955cf3c8c0a9c21 Mon Sep 17 00:00:00 2001
From: taiyang-li <654010905@qq.com>
Date: Fri, 13 Jan 2023 12:20:03 +0800
Subject: [PATCH 095/158] add fast path for like '%%' or like '%'

---
 src/Functions/MatchImpl.h | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/Functions/MatchImpl.h b/src/Functions/MatchImpl.h
index edb0df3ae34..9a6546428d3 100644
--- a/src/Functions/MatchImpl.h
+++ b/src/Functions/MatchImpl.h
@@ -118,6 +118,18 @@ struct MatchImpl
         if (haystack_offsets.empty())
             return;
 
+        /// Fast path for, because the result is always true or false
+        /// col [i]like '%%'
+        /// col not [i]like '%%'
+        /// col like '%'
+        /// col not [i]like '%'
+        if (is_like && (needle == "%%" or needle == "%"))
+        {
+            for (auto & re : res)
+                re = !negate;
+            return;
+        }
+
         /// Special case that the [I]LIKE expression reduces to finding a substring in a string
         String strstr_pattern;
         if (is_like && impl::likePatternIsSubstring(needle, strstr_pattern))

From de5474c9f9c179f892eb1d58e804fb7aa61144fa Mon Sep 17 00:00:00 2001
From: taiyang-li <654010905@qq.com>
Date: Fri, 13 Jan 2023 14:55:54 +0800
Subject: [PATCH 096/158] optimize match(a, '.*')

---
 src/Functions/MatchImpl.h | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/Functions/MatchImpl.h b/src/Functions/MatchImpl.h
index 9a6546428d3..002b876f1e8 100644
--- a/src/Functions/MatchImpl.h
+++ b/src/Functions/MatchImpl.h
@@ -118,12 +118,13 @@ struct MatchImpl
         if (haystack_offsets.empty())
             return;
 
-        /// Fast path for, because the result is always true or false
+        /// Fast path for [I]LIKE, because the result is always true or false
         /// col [i]like '%%'
         /// col not [i]like '%%'
         /// col like '%'
         /// col not [i]like '%'
-        if (is_like && (needle == "%%" or needle == "%"))
+        /// match(like, '^$')
+        if ((is_like && (needle == "%%" or needle == "%")) || (!is_like && needle == ".*"))
         {
             for (auto & re : res)
                 re = !negate;

From 296dc5006d9f40da6823b602fde9d2d38c365cae Mon Sep 17 00:00:00 2001
From: Maksim Kita <kitaetoya@gmail.com>
Date: Fri, 13 Jan 2023 10:59:26 +0100
Subject: [PATCH 097/158] Fixed tests

---
 .../Passes/AggregateFunctionsArithmericOperationsPass.cpp       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.cpp b/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.cpp
index 33ecf549363..149af61e002 100644
--- a/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.cpp
+++ b/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.cpp
@@ -144,7 +144,7 @@ public:
             auto & inner_function_clone_arguments_nodes = inner_function_clone_arguments.getNodes();
             auto inner_function_clone_left_argument = inner_function_clone_arguments_nodes[0];
             aggregate_function_arguments_nodes = {inner_function_clone_left_argument};
-            resolveAggregateFunctionNode(*aggregate_function_node, inner_function_clone_left_argument, function_name_if_constant_is_negative);
+            resolveAggregateFunctionNode(*aggregate_function_node, inner_function_clone_left_argument, lower_function_name);
             inner_function_clone_arguments_nodes[0] = node;
             node = std::move(inner_function_clone);
         }

From 0110b63a95d7c18dc7a89a575569e4dd014704ec Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Fri, 13 Jan 2023 12:12:27 +0100
Subject: [PATCH 098/158] Better fix

---
 tests/integration/test_multiple_disks/test.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tests/integration/test_multiple_disks/test.py b/tests/integration/test_multiple_disks/test.py
index 77cb9464bcb..221d830f62e 100644
--- a/tests/integration/test_multiple_disks/test.py
+++ b/tests/integration/test_multiple_disks/test.py
@@ -1625,6 +1625,11 @@ def test_rename(start_cluster):
         """
         )
 
+        # We want to check that after inserts, some parts were moved to external disk
+        # and some parts are still on the main disk, but because of merge all parts
+        # might end up on external disk.
+        node1.query("SYSTEM STOP MERGES default.renaming_table")
+
         # jbod1 disk is 40mb
         for _ in range(5):
             data = []

From 9dc9bb2ae6aa0cd93f9f3770136f717681f9c366 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Fri, 13 Jan 2023 12:23:07 +0100
Subject: [PATCH 099/158] Mention only approvals from the team members

---
 tests/ci/merge_pr.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/ci/merge_pr.py b/tests/ci/merge_pr.py
index 946eb77745a..d5569ff0d99 100644
--- a/tests/ci/merge_pr.py
+++ b/tests/ci/merge_pr.py
@@ -73,11 +73,12 @@ class Reviews:
 
         if "APPROVED" in statuses:
             logging.info(
-                "The following users approved the PR: %s",
+                "The following users from %s team approved the PR: %s",
+                TEAM_NAME,
                 ", ".join(
                     user.login
                     for user, r in self._review_per_user.items()
-                    if r.state == "APPROVED"
+                    if r.state == "APPROVED" and user in team
                 ),
             )
             # The only reliable place to get the 100% accurate last_modified

From bdb9222736e72cba6048693b633837b8d93a4509 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Fri, 13 Jan 2023 12:18:58 +0000
Subject: [PATCH 100/158] Support EXPLAIN SYNTAX oneline = 1

---
 src/Interpreters/InterpreterExplainQuery.cpp | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/src/Interpreters/InterpreterExplainQuery.cpp b/src/Interpreters/InterpreterExplainQuery.cpp
index 2341059a04f..d33df11eff1 100644
--- a/src/Interpreters/InterpreterExplainQuery.cpp
+++ b/src/Interpreters/InterpreterExplainQuery.cpp
@@ -288,6 +288,20 @@ struct ExplainSettings : public Settings
     }
 };
 
+struct QuerySyntaxSettings
+{
+    bool oneline = false;
+
+    constexpr static char name[] = "SYNTAX";
+
+    std::unordered_map<std::string, std::reference_wrapper<bool>> boolean_settings =
+    {
+        {"oneline", oneline},
+    };
+
+    std::unordered_map<std::string, std::reference_wrapper<Int64>> integer_settings;
+};
+
 template <typename Settings>
 ExplainSettings<Settings> checkAndGetSettings(const ASTPtr & ast_settings)
 {
@@ -362,13 +376,12 @@ QueryPipeline InterpreterExplainQuery::executeImpl()
         }
         case ASTExplainQuery::AnalyzedSyntax:
         {
-            if (ast.getSettings())
-                throw Exception("Settings are not supported for EXPLAIN SYNTAX query.", ErrorCodes::UNKNOWN_SETTING);
+            auto settings = checkAndGetSettings<QuerySyntaxSettings>(ast.getSettings());
 
             ExplainAnalyzedSyntaxVisitor::Data data(getContext());
             ExplainAnalyzedSyntaxVisitor(data).visit(query);
 
-            ast.getExplainedQuery()->format(IAST::FormatSettings(buf, false));
+            ast.getExplainedQuery()->format(IAST::FormatSettings(buf, settings.oneline));
             break;
         }
         case ASTExplainQuery::QueryTree:

From f881a82417ccf9830327c3b2a7134e2f57b2ff9b Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Fri, 13 Jan 2023 12:19:25 +0000
Subject: [PATCH 101/158] Fix viewExplain, add testcases

---
 src/Parsers/ExpressionElementParsers.cpp           |  4 ++--
 src/TableFunctions/TableFunctionExplain.cpp        |  4 ++++
 .../0_stateless/02421_explain_subquery.reference   |  2 ++
 .../queries/0_stateless/02421_explain_subquery.sql | 14 ++++++++++++++
 4 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp
index b102d5c4a14..790cb128415 100644
--- a/src/Parsers/ExpressionElementParsers.cpp
+++ b/src/Parsers/ExpressionElementParsers.cpp
@@ -123,8 +123,8 @@ bool ParserSubquery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
         if (explain_query.getTableFunction() || explain_query.getTableOverride())
             throw Exception("EXPLAIN in a subquery cannot have a table function or table override", ErrorCodes::BAD_ARGUMENTS);
 
-        /// Replace `SELECT * FROM (EXPLAIN <kind> <explain_settings> SELECT ...)`
-        /// with `SELECT * FROM viewExplain("<kind>", "<explain_settings>", SELECT ...)`
+        /// Replace subquery `(EXPLAIN <kind> <explain_settings> SELECT ...)`
+        /// with `(SELECT * FROM viewExplain("<kind>", "<explain_settings>", SELECT ...))`
 
         String kind_str = ASTExplainQuery::toString(explain_query.getKind());
 
diff --git a/src/TableFunctions/TableFunctionExplain.cpp b/src/TableFunctions/TableFunctionExplain.cpp
index 88b14531d9b..5b2d0e7b78f 100644
--- a/src/TableFunctions/TableFunctionExplain.cpp
+++ b/src/TableFunctions/TableFunctionExplain.cpp
@@ -69,6 +69,10 @@ void TableFunctionExplain::parseArguments(const ASTPtr & ast_function, ContextPt
                 getName(), queryToString(query_arg));
         explain_query->setExplainedQuery(query_arg);
     }
+    else if (kind != ASTExplainQuery::ExplainKind::CurrentTransaction)
+    {
+        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Table function '{}' requires a query argument", getName());
+    }
 
     query = std::move(explain_query);
 }
diff --git a/tests/queries/0_stateless/02421_explain_subquery.reference b/tests/queries/0_stateless/02421_explain_subquery.reference
index 23751ef6c1f..58501cbd0fc 100644
--- a/tests/queries/0_stateless/02421_explain_subquery.reference
+++ b/tests/queries/0_stateless/02421_explain_subquery.reference
@@ -12,3 +12,5 @@
 1
 1
 1
+1
+1
diff --git a/tests/queries/0_stateless/02421_explain_subquery.sql b/tests/queries/0_stateless/02421_explain_subquery.sql
index 72c3a3bfedd..9f83b2f56c7 100644
--- a/tests/queries/0_stateless/02421_explain_subquery.sql
+++ b/tests/queries/0_stateless/02421_explain_subquery.sql
@@ -17,6 +17,13 @@ SELECT * FROM (
     )
 ) FORMAT Null;
 
+SELECT (EXPLAIN SYNTAX oneline = 1 SELECT 1) == 'SELECT 1';
+
+SELECT * FROM viewExplain('', ''); -- { serverError BAD_ARGUMENTS }
+SELECT * FROM viewExplain('EXPLAIN AST', ''); -- { serverError BAD_ARGUMENTS }
+SELECT * FROM viewExplain('EXPLAIN AST', '', 1); -- { serverError BAD_ARGUMENTS }
+SELECT * FROM viewExplain('EXPLAIN AST', '', ''); -- { serverError BAD_ARGUMENTS }
+
 CREATE TABLE t1 ( a UInt64 ) Engine = MergeTree ORDER BY tuple() AS SELECT number AS a FROM system.numbers LIMIT 100000;
 
 SELECT rows > 1000 FROM (EXPLAIN ESTIMATE SELECT sum(a) FROM t1);
@@ -68,4 +75,11 @@ SELECT * FROM (
     )
 ) FORMAT Null;
 
+SELECT (EXPLAIN SYNTAX oneline = 1 SELECT 1) == 'SELECT 1 FROM system.one';
+
+SELECT * FROM viewExplain('', ''); -- { serverError BAD_ARGUMENTS }
+SELECT * FROM viewExplain('EXPLAIN AST', ''); -- { serverError BAD_ARGUMENTS }
+SELECT * FROM viewExplain('EXPLAIN AST', '', 1); -- { serverError BAD_ARGUMENTS }
+SELECT * FROM viewExplain('EXPLAIN AST', '', ''); -- { serverError BAD_ARGUMENTS }
+
 -- EXPLAIN ESTIMATE is not supported in experimental analyzer

From 71188c22ee019782ba16497d8e89d1151f10a5bf Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Fri, 13 Jan 2023 12:19:41 +0000
Subject: [PATCH 102/158] fix race on 'relative_data_path'

---
 src/Storages/MergeTree/MergeTreeData.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index aa735b42ba8..d69f343b827 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -1728,6 +1728,9 @@ try
             is_async ? "asynchronously" : "synchronously");
     }
 
+    /// Acquire shared lock because 'relative_data_path' is used while loading parts.
+    auto shared_lock = lockForShare(RWLockImpl::NO_QUERY, getSettings()->lock_acquire_timeout_for_background_operations);
+
     size_t num_loaded_parts = 0;
     while (true)
     {

From cf5cb0da975428ad728d520bcaf286f2e13944f9 Mon Sep 17 00:00:00 2001
From: Smita Kulkarni <Smita.Kulkarni@clickhouse.com>
Date: Fri, 13 Jan 2023 13:47:54 +0100
Subject: [PATCH 103/158] =?UTF-8?q?Record=20server=20startup=20time=20in?=
 =?UTF-8?q?=20ProfileEvents=C2=A0=20Implementation:=20*=20Added=20ProfileE?=
 =?UTF-8?q?vents::ServerStartupTime.=20*=20Recorded=20time=20from=20start?=
 =?UTF-8?q?=20of=20main=20till=20listening=20to=20sockets.=20Testing:=20*?=
 =?UTF-8?q?=20Added=20a=20test=2002532=5Fprofileevents=5Fserver=5Fstartup?=
 =?UTF-8?q?=5Ftime.sql?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 programs/server/Server.cpp                                  | 6 ++++++
 src/Common/ProfileEvents.cpp                                | 3 ++-
 .../02532_profileevents_server_startup_time.reference       | 1 +
 .../0_stateless/02532_profileevents_server_startup_time.sql | 1 +
 4 files changed, 10 insertions(+), 1 deletion(-)
 create mode 100644 tests/queries/0_stateless/02532_profileevents_server_startup_time.reference
 create mode 100644 tests/queries/0_stateless/02532_profileevents_server_startup_time.sql

diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp
index 742838d6433..a804c7bdeb8 100644
--- a/programs/server/Server.cpp
+++ b/programs/server/Server.cpp
@@ -140,6 +140,7 @@ namespace CurrentMetrics
 namespace ProfileEvents
 {
     extern const Event MainConfigLoads;
+    extern const Event ServerStartupTime;
 }
 
 namespace fs = std::filesystem;
@@ -652,6 +653,8 @@ static void sanityChecks(Server & server)
 int Server::main(const std::vector<std::string> & /*args*/)
 try
 {
+    std::chrono::system_clock::time_point startup_start_time = std::chrono::system_clock::now();
+
     Poco::Logger * log = &logger();
 
     UseSSL use_ssl;
@@ -1427,6 +1430,9 @@ try
 
     }
 
+    std::chrono::milliseconds startup_duration = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now() - startup_start_time);
+    ProfileEvents::increment(ProfileEvents::ServerStartupTime, startup_duration.count());
+
     for (auto & server : servers_to_start_before_tables)
     {
         server.start();
diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp
index abc28299f96..0fc66c2e839 100644
--- a/src/Common/ProfileEvents.cpp
+++ b/src/Common/ProfileEvents.cpp
@@ -449,7 +449,8 @@ The server successfully detected this situation and will download merged part fr
     M(OverflowBreak, "Number of times, data processing was cancelled by query complexity limitation with setting '*_overflow_mode' = 'break' and the result is incomplete.") \
     M(OverflowThrow, "Number of times, data processing was cancelled by query complexity limitation with setting '*_overflow_mode' = 'throw' and exception was thrown.") \
     M(OverflowAny, "Number of times approximate GROUP BY was in effect: when aggregation was performed only on top of first 'max_rows_to_group_by' unique keys and other keys were ignored due to 'group_by_overflow_mode' = 'any'.") \
-
+    \
+    M(ServerStartupTime, "Time elapsed from starting server to listening to sockets")\
 
 namespace ProfileEvents
 {
diff --git a/tests/queries/0_stateless/02532_profileevents_server_startup_time.reference b/tests/queries/0_stateless/02532_profileevents_server_startup_time.reference
new file mode 100644
index 00000000000..d4c9973fdbb
--- /dev/null
+++ b/tests/queries/0_stateless/02532_profileevents_server_startup_time.reference
@@ -0,0 +1 @@
+ServerStartupTime
diff --git a/tests/queries/0_stateless/02532_profileevents_server_startup_time.sql b/tests/queries/0_stateless/02532_profileevents_server_startup_time.sql
new file mode 100644
index 00000000000..6096be46e1f
--- /dev/null
+++ b/tests/queries/0_stateless/02532_profileevents_server_startup_time.sql
@@ -0,0 +1 @@
+SELECT event FROM system.events WHERE event = 'ServerStartupTime'
\ No newline at end of file

From 36c282e48e885a323ef27e2c4eeaaf42d1ce4161 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Fri, 13 Jan 2023 16:29:08 +0300
Subject: [PATCH 104/158] Update clickhouse-test

---
 tests/clickhouse-test | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index a5c6b3e0bb8..39bb9aade3c 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -232,17 +232,20 @@ def need_retry(args, stdout, stderr, total_time):
 
 
 def get_processlist(args):
-    if args.replicated_database:
-        return clickhouse_execute_json(
-            args,
-            """
-        SELECT materialize((hostName(), tcpPort())) as host, *
-        FROM clusterAllReplicas('test_cluster_database_replicated', system.processes)
-        WHERE query NOT LIKE '%system.processes%'
-        """,
-        )
-    else:
-        return clickhouse_execute_json(args, "SHOW PROCESSLIST")
+    try:
+        if args.replicated_database:
+            return clickhouse_execute_json(
+                args,
+                """
+            SELECT materialize((hostName(), tcpPort())) as host, *
+            FROM clusterAllReplicas('test_cluster_database_replicated', system.processes)
+            WHERE query NOT LIKE '%system.processes%'
+            """,
+            )
+        else:
+            return clickhouse_execute_json(args, "SHOW PROCESSLIST")
+    except Exception as e:
+        return "Failed to get processlist: " + str(e)
 
 
 def get_transactions_list(args):

From a0fe26f5069dd425f3862a3eec4cca6066c7a7d5 Mon Sep 17 00:00:00 2001
From: Smita Kulkarni <Smita.Kulkarni@clickhouse.com>
Date: Fri, 13 Jan 2023 14:38:54 +0100
Subject: [PATCH 105/158] Addressed review comments and updated name to
 ServerStartupMilliseconds - Record server startup time in ProfileEvents

---
 programs/server/Server.cpp                               | 9 ++++-----
 src/Common/ProfileEvents.cpp                             | 2 +-
 .../02532_profileevents_server_startup_time.reference    | 2 +-
 .../02532_profileevents_server_startup_time.sql          | 2 +-
 4 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp
index a804c7bdeb8..c3f0dc8c2de 100644
--- a/programs/server/Server.cpp
+++ b/programs/server/Server.cpp
@@ -140,7 +140,7 @@ namespace CurrentMetrics
 namespace ProfileEvents
 {
     extern const Event MainConfigLoads;
-    extern const Event ServerStartupTime;
+    extern const Event ServerStartupMilliseconds;
 }
 
 namespace fs = std::filesystem;
@@ -653,7 +653,7 @@ static void sanityChecks(Server & server)
 int Server::main(const std::vector<std::string> & /*args*/)
 try
 {
-    std::chrono::system_clock::time_point startup_start_time = std::chrono::system_clock::now();
+    Stopwatch startup_watch;
 
     Poco::Logger * log = &logger();
 
@@ -1430,9 +1430,6 @@ try
 
     }
 
-    std::chrono::milliseconds startup_duration = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now() - startup_start_time);
-    ProfileEvents::increment(ProfileEvents::ServerStartupTime, startup_duration.count());
-
     for (auto & server : servers_to_start_before_tables)
     {
         server.start();
@@ -1816,6 +1813,8 @@ try
                                                                      &CurrentMetrics::MaxDDLEntryID, &CurrentMetrics::MaxPushedDDLEntryID));
         }
 
+        ProfileEvents::increment(ProfileEvents::ServerStartupMilliseconds, startup_watch.elapsedMilliseconds());
+
         {
             std::lock_guard lock(servers_lock);
             for (auto & server : servers)
diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp
index 0fc66c2e839..1b387c5a080 100644
--- a/src/Common/ProfileEvents.cpp
+++ b/src/Common/ProfileEvents.cpp
@@ -450,7 +450,7 @@ The server successfully detected this situation and will download merged part fr
     M(OverflowThrow, "Number of times, data processing was cancelled by query complexity limitation with setting '*_overflow_mode' = 'throw' and exception was thrown.") \
     M(OverflowAny, "Number of times approximate GROUP BY was in effect: when aggregation was performed only on top of first 'max_rows_to_group_by' unique keys and other keys were ignored due to 'group_by_overflow_mode' = 'any'.") \
     \
-    M(ServerStartupTime, "Time elapsed from starting server to listening to sockets")\
+    M(ServerStartupMilliseconds, "Time elapsed from starting server to listening to sockets in milliseconds")\
 
 namespace ProfileEvents
 {
diff --git a/tests/queries/0_stateless/02532_profileevents_server_startup_time.reference b/tests/queries/0_stateless/02532_profileevents_server_startup_time.reference
index d4c9973fdbb..e3c0063769c 100644
--- a/tests/queries/0_stateless/02532_profileevents_server_startup_time.reference
+++ b/tests/queries/0_stateless/02532_profileevents_server_startup_time.reference
@@ -1 +1 @@
-ServerStartupTime
+ServerStartupMilliseconds
diff --git a/tests/queries/0_stateless/02532_profileevents_server_startup_time.sql b/tests/queries/0_stateless/02532_profileevents_server_startup_time.sql
index 6096be46e1f..d7c97d08c15 100644
--- a/tests/queries/0_stateless/02532_profileevents_server_startup_time.sql
+++ b/tests/queries/0_stateless/02532_profileevents_server_startup_time.sql
@@ -1 +1 @@
-SELECT event FROM system.events WHERE event = 'ServerStartupTime'
\ No newline at end of file
+SELECT event FROM system.events WHERE event = 'ServerStartupMilliseconds'
\ No newline at end of file

From abfb6b096f8cb6a8b6bc452811e84a1267c65473 Mon Sep 17 00:00:00 2001
From: avogar <pav.cruglov@yandex.ru>
Date: Fri, 13 Jan 2023 14:23:30 +0000
Subject: [PATCH 106/158] Better exception message

---
 .../TableFunctionGenerateRandom.cpp            | 18 +++++++++++++-----
 .../01256_negative_generate_random.sql         |  2 +-
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/src/TableFunctions/TableFunctionGenerateRandom.cpp b/src/TableFunctions/TableFunctionGenerateRandom.cpp
index 4b28bdf3f0f..bed3397152b 100644
--- a/src/TableFunctions/TableFunctionGenerateRandom.cpp
+++ b/src/TableFunctions/TableFunctionGenerateRandom.cpp
@@ -25,6 +25,7 @@ namespace ErrorCodes
     extern const int BAD_ARGUMENTS;
     extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
     extern const int LOGICAL_ERROR;
+    extern const int CANNOT_EXTRACT_TABLE_STRUCTURE;
 }
 
 void TableFunctionGenerateRandom::parseArguments(const ASTPtr & ast_function, ContextPtr /*context*/)
@@ -75,16 +76,23 @@ void TableFunctionGenerateRandom::parseArguments(const ASTPtr & ast_function, Co
 
 ColumnsDescription TableFunctionGenerateRandom::getActualTableStructure(ContextPtr context) const
 {
+    if (structure == "auto")
+    {
+        if (structure_hint.empty())
+            throw Exception(
+                ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE,
+                "Table function '{}' was used without structure argument but structure could not be determined automatically. Please, "
+                "provide structure manually",
+                getName());
+        return structure_hint;
+    }
+
     return parseColumnsListFromString(structure, context);
 }
 
 StoragePtr TableFunctionGenerateRandom::executeImpl(const ASTPtr & /*ast_function*/, ContextPtr context, const std::string & table_name, ColumnsDescription /*cached_columns*/) const
 {
-    ColumnsDescription columns;
-    if (structure != "auto")
-        columns = parseColumnsListFromString(structure, context);
-    else if (!structure_hint.empty())
-        columns = structure_hint;
+    ColumnsDescription columns = getActualTableStructure(context);
     auto res = std::make_shared<StorageGenerateRandom>(
         StorageID(getDatabaseName(), table_name), columns, String{}, max_array_length, max_string_length, random_seed);
     res->startup();
diff --git a/tests/queries/0_stateless/01256_negative_generate_random.sql b/tests/queries/0_stateless/01256_negative_generate_random.sql
index dc2b3d77ecc..14f1d947108 100644
--- a/tests/queries/0_stateless/01256_negative_generate_random.sql
+++ b/tests/queries/0_stateless/01256_negative_generate_random.sql
@@ -1,5 +1,5 @@
 SELECT * FROM generateRandom('i8', 1, 10, 10); -- { serverError 62 }
 SELECT * FROM generateRandom; -- { serverError 60 }
-SELECT * FROM generateRandom(); -- { serverError 90 }
+SELECT * FROM generateRandom(); -- { serverError CANNOT_EXTRACT_TABLE_STRUCTURE }
 SELECT * FROM generateRandom('i8 UInt8', 1, 10, 10, 10, 10); -- { serverError 42 }
 SELECT * FROM generateRandom('', 1, 10, 10); -- { serverError 62 }

From 6de4837580676830b5c58c1443452ac1c3677fac Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Fri, 13 Jan 2023 16:07:20 +0100
Subject: [PATCH 107/158] fix

---
 src/Databases/DDLLoadingDependencyVisitor.cpp | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/Databases/DDLLoadingDependencyVisitor.cpp b/src/Databases/DDLLoadingDependencyVisitor.cpp
index 5a9a7995bae..3a61f821629 100644
--- a/src/Databases/DDLLoadingDependencyVisitor.cpp
+++ b/src/Databases/DDLLoadingDependencyVisitor.cpp
@@ -69,10 +69,14 @@ ssize_t DDLMatcherBase::getPositionOfTableNameArgumentToVisit(const ASTFunction
 
     if (functionIsInOrGlobalInOperator(function.name))
     {
-        if (function.children.size() < 2)
+        if (function.children.empty())
             return -1;
 
-        if (function.children[1]->as<ASTFunction>())
+        const auto * args = function.children[0]->as<ASTExpressionList>();
+        if (!args || args->children.size() != 2)
+            return -1;
+
+        if (args->children[1]->as<ASTFunction>())
             return -1;
 
         return 1;

From 76c89c6d20fb8263ff0d57a5baf2a4e324b9c29b Mon Sep 17 00:00:00 2001
From: avogar <pav.cruglov@yandex.ru>
Date: Fri, 13 Jan 2023 16:31:30 +0000
Subject: [PATCH 108/158] Fix heap-use-after-free in reading from s3

---
 src/Storages/StorageS3.h                        | 17 +++++++++++++++++
 .../02534_s3_heap_use_after_free.reference      |  4 ++++
 .../02534_s3_heap_use_after_free.sql            |  2 ++
 3 files changed, 23 insertions(+)
 create mode 100644 tests/queries/0_stateless/02534_s3_heap_use_after_free.reference
 create mode 100644 tests/queries/0_stateless/02534_s3_heap_use_after_free.sql

diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h
index 0f6fc013d51..926274a4fff 100644
--- a/src/Storages/StorageS3.h
+++ b/src/Storages/StorageS3.h
@@ -176,6 +176,23 @@ private:
         }
 
         ReaderHolder() = default;
+        ReaderHolder(const ReaderHolder & other) = delete;
+        ReaderHolder & operator=(const ReaderHolder & other) = delete;
+
+        ReaderHolder(ReaderHolder && other)
+        {
+            *this = std::move(other);
+        }
+
+        ReaderHolder & operator=(ReaderHolder && other)
+        {
+            /// The order is important.
+            /// reader uses pipeline, pipeline uses read_buf.
+            reader = std::move(other.reader);
+            pipeline = std::move(other.pipeline);
+            read_buf = std::move(other.read_buf);
+            return *this;
+        }
 
         explicit operator bool() const { return reader != nullptr; }
         PullingPipelineExecutor * operator->() { return reader.get(); }
diff --git a/tests/queries/0_stateless/02534_s3_heap_use_after_free.reference b/tests/queries/0_stateless/02534_s3_heap_use_after_free.reference
new file mode 100644
index 00000000000..acd7c60768b
--- /dev/null
+++ b/tests/queries/0_stateless/02534_s3_heap_use_after_free.reference
@@ -0,0 +1,4 @@
+1	2	3
+4	5	6
+7	8	9
+0	0	0
diff --git a/tests/queries/0_stateless/02534_s3_heap_use_after_free.sql b/tests/queries/0_stateless/02534_s3_heap_use_after_free.sql
new file mode 100644
index 00000000000..535356aa634
--- /dev/null
+++ b/tests/queries/0_stateless/02534_s3_heap_use_after_free.sql
@@ -0,0 +1,2 @@
+select * from s3('http://localhost:11111/test/a.tsv', CustomSeparated);
+

From 6cb7c4d1759cfde11da9c0eaca6626dbecb98117 Mon Sep 17 00:00:00 2001
From: avogar <pav.cruglov@yandex.ru>
Date: Fri, 13 Jan 2023 16:33:11 +0000
Subject: [PATCH 109/158] Better commit, mark noexcept

---
 src/Storages/StorageS3.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h
index 926274a4fff..a5e1ed4e0b5 100644
--- a/src/Storages/StorageS3.h
+++ b/src/Storages/StorageS3.h
@@ -179,14 +179,14 @@ private:
         ReaderHolder(const ReaderHolder & other) = delete;
         ReaderHolder & operator=(const ReaderHolder & other) = delete;
 
-        ReaderHolder(ReaderHolder && other)
+        ReaderHolder(ReaderHolder && other) noexcept
         {
             *this = std::move(other);
         }
 
-        ReaderHolder & operator=(ReaderHolder && other)
+        ReaderHolder & operator=(ReaderHolder && other) noexcept
         {
-            /// The order is important.
+            /// The order of destruction is important.
             /// reader uses pipeline, pipeline uses read_buf.
             reader = std::move(other.reader);
             pipeline = std::move(other.pipeline);

From 120986da52a282c985fbbbb4b4700fc406ea90e8 Mon Sep 17 00:00:00 2001
From: avogar <pav.cruglov@yandex.ru>
Date: Fri, 13 Jan 2023 16:34:38 +0000
Subject: [PATCH 110/158] Add tags to test

---
 tests/queries/0_stateless/02534_s3_heap_use_after_free.sql | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/queries/0_stateless/02534_s3_heap_use_after_free.sql b/tests/queries/0_stateless/02534_s3_heap_use_after_free.sql
index 535356aa634..b9f815e5a07 100644
--- a/tests/queries/0_stateless/02534_s3_heap_use_after_free.sql
+++ b/tests/queries/0_stateless/02534_s3_heap_use_after_free.sql
@@ -1,2 +1,5 @@
+-- Tags: no-fasttest
+-- Tag no-fasttest: Depends on AWS
+
 select * from s3('http://localhost:11111/test/a.tsv', CustomSeparated);
 

From 5d3f0ec4a0c27aa02f4a3b7fb61a15be03f75a5e Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Fri, 13 Jan 2023 16:17:28 +0000
Subject: [PATCH 111/158] Disallow Gorilla codec on non-float columns

Cf. #45195
---
 src/Compression/CompressionCodecGorilla.cpp           | 11 ++++++++---
 .../0_stateless/02533_gorilla_on_nonfloat.reference   |  0
 .../queries/0_stateless/02533_gorilla_on_nonfloat.sql |  4 ++++
 3 files changed, 12 insertions(+), 3 deletions(-)
 create mode 100644 tests/queries/0_stateless/02533_gorilla_on_nonfloat.reference
 create mode 100644 tests/queries/0_stateless/02533_gorilla_on_nonfloat.sql

diff --git a/src/Compression/CompressionCodecGorilla.cpp b/src/Compression/CompressionCodecGorilla.cpp
index 88b8c2bc3bb..5714ba00a50 100644
--- a/src/Compression/CompressionCodecGorilla.cpp
+++ b/src/Compression/CompressionCodecGorilla.cpp
@@ -444,14 +444,19 @@ void CompressionCodecGorilla::doDecompressData(const char * source, UInt32 sourc
 void registerCodecGorilla(CompressionCodecFactory & factory)
 {
     UInt8 method_code = static_cast<UInt8>(CompressionMethodByte::Gorilla);
-    factory.registerCompressionCodecWithType("Gorilla", method_code,
-        [&](const ASTPtr & arguments, const IDataType * column_type) -> CompressionCodecPtr
+    auto codec_builder = [&](const ASTPtr & arguments, const IDataType * column_type) -> CompressionCodecPtr
     {
         if (arguments)
             throw Exception("Codec Gorilla does not accept any arguments", ErrorCodes::BAD_ARGUMENTS);
 
+        if (column_type != nullptr)
+            if (!WhichDataType(*column_type).isFloat())
+                throw Exception(ErrorCodes::BAD_ARGUMENTS, "Gorilla codec is not applicable for {} because the data type is not float",
+                        column_type->getName());
+
         UInt8 data_bytes_size = column_type ? getDataBytesSize(column_type) : 0;
         return std::make_shared<CompressionCodecGorilla>(data_bytes_size);
-    });
+    };
+    factory.registerCompressionCodecWithType("Gorilla", method_code, codec_builder);
 }
 }
diff --git a/tests/queries/0_stateless/02533_gorilla_on_nonfloat.reference b/tests/queries/0_stateless/02533_gorilla_on_nonfloat.reference
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/queries/0_stateless/02533_gorilla_on_nonfloat.sql b/tests/queries/0_stateless/02533_gorilla_on_nonfloat.sql
new file mode 100644
index 00000000000..4fb6ac6c2c0
--- /dev/null
+++ b/tests/queries/0_stateless/02533_gorilla_on_nonfloat.sql
@@ -0,0 +1,4 @@
+DROP TABLE IF EXISTS test;
+CREATE TABLE test (id UInt64, val Decimal(15,5) CODEC (Gorilla)) ENGINE = MergeTree() ORDER BY id; -- { serverError BAD_ARGUMENTS }
+CREATE TABLE test (id UInt64, val FixedString(2) CODEC (Gorilla)) ENGINE = MergeTree() ORDER BY id; -- { serverError BAD_ARGUMENTS }
+CREATE TABLE test (id UInt64, val UInt64 CODEC (Gorilla)) ENGINE = MergeTree() ORDER BY id; -- { serverError BAD_ARGUMENTS }

From 487de70d012cc8cb4483157977bcbde399ac276f Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Fri, 13 Jan 2023 17:05:32 +0000
Subject: [PATCH 112/158] fix locking at loading outdated data parts

---
 src/Storages/MergeTree/MergeTreeData.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index d69f343b827..248e0f0426a 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -1729,7 +1729,9 @@ try
     }
 
     /// Acquire shared lock because 'relative_data_path' is used while loading parts.
-    auto shared_lock = lockForShare(RWLockImpl::NO_QUERY, getSettings()->lock_acquire_timeout_for_background_operations);
+    TableLockHolder shared_lock;
+    if (is_async)
+        shared_lock = lockForShare(RWLockImpl::NO_QUERY, getSettings()->lock_acquire_timeout_for_background_operations);
 
     size_t num_loaded_parts = 0;
     while (true)

From d0a55f6dc985a55821b9c9c80b68d2482ca6757c Mon Sep 17 00:00:00 2001
From: DanRoscigno <dan@roscigno.com>
Date: Fri, 13 Jan 2023 13:17:03 -0500
Subject: [PATCH 113/158] doc grace_hash algorithm for join

---
 .../operations/settings/query-complexity.md   |  2 +-
 docs/en/operations/settings/settings.md       | 68 ++++++++++++-------
 2 files changed, 46 insertions(+), 24 deletions(-)

diff --git a/docs/en/operations/settings/query-complexity.md b/docs/en/operations/settings/query-complexity.md
index 376b7480358..7a6b2340d29 100644
--- a/docs/en/operations/settings/query-complexity.md
+++ b/docs/en/operations/settings/query-complexity.md
@@ -266,7 +266,7 @@ Default value: 0.
 
 Limits the size in bytes of the hash table used when joining tables.
 
-This settings applies to [SELECT … JOIN](../../sql-reference/statements/select/join.md#select-join) operations and [Join table engine](../../engines/table-engines/special/join.md).
+This setting applies to [SELECT … JOIN](../../sql-reference/statements/select/join.md#select-join) operations and [Join table engine](../../engines/table-engines/special/join.md).
 
 If the query contains joins, ClickHouse checks this setting for every intermediate result.
 
diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index 4ffe2bbc7c4..7eccdc9a93e 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -402,40 +402,62 @@ Default value: `ALL`.
 
 ## join_algorithm {#settings-join_algorithm}
 
-Specifies [JOIN](../../sql-reference/statements/select/join.md) algorithm.
+Specifies which [JOIN](../../sql-reference/statements/select/join.md) algorithm is used.
 
 Several algorithms can be specified, and an available one would be chosen for a particular query based on kind/strictness and table engine.
 
 Possible values:
 
-- `default` — `hash` or `direct`, if possible (same as `direct,hash`)
+### `default` 
 
-- `hash` — [Hash join algorithm](https://en.wikipedia.org/wiki/Hash_join) is used. The most generic implementation that supports all combinations of kind and strictness and multiple join keys that are combined with `OR` in the `JOIN ON` section.
+This is the equivalent of `hash` or `direct`, if possible (same as `direct,hash`)
 
-- `parallel_hash` - a variation of `hash` join that splits the data into buckets and builds several hashtables instead of one concurrently to speed up this process.
+### `grace_hash` 
+
+[Grace hash join](https://en.wikipedia.org/wiki/Hash_join#Grace_hash_join) is used.  Grace hash provides an algorithm option that provides performant complex joins while limiting memory use.
+
+The first phase of a grace join reads the right table and splits it into N buckets depending on the hash value of key columns (initially, N is `grace_hash_join_initial_buckets`). This is done in a way to ensure that each bucket can be processed independently. Rows from the first bucket are added to an in-memory hash table while the others are saved to disk. If the hash table grows beyond the memory limit (e.g., as set by [`max_bytes_in_join`](/docs/en/operations/settings/query-complexity.md/#settings-max_bytes_in_join)), the number of buckets is increased and the assigned bucket for each row. Any rows which don’t belong to the current bucket are flushed and reassigned.
+
+### `hash`
+
+[Hash join algorithm](https://en.wikipedia.org/wiki/Hash_join) is used. The most generic implementation that supports all combinations of kind and strictness and multiple join keys that are combined with `OR` in the `JOIN ON` section.
+
+### `parallel_hash` 
+
+A variation of `hash` join that splits the data into buckets and builds several hashtables instead of one concurrently to speed up this process.
 
 When using the `hash` algorithm, the right part of `JOIN` is uploaded into RAM.
 
-- `partial_merge` — a variation of the [sort-merge algorithm](https://en.wikipedia.org/wiki/Sort-merge_join), where only the right table is fully sorted.
+### `partial_merge` 
+
+A variation of the [sort-merge algorithm](https://en.wikipedia.org/wiki/Sort-merge_join), where only the right table is fully sorted.
 
 The `RIGHT JOIN` and `FULL JOIN` are supported only with `ALL` strictness (`SEMI`, `ANTI`, `ANY`, and `ASOF` are not supported).
 
-When using `partial_merge` algorithm, ClickHouse sorts the data and dumps it to the disk. The `partial_merge` algorithm in ClickHouse differs slightly from the classic realization. First, ClickHouse sorts the right table by joining keys in blocks and creates a min-max index for sorted blocks. Then it sorts parts of the left table by `join key` and joins them over the right table. The min-max index is also used to skip unneeded right table blocks.
+When using the `partial_merge` algorithm, ClickHouse sorts the data and dumps it to the disk. The `partial_merge` algorithm in ClickHouse differs slightly from the classic realization. First, ClickHouse sorts the right table by joining keys in blocks and creates a min-max index for sorted blocks. Then it sorts parts of the left table by the `join key` and joins them over the right table. The min-max index is also used to skip unneeded right table blocks.
 
-- `direct` - can be applied when the right storage supports key-value requests.
+### `direct` 
+
+This algorithm can be applied when the storage for the right table supports key-value requests.
 
 The `direct` algorithm performs a lookup in the right table using rows from the left table as keys. It's supported only by special storage such as [Dictionary](../../engines/table-engines/special/dictionary.md/#dictionary) or [EmbeddedRocksDB](../../engines/table-engines/integrations/embedded-rocksdb.md) and only the `LEFT` and `INNER` JOINs.
 
-- `auto` — try `hash` join and switch on the fly to another algorithm if the memory limit is violated.
+### `auto` 
 
-- `full_sorting_merge` — [Sort-merge algorithm](https://en.wikipedia.org/wiki/Sort-merge_join) with full sorting joined tables before joining.
+When set to `auto`, `hash` join is tried first, and the algorithm is switched on the fly to another algorithm if the memory limit is violated.
 
-- `prefer_partial_merge` — ClickHouse always tries to use `partial_merge` join if possible, otherwise, it uses `hash`. *Deprecated*, same as `partial_merge,hash`.
+### `full_sorting_merge` 
+
+[Sort-merge algorithm](https://en.wikipedia.org/wiki/Sort-merge_join) with full sorting joined tables before joining.
+
+### `prefer_partial_merge` 
+
+ClickHouse always tries to use `partial_merge` join if possible, otherwise, it uses `hash`. *Deprecated*, same as `partial_merge,hash`.
 
 
 ## join_any_take_last_row {#settings-join_any_take_last_row}
 
-Changes behaviour of join operations with `ANY` strictness.
+Changes the behaviour of join operations with `ANY` strictness.
 
 :::warning
 This setting applies only for `JOIN` operations with [Join](../../engines/table-engines/special/join.md) engine tables.
@@ -498,7 +520,7 @@ Default value: 65536.
 
 Limits the number of files allowed for parallel sorting in MergeJoin operations when they are executed on disk.
 
-The bigger the value of the setting, the more RAM used and the less disk I/O needed.
+The bigger the value of the setting, the more RAM is used and the less disk I/O is needed.
 
 Possible values:
 
@@ -514,12 +536,12 @@ Enables legacy ClickHouse server behaviour in `ANY INNER|LEFT JOIN` operations.
 Use this setting only for backward compatibility if your use cases depend on legacy `JOIN` behaviour.
 :::
 
-When the legacy behaviour enabled:
+When the legacy behaviour is enabled:
 
 -   Results of `t1 ANY LEFT JOIN t2` and `t2 ANY RIGHT JOIN t1` operations are not equal because ClickHouse uses the logic with many-to-one left-to-right table keys mapping.
 -   Results of `ANY INNER JOIN` operations contain all rows from the left table like the `SEMI LEFT JOIN` operations do.
 
-When the legacy behaviour disabled:
+When the legacy behaviour is disabled:
 
 -   Results of `t1 ANY LEFT JOIN t2` and `t2 ANY RIGHT JOIN t1` operations are equal because ClickHouse uses the logic which provides one-to-many keys mapping in `ANY RIGHT JOIN` operations.
 -   Results of `ANY INNER JOIN` operations contain one row per key from both the left and right tables.
@@ -572,7 +594,7 @@ Default value: `163840`.
 
 ## merge_tree_min_rows_for_concurrent_read_for_remote_filesystem {#merge-tree-min-rows-for-concurrent-read-for-remote-filesystem}
 
-The minimum number of lines to read from one file before [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) engine can parallelize reading, when reading from remote filesystem.
+The minimum number of lines to read from one file before the [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) engine can parallelize reading, when reading from remote filesystem.
 
 Possible values:
 
@@ -706,7 +728,7 @@ log_queries=1
 
 ## log_queries_min_query_duration_ms {#settings-log-queries-min-query-duration-ms}
 
-If enabled (non-zero), queries faster then the value of this setting will not be logged (you can think about this as a `long_query_time` for [MySQL Slow Query Log](https://dev.mysql.com/doc/refman/5.7/en/slow-query-log.html)), and this basically means that you will not find them in the following tables:
+If enabled (non-zero), queries faster than the value of this setting will not be logged (you can think about this as a `long_query_time` for [MySQL Slow Query Log](https://dev.mysql.com/doc/refman/5.7/en/slow-query-log.html)), and this basically means that you will not find them in the following tables:
 
 - `system.query_log`
 - `system.query_thread_log`
@@ -741,7 +763,7 @@ log_queries_min_type='EXCEPTION_WHILE_PROCESSING'
 
 Setting up query threads logging.
 
-Query threads log into [system.query_thread_log](../../operations/system-tables/query_thread_log.md) table. This setting have effect only when [log_queries](#settings-log-queries) is true. Queries’ threads run by ClickHouse with this setup are logged according to the rules in the [query_thread_log](../../operations/server-configuration-parameters/settings.md/#server_configuration_parameters-query_thread_log) server configuration parameter.
+Query threads log into the [system.query_thread_log](../../operations/system-tables/query_thread_log.md) table. This setting has effect only when [log_queries](#settings-log-queries) is true. Queries’ threads run by ClickHouse with this setup are logged according to the rules in the [query_thread_log](../../operations/server-configuration-parameters/settings.md/#server_configuration_parameters-query_thread_log) server configuration parameter.
 
 Possible values:
 
@@ -760,7 +782,7 @@ log_query_threads=1
 
 Setting up query views logging.
 
-When a query run by ClickHouse with this setup on has associated views (materialized or live views), they are logged in the [query_views_log](../../operations/server-configuration-parameters/settings.md/#server_configuration_parameters-query_views_log) server configuration parameter.
+When a query run by ClickHouse with this setting enabled has associated views (materialized or live views), they are logged in the [query_views_log](../../operations/server-configuration-parameters/settings.md/#server_configuration_parameters-query_views_log) server configuration parameter.
 
 Example:
 
@@ -787,7 +809,7 @@ It can be used to improve the readability of server logs. Additionally, it helps
 
 Possible values:
 
--   Any string no longer than [max_query_size](#settings-max_query_size). If length is exceeded, the server throws an exception.
+-   Any string no longer than [max_query_size](#settings-max_query_size). If the max_query_size is exceeded, the server throws an exception.
 
 Default value: empty string.
 
@@ -821,11 +843,11 @@ The setting also does not have a purpose when using INSERT SELECT, since data is
 
 Default value: 1,048,576.
 
-The default is slightly more than `max_block_size`. The reason for this is because certain table engines (`*MergeTree`) form a data part on the disk for each inserted block, which is a fairly large entity. Similarly, `*MergeTree` tables sort data during insertion, and a large enough block size allow sorting more data in RAM.
+The default is slightly more than `max_block_size`. The reason for this is that certain table engines (`*MergeTree`) form a data part on the disk for each inserted block, which is a fairly large entity. Similarly, `*MergeTree` tables sort data during insertion, and a large enough block size allow sorting more data in RAM.
 
 ## min_insert_block_size_rows {#min-insert-block-size-rows}
 
-Sets the minimum number of rows in the block which can be inserted into a table by an `INSERT` query. Smaller-sized blocks are squashed into bigger ones.
+Sets the minimum number of rows in the block that can be inserted into a table by an `INSERT` query. Smaller-sized blocks are squashed into bigger ones.
 
 Possible values:
 
@@ -891,7 +913,7 @@ Higher values will lead to higher memory usage.
 
 ## max_compress_block_size {#max-compress-block-size}
 
-The maximum size of blocks of uncompressed data before compressing for writing to a table. By default, 1,048,576 (1 MiB). Specifying smaller block size generally leads to slightly reduced compression ratio, the compression and decompression speed increases slightly due to cache locality, and memory consumption is reduced.
+The maximum size of blocks of uncompressed data before compressing for writing to a table. By default, 1,048,576 (1 MiB). Specifying a smaller block size generally leads to slightly reduced compression ratio, the compression and decompression speed increases slightly due to cache locality, and memory consumption is reduced.
 
 :::warning
 This is an expert-level setting, and you shouldn't change it if you're just getting started with ClickHouse.
@@ -935,7 +957,7 @@ Default value: 1000.
 
 ## interactive_delay {#interactive-delay}
 
-The interval in microseconds for checking whether request execution has been cancelled and sending the progress.
+The interval in microseconds for checking whether request execution has been canceled and sending the progress.
 
 Default value: 100,000 (checks for cancelling and sends the progress ten times per second).
 

From d857d62a03c861bd8da5854c0a309c9eed549c86 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Fri, 13 Jan 2023 20:34:31 +0100
Subject: [PATCH 114/158] remove another set of macros

---
 src/Common/logger_useful.h                    | 55 ++++++++---------
 .../MergeTree/MergeFromLogEntryTask.cpp       |  4 +-
 .../MergeTree/ReplicatedMergeTreeQueue.cpp    | 59 +++++++------------
 tests/clickhouse-test                         |  6 +-
 4 files changed, 50 insertions(+), 74 deletions(-)

diff --git a/src/Common/logger_useful.h b/src/Common/logger_useful.h
index ce2e2e5514a..a2f4e26d7f6 100644
--- a/src/Common/logger_useful.h
+++ b/src/Common/logger_useful.h
@@ -7,6 +7,29 @@
 #include <Poco/Message.h>
 #include <Common/CurrentThread.h>
 
+/// This wrapper is usefult to save formatted message into a String before sending it to a logger
+class LogToStrImpl
+{
+    String & out_str;
+    Poco::Logger * logger;
+    bool propagate_to_actual_log = true;
+public:
+    LogToStrImpl(String & out_str_, Poco::Logger * logger_) : out_str(out_str_) , logger(logger_) {}
+    LogToStrImpl & operator -> () { return *this; }
+    bool is(Poco::Message::Priority priority) { propagate_to_actual_log &= logger->is(priority); return true; }
+    LogToStrImpl * getChannel() {return this; }
+    const String & name() const { return logger->name(); }
+    void log(const Poco::Message & message)
+    {
+        out_str = message.getText();
+        if (!propagate_to_actual_log)
+            return;
+        if (auto * channel = logger->getChannel())
+            channel->log(message);
+    }
+};
+
+#define LogToStr(x, y) std::make_unique<LogToStrImpl>(x, y)
 
 namespace
 {
@@ -17,6 +40,7 @@ namespace
 
     [[maybe_unused]] const ::Poco::Logger * getLogger(const ::Poco::Logger * logger) { return logger; };
     [[maybe_unused]] const ::Poco::Logger * getLogger(const std::atomic<::Poco::Logger *> & logger) { return logger.load(); };
+    [[maybe_unused]] std::unique_ptr<LogToStrImpl> getLogger(std::unique_ptr<LogToStrImpl> && logger) { return logger; };
 
     template<typename T> struct is_fmt_runtime : std::false_type {};
     template<typename T> struct is_fmt_runtime<fmt::basic_runtime<T>> : std::true_type {};
@@ -58,7 +82,7 @@ namespace
     auto _logger = ::getLogger(logger);                                           \
     const bool _is_clients_log = (DB::CurrentThread::getGroup() != nullptr) &&    \
         (DB::CurrentThread::getGroup()->client_logs_level >= (priority));         \
-    if (_logger->is((PRIORITY)) || _is_clients_log)                               \
+    if (_is_clients_log || _logger->is((PRIORITY)))                               \
     {                                                                             \
         std::string formatted_message = numArgs(__VA_ARGS__) > 1 ? fmt::format(__VA_ARGS__) : firstArg(__VA_ARGS__); \
         if (auto _channel = _logger->getChannel())                                \
@@ -74,26 +98,6 @@ namespace
     }                                                                             \
 } while (false)
 
-#define LOG_IMPL_PREFORMATTED(logger, priority, PRIORITY, FORMAT_STRING, MESSAGE) do                              \
-{                                                                                 \
-    auto _logger = ::getLogger(logger);                                           \
-    const bool _is_clients_log = (DB::CurrentThread::getGroup() != nullptr) &&    \
-        (DB::CurrentThread::getGroup()->client_logs_level >= (priority));         \
-    if (_logger->is((PRIORITY)) || _is_clients_log)                               \
-    {                                                                             \
-        if (auto _channel = _logger->getChannel())                                \
-        {                                                                         \
-            std::string file_function;                                            \
-            file_function += __FILE__;                                            \
-            file_function += "; ";                                                \
-            file_function += __PRETTY_FUNCTION__;                                 \
-            Poco::Message poco_message(_logger->name(), MESSAGE,        \
-                (PRIORITY), file_function.c_str(), __LINE__, tryGetStaticFormatString(FORMAT_STRING));    \
-            _channel->log(poco_message);                                          \
-        }                                                                         \
-    }                                                                             \
-} while (false)
-
 
 #define LOG_TEST(logger, ...)    LOG_IMPL(logger, DB::LogsLevel::test, Poco::Message::PRIO_TEST, __VA_ARGS__)
 #define LOG_TRACE(logger, ...)   LOG_IMPL(logger, DB::LogsLevel::trace, Poco::Message::PRIO_TRACE, __VA_ARGS__)
@@ -102,12 +106,3 @@ namespace
 #define LOG_WARNING(logger, ...) LOG_IMPL(logger, DB::LogsLevel::warning, Poco::Message::PRIO_WARNING, __VA_ARGS__)
 #define LOG_ERROR(logger, ...)   LOG_IMPL(logger, DB::LogsLevel::error, Poco::Message::PRIO_ERROR, __VA_ARGS__)
 #define LOG_FATAL(logger, ...)   LOG_IMPL(logger, DB::LogsLevel::error, Poco::Message::PRIO_FATAL, __VA_ARGS__)
-
-
-#define LOG_TEST_PREFORMATTED(logger, FORMAT_STRING, MESSAGE)    LOG_IMPL_PREFORMATTED(logger, DB::LogsLevel::test, Poco::Message::PRIO_TEST, FORMAT_STRING, MESSAGE)
-#define LOG_TRACE_PREFORMATTED(logger, FORMAT_STRING, MESSAGE)   LOG_IMPL_PREFORMATTED(logger, DB::LogsLevel::trace, Poco::Message::PRIO_TRACE, FORMAT_STRING, MESSAGE)
-#define LOG_DEBUG_PREFORMATTED(logger, FORMAT_STRING, MESSAGE)   LOG_IMPL_PREFORMATTED(logger, DB::LogsLevel::debug, Poco::Message::PRIO_DEBUG, FORMAT_STRING, MESSAGE)
-#define LOG_INFO_PREFORMATTED(logger, FORMAT_STRING, MESSAGE)    LOG_IMPL_PREFORMATTED(logger, DB::LogsLevel::information, Poco::Message::PRIO_INFORMATION, FORMAT_STRING, MESSAGE)
-#define LOG_WARNING_PREFORMATTED(logger, FORMAT_STRING, MESSAGE) LOG_IMPL_PREFORMATTED(logger, DB::LogsLevel::warning, Poco::Message::PRIO_WARNING, FORMAT_STRING, MESSAGE)
-#define LOG_ERROR_PREFORMATTED(logger, FORMAT_STRING, MESSAGE)   LOG_IMPL_PREFORMATTED(logger, DB::LogsLevel::error, Poco::Message::PRIO_ERROR, FORMAT_STRING, MESSAGE)
-#define LOG_FATAL_PREFORMATTED(logger, FORMAT_STRING, MESSAGE)   LOG_IMPL_PREFORMATTED(logger, DB::LogsLevel::error, Poco::Message::PRIO_FATAL, FORMAT_STRING, MESSAGE)
diff --git a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp
index 5054b7f727c..97226825629 100644
--- a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp
+++ b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp
@@ -110,8 +110,8 @@ ReplicatedMergeMutateTaskBase::PrepareResult MergeFromLogEntryTask::prepare()
             /// 3. We have two intersecting parts, both cover source_part_name. It's logical error.
             /// TODO Why 1 and 2 can happen? Do we need more assertions here or somewhere else?
             constexpr auto fmt_string = "Part {} is covered by {} but should be merged into {}. This shouldn't happen often.";
-            String message = fmt::format(fmt_string, source_part_name, source_part_or_covering->name, entry.new_part_name);
-            LOG_WARNING_PREFORMATTED(log, fmt_string, message);
+            String message;
+            LOG_WARNING(LogToStr(message, log), fmt_string, source_part_name, source_part_or_covering->name, entry.new_part_name);
             if (!source_part_or_covering->info.contains(MergeTreePartInfo::fromPartName(entry.new_part_name, storage.format_version)))
                 throw Exception(ErrorCodes::LOGICAL_ERROR, message);
 
diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp
index 93994ebaa64..bd75d76109a 100644
--- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp
+++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp
@@ -1193,11 +1193,8 @@ bool ReplicatedMergeTreeQueue::isCoveredByFuturePartsImpl(const LogEntry & entry
         const LogEntry & another_entry = *entry_for_same_part_it->second;
         constexpr auto fmt_string = "Not executing log entry {} of type {} for part {} "
                                     "because another log entry {} of type {} for the same part ({}) is being processed.";
-        out_reason = fmt::format(
-            fmt_string,
-            entry.znode_name, entry.type, entry.new_part_name,
-            another_entry.znode_name, another_entry.type, another_entry.new_part_name);
-        LOG_INFO_PREFORMATTED(log, fmt_string, out_reason);
+        LOG_INFO(LogToStr(out_reason, log), fmt_string, entry.znode_name, entry.type, entry.new_part_name,
+                 another_entry.znode_name, another_entry.type, another_entry.new_part_name);
         return true;
 
         /** When the corresponding action is completed, then `isNotCoveredByFuturePart` next time, will succeed,
@@ -1241,9 +1238,7 @@ bool ReplicatedMergeTreeQueue::isCoveredByFuturePartsImpl(const LogEntry & entry
             {
                 constexpr auto fmt_string = "Not executing log entry {} for part {} "
                                             "because it is not disjoint with part {} that is currently executing and another entry {} is newer.";
-                out_reason = fmt::format(
-                    fmt_string, entry.znode_name, new_part_name, future_part_elem.first, future_part_elem.second->znode_name);
-                LOG_TRACE_PREFORMATTED(log, fmt_string, out_reason);
+                LOG_TRACE(LogToStr(out_reason, log), fmt_string, entry.znode_name, new_part_name, future_part_elem.first, future_part_elem.second->znode_name);
                 return true;
             }
 
@@ -1253,8 +1248,7 @@ bool ReplicatedMergeTreeQueue::isCoveredByFuturePartsImpl(const LogEntry & entry
 
         constexpr auto fmt_string = "Not executing log entry {} for part {} "
                                     "because it is not disjoint with part {} that is currently executing.";
-        out_reason = fmt::format(fmt_string, entry.znode_name, new_part_name, future_part_elem.first);
-        LOG_TEST_PREFORMATTED(log, fmt_string, out_reason);
+        LOG_TEST(LogToStr(out_reason, log), fmt_string, entry.znode_name, new_part_name, future_part_elem.first);
         return true;
     }
 
@@ -1339,8 +1333,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry(
             {
                 constexpr auto fmt_string = "Not executing log entry {} of type {} for part {} "
                       "because part {} is not ready yet (log entry for that part is being processed).";
-                out_postpone_reason = fmt::format(fmt_string, entry.znode_name, entry.typeToString(), entry.new_part_name, name);
-                LOG_TRACE_PREFORMATTED(log, fmt_string, out_postpone_reason);
+                LOG_TRACE(LogToStr(out_postpone_reason, log), fmt_string, entry.znode_name, entry.typeToString(), entry.new_part_name, name);
                 return false;
             }
 
@@ -1357,8 +1350,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry(
         if (merger_mutator.merges_blocker.isCancelled())
         {
             constexpr auto fmt_string = "Not executing log entry {} of type {} for part {} because merges and mutations are cancelled now.";
-            out_postpone_reason = fmt::format(fmt_string, entry.znode_name, entry.typeToString(), entry.new_part_name);
-            LOG_DEBUG_PREFORMATTED(log, fmt_string, out_postpone_reason);
+            LOG_DEBUG(LogToStr(out_postpone_reason, log), fmt_string, entry.znode_name, entry.typeToString(), entry.new_part_name);
             return false;
         }
 
@@ -1409,17 +1401,15 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry(
                 if (merger_mutator.ttl_merges_blocker.isCancelled())
                 {
                     constexpr auto fmt_string = "Not executing log entry {} for part {} because merges with TTL are cancelled now.";
-                    out_postpone_reason = fmt::format(fmt_string, entry.znode_name, entry.new_part_name);
-                    LOG_DEBUG_PREFORMATTED(log, fmt_string, out_postpone_reason);
+                    LOG_DEBUG(LogToStr(out_postpone_reason, log), fmt_string, entry.znode_name, entry.new_part_name);
                     return false;
                 }
                 size_t total_merges_with_ttl = data.getTotalMergesWithTTLInMergeList();
                 if (total_merges_with_ttl >= data_settings->max_number_of_merges_with_ttl_in_pool)
                 {
                     constexpr auto fmt_string = "Not executing log entry {} for part {} because {} merges with TTL already executing, maximum {}.";
-                    out_postpone_reason = fmt::format(fmt_string, entry.znode_name, entry.new_part_name, total_merges_with_ttl,
-                                                      data_settings->max_number_of_merges_with_ttl_in_pool);
-                    LOG_DEBUG_PREFORMATTED(log, fmt_string, out_postpone_reason);
+                    LOG_DEBUG(LogToStr(out_postpone_reason, log), fmt_string, entry.znode_name, entry.new_part_name, total_merges_with_ttl,
+                              data_settings->max_number_of_merges_with_ttl_in_pool);
                     return false;
                 }
             }
@@ -1429,10 +1419,8 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry(
         {
             constexpr auto fmt_string = "Not executing log entry {} of type {} for part {}"
                                         " because source parts size ({}) is greater than the current maximum ({}).";
-            out_postpone_reason = fmt::format(fmt_string, entry.znode_name, entry.typeToString(), entry.new_part_name,
-                ReadableSize(sum_parts_size_in_bytes), ReadableSize(max_source_parts_size));
-
-            LOG_DEBUG_PREFORMATTED(log, fmt_string, out_postpone_reason);
+            LOG_DEBUG(LogToStr(out_postpone_reason, log), fmt_string, entry.znode_name, entry.typeToString(), entry.new_part_name,
+                      ReadableSize(sum_parts_size_in_bytes), ReadableSize(max_source_parts_size));
 
             return false;
         }
@@ -1446,8 +1434,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry(
         {
             int head_alter = alter_sequence.getHeadAlterVersion(state_lock);
             constexpr auto fmt_string = "Cannot execute alter metadata {} with version {} because another alter {} must be executed before";
-            out_postpone_reason = fmt::format(fmt_string, entry.znode_name, entry.alter_version, head_alter);
-            LOG_TRACE_PREFORMATTED(log, fmt_string, out_postpone_reason);
+            LOG_TRACE(LogToStr(out_postpone_reason, log), fmt_string, entry.znode_name, entry.alter_version, head_alter);
             return false;
         }
     }
@@ -1461,14 +1448,12 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry(
             if (head_alter == entry.alter_version)
             {
                 constexpr auto fmt_string = "Cannot execute alter data {} with version {} because metadata still not altered";
-                out_postpone_reason = fmt::format(fmt_string, entry.znode_name, entry.alter_version);
-                LOG_TRACE_PREFORMATTED(log, fmt_string, out_postpone_reason);
+                LOG_TRACE(LogToStr(out_postpone_reason, log), fmt_string, entry.znode_name, entry.alter_version);
             }
             else
             {
                 constexpr auto fmt_string = "Cannot execute alter data {} with version {} because another alter {} must be executed before";
-                out_postpone_reason = fmt::format(fmt_string, entry.znode_name, entry.alter_version, head_alter);
-                LOG_TRACE_PREFORMATTED(log, fmt_string, out_postpone_reason);
+                LOG_TRACE(LogToStr(out_postpone_reason, log), fmt_string, entry.znode_name, entry.alter_version, head_alter);
             }
 
             return false;
@@ -1492,13 +1477,10 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry(
                     continue;
                 constexpr auto fmt_string = "Not executing log entry {} of type {} for part {} "
                     "because another DROP_RANGE or REPLACE_RANGE entry with not disjoint range {} is currently executing.";
-                out_postpone_reason = fmt::format(
-                    fmt_string,
-                    entry.znode_name,
-                    entry.typeToString(),
-                    entry.new_part_name,
-                    info.getPartNameForLogs());
-                LOG_TRACE_PREFORMATTED(log, fmt_string, out_postpone_reason);
+                LOG_TRACE(LogToStr(out_postpone_reason, log), fmt_string, entry.znode_name,
+                          entry.typeToString(),
+                          entry.new_part_name,
+                          info.getPartNameForLogs());
                 return false;
             }
         }
@@ -1526,9 +1508,8 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry(
                     {
                         constexpr auto fmt_string = "Not executing log entry {} of type {} for part {} "
                             "because it probably depends on {} (REPLACE_RANGE).";
-                        out_postpone_reason = fmt::format(
-                            fmt_string, entry.znode_name, entry.typeToString(), entry.new_part_name, replace_entry->znode_name);
-                        LOG_TRACE_PREFORMATTED(log, fmt_string, out_postpone_reason);
+                        LOG_TRACE(LogToStr(out_postpone_reason, log), fmt_string, entry.znode_name, entry.typeToString(),
+                                  entry.new_part_name, replace_entry->znode_name);
                         return false;
                     }
                 }
diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index 23659e9aa0d..1b9f7eeb76a 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -1814,7 +1814,7 @@ def reportCoverage(args):
         """
     )
 
-def reportLogPatterns(args):
+def reportLogStats(args):
     query = """
         WITH
             120 AS mins,
@@ -1860,7 +1860,7 @@ def reportLogPatterns(args):
         FORMAT TSVWithNamesAndTypes
     """
     value = clickhouse_execute(args, query).decode()
-    print("\nTop runtime messages:\n")
+    print("\nTop messages without format string (fmt::runtime):\n")
     print(value)
     print("\n")
 
@@ -1999,7 +1999,7 @@ def main(args):
         print("All tests have finished.")
 
     if args.report_logs_stats:
-        reportLogPatterns(args)
+        reportLogStats(args)
 
     if args.report_coverage and not reportCoverage(args):
         exit_code.value = 1

From 167d980fe2f9fccd6402070f9843cd9bb0b7d9c1 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 13 Jan 2023 22:52:05 +0100
Subject: [PATCH 115/158] Fix GitHub

---
 docker/test/fuzzer/run-fuzzer.sh | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 docker/test/fuzzer/run-fuzzer.sh

diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh
old mode 100644
new mode 100755

From ec8c38c174a4ad245ac4797fada5f027ec147cb4 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 13 Jan 2023 22:53:17 +0100
Subject: [PATCH 116/158] Add review suggestion

---
 docker/test/fuzzer/run-fuzzer.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh
index 3d34b67d4ec..3458cf905da 100755
--- a/docker/test/fuzzer/run-fuzzer.sh
+++ b/docker/test/fuzzer/run-fuzzer.sh
@@ -5,6 +5,7 @@ set -x
 
 # core.COMM.PID-TID
 sysctl kernel.core_pattern='core.%e.%p-%P'
+dmesg --clear ||:
 
 set -e
 set -u

From 3d23654720e175a049a61cc1402440b982a28947 Mon Sep 17 00:00:00 2001
From: Dmitry Novik <n0vik@clickhouse.com>
Date: Fri, 13 Jan 2023 23:10:16 +0000
Subject: [PATCH 117/158] Skip validation of function IN

---
 src/Analyzer/QueryTreePassManager.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/Analyzer/QueryTreePassManager.cpp b/src/Analyzer/QueryTreePassManager.cpp
index 6aa9365316b..5e38d46ff52 100644
--- a/src/Analyzer/QueryTreePassManager.cpp
+++ b/src/Analyzer/QueryTreePassManager.cpp
@@ -62,6 +62,9 @@ class ValidationChecker : public InDepthQueryTreeVisitor<ValidationChecker>
             "Function {} is not resolved after running {} pass",
             function->toAST()->formatForErrorMessage(), pass_name);
 
+        if (function->getFunctionName() == "in")
+            return;
+
         const auto & expected_arg_types = function->getArgumentTypes();
         auto actual_arg_columns = function->getArgumentColumns();
 

From 36ec76f53d23b28e68aa91ae4420057281f0c093 Mon Sep 17 00:00:00 2001
From: Tanya Bragin <tbragin@users.noreply.github.com>
Date: Sat, 14 Jan 2023 07:50:10 -0800
Subject: [PATCH 118/158] Update clickhouse-local.md

cc @DanRoscigno @gingerwizard Adding latest clickhouse-local blog to related content on this page.
---
 docs/en/operations/utilities/clickhouse-local.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/en/operations/utilities/clickhouse-local.md b/docs/en/operations/utilities/clickhouse-local.md
index 963ddfe7a02..a4fa5579638 100644
--- a/docs/en/operations/utilities/clickhouse-local.md
+++ b/docs/en/operations/utilities/clickhouse-local.md
@@ -120,5 +120,6 @@ Read 186 rows, 4.15 KiB in 0.035 sec., 5302 rows/sec., 118.34 KiB/sec.
 
 ## Related Content
 
+- [Extracting, converting, and querying data in local files using clickhouse-local](https://clickhouse.com/blog/extracting-converting-querying-local-files-with-sql-clickhouse-local)
 - [Getting Data Into ClickHouse - Part 1](https://clickhouse.com/blog/getting-data-into-clickhouse-part-1)
 - [Exploring massive, real-world data sets: 100+ Years of Weather Records in ClickHouse](https://clickhouse.com/blog/real-world-data-noaa-climate-data)

From a5e14a85ab1b07d017bb05e23f0840ad2701f174 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sat, 14 Jan 2023 17:29:10 +0000
Subject: [PATCH 119/158] Fix 01272_suspicious_codecs

---
 tests/queries/0_stateless/01272_suspicious_codecs.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/01272_suspicious_codecs.sql b/tests/queries/0_stateless/01272_suspicious_codecs.sql
index 5baa30e3cf4..a2cae22ccc2 100644
--- a/tests/queries/0_stateless/01272_suspicious_codecs.sql
+++ b/tests/queries/0_stateless/01272_suspicious_codecs.sql
@@ -11,7 +11,7 @@ CREATE TABLE codecs
     c Float32 CODEC(Gorilla),
     d UInt8 CODEC(Delta, LZ4),
     e Float64 CODEC(Gorilla, ZSTD),
-    f UInt32 CODEC(Delta, Delta, Gorilla),
+    f Float64 CODEC(Delta, Delta, Gorilla),
     g DateTime CODEC(DoubleDelta),
     h DateTime64 CODEC(DoubleDelta, LZ4),
     i String CODEC(NONE)

From 29eb30b49f5ec4d1a080e7b6940337fb5526b3f4 Mon Sep 17 00:00:00 2001
From: flynn <fenglv15@mails.ucas.ac.cn>
Date: Sat, 14 Jan 2023 17:58:34 +0000
Subject: [PATCH 120/158] Fix some reading avro format bugs

fix
---
 src/Processors/Formats/Impl/AvroRowInputFormat.cpp | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp
index da7f18260a9..8c6cd8bd91b 100644
--- a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp
+++ b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp
@@ -304,7 +304,9 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node
                     };
                 }
 
-                if (null_as_default)
+                /// If the Union is ['Null', Nested-Type], since the Nested-Type can not be inside
+                /// Nullable, so we will get Nested-Type, instead of Nullable type.
+                if (null_as_default || !target.isNullable())
                 {
                     auto nested_deserialize = this->createDeserializeFn(root_node->leafAt(non_null_union_index), target_type);
                     return [non_null_union_index, nested_deserialize](IColumn & column, avro::Decoder & decoder)
@@ -1001,7 +1003,7 @@ DataTypePtr AvroSchemaReader::avroNodeToDataType(avro::NodePtr node)
         case avro::Type::AVRO_STRING:
             return std::make_shared<DataTypeString>();
         case avro::Type::AVRO_BYTES:
-            return std::make_shared<DataTypeFloat32>();
+            return std::make_shared<DataTypeString>();
         case avro::Type::AVRO_ENUM:
         {
             if (node->names() < 128)

From 1c5214a19243503b3639486748c00193998a4eb8 Mon Sep 17 00:00:00 2001
From: flynn <fenglv15@mails.ucas.ac.cn>
Date: Sat, 14 Jan 2023 18:18:51 +0000
Subject: [PATCH 121/158] add test

---
 .../02521_avro_union_null_nested.reference       |  15 +++++++++++++++
 .../0_stateless/02521_avro_union_null_nested.sh  |  13 +++++++++++++
 .../02522_avro_complicate_schema.reference       |   5 +++++
 .../0_stateless/02522_avro_complicate_schema.sh  |  13 +++++++++++++
 .../data_avro/complicated_schema.avro            | Bin 0 -> 6324 bytes
 .../0_stateless/data_avro/union_null_nested.avro | Bin 0 -> 3948 bytes
 6 files changed, 46 insertions(+)
 create mode 100644 tests/queries/0_stateless/02521_avro_union_null_nested.reference
 create mode 100755 tests/queries/0_stateless/02521_avro_union_null_nested.sh
 create mode 100644 tests/queries/0_stateless/02522_avro_complicate_schema.reference
 create mode 100755 tests/queries/0_stateless/02522_avro_complicate_schema.sh
 create mode 100644 tests/queries/0_stateless/data_avro/complicated_schema.avro
 create mode 100644 tests/queries/0_stateless/data_avro/union_null_nested.avro

diff --git a/tests/queries/0_stateless/02521_avro_union_null_nested.reference b/tests/queries/0_stateless/02521_avro_union_null_nested.reference
new file mode 100644
index 00000000000..e4818b4bcac
--- /dev/null
+++ b/tests/queries/0_stateless/02521_avro_union_null_nested.reference
@@ -0,0 +1,15 @@
+manifest_path	String					
+manifest_length	Int64					
+partition_spec_id	Int32					
+added_snapshot_id	Nullable(Int64)					
+added_data_files_count	Nullable(Int32)					
+existing_data_files_count	Nullable(Int32)					
+deleted_data_files_count	Nullable(Int32)					
+partitions	Array(Tuple(contains_null Bool, contains_nan Nullable(Bool), lower_bound Nullable(String), upper_bound Nullable(String)))					
+added_rows_count	Nullable(Int64)					
+existing_rows_count	Nullable(Int64)					
+deleted_rows_count	Nullable(Int64)					
+file:/warehouse/nyc.db/taxis/metadata/f9e891e9-fbd3-4411-a5c6-0cc14a2f1392-m0.avro	6488	0	1793608066486471262	8	0	0	[(false,false,'
\0\0\0\0\0\0\0','\b\0\0\0\0\0\0\0')]	12	0	0
+file:/warehouse/nyc.db/taxis/metadata/a51dd31d-ea86-42dd-82d1-1981332a0f6d-m0.avro	6363	0	5735460159761889536	4	0	0	[(false,false,'\0\0\0\0\0\0\0','\b\0\0\0\0\0\0\0')]	4	0	0
+file:/warehouse/nyc.db/taxis/metadata/7ae325bd-fe20-4a55-917c-36cb8f6a488c-m0.avro	6370	0	7171740521400098346	4	0	0	[(false,false,'\0\0\0\0\0\0\0','\0\0\0\0\0\0\0')]	4	0	0
+file:/warehouse/nyc.db/taxis/metadata/5e3c62a9-1537-455f-98e5-0a067af5752a-m0.avro	6324	0	6850377589038341628	2	0	0	[(false,false,'
\0\0\0\0\0\0\0','\0\0\0\0\0\0\0')]	4	0	0
diff --git a/tests/queries/0_stateless/02521_avro_union_null_nested.sh b/tests/queries/0_stateless/02521_avro_union_null_nested.sh
new file mode 100755
index 00000000000..d3c6200100a
--- /dev/null
+++ b/tests/queries/0_stateless/02521_avro_union_null_nested.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# Tags: no-parallel, no-fasttest
+
+set -e
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CUR_DIR"/../shell_config.sh
+
+DATA_DIR=$CUR_DIR/data_avro
+
+$CLICKHOUSE_LOCAL -q "desc file('$DATA_DIR/union_null_nested.avro')"
+$CLICKHOUSE_LOCAL -q "select * from file('$DATA_DIR/nullable_array.avro')"
diff --git a/tests/queries/0_stateless/02522_avro_complicate_schema.reference b/tests/queries/0_stateless/02522_avro_complicate_schema.reference
new file mode 100644
index 00000000000..55c0369020f
--- /dev/null
+++ b/tests/queries/0_stateless/02522_avro_complicate_schema.reference
@@ -0,0 +1,5 @@
+status	Int32					
+snapshot_id	Nullable(Int64)					
+data_file	Tuple(file_path String, file_format String, partition Tuple(vendor_id Nullable(Int64)), record_count Int64, file_size_in_bytes Int64, block_size_in_bytes Int64, column_sizes Array(Tuple(key Int32, value Int64)), value_counts Array(Tuple(key Int32, value Int64)), null_value_counts Array(Tuple(key Int32, value Int64)), nan_value_counts Array(Tuple(key Int32, value Int64)), lower_bounds Array(Tuple(key Int32, value String)), upper_bounds Array(Tuple(key Int32, value String)), key_metadata Nullable(String), split_offsets Array(Int64), sort_order_id Nullable(Int32))					
+1	6850377589038341628	('file:/warehouse/nyc.db/taxis/data/vendor_id=1/00000-0-c070e655-dc44-43d2-a01a-484f107210cb-00001.parquet','PARQUET',(1),2,1565,67108864,[(1,87),(2,51),(3,51),(4,57),(5,51)],[(1,2),(2,2),(3,2),(4,2),(5,2)],[(1,0),(2,0),(3,0),(4,0),(5,0)],[(3,0),(4,0)],[(1,'
\0\0\0\0\0\0\0'),(2,'³C\0\0\0\0\0'),(3,'ffæ?'),(4,'¤p=\n×£.@'),(5,'N')],[(1,'
\0\0\0\0\0\0\0'),(2,'¶C\0\0\0\0\0'),(3,'ffA'),(4,'q=\n×£E@'),(5,'Y')],NULL,[4],0)
+1	6850377589038341628	('file:/warehouse/nyc.db/taxis/data/vendor_id=2/00000-0-c070e655-dc44-43d2-a01a-484f107210cb-00002.parquet','PARQUET',(2),2,1620,67108864,[(1,87),(2,51),(3,51),(4,57),(5,89)],[(1,2),(2,2),(3,2),(4,2),(5,2)],[(1,0),(2,0),(3,0),(4,0),(5,0)],[(3,0),(4,0)],[(1,'\0\0\0\0\0\0\0'),(2,'´C\0\0\0\0\0'),(3,'fff?'),(4,'…ëQ¸"@'),(5,'N')],[(1,'\0\0\0\0\0\0\0'),(2,'µC\0\0\0\0\0'),(3,'\0\0 @'),(4,'fffff&6@'),(5,'N')],NULL,[4],0)
diff --git a/tests/queries/0_stateless/02522_avro_complicate_schema.sh b/tests/queries/0_stateless/02522_avro_complicate_schema.sh
new file mode 100755
index 00000000000..fa23c7e6f34
--- /dev/null
+++ b/tests/queries/0_stateless/02522_avro_complicate_schema.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# Tags: no-parallel, no-fasttest
+
+set -e
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CUR_DIR"/../shell_config.sh
+
+DATA_DIR=$CUR_DIR/data_avro
+
+$CLICKHOUSE_LOCAL -q "desc file('$DATA_DIR/complicated_schema.avro')"
+$CLICKHOUSE_LOCAL -q "select * from file('$DATA_DIR/complicated_schema.avro')"
diff --git a/tests/queries/0_stateless/data_avro/complicated_schema.avro b/tests/queries/0_stateless/data_avro/complicated_schema.avro
new file mode 100644
index 0000000000000000000000000000000000000000..a3385f4b23b70343e857f7047ce25cfd744e3661
GIT binary patch
literal 6324
zcmb_gTWl0n7;f28Owg(nft1D!BdwxS-I*!$@*)aUNf3=dxHiMt*|Y7?*_rLkY@32j
ztj0<WAyN%cG$Kh$T7nu(RIJ9}3lS51Qxo;k@Zu%$$zUY<pEEmi&Q5oi?b4)e(wXz$
zzW;K*f1Xbqs;;}+)q8OoU0r-c^oBB6>=a$k$?BdciT;}wBPn*qB{5}SGwF(*`;Um^
zi7bg0N|W;g*h<=t1`j3C!Tnjo!SFtXOczUmPSdt}#9_%7RYFnEF*5TIRPzOshU+0q
z$0h8hOdENj;nq+-g&eG*w4Jp`pvAmS+S!DOL-F;Yxa-+O-b!kzLHIRMPYEr6r$M6c
z9$tY49Q$z})(cOBfIC>X9fDpC&@{4)6m~rgTb?rn&pGIGM<8%7>%yyC&I}p{J?H0(
zU>%&bP{!@GJs)UaV83W(O%vp4V!9<UiBl+RdSWMep0h|b$(*Jn@{pD?0FuCP(HI=1
zpx;DT%OI~e)IiFOzymJWI#km(TSuN@Tf(jvw+e%X*DG}G5GZ8CDvMB#$2rfa-jwa6
zA=`m(QO#XoTky{a1e#8Pq(c&HM<;)T408Ijn8@>1`1s+3XAtK?8^z<7<Ao96yJg<F
zh(plb!}$&>91MOXX?m`%bHzb(9N@b>n@(Uyuv3D6*cA**@Ol9mKd*f3DyNzv>>7u$
zW>{Kc$irOq<r&y%d&s2P1ah3H503M?TRGhc)7Ja6(#m<+^_;A3o7uEQg_%b|d57si
zj)O=i86Hl<3=5w}Tdz-++q3~$ZV#%!A9%S=>cd1wk(L8V&aDAtW?7r~ZI+iext(Ux
zw0jI4nLGW_n?@O!6G5t;Lm84MKSIGs3Z4x=1YHU&^axDAKJch`^TykwKKaG5Fi>>>
zR7$)Os5+u<F<f)S#N}J@i@q|bXfH@mg}Rwd!k`hzfD>SJ3)|85(%A7%5-E))bpZiX
zWCCE}IZ?XS6n-q-w$>D-qFSR!QRPsB{!LlD(qw)dTgX~)u+(+{OYNwj+SEBsP^#M0
zjtapF<|@>cKv4@vtKY#Ti%sQ+hiMODM@s<aK)8Z|a|bg`*+A;7tpezj4aFO^z*JY-
zD#DJ&E6{jLf~4?+Kby%cEJ$RUsg+DKg&!1=W-I`bTG=?G@s<M;U+pkOv^4e*VQzCO
zN%Wly@0%@_?hIup`2sm1kQ3l~PL#}x%1<LVV;Y`jr&2EV%CFwwhQuaL1MT&xr>e5c
za0gNAA>iY1CkDEUk0X_*MXv3DM@|yzg>ESk-bKiB2*M6H3v3Y13R^dqjJh?-#mJOh
zPAR@_C$U}!-kHe5jXvjW8NiMU+;NdChCH&x&0rm3T@-ILbPT+#C$}B^+7wch^42j}
z&;q(2tOjhWfrXB(VkBD9)cH%Co3|t|(X!S!elK96yw$nU(EFWRC*v#vN;Pv!LSc1L
zYF}z;2xE=2m6ciu*M;(Ibd|Y2Fnesz4HFIpRoGfN05OF4C4%WIl&kJx<_}w+BYOf<
z!RIV!Nq;XZy*Dr;+>T9ly(p<K$5hb`f>~j!I1Hx(yF)26OiO#%G?6WWiRWP&CW-KL
zNrWNk&ndZ${sY^qzlYO1AomJv_*X(_fT#bKnIM?MPUpj1Cg*1{bc8Wby^NhQ1Leg4
ziU{W{5Q74Rw=gl3CgWjHGOP4LynCLJDbV-DB7+va)5zwq<S+~s$|5(E3lQRYMxjC>
z6^TNH)ZKX$x23OPTng*H2vePBOv;9$F)4su5L3di5d98DAG9zdx9uFBSTgrn+ebJg
z(_B`|D8*Yw<yxk4_j$?}y05Zp4C;YrcSYP6LkIj3Q<CN}Gwk+?GP)vm!s%$jcUc&Z
z5h@oxfDx11@q-ggVO~ZHP`=Q;RyHD{?2c*6%VKUe`xxZ2Jm78c(-_{;l$M>WhA-W@
z_T}$e&q0<;xC0rLN#Lr6N4^TVcGl6cKlJlH^w4j1Rhw`1{-?J6{`r;TGwc3N9(|+c
z+ZsLb&tJE`|7!OiKR^7Z{^ga^Cnw%IG^M?_=j}5mw0-YhIJ<k|;;HdTZG6v(y%*1p
z?-`%gPM$e6t}Q=1k?5PYs}{}H&R(m%Q9F}5e|~n|wdUs8=Bw+vX2)tr$7Y*XG#jI}
zH@Ze^uWg+bJ{di{cx=mSQ!`C3RUdz9<lKjAwr!KXSliIh(0Kat=^x~eCofI)FYlj{
ztA->=GNnf8fF#t#mc~}cmc&-ZV)aXpRxhhRn0&cv#mM#V{yn&2^)vf^QjgS)G%f!0
zz>h0`>&VDA&&~WYIyrv*#HRP}n|$HQ1JlZ~-bd<}#OmK%+qUu8v-jTj(6K6n_O2Ow
zP<Ut6*wXu(FF*O2a8tc!mH5F&vB7UHJ^J;gnsd%$2byB9z7l(VW5Zq36C=Bx-c&OO
GxBme}aW@12

literal 0
HcmV?d00001

diff --git a/tests/queries/0_stateless/data_avro/union_null_nested.avro b/tests/queries/0_stateless/data_avro/union_null_nested.avro
new file mode 100644
index 0000000000000000000000000000000000000000..c5246c3dd644df61578f5e089980ddf45bcb3aa8
GIT binary patch
literal 3948
zcmbW3ZD<@t7{_a)wS*9ADb?E4j7yakl6aeYKMNXZ8k+_hO<P||IOcZeavSz`ue*Cm
z(==imgo+TNF$xxoV4>9dDXk4s(zHZ~3KB!5lG;Eq1dXL1Y@|pit+Q`4yY~|J^6hSS
z=J`F(%=3T#yEoDKNX16jrO0(EmqKZ{w7HkjdNPQKF$&^jg=cCQ38wY`X($Ovq-yb`
zAR_!YDIk$onb^VJycHR!rSJ+Xkg6$y)Peu=GKb@|$Q;xJSpxfC-wnD2Ed_S90fSDW
z)(Sim3;KM0HTkhcBy~8Wi?Sp3K3Rhz&=Ulza0CHC0*M|Csdl<nVAc^h0~Jls^NFh&
z#Kke0BrZrA$+``{1iN>FtcrM`$)H;iwA_5_9bqBQBR;N5Fr%hq%^CI}BV|Pq2R1Tq
zw~wXGTB<{|6x4tb0sln6K6yPEl!RGPV`6#?>kh1<{CRX9YS6eXHO|RdqZso_ND6v|
zB!dP$Wq>42=>hioWq1Jx4;>a%4PSD+L}a72&pS2fpz=sW8b0L`p*ziO-kBi>leCN8
zQ|{9w%7Kakd+=-AA!#fS@;p~jO3)X<%kqW9$&v;INsU{bVhJfB%OZl3-Ufz}_9<E1
z;z<Cd9<%3DPz4a^n*hlZbU`tj+o@4UMHFU)cGtw%OwuLApinBnEZ#nB#F2t1cOxa9
zz&%RUM|#v0FyXh_dIiQVVC(`(Spg82txI3OnPtGvsgliPRsc43l@4b6+WTr4+<a-Q
zF-x?OK2lg)Hl2o8dU`1caw&S8%huxyI}XyEI*Qy~RPpYiV@h6M??LgbbG{M9dxsIR
zd*wMNnyq#3I9ujkX#spa8w@MtWFB#qxP?Thp^f$o$2ayo8;<%zzKAar3PeJIFk2t0
zufq40hFWbGQdI0BtJx}nLkXmGSdMz(X`+8@gjr6v6w_;s7Vx?BZ~0!n=UR@cphH<f
zLD(H}C}a;HrUZx8o5iHC`4r-<ML6EPmLlp2f{Y`F%NBwBzw1(fp8(9xx9`gITM9@*
zQKiS_D#MHKQeYqlSipLUQJr<=TP;Nd34&;Bg%lz<r>9d;h=6Qyt&n3YtuQ7`Fr*3e
z30>rvh*(7YMS+*)j7##sVXH(|4kI;NJW9~i<jij`N_Pp#cDt20EIC*<QUGgsE2X*{
z(kr%r$5JYuB-WR@+fIu@1qs`Y?k&d0gco0>FgiTk7Fsf_xR<)zlE&5V3)`rf^fI1F
zbEQ+LfIZT{?kjQ4h&-F5!RZ7dYIRwZr}#vu4zHpJI&*BDmQJuG>IAg+ZN-YzzeVh^
zsasqy><<P)J~kK)huBCY8uW)&eK0<L_JRN2Sl~hPq3Y|a=GK1q=;zayn#*2Wx8~Mt
z|FPP(rS(Iz>a2S7<cxatU3T#6(Yp4|{u9SjL$ADe?gZGr^;X%RWpADy{3p@;<wyd&
z7klY`Ma$yVin*U}-rc<LOxOKn@0r`z?v(wOX?*+lcg~-@c=Y|sJuUrz-)NP=hJpTd
zAFpmc2hM*|&L2}Iz8DT>Mz3AI_r$lCV!c&2tM_g{JCyox>Z{4=Uk8t@-+y~)Quuez
zn&!W%&Q$K`eC&Ai@z0{=zodSQPSk9BgWVNk#ugS9r|0kd(Ae@(^}vntrn^rz4V?17
z6<K=<f!1bUVD!pxt=78j!o^{3B(vkEsd&>^!$UaJaCyr&52L65IKT#8er~Ga)lFAY
nFFZTZGXE4`x9PyZrwp?=I&=8?c(iJgPtV7{ADP>dduaawj&58J

literal 0
HcmV?d00001


From f98e01d538b97123296932958404ef1ea21b9798 Mon Sep 17 00:00:00 2001
From: flynn <fenglv15@mails.ucas.ac.cn>
Date: Sat, 14 Jan 2023 18:24:52 +0000
Subject: [PATCH 122/158] fix

---
 tests/queries/0_stateless/02521_avro_union_null_nested.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02521_avro_union_null_nested.sh b/tests/queries/0_stateless/02521_avro_union_null_nested.sh
index d3c6200100a..d4d14c8ca8e 100755
--- a/tests/queries/0_stateless/02521_avro_union_null_nested.sh
+++ b/tests/queries/0_stateless/02521_avro_union_null_nested.sh
@@ -10,4 +10,4 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 DATA_DIR=$CUR_DIR/data_avro
 
 $CLICKHOUSE_LOCAL -q "desc file('$DATA_DIR/union_null_nested.avro')"
-$CLICKHOUSE_LOCAL -q "select * from file('$DATA_DIR/nullable_array.avro')"
+$CLICKHOUSE_LOCAL -q "select * from file('$DATA_DIR/union_null_nested.avro')"

From fbdaca4e2ae1e993dbe8c869ebb5171371c7062c Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sat, 14 Jan 2023 19:21:07 +0000
Subject: [PATCH 123/158] Code cleanup

---
 .../CompressionFactoryAdditions.cpp           | 27 +++++++++----------
 .../0_stateless/01272_suspicious_codecs.sql   | 16 +++++------
 2 files changed, 21 insertions(+), 22 deletions(-)

diff --git a/src/Compression/CompressionFactoryAdditions.cpp b/src/Compression/CompressionFactoryAdditions.cpp
index 3e215076871..171eee48730 100644
--- a/src/Compression/CompressionFactoryAdditions.cpp
+++ b/src/Compression/CompressionFactoryAdditions.cpp
@@ -59,15 +59,15 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
     {
         ASTPtr codecs_descriptions = std::make_shared<ASTExpressionList>();
 
-        bool is_compression = false;
-        bool has_none = false;
+        bool with_compressing_codec = false;
+        bool with_none_codec = false;
         std::optional<size_t> generic_compression_codec_pos;
-        std::set<size_t> encryption_codecs;
+        std::set<size_t> encryption_codecs_pos;
 
         bool can_substitute_codec_arguments = true;
         for (size_t i = 0, size = func->arguments->children.size(); i < size; ++i)
         {
-            const auto & inner_codec_ast = func->arguments->children[i];
+            const ASTPtr & inner_codec_ast = func->arguments->children[i];
             String codec_family_name;
             ASTPtr codec_arguments;
             if (const auto * family_name = inner_codec_ast->as<ASTIdentifier>())
@@ -83,8 +83,7 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
             else
                 throw Exception("Unexpected AST element for compression codec", ErrorCodes::UNEXPECTED_AST_STRUCTURE);
 
-            /// Default codec replaced with current default codec which may depend on different
-            /// settings (and properties of data) in runtime.
+            /// Replace "Default" codec by configured default codec which may depend on different settings and data properties at runtime.
             CompressionCodecPtr result_codec;
             if (codec_family_name == DEFAULT_CODEC_NAME)
             {
@@ -136,21 +135,21 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
                 codecs_descriptions->children.emplace_back(result_codec->getCodecDesc());
             }
 
-            is_compression |= result_codec->isCompression();
-            has_none |= result_codec->isNone();
+            with_compressing_codec |= result_codec->isCompression();
+            with_none_codec |= result_codec->isNone();
 
             if (!generic_compression_codec_pos && result_codec->isGenericCompression())
                 generic_compression_codec_pos = i;
 
             if (result_codec->isEncryption())
-                encryption_codecs.insert(i);
+                encryption_codecs_pos.insert(i);
         }
 
         String codec_description = queryToString(codecs_descriptions);
 
         if (sanity_check)
         {
-            if (codecs_descriptions->children.size() > 1 && has_none)
+            if (codecs_descriptions->children.size() > 1 && with_none_codec)
                 throw Exception(
                     "It does not make sense to have codec NONE along with other compression codecs: " + codec_description
                         + ". (Note: you can enable setting 'allow_suspicious_codecs' to skip this check).",
@@ -159,7 +158,7 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
             /// Allow to explicitly specify single NONE codec if user don't want any compression.
             /// But applying other transformations solely without compression (e.g. Delta) does not make sense.
             /// It's okay to apply encryption codecs solely without anything else.
-            if (!is_compression && !has_none && encryption_codecs.size() != codecs_descriptions->children.size())
+            if (!with_compressing_codec && !with_none_codec && encryption_codecs_pos.size() != codecs_descriptions->children.size())
                 throw Exception(
                     "Compression codec " + codec_description
                         + " does not compress anything."
@@ -171,8 +170,8 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
 
             /// It does not make sense to apply any non-encryption codecs
             /// after encryption one.
-            if (!encryption_codecs.empty() &&
-                *encryption_codecs.begin() != codecs_descriptions->children.size() - encryption_codecs.size())
+            if (!encryption_codecs_pos.empty() &&
+                *encryption_codecs_pos.begin() != codecs_descriptions->children.size() - encryption_codecs_pos.size())
                 throw Exception("The combination of compression codecs " + codec_description + " is meaningless,"
                                 " because it does not make sense to apply any non-post-processing codecs after"
                                 " post-processing ones. (Note: you can enable setting 'allow_suspicious_codecs'"
@@ -181,7 +180,7 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
             /// It does not make sense to apply any transformations after generic compression algorithm
             /// So, generic compression can be only one and only at the end.
             if (generic_compression_codec_pos &&
-                *generic_compression_codec_pos != codecs_descriptions->children.size() - 1 - encryption_codecs.size())
+                *generic_compression_codec_pos != codecs_descriptions->children.size() - 1 - encryption_codecs_pos.size())
                 throw Exception("The combination of compression codecs " + codec_description + " is meaningless,"
                     " because it does not make sense to apply any transformations after generic compression algorithm."
                     " (Note: you can enable setting 'allow_suspicious_codecs' to skip this check).", ErrorCodes::BAD_ARGUMENTS);
diff --git a/tests/queries/0_stateless/01272_suspicious_codecs.sql b/tests/queries/0_stateless/01272_suspicious_codecs.sql
index a2cae22ccc2..0553c64b818 100644
--- a/tests/queries/0_stateless/01272_suspicious_codecs.sql
+++ b/tests/queries/0_stateless/01272_suspicious_codecs.sql
@@ -21,14 +21,14 @@ DROP TABLE codecs;
 
 -- test what should not work
 
-CREATE TABLE codecs (a UInt8 CODEC(NONE, NONE)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 36 }
-CREATE TABLE codecs (a UInt8 CODEC(NONE, LZ4)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 36 }
-CREATE TABLE codecs (a UInt8 CODEC(LZ4, NONE)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 36 }
-CREATE TABLE codecs (a UInt8 CODEC(LZ4, LZ4)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 36 }
-CREATE TABLE codecs (a UInt8 CODEC(LZ4, ZSTD)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 36 }
-CREATE TABLE codecs (a UInt8 CODEC(Delta)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 36 }
-CREATE TABLE codecs (a UInt8 CODEC(Delta, Delta)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 36 }
-CREATE TABLE codecs (a UInt8 CODEC(LZ4, Delta)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 36 }
+CREATE TABLE codecs (a UInt8 CODEC(NONE, NONE)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS }
+CREATE TABLE codecs (a UInt8 CODEC(NONE, LZ4)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS }
+CREATE TABLE codecs (a UInt8 CODEC(LZ4, NONE)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS }
+CREATE TABLE codecs (a UInt8 CODEC(LZ4, LZ4)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS }
+CREATE TABLE codecs (a UInt8 CODEC(LZ4, ZSTD)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS }
+CREATE TABLE codecs (a UInt8 CODEC(Delta)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS }
+CREATE TABLE codecs (a UInt8 CODEC(Delta, Delta)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS }
+CREATE TABLE codecs (a UInt8 CODEC(LZ4, Delta)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS }
 
 -- test that sanity check is not performed in ATTACH query
 

From a4a6126c9d566ea793879a30c9d35eda04165fd2 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sat, 14 Jan 2023 19:57:18 +0000
Subject: [PATCH 124/158] Prohibit manual delta compression before
 floating-point time series compression

---
 src/Compression/CompressionCodecDelta.cpp     |  1 +
 .../CompressionCodecDoubleDelta.cpp           |  1 +
 src/Compression/CompressionCodecFPC.cpp       |  1 +
 src/Compression/CompressionCodecGorilla.cpp   |  1 +
 .../CompressionFactoryAdditions.cpp           | 15 ++++++++++++
 src/Compression/ICompressionCodec.h           |  6 +++++
 .../01272_suspicious_codecs.reference         |  4 ++++
 .../0_stateless/01272_suspicious_codecs.sql   | 24 +++++++++++++++++--
 8 files changed, 51 insertions(+), 2 deletions(-)

diff --git a/src/Compression/CompressionCodecDelta.cpp b/src/Compression/CompressionCodecDelta.cpp
index 31800b6b332..88534f6082b 100644
--- a/src/Compression/CompressionCodecDelta.cpp
+++ b/src/Compression/CompressionCodecDelta.cpp
@@ -28,6 +28,7 @@ protected:
 
     bool isCompression() const override { return false; }
     bool isGenericCompression() const override { return false; }
+    bool isDelta() const override { return true; }
 
 private:
     UInt8 delta_bytes_size;
diff --git a/src/Compression/CompressionCodecDoubleDelta.cpp b/src/Compression/CompressionCodecDoubleDelta.cpp
index dd2507ab14a..a18ce81e02b 100644
--- a/src/Compression/CompressionCodecDoubleDelta.cpp
+++ b/src/Compression/CompressionCodecDoubleDelta.cpp
@@ -133,6 +133,7 @@ protected:
 
     bool isCompression() const override { return true; }
     bool isGenericCompression() const override { return false; }
+    bool isDelta() const override { return true; }
 
 private:
     UInt8 data_bytes_size;
diff --git a/src/Compression/CompressionCodecFPC.cpp b/src/Compression/CompressionCodecFPC.cpp
index 48eba210b60..0468adc4610 100644
--- a/src/Compression/CompressionCodecFPC.cpp
+++ b/src/Compression/CompressionCodecFPC.cpp
@@ -39,6 +39,7 @@ protected:
 
     bool isCompression() const override { return true; }
     bool isGenericCompression() const override { return false; }
+    bool isFloatingPointTimeSeries() const override { return true; }
 
 private:
     static constexpr UInt32 HEADER_SIZE = 2;
diff --git a/src/Compression/CompressionCodecGorilla.cpp b/src/Compression/CompressionCodecGorilla.cpp
index 5714ba00a50..a3aa5850e19 100644
--- a/src/Compression/CompressionCodecGorilla.cpp
+++ b/src/Compression/CompressionCodecGorilla.cpp
@@ -123,6 +123,7 @@ protected:
 
     bool isCompression() const override { return true; }
     bool isGenericCompression() const override { return false; }
+    bool isFloatingPointTimeSeries() const override { return true; }
 
 private:
     UInt8 data_bytes_size;
diff --git a/src/Compression/CompressionFactoryAdditions.cpp b/src/Compression/CompressionFactoryAdditions.cpp
index 171eee48730..fd2d6a67aa8 100644
--- a/src/Compression/CompressionFactoryAdditions.cpp
+++ b/src/Compression/CompressionFactoryAdditions.cpp
@@ -62,6 +62,8 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
         bool with_compressing_codec = false;
         bool with_none_codec = false;
         std::optional<size_t> generic_compression_codec_pos;
+        std::optional<size_t> first_delta_codec_pos;
+        std::optional<size_t> last_floating_point_time_series_codec_pos;
         std::set<size_t> encryption_codecs_pos;
 
         bool can_substitute_codec_arguments = true;
@@ -143,6 +145,12 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
 
             if (result_codec->isEncryption())
                 encryption_codecs_pos.insert(i);
+
+            if (result_codec->isDelta() && !first_delta_codec_pos.has_value())
+                first_delta_codec_pos = i;
+
+            if (result_codec->isFloatingPointTimeSeries())
+                last_floating_point_time_series_codec_pos = i;
         }
 
         String codec_description = queryToString(codecs_descriptions);
@@ -185,6 +193,13 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
                     " because it does not make sense to apply any transformations after generic compression algorithm."
                     " (Note: you can enable setting 'allow_suspicious_codecs' to skip this check).", ErrorCodes::BAD_ARGUMENTS);
 
+            /// Floating point time series codecs usually have implicit delta compression (or something equivalent), so it does not make
+            /// sense to run delta compression manually. Another reason for blocking such combination is occasional data corruption (#45195).
+            if (first_delta_codec_pos.has_value() && last_floating_point_time_series_codec_pos.has_value()
+                && (*first_delta_codec_pos < last_floating_point_time_series_codec_pos))
+                throw Exception("The combination of compression codecs " + codec_description + " is meaningless,"
+                    " because it does not make sense to apply delta transformations before floating point time series codecs."
+                    " (Note: you can enable setting 'allow_suspicious_codecs' to skip this check).", ErrorCodes::BAD_ARGUMENTS);
         }
 
         /// For columns with nested types like Tuple(UInt32, UInt64) we
diff --git a/src/Compression/ICompressionCodec.h b/src/Compression/ICompressionCodec.h
index f40404a84f3..c717392bbca 100644
--- a/src/Compression/ICompressionCodec.h
+++ b/src/Compression/ICompressionCodec.h
@@ -113,6 +113,12 @@ public:
     /// If it does nothing.
     virtual bool isNone() const { return false; }
 
+    /// If the only purpose of the codec is to delta (or double-delta) the data.
+    virtual bool isDelta() const { return false; }
+
+    /// If the codec is specialized for floating point time series.
+    virtual bool isFloatingPointTimeSeries() const { return false; }
+
 protected:
     /// This is used for fuzz testing
     friend int LLVMFuzzerTestOneInput(const uint8_t * data, size_t size);
diff --git a/tests/queries/0_stateless/01272_suspicious_codecs.reference b/tests/queries/0_stateless/01272_suspicious_codecs.reference
index 559b6df2693..97e60839ea7 100644
--- a/tests/queries/0_stateless/01272_suspicious_codecs.reference
+++ b/tests/queries/0_stateless/01272_suspicious_codecs.reference
@@ -6,6 +6,8 @@ CREATE TABLE default.codecs5\n(\n    `a` UInt8 CODEC(LZ4, ZSTD(1))\n)\nENGINE =
 CREATE TABLE default.codecs6\n(\n    `a` UInt8 CODEC(Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
 CREATE TABLE default.codecs7\n(\n    `a` UInt8 CODEC(Delta(1), Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
 CREATE TABLE default.codecs8\n(\n    `a` UInt8 CODEC(LZ4, Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
+CREATE TABLE default.codecs9\n(\n    `a` Float64 CODEC(Delta(8), FPC(12))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
+CREATE TABLE default.codecs10\n(\n    `a` Float64 CODEC(DoubleDelta, Gorilla)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
 CREATE TABLE default.codecs1\n(\n    `a` UInt8 CODEC(NONE, NONE)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
 CREATE TABLE default.codecs2\n(\n    `a` UInt8 CODEC(NONE, LZ4)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
 CREATE TABLE default.codecs3\n(\n    `a` UInt8 CODEC(LZ4, NONE)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
@@ -14,3 +16,5 @@ CREATE TABLE default.codecs5\n(\n    `a` UInt8 CODEC(LZ4, ZSTD(1))\n)\nENGINE =
 CREATE TABLE default.codecs6\n(\n    `a` UInt8 CODEC(Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
 CREATE TABLE default.codecs7\n(\n    `a` UInt8 CODEC(Delta(1), Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
 CREATE TABLE default.codecs8\n(\n    `a` UInt8 CODEC(LZ4, Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
+CREATE TABLE default.codecs9\n(\n    `a` Float64 CODEC(Delta(8), FPC(12))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
+CREATE TABLE default.codecs10\n(\n    `a` Float64 CODEC(DoubleDelta, Gorilla)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
diff --git a/tests/queries/0_stateless/01272_suspicious_codecs.sql b/tests/queries/0_stateless/01272_suspicious_codecs.sql
index 0553c64b818..660a9a5edf8 100644
--- a/tests/queries/0_stateless/01272_suspicious_codecs.sql
+++ b/tests/queries/0_stateless/01272_suspicious_codecs.sql
@@ -11,10 +11,12 @@ CREATE TABLE codecs
     c Float32 CODEC(Gorilla),
     d UInt8 CODEC(Delta, LZ4),
     e Float64 CODEC(Gorilla, ZSTD),
-    f Float64 CODEC(Delta, Delta, Gorilla),
+    f UInt64 CODEC(Delta, Delta, T64),
     g DateTime CODEC(DoubleDelta),
     h DateTime64 CODEC(DoubleDelta, LZ4),
-    i String CODEC(NONE)
+    i String CODEC(NONE),
+    j Float64 (Gorilla, Delta),
+    k Float32 (FPC, DoubleDelta)
 ) ENGINE = MergeTree ORDER BY tuple();
 
 DROP TABLE codecs;
@@ -29,6 +31,8 @@ CREATE TABLE codecs (a UInt8 CODEC(LZ4, ZSTD)) ENGINE = MergeTree ORDER BY tuple
 CREATE TABLE codecs (a UInt8 CODEC(Delta)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS }
 CREATE TABLE codecs (a UInt8 CODEC(Delta, Delta)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS }
 CREATE TABLE codecs (a UInt8 CODEC(LZ4, Delta)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS }
+CREATE TABLE codecs (a UInt8 CODEC(Delta, FPC)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS }
+CREATE TABLE codecs (a UInt8 CODEC(DoubleDelta, Gorilla)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS }
 
 -- test that sanity check is not performed in ATTACH query
 
@@ -40,6 +44,8 @@ DROP TABLE IF EXISTS codecs5;
 DROP TABLE IF EXISTS codecs6;
 DROP TABLE IF EXISTS codecs7;
 DROP TABLE IF EXISTS codecs8;
+DROP TABLE IF EXISTS codecs9;
+DROP TABLE IF EXISTS codecs10;
 
 SET allow_suspicious_codecs = 1;
 
@@ -51,6 +57,8 @@ CREATE TABLE codecs5 (a UInt8 CODEC(LZ4, ZSTD)) ENGINE = MergeTree ORDER BY tupl
 CREATE TABLE codecs6 (a UInt8 CODEC(Delta)) ENGINE = MergeTree ORDER BY tuple();
 CREATE TABLE codecs7 (a UInt8 CODEC(Delta, Delta)) ENGINE = MergeTree ORDER BY tuple();
 CREATE TABLE codecs8 (a UInt8 CODEC(LZ4, Delta)) ENGINE = MergeTree ORDER BY tuple();
+CREATE TABLE codecs9 (a Float64 CODEC(Delta, FPC)) ENGINE = MergeTree ORDER BY tuple();
+CREATE TABLE codecs10 (a Float64 CODEC(DoubleDelta, Gorilla)) ENGINE = MergeTree ORDER BY tuple();
 
 SET allow_suspicious_codecs = 0;
 
@@ -62,6 +70,8 @@ SHOW CREATE TABLE codecs5;
 SHOW CREATE TABLE codecs6;
 SHOW CREATE TABLE codecs7;
 SHOW CREATE TABLE codecs8;
+SHOW CREATE TABLE codecs9;
+SHOW CREATE TABLE codecs10;
 
 DETACH TABLE codecs1;
 DETACH TABLE codecs2;
@@ -71,6 +81,8 @@ DETACH TABLE codecs5;
 DETACH TABLE codecs6;
 DETACH TABLE codecs7;
 DETACH TABLE codecs8;
+DETACH TABLE codecs9;
+DETACH TABLE codecs10;
 
 ATTACH TABLE codecs1;
 ATTACH TABLE codecs2;
@@ -80,6 +92,8 @@ ATTACH TABLE codecs5;
 ATTACH TABLE codecs6;
 ATTACH TABLE codecs7;
 ATTACH TABLE codecs8;
+ATTACH TABLE codecs9;
+ATTACH TABLE codecs10;
 
 SHOW CREATE TABLE codecs1;
 SHOW CREATE TABLE codecs2;
@@ -89,6 +103,8 @@ SHOW CREATE TABLE codecs5;
 SHOW CREATE TABLE codecs6;
 SHOW CREATE TABLE codecs7;
 SHOW CREATE TABLE codecs8;
+SHOW CREATE TABLE codecs9;
+SHOW CREATE TABLE codecs10;
 
 SELECT * FROM codecs1;
 SELECT * FROM codecs2;
@@ -98,6 +114,8 @@ SELECT * FROM codecs5;
 SELECT * FROM codecs6;
 SELECT * FROM codecs7;
 SELECT * FROM codecs8;
+SELECT * FROM codecs9;
+SELECT * FROM codecs10;
 
 DROP TABLE codecs1;
 DROP TABLE codecs2;
@@ -107,3 +125,5 @@ DROP TABLE codecs5;
 DROP TABLE codecs6;
 DROP TABLE codecs7;
 DROP TABLE codecs8;
+DROP TABLE codecs9;
+DROP TABLE codecs10;

From b5773caeb232ddb3c0ac4e70d34dedee10cc5569 Mon Sep 17 00:00:00 2001
From: Tanya Bragin <tbragin@users.noreply.github.com>
Date: Sat, 14 Jan 2023 14:34:33 -0800
Subject: [PATCH 125/158] Update postgresql.md

Adding related Postgres blog here.

cc @gingerwizard @DanRoscigno
---
 docs/en/engines/table-engines/integrations/postgresql.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/en/engines/table-engines/integrations/postgresql.md b/docs/en/engines/table-engines/integrations/postgresql.md
index 7f9659400b8..b73d28c8508 100644
--- a/docs/en/engines/table-engines/integrations/postgresql.md
+++ b/docs/en/engines/table-engines/integrations/postgresql.md
@@ -175,3 +175,6 @@ CREATE TABLE pg_table_schema_with_dots (a UInt32)
 
 -   [The `postgresql` table function](../../../sql-reference/table-functions/postgresql.md)
 -   [Using PostgreSQL as a dictionary source](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md#dicts-external_dicts_dict_sources-postgresql)
+
+## Related content
+- Blog: [ClickHouse and PostgreSQL - a match made in data heaven - part 1](https://clickhouse.com/blog/migrating-data-between-clickhouse-postgres)

From d825f50c8487d3153a113ae24fcd04d9c656f524 Mon Sep 17 00:00:00 2001
From: Tanya Bragin <tbragin@users.noreply.github.com>
Date: Sat, 14 Jan 2023 14:39:41 -0800
Subject: [PATCH 126/158] Update postgresql.md

Adding Postgres blog to Postgres DB engine page

cc @DanRoscigno @gingerwizard
---
 docs/en/engines/database-engines/postgresql.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/en/engines/database-engines/postgresql.md b/docs/en/engines/database-engines/postgresql.md
index ac19794c167..939995a61c5 100644
--- a/docs/en/engines/database-engines/postgresql.md
+++ b/docs/en/engines/database-engines/postgresql.md
@@ -136,3 +136,7 @@ DESCRIBE TABLE test_database.test_table;
 │ data   │ Nullable(String)  │
 └────────┴───────────────────┘
 ```
+
+## Related content
+
+- Blog: [ClickHouse and PostgreSQL - a match made in data heaven - part 1](https://clickhouse.com/blog/migrating-data-between-clickhouse-postgres)

From 7023d685369d860236db2e664cd0a22e35576d14 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sun, 15 Jan 2023 13:31:45 +0000
Subject: [PATCH 127/158] Fix codecs_int_*.xml

---
 tests/performance/codecs_int_insert.xml | 1 -
 tests/performance/codecs_int_select.xml | 1 -
 2 files changed, 2 deletions(-)

diff --git a/tests/performance/codecs_int_insert.xml b/tests/performance/codecs_int_insert.xml
index caefaba3725..618e20160f8 100644
--- a/tests/performance/codecs_int_insert.xml
+++ b/tests/performance/codecs_int_insert.xml
@@ -13,7 +13,6 @@
                 <value>Delta</value>
                 <value>T64</value>
                 <value>DoubleDelta</value>
-                <value>Gorilla</value>
             </values>
         </substitution>
         <substitution>
diff --git a/tests/performance/codecs_int_select.xml b/tests/performance/codecs_int_select.xml
index 7d47cd300d8..62c1ee16e7b 100644
--- a/tests/performance/codecs_int_select.xml
+++ b/tests/performance/codecs_int_select.xml
@@ -13,7 +13,6 @@
                 <value>Delta</value>
                 <value>T64</value>
                 <value>DoubleDelta</value>
-                <value>Gorilla</value>
             </values>
         </substitution>
         <substitution>

From 24b686734d41f3b7c2513b2762599f3b8356d271 Mon Sep 17 00:00:00 2001
From: Nikolay Degterinsky <evillique@gmail.com>
Date: Sun, 15 Jan 2023 13:46:55 +0000
Subject: [PATCH 128/158] Fix build

---
 src/Interpreters/JoinToSubqueryTransformVisitor.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp
index 350028bf587..5500c274c23 100644
--- a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp
+++ b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp
@@ -175,9 +175,9 @@ private:
                 {
                     for (const auto & transformer : qualified_asterisk->transformers->children)
                     {
-                        if (transformer.get()->as<ASTColumnsApplyTransformer>() ||
-                            transformer.get()->as<ASTColumnsExceptTransformer>() ||
-                            transformer.get()->as<ASTColumnsReplaceTransformer>())
+                        if (transformer->as<ASTColumnsApplyTransformer>() ||
+                            transformer->as<ASTColumnsExceptTransformer>() ||
+                            transformer->as<ASTColumnsReplaceTransformer>())
                             IASTColumnsTransformer::transform(transformer, columns);
                         else
                             throw Exception("Logical error: qualified asterisk must only have children of IASTColumnsTransformer type", ErrorCodes::LOGICAL_ERROR);

From b6f12f9edd6c26873ef301b597ef45dd5ed1b06b Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sun, 15 Jan 2023 14:49:04 +0000
Subject: [PATCH 129/158] Fix unit_tests_dbms

---
 .../tests/gtest_compressionCodec.cpp          | 83 +++++--------------
 1 file changed, 21 insertions(+), 62 deletions(-)

diff --git a/src/Compression/tests/gtest_compressionCodec.cpp b/src/Compression/tests/gtest_compressionCodec.cpp
index b4c29bf9ce6..a843144c7c4 100644
--- a/src/Compression/tests/gtest_compressionCodec.cpp
+++ b/src/Compression/tests/gtest_compressionCodec.cpp
@@ -529,6 +529,13 @@ public:
 
 TEST_P(CodecTest, TranscodingWithDataType)
 {
+    /// Gorilla can only be applied to floating point columns
+    bool codec_is_gorilla = std::get<0>(GetParam()).codec_statement.find("Gorilla") != std::string::npos;
+    WhichDataType which(std::get<1>(GetParam()).data_type.get());
+    bool data_is_float = which.isFloat();
+    if (codec_is_gorilla && !data_is_float)
+        GTEST_SKIP() << "Skipping Gorilla-compressed integer column";
+
     const auto codec = makeCodec(CODEC_WITH_DATA_TYPE);
     testTranscoding(*codec);
 }
@@ -1204,68 +1211,20 @@ auto DDperformanceTestSequence()
 }
 
 // prime numbers in ascending order with some random repetitions hit all the cases of Gorilla.
-auto PrimesWithMultiplierGenerator = [](int multiplier = 1)
-{
-    return [multiplier](auto i)
-    {
-        static const int vals[] = {
-             2, 3, 5, 7, 11, 11, 13, 17, 19, 23, 29, 29, 31, 37, 41, 43,
-            47, 47, 53, 59, 61, 61, 67, 71, 73, 79, 83, 89, 89, 97, 101, 103,
-            107, 107, 109, 113, 113, 127, 127, 127
-        };
-        static const size_t count = sizeof(vals)/sizeof(vals[0]);
-
-        return static_cast<UInt64>(vals[i % count]) * multiplier;
-    };
-};
-
-template <typename ValueType>
-auto GCompatibilityTestSequence()
-{
-    // Also multiply result by some factor to test large values on types that can hold those.
-    return generateSeq<ValueType>(G(PrimesWithMultiplierGenerator(intExp10(sizeof(ValueType)))), 0, 42);
-}
-
-INSTANTIATE_TEST_SUITE_P(Gorilla,
-    CodecTestCompatibility,
-    ::testing::Combine(
-        ::testing::Values(Codec("Gorilla")),
-        ::testing::ValuesIn(std::initializer_list<std::tuple<CodecTestSequence, std::string>>{
-            {
-                GCompatibilityTestSequence<Int8>(),
-                BIN_STR("\x95\x35\x00\x00\x00\x2a\x00\x00\x00\x01\x00\x2a\x00\x00\x00\x14\xe1\xdd\x25\xe5\x7b\x29\x86\xee\x2a\x16\x5a\xc5\x0b\x23\x75\x1b\x3c\xb1\x97\x8b\x5f\xcb\x43\xd9\xc5\x48\xab\x23\xaf\x62\x93\x71\x4a\x73\x0f\xc6\x0a")
-            },
-            {
-                GCompatibilityTestSequence<UInt8>(),
-                BIN_STR("\x95\x35\x00\x00\x00\x2a\x00\x00\x00\x01\x00\x2a\x00\x00\x00\x14\xe1\xdd\x25\xe5\x7b\x29\x86\xee\x2a\x16\x5a\xc5\x0b\x23\x75\x1b\x3c\xb1\x97\x8b\x5f\xcb\x43\xd9\xc5\x48\xab\x23\xaf\x62\x93\x71\x4a\x73\x0f\xc6\x0a")
-            },
-            {
-                GCompatibilityTestSequence<Int16>(),
-                BIN_STR("\x95\x52\x00\x00\x00\x54\x00\x00\x00\x02\x00\x2a\x00\x00\x00\xc8\x00\xdc\xfe\x66\xdb\x1f\x4e\xa7\xde\xdc\xd5\xec\x6e\xf7\x37\x3a\x23\xe7\x63\xf5\x6a\x8e\x99\x37\x34\xf9\xf8\x2e\x76\x35\x2d\x51\xbb\x3b\xc3\x6d\x13\xbf\x86\x53\x9e\x25\xe4\xaf\xaf\x63\xd5\x6a\x6e\x76\x35\x3a\x27\xd3\x0f\x91\xae\x6b\x33\x57\x6e\x64\xcc\x55\x81\xe4")
-            },
-            {
-                GCompatibilityTestSequence<UInt16>(),
-                BIN_STR("\x95\x52\x00\x00\x00\x54\x00\x00\x00\x02\x00\x2a\x00\x00\x00\xc8\x00\xdc\xfe\x66\xdb\x1f\x4e\xa7\xde\xdc\xd5\xec\x6e\xf7\x37\x3a\x23\xe7\x63\xf5\x6a\x8e\x99\x37\x34\xf9\xf8\x2e\x76\x35\x2d\x51\xbb\x3b\xc3\x6d\x13\xbf\x86\x53\x9e\x25\xe4\xaf\xaf\x63\xd5\x6a\x6e\x76\x35\x3a\x27\xd3\x0f\x91\xae\x6b\x33\x57\x6e\x64\xcc\x55\x81\xe4")
-            },
-            {
-                GCompatibilityTestSequence<Int32>(),
-                BIN_STR("\x95\x65\x00\x00\x00\xa8\x00\x00\x00\x04\x00\x2a\x00\x00\x00\x20\x4e\x00\x00\xe4\x57\x63\xc0\xbb\x67\xbc\xce\x91\x97\x99\x15\x9e\xe3\x36\x3f\x89\x5f\x8e\xf2\xec\x8e\xd3\xbf\x75\x43\x58\xc4\x7e\xcf\x93\x43\x38\xc6\x91\x36\x1f\xe7\xb6\x11\x6f\x02\x73\x46\xef\xe0\xec\x50\xfb\x79\xcb\x9c\x14\xfa\x13\xea\x8d\x66\x43\x48\xa0\xde\x3a\xcf\xff\x26\xe0\x5f\x93\xde\x5e\x7f\x6e\x36\x5e\xe6\xb4\x66\x5d\xb0\x0e\xc4")
-            },
-            {
-                GCompatibilityTestSequence<UInt32>(),
-                BIN_STR("\x95\x65\x00\x00\x00\xa8\x00\x00\x00\x04\x00\x2a\x00\x00\x00\x20\x4e\x00\x00\xe4\x57\x63\xc0\xbb\x67\xbc\xce\x91\x97\x99\x15\x9e\xe3\x36\x3f\x89\x5f\x8e\xf2\xec\x8e\xd3\xbf\x75\x43\x58\xc4\x7e\xcf\x93\x43\x38\xc6\x91\x36\x1f\xe7\xb6\x11\x6f\x02\x73\x46\xef\xe0\xec\x50\xfb\x79\xcb\x9c\x14\xfa\x13\xea\x8d\x66\x43\x48\xa0\xde\x3a\xcf\xff\x26\xe0\x5f\x93\xde\x5e\x7f\x6e\x36\x5e\xe6\xb4\x66\x5d\xb0\x0e\xc4")
-            },
-            {
-                GCompatibilityTestSequence<Int64>(),
-                BIN_STR("\x95\x91\x00\x00\x00\x50\x01\x00\x00\x08\x00\x2a\x00\x00\x00\x00\xc2\xeb\x0b\x00\x00\x00\x00\xe3\x2b\xa0\xa6\x19\x85\x98\xdc\x45\x74\x74\x43\xc2\x57\x41\x4c\x6e\x42\x79\xd9\x8f\x88\xa5\x05\xf3\xf1\x94\xa3\x62\x1e\x02\xdf\x05\x10\xf1\x15\x97\x35\x2a\x50\x71\x0f\x09\x6c\x89\xf7\x65\x1d\x11\xb7\xcc\x7d\x0b\x70\xc1\x86\x88\x48\x47\x87\xb6\x32\x26\xa7\x86\x87\x88\xd3\x93\x3d\xfc\x28\x68\x85\x05\x0b\x13\xc6\x5f\xd4\x70\xe1\x5e\x76\xf1\x9f\xf3\x33\x2a\x14\x14\x5e\x40\xc1\x5c\x28\x3f\xec\x43\x03\x05\x11\x91\xe8\xeb\x8e\x0a\x0e\x27\x21\x55\xcb\x39\xbc\x6a\xff\x11\x5d\x81\xa0\xa6\x10")
-            },
-            {
-                GCompatibilityTestSequence<UInt64>(),
-                BIN_STR("\x95\x91\x00\x00\x00\x50\x01\x00\x00\x08\x00\x2a\x00\x00\x00\x00\xc2\xeb\x0b\x00\x00\x00\x00\xe3\x2b\xa0\xa6\x19\x85\x98\xdc\x45\x74\x74\x43\xc2\x57\x41\x4c\x6e\x42\x79\xd9\x8f\x88\xa5\x05\xf3\xf1\x94\xa3\x62\x1e\x02\xdf\x05\x10\xf1\x15\x97\x35\x2a\x50\x71\x0f\x09\x6c\x89\xf7\x65\x1d\x11\xb7\xcc\x7d\x0b\x70\xc1\x86\x88\x48\x47\x87\xb6\x32\x26\xa7\x86\x87\x88\xd3\x93\x3d\xfc\x28\x68\x85\x05\x0b\x13\xc6\x5f\xd4\x70\xe1\x5e\x76\xf1\x9f\xf3\x33\x2a\x14\x14\x5e\x40\xc1\x5c\x28\x3f\xec\x43\x03\x05\x11\x91\xe8\xeb\x8e\x0a\x0e\x27\x21\x55\xcb\x39\xbc\x6a\xff\x11\x5d\x81\xa0\xa6\x10")
-            },
-        })
-    )
-);
+// auto PrimesWithMultiplierGenerator = [](int multiplier = 1)
+// {
+//     return [multiplier](auto i)
+//     {
+//         static const int vals[] = {
+//              2, 3, 5, 7, 11, 11, 13, 17, 19, 23, 29, 29, 31, 37, 41, 43,
+//             47, 47, 53, 59, 61, 61, 67, 71, 73, 79, 83, 89, 89, 97, 101, 103,
+//             107, 107, 109, 113, 113, 127, 127, 127
+//         };
+//         static const size_t count = sizeof(vals)/sizeof(vals[0]);
+//
+//         return static_cast<UInt64>(vals[i % count]) * multiplier;
+//     };
+// };
 
 // These 'tests' try to measure performance of encoding and decoding and hence only make sense to be run locally,
 // also they require pretty big data to run against and generating this data slows down startup of unit test process.

From af1308083ab6938c01c59f70e47d2dc248330fe9 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 15 Jan 2023 17:02:57 +0100
Subject: [PATCH 130/158] tests/ci: encode HTML entities in the reports

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 tests/ci/report.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/ci/report.py b/tests/ci/report.py
index c2d82ee38a1..da04411632d 100644
--- a/tests/ci/report.py
+++ b/tests/ci/report.py
@@ -3,6 +3,7 @@ from ast import literal_eval
 from dataclasses import dataclass
 from pathlib import Path
 from typing import List, Optional, Tuple
+from html import escape
 import csv
 import os
 import datetime
@@ -372,9 +373,10 @@ def create_test_html_report(
             row += "</tr>"
             rows_part += row
             if test_result.raw_logs is not None:
+                raw_logs = escape(test_result.raw_logs)
                 row = (
                     '<tr class="failed-content">'
-                    f'<td colspan="{colspan}"><pre>{test_result.raw_logs}</pre></td>'
+                    f'<td colspan="{colspan}"><pre>{raw_logs}</pre></td>'
                     "</tr>"
                 )
                 rows_part += row

From 683a51633e275c1052920ae5ce010b219ed347d5 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 15 Jan 2023 16:16:17 +0100
Subject: [PATCH 131/158] tests: fix clickhouse binaries detection

Previosly only clickhouse-client binary was more or less detected, but
everything else does not, since "test -x clickhouse" does not work as
expected, "test -x" need a full path to the file, it does not search in
PATH.

Also improve clickhouse-test default --binary path, by properly search
in PATH.

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 tests/clickhouse-test         | 43 ++++++++++++++++-------------------
 tests/queries/shell_config.sh |  2 +-
 2 files changed, 21 insertions(+), 24 deletions(-)

diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index 39bb9aade3c..2c9b807574f 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -1958,19 +1958,22 @@ def main(args):
 
 
 def find_binary(name):
-    if os.path.exists(name) and os.access(name, os.X_OK):
-        return True
+    if os.access(name, os.X_OK):
+        return name
     paths = os.environ.get("PATH").split(":")
     for path in paths:
-        if os.access(os.path.join(path, name), os.X_OK):
-            return True
+        bin_path = os.path.join(path, name)
+        if os.access(bin_path, os.X_OK):
+            return bin_path
 
     # maybe it wasn't in PATH
-    if os.access(os.path.join("/usr/local/bin", name), os.X_OK):
-        return True
-    if os.access(os.path.join("/usr/bin", name), os.X_OK):
-        return True
-    return False
+    bin_path = os.path.join("/usr/local/bin", name)
+    if os.access(bin_path, os.X_OK):
+        return bin_path
+    bin_path = os.path.join("/usr/bin", name)
+    if os.access(bin_path, os.X_OK):
+        return bin_path
+    return None
 
 
 def get_additional_client_options(args):
@@ -2010,9 +2013,8 @@ if __name__ == "__main__":
     parser.add_argument(
         "-b",
         "--binary",
-        default="clickhouse",
-        help="Path to clickhouse"
-        "binary or name of binary in PATH",
+        default=find_binary("clickhouse"),
+        help="Path to clickhouse binary or name of binary in PATH",
     )
 
     parser.add_argument(
@@ -2282,18 +2284,13 @@ if __name__ == "__main__":
     if args.tmp is None:
         args.tmp = args.queries
     if args.client is None:
-        if find_binary(args.binary + "-client"):
-            args.client = args.binary + "-client"
-
-            print("Using " + args.client + " as client program")
-        elif find_binary(args.binary):
+        client_bin = find_binary(args.binary + "-client")
+        if client_bin is not None:
+            args.client = client_bin
+            print("Using {args.client} as client program")
+        elif args.binary:
             args.client = args.binary + " client"
-
-            print(
-                "Using "
-                + args.client
-                + " as client program (expecting monolithic build)"
-            )
+            print(f"Using {args.client} as client program (expecting monolithic build)")
         else:
             print(
                 "No 'clickhouse' or 'clickhouse-client' client binary found",
diff --git a/tests/queries/shell_config.sh b/tests/queries/shell_config.sh
index 8e465eb19e2..3006b74d3f9 100644
--- a/tests/queries/shell_config.sh
+++ b/tests/queries/shell_config.sh
@@ -23,7 +23,7 @@ export CLICKHOUSE_TEST_UNIQUE_NAME="${CLICKHOUSE_TEST_NAME}_${CLICKHOUSE_DATABAS
 [ -n "${CLICKHOUSE_DATABASE:-}" ] && CLICKHOUSE_BENCHMARK_OPT0+=" --database=${CLICKHOUSE_DATABASE} "
 [ -n "${CLICKHOUSE_LOG_COMMENT:-}" ] && CLICKHOUSE_BENCHMARK_OPT0+=" --log_comment $(printf '%q' ${CLICKHOUSE_LOG_COMMENT}) "
 
-export CLICKHOUSE_BINARY=${CLICKHOUSE_BINARY:="clickhouse"}
+export CLICKHOUSE_BINARY=${CLICKHOUSE_BINARY:="$(command -v clickhouse)"}
 # client
 [ -x "$CLICKHOUSE_BINARY-client" ] && CLICKHOUSE_CLIENT_BINARY=${CLICKHOUSE_CLIENT_BINARY:=$CLICKHOUSE_BINARY-client}
 [ -x "$CLICKHOUSE_BINARY" ] && CLICKHOUSE_CLIENT_BINARY=${CLICKHOUSE_CLIENT_BINARY:=$CLICKHOUSE_BINARY client}

From c33d7ffcc294b427a81a3f39192eb8bc8542554a Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Sun, 15 Jan 2023 18:53:50 +0100
Subject: [PATCH 132/158] Disable `02151_hash_table_sizes_stats_distributed.sh`
 under TSAN

---
 .../0_stateless/02151_hash_table_sizes_stats_distributed.sh     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02151_hash_table_sizes_stats_distributed.sh b/tests/queries/0_stateless/02151_hash_table_sizes_stats_distributed.sh
index 6044177ad76..237bbe9edd9 100755
--- a/tests/queries/0_stateless/02151_hash_table_sizes_stats_distributed.sh
+++ b/tests/queries/0_stateless/02151_hash_table_sizes_stats_distributed.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long, distributed
+# Tags: long, distributed, no-tsan
 
 # These tests don't use `current_database = currentDatabase()` condition, because database name isn't propagated during remote queries.
 

From bb2c0914e9a0d0699810ba5ec92401aaf71f650b Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Sun, 15 Jan 2023 19:46:09 +0100
Subject: [PATCH 133/158] Update
 src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp

Co-authored-by: alesapin <alesapin@gmail.com>
---
 src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
index 4aca646fd83..eb91ae0b775 100644
--- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
+++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
@@ -129,7 +129,7 @@ struct ReplicatedMergeTreeSinkImpl<async_insert>::DelayedChunk
                         while (start_pos < end_pos)
                         {
                             *(pos + start_pos) = 0;
-                            start_pos ++;
+                            start_pos++;
                         }
                         it++;
                     }

From c859f8dbe5b1a6ad31f0157a2dcabe51f654d5bc Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Sun, 15 Jan 2023 19:46:16 +0100
Subject: [PATCH 134/158] Update
 src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp

Co-authored-by: alesapin <alesapin@gmail.com>
---
 src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
index eb91ae0b775..c2bff9da51a 100644
--- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
+++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
@@ -76,7 +76,7 @@ struct ReplicatedMergeTreeSinkImpl<async_insert>::DelayedChunk
             if constexpr (async_insert)
             {
                 std::vector<String> dup_block_ids;
-                for (const auto & [hash_id, offset_idxes] : block_id_to_offset_idx)
+                for (const auto & [hash_id, offset_indexes] : block_id_to_offset_idx)
                 {
                     if (offset_idxes.size() > 1)
                         dup_block_ids.push_back(hash_id);

From 21b8aaeb8b94d04eab387cda1748d900c1b0c0ac Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Sat, 14 Jan 2023 03:17:31 +0100
Subject: [PATCH 135/158] Stop using HeadObject requests in S3 because they
 don't work well with endpoints without explicit region.

---
 src/Backups/BackupIO_S3.cpp                   |  16 ++-
 src/Backups/BackupIO_S3.h                     |   4 +-
 src/Common/ProfileEvents.cpp                  |   4 +
 .../ObjectStorages/S3/S3ObjectStorage.cpp     |  46 ++++----
 src/Disks/ObjectStorages/S3/S3ObjectStorage.h |   7 +-
 src/IO/S3Common.cpp                           | 102 +++++++++++++-----
 src/IO/S3Common.h                             |  38 ++++---
 src/IO/WriteBufferFromS3.cpp                  |   8 +-
 8 files changed, 144 insertions(+), 81 deletions(-)

diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp
index 4529f34e2a7..aa667fde06f 100644
--- a/src/Backups/BackupIO_S3.cpp
+++ b/src/Backups/BackupIO_S3.cpp
@@ -156,10 +156,9 @@ void BackupWriterS3::copyObjectImpl(
     const String & src_key,
     const String & dst_bucket,
     const String & dst_key,
-    const Aws::S3::Model::HeadObjectResult & head,
+    size_t size,
     const std::optional<ObjectAttributes> & metadata) const
 {
-    size_t size = head.GetContentLength();
     LOG_TRACE(log, "Copying {} bytes using single-operation copy", size);
 
     Aws::S3::Model::CopyObjectRequest request;
@@ -177,7 +176,7 @@ void BackupWriterS3::copyObjectImpl(
     if (!outcome.IsSuccess() && (outcome.GetError().GetExceptionName() == "EntityTooLarge"
             || outcome.GetError().GetExceptionName() == "InvalidRequest"))
     { // Can't come here with MinIO, MinIO allows single part upload for large objects.
-        copyObjectMultipartImpl(src_bucket, src_key, dst_bucket, dst_key, head, metadata);
+        copyObjectMultipartImpl(src_bucket, src_key, dst_bucket, dst_key, size, metadata);
         return;
     }
 
@@ -191,10 +190,9 @@ void BackupWriterS3::copyObjectMultipartImpl(
     const String & src_key,
     const String & dst_bucket,
     const String & dst_key,
-    const Aws::S3::Model::HeadObjectResult & head,
+    size_t size,
     const std::optional<ObjectAttributes> & metadata) const
 {
-    size_t size = head.GetContentLength();
     LOG_TRACE(log, "Copying {} bytes using multipart upload copy", size);
 
     String multipart_upload_id;
@@ -309,16 +307,16 @@ void BackupWriterS3::copyFileNative(DiskPtr from_disk, const String & file_name_
         std::string source_bucket = object_storage->getObjectsNamespace();
         auto file_path = fs::path(s3_uri.key) / file_name_to;
 
-        auto head = S3::headObject(*client, source_bucket, objects[0].absolute_path).GetResult();
-        if (static_cast<size_t>(head.GetContentLength()) < request_settings.getUploadSettings().max_single_operation_copy_size)
+        auto size = S3::getObjectSize(*client, source_bucket, objects[0].absolute_path);
+        if (size < request_settings.getUploadSettings().max_single_operation_copy_size)
         {
             copyObjectImpl(
-                source_bucket, objects[0].absolute_path, s3_uri.bucket, file_path, head);
+                source_bucket, objects[0].absolute_path, s3_uri.bucket, file_path, size);
         }
         else
         {
             copyObjectMultipartImpl(
-                source_bucket, objects[0].absolute_path, s3_uri.bucket, file_path, head);
+                source_bucket, objects[0].absolute_path, s3_uri.bucket, file_path, size);
         }
     }
 }
diff --git a/src/Backups/BackupIO_S3.h b/src/Backups/BackupIO_S3.h
index 634b35c1e74..70487717c48 100644
--- a/src/Backups/BackupIO_S3.h
+++ b/src/Backups/BackupIO_S3.h
@@ -67,7 +67,7 @@ private:
         const String & src_key,
         const String & dst_bucket,
         const String & dst_key,
-        const Aws::S3::Model::HeadObjectResult & head,
+        size_t size,
         const std::optional<ObjectAttributes> & metadata = std::nullopt) const;
 
     void copyObjectMultipartImpl(
@@ -75,7 +75,7 @@ private:
         const String & src_key,
         const String & dst_bucket,
         const String & dst_key,
-        const Aws::S3::Model::HeadObjectResult & head,
+        size_t size,
         const std::optional<ObjectAttributes> & metadata = std::nullopt) const;
 
     void removeFilesBatch(const Strings & file_names);
diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp
index abc28299f96..e6b71ddb82a 100644
--- a/src/Common/ProfileEvents.cpp
+++ b/src/Common/ProfileEvents.cpp
@@ -309,6 +309,8 @@ The server successfully detected this situation and will download merged part fr
     M(S3CopyObject, "Number of S3 API CopyObject calls.") \
     M(S3ListObjects, "Number of S3 API ListObjects calls.") \
     M(S3HeadObject,  "Number of S3 API HeadObject calls.") \
+    M(S3GetObjectAttributes,  "Number of S3 API GetObjectAttributes calls.") \
+    M(S3GetObjectMetadata,  "Number of S3 API GetObject calls for getting metadata.") \
     M(S3CreateMultipartUpload, "Number of S3 API CreateMultipartUpload calls.") \
     M(S3UploadPartCopy, "Number of S3 API UploadPartCopy calls.") \
     M(S3UploadPart, "Number of S3 API UploadPart calls.") \
@@ -321,6 +323,8 @@ The server successfully detected this situation and will download merged part fr
     M(DiskS3CopyObject, "Number of DiskS3 API CopyObject calls.") \
     M(DiskS3ListObjects, "Number of DiskS3 API ListObjects calls.") \
     M(DiskS3HeadObject,  "Number of DiskS3 API HeadObject calls.") \
+    M(DiskS3GetObjectAttributes,  "Number of DiskS3 API GetObjectAttributes calls.") \
+    M(DiskS3GetObjectMetadata,  "Number of DiskS3 API GetObject calls for getting metadata.") \
     M(DiskS3CreateMultipartUpload, "Number of DiskS3 API CreateMultipartUpload calls.") \
     M(DiskS3UploadPartCopy, "Number of DiskS3 API UploadPartCopy calls.") \
     M(DiskS3UploadPart, "Number of DiskS3 API UploadPart calls.") \
diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp
index 3c620ca819e..1868d558b5c 100644
--- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp
@@ -125,9 +125,9 @@ std::string S3ObjectStorage::generateBlobNameForPath(const std::string & /* path
         getRandomASCIIString(key_name_total_size - key_name_prefix_size));
 }
 
-Aws::S3::Model::HeadObjectOutcome S3ObjectStorage::requestObjectHeadData(const std::string & bucket_from, const std::string & key) const
+size_t S3ObjectStorage::getObjectSize(const std::string & bucket_from, const std::string & key) const
 {
-    return S3::headObject(*client.get(), bucket_from, key, "", true);
+    return S3::getObjectSize(*client.get(), bucket_from, key, "", true);
 }
 
 bool S3ObjectStorage::exists(const StoredObject & object) const
@@ -135,6 +135,11 @@ bool S3ObjectStorage::exists(const StoredObject & object) const
     return S3::objectExists(*client.get(), bucket, object.absolute_path, "", true);
 }
 
+std::pair<bool /* exists */, Aws::S3::S3Error> S3ObjectStorage::checkObjectExists(const std::string & bucket_from, const std::string & key) const
+{
+    return S3::checkObjectExists(*client.get(), bucket_from, key, "", true);
+}
+
 std::unique_ptr<ReadBufferFromFileBase> S3ObjectStorage::readObjects( /// NOLINT
     const StoredObjects & objects,
     const ReadSettings & read_settings,
@@ -409,13 +414,10 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path) cons
 {
     ObjectMetadata result;
 
-    auto object_head = requestObjectHeadData(bucket, path);
-    throwIfError(object_head);
-
-    auto & object_head_result = object_head.GetResult();
-    result.size_bytes = object_head_result.GetContentLength();
-    result.last_modified = object_head_result.GetLastModified().Millis();
-    result.attributes = object_head_result.GetMetadata();
+    auto object_info = S3::getObjectInfo(*client.get(), bucket, path, "", true, true);
+    result.size_bytes = object_info.size;
+    result.last_modified = object_info.last_modification_time;
+    result.attributes = S3::getObjectMetadata(*client.get(), bucket, path, "", true, true);
 
     return result;
 }
@@ -442,7 +444,7 @@ void S3ObjectStorage::copyObjectImpl(
     const String & src_key,
     const String & dst_bucket,
     const String & dst_key,
-    std::optional<Aws::S3::Model::HeadObjectResult> head,
+    size_t size,
     std::optional<ObjectAttributes> metadata) const
 {
     auto client_ptr = client.get();
@@ -464,7 +466,7 @@ void S3ObjectStorage::copyObjectImpl(
     if (!outcome.IsSuccess() && (outcome.GetError().GetExceptionName() == "EntityTooLarge"
             || outcome.GetError().GetExceptionName() == "InvalidRequest"))
     { // Can't come here with MinIO, MinIO allows single part upload for large objects.
-        copyObjectMultipartImpl(src_bucket, src_key, dst_bucket, dst_key, head, metadata);
+        copyObjectMultipartImpl(src_bucket, src_key, dst_bucket, dst_key, size, metadata);
         return;
     }
 
@@ -473,8 +475,8 @@ void S3ObjectStorage::copyObjectImpl(
     auto settings_ptr = s3_settings.get();
     if (settings_ptr->request_settings.check_objects_after_upload)
     {
-        auto object_head = requestObjectHeadData(dst_bucket, dst_key);
-        if (!object_head.IsSuccess())
+        auto [exists, error] = checkObjectExists(dst_bucket, dst_key);
+        if (!exists)
             throw Exception(ErrorCodes::S3_ERROR, "Object {} from bucket {} disappeared immediately after upload, it's a bug in S3 or S3 API.", dst_key, dst_bucket);
     }
 
@@ -485,15 +487,11 @@ void S3ObjectStorage::copyObjectMultipartImpl(
     const String & src_key,
     const String & dst_bucket,
     const String & dst_key,
-    std::optional<Aws::S3::Model::HeadObjectResult> head,
+    size_t size,
     std::optional<ObjectAttributes> metadata) const
 {
-    if (!head)
-        head = requestObjectHeadData(src_bucket, src_key).GetResult();
-
     auto settings_ptr = s3_settings.get();
     auto client_ptr = client.get();
-    size_t size = head->GetContentLength();
 
     String multipart_upload_id;
 
@@ -570,8 +568,8 @@ void S3ObjectStorage::copyObjectMultipartImpl(
 
     if (settings_ptr->request_settings.check_objects_after_upload)
     {
-        auto object_head = requestObjectHeadData(dst_bucket, dst_key);
-        if (!object_head.IsSuccess())
+        auto [exists, error] = checkObjectExists(dst_bucket, dst_key);
+        if (!exists)
             throw Exception(ErrorCodes::S3_ERROR, "Object {} from bucket {} disappeared immediately after upload, it's a bug in S3 or S3 API.", dst_key, dst_bucket);
     }
 
@@ -580,18 +578,18 @@ void S3ObjectStorage::copyObjectMultipartImpl(
 void S3ObjectStorage::copyObject( // NOLINT
     const StoredObject & object_from, const StoredObject & object_to, std::optional<ObjectAttributes> object_to_attributes)
 {
-    auto head = requestObjectHeadData(bucket, object_from.absolute_path).GetResult();
+    auto size = getObjectSize(bucket, object_from.absolute_path);
     static constexpr int64_t multipart_upload_threashold = 5UL * 1024 * 1024 * 1024;
 
-    if (head.GetContentLength() >= multipart_upload_threashold)
+    if (size >= multipart_upload_threashold)
     {
         copyObjectMultipartImpl(
-            bucket, object_from.absolute_path, bucket, object_to.absolute_path, head, object_to_attributes);
+            bucket, object_from.absolute_path, bucket, object_to.absolute_path, size, object_to_attributes);
     }
     else
     {
         copyObjectImpl(
-            bucket, object_from.absolute_path, bucket, object_to.absolute_path, head, object_to_attributes);
+            bucket, object_from.absolute_path, bucket, object_to.absolute_path, size, object_to_attributes);
     }
 }
 
diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h
index 0a07639e253..bfcfd746e07 100644
--- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h
+++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h
@@ -172,7 +172,7 @@ private:
         const String & src_key,
         const String & dst_bucket,
         const String & dst_key,
-        std::optional<Aws::S3::Model::HeadObjectResult> head = std::nullopt,
+        size_t size,
         std::optional<ObjectAttributes> metadata = std::nullopt) const;
 
     void copyObjectMultipartImpl(
@@ -180,13 +180,14 @@ private:
         const String & src_key,
         const String & dst_bucket,
         const String & dst_key,
-        std::optional<Aws::S3::Model::HeadObjectResult> head = std::nullopt,
+        size_t size,
         std::optional<ObjectAttributes> metadata = std::nullopt) const;
 
     void removeObjectImpl(const StoredObject & object, bool if_exists);
     void removeObjectsImpl(const StoredObjects & objects, bool if_exists);
 
-    Aws::S3::Model::HeadObjectOutcome requestObjectHeadData(const std::string & bucket_from, const std::string & key) const;
+    size_t getObjectSize(const std::string & bucket_from, const std::string & key) const;
+    std::pair<bool /* exists */, Aws::S3::S3Error> checkObjectExists(const std::string & bucket_from, const std::string & key) const;
 
     std::string bucket;
 
diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp
index 0c3a63b46ea..b6937e5c40d 100644
--- a/src/IO/S3Common.cpp
+++ b/src/IO/S3Common.cpp
@@ -27,7 +27,8 @@
 #    include <aws/core/utils/UUID.h>
 #    include <aws/core/http/HttpClientFactory.h>
 #    include <aws/s3/S3Client.h>
-#    include <aws/s3/model/HeadObjectRequest.h>
+#    include <aws/s3/model/GetObjectAttributesRequest.h>
+#    include <aws/s3/model/GetObjectRequest.h>
 
 #    include <IO/S3/PocoHTTPClientFactory.h>
 #    include <IO/S3/PocoHTTPClient.h>
@@ -40,8 +41,10 @@
 
 namespace ProfileEvents
 {
-    extern const Event S3HeadObject;
-    extern const Event DiskS3HeadObject;
+    extern const Event S3GetObjectAttributes;
+    extern const Event DiskS3GetObjectAttributes;
+    extern const Event S3GetObjectMetadata;
+    extern const Event DiskS3GetObjectMetadata;
 }
 
 namespace DB
@@ -699,6 +702,28 @@ public:
     }
 };
 
+/// Performs a GetObjectAttributes request.
+Aws::S3::Model::GetObjectAttributesOutcome getObjectAttributes(
+    const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3)
+{
+    ProfileEvents::increment(ProfileEvents::S3GetObjectAttributes);
+    if (for_disk_s3)
+        ProfileEvents::increment(ProfileEvents::DiskS3GetObjectAttributes);
+
+    /// We must not use the `HeadObject` request, see the comment about `HeadObjectRequest` in S3Common.h.
+
+    Aws::S3::Model::GetObjectAttributesRequest req;
+    req.SetBucket(bucket);
+    req.SetKey(key);
+
+    if (!version_id.empty())
+        req.SetVersionId(version_id);
+
+    req.SetObjectAttributes({Aws::S3::Model::ObjectAttributes::ObjectSize});
+
+    return client.GetObjectAttributes(req);
+}
+
 }
 
 
@@ -894,36 +919,19 @@ namespace S3
         return error == Aws::S3::S3Errors::RESOURCE_NOT_FOUND || error == Aws::S3::S3Errors::NO_SUCH_KEY;
     }
 
-    Aws::S3::Model::HeadObjectOutcome headObject(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3)
-    {
-        ProfileEvents::increment(ProfileEvents::S3HeadObject);
-        if (for_disk_s3)
-            ProfileEvents::increment(ProfileEvents::DiskS3HeadObject);
-
-        Aws::S3::Model::HeadObjectRequest req;
-        req.SetBucket(bucket);
-        req.SetKey(key);
-
-        if (!version_id.empty())
-            req.SetVersionId(version_id);
-
-        return client.HeadObject(req);
-    }
-
     S3::ObjectInfo getObjectInfo(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool throw_on_error, bool for_disk_s3)
     {
-        auto outcome = headObject(client, bucket, key, version_id, for_disk_s3);
-
+        auto outcome = getObjectAttributes(client, bucket, key, version_id, for_disk_s3);
         if (outcome.IsSuccess())
         {
-            auto read_result = outcome.GetResultWithOwnership();
-            return {.size = static_cast<size_t>(read_result.GetContentLength()), .last_modification_time = read_result.GetLastModified().Millis() / 1000};
+            const auto & result = outcome.GetResult();
+            return {.size = static_cast<size_t>(result.GetObjectSize()), .last_modification_time = result.GetLastModified().Millis() / 1000};
         }
         else if (throw_on_error)
         {
             const auto & error = outcome.GetError();
             throw DB::Exception(ErrorCodes::S3_ERROR,
-                "Failed to HEAD object: {}. HTTP response code: {}",
+                "Failed to get object attributes: {}. HTTP response code: {}",
                 error.GetMessage(), static_cast<size_t>(error.GetResponseCode()));
         }
         return {};
@@ -936,12 +944,11 @@ namespace S3
 
     bool objectExists(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3)
     {
-        auto outcome = headObject(client, bucket, key, version_id, for_disk_s3);
+        auto [exists, error] = checkObjectExists(client, bucket, key, version_id, for_disk_s3);
 
-        if (outcome.IsSuccess())
+        if (exists)
             return true;
 
-        const auto & error = outcome.GetError();
         if (isNotFoundError(error.GetErrorType()))
             return false;
 
@@ -949,6 +956,47 @@ namespace S3
             "Failed to check existence of key {} in bucket {}: {}",
             key, bucket, error.GetMessage());
     }
+
+    std::pair<bool /* exists */, Aws::S3::S3Error> checkObjectExists(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3)
+    {
+        auto outcome = getObjectAttributes(client, bucket, key, version_id, for_disk_s3);
+        if (outcome.IsSuccess())
+            return {true, {}};
+        return {false, std::move(outcome.GetError())};
+    }
+
+    std::map<String, String> getObjectMetadata(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool throw_on_error, bool for_disk_s3)
+    {
+        ProfileEvents::increment(ProfileEvents::S3GetObjectMetadata);
+        if (for_disk_s3)
+            ProfileEvents::increment(ProfileEvents::DiskS3GetObjectMetadata);
+
+        /// We must not use the `HeadObject` request, see the comment about `HeadObjectRequest` in S3Common.h.
+
+        Aws::S3::Model::GetObjectRequest req;
+        req.SetBucket(bucket);
+        req.SetKey(key);
+
+        /// Only the first byte will be read.
+        /// We don't need that first byte but the range should be set otherwise the entire object will be read.
+        req.SetRange("bytes=0-0");
+
+        if (!version_id.empty())
+            req.SetVersionId(version_id);
+
+        auto outcome = client.GetObject(req);
+
+        if (outcome.IsSuccess())
+            return outcome.GetResult().GetMetadata();
+        
+        if (!throw_on_error)
+            return {};
+
+        const auto & error = outcome.GetError();
+        throw S3Exception(error.GetErrorType(),
+            "Failed to get metadata of key {} in bucket {}: {}",
+            key, bucket, error.GetMessage());
+    }
 }
 
 }
diff --git a/src/IO/S3Common.h b/src/IO/S3Common.h
index f0844c05abc..081fbab777e 100644
--- a/src/IO/S3Common.h
+++ b/src/IO/S3Common.h
@@ -11,15 +11,15 @@
 #if USE_AWS_S3
 
 #include <base/types.h>
-#include <aws/core/Aws.h>
-#include <aws/core/client/ClientConfiguration.h>
-#include <aws/s3/S3Client.h>
-#include <aws/s3/S3Errors.h>
-#include <Poco/URI.h>
-
 #include <Common/Exception.h>
 #include <Common/Throttler_fwd.h>
 
+#include <Poco/URI.h>
+#include <aws/core/Aws.h>
+#include <aws/s3/S3Errors.h>
+
+
+namespace Aws::S3 { class S3Client; }
 
 namespace DB
 {
@@ -121,22 +121,36 @@ struct URI
     static void validateBucket(const String & bucket, const Poco::URI & uri);
 };
 
+/// WARNING: Don't use `HeadObjectRequest`! Use the functions below instead.
+/// Explanation: The `HeadObject` request never returns a response body (even if there is an error) however
+/// if the request was sent without specifying a region in the endpoint (i.e. for example "https://test.s3.amazonaws.com/mydata.csv"
+/// instead of "https://test.s3-us-west-2.amazonaws.com/mydata.csv") then that response body is one of the main ways
+/// to determine the correct region and try to repeat the request again with the correct region.
+/// For any other request type (`GetObject`, `ListObjects`, etc.) AWS SDK does that because they have response bodies,
+/// but for `HeadObject` there is no response body so this way doesn't work.
+/// That's why it's better to avoid using `HeadObject` requests at all.
+/// See https://github.com/aws/aws-sdk-cpp/issues/1558 and also the function S3ErrorMarshaller::ExtractRegion() for more details.
+
 struct ObjectInfo
 {
     size_t size = 0;
     time_t last_modification_time = 0;
 };
 
-bool isNotFoundError(Aws::S3::S3Errors error);
+S3::ObjectInfo getObjectInfo(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool throw_on_error = true, bool for_disk_s3 = false);
 
-Aws::S3::Model::HeadObjectOutcome headObject(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool for_disk_s3 = false);
-
-S3::ObjectInfo getObjectInfo(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool throw_on_error, bool for_disk_s3);
-
-size_t getObjectSize(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool throw_on_error, bool for_disk_s3);
+size_t getObjectSize(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool throw_on_error = true, bool for_disk_s3 = false);
 
 bool objectExists(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool for_disk_s3 = false);
 
+/// Checks if the object exists. If it doesn't exists the function returns an error without throwing any exception.
+std::pair<bool /* exists */, Aws::S3::S3Error> checkObjectExists(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool for_disk_s3 = false);
+
+bool isNotFoundError(Aws::S3::S3Errors error);
+
+/// Returns the object's metadata.
+std::map<String, String> getObjectMetadata(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool throw_on_error = true, bool for_disk_s3 = false);
+
 }
 #endif
 
diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp
index ec77fc44de6..084d3b5470d 100644
--- a/src/IO/WriteBufferFromS3.cpp
+++ b/src/IO/WriteBufferFromS3.cpp
@@ -183,11 +183,11 @@ void WriteBufferFromS3::finalizeImpl()
     {
         LOG_TRACE(log, "Checking object {} exists after upload", key);
 
-        auto response = S3::headObject(*client_ptr, bucket, key, "", write_settings.for_object_storage);
-        if (!response.IsSuccess())
-            throw S3Exception(fmt::format("Object {} from bucket {} disappeared immediately after upload, it's a bug in S3 or S3 API.", key, bucket), response.GetError().GetErrorType());
-        else
+        auto [exists, error] = S3::checkObjectExists(*client_ptr, bucket, key, "", write_settings.for_object_storage);
+        if (exists)
             LOG_TRACE(log, "Object {} exists after upload", key);
+        else
+            throw S3Exception(fmt::format("Object {} from bucket {} disappeared immediately after upload, it's a bug in S3 or S3 API.", key, bucket), error.GetErrorType());
     }
 }
 

From a9555040439a15159e27ecf9b11ab32791cb9a66 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Sun, 15 Jan 2023 20:27:21 +0100
Subject: [PATCH 136/158] Move `throw_on_error` parameter to the end.

---
 src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp | 10 +++++-----
 src/IO/ReadBufferFromS3.cpp                     |  2 +-
 src/IO/S3Common.cpp                             | 12 ++++++------
 src/IO/S3Common.h                               |  8 ++++----
 src/Storages/StorageS3.cpp                      |  8 +++-----
 5 files changed, 19 insertions(+), 21 deletions(-)

diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp
index 1868d558b5c..e5539bcf945 100644
--- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp
@@ -127,17 +127,17 @@ std::string S3ObjectStorage::generateBlobNameForPath(const std::string & /* path
 
 size_t S3ObjectStorage::getObjectSize(const std::string & bucket_from, const std::string & key) const
 {
-    return S3::getObjectSize(*client.get(), bucket_from, key, "", true);
+    return S3::getObjectSize(*client.get(), bucket_from, key, {}, /* for_disk_s3= */ true);
 }
 
 bool S3ObjectStorage::exists(const StoredObject & object) const
 {
-    return S3::objectExists(*client.get(), bucket, object.absolute_path, "", true);
+    return S3::objectExists(*client.get(), bucket, object.absolute_path, {}, /* for_disk_s3= */ true);
 }
 
 std::pair<bool /* exists */, Aws::S3::S3Error> S3ObjectStorage::checkObjectExists(const std::string & bucket_from, const std::string & key) const
 {
-    return S3::checkObjectExists(*client.get(), bucket_from, key, "", true);
+    return S3::checkObjectExists(*client.get(), bucket_from, key, {}, /* for_disk_s3= */ true);
 }
 
 std::unique_ptr<ReadBufferFromFileBase> S3ObjectStorage::readObjects( /// NOLINT
@@ -414,10 +414,10 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path) cons
 {
     ObjectMetadata result;
 
-    auto object_info = S3::getObjectInfo(*client.get(), bucket, path, "", true, true);
+    auto object_info = S3::getObjectInfo(*client.get(), bucket, path, {}, /* for_disk_s3= */ true);
     result.size_bytes = object_info.size;
     result.last_modified = object_info.last_modification_time;
-    result.attributes = S3::getObjectMetadata(*client.get(), bucket, path, "", true, true);
+    result.attributes = S3::getObjectMetadata(*client.get(), bucket, path, {}, /* for_disk_s3= */ true);
 
     return result;
 }
diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp
index 2d274435a74..69d2a244097 100644
--- a/src/IO/ReadBufferFromS3.cpp
+++ b/src/IO/ReadBufferFromS3.cpp
@@ -250,7 +250,7 @@ size_t ReadBufferFromS3::getFileSize()
     if (file_size)
         return *file_size;
 
-    auto object_size = S3::getObjectSize(*client_ptr, bucket, key, version_id, true, read_settings.for_object_storage);
+    auto object_size = S3::getObjectSize(*client_ptr, bucket, key, version_id, /* for_disk_s3= */ read_settings.for_object_storage);
 
     file_size = object_size;
     return *file_size;
diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp
index b6937e5c40d..e019086e2d1 100644
--- a/src/IO/S3Common.cpp
+++ b/src/IO/S3Common.cpp
@@ -919,7 +919,7 @@ namespace S3
         return error == Aws::S3::S3Errors::RESOURCE_NOT_FOUND || error == Aws::S3::S3Errors::NO_SUCH_KEY;
     }
 
-    S3::ObjectInfo getObjectInfo(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool throw_on_error, bool for_disk_s3)
+    S3::ObjectInfo getObjectInfo(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3, bool throw_on_error)
     {
         auto outcome = getObjectAttributes(client, bucket, key, version_id, for_disk_s3);
         if (outcome.IsSuccess())
@@ -937,19 +937,19 @@ namespace S3
         return {};
     }
 
-    size_t getObjectSize(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool throw_on_error, bool for_disk_s3)
+    size_t getObjectSize(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3, bool throw_on_error)
     {
-        return getObjectInfo(client, bucket, key, version_id, throw_on_error, for_disk_s3).size;
+        return getObjectInfo(client, bucket, key, version_id, for_disk_s3, throw_on_error).size;
     }
 
-    bool objectExists(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3)
+    bool objectExists(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3, bool throw_on_error)
     {
         auto [exists, error] = checkObjectExists(client, bucket, key, version_id, for_disk_s3);
 
         if (exists)
             return true;
 
-        if (isNotFoundError(error.GetErrorType()))
+        if (!throw_on_error || isNotFoundError(error.GetErrorType()))
             return false;
 
         throw S3Exception(error.GetErrorType(),
@@ -965,7 +965,7 @@ namespace S3
         return {false, std::move(outcome.GetError())};
     }
 
-    std::map<String, String> getObjectMetadata(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool throw_on_error, bool for_disk_s3)
+    std::map<String, String> getObjectMetadata(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3, bool throw_on_error)
     {
         ProfileEvents::increment(ProfileEvents::S3GetObjectMetadata);
         if (for_disk_s3)
diff --git a/src/IO/S3Common.h b/src/IO/S3Common.h
index 081fbab777e..b1749b636d3 100644
--- a/src/IO/S3Common.h
+++ b/src/IO/S3Common.h
@@ -137,11 +137,11 @@ struct ObjectInfo
     time_t last_modification_time = 0;
 };
 
-S3::ObjectInfo getObjectInfo(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool throw_on_error = true, bool for_disk_s3 = false);
+S3::ObjectInfo getObjectInfo(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool for_disk_s3 = false, bool throw_on_error = true);
 
-size_t getObjectSize(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool throw_on_error = true, bool for_disk_s3 = false);
+size_t getObjectSize(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool for_disk_s3 = false, bool throw_on_error = true);
 
-bool objectExists(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool for_disk_s3 = false);
+bool objectExists(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool for_disk_s3 = false, bool throw_on_error = true);
 
 /// Checks if the object exists. If it doesn't exists the function returns an error without throwing any exception.
 std::pair<bool /* exists */, Aws::S3::S3Error> checkObjectExists(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool for_disk_s3 = false);
@@ -149,7 +149,7 @@ std::pair<bool /* exists */, Aws::S3::S3Error> checkObjectExists(const Aws::S3::
 bool isNotFoundError(Aws::S3::S3Errors error);
 
 /// Returns the object's metadata.
-std::map<String, String> getObjectMetadata(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool throw_on_error = true, bool for_disk_s3 = false);
+std::map<String, String> getObjectMetadata(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool for_disk_s3 = false, bool throw_on_error = true);
 
 }
 #endif
diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp
index 9cb992bd24f..3257ccb5022 100644
--- a/src/Storages/StorageS3.cpp
+++ b/src/Storages/StorageS3.cpp
@@ -444,7 +444,7 @@ public:
             /// (which means we eventually need this info anyway, so it should be ok to do it now)
             if (object_infos_)
             {
-                info = S3::getObjectInfo(client_, bucket, key, version_id_, true, false);
+                info = S3::getObjectInfo(client_, bucket, key, version_id_);
                 total_size += info->size;
 
                 String path = fs::path(bucket) / key;
@@ -569,9 +569,7 @@ StorageS3Source::ReaderHolder StorageS3Source::createReader()
     if (current_key.empty())
         return {};
 
-    size_t object_size = info
-        ? info->size
-        : S3::getObjectSize(*client, bucket, current_key, version_id, true, false);
+    size_t object_size = info ? info->size : S3::getObjectSize(*client, bucket, current_key, version_id);
 
     int zstd_window_log_max = static_cast<int>(getContext()->getSettingsRef().zstd_window_log_max);
     auto read_buf = wrapReadBufferWithCompressionMethod(
@@ -1523,7 +1521,7 @@ std::optional<ColumnsDescription> StorageS3::tryGetColumnsFromCache(
                 /// Note that in case of exception in getObjectInfo returned info will be empty,
                 /// but schema cache will handle this case and won't return columns from cache
                 /// because we can't say that it's valid without last modification time.
-                info = S3::getObjectInfo(*s3_configuration.client, s3_configuration.uri.bucket, *it, s3_configuration.uri.version_id, false, false);
+                info = S3::getObjectInfo(*s3_configuration.client, s3_configuration.uri.bucket, *it, s3_configuration.uri.version_id, {}, /* throw_on_error= */ false);
                 if (object_infos)
                     (*object_infos)[path] = info;
             }

From 5617f7f616b620b4639294fa3f8ec9d134c10a28 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Sun, 15 Jan 2023 22:51:10 +0100
Subject: [PATCH 137/158] address comments

---
 .../MergeTree/MergeTreeDataWriter.cpp         |  7 +++--
 src/Storages/MergeTree/MergeTreeDataWriter.h  |  6 ++--
 .../MergeTree/ReplicatedMergeTreeSink.cpp     | 31 ++++++++++---------
 3 files changed, 24 insertions(+), 20 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp
index cff6da85efc..03a3d4fbd72 100644
--- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp
+++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp
@@ -193,7 +193,8 @@ BlocksWithPartition MergeTreeDataWriter::splitBlockIntoParts(
     if (!metadata_snapshot->hasPartitionKey()) /// Table is not partitioned.
     {
         result.emplace_back(Block(block), Row{});
-        result[0].offsets = chunk_offsets;
+        if (chunk_offsets != nullptr)
+            result[0].offsets = std::move(chunk_offsets->offsets);
         return result;
     }
 
@@ -230,7 +231,7 @@ BlocksWithPartition MergeTreeDataWriter::splitBlockIntoParts(
         /// do not interfere with possible calculated primary key columns of the same name.
         result.emplace_back(Block(block), get_partition(0));
         if (!chunk_offsets_with_partition.empty())
-            result[0].offsets = chunk_offsets_with_partition[0];
+            result[0].offsets = std::move(chunk_offsets_with_partition[0]->offsets);
         return result;
     }
 
@@ -245,7 +246,7 @@ BlocksWithPartition MergeTreeDataWriter::splitBlockIntoParts(
     }
 
     for (size_t i = 0; i < chunk_offsets_with_partition.size(); ++i)
-        result[i].offsets = chunk_offsets_with_partition[i];
+        result[i].offsets = std::move(chunk_offsets_with_partition[i]->offsets);
 
     return result;
 }
diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.h b/src/Storages/MergeTree/MergeTreeDataWriter.h
index cbf8094f7fd..5dc7bf40922 100644
--- a/src/Storages/MergeTree/MergeTreeDataWriter.h
+++ b/src/Storages/MergeTree/MergeTreeDataWriter.h
@@ -22,15 +22,15 @@ struct BlockWithPartition
 {
     Block block;
     Row partition;
-    ChunkOffsetsPtr offsets;
+    std::vector<size_t> offsets;
 
     BlockWithPartition(Block && block_, Row && partition_)
         : block(block_), partition(std::move(partition_))
     {
     }
 
-    BlockWithPartition(Block && block_, Row && partition_, ChunkOffsetsPtr chunk_offsets_)
-        : block(block_), partition(std::move(partition_)), offsets(chunk_offsets_)
+    BlockWithPartition(Block && block_, Row && partition_, std::vector<size_t> && offsets_)
+        : block(block_), partition(std::move(partition_)), offsets(std::move(offsets_))
     {
     }
 };
diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
index e7327286d4b..35e9cc46ebe 100644
--- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
+++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
@@ -71,27 +71,30 @@ struct ReplicatedMergeTreeSinkImpl<async_insert>::DelayedChunk
             }
         }
 
-        bool checkSelfDeduplicate()
+        /// this function check if the block contains duplicate inserts.
+        /// if so, we keep only one insert for every duplicate ones.
+        bool filterSelfDuplicate()
         {
             if constexpr (async_insert)
             {
                 std::vector<String> dup_block_ids;
                 for (const auto & [hash_id, offset_indexes] : block_id_to_offset_idx)
                 {
-                    if (offset_idxes.size() > 1)
+                    /// It means more than one inserts have the same hash id, in this case, we should keep only one of them.
+                    if (offset_indexes.size() > 1)
                         dup_block_ids.push_back(hash_id);
                 }
                 if (dup_block_ids.empty())
                     return false;
 
-                rewriteBlock(dup_block_ids, true);
+                filterBlockDuplicate(dup_block_ids, true);
                 return true;
             }
             return false;
         }
 
         /// remove the conflict parts of block for rewriting again.
-        void rewriteBlock(const std::vector<String> & block_paths, bool self_dedup)
+        void filterBlockDuplicate(const std::vector<String> & block_paths, bool self_dedup)
         {
             if constexpr (async_insert)
             {
@@ -103,12 +106,13 @@ struct ReplicatedMergeTreeSinkImpl<async_insert>::DelayedChunk
                     auto it = block_id_to_offset_idx.find(conflict_block_id);
                     if (it == block_id_to_offset_idx.end())
                         throw Exception("Unknown conflict path " + conflict_block_id, ErrorCodes::LOGICAL_ERROR);
-                    /// in case of self dedup, we should remain one insert
+                    /// if this filter is for self_dedup, that means the block paths is selected by `filterSelfDuplicate`, which is a self purge.
+                    /// in this case, we don't know if zk has this insert, then we should keep one insert, to avoid missing this insert.
                     offset_idx.insert(std::end(offset_idx), std::begin(it->second) + self_dedup, std::end(it->second));
                 }
                 std::sort(offset_idx.begin(), offset_idx.end());
 
-                auto & offsets = block_with_partition.offsets->offsets;
+                auto & offsets = block_with_partition.offsets;
                 size_t idx = 0, remove_count = 0;
                 auto it = offset_idx.begin();
                 std::vector<size_t> new_offsets;
@@ -143,7 +147,7 @@ struct ReplicatedMergeTreeSinkImpl<async_insert>::DelayedChunk
 
                 LOG_TRACE(log, "New block IDs: {}, new offsets: {}, size: {}", toString(new_block_ids), toString(new_offsets), new_offsets.size());
 
-                offsets = std::move(new_offsets);
+                block_with_partition.offsets = std::move(new_offsets);
                 block_id = std::move(new_block_ids);
                 auto cols = block_with_partition.block.getColumns();
                 for (auto & col : cols)
@@ -174,18 +178,17 @@ struct ReplicatedMergeTreeSinkImpl<async_insert>::DelayedChunk
 std::vector<Int64> testSelfDeduplicate(std::vector<Int64> data, std::vector<size_t> offsets, std::vector<String> hashes)
 {
     MutableColumnPtr column = DataTypeInt64().createColumn();
-    auto offset_ptr = std::make_shared<ChunkOffsets>(offsets);
     for (auto datum : data)
     {
         column->insert(datum);
     }
     Block block({ColumnWithTypeAndName(std::move(column), DataTypePtr(new DataTypeInt64()), "a")});
 
-    BlockWithPartition block1(std::move(block), Row(), offset_ptr);
+    BlockWithPartition block1(std::move(block), Row(), std::move(offsets));
     ReplicatedMergeTreeSinkImpl<true>::DelayedChunk::Partition part(
         &Poco::Logger::get("testSelfDeduplicate"), MergeTreeDataWriter::TemporaryPart(), 0, std::move(hashes), std::move(block1));
 
-    part.checkSelfDeduplicate();
+    part.filterSelfDuplicate();
 
     ColumnPtr col = part.block_with_partition.block.getColumns()[0];
     std::vector<Int64> result;
@@ -212,7 +215,7 @@ namespace
         size_t start = 0;
         auto cols = block.block.getColumns();
         std::vector<String> block_id_vec;
-        for (auto offset : block.offsets->offsets)
+        for (auto offset : block.offsets)
         {
             SipHash hash;
             for (size_t i = start; i < offset; ++i)
@@ -422,7 +425,7 @@ void ReplicatedMergeTreeSinkImpl<async_insert>::consume(Chunk chunk)
         {
             /// TODO consider insert_deduplication_token
             block_id = getHashesForBlocks(current_block, temp_part.part->info.partition_id);
-            LOG_TRACE(log, "async insert part, part id {}, block id {}, offsets {}, size {}", temp_part.part->info.partition_id, toString(block_id), toString(current_block.offsets->offsets), current_block.offsets->offsets.size());
+            LOG_TRACE(log, "async insert part, part id {}, block id {}, offsets {}, size {}", temp_part.part->info.partition_id, toString(block_id), toString(current_block.offsets), current_block.offsets.size());
         }
         else if (deduplicate)
         {
@@ -534,7 +537,7 @@ void ReplicatedMergeTreeSinkImpl<true>::finishDelayedChunk(const ZooKeeperWithFa
     {
         int retry_times = 0;
         /// users may have lots of same inserts. It will be helpful to deduplicate in advance.
-        if (partition.checkSelfDeduplicate())
+        if (partition.filterSelfDuplicate())
         {
             LOG_TRACE(log, "found duplicated inserts in the block");
             partition.block_with_partition.partition = std::move(partition.temp_part.part->partition.value);
@@ -550,7 +553,7 @@ void ReplicatedMergeTreeSinkImpl<true>::finishDelayedChunk(const ZooKeeperWithFa
             ++retry_times;
             LOG_DEBUG(log, "Found duplicate block IDs: {}, retry times {}", toString(conflict_block_ids), retry_times);
             /// partition clean conflict
-            partition.rewriteBlock(conflict_block_ids, false);
+            partition.filterBlockDuplicate(conflict_block_ids, false);
             if (partition.block_id.empty())
                 break;
             partition.block_with_partition.partition = std::move(partition.temp_part.part->partition.value);

From 6cf603e05f116bd2084264a5cb5877ca3e6ff3c5 Mon Sep 17 00:00:00 2001
From: Denny Crane <denis.zhuravlov@gmail.com>
Date: Sun, 15 Jan 2023 18:40:59 -0400
Subject: [PATCH 138/158] Update index.md

---
 docs/en/sql-reference/aggregate-functions/reference/index.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/en/sql-reference/aggregate-functions/reference/index.md b/docs/en/sql-reference/aggregate-functions/reference/index.md
index 5d9423e0a55..bd8e72e0fec 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/index.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/index.md
@@ -57,6 +57,7 @@ ClickHouse-specific aggregate functions:
 -   [uniqCombined](../../../sql-reference/aggregate-functions/reference/uniqcombined.md)
 -   [uniqCombined64](../../../sql-reference/aggregate-functions/reference/uniqcombined64.md)
 -   [uniqHLL12](../../../sql-reference/aggregate-functions/reference/uniqhll12.md)
+-   [uniqTheta](../../../sql-reference/aggregate-functions/reference/uniqthetasketch.md)
 -   [quantile](../../../sql-reference/aggregate-functions/reference/quantile.md)
 -   [quantiles](../../../sql-reference/aggregate-functions/reference/quantiles.md)
 -   [quantileExact](../../../sql-reference/aggregate-functions/reference/quantileexact.md)
@@ -79,4 +80,4 @@ ClickHouse-specific aggregate functions:
 -   [cramersVBiasCorrected](./cramersvbiascorrected.md)
 -   [theilsU](./theilsu.md)
 -   [maxIntersections](./maxintersections.md)
--   [maxIntersectionsPosition](./maxintersectionsposition.md)
\ No newline at end of file
+-   [maxIntersectionsPosition](./maxintersectionsposition.md)

From 674a1d18775dcb0ff9ea30576edbcf60cf3abbd4 Mon Sep 17 00:00:00 2001
From: Peignon Melvyn <melvyn@clickhouse.com>
Date: Mon, 16 Jan 2023 01:27:08 +0100
Subject: [PATCH 139/158] Update json.md

---
 docs/en/sql-reference/data-types/json.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/en/sql-reference/data-types/json.md b/docs/en/sql-reference/data-types/json.md
index ab1596b1760..76d53f72284 100644
--- a/docs/en/sql-reference/data-types/json.md
+++ b/docs/en/sql-reference/data-types/json.md
@@ -6,6 +6,10 @@ sidebar_label: JSON
 
 # JSON
 
+:::warning
+This feature is experimental and is not production ready. If you need to work with JSON document, consider using [this guide](https://clickhouse.com/docs/en/guides/developer/working-with-json) instead.
+:::
+
 Stores JavaScript Object Notation (JSON) documents in a single column.
 
 `JSON` is an alias for `Object('json')`.

From adca0b64d39766e3fbb0271b5f649267884f503a Mon Sep 17 00:00:00 2001
From: Dan Roscigno <dan@roscigno.com>
Date: Sun, 15 Jan 2023 19:31:58 -0500
Subject: [PATCH 140/158] use markdown file instead of URL to enforce 404
 checks

---
 docs/en/sql-reference/data-types/json.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/sql-reference/data-types/json.md b/docs/en/sql-reference/data-types/json.md
index 76d53f72284..d9099ba5ad3 100644
--- a/docs/en/sql-reference/data-types/json.md
+++ b/docs/en/sql-reference/data-types/json.md
@@ -7,7 +7,7 @@ sidebar_label: JSON
 # JSON
 
 :::warning
-This feature is experimental and is not production ready. If you need to work with JSON document, consider using [this guide](https://clickhouse.com/docs/en/guides/developer/working-with-json) instead.
+This feature is experimental and is not production ready. If you need to work with JSON documents, consider using [this guide](/docs/en/guides/developer/working-with-json/json-load-data.md) instead.
 :::
 
 Stores JavaScript Object Notation (JSON) documents in a single column.

From 2f7ea79d94fd1a25758c7503c6c0a6ddb7e62309 Mon Sep 17 00:00:00 2001
From: taiyang-li <654010905@qq.com>
Date: Mon, 16 Jan 2023 10:42:58 +0800
Subject: [PATCH 141/158] change as request

---
 src/Functions/MatchImpl.h | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/src/Functions/MatchImpl.h b/src/Functions/MatchImpl.h
index 002b876f1e8..3622db234b5 100644
--- a/src/Functions/MatchImpl.h
+++ b/src/Functions/MatchImpl.h
@@ -118,16 +118,13 @@ struct MatchImpl
         if (haystack_offsets.empty())
             return;
 
-        /// Fast path for [I]LIKE, because the result is always true or false
-        /// col [i]like '%%'
-        /// col not [i]like '%%'
-        /// col like '%'
-        /// col not [i]like '%'
-        /// match(like, '^$')
-        if ((is_like && (needle == "%%" or needle == "%")) || (!is_like && needle == ".*"))
+        /// Shortcut for the silly but practical case that the pattern matches everything/nothing independently of the haystack:
+        /// - col [not] [i]like '%' / '%%'
+        /// - match(col, '.*')
+        if ((is_like && (needle == "%" or needle == "%%")) || (!is_like && (needle == ".*" || needle == ".*?")))
         {
-            for (auto & re : res)
-                re = !negate;
+            for (auto & x : res)
+                x = !negate;
             return;
         }
 
@@ -280,6 +277,16 @@ struct MatchImpl
         if (haystack.empty())
             return;
 
+        /// Shortcut for the silly but practical case that the pattern matches everything/nothing independently of the haystack:
+        /// - col [not] [i]like '%' / '%%'
+        /// - match(col, '.*')
+        if ((is_like && (needle == "%" or needle == "%%")) || (!is_like && (needle == ".*" || needle == ".*?")))
+        {
+            for (auto & x : res)
+                x = !negate;
+            return;
+        }
+
         /// Special case that the [I]LIKE expression reduces to finding a substring in a string
         String strstr_pattern;
         if (is_like && impl::likePatternIsSubstring(needle, strstr_pattern))

From 175491301330b5b82720d6b599ef19092a86b378 Mon Sep 17 00:00:00 2001
From: simpleton <63032616+rgzntrade@users.noreply.github.com>
Date: Mon, 16 Jan 2023 15:01:16 +0800
Subject: [PATCH 142/158] test: Remove the temporary table in the test

---
 tests/queries/0_stateless/00647_histogram_negative.sql | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/queries/0_stateless/00647_histogram_negative.sql b/tests/queries/0_stateless/00647_histogram_negative.sql
index 91b3fb49529..2112f93bca9 100644
--- a/tests/queries/0_stateless/00647_histogram_negative.sql
+++ b/tests/queries/0_stateless/00647_histogram_negative.sql
@@ -1,4 +1,6 @@
+drop table if exists histogram;
 create table histogram(num Int64) engine=TinyLog;
 insert into histogram values(-1);
 insert into histogram values(-1);
 select histogram(2)(num) from histogram;
+drop table if exists histogram;

From 09241150a05e2beeb4a1bac97c7d0f82d4bf2472 Mon Sep 17 00:00:00 2001
From: Smita Kulkarni <Smita.Kulkarni@clickhouse.com>
Date: Mon, 16 Jan 2023 08:32:25 +0100
Subject: [PATCH 143/158] Updated recording of startup time till after starting
 last listener and added stopping the StopWatch - Record server startup time
 in ProfileEvents

---
 programs/server/Server.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp
index c3f0dc8c2de..419b80ccff2 100644
--- a/programs/server/Server.cpp
+++ b/programs/server/Server.cpp
@@ -1813,8 +1813,6 @@ try
                                                                      &CurrentMetrics::MaxDDLEntryID, &CurrentMetrics::MaxPushedDDLEntryID));
         }
 
-        ProfileEvents::increment(ProfileEvents::ServerStartupMilliseconds, startup_watch.elapsedMilliseconds());
-
         {
             std::lock_guard lock(servers_lock);
             for (auto & server : servers)
@@ -1827,6 +1825,9 @@ try
             LOG_INFO(log, "Ready for connections.");
         }
 
+        startup_watch.stop();
+        ProfileEvents::increment(ProfileEvents::ServerStartupMilliseconds, startup_watch.elapsedMilliseconds());
+
         try
         {
             global_context->startClusterDiscovery();

From e435edb4ab33a9d43279664e6896eeabed571548 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Mon, 16 Jan 2023 10:05:55 +0100
Subject: [PATCH 144/158] Make checkObjectExists() easier.

---
 src/Common/ProfileEvents.cpp                   |  8 ++++----
 .../ObjectStorages/S3/S3ObjectStorage.cpp      | 18 ++++--------------
 src/Disks/ObjectStorages/S3/S3ObjectStorage.h  |  2 +-
 src/IO/S3Common.cpp                            | 14 ++++++++------
 src/IO/S3Common.h                              |  6 +++---
 src/IO/WriteBufferFromS3.cpp                   |  8 ++------
 6 files changed, 22 insertions(+), 34 deletions(-)

diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp
index e6b71ddb82a..508c078ff96 100644
--- a/src/Common/ProfileEvents.cpp
+++ b/src/Common/ProfileEvents.cpp
@@ -309,8 +309,8 @@ The server successfully detected this situation and will download merged part fr
     M(S3CopyObject, "Number of S3 API CopyObject calls.") \
     M(S3ListObjects, "Number of S3 API ListObjects calls.") \
     M(S3HeadObject,  "Number of S3 API HeadObject calls.") \
-    M(S3GetObjectAttributes,  "Number of S3 API GetObjectAttributes calls.") \
-    M(S3GetObjectMetadata,  "Number of S3 API GetObject calls for getting metadata.") \
+    M(S3GetObjectAttributes, "Number of S3 API GetObjectAttributes calls.") \
+    M(S3GetObjectMetadata, "Number of S3 API GetObject calls for getting metadata.") \
     M(S3CreateMultipartUpload, "Number of S3 API CreateMultipartUpload calls.") \
     M(S3UploadPartCopy, "Number of S3 API UploadPartCopy calls.") \
     M(S3UploadPart, "Number of S3 API UploadPart calls.") \
@@ -323,8 +323,8 @@ The server successfully detected this situation and will download merged part fr
     M(DiskS3CopyObject, "Number of DiskS3 API CopyObject calls.") \
     M(DiskS3ListObjects, "Number of DiskS3 API ListObjects calls.") \
     M(DiskS3HeadObject,  "Number of DiskS3 API HeadObject calls.") \
-    M(DiskS3GetObjectAttributes,  "Number of DiskS3 API GetObjectAttributes calls.") \
-    M(DiskS3GetObjectMetadata,  "Number of DiskS3 API GetObject calls for getting metadata.") \
+    M(DiskS3GetObjectAttributes, "Number of DiskS3 API GetObjectAttributes calls.") \
+    M(DiskS3GetObjectMetadata, "Number of DiskS3 API GetObject calls for getting metadata.") \
     M(DiskS3CreateMultipartUpload, "Number of DiskS3 API CreateMultipartUpload calls.") \
     M(DiskS3UploadPartCopy, "Number of DiskS3 API UploadPartCopy calls.") \
     M(DiskS3UploadPart, "Number of DiskS3 API UploadPart calls.") \
diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp
index e5539bcf945..e402c18efac 100644
--- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp
@@ -135,9 +135,9 @@ bool S3ObjectStorage::exists(const StoredObject & object) const
     return S3::objectExists(*client.get(), bucket, object.absolute_path, {}, /* for_disk_s3= */ true);
 }
 
-std::pair<bool /* exists */, Aws::S3::S3Error> S3ObjectStorage::checkObjectExists(const std::string & bucket_from, const std::string & key) const
+void S3ObjectStorage::checkObjectExists(const std::string & bucket_from, const std::string & key, const std::string_view & description) const
 {
-    return S3::checkObjectExists(*client.get(), bucket_from, key, {}, /* for_disk_s3= */ true);
+    return S3::checkObjectExists(*client.get(), bucket_from, key, {}, /* for_disk_s3= */ true, description);
 }
 
 std::unique_ptr<ReadBufferFromFileBase> S3ObjectStorage::readObjects( /// NOLINT
@@ -474,12 +474,7 @@ void S3ObjectStorage::copyObjectImpl(
 
     auto settings_ptr = s3_settings.get();
     if (settings_ptr->request_settings.check_objects_after_upload)
-    {
-        auto [exists, error] = checkObjectExists(dst_bucket, dst_key);
-        if (!exists)
-            throw Exception(ErrorCodes::S3_ERROR, "Object {} from bucket {} disappeared immediately after upload, it's a bug in S3 or S3 API.", dst_key, dst_bucket);
-    }
-
+        checkObjectExists(dst_bucket, dst_key, "Immediately after upload");
 }
 
 void S3ObjectStorage::copyObjectMultipartImpl(
@@ -567,12 +562,7 @@ void S3ObjectStorage::copyObjectMultipartImpl(
     }
 
     if (settings_ptr->request_settings.check_objects_after_upload)
-    {
-        auto [exists, error] = checkObjectExists(dst_bucket, dst_key);
-        if (!exists)
-            throw Exception(ErrorCodes::S3_ERROR, "Object {} from bucket {} disappeared immediately after upload, it's a bug in S3 or S3 API.", dst_key, dst_bucket);
-    }
-
+        checkObjectExists(dst_bucket, dst_key, "Immediately after upload");
 }
 
 void S3ObjectStorage::copyObject( // NOLINT
diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h
index bfcfd746e07..9dccf3dbcf9 100644
--- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h
+++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h
@@ -187,7 +187,7 @@ private:
     void removeObjectsImpl(const StoredObjects & objects, bool if_exists);
 
     size_t getObjectSize(const std::string & bucket_from, const std::string & key) const;
-    std::pair<bool /* exists */, Aws::S3::S3Error> checkObjectExists(const std::string & bucket_from, const std::string & key) const;
+    void checkObjectExists(const std::string & bucket_from, const std::string & key, const std::string_view & description) const;
 
     std::string bucket;
 
diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp
index e019086e2d1..e41c8429891 100644
--- a/src/IO/S3Common.cpp
+++ b/src/IO/S3Common.cpp
@@ -942,14 +942,14 @@ namespace S3
         return getObjectInfo(client, bucket, key, version_id, for_disk_s3, throw_on_error).size;
     }
 
-    bool objectExists(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3, bool throw_on_error)
+    bool objectExists(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3)
     {
         auto [exists, error] = checkObjectExists(client, bucket, key, version_id, for_disk_s3);
 
         if (exists)
             return true;
 
-        if (!throw_on_error || isNotFoundError(error.GetErrorType()))
+        if (isNotFoundError(error.GetErrorType()))
             return false;
 
         throw S3Exception(error.GetErrorType(),
@@ -957,12 +957,14 @@ namespace S3
             key, bucket, error.GetMessage());
     }
 
-    std::pair<bool /* exists */, Aws::S3::S3Error> checkObjectExists(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3)
+    void checkObjectExists(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3, const std::string_view & description)
     {
         auto outcome = getObjectAttributes(client, bucket, key, version_id, for_disk_s3);
         if (outcome.IsSuccess())
-            return {true, {}};
-        return {false, std::move(outcome.GetError())};
+            return;
+        const auto & error = outcome.GetError();
+        throw S3Exception(error.GetErrorType(), "{}Object {} in bucket {} suddenly disappeared: {}",
+                          (description.empty() ? "" : (String(description) + ": ")), key, bucket, error.GetErrorMessage());
     }
 
     std::map<String, String> getObjectMetadata(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3, bool throw_on_error)
@@ -988,7 +990,7 @@ namespace S3
 
         if (outcome.IsSuccess())
             return outcome.GetResult().GetMetadata();
-        
+
         if (!throw_on_error)
             return {};
 
diff --git a/src/IO/S3Common.h b/src/IO/S3Common.h
index b1749b636d3..546a3b49a27 100644
--- a/src/IO/S3Common.h
+++ b/src/IO/S3Common.h
@@ -141,10 +141,10 @@ S3::ObjectInfo getObjectInfo(const Aws::S3::S3Client & client, const String & bu
 
 size_t getObjectSize(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool for_disk_s3 = false, bool throw_on_error = true);
 
-bool objectExists(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool for_disk_s3 = false, bool throw_on_error = true);
+bool objectExists(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool for_disk_s3 = false);
 
-/// Checks if the object exists. If it doesn't exists the function returns an error without throwing any exception.
-std::pair<bool /* exists */, Aws::S3::S3Error> checkObjectExists(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool for_disk_s3 = false);
+/// Throws an exception if a specified object doesn't exist. `description` is used as a part of the error message.
+void checkObjectExists(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool for_disk_s3 = false, const std::string_view & description = {});
 
 bool isNotFoundError(Aws::S3::S3Errors error);
 
diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp
index 084d3b5470d..7d279c07e03 100644
--- a/src/IO/WriteBufferFromS3.cpp
+++ b/src/IO/WriteBufferFromS3.cpp
@@ -182,12 +182,8 @@ void WriteBufferFromS3::finalizeImpl()
     if (check_objects_after_upload)
     {
         LOG_TRACE(log, "Checking object {} exists after upload", key);
-
-        auto [exists, error] = S3::checkObjectExists(*client_ptr, bucket, key, "", write_settings.for_object_storage);
-        if (exists)
-            LOG_TRACE(log, "Object {} exists after upload", key);
-        else
-            throw S3Exception(fmt::format("Object {} from bucket {} disappeared immediately after upload, it's a bug in S3 or S3 API.", key, bucket), error.GetErrorType());
+        S3::checkObjectExists(*client_ptr, bucket, key, {}, /* for_disk_s3= */ write_settings.for_object_storage, "Immediately after upload");
+        LOG_TRACE(log, "Object {} exists after upload", key);
     }
 }
 

From 8f5250e0009f323476d45fc385577f6898bd795c Mon Sep 17 00:00:00 2001
From: Maksim Kita <kitaetoya@gmail.com>
Date: Mon, 16 Jan 2023 10:14:34 +0100
Subject: [PATCH 145/158] Revert "Validate function arguments in query tree"

---
 src/Analyzer/FunctionNode.cpp         | 10 ---------
 src/Analyzer/FunctionNode.h           |  1 -
 src/Analyzer/QueryTreePassManager.cpp | 31 ---------------------------
 3 files changed, 42 deletions(-)

diff --git a/src/Analyzer/FunctionNode.cpp b/src/Analyzer/FunctionNode.cpp
index 7961bfbae31..84a1d5025e3 100644
--- a/src/Analyzer/FunctionNode.cpp
+++ b/src/Analyzer/FunctionNode.cpp
@@ -2,7 +2,6 @@
 
 #include <Common/SipHash.h>
 #include <Common/FieldVisitorToString.h>
-#include <DataTypes/IDataType.h>
 #include <Analyzer/ConstantNode.h>
 
 #include <IO/WriteBufferFromString.h>
@@ -32,15 +31,6 @@ FunctionNode::FunctionNode(String function_name_)
     children[arguments_child_index] = std::make_shared<ListNode>();
 }
 
-const DataTypes & FunctionNode::getArgumentTypes() const
-{
-    if (!function)
-        throw Exception(ErrorCodes::LOGICAL_ERROR,
-        "Function {} is not resolved",
-        function_name);
-    return function->getArgumentTypes();
-}
-
 ColumnsWithTypeAndName FunctionNode::getArgumentColumns() const
 {
     const auto & arguments = getArguments().getNodes();
diff --git a/src/Analyzer/FunctionNode.h b/src/Analyzer/FunctionNode.h
index 92d3ab84722..49e66ba32c1 100644
--- a/src/Analyzer/FunctionNode.h
+++ b/src/Analyzer/FunctionNode.h
@@ -85,7 +85,6 @@ public:
     /// Get arguments node
     QueryTreeNodePtr & getArgumentsNode() { return children[arguments_child_index]; }
 
-    const DataTypes & getArgumentTypes() const;
     ColumnsWithTypeAndName getArgumentColumns() const;
 
     /// Returns true if function node has window, false otherwise
diff --git a/src/Analyzer/QueryTreePassManager.cpp b/src/Analyzer/QueryTreePassManager.cpp
index 7b000310756..8efe0dd4602 100644
--- a/src/Analyzer/QueryTreePassManager.cpp
+++ b/src/Analyzer/QueryTreePassManager.cpp
@@ -25,7 +25,6 @@
 #include <Analyzer/FunctionNode.h>
 #include <Analyzer/InDepthQueryTreeVisitor.h>
 #include <Common/Exception.h>
-#include <DataTypes/IDataType.h>
 
 namespace DB
 {
@@ -62,36 +61,6 @@ class ValidationChecker : public InDepthQueryTreeVisitor<ValidationChecker>
             throw Exception(ErrorCodes::LOGICAL_ERROR,
             "Function {} is not resolved after running {} pass",
             function->toAST()->formatForErrorMessage(), pass_name);
-
-        if (function->getFunctionName() == "in")
-            return;
-
-        const auto & expected_arg_types = function->getArgumentTypes();
-        auto actual_arg_columns = function->getArgumentColumns();
-
-        if (expected_arg_types.size() != actual_arg_columns.size())
-            throw Exception(ErrorCodes::LOGICAL_ERROR,
-            "Function {} expects {} arguments but has {} after running {} pass",
-            function->toAST()->formatForErrorMessage(),
-            expected_arg_types.size(),
-            actual_arg_columns.size(),
-            pass_name);
-
-        for (size_t i = 0; i < expected_arg_types.size(); ++i)
-        {
-            // Skip lambdas
-            if (WhichDataType(expected_arg_types[i]).isFunction())
-                continue;
-
-            if (!expected_arg_types[i]->equals(*actual_arg_columns[i].type))
-                throw Exception(ErrorCodes::LOGICAL_ERROR,
-                "Function {} expects {} argument to have {} type but receives {} after running {} pass",
-                function->toAST()->formatForErrorMessage(),
-                i,
-                expected_arg_types[i]->getName(),
-                actual_arg_columns[i].type->getName(),
-                pass_name);
-        }
     }
 
 public:

From 16a20cd06e7c9e8c410be9e9293ed1e2a3e3529b Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Mon, 16 Jan 2023 10:18:04 +0100
Subject: [PATCH 146/158] Use `std::string_view` instead of `const
 std::string_view &`

---
 src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp | 2 +-
 src/Disks/ObjectStorages/S3/S3ObjectStorage.h   | 2 +-
 src/IO/S3Common.cpp                             | 2 +-
 src/IO/S3Common.h                               | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp
index e402c18efac..a56a78d6722 100644
--- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp
@@ -135,7 +135,7 @@ bool S3ObjectStorage::exists(const StoredObject & object) const
     return S3::objectExists(*client.get(), bucket, object.absolute_path, {}, /* for_disk_s3= */ true);
 }
 
-void S3ObjectStorage::checkObjectExists(const std::string & bucket_from, const std::string & key, const std::string_view & description) const
+void S3ObjectStorage::checkObjectExists(const std::string & bucket_from, const std::string & key, std::string_view description) const
 {
     return S3::checkObjectExists(*client.get(), bucket_from, key, {}, /* for_disk_s3= */ true, description);
 }
diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h
index 9dccf3dbcf9..a6318bf59b8 100644
--- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h
+++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h
@@ -187,7 +187,7 @@ private:
     void removeObjectsImpl(const StoredObjects & objects, bool if_exists);
 
     size_t getObjectSize(const std::string & bucket_from, const std::string & key) const;
-    void checkObjectExists(const std::string & bucket_from, const std::string & key, const std::string_view & description) const;
+    void checkObjectExists(const std::string & bucket_from, const std::string & key, std::string_view description) const;
 
     std::string bucket;
 
diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp
index e41c8429891..c09c1da616a 100644
--- a/src/IO/S3Common.cpp
+++ b/src/IO/S3Common.cpp
@@ -957,7 +957,7 @@ namespace S3
             key, bucket, error.GetMessage());
     }
 
-    void checkObjectExists(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3, const std::string_view & description)
+    void checkObjectExists(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3, std::string_view description)
     {
         auto outcome = getObjectAttributes(client, bucket, key, version_id, for_disk_s3);
         if (outcome.IsSuccess())
diff --git a/src/IO/S3Common.h b/src/IO/S3Common.h
index 546a3b49a27..473ed3f4bc3 100644
--- a/src/IO/S3Common.h
+++ b/src/IO/S3Common.h
@@ -144,7 +144,7 @@ size_t getObjectSize(const Aws::S3::S3Client & client, const String & bucket, co
 bool objectExists(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool for_disk_s3 = false);
 
 /// Throws an exception if a specified object doesn't exist. `description` is used as a part of the error message.
-void checkObjectExists(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool for_disk_s3 = false, const std::string_view & description = {});
+void checkObjectExists(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool for_disk_s3 = false, std::string_view description = {});
 
 bool isNotFoundError(Aws::S3::S3Errors error);
 

From 7030b6409609b309d04f028e9c07bbfec07c4623 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Mon, 16 Jan 2023 10:46:58 +0100
Subject: [PATCH 147/158] Fix build.

---
 src/IO/S3Common.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp
index c09c1da616a..297ab46017a 100644
--- a/src/IO/S3Common.cpp
+++ b/src/IO/S3Common.cpp
@@ -944,11 +944,11 @@ namespace S3
 
     bool objectExists(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3)
     {
-        auto [exists, error] = checkObjectExists(client, bucket, key, version_id, for_disk_s3);
-
-        if (exists)
+        auto outcome = getObjectAttributes(client, bucket, key, version_id, for_disk_s3);
+        if (outcome.IsSuccess())
             return true;
 
+        const auto & error = outcome.GetError();
         if (isNotFoundError(error.GetErrorType()))
             return false;
 
@@ -964,7 +964,7 @@ namespace S3
             return;
         const auto & error = outcome.GetError();
         throw S3Exception(error.GetErrorType(), "{}Object {} in bucket {} suddenly disappeared: {}",
-                          (description.empty() ? "" : (String(description) + ": ")), key, bucket, error.GetErrorMessage());
+                          (description.empty() ? "" : (String(description) + ": ")), key, bucket, error.GetMessage());
     }
 
     std::map<String, String> getObjectMetadata(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3, bool throw_on_error)

From 4a07d3682f1fc0c88548b4071f3d22ce8fd929d0 Mon Sep 17 00:00:00 2001
From: "Mikhail f. Shiryaev" <felixoid@clickhouse.com>
Date: Mon, 16 Jan 2023 11:04:57 +0100
Subject: [PATCH 148/158] Fix wrong approved_at, simplify conditions

---
 tests/ci/merge_pr.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/tests/ci/merge_pr.py b/tests/ci/merge_pr.py
index d5569ff0d99..847b49cbb28 100644
--- a/tests/ci/merge_pr.py
+++ b/tests/ci/merge_pr.py
@@ -38,13 +38,18 @@ class Reviews:
         self.approved_at = datetime.fromtimestamp(0)
         for r in self.reviews:
             user = r.user
-            if self._review_per_user.get(user):
-                if r.state in self.STATES:
-                    self._review_per_user[user] = r
-                    if r.state == "APPROVED":
-                        self.approved_at = max(r.submitted_at, self.approved_at)
+            if r.state not in self.STATES:
                 continue
-            self._review_per_user[user] = r
+
+            if r.state == "APPROVED":
+                self.approved_at = max(r.submitted_at, self.approved_at)
+
+            if not self._review_per_user.get(user):
+                self._review_per_user[user] = r
+                continue
+
+            if r.submitted_at < self._review_per_user[user].submitted_at:
+                self._review_per_user[user] = r
 
     def is_approved(self, team: List[NamedUser]) -> bool:
         """Checks if the PR is approved, and no changes made after the last approval"""

From 9ad6e1b1298782e5daf1ae3c05c448da7c4e5344 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Mon, 16 Jan 2023 14:09:55 +0300
Subject: [PATCH 149/158] Update logger_useful.h

---
 src/Common/logger_useful.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Common/logger_useful.h b/src/Common/logger_useful.h
index a2f4e26d7f6..e83245c0fa1 100644
--- a/src/Common/logger_useful.h
+++ b/src/Common/logger_useful.h
@@ -7,7 +7,7 @@
 #include <Poco/Message.h>
 #include <Common/CurrentThread.h>
 
-/// This wrapper is usefult to save formatted message into a String before sending it to a logger
+/// This wrapper is useful to save formatted message into a String before sending it to a logger
 class LogToStrImpl
 {
     String & out_str;

From 26e6cb089972251e66a19901200e4f6f6ad09487 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Mon, 16 Jan 2023 14:51:32 +0300
Subject: [PATCH 150/158] Update 02028_create_select_settings.sql

---
 tests/queries/0_stateless/02028_create_select_settings.sql | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/queries/0_stateless/02028_create_select_settings.sql b/tests/queries/0_stateless/02028_create_select_settings.sql
index 5275d335f53..35102dd68c7 100644
--- a/tests/queries/0_stateless/02028_create_select_settings.sql
+++ b/tests/queries/0_stateless/02028_create_select_settings.sql
@@ -1 +1,3 @@
+-- Tags: no-ordinary-database
+
 create table test_table engine MergeTree order by a as select a_table.a, b_table.b_arr from (select arrayJoin(range(10000)) as a) a_table cross join (select range(10000) as b_arr) b_table settings max_memory_usage = 1; -- { serverError MEMORY_LIMIT_EXCEEDED }

From 4d22b49be72cfe72c9d6da0675f2d9bd3b01f5ac Mon Sep 17 00:00:00 2001
From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com>
Date: Mon, 16 Jan 2023 14:19:18 +0100
Subject: [PATCH 151/158] Update DiskObjectStorage.cpp

---
 src/Disks/ObjectStorages/DiskObjectStorage.cpp | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.cpp b/src/Disks/ObjectStorages/DiskObjectStorage.cpp
index 24c001b3046..db8f90e777d 100644
--- a/src/Disks/ObjectStorages/DiskObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/DiskObjectStorage.cpp
@@ -520,16 +520,11 @@ std::unique_ptr<ReadBufferFromFileBase> DiskObjectStorage::readFile(
     std::optional<size_t> read_hint,
     std::optional<size_t> file_size) const
 {
-    auto result = object_storage->readObjects(
+    return object_storage->readObjects(
         metadata_storage->getStorageObjects(path),
         object_storage->getAdjustedSettingsFromMetadataFile(settings, path),
         read_hint,
         file_size);
-
-    if (read_hint)
-        result->setReadUntilPosition(*read_hint);
-
-    return result;
 }
 
 std::unique_ptr<WriteBufferFromFileBase> DiskObjectStorage::writeFile(

From 7b612da871fc976bbb2b052a08ab7a3a53e5a109 Mon Sep 17 00:00:00 2001
From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com>
Date: Mon, 16 Jan 2023 14:21:09 +0100
Subject: [PATCH 152/158] Update AsynchronousReadIndirectBufferFromRemoteFS.cpp

---
 .../IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp      | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp
index 0e50b064471..7e325aac8f7 100644
--- a/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp
+++ b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp
@@ -164,7 +164,8 @@ bool AsynchronousReadIndirectBufferFromRemoteFS::nextImpl()
     if (!hasPendingDataToRead())
         return false;
 
-    assert(file_offset_of_buffer_end <= impl->getFileSize());
+    chassert(file_offset_of_buffer_end <= impl->getFileSize());
+    chassert(file_offset_of_buffer_end >= impl->getImplementationBufferOffset());
 
     Stopwatch watch;
     CurrentMetrics::Increment metric_increment{CurrentMetrics::AsynchronousReadWait};
@@ -226,8 +227,8 @@ bool AsynchronousReadIndirectBufferFromRemoteFS::nextImpl()
     /// In case of multiple files for the same file in clickhouse (i.e. log family)
     /// file_offset_of_buffer_end will not match getImplementationBufferOffset()
     /// so we use [impl->getImplementationBufferOffset(), impl->getFileSize()]
-    assert(file_offset_of_buffer_end >= impl->getImplementationBufferOffset());
-    assert(file_offset_of_buffer_end <= impl->getFileSize());
+    chassert(file_offset_of_buffer_end >= impl->getImplementationBufferOffset());
+    chassert(file_offset_of_buffer_end <= impl->getFileSize());
 
     prefetch_future = {};
     return bytes_read;

From 04d95f487714384504510bca62245853d4c10dfa Mon Sep 17 00:00:00 2001
From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com>
Date: Mon, 16 Jan 2023 16:47:04 +0100
Subject: [PATCH 153/158] Fix

---
 src/Storages/StorageS3.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h
index a5e1ed4e0b5..02fcb7d624c 100644
--- a/src/Storages/StorageS3.h
+++ b/src/Storages/StorageS3.h
@@ -191,6 +191,7 @@ private:
             reader = std::move(other.reader);
             pipeline = std::move(other.pipeline);
             read_buf = std::move(other.read_buf);
+            path = std::move(other.path);
             return *this;
         }
 

From df75c24f014350ce23115fe29446bcb1f2fa645f Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@gmail.com>
Date: Mon, 16 Jan 2023 19:14:28 +0300
Subject: [PATCH 154/158] Revert "Disallow Gorilla codec on non-float columns"

---
 src/Compression/CompressionCodecDelta.cpp     |  1 -
 .../CompressionCodecDoubleDelta.cpp           |  1 -
 src/Compression/CompressionCodecFPC.cpp       |  1 -
 src/Compression/CompressionCodecGorilla.cpp   | 12 +--
 .../CompressionFactoryAdditions.cpp           | 42 ++++------
 src/Compression/ICompressionCodec.h           |  6 --
 .../tests/gtest_compressionCodec.cpp          | 83 ++++++++++++++-----
 tests/performance/codecs_int_insert.xml       |  1 +
 tests/performance/codecs_int_select.xml       |  1 +
 .../01272_suspicious_codecs.reference         |  4 -
 .../0_stateless/01272_suspicious_codecs.sql   | 40 +++------
 .../02533_gorilla_on_nonfloat.reference       |  0
 .../0_stateless/02533_gorilla_on_nonfloat.sql |  4 -
 13 files changed, 91 insertions(+), 105 deletions(-)
 delete mode 100644 tests/queries/0_stateless/02533_gorilla_on_nonfloat.reference
 delete mode 100644 tests/queries/0_stateless/02533_gorilla_on_nonfloat.sql

diff --git a/src/Compression/CompressionCodecDelta.cpp b/src/Compression/CompressionCodecDelta.cpp
index 88534f6082b..31800b6b332 100644
--- a/src/Compression/CompressionCodecDelta.cpp
+++ b/src/Compression/CompressionCodecDelta.cpp
@@ -28,7 +28,6 @@ protected:
 
     bool isCompression() const override { return false; }
     bool isGenericCompression() const override { return false; }
-    bool isDelta() const override { return true; }
 
 private:
     UInt8 delta_bytes_size;
diff --git a/src/Compression/CompressionCodecDoubleDelta.cpp b/src/Compression/CompressionCodecDoubleDelta.cpp
index a18ce81e02b..dd2507ab14a 100644
--- a/src/Compression/CompressionCodecDoubleDelta.cpp
+++ b/src/Compression/CompressionCodecDoubleDelta.cpp
@@ -133,7 +133,6 @@ protected:
 
     bool isCompression() const override { return true; }
     bool isGenericCompression() const override { return false; }
-    bool isDelta() const override { return true; }
 
 private:
     UInt8 data_bytes_size;
diff --git a/src/Compression/CompressionCodecFPC.cpp b/src/Compression/CompressionCodecFPC.cpp
index 0468adc4610..48eba210b60 100644
--- a/src/Compression/CompressionCodecFPC.cpp
+++ b/src/Compression/CompressionCodecFPC.cpp
@@ -39,7 +39,6 @@ protected:
 
     bool isCompression() const override { return true; }
     bool isGenericCompression() const override { return false; }
-    bool isFloatingPointTimeSeries() const override { return true; }
 
 private:
     static constexpr UInt32 HEADER_SIZE = 2;
diff --git a/src/Compression/CompressionCodecGorilla.cpp b/src/Compression/CompressionCodecGorilla.cpp
index a3aa5850e19..88b8c2bc3bb 100644
--- a/src/Compression/CompressionCodecGorilla.cpp
+++ b/src/Compression/CompressionCodecGorilla.cpp
@@ -123,7 +123,6 @@ protected:
 
     bool isCompression() const override { return true; }
     bool isGenericCompression() const override { return false; }
-    bool isFloatingPointTimeSeries() const override { return true; }
 
 private:
     UInt8 data_bytes_size;
@@ -445,19 +444,14 @@ void CompressionCodecGorilla::doDecompressData(const char * source, UInt32 sourc
 void registerCodecGorilla(CompressionCodecFactory & factory)
 {
     UInt8 method_code = static_cast<UInt8>(CompressionMethodByte::Gorilla);
-    auto codec_builder = [&](const ASTPtr & arguments, const IDataType * column_type) -> CompressionCodecPtr
+    factory.registerCompressionCodecWithType("Gorilla", method_code,
+        [&](const ASTPtr & arguments, const IDataType * column_type) -> CompressionCodecPtr
     {
         if (arguments)
             throw Exception("Codec Gorilla does not accept any arguments", ErrorCodes::BAD_ARGUMENTS);
 
-        if (column_type != nullptr)
-            if (!WhichDataType(*column_type).isFloat())
-                throw Exception(ErrorCodes::BAD_ARGUMENTS, "Gorilla codec is not applicable for {} because the data type is not float",
-                        column_type->getName());
-
         UInt8 data_bytes_size = column_type ? getDataBytesSize(column_type) : 0;
         return std::make_shared<CompressionCodecGorilla>(data_bytes_size);
-    };
-    factory.registerCompressionCodecWithType("Gorilla", method_code, codec_builder);
+    });
 }
 }
diff --git a/src/Compression/CompressionFactoryAdditions.cpp b/src/Compression/CompressionFactoryAdditions.cpp
index fd2d6a67aa8..3e215076871 100644
--- a/src/Compression/CompressionFactoryAdditions.cpp
+++ b/src/Compression/CompressionFactoryAdditions.cpp
@@ -59,17 +59,15 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
     {
         ASTPtr codecs_descriptions = std::make_shared<ASTExpressionList>();
 
-        bool with_compressing_codec = false;
-        bool with_none_codec = false;
+        bool is_compression = false;
+        bool has_none = false;
         std::optional<size_t> generic_compression_codec_pos;
-        std::optional<size_t> first_delta_codec_pos;
-        std::optional<size_t> last_floating_point_time_series_codec_pos;
-        std::set<size_t> encryption_codecs_pos;
+        std::set<size_t> encryption_codecs;
 
         bool can_substitute_codec_arguments = true;
         for (size_t i = 0, size = func->arguments->children.size(); i < size; ++i)
         {
-            const ASTPtr & inner_codec_ast = func->arguments->children[i];
+            const auto & inner_codec_ast = func->arguments->children[i];
             String codec_family_name;
             ASTPtr codec_arguments;
             if (const auto * family_name = inner_codec_ast->as<ASTIdentifier>())
@@ -85,7 +83,8 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
             else
                 throw Exception("Unexpected AST element for compression codec", ErrorCodes::UNEXPECTED_AST_STRUCTURE);
 
-            /// Replace "Default" codec by configured default codec which may depend on different settings and data properties at runtime.
+            /// Default codec replaced with current default codec which may depend on different
+            /// settings (and properties of data) in runtime.
             CompressionCodecPtr result_codec;
             if (codec_family_name == DEFAULT_CODEC_NAME)
             {
@@ -137,27 +136,21 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
                 codecs_descriptions->children.emplace_back(result_codec->getCodecDesc());
             }
 
-            with_compressing_codec |= result_codec->isCompression();
-            with_none_codec |= result_codec->isNone();
+            is_compression |= result_codec->isCompression();
+            has_none |= result_codec->isNone();
 
             if (!generic_compression_codec_pos && result_codec->isGenericCompression())
                 generic_compression_codec_pos = i;
 
             if (result_codec->isEncryption())
-                encryption_codecs_pos.insert(i);
-
-            if (result_codec->isDelta() && !first_delta_codec_pos.has_value())
-                first_delta_codec_pos = i;
-
-            if (result_codec->isFloatingPointTimeSeries())
-                last_floating_point_time_series_codec_pos = i;
+                encryption_codecs.insert(i);
         }
 
         String codec_description = queryToString(codecs_descriptions);
 
         if (sanity_check)
         {
-            if (codecs_descriptions->children.size() > 1 && with_none_codec)
+            if (codecs_descriptions->children.size() > 1 && has_none)
                 throw Exception(
                     "It does not make sense to have codec NONE along with other compression codecs: " + codec_description
                         + ". (Note: you can enable setting 'allow_suspicious_codecs' to skip this check).",
@@ -166,7 +159,7 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
             /// Allow to explicitly specify single NONE codec if user don't want any compression.
             /// But applying other transformations solely without compression (e.g. Delta) does not make sense.
             /// It's okay to apply encryption codecs solely without anything else.
-            if (!with_compressing_codec && !with_none_codec && encryption_codecs_pos.size() != codecs_descriptions->children.size())
+            if (!is_compression && !has_none && encryption_codecs.size() != codecs_descriptions->children.size())
                 throw Exception(
                     "Compression codec " + codec_description
                         + " does not compress anything."
@@ -178,8 +171,8 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
 
             /// It does not make sense to apply any non-encryption codecs
             /// after encryption one.
-            if (!encryption_codecs_pos.empty() &&
-                *encryption_codecs_pos.begin() != codecs_descriptions->children.size() - encryption_codecs_pos.size())
+            if (!encryption_codecs.empty() &&
+                *encryption_codecs.begin() != codecs_descriptions->children.size() - encryption_codecs.size())
                 throw Exception("The combination of compression codecs " + codec_description + " is meaningless,"
                                 " because it does not make sense to apply any non-post-processing codecs after"
                                 " post-processing ones. (Note: you can enable setting 'allow_suspicious_codecs'"
@@ -188,18 +181,11 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
             /// It does not make sense to apply any transformations after generic compression algorithm
             /// So, generic compression can be only one and only at the end.
             if (generic_compression_codec_pos &&
-                *generic_compression_codec_pos != codecs_descriptions->children.size() - 1 - encryption_codecs_pos.size())
+                *generic_compression_codec_pos != codecs_descriptions->children.size() - 1 - encryption_codecs.size())
                 throw Exception("The combination of compression codecs " + codec_description + " is meaningless,"
                     " because it does not make sense to apply any transformations after generic compression algorithm."
                     " (Note: you can enable setting 'allow_suspicious_codecs' to skip this check).", ErrorCodes::BAD_ARGUMENTS);
 
-            /// Floating point time series codecs usually have implicit delta compression (or something equivalent), so it does not make
-            /// sense to run delta compression manually. Another reason for blocking such combination is occasional data corruption (#45195).
-            if (first_delta_codec_pos.has_value() && last_floating_point_time_series_codec_pos.has_value()
-                && (*first_delta_codec_pos < last_floating_point_time_series_codec_pos))
-                throw Exception("The combination of compression codecs " + codec_description + " is meaningless,"
-                    " because it does not make sense to apply delta transformations before floating point time series codecs."
-                    " (Note: you can enable setting 'allow_suspicious_codecs' to skip this check).", ErrorCodes::BAD_ARGUMENTS);
         }
 
         /// For columns with nested types like Tuple(UInt32, UInt64) we
diff --git a/src/Compression/ICompressionCodec.h b/src/Compression/ICompressionCodec.h
index c717392bbca..f40404a84f3 100644
--- a/src/Compression/ICompressionCodec.h
+++ b/src/Compression/ICompressionCodec.h
@@ -113,12 +113,6 @@ public:
     /// If it does nothing.
     virtual bool isNone() const { return false; }
 
-    /// If the only purpose of the codec is to delta (or double-delta) the data.
-    virtual bool isDelta() const { return false; }
-
-    /// If the codec is specialized for floating point time series.
-    virtual bool isFloatingPointTimeSeries() const { return false; }
-
 protected:
     /// This is used for fuzz testing
     friend int LLVMFuzzerTestOneInput(const uint8_t * data, size_t size);
diff --git a/src/Compression/tests/gtest_compressionCodec.cpp b/src/Compression/tests/gtest_compressionCodec.cpp
index a843144c7c4..b4c29bf9ce6 100644
--- a/src/Compression/tests/gtest_compressionCodec.cpp
+++ b/src/Compression/tests/gtest_compressionCodec.cpp
@@ -529,13 +529,6 @@ public:
 
 TEST_P(CodecTest, TranscodingWithDataType)
 {
-    /// Gorilla can only be applied to floating point columns
-    bool codec_is_gorilla = std::get<0>(GetParam()).codec_statement.find("Gorilla") != std::string::npos;
-    WhichDataType which(std::get<1>(GetParam()).data_type.get());
-    bool data_is_float = which.isFloat();
-    if (codec_is_gorilla && !data_is_float)
-        GTEST_SKIP() << "Skipping Gorilla-compressed integer column";
-
     const auto codec = makeCodec(CODEC_WITH_DATA_TYPE);
     testTranscoding(*codec);
 }
@@ -1211,20 +1204,68 @@ auto DDperformanceTestSequence()
 }
 
 // prime numbers in ascending order with some random repetitions hit all the cases of Gorilla.
-// auto PrimesWithMultiplierGenerator = [](int multiplier = 1)
-// {
-//     return [multiplier](auto i)
-//     {
-//         static const int vals[] = {
-//              2, 3, 5, 7, 11, 11, 13, 17, 19, 23, 29, 29, 31, 37, 41, 43,
-//             47, 47, 53, 59, 61, 61, 67, 71, 73, 79, 83, 89, 89, 97, 101, 103,
-//             107, 107, 109, 113, 113, 127, 127, 127
-//         };
-//         static const size_t count = sizeof(vals)/sizeof(vals[0]);
-//
-//         return static_cast<UInt64>(vals[i % count]) * multiplier;
-//     };
-// };
+auto PrimesWithMultiplierGenerator = [](int multiplier = 1)
+{
+    return [multiplier](auto i)
+    {
+        static const int vals[] = {
+             2, 3, 5, 7, 11, 11, 13, 17, 19, 23, 29, 29, 31, 37, 41, 43,
+            47, 47, 53, 59, 61, 61, 67, 71, 73, 79, 83, 89, 89, 97, 101, 103,
+            107, 107, 109, 113, 113, 127, 127, 127
+        };
+        static const size_t count = sizeof(vals)/sizeof(vals[0]);
+
+        return static_cast<UInt64>(vals[i % count]) * multiplier;
+    };
+};
+
+template <typename ValueType>
+auto GCompatibilityTestSequence()
+{
+    // Also multiply result by some factor to test large values on types that can hold those.
+    return generateSeq<ValueType>(G(PrimesWithMultiplierGenerator(intExp10(sizeof(ValueType)))), 0, 42);
+}
+
+INSTANTIATE_TEST_SUITE_P(Gorilla,
+    CodecTestCompatibility,
+    ::testing::Combine(
+        ::testing::Values(Codec("Gorilla")),
+        ::testing::ValuesIn(std::initializer_list<std::tuple<CodecTestSequence, std::string>>{
+            {
+                GCompatibilityTestSequence<Int8>(),
+                BIN_STR("\x95\x35\x00\x00\x00\x2a\x00\x00\x00\x01\x00\x2a\x00\x00\x00\x14\xe1\xdd\x25\xe5\x7b\x29\x86\xee\x2a\x16\x5a\xc5\x0b\x23\x75\x1b\x3c\xb1\x97\x8b\x5f\xcb\x43\xd9\xc5\x48\xab\x23\xaf\x62\x93\x71\x4a\x73\x0f\xc6\x0a")
+            },
+            {
+                GCompatibilityTestSequence<UInt8>(),
+                BIN_STR("\x95\x35\x00\x00\x00\x2a\x00\x00\x00\x01\x00\x2a\x00\x00\x00\x14\xe1\xdd\x25\xe5\x7b\x29\x86\xee\x2a\x16\x5a\xc5\x0b\x23\x75\x1b\x3c\xb1\x97\x8b\x5f\xcb\x43\xd9\xc5\x48\xab\x23\xaf\x62\x93\x71\x4a\x73\x0f\xc6\x0a")
+            },
+            {
+                GCompatibilityTestSequence<Int16>(),
+                BIN_STR("\x95\x52\x00\x00\x00\x54\x00\x00\x00\x02\x00\x2a\x00\x00\x00\xc8\x00\xdc\xfe\x66\xdb\x1f\x4e\xa7\xde\xdc\xd5\xec\x6e\xf7\x37\x3a\x23\xe7\x63\xf5\x6a\x8e\x99\x37\x34\xf9\xf8\x2e\x76\x35\x2d\x51\xbb\x3b\xc3\x6d\x13\xbf\x86\x53\x9e\x25\xe4\xaf\xaf\x63\xd5\x6a\x6e\x76\x35\x3a\x27\xd3\x0f\x91\xae\x6b\x33\x57\x6e\x64\xcc\x55\x81\xe4")
+            },
+            {
+                GCompatibilityTestSequence<UInt16>(),
+                BIN_STR("\x95\x52\x00\x00\x00\x54\x00\x00\x00\x02\x00\x2a\x00\x00\x00\xc8\x00\xdc\xfe\x66\xdb\x1f\x4e\xa7\xde\xdc\xd5\xec\x6e\xf7\x37\x3a\x23\xe7\x63\xf5\x6a\x8e\x99\x37\x34\xf9\xf8\x2e\x76\x35\x2d\x51\xbb\x3b\xc3\x6d\x13\xbf\x86\x53\x9e\x25\xe4\xaf\xaf\x63\xd5\x6a\x6e\x76\x35\x3a\x27\xd3\x0f\x91\xae\x6b\x33\x57\x6e\x64\xcc\x55\x81\xe4")
+            },
+            {
+                GCompatibilityTestSequence<Int32>(),
+                BIN_STR("\x95\x65\x00\x00\x00\xa8\x00\x00\x00\x04\x00\x2a\x00\x00\x00\x20\x4e\x00\x00\xe4\x57\x63\xc0\xbb\x67\xbc\xce\x91\x97\x99\x15\x9e\xe3\x36\x3f\x89\x5f\x8e\xf2\xec\x8e\xd3\xbf\x75\x43\x58\xc4\x7e\xcf\x93\x43\x38\xc6\x91\x36\x1f\xe7\xb6\x11\x6f\x02\x73\x46\xef\xe0\xec\x50\xfb\x79\xcb\x9c\x14\xfa\x13\xea\x8d\x66\x43\x48\xa0\xde\x3a\xcf\xff\x26\xe0\x5f\x93\xde\x5e\x7f\x6e\x36\x5e\xe6\xb4\x66\x5d\xb0\x0e\xc4")
+            },
+            {
+                GCompatibilityTestSequence<UInt32>(),
+                BIN_STR("\x95\x65\x00\x00\x00\xa8\x00\x00\x00\x04\x00\x2a\x00\x00\x00\x20\x4e\x00\x00\xe4\x57\x63\xc0\xbb\x67\xbc\xce\x91\x97\x99\x15\x9e\xe3\x36\x3f\x89\x5f\x8e\xf2\xec\x8e\xd3\xbf\x75\x43\x58\xc4\x7e\xcf\x93\x43\x38\xc6\x91\x36\x1f\xe7\xb6\x11\x6f\x02\x73\x46\xef\xe0\xec\x50\xfb\x79\xcb\x9c\x14\xfa\x13\xea\x8d\x66\x43\x48\xa0\xde\x3a\xcf\xff\x26\xe0\x5f\x93\xde\x5e\x7f\x6e\x36\x5e\xe6\xb4\x66\x5d\xb0\x0e\xc4")
+            },
+            {
+                GCompatibilityTestSequence<Int64>(),
+                BIN_STR("\x95\x91\x00\x00\x00\x50\x01\x00\x00\x08\x00\x2a\x00\x00\x00\x00\xc2\xeb\x0b\x00\x00\x00\x00\xe3\x2b\xa0\xa6\x19\x85\x98\xdc\x45\x74\x74\x43\xc2\x57\x41\x4c\x6e\x42\x79\xd9\x8f\x88\xa5\x05\xf3\xf1\x94\xa3\x62\x1e\x02\xdf\x05\x10\xf1\x15\x97\x35\x2a\x50\x71\x0f\x09\x6c\x89\xf7\x65\x1d\x11\xb7\xcc\x7d\x0b\x70\xc1\x86\x88\x48\x47\x87\xb6\x32\x26\xa7\x86\x87\x88\xd3\x93\x3d\xfc\x28\x68\x85\x05\x0b\x13\xc6\x5f\xd4\x70\xe1\x5e\x76\xf1\x9f\xf3\x33\x2a\x14\x14\x5e\x40\xc1\x5c\x28\x3f\xec\x43\x03\x05\x11\x91\xe8\xeb\x8e\x0a\x0e\x27\x21\x55\xcb\x39\xbc\x6a\xff\x11\x5d\x81\xa0\xa6\x10")
+            },
+            {
+                GCompatibilityTestSequence<UInt64>(),
+                BIN_STR("\x95\x91\x00\x00\x00\x50\x01\x00\x00\x08\x00\x2a\x00\x00\x00\x00\xc2\xeb\x0b\x00\x00\x00\x00\xe3\x2b\xa0\xa6\x19\x85\x98\xdc\x45\x74\x74\x43\xc2\x57\x41\x4c\x6e\x42\x79\xd9\x8f\x88\xa5\x05\xf3\xf1\x94\xa3\x62\x1e\x02\xdf\x05\x10\xf1\x15\x97\x35\x2a\x50\x71\x0f\x09\x6c\x89\xf7\x65\x1d\x11\xb7\xcc\x7d\x0b\x70\xc1\x86\x88\x48\x47\x87\xb6\x32\x26\xa7\x86\x87\x88\xd3\x93\x3d\xfc\x28\x68\x85\x05\x0b\x13\xc6\x5f\xd4\x70\xe1\x5e\x76\xf1\x9f\xf3\x33\x2a\x14\x14\x5e\x40\xc1\x5c\x28\x3f\xec\x43\x03\x05\x11\x91\xe8\xeb\x8e\x0a\x0e\x27\x21\x55\xcb\x39\xbc\x6a\xff\x11\x5d\x81\xa0\xa6\x10")
+            },
+        })
+    )
+);
 
 // These 'tests' try to measure performance of encoding and decoding and hence only make sense to be run locally,
 // also they require pretty big data to run against and generating this data slows down startup of unit test process.
diff --git a/tests/performance/codecs_int_insert.xml b/tests/performance/codecs_int_insert.xml
index 618e20160f8..caefaba3725 100644
--- a/tests/performance/codecs_int_insert.xml
+++ b/tests/performance/codecs_int_insert.xml
@@ -13,6 +13,7 @@
                 <value>Delta</value>
                 <value>T64</value>
                 <value>DoubleDelta</value>
+                <value>Gorilla</value>
             </values>
         </substitution>
         <substitution>
diff --git a/tests/performance/codecs_int_select.xml b/tests/performance/codecs_int_select.xml
index 62c1ee16e7b..7d47cd300d8 100644
--- a/tests/performance/codecs_int_select.xml
+++ b/tests/performance/codecs_int_select.xml
@@ -13,6 +13,7 @@
                 <value>Delta</value>
                 <value>T64</value>
                 <value>DoubleDelta</value>
+                <value>Gorilla</value>
             </values>
         </substitution>
         <substitution>
diff --git a/tests/queries/0_stateless/01272_suspicious_codecs.reference b/tests/queries/0_stateless/01272_suspicious_codecs.reference
index 97e60839ea7..559b6df2693 100644
--- a/tests/queries/0_stateless/01272_suspicious_codecs.reference
+++ b/tests/queries/0_stateless/01272_suspicious_codecs.reference
@@ -6,8 +6,6 @@ CREATE TABLE default.codecs5\n(\n    `a` UInt8 CODEC(LZ4, ZSTD(1))\n)\nENGINE =
 CREATE TABLE default.codecs6\n(\n    `a` UInt8 CODEC(Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
 CREATE TABLE default.codecs7\n(\n    `a` UInt8 CODEC(Delta(1), Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
 CREATE TABLE default.codecs8\n(\n    `a` UInt8 CODEC(LZ4, Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
-CREATE TABLE default.codecs9\n(\n    `a` Float64 CODEC(Delta(8), FPC(12))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
-CREATE TABLE default.codecs10\n(\n    `a` Float64 CODEC(DoubleDelta, Gorilla)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
 CREATE TABLE default.codecs1\n(\n    `a` UInt8 CODEC(NONE, NONE)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
 CREATE TABLE default.codecs2\n(\n    `a` UInt8 CODEC(NONE, LZ4)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
 CREATE TABLE default.codecs3\n(\n    `a` UInt8 CODEC(LZ4, NONE)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
@@ -16,5 +14,3 @@ CREATE TABLE default.codecs5\n(\n    `a` UInt8 CODEC(LZ4, ZSTD(1))\n)\nENGINE =
 CREATE TABLE default.codecs6\n(\n    `a` UInt8 CODEC(Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
 CREATE TABLE default.codecs7\n(\n    `a` UInt8 CODEC(Delta(1), Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
 CREATE TABLE default.codecs8\n(\n    `a` UInt8 CODEC(LZ4, Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
-CREATE TABLE default.codecs9\n(\n    `a` Float64 CODEC(Delta(8), FPC(12))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
-CREATE TABLE default.codecs10\n(\n    `a` Float64 CODEC(DoubleDelta, Gorilla)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
diff --git a/tests/queries/0_stateless/01272_suspicious_codecs.sql b/tests/queries/0_stateless/01272_suspicious_codecs.sql
index 660a9a5edf8..5baa30e3cf4 100644
--- a/tests/queries/0_stateless/01272_suspicious_codecs.sql
+++ b/tests/queries/0_stateless/01272_suspicious_codecs.sql
@@ -11,28 +11,24 @@ CREATE TABLE codecs
     c Float32 CODEC(Gorilla),
     d UInt8 CODEC(Delta, LZ4),
     e Float64 CODEC(Gorilla, ZSTD),
-    f UInt64 CODEC(Delta, Delta, T64),
+    f UInt32 CODEC(Delta, Delta, Gorilla),
     g DateTime CODEC(DoubleDelta),
     h DateTime64 CODEC(DoubleDelta, LZ4),
-    i String CODEC(NONE),
-    j Float64 (Gorilla, Delta),
-    k Float32 (FPC, DoubleDelta)
+    i String CODEC(NONE)
 ) ENGINE = MergeTree ORDER BY tuple();
 
 DROP TABLE codecs;
 
 -- test what should not work
 
-CREATE TABLE codecs (a UInt8 CODEC(NONE, NONE)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS }
-CREATE TABLE codecs (a UInt8 CODEC(NONE, LZ4)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS }
-CREATE TABLE codecs (a UInt8 CODEC(LZ4, NONE)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS }
-CREATE TABLE codecs (a UInt8 CODEC(LZ4, LZ4)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS }
-CREATE TABLE codecs (a UInt8 CODEC(LZ4, ZSTD)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS }
-CREATE TABLE codecs (a UInt8 CODEC(Delta)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS }
-CREATE TABLE codecs (a UInt8 CODEC(Delta, Delta)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS }
-CREATE TABLE codecs (a UInt8 CODEC(LZ4, Delta)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS }
-CREATE TABLE codecs (a UInt8 CODEC(Delta, FPC)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS }
-CREATE TABLE codecs (a UInt8 CODEC(DoubleDelta, Gorilla)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS }
+CREATE TABLE codecs (a UInt8 CODEC(NONE, NONE)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 36 }
+CREATE TABLE codecs (a UInt8 CODEC(NONE, LZ4)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 36 }
+CREATE TABLE codecs (a UInt8 CODEC(LZ4, NONE)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 36 }
+CREATE TABLE codecs (a UInt8 CODEC(LZ4, LZ4)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 36 }
+CREATE TABLE codecs (a UInt8 CODEC(LZ4, ZSTD)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 36 }
+CREATE TABLE codecs (a UInt8 CODEC(Delta)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 36 }
+CREATE TABLE codecs (a UInt8 CODEC(Delta, Delta)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 36 }
+CREATE TABLE codecs (a UInt8 CODEC(LZ4, Delta)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 36 }
 
 -- test that sanity check is not performed in ATTACH query
 
@@ -44,8 +40,6 @@ DROP TABLE IF EXISTS codecs5;
 DROP TABLE IF EXISTS codecs6;
 DROP TABLE IF EXISTS codecs7;
 DROP TABLE IF EXISTS codecs8;
-DROP TABLE IF EXISTS codecs9;
-DROP TABLE IF EXISTS codecs10;
 
 SET allow_suspicious_codecs = 1;
 
@@ -57,8 +51,6 @@ CREATE TABLE codecs5 (a UInt8 CODEC(LZ4, ZSTD)) ENGINE = MergeTree ORDER BY tupl
 CREATE TABLE codecs6 (a UInt8 CODEC(Delta)) ENGINE = MergeTree ORDER BY tuple();
 CREATE TABLE codecs7 (a UInt8 CODEC(Delta, Delta)) ENGINE = MergeTree ORDER BY tuple();
 CREATE TABLE codecs8 (a UInt8 CODEC(LZ4, Delta)) ENGINE = MergeTree ORDER BY tuple();
-CREATE TABLE codecs9 (a Float64 CODEC(Delta, FPC)) ENGINE = MergeTree ORDER BY tuple();
-CREATE TABLE codecs10 (a Float64 CODEC(DoubleDelta, Gorilla)) ENGINE = MergeTree ORDER BY tuple();
 
 SET allow_suspicious_codecs = 0;
 
@@ -70,8 +62,6 @@ SHOW CREATE TABLE codecs5;
 SHOW CREATE TABLE codecs6;
 SHOW CREATE TABLE codecs7;
 SHOW CREATE TABLE codecs8;
-SHOW CREATE TABLE codecs9;
-SHOW CREATE TABLE codecs10;
 
 DETACH TABLE codecs1;
 DETACH TABLE codecs2;
@@ -81,8 +71,6 @@ DETACH TABLE codecs5;
 DETACH TABLE codecs6;
 DETACH TABLE codecs7;
 DETACH TABLE codecs8;
-DETACH TABLE codecs9;
-DETACH TABLE codecs10;
 
 ATTACH TABLE codecs1;
 ATTACH TABLE codecs2;
@@ -92,8 +80,6 @@ ATTACH TABLE codecs5;
 ATTACH TABLE codecs6;
 ATTACH TABLE codecs7;
 ATTACH TABLE codecs8;
-ATTACH TABLE codecs9;
-ATTACH TABLE codecs10;
 
 SHOW CREATE TABLE codecs1;
 SHOW CREATE TABLE codecs2;
@@ -103,8 +89,6 @@ SHOW CREATE TABLE codecs5;
 SHOW CREATE TABLE codecs6;
 SHOW CREATE TABLE codecs7;
 SHOW CREATE TABLE codecs8;
-SHOW CREATE TABLE codecs9;
-SHOW CREATE TABLE codecs10;
 
 SELECT * FROM codecs1;
 SELECT * FROM codecs2;
@@ -114,8 +98,6 @@ SELECT * FROM codecs5;
 SELECT * FROM codecs6;
 SELECT * FROM codecs7;
 SELECT * FROM codecs8;
-SELECT * FROM codecs9;
-SELECT * FROM codecs10;
 
 DROP TABLE codecs1;
 DROP TABLE codecs2;
@@ -125,5 +107,3 @@ DROP TABLE codecs5;
 DROP TABLE codecs6;
 DROP TABLE codecs7;
 DROP TABLE codecs8;
-DROP TABLE codecs9;
-DROP TABLE codecs10;
diff --git a/tests/queries/0_stateless/02533_gorilla_on_nonfloat.reference b/tests/queries/0_stateless/02533_gorilla_on_nonfloat.reference
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/queries/0_stateless/02533_gorilla_on_nonfloat.sql b/tests/queries/0_stateless/02533_gorilla_on_nonfloat.sql
deleted file mode 100644
index 4fb6ac6c2c0..00000000000
--- a/tests/queries/0_stateless/02533_gorilla_on_nonfloat.sql
+++ /dev/null
@@ -1,4 +0,0 @@
-DROP TABLE IF EXISTS test;
-CREATE TABLE test (id UInt64, val Decimal(15,5) CODEC (Gorilla)) ENGINE = MergeTree() ORDER BY id; -- { serverError BAD_ARGUMENTS }
-CREATE TABLE test (id UInt64, val FixedString(2) CODEC (Gorilla)) ENGINE = MergeTree() ORDER BY id; -- { serverError BAD_ARGUMENTS }
-CREATE TABLE test (id UInt64, val UInt64 CODEC (Gorilla)) ENGINE = MergeTree() ORDER BY id; -- { serverError BAD_ARGUMENTS }

From c8f635e40bb75c6bbf6dd3ac2bf99da7b91da484 Mon Sep 17 00:00:00 2001
From: Yakov Olkhovskiy <yakov@clickhouse.com>
Date: Mon, 16 Jan 2023 18:32:41 +0000
Subject: [PATCH 155/158] WSL1 inconsistency

---
 utils/self-extracting-executable/decompressor.cpp | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/utils/self-extracting-executable/decompressor.cpp b/utils/self-extracting-executable/decompressor.cpp
index be25d315d68..cd9cc51c45b 100644
--- a/utils/self-extracting-executable/decompressor.cpp
+++ b/utils/self-extracting-executable/decompressor.cpp
@@ -363,7 +363,7 @@ int decompressFiles(int input_fd, char * path, char * name, bool & have_compress
 
 #if !defined(OS_DARWIN) && !defined(OS_FREEBSD)
 
-uint32_t getInode(const char * self)
+uint64_t getInode(const char * self)
 {
     std::ifstream maps("/proc/self/maps");
     if (maps.fail())
@@ -380,7 +380,7 @@ uint32_t getInode(const char * self)
     {
         std::stringstream ss(line); // STYLE_CHECK_ALLOW_STD_STRING_STREAM
         std::string addr, mode, offset, id, path;
-        uint32_t inode = 0;
+        uint64_t inode = 0;
         if (ss >> addr >> mode >> offset >> id >> inode >> path && path == self)
             return inode;
     }
@@ -415,7 +415,7 @@ int main(int/* argc*/, char* argv[])
 
 #if !defined(OS_DARWIN) && !defined(OS_FREEBSD)
     /// get inode of this executable
-    uint32_t inode = getInode(self);
+    uint64_t inode = getInode(self);
     if (inode == 0)
     {
         std::cerr << "Unable to obtain inode." << std::endl;
@@ -447,6 +447,11 @@ int main(int/* argc*/, char* argv[])
         return 1;
     }
 
+    /// inconsistency in WSL1 Ubuntu - inode reported in /proc/self/maps is a 64bit to
+    /// 32bit conversion of input_info.st_ino
+    if (input_info.st_ino & 0xFFFFFFFF00000000 && !(inode & 0xFFFFFFFF00000000))
+        input_info.st_ino &= 0x00000000FFFFFFFF;
+
     /// if decompression was performed by another process since this copy was started
     /// then file referred by path "self" is already pointing to different inode
     if (input_info.st_ino != inode)

From 9a520879898ec6deb026722ea6cec0194f383e6d Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Mon, 16 Jan 2023 20:14:39 +0100
Subject: [PATCH 156/158] More complex logic: GetObjectAttributes requests will
 be used only if the endpoint is "*.amazonaws.com", otherwise HeadObject
 requests will be used.

---
 src/IO/S3Common.cpp | 115 ++++++++++++++++++++++++++++++++++----------
 src/IO/S3Common.h   |  11 +----
 2 files changed, 91 insertions(+), 35 deletions(-)

diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp
index 297ab46017a..a18fcf70566 100644
--- a/src/IO/S3Common.cpp
+++ b/src/IO/S3Common.cpp
@@ -29,6 +29,7 @@
 #    include <aws/s3/S3Client.h>
 #    include <aws/s3/model/GetObjectAttributesRequest.h>
 #    include <aws/s3/model/GetObjectRequest.h>
+#    include <aws/s3/model/HeadObjectRequest.h>
 
 #    include <IO/S3/PocoHTTPClientFactory.h>
 #    include <IO/S3/PocoHTTPClient.h>
@@ -42,9 +43,11 @@
 namespace ProfileEvents
 {
     extern const Event S3GetObjectAttributes;
-    extern const Event DiskS3GetObjectAttributes;
     extern const Event S3GetObjectMetadata;
+    extern const Event S3HeadObject;
+    extern const Event DiskS3GetObjectAttributes;
     extern const Event DiskS3GetObjectMetadata;
+    extern const Event DiskS3HeadObject;
 }
 
 namespace DB
@@ -702,26 +705,90 @@ public:
     }
 };
 
-/// Performs a GetObjectAttributes request.
-Aws::S3::Model::GetObjectAttributesOutcome getObjectAttributes(
+/// Extracts the endpoint from a constructed S3 client.
+String getEndpoint(const Aws::S3::S3Client & client)
+{
+    const auto * endpoint_provider = dynamic_cast<const Aws::S3::Endpoint::S3DefaultEpProviderBase *>(const_cast<Aws::S3::S3Client &>(client).accessEndpointProvider().get());
+    if (!endpoint_provider)
+        return {};
+    String endpoint;
+    endpoint_provider->GetBuiltInParameters().GetParameter("Endpoint").GetString(endpoint);
+    return endpoint;
+}
+
+/// Performs a request to get the size and last modification time of an object.
+/// The function performs either HeadObject or GetObjectAttributes request depending on the endpoint.
+std::pair<std::optional<DB::S3::ObjectInfo>, Aws::S3::S3Error> tryGetObjectInfo(
     const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3)
 {
-    ProfileEvents::increment(ProfileEvents::S3GetObjectAttributes);
-    if (for_disk_s3)
-        ProfileEvents::increment(ProfileEvents::DiskS3GetObjectAttributes);
+    auto endpoint = getEndpoint(client);
+    bool use_get_object_attributes_request = (endpoint.find(".amazonaws.com") != String::npos);
 
-    /// We must not use the `HeadObject` request, see the comment about `HeadObjectRequest` in S3Common.h.
+    if (use_get_object_attributes_request)
+    {
+        /// It's better not to use `HeadObject` requests for AWS S3 because they don't work well with the global region.
+        /// Details: `HeadObject` request never returns a response body (even if there is an error) however
+        /// if the request was sent without specifying a region in the endpoint (i.e. for example "https://test.s3.amazonaws.com/mydata.csv"
+        /// instead of "https://test.s3-us-west-2.amazonaws.com/mydata.csv") then that response body is one of the main ways
+        /// to determine the correct region and try to repeat the request again with the correct region.
+        /// For any other request type (`GetObject`, `ListObjects`, etc.) AWS SDK does that because they have response bodies,
+        /// but for `HeadObject` there is no response body so this way doesn't work. That's why we use `GetObjectAttributes` request instead.
+        /// See https://github.com/aws/aws-sdk-cpp/issues/1558 and also the function S3ErrorMarshaller::ExtractRegion() for more information.
 
-    Aws::S3::Model::GetObjectAttributesRequest req;
-    req.SetBucket(bucket);
-    req.SetKey(key);
+        ProfileEvents::increment(ProfileEvents::S3GetObjectAttributes);
+        if (for_disk_s3)
+            ProfileEvents::increment(ProfileEvents::DiskS3GetObjectAttributes);
 
-    if (!version_id.empty())
-        req.SetVersionId(version_id);
+        Aws::S3::Model::GetObjectAttributesRequest req;
+        req.SetBucket(bucket);
+        req.SetKey(key);
 
-    req.SetObjectAttributes({Aws::S3::Model::ObjectAttributes::ObjectSize});
+        if (!version_id.empty())
+            req.SetVersionId(version_id);
 
-    return client.GetObjectAttributes(req);
+        req.SetObjectAttributes({Aws::S3::Model::ObjectAttributes::ObjectSize});
+
+        auto outcome = client.GetObjectAttributes(req);
+        if (outcome.IsSuccess())
+        {
+            const auto & result = outcome.GetResult();
+            DB::S3::ObjectInfo object_info;
+            object_info.size = static_cast<size_t>(result.GetObjectSize());
+            object_info.last_modification_time = result.GetLastModified().Millis() / 1000;
+            return {object_info, {}};
+        }
+
+        return {std::nullopt, outcome.GetError()};
+    }
+    else
+    {
+        /// By default we use `HeadObject` requests.
+        /// We cannot just use `GetObjectAttributes` requests always because some S3 providers (e.g. Minio)
+        /// don't support `GetObjectAttributes` requests.
+
+        ProfileEvents::increment(ProfileEvents::S3HeadObject);
+        if (for_disk_s3)
+            ProfileEvents::increment(ProfileEvents::DiskS3HeadObject);
+
+        Aws::S3::Model::HeadObjectRequest req;
+        req.SetBucket(bucket);
+        req.SetKey(key);
+
+        if (!version_id.empty())
+            req.SetVersionId(version_id);
+
+        auto outcome = client.HeadObject(req);
+        if (outcome.IsSuccess())
+        {
+            const auto & result = outcome.GetResult();
+            DB::S3::ObjectInfo object_info;
+            object_info.size = static_cast<size_t>(result.GetContentLength());
+            object_info.last_modification_time = result.GetLastModified().Millis() / 1000;
+            return {object_info, {}};
+        }
+
+        return {std::nullopt, outcome.GetError()};
+    }
 }
 
 }
@@ -919,17 +986,15 @@ namespace S3
         return error == Aws::S3::S3Errors::RESOURCE_NOT_FOUND || error == Aws::S3::S3Errors::NO_SUCH_KEY;
     }
 
-    S3::ObjectInfo getObjectInfo(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3, bool throw_on_error)
+    ObjectInfo getObjectInfo(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3, bool throw_on_error)
     {
-        auto outcome = getObjectAttributes(client, bucket, key, version_id, for_disk_s3);
-        if (outcome.IsSuccess())
+        auto [object_info, error] = tryGetObjectInfo(client, bucket, key, version_id, for_disk_s3);
+        if (object_info)
         {
-            const auto & result = outcome.GetResult();
-            return {.size = static_cast<size_t>(result.GetObjectSize()), .last_modification_time = result.GetLastModified().Millis() / 1000};
+            return *object_info;
         }
         else if (throw_on_error)
         {
-            const auto & error = outcome.GetError();
             throw DB::Exception(ErrorCodes::S3_ERROR,
                 "Failed to get object attributes: {}. HTTP response code: {}",
                 error.GetMessage(), static_cast<size_t>(error.GetResponseCode()));
@@ -944,11 +1009,10 @@ namespace S3
 
     bool objectExists(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3)
     {
-        auto outcome = getObjectAttributes(client, bucket, key, version_id, for_disk_s3);
-        if (outcome.IsSuccess())
+        auto [object_info, error] = tryGetObjectInfo(client, bucket, key, version_id, for_disk_s3);
+        if (object_info)
             return true;
 
-        const auto & error = outcome.GetError();
         if (isNotFoundError(error.GetErrorType()))
             return false;
 
@@ -959,10 +1023,9 @@ namespace S3
 
     void checkObjectExists(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3, std::string_view description)
     {
-        auto outcome = getObjectAttributes(client, bucket, key, version_id, for_disk_s3);
-        if (outcome.IsSuccess())
+        auto [object_info, error] = tryGetObjectInfo(client, bucket, key, version_id, for_disk_s3);
+        if (object_info)
             return;
-        const auto & error = outcome.GetError();
         throw S3Exception(error.GetErrorType(), "{}Object {} in bucket {} suddenly disappeared: {}",
                           (description.empty() ? "" : (String(description) + ": ")), key, bucket, error.GetMessage());
     }
diff --git a/src/IO/S3Common.h b/src/IO/S3Common.h
index 473ed3f4bc3..69ae1cbb4f4 100644
--- a/src/IO/S3Common.h
+++ b/src/IO/S3Common.h
@@ -122,14 +122,7 @@ struct URI
 };
 
 /// WARNING: Don't use `HeadObjectRequest`! Use the functions below instead.
-/// Explanation: The `HeadObject` request never returns a response body (even if there is an error) however
-/// if the request was sent without specifying a region in the endpoint (i.e. for example "https://test.s3.amazonaws.com/mydata.csv"
-/// instead of "https://test.s3-us-west-2.amazonaws.com/mydata.csv") then that response body is one of the main ways
-/// to determine the correct region and try to repeat the request again with the correct region.
-/// For any other request type (`GetObject`, `ListObjects`, etc.) AWS SDK does that because they have response bodies,
-/// but for `HeadObject` there is no response body so this way doesn't work.
-/// That's why it's better to avoid using `HeadObject` requests at all.
-/// See https://github.com/aws/aws-sdk-cpp/issues/1558 and also the function S3ErrorMarshaller::ExtractRegion() for more details.
+/// For explanation see the comment about `HeadObject` request in the function tryGetObjectInfo().
 
 struct ObjectInfo
 {
@@ -137,7 +130,7 @@ struct ObjectInfo
     time_t last_modification_time = 0;
 };
 
-S3::ObjectInfo getObjectInfo(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool for_disk_s3 = false, bool throw_on_error = true);
+ObjectInfo getObjectInfo(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool for_disk_s3 = false, bool throw_on_error = true);
 
 size_t getObjectSize(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool for_disk_s3 = false, bool throw_on_error = true);
 

From 6a02bdc917435eaa32a39955a8e162475974c68e Mon Sep 17 00:00:00 2001
From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com>
Date: Tue, 17 Jan 2023 00:37:47 +0100
Subject: [PATCH 157/158] Update AsynchronousReadIndirectBufferFromRemoteFS.cpp

---
 src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp
index 07b74d04947..711f1553ce6 100644
--- a/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp
+++ b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp
@@ -172,11 +172,8 @@ bool AsynchronousReadIndirectBufferFromRemoteFS::nextImpl()
         return false;
 
     chassert(file_offset_of_buffer_end <= impl->getFileSize());
-    chassert(file_offset_of_buffer_end >= impl->getImplementationBufferOffset());
 
     size_t size, offset;
-
-
     if (prefetch_future.valid())
     {
         ProfileEventTimeIncrement<Microseconds> watch(ProfileEvents::AsynchronousRemoteReadWaitMicroseconds);

From 0fc755806e0ddcf5862df5758ccc30eb524929be Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <mikhaylovnikitka@gmail.com>
Date: Tue, 17 Jan 2023 16:17:14 +0100
Subject: [PATCH 158/158] One more attempt to fix race in TCPHandler (#45240)

---
 src/Server/TCPHandler.cpp | 28 +++++++++++++++++++---------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp
index d8ca1c124ae..5b3d18c66f6 100644
--- a/src/Server/TCPHandler.cpp
+++ b/src/Server/TCPHandler.cpp
@@ -402,14 +402,10 @@ void TCPHandler::runImpl()
                     {
                         auto callback = [this]()
                         {
-                            {
-                                std::lock_guard task_callback_lock(task_callback_mutex);
+                            std::scoped_lock lock(task_callback_mutex, fatal_error_mutex);
 
-                                if (isQueryCancelled())
-                                    return true;
-                            }
-
-                            std::lock_guard lock(fatal_error_mutex);
+                            if (isQueryCancelled())
+                                return true;
 
                             sendProgress();
                             sendSelectProfileEvents();
@@ -424,6 +420,9 @@ void TCPHandler::runImpl()
                 }
 
                 state.io.onFinish();
+
+                std::lock_guard lock(task_callback_mutex);
+
                 /// Send final progress after calling onFinish(), since it will update the progress.
                 ///
                 /// NOTE: we cannot send Progress for regular INSERT (with VALUES)
@@ -446,8 +445,11 @@ void TCPHandler::runImpl()
             if (state.is_connection_closed)
                 break;
 
-            sendLogs();
-            sendEndOfStream();
+            {
+                std::lock_guard lock(task_callback_mutex);
+                sendLogs();
+                sendEndOfStream();
+            }
 
             /// QueryState should be cleared before QueryScope, since otherwise
             /// the MemoryTracker will be wrong for possible deallocations.
@@ -760,6 +762,9 @@ void TCPHandler::processOrdinaryQueryWithProcessors()
         }
     }
 
+    /// Defer locking to cover a part of the scope below and everything after it
+    std::unique_lock progress_lock(task_callback_mutex, std::defer_lock);
+
     {
         PullingAsyncPipelineExecutor executor(pipeline);
         CurrentMetrics::Increment query_thread_metric_increment{CurrentMetrics::QueryThread};
@@ -796,6 +801,11 @@ void TCPHandler::processOrdinaryQueryWithProcessors()
             }
         }
 
+        /// This lock wasn't acquired before and we make .lock() call here
+        /// so everything under this line is covered even together
+        /// with sendProgress() out of the scope
+        progress_lock.lock();
+
         /** If data has run out, we will send the profiling data and total values to
           * the last zero block to be able to use
           * this information in the suffix output of stream.