From 043af0a5ca6b86b94fc8ef00d9e3900108003af6 Mon Sep 17 00:00:00 2001
From: redclusive <nia4859@ya.ru>
Date: Wed, 14 Apr 2021 23:38:56 +0300
Subject: [PATCH 01/76] add quantileBfloat16

---
 .../AggregateFunctionQuantile.cpp             |   7 +
 .../AggregateFunctionQuantile.h               |   4 +
 src/AggregateFunctions/Bfloat16Histogram.h    | 171 ++++++++++++++++++
 .../QuantileBfloat16Histogram.h               |  65 +++++++
 4 files changed, 247 insertions(+)
 create mode 100644 src/AggregateFunctions/Bfloat16Histogram.h
 create mode 100644 src/AggregateFunctions/QuantileBfloat16Histogram.h

diff --git a/src/AggregateFunctions/AggregateFunctionQuantile.cpp b/src/AggregateFunctions/AggregateFunctionQuantile.cpp
index 1dcdb288339..5594beb90e3 100644
--- a/src/AggregateFunctions/AggregateFunctionQuantile.cpp
+++ b/src/AggregateFunctions/AggregateFunctionQuantile.cpp
@@ -52,6 +52,9 @@ template <typename Value, bool float_return> using FuncQuantilesTDigest = Aggreg
 template <typename Value, bool float_return> using FuncQuantileTDigestWeighted = AggregateFunctionQuantile<Value, QuantileTDigest<Value>, NameQuantileTDigestWeighted, true, std::conditional_t<float_return, Float32, void>, false>;
 template <typename Value, bool float_return> using FuncQuantilesTDigestWeighted = AggregateFunctionQuantile<Value, QuantileTDigest<Value>, NameQuantilesTDigestWeighted, true, std::conditional_t<float_return, Float32, void>, true>;
 
+template <typename Value, bool float_return> using FuncQuantileBfloat16 = AggregateFunctionQuantile<Value, QuantileBfloat16Histogram<Value>, NameQuantileBfloat16, false, std::conditional_t<float_return, Float64, void>, false>;
+template <typename Value, bool float_return> using FuncQuantilesBfloat16 = AggregateFunctionQuantile<Value, QuantileBfloat16Histogram<Value>, NameQuantilesBfloat16, false, std::conditional_t<float_return, Float64, void>, true>;
+
 
 template <template <typename, bool> class Function>
 static constexpr bool supportDecimal()
@@ -155,6 +158,9 @@ void registerAggregateFunctionsQuantile(AggregateFunctionFactory & factory)
     factory.registerFunction(NameQuantileTDigestWeighted::name, createAggregateFunctionQuantile<FuncQuantileTDigestWeighted>);
     factory.registerFunction(NameQuantilesTDigestWeighted::name, createAggregateFunctionQuantile<FuncQuantilesTDigestWeighted>);
 
+    factory.registerFunction(NameQuantileBfloat16::name, createAggregateFunctionQuantile<FuncQuantileBfloat16>);
+    factory.registerFunction(NameQuantilesBfloat16::name, createAggregateFunctionQuantile<FuncQuantilesBfloat16>);
+
     /// 'median' is an alias for 'quantile'
     factory.registerAlias("median", NameQuantile::name);
     factory.registerAlias("medianDeterministic", NameQuantileDeterministic::name);
@@ -166,6 +172,7 @@ void registerAggregateFunctionsQuantile(AggregateFunctionFactory & factory)
     factory.registerAlias("medianTimingWeighted", NameQuantileTimingWeighted::name);
     factory.registerAlias("medianTDigest", NameQuantileTDigest::name);
     factory.registerAlias("medianTDigestWeighted", NameQuantileTDigestWeighted::name);
+    factory.registerAlias("medianBfloat16", NameQuantileBfloat16::name);
 }
 
 }
diff --git a/src/AggregateFunctions/AggregateFunctionQuantile.h b/src/AggregateFunctions/AggregateFunctionQuantile.h
index 209784361dd..8e5a26fb267 100644
--- a/src/AggregateFunctions/AggregateFunctionQuantile.h
+++ b/src/AggregateFunctions/AggregateFunctionQuantile.h
@@ -9,6 +9,7 @@
 #include <AggregateFunctions/QuantileExactWeighted.h>
 #include <AggregateFunctions/QuantileTiming.h>
 #include <AggregateFunctions/QuantileTDigest.h>
+#include <AggregateFunctions/QuantileBfloat16Histogram.h>
 
 #include <AggregateFunctions/IAggregateFunction.h>
 #include <AggregateFunctions/QuantilesCommon.h>
@@ -228,4 +229,7 @@ struct NameQuantileTDigestWeighted { static constexpr auto name = "quantileTDige
 struct NameQuantilesTDigest { static constexpr auto name = "quantilesTDigest"; };
 struct NameQuantilesTDigestWeighted { static constexpr auto name = "quantilesTDigestWeighted"; };
 
+struct NameQuantileBfloat16 { static constexpr auto name = "quantileBfloat16"; };
+struct NameQuantilesBfloat16 { static constexpr auto name = "quantilesBfloat16"; };
+
 }
diff --git a/src/AggregateFunctions/Bfloat16Histogram.h b/src/AggregateFunctions/Bfloat16Histogram.h
new file mode 100644
index 00000000000..1b80c8b4343
--- /dev/null
+++ b/src/AggregateFunctions/Bfloat16Histogram.h
@@ -0,0 +1,171 @@
+#pragma once
+
+#include <IO/ReadBuffer.h>
+#include <IO/VarInt.h>
+#include <IO/WriteBuffer.h>
+#include <Common/HashTable/HashMap.h>
+#include <Common/PODArray.h>
+#include <common/types.h>
+#include <ext/bit_cast.h>
+
+namespace DB
+{
+namespace ErrorCodes
+{
+    extern const int NOT_IMPLEMENTED;
+}
+
+template <typename Value>
+struct Bfloat16Histogram
+{
+    using bfloat16 = UInt16;
+    using Data = HashMap<bfloat16, size_t>;
+    using Array = PODArrayWithStackMemory<Value, 64>;
+    Data data;
+    Array array;
+
+    void add(const Value & x, size_t to_add = 1)
+    {
+        if (isNaN(x))
+            return;
+
+        bfloat16 val = to_bfloat16(x);
+        if (!data.find(val))
+        {
+            sorted = false;
+            count += to_add;
+            array.push_back(x);
+        }
+        data[val] += to_add;
+    }
+
+    void merge(const Bfloat16Histogram & rhs)
+    {
+        for (const Value & value : rhs.array)
+        {
+            add(value, rhs.data.find(to_bfloat16(value))->getMapped());
+        }
+    }
+
+    void write(WriteBuffer & buf) const
+    {
+        writeVarUInt(count, buf);
+        data.write(buf);
+        size_t size = array.size();
+        writeVarUInt(size, buf);
+        buf.write(reinterpret_cast<const char *>(array.data()), size * sizeof(array[0]));
+    }
+
+    void read(ReadBuffer & buf)
+    {
+        count = 0;
+        readVarUInt(count, buf);
+        data.read(buf);
+        size_t size = 0;
+        readVarUInt(size, buf);
+        array.resize(size);
+        buf.read(reinterpret_cast<char *>(array.data()), size * sizeof(array[0]));
+    }
+
+    template <typename T>
+    T quantile(const Float64 & level)
+    {
+        if (array.empty())
+        {
+            return onEmpty<T>();
+        }
+        sortIfNeeded();
+
+        size_t sum = 0;
+        size_t need = level * count;
+        for (const Value & value : array)
+        {
+            sum += data.find(to_bfloat16(value))->getMapped();
+            if (sum >= need)
+                return value;
+        }
+
+        return array[array.size() - 1];
+    }
+
+    // levels[indicies[i]] must be sorted
+    template <typename T>
+    void quantilesManySorted(const Float64 * levels, const size_t * indices, size_t size, T * result)
+    {
+        if (array.empty())
+        {
+            for (size_t i = 0; i < size; ++i)
+            {
+                result[indices[i]] = onEmpty<T>();
+            }
+            return;
+        }
+        sortIfNeeded();
+
+        size_t sum = 0;
+        size_t it = 0;
+        for (const auto & value : array)
+        {
+            sum += data.find(to_bfloat16(value))->getMapped();
+            while (it < size && sum >= static_cast<size_t>(levels[indices[it]] * count))
+            {
+                result[indices[it++]] = value;
+            }
+        }
+
+        for (size_t i = it; i < size; ++i)
+        {
+            result[indices[i]] = array[array.size() - 1];
+        }
+    }
+
+    template <typename T>
+    void quantilesMany(const Float64 * levels, const size_t * indices, size_t size, T * result)
+    {
+        if (is_sorted_r(levels, indices, size))
+        {
+            quantilesManySorted(levels, indices, size, result);
+        }
+        else
+        {
+            for (size_t i = 0; i < size; ++i)
+                result[indices[i]] = quantile<T>(levels[indices[i]]);
+        }
+    }
+
+private:
+    size_t count = 0;
+    bool sorted = false;
+
+    bfloat16 to_bfloat16(const Value & x) const
+    { 
+        return ext::bit_cast<UInt32>(static_cast<Float32>(x)) >> 16; 
+    }
+
+    void sortIfNeeded()
+    {
+        if (sorted)
+            return;
+        sorted = true;
+        std::sort(array.begin(), array.end());
+    }
+
+    bool is_sorted_r(const Float64 * levels, const size_t * indices, size_t size) const
+    {
+        for (size_t i = 0; i < size - 1; ++i)
+        {
+            if (levels[indices[i]] > levels[indices[i + 1]])
+                return false;
+        }
+
+        return true;
+    }
+
+    template <typename ResultType>
+    ResultType onEmpty() const 
+    {
+        return std::numeric_limits<ResultType>::quiet_NaN();
+    }
+};
+
+}
\ No newline at end of file
diff --git a/src/AggregateFunctions/QuantileBfloat16Histogram.h b/src/AggregateFunctions/QuantileBfloat16Histogram.h
new file mode 100644
index 00000000000..a0827c2cc37
--- /dev/null
+++ b/src/AggregateFunctions/QuantileBfloat16Histogram.h
@@ -0,0 +1,65 @@
+#pragma once
+
+#include <AggregateFunctions/Bfloat16Histogram.h>
+
+namespace DB 
+{
+
+namespace ErrorCodes
+{
+    extern const int NOT_IMPLEMENTED;
+}
+
+template<typename Value>
+struct QuantileBfloat16Histogram 
+{
+    using Hist = Bfloat16Histogram<Value>;
+    Hist data;
+
+    void add(const Value & x)
+    {
+        data.add(x);
+    }
+
+    template <typename Weight>
+    void add(const Value &, const Weight &)
+    {
+        throw Exception("Method add with weight is not implemented for QuantileBfloat16Histogram", ErrorCodes::NOT_IMPLEMENTED);
+    }
+
+    void merge(const QuantileBfloat16Histogram & rhs)
+    {
+        data.merge(rhs.data);
+    }
+
+    void serialize(WriteBuffer & buf) const
+    {
+        data.write(buf);
+    }
+
+    void deserialize(ReadBuffer & buf)
+    {
+        data.read(buf);
+    }
+
+    Value get(Float64 level)
+    {
+        return data.template quantile<Value>(level);
+    }
+
+    void getMany(const Float64 * levels, const size_t * indices, size_t size, Value * result)
+    {
+        data.quantilesMany(levels, indices, size, result);
+    }
+
+    Float64 getFloat(Float64 level) {
+        return data.template quantile<Float64>(level);
+    }
+
+    void getManyFloat(const Float64 * levels, const size_t * indices, size_t size, Float64 * result)
+    {
+        data.quantilesMany(levels, indices, size, result);
+    }
+};
+
+}
\ No newline at end of file

From 8db89e493b4e965dfb646f9c4a9b1f1da81a66d5 Mon Sep 17 00:00:00 2001
From: redclusive <nia4859@ya.ru>
Date: Thu, 15 Apr 2021 00:06:22 +0300
Subject: [PATCH 02/76] fix styles

---
 src/AggregateFunctions/Bfloat16Histogram.h         | 2 +-
 src/AggregateFunctions/QuantileBfloat16Histogram.h | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/AggregateFunctions/Bfloat16Histogram.h b/src/AggregateFunctions/Bfloat16Histogram.h
index 1b80c8b4343..3a32a8aee46 100644
--- a/src/AggregateFunctions/Bfloat16Histogram.h
+++ b/src/AggregateFunctions/Bfloat16Histogram.h
@@ -168,4 +168,4 @@ private:
     }
 };
 
-}
\ No newline at end of file
+}
diff --git a/src/AggregateFunctions/QuantileBfloat16Histogram.h b/src/AggregateFunctions/QuantileBfloat16Histogram.h
index a0827c2cc37..23d985fa50a 100644
--- a/src/AggregateFunctions/QuantileBfloat16Histogram.h
+++ b/src/AggregateFunctions/QuantileBfloat16Histogram.h
@@ -62,4 +62,4 @@ struct QuantileBfloat16Histogram
     }
 };
 
-}
\ No newline at end of file
+}

From be547193ae455e1b750c1f83b516bce90ec4f3d6 Mon Sep 17 00:00:00 2001
From: redclusive <nia4859@ya.ru>
Date: Thu, 15 Apr 2021 17:30:41 +0300
Subject: [PATCH 03/76] fix add method

---
 src/AggregateFunctions/Bfloat16Histogram.h | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/src/AggregateFunctions/Bfloat16Histogram.h b/src/AggregateFunctions/Bfloat16Histogram.h
index 3a32a8aee46..8b056a1fbac 100644
--- a/src/AggregateFunctions/Bfloat16Histogram.h
+++ b/src/AggregateFunctions/Bfloat16Histogram.h
@@ -20,7 +20,7 @@ struct Bfloat16Histogram
 {
     using bfloat16 = UInt16;
     using Data = HashMap<bfloat16, size_t>;
-    using Array = PODArrayWithStackMemory<Value, 64>;
+    using Array = PODArrayWithStackMemory<Float32, 64>;
     Data data;
     Array array;
 
@@ -33,15 +33,15 @@ struct Bfloat16Histogram
         if (!data.find(val))
         {
             sorted = false;
-            count += to_add;
-            array.push_back(x);
+            array.push_back(to_Float32(val));
         }
+        count += to_add;
         data[val] += to_add;
     }
 
     void merge(const Bfloat16Histogram & rhs)
     {
-        for (const Value & value : rhs.array)
+        for (const Float32 & value : rhs.array)
         {
             add(value, rhs.data.find(to_bfloat16(value))->getMapped());
         }
@@ -78,9 +78,9 @@ struct Bfloat16Histogram
 
         size_t sum = 0;
         size_t need = level * count;
-        for (const Value & value : array)
+        for (const Float32 & value : array)
         {
-            sum += data.find(to_bfloat16(value))->getMapped();
+            sum += data[to_bfloat16(value)];
             if (sum >= need)
                 return value;
         }
@@ -104,9 +104,9 @@ struct Bfloat16Histogram
 
         size_t sum = 0;
         size_t it = 0;
-        for (const auto & value : array)
+        for (const Float32 & value : array)
         {
-            sum += data.find(to_bfloat16(value))->getMapped();
+            sum += data[to_bfloat16(value)];
             while (it < size && sum >= static_cast<size_t>(levels[indices[it]] * count))
             {
                 result[indices[it++]] = value;
@@ -142,6 +142,11 @@ private:
         return ext::bit_cast<UInt32>(static_cast<Float32>(x)) >> 16; 
     }
 
+    Float32 to_Float32(const bfloat & x) const
+    {
+        return ext::bit_cast<Float32>(x << 16);
+    }
+
     void sortIfNeeded()
     {
         if (sorted)

From d8af0c5a2306e3e2f129c29c2c23d428d3a25be0 Mon Sep 17 00:00:00 2001
From: redclusive <nia4859@ya.ru>
Date: Thu, 15 Apr 2021 17:45:11 +0300
Subject: [PATCH 04/76] fix typo

---
 src/AggregateFunctions/Bfloat16Histogram.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/AggregateFunctions/Bfloat16Histogram.h b/src/AggregateFunctions/Bfloat16Histogram.h
index 8b056a1fbac..d29d809618c 100644
--- a/src/AggregateFunctions/Bfloat16Histogram.h
+++ b/src/AggregateFunctions/Bfloat16Histogram.h
@@ -142,7 +142,7 @@ private:
         return ext::bit_cast<UInt32>(static_cast<Float32>(x)) >> 16; 
     }
 
-    Float32 to_Float32(const bfloat & x) const
+    Float32 to_Float32(const bfloat16 & x) const
     {
         return ext::bit_cast<Float32>(x << 16);
     }

From 8737c533c0c432c597f082882b38a35b4409b22b Mon Sep 17 00:00:00 2001
From: redclusive <nia4859@ya.ru>
Date: Thu, 15 Apr 2021 20:07:14 +0300
Subject: [PATCH 05/76] fix logic

---
 src/AggregateFunctions/Bfloat16Histogram.h         | 5 ++---
 src/AggregateFunctions/QuantileBfloat16Histogram.h | 4 ++--
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/AggregateFunctions/Bfloat16Histogram.h b/src/AggregateFunctions/Bfloat16Histogram.h
index d29d809618c..424ae6c1f4b 100644
--- a/src/AggregateFunctions/Bfloat16Histogram.h
+++ b/src/AggregateFunctions/Bfloat16Histogram.h
@@ -67,12 +67,11 @@ struct Bfloat16Histogram
         buf.read(reinterpret_cast<char *>(array.data()), size * sizeof(array[0]));
     }
 
-    template <typename T>
-    T quantile(const Float64 & level)
+    Float32 quantile(const Float64 & level)
     {
         if (array.empty())
         {
-            return onEmpty<T>();
+            return onEmpty<Value>();
         }
         sortIfNeeded();
 
diff --git a/src/AggregateFunctions/QuantileBfloat16Histogram.h b/src/AggregateFunctions/QuantileBfloat16Histogram.h
index 23d985fa50a..2b5bb9d9ddc 100644
--- a/src/AggregateFunctions/QuantileBfloat16Histogram.h
+++ b/src/AggregateFunctions/QuantileBfloat16Histogram.h
@@ -44,7 +44,7 @@ struct QuantileBfloat16Histogram
 
     Value get(Float64 level)
     {
-        return data.template quantile<Value>(level);
+        return data.quantile(level);
     }
 
     void getMany(const Float64 * levels, const size_t * indices, size_t size, Value * result)
@@ -53,7 +53,7 @@ struct QuantileBfloat16Histogram
     }
 
     Float64 getFloat(Float64 level) {
-        return data.template quantile<Float64>(level);
+        return data.quantile(level);
     }
 
     void getManyFloat(const Float64 * levels, const size_t * indices, size_t size, Float64 * result)

From f286692497e87146f7e5aa4c406925e8907b101c Mon Sep 17 00:00:00 2001
From: redclusive <nia4859@ya.ru>
Date: Thu, 15 Apr 2021 20:13:45 +0300
Subject: [PATCH 06/76] fix typo

---
 src/AggregateFunctions/Bfloat16Histogram.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/AggregateFunctions/Bfloat16Histogram.h b/src/AggregateFunctions/Bfloat16Histogram.h
index 424ae6c1f4b..eaf6e56ff23 100644
--- a/src/AggregateFunctions/Bfloat16Histogram.h
+++ b/src/AggregateFunctions/Bfloat16Histogram.h
@@ -128,7 +128,7 @@ struct Bfloat16Histogram
         else
         {
             for (size_t i = 0; i < size; ++i)
-                result[indices[i]] = quantile<T>(levels[indices[i]]);
+                result[indices[i]] = quantile(levels[indices[i]]);
         }
     }
 

From dbb0a4aab7e099616b3a0b118d41bd77d5f7c599 Mon Sep 17 00:00:00 2001
From: RedClusive <nia4859@ya.ru>
Date: Sat, 24 Apr 2021 19:11:56 +0000
Subject: [PATCH 07/76] fix style

---
 src/AggregateFunctions/Bfloat16Histogram.h    | 14 ++-----
 .../QuantileBfloat16Histogram.h               | 40 +++++--------------
 2 files changed, 15 insertions(+), 39 deletions(-)

diff --git a/src/AggregateFunctions/Bfloat16Histogram.h b/src/AggregateFunctions/Bfloat16Histogram.h
index eaf6e56ff23..653965c3658 100644
--- a/src/AggregateFunctions/Bfloat16Histogram.h
+++ b/src/AggregateFunctions/Bfloat16Histogram.h
@@ -87,7 +87,7 @@ struct Bfloat16Histogram
         return array[array.size() - 1];
     }
 
-    // levels[indicies[i]] must be sorted
+    // levels[indices[i]] must be sorted
     template <typename T>
     void quantilesManySorted(const Float64 * levels, const size_t * indices, size_t size, T * result)
     {
@@ -136,15 +136,9 @@ private:
     size_t count = 0;
     bool sorted = false;
 
-    bfloat16 to_bfloat16(const Value & x) const
-    { 
-        return ext::bit_cast<UInt32>(static_cast<Float32>(x)) >> 16; 
-    }
+    bfloat16 to_bfloat16(const Value & x) const { return ext::bit_cast<UInt32>(static_cast<Float32>(x)) >> 16; }
 
-    Float32 to_Float32(const bfloat16 & x) const
-    {
-        return ext::bit_cast<Float32>(x << 16);
-    }
+    Float32 to_Float32(const bfloat16 & x) const { return ext::bit_cast<Float32>(x << 16); }
 
     void sortIfNeeded()
     {
@@ -166,7 +160,7 @@ private:
     }
 
     template <typename ResultType>
-    ResultType onEmpty() const 
+    ResultType onEmpty() const
     {
         return std::numeric_limits<ResultType>::quiet_NaN();
     }
diff --git a/src/AggregateFunctions/QuantileBfloat16Histogram.h b/src/AggregateFunctions/QuantileBfloat16Histogram.h
index 2b5bb9d9ddc..cae734efa8b 100644
--- a/src/AggregateFunctions/QuantileBfloat16Histogram.h
+++ b/src/AggregateFunctions/QuantileBfloat16Histogram.h
@@ -2,24 +2,20 @@
 
 #include <AggregateFunctions/Bfloat16Histogram.h>
 
-namespace DB 
+namespace DB
 {
-
 namespace ErrorCodes
 {
     extern const int NOT_IMPLEMENTED;
 }
 
-template<typename Value>
-struct QuantileBfloat16Histogram 
+template <typename Value>
+struct QuantileBfloat16Histogram
 {
-    using Hist = Bfloat16Histogram<Value>;
-    Hist data;
+    using Histogram = Bfloat16Histogram<Value>;
+    Histogram data;
 
-    void add(const Value & x)
-    {
-        data.add(x);
-    }
+    void add(const Value & x) { data.add(x); }
 
     template <typename Weight>
     void add(const Value &, const Weight &)
@@ -27,34 +23,20 @@ struct QuantileBfloat16Histogram
         throw Exception("Method add with weight is not implemented for QuantileBfloat16Histogram", ErrorCodes::NOT_IMPLEMENTED);
     }
 
-    void merge(const QuantileBfloat16Histogram & rhs)
-    {
-        data.merge(rhs.data);
-    }
+    void merge(const QuantileBfloat16Histogram & rhs) { data.merge(rhs.data); }
 
-    void serialize(WriteBuffer & buf) const
-    {
-        data.write(buf);
-    }
+    void serialize(WriteBuffer & buf) const { data.write(buf); }
 
-    void deserialize(ReadBuffer & buf)
-    {
-        data.read(buf);
-    }
+    void deserialize(ReadBuffer & buf) { data.read(buf); }
 
-    Value get(Float64 level)
-    {
-        return data.quantile(level);
-    }
+    Value get(Float64 level) { return data.quantile(level); }
 
     void getMany(const Float64 * levels, const size_t * indices, size_t size, Value * result)
     {
         data.quantilesMany(levels, indices, size, result);
     }
 
-    Float64 getFloat(Float64 level) {
-        return data.quantile(level);
-    }
+    Float64 getFloat(Float64 level) { return data.quantile(level); }
 
     void getManyFloat(const Float64 * levels, const size_t * indices, size_t size, Float64 * result)
     {

From 907115a4c47d8804bcb7467867e9a75d2d6ad251 Mon Sep 17 00:00:00 2001
From: RedClusive <nia4859@ya.ru>
Date: Sun, 25 Apr 2021 21:49:02 +0000
Subject: [PATCH 08/76] add tests

---
 .../00753_quantile_format.reference           | 10 +++--
 .../0_stateless/00753_quantile_format.sql     |  3 ++
 .../01813_quantileBfloat16_nans.reference     |  1 +
 .../01813_quantileBfloat16_nans.sql           | 16 ++++++++
 .../00164_quantileBfloat16.reference          | 40 +++++++++++++++++++
 .../1_stateful/00164_quantileBfloat16.sql     |  6 +++
 6 files changed, 72 insertions(+), 4 deletions(-)
 create mode 100644 tests/queries/0_stateless/01813_quantileBfloat16_nans.reference
 create mode 100644 tests/queries/0_stateless/01813_quantileBfloat16_nans.sql
 create mode 100644 tests/queries/1_stateful/00164_quantileBfloat16.reference
 create mode 100644 tests/queries/1_stateful/00164_quantileBfloat16.sql

diff --git a/tests/queries/0_stateless/00753_quantile_format.reference b/tests/queries/0_stateless/00753_quantile_format.reference
index 04bb0000b3e..d3503745e4b 100644
--- a/tests/queries/0_stateless/00753_quantile_format.reference
+++ b/tests/queries/0_stateless/00753_quantile_format.reference
@@ -10,7 +10,9 @@
 [30000]
 30000
 [30000]
-2016-06-15 23:01:04
-['2016-06-15 23:01:04']
-2016-06-15 23:01:04
-['2016-06-15 23:01:04']
+2016-06-15 23:00:16
+['2016-06-15 23:00:16']
+2016-06-15 23:00:16
+['2016-06-15 23:00:16']
+2016-04-02 17:23:12
+['2016-04-02 17:23:12']
diff --git a/tests/queries/0_stateless/00753_quantile_format.sql b/tests/queries/0_stateless/00753_quantile_format.sql
index 24cf0e4bf90..c36028627e0 100644
--- a/tests/queries/0_stateless/00753_quantile_format.sql
+++ b/tests/queries/0_stateless/00753_quantile_format.sql
@@ -27,4 +27,7 @@ SELECT quantilesTDigest(0.2)(d) FROM datetime;
 SELECT quantileTDigestWeighted(0.2)(d, 1) FROM datetime;
 SELECT quantilesTDigestWeighted(0.2)(d, 1) FROM datetime;
 
+SELECT quantileBfloat16(0.2)(d) FROM datetime;
+SELECT quantilesBfloat16(0.2)(d) FROM datetime;
+
 DROP TABLE datetime;
diff --git a/tests/queries/0_stateless/01813_quantileBfloat16_nans.reference b/tests/queries/0_stateless/01813_quantileBfloat16_nans.reference
new file mode 100644
index 00000000000..d00491fd7e5
--- /dev/null
+++ b/tests/queries/0_stateless/01813_quantileBfloat16_nans.reference
@@ -0,0 +1 @@
+1
diff --git a/tests/queries/0_stateless/01813_quantileBfloat16_nans.sql b/tests/queries/0_stateless/01813_quantileBfloat16_nans.sql
new file mode 100644
index 00000000000..19a8f8da8d6
--- /dev/null
+++ b/tests/queries/0_stateless/01813_quantileBfloat16_nans.sql
@@ -0,0 +1,16 @@
+SELECT DISTINCT
+    eq
+FROM
+    (
+        WITH
+            range(2 + number % 10) AS arr, -- minimum two elements, to avoid nan result --
+            arrayMap(x -> x = intDiv(number, 10) ? nan : x, arr) AS arr_with_nan,
+            arrayFilter(x -> x != intDiv(number, 10), arr) AS arr_filtered
+        SELECT
+            number,
+            arrayReduce('quantileBfloat16', arr_with_nan) AS q1,
+            arrayReduce('quantileBfloat16', arr_filtered) AS q2,
+            q1 = q2 AS eq
+        FROM
+            numbers(100)
+    );
diff --git a/tests/queries/1_stateful/00164_quantileBfloat16.reference b/tests/queries/1_stateful/00164_quantileBfloat16.reference
new file mode 100644
index 00000000000..f0aaff5cef6
--- /dev/null
+++ b/tests/queries/1_stateful/00164_quantileBfloat16.reference
@@ -0,0 +1,40 @@
+1704509	1384
+732797	1336
+598875	1384
+792887	1336
+3807842	1336
+25703952	1336
+716829	1384
+59183	1336
+33010362	1336
+800784	1336
+1704509	[1296,1384,1840,1960,3696]
+732797	[1232,1336,1840,1944,3664]
+598875	[1232,1384,1840,1944,3536]
+792887	[1296,1336,1840,1888,3696]
+3807842	[1232,1336,1840,1936,2032]
+25703952	[1012,1336,1840,1944,3696]
+716829	[1232,1384,1840,1944,3696]
+59183	[316,1336,1840,2008,2032]
+33010362	[1232,1336,1840,1936,2032]
+800784	[1232,1336,1840,1928,2032]
+1704509	1384
+732797	1336
+598875	1384
+792887	1336
+3807842	1336
+25703952	1336
+716829	1384
+59183	1336
+33010362	1336
+800784	1336
+1704509	[1296,1384,1840,1960,3696]
+732797	[1232,1336,1840,1944,3664]
+598875	[1232,1384,1840,1944,3536]
+792887	[1296,1336,1840,1888,3696]
+3807842	[1232,1336,1840,1936,2032]
+25703952	[1012,1336,1840,1944,3696]
+716829	[1232,1384,1840,1944,3696]
+59183	[316,1336,1840,2008,2032]
+33010362	[1232,1336,1840,1936,2032]
+800784	[1232,1336,1840,1928,2032]
diff --git a/tests/queries/1_stateful/00164_quantileBfloat16.sql b/tests/queries/1_stateful/00164_quantileBfloat16.sql
new file mode 100644
index 00000000000..59715f5d6a0
--- /dev/null
+++ b/tests/queries/1_stateful/00164_quantileBfloat16.sql
@@ -0,0 +1,6 @@
+SELECT CounterID AS k, quantileBfloat16(0.5)(ResolutionWidth) FROM test.hits GROUP BY k ORDER BY count() DESC, CounterID LIMIT 10;
+SELECT CounterID AS k, quantilesBfloat16(0.1, 0.5, 0.9, 0.99, 0.999)(ResolutionWidth) FROM test.hits GROUP BY k ORDER BY count() DESC, CounterID LIMIT 10;
+
+
+SELECT CounterID AS k, quantileBfloat16(0.5)(ResolutionWidth) FROM remote('127.0.0.{1,2}', test.hits) GROUP BY k ORDER BY count() DESC, CounterID LIMIT 10;
+SELECT CounterID AS k, quantilesBfloat16(0.1, 0.5, 0.9, 0.99, 0.999)(ResolutionWidth) FROM remote('127.0.0.{1,2}', test.hits) GROUP BY k ORDER BY count() DESC, CounterID LIMIT 10;

From 8ed90864cbbc05bd393cdbfed06b408e3a270395 Mon Sep 17 00:00:00 2001
From: RedClusive <nia4859@ya.ru>
Date: Mon, 26 Apr 2021 09:39:08 +0000
Subject: [PATCH 09/76] refactored

---
 src/AggregateFunctions/AggregateFunctionQuantile.cpp   | 10 +++++-----
 src/AggregateFunctions/AggregateFunctionQuantile.h     |  6 +++---
 .../{Bfloat16Histogram.h => BFloat16Histogram.h}       |  4 ++--
 ...Bfloat16Histogram.h => QuantileBFloat16Histogram.h} | 10 +++++-----
 tests/queries/0_stateless/00753_quantile_format.sql    |  4 ++--
 .../0_stateless/01813_quantileBfloat16_nans.sql        |  4 ++--
 tests/queries/1_stateful/00164_quantileBfloat16.sql    |  8 ++++----
 7 files changed, 23 insertions(+), 23 deletions(-)
 rename src/AggregateFunctions/{Bfloat16Histogram.h => BFloat16Histogram.h} (98%)
 rename src/AggregateFunctions/{QuantileBfloat16Histogram.h => QuantileBFloat16Histogram.h} (78%)

diff --git a/src/AggregateFunctions/AggregateFunctionQuantile.cpp b/src/AggregateFunctions/AggregateFunctionQuantile.cpp
index 5594beb90e3..90563e51ddf 100644
--- a/src/AggregateFunctions/AggregateFunctionQuantile.cpp
+++ b/src/AggregateFunctions/AggregateFunctionQuantile.cpp
@@ -52,8 +52,8 @@ template <typename Value, bool float_return> using FuncQuantilesTDigest = Aggreg
 template <typename Value, bool float_return> using FuncQuantileTDigestWeighted = AggregateFunctionQuantile<Value, QuantileTDigest<Value>, NameQuantileTDigestWeighted, true, std::conditional_t<float_return, Float32, void>, false>;
 template <typename Value, bool float_return> using FuncQuantilesTDigestWeighted = AggregateFunctionQuantile<Value, QuantileTDigest<Value>, NameQuantilesTDigestWeighted, true, std::conditional_t<float_return, Float32, void>, true>;
 
-template <typename Value, bool float_return> using FuncQuantileBfloat16 = AggregateFunctionQuantile<Value, QuantileBfloat16Histogram<Value>, NameQuantileBfloat16, false, std::conditional_t<float_return, Float64, void>, false>;
-template <typename Value, bool float_return> using FuncQuantilesBfloat16 = AggregateFunctionQuantile<Value, QuantileBfloat16Histogram<Value>, NameQuantilesBfloat16, false, std::conditional_t<float_return, Float64, void>, true>;
+template <typename Value, bool float_return> using FuncQuantileBFloat16 = AggregateFunctionQuantile<Value, QuantileBFloat16Histogram<Value>, NameQuantileBFloat16, false, std::conditional_t<float_return, Float64, void>, false>;
+template <typename Value, bool float_return> using FuncQuantilesBFloat16 = AggregateFunctionQuantile<Value, QuantileBFloat16Histogram<Value>, NameQuantilesBFloat16, false, std::conditional_t<float_return, Float64, void>, true>;
 
 
 template <template <typename, bool> class Function>
@@ -158,8 +158,8 @@ void registerAggregateFunctionsQuantile(AggregateFunctionFactory & factory)
     factory.registerFunction(NameQuantileTDigestWeighted::name, createAggregateFunctionQuantile<FuncQuantileTDigestWeighted>);
     factory.registerFunction(NameQuantilesTDigestWeighted::name, createAggregateFunctionQuantile<FuncQuantilesTDigestWeighted>);
 
-    factory.registerFunction(NameQuantileBfloat16::name, createAggregateFunctionQuantile<FuncQuantileBfloat16>);
-    factory.registerFunction(NameQuantilesBfloat16::name, createAggregateFunctionQuantile<FuncQuantilesBfloat16>);
+    factory.registerFunction(NameQuantileBFloat16::name, createAggregateFunctionQuantile<FuncQuantileBFloat16>);
+    factory.registerFunction(NameQuantilesBFloat16::name, createAggregateFunctionQuantile<FuncQuantilesBFloat16>);
 
     /// 'median' is an alias for 'quantile'
     factory.registerAlias("median", NameQuantile::name);
@@ -172,7 +172,7 @@ void registerAggregateFunctionsQuantile(AggregateFunctionFactory & factory)
     factory.registerAlias("medianTimingWeighted", NameQuantileTimingWeighted::name);
     factory.registerAlias("medianTDigest", NameQuantileTDigest::name);
     factory.registerAlias("medianTDigestWeighted", NameQuantileTDigestWeighted::name);
-    factory.registerAlias("medianBfloat16", NameQuantileBfloat16::name);
+    factory.registerAlias("medianBFloat16", NameQuantileBFloat16::name);
 }
 
 }
diff --git a/src/AggregateFunctions/AggregateFunctionQuantile.h b/src/AggregateFunctions/AggregateFunctionQuantile.h
index 8e5a26fb267..502905e8b48 100644
--- a/src/AggregateFunctions/AggregateFunctionQuantile.h
+++ b/src/AggregateFunctions/AggregateFunctionQuantile.h
@@ -9,7 +9,7 @@
 #include <AggregateFunctions/QuantileExactWeighted.h>
 #include <AggregateFunctions/QuantileTiming.h>
 #include <AggregateFunctions/QuantileTDigest.h>
-#include <AggregateFunctions/QuantileBfloat16Histogram.h>
+#include <AggregateFunctions/QuantileBFloat16Histogram.h>
 
 #include <AggregateFunctions/IAggregateFunction.h>
 #include <AggregateFunctions/QuantilesCommon.h>
@@ -229,7 +229,7 @@ struct NameQuantileTDigestWeighted { static constexpr auto name = "quantileTDige
 struct NameQuantilesTDigest { static constexpr auto name = "quantilesTDigest"; };
 struct NameQuantilesTDigestWeighted { static constexpr auto name = "quantilesTDigestWeighted"; };
 
-struct NameQuantileBfloat16 { static constexpr auto name = "quantileBfloat16"; };
-struct NameQuantilesBfloat16 { static constexpr auto name = "quantilesBfloat16"; };
+struct NameQuantileBFloat16 { static constexpr auto name = "quantileBFloat16"; };
+struct NameQuantilesBFloat16 { static constexpr auto name = "quantilesBFloat16"; };
 
 }
diff --git a/src/AggregateFunctions/Bfloat16Histogram.h b/src/AggregateFunctions/BFloat16Histogram.h
similarity index 98%
rename from src/AggregateFunctions/Bfloat16Histogram.h
rename to src/AggregateFunctions/BFloat16Histogram.h
index 653965c3658..f07a6d4e019 100644
--- a/src/AggregateFunctions/Bfloat16Histogram.h
+++ b/src/AggregateFunctions/BFloat16Histogram.h
@@ -16,7 +16,7 @@ namespace ErrorCodes
 }
 
 template <typename Value>
-struct Bfloat16Histogram
+struct BFloat16Histogram
 {
     using bfloat16 = UInt16;
     using Data = HashMap<bfloat16, size_t>;
@@ -39,7 +39,7 @@ struct Bfloat16Histogram
         data[val] += to_add;
     }
 
-    void merge(const Bfloat16Histogram & rhs)
+    void merge(const BFloat16Histogram & rhs)
     {
         for (const Float32 & value : rhs.array)
         {
diff --git a/src/AggregateFunctions/QuantileBfloat16Histogram.h b/src/AggregateFunctions/QuantileBFloat16Histogram.h
similarity index 78%
rename from src/AggregateFunctions/QuantileBfloat16Histogram.h
rename to src/AggregateFunctions/QuantileBFloat16Histogram.h
index cae734efa8b..87be0d4679d 100644
--- a/src/AggregateFunctions/QuantileBfloat16Histogram.h
+++ b/src/AggregateFunctions/QuantileBFloat16Histogram.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#include <AggregateFunctions/Bfloat16Histogram.h>
+#include <AggregateFunctions/BFloat16Histogram.h>
 
 namespace DB
 {
@@ -10,9 +10,9 @@ namespace ErrorCodes
 }
 
 template <typename Value>
-struct QuantileBfloat16Histogram
+struct QuantileBFloat16Histogram
 {
-    using Histogram = Bfloat16Histogram<Value>;
+    using Histogram = BFloat16Histogram<Value>;
     Histogram data;
 
     void add(const Value & x) { data.add(x); }
@@ -20,10 +20,10 @@ struct QuantileBfloat16Histogram
     template <typename Weight>
     void add(const Value &, const Weight &)
     {
-        throw Exception("Method add with weight is not implemented for QuantileBfloat16Histogram", ErrorCodes::NOT_IMPLEMENTED);
+        throw Exception("Method add with weight is not implemented for QuantileBFloat16Histogram", ErrorCodes::NOT_IMPLEMENTED);
     }
 
-    void merge(const QuantileBfloat16Histogram & rhs) { data.merge(rhs.data); }
+    void merge(const QuantileBFloat16Histogram & rhs) { data.merge(rhs.data); }
 
     void serialize(WriteBuffer & buf) const { data.write(buf); }
 
diff --git a/tests/queries/0_stateless/00753_quantile_format.sql b/tests/queries/0_stateless/00753_quantile_format.sql
index c36028627e0..017aea33aba 100644
--- a/tests/queries/0_stateless/00753_quantile_format.sql
+++ b/tests/queries/0_stateless/00753_quantile_format.sql
@@ -27,7 +27,7 @@ SELECT quantilesTDigest(0.2)(d) FROM datetime;
 SELECT quantileTDigestWeighted(0.2)(d, 1) FROM datetime;
 SELECT quantilesTDigestWeighted(0.2)(d, 1) FROM datetime;
 
-SELECT quantileBfloat16(0.2)(d) FROM datetime;
-SELECT quantilesBfloat16(0.2)(d) FROM datetime;
+SELECT quantileBFloat16(0.2)(d) FROM datetime;
+SELECT quantilesBFloat16(0.2)(d) FROM datetime;
 
 DROP TABLE datetime;
diff --git a/tests/queries/0_stateless/01813_quantileBfloat16_nans.sql b/tests/queries/0_stateless/01813_quantileBfloat16_nans.sql
index 19a8f8da8d6..1f823f0b990 100644
--- a/tests/queries/0_stateless/01813_quantileBfloat16_nans.sql
+++ b/tests/queries/0_stateless/01813_quantileBfloat16_nans.sql
@@ -8,8 +8,8 @@ FROM
             arrayFilter(x -> x != intDiv(number, 10), arr) AS arr_filtered
         SELECT
             number,
-            arrayReduce('quantileBfloat16', arr_with_nan) AS q1,
-            arrayReduce('quantileBfloat16', arr_filtered) AS q2,
+            arrayReduce('quantileBFloat16', arr_with_nan) AS q1,
+            arrayReduce('quantileBFloat16', arr_filtered) AS q2,
             q1 = q2 AS eq
         FROM
             numbers(100)
diff --git a/tests/queries/1_stateful/00164_quantileBfloat16.sql b/tests/queries/1_stateful/00164_quantileBfloat16.sql
index 59715f5d6a0..b39bae3803c 100644
--- a/tests/queries/1_stateful/00164_quantileBfloat16.sql
+++ b/tests/queries/1_stateful/00164_quantileBfloat16.sql
@@ -1,6 +1,6 @@
-SELECT CounterID AS k, quantileBfloat16(0.5)(ResolutionWidth) FROM test.hits GROUP BY k ORDER BY count() DESC, CounterID LIMIT 10;
-SELECT CounterID AS k, quantilesBfloat16(0.1, 0.5, 0.9, 0.99, 0.999)(ResolutionWidth) FROM test.hits GROUP BY k ORDER BY count() DESC, CounterID LIMIT 10;
+SELECT CounterID AS k, quantileBFloat16(0.5)(ResolutionWidth) FROM test.hits GROUP BY k ORDER BY count() DESC, CounterID LIMIT 10;
+SELECT CounterID AS k, quantilesBFloat16(0.1, 0.5, 0.9, 0.99, 0.999)(ResolutionWidth) FROM test.hits GROUP BY k ORDER BY count() DESC, CounterID LIMIT 10;
 
 
-SELECT CounterID AS k, quantileBfloat16(0.5)(ResolutionWidth) FROM remote('127.0.0.{1,2}', test.hits) GROUP BY k ORDER BY count() DESC, CounterID LIMIT 10;
-SELECT CounterID AS k, quantilesBfloat16(0.1, 0.5, 0.9, 0.99, 0.999)(ResolutionWidth) FROM remote('127.0.0.{1,2}', test.hits) GROUP BY k ORDER BY count() DESC, CounterID LIMIT 10;
+SELECT CounterID AS k, quantileBFloat16(0.5)(ResolutionWidth) FROM remote('127.0.0.{1,2}', test.hits) GROUP BY k ORDER BY count() DESC, CounterID LIMIT 10;
+SELECT CounterID AS k, quantilesBFloat16(0.1, 0.5, 0.9, 0.99, 0.999)(ResolutionWidth) FROM remote('127.0.0.{1,2}', test.hits) GROUP BY k ORDER BY count() DESC, CounterID LIMIT 10;

From 6766dddbf4c724069fb80fbf249cbb47178ec6cb Mon Sep 17 00:00:00 2001
From: RedClusive <nia4859@ya.ru>
Date: Tue, 27 Apr 2021 07:40:29 +0000
Subject: [PATCH 10/76] fix logic

---
 src/AggregateFunctions/BFloat16Histogram.h         | 7 ++++---
 src/AggregateFunctions/QuantileBFloat16Histogram.h | 4 ++--
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/AggregateFunctions/BFloat16Histogram.h b/src/AggregateFunctions/BFloat16Histogram.h
index f07a6d4e019..c65aa10cb50 100644
--- a/src/AggregateFunctions/BFloat16Histogram.h
+++ b/src/AggregateFunctions/BFloat16Histogram.h
@@ -67,11 +67,12 @@ struct BFloat16Histogram
         buf.read(reinterpret_cast<char *>(array.data()), size * sizeof(array[0]));
     }
 
-    Float32 quantile(const Float64 & level)
+    template <typename ResultType>
+    ResultType quantile(const Float64 & level)
     {
         if (array.empty())
         {
-            return onEmpty<Value>();
+            return onEmpty<ResultType>();
         }
         sortIfNeeded();
 
@@ -128,7 +129,7 @@ struct BFloat16Histogram
         else
         {
             for (size_t i = 0; i < size; ++i)
-                result[indices[i]] = quantile(levels[indices[i]]);
+                result[indices[i]] = quantile<T>(levels[indices[i]]);
         }
     }
 
diff --git a/src/AggregateFunctions/QuantileBFloat16Histogram.h b/src/AggregateFunctions/QuantileBFloat16Histogram.h
index 87be0d4679d..4a1dd1c8bb3 100644
--- a/src/AggregateFunctions/QuantileBFloat16Histogram.h
+++ b/src/AggregateFunctions/QuantileBFloat16Histogram.h
@@ -29,14 +29,14 @@ struct QuantileBFloat16Histogram
 
     void deserialize(ReadBuffer & buf) { data.read(buf); }
 
-    Value get(Float64 level) { return data.quantile(level); }
+    Value get(Float64 level) { return data.template quantile<Value>(level); }
 
     void getMany(const Float64 * levels, const size_t * indices, size_t size, Value * result)
     {
         data.quantilesMany(levels, indices, size, result);
     }
 
-    Float64 getFloat(Float64 level) { return data.quantile(level); }
+    Float64 getFloat(Float64 level) { return data.template quantile<Float64>(level); }
 
     void getManyFloat(const Float64 * levels, const size_t * indices, size_t size, Float64 * result)
     {

From 508efc83214ba8212c905e8a9e1c7ce9c4c04cfd Mon Sep 17 00:00:00 2001
From: RedClusive <nia4859@ya.ru>
Date: Tue, 27 Apr 2021 08:44:48 +0000
Subject: [PATCH 11/76] fix test

---
 tests/queries/0_stateless/00753_quantile_format.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/00753_quantile_format.sql b/tests/queries/0_stateless/00753_quantile_format.sql
index 017aea33aba..e737498c114 100644
--- a/tests/queries/0_stateless/00753_quantile_format.sql
+++ b/tests/queries/0_stateless/00753_quantile_format.sql
@@ -1,6 +1,6 @@
 DROP TABLE IF EXISTS datetime;
 
-CREATE TABLE datetime (d DateTime) ENGINE = Memory;
+CREATE TABLE datetime (d DateTime('UTC')) ENGINE = Memory;
 INSERT INTO datetime(d) VALUES(toDateTime('2016-06-15 23:00:00'));
 
 SELECT quantile(0.2)(d) FROM datetime;

From 1a8871be36772214c3c07dfc77387d29865083e5 Mon Sep 17 00:00:00 2001
From: RedClusive <nia4859@ya.ru>
Date: Tue, 27 Apr 2021 09:49:24 +0000
Subject: [PATCH 12/76] fix test

---
 tests/queries/0_stateless/00753_quantile_format.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/00753_quantile_format.sql b/tests/queries/0_stateless/00753_quantile_format.sql
index e737498c114..f72ecc8a49f 100644
--- a/tests/queries/0_stateless/00753_quantile_format.sql
+++ b/tests/queries/0_stateless/00753_quantile_format.sql
@@ -1,7 +1,7 @@
 DROP TABLE IF EXISTS datetime;
 
 CREATE TABLE datetime (d DateTime('UTC')) ENGINE = Memory;
-INSERT INTO datetime(d) VALUES(toDateTime('2016-06-15 23:00:00'));
+INSERT INTO datetime(d) VALUES(toDateTime('2016-06-15 23:00:00', 'UTC'));
 
 SELECT quantile(0.2)(d) FROM datetime;
 SELECT quantiles(0.2)(d) FROM datetime;

From 6ee46fe936c57741e228ebb469f038db577715e7 Mon Sep 17 00:00:00 2001
From: RedClusive <nia4859@ya.ru>
Date: Tue, 27 Apr 2021 10:46:16 +0000
Subject: [PATCH 13/76] fix style

---
 src/AggregateFunctions/BFloat16Histogram.h | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/src/AggregateFunctions/BFloat16Histogram.h b/src/AggregateFunctions/BFloat16Histogram.h
index c65aa10cb50..d5c5d27eb95 100644
--- a/src/AggregateFunctions/BFloat16Histogram.h
+++ b/src/AggregateFunctions/BFloat16Histogram.h
@@ -10,10 +10,6 @@
 
 namespace DB
 {
-namespace ErrorCodes
-{
-    extern const int NOT_IMPLEMENTED;
-}
 
 template <typename Value>
 struct BFloat16Histogram

From de2523cabd77e9a2f1811d0121c42306752f389b Mon Sep 17 00:00:00 2001
From: RedClusive <nia4859@ya.ru>
Date: Wed, 28 Apr 2021 14:54:10 +0000
Subject: [PATCH 14/76] add weight and change logic

---
 src/AggregateFunctions/BFloat16Histogram.h    | 166 ------------------
 .../QuantileBFloat16Histogram.h               | 142 +++++++++++++--
 2 files changed, 123 insertions(+), 185 deletions(-)
 delete mode 100644 src/AggregateFunctions/BFloat16Histogram.h

diff --git a/src/AggregateFunctions/BFloat16Histogram.h b/src/AggregateFunctions/BFloat16Histogram.h
deleted file mode 100644
index d5c5d27eb95..00000000000
--- a/src/AggregateFunctions/BFloat16Histogram.h
+++ /dev/null
@@ -1,166 +0,0 @@
-#pragma once
-
-#include <IO/ReadBuffer.h>
-#include <IO/VarInt.h>
-#include <IO/WriteBuffer.h>
-#include <Common/HashTable/HashMap.h>
-#include <Common/PODArray.h>
-#include <common/types.h>
-#include <ext/bit_cast.h>
-
-namespace DB
-{
-
-template <typename Value>
-struct BFloat16Histogram
-{
-    using bfloat16 = UInt16;
-    using Data = HashMap<bfloat16, size_t>;
-    using Array = PODArrayWithStackMemory<Float32, 64>;
-    Data data;
-    Array array;
-
-    void add(const Value & x, size_t to_add = 1)
-    {
-        if (isNaN(x))
-            return;
-
-        bfloat16 val = to_bfloat16(x);
-        if (!data.find(val))
-        {
-            sorted = false;
-            array.push_back(to_Float32(val));
-        }
-        count += to_add;
-        data[val] += to_add;
-    }
-
-    void merge(const BFloat16Histogram & rhs)
-    {
-        for (const Float32 & value : rhs.array)
-        {
-            add(value, rhs.data.find(to_bfloat16(value))->getMapped());
-        }
-    }
-
-    void write(WriteBuffer & buf) const
-    {
-        writeVarUInt(count, buf);
-        data.write(buf);
-        size_t size = array.size();
-        writeVarUInt(size, buf);
-        buf.write(reinterpret_cast<const char *>(array.data()), size * sizeof(array[0]));
-    }
-
-    void read(ReadBuffer & buf)
-    {
-        count = 0;
-        readVarUInt(count, buf);
-        data.read(buf);
-        size_t size = 0;
-        readVarUInt(size, buf);
-        array.resize(size);
-        buf.read(reinterpret_cast<char *>(array.data()), size * sizeof(array[0]));
-    }
-
-    template <typename ResultType>
-    ResultType quantile(const Float64 & level)
-    {
-        if (array.empty())
-        {
-            return onEmpty<ResultType>();
-        }
-        sortIfNeeded();
-
-        size_t sum = 0;
-        size_t need = level * count;
-        for (const Float32 & value : array)
-        {
-            sum += data[to_bfloat16(value)];
-            if (sum >= need)
-                return value;
-        }
-
-        return array[array.size() - 1];
-    }
-
-    // levels[indices[i]] must be sorted
-    template <typename T>
-    void quantilesManySorted(const Float64 * levels, const size_t * indices, size_t size, T * result)
-    {
-        if (array.empty())
-        {
-            for (size_t i = 0; i < size; ++i)
-            {
-                result[indices[i]] = onEmpty<T>();
-            }
-            return;
-        }
-        sortIfNeeded();
-
-        size_t sum = 0;
-        size_t it = 0;
-        for (const Float32 & value : array)
-        {
-            sum += data[to_bfloat16(value)];
-            while (it < size && sum >= static_cast<size_t>(levels[indices[it]] * count))
-            {
-                result[indices[it++]] = value;
-            }
-        }
-
-        for (size_t i = it; i < size; ++i)
-        {
-            result[indices[i]] = array[array.size() - 1];
-        }
-    }
-
-    template <typename T>
-    void quantilesMany(const Float64 * levels, const size_t * indices, size_t size, T * result)
-    {
-        if (is_sorted_r(levels, indices, size))
-        {
-            quantilesManySorted(levels, indices, size, result);
-        }
-        else
-        {
-            for (size_t i = 0; i < size; ++i)
-                result[indices[i]] = quantile<T>(levels[indices[i]]);
-        }
-    }
-
-private:
-    size_t count = 0;
-    bool sorted = false;
-
-    bfloat16 to_bfloat16(const Value & x) const { return ext::bit_cast<UInt32>(static_cast<Float32>(x)) >> 16; }
-
-    Float32 to_Float32(const bfloat16 & x) const { return ext::bit_cast<Float32>(x << 16); }
-
-    void sortIfNeeded()
-    {
-        if (sorted)
-            return;
-        sorted = true;
-        std::sort(array.begin(), array.end());
-    }
-
-    bool is_sorted_r(const Float64 * levels, const size_t * indices, size_t size) const
-    {
-        for (size_t i = 0; i < size - 1; ++i)
-        {
-            if (levels[indices[i]] > levels[indices[i + 1]])
-                return false;
-        }
-
-        return true;
-    }
-
-    template <typename ResultType>
-    ResultType onEmpty() const
-    {
-        return std::numeric_limits<ResultType>::quiet_NaN();
-    }
-};
-
-}
diff --git a/src/AggregateFunctions/QuantileBFloat16Histogram.h b/src/AggregateFunctions/QuantileBFloat16Histogram.h
index 4a1dd1c8bb3..be6b30bec90 100644
--- a/src/AggregateFunctions/QuantileBFloat16Histogram.h
+++ b/src/AggregateFunctions/QuantileBFloat16Histogram.h
@@ -1,46 +1,150 @@
 #pragma once
 
-#include <AggregateFunctions/BFloat16Histogram.h>
+#include <IO/ReadBuffer.h>
+#include <IO/VarInt.h>
+#include <IO/WriteBuffer.h>
+#include <Common/HashTable/HashMap.h>
+#include <Common/NaNUtils.h>
+#include <common/types.h>
+#include <ext/bit_cast.h>
 
 namespace DB
 {
-namespace ErrorCodes
-{
-    extern const int NOT_IMPLEMENTED;
-}
-
 template <typename Value>
 struct QuantileBFloat16Histogram
 {
-    using Histogram = BFloat16Histogram<Value>;
-    Histogram data;
+    using bfloat16 = UInt16;
+    using Weight = UInt64;
+    using Data = HashMap<bfloat16, Weight>;
+    Data data;
 
-    void add(const Value & x) { data.add(x); }
+    void add(const Value & x) { add(x, 1); }
 
-    template <typename Weight>
-    void add(const Value &, const Weight &)
+    void add(const Value & x, Weight w)
     {
-        throw Exception("Method add with weight is not implemented for QuantileBFloat16Histogram", ErrorCodes::NOT_IMPLEMENTED);
+        if (!isNaN(x))
+            data[to_bfloat16(x)] += w;
     }
 
-    void merge(const QuantileBFloat16Histogram & rhs) { data.merge(rhs.data); }
+    void merge(const QuantileBFloat16Histogram & rhs)
+    {
+        for (const auto & pair : rhs.data)
+            data[pair.getKey()] += pair.getMapped();
+    }
 
     void serialize(WriteBuffer & buf) const { data.write(buf); }
 
     void deserialize(ReadBuffer & buf) { data.read(buf); }
 
-    Value get(Float64 level) { return data.template quantile<Value>(level); }
+    Value get(Float64 level) const { return getImpl<Value>(level); }
 
-    void getMany(const Float64 * levels, const size_t * indices, size_t size, Value * result)
+    void getMany(const Float64 * levels, const size_t * indices, size_t size, Value * result) const
     {
-        data.quantilesMany(levels, indices, size, result);
+        getManyImpl(levels, indices, size, result);
     }
 
-    Float64 getFloat(Float64 level) { return data.template quantile<Float64>(level); }
+    Float64 getFloat(Float64 level) const { return getImpl<Float64>(level); }
 
-    void getManyFloat(const Float64 * levels, const size_t * indices, size_t size, Float64 * result)
+    void getManyFloat(const Float64 * levels, const size_t * indices, size_t size, Float64 * result) const
     {
-        data.quantilesMany(levels, indices, size, result);
+        getManyImpl(levels, indices, size, result);
+    }
+
+private:
+    bfloat16 to_bfloat16(const Value & x) const { return ext::bit_cast<UInt32>(static_cast<Float32>(x)) >> 16; }
+
+    Float32 to_Float32(const bfloat16 & x) const { return ext::bit_cast<Float32>(x << 16); }
+
+    using Pair = PairNoInit<Float32, Weight>;
+
+    template <typename T>
+    T getImpl(Float64 level) const
+    {
+        size_t size = data.size();
+
+        if (0 == size)
+            return std::numeric_limits<T>::quiet_NaN();
+
+        std::unique_ptr<Pair[]> array_holder(new Pair[size]);
+        Pair * array = array_holder.get();
+
+        Float64 sum_weight = 0;
+        Pair * arr_it = array;
+        for (const auto & pair : data)
+        {
+            sum_weight += pair.getMapped();
+            *arr_it = {to_Float32(pair.getKey()), pair.getMapped()};
+            ++arr_it;
+        }
+
+        std::sort(array, array + size, [](const Pair & a, const Pair & b) { return a.first < b.first; });
+
+        Float64 threshold = std::ceil(sum_weight * level);
+        Float64 accumulated = 0;
+
+        for (const Pair * p = array; p != (array + size); ++p)
+        {
+            accumulated += p->second;
+
+            if (accumulated >= threshold)
+                return p->first;
+        }
+
+        return array[size - 1].first;
+    }
+
+    template <typename T>
+    void getManyImpl(const Float64 * levels, const size_t * indices, size_t num_levels, T * result) const
+    {
+        size_t size = data.size();
+
+        if (0 == size)
+        {
+            for (size_t i = 0; i < num_levels; ++i)
+                result[i] = std::numeric_limits<T>::quiet_NaN();
+
+            return;
+        }
+
+        std::unique_ptr<Pair[]> array_holder(new Pair[size]);
+        Pair * array = array_holder.get();
+
+        Float64 sum_weight = 0;
+        Pair * arr_it = array;
+        for (const auto & pair : data)
+        {
+            sum_weight += pair.getMapped();
+            *arr_it = {to_Float32(pair.getKey()), pair.getMapped()};
+            ++arr_it;
+        }
+
+        std::sort(array, array + size, [](const Pair & a, const Pair & b) { return a.first < b.first; });
+
+        size_t level_index = 0;
+        Float64 accumulated = 0;
+        Float64 threshold = std::ceil(sum_weight * levels[indices[level_index]]);
+
+        for (const Pair * p = array; p != (array + size); ++p)
+        {
+            accumulated += p->second;
+
+            while (accumulated >= threshold)
+            {
+                result[indices[level_index]] = p->first;
+                ++level_index;
+
+                if (level_index == num_levels)
+                    return;
+
+                threshold = std::ceil(sum_weight * levels[indices[level_index]]);
+            }
+        }
+
+        while (level_index < num_levels)
+        {
+            result[indices[level_index]] = array[size - 1].first;
+            ++level_index;
+        }
     }
 };
 

From f77506c9de1396564b1d4e15903a391fe57a4031 Mon Sep 17 00:00:00 2001
From: RedClusive <nia4859@ya.ru>
Date: Wed, 28 Apr 2021 14:59:31 +0000
Subject: [PATCH 15/76] remove useless includes

---
 src/AggregateFunctions/QuantileBFloat16Histogram.h | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/AggregateFunctions/QuantileBFloat16Histogram.h b/src/AggregateFunctions/QuantileBFloat16Histogram.h
index be6b30bec90..8072350caac 100644
--- a/src/AggregateFunctions/QuantileBFloat16Histogram.h
+++ b/src/AggregateFunctions/QuantileBFloat16Histogram.h
@@ -1,10 +1,8 @@
 #pragma once
 
 #include <IO/ReadBuffer.h>
-#include <IO/VarInt.h>
 #include <IO/WriteBuffer.h>
 #include <Common/HashTable/HashMap.h>
-#include <Common/NaNUtils.h>
 #include <common/types.h>
 #include <ext/bit_cast.h>
 

From 9744925aa99991af7ac6b4b873a75a58a07f1a8b Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <mikhaylovnikitka@gmail.com>
Date: Thu, 13 May 2021 19:33:07 +0300
Subject: [PATCH 16/76] done

---
 CHANGELOG.md | 145 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 145 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index cc1ec835a7b..6a2c7cb66b8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,148 @@
+## ClickHouse release 21.5
+
+#### Backward Incompatible Change
+
+* Change comparison of integers and floating point numbers when integer is not exactly representable in the floating point data type. In new version comparison will return false as the rounding error will occur. Example: `9223372036854775808.0 != 9223372036854775808`, because the number `9223372036854775808` is not representable as floating point number exactly (and `9223372036854775808.0` is rounded to `9223372036854776000.0`). But in previous version the comparison will return as the numbers are equal, because if the floating point number `9223372036854776000.0` get converted back to UInt64, it will yield `9223372036854775808`. For the reference, the Python programming language also treats these numbers as equal. But this behaviour was dependend on CPU model (different results on AMD64 and AArch64 for some out-of-range numbers), so we make the comparison more precise. It will treat int and float numbers equal only if int is represented in floating point type exactly. [#22595](https://github.com/ClickHouse/ClickHouse/pull/22595) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Now replicas that are processing the `ALTER TABLE ATTACH PART[ITION]` command search in their `detached/` folders before fetching the data from other replicas. As an implementation detail, a new command `ATTACH_PART` is introduced in the replicated log. Parts are searched and compared by their checksums. [#18978](https://github.com/ClickHouse/ClickHouse/pull/18978) ([Mike Kot](https://github.com/myrrc)).
+
+#### New Feature
+
+* Added `Decimal256` type support in dictionaries. Closes [#20979](https://github.com/ClickHouse/ClickHouse/issues/20979). [#22960](https://github.com/ClickHouse/ClickHouse/pull/22960) ([Maksim Kita](https://github.com/kitaisreal)).
+* Added `ALTER TABLE ... FETCH PART ...` query. It's similar to `FETCH PARTITION`, but fetches only one part. [#22706](https://github.com/ClickHouse/ClickHouse/pull/22706) ([turbo jason](https://github.com/songenjie)).
+* FlatDictionary added `initial_array_size`, `max_array_size` options. [#22521](https://github.com/ClickHouse/ClickHouse/pull/22521) ([Maksim Kita](https://github.com/kitaisreal)).
+* Add new setting `non_replicated_deduplication_window` for non-replicated MergeTree inserts deduplication. [#22514](https://github.com/ClickHouse/ClickHouse/pull/22514) ([alesapin](https://github.com/alesapin)).
+* Update paths to the `CatBoost` model configs in config reloading. [#22434](https://github.com/ClickHouse/ClickHouse/pull/22434) ([Kruglov Pavel](https://github.com/Avogar)).
+* Added function `dictGetOrNull`. It works like `dictGet`, but return `Null` in case key was not found in dictionary. Closes [#22375](https://github.com/ClickHouse/ClickHouse/issues/22375). [#22413](https://github.com/ClickHouse/ClickHouse/pull/22413) ([Maksim Kita](https://github.com/kitaisreal)).
+* Added support for replicas and shards in MySQL/PostgreSQL table engine / table function. Replicas and shards, if used together, are added via new table engine ExternalDistributed. Closes [#20969](https://github.com/ClickHouse/ClickHouse/issues/20969). [#22217](https://github.com/ClickHouse/ClickHouse/pull/22217) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Improved performance of `dictGetHierarchy`, `dictIsIn` functions. Added functions `dictGetChildren(dictionary, key)`, `dictGetDescendants(dictionary, key, level)`. Function `dictGetChildren` return all children as an array if indexes. It is a inverse transformation for `dictGetHierarchy`. Function `dictGetDescendants` return all descendants as if `dictGetChildren` was applied `level` times recursively. Zero `level` value is equivalent to infinity. Closes [#14656](https://github.com/ClickHouse/ClickHouse/issues/14656). [#22096](https://github.com/ClickHouse/ClickHouse/pull/22096) ([Maksim Kita](https://github.com/kitaisreal)).
+* Added a table function `s3Cluster`, which allows to process files from `s3` in parallel on every node of a specified cluster. [#22012](https://github.com/ClickHouse/ClickHouse/pull/22012) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Added a setting `max_distributed_depth` that limits the depth of recursive queries to `Distributed` tables. Closes [#20229](https://github.com/ClickHouse/ClickHouse/issues/20229). [#21942](https://github.com/ClickHouse/ClickHouse/pull/21942) ([flynn](https://github.com/ucasFL)).
+* Added function `arrayFold(x1,...,xn,accum -> expression, array1,...,arrayn, init_accum)` that applies the expression to each element of the array (or set of parallel arrays) and collect result in accumulator. [#21589](https://github.com/ClickHouse/ClickHouse/pull/21589) ([Dmitry Krylov](https://github.com/dmalkr)).
+
+#### Performance Improvement
+
+* Improved performance of `intDiv` by dynamic dispatch for AVX2. This closes [#22314](https://github.com/ClickHouse/ClickHouse/issues/22314). [#23000](https://github.com/ClickHouse/ClickHouse/pull/23000) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Improved performance of reading from `ArrowStream` input format for sources other then local file (e.g. URL). [#22673](https://github.com/ClickHouse/ClickHouse/pull/22673) ([nvartolomei](https://github.com/nvartolomei)).
+* Disabled compression by default when interacting with localhost (with clickhouse-client or server to server with distributed queries) via native protocol. It may improve performance of some import/export operations. This closes [#22234](https://github.com/ClickHouse/ClickHouse/issues/22234). [#22237](https://github.com/ClickHouse/ClickHouse/pull/22237) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Exclude values that does not belong to the shard from right part of IN section for distributed queries. [#21511](https://github.com/ClickHouse/ClickHouse/pull/21511) ([Azat Khuzhin](https://github.com/azat)).
+* Improved performance of reading a subset of columns with File-like table engine and column-oriented format like Parquet, Arrow or ORC. This closes [#issue:20129](https://github.com/ClickHouse/ClickHouse/issues/20129). [#21302](https://github.com/ClickHouse/ClickHouse/pull/21302) ([keenwolf](https://github.com/keen-wolf)).
+
+#### Improvement
+
+* Enabled `async_socket_for_remote` by default. [#23683](https://github.com/ClickHouse/ClickHouse/pull/23683) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fixed `quantile(s)TDigest`. Added special handling of singleton centroids according to tdunning/t-digest 3.2+. Also a bug with over-compression of centroids in implementation of earlier version of the algorithm was fixed. [#23314](https://github.com/ClickHouse/ClickHouse/pull/23314) ([Vladimir Chebotarev](https://github.com/excitoon)).
+* Disable settings `use_hedged_requests` and `async_socket_for_remote` because there is an evidence that it may cause issues. [#23261](https://github.com/ClickHouse/ClickHouse/pull/23261) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Make function `unhex` case insensitive for compatibility with MySQL. [#23229](https://github.com/ClickHouse/ClickHouse/pull/23229) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Fixed memory tracking with min_bytes_to_use_mmap_io. [#23211](https://github.com/ClickHouse/ClickHouse/pull/23211) ([Azat Khuzhin](https://github.com/azat)).
+* Implement functions `arrayHasAny`, `arrayHasAll`, `has`, `indexOf`, `countEqual` for generic case when types of array elements are different. In previous versions the functions `arrayHasAny`, `arrayHasAll` returned false and `has`, `indexOf`, `countEqual` thrown exception. Also add support for `Decimal` and big integer types in functions `has` and similar. This closes [#20272](https://github.com/ClickHouse/ClickHouse/issues/20272). [#23044](https://github.com/ClickHouse/ClickHouse/pull/23044) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Raised the threshold on max number of matches in result of the function `extractAllGroupsHorizontal`. [#23036](https://github.com/ClickHouse/ClickHouse/pull/23036) ([Vasily Nemkov](https://github.com/Enmk)).
+* Do not perform `optimize_skip_unused_shards` for cluster with one node. [#22999](https://github.com/ClickHouse/ClickHouse/pull/22999) ([Azat Khuzhin](https://github.com/azat)).
+* Increase `max_uri_size` (the maximum size of URL in HTTP interface) to 1 MiB by default. This closes [#21197](https://github.com/ClickHouse/ClickHouse/issues/21197). [#22997](https://github.com/ClickHouse/ClickHouse/pull/22997) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Added ability to run clickhouse-keeper with SSL. Config settings `keeper_server.tcp_port_secure` can be used for secure interaction between client and keeper-server. `keeper_server.raft_configuration.secure` can be used to enable internal secure communication between nodes. [#22992](https://github.com/ClickHouse/ClickHouse/pull/22992) ([alesapin](https://github.com/alesapin)).
+* Added ability to flush buffer only in background for StorageBuffer. [#22986](https://github.com/ClickHouse/ClickHouse/pull/22986) ([Azat Khuzhin](https://github.com/azat)).
+* When selecting from MergeTree table with NULL in WHERE condition, in rare cases, exception was thrown. This closes [#20019](https://github.com/ClickHouse/ClickHouse/issues/20019). [#22978](https://github.com/ClickHouse/ClickHouse/pull/22978) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Fixed an error handling in Poco HTTP Client for AWS. [#22973](https://github.com/ClickHouse/ClickHouse/pull/22973) ([kreuzerkrieg](https://github.com/kreuzerkrieg)).
+* Respect `max_part_removal_threads` for ReplicatedMergeTree. [#22971](https://github.com/ClickHouse/ClickHouse/pull/22971) ([Azat Khuzhin](https://github.com/azat)).
+* Fix inactive_parts_to_throw_insert=0 with inactive_parts_to_delay_insert>0. [#22947](https://github.com/ClickHouse/ClickHouse/pull/22947) ([Azat Khuzhin](https://github.com/azat)).
+* Set `background_fetches_pool_size` to 8 that is better for production usage with frequent small insertions or slow ZooKeeper cluster. [#22945](https://github.com/ClickHouse/ClickHouse/pull/22945) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* `dateDiff` now works with `DateTime64` arguments (even for values outside of `DateTime` range) [#22931](https://github.com/ClickHouse/ClickHouse/pull/22931) ([Vasily Nemkov](https://github.com/Enmk)).
+* Added an ability to replicate MySQL databases containing views without failing. This is accomplished by ignoring the views. [#22760](https://github.com/ClickHouse/ClickHouse/pull/22760) ([Christian](https://github.com/cfroystad)).
+* Allow RBAC row policy via postgresql protocol. Closes [#22658](https://github.com/ClickHouse/ClickHouse/issues/22658). PostgreSQL protocol is enabled in configuration by default. [#22755](https://github.com/ClickHouse/ClickHouse/pull/22755) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Add metric to track how much time is spend during waiting for Buffer layer lock. [#22725](https://github.com/ClickHouse/ClickHouse/pull/22725) ([Azat Khuzhin](https://github.com/azat)).
+* Allow to use CTE in VIEW definition. This closes [#22491](https://github.com/ClickHouse/ClickHouse/issues/22491). [#22657](https://github.com/ClickHouse/ClickHouse/pull/22657) ([Amos Bird](https://github.com/amosbird)).
+* Clear the rest of the screen and show cursor in `clickhouse-client` if previous program has left garbage in terminal. This closes [#16518](https://github.com/ClickHouse/ClickHouse/issues/16518). [#22634](https://github.com/ClickHouse/ClickHouse/pull/22634) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Make `round` function to behave consistently on non-x86_64 platforms. Rounding half to nearest even (Banker's rounding) is used. [#22582](https://github.com/ClickHouse/ClickHouse/pull/22582) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Correctly check structure of async distributed blocks. [#22325](https://github.com/ClickHouse/ClickHouse/pull/22325) ([Azat Khuzhin](https://github.com/azat)).
+* Added nanodbc instead of Poco::ODBC. Closes [#9678](https://github.com/ClickHouse/ClickHouse/issues/9678). Add support for DateTime64 and Decimal* for ODBC table engine. Closes [#21961](https://github.com/ClickHouse/ClickHouse/issues/21961). Fixed issue with cyrillic text being truncated. Closes [#16246](https://github.com/ClickHouse/ClickHouse/issues/16246). Added connection pools for odbc bridge. [#21972](https://github.com/ClickHouse/ClickHouse/pull/21972) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Allow publishing Kafka errors to a virtual column of Kafka engine, controlled by the `kafka_handle_error_mode` setting. [#21850](https://github.com/ClickHouse/ClickHouse/pull/21850) ([fastio](https://github.com/fastio)).
+* Add aliases `simpleJSONExtract/simpleJSONHas` to `visitParam/visitParamExtract{UInt, Int, Bool, Float, Raw, String}`. Fixes [#21383](https://github.com/ClickHouse/ClickHouse/issues/21383). [#21519](https://github.com/ClickHouse/ClickHouse/pull/21519) ([fastio](https://github.com/fastio)).
+* Add `clickhouse-library-bridge` for library dictionary source. Closes [#9502](https://github.com/ClickHouse/ClickHouse/issues/9502). [#21509](https://github.com/ClickHouse/ClickHouse/pull/21509) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Forbid to drop a column if it's referenced by materialized view. Closes [#21164](https://github.com/ClickHouse/ClickHouse/issues/21164). [#21303](https://github.com/ClickHouse/ClickHouse/pull/21303) ([flynn](https://github.com/ucasFL)).
+* Support dynamic interserver credentials. [#14113](https://github.com/ClickHouse/ClickHouse/pull/14113) ([johnskopis](https://github.com/johnskopis)).
+
+#### Bug Fix
+
+* HashedDictionary complex key update field initial load fix. Closes [#23800](https://github.com/ClickHouse/ClickHouse/issues/23800). [#23824](https://github.com/ClickHouse/ClickHouse/pull/23824) ([Maksim Kita](https://github.com/kitaisreal)).
+* Fixed an error `Can't initialize pipeline with empty pipe` for queries with `GLOBAL IN/JOIN` and `use_hedged_requests`. Fixes [#23431](https://github.com/ClickHouse/ClickHouse/issues/23431). [#23805](https://github.com/ClickHouse/ClickHouse/pull/23805) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fixed remote JDBC bridge timeout connection issue. Closes [#9609](https://github.com/ClickHouse/ClickHouse/issues/9609). [#23771](https://github.com/ClickHouse/ClickHouse/pull/23771) ([Maksim Kita](https://github.com/kitaisreal)).
+* Fixed crash when `PREWHERE` and row policy filter are both in effect with empty result. [#23763](https://github.com/ClickHouse/ClickHouse/pull/23763) ([Amos Bird](https://github.com/amosbird)).
+* Avoid possible "Cannot schedule a task" error (in case some exception had been occurred) on INSERT into Distributed. [#23744](https://github.com/ClickHouse/ClickHouse/pull/23744) ([Azat Khuzhin](https://github.com/azat)).
+* Added an exception in case of completely the same values in both samples in aggregate function `mannWhitneyUTest`. This fixes [#23646](https://github.com/ClickHouse/ClickHouse/issues/23646). [#23654](https://github.com/ClickHouse/ClickHouse/pull/23654) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Fixed server fault when inserting data through HTTP caused an exception. This fixes [#23512](https://github.com/ClickHouse/ClickHouse/issues/23512). [#23643](https://github.com/ClickHouse/ClickHouse/pull/23643) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Fixed misinterpretation of some `LIKE` expressions with escape sequences. [#23610](https://github.com/ClickHouse/ClickHouse/pull/23610) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Fixed restart / stop command hanging. Closes [#20214](https://github.com/ClickHouse/ClickHouse/issues/20214). [#23552](https://github.com/ClickHouse/ClickHouse/pull/23552) ([filimonov](https://github.com/filimonov)).
+* Fixed `columns` function when multiple joins in select query. Closes [#22736](https://github.com/ClickHouse/ClickHouse/issues/22736). [#23501](https://github.com/ClickHouse/ClickHouse/pull/23501) ([Maksim Kita](https://github.com/kitaisreal)).
+* Fixed a crash when modifying column's default value when a column itself is used as ReplacingMergeTree's parameter. [#23483](https://github.com/ClickHouse/ClickHouse/pull/23483) ([hexiaoting](https://github.com/hexiaoting)).
+* Fixed corner cases in vertical merges with `ReplacingMergeTree`. In rare cases they could lead to fails of merges with exceptions like `Incomplete granules are not allowed while blocks are granules size`. [#23459](https://github.com/ClickHouse/ClickHouse/pull/23459) ([Anton Popov](https://github.com/CurtizJ)).
+* Fixed bug that does not allow cast from empty array literal, to array with dimensions greater than 1. Closes [#14476](https://github.com/ClickHouse/ClickHouse/issues/14476). [#23456](https://github.com/ClickHouse/ClickHouse/pull/23456) ([Maksim Kita](https://github.com/kitaisreal)).
+* Fixed a bug when `deltaSum` aggregate function produced incorrect result after resetting the counter. [#23437](https://github.com/ClickHouse/ClickHouse/pull/23437) ([Russ Frank](https://github.com/rf)).
+* Fixed `Cannot unlink file` error on unsuccessful creation of ReplicatedMergeTree table with multidisk configuration. This closes [#21755](https://github.com/ClickHouse/ClickHouse/issues/21755). [#23433](https://github.com/ClickHouse/ClickHouse/pull/23433) ([tavplubix](https://github.com/tavplubix)).
+* Fixed a lack of support for Kafka storage with `arrow` and `arrowstream` format messages. [#23415](https://github.com/ClickHouse/ClickHouse/pull/23415) ([Chao Ma](https://github.com/godliness)).
+* Allow to move more conditions to `PREWHERE` as it was before version 21.1. Insufficient number of moved condtions could lead to worse performance. [#23397](https://github.com/ClickHouse/ClickHouse/pull/23397) ([Anton Popov](https://github.com/CurtizJ)).
+* Remove support for `argMin` and `argMax` for single `Tuple` argument. The code was not memory-safe. The feature was added by mistake and it is confusing for people. These functions can be reintroduced under different names later. This fixes [#22384](https://github.com/ClickHouse/ClickHouse/issues/22384) and reverts [#17359](https://github.com/ClickHouse/ClickHouse/issues/17359). [#23393](https://github.com/ClickHouse/ClickHouse/pull/23393) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Fixed incompatible constant expression generation during partition pruning based on virtual columns. This fixes https://github.com/ClickHouse/ClickHouse/pull/21401#discussion_r611888913. [#23366](https://github.com/ClickHouse/ClickHouse/pull/23366) ([Amos Bird](https://github.com/amosbird)).
+* Fixed a bug in dict join with join_algorithm = 'auto'. Close [#23002](https://github.com/ClickHouse/ClickHouse/issues/23002). [#23312](https://github.com/ClickHouse/ClickHouse/pull/23312) ([Vladimir](https://github.com/vdimir)).
+* Don't relax NOT conditions during partition pruning. This fixes [#23305](https://github.com/ClickHouse/ClickHouse/issues/23305) and [#21539](https://github.com/ClickHouse/ClickHouse/issues/21539). [#23310](https://github.com/ClickHouse/ClickHouse/pull/23310) ([Amos Bird](https://github.com/amosbird)).
+* Fix possible crash in case if `unknown packet` was received form remote query (with `async_socket_for_remote` enabled). Maybe fixes [#21167](https://github.com/ClickHouse/ClickHouse/issues/21167). [#23309](https://github.com/ClickHouse/ClickHouse/pull/23309) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fixed very rare race condition on background cleanup of old blocks. It might cause a block not to be deduplicated if it's too close to the end of deduplication window. [#23301](https://github.com/ClickHouse/ClickHouse/pull/23301) ([tavplubix](https://github.com/tavplubix)).
+* Fixed very rare (distributed) race condition between creation and removal of ReplicatedMergeTree tables. It might cause exceptions like `node doesn't exist` on attempt to create replicated table. Fixes [#21419](https://github.com/ClickHouse/ClickHouse/issues/21419). [#23294](https://github.com/ClickHouse/ClickHouse/pull/23294) ([tavplubix](https://github.com/tavplubix)).
+* Fixed simple key dictionary from DDL creation if primary key is not first attribute. Fixes [#23236](https://github.com/ClickHouse/ClickHouse/issues/23236). [#23262](https://github.com/ClickHouse/ClickHouse/pull/23262) ([Maksim Kita](https://github.com/kitaisreal)).
+* Add type conversion for `optimize_skip_unused_shards_rewrite_in` (fixes `use-of-uninitialized-value` with `optimize_skip_unused_shards_rewrite_in`). [#23219](https://github.com/ClickHouse/ClickHouse/pull/23219) ([Azat Khuzhin](https://github.com/azat)).
+* Fixed reading from ODBC when there are many long column names in a table. Closes [#8853](https://github.com/ClickHouse/ClickHouse/issues/8853). [#23215](https://github.com/ClickHouse/ClickHouse/pull/23215) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fixed missing semicolon in exception message. The user may find this exception message unpleasant to read. [#23208](https://github.com/ClickHouse/ClickHouse/pull/23208) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Fixed missing whitespace in some exception messages about `LowCardinality` type. [#23207](https://github.com/ClickHouse/ClickHouse/pull/23207) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Fixed the behavior when disabling `input_format_with_names_use_header ` setting discards all the input with CSVWithNames format. This fixes [#22406](https://github.com/ClickHouse/ClickHouse/issues/22406). [#23202](https://github.com/ClickHouse/ClickHouse/pull/23202) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Fixed `Not found column` error when selecting from `MaterializeMySQL` with condition on key column. Fixes [#22432](https://github.com/ClickHouse/ClickHouse/issues/22432). [#23200](https://github.com/ClickHouse/ClickHouse/pull/23200) ([tavplubix](https://github.com/tavplubix)).
+* QueryAliasVisitor to prefer alias for ASTWithAlias if subquery was optimized to constant. Fixes [#22924](https://github.com/ClickHouse/ClickHouse/issues/22924). Fixes [#10401](https://github.com/ClickHouse/ClickHouse/issues/10401). [#23191](https://github.com/ClickHouse/ClickHouse/pull/23191) ([Maksim Kita](https://github.com/kitaisreal)).
+* Server might fail to start if `data_type_default_nullable` setting is enabled in default profile, it's fixed. Fixes [#22573](https://github.com/ClickHouse/ClickHouse/issues/22573). [#23185](https://github.com/ClickHouse/ClickHouse/pull/23185) ([tavplubix](https://github.com/tavplubix)).
+* Fixed a crash on shutdown which happened because of wrong accounting of current connections. [#23154](https://github.com/ClickHouse/ClickHouse/pull/23154) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Some values were formatted with alignment in center in table cells in `Markdown` format. Not anymore. [#23096](https://github.com/ClickHouse/ClickHouse/pull/23096) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Fixed `Table .inner_id... doesn't exist` error when selecting from Materialized View after detaching it from Atomic database and attaching back. [#23047](https://github.com/ClickHouse/ClickHouse/pull/23047) ([tavplubix](https://github.com/tavplubix)).
+* Remove non-essential details from suggestions in clickhouse-client. This closes [#22158](https://github.com/ClickHouse/ClickHouse/issues/22158). [#23040](https://github.com/ClickHouse/ClickHouse/pull/23040) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Fix error `Cannot find column in ActionsDAG result` which may happen if subquery uses `untuple`. Fixes [#22290](https://github.com/ClickHouse/ClickHouse/issues/22290). [#22991](https://github.com/ClickHouse/ClickHouse/pull/22991) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix usage of constant columns of type `Map` with nullable values. [#22939](https://github.com/ClickHouse/ClickHouse/pull/22939) ([Anton Popov](https://github.com/CurtizJ)).
+* fixed `formatDateTime()` on `DateTime64` and "%C" format specifier fixed `toDateTime64()` for large values and non-zero scale. ... [#22937](https://github.com/ClickHouse/ClickHouse/pull/22937) ([Vasily Nemkov](https://github.com/Enmk)).
+* Fixed a crash when using `mannWhitneyUTest` and `rankCorr` with window functions. This fixes [#22728](https://github.com/ClickHouse/ClickHouse/issues/22728). [#22876](https://github.com/ClickHouse/ClickHouse/pull/22876) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Fixed `bytes_allocated` field in system.dictionaries for sparse_hashed dictionaries. [#22867](https://github.com/ClickHouse/ClickHouse/pull/22867) ([Azat Khuzhin](https://github.com/azat)).
+* Fixed possible hanging in concurrent DROP/CREATE of TEMPORARY LIVE VIEW in `TemporaryLiveViewCleaner`, see https://gist.github.com/vzakaznikov/0c03195960fc86b56bfe2bc73a90019e. [#22858](https://github.com/ClickHouse/ClickHouse/pull/22858) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Fixed pushdown of `HAVING` in case, when filter column is used in aggregation. [#22763](https://github.com/ClickHouse/ClickHouse/pull/22763) ([Anton Popov](https://github.com/CurtizJ)).
+* Fixed approximate total rows accounting for reverse reading from MergeTree. [#22726](https://github.com/ClickHouse/ClickHouse/pull/22726) ([Azat Khuzhin](https://github.com/azat)).
+* Fixed possible hangs in Zookeeper requests in case of OOM exception. Fixes [#22438](https://github.com/ClickHouse/ClickHouse/issues/22438). [#22684](https://github.com/ClickHouse/ClickHouse/pull/22684) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fixed wait for mutations on several replicas for ReplicatedMergeTree table engines. Previously, mutation/alter query may finish before mutation actually executed on other replicas. [#22669](https://github.com/ClickHouse/ClickHouse/pull/22669) ([alesapin](https://github.com/alesapin)).
+* Fixed LOGICAL_ERROR for Log with nested types w/o columns in the SELECT clause. [#22654](https://github.com/ClickHouse/ClickHouse/pull/22654) ([Azat Khuzhin](https://github.com/azat)).
+* Fixed a bug with unlimited wait for auxiliary AWS requests. [#22594](https://github.com/ClickHouse/ClickHouse/pull/22594) ([Vladimir Chebotarev](https://github.com/excitoon)).
+* Fixed a crash when client closes connection very early [#22579](https://github.com/ClickHouse/ClickHouse/issues/22579). [#22591](https://github.com/ClickHouse/ClickHouse/pull/22591) ([nvartolomei](https://github.com/nvartolomei)).
+* Fixed an incorrect formatting of function `map` in distributed queries. [#22588](https://github.com/ClickHouse/ClickHouse/pull/22588) ([foolchi](https://github.com/foolchi)).
+* Avoid UB in *Log engines for rwlock unlock due to unlock from another thread. [#22583](https://github.com/ClickHouse/ClickHouse/pull/22583) ([Azat Khuzhin](https://github.com/azat)).
+* Fixed UB by unlocking the rwlock of the TinyLog from the same thread. [#22560](https://github.com/ClickHouse/ClickHouse/pull/22560) ([Azat Khuzhin](https://github.com/azat)).
+* Fixed deserialization of empty string without newline at end of TSV format. This closes [#20244](https://github.com/ClickHouse/ClickHouse/issues/20244). Possible workaround without version update: set `input_format_null_as_default` to zero. It was zero in old versions. [#22527](https://github.com/ClickHouse/ClickHouse/pull/22527) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Fix bug in partial merge join with `LowCardinality`. Close [#22386](https://github.com/ClickHouse/ClickHouse/issues/22386), close [#22388](https://github.com/ClickHouse/ClickHouse/issues/22388). [#22510](https://github.com/ClickHouse/ClickHouse/pull/22510) ([Vladimir](https://github.com/vdimir)).
+* Fixed `ClickHouseDictionarySource` configuration loop. Closes [#14314](https://github.com/ClickHouse/ClickHouse/issues/14314). [#22479](https://github.com/ClickHouse/ClickHouse/pull/22479) ([Maksim Kita](https://github.com/kitaisreal)).
+* Fixed a race condition in HedgedConnections which leads to crash. This fixes [#22161](https://github.com/ClickHouse/ClickHouse/issues/22161). [#22443](https://github.com/ClickHouse/ClickHouse/pull/22443) ([Kruglov Pavel](https://github.com/Avogar)).
+* Buffer overflow (on read) was possible in `tokenbf_v1` full text index. The excessive bytes are not used but the read operation may lead to crash in rare cases. This closes [#19233](https://github.com/ClickHouse/ClickHouse/issues/19233). [#22421](https://github.com/ClickHouse/ClickHouse/pull/22421) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Do not limit HTTP chunk size. Fixes [#21907](https://github.com/ClickHouse/ClickHouse/issues/21907). [#22322](https://github.com/ClickHouse/ClickHouse/pull/22322) ([Ivan](https://github.com/abyss7)).
+* Fixed a bug, which leads to underaggregation of data in case of enabled `optimize_aggregation_in_order` and many parts in table. Slightly improve performance of aggregation with enabled `optimize_aggregation_in_order`. [#21889](https://github.com/ClickHouse/ClickHouse/pull/21889) ([Anton Popov](https://github.com/CurtizJ)).
+* Check if table function view is used as a column. This complements https://github.com/ClickHouse/ClickHouse/pull/20350. [#21465](https://github.com/ClickHouse/ClickHouse/pull/21465) ([Amos Bird](https://github.com/amosbird)).
+* * Fix "unknown column" error for tables with `Merge` engine in queris with `JOIN` and aggregation. Closes [#18368](https://github.com/ClickHouse/ClickHouse/issues/18368), close [#22226](https://github.com/ClickHouse/ClickHouse/issues/22226). [#21370](https://github.com/ClickHouse/ClickHouse/pull/21370) ([Vladimir](https://github.com/vdimir)).
+* Fixed name clashes in `PredicateRewriteVisitor`. It caused incorrect `WHERE` filtration after full join. Close [#20497](https://github.com/ClickHouse/ClickHouse/issues/20497). [#20622](https://github.com/ClickHouse/ClickHouse/pull/20622) ([Vladimir](https://github.com/vdimir)).
+* Fixed very rare bug when quorum insert with `quorum_parallel=1` is not really "quorum" because of deduplication. [#18215](https://github.com/ClickHouse/ClickHouse/pull/18215) ([filimonov](https://github.com/filimonov)).
+
+#### Build/Testing/Packaging Improvement
+
+* Simplify debian packages. This fixes [#21698](https://github.com/ClickHouse/ClickHouse/issues/21698). [#22976](https://github.com/ClickHouse/ClickHouse/pull/22976) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Added support for ClickHouse build on Apple M1. [#21639](https://github.com/ClickHouse/ClickHouse/pull/21639) ([changvvb](https://github.com/changvvb)).
+* Fixed ClickHouse Keeper build for MacOS. [#22860](https://github.com/ClickHouse/ClickHouse/pull/22860) ([alesapin](https://github.com/alesapin)).
+* Fixed some tests on AArch64 platform. [#22596](https://github.com/ClickHouse/ClickHouse/pull/22596) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Added function alignment for possibly better performance. [#21431](https://github.com/ClickHouse/ClickHouse/pull/21431) ([Danila Kutenin](https://github.com/danlark1)).
+* Adjust some tests to output identical results on amd64 and aarch64 (qemu). The result was depending on implementation specific CPU behaviour. [#22590](https://github.com/ClickHouse/ClickHouse/pull/22590) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Allow query profiling only on x86_64. See [#15174](https://github.com/ClickHouse/ClickHouse/issues/15174)#issuecomment-812954965 and [#15638](https://github.com/ClickHouse/ClickHouse/issues/15638)#issuecomment-703805337. This closes [#15638](https://github.com/ClickHouse/ClickHouse/issues/15638). [#22580](https://github.com/ClickHouse/ClickHouse/pull/22580) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Allow building with unbundled xz (lzma) using USE_INTERNAL_XZ_LIBRARY=OFF ... [#22571](https://github.com/ClickHouse/ClickHouse/pull/22571) ([Kfir Itzhak](https://github.com/mastertheknife)).
+* Enable the bundled openldap on ppc64le ... [#22487](https://github.com/ClickHouse/ClickHouse/pull/22487) ([Kfir Itzhak](https://github.com/mastertheknife)).
+* Disable incompatible libraries (platform specific typically) on ppc64le ... [#22475](https://github.com/ClickHouse/ClickHouse/pull/22475) ([Kfir Itzhak](https://github.com/mastertheknife)).
+* Add on-demand check for clickhouse Keeper. [#22373](https://github.com/ClickHouse/ClickHouse/pull/22373) ([alesapin](https://github.com/alesapin)).
+* Run stateless tests in parallel in CI. Depends on [#22181](https://github.com/ClickHouse/ClickHouse/issues/22181). [#22300](https://github.com/ClickHouse/ClickHouse/pull/22300) ([alesapin](https://github.com/alesapin)).
+* Build `jemalloc` with support for [heap profiling](https://github.com/jemalloc/jemalloc/wiki/Use-Case%3A-Heap-Profiling). [#22834](https://github.com/ClickHouse/ClickHouse/pull/22834) ([nvartolomei](https://github.com/nvartolomei)).
+
 ## ClickHouse release 21.4
 
 ### ClickHouse release 21.4.1 2021-04-12

From c314d5393eb6be7da5c8cd739a167080f484b223 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <mikhaylovnikitka@gmail.com>
Date: Thu, 13 May 2021 20:44:55 +0300
Subject: [PATCH 17/76] Update CHANGELOG.md

Co-authored-by: tavplubix <tavplubix@gmail.com>
---
 CHANGELOG.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6a2c7cb66b8..b049227a6c8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,7 +3,6 @@
 #### Backward Incompatible Change
 
 * Change comparison of integers and floating point numbers when integer is not exactly representable in the floating point data type. In new version comparison will return false as the rounding error will occur. Example: `9223372036854775808.0 != 9223372036854775808`, because the number `9223372036854775808` is not representable as floating point number exactly (and `9223372036854775808.0` is rounded to `9223372036854776000.0`). But in previous version the comparison will return as the numbers are equal, because if the floating point number `9223372036854776000.0` get converted back to UInt64, it will yield `9223372036854775808`. For the reference, the Python programming language also treats these numbers as equal. But this behaviour was dependend on CPU model (different results on AMD64 and AArch64 for some out-of-range numbers), so we make the comparison more precise. It will treat int and float numbers equal only if int is represented in floating point type exactly. [#22595](https://github.com/ClickHouse/ClickHouse/pull/22595) ([alexey-milovidov](https://github.com/alexey-milovidov)).
-* Now replicas that are processing the `ALTER TABLE ATTACH PART[ITION]` command search in their `detached/` folders before fetching the data from other replicas. As an implementation detail, a new command `ATTACH_PART` is introduced in the replicated log. Parts are searched and compared by their checksums. [#18978](https://github.com/ClickHouse/ClickHouse/pull/18978) ([Mike Kot](https://github.com/myrrc)).
 
 #### New Feature
 

From b4eb181378d082a80cc1abf5d39ab2afd61b0448 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <mikhaylovnikitka@gmail.com>
Date: Thu, 13 May 2021 20:51:16 +0300
Subject: [PATCH 18/76] Update CHANGELOG.md

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b049227a6c8..d3a8c13f5e4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -80,7 +80,7 @@
 * Allow to move more conditions to `PREWHERE` as it was before version 21.1. Insufficient number of moved condtions could lead to worse performance. [#23397](https://github.com/ClickHouse/ClickHouse/pull/23397) ([Anton Popov](https://github.com/CurtizJ)).
 * Remove support for `argMin` and `argMax` for single `Tuple` argument. The code was not memory-safe. The feature was added by mistake and it is confusing for people. These functions can be reintroduced under different names later. This fixes [#22384](https://github.com/ClickHouse/ClickHouse/issues/22384) and reverts [#17359](https://github.com/ClickHouse/ClickHouse/issues/17359). [#23393](https://github.com/ClickHouse/ClickHouse/pull/23393) ([alexey-milovidov](https://github.com/alexey-milovidov)).
 * Fixed incompatible constant expression generation during partition pruning based on virtual columns. This fixes https://github.com/ClickHouse/ClickHouse/pull/21401#discussion_r611888913. [#23366](https://github.com/ClickHouse/ClickHouse/pull/23366) ([Amos Bird](https://github.com/amosbird)).
-* Fixed a bug in dict join with join_algorithm = 'auto'. Close [#23002](https://github.com/ClickHouse/ClickHouse/issues/23002). [#23312](https://github.com/ClickHouse/ClickHouse/pull/23312) ([Vladimir](https://github.com/vdimir)).
+* Fixed a crash when setting join_algorithm is set to 'auto' and Join is performed with a Dictionary. Close [#23002](https://github.com/ClickHouse/ClickHouse/issues/23002). [#23312](https://github.com/ClickHouse/ClickHouse/pull/23312) ([Vladimir](https://github.com/vdimir)).
 * Don't relax NOT conditions during partition pruning. This fixes [#23305](https://github.com/ClickHouse/ClickHouse/issues/23305) and [#21539](https://github.com/ClickHouse/ClickHouse/issues/21539). [#23310](https://github.com/ClickHouse/ClickHouse/pull/23310) ([Amos Bird](https://github.com/amosbird)).
 * Fix possible crash in case if `unknown packet` was received form remote query (with `async_socket_for_remote` enabled). Maybe fixes [#21167](https://github.com/ClickHouse/ClickHouse/issues/21167). [#23309](https://github.com/ClickHouse/ClickHouse/pull/23309) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
 * Fixed very rare race condition on background cleanup of old blocks. It might cause a block not to be deduplicated if it's too close to the end of deduplication window. [#23301](https://github.com/ClickHouse/ClickHouse/pull/23301) ([tavplubix](https://github.com/tavplubix)).

From 2dd4aa7ea28cbf209ff925d29291b81cc5b842ca Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <mikhaylovnikitka@gmail.com>
Date: Thu, 13 May 2021 20:54:22 +0300
Subject: [PATCH 19/76] Update CHANGELOG.md

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d3a8c13f5e4..e5e6b692bb0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -115,7 +115,7 @@
 * Avoid UB in *Log engines for rwlock unlock due to unlock from another thread. [#22583](https://github.com/ClickHouse/ClickHouse/pull/22583) ([Azat Khuzhin](https://github.com/azat)).
 * Fixed UB by unlocking the rwlock of the TinyLog from the same thread. [#22560](https://github.com/ClickHouse/ClickHouse/pull/22560) ([Azat Khuzhin](https://github.com/azat)).
 * Fixed deserialization of empty string without newline at end of TSV format. This closes [#20244](https://github.com/ClickHouse/ClickHouse/issues/20244). Possible workaround without version update: set `input_format_null_as_default` to zero. It was zero in old versions. [#22527](https://github.com/ClickHouse/ClickHouse/pull/22527) ([alexey-milovidov](https://github.com/alexey-milovidov)).
-* Fix bug in partial merge join with `LowCardinality`. Close [#22386](https://github.com/ClickHouse/ClickHouse/issues/22386), close [#22388](https://github.com/ClickHouse/ClickHouse/issues/22388). [#22510](https://github.com/ClickHouse/ClickHouse/pull/22510) ([Vladimir](https://github.com/vdimir)).
+* Fixed wrong cast of a column of `LowCardinality` type in MergeJoin algorithm. Close [#22386](https://github.com/ClickHouse/ClickHouse/issues/22386), close [#22388](https://github.com/ClickHouse/ClickHouse/issues/22388). [#22510](https://github.com/ClickHouse/ClickHouse/pull/22510) ([Vladimir](https://github.com/vdimir)).
 * Fixed `ClickHouseDictionarySource` configuration loop. Closes [#14314](https://github.com/ClickHouse/ClickHouse/issues/14314). [#22479](https://github.com/ClickHouse/ClickHouse/pull/22479) ([Maksim Kita](https://github.com/kitaisreal)).
 * Fixed a race condition in HedgedConnections which leads to crash. This fixes [#22161](https://github.com/ClickHouse/ClickHouse/issues/22161). [#22443](https://github.com/ClickHouse/ClickHouse/pull/22443) ([Kruglov Pavel](https://github.com/Avogar)).
 * Buffer overflow (on read) was possible in `tokenbf_v1` full text index. The excessive bytes are not used but the read operation may lead to crash in rare cases. This closes [#19233](https://github.com/ClickHouse/ClickHouse/issues/19233). [#22421](https://github.com/ClickHouse/ClickHouse/pull/22421) ([alexey-milovidov](https://github.com/alexey-milovidov)).

From 53b9ead1a9f224409576c51cd30153fe9a467f01 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <mikhaylovnikitka@gmail.com>
Date: Fri, 14 May 2021 00:12:36 +0300
Subject: [PATCH 20/76] Update CHANGELOG.md

Co-authored-by: Azat Khuzhin <a3at.mail@gmail.com>
---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e5e6b692bb0..f3dd92f4d32 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -23,7 +23,7 @@
 * Improved performance of `intDiv` by dynamic dispatch for AVX2. This closes [#22314](https://github.com/ClickHouse/ClickHouse/issues/22314). [#23000](https://github.com/ClickHouse/ClickHouse/pull/23000) ([alexey-milovidov](https://github.com/alexey-milovidov)).
 * Improved performance of reading from `ArrowStream` input format for sources other then local file (e.g. URL). [#22673](https://github.com/ClickHouse/ClickHouse/pull/22673) ([nvartolomei](https://github.com/nvartolomei)).
 * Disabled compression by default when interacting with localhost (with clickhouse-client or server to server with distributed queries) via native protocol. It may improve performance of some import/export operations. This closes [#22234](https://github.com/ClickHouse/ClickHouse/issues/22234). [#22237](https://github.com/ClickHouse/ClickHouse/pull/22237) ([alexey-milovidov](https://github.com/alexey-milovidov)).
-* Exclude values that does not belong to the shard from right part of IN section for distributed queries. [#21511](https://github.com/ClickHouse/ClickHouse/pull/21511) ([Azat Khuzhin](https://github.com/azat)).
+* Exclude values that does not belong to the shard from right part of IN section for distributed queries (under `optimize_skip_unused_shards_rewrite_in`, enabled by default, since it still requires `optimize_skip_unused_shards`). [#21511](https://github.com/ClickHouse/ClickHouse/pull/21511) ([Azat Khuzhin](https://github.com/azat)).
 * Improved performance of reading a subset of columns with File-like table engine and column-oriented format like Parquet, Arrow or ORC. This closes [#issue:20129](https://github.com/ClickHouse/ClickHouse/issues/20129). [#21302](https://github.com/ClickHouse/ClickHouse/pull/21302) ([keenwolf](https://github.com/keen-wolf)).
 
 #### Improvement

From ba754b51f95992317dbb93f2ac02061af96840bd Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <mikhaylovnikitka@gmail.com>
Date: Fri, 14 May 2021 00:54:43 +0300
Subject: [PATCH 21/76] Update CHANGELOG.md

Co-authored-by: Azat Khuzhin <a3at.mail@gmail.com>
---
 CHANGELOG.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f3dd92f4d32..c9d5511f383 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -86,7 +86,6 @@
 * Fixed very rare race condition on background cleanup of old blocks. It might cause a block not to be deduplicated if it's too close to the end of deduplication window. [#23301](https://github.com/ClickHouse/ClickHouse/pull/23301) ([tavplubix](https://github.com/tavplubix)).
 * Fixed very rare (distributed) race condition between creation and removal of ReplicatedMergeTree tables. It might cause exceptions like `node doesn't exist` on attempt to create replicated table. Fixes [#21419](https://github.com/ClickHouse/ClickHouse/issues/21419). [#23294](https://github.com/ClickHouse/ClickHouse/pull/23294) ([tavplubix](https://github.com/tavplubix)).
 * Fixed simple key dictionary from DDL creation if primary key is not first attribute. Fixes [#23236](https://github.com/ClickHouse/ClickHouse/issues/23236). [#23262](https://github.com/ClickHouse/ClickHouse/pull/23262) ([Maksim Kita](https://github.com/kitaisreal)).
-* Add type conversion for `optimize_skip_unused_shards_rewrite_in` (fixes `use-of-uninitialized-value` with `optimize_skip_unused_shards_rewrite_in`). [#23219](https://github.com/ClickHouse/ClickHouse/pull/23219) ([Azat Khuzhin](https://github.com/azat)).
 * Fixed reading from ODBC when there are many long column names in a table. Closes [#8853](https://github.com/ClickHouse/ClickHouse/issues/8853). [#23215](https://github.com/ClickHouse/ClickHouse/pull/23215) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * Fixed missing semicolon in exception message. The user may find this exception message unpleasant to read. [#23208](https://github.com/ClickHouse/ClickHouse/pull/23208) ([alexey-milovidov](https://github.com/alexey-milovidov)).
 * Fixed missing whitespace in some exception messages about `LowCardinality` type. [#23207](https://github.com/ClickHouse/ClickHouse/pull/23207) ([alexey-milovidov](https://github.com/alexey-milovidov)).

From 6c064d62e92496c5eba383261576233552156585 Mon Sep 17 00:00:00 2001
From: feng lv <fenglv15@mails.ucas.ac.cn>
Date: Wed, 12 May 2021 12:29:43 +0000
Subject: [PATCH 22/76] add rangeSum udaf

fix
---
 .../AggregateFunctionRangeSum.cpp             |  56 +++++
 .../AggregateFunctionRangeSum.h               | 194 ++++++++++++++++++
 src/AggregateFunctions/Helpers.h              |  11 +
 .../registerAggregateFunctions.cpp            |   3 +
 4 files changed, 264 insertions(+)
 create mode 100644 src/AggregateFunctions/AggregateFunctionRangeSum.cpp
 create mode 100644 src/AggregateFunctions/AggregateFunctionRangeSum.h

diff --git a/src/AggregateFunctions/AggregateFunctionRangeSum.cpp b/src/AggregateFunctions/AggregateFunctionRangeSum.cpp
new file mode 100644
index 00000000000..4584eea7a39
--- /dev/null
+++ b/src/AggregateFunctions/AggregateFunctionRangeSum.cpp
@@ -0,0 +1,56 @@
+#include <AggregateFunctions/AggregateFunctionFactory.h>
+#include <AggregateFunctions/AggregateFunctionRangeSum.h>
+#include <AggregateFunctions/FactoryHelpers.h>
+#include <AggregateFunctions/Helpers.h>
+#include <DataTypes/DataTypeDate.h>
+#include <DataTypes/DataTypeDateTime.h>
+
+#include <ext/range.h>
+
+
+namespace DB
+{
+namespace ErrorCodes
+{
+    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+}
+
+namespace
+{
+    template <template <typename> class Data>
+    AggregateFunctionPtr createAggregateFunctionRangeSum(const std::string & name, const DataTypes & arguments, const Array &)
+    {
+        if (arguments.size() != 2)
+            throw Exception(
+                "Aggregate function " + name + " requires two timestamps argument.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
+
+        for (const auto i : ext::range(1, arguments.size()))
+        {
+            const auto * arg = arguments[i].get();
+            if (!WhichDataType(arg).isNativeUInt())
+                throw Exception{
+                    "Illegal type " + arg->getName() + " of argument " + toString(i + 1) + " of aggregate function " + name
+                        + ", must be Native Unsigned Number",
+                    ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
+        }
+
+        AggregateFunctionPtr res(createWithNativeUnsignedIntegerType<AggregateFunctionRangeSum, Data>(*arguments[0], arguments));
+
+        if (res)
+            return res;
+
+        throw Exception{
+            "Illegal type " + arguments.front().get()->getName() + " of first argument of aggregate function " + name
+                + ", must be Native Unsigned Number",
+            ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
+}
+
+}
+
+void registerAggregateFunctionRangeSum(AggregateFunctionFactory & factory)
+{
+    factory.registerFunction("rangeSum", createAggregateFunctionRangeSum<AggregateFunctionRangeSumData>);
+}
+
+}
diff --git a/src/AggregateFunctions/AggregateFunctionRangeSum.h b/src/AggregateFunctions/AggregateFunctionRangeSum.h
new file mode 100644
index 00000000000..66968bf6a3e
--- /dev/null
+++ b/src/AggregateFunctions/AggregateFunctionRangeSum.h
@@ -0,0 +1,194 @@
+#pragma once
+
+#include <unordered_set>
+#include <Columns/ColumnsNumber.h>
+#include <DataTypes/DataTypeDateTime.h>
+#include <DataTypes/DataTypesNumber.h>
+#include <IO/ReadHelpers.h>
+#include <IO/WriteHelpers.h>
+#include <Common/ArenaAllocator.h>
+#include <Common/assert_cast.h>
+
+#include <AggregateFunctions/AggregateFunctionNull.h>
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int BAD_ARGUMENTS;
+}
+
+template <typename T>
+struct AggregateFunctionRangeSumData
+{
+    using TimeGap = std::pair<T, T>;
+    using TimeGaps = PODArrayWithStackMemory<TimeGap, 64>;
+
+    bool sorted = false;
+
+    TimeGaps gap_list;
+
+    size_t size() const { return gap_list.size(); }
+
+    void add(T start, T end)
+    {
+        if (sorted && gap_list.size() > 0)
+        {
+            sorted = gap_list.back().first <= start;
+        }
+        gap_list.emplace_back(start, end);
+    }
+
+    void merge(const AggregateFunctionRangeSumData & other)
+    {
+        if (other.gap_list.empty())
+            return;
+
+        const auto size = gap_list.size();
+
+        gap_list.insert(std::begin(other.gap_list), std::end(other.gap_list));
+
+        /// either sort whole container or do so partially merging ranges afterwards
+        if (!sorted && !other.sorted)
+            std::stable_sort(std::begin(gap_list), std::end(gap_list));
+        else
+        {
+            const auto begin = std::begin(gap_list);
+            const auto middle = std::next(begin, size);
+            const auto end = std::end(gap_list);
+
+            if (!sorted)
+                std::stable_sort(begin, middle);
+
+            if (!other.sorted)
+                std::stable_sort(middle, end);
+
+            std::inplace_merge(begin, middle, end);
+        }
+
+        sorted = true;
+    }
+
+    void sort()
+    {
+        if (!sorted)
+        {
+            std::stable_sort(std::begin(gap_list), std::end(gap_list));
+            sorted = true;
+        }
+    }
+
+    void serialize(WriteBuffer & buf) const
+    {
+        writeBinary(sorted, buf);
+        writeBinary(gap_list.size(), buf);
+
+        for (const auto & time_gap : gap_list)
+        {
+            writeBinary(time_gap.first, buf);
+            writeBinary(time_gap.second, buf);
+        }
+    }
+
+    void deserialize(ReadBuffer & buf)
+    {
+        readBinary(sorted, buf);
+
+        size_t size;
+        readBinary(size, buf);
+
+        gap_list.clear();
+        gap_list.reserve(size);
+
+        T start, end;
+
+        for (size_t i = 0; i < size; ++i)
+        {
+            readBinary(start, buf);
+            readBinary(end, buf);
+            gap_list.emplace_back(start, end);
+        }
+    }
+};
+
+template <typename T, typename Data>
+class AggregateFunctionRangeSum final : public IAggregateFunctionDataHelper<Data, AggregateFunctionRangeSum<T, Data>>
+{
+private:
+    UInt64 getRangeSum(Data & data) const
+    {
+        if (data.size() == 0)
+            return 0;
+
+        data.sort();
+
+        typename Data::TimeGaps merged;
+
+        for (const auto & gap : data.gap_list)
+        {
+            if (!merged.size() || merged.back().second < gap.first)
+                merged.emplace_back(gap.first, gap.second);
+            else
+                merged.back().second = std::max(merged.back().second, gap.second);
+        }
+
+        UInt64 res = 0;
+        for (const auto & gap : merged)
+        {
+            res += gap.second - gap.first;
+        }
+
+        return res;
+    }
+
+public:
+    String getName() const override { return "rangeSum"; }
+
+    explicit AggregateFunctionRangeSum(const DataTypes & arguments)
+        : IAggregateFunctionDataHelper<Data, AggregateFunctionRangeSum<T, Data>>(arguments, {})
+    {
+    }
+
+    DataTypePtr getReturnType() const override { return std::make_shared<DataTypeUInt64>(); }
+
+    bool allocatesMemoryInArena() const override { return false; }
+
+    AggregateFunctionPtr getOwnNullAdapter(
+        const AggregateFunctionPtr & nested_function,
+        const DataTypes & arguments,
+        const Array & params,
+        const AggregateFunctionProperties & /*properties*/) const override
+    {
+        return std::make_shared<AggregateFunctionNullVariadic<false, false, false>>(nested_function, arguments, params);
+    }
+
+    void add(AggregateDataPtr __restrict place, const IColumn ** columns, const size_t row_num, Arena *) const override
+    {
+        auto start = assert_cast<const ColumnVector<T> *>(columns[0])->getData()[row_num];
+        auto end = assert_cast<const ColumnVector<T> *>(columns[1])->getData()[row_num];
+        this->data(place).add(start, end);
+    }
+
+    void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena *) const override
+    {
+        this->data(place).merge(this->data(rhs));
+    }
+
+    void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf) const override
+    {
+        this->data(place).serialize(buf);
+    }
+
+    void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, Arena *) const override
+    {
+        this->data(place).deserialize(buf);
+    }
+
+    void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override
+    {
+        assert_cast<ColumnUInt64 &>(to).getData().push_back(getRangeSum(this->data(place)));
+    }
+};
+
+}
diff --git a/src/AggregateFunctions/Helpers.h b/src/AggregateFunctions/Helpers.h
index d1f1830eb27..c2f1434b515 100644
--- a/src/AggregateFunctions/Helpers.h
+++ b/src/AggregateFunctions/Helpers.h
@@ -114,6 +114,17 @@ static IAggregateFunction * createWithUnsignedIntegerType(const IDataType & argu
     return nullptr;
 }
 
+template <template <typename, typename> class AggregateFunctionTemplate, template <typename> class Data, typename... TArgs>
+static IAggregateFunction * createWithNativeUnsignedIntegerType(const IDataType & argument_type, TArgs &&... args)
+{
+    WhichDataType which(argument_type);
+    if (which.idx == TypeIndex::UInt8) return new AggregateFunctionTemplate<UInt8, Data<UInt8>>(std::forward<TArgs>(args)...);
+    if (which.idx == TypeIndex::UInt16) return new AggregateFunctionTemplate<UInt16, Data<UInt16>>(std::forward<TArgs>(args)...);
+    if (which.idx == TypeIndex::UInt32) return new AggregateFunctionTemplate<UInt32, Data<UInt32>>(std::forward<TArgs>(args)...);
+    if (which.idx == TypeIndex::UInt64) return new AggregateFunctionTemplate<UInt64, Data<UInt64>>(std::forward<TArgs>(args)...);
+    return nullptr;
+}
+
 template <template <typename> class AggregateFunctionTemplate, typename... TArgs>
 static IAggregateFunction * createWithNumericBasedType(const IDataType & argument_type, TArgs && ... args)
 {
diff --git a/src/AggregateFunctions/registerAggregateFunctions.cpp b/src/AggregateFunctions/registerAggregateFunctions.cpp
index 7663eb55955..0a94424dbbc 100644
--- a/src/AggregateFunctions/registerAggregateFunctions.cpp
+++ b/src/AggregateFunctions/registerAggregateFunctions.cpp
@@ -62,6 +62,7 @@ void registerAggregateFunctionCombinatorDistinct(AggregateFunctionCombinatorFact
 
 void registerWindowFunctions(AggregateFunctionFactory & factory);
 
+void registerAggregateFunctionRangeSum(AggregateFunctionFactory &);
 
 void registerAggregateFunctions()
 {
@@ -111,6 +112,8 @@ void registerAggregateFunctions()
         registerAggregateFunctionStudentTTest(factory);
 
         registerWindowFunctions(factory);
+
+        registerAggregateFunctionRangeSum(factory);
     }
 
     {

From 5e4cde85400dd73b82f45ffc74eaafe7c0e46789 Mon Sep 17 00:00:00 2001
From: feng lv <fenglv15@mails.ucas.ac.cn>
Date: Wed, 19 May 2021 10:24:53 +0000
Subject: [PATCH 23/76] fix

---
 .../AggregateFunctionRangeSum.cpp             |  8 +++----
 .../AggregateFunctionRangeSum.h               |  5 -----
 ...866_aggregate_function_range_sum.reference |  2 ++
 .../01866_aggregate_function_range_sum.sql    | 21 +++++++++++++++++++
 4 files changed, 27 insertions(+), 9 deletions(-)
 create mode 100644 tests/queries/0_stateless/01866_aggregate_function_range_sum.reference
 create mode 100644 tests/queries/0_stateless/01866_aggregate_function_range_sum.sql

diff --git a/src/AggregateFunctions/AggregateFunctionRangeSum.cpp b/src/AggregateFunctions/AggregateFunctionRangeSum.cpp
index 4584eea7a39..0c1cfd289a6 100644
--- a/src/AggregateFunctions/AggregateFunctionRangeSum.cpp
+++ b/src/AggregateFunctions/AggregateFunctionRangeSum.cpp
@@ -29,10 +29,10 @@ namespace
         {
             const auto * arg = arguments[i].get();
             if (!WhichDataType(arg).isNativeUInt())
-                throw Exception{
+                throw Exception(
                     "Illegal type " + arg->getName() + " of argument " + toString(i + 1) + " of aggregate function " + name
                         + ", must be Native Unsigned Number",
-                    ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
+                    ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
         }
 
         AggregateFunctionPtr res(createWithNativeUnsignedIntegerType<AggregateFunctionRangeSum, Data>(*arguments[0], arguments));
@@ -40,10 +40,10 @@ namespace
         if (res)
             return res;
 
-        throw Exception{
+        throw Exception(
             "Illegal type " + arguments.front().get()->getName() + " of first argument of aggregate function " + name
                 + ", must be Native Unsigned Number",
-            ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
+            ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
 }
 
 }
diff --git a/src/AggregateFunctions/AggregateFunctionRangeSum.h b/src/AggregateFunctions/AggregateFunctionRangeSum.h
index 66968bf6a3e..616f974462c 100644
--- a/src/AggregateFunctions/AggregateFunctionRangeSum.h
+++ b/src/AggregateFunctions/AggregateFunctionRangeSum.h
@@ -14,11 +14,6 @@
 namespace DB
 {
 
-namespace ErrorCodes
-{
-    extern const int BAD_ARGUMENTS;
-}
-
 template <typename T>
 struct AggregateFunctionRangeSumData
 {
diff --git a/tests/queries/0_stateless/01866_aggregate_function_range_sum.reference b/tests/queries/0_stateless/01866_aggregate_function_range_sum.reference
new file mode 100644
index 00000000000..a830f800286
--- /dev/null
+++ b/tests/queries/0_stateless/01866_aggregate_function_range_sum.reference
@@ -0,0 +1,2 @@
+a	5
+b	8
diff --git a/tests/queries/0_stateless/01866_aggregate_function_range_sum.sql b/tests/queries/0_stateless/01866_aggregate_function_range_sum.sql
new file mode 100644
index 00000000000..cd365e4c2fc
--- /dev/null
+++ b/tests/queries/0_stateless/01866_aggregate_function_range_sum.sql
@@ -0,0 +1,21 @@
+DROP TABLE IF EXISTS range;
+
+CREATE TABLE range
+(
+    `id` String,
+    `start` UInt64,
+    `end` UInt64
+)
+ENGINE = MergeTree
+ORDER BY start;
+
+INSERT INTO range VALUES ('a', 1, 3), ('a', 2, 4), ('a', 5, 6), ('a', 5, 7), ('b', 10, 12), ('b', 13, 19), ('b', 14, 16);
+
+SELECT
+    id,
+    rangeSum(start, end)
+FROM range
+GROUP BY id
+ORDER BY id;
+
+DROP TABLE range;

From 33f816d28d0b26c2f5bbbf88f0b0afc11846fa21 Mon Sep 17 00:00:00 2001
From: Ivan Lezhankin <ilezhankin@yandex-team.ru>
Date: Wed, 19 May 2021 16:20:23 +0300
Subject: [PATCH 24/76] Allow empty HTTP headers

---
 src/Common/StringUtils/StringUtils.h                    | 2 +-
 src/Server/HTTP/ReadHeaders.cpp                         | 3 ---
 tests/queries/0_stateless/01399_http_request_headers.sh | 1 +
 3 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/Common/StringUtils/StringUtils.h b/src/Common/StringUtils/StringUtils.h
index cb2227f01a8..20c0a5ca380 100644
--- a/src/Common/StringUtils/StringUtils.h
+++ b/src/Common/StringUtils/StringUtils.h
@@ -123,7 +123,7 @@ inline bool isWhitespaceASCII(char c)
 /// Since |isWhiteSpaceASCII()| is used inside algorithms it's easier to implement another function than add extra argument.
 inline bool isWhitespaceASCIIOneLine(char c)
 {
-    return c == ' ' || c == '\t' || c == '\r' || c == '\f' || c == '\v';
+    return c == ' ' || c == '\t' || c == '\f' || c == '\v';
 }
 
 inline bool isControlASCII(char c)
diff --git a/src/Server/HTTP/ReadHeaders.cpp b/src/Server/HTTP/ReadHeaders.cpp
index 2fc2de8321a..b7057501064 100644
--- a/src/Server/HTTP/ReadHeaders.cpp
+++ b/src/Server/HTTP/ReadHeaders.cpp
@@ -68,9 +68,6 @@ void readHeaders(
         if (in.eof())
             throw Poco::Net::MessageException("Field is invalid");
 
-        if (value.empty())
-            throw Poco::Net::MessageException("Field value is empty");
-
         if (ch == '\n')
             throw Poco::Net::MessageException("No CRLF found");
 
diff --git a/tests/queries/0_stateless/01399_http_request_headers.sh b/tests/queries/0_stateless/01399_http_request_headers.sh
index 9b07f018230..f06e7ffc32b 100755
--- a/tests/queries/0_stateless/01399_http_request_headers.sh
+++ b/tests/queries/0_stateless/01399_http_request_headers.sh
@@ -4,6 +4,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
+${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -H 'EmptyHeader;' -d 'SELECT 1'
 ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -H 'X-ClickHouse-User: default' -d 'SELECT 1'
 ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -H 'X-ClickHouse-User: header_test' -d 'SELECT 1' | grep -o 'Code: 516'
 ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -H 'X-ClickHouse-Key: ' -d 'SELECT 1'

From d6e357d78ce6d66036be4df5f06948b5a53f75dd Mon Sep 17 00:00:00 2001
From: Alexander Kuzmenkov <akuzm@yandex-team.ru>
Date: Wed, 19 May 2021 17:35:23 +0300
Subject: [PATCH 25/76] consolidate connection loss handling in fuzzer

---
 programs/client/Client.cpp | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp
index ccf92ebc419..7973fc05205 100644
--- a/programs/client/Client.cpp
+++ b/programs/client/Client.cpp
@@ -1366,6 +1366,27 @@ private:
             {
                 const auto * exception = server_exception ? server_exception.get() : client_exception.get();
                 fmt::print(stderr, "Error on processing query '{}': {}\n", ast_to_process->formatForErrorMessage(), exception->message());
+
+                // Try to reconnect after errors, for two reasons:
+                // 1. We might not have realized that the server died, e.g. if
+                //    it sent us a <Fatal> trace and closed connection properly.
+                // 2. The connection might have gotten into a wrong state and
+                //    the next query will get false positive about
+                //    "Unknown packet from server".
+                try
+                {
+                    connection->forceConnected(connection_parameters.timeouts);
+                }
+                catch (...)
+                {
+                    // Just report it, we'll terminate below.
+                    fmt::print(stderr,
+                        "Error while reconnecting to the  server: Code: {}: {}\n",
+                        getCurrentExceptionCode(),
+                        getCurrentExceptionMessage(true));
+
+                    assert(!connection->isConnected());
+                }
             }
 
             if (!connection->isConnected())
@@ -1469,11 +1490,6 @@ private:
                 server_exception.reset();
                 client_exception.reset();
                 have_error = false;
-
-                // We have to reinitialize connection after errors, because it
-                // might have gotten into a wrong state and we'll get false
-                // positives about "Unknown packet from server".
-                connection->forceConnected(connection_parameters.timeouts);
             }
             else if (ast_to_process->formatForErrorMessage().size() > 500)
             {

From 9de73997379a2e3200cfd8e6374ca124186e55b1 Mon Sep 17 00:00:00 2001
From: feng lv <fenglv15@mails.ucas.ac.cn>
Date: Wed, 19 May 2021 13:32:53 +0000
Subject: [PATCH 26/76] update

fix test
---
 .../AggregateFunctionRangeSum.cpp             | 56 -------------
 .../AggregateFunctionSegmentLengthSum.cpp     | 63 +++++++++++++++
 ....h => AggregateFunctionSegmentLengthSum.h} | 79 ++++++++++---------
 src/AggregateFunctions/Helpers.h              | 17 ++--
 .../registerAggregateFunctions.cpp            |  4 +-
 .../01866_aggregate_function_range_sum.sql    | 21 -----
 ...ate_function_segment_length_sum.reference} |  0
 ..._aggregate_function_segment_length_sum.sql | 21 +++++
 8 files changed, 138 insertions(+), 123 deletions(-)
 delete mode 100644 src/AggregateFunctions/AggregateFunctionRangeSum.cpp
 create mode 100644 src/AggregateFunctions/AggregateFunctionSegmentLengthSum.cpp
 rename src/AggregateFunctions/{AggregateFunctionRangeSum.h => AggregateFunctionSegmentLengthSum.h} (61%)
 delete mode 100644 tests/queries/0_stateless/01866_aggregate_function_range_sum.sql
 rename tests/queries/0_stateless/{01866_aggregate_function_range_sum.reference => 01866_aggregate_function_segment_length_sum.reference} (100%)
 create mode 100644 tests/queries/0_stateless/01866_aggregate_function_segment_length_sum.sql

diff --git a/src/AggregateFunctions/AggregateFunctionRangeSum.cpp b/src/AggregateFunctions/AggregateFunctionRangeSum.cpp
deleted file mode 100644
index 0c1cfd289a6..00000000000
--- a/src/AggregateFunctions/AggregateFunctionRangeSum.cpp
+++ /dev/null
@@ -1,56 +0,0 @@
-#include <AggregateFunctions/AggregateFunctionFactory.h>
-#include <AggregateFunctions/AggregateFunctionRangeSum.h>
-#include <AggregateFunctions/FactoryHelpers.h>
-#include <AggregateFunctions/Helpers.h>
-#include <DataTypes/DataTypeDate.h>
-#include <DataTypes/DataTypeDateTime.h>
-
-#include <ext/range.h>
-
-
-namespace DB
-{
-namespace ErrorCodes
-{
-    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
-    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
-}
-
-namespace
-{
-    template <template <typename> class Data>
-    AggregateFunctionPtr createAggregateFunctionRangeSum(const std::string & name, const DataTypes & arguments, const Array &)
-    {
-        if (arguments.size() != 2)
-            throw Exception(
-                "Aggregate function " + name + " requires two timestamps argument.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
-
-        for (const auto i : ext::range(1, arguments.size()))
-        {
-            const auto * arg = arguments[i].get();
-            if (!WhichDataType(arg).isNativeUInt())
-                throw Exception(
-                    "Illegal type " + arg->getName() + " of argument " + toString(i + 1) + " of aggregate function " + name
-                        + ", must be Native Unsigned Number",
-                    ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
-        }
-
-        AggregateFunctionPtr res(createWithNativeUnsignedIntegerType<AggregateFunctionRangeSum, Data>(*arguments[0], arguments));
-
-        if (res)
-            return res;
-
-        throw Exception(
-            "Illegal type " + arguments.front().get()->getName() + " of first argument of aggregate function " + name
-                + ", must be Native Unsigned Number",
-            ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
-}
-
-}
-
-void registerAggregateFunctionRangeSum(AggregateFunctionFactory & factory)
-{
-    factory.registerFunction("rangeSum", createAggregateFunctionRangeSum<AggregateFunctionRangeSumData>);
-}
-
-}
diff --git a/src/AggregateFunctions/AggregateFunctionSegmentLengthSum.cpp b/src/AggregateFunctions/AggregateFunctionSegmentLengthSum.cpp
new file mode 100644
index 00000000000..fcd4f3aaea0
--- /dev/null
+++ b/src/AggregateFunctions/AggregateFunctionSegmentLengthSum.cpp
@@ -0,0 +1,63 @@
+#include <AggregateFunctions/AggregateFunctionFactory.h>
+#include <AggregateFunctions/AggregateFunctionSegmentLengthSum.h>
+#include <AggregateFunctions/FactoryHelpers.h>
+#include <AggregateFunctions/Helpers.h>
+#include <DataTypes/DataTypeDate.h>
+#include <DataTypes/DataTypeDateTime.h>
+
+#include <ext/range.h>
+
+
+namespace DB
+{
+namespace ErrorCodes
+{
+    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+}
+
+namespace
+{
+    template <template <typename> class Data>
+    AggregateFunctionPtr createAggregateFunctionSegmentLengthSum(const std::string & name, const DataTypes & arguments, const Array &)
+    {
+        if (arguments.size() != 2)
+            throw Exception(
+                "Aggregate function " + name + " requires two timestamps argument.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
+
+        auto args = {arguments[0].get(), arguments[1].get()};
+
+        if (WhichDataType{args.begin()[0]}.idx != WhichDataType{args.begin()[1]}.idx)
+            throw Exception(
+                "Illegal type " + args.begin()[0]->getName() + " and " + args.begin()[1]->getName() + " of arguments of aggregate function "
+                    + name + ", there two arguments should have same DataType",
+                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+
+        for (const auto & arg : args)
+        {
+            if (!isNativeNumber(arg) && !isDateOrDateTime(arg))
+                throw Exception(
+                    "Illegal type " + arg->getName() + " of argument of aggregate function " + name
+                        + ", must be Number, Date, DateTime or DateTime64",
+                    ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+        }
+
+        AggregateFunctionPtr res(createWithBasicNumberOrDateOrDateTime<AggregateFunctionSegmentLengthSum, Data>(*arguments[0], arguments));
+
+        if (res)
+            return res;
+
+        throw Exception(
+            "Illegal type " + arguments.front().get()->getName() + " of first argument of aggregate function " + name
+                + ", must be Native Unsigned Number",
+            ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+}
+
+}
+
+void registerAggregateFunctionSegmentLengthSum(AggregateFunctionFactory & factory)
+{
+    factory.registerFunction("segmentLengthSum", createAggregateFunctionSegmentLengthSum<AggregateFunctionSegmentLengthSumData>);
+}
+
+}
diff --git a/src/AggregateFunctions/AggregateFunctionRangeSum.h b/src/AggregateFunctions/AggregateFunctionSegmentLengthSum.h
similarity index 61%
rename from src/AggregateFunctions/AggregateFunctionRangeSum.h
rename to src/AggregateFunctions/AggregateFunctionSegmentLengthSum.h
index 616f974462c..ff388316289 100644
--- a/src/AggregateFunctions/AggregateFunctionRangeSum.h
+++ b/src/AggregateFunctions/AggregateFunctionSegmentLengthSum.h
@@ -15,43 +15,43 @@ namespace DB
 {
 
 template <typename T>
-struct AggregateFunctionRangeSumData
+struct AggregateFunctionSegmentLengthSumData
 {
-    using TimeGap = std::pair<T, T>;
-    using TimeGaps = PODArrayWithStackMemory<TimeGap, 64>;
+    using Segment = std::pair<T, T>;
+    using Segments = PODArrayWithStackMemory<Segment, 64>;
 
     bool sorted = false;
 
-    TimeGaps gap_list;
+    Segments segments;
 
-    size_t size() const { return gap_list.size(); }
+    size_t size() const { return segments.size(); }
 
     void add(T start, T end)
     {
-        if (sorted && gap_list.size() > 0)
+        if (sorted && segments.size() > 0)
         {
-            sorted = gap_list.back().first <= start;
+            sorted = segments.back().first <= start;
         }
-        gap_list.emplace_back(start, end);
+        segments.emplace_back(start, end);
     }
 
-    void merge(const AggregateFunctionRangeSumData & other)
+    void merge(const AggregateFunctionSegmentLengthSumData & other)
     {
-        if (other.gap_list.empty())
+        if (other.segments.empty())
             return;
 
-        const auto size = gap_list.size();
+        const auto size = segments.size();
 
-        gap_list.insert(std::begin(other.gap_list), std::end(other.gap_list));
+        segments.insert(std::begin(other.segments), std::end(other.segments));
 
         /// either sort whole container or do so partially merging ranges afterwards
         if (!sorted && !other.sorted)
-            std::stable_sort(std::begin(gap_list), std::end(gap_list));
+            std::stable_sort(std::begin(segments), std::end(segments));
         else
         {
-            const auto begin = std::begin(gap_list);
+            const auto begin = std::begin(segments);
             const auto middle = std::next(begin, size);
-            const auto end = std::end(gap_list);
+            const auto end = std::end(segments);
 
             if (!sorted)
                 std::stable_sort(begin, middle);
@@ -69,7 +69,7 @@ struct AggregateFunctionRangeSumData
     {
         if (!sorted)
         {
-            std::stable_sort(std::begin(gap_list), std::end(gap_list));
+            std::stable_sort(std::begin(segments), std::end(segments));
             sorted = true;
         }
     }
@@ -77,9 +77,9 @@ struct AggregateFunctionRangeSumData
     void serialize(WriteBuffer & buf) const
     {
         writeBinary(sorted, buf);
-        writeBinary(gap_list.size(), buf);
+        writeBinary(segments.size(), buf);
 
-        for (const auto & time_gap : gap_list)
+        for (const auto & time_gap : segments)
         {
             writeBinary(time_gap.first, buf);
             writeBinary(time_gap.second, buf);
@@ -93,8 +93,8 @@ struct AggregateFunctionRangeSumData
         size_t size;
         readBinary(size, buf);
 
-        gap_list.clear();
-        gap_list.reserve(size);
+        segments.clear();
+        segments.reserve(size);
 
         T start, end;
 
@@ -102,46 +102,47 @@ struct AggregateFunctionRangeSumData
         {
             readBinary(start, buf);
             readBinary(end, buf);
-            gap_list.emplace_back(start, end);
+            segments.emplace_back(start, end);
         }
     }
 };
 
 template <typename T, typename Data>
-class AggregateFunctionRangeSum final : public IAggregateFunctionDataHelper<Data, AggregateFunctionRangeSum<T, Data>>
+class AggregateFunctionSegmentLengthSum final : public IAggregateFunctionDataHelper<Data, AggregateFunctionSegmentLengthSum<T, Data>>
 {
 private:
-    UInt64 getRangeSum(Data & data) const
+    UInt64 getSegmentLengthSum(Data & data) const
     {
         if (data.size() == 0)
             return 0;
 
         data.sort();
 
-        typename Data::TimeGaps merged;
-
-        for (const auto & gap : data.gap_list)
-        {
-            if (!merged.size() || merged.back().second < gap.first)
-                merged.emplace_back(gap.first, gap.second);
-            else
-                merged.back().second = std::max(merged.back().second, gap.second);
-        }
-
         UInt64 res = 0;
-        for (const auto & gap : merged)
+
+        typename Data::Segment cur_segment = data.segments[0];
+
+        for (size_t i = 1; i < data.segments.size(); ++i)
         {
-            res += gap.second - gap.first;
+            if (cur_segment.second < data.segments[i].first)
+            {
+                res += cur_segment.second - cur_segment.first;
+                cur_segment = data.segments[i];
+            }
+            else
+                cur_segment.second = std::max(cur_segment.second, data.segments[i].second);
         }
 
+        res += cur_segment.second - cur_segment.first;
+
         return res;
     }
 
 public:
-    String getName() const override { return "rangeSum"; }
+    String getName() const override { return "segmentLengthSum"; }
 
-    explicit AggregateFunctionRangeSum(const DataTypes & arguments)
-        : IAggregateFunctionDataHelper<Data, AggregateFunctionRangeSum<T, Data>>(arguments, {})
+    explicit AggregateFunctionSegmentLengthSum(const DataTypes & arguments)
+        : IAggregateFunctionDataHelper<Data, AggregateFunctionSegmentLengthSum<T, Data>>(arguments, {})
     {
     }
 
@@ -182,7 +183,7 @@ public:
 
     void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override
     {
-        assert_cast<ColumnUInt64 &>(to).getData().push_back(getRangeSum(this->data(place)));
+        assert_cast<ColumnUInt64 &>(to).getData().push_back(getSegmentLengthSum(this->data(place)));
     }
 };
 
diff --git a/src/AggregateFunctions/Helpers.h b/src/AggregateFunctions/Helpers.h
index c2f1434b515..e33b32585a0 100644
--- a/src/AggregateFunctions/Helpers.h
+++ b/src/AggregateFunctions/Helpers.h
@@ -115,13 +115,20 @@ static IAggregateFunction * createWithUnsignedIntegerType(const IDataType & argu
 }
 
 template <template <typename, typename> class AggregateFunctionTemplate, template <typename> class Data, typename... TArgs>
-static IAggregateFunction * createWithNativeUnsignedIntegerType(const IDataType & argument_type, TArgs &&... args)
+static IAggregateFunction * createWithBasicNumberOrDateOrDateTime(const IDataType & argument_type, TArgs &&... args)
 {
     WhichDataType which(argument_type);
-    if (which.idx == TypeIndex::UInt8) return new AggregateFunctionTemplate<UInt8, Data<UInt8>>(std::forward<TArgs>(args)...);
-    if (which.idx == TypeIndex::UInt16) return new AggregateFunctionTemplate<UInt16, Data<UInt16>>(std::forward<TArgs>(args)...);
-    if (which.idx == TypeIndex::UInt32) return new AggregateFunctionTemplate<UInt32, Data<UInt32>>(std::forward<TArgs>(args)...);
-    if (which.idx == TypeIndex::UInt64) return new AggregateFunctionTemplate<UInt64, Data<UInt64>>(std::forward<TArgs>(args)...);
+#define DISPATCH(TYPE) \
+    if (which.idx == TypeIndex::TYPE) \
+        return new AggregateFunctionTemplate<TYPE, Data<TYPE>>(std::forward<TArgs>(args)...);
+    FOR_BASIC_NUMERIC_TYPES(DISPATCH)
+#undef DISPATCH
+
+    if (which.idx == TypeIndex::Date)
+        return new AggregateFunctionTemplate<UInt16, Data<UInt16>>(std::forward<TArgs>(args)...);
+    if (which.idx == TypeIndex::DateTime)
+        return new AggregateFunctionTemplate<UInt32, Data<UInt32>>(std::forward<TArgs>(args)...);
+
     return nullptr;
 }
 
diff --git a/src/AggregateFunctions/registerAggregateFunctions.cpp b/src/AggregateFunctions/registerAggregateFunctions.cpp
index 0a94424dbbc..cf757cada4e 100644
--- a/src/AggregateFunctions/registerAggregateFunctions.cpp
+++ b/src/AggregateFunctions/registerAggregateFunctions.cpp
@@ -62,7 +62,7 @@ void registerAggregateFunctionCombinatorDistinct(AggregateFunctionCombinatorFact
 
 void registerWindowFunctions(AggregateFunctionFactory & factory);
 
-void registerAggregateFunctionRangeSum(AggregateFunctionFactory &);
+void registerAggregateFunctionSegmentLengthSum(AggregateFunctionFactory &);
 
 void registerAggregateFunctions()
 {
@@ -113,7 +113,7 @@ void registerAggregateFunctions()
 
         registerWindowFunctions(factory);
 
-        registerAggregateFunctionRangeSum(factory);
+        registerAggregateFunctionSegmentLengthSum(factory);
     }
 
     {
diff --git a/tests/queries/0_stateless/01866_aggregate_function_range_sum.sql b/tests/queries/0_stateless/01866_aggregate_function_range_sum.sql
deleted file mode 100644
index cd365e4c2fc..00000000000
--- a/tests/queries/0_stateless/01866_aggregate_function_range_sum.sql
+++ /dev/null
@@ -1,21 +0,0 @@
-DROP TABLE IF EXISTS range;
-
-CREATE TABLE range
-(
-    `id` String,
-    `start` UInt64,
-    `end` UInt64
-)
-ENGINE = MergeTree
-ORDER BY start;
-
-INSERT INTO range VALUES ('a', 1, 3), ('a', 2, 4), ('a', 5, 6), ('a', 5, 7), ('b', 10, 12), ('b', 13, 19), ('b', 14, 16);
-
-SELECT
-    id,
-    rangeSum(start, end)
-FROM range
-GROUP BY id
-ORDER BY id;
-
-DROP TABLE range;
diff --git a/tests/queries/0_stateless/01866_aggregate_function_range_sum.reference b/tests/queries/0_stateless/01866_aggregate_function_segment_length_sum.reference
similarity index 100%
rename from tests/queries/0_stateless/01866_aggregate_function_range_sum.reference
rename to tests/queries/0_stateless/01866_aggregate_function_segment_length_sum.reference
diff --git a/tests/queries/0_stateless/01866_aggregate_function_segment_length_sum.sql b/tests/queries/0_stateless/01866_aggregate_function_segment_length_sum.sql
new file mode 100644
index 00000000000..ed2c8b66bd7
--- /dev/null
+++ b/tests/queries/0_stateless/01866_aggregate_function_segment_length_sum.sql
@@ -0,0 +1,21 @@
+DROP TABLE IF EXISTS segment;
+
+CREATE TABLE segment
+(
+    `id` String,
+    `start` UInt64,
+    `end` UInt64
+)
+ENGINE = MergeTree
+ORDER BY start;
+
+INSERT INTO segment VALUES ('a', 1, 3), ('a', 2, 4), ('a', 5, 6), ('a', 5, 7), ('b', 10, 12), ('b', 13, 19), ('b', 14, 16);
+
+SELECT
+    id,
+    segmentLengthSum(start, end)
+FROM segment
+GROUP BY id
+ORDER BY id;
+
+DROP TABLE segment;

From 54ee3a4e48d9ee0c127eb320e529df9c0b97effe Mon Sep 17 00:00:00 2001
From: Alexander Kuzmenkov <akuzm@yandex-team.ru>
Date: Wed, 19 May 2021 18:01:11 +0300
Subject: [PATCH 27/76] space

---
 programs/client/Client.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp
index 7973fc05205..098f7e689c5 100644
--- a/programs/client/Client.cpp
+++ b/programs/client/Client.cpp
@@ -1381,7 +1381,7 @@ private:
                 {
                     // Just report it, we'll terminate below.
                     fmt::print(stderr,
-                        "Error while reconnecting to the  server: Code: {}: {}\n",
+                        "Error while reconnecting to the server: Code: {}: {}\n",
                         getCurrentExceptionCode(),
                         getCurrentExceptionMessage(true));
 

From 1006a970f74308a9ee5065b3a022bbd31dd78b09 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Thu, 20 May 2021 09:30:13 +0300
Subject: [PATCH 28/76] Remove AutoArray

---
 src/Common/AutoArray.h                        | 309 ------------------
 src/Common/examples/CMakeLists.txt            |   3 -
 src/Common/examples/auto_array.cpp            | 197 -----------
 .../MergingSortedBlockInputStream.h           |   1 -
 src/DataStreams/TTLAggregationAlgorithm.h     |   2 +-
 src/Functions/partitionId.cpp                 |   2 +-
 src/Interpreters/ActionsVisitor.cpp           |   5 +-
 src/Interpreters/Set.cpp                      |   1 -
 src/Interpreters/examples/hash_map.cpp        |  27 --
 .../Algorithms/SummingSortedAlgorithm.cpp     |   4 +
 .../Algorithms/SummingSortedAlgorithm.h       |   4 +-
 src/Storages/MergeTree/IMergeTreeDataPart.h   |   1 -
 .../MergeTreeBaseSelectProcessor.cpp          |  15 +-
 src/Storages/MergeTree/MergeTreeData.cpp      |   2 +-
 .../MergeTree/MergeTreeDataPartInMemory.cpp   |   2 +-
 .../MergeTree/MergeTreeDataWriter.cpp         |   4 +-
 src/Storages/MergeTree/MergeTreeDataWriter.h  |   5 +-
 src/Storages/MergeTree/MergeTreePartition.h   |   9 +-
 18 files changed, 30 insertions(+), 563 deletions(-)
 delete mode 100644 src/Common/AutoArray.h
 delete mode 100644 src/Common/examples/auto_array.cpp

diff --git a/src/Common/AutoArray.h b/src/Common/AutoArray.h
deleted file mode 100644
index 1df8bde8c12..00000000000
--- a/src/Common/AutoArray.h
+++ /dev/null
@@ -1,309 +0,0 @@
-#pragma once
-
-#include <cstddef>
-#include <cstdlib>
-
-#include <Common/Exception.h>
-#include <Common/formatReadable.h>
-
-
-namespace DB
-{
-
-namespace ErrorCodes
-{
-    extern const int CANNOT_ALLOCATE_MEMORY;
-}
-
-/** An array of (almost) unchangeable size:
-  *  the size is specified in the constructor;
-  *  `resize` method removes old data, and necessary only for
-  *  so that you can first create an empty object using the default constructor,
-  *  and then decide on the size.
-  *
-  * There is a possibility to not initialize elements by default, but create them inplace.
-  * Member destructors are called automatically.
-  *
-  * `sizeof` is equal to the size of one pointer.
-  *
-  * Not exception-safe.
-  *
-  * Copying is supported via assign() method. Moving empties the original object.
-  * That is, it is inconvenient to use this array in many cases.
-  *
-  * Designed for situations in which many arrays of the same small size are created,
-  *  but the size is not known at compile time.
-  * Also gives a significant advantage in cases where it is important that `sizeof` is minimal.
-  * For example, if arrays are put in an open-addressing hash table with inplace storage of values (like HashMap)
-  *
-  * In this case, compared to std::vector:
-  * - for arrays of 1 element size - an advantage of about 2 times;
-  * - for arrays of 5 elements - an advantage of about 1.5 times
-  *   (DB::Field, containing UInt64 and String, used as T);
-  */
-
-const size_t empty_auto_array_helper = 0;
-
-template <typename T>
-class AutoArray
-{
-public:
-    /// For deferred creation.
-    AutoArray()
-    {
-        setEmpty();
-    }
-
-    explicit AutoArray(size_t size_)
-    {
-        init(size_, false);
-    }
-
-    /** Initializes all elements with a copy constructor with the `value` parameter.
-      */
-    AutoArray(size_t size_, const T & value)
-    {
-        init(size_, true);
-
-        for (size_t i = 0; i < size_; ++i)
-        {
-            new (place(i)) T(value);
-        }
-    }
-
-    /** `resize` removes all existing items.
-      */
-    void resize(size_t size_, bool dont_init_elems = false)
-    {
-        uninit();
-        init(size_, dont_init_elems);
-    }
-
-    /** Move operations.
-      */
-    AutoArray(AutoArray && src)
-    {
-        if (this == &src)
-            return;
-        setEmpty();
-        data_ptr = src.data_ptr;
-        src.setEmpty();
-    }
-
-    AutoArray & operator= (AutoArray && src)
-    {
-        if (this == &src)
-            return *this;
-        uninit();
-        data_ptr = src.data_ptr;
-        src.setEmpty();
-
-        return *this;
-    }
-
-    ~AutoArray()
-    {
-        uninit();
-    }
-
-    size_t size() const
-    {
-        return m_size();
-    }
-
-    bool empty() const
-    {
-        return size() == 0;
-    }
-
-    void clear()
-    {
-        uninit();
-        setEmpty();
-    }
-
-    template <typename It>
-    void assign(It from_begin, It from_end)
-    {
-        uninit();
-
-        size_t size = from_end - from_begin;
-        init(size, /* dont_init_elems = */ true);
-
-        It it = from_begin;
-        for (size_t i = 0; i < size; ++i, ++it)
-            new (place(i)) T(*it);
-    }
-
-    void assign(const AutoArray & from)
-    {
-        assign(from.begin(), from.end());
-    }
-
-    /** You can read and modify elements using the [] operator
-      *  only if items were initialized
-      *  (that is, into the constructor was not passed DontInitElemsTag,
-      *   or you initialized them using `place` and `placement new`).
-      */
-    T & operator[](size_t i)
-    {
-        return elem(i);
-    }
-
-    const T & operator[](size_t i) const
-    {
-        return elem(i);
-    }
-
-    T * data()
-    {
-        return elemPtr(0);
-    }
-
-    const T * data() const
-    {
-        return elemPtr(0);
-    }
-
-    /** Get the piece of memory in which the element should be located.
-      * The function is intended to initialize an element,
-      *  which has not yet been initialized
-      * new (arr.place(i)) T(args);
-      */
-    char * place(size_t i)
-    {
-        return data_ptr + sizeof(T) * i;
-    }
-
-    using iterator = T *;
-    using const_iterator = const T *;
-
-    iterator begin() { return elemPtr(0); }
-    iterator end() { return elemPtr(size()); }
-
-    const_iterator begin() const { return elemPtr(0); }
-    const_iterator end() const { return elemPtr(size()); }
-
-    bool operator== (const AutoArray<T> & rhs) const
-    {
-        size_t s = size();
-
-        if (s != rhs.size())
-            return false;
-
-        for (size_t i = 0; i < s; ++i)
-            if (elem(i) != rhs.elem(i))
-                return false;
-
-        return true;
-    }
-
-    bool operator!= (const AutoArray<T> & rhs) const
-    {
-        return !(*this == rhs);
-    }
-
-    bool operator< (const AutoArray<T> & rhs) const
-    {
-        size_t s = size();
-        size_t rhs_s = rhs.size();
-
-        if (s < rhs_s)
-            return true;
-        if (s > rhs_s)
-            return false;
-
-        for (size_t i = 0; i < s; ++i)
-        {
-            if (elem(i) < rhs.elem(i))
-                return true;
-            if (elem(i) > rhs.elem(i))
-                return false;
-        }
-
-        return false;
-    }
-
-private:
-    static constexpr size_t alignment = alignof(T);
-    /// Bytes allocated to store size of array before data. It is padded to have minimum size as alignment.
-    /// Padding is at left and the size is stored at right (just before the first data element).
-    static constexpr size_t prefix_size = std::max(sizeof(size_t), alignment);
-
-    char * data_ptr;
-
-    size_t & m_size()
-    {
-        return reinterpret_cast<size_t *>(data_ptr)[-1];
-    }
-
-    size_t m_size() const
-    {
-        return reinterpret_cast<const size_t *>(data_ptr)[-1];
-    }
-
-    T * elemPtr(size_t i)
-    {
-        return reinterpret_cast<T *>(data_ptr) + i;
-    }
-
-    const T * elemPtr(size_t i) const
-    {
-        return reinterpret_cast<const T *>(data_ptr) + i;
-    }
-
-    T & elem(size_t i)
-    {
-        return *elemPtr(i);
-    }
-
-    const T & elem(size_t i) const
-    {
-        return *elemPtr(i);
-    }
-
-    void setEmpty()
-    {
-        data_ptr = const_cast<char *>(reinterpret_cast<const char *>(&empty_auto_array_helper)) + sizeof(size_t);
-    }
-
-    void init(size_t new_size, bool dont_init_elems)
-    {
-        if (!new_size)
-        {
-            setEmpty();
-            return;
-        }
-
-        void * new_data = nullptr;
-        int res = posix_memalign(&new_data, alignment, prefix_size + new_size * sizeof(T));
-        if (0 != res)
-            throwFromErrno(fmt::format("Cannot allocate memory (posix_memalign) {}.", ReadableSize(new_size)),
-                ErrorCodes::CANNOT_ALLOCATE_MEMORY, res);
-
-        data_ptr = static_cast<char *>(new_data);
-        data_ptr += prefix_size;
-
-        m_size() = new_size;
-
-        if (!dont_init_elems)
-            for (size_t i = 0; i < new_size; ++i)
-                new (place(i)) T();
-    }
-
-    void uninit()
-    {
-        size_t s = size();
-
-        if (s)
-        {
-            for (size_t i = 0; i < s; ++i)
-                elem(i).~T();
-
-            data_ptr -= prefix_size;
-            free(data_ptr);
-        }
-    }
-};
-
-}
diff --git a/src/Common/examples/CMakeLists.txt b/src/Common/examples/CMakeLists.txt
index c94ce580942..64d28fec5c2 100644
--- a/src/Common/examples/CMakeLists.txt
+++ b/src/Common/examples/CMakeLists.txt
@@ -7,9 +7,6 @@ endif()
 add_executable (sip_hash_perf sip_hash_perf.cpp)
 target_link_libraries (sip_hash_perf PRIVATE clickhouse_common_io)
 
-add_executable (auto_array auto_array.cpp)
-target_link_libraries (auto_array PRIVATE clickhouse_common_io)
-
 add_executable (small_table small_table.cpp)
 target_link_libraries (small_table PRIVATE clickhouse_common_io)
 
diff --git a/src/Common/examples/auto_array.cpp b/src/Common/examples/auto_array.cpp
deleted file mode 100644
index 9e9a18ffec9..00000000000
--- a/src/Common/examples/auto_array.cpp
+++ /dev/null
@@ -1,197 +0,0 @@
-#include <iostream>
-#include <iomanip>
-#include <map>
-
-#include <pcg_random.hpp>
-#include <Core/Field.h>
-#include <Common/HashTable/HashMap.h>
-#include <Common/AutoArray.h>
-#include <IO/WriteHelpers.h>
-
-#include <Common/Stopwatch.h>
-
-
-int main(int argc, char ** argv)
-{
-    pcg64 rng;
-
-    {
-        size_t n = 10;
-        using T = std::string;
-        DB::AutoArray<T> arr(n);
-
-        for (size_t i = 0; i < arr.size(); ++i)
-            arr[i] = "Hello, world! " + DB::toString(i);
-
-        for (auto & elem : arr)
-            std::cerr << elem << std::endl;
-    }
-
-    std::cerr << std::endl;
-
-    {
-        size_t n = 10;
-        using T = std::string;
-        using Arr = DB::AutoArray<T>;
-        Arr arr;
-
-        arr.resize(n);
-        for (size_t i = 0; i < arr.size(); ++i)
-            arr[i] = "Hello, world! " + DB::toString(i);
-
-        for (auto & elem : arr)
-            std::cerr << elem << std::endl;
-
-        std::cerr << std::endl;
-
-        Arr arr2 = std::move(arr);
-
-        std::cerr << arr.size() << ", " << arr2.size() << std::endl;  // NOLINT
-
-        for (auto & elem : arr2)
-            std::cerr << elem << std::endl;
-    }
-
-    std::cerr << std::endl;
-
-    {
-        size_t n = 10;
-        size_t keys = 10;
-        using T = std::string;
-        using Arr = DB::AutoArray<T>;
-        using Map = std::map<Arr, T>;
-        Map map;
-
-        for (size_t i = 0; i < keys; ++i)
-        {
-            Arr key(n);
-            for (size_t j = 0; j < n; ++j)
-                key[j] = DB::toString(rng());
-
-            map[std::move(key)] = "Hello, world! " + DB::toString(i);
-        }
-
-        for (const auto & kv : map)
-        {
-            std::cerr << "[";
-            for (size_t j = 0; j < n; ++j)
-                std::cerr << (j == 0 ? "" : ", ") << kv.first[j];
-            std::cerr << "]";
-
-            std::cerr << ":\t" << kv.second << std::endl;
-        }
-
-        std::cerr << std::endl;
-
-        Map map2 = std::move(map);
-
-        for (const auto & kv : map2)
-        {
-            std::cerr << "[";
-            for (size_t j = 0; j < n; ++j)
-                std::cerr << (j == 0 ? "" : ", ") << kv.first[j];
-            std::cerr << "]";
-
-            std::cerr << ":\t" << kv.second << std::endl;
-        }
-    }
-
-    std::cerr << std::endl;
-
-    {
-        size_t n = 10;
-        size_t keys = 10;
-        using T = std::string;
-        using Arr = DB::AutoArray<T>;
-        using Vec = std::vector<Arr>;
-        Vec vec;
-
-        for (size_t i = 0; i < keys; ++i)
-        {
-            Arr key(n);
-            for (size_t j = 0; j < n; ++j)
-                key[j] = DB::toString(rng());
-
-            vec.push_back(std::move(key));
-        }
-
-        for (const auto & elem : vec)
-        {
-            std::cerr << "[";
-            for (size_t j = 0; j < n; ++j)
-                std::cerr << (j == 0 ? "" : ", ") << elem[j];
-            std::cerr << "]" << std::endl;
-        }
-
-        std::cerr << std::endl;
-
-        Vec vec2 = std::move(vec);
-
-        for (const auto & elem : vec2)
-        {
-            std::cerr << "[";
-            for (size_t j = 0; j < n; ++j)
-                std::cerr << (j == 0 ? "" : ", ") << elem[j];
-            std::cerr << "]" << std::endl;
-        }
-    }
-
-    if (argc == 2 && !strcmp(argv[1], "1"))
-    {
-        size_t n = 5;
-        size_t map_size = 1000000;
-
-        using T = DB::Field;
-        T field = std::string("Hello, world");
-
-        using Arr = std::vector<T>;
-        using Map = HashMap<UInt64, Arr>;
-
-        Stopwatch watch;
-
-        Map map;
-        for (size_t i = 0; i < map_size; ++i)
-        {
-            Map::LookupResult it;
-            bool inserted;
-
-            map.emplace(rng(), it, inserted);
-            if (inserted)
-            {
-                new (&it->getMapped()) Arr(n);
-
-                for (size_t j = 0; j < n; ++j)
-                    (it->getMapped())[j] = field;
-            }
-        }
-
-        std::cerr << std::fixed << std::setprecision(2)
-            << "Vector:    Elapsed: " << watch.elapsedSeconds()
-            << " (" << map_size / watch.elapsedSeconds() << " rows/sec., "
-            << "sizeof(Map::value_type) = " << sizeof(Map::value_type)
-            << std::endl;
-    }
-
-    {
-        size_t n = 10000;
-        using Arr = DB::AutoArray<std::string>;
-        Arr arr1(n);
-        Arr arr2(n);
-
-        for (size_t i = 0; i < n; ++i)
-        {
-            arr1[i] = "Hello, world! " + DB::toString(i);
-            arr2[i] = "Goodbye, world! " + DB::toString(i);
-        }
-
-        arr2 = std::move(arr1);
-        arr1.resize(n); // NOLINT
-
-        std::cerr
-            << "arr1.size(): " << arr1.size() << ", arr2.size(): " << arr2.size() << std::endl
-            << "arr1.data(): " << arr1.data() << ", arr2.data(): " << arr2.data() << std::endl
-            << "arr1[0]: " << arr1[0] << ", arr2[0]: " << arr2[0] << std::endl;
-    }
-
-    return 0;
-}
diff --git a/src/DataStreams/MergingSortedBlockInputStream.h b/src/DataStreams/MergingSortedBlockInputStream.h
index 7f8d594416d..582b41ff3af 100644
--- a/src/DataStreams/MergingSortedBlockInputStream.h
+++ b/src/DataStreams/MergingSortedBlockInputStream.h
@@ -1,6 +1,5 @@
 #pragma once
 
-#include <Core/Row.h>
 #include <Core/SortDescription.h>
 #include <Core/SortCursor.h>
 
diff --git a/src/DataStreams/TTLAggregationAlgorithm.h b/src/DataStreams/TTLAggregationAlgorithm.h
index c2f40bab6b9..7231506b7da 100644
--- a/src/DataStreams/TTLAggregationAlgorithm.h
+++ b/src/DataStreams/TTLAggregationAlgorithm.h
@@ -32,7 +32,7 @@ private:
 
     const Block header;
     std::unique_ptr<Aggregator> aggregator;
-    Row current_key_value;
+    std::vector<Field> current_key_value;
     AggregatedDataVariants aggregation_result;
     ColumnRawPtrs key_columns;
     Aggregator::AggregateColumns columns_for_aggregator;
diff --git a/src/Functions/partitionId.cpp b/src/Functions/partitionId.cpp
index cf679452da1..70ece96f983 100644
--- a/src/Functions/partitionId.cpp
+++ b/src/Functions/partitionId.cpp
@@ -52,7 +52,7 @@ public:
         auto result_column = ColumnString::create();
         for (size_t j = 0; j < input_rows_count; ++j)
         {
-            Row row(size);
+            std::vector<Field> row(size);
             for (size_t i = 0; i < size; ++i)
                 arguments[i].column->get(j, row[i]);
             MergeTreePartition partition(std::move(row));
diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp
index dace0beea11..2c66d2a9841 100644
--- a/src/Interpreters/ActionsVisitor.cpp
+++ b/src/Interpreters/ActionsVisitor.cpp
@@ -1,6 +1,5 @@
 #include <Common/quoteString.h>
 #include <Common/typeid_cast.h>
-#include <Core/Row.h>
 
 #include <Functions/FunctionFactory.h>
 #include <Functions/FunctionsMiscellaneous.h>
@@ -80,7 +79,7 @@ static Block createBlockFromCollection(const Collection & collection, const Data
     for (size_t i = 0; i < columns_num; ++i)
         columns[i] = types[i]->createColumn();
 
-    Row tuple_values;
+    std::vector<Field> tuple_values;
     for (const auto & value : collection)
     {
         if (columns_num == 1)
@@ -154,7 +153,7 @@ static Block createBlockFromAST(const ASTPtr & node, const DataTypes & types, Co
     MutableColumns columns = header.cloneEmptyColumns();
 
     DataTypePtr tuple_type;
-    Row tuple_values;
+    std::vector<Field> tuple_values;
     const auto & list = node->as<ASTExpressionList &>();
     bool transform_null_in = context->getSettingsRef().transform_null_in;
     for (const auto & elem : list.children)
diff --git a/src/Interpreters/Set.cpp b/src/Interpreters/Set.cpp
index 713be3c1447..65896edfa69 100644
--- a/src/Interpreters/Set.cpp
+++ b/src/Interpreters/Set.cpp
@@ -1,7 +1,6 @@
 #include <optional>
 
 #include <Core/Field.h>
-#include <Core/Row.h>
 
 #include <Columns/ColumnsNumber.h>
 #include <Columns/ColumnTuple.h>
diff --git a/src/Interpreters/examples/hash_map.cpp b/src/Interpreters/examples/hash_map.cpp
index 620f2515825..66704ab432a 100644
--- a/src/Interpreters/examples/hash_map.cpp
+++ b/src/Interpreters/examples/hash_map.cpp
@@ -12,7 +12,6 @@
 #define DBMS_HASH_MAP_COUNT_COLLISIONS
 */
 #include <common/types.h>
-#include <Core/Row.h>
 #include <IO/ReadBufferFromFile.h>
 #include <Compression/CompressedReadBuffer.h>
 #include <Common/HashTable/HashMap.h>
@@ -27,19 +26,6 @@
   * This is important, because if you run all the tests one by one, the results will be incorrect.
   * (Due to the peculiarities of the work of the allocator, the first test takes advantage.)
   *
-  * Depending on USE_AUTO_ARRAY, one of the structures is selected as the value.
-  * USE_AUTO_ARRAY = 0 - uses std::vector (hard-copy structure, sizeof = 24 bytes).
-  * USE_AUTO_ARRAY = 1 - uses AutoArray (a structure specially designed for such cases, sizeof = 8 bytes).
-  *
-  * That is, the test also allows you to compare AutoArray and std::vector.
-  *
-  * If USE_AUTO_ARRAY = 0, then HashMap confidently overtakes all.
-  * If USE_AUTO_ARRAY = 1, then HashMap is slightly less serious (20%) ahead of google::dense_hash_map.
-  *
-  * When using HashMap, AutoArray has a rather serious (40%) advantage over std::vector.
-  * And when using other hash tables, AutoArray even more seriously overtakes std::vector
-  *  (up to three and a half times in the case of std::unordered_map and google::sparse_hash_map).
-  *
   * HashMap, unlike google::dense_hash_map, much more depends on the quality of the hash function.
   *
   * PS. Measure everything yourself, otherwise I'm almost confused.
@@ -49,9 +35,6 @@
   * But in this test, there was something similar to the old scenario of using hash tables in the aggregation.
   */
 
-#define USE_AUTO_ARRAY    0
-
-
 struct AlternativeHash
 {
     size_t operator() (UInt64 x) const
@@ -85,12 +68,7 @@ int main(int argc, char ** argv)
     using namespace DB;
 
     using Key = UInt64;
-
-#if USE_AUTO_ARRAY
-    using Value = AutoArray<IAggregateFunction*>;
-#else
     using Value = std::vector<IAggregateFunction*>;
-#endif
 
     size_t n = argc < 2 ? 10000000 : std::stol(argv[1]);
     //size_t m = std::stol(argv[2]);
@@ -119,13 +97,8 @@ int main(int argc, char ** argv)
 
     INIT
 
-#ifndef USE_AUTO_ARRAY
     #undef INIT
     #define INIT
-#endif
-
-    Row row(1);
-    row[0] = UInt64(0);
 
     std::cerr << "sizeof(Key) = " << sizeof(Key) << ", sizeof(Value) = " << sizeof(Value) << std::endl;
 
diff --git a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp
index 749b460a934..19ead09ae4b 100644
--- a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp
+++ b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp
@@ -12,6 +12,7 @@
 #include <DataTypes/NestedUtils.h>
 #include <IO/WriteHelpers.h>
 
+
 namespace DB
 {
 
@@ -109,6 +110,9 @@ static bool isInPartitionKey(const std::string & column_name, const Names & part
     return is_in_partition_key != partition_key_columns.end();
 }
 
+
+using Row = std::vector<Field>;
+
 /// Returns true if merge result is not empty
 static bool mergeMap(const SummingSortedAlgorithm::MapDescription & desc,
                      Row & row, const ColumnRawPtrs & raw_columns, size_t row_number)
diff --git a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.h b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.h
index df3749b5e88..3e03f5e9ae3 100644
--- a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.h
+++ b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.h
@@ -2,7 +2,7 @@
 
 #include <Processors/Merges/Algorithms/IMergingAlgorithmWithDelayedChunk.h>
 #include <Processors/Merges/Algorithms/MergedData.h>
-#include <Core/Row.h>
+
 
 namespace DB
 {
@@ -83,7 +83,7 @@ public:
 
         bool is_group_started = false;
 
-        Row current_row;
+        std::vector<Field> current_row;
         bool current_row_is_zero = true;    /// Are all summed columns zero (or empty)? It is updated incrementally.
 
         void addRowImpl(ColumnRawPtrs & raw_columns, size_t row);
diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h
index 38aeb15ebeb..5f12498c08c 100644
--- a/src/Storages/MergeTree/IMergeTreeDataPart.h
+++ b/src/Storages/MergeTree/IMergeTreeDataPart.h
@@ -2,7 +2,6 @@
 
 #include <DataStreams/IBlockInputStream.h>
 
-#include <Core/Row.h>
 #include <Core/Block.h>
 #include <common/types.h>
 #include <Core/NamesAndTypes.h>
diff --git a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp
index 5def458a382..1280138d38b 100644
--- a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp
+++ b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp
@@ -211,9 +211,12 @@ namespace
         virtual void insertStringColumn(const ColumnPtr & column, const String & name) = 0;
         virtual void insertUInt64Column(const ColumnPtr & column, const String & name) = 0;
         virtual void insertUUIDColumn(const ColumnPtr & column, const String & name) = 0;
-        virtual void
-        insertPartitionValueColumn(size_t rows, const Row & partition_value, const DataTypePtr & partition_value_type, const String & name)
-            = 0;
+
+        virtual void insertPartitionValueColumn(
+            size_t rows,
+            const std::vector<Field> & partition_value,
+            const DataTypePtr & partition_value_type,
+            const String & name) = 0;
     };
 }
 
@@ -319,7 +322,7 @@ namespace
         }
 
         void insertPartitionValueColumn(
-            size_t rows, const Row & partition_value, const DataTypePtr & partition_value_type, const String & name) final
+            size_t rows, const std::vector<Field> & partition_value, const DataTypePtr & partition_value_type, const String & name) final
         {
             ColumnPtr column;
             if (rows)
@@ -358,8 +361,8 @@ namespace
             columns.push_back(column);
         }
 
-        void
-        insertPartitionValueColumn(size_t rows, const Row & partition_value, const DataTypePtr & partition_value_type, const String &) final
+        void insertPartitionValueColumn(
+            size_t rows, const std::vector<Field> & partition_value, const DataTypePtr & partition_value_type, const String &) final
         {
             ColumnPtr column;
             if (rows)
diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 51077996212..e884ff7f211 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -3144,7 +3144,7 @@ String MergeTreeData::getPartitionIDFromQuery(const ASTPtr & ast, ContextPtr loc
             ErrorCodes::INVALID_PARTITION_VALUE);
 
     const FormatSettings format_settings;
-    Row partition_row(fields_count);
+    std::vector<Field> partition_row(fields_count);
 
     if (fields_count)
     {
diff --git a/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp b/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp
index e311cf8fa28..473f2e598a9 100644
--- a/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp
+++ b/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp
@@ -79,7 +79,7 @@ void MergeTreeDataPartInMemory::flushToDisk(const String & base_path, const Stri
 
     new_data_part->uuid = uuid;
     new_data_part->setColumns(columns);
-    new_data_part->partition.value.assign(partition.value);
+    new_data_part->partition.value = partition.value;
     new_data_part->minmax_idx = minmax_idx;
 
     if (disk->exists(destination_path))
diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp
index 2a85faa7083..33d0291946c 100644
--- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp
+++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp
@@ -150,7 +150,7 @@ BlocksWithPartition MergeTreeDataWriter::splitBlockIntoParts(const Block & block
 
     if (!metadata_snapshot->hasPartitionKey()) /// Table is not partitioned.
     {
-        result.emplace_back(Block(block), Row());
+        result.emplace_back(Block(block), std::vector<Field>{});
         return result;
     }
 
@@ -172,7 +172,7 @@ BlocksWithPartition MergeTreeDataWriter::splitBlockIntoParts(const Block & block
 
     auto get_partition = [&](size_t num)
     {
-        Row partition(partition_columns.size());
+        std::vector<Field> partition(partition_columns.size());
         for (size_t i = 0; i < partition_columns.size(); ++i)
             partition[i] = Field((*partition_columns[i])[partition_num_to_first_row[num]]);
         return partition;
diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.h b/src/Storages/MergeTree/MergeTreeDataWriter.h
index 4c5b75657ee..f5eb0cd98e9 100644
--- a/src/Storages/MergeTree/MergeTreeDataWriter.h
+++ b/src/Storages/MergeTree/MergeTreeDataWriter.h
@@ -1,7 +1,6 @@
 #pragma once
 
 #include <Core/Block.h>
-#include <Core/Row.h>
 
 #include <IO/WriteBufferFromFile.h>
 #include <Compression/CompressedWriteBuffer.h>
@@ -19,9 +18,9 @@ namespace DB
 struct BlockWithPartition
 {
     Block block;
-    Row partition;
+    std::vector<Field> partition;
 
-    BlockWithPartition(Block && block_, Row && partition_)
+    BlockWithPartition(Block && block_, std::vector<Field> && partition_)
         : block(block_), partition(std::move(partition_))
     {
     }
diff --git a/src/Storages/MergeTree/MergeTreePartition.h b/src/Storages/MergeTree/MergeTreePartition.h
index f89b6f22d4f..c55eb8ce142 100644
--- a/src/Storages/MergeTree/MergeTreePartition.h
+++ b/src/Storages/MergeTree/MergeTreePartition.h
@@ -1,9 +1,10 @@
 #pragma once
 
-#include <Core/Row.h>
 #include <common/types.h>
 #include <Disks/IDisk.h>
 #include <IO/WriteBuffer.h>
+#include <Core/Field.h>
+
 
 namespace DB
 {
@@ -19,12 +20,12 @@ using StorageMetadataPtr = std::shared_ptr<const StorageInMemoryMetadata>;
 /// This class represents a partition value of a single part and encapsulates its loading/storing logic.
 struct MergeTreePartition
 {
-    Row value;
+    std::vector<Field> value;
 
 public:
     MergeTreePartition() = default;
 
-    explicit MergeTreePartition(Row value_) : value(std::move(value_)) {}
+    explicit MergeTreePartition(std::vector<Field> value_) : value(std::move(value_)) {}
 
     /// For month-based partitioning.
     explicit MergeTreePartition(UInt32 yyyymm) : value(1, yyyymm) {}
@@ -38,7 +39,7 @@ public:
     void store(const MergeTreeData & storage, const DiskPtr & disk, const String & part_path, MergeTreeDataPartChecksums & checksums) const;
     void store(const Block & partition_key_sample, const DiskPtr & disk, const String & part_path, MergeTreeDataPartChecksums & checksums) const;
 
-    void assign(const MergeTreePartition & other) { value.assign(other.value); }
+    void assign(const MergeTreePartition & other) { value = other.value; }
 
     void create(const StorageMetadataPtr & metadata_snapshot, Block block, size_t row);
 };

From 2b6ffc557ae42cea9e96bde21977d3352752208a Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Thu, 20 May 2021 09:31:17 +0300
Subject: [PATCH 29/76] Remove useless file

---
 src/Core/Row.h | 18 ------------------
 1 file changed, 18 deletions(-)
 delete mode 100644 src/Core/Row.h

diff --git a/src/Core/Row.h b/src/Core/Row.h
deleted file mode 100644
index 0dc105922ff..00000000000
--- a/src/Core/Row.h
+++ /dev/null
@@ -1,18 +0,0 @@
-#pragma once
-
-#include <vector>
-
-#include <Common/AutoArray.h>
-#include <Core/Field.h>
-
-
-namespace DB
-{
-
-/** The data type for representing one row of the table in the RAM.
-  * Warning! It is preferable to store column blocks instead of single rows. See Block.h
-  */
-
-using Row = AutoArray<Field>;
-
-}

From 3c6b69a430db90e52bc465b42d1233d9bb673ea4 Mon Sep 17 00:00:00 2001
From: Ivan Lezhankin <ilezhankin@yandex-team.ru>
Date: Thu, 20 May 2021 15:12:09 +0300
Subject: [PATCH 30/76] Fix test

---
 tests/queries/0_stateless/01399_http_request_headers.reference | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/queries/0_stateless/01399_http_request_headers.reference b/tests/queries/0_stateless/01399_http_request_headers.reference
index 99ba2b1787f..90a10a9818d 100644
--- a/tests/queries/0_stateless/01399_http_request_headers.reference
+++ b/tests/queries/0_stateless/01399_http_request_headers.reference
@@ -1,4 +1,5 @@
 1
+1
 Code: 516
 1
 Code: 516

From cf5f36b70e2fbefbe34a0b2c20cf1db11ece0d30 Mon Sep 17 00:00:00 2001
From: Maksim Kita <maksim-kita@yandex-team.ru>
Date: Thu, 20 May 2021 14:09:32 +0000
Subject: [PATCH 31/76] LLVM JIT support AARCH64

---
 src/Interpreters/examples/jit_example.cpp | 68 +++++++++++------------
 utils/CMakeLists.txt                      |  2 +-
 2 files changed, 35 insertions(+), 35 deletions(-)

diff --git a/src/Interpreters/examples/jit_example.cpp b/src/Interpreters/examples/jit_example.cpp
index a4f19838bbe..92215429bfc 100644
--- a/src/Interpreters/examples/jit_example.cpp
+++ b/src/Interpreters/examples/jit_example.cpp
@@ -1,57 +1,57 @@
 #include <iostream>
 
-#include <llvm/IR/IRBuilder.h>
+// #include <llvm/IR/IRBuilder.h>
 
-#include <Interpreters/JIT/CHJIT.h>
+// #include <Interpreters/JIT/CHJIT.h>
 
-void test_function()
-{
-    std::cerr << "Test function" << std::endl;
-}
+// void test_function()
+// {
+//     std::cerr << "Test function" << std::endl;
+// }
 
 int main(int argc, char **argv)
 {
     (void)(argc);
     (void)(argv);
 
-    auto jit = DB::CHJIT();
+    // auto jit = DB::CHJIT();
 
-    jit.registerExternalSymbol("test_function", reinterpret_cast<void *>(&test_function));
+    // jit.registerExternalSymbol("test_function", reinterpret_cast<void *>(&test_function));
 
-    auto compiled_module_info = jit.compileModule([](llvm::Module & module)
-    {
-        auto & context = module.getContext();
-        llvm::IRBuilder<> b (context);
+    // auto compiled_module_info = jit.compileModule([](llvm::Module & module)
+    // {
+    //     auto & context = module.getContext();
+    //     llvm::IRBuilder<> b (context);
 
-        auto * func_declaration_type = llvm::FunctionType::get(b.getVoidTy(), { }, /*isVarArg=*/false);
-        auto * func_declaration = llvm::Function::Create(func_declaration_type, llvm::Function::ExternalLinkage, "test_function", module);
+    //     auto * func_declaration_type = llvm::FunctionType::get(b.getVoidTy(), { }, /*isVarArg=*/false);
+    //     auto * func_declaration = llvm::Function::Create(func_declaration_type, llvm::Function::ExternalLinkage, "test_function", module);
 
-        auto * value_type = b.getInt64Ty();
-        auto * pointer_type = value_type->getPointerTo();
+    //     auto * value_type = b.getInt64Ty();
+    //     auto * pointer_type = value_type->getPointerTo();
 
-        auto * func_type = llvm::FunctionType::get(b.getVoidTy(), { pointer_type }, /*isVarArg=*/false);
-        auto * function = llvm::Function::Create(func_type, llvm::Function::ExternalLinkage, "test_name", module);
-        auto * entry = llvm::BasicBlock::Create(context, "entry", function);
+    //     auto * func_type = llvm::FunctionType::get(b.getVoidTy(), { pointer_type }, /*isVarArg=*/false);
+    //     auto * function = llvm::Function::Create(func_type, llvm::Function::ExternalLinkage, "test_name", module);
+    //     auto * entry = llvm::BasicBlock::Create(context, "entry", function);
 
-        auto * argument = function->args().begin();
-        b.SetInsertPoint(entry);
+    //     auto * argument = function->args().begin();
+    //     b.SetInsertPoint(entry);
 
-        b.CreateCall(func_declaration);
+    //     b.CreateCall(func_declaration);
 
-        auto * load_argument = b.CreateLoad(argument);
-        auto * value = b.CreateAdd(load_argument, load_argument);
-        b.CreateRet(value);
-    });
+    //     auto * load_argument = b.CreateLoad(argument);
+    //     auto * value = b.CreateAdd(load_argument, load_argument);
+    //     b.CreateRet(value);
+    // });
 
-    for (const auto & compiled_function_name : compiled_module_info.compiled_functions)
-    {
-        std::cerr << compiled_function_name << std::endl;
-    }
+    // for (const auto & compiled_function_name : compiled_module_info.compiled_functions)
+    // {
+    //     std::cerr << compiled_function_name << std::endl;
+    // }
 
-    int64_t value = 5;
-    auto * test_name_function = reinterpret_cast<int64_t (*)(int64_t *)>(jit.findCompiledFunction(compiled_module_info, "test_name"));
-    auto result = test_name_function(&value);
-    std::cerr << "Result " << result << std::endl;
+    // int64_t value = 5;
+    // auto * test_name_function = reinterpret_cast<int64_t (*)(int64_t *)>(jit.findCompiledFunction(compiled_module_info, "test_name"));
+    // auto result = test_name_function(&value);
+    // std::cerr << "Result " << result << std::endl;
 
     return 0;
 }
diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt
index 3da8612e6c1..bd6453e406b 100644
--- a/utils/CMakeLists.txt
+++ b/utils/CMakeLists.txt
@@ -39,7 +39,7 @@ if (NOT DEFINED ENABLE_UTILS OR ENABLE_UTILS)
     endif ()
 
     # memcpy_jart.S contains position dependent code
-    if (NOT CMAKE_POSITION_INDEPENDENT_CODE AND NOT OS_DARWIN AND NOT OS_SUNOS)
+    if (NOT CMAKE_POSITION_INDEPENDENT_CODE AND NOT OS_DARWIN AND NOT OS_SUNOS AND NOT ARCH_AARCH64)
         add_subdirectory (memcpy-bench)
     endif ()
 endif ()

From 09d63545b019710f6ad0f956b252cb248edf7352 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@yandex-team.ru>
Date: Thu, 20 May 2021 17:43:24 +0300
Subject: [PATCH 32/76] Return float in segmentLengthSum for float args, add
 tests

---
 .../AggregateFunctionSegmentLengthSum.h       | 17 ++++++---
 ...gate_function_segment_length_sum.reference |  8 +++--
 ..._aggregate_function_segment_length_sum.sql | 35 +++++++++++--------
 3 files changed, 39 insertions(+), 21 deletions(-)

diff --git a/src/AggregateFunctions/AggregateFunctionSegmentLengthSum.h b/src/AggregateFunctions/AggregateFunctionSegmentLengthSum.h
index ff388316289..0ca70cc7367 100644
--- a/src/AggregateFunctions/AggregateFunctionSegmentLengthSum.h
+++ b/src/AggregateFunctions/AggregateFunctionSegmentLengthSum.h
@@ -111,14 +111,15 @@ template <typename T, typename Data>
 class AggregateFunctionSegmentLengthSum final : public IAggregateFunctionDataHelper<Data, AggregateFunctionSegmentLengthSum<T, Data>>
 {
 private:
-    UInt64 getSegmentLengthSum(Data & data) const
+    template <typename TResult>
+    TResult getSegmentLengthSum(Data & data) const
     {
         if (data.size() == 0)
             return 0;
 
         data.sort();
 
-        UInt64 res = 0;
+        TResult res = 0;
 
         typename Data::Segment cur_segment = data.segments[0];
 
@@ -146,7 +147,12 @@ public:
     {
     }
 
-    DataTypePtr getReturnType() const override { return std::make_shared<DataTypeUInt64>(); }
+    DataTypePtr getReturnType() const override
+    {
+        if constexpr (std::is_floating_point_v<T>)
+            return std::make_shared<DataTypeFloat64>();
+        return std::make_shared<DataTypeUInt64>();
+    }
 
     bool allocatesMemoryInArena() const override { return false; }
 
@@ -183,7 +189,10 @@ public:
 
     void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override
     {
-        assert_cast<ColumnUInt64 &>(to).getData().push_back(getSegmentLengthSum(this->data(place)));
+        if constexpr (std::is_floating_point_v<T>)
+            assert_cast<ColumnFloat64 &>(to).getData().push_back(getSegmentLengthSum<Float64>(this->data(place)));
+        else
+            assert_cast<ColumnUInt64 &>(to).getData().push_back(getSegmentLengthSum<UInt64>(this->data(place)));
     }
 };
 
diff --git a/tests/queries/0_stateless/01866_aggregate_function_segment_length_sum.reference b/tests/queries/0_stateless/01866_aggregate_function_segment_length_sum.reference
index a830f800286..17448ee4ac3 100644
--- a/tests/queries/0_stateless/01866_aggregate_function_segment_length_sum.reference
+++ b/tests/queries/0_stateless/01866_aggregate_function_segment_length_sum.reference
@@ -1,2 +1,6 @@
-a	5
-b	8
+a	5	UInt64
+b	8	UInt64
+c	3	UInt64
+a	1	Float64
+a	7201	UInt64
+a	7	UInt64
diff --git a/tests/queries/0_stateless/01866_aggregate_function_segment_length_sum.sql b/tests/queries/0_stateless/01866_aggregate_function_segment_length_sum.sql
index ed2c8b66bd7..c4e4f8afa85 100644
--- a/tests/queries/0_stateless/01866_aggregate_function_segment_length_sum.sql
+++ b/tests/queries/0_stateless/01866_aggregate_function_segment_length_sum.sql
@@ -1,21 +1,26 @@
 DROP TABLE IF EXISTS segment;
+DROP TABLE IF EXISTS fl_segment;
+DROP TABLE IF EXISTS dt_segment;
+DROP TABLE IF EXISTS date_segment;
 
-CREATE TABLE segment
-(
-    `id` String,
-    `start` UInt64,
-    `end` UInt64
-)
-ENGINE = MergeTree
-ORDER BY start;
+CREATE TABLE segment ( `id` String, `start` Int64, `end` Int64 ) ENGINE = MergeTree ORDER BY start;
+INSERT INTO segment VALUES ('a', 1, 3), ('a', 1, 3), ('a', 2, 4), ('a', 1, 1), ('a', 5, 6), ('a', 5, 7), ('b', 10, 12), ('b', 13, 19), ('b', 14, 16), ('c', -1, 1), ('c', -2, -1);
 
-INSERT INTO segment VALUES ('a', 1, 3), ('a', 2, 4), ('a', 5, 6), ('a', 5, 7), ('b', 10, 12), ('b', 13, 19), ('b', 14, 16);
+CREATE TABLE fl_segment ( `id` String, `start` Float, `end` Float ) ENGINE = MergeTree ORDER BY start;
+INSERT INTO fl_segment VALUES ('a', 1.1, 3.2), ('a', 1.5, 3.6), ('a', 4.0, 5.0);
 
-SELECT
-    id,
-    segmentLengthSum(start, end)
-FROM segment
-GROUP BY id
-ORDER BY id;
+CREATE TABLE dt_segment ( `id` String, `start` DateTime, `end` DateTime ) ENGINE = MergeTree ORDER BY start;
+INSERT INTO dt_segment VALUES ('a', '2020-01-01 02:11:22', '2020-01-01 03:12:31'), ('a', '2020-01-01 01:12:30', '2020-01-01 02:50:11');
+
+CREATE TABLE date_segment ( `id` String, `start` Date, `end` Date ) ENGINE = MergeTree ORDER BY start;
+INSERT INTO date_segment VALUES ('a', '2020-01-01', '2020-01-04'), ('a', '2020-01-03', '2020-01-08 02:50:11');
+
+SELECT id, segmentLengthSum(start, end), toTypeName(segmentLengthSum(start, end)) FROM segment GROUP BY id ORDER BY id;
+SELECT id, 3.4 < segmentLengthSum(start, end) AND segmentLengthSum(start, end) < 3.6, toTypeName(segmentLengthSum(start, end)) FROM fl_segment GROUP BY id ORDER BY id;
+SELECT id, segmentLengthSum(start, end), toTypeName(segmentLengthSum(start, end)) FROM dt_segment GROUP BY id ORDER BY id;
+SELECT id, segmentLengthSum(start, end), toTypeName(segmentLengthSum(start, end)) FROM date_segment GROUP BY id ORDER BY id;
 
 DROP TABLE segment;
+DROP TABLE fl_segment;
+DROP TABLE dt_segment;
+DROP TABLE date_segment;

From fa88e8725a5cdeb05d3da1ac85b5b81293a35428 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Thu, 20 May 2021 19:02:46 +0300
Subject: [PATCH 33/76] Better parallelizm for fast tests.

---
 tests/clickhouse-test | 36 +++++++++++++++++++++++++++---------
 1 file changed, 27 insertions(+), 9 deletions(-)

diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index ecc2001fe87..0b486e4614c 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -27,6 +27,7 @@ except ImportError:
 import random
 import string
 import multiprocessing
+import multiprocessing.queues
 from contextlib import closing
 
 
@@ -318,7 +319,7 @@ stop_time = None
 
 # def run_tests_array(all_tests, suite, suite_dir, suite_tmp_dir, run_total):
 def run_tests_array(all_tests_with_params):
-    all_tests, suite, suite_dir, suite_tmp_dir = all_tests_with_params
+    tests_queue, num_tests, suite, suite_dir, suite_tmp_dir = all_tests_with_params
     global exit_code
     global SERVER_DIED
     global stop_time
@@ -348,10 +349,22 @@ def run_tests_array(all_tests_with_params):
         else:
             return ''
 
-    if all_tests:
-        print(f"\nRunning {len(all_tests)} {suite} tests ({multiprocessing.current_process().name}).\n")
+    if num_tests > 0:
+        about = 'about ' if is_concurrent else ''
+        print(f"\nRunning {about}{num_tests} {suite} tests ({multiprocessing.current_process().name}).\n")
+
+    while True:
+        if isinstance(tests_queue, multiprocessing.queues.Queue):
+            try:
+                case = tests_queue.get()
+            except (ValueError, OSError):
+                break
+        else:
+            if tests_queue:
+                case = tests_queue.pop(0)
+            else:
+                break
 
-    for case in all_tests:
         if SERVER_DIED:
             stop_tests()
             break
@@ -856,17 +869,22 @@ def main(args):
 
                 # Create two batches per process for more uniform execution time.
                 batch_size = max(1, len(parallel_tests) // (jobs * 2))
+                queue = multiprocessing.queues.Queue()
                 parallel_tests_array = []
-                for i in range(0, len(parallel_tests), batch_size):
-                    parallel_tests_array.append((parallel_tests[i:i+batch_size], suite, suite_dir, suite_tmp_dir))
+                for i in range(jobs):
+                    parallel_tests_array.append((queue, batch_size, suite, suite_dir, suite_tmp_dir))
 
                 with closing(multiprocessing.Pool(processes=jobs)) as pool:
-                    pool.map(run_tests_array, parallel_tests_array)
+                    pool.map_async(run_tests_array, parallel_tests_array)
 
-                run_tests_array((sequential_tests, suite, suite_dir, suite_tmp_dir))
+                    for suit in parallel_tests:
+                        queue.put(suit)
+                    queue.close()
+
+                run_tests_array((sequential_tests, len(sequential_tests), suite, suite_dir, suite_tmp_dir))
                 total_tests_run += len(sequential_tests) + len(parallel_tests)
             else:
-                run_tests_array((all_tests, suite, suite_dir, suite_tmp_dir))
+                run_tests_array((all_tests, len(all_tests), suite, suite_dir, suite_tmp_dir))
                 total_tests_run += len(all_tests)
 
     if args.hung_check:

From 0037713dd313507cdcea8e5bcf1e6ba41f14dc9a Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@yandex-team.ru>
Date: Thu, 20 May 2021 19:21:45 +0300
Subject: [PATCH 34/76] Update version_date.tsv after release 21.5.5.12

---
 utils/list-versions/version_date.tsv | 1 +
 1 file changed, 1 insertion(+)

diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv
index 9aae71965f6..a2773b22cec 100644
--- a/utils/list-versions/version_date.tsv
+++ b/utils/list-versions/version_date.tsv
@@ -1,3 +1,4 @@
+v21.5.5.12-stable	2021-05-20
 v21.4.7.3-stable	2021-05-19
 v21.4.6.55-stable	2021-04-30
 v21.4.5.46-stable	2021-04-24

From 9d4927a5b97771341c2de828d18d48668e248aab Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Thu, 20 May 2021 19:44:35 +0300
Subject: [PATCH 35/76] Better parallelizm for fast tests.

---
 tests/clickhouse-test | 29 +++++++++++++++++------------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index 0b486e4614c..66831d4cd5c 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -27,7 +27,8 @@ except ImportError:
 import random
 import string
 import multiprocessing
-import multiprocessing.queues
+from collections.abc import Iterable
+#import multiprocessing.queues
 from contextlib import closing
 
 
@@ -315,11 +316,12 @@ def colored(text, args, color=None, on_color=None, attrs=None):
 SERVER_DIED = False
 exit_code = 0
 stop_time = None
+queue = multiprocessing.Queue(maxsize=1)
 
 
 # def run_tests_array(all_tests, suite, suite_dir, suite_tmp_dir, run_total):
 def run_tests_array(all_tests_with_params):
-    tests_queue, num_tests, suite, suite_dir, suite_tmp_dir = all_tests_with_params
+    all_tests, num_tests, suite, suite_dir, suite_tmp_dir = all_tests_with_params
     global exit_code
     global SERVER_DIED
     global stop_time
@@ -354,14 +356,13 @@ def run_tests_array(all_tests_with_params):
         print(f"\nRunning {about}{num_tests} {suite} tests ({multiprocessing.current_process().name}).\n")
 
     while True:
-        if isinstance(tests_queue, multiprocessing.queues.Queue):
-            try:
-                case = tests_queue.get()
-            except (ValueError, OSError):
+        if is_concurrent:
+            case = queue.get()
+            if not case:
                 break
         else:
-            if tests_queue:
-                case = tests_queue.pop(0)
+            if all_tests:
+                case = all_tests.pop(0)
             else:
                 break
 
@@ -867,20 +868,24 @@ def main(args):
                 if jobs > run_total:
                     run_total = jobs
 
-                # Create two batches per process for more uniform execution time.
-                batch_size = max(1, len(parallel_tests) // (jobs * 2))
-                queue = multiprocessing.queues.Queue()
+                batch_size = max(1, len(parallel_tests) // jobs)
                 parallel_tests_array = []
                 for i in range(jobs):
-                    parallel_tests_array.append((queue, batch_size, suite, suite_dir, suite_tmp_dir))
+                    parallel_tests_array.append((None, batch_size, suite, suite_dir, suite_tmp_dir))
 
                 with closing(multiprocessing.Pool(processes=jobs)) as pool:
                     pool.map_async(run_tests_array, parallel_tests_array)
 
                     for suit in parallel_tests:
                         queue.put(suit)
+
+                    for i in range(jobs):
+                        queue.put(None)
+
                     queue.close()
 
+                pool.join()
+
                 run_tests_array((sequential_tests, len(sequential_tests), suite, suite_dir, suite_tmp_dir))
                 total_tests_run += len(sequential_tests) + len(parallel_tests)
             else:

From 386cbf751c1d3315a5b16d47e28ed1e281d174e5 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Thu, 20 May 2021 19:46:15 +0300
Subject: [PATCH 36/76] Better parallelizm for fast tests.

---
 tests/clickhouse-test | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index 66831d4cd5c..21034931bc4 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -27,8 +27,6 @@ except ImportError:
 import random
 import string
 import multiprocessing
-from collections.abc import Iterable
-#import multiprocessing.queues
 from contextlib import closing
 
 

From 8b438bcd3c7ff07127816aaa61f775718dec2204 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sat, 15 May 2021 12:11:13 +0300
Subject: [PATCH 37/76] Change formatting of subqueries (make it more human
 friendly)

Fix trailing whitespaces in FROM/IN clause with subqueries in multiline
mode, and also changes the output of the queries slightly in a more
human friendly way.

Before:

    $ clickhouse-format <<<'select * from system.one, (select * from system.one)'
    SELECT *
    FROM system.one
    ,
    (
        SELECT *
        FROM system.one
    )

After:

    $ clickhouse-format <<<'select * from system.one, (select * from system.one)'
    SELECT *
    FROM system.one,
    (
        SELECT *
        FROM system.one
    )

v2: Fix subqueries formatting in a different way
v3: Adjust *.reference in tests
v4: Fix modernize-loop-convert in ASTTablesInSelectQuery
---
 src/Parsers/ASTSelectQuery.cpp                |   2 +-
 src/Parsers/ASTSelectWithUnionQuery.cpp       |  10 +-
 src/Parsers/ASTSubquery.cpp                   |   7 +-
 src/Parsers/ASTTablesInSelectQuery.cpp        |  24 ++--
 src/Parsers/ASTWithElement.cpp                |   5 +-
 ...0077_set_keys_fit_128_bits_many_blocks.sql |   2 +-
 ...316_rounding_functions_and_empty_block.sql |   2 +-
 ...ll_subquery_aggregation_column_removal.sql |  24 ++--
 .../00597_push_down_predicate.reference       |  98 +++++++--------
 .../00599_create_view_with_subquery.reference |   2 +-
 .../queries/0_stateless/00618_nullable_in.sql |   2 +-
 .../00743_limit_by_not_found_column.sql       |   2 +-
 ...51_default_databasename_for_view.reference |   9 +-
 .../00808_not_optimize_predicate.reference    |   2 +-
 .../00849_multiple_comma_join_2.reference     |  40 +++---
 ...0957_format_with_clashed_aliases.reference |   2 +-
 .../01029_early_constant_folding.reference    |   3 +-
 .../01054_cache_dictionary_overflow_cell.sql  |   2 +-
 .../01056_predicate_optimizer_bugs.reference  |  28 ++---
 tests/queries/0_stateless/01067_join_null.sql |   6 +-
 ...76_predicate_optimizer_with_view.reference |   8 +-
 ..._expressions_in_engine_arguments.reference |   2 +-
 .../01245_limit_infinite_sources.sql          |   2 +-
 ...ithmetic_operations_in_aggr_func.reference |  96 +++++++--------
 ..._duplicate_order_by_and_distinct.reference |  16 +--
 ...egate_functions_of_group_by_keys.reference |   4 +-
 ..._redundant_functions_in_order_by.reference |  16 +--
 .../01372_wrong_order_by_removal.reference    |   2 +-
 .../01390_remove_injective_in_uniq.reference  |  28 ++---
 .../0_stateless/01412_row_from_totals.sql     |   2 +-
 ...dicate_when_contains_with_clause.reference |   2 +-
 ..._duplicate_distinct_optimization.reference |  36 +++---
 ...5_subqueries_in_with_statement_4.reference |   5 +-
 ...with_global_and_with_propagation.reference |   9 +-
 .../0_stateless/01532_having_with_totals.sql  |   4 +-
 .../01532_min_max_with_modifiers.sql          |   2 +-
 .../01559_aggregate_null_for_empty_fix.sql    |  16 +--
 ...60_DateTime_and_DateTime64_comparision.sql |   2 +-
 ...0_monotonicity_check_multiple_args_bug.sql |   4 +-
 ...ministic_function_with_predicate.reference |   4 +-
 .../01593_functions_in_order_by.reference     |   2 +-
 .../01593_functions_in_order_by.sql           |   2 +-
 .../0_stateless/01635_nullable_fuzz.sql       |   2 +-
 .../0_stateless/01636_nullable_fuzz2.sql      |   2 +-
 .../01650_expressions_merge_bug.sql           |   2 +-
 .../0_stateless/01670_neighbor_lc_bug.sql     |   2 +-
 ...01732_explain_syntax_union_query.reference |  16 +--
 .../01732_explain_syntax_union_query.sql      |  14 +--
 ...1754_clickhouse_format_backslash.reference |   2 +-
 .../01798_uniq_theta_sketch.reference         |  24 ++--
 ...select_from_trailing_whitespaces.reference | 114 ++++++++++++++++++
 .../01874_select_from_trailing_whitespaces.sh |  30 +++++
 .../1_stateful/00031_array_enumerate_uniq.sql |   2 +-
 .../1_stateful/00063_loyalty_joins.sql        |   8 +-
 54 files changed, 453 insertions(+), 301 deletions(-)
 create mode 100644 tests/queries/0_stateless/01874_select_from_trailing_whitespaces.reference
 create mode 100755 tests/queries/0_stateless/01874_select_from_trailing_whitespaces.sh

diff --git a/src/Parsers/ASTSelectQuery.cpp b/src/Parsers/ASTSelectQuery.cpp
index 4715c7f201b..b5cfa17f035 100644
--- a/src/Parsers/ASTSelectQuery.cpp
+++ b/src/Parsers/ASTSelectQuery.cpp
@@ -95,7 +95,7 @@ void ASTSelectQuery::formatImpl(const FormatSettings & s, FormatState & state, F
 
     if (tables())
     {
-        s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "FROM " << (s.hilite ? hilite_none : "");
+        s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "FROM" << (s.hilite ? hilite_none : "");
         tables()->formatImpl(s, state, frame);
     }
 
diff --git a/src/Parsers/ASTSelectWithUnionQuery.cpp b/src/Parsers/ASTSelectWithUnionQuery.cpp
index 6a990a3d154..fa7359574f8 100644
--- a/src/Parsers/ASTSelectWithUnionQuery.cpp
+++ b/src/Parsers/ASTSelectWithUnionQuery.cpp
@@ -35,24 +35,24 @@ void ASTSelectWithUnionQuery::formatQueryImpl(const FormatSettings & settings, F
         if (mode == Mode::Unspecified)
             return "";
         else if (mode == Mode::ALL)
-            return "ALL";
+            return " ALL";
         else
-            return "DISTINCT";
+            return " DISTINCT";
     };
 
     for (ASTs::const_iterator it = list_of_selects->children.begin(); it != list_of_selects->children.end(); ++it)
     {
         if (it != list_of_selects->children.begin())
-            settings.ostr << settings.nl_or_ws << indent_str << (settings.hilite ? hilite_keyword : "") << "UNION "
+            settings.ostr << settings.nl_or_ws << indent_str << (settings.hilite ? hilite_keyword : "") << "UNION"
                           << mode_to_str((is_normalized) ? union_mode : list_of_modes[it - list_of_selects->children.begin() - 1])
                           << (settings.hilite ? hilite_none : "");
 
         if (auto * node = (*it)->as<ASTSelectWithUnionQuery>())
         {
+            settings.ostr << settings.nl_or_ws << indent_str;
+
             if (node->list_of_selects->children.size() == 1)
             {
-                if (it != list_of_selects->children.begin())
-                    settings.ostr << settings.nl_or_ws;
                 (node->list_of_selects->children.at(0))->formatImpl(settings, state, frame);
             }
             else
diff --git a/src/Parsers/ASTSubquery.cpp b/src/Parsers/ASTSubquery.cpp
index 142072050a6..58f334376e6 100644
--- a/src/Parsers/ASTSubquery.cpp
+++ b/src/Parsers/ASTSubquery.cpp
@@ -29,6 +29,11 @@ void ASTSubquery::appendColumnNameImpl(WriteBuffer & ostr) const
 
 void ASTSubquery::formatImplWithoutAlias(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
 {
+    /// NOTE: due to trickery of filling cte_name (in interpreters) it is hard
+    /// to print it w/o newline (for !oneline case), since if nl_or_ws
+    /// prepended here, then formatting will be incorrect with alias:
+    ///
+    ///   (select 1 in ((select 1) as sub))
     if (!cte_name.empty())
     {
         settings.ostr << (settings.hilite ? hilite_identifier : "");
@@ -40,7 +45,7 @@ void ASTSubquery::formatImplWithoutAlias(const FormatSettings & settings, Format
     std::string indent_str = settings.one_line ? "" : std::string(4u * frame.indent, ' ');
     std::string nl_or_nothing = settings.one_line ? "" : "\n";
 
-    settings.ostr << nl_or_nothing << indent_str << "(" << nl_or_nothing;
+    settings.ostr << "(" << nl_or_nothing;
     FormatStateStacked frame_nested = frame;
     frame_nested.need_parens = false;
     ++frame_nested.indent;
diff --git a/src/Parsers/ASTTablesInSelectQuery.cpp b/src/Parsers/ASTTablesInSelectQuery.cpp
index 8d131a848f7..e063b9f7125 100644
--- a/src/Parsers/ASTTablesInSelectQuery.cpp
+++ b/src/Parsers/ASTTablesInSelectQuery.cpp
@@ -109,14 +109,17 @@ void ASTTableExpression::formatImpl(const FormatSettings & settings, FormatState
 
     if (database_and_table_name)
     {
+        settings.ostr << " ";
         database_and_table_name->formatImpl(settings, state, frame);
     }
     else if (table_function)
     {
+        settings.ostr << " ";
         table_function->formatImpl(settings, state, frame);
     }
     else if (subquery)
     {
+        settings.ostr << settings.nl_or_ws << indent_str;
         subquery->formatImpl(settings, state, frame);
     }
 
@@ -142,9 +145,15 @@ void ASTTableExpression::formatImpl(const FormatSettings & settings, FormatState
 }
 
 
-void ASTTableJoin::formatImplBeforeTable(const FormatSettings & settings, FormatState &, FormatStateStacked) const
+void ASTTableJoin::formatImplBeforeTable(const FormatSettings & settings, FormatState &, FormatStateStacked frame) const
 {
     settings.ostr << (settings.hilite ? hilite_keyword : "");
+    std::string indent_str = settings.one_line ? "" : std::string(4 * frame.indent, ' ');
+
+    if (kind != Kind::Comma)
+    {
+        settings.ostr << settings.nl_or_ws << indent_str;
+    }
 
     switch (locality)
     {
@@ -241,6 +250,7 @@ void ASTArrayJoin::formatImpl(const FormatSettings & settings, FormatState & sta
     frame.expression_list_prepend_whitespace = true;
 
     settings.ostr << (settings.hilite ? hilite_keyword : "")
+        << settings.nl_or_ws
         << (kind == Kind::Left ? "LEFT " : "") << "ARRAY JOIN" << (settings.hilite ? hilite_none : "");
 
     settings.one_line
@@ -254,10 +264,7 @@ void ASTTablesInSelectQueryElement::formatImpl(const FormatSettings & settings,
     if (table_expression)
     {
         if (table_join)
-        {
             table_join->as<ASTTableJoin &>().formatImplBeforeTable(settings, state, frame);
-            settings.ostr << " ";
-        }
 
         table_expression->formatImpl(settings, state, frame);
 
@@ -275,13 +282,8 @@ void ASTTablesInSelectQuery::formatImpl(const FormatSettings & settings, FormatS
 {
     std::string indent_str = settings.one_line ? "" : std::string(4 * frame.indent, ' ');
 
-    for (ASTs::const_iterator it = children.begin(); it != children.end(); ++it)
-    {
-        if (it != children.begin())
-            settings.ostr << settings.nl_or_ws << indent_str;
-
-        (*it)->formatImpl(settings, state, frame);
-    }
+    for (const auto & child : children)
+        child->formatImpl(settings, state, frame);
 }
 
 }
diff --git a/src/Parsers/ASTWithElement.cpp b/src/Parsers/ASTWithElement.cpp
index ce39086eb4a..b517509c4bc 100644
--- a/src/Parsers/ASTWithElement.cpp
+++ b/src/Parsers/ASTWithElement.cpp
@@ -16,8 +16,11 @@ ASTPtr ASTWithElement::clone() const
 
 void ASTWithElement::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
 {
+    std::string indent_str = settings.one_line ? "" : std::string(4 * frame.indent, ' ');
+
     settings.writeIdentifier(name);
-    settings.ostr << (settings.hilite ? hilite_keyword : "") << " AS " << (settings.hilite ? hilite_none : "");
+    settings.ostr << (settings.hilite ? hilite_keyword : "") << " AS" << (settings.hilite ? hilite_none : "");
+    settings.ostr << settings.nl_or_ws << indent_str;
     dynamic_cast<const ASTWithAlias &>(*subquery).formatImplWithoutAlias(settings, state, frame);
 }
 
diff --git a/tests/queries/0_stateless/00077_set_keys_fit_128_bits_many_blocks.sql b/tests/queries/0_stateless/00077_set_keys_fit_128_bits_many_blocks.sql
index 02f0a6648a8..fe6a0cefd31 100644
--- a/tests/queries/0_stateless/00077_set_keys_fit_128_bits_many_blocks.sql
+++ b/tests/queries/0_stateless/00077_set_keys_fit_128_bits_many_blocks.sql
@@ -1,6 +1,6 @@
 SET max_block_size = 1000;
 
-SELECT number FROM 
+SELECT number FROM
 (
     SELECT * FROM system.numbers LIMIT 10000
 ) 
diff --git a/tests/queries/0_stateless/00316_rounding_functions_and_empty_block.sql b/tests/queries/0_stateless/00316_rounding_functions_and_empty_block.sql
index 9f91cd713ae..08c30324da4 100644
--- a/tests/queries/0_stateless/00316_rounding_functions_and_empty_block.sql
+++ b/tests/queries/0_stateless/00316_rounding_functions_and_empty_block.sql
@@ -2,7 +2,7 @@ SET any_join_distinct_right_table_keys = 1;
 
 SELECT
     floor((ReferrerTimestamp - InstallTimestamp) / 86400) AS DaysSinceInstallations
-FROM 
+FROM
 (
     SELECT 6534090703218709881 AS DeviceIDHash, 1458586663 AS InstallTimestamp
     UNION ALL SELECT 2697418689476658272, 1458561552
diff --git a/tests/queries/0_stateless/00585_union_all_subquery_aggregation_column_removal.sql b/tests/queries/0_stateless/00585_union_all_subquery_aggregation_column_removal.sql
index bf5d2251470..49975daaa7e 100644
--- a/tests/queries/0_stateless/00585_union_all_subquery_aggregation_column_removal.sql
+++ b/tests/queries/0_stateless/00585_union_all_subquery_aggregation_column_removal.sql
@@ -11,7 +11,7 @@ INSERT INTO transactions VALUES ('facebook.com'), ('yandex.ru'), ('baidu.com');
 SELECT
     sum(total_count) AS total, 
     domain
-FROM 
+FROM
 (
     SELECT
         COUNT(*) AS total_count, 
@@ -36,7 +36,7 @@ FORMAT JSONEachRow;
 SELECT
     sum(total_count) AS total, 
     domain
-FROM 
+FROM
 (
     SELECT
         COUNT(*) AS total_count, 
@@ -63,7 +63,7 @@ SELECT DISTINCT * FROM
 SELECT
     sum(total_count) AS total, 
     domain
-FROM 
+FROM
 (
     SELECT
         COUNT(*) AS total_count, 
@@ -88,7 +88,7 @@ UNION ALL
 SELECT
     sum(total_count) AS total, 
     domain
-FROM 
+FROM
 (
     SELECT
         COUNT(*) AS total_count, 
@@ -116,7 +116,7 @@ SELECT
     sum(total_count) AS total, 
     sum(facebookHits) AS facebook,
     domain
-FROM 
+FROM
 (
     SELECT
         COUNT(*) AS total_count, 
@@ -142,7 +142,7 @@ SELECT
     sum(total_count) AS total, 
     max(facebookHits) AS facebook,
     domain
-FROM 
+FROM
 (
     SELECT
         COUNT(*) AS total_count, 
@@ -170,7 +170,7 @@ SELECT * FROM
 SELECT
     sum(total_count) AS total, 
     domain
-FROM 
+FROM
 (
     SELECT
         COUNT(*) AS total_count, 
@@ -195,7 +195,7 @@ ALL FULL OUTER JOIN
 SELECT
     sum(total_count) AS total, 
     domain
-FROM 
+FROM
 (
     SELECT
         COUNT(*) AS total_count, 
@@ -224,7 +224,7 @@ SELECT total FROM
 SELECT
     sum(total_count) AS total, 
     domain
-FROM 
+FROM
 (
     SELECT
         COUNT(*) AS total_count, 
@@ -249,7 +249,7 @@ ALL FULL OUTER JOIN
 SELECT
     sum(total_count) AS total, 
     domain
-FROM 
+FROM
 (
     SELECT
         COUNT(*) AS total_count, 
@@ -278,7 +278,7 @@ SELECT domain FROM
 SELECT
     sum(total_count) AS total, 
     domain
-FROM 
+FROM
 (
     SELECT
         COUNT(*) AS total_count, 
@@ -303,7 +303,7 @@ ALL FULL OUTER JOIN
 SELECT
     sum(total_count) AS total, 
     domain
-FROM 
+FROM
 (
     SELECT
         COUNT(*) AS total_count, 
diff --git a/tests/queries/0_stateless/00597_push_down_predicate.reference b/tests/queries/0_stateless/00597_push_down_predicate.reference
index 59313c35b81..bd7d3cd81d4 100644
--- a/tests/queries/0_stateless/00597_push_down_predicate.reference
+++ b/tests/queries/0_stateless/00597_push_down_predicate.reference
@@ -5,7 +5,7 @@
 2000-01-01	1	test string 1	1
 -------Forbid push down-------
 SELECT count()
-FROM 
+FROM
 (
     SELECT
         [number] AS a,
@@ -21,11 +21,11 @@ WHERE NOT ignore(a + b)
 SELECT
     a,
     b
-FROM 
+FROM
 (
     SELECT 1 AS a
 )
-ANY LEFT JOIN 
+ANY LEFT JOIN
 (
     SELECT
         1 AS a,
@@ -35,13 +35,13 @@ WHERE b = 0
 SELECT
     a,
     b
-FROM 
+FROM
 (
     SELECT
         1 AS a,
         1 AS b
 )
-ANY RIGHT JOIN 
+ANY RIGHT JOIN
 (
     SELECT 1 AS a
 ) USING (a)
@@ -49,11 +49,11 @@ WHERE b = 0
 SELECT
     a,
     b
-FROM 
+FROM
 (
     SELECT 1 AS a
 )
-ANY FULL OUTER JOIN 
+ANY FULL OUTER JOIN
 (
     SELECT
         1 AS a,
@@ -63,26 +63,26 @@ WHERE b = 0
 SELECT
     a,
     b
-FROM 
+FROM
 (
     SELECT
         1 AS a,
         1 AS b
 )
-ANY FULL OUTER JOIN 
+ANY FULL OUTER JOIN
 (
     SELECT 1 AS a
 ) USING (a)
 WHERE b = 0
 -------Need push down-------
 SELECT toString(value) AS value
-FROM 
+FROM
 (
     SELECT 1 AS value
 )
 1
 SELECT id
-FROM 
+FROM
 (
     SELECT 1 AS id
     UNION ALL
@@ -92,7 +92,7 @@ FROM
 WHERE id = 1
 1
 SELECT id
-FROM 
+FROM
 (
     SELECT arrayJoin([1, 2, 3]) AS id
     WHERE id = 1
@@ -100,7 +100,7 @@ FROM
 WHERE id = 1
 1
 SELECT id
-FROM 
+FROM
 (
     SELECT arrayJoin([1, 2, 3]) AS id
     WHERE id = 1
@@ -110,7 +110,7 @@ WHERE id = 1
 SELECT
     id,
     subquery
-FROM 
+FROM
 (
     SELECT
         1 AS id,
@@ -122,7 +122,7 @@ WHERE subquery = 1
 SELECT
     a,
     b
-FROM 
+FROM
 (
     SELECT
         toUInt64(sum(id) AS b) AS a,
@@ -137,7 +137,7 @@ SELECT
     id,
     name,
     value
-FROM 
+FROM
 (
     SELECT
         date,
@@ -156,7 +156,7 @@ WHERE id = 1
 SELECT
     a,
     b
-FROM 
+FROM
 (
     SELECT
         toUInt64(sum(id) AS b) AS a,
@@ -171,7 +171,7 @@ SELECT
     id,
     name,
     value
-FROM 
+FROM
 (
     SELECT
         date,
@@ -188,14 +188,14 @@ SELECT
     id,
     name,
     value
-FROM 
+FROM
 (
     SELECT
         date,
         id,
         name,
         value
-    FROM 
+    FROM
     (
         SELECT
             date,
@@ -214,14 +214,14 @@ SELECT
     id,
     name,
     value
-FROM 
+FROM
 (
     SELECT
         date,
         id,
         name,
         value
-    FROM 
+    FROM
     (
         SELECT
             date,
@@ -240,7 +240,7 @@ SELECT
     id,
     name,
     value
-FROM 
+FROM
 (
     SELECT
         date,
@@ -257,14 +257,14 @@ SELECT
     id,
     name,
     value
-FROM 
+FROM
 (
     SELECT
         date,
         id,
         name,
         value
-    FROM 
+    FROM
     (
         SELECT
             date,
@@ -283,7 +283,7 @@ SELECT
     id,
     name,
     value
-FROM 
+FROM
 (
     SELECT
         date,
@@ -300,14 +300,14 @@ SELECT
     id,
     name,
     value
-FROM 
+FROM
 (
     SELECT
         date,
         id,
         name,
         value
-    FROM 
+    FROM
     (
         SELECT
             date,
@@ -325,7 +325,7 @@ SELECT
     id,
     date,
     value
-FROM 
+FROM
 (
     SELECT
         id,
@@ -344,7 +344,7 @@ SELECT
     id,
     name,
     value
-FROM 
+FROM
 (
     SELECT
         date,
@@ -373,7 +373,7 @@ SELECT
     date,
     name,
     value
-FROM 
+FROM
 (
     SELECT
         date,
@@ -383,7 +383,7 @@ FROM
     FROM test_00597
     WHERE id = 1
 )
-ANY LEFT JOIN 
+ANY LEFT JOIN
 (
     SELECT id
     FROM test_00597
@@ -395,11 +395,11 @@ SELECT
     date,
     name,
     value
-FROM 
+FROM
 (
     SELECT toInt8(1) AS id
 )
-ANY LEFT JOIN 
+ANY LEFT JOIN
 (
     SELECT
         date,
@@ -411,7 +411,7 @@ ANY LEFT JOIN
 WHERE value = 1
 1	2000-01-01	test string 1	1
 SELECT value
-FROM 
+FROM
 (
     SELECT toInt8(1) AS id
 )
@@ -423,7 +423,7 @@ SELECT
     id,
     name,
     value
-FROM 
+FROM
 (
     SELECT
         date,
@@ -433,7 +433,7 @@ FROM
         date,
         name,
         value
-    FROM 
+    FROM
     (
         SELECT
             date,
@@ -443,7 +443,7 @@ FROM
         FROM test_00597
         WHERE id = 1
     )
-    ANY LEFT JOIN 
+    ANY LEFT JOIN
     (
         SELECT id
         FROM test_00597
@@ -460,7 +460,7 @@ SELECT
     b.date,
     b.name,
     b.value
-FROM 
+FROM
 (
     SELECT
         date,
@@ -469,7 +469,7 @@ FROM
         value
     FROM test_00597
 )
-ANY LEFT JOIN 
+ANY LEFT JOIN
 (
     SELECT
         date,
@@ -485,7 +485,7 @@ SELECT
     date,
     name,
     value
-FROM 
+FROM
 (
     SELECT
         toInt8(1) AS id,
@@ -493,7 +493,7 @@ FROM
     FROM system.numbers
     LIMIT 1
 )
-ANY LEFT JOIN 
+ANY LEFT JOIN
 (
     SELECT
         date,
@@ -513,7 +513,7 @@ SELECT
     `b.id`,
     `b.name`,
     `b.value`
-FROM 
+FROM
 (
     SELECT
         date,
@@ -524,7 +524,7 @@ FROM
         b.id,
         b.name,
         b.value
-    FROM 
+    FROM
     (
         SELECT
             date,
@@ -534,7 +534,7 @@ FROM
         FROM test_00597
         WHERE id = 1
     ) AS a
-    ANY LEFT JOIN 
+    ANY LEFT JOIN
     (
         SELECT
             date,
@@ -555,7 +555,7 @@ SELECT
     r.date,
     r.name,
     r.value
-FROM 
+FROM
 (
     SELECT
         date,
@@ -564,14 +564,14 @@ FROM
         value
     FROM test_00597
 )
-SEMI LEFT JOIN 
+SEMI LEFT JOIN
 (
     SELECT
         date,
         id,
         name,
         value
-    FROM 
+    FROM
     (
         SELECT
             date,
@@ -586,7 +586,7 @@ SEMI LEFT JOIN
 WHERE r.id = 1
 2000-01-01	1	test string 1	1	2000-01-01	test string 1	1
 SELECT value + t1.value AS expr
-FROM 
+FROM
 (
     SELECT
         value,
diff --git a/tests/queries/0_stateless/00599_create_view_with_subquery.reference b/tests/queries/0_stateless/00599_create_view_with_subquery.reference
index d83d2837a18..0458f650fd0 100644
--- a/tests/queries/0_stateless/00599_create_view_with_subquery.reference
+++ b/tests/queries/0_stateless/00599_create_view_with_subquery.reference
@@ -1 +1 @@
-CREATE VIEW default.test_view_00599\n(\n    `id` UInt64\n) AS\nSELECT *\nFROM default.test_00599\nWHERE id = \n(\n    SELECT 1\n)
+CREATE VIEW default.test_view_00599\n(\n    `id` UInt64\n) AS\nSELECT *\nFROM default.test_00599\nWHERE id = (\n    SELECT 1\n)
diff --git a/tests/queries/0_stateless/00618_nullable_in.sql b/tests/queries/0_stateless/00618_nullable_in.sql
index 72e166dc0f5..f039f1fb9d5 100644
--- a/tests/queries/0_stateless/00618_nullable_in.sql
+++ b/tests/queries/0_stateless/00618_nullable_in.sql
@@ -5,7 +5,7 @@ SELECT
     uniqExact(x) AS u, 
     uniqExactIf(x, name = 'a') AS ue, 
     uniqExactIf(x, name IN ('a', 'b')) AS ui
-FROM 
+FROM
 (
     SELECT
         toNullable('a') AS name, 
diff --git a/tests/queries/0_stateless/00743_limit_by_not_found_column.sql b/tests/queries/0_stateless/00743_limit_by_not_found_column.sql
index d20b3b0209e..831d67f624b 100644
--- a/tests/queries/0_stateless/00743_limit_by_not_found_column.sql
+++ b/tests/queries/0_stateless/00743_limit_by_not_found_column.sql
@@ -22,7 +22,7 @@ DROP TABLE installation_stats;
 CREATE TEMPORARY TABLE Accounts (AccountID UInt64, Currency String);
 
 SELECT AccountID
-FROM 
+FROM
 (
     SELECT
         AccountID, 
diff --git a/tests/queries/0_stateless/00751_default_databasename_for_view.reference b/tests/queries/0_stateless/00751_default_databasename_for_view.reference
index 76d5cee02e2..b3f1875ae91 100644
--- a/tests/queries/0_stateless/00751_default_databasename_for_view.reference
+++ b/tests/queries/0_stateless/00751_default_databasename_for_view.reference
@@ -12,14 +12,11 @@ SELECT
     platform,
     app
 FROM test_00751.t_00751
-WHERE (app = 
-(
+WHERE (app = (
     SELECT min(app)
     FROM test_00751.u_00751
-)) AND (platform = 
-(
-    SELECT 
-        (
+)) AND (platform = (
+    SELECT (
             SELECT min(platform)
             FROM test_00751.v_00751
         )
diff --git a/tests/queries/0_stateless/00808_not_optimize_predicate.reference b/tests/queries/0_stateless/00808_not_optimize_predicate.reference
index d8ab9425aab..647c6d91890 100644
--- a/tests/queries/0_stateless/00808_not_optimize_predicate.reference
+++ b/tests/queries/0_stateless/00808_not_optimize_predicate.reference
@@ -18,7 +18,7 @@
 SELECT
     n,
     `finalizeAggregation(s)`
-FROM 
+FROM
 (
     SELECT
         n,
diff --git a/tests/queries/0_stateless/00849_multiple_comma_join_2.reference b/tests/queries/0_stateless/00849_multiple_comma_join_2.reference
index fc39ef13935..5709db44eb1 100644
--- a/tests/queries/0_stateless/00849_multiple_comma_join_2.reference
+++ b/tests/queries/0_stateless/00849_multiple_comma_join_2.reference
@@ -10,7 +10,7 @@ FROM t1
 ALL INNER JOIN t2 ON b = t2.b
 WHERE b = t2.b
 SELECT `--t1.a` AS `t1.a`
-FROM 
+FROM
 (
     SELECT
         a AS `--t1.a`,
@@ -21,7 +21,7 @@ FROM
 ALL INNER JOIN t3 ON `--t1.a` = a
 WHERE (`--t1.a` = `--t2.a`) AND (`--t1.a` = a)
 SELECT `--t1.a` AS `t1.a`
-FROM 
+FROM
 (
     SELECT
         b AS `--t1.b`,
@@ -33,13 +33,13 @@ FROM
 ALL INNER JOIN t3 ON `--t1.b` = b
 WHERE (`--t1.b` = `--t2.b`) AND (`--t1.b` = b)
 SELECT `--t1.a` AS `t1.a`
-FROM 
+FROM
 (
     SELECT
         `--t1.a`,
         `--t2.a`,
         a AS `--t3.a`
-    FROM 
+    FROM
     (
         SELECT
             a AS `--t1.a`,
@@ -52,14 +52,14 @@ FROM
 ALL INNER JOIN t4 ON `--t1.a` = a
 WHERE (`--t1.a` = `--t2.a`) AND (`--t1.a` = `--t3.a`) AND (`--t1.a` = a)
 SELECT `--t1.a` AS `t1.a`
-FROM 
+FROM
 (
     SELECT
         `--t1.b`,
         `--t1.a`,
         `--t2.b`,
         b AS `--t3.b`
-    FROM 
+    FROM
     (
         SELECT
             b AS `--t1.b`,
@@ -73,13 +73,13 @@ FROM
 ALL INNER JOIN t4 ON `--t1.b` = b
 WHERE (`--t1.b` = `--t2.b`) AND (`--t1.b` = `--t3.b`) AND (`--t1.b` = b)
 SELECT `--t1.a` AS `t1.a`
-FROM 
+FROM
 (
     SELECT
         `--t1.a`,
         `--t2.a`,
         a AS `--t3.a`
-    FROM 
+    FROM
     (
         SELECT
             a AS `--t1.a`,
@@ -92,13 +92,13 @@ FROM
 ALL INNER JOIN t4 ON `--t2.a` = a
 WHERE (`--t2.a` = `--t1.a`) AND (`--t2.a` = `--t3.a`) AND (`--t2.a` = a)
 SELECT `--t1.a` AS `t1.a`
-FROM 
+FROM
 (
     SELECT
         `--t1.a`,
         `--t2.a`,
         a AS `--t3.a`
-    FROM 
+    FROM
     (
         SELECT
             a AS `--t1.a`,
@@ -111,13 +111,13 @@ FROM
 ALL INNER JOIN t4 ON `--t3.a` = a
 WHERE (`--t3.a` = `--t1.a`) AND (`--t3.a` = `--t2.a`) AND (`--t3.a` = a)
 SELECT `--t1.a` AS `t1.a`
-FROM 
+FROM
 (
     SELECT
         `--t1.a`,
         `--t2.a`,
         a AS `--t3.a`
-    FROM 
+    FROM
     (
         SELECT
             a AS `--t1.a`,
@@ -130,13 +130,13 @@ FROM
 ALL INNER JOIN t4 ON (a = `--t1.a`) AND (a = `--t2.a`) AND (a = `--t3.a`)
 WHERE (a = `--t1.a`) AND (a = `--t2.a`) AND (a = `--t3.a`)
 SELECT `--t1.a` AS `t1.a`
-FROM 
+FROM
 (
     SELECT
         `--t1.a`,
         `--t2.a`,
         a AS `--t3.a`
-    FROM 
+    FROM
     (
         SELECT
             a AS `--t1.a`,
@@ -149,10 +149,10 @@ FROM
 ALL INNER JOIN t4 ON `--t3.a` = a
 WHERE (`--t1.a` = `--t2.a`) AND (`--t2.a` = `--t3.a`) AND (`--t3.a` = a)
 SELECT `--t1.a` AS `t1.a`
-FROM 
+FROM
 (
     SELECT `--t1.a`
-    FROM 
+    FROM
     (
         SELECT a AS `--t1.a`
         FROM t1
@@ -162,10 +162,10 @@ FROM
 ) AS `--.s`
 CROSS JOIN t4
 SELECT `--t1.a` AS `t1.a`
-FROM 
+FROM
 (
     SELECT `--t1.a`
-    FROM 
+    FROM
     (
         SELECT a AS `--t1.a`
         FROM t1
@@ -175,7 +175,7 @@ FROM
 ) AS `--.s`
 CROSS JOIN t4
 SELECT `--t1.a` AS `t1.a`
-FROM 
+FROM
 (
     SELECT a AS `--t1.a`
     FROM t1
@@ -183,7 +183,7 @@ FROM
 ) AS `--.s`
 CROSS JOIN t3
 SELECT `--t1.a` AS `t1.a`
-FROM 
+FROM
 (
     SELECT
         a AS `--t1.a`,
diff --git a/tests/queries/0_stateless/00957_format_with_clashed_aliases.reference b/tests/queries/0_stateless/00957_format_with_clashed_aliases.reference
index d6e53c8b48b..cffcb8482ad 100644
--- a/tests/queries/0_stateless/00957_format_with_clashed_aliases.reference
+++ b/tests/queries/0_stateless/00957_format_with_clashed_aliases.reference
@@ -1,7 +1,7 @@
 SELECT
     1 AS x,
     x.y
-FROM 
+FROM
 (
     SELECT 'Hello, world' AS y
 ) AS x
diff --git a/tests/queries/0_stateless/01029_early_constant_folding.reference b/tests/queries/0_stateless/01029_early_constant_folding.reference
index 8a2d7e6c61a..6063e08afe0 100644
--- a/tests/queries/0_stateless/01029_early_constant_folding.reference
+++ b/tests/queries/0_stateless/01029_early_constant_folding.reference
@@ -4,8 +4,7 @@ SELECT 1
 SELECT 1
 WHERE (1 IN (0, 2)) AND (2 = (identity(CAST(2, \'UInt8\')) AS subquery))
 SELECT 1
-WHERE 1 IN (
-(
+WHERE 1 IN ((
     SELECT arrayJoin([1, 2, 3])
 ) AS subquery)
 SELECT 1
diff --git a/tests/queries/0_stateless/01054_cache_dictionary_overflow_cell.sql b/tests/queries/0_stateless/01054_cache_dictionary_overflow_cell.sql
index b040a0e7a50..d8d1d61be63 100644
--- a/tests/queries/0_stateless/01054_cache_dictionary_overflow_cell.sql
+++ b/tests/queries/0_stateless/01054_cache_dictionary_overflow_cell.sql
@@ -48,7 +48,7 @@ dictGet('one_cell_cache_ints_overflow', 'i8', toUInt64(19)),
 dictGet('one_cell_cache_ints_overflow', 'i8', toUInt64(20));
 
 SELECT arrayMap(x -> dictGet('one_cell_cache_ints_overflow', 'i8', toUInt64(x)), array)
-FROM 
+FROM
 (
     SELECT [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20] AS array
 );
diff --git a/tests/queries/0_stateless/01056_predicate_optimizer_bugs.reference b/tests/queries/0_stateless/01056_predicate_optimizer_bugs.reference
index 4227af86be7..7df4dc7ead5 100644
--- a/tests/queries/0_stateless/01056_predicate_optimizer_bugs.reference
+++ b/tests/queries/0_stateless/01056_predicate_optimizer_bugs.reference
@@ -3,14 +3,14 @@ SELECT
     v,
     d,
     i
-FROM 
+FROM
 (
     SELECT
         t.1 AS k,
         t.2 AS v,
         runningDifference(v) AS d,
         runningDifference(cityHash64(t.1)) AS i
-    FROM 
+    FROM
     (
         SELECT arrayJoin([(\'a\', 1), (\'a\', 2), (\'a\', 3), (\'b\', 11), (\'b\', 13), (\'b\', 15)]) AS t
     )
@@ -26,14 +26,14 @@ SELECT
     co2,
     co3,
     num
-FROM 
+FROM
 (
     SELECT
         co,
         co2,
         co3,
         count() AS num
-    FROM 
+    FROM
     (
         SELECT
             1 AS co,
@@ -51,13 +51,13 @@ WHERE (co != 0) AND (co2 != 2)
 1	0	3	1
 1	0	0	1
 SELECT alias AS name
-FROM 
+FROM
 (
     SELECT name AS alias
     FROM system.settings
     WHERE alias = \'enable_optimize_predicate_expression\'
 )
-ANY INNER JOIN 
+ANY INNER JOIN
 (
     SELECT name
     FROM system.settings
@@ -66,17 +66,17 @@ WHERE name = \'enable_optimize_predicate_expression\'
 enable_optimize_predicate_expression
 1	val11	val21	val31
 SELECT ccc
-FROM 
+FROM
 (
     SELECT 1 AS ccc
     WHERE 0
     UNION ALL
     SELECT ccc
-    FROM 
+    FROM
     (
         SELECT 2 AS ccc
     )
-    ANY INNER JOIN 
+    ANY INNER JOIN
     (
         SELECT 2 AS ccc
     ) USING (ccc)
@@ -91,7 +91,7 @@ SELECT
     b.ts,
     b.id,
     id_c
-FROM 
+FROM
 (
     SELECT
         ts,
@@ -109,7 +109,7 @@ SELECT
     b.ts AS `--b.ts`,
     b.id AS `--b.id`,
     id_c AS `--b.id_c`
-FROM 
+FROM
 (
     SELECT
         ts,
@@ -129,7 +129,7 @@ WHERE `--a.ts` <= toDateTime(\'1970-01-01 03:00:00\')
 2	3
 4	5
 SELECT dummy
-FROM 
+FROM
 (
     SELECT dummy
     FROM system.one
@@ -141,13 +141,13 @@ SELECT
     id,
     value,
     value_1
-FROM 
+FROM
 (
     SELECT
         1 AS id,
         2 AS value
 )
-ALL INNER JOIN 
+ALL INNER JOIN
 (
     SELECT
         1 AS id,
diff --git a/tests/queries/0_stateless/01067_join_null.sql b/tests/queries/0_stateless/01067_join_null.sql
index bc888fe01ba..a00c345128c 100644
--- a/tests/queries/0_stateless/01067_join_null.sql
+++ b/tests/queries/0_stateless/01067_join_null.sql
@@ -1,5 +1,5 @@
 SELECT id
-FROM 
+FROM
 (
     SELECT 1 AS id
     UNION ALL
@@ -20,7 +20,7 @@ ORDER BY id;
 SELECT '---';
 
 SELECT *
-FROM 
+FROM
 (
     SELECT NULL AS x
 ) js1
@@ -32,7 +32,7 @@ INNER JOIN
 SELECT '---';
 
 SELECT *
-FROM 
+FROM
 (
     SELECT NULL AS x
 ) js1
diff --git a/tests/queries/0_stateless/01076_predicate_optimizer_with_view.reference b/tests/queries/0_stateless/01076_predicate_optimizer_with_view.reference
index dfab41b5e4c..620c5c7c8d1 100644
--- a/tests/queries/0_stateless/01076_predicate_optimizer_with_view.reference
+++ b/tests/queries/0_stateless/01076_predicate_optimizer_with_view.reference
@@ -3,7 +3,7 @@ SELECT
     id,
     name,
     value
-FROM 
+FROM
 (
     SELECT *
     FROM default.test
@@ -15,7 +15,7 @@ SELECT
     id,
     name,
     value
-FROM 
+FROM
 (
     SELECT *
     FROM default.test
@@ -23,7 +23,7 @@ FROM
 ) AS test_view
 WHERE id = 2
 SELECT id
-FROM 
+FROM
 (
     SELECT *
     FROM default.test
@@ -31,7 +31,7 @@ FROM
 ) AS test_view
 WHERE id = 1
 SELECT id
-FROM 
+FROM
 (
     SELECT *
     FROM default.test
diff --git a/tests/queries/0_stateless/01083_expressions_in_engine_arguments.reference b/tests/queries/0_stateless/01083_expressions_in_engine_arguments.reference
index 2a5d7e6da32..b25cfadd0ec 100644
--- a/tests/queries/0_stateless/01083_expressions_in_engine_arguments.reference
+++ b/tests/queries/0_stateless/01083_expressions_in_engine_arguments.reference
@@ -6,6 +6,6 @@ CREATE TABLE default.distributed\n(\n    `n` Int8\n)\nENGINE = Distributed(\'tes
 CREATE TABLE default.distributed_tf\n(\n    `n` Int8\n) AS cluster(\'test_shard_localhost\', \'default\', \'buffer\')
 CREATE TABLE default.url\n(\n    `n` UInt64,\n    `col` String\n)\nENGINE = URL(\'https://localhost:8443/?query=select+n,+_table+from+default.merge+format+CSV\', \'CSV\')
 CREATE TABLE default.rich_syntax\n(\n    `n` Int64\n) AS remote(\'localhos{x|y|t}\', cluster(\'test_shard_localhost\', remote(\'127.0.0.{1..4}\', \'default\', \'view\')))
-CREATE VIEW default.view\n(\n    `n` Int64\n) AS\nSELECT toInt64(n) AS n\nFROM \n(\n    SELECT toString(n) AS n\n    FROM default.merge\n    WHERE _table != \'qwerty\'\n    ORDER BY _table ASC\n)\nUNION ALL\nSELECT *\nFROM default.file
+CREATE VIEW default.view\n(\n    `n` Int64\n) AS\nSELECT toInt64(n) AS n\nFROM\n(\n    SELECT toString(n) AS n\n    FROM default.merge\n    WHERE _table != \'qwerty\'\n    ORDER BY _table ASC\n)\nUNION ALL\nSELECT *\nFROM default.file
 CREATE DICTIONARY default.dict\n(\n    `n` UInt64,\n    `col` String DEFAULT \'42\'\n)\nPRIMARY KEY n\nSOURCE(CLICKHOUSE(HOST \'localhost\' PORT 9440 SECURE 1 USER \'default\' TABLE \'url\'))\nLIFETIME(MIN 0 MAX 1)\nLAYOUT(CACHE(SIZE_IN_CELLS 1))
 16
diff --git a/tests/queries/0_stateless/01245_limit_infinite_sources.sql b/tests/queries/0_stateless/01245_limit_infinite_sources.sql
index 803a2d14c39..05680d86a33 100644
--- a/tests/queries/0_stateless/01245_limit_infinite_sources.sql
+++ b/tests/queries/0_stateless/01245_limit_infinite_sources.sql
@@ -1,5 +1,5 @@
 SELECT number
-FROM 
+FROM
 (
     SELECT zero AS number
     FROM remote('127.0.0.2', system.zeros)
diff --git a/tests/queries/0_stateless/01271_optimize_arithmetic_operations_in_aggr_func.reference b/tests/queries/0_stateless/01271_optimize_arithmetic_operations_in_aggr_func.reference
index 669221005f4..b50519b9b3a 100644
--- a/tests/queries/0_stateless/01271_optimize_arithmetic_operations_in_aggr_func.reference
+++ b/tests/queries/0_stateless/01271_optimize_arithmetic_operations_in_aggr_func.reference
@@ -3,7 +3,7 @@ SELECT
     sum(1 + n),
     sum(n - 1),
     sum(1 - n)
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -13,7 +13,7 @@ SELECT
     2 * sum(n),
     sum(n) / 2,
     sum(1 / n)
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -23,7 +23,7 @@ SELECT
     1 + min(n),
     min(n) - 1,
     1 - min(n)
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -33,7 +33,7 @@ SELECT
     2 * min(n),
     min(n) / 2,
     min(1 / n)
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -43,7 +43,7 @@ SELECT
     1 + max(n),
     max(n) - 1,
     1 - max(n)
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -53,7 +53,7 @@ SELECT
     2 * max(n),
     max(n) / 2,
     max(1 / n)
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -63,7 +63,7 @@ SELECT
     sum(-1 + n),
     sum(n - -1),
     sum(-1 - n)
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -73,7 +73,7 @@ SELECT
     -2 * sum(n),
     sum(n) / -2,
     sum(-1 / n)
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -83,7 +83,7 @@ SELECT
     -1 + min(n),
     min(n) - -1,
     -1 - min(n)
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -93,7 +93,7 @@ SELECT
     -2 * max(n),
     max(n) / -2,
     min(-1 / n)
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -103,7 +103,7 @@ SELECT
     -1 + max(n),
     max(n) - -1,
     -1 - max(n)
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -113,7 +113,7 @@ SELECT
     -2 * min(n),
     min(n) / -2,
     max(-1 / n)
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -123,7 +123,7 @@ SELECT
     sum(abs(2) + n),
     sum(n - abs(2)),
     sum(1 - abs(2))
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -133,7 +133,7 @@ SELECT
     sum(abs(2) * n),
     sum(n / abs(2)),
     sum(1 / abs(2))
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -143,7 +143,7 @@ SELECT
     min(abs(2) + n),
     min(n - abs(2)),
     1 - min(abs(2))
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -153,7 +153,7 @@ SELECT
     min(abs(2) * n),
     min(n / abs(2)),
     min(1 / abs(2))
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -163,7 +163,7 @@ SELECT
     max(abs(2) + n),
     max(n - abs(2)),
     1 - max(abs(2))
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -173,7 +173,7 @@ SELECT
     max(abs(2) * n),
     max(n / abs(2)),
     max(1 / abs(2))
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -183,7 +183,7 @@ SELECT
     sum(abs(n) + n),
     sum(n - abs(n)),
     sum(1 - abs(n))
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -193,7 +193,7 @@ SELECT
     sum(abs(n) * n),
     sum(n / abs(n)),
     sum(1 / abs(n))
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -203,7 +203,7 @@ SELECT
     min(abs(n) + n),
     min(n - abs(n)),
     1 - min(abs(n))
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -213,7 +213,7 @@ SELECT
     min(abs(n) * n),
     min(n / abs(n)),
     min(1 / abs(n))
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -223,7 +223,7 @@ SELECT
     max(abs(n) + n),
     max(n - abs(n)),
     1 - max(abs(n))
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -233,7 +233,7 @@ SELECT
     max(abs(n) * n),
     max(n / abs(n)),
     max(1 / abs(n))
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -243,7 +243,7 @@ SELECT
     sum(1 + (n * n)),
     sum((n * n) - 1),
     sum(1 - (n * n))
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -253,7 +253,7 @@ SELECT
     sum((2 * n) * n),
     sum(n * n) / 2,
     sum((1 / n) * n)
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -263,7 +263,7 @@ SELECT
     1 + min(n * n),
     min(n * n) - 1,
     1 - min(n * n)
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -273,7 +273,7 @@ SELECT
     min((2 * n) * n),
     min(n * n) / 2,
     min((1 / n) * n)
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -283,7 +283,7 @@ SELECT
     1 + max(n * n),
     max(n * n) - 1,
     1 - max(n * n)
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -293,7 +293,7 @@ SELECT
     max((2 * n) * n),
     max(n * n) / 2,
     max((1 / n) * n)
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -303,7 +303,7 @@ SELECT
     sum((1 + 1) + n),
     sum((1 + n) - 1),
     sum((1 + 1) - n)
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -313,7 +313,7 @@ SELECT
     sum(1 + (2 * n)),
     sum(1 + (n / 2)),
     sum(1 + (1 / n))
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -323,7 +323,7 @@ SELECT
     min((1 + 1) + n),
     (1 + min(n)) - 1,
     min((1 + 1) - n)
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -333,7 +333,7 @@ SELECT
     1 + min(2 * n),
     1 + min(n / 2),
     1 + min(1 / n)
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -343,7 +343,7 @@ SELECT
     max((1 + 1) + n),
     (1 + max(n)) - 1,
     max((1 + 1) - n)
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -353,7 +353,7 @@ SELECT
     1 + max(2 * n),
     1 + max(n / 2),
     1 + max(1 / n)
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -363,7 +363,7 @@ SELECT
     sum((-1 + n) + -1),
     sum((n - -1) + -1),
     sum((-1 - n) + -1)
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -373,7 +373,7 @@ SELECT
     (-2 * sum(n)) * -1,
     (sum(n) / -2) / -1,
     sum(-1 / n) / -1
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -383,7 +383,7 @@ SELECT
     (-1 + min(n)) + -1,
     (min(n) - -1) + -1,
     (-1 - min(n)) + -1
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -393,7 +393,7 @@ SELECT
     (-2 * min(n)) * -1,
     (min(n) / -2) / -1,
     max(-1 / n) / -1
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -403,7 +403,7 @@ SELECT
     (-1 + max(n)) + -1,
     (max(n) - -1) + -1,
     (-1 - max(n)) + -1
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
@@ -413,43 +413,43 @@ SELECT
     (-2 * max(n)) * -1,
     (max(n) / -2) / -1,
     min(-1 / n) / -1
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
 )
 SELECT ((sum(n + 1) + sum(1 + n)) + sum(n - 1)) + sum(1 - n)
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
 )
 SELECT (((sum(n) * 2) + (2 * sum(n))) + (sum(n) / 2)) + sum(1 / n)
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
 )
 SELECT (((min(n) + 1) + (1 + min(n))) + (min(n) - 1)) + (1 - min(n))
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
 )
 SELECT (((min(n) * 2) + (2 * min(n))) + (min(n) / 2)) + min(1 / n)
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
 )
 SELECT (((max(n) + 1) + (1 + max(n))) + (max(n) - 1)) + (1 - max(n))
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
 )
 SELECT (((max(n) * 2) + (2 * max(n))) + (max(n) / 2)) + max(1 / n)
-FROM 
+FROM
 (
     SELECT number AS n
     FROM numbers(10)
diff --git a/tests/queries/0_stateless/01305_duplicate_order_by_and_distinct.reference b/tests/queries/0_stateless/01305_duplicate_order_by_and_distinct.reference
index c076e56a91b..10f8bbfd392 100644
--- a/tests/queries/0_stateless/01305_duplicate_order_by_and_distinct.reference
+++ b/tests/queries/0_stateless/01305_duplicate_order_by_and_distinct.reference
@@ -1,8 +1,8 @@
 SELECT number
-FROM 
+FROM
 (
     SELECT number
-    FROM 
+    FROM
     (
         SELECT DISTINCT number
         FROM numbers(3)
@@ -13,10 +13,10 @@ ORDER BY number ASC
 1
 2
 SELECT DISTINCT number
-FROM 
+FROM
 (
     SELECT DISTINCT number
-    FROM 
+    FROM
     (
         SELECT DISTINCT number
         FROM numbers(3)
@@ -29,10 +29,10 @@ ORDER BY number ASC
 1
 2
 SELECT number
-FROM 
+FROM
 (
     SELECT number
-    FROM 
+    FROM
     (
         SELECT DISTINCT number % 2 AS number
         FROM numbers(3)
@@ -42,10 +42,10 @@ ORDER BY number ASC
 0
 1
 SELECT DISTINCT number
-FROM 
+FROM
 (
     SELECT DISTINCT number
-    FROM 
+    FROM
     (
         SELECT DISTINCT number % 2 AS number
         FROM numbers(3)
diff --git a/tests/queries/0_stateless/01321_aggregate_functions_of_group_by_keys.reference b/tests/queries/0_stateless/01321_aggregate_functions_of_group_by_keys.reference
index 5eaaf24208e..555593d04bc 100644
--- a/tests/queries/0_stateless/01321_aggregate_functions_of_group_by_keys.reference
+++ b/tests/queries/0_stateless/01321_aggregate_functions_of_group_by_keys.reference
@@ -74,7 +74,7 @@ GROUP BY
     number % 5
 ORDER BY a ASC
 SELECT foo
-FROM 
+FROM
 (
     SELECT number AS foo
     FROM numbers(1)
@@ -155,7 +155,7 @@ GROUP BY
     number % 5
 ORDER BY a ASC
 SELECT foo
-FROM 
+FROM
 (
     SELECT anyLast(number) AS foo
     FROM numbers(1)
diff --git a/tests/queries/0_stateless/01323_redundant_functions_in_order_by.reference b/tests/queries/0_stateless/01323_redundant_functions_in_order_by.reference
index c8421b9869e..b32ad433730 100644
--- a/tests/queries/0_stateless/01323_redundant_functions_in_order_by.reference
+++ b/tests/queries/0_stateless/01323_redundant_functions_in_order_by.reference
@@ -16,21 +16,21 @@
 2	2
 3	3
 SELECT groupArray(x)
-FROM 
+FROM
 (
     SELECT number AS x
     FROM numbers(3)
     ORDER BY x ASC
 )
 SELECT groupArray(x)
-FROM 
+FROM
 (
     SELECT number AS x
     FROM numbers(3)
     ORDER BY x ASC
 )
 SELECT groupArray(x)
-FROM 
+FROM
 (
     SELECT number AS x
     FROM numbers(3)
@@ -43,7 +43,7 @@ SELECT
     a,
     b,
     c
-FROM 
+FROM
 (
     SELECT number + 2 AS key
     FROM numbers(4)
@@ -84,7 +84,7 @@ ORDER BY
 2	2
 3	3
 SELECT groupArray(x)
-FROM 
+FROM
 (
     SELECT number AS x
     FROM numbers(3)
@@ -93,7 +93,7 @@ FROM
         exp(x) ASC
 )
 SELECT groupArray(x)
-FROM 
+FROM
 (
     SELECT number AS x
     FROM numbers(3)
@@ -102,7 +102,7 @@ FROM
         exp(exp(x)) ASC
 )
 SELECT groupArray(x)
-FROM 
+FROM
 (
     SELECT number AS x
     FROM numbers(3)
@@ -115,7 +115,7 @@ SELECT
     a,
     b,
     c
-FROM 
+FROM
 (
     SELECT number + 2 AS key
     FROM numbers(4)
diff --git a/tests/queries/0_stateless/01372_wrong_order_by_removal.reference b/tests/queries/0_stateless/01372_wrong_order_by_removal.reference
index f1f1bcef6e5..0c4d92a26b9 100644
--- a/tests/queries/0_stateless/01372_wrong_order_by_removal.reference
+++ b/tests/queries/0_stateless/01372_wrong_order_by_removal.reference
@@ -1,7 +1,7 @@
 SELECT
     k,
     groupArrayMovingSum(v)
-FROM 
+FROM
 (
     SELECT
         k,
diff --git a/tests/queries/0_stateless/01390_remove_injective_in_uniq.reference b/tests/queries/0_stateless/01390_remove_injective_in_uniq.reference
index 94e1dbc5da7..eaff98b39cb 100644
--- a/tests/queries/0_stateless/01390_remove_injective_in_uniq.reference
+++ b/tests/queries/0_stateless/01390_remove_injective_in_uniq.reference
@@ -4,7 +4,7 @@ SELECT
     uniqHLL12(x),
     uniqCombined(x),
     uniqCombined64(x)
-FROM 
+FROM
 (
     SELECT number % 2 AS x
     FROM numbers(10)
@@ -15,7 +15,7 @@ SELECT
     uniqHLL12(x + y),
     uniqCombined(x + y),
     uniqCombined64(x + y)
-FROM 
+FROM
 (
     SELECT
         number % 2 AS x,
@@ -28,7 +28,7 @@ SELECT
     uniqHLL12(x),
     uniqCombined(x),
     uniqCombined64(x)
-FROM 
+FROM
 (
     SELECT number % 2 AS x
     FROM numbers(10)
@@ -39,7 +39,7 @@ SELECT
     uniqHLL12(x),
     uniqCombined(x),
     uniqCombined64(x)
-FROM 
+FROM
 (
     SELECT number % 2 AS x
     FROM numbers(10)
@@ -50,7 +50,7 @@ SELECT
     uniqHLL12(x),
     uniqCombined(x),
     uniqCombined64(x)
-FROM 
+FROM
 (
     SELECT number % 2 AS x
     FROM numbers(10)
@@ -61,13 +61,13 @@ SELECT
     uniqHLL12(x),
     uniqCombined(x),
     uniqCombined64(x)
-FROM 
+FROM
 (
     SELECT number % 2 AS x
     FROM numbers(10)
 )
 SELECT uniqExact(x)
-FROM 
+FROM
 (
     SELECT number % 2 AS x
     FROM numbers(10)
@@ -80,7 +80,7 @@ SELECT
     uniqHLL12(x),
     uniqCombined(x),
     uniqCombined64(x)
-FROM 
+FROM
 (
     SELECT number % 2 AS x
     FROM numbers(10)
@@ -91,7 +91,7 @@ SELECT
     uniqHLL12(x + y),
     uniqCombined(x + y),
     uniqCombined64(x + y)
-FROM 
+FROM
 (
     SELECT
         number % 2 AS x,
@@ -104,7 +104,7 @@ SELECT
     uniqHLL12(-x),
     uniqCombined(-x),
     uniqCombined64(-x)
-FROM 
+FROM
 (
     SELECT number % 2 AS x
     FROM numbers(10)
@@ -115,7 +115,7 @@ SELECT
     uniqHLL12(bitNot(x)),
     uniqCombined(bitNot(x)),
     uniqCombined64(bitNot(x))
-FROM 
+FROM
 (
     SELECT number % 2 AS x
     FROM numbers(10)
@@ -126,7 +126,7 @@ SELECT
     uniqHLL12(bitNot(-x)),
     uniqCombined(bitNot(-x)),
     uniqCombined64(bitNot(-x))
-FROM 
+FROM
 (
     SELECT number % 2 AS x
     FROM numbers(10)
@@ -137,13 +137,13 @@ SELECT
     uniqHLL12(-bitNot(-x)),
     uniqCombined(-bitNot(-x)),
     uniqCombined64(-bitNot(-x))
-FROM 
+FROM
 (
     SELECT number % 2 AS x
     FROM numbers(10)
 )
 SELECT uniqExact(-bitNot(-x))
-FROM 
+FROM
 (
     SELECT number % 2 AS x
     FROM numbers(10)
diff --git a/tests/queries/0_stateless/01412_row_from_totals.sql b/tests/queries/0_stateless/01412_row_from_totals.sql
index 170eb06721a..63c2a111369 100644
--- a/tests/queries/0_stateless/01412_row_from_totals.sql
+++ b/tests/queries/0_stateless/01412_row_from_totals.sql
@@ -11,7 +11,7 @@ insert into tracking_events_tmp select 2, '2020-07-10' from numbers(1881);
 insert into tracking_events_tmp select 2, '2020-07-11' from numbers(1623);
 
 SELECT EventDate
-FROM 
+FROM
 (
     SELECT EventDate
     FROM tracking_events_tmp AS t1
diff --git a/tests/queries/0_stateless/01414_push_predicate_when_contains_with_clause.reference b/tests/queries/0_stateless/01414_push_predicate_when_contains_with_clause.reference
index 13c8fe551c7..bbda231662e 100644
--- a/tests/queries/0_stateless/01414_push_predicate_when_contains_with_clause.reference
+++ b/tests/queries/0_stateless/01414_push_predicate_when_contains_with_clause.reference
@@ -3,7 +3,7 @@
 SELECT
     number,
     square_number
-FROM 
+FROM
 (
     WITH number * 2 AS square_number
     SELECT
diff --git a/tests/queries/0_stateless/01455_duplicate_distinct_optimization.reference b/tests/queries/0_stateless/01455_duplicate_distinct_optimization.reference
index 2c54899f9f5..82e887e1b92 100644
--- a/tests/queries/0_stateless/01455_duplicate_distinct_optimization.reference
+++ b/tests/queries/0_stateless/01455_duplicate_distinct_optimization.reference
@@ -1,13 +1,13 @@
 SELECT DISTINCT number
 FROM numbers(1)
 SELECT number
-FROM 
+FROM
 (
     SELECT DISTINCT number
     FROM numbers(1)
 )
 SELECT DISTINCT number * 2
-FROM 
+FROM
 (
     SELECT DISTINCT
         number * 2,
@@ -15,7 +15,7 @@ FROM
     FROM numbers(1)
 )
 SELECT number
-FROM 
+FROM
 (
     SELECT DISTINCT number * 2 AS number
     FROM numbers(1)
@@ -23,7 +23,7 @@ FROM
 SELECT
     b,
     a
-FROM 
+FROM
 (
     SELECT DISTINCT
         number % 2 AS a,
@@ -31,7 +31,7 @@ FROM
     FROM numbers(100)
 )
 SELECT DISTINCT a
-FROM 
+FROM
 (
     SELECT DISTINCT
         number % 2 AS a,
@@ -39,10 +39,10 @@ FROM
     FROM numbers(100)
 )
 SELECT a
-FROM 
+FROM
 (
     SELECT DISTINCT a
-    FROM 
+    FROM
     (
         SELECT DISTINCT
             number % 2 AS a,
@@ -51,12 +51,12 @@ FROM
     )
 )
 SELECT DISTINCT a
-FROM 
+FROM
 (
     SELECT
         a,
         b
-    FROM 
+    FROM
     (
         SELECT DISTINCT
             number % 2 AS a,
@@ -67,12 +67,12 @@ FROM
 SELECT
     a,
     b
-FROM 
+FROM
 (
     SELECT
         b,
         a
-    FROM 
+    FROM
     (
         SELECT DISTINCT
             number AS a,
@@ -83,13 +83,13 @@ FROM
 SELECT
     a,
     b
-FROM 
+FROM
 (
     SELECT
         b,
         a,
         a + b
-    FROM 
+    FROM
     (
         SELECT DISTINCT
             number % 2 AS a,
@@ -98,10 +98,10 @@ FROM
     )
 )
 SELECT DISTINCT a
-FROM 
+FROM
 (
     SELECT a
-    FROM 
+    FROM
     (
         SELECT DISTINCT
             number % 2 AS a,
@@ -110,21 +110,21 @@ FROM
     )
 )
 SELECT DISTINCT number
-FROM 
+FROM
 (
     SELECT DISTINCT number
     FROM numbers(1)
 ) AS t1
 CROSS JOIN numbers(2) AS t2
 SELECT number
-FROM 
+FROM
 (
     SELECT DISTINCT number
     FROM numbers(1) AS t1
     CROSS JOIN numbers(2) AS t2
 )
 SELECT DISTINCT number
-FROM 
+FROM
 (
     SELECT DISTINCT number
     FROM numbers(1)
diff --git a/tests/queries/0_stateless/01495_subqueries_in_with_statement_4.reference b/tests/queries/0_stateless/01495_subqueries_in_with_statement_4.reference
index 6a713b14181..c83f681fc54 100644
--- a/tests/queries/0_stateless/01495_subqueries_in_with_statement_4.reference
+++ b/tests/queries/0_stateless/01495_subqueries_in_with_statement_4.reference
@@ -1,5 +1,5 @@
 0	0
-WITH it AS 
+WITH it AS
     (
         SELECT *
         FROM numbers(1)
@@ -7,4 +7,5 @@ WITH it AS
 SELECT
     number,
     number
-FROM it AS i
+FROM
+it AS i
diff --git a/tests/queries/0_stateless/01515_with_global_and_with_propagation.reference b/tests/queries/0_stateless/01515_with_global_and_with_propagation.reference
index 76c46dd798f..dd611b45ccd 100644
--- a/tests/queries/0_stateless/01515_with_global_and_with_propagation.reference
+++ b/tests/queries/0_stateless/01515_with_global_and_with_propagation.reference
@@ -13,7 +13,7 @@ WITH 1 AS x
 SELECT x
 WITH 1 AS x
 SELECT x
-FROM 
+FROM
 (
     WITH 1 AS x
     SELECT x
@@ -22,7 +22,7 @@ WITH 1 AS x
 SELECT
     y,
     x
-FROM 
+FROM
 (
     WITH 2 AS x
     SELECT 2 AS y
@@ -39,7 +39,7 @@ WITH 2 AS x
 SELECT x
 WITH
     5 AS q1,
-    x AS 
+    x AS
     (
         WITH 5 AS q1
         SELECT
@@ -51,4 +51,5 @@ WITH
 SELECT
     b,
     a
-FROM x
+FROM
+x
diff --git a/tests/queries/0_stateless/01532_having_with_totals.sql b/tests/queries/0_stateless/01532_having_with_totals.sql
index 10f55c8c135..290799c1354 100644
--- a/tests/queries/0_stateless/01532_having_with_totals.sql
+++ b/tests/queries/0_stateless/01532_having_with_totals.sql
@@ -3,7 +3,7 @@ create table local_t engine Log as select 1 a;
 
 SELECT '127.0.0.{1,2}';
 SELECT *
-FROM 
+FROM
 (
     SELECT a
     FROM remote('127.0.0.{1,2}', currentDatabase(), local_t)
@@ -17,7 +17,7 @@ WHERE a IN
 
 SELECT '127.0.0.1';
 SELECT *
-FROM 
+FROM
 (
     SELECT a
     FROM remote('127.0.0.1', currentDatabase(), local_t)
diff --git a/tests/queries/0_stateless/01532_min_max_with_modifiers.sql b/tests/queries/0_stateless/01532_min_max_with_modifiers.sql
index 0c8651c0f01..364b110d8c1 100644
--- a/tests/queries/0_stateless/01532_min_max_with_modifiers.sql
+++ b/tests/queries/0_stateless/01532_min_max_with_modifiers.sql
@@ -11,7 +11,7 @@ SELECT
     min(x) AS lower,
     max(x) + 1 AS upper,
     upper - lower AS range
-FROM 
+FROM
 (
     SELECT arrayJoin([1, 2]) AS x
 ) 
diff --git a/tests/queries/0_stateless/01559_aggregate_null_for_empty_fix.sql b/tests/queries/0_stateless/01559_aggregate_null_for_empty_fix.sql
index 5955dee37f8..3434a049073 100644
--- a/tests/queries/0_stateless/01559_aggregate_null_for_empty_fix.sql
+++ b/tests/queries/0_stateless/01559_aggregate_null_for_empty_fix.sql
@@ -1,5 +1,5 @@
 SELECT MAX(aggr)
-FROM 
+FROM
 (
 	SELECT MAX(-1) AS aggr
 	FROM system.one
@@ -11,7 +11,7 @@ FROM
 
 );
 SELECT MaX(aggr)
-FROM 
+FROM
 (
     SELECT mAX(-1) AS aggr
     FROM system.one
@@ -22,7 +22,7 @@ FROM
     WHERE 1
 );
 SELECT MaX(aggr)
-FROM 
+FROM
 (
     SELECT mAX(-1) AS aggr
     FROM system.one
@@ -33,7 +33,7 @@ FROM
     WHERE 1
 );
 SELECT MaX(aggr)
-FROM 
+FROM
 (
     SELECT mAX(-1) AS aggr
     FROM system.one
@@ -45,7 +45,7 @@ FROM
 );
 SET aggregate_functions_null_for_empty=1;
 SELECT MAX(aggr)
-FROM 
+FROM
 (
 	SELECT MAX(-1) AS aggr
 	FROM system.one
@@ -57,7 +57,7 @@ FROM
 
 );
 SELECT MaX(aggr)
-FROM 
+FROM
 (
     SELECT mAX(-1) AS aggr
     FROM system.one
@@ -68,7 +68,7 @@ FROM
     WHERE 1
 );
 SELECT MaX(aggr)
-FROM 
+FROM
 (
     SELECT mAX(-1) AS aggr
     FROM system.one
@@ -79,7 +79,7 @@ FROM
     WHERE 1
 );
 SELECT MaX(aggr)
-FROM 
+FROM
 (
     SELECT mAX(-1) AS aggr
     FROM system.one
diff --git a/tests/queries/0_stateless/01560_DateTime_and_DateTime64_comparision.sql b/tests/queries/0_stateless/01560_DateTime_and_DateTime64_comparision.sql
index c5e677570ea..2c3ae41864e 100644
--- a/tests/queries/0_stateless/01560_DateTime_and_DateTime64_comparision.sql
+++ b/tests/queries/0_stateless/01560_DateTime_and_DateTime64_comparision.sql
@@ -31,7 +31,7 @@ SELECT
     dt64 != dt,
     toDateTime(dt64) != dt,
     dt64 != toDateTime64(dt, 1, 'UTC')
-FROM 
+FROM
 (
     WITH toDateTime('2015-05-18 07:40:11') as value
     SELECT
diff --git a/tests/queries/0_stateless/01560_monotonicity_check_multiple_args_bug.sql b/tests/queries/0_stateless/01560_monotonicity_check_multiple_args_bug.sql
index befc13be8eb..b475a5bdd3c 100644
--- a/tests/queries/0_stateless/01560_monotonicity_check_multiple_args_bug.sql
+++ b/tests/queries/0_stateless/01560_monotonicity_check_multiple_args_bug.sql
@@ -1,7 +1,7 @@
 WITH arrayJoin(range(2)) AS delta
 SELECT
     toDate(time) + toIntervalDay(delta) AS dt
-FROM 
+FROM
 (
     SELECT toDateTime('2020.11.12 19:02:04') AS time
 )
@@ -10,7 +10,7 @@ ORDER BY dt ASC;
 WITH arrayJoin([0, 1]) AS delta
 SELECT
     toDate(time) + toIntervalDay(delta) AS dt
-FROM 
+FROM
 (
     SELECT toDateTime('2020.11.12 19:02:04') AS time
 )
diff --git a/tests/queries/0_stateless/01582_deterministic_function_with_predicate.reference b/tests/queries/0_stateless/01582_deterministic_function_with_predicate.reference
index a1b4bfdcc03..b764691f5b6 100644
--- a/tests/queries/0_stateless/01582_deterministic_function_with_predicate.reference
+++ b/tests/queries/0_stateless/01582_deterministic_function_with_predicate.reference
@@ -1,8 +1,8 @@
 SELECT count()
-FROM 
+FROM
 (
     SELECT number
-    FROM 
+    FROM
     (
         SELECT number
         FROM numbers(1000000)
diff --git a/tests/queries/0_stateless/01593_functions_in_order_by.reference b/tests/queries/0_stateless/01593_functions_in_order_by.reference
index 534b29af3e6..4f4ea02f95c 100644
--- a/tests/queries/0_stateless/01593_functions_in_order_by.reference
+++ b/tests/queries/0_stateless/01593_functions_in_order_by.reference
@@ -1,7 +1,7 @@
 SELECT
     msg,
     toDateTime(intDiv(ms, 1000)) AS time
-FROM 
+FROM
 (
     SELECT
         \'hello\' AS msg,
diff --git a/tests/queries/0_stateless/01593_functions_in_order_by.sql b/tests/queries/0_stateless/01593_functions_in_order_by.sql
index 2d38e45fff7..5aa6aef9e12 100644
--- a/tests/queries/0_stateless/01593_functions_in_order_by.sql
+++ b/tests/queries/0_stateless/01593_functions_in_order_by.sql
@@ -1,6 +1,6 @@
 EXPLAIN SYNTAX
 SELECT msg, toDateTime(intDiv(ms, 1000)) AS time
-FROM 
+FROM
 (
     SELECT
         'hello' AS msg,
diff --git a/tests/queries/0_stateless/01635_nullable_fuzz.sql b/tests/queries/0_stateless/01635_nullable_fuzz.sql
index c134578b221..f49fe39d350 100644
--- a/tests/queries/0_stateless/01635_nullable_fuzz.sql
+++ b/tests/queries/0_stateless/01635_nullable_fuzz.sql
@@ -4,7 +4,7 @@ SELECT
     '',
     number,
     NULL AS k
-FROM 
+FROM
 (
     SELECT
         materialize(NULL) OR materialize(-9223372036854775808),
diff --git a/tests/queries/0_stateless/01636_nullable_fuzz2.sql b/tests/queries/0_stateless/01636_nullable_fuzz2.sql
index a40da51c38c..49ee7626b4e 100644
--- a/tests/queries/0_stateless/01636_nullable_fuzz2.sql
+++ b/tests/queries/0_stateless/01636_nullable_fuzz2.sql
@@ -13,7 +13,7 @@ insert into tracking_events_tmp select 2, '2020-07-10' from numbers(1881);
 insert into tracking_events_tmp select 2, '2020-07-11' from numbers(1623);
 
 SELECT EventDate
-FROM 
+FROM
 (
     SELECT EventDate
     FROM tracking_events_tmp AS t1
diff --git a/tests/queries/0_stateless/01650_expressions_merge_bug.sql b/tests/queries/0_stateless/01650_expressions_merge_bug.sql
index f28f663b2d4..3cab4dbd5a6 100644
--- a/tests/queries/0_stateless/01650_expressions_merge_bug.sql
+++ b/tests/queries/0_stateless/01650_expressions_merge_bug.sql
@@ -7,7 +7,7 @@ SELECT
             9223372036854775807
     ),
     NULL
-FROM 
+FROM
 (
     SELECT DISTINCT
         NULL,
diff --git a/tests/queries/0_stateless/01670_neighbor_lc_bug.sql b/tests/queries/0_stateless/01670_neighbor_lc_bug.sql
index de9afcf1495..2d99225aa89 100644
--- a/tests/queries/0_stateless/01670_neighbor_lc_bug.sql
+++ b/tests/queries/0_stateless/01670_neighbor_lc_bug.sql
@@ -33,7 +33,7 @@ SELECT
     val_low,
     neighbor(val_low, -1) AS low_m1,
     neighbor(val_low, 1) AS low_p1
-FROM 
+FROM
 (
     SELECT *
     FROM neighbor_test
diff --git a/tests/queries/0_stateless/01732_explain_syntax_union_query.reference b/tests/queries/0_stateless/01732_explain_syntax_union_query.reference
index fe5eb01a7ed..ccafa916b9f 100644
--- a/tests/queries/0_stateless/01732_explain_syntax_union_query.reference
+++ b/tests/queries/0_stateless/01732_explain_syntax_union_query.reference
@@ -7,7 +7,7 @@ UNION ALL
 SELECT 1
 UNION ALL
 SELECT 1
- 
+-
 SELECT 1
 UNION ALL
 (
@@ -19,9 +19,9 @@ UNION ALL
 )
 UNION ALL
 SELECT 1
- 
+-
 SELECT x
-FROM 
+FROM
 (
     SELECT 1 AS x
     UNION ALL
@@ -35,9 +35,9 @@ FROM
     UNION ALL
     SELECT 1
 )
- 
+-
 SELECT x
-FROM 
+FROM
 (
     SELECT 1 AS x
     UNION ALL
@@ -45,15 +45,15 @@ FROM
     UNION ALL
     SELECT 1
 )
- 
+-
 SELECT 1
 UNION DISTINCT
 SELECT 1
 UNION DISTINCT
 SELECT 1
- 
+-
 SELECT 1
- 
+-
 
 (
     SELECT 1
diff --git a/tests/queries/0_stateless/01732_explain_syntax_union_query.sql b/tests/queries/0_stateless/01732_explain_syntax_union_query.sql
index 0dd1e19e765..c35021090f1 100644
--- a/tests/queries/0_stateless/01732_explain_syntax_union_query.sql
+++ b/tests/queries/0_stateless/01732_explain_syntax_union_query.sql
@@ -13,7 +13,7 @@ UNION ALL
     SELECT 1
 );
 
-SELECT ' ';
+SELECT '-';
 
 EXPLAIN SYNTAX
 SELECT 1
@@ -30,7 +30,7 @@ UNION ALL
     SELECT 1
 );
 
-SELECT ' ';
+SELECT '-';
 
 EXPLAIN SYNTAX
 SELECT x
@@ -51,11 +51,11 @@ FROM
     )
 );
 
-SELECT ' ';
+SELECT '-';
 
 EXPLAIN SYNTAX
 SELECT x
-FROM 
+FROM
 (
     SELECT 1 AS x
     UNION ALL
@@ -66,7 +66,7 @@ FROM
     )
 );
 
-SELECT ' ';
+SELECT '-';
 
 EXPLAIN SYNTAX
 SELECT 1
@@ -75,12 +75,12 @@ SELECT 1
 UNION DISTINCT
 SELECT 1;
 
-SELECT ' ';
+SELECT '-';
 
 EXPLAIN SYNTAX
 (((((((((((((((SELECT 1)))))))))))))));
 
-SELECT ' ';
+SELECT '-';
 
 EXPLAIN SYNTAX
 (((((((((((((((SELECT 1 UNION DISTINCT SELECT 1))) UNION DISTINCT SELECT 1)))) UNION ALL SELECT 1))))))));
diff --git a/tests/queries/0_stateless/01754_clickhouse_format_backslash.reference b/tests/queries/0_stateless/01754_clickhouse_format_backslash.reference
index 328483d9867..28519b2f92f 100644
--- a/tests/queries/0_stateless/01754_clickhouse_format_backslash.reference
+++ b/tests/queries/0_stateless/01754_clickhouse_format_backslash.reference
@@ -1,5 +1,5 @@
 SELECT * \
-FROM  \
+FROM \
 ( \
     SELECT 1 AS x \
     UNION ALL \
diff --git a/tests/queries/0_stateless/01798_uniq_theta_sketch.reference b/tests/queries/0_stateless/01798_uniq_theta_sketch.reference
index e5f3fe4911e..0455a06036f 100644
--- a/tests/queries/0_stateless/01798_uniq_theta_sketch.reference
+++ b/tests/queries/0_stateless/01798_uniq_theta_sketch.reference
@@ -32,13 +32,13 @@ uniqTheta decimals
 (101,101,101)
 uniqTheta remove injective
 SELECT uniqTheta(x)
-FROM 
+FROM
 (
     SELECT number % 2 AS x
     FROM numbers(10)
 )
 SELECT uniqTheta(x + y)
-FROM 
+FROM
 (
     SELECT
         number % 2 AS x,
@@ -46,37 +46,37 @@ FROM
     FROM numbers(10)
 )
 SELECT uniqTheta(x)
-FROM 
+FROM
 (
     SELECT number % 2 AS x
     FROM numbers(10)
 )
 SELECT uniqTheta(x)
-FROM 
+FROM
 (
     SELECT number % 2 AS x
     FROM numbers(10)
 )
 SELECT uniqTheta(x)
-FROM 
+FROM
 (
     SELECT number % 2 AS x
     FROM numbers(10)
 )
 SELECT uniqTheta(x)
-FROM 
+FROM
 (
     SELECT number % 2 AS x
     FROM numbers(10)
 )
 SELECT uniqTheta(x)
-FROM 
+FROM
 (
     SELECT number % 2 AS x
     FROM numbers(10)
 )
 SELECT uniqTheta(x + y)
-FROM 
+FROM
 (
     SELECT
         number % 2 AS x,
@@ -84,25 +84,25 @@ FROM
     FROM numbers(10)
 )
 SELECT uniqTheta(-x)
-FROM 
+FROM
 (
     SELECT number % 2 AS x
     FROM numbers(10)
 )
 SELECT uniqTheta(bitNot(x))
-FROM 
+FROM
 (
     SELECT number % 2 AS x
     FROM numbers(10)
 )
 SELECT uniqTheta(bitNot(-x))
-FROM 
+FROM
 (
     SELECT number % 2 AS x
     FROM numbers(10)
 )
 SELECT uniqTheta(-bitNot(-x))
-FROM 
+FROM
 (
     SELECT number % 2 AS x
     FROM numbers(10)
diff --git a/tests/queries/0_stateless/01874_select_from_trailing_whitespaces.reference b/tests/queries/0_stateless/01874_select_from_trailing_whitespaces.reference
new file mode 100644
index 00000000000..a52505659d1
--- /dev/null
+++ b/tests/queries/0_stateless/01874_select_from_trailing_whitespaces.reference
@@ -0,0 +1,114 @@
+# select * from system.one a
+SELECT *
+FROM system.one AS a
+# /* oneline */ select * from system.one a
+SELECT * FROM system.one AS a
+# select * from (select * from system.one) b, system.one a
+SELECT *
+FROM
+(
+    SELECT *
+    FROM system.one
+) AS b, system.one AS a
+# /* oneline */ select * from (select * from system.one) b, system.one a
+SELECT * FROM (SELECT * FROM system.one) AS b, system.one AS a
+# select * from system.one a, (select * from system.one) b, system.one c
+SELECT *
+FROM system.one AS a,
+(
+    SELECT *
+    FROM system.one
+) AS b, system.one AS c
+# /* oneline */ select * from system.one a, (select * from system.one) b, system.one c
+SELECT * FROM system.one AS a, (SELECT * FROM system.one) AS b, system.one AS c
+# select * from system.one a, (select * from system.one) b, system.one c, (select * from system.one) d
+SELECT *
+FROM system.one AS a,
+(
+    SELECT *
+    FROM system.one
+) AS b, system.one AS c,
+(
+    SELECT *
+    FROM system.one
+) AS d
+# /* oneline */ select * from system.one a, (select * from system.one) b, system.one c, (select * from system.one) d
+SELECT * FROM system.one AS a, (SELECT * FROM system.one) AS b, system.one AS c, (SELECT * FROM system.one) AS d
+# select * from system.one union all select * from system.one
+SELECT *
+FROM system.one
+UNION ALL
+SELECT *
+FROM system.one
+# /* oneline */ select * from system.one union all select * from system.one
+SELECT * FROM system.one UNION ALL SELECT * FROM system.one
+# select * from system.one union all (select * from system.one)
+SELECT *
+FROM system.one
+UNION ALL
+SELECT *
+FROM system.one
+# /* oneline */ select * from system.one union all (select * from system.one)
+SELECT * FROM system.one UNION ALL SELECT * FROM system.one
+# select 1 union all (select 1 union distinct select 1)
+SELECT 1
+UNION ALL
+(
+    SELECT 1
+    UNION DISTINCT
+    SELECT 1
+)
+# /* oneline */ select 1 union all (select 1 union distinct select 1)
+SELECT 1 UNION ALL (SELECT 1 UNION DISTINCT SELECT 1)
+# select * from system.one array join arr as row
+SELECT *
+FROM system.one
+ARRAY JOIN arr AS row
+# /* oneline */ select * from system.one array join arr as row
+SELECT * FROM system.one ARRAY JOIN arr AS row
+# select 1 in 1
+SELECT 1 IN (1)
+# /* oneline */ select 1 in 1
+SELECT 1 IN (1)
+# select 1 in (select 1)
+SELECT 1 IN (
+        SELECT 1
+    )
+# /* oneline */ select 1 in (select 1)
+SELECT 1 IN (SELECT 1)
+# select 1 in f(1)
+SELECT 1 IN f(1)
+# /* oneline */ select 1 in f(1)
+SELECT 1 IN f(1)
+# select 1 in ((select 1) as sub)
+SELECT 1 IN ((
+        SELECT 1
+    ) AS sub)
+# /* oneline */ select 1 in ((select 1) as sub)
+SELECT 1 IN ((SELECT 1) AS sub)
+# with it as ( select * from numbers(1) ) select it.number, i.number from it as i
+WITH it AS
+    (
+        SELECT *
+        FROM numbers(1)
+    )
+SELECT
+    it.number,
+    i.number
+FROM it AS i
+# /* oneline */ with it as ( select * from numbers(1) ) select it.number, i.number from it as i
+WITH it AS (SELECT * FROM numbers(1)) SELECT it.number, i.number FROM it AS i
+# SELECT x FROM ( SELECT 1 AS x UNION ALL ( SELECT 1 UNION ALL SELECT 1))
+SELECT x
+FROM
+(
+    SELECT 1 AS x
+    UNION ALL
+    (
+        SELECT 1
+        UNION ALL
+        SELECT 1
+    )
+)
+# /* oneline */ SELECT x FROM ( SELECT 1 AS x UNION ALL ( SELECT 1 UNION ALL SELECT 1))
+SELECT x FROM (SELECT 1 AS x UNION ALL (SELECT 1 UNION ALL SELECT 1))
diff --git a/tests/queries/0_stateless/01874_select_from_trailing_whitespaces.sh b/tests/queries/0_stateless/01874_select_from_trailing_whitespaces.sh
new file mode 100755
index 00000000000..f416ac33d69
--- /dev/null
+++ b/tests/queries/0_stateless/01874_select_from_trailing_whitespaces.sh
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+set -e
+
+queries=(
+    "select * from system.one a"
+    "select * from (select * from system.one) b, system.one a"
+    "select * from system.one a, (select * from system.one) b, system.one c"
+    "select * from system.one a, (select * from system.one) b, system.one c, (select * from system.one) d"
+    "select * from system.one union all select * from system.one"
+    "select * from system.one union all (select * from system.one)"
+    "select 1 union all (select 1 union distinct select 1)"
+    "select * from system.one array join arr as row"
+    "select 1 in 1"
+    "select 1 in (select 1)"
+    "select 1 in f(1)"
+    "select 1 in ((select 1) as sub)"
+    "with it as ( select * from numbers(1) ) select it.number, i.number from it as i"
+    "SELECT x FROM ( SELECT 1 AS x UNION ALL ( SELECT 1 UNION ALL SELECT 1))"
+)
+for q in "${queries[@]}"; do
+    echo "# $q"
+    $CLICKHOUSE_FORMAT <<<"$q"
+    echo "# /* oneline */ $q"
+    $CLICKHOUSE_FORMAT --oneline <<<"$q"
+done
diff --git a/tests/queries/1_stateful/00031_array_enumerate_uniq.sql b/tests/queries/1_stateful/00031_array_enumerate_uniq.sql
index 1898fc20579..f0392a17024 100644
--- a/tests/queries/1_stateful/00031_array_enumerate_uniq.sql
+++ b/tests/queries/1_stateful/00031_array_enumerate_uniq.sql
@@ -1,5 +1,5 @@
 SELECT UserID, arrayEnumerateUniq(groupArray(SearchPhrase)) AS arr
-FROM 
+FROM
 (
     SELECT UserID, SearchPhrase
     FROM test.hits
diff --git a/tests/queries/1_stateful/00063_loyalty_joins.sql b/tests/queries/1_stateful/00063_loyalty_joins.sql
index 7713c65838c..1e7011ea909 100644
--- a/tests/queries/1_stateful/00063_loyalty_joins.sql
+++ b/tests/queries/1_stateful/00063_loyalty_joins.sql
@@ -23,7 +23,7 @@ ORDER BY loyalty ASC;
 SELECT
     loyalty, 
     count()
-FROM 
+FROM
 (
     SELECT UserID
     FROM test.hits
@@ -46,12 +46,12 @@ ORDER BY loyalty ASC;
 SELECT
     loyalty, 
     count()
-FROM 
+FROM
 (
     SELECT
         loyalty, 
         UserID
-    FROM 
+    FROM
     (
         SELECT UserID
         FROM test.hits
@@ -81,7 +81,7 @@ FROM test.hits ANY INNER JOIN
     SELECT
         UserID, 
         toInt8(if(yandex > google, yandex / (yandex + google), -google / (yandex + google)) * 10) AS loyalty
-    FROM 
+    FROM
     (
         SELECT
             UserID, 

From 59d4d42b5ef73e48f6f8ba9ba2617852ad4151cf Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Tue, 18 May 2021 07:02:55 +0300
Subject: [PATCH 38/76] Mark some tests as long

---
 ...s4.reference => 00534_functions_bad_arguments4_long.reference} | 0
 ...s_bad_arguments4.sh => 00534_functions_bad_arguments4_long.sh} | 0
 ...edicate.reference => 00597_push_down_predicate_long.reference} | 0
 ...push_down_predicate.sql => 00597_push_down_predicate_long.sql} | 0
 ...71_optimize_arithmetic_operations_in_aggr_func_long.reference} | 0
 ...=> 01271_optimize_arithmetic_operations_in_aggr_func_long.sql} | 0
 6 files changed, 0 insertions(+), 0 deletions(-)
 rename tests/queries/0_stateless/{00534_functions_bad_arguments4.reference => 00534_functions_bad_arguments4_long.reference} (100%)
 rename tests/queries/0_stateless/{00534_functions_bad_arguments4.sh => 00534_functions_bad_arguments4_long.sh} (100%)
 rename tests/queries/0_stateless/{00597_push_down_predicate.reference => 00597_push_down_predicate_long.reference} (100%)
 rename tests/queries/0_stateless/{00597_push_down_predicate.sql => 00597_push_down_predicate_long.sql} (100%)
 rename tests/queries/0_stateless/{01271_optimize_arithmetic_operations_in_aggr_func.reference => 01271_optimize_arithmetic_operations_in_aggr_func_long.reference} (100%)
 rename tests/queries/0_stateless/{01271_optimize_arithmetic_operations_in_aggr_func.sql => 01271_optimize_arithmetic_operations_in_aggr_func_long.sql} (100%)

diff --git a/tests/queries/0_stateless/00534_functions_bad_arguments4.reference b/tests/queries/0_stateless/00534_functions_bad_arguments4_long.reference
similarity index 100%
rename from tests/queries/0_stateless/00534_functions_bad_arguments4.reference
rename to tests/queries/0_stateless/00534_functions_bad_arguments4_long.reference
diff --git a/tests/queries/0_stateless/00534_functions_bad_arguments4.sh b/tests/queries/0_stateless/00534_functions_bad_arguments4_long.sh
similarity index 100%
rename from tests/queries/0_stateless/00534_functions_bad_arguments4.sh
rename to tests/queries/0_stateless/00534_functions_bad_arguments4_long.sh
diff --git a/tests/queries/0_stateless/00597_push_down_predicate.reference b/tests/queries/0_stateless/00597_push_down_predicate_long.reference
similarity index 100%
rename from tests/queries/0_stateless/00597_push_down_predicate.reference
rename to tests/queries/0_stateless/00597_push_down_predicate_long.reference
diff --git a/tests/queries/0_stateless/00597_push_down_predicate.sql b/tests/queries/0_stateless/00597_push_down_predicate_long.sql
similarity index 100%
rename from tests/queries/0_stateless/00597_push_down_predicate.sql
rename to tests/queries/0_stateless/00597_push_down_predicate_long.sql
diff --git a/tests/queries/0_stateless/01271_optimize_arithmetic_operations_in_aggr_func.reference b/tests/queries/0_stateless/01271_optimize_arithmetic_operations_in_aggr_func_long.reference
similarity index 100%
rename from tests/queries/0_stateless/01271_optimize_arithmetic_operations_in_aggr_func.reference
rename to tests/queries/0_stateless/01271_optimize_arithmetic_operations_in_aggr_func_long.reference
diff --git a/tests/queries/0_stateless/01271_optimize_arithmetic_operations_in_aggr_func.sql b/tests/queries/0_stateless/01271_optimize_arithmetic_operations_in_aggr_func_long.sql
similarity index 100%
rename from tests/queries/0_stateless/01271_optimize_arithmetic_operations_in_aggr_func.sql
rename to tests/queries/0_stateless/01271_optimize_arithmetic_operations_in_aggr_func_long.sql

From e66e3e92714ec9b371b2b7d1112d3924b729d754 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Thu, 20 May 2021 21:11:12 +0300
Subject: [PATCH 39/76] Fix style

---
 tests/clickhouse-test | 78 +++++++++++++++++++++++--------------------
 1 file changed, 41 insertions(+), 37 deletions(-)

diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index 21034931bc4..e1231a61ad9 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -689,6 +689,46 @@ def collect_build_flags(client):
     return result
 
 
+def do_run_tests(jobs, suite, suite_dir, suite_tmp_dir, all_tests, parallel_tests, sequential_tests, parallel):
+    if jobs > 1 and len(parallel_tests) > 0:
+        print("Found", len(parallel_tests), "parallel tests and", len(sequential_tests), "sequential tests")
+        run_n, run_total = parallel.split('/')
+        run_n = float(run_n)
+        run_total = float(run_total)
+        tests_n = len(parallel_tests)
+        if run_total > tests_n:
+            run_total = tests_n
+
+        if jobs > tests_n:
+            jobs = tests_n
+        if jobs > run_total:
+            run_total = jobs
+
+        batch_size = max(1, len(parallel_tests) // jobs)
+        parallel_tests_array = []
+        for _ in range(jobs):
+            parallel_tests_array.append((None, batch_size, suite, suite_dir, suite_tmp_dir))
+
+        with closing(multiprocessing.Pool(processes=jobs)) as pool:
+            pool.map_async(run_tests_array, parallel_tests_array)
+
+            for suit in parallel_tests:
+                queue.put(suit)
+
+            for _ in range(jobs):
+                queue.put(None)
+
+            queue.close()
+
+        pool.join()
+
+        run_tests_array((sequential_tests, len(sequential_tests), suite, suite_dir, suite_tmp_dir))
+        total_tests_run += len(sequential_tests) + len(parallel_tests)
+    else:
+        run_tests_array((all_tests, len(all_tests), suite, suite_dir, suite_tmp_dir))
+        total_tests_run += len(all_tests)
+
+
 def main(args):
     global SERVER_DIED
     global stop_time
@@ -852,43 +892,7 @@ def main(args):
                 else:
                     parallel_tests.append(test)
 
-            if jobs > 1 and len(parallel_tests) > 0:
-                print("Found", len(parallel_tests), "parallel tests and", len(sequential_tests), "sequential tests")
-                run_n, run_total = args.parallel.split('/')
-                run_n = float(run_n)
-                run_total = float(run_total)
-                tests_n = len(parallel_tests)
-                if run_total > tests_n:
-                    run_total = tests_n
-
-                if jobs > tests_n:
-                    jobs = tests_n
-                if jobs > run_total:
-                    run_total = jobs
-
-                batch_size = max(1, len(parallel_tests) // jobs)
-                parallel_tests_array = []
-                for i in range(jobs):
-                    parallel_tests_array.append((None, batch_size, suite, suite_dir, suite_tmp_dir))
-
-                with closing(multiprocessing.Pool(processes=jobs)) as pool:
-                    pool.map_async(run_tests_array, parallel_tests_array)
-
-                    for suit in parallel_tests:
-                        queue.put(suit)
-
-                    for i in range(jobs):
-                        queue.put(None)
-
-                    queue.close()
-
-                pool.join()
-
-                run_tests_array((sequential_tests, len(sequential_tests), suite, suite_dir, suite_tmp_dir))
-                total_tests_run += len(sequential_tests) + len(parallel_tests)
-            else:
-                run_tests_array((all_tests, len(all_tests), suite, suite_dir, suite_tmp_dir))
-                total_tests_run += len(all_tests)
+            do_run_tests(jobs, suite, suite_dir, suite_tmp_dir, all_tests, parallel_tests, sequential_tests, args.parallel)
 
     if args.hung_check:
 

From 87e73a970e81ccbaf8e15168b93857159bde6c48 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Thu, 20 May 2021 21:18:44 +0300
Subject: [PATCH 40/76] Fix style

---
 tests/clickhouse-test | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index e1231a61ad9..6e4ae145e76 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -723,10 +723,10 @@ def do_run_tests(jobs, suite, suite_dir, suite_tmp_dir, all_tests, parallel_test
         pool.join()
 
         run_tests_array((sequential_tests, len(sequential_tests), suite, suite_dir, suite_tmp_dir))
-        total_tests_run += len(sequential_tests) + len(parallel_tests)
+        return len(sequential_tests) + len(parallel_tests)
     else:
         run_tests_array((all_tests, len(all_tests), suite, suite_dir, suite_tmp_dir))
-        total_tests_run += len(all_tests)
+        return len(all_tests)
 
 
 def main(args):
@@ -892,7 +892,8 @@ def main(args):
                 else:
                     parallel_tests.append(test)
 
-            do_run_tests(jobs, suite, suite_dir, suite_tmp_dir, all_tests, parallel_tests, sequential_tests, args.parallel)
+            total_tests_run += do_run_tests(
+                jobs, suite, suite_dir, suite_tmp_dir, all_tests, parallel_tests, sequential_tests, args.parallel)
 
     if args.hung_check:
 

From 7f8b8e6ade44429aee2f141bbcbdeb0d8b2dc55e Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@yandex-team.ru>
Date: Thu, 20 May 2021 22:20:07 +0300
Subject: [PATCH 41/76] Auto version update to [21.6.1.6891] [54451]

---
 cmake/autogenerated_versions.txt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt
index 51f4b974161..22d75049e8d 100644
--- a/cmake/autogenerated_versions.txt
+++ b/cmake/autogenerated_versions.txt
@@ -3,7 +3,7 @@ SET(VERSION_REVISION 54451)
 SET(VERSION_MAJOR 21)
 SET(VERSION_MINOR 6)
 SET(VERSION_PATCH 1)
-SET(VERSION_GITHASH 96fced4c3cf432fb0b401d2ab01f0c56e5f74a96)
-SET(VERSION_DESCRIBE v21.6.1.1-prestable)
-SET(VERSION_STRING 21.6.1.1)
+SET(VERSION_GITHASH 976ccc2e908ac3bc28f763bfea8134ea0a121b40)
+SET(VERSION_DESCRIBE v21.6.1.6891-prestable)
+SET(VERSION_STRING 21.6.1.6891)
 # end of autochange

From 471142043ff3515c3d425eb7d294fc43c0c40e2a Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@yandex-team.ru>
Date: Thu, 20 May 2021 22:23:34 +0300
Subject: [PATCH 42/76] Auto version update to [21.7.1.1] [54452]

---
 cmake/autogenerated_versions.txt              |   8 +-
 debian/changelog                              |   4 +-
 docker/client/Dockerfile                      |   2 +-
 docker/server/Dockerfile                      |   2 +-
 docker/test/Dockerfile                        |   2 +-
 .../StorageSystemContributors.generated.cpp   | 693 +++++++++---------
 6 files changed, 371 insertions(+), 340 deletions(-)

diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt
index 22d75049e8d..34de50e9f8a 100644
--- a/cmake/autogenerated_versions.txt
+++ b/cmake/autogenerated_versions.txt
@@ -1,9 +1,9 @@
 # This strings autochanged from release_lib.sh:
-SET(VERSION_REVISION 54451)
+SET(VERSION_REVISION 54452)
 SET(VERSION_MAJOR 21)
-SET(VERSION_MINOR 6)
+SET(VERSION_MINOR 7)
 SET(VERSION_PATCH 1)
 SET(VERSION_GITHASH 976ccc2e908ac3bc28f763bfea8134ea0a121b40)
-SET(VERSION_DESCRIBE v21.6.1.6891-prestable)
-SET(VERSION_STRING 21.6.1.6891)
+SET(VERSION_DESCRIBE v21.7.1.1-prestable)
+SET(VERSION_STRING 21.7.1.1)
 # end of autochange
diff --git a/debian/changelog b/debian/changelog
index 8b6626416a9..e1c46dae3a8 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,5 @@
-clickhouse (21.6.1.1) unstable; urgency=low
+clickhouse (21.7.1.1) unstable; urgency=low
 
   * Modified source code
 
- -- clickhouse-release <clickhouse-release@yandex-team.ru>  Tue, 20 Apr 2021 01:48:16 +0300
+ -- clickhouse-release <clickhouse-release@yandex-team.ru>  Thu, 20 May 2021 22:23:29 +0300
diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile
index 569025dec1c..79ac92f2277 100644
--- a/docker/client/Dockerfile
+++ b/docker/client/Dockerfile
@@ -1,7 +1,7 @@
 FROM ubuntu:18.04
 
 ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/"
-ARG version=21.6.1.*
+ARG version=21.7.1.*
 
 RUN apt-get update \
     && apt-get install --yes --no-install-recommends \
diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile
index d302fec7417..52dcb6caae5 100644
--- a/docker/server/Dockerfile
+++ b/docker/server/Dockerfile
@@ -1,7 +1,7 @@
 FROM ubuntu:20.04
 
 ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/"
-ARG version=21.6.1.*
+ARG version=21.7.1.*
 ARG gosu_ver=1.10
 
 # set non-empty deb_location_url url to create a docker image
diff --git a/docker/test/Dockerfile b/docker/test/Dockerfile
index 0e4646386ce..9809a36395d 100644
--- a/docker/test/Dockerfile
+++ b/docker/test/Dockerfile
@@ -1,7 +1,7 @@
 FROM ubuntu:18.04
 
 ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/"
-ARG version=21.6.1.*
+ARG version=21.7.1.*
 
 RUN apt-get update && \
     apt-get install -y apt-transport-https dirmngr && \
diff --git a/src/Storages/System/StorageSystemContributors.generated.cpp b/src/Storages/System/StorageSystemContributors.generated.cpp
index b8741e6951c..a5524aee6ae 100644
--- a/src/Storages/System/StorageSystemContributors.generated.cpp
+++ b/src/Storages/System/StorageSystemContributors.generated.cpp
@@ -1,34 +1,43 @@
 // autogenerated by ./StorageSystemContributors.sh
 const char * auto_contributors[] {
+    "박현우",
     "0xflotus",
     "20018712",
     "243f6a88 85a308d3",
     "243f6a8885a308d313198a2e037",
     "3ldar-nasyrov",
     "821008736@qq.com",
+    "abdrakhmanov",
+    "abel-wang",
+    "abyss7",
+    "achimbab",
+    "achulkov2",
+    "adevyatova",
+    "ageraab",
+    "akazz",
     "Akazz",
+    "akonyaev",
+    "akuzm",
     "Alain BERRIER",
     "Albert Kidrachev",
     "Alberto",
-    "Aleksandr Karo",
     "Aleksandra (Ася)",
+    "Aleksandr Karo",
     "Aleksandrov Vladimir",
+    "alekseik1",
     "Aleksei Levushkin",
     "Aleksei Semiglazov",
     "Aleksey",
     "Aleksey Akulovich",
+    "alesapin",
     "Alex",
-    "Alex Bocharov",
-    "Alex Karo",
-    "Alex Krash",
-    "Alex Ryndin",
-    "Alex Zatelepin",
     "Alexander Avdonkin",
     "Alexander Burmak",
     "Alexander Ermolaev",
-    "Alexander GQ Gerasiov",
     "Alexander Gololobov",
+    "Alexander GQ Gerasiov",
     "Alexander Kazakov",
+    "alexander kozhikhov",
     "Alexander Kozhikhov",
     "Alexander Krasheninnikov",
     "Alexander Kuranoff",
@@ -44,37 +53,53 @@ const char * auto_contributors[] {
     "Alexander Sapin",
     "Alexander Tokmakov",
     "Alexander Tretiakov",
+    "Alexandra Latysheva",
+    "Alexandre Snarskii",
     "Alexandr Kondratev",
     "Alexandr Krasheninnikov",
     "Alexandr Orlov",
-    "Alexandra Latysheva",
-    "Alexandre Snarskii",
+    "Alex Bocharov",
     "Alexei Averchenko",
     "Alexey",
     "Alexey Arno",
     "Alexey Dushechkin",
     "Alexey Elymanov",
     "Alexey Ilyukhov",
+    "alexey-milovidov",
     "Alexey Milovidov",
     "Alexey Tronov",
     "Alexey Vasiliev",
     "Alexey Zatelepin",
+    "Alex Karo",
+    "Alex Krash",
+    "alex.lvxin",
+    "Alex Ryndin",
     "Alexsey Shestakov",
-    "Ali Demirci",
+    "alex-zaitsev",
+    "Alex Zatelepin",
+    "alfredlu",
     "Aliaksandr Pliutau",
     "Aliaksandr Shylau",
+    "Ali Demirci",
+    "amesaru",
+    "Amesaru",
     "Amos Bird",
+    "amoschen",
+    "amudong",
     "Amy Krishnevsky",
-    "AnaUvarova",
     "Anastasiya Rodigina",
     "Anastasiya Tsarkova",
     "Anatoly Pugachev",
+    "ana-uvarova",
+    "AnaUvarova",
     "AndreevDm",
     "Andrei Bodrov",
     "Andrei Chulkov",
+    "andrei-karpliuk",
     "Andrei Nekrashevich",
     "Andrew Grigorev",
     "Andrew Onyshchuk",
+    "andrewsg",
     "Andrey",
     "Andrey Chulkov",
     "Andrey Dudin",
@@ -85,11 +110,16 @@ const char * auto_contributors[] {
     "Andrey Mironov",
     "Andrey Skobtsov",
     "Andrey Urusov",
+    "Andrey Z",
     "Andy Yang",
     "Anmol Arora",
     "Anna",
     "Anna Shakhova",
+    "annvsh",
+    "anrodigina",
     "Anthony N. Simon",
+    "antikvist",
+    "anton",
     "Anton Ivashkin",
     "Anton Kobzev",
     "Anton Kvasha",
@@ -101,83 +131,136 @@ const char * auto_contributors[] {
     "Anton Tikhonov",
     "Anton Yuzhaninov",
     "Anton Zhabolenko",
+    "ap11",
+    "a.palagashvili",
+    "aprudaev",
     "Ariel Robaldo",
     "Arsen Hakobyan",
     "ArtCorp",
     "Artem Andreenko",
+    "Artemeey",
     "Artem Gavrilov",
     "Artem Hnilov",
+    "Artemkin Pavel",
     "Artem Konovalov",
     "Artem Streltsov",
     "Artem Zuikov",
-    "Artemeey",
-    "Artemkin Pavel",
     "Arthur Petukhovsky",
     "Arthur Tokarchuk",
     "Arthur Wong",
+    "artpaul",
     "Artur",
     "Artur Beglaryan",
     "AsiaKorushkina",
+    "asiana21",
     "Atri Sharma",
+    "avasiliev",
+    "avogar",
     "Avogar",
+    "avsharapov",
+    "awesomeleo",
     "Azat Khuzhin",
-    "BSD_Conqueror",
     "Babacar Diassé",
     "Bakhtiyor Ruziev",
     "BanyRule",
     "Baudouin Giard",
     "BayoNet",
+    "benamazing",
+    "benbiti",
+    "Benjamin Naecker",
     "Bertrand Junqua",
+    "bgranvea",
     "Bharat Nallan",
+    "bharatnc",
     "Big Elephant",
     "Bill",
     "BlahGeek",
+    "blazerer",
+    "bluebirddm",
+    "bobrovskij artemij",
     "Bogdan",
     "Bogdan Voronin",
     "BohuTANG",
     "Bolinov",
+    "booknouse",
     "Boris Granveaud",
     "Bowen Masco",
+    "bo zeng",
     "Brett Hoerner",
+    "BSD_Conqueror",
+    "bseng",
     "Bulat Gaifullin",
     "Carbyn",
+    "cekc",
+    "centos7",
+    "champtar",
+    "chang.chen",
+    "changvvb",
     "Chao Ma",
     "Chao Wang",
+    "chasingegg",
+    "chengy8934",
+    "chenqi",
+    "chenxing-xc",
+    "chenxing.xc",
     "Chen Yufei",
+    "chertus",
     "Chienlung Cheung",
+    "chou.fan",
     "Christian",
     "Ciprian Hacman",
     "Clement Rodriguez",
     "Clément Rodriguez",
     "Colum",
+    "comunodi",
     "Constantin S. Pan",
+    "coraxster",
     "CurtizJ",
-    "DIAOZHAFENG",
+    "damozhaeva",
     "Daniel Bershatsky",
     "Daniel Dao",
     "Daniel Qin",
     "Danila Kutenin",
+    "dankondr",
     "Dao Minh Thuc",
+    "daoready",
     "Daria Mozhaeva",
     "Dario",
-    "DarkWanderer",
     "Darío",
+    "DarkWanderer",
+    "dasmfm",
+    "davydovska",
+    "decaseal",
     "Denis Burlaka",
     "Denis Glazachev",
     "Denis Krivak",
     "Denis Zhuravlev",
     "Denny Crane",
+    "dependabot[bot]",
+    "dependabot-preview[bot]",
     "Derek Perkins",
+    "detailyang",
+    "dfenelonov",
+    "dgrr",
+    "DIAOZHAFENG",
+    "dimarub2000",
     "Ding Xiang Fei",
+    "dinosaur",
+    "divanorama",
+    "dkxiaohei",
+    "dmi-feo",
     "Dmitriev Mikhail",
+    "dmitrii",
     "Dmitrii Kovalkov",
     "Dmitrii Raev",
+    "dmitriiut",
     "Dmitriy",
     "Dmitry",
     "Dmitry Belyavtsev",
     "Dmitry Bilunov",
     "Dmitry Galuza",
     "Dmitry Krylov",
+    "dmitry kuzmin",
     "Dmitry Luhtionov",
     "Dmitry Moskowski",
     "Dmitry Muzyka",
@@ -188,69 +271,126 @@ const char * auto_contributors[] {
     "Dongdong Yang",
     "DoomzD",
     "Dr. Strange Looker",
+    "eaxdev",
+    "eejoin",
+    "egatov",
     "Egor O'Sten",
+    "Egor Savin",
     "Ekaterina",
+    "elBroom",
     "Eldar Zaitov",
     "Elena Baskakova",
+    "elenaspb2019",
     "Elghazal Ahmed",
     "Elizaveta Mironyuk",
+    "emakarov",
+    "emhlbmc",
+    "emironyuk",
     "Emmanuel Donin de Rosière",
     "Eric",
     "Eric Daniel",
     "Erixonich",
+    "ermaotech",
     "Ernest Poletaev",
     "Eugene Klimov",
     "Eugene Konkov",
     "Evgenia Sudarikova",
-    "Evgenii Pravda",
     "Evgeniia Sudarikova",
+    "Evgenii Pravda",
     "Evgeniy Gatov",
     "Evgeniy Udodov",
     "Evgeny Konkov",
     "Evgeny Markov",
+    "evtan",
     "Ewout",
-    "Fabian Stäber",
+    "exprmntr",
+    "ezhaka",
+    "f1yegor",
     "Fabiano Francesconi",
+    "Fabian Stäber",
     "Fadi Hadzh",
     "Fan()",
+    "fancno",
+    "FArthur-cmd",
+    "fastio",
+    "favstovol",
     "FawnD2",
     "FeehanG",
+    "felixoid",
+    "felixxdu",
+    "feng lv",
+    "fenglv",
+    "fessmage",
     "FgoDt",
+    "filimonov",
+    "filipe",
     "Filipe Caixeta",
+    "flow",
     "Flowyi",
+    "flynn",
+    "foxxmary",
     "Francisco Barón",
+    "frank",
+    "franklee",
     "Frank Zhao",
+    "fredchenbj",
     "Fruit of Eden",
     "Fullstop000",
+    "fuqi",
+    "Fuwang Hu",
+    "fuwhu",
+    "Fu Zhe",
+    "fuzhe1989",
     "Gagan Arneja",
     "Gao Qiang",
+    "g-arslan",
     "Gary Dotzler",
     "George",
-    "George G",
     "George3d6",
+    "George G",
     "Gervasio Varela",
+    "ggerogery",
+    "giordyb",
     "Gleb Kanterov",
     "Gleb Novikov",
     "Gleb-Tretyakov",
+    "glockbender",
+    "glushkovds",
     "Gregory",
     "Grigory",
     "Grigory Buteyko",
     "Grigory Pervakov",
     "Guillaume Tassery",
+    "guoleiyi",
+    "gyuton",
     "Haavard Kvaalen",
     "Habibullah Oladepo",
     "Hamoon",
+    "hao.he",
     "Hasitha Kanchana",
     "Hasnat",
+    "hchen9",
+    "hcz",
+    "heng zhao",
+    "hexiaoting",
     "Hiroaki Nakamura",
+    "hotid",
     "HuFuwang",
     "Hui Wang",
+    "hustnn",
+    "huzhichengdd",
+    "ice1x",
+    "idfer",
+    "igor",
     "Igor",
     "Igor Hatarist",
+    "igor.lapko",
     "Igor Mineev",
     "Igor Strykhar",
     "Igr",
     "Igr Mineev",
+    "ikarishinjieva",
+    "ikopylov",
     "Ildar Musin",
     "Ildus Kurbangaliev",
     "Ilya",
@@ -265,42 +405,60 @@ const char * auto_contributors[] {
     "Ilya Skrypitsa",
     "Ilya Yatsishin",
     "ImgBotApp",
+    "imgbot[bot]",
+    "ip",
     "Islam Israfilov",
     "Islam Israfilov (Islam93)",
+    "it1804",
     "Ivan",
     "Ivan A. Torgashov",
     "Ivan Babrou",
     "Ivan Blinkov",
     "Ivan He",
+    "ivan-kush",
     "Ivan Kush",
     "Ivan Kushnarenko",
     "Ivan Lezhankin",
     "Ivan Remen",
     "Ivan Starkov",
+    "ivanzhukov",
     "Ivan Zhukov",
     "JackyWoo",
     "Jacob Hayes",
+    "jakalletti",
     "JaosnHsieh",
     "Jason",
+    "javartisan",
+    "javi",
+    "javi santana",
     "Javi Santana",
     "Javi santana bot",
     "Jean Baptiste Favre",
+    "jennyma",
+    "jetgm",
     "Jiading Guo",
     "Jiang Tao",
+    "jianmei zhang",
     "Jochen Schalanda",
     "John",
     "John Hummel",
     "John Skopis",
     "Jonatas Freitas",
+    "Julian Zhou",
+    "jyz0309",
     "Kang Liu",
     "Karl Pietrzak",
+    "keenwolf",
     "Keiji Yoshida",
     "Ken Chen",
     "Kevin Chiang",
+    "kevin wan",
     "Kiran",
     "Kirill Danshin",
+    "kirillikoff",
     "Kirill Malev",
     "Kirill Shvakov",
+    "kmeaw",
     "Koblikov Mihail",
     "KochetovNicolai",
     "Konstantin Grabar",
@@ -308,30 +466,60 @@ const char * auto_contributors[] {
     "Konstantin Malanchev",
     "Konstantin Podshumok",
     "Korviakov Andrey",
+    "koshachy",
     "Kozlov Ivan",
+    "kreuzerkrieg",
     "Kruglov Pavel",
+    "ks1322",
     "Kseniia Sumarokova",
+    "kshvakov",
+    "kssenii",
+    "l",
+    "lalex",
     "Latysheva Alexandra",
+    "lehasm",
+    "Léo Ercolanelli",
     "Leonardo Cecchi",
     "Leopold Schabel",
+    "leozhang",
     "Lev Borodin",
+    "levushkin aleksej",
+    "levysh",
     "Lewinma",
+    "liangqian",
+    "libenwang",
+    "lichengxiang",
+    "linceyou",
+    "litao91",
+    "liu-bov",
     "Liu Cong",
     "LiuCong",
+    "liuyangkuan",
     "LiuYangkuan",
+    "liuyimin",
+    "liyang",
+    "lomberts",
+    "long2ice",
     "Lopatin Konstantin",
     "Loud_Scream",
+    "luc1ph3r",
+    "Lucid Dreams",
     "Luis Bosque",
+    "lulichao",
     "Lv Feng",
-    "Léo Ercolanelli",
     "M0r64n",
-    "Maks Skorokhod",
+    "madianjun",
+    "maiha",
     "Maksim",
     "Maksim Fedotov",
     "Maksim Kita",
+    "Maks Skorokhod",
+    "malkfilipp",
+    "manmitya",
+    "maqroll",
     "Marat IDRISOV",
-    "Marek Vavrusa",
     "Marek Vavruša",
+    "Marek Vavrusa",
     "Marek Vavruša",
     "Mariano Benítez Mulet",
     "Mark Andreev",
@@ -340,16 +528,19 @@ const char * auto_contributors[] {
     "Maroun Maroun",
     "Marquitos",
     "Marsel Arduanov",
-    "Marti Raudsepp",
     "Martijn Bakker",
+    "Marti Raudsepp",
     "Marvin Taschenberger",
     "Masha",
+    "mastertheknife",
     "Matthew Peveler",
     "Matwey V. Kornilov",
     "Max",
     "Max Akhmedov",
-    "Max Vetrov",
+    "maxim",
     "Maxim Akhmedov",
+    "MaximAL",
+    "maxim-babenko",
     "Maxim Babenko",
     "Maxim Fedotov",
     "Maxim Fridental",
@@ -360,55 +551,81 @@ const char * auto_contributors[] {
     "Maxim Serebryakov",
     "Maxim Smirnov",
     "Maxim Ulanovskiy",
-    "MaximAL",
+    "maxkuzn",
+    "maxulan",
+    "Max Vetrov",
     "Mc.Spring",
+    "mehanizm",
     "MeiK",
+    "melin",
+    "memo",
+    "meo",
+    "mergify[bot]",
     "Metehan Çetinkaya",
     "Metikov Vadim",
+    "mf5137",
+    "mfridental",
     "Michael Furmur",
     "Michael Kolupaev",
     "Michael Monashev",
     "Michael Razuvaev",
     "Michael Smitasin",
     "Michal Lisowski",
+    "michon470",
     "MicrochipQ",
+    "miha-g",
     "Mihail Fandyushin",
     "Mikahil Nacharov",
     "Mike",
     "Mike F",
     "Mike Kot",
+    "mikepop7",
     "Mikhail",
     "Mikhail Cheshkov",
     "Mikhail Fandyushin",
     "Mikhail Filimonov",
+    "Mikhail f. Shiryaev",
     "Mikhail Gaidamaka",
     "Mikhail Korotov",
     "Mikhail Malafeev",
     "Mikhail Nacharov",
     "Mikhail Salosin",
     "Mikhail Surin",
-    "Mikhail f. Shiryaev",
     "MikuSugar",
     "Milad Arabi",
+    "millb",
+    "mnkonkova",
     "Mohammad Hossein Sekhavat",
+    "morty",
+    "moscas",
     "MovElb",
     "Mr.General",
     "Murat Kabilov",
+    "m-ves",
+    "mwish",
     "MyroTk",
-    "NIKITA MIKHAILOV",
+    "myrrc",
+    "nagorny",
     "Narek Galstyan",
-    "NeZeD [Mac Pro]",
+    "nauta",
+    "nautaa",
     "Neeke Gao",
     "Neng Liu",
+    "never lee",
+    "NeZeD [Mac Pro]",
+    "nicelulu",
     "Nickolay Yastrebov",
+    "Nicolae Vartolomei",
     "Nico Mandery",
     "Nico Piderman",
-    "Nicolae Vartolomei",
     "Nik",
     "Nikhil Nadig",
     "Nikhil Raman",
     "Nikita Lapkov",
     "Nikita Mikhailov",
+    "NIKITA MIKHAILOV",
+    "Nikita Mikhalev",
+    "nikitamikhaylov",
     "Nikita Mikhaylov",
     "Nikita Orlov",
     "Nikita Vasilev",
@@ -421,18 +638,29 @@ const char * auto_contributors[] {
     "Nikolay Vasiliev",
     "Nikolay Volosatov",
     "Niu Zhaojie",
+    "nonexistence",
+    "ns-vasilev",
+    "nvartolomei",
+    "oandrew",
+    "objatie_groba",
+    "ocadaruma",
     "Odin Hultgren Van Der Horst",
+    "ogorbacheva",
     "Okada Haruki",
     "Oleg Favstov",
     "Oleg Komarov",
+    "olegkv",
     "Oleg Matrokhin",
     "Oleg Obleukhov",
+    "Oleg Strokachuk",
     "Olga Khvostikova",
+    "olgarev",
     "Olga Revyakina",
+    "orantius",
     "Orivej Desh",
     "Oskar Wojciski",
     "OuO",
-    "PHO",
+    "palasonicq",
     "Paramtamtam",
     "Patrick Zippenfenig",
     "Pavel",
@@ -449,41 +677,67 @@ const char * auto_contributors[] {
     "Persiyanov Dmitriy Andreevich",
     "Pervakov Grigorii",
     "Pervakov Grigory",
+    "peshkurov",
+    "philip.han",
     "Philippe Ombredanne",
+    "PHO",
+    "pingyu",
+    "potya",
     "Potya",
     "Pradeep Chhetri",
+    "proller",
+    "pufit",
+    "pyos",
     "Pysaoke",
+    "qianlixiang",
+    "qianmoQ",
+    "quid",
     "Quid37",
+    "r1j1k",
     "Rafael David Tinoco",
+    "rainbowsysu",
     "Ramazan Polat",
+    "Raúl Marín",
     "Ravengg",
     "RegulusZ",
     "Reilee",
     "Reto Kromer",
     "Ri",
+    "ritaank",
+    "robert",
     "Robert Hodges",
+    "robot-clickhouse",
+    "robot-metrika-test",
+    "rodrigargar",
+    "roman",
     "Roman Bug",
     "Roman Lipovsky",
     "Roman Nikolaev",
     "Roman Nozdrin",
     "Roman Peshkurov",
     "Roman Tsisyk",
+    "romanzhukov",
+    "root",
+    "roverxu",
     "Ruslan",
     "Ruslan Savchenko",
     "Russ Frank",
     "Ruzal Ibragimov",
-    "S.M.A. Djawadi",
     "Sabyanin Maxim",
     "SaltTan",
     "Sami Kerola",
     "Samuel Chou",
+    "santaux",
+    "satanson",
     "Saulius Valatka",
-    "Serg Kulakov",
-    "Serge Rider",
+    "sdk2",
+    "Sébastien Launay",
+    "serebrserg",
     "Sergei Bocharov",
     "Sergei Semin",
     "Sergei Shtykov",
     "Sergei Tsetlin (rekub)",
+    "Serge Rider",
     "Sergey Demurin",
     "Sergey Elantsev",
     "Sergey Fedorov",
@@ -497,63 +751,102 @@ const char * auto_contributors[] {
     "Sergey Zaikin",
     "Sergi Almacellas Abellana",
     "Sergi Vladykin",
+    "Serg Kulakov",
+    "sev7e0",
     "SevaCode",
+    "sevirov",
+    "sfod",
+    "shangshujie",
+    "shedx",
     "Sherry Wang",
     "Silviu Caragea",
     "Simeon Emanuilov",
     "Simon Liu",
     "Simon Podlipsky",
+    "Šimon Podlipský",
+    "simon-says",
     "Sina",
     "Sjoerd Mulder",
     "Slach",
+    "S.M.A. Djawadi",
     "Snow",
     "Sofia Antipushina",
+    "songenjie",
+    "spff",
+    "spongedc",
+    "spyros87",
     "Stanislav Pavlovichev",
     "Stas Pavlovichev",
+    "stavrolia",
     "Stefan Thies",
     "Stepan",
     "Stepan Herold",
+    "stepenhu",
     "Steve-金勇",
     "Stig Bakken",
     "Stupnikov Andrey",
+    "su-houzhen",
+    "sundy",
+    "sundy-li",
+    "sundyli",
     "SuperBot",
-    "Sébastien Launay",
+    "svladykin",
     "TAC",
-    "TCeason",
     "Tagir Kuskarov",
+    "tai",
+    "taichong",
     "Tai White",
+    "taiyang-li",
     "Taleh Zaliyev",
     "Tangaev",
+    "tao jiang",
     "Tatiana Kirillova",
+    "tavplubix",
+    "TCeason",
     "Tema Novikov",
+    "templarzq",
     "The-Alchemist",
+    "tiger.yan",
+    "tison",
     "TiunovNN",
     "Tobias Adamson",
     "Tom Bombadil",
+    "topvisor",
     "Tsarkova Anastasia",
     "TszkitLo40",
+    "turbo jason",
+    "tyrionhuang",
+    "ubuntu",
     "Ubuntu",
     "Ubus",
     "UnamedRus",
+    "unegare",
+    "unknown",
+    "urgordeadbeef",
     "V",
-    "VDimir",
     "Vadim",
+    "VadimPE",
     "Vadim Plakhtinskiy",
     "Vadim Skipin",
-    "VadimPE",
     "Val",
     "Valera Ryaboshapko",
+    "Vasilyev Nikita",
     "Vasily Kozhukhovskiy",
     "Vasily Morozov",
     "Vasily Nemkov",
     "Vasily Okunev",
     "Vasily Vasilkov",
-    "Vasilyev Nikita",
+    "vdimir",
+    "VDimir",
+    "velom",
     "Veloman Yunkan",
     "Veniamin Gvozdikov",
     "Veselkov Konstantin",
+    "vic",
+    "vicdashkov",
     "Victor Tarnavsky",
     "Viktor Taranenko",
+    "vinity",
     "Vitaliy Fedorchenko",
     "Vitaliy Karnienko",
     "Vitaliy Kozlovskiy",
@@ -562,12 +855,15 @@ const char * auto_contributors[] {
     "Vitaly",
     "Vitaly Baranov",
     "Vitaly Samigullin",
+    "vitstn",
+    "vivarum",
     "Vivien Maisonneuve",
     "Vlad Arkhipov",
     "Vladimir",
     "Vladimir Bunchuk",
     "Vladimir Ch",
     "Vladimir Chebotarev",
+    "vladimir golovchenko",
     "Vladimir Golovchenko",
     "Vladimir Goncharov",
     "Vladimir Klimontovich",
@@ -580,22 +876,39 @@ const char * auto_contributors[] {
     "Vojtech Splichal",
     "Volodymyr Kuznetsov",
     "Vsevolod Orlov",
+    "vxider",
     "Vxider",
     "Vyacheslav Alipov",
+    "vzakaznikov",
+    "wangchao",
     "Wang Fenjin",
+    "weeds085490",
     "Weiqing Xu",
     "William Shallum",
     "Winter Zhang",
+    "wzl",
     "Xianda Ke",
     "Xiang Zhou",
-    "Y Lu",
-    "Yangkuan Liu",
-    "Yatsishin Ilya",
+    "xPoSx",
     "Yağızcan Değirmenci",
+    "Yangkuan Liu",
+    "yangshuai",
+    "Yatsishin Ilya",
     "Yegor Andreenko",
-    "Yingchun Lai",
+    "Yegor Levankov",
+    "ygrek",
+    "yhgcn",
     "Yiğit Konur",
+    "yiguolei",
+    "Yingchun Lai",
+    "yingjinghan",
+    "ylchou",
+    "Y Lu",
     "Yohann Jardin",
+    "yonesko",
+    "yuefoo",
+    "yulu86",
+    "yuluxu",
     "Yuntao Wu",
     "Yuri Dyachenko",
     "Yurii Vlasenko",
@@ -605,315 +918,34 @@ const char * auto_contributors[] {
     "Yuriy Korzhenevskiy",
     "Yury Karpovich",
     "Yury Stankevich",
-    "Zhichang Yu",
-    "Zhipeng",
-    "Zoran Pandovski",
-    "a.palagashvili",
-    "abdrakhmanov",
-    "abyss7",
-    "achimbab",
-    "achulkov2",
-    "adevyatova",
-    "ageraab",
-    "akazz",
-    "akonyaev",
-    "akuzm",
-    "alekseik1",
-    "alesapin",
-    "alex-zaitsev",
-    "alex.lvxin",
-    "alexander kozhikhov",
-    "alexey-milovidov",
-    "alfredlu",
-    "amoschen",
-    "amudong",
-    "ana-uvarova",
-    "andrei-karpliuk",
-    "andrewsg",
-    "annvsh",
-    "anrodigina",
-    "antikvist",
-    "anton",
-    "ap11",
-    "aprudaev",
-    "artpaul",
-    "asiana21",
-    "avasiliev",
-    "avogar",
-    "avsharapov",
-    "awesomeleo",
-    "benamazing",
-    "benbiti",
-    "bgranvea",
-    "bharatnc",
-    "blazerer",
-    "bluebirddm",
-    "bo zeng",
-    "bobrovskij artemij",
-    "booknouse",
-    "bseng",
-    "cekc",
-    "centos7",
-    "champtar",
-    "chang.chen",
-    "changvvb",
-    "chasingegg",
-    "chengy8934",
-    "chenqi",
-    "chenxing-xc",
-    "chenxing.xc",
-    "chertus",
-    "comunodi",
-    "coraxster",
-    "damozhaeva",
-    "dankondr",
-    "daoready",
-    "dasmfm",
-    "davydovska",
-    "decaseal",
-    "dependabot-preview[bot]",
-    "dependabot[bot]",
-    "detailyang",
-    "dfenelonov",
-    "dgrr",
-    "dimarub2000",
-    "dinosaur",
-    "dkxiaohei",
-    "dmi-feo",
-    "dmitrii",
-    "dmitriiut",
-    "dmitry kuzmin",
-    "eejoin",
-    "egatov",
-    "elBroom",
-    "elenaspb2019",
-    "emakarov",
-    "emhlbmc",
-    "emironyuk",
-    "evtan",
-    "exprmntr",
-    "ezhaka",
-    "f1yegor",
-    "fastio",
-    "favstovol",
-    "felixoid",
-    "felixxdu",
-    "feng lv",
-    "fenglv",
-    "fessmage",
-    "filimonov",
-    "filipe",
-    "flow",
-    "flynn",
-    "foxxmary",
-    "frank",
-    "franklee",
-    "fredchenbj",
-    "fuqi",
-    "fuwhu",
-    "g-arslan",
-    "ggerogery",
-    "giordyb",
-    "glockbender",
-    "glushkovds",
-    "guoleiyi",
-    "gyuton",
-    "hao.he",
-    "hchen9",
-    "hcz",
-    "heng zhao",
-    "hexiaoting",
-    "hotid",
-    "hustnn",
-    "idfer",
-    "igor",
-    "igor.lapko",
-    "ikarishinjieva",
-    "ikopylov",
-    "imgbot[bot]",
-    "ip",
-    "it1804",
-    "ivan-kush",
-    "ivanzhukov",
-    "jakalletti",
-    "javartisan",
-    "javi",
-    "javi santana",
-    "jennyma",
-    "jetgm",
-    "jianmei zhang",
-    "jyz0309",
-    "keenwolf",
-    "kevin wan",
-    "kirillikoff",
-    "kmeaw",
-    "koshachy",
-    "kreuzerkrieg",
-    "ks1322",
-    "kshvakov",
-    "kssenii",
-    "l",
-    "lalex",
-    "lehasm",
-    "leozhang",
-    "levushkin aleksej",
-    "levysh",
-    "liangqian",
-    "libenwang",
-    "lichengxiang",
-    "linceyou",
-    "litao91",
-    "liu-bov",
-    "liuyangkuan",
-    "liuyimin",
-    "liyang",
-    "lomberts",
-    "long2ice",
-    "luc1ph3r",
-    "madianjun",
-    "maiha",
-    "malkfilipp",
-    "manmitya",
-    "maqroll",
-    "mastertheknife",
-    "maxim",
-    "maxim-babenko",
-    "maxkuzn",
-    "maxulan",
-    "mehanizm",
-    "melin",
-    "memo",
-    "meo",
-    "mergify[bot]",
-    "mf5137",
-    "mfridental",
-    "michon470",
-    "miha-g",
-    "mikepop7",
-    "millb",
-    "mnkonkova",
-    "morty",
-    "moscas",
-    "myrrc",
-    "nagorny",
-    "nauta",
-    "nautaa",
-    "never lee",
-    "nicelulu",
-    "nikitamikhaylov",
-    "nonexistence",
-    "ns-vasilev",
-    "nvartolomei",
-    "oandrew",
-    "objatie_groba",
-    "ocadaruma",
-    "ogorbacheva",
-    "olegkv",
-    "olgarev",
-    "orantius",
-    "palasonicq",
-    "peshkurov",
-    "philip.han",
-    "pingyu",
-    "potya",
-    "proller",
-    "pufit",
-    "pyos",
-    "qianlixiang",
-    "qianmoQ",
-    "quid",
-    "r1j1k",
-    "rainbowsysu",
-    "ritaank",
-    "robert",
-    "robot-clickhouse",
-    "robot-metrika-test",
-    "rodrigargar",
-    "roman",
-    "romanzhukov",
-    "root",
-    "roverxu",
-    "santaux",
-    "satanson",
-    "sdk2",
-    "serebrserg",
-    "sev7e0",
-    "sevirov",
-    "sfod",
-    "shangshujie",
-    "shedx",
-    "simon-says",
-    "songenjie",
-    "spff",
-    "spongedc",
-    "spyros87",
-    "stavrolia",
-    "stepenhu",
-    "su-houzhen",
-    "sundy",
-    "sundy-li",
-    "sundyli",
-    "svladykin",
-    "tai",
-    "taichong",
-    "taiyang-li",
-    "tao jiang",
-    "tavplubix",
-    "templarzq",
-    "tiger.yan",
-    "tison",
-    "topvisor",
-    "turbo jason",
-    "tyrionhuang",
-    "ubuntu",
-    "unegare",
-    "unknown",
-    "urgordeadbeef",
-    "vdimir",
-    "velom",
-    "vic",
-    "vicdashkov",
-    "vinity",
-    "vitstn",
-    "vivarum",
-    "vladimir golovchenko",
-    "vxider",
-    "vzakaznikov",
-    "wangchao",
-    "weeds085490",
-    "xPoSx",
-    "yangshuai",
-    "ygrek",
-    "yhgcn",
-    "yiguolei",
-    "yingjinghan",
-    "ylchou",
-    "yonesko",
-    "yuefoo",
-    "yulu86",
-    "yuluxu",
     "zamulla",
     "zhang2014",
     "zhangshengyu",
     "zhangxiao018",
     "zhangxiao871",
     "zhen ni",
+    "Zhichang Yu",
+    "Zhichun Wu",
+    "Zhipeng",
     "zhukai",
     "zlx19950903",
+    "Zoran Pandovski",
     "zvonand",
     "zvrr",
     "zvvr",
     "zzsmdfj",
-    "Šimon Podlipský",
     "Артем Стрельцов",
     "Георгий Кондратьев",
     "Дмитрий Канатников",
     "Иванов Евгений",
+    "Илья Исаев",
     "Павел Литвиненко",
     "Смитюх Вячеслав",
     "Сундуков Алексей",
+    "万康",
     "吴健",
     "小路",
+    "张中南",
     "张健",
     "张风啸",
     "徐炘",
@@ -925,5 +957,4 @@ const char * auto_contributors[] {
     "靳阳",
     "黄朝晖",
     "黄璞",
-    "박현우",
     nullptr};

From 4f41ebcae365fdb8a0b3f824e215672a625c8a51 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sat, 15 May 2021 07:40:43 +0300
Subject: [PATCH 43/76] Add settings (for connections) for MySQL storage engine

Default settings are not very efficient, since they do not even reuse
connections.
And when each query requires connection you can have only ~80 QPS, while
by simply enabling connection reuse (connection_auto_close=false) you
can have ~500 QPS (and by increasing connection_pool_size you can have
better QPS throughput).

So this patch allows to pass through some connection related settings
for the StorageMySQL engine, like:
- connection_pool_size=16
- connection_max_tries=3
- connection_auto_close=true

v2: remove connection_pool_default_size
v3: remove num_tries_on_connection_loss
---
 base/mysqlxx/PoolWithFailover.cpp             | 10 ++++-
 base/mysqlxx/PoolWithFailover.h               |  2 +
 .../table-engines/integrations/mysql.md       |  7 +++-
 src/CMakeLists.txt                            |  1 +
 .../MySQL/DatabaseConnectionMySQL.cpp         |  8 ++--
 src/Storages/MySQL/MySQLSettings.cpp          | 42 +++++++++++++++++++
 src/Storages/MySQL/MySQLSettings.h            | 32 ++++++++++++++
 src/Storages/StorageExternalDistributed.cpp   |  4 +-
 src/Storages/StorageMySQL.cpp                 | 28 +++++++++++--
 src/Storages/StorageMySQL.h                   |  6 ++-
 src/Storages/ya.make                          |  1 +
 src/TableFunctions/TableFunctionMySQL.cpp     |  4 +-
 12 files changed, 133 insertions(+), 12 deletions(-)
 create mode 100644 src/Storages/MySQL/MySQLSettings.cpp
 create mode 100644 src/Storages/MySQL/MySQLSettings.h

diff --git a/base/mysqlxx/PoolWithFailover.cpp b/base/mysqlxx/PoolWithFailover.cpp
index ea2d060e596..e317ab7f228 100644
--- a/base/mysqlxx/PoolWithFailover.cpp
+++ b/base/mysqlxx/PoolWithFailover.cpp
@@ -78,6 +78,8 @@ PoolWithFailover::PoolWithFailover(
         const RemoteDescription & addresses,
         const std::string & user,
         const std::string & password,
+        unsigned default_connections_,
+        unsigned max_connections_,
         size_t max_tries_)
     : max_tries(max_tries_)
     , shareable(false)
@@ -85,7 +87,13 @@ PoolWithFailover::PoolWithFailover(
     /// Replicas have the same priority, but traversed replicas are moved to the end of the queue.
     for (const auto & [host, port] : addresses)
     {
-        replicas_by_priority[0].emplace_back(std::make_shared<Pool>(database, host, user, password, port));
+        replicas_by_priority[0].emplace_back(std::make_shared<Pool>(database,
+            host, user, password, port,
+            /* socket_ = */ "",
+            MYSQLXX_DEFAULT_TIMEOUT,
+            MYSQLXX_DEFAULT_RW_TIMEOUT,
+            default_connections_,
+            max_connections_));
     }
 }
 
diff --git a/base/mysqlxx/PoolWithFailover.h b/base/mysqlxx/PoolWithFailover.h
index 5154fc3e253..1c7a63e76c0 100644
--- a/base/mysqlxx/PoolWithFailover.h
+++ b/base/mysqlxx/PoolWithFailover.h
@@ -115,6 +115,8 @@ namespace mysqlxx
             const RemoteDescription & addresses,
             const std::string & user,
             const std::string & password,
+            unsigned default_connections_ = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_START_CONNECTIONS,
+            unsigned max_connections_ = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_MAX_CONNECTIONS,
             size_t max_tries_ = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES);
 
         PoolWithFailover(const PoolWithFailover & other);
diff --git a/docs/en/engines/table-engines/integrations/mysql.md b/docs/en/engines/table-engines/integrations/mysql.md
index 3847e7a9e0e..9bd12e97dd8 100644
--- a/docs/en/engines/table-engines/integrations/mysql.md
+++ b/docs/en/engines/table-engines/integrations/mysql.md
@@ -15,7 +15,12 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
     name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [TTL expr1],
     name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2] [TTL expr2],
     ...
-) ENGINE = MySQL('host:port', 'database', 'table', 'user', 'password'[, replace_query, 'on_duplicate_clause']);
+) ENGINE = MySQL('host:port', 'database', 'table', 'user', 'password'[, replace_query, 'on_duplicate_clause'])
+SETTINGS
+    [connection_pool_size=16, ]
+    [connection_max_tries=3, ]
+    [connection_auto_close=true ]
+;
 ```
 
 See a detailed description of the [CREATE TABLE](../../../sql-reference/statements/create/table.md#create-table-query) query.
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index a0f36163d68..a32817928fc 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -187,6 +187,7 @@ add_object_library(clickhouse_interpreters_clusterproxy Interpreters/ClusterProx
 add_object_library(clickhouse_interpreters_jit Interpreters/JIT)
 add_object_library(clickhouse_columns Columns)
 add_object_library(clickhouse_storages Storages)
+add_object_library(clickhouse_storages_mysql Storages/MySQL)
 add_object_library(clickhouse_storages_distributed Storages/Distributed)
 add_object_library(clickhouse_storages_mergetree Storages/MergeTree)
 add_object_library(clickhouse_storages_liveview Storages/LiveView)
diff --git a/src/Databases/MySQL/DatabaseConnectionMySQL.cpp b/src/Databases/MySQL/DatabaseConnectionMySQL.cpp
index 5cd59f8a7c8..9b71fe537ec 100644
--- a/src/Databases/MySQL/DatabaseConnectionMySQL.cpp
+++ b/src/Databases/MySQL/DatabaseConnectionMySQL.cpp
@@ -20,6 +20,7 @@
 #    include <Parsers/parseQuery.h>
 #    include <Parsers/queryToString.h>
 #    include <Storages/StorageMySQL.h>
+#    include <Storages/MySQL/MySQLSettings.h>
 #    include <Common/escapeForFileName.h>
 #    include <Common/parseAddress.h>
 #    include <Common/setThreadName.h>
@@ -253,12 +254,13 @@ void DatabaseConnectionMySQL::fetchLatestTablesStructureIntoCache(
                 std::move(mysql_pool),
                 database_name_in_mysql,
                 table_name,
-                false,
-                "",
+                /* replace_query_ */ false,
+                /* on_duplicate_clause = */ "",
                 ColumnsDescription{columns_name_and_type},
                 ConstraintsDescription{},
                 String{},
-                getContext()));
+                getContext(),
+                MySQLSettings{}));
     }
 }
 
diff --git a/src/Storages/MySQL/MySQLSettings.cpp b/src/Storages/MySQL/MySQLSettings.cpp
new file mode 100644
index 00000000000..1a8f0804777
--- /dev/null
+++ b/src/Storages/MySQL/MySQLSettings.cpp
@@ -0,0 +1,42 @@
+#include <Storages/MySQL/MySQLSettings.h>
+#include <Parsers/ASTCreateQuery.h>
+#include <Parsers/ASTSetQuery.h>
+#include <Parsers/ASTFunction.h>
+#include <Common/Exception.h>
+
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int UNKNOWN_SETTING;
+}
+
+IMPLEMENT_SETTINGS_TRAITS(MySQLSettingsTraits, LIST_OF_MYSQL_SETTINGS)
+
+void MySQLSettings::loadFromQuery(ASTStorage & storage_def)
+{
+    if (storage_def.settings)
+    {
+        try
+        {
+            applyChanges(storage_def.settings->changes);
+        }
+        catch (Exception & e)
+        {
+            if (e.code() == ErrorCodes::UNKNOWN_SETTING)
+                e.addMessage("for storage " + storage_def.engine->name);
+            throw;
+        }
+    }
+    else
+    {
+        auto settings_ast = std::make_shared<ASTSetQuery>();
+        settings_ast->is_standalone = false;
+        storage_def.set(storage_def.settings, settings_ast);
+    }
+}
+
+}
+
diff --git a/src/Storages/MySQL/MySQLSettings.h b/src/Storages/MySQL/MySQLSettings.h
new file mode 100644
index 00000000000..da8723c2ea6
--- /dev/null
+++ b/src/Storages/MySQL/MySQLSettings.h
@@ -0,0 +1,32 @@
+#pragma once
+
+#include <Core/Defines.h>
+#include <Core/BaseSettings.h>
+
+
+namespace Poco::Util
+{
+    class AbstractConfiguration;
+}
+
+
+namespace DB
+{
+class ASTStorage;
+
+#define LIST_OF_MYSQL_SETTINGS(M) \
+    M(UInt64, connection_pool_size, 16, "Size of connection pool (if all connections are in use, the query will wait until some connection will be freed).", 0) \
+    M(UInt64, connection_max_tries, 3, "Number of retries for pool with failover", 0) \
+    M(Bool, connection_auto_close, true, "Auto-close connection after query execution, i.e. disable connection reuse.", 0) \
+
+DECLARE_SETTINGS_TRAITS(MySQLSettingsTraits, LIST_OF_MYSQL_SETTINGS)
+
+
+/** Settings for the MySQL family of engines.
+  */
+struct MySQLSettings : public BaseSettings<MySQLSettingsTraits>
+{
+    void loadFromQuery(ASTStorage & storage_def);
+};
+
+}
diff --git a/src/Storages/StorageExternalDistributed.cpp b/src/Storages/StorageExternalDistributed.cpp
index 5a153f16a0a..32b9c7e9245 100644
--- a/src/Storages/StorageExternalDistributed.cpp
+++ b/src/Storages/StorageExternalDistributed.cpp
@@ -12,6 +12,7 @@
 #include <Processors/Pipe.h>
 #include <Common/parseRemoteDescription.h>
 #include <Storages/StorageMySQL.h>
+#include <Storages/MySQL/MySQLSettings.h>
 #include <Storages/StoragePostgreSQL.h>
 #include <Storages/StorageURL.h>
 #include <common/logger_useful.h>
@@ -79,7 +80,8 @@ StorageExternalDistributed::StorageExternalDistributed(
                     columns_,
                     constraints_,
                     String{},
-                    context);
+                    context,
+                    MySQLSettings{});
                 break;
             }
 #endif
diff --git a/src/Storages/StorageMySQL.cpp b/src/Storages/StorageMySQL.cpp
index 4cf69d7dd77..1dadcfe986b 100644
--- a/src/Storages/StorageMySQL.cpp
+++ b/src/Storages/StorageMySQL.cpp
@@ -15,6 +15,7 @@
 #include <IO/Operators.h>
 #include <IO/WriteHelpers.h>
 #include <Parsers/ASTLiteral.h>
+#include <Parsers/ASTCreateQuery.h>
 #include <mysqlxx/Transaction.h>
 #include <Processors/Sources/SourceFromInputStream.h>
 #include <Processors/Pipe.h>
@@ -50,13 +51,15 @@ StorageMySQL::StorageMySQL(
     const ColumnsDescription & columns_,
     const ConstraintsDescription & constraints_,
     const String & comment,
-    ContextPtr context_)
+    ContextPtr context_,
+    const MySQLSettings & mysql_settings_)
     : IStorage(table_id_)
     , WithContext(context_->getGlobalContext())
     , remote_database_name(remote_database_name_)
     , remote_table_name(remote_table_name_)
     , replace_query{replace_query_}
     , on_duplicate_clause{on_duplicate_clause_}
+    , mysql_settings(mysql_settings_)
     , pool(std::make_shared<mysqlxx::PoolWithFailover>(pool_))
 {
     StorageInMemoryMetadata storage_metadata;
@@ -98,7 +101,8 @@ Pipe StorageMySQL::read(
     }
 
 
-    StreamSettings mysql_input_stream_settings(context_->getSettingsRef(), true, false);
+    StreamSettings mysql_input_stream_settings(context_->getSettingsRef(),
+        mysql_settings.connection_auto_close);
     return Pipe(std::make_shared<SourceFromInputStream>(
             std::make_shared<MySQLWithFailoverBlockInputStream>(pool, query, sample_block, mysql_input_stream_settings)));
 }
@@ -250,8 +254,22 @@ void registerStorageMySQL(StorageFactory & factory)
         const String & password = engine_args[4]->as<ASTLiteral &>().value.safeGet<String>();
         size_t max_addresses = args.getContext()->getSettingsRef().glob_expansion_max_elements;
 
+        /// TODO: move some arguments from the arguments to the SETTINGS.
+        MySQLSettings mysql_settings;
+        if (args.storage_def->settings)
+        {
+            mysql_settings.loadFromQuery(*args.storage_def);
+        }
+
+        if (!mysql_settings.connection_pool_size)
+            throw Exception("connection_pool_size cannot be zero.", ErrorCodes::BAD_ARGUMENTS);
+
         auto addresses = parseRemoteDescriptionForExternalDatabase(host_port, max_addresses, 3306);
-        mysqlxx::PoolWithFailover pool(remote_database, addresses, username, password);
+        mysqlxx::PoolWithFailover pool(remote_database, addresses,
+            username, password,
+            MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_START_CONNECTIONS,
+            mysql_settings.connection_pool_size,
+            mysql_settings.connection_max_tries);
 
         bool replace_query = false;
         std::string on_duplicate_clause;
@@ -275,9 +293,11 @@ void registerStorageMySQL(StorageFactory & factory)
             args.columns,
             args.constraints,
             args.comment,
-            args.getContext());
+            args.getContext(),
+            mysql_settings);
     },
     {
+        .supports_settings = true,
         .source_access_type = AccessType::MYSQL,
     });
 }
diff --git a/src/Storages/StorageMySQL.h b/src/Storages/StorageMySQL.h
index a7aca48197e..5eb9ed14524 100644
--- a/src/Storages/StorageMySQL.h
+++ b/src/Storages/StorageMySQL.h
@@ -9,6 +9,7 @@
 #include <ext/shared_ptr_helper.h>
 
 #include <Storages/IStorage.h>
+#include <Storages/MySQL/MySQLSettings.h>
 #include <mysqlxx/PoolWithFailover.h>
 
 
@@ -33,7 +34,8 @@ public:
         const ColumnsDescription & columns_,
         const ConstraintsDescription & constraints_,
         const String & comment,
-        ContextPtr context_);
+        ContextPtr context_,
+        const MySQLSettings & mysql_settings_);
 
     std::string getName() const override { return "MySQL"; }
 
@@ -56,6 +58,8 @@ private:
     bool replace_query;
     std::string on_duplicate_clause;
 
+    MySQLSettings mysql_settings;
+
     mysqlxx::PoolWithFailoverPtr pool;
 };
 
diff --git a/src/Storages/ya.make b/src/Storages/ya.make
index d83ba7f6490..8e0efac8c6e 100644
--- a/src/Storages/ya.make
+++ b/src/Storages/ya.make
@@ -112,6 +112,7 @@ SRCS(
     MergeTree/localBackup.cpp
     MergeTree/registerStorageMergeTree.cpp
     MutationCommands.cpp
+    MySQL/MySQLSettings.cpp
     PartitionCommands.cpp
     ProjectionsDescription.cpp
     ReadInOrderOptimizer.cpp
diff --git a/src/TableFunctions/TableFunctionMySQL.cpp b/src/TableFunctions/TableFunctionMySQL.cpp
index 325b2dc44c6..0b60e11f490 100644
--- a/src/TableFunctions/TableFunctionMySQL.cpp
+++ b/src/TableFunctions/TableFunctionMySQL.cpp
@@ -15,6 +15,7 @@
 #include <Parsers/ASTFunction.h>
 #include <Parsers/ASTLiteral.h>
 #include <Storages/StorageMySQL.h>
+#include <Storages/MySQL/MySQLSettings.h>
 #include <TableFunctions/ITableFunction.h>
 #include <TableFunctions/TableFunctionFactory.h>
 #include <TableFunctions/TableFunctionMySQL.h>
@@ -107,7 +108,8 @@ StoragePtr TableFunctionMySQL::executeImpl(
         columns,
         ConstraintsDescription{},
         String{},
-        context);
+        context,
+        MySQLSettings{});
 
     pool.reset();
 

From 48ce2ce500ba9a37539856feba75aba9e8a7c69e Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Thu, 20 May 2021 22:57:06 +0300
Subject: [PATCH 44/76] Fix script

---
 tests/clickhouse-test | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index 6e4ae145e76..d0866dd3bbe 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -725,8 +725,9 @@ def do_run_tests(jobs, suite, suite_dir, suite_tmp_dir, all_tests, parallel_test
         run_tests_array((sequential_tests, len(sequential_tests), suite, suite_dir, suite_tmp_dir))
         return len(sequential_tests) + len(parallel_tests)
     else:
-        run_tests_array((all_tests, len(all_tests), suite, suite_dir, suite_tmp_dir))
-        return len(all_tests)
+        num_tests = len(all_tests)
+        run_tests_array((all_tests, num_tests, suite, suite_dir, suite_tmp_dir))
+        return num_tests
 
 
 def main(args):

From d17adeecaf2ad58a8d72a725f5358521fe98de64 Mon Sep 17 00:00:00 2001
From: Maksim Kita <maksim-kita@yandex-team.ru>
Date: Thu, 20 May 2021 20:50:37 +0000
Subject: [PATCH 45/76] AARCH64 hash tables benchmark

---
 .../integer_hash_tables_benchmark.cpp         | 58 ++++++++++++++++++-
 1 file changed, 57 insertions(+), 1 deletion(-)

diff --git a/src/Common/examples/integer_hash_tables_benchmark.cpp b/src/Common/examples/integer_hash_tables_benchmark.cpp
index cb7467ce909..a245c990f2e 100644
--- a/src/Common/examples/integer_hash_tables_benchmark.cpp
+++ b/src/Common/examples/integer_hash_tables_benchmark.cpp
@@ -61,6 +61,10 @@ static void NO_INLINE testForType(size_t method, size_t rows_size)
         test<Key, ::absl::flat_hash_map<Key, UInt64>>(data.data(), data.size(), "Abseil HashMap");
     }
     else if (method == 3)
+    {
+        test<Key, ::absl::flat_hash_map<Key, UInt64, DefaultHash<Key>>>(data.data(), data.size(), "Abseil HashMap with CH Hash");
+    }
+    else if (method == 4)
     {
         test<Key, std::unordered_map<Key, UInt64>>(data.data(), data.size(), "std::unordered_map");
     }
@@ -81,8 +85,9 @@ static void NO_INLINE testForType(size_t method, size_t rows_size)
  * ./integer_hash_tables_benchmark 1 $2 100000000 < $1
  * ./integer_hash_tables_benchmark 2 $2 100000000 < $1
  * ./integer_hash_tables_benchmark 3 $2 100000000 < $1
+ * ./integer_hash_tables_benchmark 4 $2 100000000 < $1
  *
- * Results of this benchmark on hits_100m_obfuscated
+ * Results of this benchmark on hits_100m_obfuscated X86-64
  *
  * File hits_100m_obfuscated/201307_1_96_4/WatchID.bin
  * CH HashMap: Elapsed: 7.366 (13575745.933 elem/sec.), map size: 99997493
@@ -125,6 +130,57 @@ static void NO_INLINE testForType(size_t method, size_t rows_size)
  * Google DenseMap: Elapsed: 1.673 (59757144.027 elem/sec.), map size: 32260732 /// Broken because there is 0 key in dataset
  * Abseil HashMap: Elapsed: 0.297 (336589258.845 elem/sec.), map size: 19
  * std::unordered_map: Elapsed: 0.332 (301114451.384 elem/sec.), map size: 19
+ *
+ *
+ * Results of this benchmark on hits_100m_obfuscated AARCH64
+ * File hits_100m_obfuscated/201307_1_96_4/WatchID.bin
+ * CH HashMap: Elapsed: 9.530 (10493528.533 elem/sec.), map size: 99997493
+ * Google DenseMap: Elapsed: 14.436 (6927091.135 elem/sec.), map size: 99997493
+ * Abseil HashMap: Elapsed: 16.671 (5998504.085 elem/sec.), map size: 99997493
+ * Abseil HashMap with CH Hash: Elapsed: 16.803 (5951365.711 elem/sec.), map size: 99997493
+ * std::unordered_map: Elapsed: 50.805 (1968305.658 elem/sec.), map size: 99997493
+ *
+ * File hits_100m_obfuscated/201307_1_96_4/URLHash.bin
+ * CH HashMap: Elapsed: 3.693 (27076878.092 elem/sec.), map size: 20714865
+ * Google DenseMap: Elapsed: 5.051 (19796401.694 elem/sec.), map size: 20714865
+ * Abseil HashMap: Elapsed: 5.617 (17804528.625 elem/sec.), map size: 20714865
+ * Abseil HashMap with CH Hash: Elapsed: 5.702 (17537013.639 elem/sec.), map size: 20714865
+ * std::unordered_map: Elapsed: 10.757 (9296040.953 elem/sec.), map size: 2071486
+ *
+ * File hits_100m_obfuscated/201307_1_96_4/UserID.bin
+ * CH HashMap: Elapsed: 2.982 (33535795.695 elem/sec.), map size: 17630976
+ * Google DenseMap: Elapsed: 3.940 (25381557.959 elem/sec.), map size: 17630976
+ * Abseil HashMap: Elapsed: 4.493 (22259078.458 elem/sec.), map size: 17630976
+ * Abseil HashMap with CH Hash: Elapsed: 4.596 (21759738.710 elem/sec.), map size: 17630976
+ * std::unordered_map: Elapsed: 9.035 (11067903.596 elem/sec.), map size: 17630976
+ *
+ * File hits_100m_obfuscated/201307_1_96_4/RegionID.bin
+ * CH HashMap: Elapsed: 0.302 (331026285.361 elem/sec.), map size: 9040
+ * Google DenseMap: Elapsed: 0.623 (160419421.840 elem/sec.), map size: 9046
+ * Abseil HashMap: Elapsed: 0.981 (101971186.758 elem/sec.), map size: 9040
+ * Abseil HashMap with CH Hash: Elapsed: 0.991 (100932993.199 elem/sec.), map size: 9040
+ * std::unordered_map: Elapsed: 0.809 (123541402.715 elem/sec.), map size: 9040
+ *
+ * File hits_100m_obfuscated/201307_1_96_4/CounterID.bin
+ * CH HashMap: Elapsed: 0.343 (291821742.078 elem/sec.), map size: 6506
+ * Google DenseMap: Elapsed: 0.718 (139191105.450 elem/sec.), map size: 6506
+ * Abseil HashMap: Elapsed: 1.019 (98148285.278 elem/sec.), map size: 6506
+ * Abseil HashMap with CH Hash: Elapsed: 1.048 (95446843.667 elem/sec.), map size: 6506
+ * std::unordered_map: Elapsed: 0.701 (142701070.085 elem/sec.), map size: 6506
+ *
+ * File hits_100m_obfuscated/201307_1_96_4/TraficSourceID.bin
+ * CH HashMap: Elapsed: 0.376 (265905243.103 elem/sec.), map size: 10
+ * Google DenseMap: Elapsed: 1.309 (76420707.298 elem/sec.), map size: 1565609 /// Broken because there is 0 key in dataset
+ * Abseil HashMap: Elapsed: 0.955 (104668109.775 elem/sec.), map size: 10
+ * Abseil HashMap with CH Hash: Elapsed: 0.967 (103456305.391 elem/sec.), map size: 10
+ * std::unordered_map: Elapsed: 1.241 (80591305.890 elem/sec.), map size: 10
+ *
+ * File hits_100m_obfuscated/201307_1_96_4/AdvEngineID.bin
+ * CH HashMap: Elapsed: 0.213 (470208130.105 elem/sec.), map size: 19
+ * Google DenseMap: Elapsed: 2.525 (39607131.523 elem/sec.), map size: 32260732 /// Broken because there is 0 key in dataset
+ * Abseil HashMap: Elapsed: 0.950 (105233678.618 elem/sec.), map size: 19
+ * Abseil HashMap with CH Hash: Elapsed: 0.962 (104001230.717 elem/sec.), map size: 19
+ * std::unordered_map: Elapsed: 0.585 (171059989.837 elem/sec.), map size: 19
  */
 
 int main(int argc, char ** argv)

From 0d174831c963761c7bca6ba39700405ce78b3a16 Mon Sep 17 00:00:00 2001
From: Maksim Kita <kitaetoya@gmail.com>
Date: Fri, 21 May 2021 00:56:33 +0300
Subject: [PATCH 46/76] Updated X86 benchmarks

---
 .../integer_hash_tables_benchmark.cpp         | 63 ++++++++++---------
 1 file changed, 35 insertions(+), 28 deletions(-)

diff --git a/src/Common/examples/integer_hash_tables_benchmark.cpp b/src/Common/examples/integer_hash_tables_benchmark.cpp
index a245c990f2e..948ff9cc057 100644
--- a/src/Common/examples/integer_hash_tables_benchmark.cpp
+++ b/src/Common/examples/integer_hash_tables_benchmark.cpp
@@ -90,46 +90,53 @@ static void NO_INLINE testForType(size_t method, size_t rows_size)
  * Results of this benchmark on hits_100m_obfuscated X86-64
  *
  * File hits_100m_obfuscated/201307_1_96_4/WatchID.bin
- * CH HashMap: Elapsed: 7.366 (13575745.933 elem/sec.), map size: 99997493
- * Google DenseMap: Elapsed: 10.089 (9911817.125 elem/sec.), map size: 99997493
- * Abseil HashMap: Elapsed: 9.011 (11097794.073 elem/sec.), map size: 99997493
- * std::unordered_map: Elapsed: 44.758 (2234223.189 elem/sec.), map size: 99997493
+ * CH HashMap: Elapsed: 7.416 (13484217.815 elem/sec.), map size: 99997493
+ * Google DenseMap: Elapsed: 10.303 (9706022.031 elem/sec.), map size: 99997493
+ * Abseil HashMap: Elapsed: 9.106 (10982139.229 elem/sec.), map size: 99997493
+ * Abseil HashMap with CH Hash: Elapsed: 9.221 (10845360.669 elem/sec.), map size: 99997493
+ * std::unordered_map: Elapsed: 45.213 (2211758.706 elem/sec.), map size: 9999749
  *
  * File hits_100m_obfuscated/201307_1_96_4/URLHash.bin
- * CH HashMap: Elapsed: 2.672 (37421588.347 elem/sec.), map size: 20714865
- * Google DenseMap: Elapsed: 3.409 (29333308.209 elem/sec.), map size: 20714865
- * Abseil HashMap: Elapsed: 2.778 (36000540.035 elem/sec.), map size: 20714865
- * std::unordered_map: Elapsed: 8.643 (11570012.207 elem/sec.), map size: 20714865
+ * CH HashMap: Elapsed: 2.620 (38168135.308 elem/sec.), map size: 20714865
+ * Google DenseMap: Elapsed: 3.426 (29189309.058 elem/sec.), map size: 20714865
+ * Abseil HashMap: Elapsed: 2.788 (35870495.097 elem/sec.), map size: 20714865
+ * Abseil HashMap with CH Hash: Elapsed: 2.991 (33428850.155 elem/sec.), map size: 20714865
+ * std::unordered_map: Elapsed: 8.503 (11760331.346 elem/sec.), map size: 20714865
  *
  * File hits_100m_obfuscated/201307_1_96_4/UserID.bin
- * CH HashMap: Elapsed: 2.116 (47267659.076 elem/sec.), map size: 17630976
- * Google DenseMap: Elapsed: 2.722 (36740693.786 elem/sec.), map size: 17630976
- * Abseil HashMap: Elapsed: 2.597 (38509988.663 elem/sec.), map size: 17630976
- * std::unordered_map: Elapsed: 7.327 (13647271.471 elem/sec.), map size: 17630976
+ * CH HashMap: Elapsed: 2.157 (46352039.753 elem/sec.), map size: 17630976
+ * Google DenseMap: Elapsed: 2.725 (36694226.782 elem/sec.), map size: 17630976
+ * Abseil HashMap: Elapsed: 2.590 (38604284.187 elem/sec.), map size: 17630976
+ * Abseil HashMap with CH Hash: Elapsed: 2.785 (35904856.137 elem/sec.), map size: 17630976
+ * std::unordered_map: Elapsed: 7.268 (13759557.609 elem/sec.), map size: 17630976
  *
  * File hits_100m_obfuscated/201307_1_96_4/RegionID.bin
- * CH HashMap: Elapsed: 0.201 (498144193.695 elem/sec.), map size: 9040
- * Google DenseMap: Elapsed: 0.261 (382656387.016 elem/sec.), map size: 9046
- * Abseil HashMap: Elapsed: 0.307 (325874545.117 elem/sec.), map size: 9040
- * std::unordered_map: Elapsed: 0.466 (214379083.420 elem/sec.), map size: 9040
+ * CH HashMap: Elapsed: 0.192 (521583315.810 elem/sec.), map size: 9040
+ * Google DenseMap: Elapsed: 0.297 (337081407.799 elem/sec.), map size: 9046
+ * Abseil HashMap: Elapsed: 0.295 (338805623.511 elem/sec.), map size: 9040
+ * Abseil HashMap with CH Hash: Elapsed: 0.331 (302155391.036 elem/sec.), map size: 9040
+ * std::unordered_map: Elapsed: 0.455 (219971555.390 elem/sec.), map size: 9040
  *
  * File hits_100m_obfuscated/201307_1_96_4/CounterID.bin
- * CH HashMap: Elapsed: 0.220 (455344735.648 elem/sec.), map size: 6506
- * Google DenseMap: Elapsed: 0.297 (336187522.818 elem/sec.), map size: 6506
- * Abseil HashMap: Elapsed: 0.307 (325264214.480 elem/sec.), map size: 6506
- * std::unordered_map: Elapsed: 0.389 (257195996.114 elem/sec.), map size: 6506
+ * CH HashMap: Elapsed: 0.217 (460216823.609 elem/sec.), map size: 6506
+ * Google DenseMap: Elapsed: 0.373 (267838665.098 elem/sec.), map size: 6506
+ * Abseil HashMap: Elapsed: 0.325 (308124728.989 elem/sec.), map size: 6506
+ * Abseil HashMap with CH Hash: Elapsed: 0.354 (282167144.801 elem/sec.), map size: 6506
+ * std::unordered_map: Elapsed: 0.390 (256573354.171 elem/sec.), map size: 6506
  *
  * File hits_100m_obfuscated/201307_1_96_4/TraficSourceID.bin
- * CH HashMap: Elapsed: 0.274 (365196673.729 elem/sec.), map size: 10
- * Google DenseMap: Elapsed: 0.782 (127845746.927 elem/sec.), map size: 1565609 /// Broken because there is 0 key in dataset
- * Abseil HashMap: Elapsed: 0.303 (330461565.053 elem/sec.), map size: 10
- * std::unordered_map: Elapsed: 0.843 (118596530.649 elem/sec.), map size: 10
+ * CH HashMap: Elapsed: 0.246 (406714566.282 elem/sec.), map size: 10
+ * Google DenseMap: Elapsed: 0.760 (131615151.233 elem/sec.), map size: 1565609 /// Broken because there is 0 key in dataset
+ * Abseil HashMap: Elapsed: 0.309 (324068156.680 elem/sec.), map size: 10
+ * Abseil HashMap with CH Hash: Elapsed: 0.339 (295108223.814 elem/sec.), map size: 10
+ * std::unordered_map: Elapsed: 0.811 (123304031.195 elem/sec.), map size: 10
  *
  * File hits_100m_obfuscated/201307_1_96_4/AdvEngineID.bin
- * CH HashMap: Elapsed: 0.160 (623399865.019 elem/sec.), map size: 19
- * Google DenseMap: Elapsed: 1.673 (59757144.027 elem/sec.), map size: 32260732 /// Broken because there is 0 key in dataset
- * Abseil HashMap: Elapsed: 0.297 (336589258.845 elem/sec.), map size: 19
- * std::unordered_map: Elapsed: 0.332 (301114451.384 elem/sec.), map size: 19
+ * CH HashMap: Elapsed: 0.155 (643245257.748 elem/sec.), map size: 19
+ * Google DenseMap: Elapsed: 1.629 (61395025.417 elem/sec.), map size: 32260732 // Broken because there is 0 key in dataset
+ * Abseil HashMap: Elapsed: 0.292 (342765027.204 elem/sec.), map size: 19
+ * Abseil HashMap with CH Hash: Elapsed: 0.330 (302822020.210 elem/sec.), map size: 19
+ * std::unordered_map: Elapsed: 0.308 (325059333.730 elem/sec.), map size: 19
  *
  *
  * Results of this benchmark on hits_100m_obfuscated AARCH64

From a896af572e86955c77c99d7ec6a5daad1659a525 Mon Sep 17 00:00:00 2001
From: Maksim Kita <kitaetoya@gmail.com>
Date: Fri, 21 May 2021 01:06:29 +0300
Subject: [PATCH 47/76] Fixed comment

---
 src/Common/examples/integer_hash_tables_benchmark.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Common/examples/integer_hash_tables_benchmark.cpp b/src/Common/examples/integer_hash_tables_benchmark.cpp
index 948ff9cc057..f81add6d6cb 100644
--- a/src/Common/examples/integer_hash_tables_benchmark.cpp
+++ b/src/Common/examples/integer_hash_tables_benchmark.cpp
@@ -140,6 +140,7 @@ static void NO_INLINE testForType(size_t method, size_t rows_size)
  *
  *
  * Results of this benchmark on hits_100m_obfuscated AARCH64
+ *
  * File hits_100m_obfuscated/201307_1_96_4/WatchID.bin
  * CH HashMap: Elapsed: 9.530 (10493528.533 elem/sec.), map size: 99997493
  * Google DenseMap: Elapsed: 14.436 (6927091.135 elem/sec.), map size: 99997493

From b2c828f0b6fa8b5b50f7c6e3f1eef1810377de4e Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Fri, 21 May 2021 03:52:18 +0300
Subject: [PATCH 48/76] Update CHANGELOG.md

---
 CHANGELOG.md | 122 +++++++++++++++++++++++++--------------------------
 1 file changed, 60 insertions(+), 62 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index c9d5511f383..eaa3682051e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,22 +1,18 @@
-## ClickHouse release 21.5
+## ClickHouse release 21.5, 2021-05-20
 
 #### Backward Incompatible Change
 
 * Change comparison of integers and floating point numbers when integer is not exactly representable in the floating point data type. In new version comparison will return false as the rounding error will occur. Example: `9223372036854775808.0 != 9223372036854775808`, because the number `9223372036854775808` is not representable as floating point number exactly (and `9223372036854775808.0` is rounded to `9223372036854776000.0`). But in previous version the comparison will return as the numbers are equal, because if the floating point number `9223372036854776000.0` get converted back to UInt64, it will yield `9223372036854775808`. For the reference, the Python programming language also treats these numbers as equal. But this behaviour was dependend on CPU model (different results on AMD64 and AArch64 for some out-of-range numbers), so we make the comparison more precise. It will treat int and float numbers equal only if int is represented in floating point type exactly. [#22595](https://github.com/ClickHouse/ClickHouse/pull/22595) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Remove support for `argMin` and `argMax` for single `Tuple` argument. The code was not memory-safe. The feature was added by mistake and it is confusing for people. These functions can be reintroduced under different names later. This fixes [#22384](https://github.com/ClickHouse/ClickHouse/issues/22384) and reverts [#17359](https://github.com/ClickHouse/ClickHouse/issues/17359). [#23393](https://github.com/ClickHouse/ClickHouse/pull/23393) ([alexey-milovidov](https://github.com/alexey-milovidov)).
 
 #### New Feature
 
-* Added `Decimal256` type support in dictionaries. Closes [#20979](https://github.com/ClickHouse/ClickHouse/issues/20979). [#22960](https://github.com/ClickHouse/ClickHouse/pull/22960) ([Maksim Kita](https://github.com/kitaisreal)).
-* Added `ALTER TABLE ... FETCH PART ...` query. It's similar to `FETCH PARTITION`, but fetches only one part. [#22706](https://github.com/ClickHouse/ClickHouse/pull/22706) ([turbo jason](https://github.com/songenjie)).
-* FlatDictionary added `initial_array_size`, `max_array_size` options. [#22521](https://github.com/ClickHouse/ClickHouse/pull/22521) ([Maksim Kita](https://github.com/kitaisreal)).
-* Add new setting `non_replicated_deduplication_window` for non-replicated MergeTree inserts deduplication. [#22514](https://github.com/ClickHouse/ClickHouse/pull/22514) ([alesapin](https://github.com/alesapin)).
-* Update paths to the `CatBoost` model configs in config reloading. [#22434](https://github.com/ClickHouse/ClickHouse/pull/22434) ([Kruglov Pavel](https://github.com/Avogar)).
+* Added functions `dictGetChildren(dictionary, key)`, `dictGetDescendants(dictionary, key, level)`. Function `dictGetChildren` return all children as an array if indexes. It is a inverse transformation for `dictGetHierarchy`. Function `dictGetDescendants` return all descendants as if `dictGetChildren` was applied `level` times recursively. Zero `level` value is equivalent to infinity. Improved performance of `dictGetHierarchy`, `dictIsIn` functions. Closes [#14656](https://github.com/ClickHouse/ClickHouse/issues/14656). [#22096](https://github.com/ClickHouse/ClickHouse/pull/22096) ([Maksim Kita](https://github.com/kitaisreal)).
 * Added function `dictGetOrNull`. It works like `dictGet`, but return `Null` in case key was not found in dictionary. Closes [#22375](https://github.com/ClickHouse/ClickHouse/issues/22375). [#22413](https://github.com/ClickHouse/ClickHouse/pull/22413) ([Maksim Kita](https://github.com/kitaisreal)).
-* Added support for replicas and shards in MySQL/PostgreSQL table engine / table function. Replicas and shards, if used together, are added via new table engine ExternalDistributed. Closes [#20969](https://github.com/ClickHouse/ClickHouse/issues/20969). [#22217](https://github.com/ClickHouse/ClickHouse/pull/22217) ([Kseniia Sumarokova](https://github.com/kssenii)).
-* Improved performance of `dictGetHierarchy`, `dictIsIn` functions. Added functions `dictGetChildren(dictionary, key)`, `dictGetDescendants(dictionary, key, level)`. Function `dictGetChildren` return all children as an array if indexes. It is a inverse transformation for `dictGetHierarchy`. Function `dictGetDescendants` return all descendants as if `dictGetChildren` was applied `level` times recursively. Zero `level` value is equivalent to infinity. Closes [#14656](https://github.com/ClickHouse/ClickHouse/issues/14656). [#22096](https://github.com/ClickHouse/ClickHouse/pull/22096) ([Maksim Kita](https://github.com/kitaisreal)).
 * Added a table function `s3Cluster`, which allows to process files from `s3` in parallel on every node of a specified cluster. [#22012](https://github.com/ClickHouse/ClickHouse/pull/22012) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Added support for replicas and shards in MySQL/PostgreSQL table engine / table function. You can write `SELECT * FROM mysql('host{1,2}-{1|2}', ...)`. Closes [#20969](https://github.com/ClickHouse/ClickHouse/issues/20969). [#22217](https://github.com/ClickHouse/ClickHouse/pull/22217) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Added `ALTER TABLE ... FETCH PART ...` query. It's similar to `FETCH PARTITION`, but fetches only one part. [#22706](https://github.com/ClickHouse/ClickHouse/pull/22706) ([turbo jason](https://github.com/songenjie)).
 * Added a setting `max_distributed_depth` that limits the depth of recursive queries to `Distributed` tables. Closes [#20229](https://github.com/ClickHouse/ClickHouse/issues/20229). [#21942](https://github.com/ClickHouse/ClickHouse/pull/21942) ([flynn](https://github.com/ucasFL)).
-* Added function `arrayFold(x1,...,xn,accum -> expression, array1,...,arrayn, init_accum)` that applies the expression to each element of the array (or set of parallel arrays) and collect result in accumulator. [#21589](https://github.com/ClickHouse/ClickHouse/pull/21589) ([Dmitry Krylov](https://github.com/dmalkr)).
 
 #### Performance Improvement
 
@@ -25,108 +21,108 @@
 * Disabled compression by default when interacting with localhost (with clickhouse-client or server to server with distributed queries) via native protocol. It may improve performance of some import/export operations. This closes [#22234](https://github.com/ClickHouse/ClickHouse/issues/22234). [#22237](https://github.com/ClickHouse/ClickHouse/pull/22237) ([alexey-milovidov](https://github.com/alexey-milovidov)).
 * Exclude values that does not belong to the shard from right part of IN section for distributed queries (under `optimize_skip_unused_shards_rewrite_in`, enabled by default, since it still requires `optimize_skip_unused_shards`). [#21511](https://github.com/ClickHouse/ClickHouse/pull/21511) ([Azat Khuzhin](https://github.com/azat)).
 * Improved performance of reading a subset of columns with File-like table engine and column-oriented format like Parquet, Arrow or ORC. This closes [#issue:20129](https://github.com/ClickHouse/ClickHouse/issues/20129). [#21302](https://github.com/ClickHouse/ClickHouse/pull/21302) ([keenwolf](https://github.com/keen-wolf)).
+* Allow to move more conditions to `PREWHERE` as it was before version 21.1 (adjustment of internal heuristics). Insufficient number of moved condtions could lead to worse performance. [#23397](https://github.com/ClickHouse/ClickHouse/pull/23397) ([Anton Popov](https://github.com/CurtizJ)).
+* Added nanodbc instead of Poco::ODBC. Closes [#9678](https://github.com/ClickHouse/ClickHouse/issues/9678). Add support for DateTime64 and Decimal* for ODBC table engine. Closes [#21961](https://github.com/ClickHouse/ClickHouse/issues/21961). Fixed issue with cyrillic text being truncated. Closes [#16246](https://github.com/ClickHouse/ClickHouse/issues/16246). Added connection pools for odbc bridge. [#21972](https://github.com/ClickHouse/ClickHouse/pull/21972) ([Kseniia Sumarokova](https://github.com/kssenii)).
 
 #### Improvement
 
-* Enabled `async_socket_for_remote` by default. [#23683](https://github.com/ClickHouse/ClickHouse/pull/23683) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Increase `max_uri_size` (the maximum size of URL in HTTP interface) to 1 MiB by default. This closes [#21197](https://github.com/ClickHouse/ClickHouse/issues/21197). [#22997](https://github.com/ClickHouse/ClickHouse/pull/22997) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Set `background_fetches_pool_size` to `8` that is better for production usage with frequent small insertions or slow ZooKeeper cluster. [#22945](https://github.com/ClickHouse/ClickHouse/pull/22945) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* FlatDictionary added `initial_array_size`, `max_array_size` options. [#22521](https://github.com/ClickHouse/ClickHouse/pull/22521) ([Maksim Kita](https://github.com/kitaisreal)).
+* Add new setting `non_replicated_deduplication_window` for non-replicated MergeTree inserts deduplication. [#22514](https://github.com/ClickHouse/ClickHouse/pull/22514) ([alesapin](https://github.com/alesapin)).
+* Update paths to the `CatBoost` model configs in config reloading. [#22434](https://github.com/ClickHouse/ClickHouse/pull/22434) ([Kruglov Pavel](https://github.com/Avogar)).
+* Added `Decimal256` type support in dictionaries. `Decimal256` is experimental feature. Closes [#20979](https://github.com/ClickHouse/ClickHouse/issues/20979). [#22960](https://github.com/ClickHouse/ClickHouse/pull/22960) ([Maksim Kita](https://github.com/kitaisreal)).
+* Enabled `async_socket_for_remote` by default (using less amount of OS threads for distributed queries). [#23683](https://github.com/ClickHouse/ClickHouse/pull/23683) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
 * Fixed `quantile(s)TDigest`. Added special handling of singleton centroids according to tdunning/t-digest 3.2+. Also a bug with over-compression of centroids in implementation of earlier version of the algorithm was fixed. [#23314](https://github.com/ClickHouse/ClickHouse/pull/23314) ([Vladimir Chebotarev](https://github.com/excitoon)).
-* Disable settings `use_hedged_requests` and `async_socket_for_remote` because there is an evidence that it may cause issues. [#23261](https://github.com/ClickHouse/ClickHouse/pull/23261) ([alexey-milovidov](https://github.com/alexey-milovidov)).
-* Make function `unhex` case insensitive for compatibility with MySQL. [#23229](https://github.com/ClickHouse/ClickHouse/pull/23229) ([alexey-milovidov](https://github.com/alexey-milovidov)).
-* Fixed memory tracking with min_bytes_to_use_mmap_io. [#23211](https://github.com/ClickHouse/ClickHouse/pull/23211) ([Azat Khuzhin](https://github.com/azat)).
+* Make function name `unhex` case insensitive for compatibility with MySQL. [#23229](https://github.com/ClickHouse/ClickHouse/pull/23229) ([alexey-milovidov](https://github.com/alexey-milovidov)).
 * Implement functions `arrayHasAny`, `arrayHasAll`, `has`, `indexOf`, `countEqual` for generic case when types of array elements are different. In previous versions the functions `arrayHasAny`, `arrayHasAll` returned false and `has`, `indexOf`, `countEqual` thrown exception. Also add support for `Decimal` and big integer types in functions `has` and similar. This closes [#20272](https://github.com/ClickHouse/ClickHouse/issues/20272). [#23044](https://github.com/ClickHouse/ClickHouse/pull/23044) ([alexey-milovidov](https://github.com/alexey-milovidov)).
 * Raised the threshold on max number of matches in result of the function `extractAllGroupsHorizontal`. [#23036](https://github.com/ClickHouse/ClickHouse/pull/23036) ([Vasily Nemkov](https://github.com/Enmk)).
 * Do not perform `optimize_skip_unused_shards` for cluster with one node. [#22999](https://github.com/ClickHouse/ClickHouse/pull/22999) ([Azat Khuzhin](https://github.com/azat)).
-* Increase `max_uri_size` (the maximum size of URL in HTTP interface) to 1 MiB by default. This closes [#21197](https://github.com/ClickHouse/ClickHouse/issues/21197). [#22997](https://github.com/ClickHouse/ClickHouse/pull/22997) ([alexey-milovidov](https://github.com/alexey-milovidov)).
-* Added ability to run clickhouse-keeper with SSL. Config settings `keeper_server.tcp_port_secure` can be used for secure interaction between client and keeper-server. `keeper_server.raft_configuration.secure` can be used to enable internal secure communication between nodes. [#22992](https://github.com/ClickHouse/ClickHouse/pull/22992) ([alesapin](https://github.com/alesapin)).
-* Added ability to flush buffer only in background for StorageBuffer. [#22986](https://github.com/ClickHouse/ClickHouse/pull/22986) ([Azat Khuzhin](https://github.com/azat)).
+* Added ability to run clickhouse-keeper (experimental drop-in replacement to ZooKeeper) with SSL. Config settings `keeper_server.tcp_port_secure` can be used for secure interaction between client and keeper-server. `keeper_server.raft_configuration.secure` can be used to enable internal secure communication between nodes. [#22992](https://github.com/ClickHouse/ClickHouse/pull/22992) ([alesapin](https://github.com/alesapin)).
+* Added ability to flush buffer only in background for `Buffer` tables. [#22986](https://github.com/ClickHouse/ClickHouse/pull/22986) ([Azat Khuzhin](https://github.com/azat)).
 * When selecting from MergeTree table with NULL in WHERE condition, in rare cases, exception was thrown. This closes [#20019](https://github.com/ClickHouse/ClickHouse/issues/20019). [#22978](https://github.com/ClickHouse/ClickHouse/pull/22978) ([alexey-milovidov](https://github.com/alexey-milovidov)).
-* Fixed an error handling in Poco HTTP Client for AWS. [#22973](https://github.com/ClickHouse/ClickHouse/pull/22973) ([kreuzerkrieg](https://github.com/kreuzerkrieg)).
-* Respect `max_part_removal_threads` for ReplicatedMergeTree. [#22971](https://github.com/ClickHouse/ClickHouse/pull/22971) ([Azat Khuzhin](https://github.com/azat)).
-* Fix inactive_parts_to_throw_insert=0 with inactive_parts_to_delay_insert>0. [#22947](https://github.com/ClickHouse/ClickHouse/pull/22947) ([Azat Khuzhin](https://github.com/azat)).
-* Set `background_fetches_pool_size` to 8 that is better for production usage with frequent small insertions or slow ZooKeeper cluster. [#22945](https://github.com/ClickHouse/ClickHouse/pull/22945) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Fix error handling in Poco HTTP Client for AWS. [#22973](https://github.com/ClickHouse/ClickHouse/pull/22973) ([kreuzerkrieg](https://github.com/kreuzerkrieg)).
+* Respect `max_part_removal_threads` for `ReplicatedMergeTree`. [#22971](https://github.com/ClickHouse/ClickHouse/pull/22971) ([Azat Khuzhin](https://github.com/azat)).
+* Fix obscure corner case of MergeTree settings inactive_parts_to_throw_insert = 0 with inactive_parts_to_delay_insert > 0. [#22947](https://github.com/ClickHouse/ClickHouse/pull/22947) ([Azat Khuzhin](https://github.com/azat)).
 * `dateDiff` now works with `DateTime64` arguments (even for values outside of `DateTime` range) [#22931](https://github.com/ClickHouse/ClickHouse/pull/22931) ([Vasily Nemkov](https://github.com/Enmk)).
-* Added an ability to replicate MySQL databases containing views without failing. This is accomplished by ignoring the views. [#22760](https://github.com/ClickHouse/ClickHouse/pull/22760) ([Christian](https://github.com/cfroystad)).
+* MaterializeMySQL (experimental feature): added an ability to replicate MySQL databases containing views without failing. This is accomplished by ignoring the views. [#22760](https://github.com/ClickHouse/ClickHouse/pull/22760) ([Christian](https://github.com/cfroystad)).
 * Allow RBAC row policy via postgresql protocol. Closes [#22658](https://github.com/ClickHouse/ClickHouse/issues/22658). PostgreSQL protocol is enabled in configuration by default. [#22755](https://github.com/ClickHouse/ClickHouse/pull/22755) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * Add metric to track how much time is spend during waiting for Buffer layer lock. [#22725](https://github.com/ClickHouse/ClickHouse/pull/22725) ([Azat Khuzhin](https://github.com/azat)).
 * Allow to use CTE in VIEW definition. This closes [#22491](https://github.com/ClickHouse/ClickHouse/issues/22491). [#22657](https://github.com/ClickHouse/ClickHouse/pull/22657) ([Amos Bird](https://github.com/amosbird)).
 * Clear the rest of the screen and show cursor in `clickhouse-client` if previous program has left garbage in terminal. This closes [#16518](https://github.com/ClickHouse/ClickHouse/issues/16518). [#22634](https://github.com/ClickHouse/ClickHouse/pull/22634) ([alexey-milovidov](https://github.com/alexey-milovidov)).
 * Make `round` function to behave consistently on non-x86_64 platforms. Rounding half to nearest even (Banker's rounding) is used. [#22582](https://github.com/ClickHouse/ClickHouse/pull/22582) ([alexey-milovidov](https://github.com/alexey-milovidov)).
-* Correctly check structure of async distributed blocks. [#22325](https://github.com/ClickHouse/ClickHouse/pull/22325) ([Azat Khuzhin](https://github.com/azat)).
-* Added nanodbc instead of Poco::ODBC. Closes [#9678](https://github.com/ClickHouse/ClickHouse/issues/9678). Add support for DateTime64 and Decimal* for ODBC table engine. Closes [#21961](https://github.com/ClickHouse/ClickHouse/issues/21961). Fixed issue with cyrillic text being truncated. Closes [#16246](https://github.com/ClickHouse/ClickHouse/issues/16246). Added connection pools for odbc bridge. [#21972](https://github.com/ClickHouse/ClickHouse/pull/21972) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Correctly check structure of blocks of data that are sending by Distributed tables. [#22325](https://github.com/ClickHouse/ClickHouse/pull/22325) ([Azat Khuzhin](https://github.com/azat)).
 * Allow publishing Kafka errors to a virtual column of Kafka engine, controlled by the `kafka_handle_error_mode` setting. [#21850](https://github.com/ClickHouse/ClickHouse/pull/21850) ([fastio](https://github.com/fastio)).
 * Add aliases `simpleJSONExtract/simpleJSONHas` to `visitParam/visitParamExtract{UInt, Int, Bool, Float, Raw, String}`. Fixes [#21383](https://github.com/ClickHouse/ClickHouse/issues/21383). [#21519](https://github.com/ClickHouse/ClickHouse/pull/21519) ([fastio](https://github.com/fastio)).
 * Add `clickhouse-library-bridge` for library dictionary source. Closes [#9502](https://github.com/ClickHouse/ClickHouse/issues/9502). [#21509](https://github.com/ClickHouse/ClickHouse/pull/21509) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * Forbid to drop a column if it's referenced by materialized view. Closes [#21164](https://github.com/ClickHouse/ClickHouse/issues/21164). [#21303](https://github.com/ClickHouse/ClickHouse/pull/21303) ([flynn](https://github.com/ucasFL)).
-* Support dynamic interserver credentials. [#14113](https://github.com/ClickHouse/ClickHouse/pull/14113) ([johnskopis](https://github.com/johnskopis)).
+* Support dynamic interserver credentials (rotating credentials without downtime). [#14113](https://github.com/ClickHouse/ClickHouse/pull/14113) ([johnskopis](https://github.com/johnskopis)).
+* Add support for Kafka storage with `Arrow` and `ArrowStream` format messages. [#23415](https://github.com/ClickHouse/ClickHouse/pull/23415) ([Chao Ma](https://github.com/godliness)).
+* * Fixed missing semicolon in exception message. The user may find this exception message unpleasant to read. [#23208](https://github.com/ClickHouse/ClickHouse/pull/23208) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Fixed missing whitespace in some exception messages about `LowCardinality` type. [#23207](https://github.com/ClickHouse/ClickHouse/pull/23207) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Some values were formatted with alignment in center in table cells in `Markdown` format. Not anymore. [#23096](https://github.com/ClickHouse/ClickHouse/pull/23096) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Remove non-essential details from suggestions in clickhouse-client. This closes [#22158](https://github.com/ClickHouse/ClickHouse/issues/22158). [#23040](https://github.com/ClickHouse/ClickHouse/pull/23040) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Correct calculation of `bytes_allocated` field in system.dictionaries for sparse_hashed dictionaries. [#22867](https://github.com/ClickHouse/ClickHouse/pull/22867) ([Azat Khuzhin](https://github.com/azat)).
+* Fixed approximate total rows accounting for reverse reading from MergeTree. [#22726](https://github.com/ClickHouse/ClickHouse/pull/22726) ([Azat Khuzhin](https://github.com/azat)).
+* Fix the case when it was possible to configure dictionary with clickhouse source that was looking to itself that leads to infinite loop. Closes [#14314](https://github.com/ClickHouse/ClickHouse/issues/14314). [#22479](https://github.com/ClickHouse/ClickHouse/pull/22479) ([Maksim Kita](https://github.com/kitaisreal)).
 
 #### Bug Fix
 
-* HashedDictionary complex key update field initial load fix. Closes [#23800](https://github.com/ClickHouse/ClickHouse/issues/23800). [#23824](https://github.com/ClickHouse/ClickHouse/pull/23824) ([Maksim Kita](https://github.com/kitaisreal)).
-* Fixed an error `Can't initialize pipeline with empty pipe` for queries with `GLOBAL IN/JOIN` and `use_hedged_requests`. Fixes [#23431](https://github.com/ClickHouse/ClickHouse/issues/23431). [#23805](https://github.com/ClickHouse/ClickHouse/pull/23805) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
-* Fixed remote JDBC bridge timeout connection issue. Closes [#9609](https://github.com/ClickHouse/ClickHouse/issues/9609). [#23771](https://github.com/ClickHouse/ClickHouse/pull/23771) ([Maksim Kita](https://github.com/kitaisreal)).
+* Fixed the behavior when disabling `input_format_with_names_use_header ` setting discards all the input with CSVWithNames format. This fixes [#22406](https://github.com/ClickHouse/ClickHouse/issues/22406). [#23202](https://github.com/ClickHouse/ClickHouse/pull/23202) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Fixed remote JDBC bridge timeout connection issue. Closes [#9609](https://github.com/ClickHouse/ClickHouse/issues/9609). [#23771](https://github.com/ClickHouse/ClickHouse/pull/23771) ([Maksim Kita](https://github.com/kitaisreal), [alexey-milovidov](https://github.com/alexey-milovidov)).
+* Fix the logic of initial load of `complex_key_hashed` if `update_field` is specified. Closes [#23800](https://github.com/ClickHouse/ClickHouse/issues/23800). [#23824](https://github.com/ClickHouse/ClickHouse/pull/23824) ([Maksim Kita](https://github.com/kitaisreal)).
+* Fixed an error `Can't initialize pipeline with empty pipe` for queries with `GLOBAL IN/JOIN` when the setting `use_hedged_requests` is enabled. Fixes [#23431](https://github.com/ClickHouse/ClickHouse/issues/23431). [#23805](https://github.com/ClickHouse/ClickHouse/pull/23805) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
 * Fixed crash when `PREWHERE` and row policy filter are both in effect with empty result. [#23763](https://github.com/ClickHouse/ClickHouse/pull/23763) ([Amos Bird](https://github.com/amosbird)).
 * Avoid possible "Cannot schedule a task" error (in case some exception had been occurred) on INSERT into Distributed. [#23744](https://github.com/ClickHouse/ClickHouse/pull/23744) ([Azat Khuzhin](https://github.com/azat)).
 * Added an exception in case of completely the same values in both samples in aggregate function `mannWhitneyUTest`. This fixes [#23646](https://github.com/ClickHouse/ClickHouse/issues/23646). [#23654](https://github.com/ClickHouse/ClickHouse/pull/23654) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
 * Fixed server fault when inserting data through HTTP caused an exception. This fixes [#23512](https://github.com/ClickHouse/ClickHouse/issues/23512). [#23643](https://github.com/ClickHouse/ClickHouse/pull/23643) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
 * Fixed misinterpretation of some `LIKE` expressions with escape sequences. [#23610](https://github.com/ClickHouse/ClickHouse/pull/23610) ([alexey-milovidov](https://github.com/alexey-milovidov)).
 * Fixed restart / stop command hanging. Closes [#20214](https://github.com/ClickHouse/ClickHouse/issues/20214). [#23552](https://github.com/ClickHouse/ClickHouse/pull/23552) ([filimonov](https://github.com/filimonov)).
-* Fixed `columns` function when multiple joins in select query. Closes [#22736](https://github.com/ClickHouse/ClickHouse/issues/22736). [#23501](https://github.com/ClickHouse/ClickHouse/pull/23501) ([Maksim Kita](https://github.com/kitaisreal)).
-* Fixed a crash when modifying column's default value when a column itself is used as ReplacingMergeTree's parameter. [#23483](https://github.com/ClickHouse/ClickHouse/pull/23483) ([hexiaoting](https://github.com/hexiaoting)).
+* Fixed `COLUMNS` matcher in case of multiple JOINs in select query. Closes [#22736](https://github.com/ClickHouse/ClickHouse/issues/22736). [#23501](https://github.com/ClickHouse/ClickHouse/pull/23501) ([Maksim Kita](https://github.com/kitaisreal)).
+* Fixed a crash when modifying column's default value when a column itself is used as `ReplacingMergeTree`'s parameter. [#23483](https://github.com/ClickHouse/ClickHouse/pull/23483) ([hexiaoting](https://github.com/hexiaoting)).
 * Fixed corner cases in vertical merges with `ReplacingMergeTree`. In rare cases they could lead to fails of merges with exceptions like `Incomplete granules are not allowed while blocks are granules size`. [#23459](https://github.com/ClickHouse/ClickHouse/pull/23459) ([Anton Popov](https://github.com/CurtizJ)).
-* Fixed bug that does not allow cast from empty array literal, to array with dimensions greater than 1. Closes [#14476](https://github.com/ClickHouse/ClickHouse/issues/14476). [#23456](https://github.com/ClickHouse/ClickHouse/pull/23456) ([Maksim Kita](https://github.com/kitaisreal)).
+* Fixed bug that does not allow cast from empty array literal, to array with dimensions greater than 1, e.g. `CAST([] AS Array(Array(String)))`. Closes [#14476](https://github.com/ClickHouse/ClickHouse/issues/14476). [#23456](https://github.com/ClickHouse/ClickHouse/pull/23456) ([Maksim Kita](https://github.com/kitaisreal)).
 * Fixed a bug when `deltaSum` aggregate function produced incorrect result after resetting the counter. [#23437](https://github.com/ClickHouse/ClickHouse/pull/23437) ([Russ Frank](https://github.com/rf)).
 * Fixed `Cannot unlink file` error on unsuccessful creation of ReplicatedMergeTree table with multidisk configuration. This closes [#21755](https://github.com/ClickHouse/ClickHouse/issues/21755). [#23433](https://github.com/ClickHouse/ClickHouse/pull/23433) ([tavplubix](https://github.com/tavplubix)).
-* Fixed a lack of support for Kafka storage with `arrow` and `arrowstream` format messages. [#23415](https://github.com/ClickHouse/ClickHouse/pull/23415) ([Chao Ma](https://github.com/godliness)).
-* Allow to move more conditions to `PREWHERE` as it was before version 21.1. Insufficient number of moved condtions could lead to worse performance. [#23397](https://github.com/ClickHouse/ClickHouse/pull/23397) ([Anton Popov](https://github.com/CurtizJ)).
-* Remove support for `argMin` and `argMax` for single `Tuple` argument. The code was not memory-safe. The feature was added by mistake and it is confusing for people. These functions can be reintroduced under different names later. This fixes [#22384](https://github.com/ClickHouse/ClickHouse/issues/22384) and reverts [#17359](https://github.com/ClickHouse/ClickHouse/issues/17359). [#23393](https://github.com/ClickHouse/ClickHouse/pull/23393) ([alexey-milovidov](https://github.com/alexey-milovidov)).
 * Fixed incompatible constant expression generation during partition pruning based on virtual columns. This fixes https://github.com/ClickHouse/ClickHouse/pull/21401#discussion_r611888913. [#23366](https://github.com/ClickHouse/ClickHouse/pull/23366) ([Amos Bird](https://github.com/amosbird)).
 * Fixed a crash when setting join_algorithm is set to 'auto' and Join is performed with a Dictionary. Close [#23002](https://github.com/ClickHouse/ClickHouse/issues/23002). [#23312](https://github.com/ClickHouse/ClickHouse/pull/23312) ([Vladimir](https://github.com/vdimir)).
 * Don't relax NOT conditions during partition pruning. This fixes [#23305](https://github.com/ClickHouse/ClickHouse/issues/23305) and [#21539](https://github.com/ClickHouse/ClickHouse/issues/21539). [#23310](https://github.com/ClickHouse/ClickHouse/pull/23310) ([Amos Bird](https://github.com/amosbird)).
-* Fix possible crash in case if `unknown packet` was received form remote query (with `async_socket_for_remote` enabled). Maybe fixes [#21167](https://github.com/ClickHouse/ClickHouse/issues/21167). [#23309](https://github.com/ClickHouse/ClickHouse/pull/23309) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix possible crash in case if `unknown packet` was received from remote query (with `async_socket_for_remote` enabled). Maybe fixes [#21167](https://github.com/ClickHouse/ClickHouse/issues/21167). [#23309](https://github.com/ClickHouse/ClickHouse/pull/23309) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
 * Fixed very rare race condition on background cleanup of old blocks. It might cause a block not to be deduplicated if it's too close to the end of deduplication window. [#23301](https://github.com/ClickHouse/ClickHouse/pull/23301) ([tavplubix](https://github.com/tavplubix)).
 * Fixed very rare (distributed) race condition between creation and removal of ReplicatedMergeTree tables. It might cause exceptions like `node doesn't exist` on attempt to create replicated table. Fixes [#21419](https://github.com/ClickHouse/ClickHouse/issues/21419). [#23294](https://github.com/ClickHouse/ClickHouse/pull/23294) ([tavplubix](https://github.com/tavplubix)).
 * Fixed simple key dictionary from DDL creation if primary key is not first attribute. Fixes [#23236](https://github.com/ClickHouse/ClickHouse/issues/23236). [#23262](https://github.com/ClickHouse/ClickHouse/pull/23262) ([Maksim Kita](https://github.com/kitaisreal)).
 * Fixed reading from ODBC when there are many long column names in a table. Closes [#8853](https://github.com/ClickHouse/ClickHouse/issues/8853). [#23215](https://github.com/ClickHouse/ClickHouse/pull/23215) ([Kseniia Sumarokova](https://github.com/kssenii)).
-* Fixed missing semicolon in exception message. The user may find this exception message unpleasant to read. [#23208](https://github.com/ClickHouse/ClickHouse/pull/23208) ([alexey-milovidov](https://github.com/alexey-milovidov)).
-* Fixed missing whitespace in some exception messages about `LowCardinality` type. [#23207](https://github.com/ClickHouse/ClickHouse/pull/23207) ([alexey-milovidov](https://github.com/alexey-milovidov)).
-* Fixed the behavior when disabling `input_format_with_names_use_header ` setting discards all the input with CSVWithNames format. This fixes [#22406](https://github.com/ClickHouse/ClickHouse/issues/22406). [#23202](https://github.com/ClickHouse/ClickHouse/pull/23202) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
-* Fixed `Not found column` error when selecting from `MaterializeMySQL` with condition on key column. Fixes [#22432](https://github.com/ClickHouse/ClickHouse/issues/22432). [#23200](https://github.com/ClickHouse/ClickHouse/pull/23200) ([tavplubix](https://github.com/tavplubix)).
-* QueryAliasVisitor to prefer alias for ASTWithAlias if subquery was optimized to constant. Fixes [#22924](https://github.com/ClickHouse/ClickHouse/issues/22924). Fixes [#10401](https://github.com/ClickHouse/ClickHouse/issues/10401). [#23191](https://github.com/ClickHouse/ClickHouse/pull/23191) ([Maksim Kita](https://github.com/kitaisreal)).
+* MaterializeMySQL (experimental feature): fixed `Not found column` error when selecting from `MaterializeMySQL` with condition on key column. Fixes [#22432](https://github.com/ClickHouse/ClickHouse/issues/22432). [#23200](https://github.com/ClickHouse/ClickHouse/pull/23200) ([tavplubix](https://github.com/tavplubix)).
+* Correct aliases handling if subquery was optimized to constant. Fixes [#22924](https://github.com/ClickHouse/ClickHouse/issues/22924). Fixes [#10401](https://github.com/ClickHouse/ClickHouse/issues/10401). [#23191](https://github.com/ClickHouse/ClickHouse/pull/23191) ([Maksim Kita](https://github.com/kitaisreal)).
 * Server might fail to start if `data_type_default_nullable` setting is enabled in default profile, it's fixed. Fixes [#22573](https://github.com/ClickHouse/ClickHouse/issues/22573). [#23185](https://github.com/ClickHouse/ClickHouse/pull/23185) ([tavplubix](https://github.com/tavplubix)).
 * Fixed a crash on shutdown which happened because of wrong accounting of current connections. [#23154](https://github.com/ClickHouse/ClickHouse/pull/23154) ([Vitaly Baranov](https://github.com/vitlibar)).
-* Some values were formatted with alignment in center in table cells in `Markdown` format. Not anymore. [#23096](https://github.com/ClickHouse/ClickHouse/pull/23096) ([alexey-milovidov](https://github.com/alexey-milovidov)).
 * Fixed `Table .inner_id... doesn't exist` error when selecting from Materialized View after detaching it from Atomic database and attaching back. [#23047](https://github.com/ClickHouse/ClickHouse/pull/23047) ([tavplubix](https://github.com/tavplubix)).
-* Remove non-essential details from suggestions in clickhouse-client. This closes [#22158](https://github.com/ClickHouse/ClickHouse/issues/22158). [#23040](https://github.com/ClickHouse/ClickHouse/pull/23040) ([alexey-milovidov](https://github.com/alexey-milovidov)).
 * Fix error `Cannot find column in ActionsDAG result` which may happen if subquery uses `untuple`. Fixes [#22290](https://github.com/ClickHouse/ClickHouse/issues/22290). [#22991](https://github.com/ClickHouse/ClickHouse/pull/22991) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
 * Fix usage of constant columns of type `Map` with nullable values. [#22939](https://github.com/ClickHouse/ClickHouse/pull/22939) ([Anton Popov](https://github.com/CurtizJ)).
-* fixed `formatDateTime()` on `DateTime64` and "%C" format specifier fixed `toDateTime64()` for large values and non-zero scale. ... [#22937](https://github.com/ClickHouse/ClickHouse/pull/22937) ([Vasily Nemkov](https://github.com/Enmk)).
+* fixed `formatDateTime()` on `DateTime64` and "%C" format specifier fixed `toDateTime64()` for large values and non-zero scale. [#22937](https://github.com/ClickHouse/ClickHouse/pull/22937) ([Vasily Nemkov](https://github.com/Enmk)).
 * Fixed a crash when using `mannWhitneyUTest` and `rankCorr` with window functions. This fixes [#22728](https://github.com/ClickHouse/ClickHouse/issues/22728). [#22876](https://github.com/ClickHouse/ClickHouse/pull/22876) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
-* Fixed `bytes_allocated` field in system.dictionaries for sparse_hashed dictionaries. [#22867](https://github.com/ClickHouse/ClickHouse/pull/22867) ([Azat Khuzhin](https://github.com/azat)).
-* Fixed possible hanging in concurrent DROP/CREATE of TEMPORARY LIVE VIEW in `TemporaryLiveViewCleaner`, see https://gist.github.com/vzakaznikov/0c03195960fc86b56bfe2bc73a90019e. [#22858](https://github.com/ClickHouse/ClickHouse/pull/22858) ([Vitaly Baranov](https://github.com/vitlibar)).
+* LIVE VIEW (experimental feature): fixed possible hanging in concurrent DROP/CREATE of TEMPORARY LIVE VIEW in `TemporaryLiveViewCleaner`, [see](https://gist.github.com/vzakaznikov/0c03195960fc86b56bfe2bc73a90019e). [#22858](https://github.com/ClickHouse/ClickHouse/pull/22858) ([Vitaly Baranov](https://github.com/vitlibar)).
 * Fixed pushdown of `HAVING` in case, when filter column is used in aggregation. [#22763](https://github.com/ClickHouse/ClickHouse/pull/22763) ([Anton Popov](https://github.com/CurtizJ)).
-* Fixed approximate total rows accounting for reverse reading from MergeTree. [#22726](https://github.com/ClickHouse/ClickHouse/pull/22726) ([Azat Khuzhin](https://github.com/azat)).
 * Fixed possible hangs in Zookeeper requests in case of OOM exception. Fixes [#22438](https://github.com/ClickHouse/ClickHouse/issues/22438). [#22684](https://github.com/ClickHouse/ClickHouse/pull/22684) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
 * Fixed wait for mutations on several replicas for ReplicatedMergeTree table engines. Previously, mutation/alter query may finish before mutation actually executed on other replicas. [#22669](https://github.com/ClickHouse/ClickHouse/pull/22669) ([alesapin](https://github.com/alesapin)).
-* Fixed LOGICAL_ERROR for Log with nested types w/o columns in the SELECT clause. [#22654](https://github.com/ClickHouse/ClickHouse/pull/22654) ([Azat Khuzhin](https://github.com/azat)).
-* Fixed a bug with unlimited wait for auxiliary AWS requests. [#22594](https://github.com/ClickHouse/ClickHouse/pull/22594) ([Vladimir Chebotarev](https://github.com/excitoon)).
+* Fixed exception for Log with nested types without columns in the SELECT clause. [#22654](https://github.com/ClickHouse/ClickHouse/pull/22654) ([Azat Khuzhin](https://github.com/azat)).
+* Fix unlimited wait for auxiliary AWS requests. [#22594](https://github.com/ClickHouse/ClickHouse/pull/22594) ([Vladimir Chebotarev](https://github.com/excitoon)).
 * Fixed a crash when client closes connection very early [#22579](https://github.com/ClickHouse/ClickHouse/issues/22579). [#22591](https://github.com/ClickHouse/ClickHouse/pull/22591) ([nvartolomei](https://github.com/nvartolomei)).
-* Fixed an incorrect formatting of function `map` in distributed queries. [#22588](https://github.com/ClickHouse/ClickHouse/pull/22588) ([foolchi](https://github.com/foolchi)).
-* Avoid UB in *Log engines for rwlock unlock due to unlock from another thread. [#22583](https://github.com/ClickHouse/ClickHouse/pull/22583) ([Azat Khuzhin](https://github.com/azat)).
-* Fixed UB by unlocking the rwlock of the TinyLog from the same thread. [#22560](https://github.com/ClickHouse/ClickHouse/pull/22560) ([Azat Khuzhin](https://github.com/azat)).
+* `Map` data type (experimental feature): fixed an incorrect formatting of function `map` in distributed queries. [#22588](https://github.com/ClickHouse/ClickHouse/pull/22588) ([foolchi](https://github.com/foolchi)).
 * Fixed deserialization of empty string without newline at end of TSV format. This closes [#20244](https://github.com/ClickHouse/ClickHouse/issues/20244). Possible workaround without version update: set `input_format_null_as_default` to zero. It was zero in old versions. [#22527](https://github.com/ClickHouse/ClickHouse/pull/22527) ([alexey-milovidov](https://github.com/alexey-milovidov)).
-* Fixed wrong cast of a column of `LowCardinality` type in MergeJoin algorithm. Close [#22386](https://github.com/ClickHouse/ClickHouse/issues/22386), close [#22388](https://github.com/ClickHouse/ClickHouse/issues/22388). [#22510](https://github.com/ClickHouse/ClickHouse/pull/22510) ([Vladimir](https://github.com/vdimir)).
-* Fixed `ClickHouseDictionarySource` configuration loop. Closes [#14314](https://github.com/ClickHouse/ClickHouse/issues/14314). [#22479](https://github.com/ClickHouse/ClickHouse/pull/22479) ([Maksim Kita](https://github.com/kitaisreal)).
-* Fixed a race condition in HedgedConnections which leads to crash. This fixes [#22161](https://github.com/ClickHouse/ClickHouse/issues/22161). [#22443](https://github.com/ClickHouse/ClickHouse/pull/22443) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fixed wrong cast of a column of `LowCardinality` type in Merge Join algorithm. Close [#22386](https://github.com/ClickHouse/ClickHouse/issues/22386), close [#22388](https://github.com/ClickHouse/ClickHouse/issues/22388). [#22510](https://github.com/ClickHouse/ClickHouse/pull/22510) ([Vladimir](https://github.com/vdimir)).
+* Fixed a race condition in hedged connections which leads to crash. This fixes [#22161](https://github.com/ClickHouse/ClickHouse/issues/22161). [#22443](https://github.com/ClickHouse/ClickHouse/pull/22443) ([Kruglov Pavel](https://github.com/Avogar)).
 * Buffer overflow (on read) was possible in `tokenbf_v1` full text index. The excessive bytes are not used but the read operation may lead to crash in rare cases. This closes [#19233](https://github.com/ClickHouse/ClickHouse/issues/19233). [#22421](https://github.com/ClickHouse/ClickHouse/pull/22421) ([alexey-milovidov](https://github.com/alexey-milovidov)).
 * Do not limit HTTP chunk size. Fixes [#21907](https://github.com/ClickHouse/ClickHouse/issues/21907). [#22322](https://github.com/ClickHouse/ClickHouse/pull/22322) ([Ivan](https://github.com/abyss7)).
 * Fixed a bug, which leads to underaggregation of data in case of enabled `optimize_aggregation_in_order` and many parts in table. Slightly improve performance of aggregation with enabled `optimize_aggregation_in_order`. [#21889](https://github.com/ClickHouse/ClickHouse/pull/21889) ([Anton Popov](https://github.com/CurtizJ)).
-* Check if table function view is used as a column. This complements https://github.com/ClickHouse/ClickHouse/pull/20350. [#21465](https://github.com/ClickHouse/ClickHouse/pull/21465) ([Amos Bird](https://github.com/amosbird)).
-* * Fix "unknown column" error for tables with `Merge` engine in queris with `JOIN` and aggregation. Closes [#18368](https://github.com/ClickHouse/ClickHouse/issues/18368), close [#22226](https://github.com/ClickHouse/ClickHouse/issues/22226). [#21370](https://github.com/ClickHouse/ClickHouse/pull/21370) ([Vladimir](https://github.com/vdimir)).
-* Fixed name clashes in `PredicateRewriteVisitor`. It caused incorrect `WHERE` filtration after full join. Close [#20497](https://github.com/ClickHouse/ClickHouse/issues/20497). [#20622](https://github.com/ClickHouse/ClickHouse/pull/20622) ([Vladimir](https://github.com/vdimir)).
-* Fixed very rare bug when quorum insert with `quorum_parallel=1` is not really "quorum" because of deduplication. [#18215](https://github.com/ClickHouse/ClickHouse/pull/18215) ([filimonov](https://github.com/filimonov)).
+* Check if table function view is used as a column. This complements #20350. [#21465](https://github.com/ClickHouse/ClickHouse/pull/21465) ([Amos Bird](https://github.com/amosbird)).
+* Fix "unknown column" error for tables with `Merge` engine in queris with `JOIN` and aggregation. Closes [#18368](https://github.com/ClickHouse/ClickHouse/issues/18368), close [#22226](https://github.com/ClickHouse/ClickHouse/issues/22226). [#21370](https://github.com/ClickHouse/ClickHouse/pull/21370) ([Vladimir](https://github.com/vdimir)).
+* Fixed name clashes in pushdown optimization. It caused incorrect `WHERE` filtration after FULL JOIN. Close [#20497](https://github.com/ClickHouse/ClickHouse/issues/20497). [#20622](https://github.com/ClickHouse/ClickHouse/pull/20622) ([Vladimir](https://github.com/vdimir)).
+* Fixed very rare bug when quorum insert with `quorum_parallel=1` is not really "quorum" because of deduplication. [#18215](https://github.com/ClickHouse/ClickHouse/pull/18215) ([filimonov](https://github.com/filimonov) - reported, [alesapin](https://github.com/alesapin) - fixed).
 
 #### Build/Testing/Packaging Improvement
 
+* Run stateless tests in parallel in CI. [#22300](https://github.com/ClickHouse/ClickHouse/pull/22300) ([alesapin](https://github.com/alesapin)).
 * Simplify debian packages. This fixes [#21698](https://github.com/ClickHouse/ClickHouse/issues/21698). [#22976](https://github.com/ClickHouse/ClickHouse/pull/22976) ([alexey-milovidov](https://github.com/alexey-milovidov)).
 * Added support for ClickHouse build on Apple M1. [#21639](https://github.com/ClickHouse/ClickHouse/pull/21639) ([changvvb](https://github.com/changvvb)).
 * Fixed ClickHouse Keeper build for MacOS. [#22860](https://github.com/ClickHouse/ClickHouse/pull/22860) ([alesapin](https://github.com/alesapin)).
@@ -134,12 +130,14 @@
 * Added function alignment for possibly better performance. [#21431](https://github.com/ClickHouse/ClickHouse/pull/21431) ([Danila Kutenin](https://github.com/danlark1)).
 * Adjust some tests to output identical results on amd64 and aarch64 (qemu). The result was depending on implementation specific CPU behaviour. [#22590](https://github.com/ClickHouse/ClickHouse/pull/22590) ([alexey-milovidov](https://github.com/alexey-milovidov)).
 * Allow query profiling only on x86_64. See [#15174](https://github.com/ClickHouse/ClickHouse/issues/15174)#issuecomment-812954965 and [#15638](https://github.com/ClickHouse/ClickHouse/issues/15638)#issuecomment-703805337. This closes [#15638](https://github.com/ClickHouse/ClickHouse/issues/15638). [#22580](https://github.com/ClickHouse/ClickHouse/pull/22580) ([alexey-milovidov](https://github.com/alexey-milovidov)).
-* Allow building with unbundled xz (lzma) using USE_INTERNAL_XZ_LIBRARY=OFF ... [#22571](https://github.com/ClickHouse/ClickHouse/pull/22571) ([Kfir Itzhak](https://github.com/mastertheknife)).
-* Enable the bundled openldap on ppc64le ... [#22487](https://github.com/ClickHouse/ClickHouse/pull/22487) ([Kfir Itzhak](https://github.com/mastertheknife)).
-* Disable incompatible libraries (platform specific typically) on ppc64le ... [#22475](https://github.com/ClickHouse/ClickHouse/pull/22475) ([Kfir Itzhak](https://github.com/mastertheknife)).
-* Add on-demand check for clickhouse Keeper. [#22373](https://github.com/ClickHouse/ClickHouse/pull/22373) ([alesapin](https://github.com/alesapin)).
-* Run stateless tests in parallel in CI. Depends on [#22181](https://github.com/ClickHouse/ClickHouse/issues/22181). [#22300](https://github.com/ClickHouse/ClickHouse/pull/22300) ([alesapin](https://github.com/alesapin)).
+* Allow building with unbundled xz (lzma) using `USE_INTERNAL_XZ_LIBRARY=OFF` CMake option. [#22571](https://github.com/ClickHouse/ClickHouse/pull/22571) ([Kfir Itzhak](https://github.com/mastertheknife)).
+* Enable bundled `openldap` on `ppc64le` [#22487](https://github.com/ClickHouse/ClickHouse/pull/22487) ([Kfir Itzhak](https://github.com/mastertheknife)).
+* Disable incompatible libraries (platform specific typically) on `ppc64le` [#22475](https://github.com/ClickHouse/ClickHouse/pull/22475) ([Kfir Itzhak](https://github.com/mastertheknife)).
+* Add Jepsen test in CI for clickhouse Keeper. [#22373](https://github.com/ClickHouse/ClickHouse/pull/22373) ([alesapin](https://github.com/alesapin)).
 * Build `jemalloc` with support for [heap profiling](https://github.com/jemalloc/jemalloc/wiki/Use-Case%3A-Heap-Profiling). [#22834](https://github.com/ClickHouse/ClickHouse/pull/22834) ([nvartolomei](https://github.com/nvartolomei)).
+* Avoid UB in `*Log` engines for rwlock unlock due to unlock from another thread. [#22583](https://github.com/ClickHouse/ClickHouse/pull/22583) ([Azat Khuzhin](https://github.com/azat)).
+* Fixed UB by unlocking the rwlock of the TinyLog from the same thread. [#22560](https://github.com/ClickHouse/ClickHouse/pull/22560) ([Azat Khuzhin](https://github.com/azat)).
+
 
 ## ClickHouse release 21.4
 

From 4d42acba6a212a62c069706f720552dab63f262e Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Fri, 21 May 2021 03:54:25 +0300
Subject: [PATCH 49/76] Update CHANGELOG.md

---
 CHANGELOG.md | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index eaa3682051e..298cbd2ccf4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -68,10 +68,10 @@
 
 #### Bug Fix
 
+* Multiple fixes for hedged requests. Fixed an error `Can't initialize pipeline with empty pipe` for queries with `GLOBAL IN/JOIN` when the setting `use_hedged_requests` is enabled. Fixes [#23431](https://github.com/ClickHouse/ClickHouse/issues/23431). [#23805](https://github.com/ClickHouse/ClickHouse/pull/23805) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). Fixed a race condition in hedged connections which leads to crash. This fixes [#22161](https://github.com/ClickHouse/ClickHouse/issues/22161). [#22443](https://github.com/ClickHouse/ClickHouse/pull/22443) ([Kruglov Pavel](https://github.com/Avogar)). Fix possible crash in case if `unknown packet` was received from remote query (with `async_socket_for_remote` enabled). Fixes [#21167](https://github.com/ClickHouse/ClickHouse/issues/21167). [#23309](https://github.com/ClickHouse/ClickHouse/pull/23309) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
 * Fixed the behavior when disabling `input_format_with_names_use_header ` setting discards all the input with CSVWithNames format. This fixes [#22406](https://github.com/ClickHouse/ClickHouse/issues/22406). [#23202](https://github.com/ClickHouse/ClickHouse/pull/23202) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
 * Fixed remote JDBC bridge timeout connection issue. Closes [#9609](https://github.com/ClickHouse/ClickHouse/issues/9609). [#23771](https://github.com/ClickHouse/ClickHouse/pull/23771) ([Maksim Kita](https://github.com/kitaisreal), [alexey-milovidov](https://github.com/alexey-milovidov)).
 * Fix the logic of initial load of `complex_key_hashed` if `update_field` is specified. Closes [#23800](https://github.com/ClickHouse/ClickHouse/issues/23800). [#23824](https://github.com/ClickHouse/ClickHouse/pull/23824) ([Maksim Kita](https://github.com/kitaisreal)).
-* Fixed an error `Can't initialize pipeline with empty pipe` for queries with `GLOBAL IN/JOIN` when the setting `use_hedged_requests` is enabled. Fixes [#23431](https://github.com/ClickHouse/ClickHouse/issues/23431). [#23805](https://github.com/ClickHouse/ClickHouse/pull/23805) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
 * Fixed crash when `PREWHERE` and row policy filter are both in effect with empty result. [#23763](https://github.com/ClickHouse/ClickHouse/pull/23763) ([Amos Bird](https://github.com/amosbird)).
 * Avoid possible "Cannot schedule a task" error (in case some exception had been occurred) on INSERT into Distributed. [#23744](https://github.com/ClickHouse/ClickHouse/pull/23744) ([Azat Khuzhin](https://github.com/azat)).
 * Added an exception in case of completely the same values in both samples in aggregate function `mannWhitneyUTest`. This fixes [#23646](https://github.com/ClickHouse/ClickHouse/issues/23646). [#23654](https://github.com/ClickHouse/ClickHouse/pull/23654) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
@@ -87,7 +87,6 @@
 * Fixed incompatible constant expression generation during partition pruning based on virtual columns. This fixes https://github.com/ClickHouse/ClickHouse/pull/21401#discussion_r611888913. [#23366](https://github.com/ClickHouse/ClickHouse/pull/23366) ([Amos Bird](https://github.com/amosbird)).
 * Fixed a crash when setting join_algorithm is set to 'auto' and Join is performed with a Dictionary. Close [#23002](https://github.com/ClickHouse/ClickHouse/issues/23002). [#23312](https://github.com/ClickHouse/ClickHouse/pull/23312) ([Vladimir](https://github.com/vdimir)).
 * Don't relax NOT conditions during partition pruning. This fixes [#23305](https://github.com/ClickHouse/ClickHouse/issues/23305) and [#21539](https://github.com/ClickHouse/ClickHouse/issues/21539). [#23310](https://github.com/ClickHouse/ClickHouse/pull/23310) ([Amos Bird](https://github.com/amosbird)).
-* Fix possible crash in case if `unknown packet` was received from remote query (with `async_socket_for_remote` enabled). Maybe fixes [#21167](https://github.com/ClickHouse/ClickHouse/issues/21167). [#23309](https://github.com/ClickHouse/ClickHouse/pull/23309) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
 * Fixed very rare race condition on background cleanup of old blocks. It might cause a block not to be deduplicated if it's too close to the end of deduplication window. [#23301](https://github.com/ClickHouse/ClickHouse/pull/23301) ([tavplubix](https://github.com/tavplubix)).
 * Fixed very rare (distributed) race condition between creation and removal of ReplicatedMergeTree tables. It might cause exceptions like `node doesn't exist` on attempt to create replicated table. Fixes [#21419](https://github.com/ClickHouse/ClickHouse/issues/21419). [#23294](https://github.com/ClickHouse/ClickHouse/pull/23294) ([tavplubix](https://github.com/tavplubix)).
 * Fixed simple key dictionary from DDL creation if primary key is not first attribute. Fixes [#23236](https://github.com/ClickHouse/ClickHouse/issues/23236). [#23262](https://github.com/ClickHouse/ClickHouse/pull/23262) ([Maksim Kita](https://github.com/kitaisreal)).
@@ -111,7 +110,6 @@
 * `Map` data type (experimental feature): fixed an incorrect formatting of function `map` in distributed queries. [#22588](https://github.com/ClickHouse/ClickHouse/pull/22588) ([foolchi](https://github.com/foolchi)).
 * Fixed deserialization of empty string without newline at end of TSV format. This closes [#20244](https://github.com/ClickHouse/ClickHouse/issues/20244). Possible workaround without version update: set `input_format_null_as_default` to zero. It was zero in old versions. [#22527](https://github.com/ClickHouse/ClickHouse/pull/22527) ([alexey-milovidov](https://github.com/alexey-milovidov)).
 * Fixed wrong cast of a column of `LowCardinality` type in Merge Join algorithm. Close [#22386](https://github.com/ClickHouse/ClickHouse/issues/22386), close [#22388](https://github.com/ClickHouse/ClickHouse/issues/22388). [#22510](https://github.com/ClickHouse/ClickHouse/pull/22510) ([Vladimir](https://github.com/vdimir)).
-* Fixed a race condition in hedged connections which leads to crash. This fixes [#22161](https://github.com/ClickHouse/ClickHouse/issues/22161). [#22443](https://github.com/ClickHouse/ClickHouse/pull/22443) ([Kruglov Pavel](https://github.com/Avogar)).
 * Buffer overflow (on read) was possible in `tokenbf_v1` full text index. The excessive bytes are not used but the read operation may lead to crash in rare cases. This closes [#19233](https://github.com/ClickHouse/ClickHouse/issues/19233). [#22421](https://github.com/ClickHouse/ClickHouse/pull/22421) ([alexey-milovidov](https://github.com/alexey-milovidov)).
 * Do not limit HTTP chunk size. Fixes [#21907](https://github.com/ClickHouse/ClickHouse/issues/21907). [#22322](https://github.com/ClickHouse/ClickHouse/pull/22322) ([Ivan](https://github.com/abyss7)).
 * Fixed a bug, which leads to underaggregation of data in case of enabled `optimize_aggregation_in_order` and many parts in table. Slightly improve performance of aggregation with enabled `optimize_aggregation_in_order`. [#21889](https://github.com/ClickHouse/ClickHouse/pull/21889) ([Anton Popov](https://github.com/CurtizJ)).

From b9e61c6e696b062ed6832025220c29b67b518449 Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Fri, 21 May 2021 03:57:27 +0300
Subject: [PATCH 50/76] Update CHANGELOG.md

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 298cbd2ccf4..ed6a7182dad 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -22,7 +22,7 @@
 * Exclude values that does not belong to the shard from right part of IN section for distributed queries (under `optimize_skip_unused_shards_rewrite_in`, enabled by default, since it still requires `optimize_skip_unused_shards`). [#21511](https://github.com/ClickHouse/ClickHouse/pull/21511) ([Azat Khuzhin](https://github.com/azat)).
 * Improved performance of reading a subset of columns with File-like table engine and column-oriented format like Parquet, Arrow or ORC. This closes [#issue:20129](https://github.com/ClickHouse/ClickHouse/issues/20129). [#21302](https://github.com/ClickHouse/ClickHouse/pull/21302) ([keenwolf](https://github.com/keen-wolf)).
 * Allow to move more conditions to `PREWHERE` as it was before version 21.1 (adjustment of internal heuristics). Insufficient number of moved condtions could lead to worse performance. [#23397](https://github.com/ClickHouse/ClickHouse/pull/23397) ([Anton Popov](https://github.com/CurtizJ)).
-* Added nanodbc instead of Poco::ODBC. Closes [#9678](https://github.com/ClickHouse/ClickHouse/issues/9678). Add support for DateTime64 and Decimal* for ODBC table engine. Closes [#21961](https://github.com/ClickHouse/ClickHouse/issues/21961). Fixed issue with cyrillic text being truncated. Closes [#16246](https://github.com/ClickHouse/ClickHouse/issues/16246). Added connection pools for odbc bridge. [#21972](https://github.com/ClickHouse/ClickHouse/pull/21972) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Improved performance of ODBC connections and fixed all the outstanding issues from the backlog. Using `nanodbc` library instead of `Poco::ODBC`. Closes [#9678](https://github.com/ClickHouse/ClickHouse/issues/9678). Add support for DateTime64 and Decimal* for ODBC table engine. Closes [#21961](https://github.com/ClickHouse/ClickHouse/issues/21961). Fixed issue with cyrillic text being truncated. Closes [#16246](https://github.com/ClickHouse/ClickHouse/issues/16246). Added connection pools for odbc bridge. [#21972](https://github.com/ClickHouse/ClickHouse/pull/21972) ([Kseniia Sumarokova](https://github.com/kssenii)).
 
 #### Improvement
 

From b1d13198945e5ce0d45af3c168e6cf75470d2254 Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Fri, 21 May 2021 03:57:52 +0300
Subject: [PATCH 51/76] Update CHANGELOG.md

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ed6a7182dad..636fdb5cbb3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -58,7 +58,7 @@
 * Forbid to drop a column if it's referenced by materialized view. Closes [#21164](https://github.com/ClickHouse/ClickHouse/issues/21164). [#21303](https://github.com/ClickHouse/ClickHouse/pull/21303) ([flynn](https://github.com/ucasFL)).
 * Support dynamic interserver credentials (rotating credentials without downtime). [#14113](https://github.com/ClickHouse/ClickHouse/pull/14113) ([johnskopis](https://github.com/johnskopis)).
 * Add support for Kafka storage with `Arrow` and `ArrowStream` format messages. [#23415](https://github.com/ClickHouse/ClickHouse/pull/23415) ([Chao Ma](https://github.com/godliness)).
-* * Fixed missing semicolon in exception message. The user may find this exception message unpleasant to read. [#23208](https://github.com/ClickHouse/ClickHouse/pull/23208) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Fixed missing semicolon in exception message. The user may find this exception message unpleasant to read. [#23208](https://github.com/ClickHouse/ClickHouse/pull/23208) ([alexey-milovidov](https://github.com/alexey-milovidov)).
 * Fixed missing whitespace in some exception messages about `LowCardinality` type. [#23207](https://github.com/ClickHouse/ClickHouse/pull/23207) ([alexey-milovidov](https://github.com/alexey-milovidov)).
 * Some values were formatted with alignment in center in table cells in `Markdown` format. Not anymore. [#23096](https://github.com/ClickHouse/ClickHouse/pull/23096) ([alexey-milovidov](https://github.com/alexey-milovidov)).
 * Remove non-essential details from suggestions in clickhouse-client. This closes [#22158](https://github.com/ClickHouse/ClickHouse/issues/22158). [#23040](https://github.com/ClickHouse/ClickHouse/pull/23040) ([alexey-milovidov](https://github.com/alexey-milovidov)).

From ea30bb7abfb53b9b7d0fa8861c4ac470b0ae42a2 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 21 May 2021 04:17:18 +0300
Subject: [PATCH 52/76] Improve implementation

---
 src/Core/Field.h                                          | 3 +++
 src/Core/MySQL/MySQLReplication.h                         | 2 +-
 src/DataStreams/PostgreSQLBlockInputStream.cpp            | 2 +-
 src/DataStreams/TTLAggregationAlgorithm.h                 | 2 +-
 src/Databases/MySQL/MaterializeMySQLSyncThread.cpp        | 6 +++---
 src/Dictionaries/CacheDictionaryStorage.h                 | 8 ++++----
 src/Functions/partitionId.cpp                             | 2 +-
 src/Interpreters/ActionsVisitor.cpp                       | 4 ++--
 src/Interpreters/FillingRow.h                             | 2 +-
 src/Processors/Formats/Impl/AvroRowInputFormat.cpp        | 2 +-
 src/Processors/Merges/Algorithms/SummingSortedAlgorithm.h | 2 +-
 src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp   | 6 +++---
 src/Storages/MergeTree/MergeTreeData.cpp                  | 2 +-
 src/Storages/MergeTree/MergeTreeDataWriter.cpp            | 4 ++--
 src/Storages/MergeTree/MergeTreeDataWriter.h              | 4 ++--
 src/Storages/MergeTree/MergeTreePartition.h               | 4 ++--
 16 files changed, 29 insertions(+), 26 deletions(-)

diff --git a/src/Core/Field.h b/src/Core/Field.h
index fb9f811ddf3..9c970fbbb31 100644
--- a/src/Core/Field.h
+++ b/src/Core/Field.h
@@ -721,6 +721,9 @@ private:
 #undef DBMS_MIN_FIELD_SIZE
 
 
+using Row = std::vector<Field>;
+
+
 template <> struct Field::TypeToEnum<Null>    { static const Types::Which value = Types::Null; };
 template <> struct Field::TypeToEnum<UInt64>  { static const Types::Which value = Types::UInt64; };
 template <> struct Field::TypeToEnum<UInt128> { static const Types::Which value = Types::UInt128; };
diff --git a/src/Core/MySQL/MySQLReplication.h b/src/Core/MySQL/MySQLReplication.h
index d415bdda70d..cd24979caaa 100644
--- a/src/Core/MySQL/MySQLReplication.h
+++ b/src/Core/MySQL/MySQLReplication.h
@@ -452,7 +452,7 @@ namespace MySQLReplication
         UInt32 number_columns;
         String schema;
         String table;
-        std::vector<Field> rows;
+        Row rows;
 
         RowsEvent(std::shared_ptr<TableMapEvent> table_map_, EventHeader && header_, const RowsEventHeader & rows_header)
             : EventBase(std::move(header_)), number_columns(0), table_map(table_map_)
diff --git a/src/DataStreams/PostgreSQLBlockInputStream.cpp b/src/DataStreams/PostgreSQLBlockInputStream.cpp
index 63a8e9edb79..9ef0a45acc0 100644
--- a/src/DataStreams/PostgreSQLBlockInputStream.cpp
+++ b/src/DataStreams/PostgreSQLBlockInputStream.cpp
@@ -193,7 +193,7 @@ void PostgreSQLBlockInputStream::insertValue(IColumn & column, std::string_view
 
             size_t dimension = 0, max_dimension = 0, expected_dimensions = array_info[idx].num_dimensions;
             const auto parse_value = array_info[idx].pqxx_parser;
-            std::vector<std::vector<Field>> dimensions(expected_dimensions + 1);
+            std::vector<Row> dimensions(expected_dimensions + 1);
 
             while (parsed.first != pqxx::array_parser::juncture::done)
             {
diff --git a/src/DataStreams/TTLAggregationAlgorithm.h b/src/DataStreams/TTLAggregationAlgorithm.h
index 7231506b7da..c2f40bab6b9 100644
--- a/src/DataStreams/TTLAggregationAlgorithm.h
+++ b/src/DataStreams/TTLAggregationAlgorithm.h
@@ -32,7 +32,7 @@ private:
 
     const Block header;
     std::unique_ptr<Aggregator> aggregator;
-    std::vector<Field> current_key_value;
+    Row current_key_value;
     AggregatedDataVariants aggregation_result;
     ColumnRawPtrs key_columns;
     Aggregator::AggregateColumns columns_for_aggregator;
diff --git a/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp b/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp
index fff4739eff9..a7ebcb0d3f2 100644
--- a/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp
+++ b/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp
@@ -477,7 +477,7 @@ static inline void fillSignAndVersionColumnsData(Block & data, Int8 sign_value,
 
 template <bool assert_nullable = false>
 static void writeFieldsToColumn(
-    IColumn & column_to, const std::vector<Field> & rows_data, size_t column_index, const std::vector<bool> & mask, ColumnUInt8 * null_map_column = nullptr)
+    IColumn & column_to, const Row & rows_data, size_t column_index, const std::vector<bool> & mask, ColumnUInt8 * null_map_column = nullptr)
 {
     if (ColumnNullable * column_nullable = typeid_cast<ColumnNullable *>(&column_to))
         writeFieldsToColumn<true>(column_nullable->getNestedColumn(), rows_data, column_index, mask, &column_nullable->getNullMapColumn());
@@ -599,7 +599,7 @@ static void writeFieldsToColumn(
 }
 
 template <Int8 sign>
-static size_t onWriteOrDeleteData(const std::vector<Field> & rows_data, Block & buffer, size_t version)
+static size_t onWriteOrDeleteData(const Row & rows_data, Block & buffer, size_t version)
 {
     size_t prev_bytes = buffer.bytes();
     for (size_t column = 0; column < buffer.columns() - 2; ++column)
@@ -623,7 +623,7 @@ static inline bool differenceSortingKeys(const Tuple & row_old_data, const Tuple
     return false;
 }
 
-static inline size_t onUpdateData(const std::vector<Field> & rows_data, Block & buffer, size_t version, const std::vector<size_t> & sorting_columns_index)
+static inline size_t onUpdateData(const Row & rows_data, Block & buffer, size_t version, const std::vector<size_t> & sorting_columns_index)
 {
     if (rows_data.size() % 2 != 0)
         throw Exception("LOGICAL ERROR: It is a bug.", ErrorCodes::LOGICAL_ERROR);
diff --git a/src/Dictionaries/CacheDictionaryStorage.h b/src/Dictionaries/CacheDictionaryStorage.h
index a79d83eea05..07217369dc1 100644
--- a/src/Dictionaries/CacheDictionaryStorage.h
+++ b/src/Dictionaries/CacheDictionaryStorage.h
@@ -232,7 +232,7 @@ private:
 
             if (unlikely(attribute.is_complex_type))
             {
-                auto & container = std::get<std::vector<Field>>(attribute.attribute_container);
+                auto & container = std::get<Row>(attribute.attribute_container);
 
                 for (size_t fetched_key_index = 0; fetched_key_index < fetched_columns_index; ++fetched_key_index)
                 {
@@ -506,7 +506,7 @@ private:
 
         if (unlikely(attribute.is_complex_type))
         {
-            auto & container = std::get<std::vector<Field>>(attribute.attribute_container);
+            auto & container = std::get<Row>(attribute.attribute_container);
             std::forward<GetContainerFunc>(func)(container);
         }
         else
@@ -564,7 +564,7 @@ private:
                 last_attribute.is_complex_type = dictionary_attribute.is_nullable || dictionary_attribute.is_array;
 
                 if (dictionary_attribute.is_nullable)
-                    last_attribute.attribute_container = std::vector<Field>();
+                    last_attribute.attribute_container = Row();
                 else
                     last_attribute.attribute_container = PaddedPODArray<ValueType>();
             };
@@ -609,7 +609,7 @@ private:
             PaddedPODArray<Float64>,
             PaddedPODArray<UUID>,
             PaddedPODArray<StringRef>,
-            std::vector<Field>> attribute_container;
+            Row> attribute_container;
     };
 
     CacheDictionaryStorageConfiguration configuration;
diff --git a/src/Functions/partitionId.cpp b/src/Functions/partitionId.cpp
index 70ece96f983..cf679452da1 100644
--- a/src/Functions/partitionId.cpp
+++ b/src/Functions/partitionId.cpp
@@ -52,7 +52,7 @@ public:
         auto result_column = ColumnString::create();
         for (size_t j = 0; j < input_rows_count; ++j)
         {
-            std::vector<Field> row(size);
+            Row row(size);
             for (size_t i = 0; i < size; ++i)
                 arguments[i].column->get(j, row[i]);
             MergeTreePartition partition(std::move(row));
diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp
index 2c66d2a9841..a65e9337ecd 100644
--- a/src/Interpreters/ActionsVisitor.cpp
+++ b/src/Interpreters/ActionsVisitor.cpp
@@ -79,7 +79,7 @@ static Block createBlockFromCollection(const Collection & collection, const Data
     for (size_t i = 0; i < columns_num; ++i)
         columns[i] = types[i]->createColumn();
 
-    std::vector<Field> tuple_values;
+    Row tuple_values;
     for (const auto & value : collection)
     {
         if (columns_num == 1)
@@ -153,7 +153,7 @@ static Block createBlockFromAST(const ASTPtr & node, const DataTypes & types, Co
     MutableColumns columns = header.cloneEmptyColumns();
 
     DataTypePtr tuple_type;
-    std::vector<Field> tuple_values;
+    Row tuple_values;
     const auto & list = node->as<ASTExpressionList &>();
     bool transform_null_in = context->getSettingsRef().transform_null_in;
     for (const auto & elem : list.children)
diff --git a/src/Interpreters/FillingRow.h b/src/Interpreters/FillingRow.h
index 0e1d60d0d7a..434a9270718 100644
--- a/src/Interpreters/FillingRow.h
+++ b/src/Interpreters/FillingRow.h
@@ -34,7 +34,7 @@ public:
     FillColumnDescription & getFillDescription(size_t ind) { return description[ind].fill_description; }
 
 private:
-    std::vector<Field> row;
+    Row row;
     SortDescription description;
 };
 
diff --git a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp
index 95ee42b4d09..cd576916364 100644
--- a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp
+++ b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp
@@ -344,7 +344,7 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node
             if (target.isEnum())
             {
                 const auto & enum_type = dynamic_cast<const IDataTypeEnum &>(*target_type);
-                std::vector<Field> symbol_mapping;
+                Row symbol_mapping;
                 for (size_t i = 0; i < root_node->names(); i++)
                 {
                     symbol_mapping.push_back(enum_type.castToValue(root_node->nameAt(i)));
diff --git a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.h b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.h
index 3e03f5e9ae3..c77bf7c0ba5 100644
--- a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.h
+++ b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.h
@@ -83,7 +83,7 @@ public:
 
         bool is_group_started = false;
 
-        std::vector<Field> current_row;
+        Row current_row;
         bool current_row_is_zero = true;    /// Are all summed columns zero (or empty)? It is updated incrementally.
 
         void addRowImpl(ColumnRawPtrs & raw_columns, size_t row);
diff --git a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp
index 1280138d38b..d9cb949042c 100644
--- a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp
+++ b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp
@@ -214,7 +214,7 @@ namespace
 
         virtual void insertPartitionValueColumn(
             size_t rows,
-            const std::vector<Field> & partition_value,
+            const Row & partition_value,
             const DataTypePtr & partition_value_type,
             const String & name) = 0;
     };
@@ -322,7 +322,7 @@ namespace
         }
 
         void insertPartitionValueColumn(
-            size_t rows, const std::vector<Field> & partition_value, const DataTypePtr & partition_value_type, const String & name) final
+            size_t rows, const Row & partition_value, const DataTypePtr & partition_value_type, const String & name) final
         {
             ColumnPtr column;
             if (rows)
@@ -362,7 +362,7 @@ namespace
         }
 
         void insertPartitionValueColumn(
-            size_t rows, const std::vector<Field> & partition_value, const DataTypePtr & partition_value_type, const String &) final
+            size_t rows, const Row & partition_value, const DataTypePtr & partition_value_type, const String &) final
         {
             ColumnPtr column;
             if (rows)
diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index e884ff7f211..51077996212 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -3144,7 +3144,7 @@ String MergeTreeData::getPartitionIDFromQuery(const ASTPtr & ast, ContextPtr loc
             ErrorCodes::INVALID_PARTITION_VALUE);
 
     const FormatSettings format_settings;
-    std::vector<Field> partition_row(fields_count);
+    Row partition_row(fields_count);
 
     if (fields_count)
     {
diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp
index 33d0291946c..b875b3b2785 100644
--- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp
+++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp
@@ -150,7 +150,7 @@ BlocksWithPartition MergeTreeDataWriter::splitBlockIntoParts(const Block & block
 
     if (!metadata_snapshot->hasPartitionKey()) /// Table is not partitioned.
     {
-        result.emplace_back(Block(block), std::vector<Field>{});
+        result.emplace_back(Block(block), Row{});
         return result;
     }
 
@@ -172,7 +172,7 @@ BlocksWithPartition MergeTreeDataWriter::splitBlockIntoParts(const Block & block
 
     auto get_partition = [&](size_t num)
     {
-        std::vector<Field> partition(partition_columns.size());
+        Row partition(partition_columns.size());
         for (size_t i = 0; i < partition_columns.size(); ++i)
             partition[i] = Field((*partition_columns[i])[partition_num_to_first_row[num]]);
         return partition;
diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.h b/src/Storages/MergeTree/MergeTreeDataWriter.h
index f5eb0cd98e9..8b7bb9d9165 100644
--- a/src/Storages/MergeTree/MergeTreeDataWriter.h
+++ b/src/Storages/MergeTree/MergeTreeDataWriter.h
@@ -18,9 +18,9 @@ namespace DB
 struct BlockWithPartition
 {
     Block block;
-    std::vector<Field> partition;
+    Row partition;
 
-    BlockWithPartition(Block && block_, std::vector<Field> && partition_)
+    BlockWithPartition(Block && block_, Row && partition_)
         : block(block_), partition(std::move(partition_))
     {
     }
diff --git a/src/Storages/MergeTree/MergeTreePartition.h b/src/Storages/MergeTree/MergeTreePartition.h
index c55eb8ce142..f8ce4b60a34 100644
--- a/src/Storages/MergeTree/MergeTreePartition.h
+++ b/src/Storages/MergeTree/MergeTreePartition.h
@@ -20,12 +20,12 @@ using StorageMetadataPtr = std::shared_ptr<const StorageInMemoryMetadata>;
 /// This class represents a partition value of a single part and encapsulates its loading/storing logic.
 struct MergeTreePartition
 {
-    std::vector<Field> value;
+    Row value;
 
 public:
     MergeTreePartition() = default;
 
-    explicit MergeTreePartition(std::vector<Field> value_) : value(std::move(value_)) {}
+    explicit MergeTreePartition(Row value_) : value(std::move(value_)) {}
 
     /// For month-based partitioning.
     explicit MergeTreePartition(UInt32 yyyymm) : value(1, yyyymm) {}

From 56871aafec906e913c228af9a7e951cab553f5c4 Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Fri, 21 May 2021 07:47:18 +0300
Subject: [PATCH 53/76] Update CHANGELOG.md

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 636fdb5cbb3..2eaecaa4c9b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -127,7 +127,7 @@
 * Fixed some tests on AArch64 platform. [#22596](https://github.com/ClickHouse/ClickHouse/pull/22596) ([alexey-milovidov](https://github.com/alexey-milovidov)).
 * Added function alignment for possibly better performance. [#21431](https://github.com/ClickHouse/ClickHouse/pull/21431) ([Danila Kutenin](https://github.com/danlark1)).
 * Adjust some tests to output identical results on amd64 and aarch64 (qemu). The result was depending on implementation specific CPU behaviour. [#22590](https://github.com/ClickHouse/ClickHouse/pull/22590) ([alexey-milovidov](https://github.com/alexey-milovidov)).
-* Allow query profiling only on x86_64. See [#15174](https://github.com/ClickHouse/ClickHouse/issues/15174)#issuecomment-812954965 and [#15638](https://github.com/ClickHouse/ClickHouse/issues/15638)#issuecomment-703805337. This closes [#15638](https://github.com/ClickHouse/ClickHouse/issues/15638). [#22580](https://github.com/ClickHouse/ClickHouse/pull/22580) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Allow query profiling only on x86_64. See [#15174](https://github.com/ClickHouse/ClickHouse/issues/15174#issuecomment-812954965) and [#15638](https://github.com/ClickHouse/ClickHouse/issues/15638#issuecomment-703805337). This closes [#15638](https://github.com/ClickHouse/ClickHouse/issues/15638). [#22580](https://github.com/ClickHouse/ClickHouse/pull/22580) ([alexey-milovidov](https://github.com/alexey-milovidov)).
 * Allow building with unbundled xz (lzma) using `USE_INTERNAL_XZ_LIBRARY=OFF` CMake option. [#22571](https://github.com/ClickHouse/ClickHouse/pull/22571) ([Kfir Itzhak](https://github.com/mastertheknife)).
 * Enable bundled `openldap` on `ppc64le` [#22487](https://github.com/ClickHouse/ClickHouse/pull/22487) ([Kfir Itzhak](https://github.com/mastertheknife)).
 * Disable incompatible libraries (platform specific typically) on `ppc64le` [#22475](https://github.com/ClickHouse/ClickHouse/pull/22475) ([Kfir Itzhak](https://github.com/mastertheknife)).

From a66c16ed03c4e62f268e4920a17e148cc39ac19b Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 21 May 2021 07:53:00 +0300
Subject: [PATCH 54/76] Fix diagnostics in documentation script

---
 docs/tools/nav.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/docs/tools/nav.py b/docs/tools/nav.py
index 291797a1633..db64d1ba404 100644
--- a/docs/tools/nav.py
+++ b/docs/tools/nav.py
@@ -31,7 +31,16 @@ def build_nav_entry(root, args):
                 result_items.append((prio, title, payload))
         elif filename.endswith('.md'):
             path = os.path.join(root, filename)
-            meta, content = util.read_md_file(path)
+
+            meta = ''
+            content = ''
+
+            try:
+                meta, content = util.read_md_file(path)
+            except:
+                print('Error in file: {}'.format(path))
+                raise
+
             path = path.split('/', 2)[-1]
             title = meta.get('toc_title', find_first_header(content))
             if title:

From b553c5a117f5aa8b818e6ed20bf1e0514820eb33 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Thu, 20 May 2021 06:24:42 +0300
Subject: [PATCH 55/76] Recognize more addresses as local

---
 src/Common/isLocalAddress.cpp | 36 +++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/src/Common/isLocalAddress.cpp b/src/Common/isLocalAddress.cpp
index 4167a24ab21..2a76241300c 100644
--- a/src/Common/isLocalAddress.cpp
+++ b/src/Common/isLocalAddress.cpp
@@ -4,6 +4,7 @@
 #include <cstring>
 #include <optional>
 #include <common/types.h>
+#include <common/unaligned.h>
 #include <Common/Exception.h>
 #include <Poco/Net/IPAddress.h>
 #include <Poco/Net/SocketAddress.h>
@@ -81,10 +82,45 @@ struct NetworkInterfaces
 
 bool isLocalAddress(const Poco::Net::IPAddress & address)
 {
+    /** 127.0.0.1 is treat as local address unconditionally.
+      * ::1 is also treat as local address unconditionally.
+      *
+      * 127.0.0.{2..255} are not treat as local addresses, because they are used in tests
+      *  to emulate distributed queries across localhost.
+      *
+      * But 127.{0,1}.{0,1}.{0,1} are treat as local addresses,
+      *  because they are used in Debian for localhost.
+      */
+    if (address.isLoopback())
+    {
+        if (address.family() == Poco::Net::AddressFamily::IPv4)
+        {
+            union
+            {
+                UInt32 word;
+                unsigned char digits[4];
+            } digits_union;
+
+            digits_union.word = ntohl(unalignedLoad<UInt32>(address.addr()));
+            if (digits_union.digits[0] == 127
+                && digits_union.digits[1] <= 1
+                && digits_union.digits[2] <= 1
+                && digits_union.digits[3] <= 1)
+            {
+                return true;
+            }
+        }
+        else if (address.family() == Poco::Net::AddressFamily::IPv6)
+        {
+            return true;
+        }
+    }
+
     NetworkInterfaces interfaces;
     return interfaces.hasAddress(address);
 }
 
+
 bool isLocalAddress(const Poco::Net::SocketAddress & address, UInt16 clickhouse_port)
 {
     return clickhouse_port == address.port() && isLocalAddress(address.host());

From 87a713365925ff62b996d78b08e99cdd36695206 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 21 May 2021 08:17:54 +0300
Subject: [PATCH 56/76] Add a test and fix the code

---
 src/Common/isLocalAddress.cpp            | 17 +++++++----------
 src/Common/tests/gtest_local_address.cpp | 23 ++++++++++++++++++++++-
 2 files changed, 29 insertions(+), 11 deletions(-)

diff --git a/src/Common/isLocalAddress.cpp b/src/Common/isLocalAddress.cpp
index 2a76241300c..fba92f5d763 100644
--- a/src/Common/isLocalAddress.cpp
+++ b/src/Common/isLocalAddress.cpp
@@ -1,6 +1,7 @@
 #include <Common/isLocalAddress.h>
 
 #include <ifaddrs.h>
+#include <array>
 #include <cstring>
 #include <optional>
 #include <common/types.h>
@@ -95,17 +96,13 @@ bool isLocalAddress(const Poco::Net::IPAddress & address)
     {
         if (address.family() == Poco::Net::AddressFamily::IPv4)
         {
-            union
-            {
-                UInt32 word;
-                unsigned char digits[4];
-            } digits_union;
+            using Digits = std::array<UInt8, 4>;
+            Digits digits = unalignedLoad<Digits>(address.addr());  /// The address is located in memory in big endian form.
 
-            digits_union.word = ntohl(unalignedLoad<UInt32>(address.addr()));
-            if (digits_union.digits[0] == 127
-                && digits_union.digits[1] <= 1
-                && digits_union.digits[2] <= 1
-                && digits_union.digits[3] <= 1)
+            if (digits[0] == 127
+                && digits[1] <= 1
+                && digits[2] <= 1
+                && digits[3] <= 1)
             {
                 return true;
             }
diff --git a/src/Common/tests/gtest_local_address.cpp b/src/Common/tests/gtest_local_address.cpp
index 504fba19713..47824a7fa27 100644
--- a/src/Common/tests/gtest_local_address.cpp
+++ b/src/Common/tests/gtest_local_address.cpp
@@ -11,9 +11,30 @@ TEST(LocalAddress, SmokeTest)
     std::string address_str;
     DB::readString(address_str, cmd->out);
     cmd->wait();
-    std::cerr << "Got Address:" << address_str << std::endl;
+    std::cerr << "Got Address: " << address_str << std::endl;
 
     Poco::Net::IPAddress address(address_str);
 
     EXPECT_TRUE(DB::isLocalAddress(address));
 }
+
+TEST(LocalAddress, Localhost)
+{
+    EXPECT_TRUE(DB::isLocalAddress(Poco::Net::IPAddress{"127.0.0.1"}));
+    EXPECT_TRUE(DB::isLocalAddress(Poco::Net::IPAddress{"127.0.1.1"}));
+    EXPECT_TRUE(DB::isLocalAddress(Poco::Net::IPAddress{"127.1.1.1"}));
+    EXPECT_TRUE(DB::isLocalAddress(Poco::Net::IPAddress{"127.1.0.1"}));
+    EXPECT_TRUE(DB::isLocalAddress(Poco::Net::IPAddress{"127.1.0.0"}));
+    EXPECT_TRUE(DB::isLocalAddress(Poco::Net::IPAddress{"::1"}));
+
+    /// Make sure we don't mess with the byte order.
+    EXPECT_FALSE(DB::isLocalAddress(Poco::Net::IPAddress{"1.0.0.127"}));
+    EXPECT_FALSE(DB::isLocalAddress(Poco::Net::IPAddress{"1.1.1.127"}));
+
+    EXPECT_FALSE(DB::isLocalAddress(Poco::Net::IPAddress{"0.0.0.0"}));
+    EXPECT_FALSE(DB::isLocalAddress(Poco::Net::IPAddress{"::"}));
+    EXPECT_FALSE(DB::isLocalAddress(Poco::Net::IPAddress{"::2"}));
+
+    /// The the comment in the implementation of isLocalAddress.
+    EXPECT_FALSE(DB::isLocalAddress(Poco::Net::IPAddress{"127.0.0.2"}));
+}

From ccc19f1201bffcea92374ebed219d05360c07288 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 21 May 2021 08:21:19 +0300
Subject: [PATCH 57/76] Simplify code

---
 src/Common/isLocalAddress.cpp | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/Common/isLocalAddress.cpp b/src/Common/isLocalAddress.cpp
index fba92f5d763..b8483230272 100644
--- a/src/Common/isLocalAddress.cpp
+++ b/src/Common/isLocalAddress.cpp
@@ -5,7 +5,6 @@
 #include <cstring>
 #include <optional>
 #include <common/types.h>
-#include <common/unaligned.h>
 #include <Common/Exception.h>
 #include <Poco/Net/IPAddress.h>
 #include <Poco/Net/SocketAddress.h>
@@ -96,8 +95,8 @@ bool isLocalAddress(const Poco::Net::IPAddress & address)
     {
         if (address.family() == Poco::Net::AddressFamily::IPv4)
         {
-            using Digits = std::array<UInt8, 4>;
-            Digits digits = unalignedLoad<Digits>(address.addr());  /// The address is located in memory in big endian form.
+            /// The address is located in memory in big endian form (network byte order).
+            const unsigned char * digits = static_cast<const unsigned char *>(address.addr());
 
             if (digits[0] == 127
                 && digits[1] <= 1

From ee247aaa5ba9211ee4d12bcc299309d3738b31c9 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 21 May 2021 08:21:37 +0300
Subject: [PATCH 58/76] Simplify code

---
 src/Common/isLocalAddress.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/Common/isLocalAddress.cpp b/src/Common/isLocalAddress.cpp
index b8483230272..9e22c58f94c 100644
--- a/src/Common/isLocalAddress.cpp
+++ b/src/Common/isLocalAddress.cpp
@@ -1,7 +1,6 @@
 #include <Common/isLocalAddress.h>
 
 #include <ifaddrs.h>
-#include <array>
 #include <cstring>
 #include <optional>
 #include <common/types.h>

From 7ae047a7217f8cc1126945c012be001c56f324ba Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Fri, 21 May 2021 08:22:08 +0300
Subject: [PATCH 59/76] Update gtest_local_address.cpp

---
 src/Common/tests/gtest_local_address.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Common/tests/gtest_local_address.cpp b/src/Common/tests/gtest_local_address.cpp
index 47824a7fa27..7f1d2dfc1a7 100644
--- a/src/Common/tests/gtest_local_address.cpp
+++ b/src/Common/tests/gtest_local_address.cpp
@@ -35,6 +35,6 @@ TEST(LocalAddress, Localhost)
     EXPECT_FALSE(DB::isLocalAddress(Poco::Net::IPAddress{"::"}));
     EXPECT_FALSE(DB::isLocalAddress(Poco::Net::IPAddress{"::2"}));
 
-    /// The the comment in the implementation of isLocalAddress.
+    /// See the comment in the implementation of isLocalAddress.
     EXPECT_FALSE(DB::isLocalAddress(Poco::Net::IPAddress{"127.0.0.2"}));
 }

From 28c0e85dab5df5ea705d91267be4492af2d44f5b Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 21 May 2021 09:29:56 +0300
Subject: [PATCH 60/76] Add performance test

---
 tests/performance/quantile.xml | 29 +++++++++++++++++++++++++++++
 tests/performance/uniq.xml     |  5 +----
 2 files changed, 30 insertions(+), 4 deletions(-)
 create mode 100644 tests/performance/quantile.xml

diff --git a/tests/performance/quantile.xml b/tests/performance/quantile.xml
new file mode 100644
index 00000000000..4718c129db6
--- /dev/null
+++ b/tests/performance/quantile.xml
@@ -0,0 +1,29 @@
+<test>
+    <preconditions>
+        <table_exists>hits_100m_single</table_exists>
+    </preconditions>
+
+    <substitutions>
+        <substitution>
+           <name>key</name>
+           <values>
+               <value>SearchEngineID</value>
+               <value>RegionID</value>
+               <value>SearchPhrase</value>
+               <value>ClientIP</value>
+           </values>
+        </substitution>
+        <substitution>
+           <name>func</name>
+           <values>
+               <value>quantile</value>
+               <value>quantileExact</value>
+               <value>quantileTDigest</value>
+               <value>quantileTiming</value>
+               <value>quantileBFloat16</value>
+           </values>
+       </substitution>
+    </substitutions>
+
+    <query>SELECT {key} AS k, {func}(ResolutionWidth) FROM hits_100m_single GROUP BY k FORMAT Null</query>
+</test>
diff --git a/tests/performance/uniq.xml b/tests/performance/uniq.xml
index 0ddd2321ca6..d4521961b9a 100644
--- a/tests/performance/uniq.xml
+++ b/tests/performance/uniq.xml
@@ -1,12 +1,9 @@
 <test>
-
     <preconditions>
         <table_exists>hits_100m_single</table_exists>
         <ram_size>30000000000</ram_size>
     </preconditions>
 
-
-
     <settings>
         <max_memory_usage>30000000000</max_memory_usage>
         <!--
@@ -36,7 +33,7 @@
                <value>SearchPhrase</value>
                <value>ClientIP</value>
            </values>
-       </substitution>
+        </substitution>
         <substitution>
            <name>func</name>
            <values>

From 792dbbe6aea881ad67d92fb62a82b272e9b2a98a Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 21 May 2021 09:30:13 +0300
Subject: [PATCH 61/76] Refinements

---
 .../QuantileBFloat16Histogram.h               | 79 ++++++++++++++++---
 1 file changed, 67 insertions(+), 12 deletions(-)

diff --git a/src/AggregateFunctions/QuantileBFloat16Histogram.h b/src/AggregateFunctions/QuantileBFloat16Histogram.h
index 8072350caac..1c648ebc326 100644
--- a/src/AggregateFunctions/QuantileBFloat16Histogram.h
+++ b/src/AggregateFunctions/QuantileBFloat16Histogram.h
@@ -6,22 +6,57 @@
 #include <common/types.h>
 #include <ext/bit_cast.h>
 
+
 namespace DB
 {
+
+/** `bfloat16` is a 16-bit floating point data type that is the same as the corresponding most significant 16 bits of the `float`.
+  * https://en.wikipedia.org/wiki/Bfloat16_floating-point_format
+  *
+  * To calculate quantile, simply convert input value to 16 bit (convert to float, then take the most significant 16 bits),
+  * and calculate the histogram of these values.
+  *
+  * Hash table is the preferred way to store histogram, because the number of distinct values is small:
+  * ```
+  * SELECT uniq(bfloat)
+  * FROM
+  * (
+  *     SELECT
+  *         number,
+  *         toFloat32(number) AS f,
+  *         bitShiftRight(bitAnd(reinterpretAsUInt32(reinterpretAsFixedString(f)), 4294901760) AS cut, 16),
+  *         reinterpretAsFloat32(reinterpretAsFixedString(cut)) AS bfloat
+  *     FROM numbers(100000000)
+  * )
+  *
+  * ┌─uniq(bfloat)─┐
+  * │         2623 │
+  * └──────────────┘
+  * ```
+  * (when increasing the range of values 1000 times, the number of distinct bfloat16 values increases just by 1280).
+  *
+  * Then calculate quantile from the histogram.
+  *
+  * This sketch is very simple and rough. Its relative precision is constant 1 / 256 = 0.390625%.
+  */
 template <typename Value>
 struct QuantileBFloat16Histogram
 {
-    using bfloat16 = UInt16;
+    using BFloat16 = UInt16;
     using Weight = UInt64;
-    using Data = HashMap<bfloat16, Weight>;
+    using Data = HashMapWithStackMemory<BFloat16, Weight, TrivialHash, 4>;
+
     Data data;
 
-    void add(const Value & x) { add(x, 1); }
+    void add(const Value & x)
+    {
+        add(x, 1);
+    }
 
     void add(const Value & x, Weight w)
     {
         if (!isNaN(x))
-            data[to_bfloat16(x)] += w;
+            data[toBFloat16(x)] += w;
     }
 
     void merge(const QuantileBFloat16Histogram & rhs)
@@ -30,18 +65,30 @@ struct QuantileBFloat16Histogram
             data[pair.getKey()] += pair.getMapped();
     }
 
-    void serialize(WriteBuffer & buf) const { data.write(buf); }
+    void serialize(WriteBuffer & buf) const
+    {
+        data.write(buf);
+    }
 
-    void deserialize(ReadBuffer & buf) { data.read(buf); }
+    void deserialize(ReadBuffer & buf)
+    {
+        data.read(buf);
+    }
 
-    Value get(Float64 level) const { return getImpl<Value>(level); }
+    Value get(Float64 level) const
+    {
+        return getImpl<Value>(level);
+    }
 
     void getMany(const Float64 * levels, const size_t * indices, size_t size, Value * result) const
     {
         getManyImpl(levels, indices, size, result);
     }
 
-    Float64 getFloat(Float64 level) const { return getImpl<Float64>(level); }
+    Float64 getFloat(Float64 level) const
+    {
+        return getImpl<Float64>(level);
+    }
 
     void getManyFloat(const Float64 * levels, const size_t * indices, size_t size, Float64 * result) const
     {
@@ -49,9 +96,17 @@ struct QuantileBFloat16Histogram
     }
 
 private:
-    bfloat16 to_bfloat16(const Value & x) const { return ext::bit_cast<UInt32>(static_cast<Float32>(x)) >> 16; }
+    /// Take the most significant 16 bits of the floating point number.
+    BFloat16 toBFloat16(const Value & x) const
+    {
+        return ext::bit_cast<UInt32>(static_cast<Float32>(x)) >> 16;
+    }
 
-    Float32 to_Float32(const bfloat16 & x) const { return ext::bit_cast<Float32>(x << 16); }
+    /// Put the bits into most significant 16 bits of the floating point number and fill other bits with zeros.
+    Float32 toFloat32(const BFloat16 & x) const
+    {
+        return ext::bit_cast<Float32>(x << 16);
+    }
 
     using Pair = PairNoInit<Float32, Weight>;
 
@@ -71,7 +126,7 @@ private:
         for (const auto & pair : data)
         {
             sum_weight += pair.getMapped();
-            *arr_it = {to_Float32(pair.getKey()), pair.getMapped()};
+            *arr_it = {toFloat32(pair.getKey()), pair.getMapped()};
             ++arr_it;
         }
 
@@ -112,7 +167,7 @@ private:
         for (const auto & pair : data)
         {
             sum_weight += pair.getMapped();
-            *arr_it = {to_Float32(pair.getKey()), pair.getMapped()};
+            *arr_it = {toFloat32(pair.getKey()), pair.getMapped()};
             ++arr_it;
         }
 

From 2b01b46b25feded1cead14f3d4055df9f3f9ed55 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 21 May 2021 09:33:00 +0300
Subject: [PATCH 62/76] Add comment

---
 src/AggregateFunctions/QuantileBFloat16Histogram.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/AggregateFunctions/QuantileBFloat16Histogram.h b/src/AggregateFunctions/QuantileBFloat16Histogram.h
index 1c648ebc326..559a87a5591 100644
--- a/src/AggregateFunctions/QuantileBFloat16Histogram.h
+++ b/src/AggregateFunctions/QuantileBFloat16Histogram.h
@@ -44,6 +44,9 @@ struct QuantileBFloat16Histogram
 {
     using BFloat16 = UInt16;
     using Weight = UInt64;
+
+    /// Make automatic memory for 16 elements to avoid allocations for small states.
+    /// The usage of trivial hash is ok, because we effectively take logarithm of the values and pathological cases are unlikely.
     using Data = HashMapWithStackMemory<BFloat16, Weight, TrivialHash, 4>;
 
     Data data;

From 11cbae478506458cbd33108142c8f4b406e41ffe Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 21 May 2021 09:49:54 +0300
Subject: [PATCH 63/76] Fix copyright in documentation

---
 docs/tools/blog.py  | 2 +-
 docs/tools/build.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/tools/blog.py b/docs/tools/blog.py
index c3261f61d4d..a8f7eccccb1 100644
--- a/docs/tools/blog.py
+++ b/docs/tools/blog.py
@@ -62,7 +62,7 @@ def build_for_lang(lang, args):
             strict=True,
             theme=theme_cfg,
             nav=blog_nav,
-            copyright='©2016–2020 Yandex LLC',
+            copyright='©2016–2021 Yandex LLC',
             use_directory_urls=True,
             repo_name='ClickHouse/ClickHouse',
             repo_url='https://github.com/ClickHouse/ClickHouse/',
diff --git a/docs/tools/build.py b/docs/tools/build.py
index 5a1f10268ab..61112d5a4f5 100755
--- a/docs/tools/build.py
+++ b/docs/tools/build.py
@@ -94,7 +94,7 @@ def build_for_lang(lang, args):
             site_dir=site_dir,
             strict=True,
             theme=theme_cfg,
-            copyright='©2016–2020 Yandex LLC',
+            copyright='©2016–2021 Yandex LLC',
             use_directory_urls=True,
             repo_name='ClickHouse/ClickHouse',
             repo_url='https://github.com/ClickHouse/ClickHouse/',

From 823e1f222142881240bf2449e3b82f1169fed8fb Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 21 May 2021 09:54:35 +0300
Subject: [PATCH 64/76] Fix strange whitespace

---
 docs/tools/blog.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/tools/blog.py b/docs/tools/blog.py
index a8f7eccccb1..d0f2496f914 100644
--- a/docs/tools/blog.py
+++ b/docs/tools/blog.py
@@ -40,7 +40,7 @@ def build_for_lang(lang, args):
 
         site_names = {
             'en': 'ClickHouse Blog',
-            'ru': 'Блог ClickHouse '
+            'ru': 'Блог ClickHouse'
         }
 
         assert len(site_names) == len(languages)

From be65d588a448c9e6dd263c42f82e84113fdc501b Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 21 May 2021 10:19:53 +0300
Subject: [PATCH 65/76] Debug strange errors in documentation

---
 docs/tools/test.py | 16 ++--------------
 1 file changed, 2 insertions(+), 14 deletions(-)

diff --git a/docs/tools/test.py b/docs/tools/test.py
index 00d1d47137f..290126f3ed1 100755
--- a/docs/tools/test.py
+++ b/docs/tools/test.py
@@ -3,25 +3,13 @@
 import logging
 import os
 import sys
-
 import bs4
-
-import logging
-import os
 import subprocess
 
-import bs4
-
 
 def test_amp(paths, lang):
-    try:
-        # Get latest amp validator version
-        subprocess.check_call('amphtml-validator --help',
-                              stdout=subprocess.DEVNULL,
-                              stderr=subprocess.DEVNULL,
-                              shell=True)
-    except subprocess.CalledProcessError:
-        subprocess.check_call('npm i -g amphtml-validator', stderr=subprocess.DEVNULL, shell=True)
+    # Get latest amp validator version
+    subprocess.check_call('command -v sudo && sudo npm install --global amphtml-validator || npm install --global amphtml-validator', shell=True)
 
     paths = ' '.join(paths)
     command = f'amphtml-validator {paths}'

From 24f1856d5ad1d3bc484170777a7b05ee20dcee5b Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 21 May 2021 10:25:48 +0300
Subject: [PATCH 66/76] Add comment out of despair about low quality
 documentation scripts

---
 docs/tools/test.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/docs/tools/test.py b/docs/tools/test.py
index 290126f3ed1..e95ce444ae5 100755
--- a/docs/tools/test.py
+++ b/docs/tools/test.py
@@ -8,16 +8,16 @@ import subprocess
 
 
 def test_amp(paths, lang):
-    # Get latest amp validator version
+    # Get latest amp validator version # TODO This makes "leftpad" attack possible, remove it.
     subprocess.check_call('command -v sudo && sudo npm install --global amphtml-validator || npm install --global amphtml-validator', shell=True)
 
-    paths = ' '.join(paths)
-    command = f'amphtml-validator {paths}'
-    try:
-        subprocess.check_output(command, shell=True).decode('utf-8')
-    except subprocess.CalledProcessError:
-        logging.error(f'Invalid AMP for {lang}')
-        raise
+    for path in paths:
+        command = f'amphtml-validator {path}'
+        try:
+            subprocess.check_output(command, shell=True).decode('utf-8')
+        except subprocess.CalledProcessError:
+            logging.error(f'Invalid AMP for {lang}, {path}')
+            raise
 
 
 def test_template(template_path):

From cec9d48edcdb96ba85975f90a6dbadad9a08f7cb Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 21 May 2021 10:36:45 +0300
Subject: [PATCH 67/76] Simply avoid validating AMP

---
 docs/tools/amp.py  |  1 -
 docs/tools/test.py | 13 -------------
 2 files changed, 14 deletions(-)

diff --git a/docs/tools/amp.py b/docs/tools/amp.py
index b08b58d3cba..22417407946 100644
--- a/docs/tools/amp.py
+++ b/docs/tools/amp.py
@@ -62,7 +62,6 @@ def build_amp(lang, args, cfg):
         for root, _, filenames in os.walk(site_temp):
             if 'index.html' in filenames:
                 paths.append(prepare_amp_html(lang, args, root, site_temp, main_site_dir))
-        test.test_amp(paths, lang)
     logging.info(f'Finished building AMP version for {lang}')
 
 
diff --git a/docs/tools/test.py b/docs/tools/test.py
index e95ce444ae5..ada4df29644 100755
--- a/docs/tools/test.py
+++ b/docs/tools/test.py
@@ -7,19 +7,6 @@ import bs4
 import subprocess
 
 
-def test_amp(paths, lang):
-    # Get latest amp validator version # TODO This makes "leftpad" attack possible, remove it.
-    subprocess.check_call('command -v sudo && sudo npm install --global amphtml-validator || npm install --global amphtml-validator', shell=True)
-
-    for path in paths:
-        command = f'amphtml-validator {path}'
-        try:
-            subprocess.check_output(command, shell=True).decode('utf-8')
-        except subprocess.CalledProcessError:
-            logging.error(f'Invalid AMP for {lang}, {path}')
-            raise
-
-
 def test_template(template_path):
     if template_path.endswith('amp.html'):
         # Inline CSS/JS is ok for AMP pages

From 2aec0e37e42cb81f9e41ff8f36077c353a871b91 Mon Sep 17 00:00:00 2001
From: Maksim Kita <maksim-kita@yandex-team.ru>
Date: Fri, 21 May 2021 15:27:41 +0300
Subject: [PATCH 68/76] CacheDictionaryStorage update

---
 src/Dictionaries/CacheDictionaryStorage.h | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/Dictionaries/CacheDictionaryStorage.h b/src/Dictionaries/CacheDictionaryStorage.h
index 07217369dc1..a79d83eea05 100644
--- a/src/Dictionaries/CacheDictionaryStorage.h
+++ b/src/Dictionaries/CacheDictionaryStorage.h
@@ -232,7 +232,7 @@ private:
 
             if (unlikely(attribute.is_complex_type))
             {
-                auto & container = std::get<Row>(attribute.attribute_container);
+                auto & container = std::get<std::vector<Field>>(attribute.attribute_container);
 
                 for (size_t fetched_key_index = 0; fetched_key_index < fetched_columns_index; ++fetched_key_index)
                 {
@@ -506,7 +506,7 @@ private:
 
         if (unlikely(attribute.is_complex_type))
         {
-            auto & container = std::get<Row>(attribute.attribute_container);
+            auto & container = std::get<std::vector<Field>>(attribute.attribute_container);
             std::forward<GetContainerFunc>(func)(container);
         }
         else
@@ -564,7 +564,7 @@ private:
                 last_attribute.is_complex_type = dictionary_attribute.is_nullable || dictionary_attribute.is_array;
 
                 if (dictionary_attribute.is_nullable)
-                    last_attribute.attribute_container = Row();
+                    last_attribute.attribute_container = std::vector<Field>();
                 else
                     last_attribute.attribute_container = PaddedPODArray<ValueType>();
             };
@@ -609,7 +609,7 @@ private:
             PaddedPODArray<Float64>,
             PaddedPODArray<UUID>,
             PaddedPODArray<StringRef>,
-            Row> attribute_container;
+            std::vector<Field>> attribute_container;
     };
 
     CacheDictionaryStorageConfiguration configuration;

From 1a18bb57ceba4d02d9287735fc8b87246c2ce6db Mon Sep 17 00:00:00 2001
From: Mikhail Andreev <x11org@gmail.com>
Date: Fri, 21 May 2021 17:46:35 +0300
Subject: [PATCH 69/76] =?UTF-8?q?=D0=9F=D0=BE=D0=BF=D1=80=D0=B0=D0=B2?=
 =?UTF-8?q?=D0=B8=D0=BB=20=D0=BE=D0=BF=D0=B5=D1=87=D0=B0=D1=82=D0=BA=D0=B8?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docs/ru/sql-reference/functions/encryption-functions.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/ru/sql-reference/functions/encryption-functions.md b/docs/ru/sql-reference/functions/encryption-functions.md
index 844f9cc3197..44957fde152 100644
--- a/docs/ru/sql-reference/functions/encryption-functions.md
+++ b/docs/ru/sql-reference/functions/encryption-functions.md
@@ -5,11 +5,11 @@ toc_title: "Функции для шифрования"
 
 # Функции шифрования {#encryption-functions}
 
-Даннвые функции реализуют шифрование и расшифровку данных с помощью AES (Advanced Encryption Standard) алгоритма.
+Данные функции реализуют шифрование и расшифровку данных с помощью AES (Advanced Encryption Standard) алгоритма.
 
 Длина ключа зависит от режима шифрования. Он может быть длинной в 16, 24 и 32 байта для режимов шифрования `-128-`, `-196-` и `-256-` соответственно.
 
-Длина инициализирующего вектора всегда 16 байт (лишнии байты игнорируются). 
+Длина инициализирующего вектора всегда 16 байт (лишние байты игнорируются). 
 
 Обратите внимание, что до версии Clickhouse 21.1 эти функции работали медленно.
 

From 33755f1f3d02e1557184fe5e364ea0906225cefe Mon Sep 17 00:00:00 2001
From: BoloniniD <denis@bolonin.ru>
Date: Fri, 21 May 2021 21:56:22 +0300
Subject: [PATCH 70/76] Implemented YAMLParser class

---
 .gitmodules                                   |    4 +
 CMakeLists.txt                                |    1 +
 base/bridge/CMakeLists.txt                    |    8 +-
 cmake/find/yaml-cpp.cmake                     |    9 +
 contrib/CMakeLists.txt                        |    4 +
 contrib/yaml-cpp                              |    1 +
 contrib/yaml-cpp-cmake/CMakeLists.txt         |   39 +
 programs/server/config-example.yaml.disabled  |   86 ++
 src/Common/Config/CMakeLists.txt              |    8 +
 src/Common/Config/ConfigProcessor.cpp         |   39 +-
 src/Common/Config/ConfigProcessor.h           |    5 +
 src/Common/Config/YAMLParser.cpp              |  166 +++
 src/Common/Config/YAMLParser.h                |   55 +
 src/Common/ErrorCodes.cpp                     |    1 +
 src/Common/config.h.in                        |    1 +
 tests/integration/helpers/cluster.py          |   40 +-
 tests/integration/runner                      |    8 +-
 .../test_config_xml_full/__init__.py          |    0
 .../configs/config.d/access_control.xml       |   13 +
 .../configs/config.d/keeper_port.xml          |   23 +
 .../configs/config.d/log_to_console.xml       |    7 +
 .../configs/config.d/logging_no_rotate.xml    |    8 +
 .../configs/config.d/macros.xml               |    9 +
 .../configs/config.d/metric_log.xml           |    8 +
 .../configs/config.d/more_clusters.xml        |   49 +
 .../configs/config.d/part_log.xml             |    8 +
 .../configs/config.d/path.xml                 |    8 +
 .../configs/config.d/query_masking_rules.xml  |   10 +
 .../configs/config.d/tcp_with_proxy.xml       |    3 +
 .../configs/config.d/text_log.xml             |    7 +
 .../configs/config.d/zookeeper.xml            |    8 +
 .../test_config_xml_full/configs/config.xml   | 1118 +++++++++++++++++
 .../test_config_xml_full/configs/embedded.xml |   40 +
 .../users.d/allow_introspection_functions.xml |    8 +
 .../configs/users.d/log_queries.xml           |    7 +
 .../test_config_xml_full/configs/users.xml    |  120 ++
 .../integration/test_config_xml_full/test.py  |   40 +
 .../test_config_xml_main/__init__.py          |    0
 .../configs/config.d/access_control.yaml      |    7 +
 .../configs/config.d/keeper_port.yaml         |   15 +
 .../configs/config.d/log_to_console.yaml      |    7 +
 .../configs/config.d/logging_no_rotate.yaml   |    2 +
 .../configs/config.d/macros.yaml              |    7 +
 .../configs/config.d/metric_log.yaml          |    6 +
 .../configs/config.d/more_clusters.yaml       |   23 +
 .../configs/config.d/part_log.yaml            |    5 +
 .../configs/config.d/path.yaml                |   18 +
 .../configs/config.d/query_masking_rules.yaml |    4 +
 .../configs/config.d/tcp_with_proxy.yaml      |    1 +
 .../test_cluster_with_incorrect_pw.yaml       |   11 +
 .../configs/config.d/text_log.yaml            |    4 +
 .../configs/config.d/zookeeper.yaml           |    5 +
 .../test_config_xml_main/configs/config.xml   |  277 ++++
 .../test_config_xml_main/configs/embedded.xml |   40 +
 .../allow_introspection_functions.yaml        |    3 +
 .../configs/users.d/log_queries.yaml          |    3 +
 .../test_config_xml_main/configs/users.xml    |   19 +
 .../integration/test_config_xml_main/test.py  |   43 +
 .../test_config_xml_yaml_mix/__init__.py      |    0
 .../config.d/0_common_instance_config.yaml    |    6 +
 .../configs/config.d/access_control.yaml      |    7 +
 .../configs/config.d/keeper_port.xml          |   23 +
 .../configs/config.d/log_to_console.yaml      |    7 +
 .../configs/config.d/logging_no_rotate.xml    |    8 +
 .../configs/config.d/macros.yaml              |    7 +
 .../configs/config.d/metric_log.xml           |    8 +
 .../configs/config.d/more_clusters.yaml       |   23 +
 .../configs/config.d/part_log.xml             |    8 +
 .../configs/config.d/path.yaml                |   18 +
 .../configs/config.d/query_masking_rules.xml  |   10 +
 .../configs/config.d/tcp_with_proxy.yaml      |    1 +
 .../test_cluster_with_incorrect_pw.xml        |   21 +
 .../configs/config.d/text_log.yaml            |    4 +
 .../configs/config.d/zookeeper.xml            |    8 +
 .../configs/config.xml                        |  277 ++++
 .../configs/embedded.xml                      |   40 +
 .../users.d/allow_introspection_functions.xml |    8 +
 .../configs/users.d/log_queries.yaml          |    3 +
 .../configs/users.yaml                        |   12 +
 .../test_config_xml_yaml_mix/test.py          |   43 +
 .../test_config_yaml_full/__init__.py         |    0
 .../config.d/0_common_instance_config.yaml    |    6 +
 .../configs/config.d/access_control.yaml      |    7 +
 .../configs/config.d/keeper_port.yaml         |   15 +
 .../configs/config.d/log_to_console.yaml      |    7 +
 .../configs/config.d/logging_no_rotate.yaml   |    2 +
 .../configs/config.d/macros.yaml              |    7 +
 .../configs/config.d/metric_log.yaml          |    6 +
 .../configs/config.d/more_clusters.yaml       |   23 +
 .../configs/config.d/part_log.yaml            |    5 +
 .../configs/config.d/path.yaml                |   18 +
 .../configs/config.d/query_masking_rules.yaml |    4 +
 .../configs/config.d/tcp_with_proxy.yaml      |    1 +
 .../test_cluster_with_incorrect_pw.yaml       |   11 +
 .../configs/config.d/text_log.yaml            |    4 +
 .../configs/config.d/zookeeper.yaml           |    5 +
 .../test_config_yaml_full/configs/config.yaml |  183 +++
 .../configs/embedded.xml                      |   40 +
 .../allow_introspection_functions.yaml        |    3 +
 .../configs/users.d/log_queries.yaml          |    3 +
 .../test_config_yaml_full/configs/users.yaml  |   12 +
 .../integration/test_config_yaml_full/test.py |   42 +
 .../test_config_yaml_main/__init__.py         |    0
 .../config.d/0_common_instance_config.yaml    |    6 +
 .../configs/config.d/access_control.xml       |   13 +
 .../configs/config.d/keeper_port.xml          |   23 +
 .../configs/config.d/log_to_console.xml       |    7 +
 .../configs/config.d/logging_no_rotate.xml    |    8 +
 .../configs/config.d/macros.xml               |    9 +
 .../configs/config.d/metric_log.xml           |    8 +
 .../configs/config.d/more_clusters.xml        |   49 +
 .../configs/config.d/part_log.xml             |    8 +
 .../configs/config.d/path.xml                 |    8 +
 .../configs/config.d/query_masking_rules.xml  |   10 +
 .../configs/config.d/tcp_with_proxy.xml       |    3 +
 .../test_cluster_with_incorrect_pw.xml        |   21 +
 .../configs/config.d/text_log.xml             |    7 +
 .../configs/config.d/zookeeper.xml            |    8 +
 .../test_config_yaml_main/configs/config.yaml |  183 +++
 .../configs/embedded.xml                      |   40 +
 .../users.d/allow_introspection_functions.xml |    8 +
 .../configs/users.d/log_queries.xml           |    7 +
 .../test_config_yaml_main/configs/users.yaml  |   12 +
 .../integration/test_config_yaml_main/test.py |   43 +
 124 files changed, 3873 insertions(+), 23 deletions(-)
 create mode 100644 cmake/find/yaml-cpp.cmake
 create mode 160000 contrib/yaml-cpp
 create mode 100644 contrib/yaml-cpp-cmake/CMakeLists.txt
 create mode 100644 programs/server/config-example.yaml.disabled
 create mode 100644 src/Common/Config/YAMLParser.cpp
 create mode 100644 src/Common/Config/YAMLParser.h
 create mode 100644 tests/integration/test_config_xml_full/__init__.py
 create mode 100644 tests/integration/test_config_xml_full/configs/config.d/access_control.xml
 create mode 100644 tests/integration/test_config_xml_full/configs/config.d/keeper_port.xml
 create mode 100644 tests/integration/test_config_xml_full/configs/config.d/log_to_console.xml
 create mode 100644 tests/integration/test_config_xml_full/configs/config.d/logging_no_rotate.xml
 create mode 100644 tests/integration/test_config_xml_full/configs/config.d/macros.xml
 create mode 100644 tests/integration/test_config_xml_full/configs/config.d/metric_log.xml
 create mode 100644 tests/integration/test_config_xml_full/configs/config.d/more_clusters.xml
 create mode 100644 tests/integration/test_config_xml_full/configs/config.d/part_log.xml
 create mode 100644 tests/integration/test_config_xml_full/configs/config.d/path.xml
 create mode 100644 tests/integration/test_config_xml_full/configs/config.d/query_masking_rules.xml
 create mode 100644 tests/integration/test_config_xml_full/configs/config.d/tcp_with_proxy.xml
 create mode 100644 tests/integration/test_config_xml_full/configs/config.d/text_log.xml
 create mode 100644 tests/integration/test_config_xml_full/configs/config.d/zookeeper.xml
 create mode 100644 tests/integration/test_config_xml_full/configs/config.xml
 create mode 100644 tests/integration/test_config_xml_full/configs/embedded.xml
 create mode 100644 tests/integration/test_config_xml_full/configs/users.d/allow_introspection_functions.xml
 create mode 100644 tests/integration/test_config_xml_full/configs/users.d/log_queries.xml
 create mode 100644 tests/integration/test_config_xml_full/configs/users.xml
 create mode 100644 tests/integration/test_config_xml_full/test.py
 create mode 100644 tests/integration/test_config_xml_main/__init__.py
 create mode 100644 tests/integration/test_config_xml_main/configs/config.d/access_control.yaml
 create mode 100644 tests/integration/test_config_xml_main/configs/config.d/keeper_port.yaml
 create mode 100644 tests/integration/test_config_xml_main/configs/config.d/log_to_console.yaml
 create mode 100644 tests/integration/test_config_xml_main/configs/config.d/logging_no_rotate.yaml
 create mode 100644 tests/integration/test_config_xml_main/configs/config.d/macros.yaml
 create mode 100644 tests/integration/test_config_xml_main/configs/config.d/metric_log.yaml
 create mode 100644 tests/integration/test_config_xml_main/configs/config.d/more_clusters.yaml
 create mode 100644 tests/integration/test_config_xml_main/configs/config.d/part_log.yaml
 create mode 100644 tests/integration/test_config_xml_main/configs/config.d/path.yaml
 create mode 100644 tests/integration/test_config_xml_main/configs/config.d/query_masking_rules.yaml
 create mode 100644 tests/integration/test_config_xml_main/configs/config.d/tcp_with_proxy.yaml
 create mode 100644 tests/integration/test_config_xml_main/configs/config.d/test_cluster_with_incorrect_pw.yaml
 create mode 100644 tests/integration/test_config_xml_main/configs/config.d/text_log.yaml
 create mode 100644 tests/integration/test_config_xml_main/configs/config.d/zookeeper.yaml
 create mode 100644 tests/integration/test_config_xml_main/configs/config.xml
 create mode 100644 tests/integration/test_config_xml_main/configs/embedded.xml
 create mode 100644 tests/integration/test_config_xml_main/configs/users.d/allow_introspection_functions.yaml
 create mode 100644 tests/integration/test_config_xml_main/configs/users.d/log_queries.yaml
 create mode 100644 tests/integration/test_config_xml_main/configs/users.xml
 create mode 100644 tests/integration/test_config_xml_main/test.py
 create mode 100644 tests/integration/test_config_xml_yaml_mix/__init__.py
 create mode 100644 tests/integration/test_config_xml_yaml_mix/configs/config.d/0_common_instance_config.yaml
 create mode 100644 tests/integration/test_config_xml_yaml_mix/configs/config.d/access_control.yaml
 create mode 100644 tests/integration/test_config_xml_yaml_mix/configs/config.d/keeper_port.xml
 create mode 100644 tests/integration/test_config_xml_yaml_mix/configs/config.d/log_to_console.yaml
 create mode 100644 tests/integration/test_config_xml_yaml_mix/configs/config.d/logging_no_rotate.xml
 create mode 100644 tests/integration/test_config_xml_yaml_mix/configs/config.d/macros.yaml
 create mode 100644 tests/integration/test_config_xml_yaml_mix/configs/config.d/metric_log.xml
 create mode 100644 tests/integration/test_config_xml_yaml_mix/configs/config.d/more_clusters.yaml
 create mode 100644 tests/integration/test_config_xml_yaml_mix/configs/config.d/part_log.xml
 create mode 100644 tests/integration/test_config_xml_yaml_mix/configs/config.d/path.yaml
 create mode 100644 tests/integration/test_config_xml_yaml_mix/configs/config.d/query_masking_rules.xml
 create mode 100644 tests/integration/test_config_xml_yaml_mix/configs/config.d/tcp_with_proxy.yaml
 create mode 100644 tests/integration/test_config_xml_yaml_mix/configs/config.d/test_cluster_with_incorrect_pw.xml
 create mode 100644 tests/integration/test_config_xml_yaml_mix/configs/config.d/text_log.yaml
 create mode 100644 tests/integration/test_config_xml_yaml_mix/configs/config.d/zookeeper.xml
 create mode 100644 tests/integration/test_config_xml_yaml_mix/configs/config.xml
 create mode 100644 tests/integration/test_config_xml_yaml_mix/configs/embedded.xml
 create mode 100644 tests/integration/test_config_xml_yaml_mix/configs/users.d/allow_introspection_functions.xml
 create mode 100644 tests/integration/test_config_xml_yaml_mix/configs/users.d/log_queries.yaml
 create mode 100644 tests/integration/test_config_xml_yaml_mix/configs/users.yaml
 create mode 100644 tests/integration/test_config_xml_yaml_mix/test.py
 create mode 100644 tests/integration/test_config_yaml_full/__init__.py
 create mode 100644 tests/integration/test_config_yaml_full/configs/config.d/0_common_instance_config.yaml
 create mode 100644 tests/integration/test_config_yaml_full/configs/config.d/access_control.yaml
 create mode 100644 tests/integration/test_config_yaml_full/configs/config.d/keeper_port.yaml
 create mode 100644 tests/integration/test_config_yaml_full/configs/config.d/log_to_console.yaml
 create mode 100644 tests/integration/test_config_yaml_full/configs/config.d/logging_no_rotate.yaml
 create mode 100644 tests/integration/test_config_yaml_full/configs/config.d/macros.yaml
 create mode 100644 tests/integration/test_config_yaml_full/configs/config.d/metric_log.yaml
 create mode 100644 tests/integration/test_config_yaml_full/configs/config.d/more_clusters.yaml
 create mode 100644 tests/integration/test_config_yaml_full/configs/config.d/part_log.yaml
 create mode 100644 tests/integration/test_config_yaml_full/configs/config.d/path.yaml
 create mode 100644 tests/integration/test_config_yaml_full/configs/config.d/query_masking_rules.yaml
 create mode 100644 tests/integration/test_config_yaml_full/configs/config.d/tcp_with_proxy.yaml
 create mode 100644 tests/integration/test_config_yaml_full/configs/config.d/test_cluster_with_incorrect_pw.yaml
 create mode 100644 tests/integration/test_config_yaml_full/configs/config.d/text_log.yaml
 create mode 100644 tests/integration/test_config_yaml_full/configs/config.d/zookeeper.yaml
 create mode 100644 tests/integration/test_config_yaml_full/configs/config.yaml
 create mode 100644 tests/integration/test_config_yaml_full/configs/embedded.xml
 create mode 100644 tests/integration/test_config_yaml_full/configs/users.d/allow_introspection_functions.yaml
 create mode 100644 tests/integration/test_config_yaml_full/configs/users.d/log_queries.yaml
 create mode 100644 tests/integration/test_config_yaml_full/configs/users.yaml
 create mode 100644 tests/integration/test_config_yaml_full/test.py
 create mode 100644 tests/integration/test_config_yaml_main/__init__.py
 create mode 100644 tests/integration/test_config_yaml_main/configs/config.d/0_common_instance_config.yaml
 create mode 100644 tests/integration/test_config_yaml_main/configs/config.d/access_control.xml
 create mode 100644 tests/integration/test_config_yaml_main/configs/config.d/keeper_port.xml
 create mode 100644 tests/integration/test_config_yaml_main/configs/config.d/log_to_console.xml
 create mode 100644 tests/integration/test_config_yaml_main/configs/config.d/logging_no_rotate.xml
 create mode 100644 tests/integration/test_config_yaml_main/configs/config.d/macros.xml
 create mode 100644 tests/integration/test_config_yaml_main/configs/config.d/metric_log.xml
 create mode 100644 tests/integration/test_config_yaml_main/configs/config.d/more_clusters.xml
 create mode 100644 tests/integration/test_config_yaml_main/configs/config.d/part_log.xml
 create mode 100644 tests/integration/test_config_yaml_main/configs/config.d/path.xml
 create mode 100644 tests/integration/test_config_yaml_main/configs/config.d/query_masking_rules.xml
 create mode 100644 tests/integration/test_config_yaml_main/configs/config.d/tcp_with_proxy.xml
 create mode 100644 tests/integration/test_config_yaml_main/configs/config.d/test_cluster_with_incorrect_pw.xml
 create mode 100644 tests/integration/test_config_yaml_main/configs/config.d/text_log.xml
 create mode 100644 tests/integration/test_config_yaml_main/configs/config.d/zookeeper.xml
 create mode 100644 tests/integration/test_config_yaml_main/configs/config.yaml
 create mode 100644 tests/integration/test_config_yaml_main/configs/embedded.xml
 create mode 100644 tests/integration/test_config_yaml_main/configs/users.d/allow_introspection_functions.xml
 create mode 100644 tests/integration/test_config_yaml_main/configs/users.d/log_queries.xml
 create mode 100644 tests/integration/test_config_yaml_main/configs/users.yaml
 create mode 100644 tests/integration/test_config_yaml_main/test.py

diff --git a/.gitmodules b/.gitmodules
index 66a2370f0da..2ccce88e5e4 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -228,3 +228,7 @@
 [submodule "contrib/datasketches-cpp"]
 	path = contrib/datasketches-cpp
 	url = https://github.com/ClickHouse-Extras/datasketches-cpp.git
+
+[submodule "contrib/yaml-cpp"]
+	path = contrib/yaml-cpp
+	url = https://github.com/ClickHouse-Extras/yaml-cpp.git
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2c3fa088995..866d9f542e1 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -527,6 +527,7 @@ include (cmake/find/nanodbc.cmake)
 include (cmake/find/rocksdb.cmake)
 include (cmake/find/libpqxx.cmake)
 include (cmake/find/nuraft.cmake)
+include (cmake/find/yaml-cpp.cmake)
 
 
 if(NOT USE_INTERNAL_PARQUET_LIBRARY)
diff --git a/base/bridge/CMakeLists.txt b/base/bridge/CMakeLists.txt
index 20b0b651677..bcba43e8c2e 100644
--- a/base/bridge/CMakeLists.txt
+++ b/base/bridge/CMakeLists.txt
@@ -3,5 +3,11 @@ add_library (bridge
 )
 
 target_include_directories (daemon PUBLIC ..)
-target_link_libraries (bridge PRIVATE daemon dbms Poco::Data Poco::Data::ODBC)
+target_link_libraries (bridge 
+    PRIVATE 
+	daemon
+	dbms
+	Poco::Data
+	Poco::Data::ODBC
+)
 
diff --git a/cmake/find/yaml-cpp.cmake b/cmake/find/yaml-cpp.cmake
new file mode 100644
index 00000000000..9b9d9bd39d6
--- /dev/null
+++ b/cmake/find/yaml-cpp.cmake
@@ -0,0 +1,9 @@
+option(USE_YAML_CPP "Enable yaml-cpp" ${ENABLE_LIBRARIES})
+
+if (NOT USE_YAML_CPP)
+    return()
+endif()
+
+if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/yaml-cpp")
+    message (ERROR "submodule contrib/yaml-cpp is missing. to fix try run: \n git submodule update --init --recursive")
+endif()
diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
index 9eafec23f51..a9438aa4b76 100644
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@@ -50,6 +50,10 @@ add_subdirectory (replxx-cmake)
 add_subdirectory (unixodbc-cmake)
 add_subdirectory (nanodbc-cmake)
 
+if (USE_YAML_CPP)
+    add_subdirectory (yaml-cpp-cmake)
+endif()
+
 if (USE_INTERNAL_XZ_LIBRARY)
     add_subdirectory (xz)
 endif()
diff --git a/contrib/yaml-cpp b/contrib/yaml-cpp
new file mode 160000
index 00000000000..0c86adac6d1
--- /dev/null
+++ b/contrib/yaml-cpp
@@ -0,0 +1 @@
+Subproject commit 0c86adac6d117ee2b4afcedb8ade19036ca0327d
diff --git a/contrib/yaml-cpp-cmake/CMakeLists.txt b/contrib/yaml-cpp-cmake/CMakeLists.txt
new file mode 100644
index 00000000000..ed0287de110
--- /dev/null
+++ b/contrib/yaml-cpp-cmake/CMakeLists.txt
@@ -0,0 +1,39 @@
+set (LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/yaml-cpp)
+
+set (SRCS
+	${LIBRARY_DIR}/src/binary.cpp
+	${LIBRARY_DIR}/src/emitterutils.cpp
+	${LIBRARY_DIR}/src/null.cpp
+	${LIBRARY_DIR}/src/scantoken.cpp
+	${LIBRARY_DIR}/src/convert.cpp
+	${LIBRARY_DIR}/src/exceptions.cpp
+	${LIBRARY_DIR}/src/ostream_wrapper.cpp
+	${LIBRARY_DIR}/src/simplekey.cpp
+	${LIBRARY_DIR}/src/depthguard.cpp 
+	${LIBRARY_DIR}/src/exp.cpp
+	${LIBRARY_DIR}/src/parse.cpp
+	${LIBRARY_DIR}/src/singledocparser.cpp
+	${LIBRARY_DIR}/src/directives.cpp
+	${LIBRARY_DIR}/src/memory.cpp
+	${LIBRARY_DIR}/src/parser.cpp
+	${LIBRARY_DIR}/src/stream.cpp
+	${LIBRARY_DIR}/src/emit.cpp
+	${LIBRARY_DIR}/src/nodebuilder.cpp
+	${LIBRARY_DIR}/src/regex_yaml.cpp
+	${LIBRARY_DIR}/src/tag.cpp
+	${LIBRARY_DIR}/src/emitfromevents.cpp
+	${LIBRARY_DIR}/src/node.cpp
+	${LIBRARY_DIR}/src/scanner.cpp
+	${LIBRARY_DIR}/src/emitter.cpp
+	${LIBRARY_DIR}/src/node_data.cpp
+	${LIBRARY_DIR}/src/scanscalar.cpp
+	${LIBRARY_DIR}/src/emitterstate.cpp
+	${LIBRARY_DIR}/src/nodeevents.cpp
+	${LIBRARY_DIR}/src/scantag.cpp
+)
+
+add_library (yaml-cpp ${SRCS})
+
+
+target_include_directories(yaml-cpp PRIVATE ${LIBRARY_DIR}/include/yaml-cpp)
+target_include_directories(yaml-cpp SYSTEM BEFORE PUBLIC ${LIBRARY_DIR}/include)
diff --git a/programs/server/config-example.yaml.disabled b/programs/server/config-example.yaml.disabled
new file mode 100644
index 00000000000..a83acf50de0
--- /dev/null
+++ b/programs/server/config-example.yaml.disabled
@@ -0,0 +1,86 @@
+# We can use 3 main node types in YAML: Scalar, Map and Sequence.
+
+
+
+# A Scalar is a simple key-value pair:
+
+scalar: 123
+
+# Here we have a key "scalar" and value "123"
+# If we rewrite this in XML, we will get <scalar>123</scalar>
+
+# We can also represent an empty value with '':
+
+key: ''
+
+
+
+# A Map is a node, which contains other nodes:
+
+map:
+  key1: value1
+  key2: value2
+  small_map:
+    key3: value3
+    
+# This map can be converted into:
+# <map>
+#     <key1>value1</key1>
+#     <key2>value2</key2>
+#     <small_map>
+#         <key3>value3</key3>
+#     </small_map>
+# </map>
+
+
+
+# A Sequence is a node, which contains also other nodes.
+# The main difference from Map is that Sequence can also contain simple values.
+
+sequence:
+  - val1
+  - val2
+  - key: 123
+  - map:
+      mkey1: foo
+      mkey2: bar
+      
+# We can represent it in XML this way:
+# <sequence>val1</sequence>
+# <sequence>val2</sequence>
+# <sequence>
+#     <key>123</key>
+# </sequence>
+# <sequence>
+#     <map>
+#         <mkey1>foo</mkey1>
+#         <mkey2>bar</mkey2>
+#     </map>
+# </sequence>
+
+
+
+# YAML does not have direct support for structures like XML attributes.
+# We represent them as nodes with @ prefix in key. Note, that @ is reserved by YAML standard,
+# so you will need to write double quotes around the key. Both Map and Sequence can have
+# attributes as children nodes
+
+map:
+  "@attr1": value1
+  "@attr2": value2
+  key: 123
+  
+# This gives us:
+# <map attr1="value1" attr2="value2">
+#     <key>123</key>
+# </map>
+
+sequence:
+  - "@attr1": value1
+  - "@attr2": value2
+  - 123
+  - abc
+  
+# And this gives us:
+# <map attr1="value1" attr2="value2">123</map>
+# <map attr1="value1" attr2="value2">abc</map>
diff --git a/src/Common/Config/CMakeLists.txt b/src/Common/Config/CMakeLists.txt
index a7914fb17ec..3da44be2af6 100644
--- a/src/Common/Config/CMakeLists.txt
+++ b/src/Common/Config/CMakeLists.txt
@@ -3,6 +3,7 @@ set (SRCS
     ConfigProcessor.cpp
     configReadClient.cpp
     ConfigReloader.cpp
+    YAMLParser.cpp
 )
 
 add_library(clickhouse_common_config ${SRCS})
@@ -15,3 +16,10 @@ target_link_libraries(clickhouse_common_config
     PRIVATE
         string_utils
 )
+
+if (USE_YAML_CPP)
+target_link_libraries(clickhouse_common_config
+    PRIVATE
+        yaml-cpp
+)
+endif()
diff --git a/src/Common/Config/ConfigProcessor.cpp b/src/Common/Config/ConfigProcessor.cpp
index bc2a8a27943..fa9e9b72087 100644
--- a/src/Common/Config/ConfigProcessor.cpp
+++ b/src/Common/Config/ConfigProcessor.cpp
@@ -1,4 +1,8 @@
+#if !defined(ARCADIA_BUILD)
+    #include <Common/config.h>
+#endif
 #include "ConfigProcessor.h"
+#include "YAMLParser.h"
 
 #include <sys/utsname.h>
 #include <cerrno>
@@ -20,10 +24,8 @@
 #include <IO/WriteBufferFromString.h>
 #include <IO/Operators.h>
 
-
 #define PREPROCESSED_SUFFIX "-preprocessed"
 
-
 namespace fs = std::filesystem;
 
 using namespace Poco::XML;
@@ -438,8 +440,10 @@ ConfigProcessor::Files ConfigProcessor::getConfigMergeFiles(const std::string &
             std::string base_name = path.getBaseName();
 
             // Skip non-config and temporary files
-            if (file.isFile() && (extension == "xml" || extension == "conf") && !startsWith(base_name, "."))
-                files.push_back(file.path());
+            if (file.isFile() && (extension == "xml" || extension == "conf" || extension == "yaml" || extension == "yml") && !startsWith(base_name, "."))
+            {
+               files.push_back(file.path());
+            }
         }
     }
 
@@ -453,12 +457,21 @@ XMLDocumentPtr ConfigProcessor::processConfig(
     zkutil::ZooKeeperNodeCache * zk_node_cache,
     const zkutil::EventPtr & zk_changed_event)
 {
-    XMLDocumentPtr config;
     LOG_DEBUG(log, "Processing configuration file '{}'.", path);
 
+    XMLDocumentPtr config;
+
     if (fs::exists(path))
     {
-        config = dom_parser.parse(path);
+        fs::path p(path);
+        if (p.extension() == ".xml")
+        {
+            config = dom_parser.parse(path);
+        }
+        else if (p.extension() == ".yaml" || p.extension() == ".yml")
+        {
+            config = YAMLParser::parse(path);
+        }
     }
     else
     {
@@ -493,8 +506,20 @@ XMLDocumentPtr ConfigProcessor::processConfig(
         {
             LOG_DEBUG(log, "Merging configuration file '{}'.", merge_file);
 
-            XMLDocumentPtr with = dom_parser.parse(merge_file);
+            XMLDocumentPtr with;
+
+            fs::path p(merge_file);
+            if (p.extension() == ".yaml" || p.extension() == ".yml")
+            {
+                with = YAMLParser::parse(merge_file);
+            }
+            else
+            {
+                with = dom_parser.parse(merge_file);
+            }
+
             merge(config, with);
+
             contributing_files.push_back(merge_file);
         }
         catch (Exception & e)
diff --git a/src/Common/Config/ConfigProcessor.h b/src/Common/Config/ConfigProcessor.h
index 7a4102140d9..5b16bc0cb1b 100644
--- a/src/Common/Config/ConfigProcessor.h
+++ b/src/Common/Config/ConfigProcessor.h
@@ -1,5 +1,9 @@
 #pragma once
 
+#if !defined(ARCADIA_BUILD)
+    #include <Common/config.h>
+#endif
+
 #include <string>
 #include <unordered_set>
 #include <vector>
@@ -141,3 +145,4 @@ private:
 };
 
 }
+
diff --git a/src/Common/Config/YAMLParser.cpp b/src/Common/Config/YAMLParser.cpp
new file mode 100644
index 00000000000..9eaf1cdc1ad
--- /dev/null
+++ b/src/Common/Config/YAMLParser.cpp
@@ -0,0 +1,166 @@
+#if !defined(ARCADIA_BUILD)
+    #include <Common/config.h>
+#endif
+
+#if USE_YAML_CPP
+#include "YAMLParser.h"
+
+#include <string>
+#include <cstring>
+#include <vector>
+
+#include <Poco/DOM/Document.h>
+#include <Poco/DOM/DOMParser.h>
+#include <Poco/DOM/DOMWriter.h>
+#include <Poco/DOM/NodeList.h>
+#include <Poco/DOM/Element.h>
+#include <Poco/DOM/AutoPtr.h>
+#include <Poco/DOM/NamedNodeMap.h>
+#include <Poco/DOM/Text.h>
+#include <Common/Exception.h>
+
+#include <yaml-cpp/yaml.h> // Y_IGNORE
+
+#include <common/logger_useful.h>
+
+using namespace Poco::XML;
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int CANNOT_OPEN_FILE;
+    extern const int CANNOT_PARSE_YAML;
+}
+
+/// A prefix symbol in yaml key
+/// We add attributes to nodes by using a prefix symbol in the key part.
+/// Currently we use @ as a prefix symbol. Note, that @ is reserved
+/// by YAML standard, so we need to write a key-value pair like this: "@attribute": attr_value
+const char YAML_ATTRIBUTE_PREFIX = '@';
+
+namespace
+{
+
+Poco::AutoPtr<Poco::XML::Element> createCloneNode(Poco::XML::Element & original_node)
+{
+    Poco::AutoPtr<Poco::XML::Element> clone_node = original_node.ownerDocument()->createElement(original_node.nodeName());
+    original_node.parentNode()->appendChild(clone_node);
+    return clone_node;
+}
+
+void processNode(const YAML::Node & node, Poco::XML::Element & parent_xml_element)
+{
+    auto * xml_document = parent_xml_element.ownerDocument();
+    switch (node.Type())
+    {
+        case YAML::NodeType::Scalar:
+        {
+            auto value = node.as<std::string>();
+            Poco::AutoPtr<Poco::XML::Text> xml_value = xml_document->createTextNode(value);
+            parent_xml_element.appendChild(xml_value);
+            break;
+        }
+
+        /// We process YAML Sequences as a
+        /// list of <key>value</key> tags with same key and different values.
+        /// For example, we translate this sequence
+        /// seq:
+        ///     - val1
+        ///     - val2
+        ///
+        /// into this:
+        /// <seq>val1</seq>
+        /// <seq>val2</seq>
+        case YAML::NodeType::Sequence:
+        {
+            for (const auto & child_node : node)
+                if (parent_xml_element.hasChildNodes())
+                {
+                    /// We want to process sequences like that:
+                    /// seq:
+                    ///     - val1
+                    ///     - k2: val2
+                    ///     - val3
+                    ///     - k4: val4
+                    ///     - val5
+                    /// into xml like this:
+                    /// <seq>val1</seq>
+                    /// <seq>
+                    ///     <k2>val2</k2>
+                    /// </seq>
+                    /// <seq>val3</seq>
+                    /// <seq>
+                    ///     <k4>val4</k4>
+                    /// </seq>
+                    /// <seq>val5</seq>
+                    /// So, we create a new parent node with same tag for each child node
+                    processNode(child_node, *createCloneNode(parent_xml_element));
+                }
+                else
+                {
+                    processNode(child_node, parent_xml_element);
+                }
+            break;
+        }
+        case YAML::NodeType::Map:
+        {
+            for (const auto & key_value_pair : node)
+            {
+                const auto & key_node = key_value_pair.first;
+                const auto & value_node = key_value_pair.second;
+                auto key = key_node.as<std::string>();
+                bool is_attribute = (key.starts_with(YAML_ATTRIBUTE_PREFIX) && value_node.IsScalar());
+                if (is_attribute)
+                {
+                    /// we use substr(1) here to remove YAML_ATTRIBUTE_PREFIX from key
+                    auto attribute_name = key.substr(1);
+                    auto value = value_node.as<std::string>();
+                    parent_xml_element.setAttribute(attribute_name, value);
+                }
+                else
+                {
+                    Poco::AutoPtr<Poco::XML::Element> xml_key = xml_document->createElement(key);
+                    parent_xml_element.appendChild(xml_key);
+                    processNode(value_node, *xml_key);
+                }
+            }
+            break;
+        }
+        case YAML::NodeType::Null: break;
+        case YAML::NodeType::Undefined:
+        {
+            throw Exception(ErrorCodes::CANNOT_PARSE_YAML, "YAMLParser has encountered node with undefined type and cannot continue parsing of the file");
+        }
+    }
+}
+
+}
+
+Poco::AutoPtr<Poco::XML::Document> YAMLParser::parse(const String& path)
+{
+    YAML::Node node_yml;
+    try
+    {
+        node_yml = YAML::LoadFile(path);
+    }
+    catch (const YAML::ParserException& e)
+    {
+        /// yaml-cpp cannot parse the file because its contents are incorrect
+        throw Exception(ErrorCodes::CANNOT_PARSE_YAML, "Unable to parse YAML configuration file {}", path, e.what());
+    }
+    catch (const YAML::BadFile&)
+    {
+        /// yaml-cpp cannot open the file even though it exists
+        throw Exception(ErrorCodes::CANNOT_OPEN_FILE, "Unable to open YAML configuration file {}", path);
+    }
+    Poco::AutoPtr<Poco::XML::Document> xml = new Document;
+    Poco::AutoPtr<Poco::XML::Element> root_node = xml->createElement("yandex");
+    xml->appendChild(root_node);
+    processNode(node_yml, *root_node);
+    return xml;
+}
+
+}
+#endif
diff --git a/src/Common/Config/YAMLParser.h b/src/Common/Config/YAMLParser.h
new file mode 100644
index 00000000000..a716f3de9b1
--- /dev/null
+++ b/src/Common/Config/YAMLParser.h
@@ -0,0 +1,55 @@
+#pragma once
+
+#if !defined(ARCADIA_BUILD)
+    #include <Common/config.h>
+#endif
+
+#include <string>
+
+#include <Poco/DOM/Document.h>
+#include "Poco/DOM/AutoPtr.h"
+#include <common/logger_useful.h>
+
+#if USE_YAML_CPP
+
+namespace DB
+{
+
+/// Real YAML parser: loads yaml file into a YAML::Node
+class YAMLParserImpl
+{
+public:
+    static Poco::AutoPtr<Poco::XML::Document> parse(const String& path);
+};
+
+using YAMLParser = YAMLParserImpl;
+
+}
+
+#else
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int CANNOT_PARSE_YAML;
+}
+
+/// Fake YAML parser: throws an exception if we try to parse YAML configs in a build without yaml-cpp
+class DummyYAMLParser
+{
+public:
+    static Poco::AutoPtr<Poco::XML::Document> parse(const String& path)
+    {
+        Poco::AutoPtr<Poco::XML::Document> xml = new Poco::XML::Document;
+        throw Exception(ErrorCodes::CANNOT_PARSE_YAML, "Unable to parse YAML configuration file {} without usage of yaml-cpp library", path);
+        return xml;
+    }
+};
+
+using YAMLParser = DummyYAMLParser;
+
+}
+
+#endif
diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp
index b1efd791bf8..330a193212b 100644
--- a/src/Common/ErrorCodes.cpp
+++ b/src/Common/ErrorCodes.cpp
@@ -552,6 +552,7 @@
     M(582, NO_SUCH_PROJECTION_IN_TABLE) \
     M(583, ILLEGAL_PROJECTION) \
     M(584, PROJECTION_NOT_USED) \
+    M(585, CANNOT_PARSE_YAML) \
     \
     M(998, POSTGRESQL_CONNECTION_FAILURE) \
     M(999, KEEPER_EXCEPTION) \
diff --git a/src/Common/config.h.in b/src/Common/config.h.in
index 28a21ea7764..6844f0fa9e3 100644
--- a/src/Common/config.h.in
+++ b/src/Common/config.h.in
@@ -16,3 +16,4 @@
 #cmakedefine01 USE_STATS
 #cmakedefine01 CLICKHOUSE_SPLIT_BINARY
 #cmakedefine01 USE_DATASKETCHES
+#cmakedefine01 USE_YAML_CPP
diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py
index 5bd608ef758..6287064b616 100644
--- a/tests/integration/helpers/cluster.py
+++ b/tests/integration/helpers/cluster.py
@@ -126,7 +126,8 @@ class ClickHouseCluster:
     """
 
     def __init__(self, base_path, name=None, base_config_dir=None, server_bin_path=None, client_bin_path=None,
-                 odbc_bridge_bin_path=None, library_bridge_bin_path=None, zookeeper_config_path=None, custom_dockerd_host=None):
+                 odbc_bridge_bin_path=None, library_bridge_bin_path=None, zookeeper_config_path=None, 
+                 custom_dockerd_host=None):
         for param in list(os.environ.keys()):
             print("ENV %40s %s" % (param, os.environ[param]))
         self.base_dir = p.dirname(base_path)
@@ -219,7 +220,9 @@ class ClickHouseCluster:
                      with_redis=False, with_minio=False, with_cassandra=False,
                      hostname=None, env_variables=None, image="yandex/clickhouse-integration-test", tag=None,
                      stay_alive=False, ipv4_address=None, ipv6_address=None, with_installed_binary=False, tmpfs=None,
-                     zookeeper_docker_compose_path=None, zookeeper_use_tmpfs=True, minio_certs_dir=None, use_keeper=True):
+                     zookeeper_docker_compose_path=None, zookeeper_use_tmpfs=True, minio_certs_dir=None, use_keeper=True,
+                     main_config_name="config.xml", users_config_name="users.xml", copy_common_configs=True):
+
         """Add an instance to the cluster.
 
         name - the name of the instance directory and the value of the 'instance' macro in ClickHouse.
@@ -280,6 +283,9 @@ class ClickHouseCluster:
             ipv4_address=ipv4_address,
             ipv6_address=ipv6_address,
             with_installed_binary=with_installed_binary,
+            main_config_name=main_config_name,
+            users_config_name=users_config_name,
+            copy_common_configs=copy_common_configs,
             tmpfs=tmpfs or [])
 
         docker_compose_yml_dir = get_docker_compose_path()
@@ -944,7 +950,7 @@ class ClickHouseCluster:
             subprocess_check_call(self.base_zookeeper_cmd + ["start", n])
 
 
-CLICKHOUSE_START_COMMAND = "clickhouse server --config-file=/etc/clickhouse-server/config.xml --log-file=/var/log/clickhouse-server/clickhouse-server.log --errorlog-file=/var/log/clickhouse-server/clickhouse-server.err.log"
+CLICKHOUSE_START_COMMAND = "clickhouse server --config-file=/etc/clickhouse-server/{main_config_file} --log-file=/var/log/clickhouse-server/clickhouse-server.log --errorlog-file=/var/log/clickhouse-server/clickhouse-server.err.log"
 
 CLICKHOUSE_STAY_ALIVE_COMMAND = 'bash -c "{} --daemon; tail -f /dev/null"'.format(CLICKHOUSE_START_COMMAND)
 
@@ -1000,6 +1006,8 @@ class ClickHouseInstance:
             macros, with_zookeeper, zookeeper_config_path, with_mysql, with_mysql_cluster, with_kafka, with_kerberized_kafka, with_rabbitmq, with_kerberized_hdfs,
             with_mongo, with_redis, with_minio,
             with_cassandra, server_bin_path, odbc_bridge_bin_path, library_bridge_bin_path, clickhouse_path_dir, with_odbc_drivers,
+            clickhouse_start_command=CLICKHOUSE_START_COMMAND,
+            main_config_name="config.xml", users_config_name="users.xml", copy_common_configs=True,
             hostname=None, env_variables=None,
             image="yandex/clickhouse-integration-test", tag="latest",
             stay_alive=False, ipv4_address=None, ipv6_address=None, with_installed_binary=False, tmpfs=None):
@@ -1036,6 +1044,12 @@ class ClickHouseInstance:
         self.with_minio = with_minio
         self.with_cassandra = with_cassandra
 
+        self.main_config_name = main_config_name
+        self.users_config_name = users_config_name
+        self.copy_common_configs = copy_common_configs
+
+        self.clickhouse_start_command = clickhouse_start_command.replace("{main_config_file}", self.main_config_name)
+
         self.path = p.join(self.cluster.instances_dir, name)
         self.docker_compose_path = p.join(self.path, 'docker-compose.yml')
         self.env_variables = env_variables or {}
@@ -1177,7 +1191,7 @@ class ClickHouseInstance:
         if not self.stay_alive:
             raise Exception("clickhouse can be started again only with stay_alive=True instance")
 
-        self.exec_in_container(["bash", "-c", "{} --daemon".format(CLICKHOUSE_START_COMMAND)], user=str(os.getuid()))
+        self.exec_in_container(["bash", "-c", "{} --daemon".format(self.clickhouse_start_command)], user=str(os.getuid()))
         # wait start
         from helpers.test_tools import assert_eq_with_retry
         assert_eq_with_retry(self, "select 1", "1", retry_count=int(start_wait_sec / 0.5), sleep_time=0.5)
@@ -1263,7 +1277,7 @@ class ClickHouseInstance:
         self.exec_in_container(["bash", "-c",
                                 "cp /usr/share/clickhouse-odbc-bridge_fresh /usr/bin/clickhouse-odbc-bridge && chmod 777 /usr/bin/clickhouse"],
                                user='root')
-        self.exec_in_container(["bash", "-c", "{} --daemon".format(CLICKHOUSE_START_COMMAND)], user=str(os.getuid()))
+        self.exec_in_container(["bash", "-c", "{} --daemon".format(self.clickhouse_start_command)], user=str(os.getuid()))
         from helpers.test_tools import assert_eq_with_retry
         # wait start
         assert_eq_with_retry(self, "select 1", "1", retry_count=retries)
@@ -1404,8 +1418,10 @@ class ClickHouseInstance:
         os.makedirs(instance_config_dir)
 
         print("Copy common default production configuration from {}".format(self.base_config_dir))
-        shutil.copyfile(p.join(self.base_config_dir, 'config.xml'), p.join(instance_config_dir, 'config.xml'))
-        shutil.copyfile(p.join(self.base_config_dir, 'users.xml'), p.join(instance_config_dir, 'users.xml'))
+
+        shutil.copyfile(p.join(self.base_config_dir, self.main_config_name), p.join(instance_config_dir, self.main_config_name))
+
+        shutil.copyfile(p.join(self.base_config_dir, self.users_config_name), p.join(instance_config_dir, self.users_config_name))
 
         print("Create directory for configuration generated in this helper")
         # used by all utils with any config
@@ -1423,7 +1439,9 @@ class ClickHouseInstance:
 
         print("Copy common configuration from helpers")
         # The file is named with 0_ prefix to be processed before other configuration overloads.
-        shutil.copy(p.join(HELPERS_DIR, '0_common_instance_config.xml'), self.config_d_dir)
+        if self.copy_common_configs:
+            shutil.copy(p.join(HELPERS_DIR, '0_common_instance_config.xml'), self.config_d_dir)
+
         shutil.copy(p.join(HELPERS_DIR, '0_common_instance_users.xml'), users_d_dir)
         if len(self.custom_dictionaries_paths):
             shutil.copy(p.join(HELPERS_DIR, '0_common_enable_dictionaries.xml'), self.config_d_dir)
@@ -1502,11 +1520,11 @@ class ClickHouseInstance:
             self._create_odbc_config_file()
             odbc_ini_path = '- ' + self.odbc_ini_path
 
-        entrypoint_cmd = CLICKHOUSE_START_COMMAND
+        entrypoint_cmd = self.clickhouse_start_command
 
         if self.stay_alive:
-            entrypoint_cmd = CLICKHOUSE_STAY_ALIVE_COMMAND
-
+            entrypoint_cmd = CLICKHOUSE_STAY_ALIVE_COMMAND.replace("{main_config_file}", self.main_config_name)
+        
         print("Entrypoint cmd: {}".format(entrypoint_cmd))
 
         networks = app_net = ipv4_address = ipv6_address = net_aliases = net_alias1 = ""
diff --git a/tests/integration/runner b/tests/integration/runner
index ee116c29aa5..16eb31dfd5e 100755
--- a/tests/integration/runner
+++ b/tests/integration/runner
@@ -70,11 +70,11 @@ def check_args_and_update_paths(args):
         if not os.path.exists(path):
             raise Exception("Path {} doesn't exist".format(path))
 
-    if not os.path.exists(os.path.join(args.base_configs_dir, "config.xml")):
-        raise Exception("No configs.xml in {}".format(args.base_configs_dir))
+    if (not os.path.exists(os.path.join(args.base_configs_dir, "config.xml"))) and (not os.path.exists(os.path.join(args.base_configs_dir, "config.yaml"))):
+        raise Exception("No configs.xml or configs.yaml in {}".format(args.base_configs_dir))
 
-    if not os.path.exists(os.path.join(args.base_configs_dir, "users.xml")):
-        raise Exception("No users.xml in {}".format(args.base_configs_dir))
+    if (not os.path.exists(os.path.join(args.base_configs_dir, "users.xml"))) and (not os.path.exists(os.path.join(args.base_configs_dir, "users.yaml"))):
+        raise Exception("No users.xml or users.yaml in {}".format(args.base_configs_dir))
 
 def docker_kill_handler_handler(signum, frame):
     subprocess.check_call('docker kill $(docker ps -a -q --filter name={name} --format="{{{{.ID}}}}")'.format(name=CONTAINER_NAME), shell=True)
diff --git a/tests/integration/test_config_xml_full/__init__.py b/tests/integration/test_config_xml_full/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/integration/test_config_xml_full/configs/config.d/access_control.xml b/tests/integration/test_config_xml_full/configs/config.d/access_control.xml
new file mode 100644
index 00000000000..6567c39f171
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/config.d/access_control.xml
@@ -0,0 +1,13 @@
+<yandex>
+    <!-- Sources to read users, roles, access rights, profiles of settings, quotas. -->
+    <user_directories replace="replace">
+        <users_xml>
+            <!-- Path to configuration file with predefined users. -->
+            <path>users.xml</path>
+        </users_xml>
+        <local_directory>
+            <!-- Path to folder where users created by SQL commands are stored. -->
+            <path>access/</path>
+        </local_directory>
+    </user_directories>
+</yandex>
diff --git a/tests/integration/test_config_xml_full/configs/config.d/keeper_port.xml b/tests/integration/test_config_xml_full/configs/config.d/keeper_port.xml
new file mode 100644
index 00000000000..b21df47bc85
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/config.d/keeper_port.xml
@@ -0,0 +1,23 @@
+<yandex>
+    <keeper_server>
+        <tcp_port>9181</tcp_port>
+        <server_id>1</server_id>
+
+        <coordination_settings>
+            <operation_timeout_ms>10000</operation_timeout_ms>
+            <session_timeout_ms>30000</session_timeout_ms>
+            <force_sync>false</force_sync>
+            <startup_timeout>60000</startup_timeout>
+            <!-- we want all logs for complex problems investigation -->
+            <reserved_log_items>1000000000000000</reserved_log_items>
+        </coordination_settings>
+
+        <raft_configuration>
+            <server>
+                <id>1</id>
+                <hostname>localhost</hostname>
+                <port>44444</port>
+            </server>
+        </raft_configuration>
+    </keeper_server>
+</yandex>
diff --git a/tests/integration/test_config_xml_full/configs/config.d/log_to_console.xml b/tests/integration/test_config_xml_full/configs/config.d/log_to_console.xml
new file mode 100644
index 00000000000..227c53647f3
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/config.d/log_to_console.xml
@@ -0,0 +1,7 @@
+<yandex>
+    <logger>
+        <console>true</console>
+        <log remove="remove"/>
+        <errorlog remove="remove"/>
+    </logger>
+</yandex>
diff --git a/tests/integration/test_config_xml_full/configs/config.d/logging_no_rotate.xml b/tests/integration/test_config_xml_full/configs/config.d/logging_no_rotate.xml
new file mode 100644
index 00000000000..2c34585437b
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/config.d/logging_no_rotate.xml
@@ -0,0 +1,8 @@
+<yandex>
+    <logger>
+        <!-- Disable rotation
+             https://pocoproject.org/docs/Poco.FileChannel.html
+        -->
+        <size>never</size>
+    </logger>
+</yandex>
diff --git a/tests/integration/test_config_xml_full/configs/config.d/macros.xml b/tests/integration/test_config_xml_full/configs/config.d/macros.xml
new file mode 100644
index 00000000000..4902b12bc81
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/config.d/macros.xml
@@ -0,0 +1,9 @@
+<yandex>
+    <macros>
+        <test>Hello, world!</test>
+        <shard>s1</shard>
+        <replica>r1</replica>
+        <default_path_test>/clickhouse/tables/{database}/{shard}/</default_path_test>
+        <default_name_test>table_{table}</default_name_test>
+    </macros>
+</yandex>
diff --git a/tests/integration/test_config_xml_full/configs/config.d/metric_log.xml b/tests/integration/test_config_xml_full/configs/config.d/metric_log.xml
new file mode 100644
index 00000000000..0ca9f162416
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/config.d/metric_log.xml
@@ -0,0 +1,8 @@
+<yandex>
+    <metric_log>
+        <database>system</database>
+        <table>metric_log</table>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+        <collect_interval_milliseconds>1000</collect_interval_milliseconds>
+    </metric_log>
+</yandex>
diff --git a/tests/integration/test_config_xml_full/configs/config.d/more_clusters.xml b/tests/integration/test_config_xml_full/configs/config.d/more_clusters.xml
new file mode 100644
index 00000000000..aecbf9e0ba7
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/config.d/more_clusters.xml
@@ -0,0 +1,49 @@
+<yandex>
+    <remote_servers>
+
+        <![CDATA[
+            You can run additional servers simply as
+             ./clickhouse-server -- --path=9001 --tcp_port=9001
+        ]]>
+
+        <single_remote_shard_at_port_9001>
+            <shard>
+                <replica>
+                    <host>localhost</host>
+                    <port>9001</port>
+                </replica>
+            </shard>
+        </single_remote_shard_at_port_9001>
+
+        <two_remote_shards_at_port_9001_9002>
+            <shard>
+                <replica>
+                    <host>localhost</host>
+                    <port>9001</port>
+                </replica>
+            </shard>
+            <shard>
+                <replica>
+                    <host>localhost</host>
+                    <port>9002</port>
+                </replica>
+            </shard>
+        </two_remote_shards_at_port_9001_9002>
+
+        <two_shards_one_local_one_remote_at_port_9001>
+            <shard>
+                <replica>
+                    <host>localhost</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+            <shard>
+                <replica>
+                    <host>localhost</host>
+                    <port>9001</port>
+                </replica>
+            </shard>
+        </two_shards_one_local_one_remote_at_port_9001>
+
+    </remote_servers>
+</yandex>
diff --git a/tests/integration/test_config_xml_full/configs/config.d/part_log.xml b/tests/integration/test_config_xml_full/configs/config.d/part_log.xml
new file mode 100644
index 00000000000..6c6fc9c6982
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/config.d/part_log.xml
@@ -0,0 +1,8 @@
+<yandex>
+    <part_log>
+        <database>system</database>
+        <table>part_log</table>
+
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </part_log>
+</yandex>
diff --git a/tests/integration/test_config_xml_full/configs/config.d/path.xml b/tests/integration/test_config_xml_full/configs/config.d/path.xml
new file mode 100644
index 00000000000..466ed0d1663
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/config.d/path.xml
@@ -0,0 +1,8 @@
+<yandex>
+    <path replace="replace">./</path>
+    <tmp_path replace="replace">./tmp/</tmp_path>
+    <user_files_path replace="replace">./user_files/</user_files_path>
+    <format_schema_path replace="replace">./format_schemas/</format_schema_path>
+    <access_control_path replace="replace">./access/</access_control_path>
+    <top_level_domains_path replace="replace">./top_level_domains/</top_level_domains_path>
+</yandex>
diff --git a/tests/integration/test_config_xml_full/configs/config.d/query_masking_rules.xml b/tests/integration/test_config_xml_full/configs/config.d/query_masking_rules.xml
new file mode 100644
index 00000000000..5a854848f3d
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/config.d/query_masking_rules.xml
@@ -0,0 +1,10 @@
+<?xml version="1.0"?>
+<!-- Config for test server -->
+<yandex>
+    <query_masking_rules>
+        <rule>
+            <regexp>TOPSECRET.TOPSECRET</regexp>
+            <replace>[hidden]</replace>
+        </rule>
+    </query_masking_rules>
+</yandex>
diff --git a/tests/integration/test_config_xml_full/configs/config.d/tcp_with_proxy.xml b/tests/integration/test_config_xml_full/configs/config.d/tcp_with_proxy.xml
new file mode 100644
index 00000000000..19046054c16
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/config.d/tcp_with_proxy.xml
@@ -0,0 +1,3 @@
+<yandex>
+    <tcp_with_proxy_port>9010</tcp_with_proxy_port>
+</yandex>
diff --git a/tests/integration/test_config_xml_full/configs/config.d/text_log.xml b/tests/integration/test_config_xml_full/configs/config.d/text_log.xml
new file mode 100644
index 00000000000..3699a23578c
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/config.d/text_log.xml
@@ -0,0 +1,7 @@
+<yandex>
+    <text_log>
+        <database>system</database>
+        <table>text_log</table>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </text_log>
+</yandex>
diff --git a/tests/integration/test_config_xml_full/configs/config.d/zookeeper.xml b/tests/integration/test_config_xml_full/configs/config.d/zookeeper.xml
new file mode 100644
index 00000000000..06ed7fcd39f
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/config.d/zookeeper.xml
@@ -0,0 +1,8 @@
+<yandex>
+    <zookeeper>
+        <node index="1">
+            <host>localhost</host>
+            <port>9181</port>
+        </node>
+    </zookeeper>
+</yandex>
diff --git a/tests/integration/test_config_xml_full/configs/config.xml b/tests/integration/test_config_xml_full/configs/config.xml
new file mode 100644
index 00000000000..a7bb9d49e95
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/config.xml
@@ -0,0 +1,1118 @@
+<?xml version="1.0"?>
+<!--
+  NOTE: User and query level settings are set up in "users.xml" file.
+  If you have accidentally specified user-level settings here, server won't start.
+  You can either move the settings to the right place inside "users.xml" file
+   or add <skip_check_for_incorrect_settings>1</skip_check_for_incorrect_settings> here.
+-->
+<yandex>
+    <logger>
+        <!-- Possible levels [1]:
+
+          - none (turns off logging)
+          - fatal
+          - critical
+          - error
+          - warning
+          - notice
+          - information
+          - debug
+          - trace
+
+            [1]: https://github.com/pocoproject/poco/blob/poco-1.9.4-release/Foundation/include/Poco/Logger.h#L105-L114
+        -->
+        <level>trace</level>
+        <log>/var/log/clickhouse-server/clickhouse-server.log</log>
+        <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
+        <!-- Rotation policy
+             See https://github.com/pocoproject/poco/blob/poco-1.9.4-release/Foundation/include/Poco/FileChannel.h#L54-L85
+          -->
+        <size>1000M</size>
+        <count>10</count>
+        <!-- <console>1</console> --> <!-- Default behavior is autodetection (log to console if not daemon mode and is tty) -->
+
+        <!-- Per level overrides (legacy):
+
+        For example to suppress logging of the ConfigReloader you can use:
+        NOTE: levels.logger is reserved, see below.
+        -->
+        <!--
+        <levels>
+          <ConfigReloader>none</ConfigReloader>
+        </levels>
+        -->
+
+        <!-- Per level overrides:
+
+        For example to suppress logging of the RBAC for default user you can use:
+        (But please note that the logger name maybe changed from version to version, even after minor upgrade)
+        -->
+        <!--
+        <levels>
+          <logger>
+            <name>ContextAccess (default)</name>
+            <level>none</level>
+          </logger>
+          <logger>
+            <name>DatabaseOrdinary (test)</name>
+            <level>none</level>
+          </logger>
+        </levels>
+        -->
+    </logger>
+
+    <!-- It is the name that will be shown in the clickhouse-client.
+         By default, anything with "production" will be highlighted in red in query prompt.
+    -->
+    <!--display_name>production</display_name-->
+
+    <!-- Port for HTTP API. See also 'https_port' for secure connections.
+         This interface is also used by ODBC and JDBC drivers (DataGrip, Dbeaver, ...)
+         and by most of web interfaces (embedded UI, Grafana, Redash, ...).
+      -->
+    <http_port>8123</http_port>
+
+    <!-- Port for interaction by native protocol with:
+         - clickhouse-client and other native ClickHouse tools (clickhouse-benchmark, clickhouse-copier);
+         - clickhouse-server with other clickhouse-servers for distributed query processing;
+         - ClickHouse drivers and applications supporting native protocol
+         (this protocol is also informally called as "the TCP protocol");
+         See also 'tcp_port_secure' for secure connections.
+    -->
+    <tcp_port>9000</tcp_port>
+
+    <!-- Compatibility with MySQL protocol.
+         ClickHouse will pretend to be MySQL for applications connecting to this port.
+    -->
+    <mysql_port>9004</mysql_port>
+
+    <!-- Compatibility with PostgreSQL protocol.
+         ClickHouse will pretend to be PostgreSQL for applications connecting to this port.
+    -->
+    <postgresql_port>9005</postgresql_port>
+
+    <!-- HTTP API with TLS (HTTPS).
+         You have to configure certificate to enable this interface.
+         See the openSSL section below.
+    -->
+    <!-- <https_port>8443</https_port> -->
+
+    <!-- Native interface with TLS.
+         You have to configure certificate to enable this interface.
+         See the openSSL section below.
+    -->
+    <!-- <tcp_port_secure>9440</tcp_port_secure> -->
+
+    <!-- Native interface wrapped with PROXYv1 protocol
+         PROXYv1 header sent for every connection.
+         ClickHouse will extract information about proxy-forwarded client address from the header.
+    -->
+    <!-- <tcp_with_proxy_port>9011</tcp_with_proxy_port> -->
+
+    <!-- Port for communication between replicas. Used for data exchange.
+         It provides low-level data access between servers.
+         This port should not be accessible from untrusted networks.
+         See also 'interserver_http_credentials'.
+         Data transferred over connections to this port should not go through untrusted networks.
+         See also 'interserver_https_port'.
+      -->
+    <interserver_http_port>9009</interserver_http_port>
+
+    <!-- Port for communication between replicas with TLS.
+         You have to configure certificate to enable this interface.
+         See the openSSL section below.
+         See also 'interserver_http_credentials'.
+      -->
+    <!-- <interserver_https_port>9010</interserver_https_port> -->
+
+    <!-- Hostname that is used by other replicas to request this server.
+         If not specified, than it is determined analogous to 'hostname -f' command.
+         This setting could be used to switch replication to another network interface
+         (the server may be connected to multiple networks via multiple addresses)
+      -->
+    <!--
+    <interserver_http_host>example.yandex.ru</interserver_http_host>
+    -->
+
+    <!-- You can specify credentials for authenthication between replicas.
+         This is required when interserver_https_port is accessible from untrusted networks,
+         and also recommended to avoid SSRF attacks from possibly compromised services in your network.
+      -->
+    <!--<interserver_http_credentials>
+        <user>interserver</user>
+        <password></password>
+    </interserver_http_credentials>-->
+
+    <!-- Listen specified address.
+         Use :: (wildcard IPv6 address), if you want to accept connections both with IPv4 and IPv6 from everywhere.
+         Notes:
+         If you open connections from wildcard address, make sure that at least one of the following measures applied:
+         - server is protected by firewall and not accessible from untrusted networks;
+         - all users are restricted to subset of network addresses (see users.xml);
+         - all users have strong passwords, only secure (TLS) interfaces are accessible, or connections are only made via TLS interfaces.
+         - users without password have readonly access.
+         See also: https://www.shodan.io/search?query=clickhouse
+      -->
+    <!-- <listen_host>::</listen_host> -->
+
+    <!-- Same for hosts without support for IPv6: -->
+    <!-- <listen_host>0.0.0.0</listen_host> -->
+
+    <!-- Default values - try listen localhost on IPv4 and IPv6. -->
+    <!--
+    <listen_host>::1</listen_host>
+    <listen_host>127.0.0.1</listen_host>
+    -->
+
+    <!-- Don't exit if IPv6 or IPv4 networks are unavailable while trying to listen. -->
+    <!-- <listen_try>0</listen_try> -->
+
+    <!-- Allow multiple servers to listen on the same address:port. This is not recommended.
+      -->
+    <!-- <listen_reuse_port>0</listen_reuse_port> -->
+
+    <!-- <listen_backlog>64</listen_backlog> -->
+
+    <max_connections>4096</max_connections>
+
+    <!-- For 'Connection: keep-alive' in HTTP 1.1 -->
+    <keep_alive_timeout>3</keep_alive_timeout>
+
+    <!-- gRPC protocol (see src/Server/grpc_protos/clickhouse_grpc.proto for the API) -->
+    <!-- <grpc_port>9100</grpc_port> -->
+    <grpc>
+        <enable_ssl>false</enable_ssl>
+
+        <!-- The following two files are used only if enable_ssl=1 -->
+        <ssl_cert_file>/path/to/ssl_cert_file</ssl_cert_file>
+        <ssl_key_file>/path/to/ssl_key_file</ssl_key_file>
+
+        <!-- Whether server will request client for a certificate -->
+        <ssl_require_client_auth>false</ssl_require_client_auth>
+
+        <!-- The following file is used only if ssl_require_client_auth=1 -->
+        <ssl_ca_cert_file>/path/to/ssl_ca_cert_file</ssl_ca_cert_file>
+
+        <!-- Default compression algorithm (applied if client doesn't specify another algorithm).
+             Supported algorithms: none, deflate, gzip, stream_gzip -->
+        <compression>deflate</compression>
+
+        <!-- Default compression level (applied if client doesn't specify another level).
+             Supported levels: none, low, medium, high -->
+        <compression_level>medium</compression_level>
+
+        <!-- Send/receive message size limits in bytes. -1 means unlimited -->
+        <max_send_message_size>-1</max_send_message_size>
+        <max_receive_message_size>-1</max_receive_message_size>
+
+        <!-- Enable if you want very detailed logs -->
+        <verbose_logs>false</verbose_logs>
+    </grpc>
+
+    <!-- Used with https_port and tcp_port_secure. Full ssl options list: https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h#L71 -->
+    <openSSL>
+        <server> <!-- Used for https server AND secure tcp port -->
+            <!-- openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout /etc/clickhouse-server/server.key -out /etc/clickhouse-server/server.crt -->
+            <certificateFile>/etc/clickhouse-server/server.crt</certificateFile>
+            <privateKeyFile>/etc/clickhouse-server/server.key</privateKeyFile>
+            <!-- dhparams are optional. You can delete the <dhParamsFile> element.
+                 To generate dhparams, use the following command:
+                  openssl dhparam -out /etc/clickhouse-server/dhparam.pem 4096
+                 Only file format with BEGIN DH PARAMETERS is supported.
+              -->
+            <dhParamsFile>/etc/clickhouse-server/dhparam.pem</dhParamsFile>
+            <verificationMode>none</verificationMode>
+            <loadDefaultCAFile>true</loadDefaultCAFile>
+            <cacheSessions>true</cacheSessions>
+            <disableProtocols>sslv2,sslv3</disableProtocols>
+            <preferServerCiphers>true</preferServerCiphers>
+        </server>
+
+        <client> <!-- Used for connecting to https dictionary source and secured Zookeeper communication -->
+            <loadDefaultCAFile>true</loadDefaultCAFile>
+            <cacheSessions>true</cacheSessions>
+            <disableProtocols>sslv2,sslv3</disableProtocols>
+            <preferServerCiphers>true</preferServerCiphers>
+            <!-- Use for self-signed: <verificationMode>none</verificationMode> -->
+            <invalidCertificateHandler>
+                <!-- Use for self-signed: <name>AcceptCertificateHandler</name> -->
+                <name>RejectCertificateHandler</name>
+            </invalidCertificateHandler>
+        </client>
+    </openSSL>
+
+    <!-- Default root page on http[s] server. For example load UI from https://tabix.io/ when opening http://localhost:8123 -->
+    <!--
+    <http_server_default_response><![CDATA[<html ng-app="SMI2"><head><base href="http://ui.tabix.io/"></head><body><div ui-view="" class="content-ui"></div><script src="http://loader.tabix.io/master.js"></script></body></html>]]></http_server_default_response>
+    -->
+
+    <!-- Maximum number of concurrent queries. -->
+    <max_concurrent_queries>100</max_concurrent_queries>
+
+    <!-- Maximum memory usage (resident set size) for server process.
+         Zero value or unset means default. Default is "max_server_memory_usage_to_ram_ratio" of available physical RAM.
+         If the value is larger than "max_server_memory_usage_to_ram_ratio" of available physical RAM, it will be cut down.
+
+         The constraint is checked on query execution time.
+         If a query tries to allocate memory and the current memory usage plus allocation is greater
+          than specified threshold, exception will be thrown.
+
+         It is not practical to set this constraint to small values like just a few gigabytes,
+          because memory allocator will keep this amount of memory in caches and the server will deny service of queries.
+      -->
+    <max_server_memory_usage>0</max_server_memory_usage>
+
+    <!-- Maximum number of threads in the Global thread pool.
+    This will default to a maximum of 10000 threads if not specified.
+    This setting will be useful in scenarios where there are a large number
+    of distributed queries that are running concurrently but are idling most
+    of the time, in which case a higher number of threads might be required.
+    -->
+
+    <max_thread_pool_size>10000</max_thread_pool_size>
+
+    <!-- On memory constrained environments you may have to set this to value larger than 1.
+      -->
+    <max_server_memory_usage_to_ram_ratio>0.9</max_server_memory_usage_to_ram_ratio>
+
+    <!-- Simple server-wide memory profiler. Collect a stack trace at every peak allocation step (in bytes).
+         Data will be stored in system.trace_log table with query_id = empty string.
+         Zero means disabled.
+      -->
+    <total_memory_profiler_step>4194304</total_memory_profiler_step>
+
+    <!-- Collect random allocations and deallocations and write them into system.trace_log with 'MemorySample' trace_type.
+         The probability is for every alloc/free regardless to the size of the allocation.
+         Note that sampling happens only when the amount of untracked memory exceeds the untracked memory limit,
+          which is 4 MiB by default but can be lowered if 'total_memory_profiler_step' is lowered.
+         You may want to set 'total_memory_profiler_step' to 1 for extra fine grained sampling.
+      -->
+    <total_memory_tracker_sample_probability>0</total_memory_tracker_sample_probability>
+
+    <!-- Set limit on number of open files (default: maximum). This setting makes sense on Mac OS X because getrlimit() fails to retrieve
+         correct maximum value. -->
+    <!-- <max_open_files>262144</max_open_files> -->
+
+    <!-- Size of cache of uncompressed blocks of data, used in tables of MergeTree family.
+         In bytes. Cache is single for server. Memory is allocated only on demand.
+         Cache is used when 'use_uncompressed_cache' user setting turned on (off by default).
+         Uncompressed cache is advantageous only for very short queries and in rare cases.
+
+         Note: uncompressed cache can be pointless for lz4, because memory bandwidth
+         is slower than multi-core decompression on some server configurations.
+         Enabling it can sometimes paradoxically make queries slower.
+      -->
+    <uncompressed_cache_size>8589934592</uncompressed_cache_size>
+
+    <!-- Approximate size of mark cache, used in tables of MergeTree family.
+         In bytes. Cache is single for server. Memory is allocated only on demand.
+         You should not lower this value.
+      -->
+    <mark_cache_size>5368709120</mark_cache_size>
+
+
+    <!-- If you enable the `min_bytes_to_use_mmap_io` setting,
+         the data in MergeTree tables can be read with mmap to avoid copying from kernel to userspace.
+         It makes sense only for large files and helps only if data reside in page cache.
+         To avoid frequent open/mmap/munmap/close calls (which are very expensive due to consequent page faults)
+         and to reuse mappings from several threads and queries,
+         the cache of mapped files is maintained. Its size is the number of mapped regions (usually equal to the number of mapped files).
+         The amount of data in mapped files can be monitored
+         in system.metrics, system.metric_log by the MMappedFiles, MMappedFileBytes metrics
+         and in system.asynchronous_metrics, system.asynchronous_metrics_log by the MMapCacheCells metric,
+         and also in system.events, system.processes, system.query_log, system.query_thread_log by the
+         CreatedReadBufferMMap, CreatedReadBufferMMapFailed, MMappedFileCacheHits, MMappedFileCacheMisses events.
+         Note that the amount of data in mapped files does not consume memory directly and is not accounted
+         in query or server memory usage - because this memory can be discarded similar to OS page cache.
+         The cache is dropped (the files are closed) automatically on removal of old parts in MergeTree,
+         also it can be dropped manually by the SYSTEM DROP MMAP CACHE query.
+      -->
+    <mmap_cache_size>1000</mmap_cache_size>
+
+
+    <!-- Path to data directory, with trailing slash. -->
+    <path>/var/lib/clickhouse/</path>
+
+    <!-- Path to temporary data for processing hard queries. -->
+    <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
+
+    <!-- Policy from the <storage_configuration> for the temporary files.
+         If not set <tmp_path> is used, otherwise <tmp_path> is ignored.
+
+         Notes:
+         - move_factor              is ignored
+         - keep_free_space_bytes    is ignored
+         - max_data_part_size_bytes is ignored
+         - you must have exactly one volume in that policy
+    -->
+    <!-- <tmp_policy>tmp</tmp_policy> -->
+
+    <!-- Directory with user provided files that are accessible by 'file' table function. -->
+    <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
+
+    <!-- LDAP server definitions. -->
+    <ldap_servers>
+        <!-- List LDAP servers with their connection parameters here to later 1) use them as authenticators for dedicated local users,
+              who have 'ldap' authentication mechanism specified instead of 'password', or to 2) use them as remote user directories.
+             Parameters:
+                host - LDAP server hostname or IP, this parameter is mandatory and cannot be empty.
+                port - LDAP server port, default is 636 if enable_tls is set to true, 389 otherwise.
+                bind_dn - template used to construct the DN to bind to.
+                        The resulting DN will be constructed by replacing all '{user_name}' substrings of the template with the actual
+                         user name during each authentication attempt.
+                verification_cooldown - a period of time, in seconds, after a successful bind attempt, during which a user will be assumed
+                         to be successfully authenticated for all consecutive requests without contacting the LDAP server.
+                        Specify 0 (the default) to disable caching and force contacting the LDAP server for each authentication request.
+                enable_tls - flag to trigger use of secure connection to the LDAP server.
+                        Specify 'no' for plain text (ldap://) protocol (not recommended).
+                        Specify 'yes' for LDAP over SSL/TLS (ldaps://) protocol (recommended, the default).
+                        Specify 'starttls' for legacy StartTLS protocol (plain text (ldap://) protocol, upgraded to TLS).
+                tls_minimum_protocol_version - the minimum protocol version of SSL/TLS.
+                        Accepted values are: 'ssl2', 'ssl3', 'tls1.0', 'tls1.1', 'tls1.2' (the default).
+                tls_require_cert - SSL/TLS peer certificate verification behavior.
+                        Accepted values are: 'never', 'allow', 'try', 'demand' (the default).
+                tls_cert_file - path to certificate file.
+                tls_key_file - path to certificate key file.
+                tls_ca_cert_file - path to CA certificate file.
+                tls_ca_cert_dir - path to the directory containing CA certificates.
+                tls_cipher_suite - allowed cipher suite (in OpenSSL notation).
+             Example:
+                <my_ldap_server>
+                    <host>localhost</host>
+                    <port>636</port>
+                    <bind_dn>uid={user_name},ou=users,dc=example,dc=com</bind_dn>
+                    <verification_cooldown>300</verification_cooldown>
+                    <enable_tls>yes</enable_tls>
+                    <tls_minimum_protocol_version>tls1.2</tls_minimum_protocol_version>
+                    <tls_require_cert>demand</tls_require_cert>
+                    <tls_cert_file>/path/to/tls_cert_file</tls_cert_file>
+                    <tls_key_file>/path/to/tls_key_file</tls_key_file>
+                    <tls_ca_cert_file>/path/to/tls_ca_cert_file</tls_ca_cert_file>
+                    <tls_ca_cert_dir>/path/to/tls_ca_cert_dir</tls_ca_cert_dir>
+                    <tls_cipher_suite>ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384</tls_cipher_suite>
+                </my_ldap_server>
+        -->
+    </ldap_servers>
+
+    <!-- To enable Kerberos authentication support for HTTP requests (GSS-SPNEGO), for those users who are explicitly configured
+          to authenticate via Kerberos, define a single 'kerberos' section here.
+         Parameters:
+            principal - canonical service principal name, that will be acquired and used when accepting security contexts.
+                    This parameter is optional, if omitted, the default principal will be used.
+                    This parameter cannot be specified together with 'realm' parameter.
+            realm - a realm, that will be used to restrict authentication to only those requests whose initiator's realm matches it.
+                    This parameter is optional, if omitted, no additional filtering by realm will be applied.
+                    This parameter cannot be specified together with 'principal' parameter.
+         Example:
+            <kerberos />
+         Example:
+            <kerberos>
+                <principal>HTTP/clickhouse.example.com@EXAMPLE.COM</principal>
+            </kerberos>
+         Example:
+            <kerberos>
+                <realm>EXAMPLE.COM</realm>
+            </kerberos>
+    -->
+
+    <!-- Sources to read users, roles, access rights, profiles of settings, quotas. -->
+    <user_directories>
+        <users_xml>
+            <!-- Path to configuration file with predefined users. -->
+            <path>users.xml</path>
+        </users_xml>
+        <local_directory>
+            <!-- Path to folder where users created by SQL commands are stored. -->
+            <path>/var/lib/clickhouse/access/</path>
+        </local_directory>
+
+        <!-- To add an LDAP server as a remote user directory of users that are not defined locally, define a single 'ldap' section
+              with the following parameters:
+                server - one of LDAP server names defined in 'ldap_servers' config section above.
+                        This parameter is mandatory and cannot be empty.
+                roles - section with a list of locally defined roles that will be assigned to each user retrieved from the LDAP server.
+                        If no roles are specified here or assigned during role mapping (below), user will not be able to perform any
+                         actions after authentication.
+                role_mapping - section with LDAP search parameters and mapping rules.
+                        When a user authenticates, while still bound to LDAP, an LDAP search is performed using search_filter and the
+                         name of the logged in user. For each entry found during that search, the value of the specified attribute is
+                         extracted. For each attribute value that has the specified prefix, the prefix is removed, and the rest of the
+                         value becomes the name of a local role defined in ClickHouse, which is expected to be created beforehand by
+                         CREATE ROLE command.
+                        There can be multiple 'role_mapping' sections defined inside the same 'ldap' section. All of them will be
+                         applied.
+                    base_dn - template used to construct the base DN for the LDAP search.
+                            The resulting DN will be constructed by replacing all '{user_name}' and '{bind_dn}' substrings
+                             of the template with the actual user name and bind DN during each LDAP search.
+                    scope - scope of the LDAP search.
+                            Accepted values are: 'base', 'one_level', 'children', 'subtree' (the default).
+                    search_filter - template used to construct the search filter for the LDAP search.
+                            The resulting filter will be constructed by replacing all '{user_name}', '{bind_dn}', and '{base_dn}'
+                             substrings of the template with the actual user name, bind DN, and base DN during each LDAP search.
+                            Note, that the special characters must be escaped properly in XML.
+                    attribute - attribute name whose values will be returned by the LDAP search.
+                    prefix - prefix, that will be expected to be in front of each string in the original list of strings returned by
+                             the LDAP search. Prefix will be removed from the original strings and resulting strings will be treated
+                             as local role names. Empty, by default.
+             Example:
+                <ldap>
+                    <server>my_ldap_server</server>
+                    <roles>
+                        <my_local_role1 />
+                        <my_local_role2 />
+                    </roles>
+                    <role_mapping>
+                        <base_dn>ou=groups,dc=example,dc=com</base_dn>
+                        <scope>subtree</scope>
+                        <search_filter>(&amp;(objectClass=groupOfNames)(member={bind_dn}))</search_filter>
+                        <attribute>cn</attribute>
+                        <prefix>clickhouse_</prefix>
+                    </role_mapping>
+                </ldap>
+        -->
+    </user_directories>
+
+    <!-- Default profile of settings. -->
+    <default_profile>default</default_profile>
+
+    <!-- Comma-separated list of prefixes for user-defined settings. -->
+    <custom_settings_prefixes></custom_settings_prefixes>
+
+    <!-- System profile of settings. This settings are used by internal processes (Distributed DDL worker and so on). -->
+    <!-- <system_profile>default</system_profile> -->
+
+    <!-- Buffer profile of settings.
+         This settings are used by Buffer storage to flush data to the underlying table.
+         Default: used from system_profile directive.
+    -->
+    <!-- <buffer_profile>default</buffer_profile> -->
+
+    <!-- Default database. -->
+    <default_database>default</default_database>
+
+    <!-- Server time zone could be set here.
+
+         Time zone is used when converting between String and DateTime types,
+          when printing DateTime in text formats and parsing DateTime from text,
+          it is used in date and time related functions, if specific time zone was not passed as an argument.
+
+         Time zone is specified as identifier from IANA time zone database, like UTC or Africa/Abidjan.
+         If not specified, system time zone at server startup is used.
+
+         Please note, that server could display time zone alias instead of specified name.
+         Example: W-SU is an alias for Europe/Moscow and Zulu is an alias for UTC.
+    -->
+    <!-- <timezone>Europe/Moscow</timezone> -->
+
+    <!-- You can specify umask here (see "man umask"). Server will apply it on startup.
+         Number is always parsed as octal. Default umask is 027 (other users cannot read logs, data files, etc; group can only read).
+    -->
+    <!-- <umask>022</umask> -->
+
+    <!-- Perform mlockall after startup to lower first queries latency
+          and to prevent clickhouse executable from being paged out under high IO load.
+         Enabling this option is recommended but will lead to increased startup time for up to a few seconds.
+    -->
+    <mlock_executable>true</mlock_executable>
+
+    <!-- Reallocate memory for machine code ("text") using huge pages. Highly experimental. -->
+    <remap_executable>false</remap_executable>
+
+    <![CDATA[
+         Uncomment below in order to use JDBC table engine and function.
+
+         To install and run JDBC bridge in background:
+         * [Debian/Ubuntu]
+           export MVN_URL=https://repo1.maven.org/maven2/ru/yandex/clickhouse/clickhouse-jdbc-bridge
+           export PKG_VER=$(curl -sL $MVN_URL/maven-metadata.xml | grep '<release>' | sed -e 's|.*>\(.*\)<.*|\1|')
+           wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge_$PKG_VER-1_all.deb
+           apt install --no-install-recommends -f ./clickhouse-jdbc-bridge_$PKG_VER-1_all.deb
+           clickhouse-jdbc-bridge &
+
+         * [CentOS/RHEL]
+           export MVN_URL=https://repo1.maven.org/maven2/ru/yandex/clickhouse/clickhouse-jdbc-bridge
+           export PKG_VER=$(curl -sL $MVN_URL/maven-metadata.xml | grep '<release>' | sed -e 's|.*>\(.*\)<.*|\1|')
+           wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm
+           yum localinstall -y clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm
+           clickhouse-jdbc-bridge &
+
+         Please refer to https://github.com/ClickHouse/clickhouse-jdbc-bridge#usage for more information.
+    ]]>
+    <!--
+    <jdbc_bridge>
+        <host>127.0.0.1</host>
+        <port>9019</port>
+    </jdbc_bridge>
+    -->
+  
+    <!-- Configuration of clusters that could be used in Distributed tables.
+         https://clickhouse.tech/docs/en/operations/table_engines/distributed/
+      -->
+    <remote_servers>
+        <!-- Test only shard config for testing distributed storage -->
+        <test_shard_localhost>
+            <!-- Inter-server per-cluster secret for Distributed queries
+                 default: no secret (no authentication will be performed)
+
+                 If set, then Distributed queries will be validated on shards, so at least:
+                 - such cluster should exist on the shard,
+                 - such cluster should have the same secret.
+
+                 And also (and which is more important), the initial_user will
+                 be used as current user for the query.
+
+                 Right now the protocol is pretty simple and it only takes into account:
+                 - cluster name
+                 - query
+
+                 Also it will be nice if the following will be implemented:
+                 - source hostname (see interserver_http_host), but then it will depends from DNS,
+                   it can use IP address instead, but then the you need to get correct on the initiator node.
+                 - target hostname / ip address (same notes as for source hostname)
+                 - time-based security tokens
+            -->
+            <!-- <secret></secret> -->
+
+            <shard>
+                <!-- Optional. Whether to write data to just one of the replicas. Default: false (write data to all replicas). -->
+                <!-- <internal_replication>false</internal_replication> -->
+                <!-- Optional. Shard weight when writing data. Default: 1. -->
+                <!-- <weight>1</weight> -->
+                <replica>
+                    <host>localhost</host>
+                    <port>9000</port>
+                    <!-- Optional. Priority of the replica for load_balancing. Default: 1 (less value has more priority). -->
+                    <!-- <priority>1</priority> -->
+                </replica>
+            </shard>
+        </test_shard_localhost>
+        <test_cluster_two_shards_localhost>
+             <shard>
+                 <replica>
+                     <host>localhost</host>
+                     <port>9000</port>
+                 </replica>
+             </shard>
+             <shard>
+                 <replica>
+                     <host>localhost</host>
+                     <port>9000</port>
+                 </replica>
+             </shard>
+        </test_cluster_two_shards_localhost>
+        <test_cluster_two_shards>
+            <shard>
+                <replica>
+                    <host>127.0.0.1</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+            <shard>
+                <replica>
+                    <host>127.0.0.2</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </test_cluster_two_shards>
+        <test_cluster_two_shards_internal_replication>
+            <shard>
+                <internal_replication>true</internal_replication>
+                <replica>
+                    <host>127.0.0.1</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+            <shard>
+                <internal_replication>true</internal_replication>
+                <replica>
+                    <host>127.0.0.2</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </test_cluster_two_shards_internal_replication>
+        <test_shard_localhost_secure>
+            <shard>
+                <replica>
+                    <host>localhost</host>
+                    <port>9440</port>
+                    <secure>1</secure>
+                </replica>
+            </shard>
+        </test_shard_localhost_secure>
+        <test_unavailable_shard>
+            <shard>
+                <replica>
+                    <host>localhost</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+            <shard>
+                <replica>
+                    <host>localhost</host>
+                    <port>1</port>
+                </replica>
+            </shard>
+        </test_unavailable_shard>
+    </remote_servers>
+
+    <!-- The list of hosts allowed to use in URL-related storage engines and table functions.
+        If this section is not present in configuration, all hosts are allowed.
+    -->
+    <!--<remote_url_allow_hosts>-->
+        <!-- Host should be specified exactly as in URL. The name is checked before DNS resolution.
+            Example: "yandex.ru", "yandex.ru." and "www.yandex.ru" are different hosts.
+                    If port is explicitly specified in URL, the host:port is checked as a whole.
+                    If host specified here without port, any port with this host allowed.
+                    "yandex.ru" -> "yandex.ru:443", "yandex.ru:80" etc. is allowed, but "yandex.ru:80" -> only "yandex.ru:80" is allowed.
+            If the host is specified as IP address, it is checked as specified in URL. Example: "[2a02:6b8:a::a]".
+            If there are redirects and support for redirects is enabled, every redirect (the Location field) is checked.
+        -->
+
+        <!-- Regular expression can be specified. RE2 engine is used for regexps.
+            Regexps are not aligned: don't forget to add ^ and $. Also don't forget to escape dot (.) metacharacter
+            (forgetting to do so is a common source of error).
+        -->
+    <!--</remote_url_allow_hosts>-->
+
+    <!-- If element has 'incl' attribute, then for it's value will be used corresponding substitution from another file.
+         By default, path to file with substitutions is /etc/metrika.xml. It could be changed in config in 'include_from' element.
+         Values for substitutions are specified in /yandex/name_of_substitution elements in that file.
+      -->
+
+    <!-- ZooKeeper is used to store metadata about replicas, when using Replicated tables.
+         Optional. If you don't use replicated tables, you could omit that.
+
+         See https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/replication/
+      -->
+
+    <!--
+    <zookeeper>
+        <node>
+            <host>example1</host>
+            <port>2181</port>
+        </node>
+        <node>
+            <host>example2</host>
+            <port>2181</port>
+        </node>
+        <node>
+            <host>example3</host>
+            <port>2181</port>
+        </node>
+    </zookeeper>
+    -->
+
+    <!-- Substitutions for parameters of replicated tables.
+          Optional. If you don't use replicated tables, you could omit that.
+
+         See https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/replication/#creating-replicated-tables
+      -->
+    <!--
+    <macros>
+        <shard>01</shard>
+        <replica>example01-01-1</replica>
+    </macros>
+    -->
+
+
+    <!-- Reloading interval for embedded dictionaries, in seconds. Default: 3600. -->
+    <builtin_dictionaries_reload_interval>3600</builtin_dictionaries_reload_interval>
+
+
+    <!-- Maximum session timeout, in seconds. Default: 3600. -->
+    <max_session_timeout>3600</max_session_timeout>
+
+    <!-- Default session timeout, in seconds. Default: 60. -->
+    <default_session_timeout>60</default_session_timeout>
+
+    <!-- Sending data to Graphite for monitoring. Several sections can be defined. -->
+    <!--
+        interval - send every X second
+        root_path - prefix for keys
+        hostname_in_path - append hostname to root_path (default = true)
+        metrics - send data from table system.metrics
+        events - send data from table system.events
+        asynchronous_metrics - send data from table system.asynchronous_metrics
+    -->
+    <!--
+    <graphite>
+        <host>localhost</host>
+        <port>42000</port>
+        <timeout>0.1</timeout>
+        <interval>60</interval>
+        <root_path>one_min</root_path>
+        <hostname_in_path>true</hostname_in_path>
+
+        <metrics>true</metrics>
+        <events>true</events>
+        <events_cumulative>false</events_cumulative>
+        <asynchronous_metrics>true</asynchronous_metrics>
+    </graphite>
+    <graphite>
+        <host>localhost</host>
+        <port>42000</port>
+        <timeout>0.1</timeout>
+        <interval>1</interval>
+        <root_path>one_sec</root_path>
+
+        <metrics>true</metrics>
+        <events>true</events>
+        <events_cumulative>false</events_cumulative>
+        <asynchronous_metrics>false</asynchronous_metrics>
+    </graphite>
+    -->
+
+    <!-- Serve endpoint for Prometheus monitoring. -->
+    <!--
+        endpoint - mertics path (relative to root, statring with "/")
+        port - port to setup server. If not defined or 0 than http_port used
+        metrics - send data from table system.metrics
+        events - send data from table system.events
+        asynchronous_metrics - send data from table system.asynchronous_metrics
+        status_info - send data from different component from CH, ex: Dictionaries status
+    -->
+    <!--
+    <prometheus>
+        <endpoint>/metrics</endpoint>
+        <port>9363</port>
+
+        <metrics>true</metrics>
+        <events>true</events>
+        <asynchronous_metrics>true</asynchronous_metrics>
+        <status_info>true</status_info>
+    </prometheus>
+    -->
+
+    <!-- Query log. Used only for queries with setting log_queries = 1. -->
+    <query_log>
+        <!-- What table to insert data. If table is not exist, it will be created.
+             When query log structure is changed after system update,
+              then old table will be renamed and new table will be created automatically.
+        -->
+        <database>system</database>
+        <table>query_log</table>
+        <!--
+            PARTITION BY expr: https://clickhouse.yandex/docs/en/table_engines/mergetree-family/custom_partitioning_key/
+            Example:
+                event_date
+                toMonday(event_date)
+                toYYYYMM(event_date)
+                toStartOfHour(event_time)
+        -->
+        <partition_by>toYYYYMM(event_date)</partition_by>
+        <!--
+            Table TTL specification: https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/mergetree/#mergetree-table-ttl
+            Example:
+                event_date + INTERVAL 1 WEEK
+                event_date + INTERVAL 7 DAY DELETE
+                event_date + INTERVAL 2 WEEK TO DISK 'bbb'
+
+        <ttl>event_date + INTERVAL 30 DAY DELETE</ttl>
+        -->
+
+        <!-- Instead of partition_by, you can provide full engine expression (starting with ENGINE = ) with parameters,
+             Example: <engine>ENGINE = MergeTree PARTITION BY toYYYYMM(event_date) ORDER BY (event_date, event_time) SETTINGS index_granularity = 1024</engine>
+          -->
+
+        <!-- Interval of flushing data. -->
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </query_log>
+
+    <!-- Trace log. Stores stack traces collected by query profilers.
+         See query_profiler_real_time_period_ns and query_profiler_cpu_time_period_ns settings. -->
+    <trace_log>
+        <database>system</database>
+        <table>trace_log</table>
+
+        <partition_by>toYYYYMM(event_date)</partition_by>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </trace_log>
+
+    <!-- Query thread log. Has information about all threads participated in query execution.
+         Used only for queries with setting log_query_threads = 1. -->
+    <query_thread_log>
+        <database>system</database>
+        <table>query_thread_log</table>
+        <partition_by>toYYYYMM(event_date)</partition_by>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </query_thread_log>
+
+    <!-- Uncomment if use part log.
+         Part log contains information about all actions with parts in MergeTree tables (creation, deletion, merges, downloads).
+    <part_log>
+        <database>system</database>
+        <table>part_log</table>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </part_log>
+    -->
+
+    <!-- Uncomment to write text log into table.
+         Text log contains all information from usual server log but stores it in structured and efficient way.
+         The level of the messages that goes to the table can be limited (<level>), if not specified all messages will go to the table.
+    <text_log>
+        <database>system</database>
+        <table>text_log</table>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+        <level></level>
+    </text_log>
+    -->
+
+    <!-- Metric log contains rows with current values of ProfileEvents, CurrentMetrics collected with "collect_interval_milliseconds" interval. -->
+    <metric_log>
+        <database>system</database>
+        <table>metric_log</table>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+        <collect_interval_milliseconds>1000</collect_interval_milliseconds>
+    </metric_log>
+
+    <!--
+        Asynchronous metric log contains values of metrics from
+        system.asynchronous_metrics.
+    -->
+    <asynchronous_metric_log>
+        <database>system</database>
+        <table>asynchronous_metric_log</table>
+        <!--
+            Asynchronous metrics are updated once a minute, so there is
+            no need to flush more often.
+        -->
+        <flush_interval_milliseconds>60000</flush_interval_milliseconds>
+    </asynchronous_metric_log>
+
+    <!--
+        OpenTelemetry log contains OpenTelemetry trace spans.
+    -->
+    <opentelemetry_span_log>
+        <!--
+            The default table creation code is insufficient, this <engine> spec
+            is a workaround. There is no 'event_time' for this log, but two times,
+            start and finish. It is sorted by finish time, to avoid inserting
+            data too far away in the past (probably we can sometimes insert a span
+            that is seconds earlier than the last span in the table, due to a race
+            between several spans inserted in parallel). This gives the spans a
+            global order that we can use to e.g. retry insertion into some external
+            system.
+        -->
+        <engine>
+            engine MergeTree
+            partition by toYYYYMM(finish_date)
+            order by (finish_date, finish_time_us, trace_id)
+        </engine>
+        <database>system</database>
+        <table>opentelemetry_span_log</table>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </opentelemetry_span_log>
+
+
+    <!-- Crash log. Stores stack traces for fatal errors.
+         This table is normally empty. -->
+    <crash_log>
+        <database>system</database>
+        <table>crash_log</table>
+
+        <partition_by />
+        <flush_interval_milliseconds>1000</flush_interval_milliseconds>
+    </crash_log>
+
+    <!-- Parameters for embedded dictionaries, used in Yandex.Metrica.
+         See https://clickhouse.yandex/docs/en/dicts/internal_dicts/
+    -->
+
+    <!-- Path to file with region hierarchy. -->
+    <!-- <path_to_regions_hierarchy_file>/opt/geo/regions_hierarchy.txt</path_to_regions_hierarchy_file> -->
+
+    <!-- Path to directory with files containing names of regions -->
+    <!-- <path_to_regions_names_files>/opt/geo/</path_to_regions_names_files> -->
+
+
+    <!-- <top_level_domains_path>/var/lib/clickhouse/top_level_domains/</top_level_domains_path> -->
+    <!-- Custom TLD lists.
+         Format: <name>/path/to/file</name>
+
+         Changes will not be applied w/o server restart.
+         Path to the list is under top_level_domains_path (see above).
+    -->
+    <top_level_domains_lists>
+        <!--
+        <public_suffix_list>/path/to/public_suffix_list.dat</public_suffix_list>
+        -->
+    </top_level_domains_lists>
+
+    <!-- Configuration of external dictionaries. See:
+         https://clickhouse.tech/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts
+    -->
+    <dictionaries_config>*_dictionary.xml</dictionaries_config>
+
+    <!-- Uncomment if you want data to be compressed 30-100% better.
+         Don't do that if you just started using ClickHouse.
+      -->
+    <!--
+    <compression>
+        <!- - Set of variants. Checked in order. Last matching case wins. If nothing matches, lz4 will be used. - ->
+        <case>
+
+            <!- - Conditions. All must be satisfied. Some conditions may be omitted. - ->
+            <min_part_size>10000000000</min_part_size>        <!- - Min part size in bytes. - ->
+            <min_part_size_ratio>0.01</min_part_size_ratio>   <!- - Min size of part relative to whole table size. - ->
+
+            <!- - What compression method to use. - ->
+            <method>zstd</method>
+        </case>
+    </compression>
+    -->
+
+    <!-- Allow to execute distributed DDL queries (CREATE, DROP, ALTER, RENAME) on cluster.
+         Works only if ZooKeeper is enabled. Comment it if such functionality isn't required. -->
+    <distributed_ddl>
+        <!-- Path in ZooKeeper to queue with DDL queries -->
+        <path>/clickhouse/task_queue/ddl</path>
+
+        <!-- Settings from this profile will be used to execute DDL queries -->
+        <!-- <profile>default</profile> -->
+
+        <!-- Controls how much ON CLUSTER queries can be run simultaneously. -->
+        <!-- <pool_size>1</pool_size> -->
+
+        <!--
+             Cleanup settings (active tasks will not be removed)
+        -->
+
+        <!-- Controls task TTL (default 1 week) -->
+        <!-- <task_max_lifetime>604800</task_max_lifetime> -->
+
+        <!-- Controls how often cleanup should be performed (in seconds) -->
+        <!-- <cleanup_delay_period>60</cleanup_delay_period> -->
+
+        <!-- Controls how many tasks could be in the queue -->
+        <!-- <max_tasks_in_queue>1000</max_tasks_in_queue> -->
+    </distributed_ddl>
+
+    <!-- Settings to fine tune MergeTree tables. See documentation in source code, in MergeTreeSettings.h -->
+    <!--
+    <merge_tree>
+        <max_suspicious_broken_parts>5</max_suspicious_broken_parts>
+    </merge_tree>
+    -->
+
+    <!-- Protection from accidental DROP.
+         If size of a MergeTree table is greater than max_table_size_to_drop (in bytes) than table could not be dropped with any DROP query.
+         If you want do delete one table and don't want to change clickhouse-server config, you could create special file <clickhouse-path>/flags/force_drop_table and make DROP once.
+         By default max_table_size_to_drop is 50GB; max_table_size_to_drop=0 allows to DROP any tables.
+         The same for max_partition_size_to_drop.
+         Uncomment to disable protection.
+    -->
+    <!-- <max_table_size_to_drop>0</max_table_size_to_drop> -->
+    <!-- <max_partition_size_to_drop>0</max_partition_size_to_drop> -->
+
+    <!-- Example of parameters for GraphiteMergeTree table engine -->
+    <graphite_rollup_example>
+        <pattern>
+            <regexp>click_cost</regexp>
+            <function>any</function>
+            <retention>
+                <age>0</age>
+                <precision>3600</precision>
+            </retention>
+            <retention>
+                <age>86400</age>
+                <precision>60</precision>
+            </retention>
+        </pattern>
+        <default>
+            <function>max</function>
+            <retention>
+                <age>0</age>
+                <precision>60</precision>
+            </retention>
+            <retention>
+                <age>3600</age>
+                <precision>300</precision>
+            </retention>
+            <retention>
+                <age>86400</age>
+                <precision>3600</precision>
+            </retention>
+        </default>
+    </graphite_rollup_example>
+
+    <!-- Directory in <clickhouse-path> containing schema files for various input formats.
+         The directory will be created if it doesn't exist.
+      -->
+    <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
+
+    <!-- Default query masking rules, matching lines would be replaced with something else in the logs
+        (both text logs and system.query_log).
+        name - name for the rule (optional)
+        regexp - RE2 compatible regular expression (mandatory)
+        replace - substitution string for sensitive data (optional, by default - six asterisks)
+    -->
+    <query_masking_rules>
+        <rule>
+            <name>hide encrypt/decrypt arguments</name>
+            <regexp>((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:'(?:\\'|.)+'|.*?)\s*\)</regexp>
+            <!-- or more secure, but also more invasive:
+                (aes_\w+)\s*\(.*\)
+            -->
+            <replace>\1(???)</replace>
+        </rule>
+    </query_masking_rules>
+
+    <!-- Uncomment to use custom http handlers.
+        rules are checked from top to bottom, first match runs the handler
+            url - to match request URL, you can use 'regex:' prefix to use regex match(optional)
+            methods - to match request method, you can use commas to separate multiple method matches(optional)
+            headers - to match request headers, match each child element(child element name is header name), you can use 'regex:' prefix to use regex match(optional)
+        handler is request handler
+            type - supported types: static, dynamic_query_handler, predefined_query_handler
+            query - use with predefined_query_handler type, executes query when the handler is called
+            query_param_name - use with dynamic_query_handler type, extracts and executes the value corresponding to the <query_param_name> value in HTTP request params
+            status - use with static type, response status code
+            content_type - use with static type, response content-type
+            response_content - use with static type, Response content sent to client, when using the prefix 'file://' or 'config://', find the content from the file or configuration send to client.
+
+    <http_handlers>
+        <rule>
+            <url>/</url>
+            <methods>POST,GET</methods>
+            <headers><pragma>no-cache</pragma></headers>
+            <handler>
+                <type>dynamic_query_handler</type>
+                <query_param_name>query</query_param_name>
+            </handler>
+        </rule>
+
+        <rule>
+            <url>/predefined_query</url>
+            <methods>POST,GET</methods>
+            <handler>
+                <type>predefined_query_handler</type>
+                <query>SELECT * FROM system.settings</query>
+            </handler>
+        </rule>
+
+        <rule>
+            <handler>
+                <type>static</type>
+                <status>200</status>
+                <content_type>text/plain; charset=UTF-8</content_type>
+                <response_content>config://http_server_default_response</response_content>
+            </handler>
+        </rule>
+    </http_handlers>
+    -->
+
+    <send_crash_reports>
+        <!-- Changing <enabled> to true allows sending crash reports to -->
+        <!-- the ClickHouse core developers team via Sentry https://sentry.io -->
+        <!-- Doing so at least in pre-production environments is highly appreciated -->
+        <enabled>false</enabled>
+        <!-- Change <anonymize> to true if you don't feel comfortable attaching the server hostname to the crash report -->
+        <anonymize>false</anonymize>
+        <!-- Default endpoint should be changed to different Sentry DSN only if you have -->
+        <!-- some in-house engineers or hired consultants who're going to debug ClickHouse issues for you -->
+        <endpoint>https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277</endpoint>
+    </send_crash_reports>
+
+    <!-- Uncomment to disable ClickHouse internal DNS caching. -->
+    <!-- <disable_internal_dns_cache>1</disable_internal_dns_cache> -->
+</yandex>
diff --git a/tests/integration/test_config_xml_full/configs/embedded.xml b/tests/integration/test_config_xml_full/configs/embedded.xml
new file mode 100644
index 00000000000..a66f57d1eb7
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/embedded.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0"?>
+<!-- Config that is used when server is run without config file. -->
+<yandex>
+    <logger>
+        <level>trace</level>
+        <console>true</console>
+    </logger>
+
+    <http_port>8123</http_port>
+    <tcp_port>9000</tcp_port>
+    <mysql_port>9004</mysql_port>
+
+    <path>./</path>
+
+    <uncompressed_cache_size>8589934592</uncompressed_cache_size>
+    <mark_cache_size>5368709120</mark_cache_size>
+    <mlock_executable>true</mlock_executable>
+
+    <users>
+        <default>
+            <password></password>
+
+            <networks>
+                <ip>::/0</ip>
+            </networks>
+
+            <profile>default</profile>
+            <quota>default</quota>
+            <access_management>1</access_management>
+        </default>
+    </users>
+
+    <profiles>
+        <default/>
+    </profiles>
+
+    <quotas>
+        <default />
+    </quotas>
+</yandex>
diff --git a/tests/integration/test_config_xml_full/configs/users.d/allow_introspection_functions.xml b/tests/integration/test_config_xml_full/configs/users.d/allow_introspection_functions.xml
new file mode 100644
index 00000000000..b94e95bc043
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/users.d/allow_introspection_functions.xml
@@ -0,0 +1,8 @@
+<?xml version="1.0"?>
+<yandex>
+    <profiles>
+        <default>
+            <allow_introspection_functions>1</allow_introspection_functions>
+        </default>
+    </profiles>
+</yandex>
diff --git a/tests/integration/test_config_xml_full/configs/users.d/log_queries.xml b/tests/integration/test_config_xml_full/configs/users.d/log_queries.xml
new file mode 100644
index 00000000000..25261072ade
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/users.d/log_queries.xml
@@ -0,0 +1,7 @@
+<yandex>
+    <profiles>
+        <default>
+            <log_queries>1</log_queries>
+        </default>
+    </profiles>
+</yandex>
diff --git a/tests/integration/test_config_xml_full/configs/users.xml b/tests/integration/test_config_xml_full/configs/users.xml
new file mode 100644
index 00000000000..829748dafb1
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/users.xml
@@ -0,0 +1,120 @@
+<?xml version="1.0"?>
+<yandex>
+    <!-- Profiles of settings. -->
+    <profiles>
+        <!-- Default settings. -->
+        <default>
+            <!-- Maximum memory usage for processing single query, in bytes. -->
+            <max_memory_usage>10000000000</max_memory_usage>
+            <max_block_size>64999</max_block_size>
+
+            <!-- How to choose between replicas during distributed query processing.
+                 random - choose random replica from set of replicas with minimum number of errors
+                 nearest_hostname - from set of replicas with minimum number of errors, choose replica
+                  with minimum number of different symbols between replica's hostname and local hostname
+                  (Hamming distance).
+                 in_order - first live replica is chosen in specified order.
+                 first_or_random - if first replica one has higher number of errors, pick a random one from replicas with minimum number of errors.
+            -->
+            <load_balancing>random</load_balancing>
+        </default>
+
+        <!-- Profile that allows only read queries. -->
+        <readonly>
+            <readonly>1</readonly>
+        </readonly>
+    </profiles>
+
+    <!-- Users and ACL. -->
+    <users>
+        <!-- If user name was not specified, 'default' user is used. -->
+        <default>
+            <!-- Password could be specified in plaintext or in SHA256 (in hex format).
+
+                 If you want to specify password in plaintext (not recommended), place it in 'password' element.
+                 Example: <password>qwerty</password>.
+                 Password could be empty.
+
+                 If you want to specify SHA256, place it in 'password_sha256_hex' element.
+                 Example: <password_sha256_hex>65e84be33532fb784c48129675f9eff3a682b27168c0ea744b2cf58ee02337c5</password_sha256_hex>
+                 Restrictions of SHA256: impossibility to connect to ClickHouse using MySQL JS client (as of July 2019).
+
+                 If you want to specify double SHA1, place it in 'password_double_sha1_hex' element.
+                 Example: <password_double_sha1_hex>e395796d6546b1b65db9d665cd43f0e858dd4303</password_double_sha1_hex>
+
+                 If you want to specify a previously defined LDAP server (see 'ldap_servers' in the main config) for authentication,
+                  place its name in 'server' element inside 'ldap' element.
+                 Example: <ldap><server>my_ldap_server</server></ldap>
+
+                 If you want to authenticate the user via Kerberos (assuming Kerberos is enabled, see 'kerberos' in the main config),
+                  place 'kerberos' element instead of 'password' (and similar) elements.
+                 The name part of the canonical principal name of the initiator must match the user name for authentication to succeed.
+                 You can also place 'realm' element inside 'kerberos' element to further restrict authentication to only those requests
+                  whose initiator's realm matches it. 
+                 Example: <kerberos />
+                 Example: <kerberos><realm>EXAMPLE.COM</realm></kerberos>
+
+                 How to generate decent password:
+                 Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha256sum | tr -d '-'
+                 In first line will be password and in second - corresponding SHA256.
+
+                 How to generate double SHA1:
+                 Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha1sum | tr -d '-' | xxd -r -p | sha1sum | tr -d '-'
+                 In first line will be password and in second - corresponding double SHA1.
+            -->
+            <password></password>
+
+            <!-- List of networks with open access.
+
+                 To open access from everywhere, specify:
+                    <ip>::/0</ip>
+
+                 To open access only from localhost, specify:
+                    <ip>::1</ip>
+                    <ip>127.0.0.1</ip>
+
+                 Each element of list has one of the following forms:
+                 <ip> IP-address or network mask. Examples: 213.180.204.3 or 10.0.0.1/8 or 10.0.0.1/255.255.255.0
+                     2a02:6b8::3 or 2a02:6b8::3/64 or 2a02:6b8::3/ffff:ffff:ffff:ffff::.
+                 <host> Hostname. Example: server01.yandex.ru.
+                     To check access, DNS query is performed, and all received addresses compared to peer address.
+                 <host_regexp> Regular expression for host names. Example, ^server\d\d-\d\d-\d\.yandex\.ru$
+                     To check access, DNS PTR query is performed for peer address and then regexp is applied.
+                     Then, for result of PTR query, another DNS query is performed and all received addresses compared to peer address.
+                     Strongly recommended that regexp is ends with $
+                 All results of DNS requests are cached till server restart.
+            -->
+            <networks>
+                <ip>::/0</ip>
+            </networks>
+
+            <!-- Settings profile for user. -->
+            <profile>default</profile>
+
+            <!-- Quota for user. -->
+            <quota>default</quota>
+
+            <!-- User can create other users and grant rights to them. -->
+            <!-- <access_management>1</access_management> -->
+        </default>
+    </users>
+
+    <!-- Quotas. -->
+    <quotas>
+        <!-- Name of quota. -->
+        <default>
+            <!-- Limits for time interval. You could specify many intervals with different limits. -->
+            <interval>
+                <!-- Length of interval. -->
+                <duration>3600</duration>
+
+                <!-- No limits. Just calculate resource usage for time interval. -->
+                <queries>0</queries>
+                <errors>0</errors>
+                <result_rows>0</result_rows>
+                <read_rows>0</read_rows>
+                <execution_time>0</execution_time>
+            </interval>
+        </default>
+    </quotas>
+</yandex>
diff --git a/tests/integration/test_config_xml_full/test.py b/tests/integration/test_config_xml_full/test.py
new file mode 100644
index 00000000000..a8650a0dc55
--- /dev/null
+++ b/tests/integration/test_config_xml_full/test.py
@@ -0,0 +1,40 @@
+import time
+import threading
+from os import path as p, unlink
+from tempfile import NamedTemporaryFile
+
+import helpers
+import pytest
+from helpers.cluster import ClickHouseCluster
+
+
+def test_xml_full_conf():
+    # all configs are in XML
+    cluster = ClickHouseCluster(__file__, zookeeper_config_path='configs/config.d/zookeeper.xml')
+
+    all_confd = ['configs/config.d/access_control.xml',
+                'configs/config.d/keeper_port.xml',
+                'configs/config.d/logging_no_rotate.xml',
+                'configs/config.d/log_to_console.xml',
+                'configs/config.d/macros.xml',
+                'configs/config.d/metric_log.xml',
+                'configs/config.d/more_clusters.xml',
+                'configs/config.d/part_log.xml',
+                'configs/config.d/path.xml',
+                'configs/config.d/query_masking_rules.xml',
+                'configs/config.d/tcp_with_proxy.xml',
+                'configs/config.d/text_log.xml',
+                'configs/config.d/zookeeper.xml']
+
+    all_userd = ['configs/users.d/allow_introspection_functions.xml',
+                'configs/users.d/log_queries.xml']
+
+    node = cluster.add_instance('node', base_config_dir='configs', main_configs=all_confd, user_configs=all_userd, with_zookeeper=False)
+
+    try:
+        cluster.start()
+        assert(node.query("select value from system.settings where name = 'max_memory_usage'") == "10000000000\n")
+        assert(node.query("select value from system.settings where name = 'max_block_size'") == "64999\n")
+
+    finally:
+        cluster.shutdown()
diff --git a/tests/integration/test_config_xml_main/__init__.py b/tests/integration/test_config_xml_main/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/integration/test_config_xml_main/configs/config.d/access_control.yaml b/tests/integration/test_config_xml_main/configs/config.d/access_control.yaml
new file mode 100644
index 00000000000..d8ead517c86
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/config.d/access_control.yaml
@@ -0,0 +1,7 @@
+user_directories:
+  users_xml:
+    path: users.xml
+  local_directory:
+    path: access/
+  "@replace": replace
+
diff --git a/tests/integration/test_config_xml_main/configs/config.d/keeper_port.yaml b/tests/integration/test_config_xml_main/configs/config.d/keeper_port.yaml
new file mode 100644
index 00000000000..91723bc372f
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/config.d/keeper_port.yaml
@@ -0,0 +1,15 @@
+keeper_server:
+  tcp_port: 9181
+  server_id: 1
+  coordination_settings:
+    operation_timeout_ms: 10000
+    session_timeout_ms: 30000
+    force_sync: false
+    startup_timeout: 60000
+    reserved_log_items: 1000000000000000
+  raft_configuration:
+    server:
+      id: 1
+      hostname: localhost
+      port: 44444
+
diff --git a/tests/integration/test_config_xml_main/configs/config.d/log_to_console.yaml b/tests/integration/test_config_xml_main/configs/config.d/log_to_console.yaml
new file mode 100644
index 00000000000..7b59339800b
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/config.d/log_to_console.yaml
@@ -0,0 +1,7 @@
+logger:
+  console: true
+  log:
+    "@remove": remove
+  errorlog:
+    "@remove": remove
+
diff --git a/tests/integration/test_config_xml_main/configs/config.d/logging_no_rotate.yaml b/tests/integration/test_config_xml_main/configs/config.d/logging_no_rotate.yaml
new file mode 100644
index 00000000000..513cd3f7dc5
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/config.d/logging_no_rotate.yaml
@@ -0,0 +1,2 @@
+logger:
+  size: never
diff --git a/tests/integration/test_config_xml_main/configs/config.d/macros.yaml b/tests/integration/test_config_xml_main/configs/config.d/macros.yaml
new file mode 100644
index 00000000000..a9c61e270ab
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/config.d/macros.yaml
@@ -0,0 +1,7 @@
+macros:
+  test: 'Hello, world!'
+  shard: s1
+  replica: r1
+  default_path_test: '/clickhouse/tables/{database}/{shard}/'
+  default_name_test: 'table_{table}'
+
diff --git a/tests/integration/test_config_xml_main/configs/config.d/metric_log.yaml b/tests/integration/test_config_xml_main/configs/config.d/metric_log.yaml
new file mode 100644
index 00000000000..aafad939c37
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/config.d/metric_log.yaml
@@ -0,0 +1,6 @@
+metric_log:
+  database: system
+  table: metric_log
+  flush_interval_milliseconds: 7500
+  collect_interval_milliseconds: 1000
+
diff --git a/tests/integration/test_config_xml_main/configs/config.d/more_clusters.yaml b/tests/integration/test_config_xml_main/configs/config.d/more_clusters.yaml
new file mode 100644
index 00000000000..7da07190894
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/config.d/more_clusters.yaml
@@ -0,0 +1,23 @@
+remote_servers:
+  single_remote_shard_at_port_9001:
+    shard:
+      replica:
+        host: localhost
+        port: 9001
+  two_remote_shards_at_port_9001_9002:
+    shard:
+      - replica:
+          host: localhost
+          port: 9001
+      - replica:
+          host: localhost
+          port: 9002
+  two_shards_one_local_one_remote_at_port_9001:
+    shard:
+      - replica:
+          host: localhost
+          port: 9000
+      - replica:
+          host: localhost
+          port: 9001
+
diff --git a/tests/integration/test_config_xml_main/configs/config.d/part_log.yaml b/tests/integration/test_config_xml_main/configs/config.d/part_log.yaml
new file mode 100644
index 00000000000..43f60146dca
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/config.d/part_log.yaml
@@ -0,0 +1,5 @@
+part_log:
+  database: system
+  table: part_log
+  flush_interval_milliseconds: 7500
+
diff --git a/tests/integration/test_config_xml_main/configs/config.d/path.yaml b/tests/integration/test_config_xml_main/configs/config.d/path.yaml
new file mode 100644
index 00000000000..3e26e8906ee
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/config.d/path.yaml
@@ -0,0 +1,18 @@
+path:
+  - ./
+  - "@replace": replace
+tmp_path:
+  - ./tmp/
+  - "@replace": replace
+user_files_path:
+  - ./user_files/
+  - "@replace": replace
+format_schema_path:
+  - ./format_schemas/
+  - "@replace": replace
+access_control_path:
+  - ./access/
+  - "@replace": replace
+top_level_domains_path:
+  - ./top_level_domains/
+  - "@replace": replace
diff --git a/tests/integration/test_config_xml_main/configs/config.d/query_masking_rules.yaml b/tests/integration/test_config_xml_main/configs/config.d/query_masking_rules.yaml
new file mode 100644
index 00000000000..38163429d81
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/config.d/query_masking_rules.yaml
@@ -0,0 +1,4 @@
+query_masking_rules:
+  rule:
+    regexp: TOPSECRET.TOPSECRET
+    replace: '[hidden]'
diff --git a/tests/integration/test_config_xml_main/configs/config.d/tcp_with_proxy.yaml b/tests/integration/test_config_xml_main/configs/config.d/tcp_with_proxy.yaml
new file mode 100644
index 00000000000..b0349f5a9b9
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/config.d/tcp_with_proxy.yaml
@@ -0,0 +1 @@
+tcp_with_proxy_port: 9010
diff --git a/tests/integration/test_config_xml_main/configs/config.d/test_cluster_with_incorrect_pw.yaml b/tests/integration/test_config_xml_main/configs/config.d/test_cluster_with_incorrect_pw.yaml
new file mode 100644
index 00000000000..309a7daa81d
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/config.d/test_cluster_with_incorrect_pw.yaml
@@ -0,0 +1,11 @@
+remote_servers:
+  test_cluster_with_incorrect_pw:
+    shard:
+      internal_replication: true
+      replica:
+        - host: 127.0.0.1
+          port: 9000
+          password: foo
+        - host: 127.0.0.2
+          port: 9000
+          password: foo
diff --git a/tests/integration/test_config_xml_main/configs/config.d/text_log.yaml b/tests/integration/test_config_xml_main/configs/config.d/text_log.yaml
new file mode 100644
index 00000000000..4d188020e37
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/config.d/text_log.yaml
@@ -0,0 +1,4 @@
+text_log:
+  database: system
+  table: text_log
+  flush_interval_milliseconds: 7500
diff --git a/tests/integration/test_config_xml_main/configs/config.d/zookeeper.yaml b/tests/integration/test_config_xml_main/configs/config.d/zookeeper.yaml
new file mode 100644
index 00000000000..be02c516798
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/config.d/zookeeper.yaml
@@ -0,0 +1,5 @@
+zookeeper:
+  node:
+    host: localhost
+    port: 9181
+    "@index": 1
diff --git a/tests/integration/test_config_xml_main/configs/config.xml b/tests/integration/test_config_xml_main/configs/config.xml
new file mode 100644
index 00000000000..3b5dab50ffe
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/config.xml
@@ -0,0 +1,277 @@
+<?xml version="1.0"?>
+<yandex>
+    <logger>
+        <level>trace</level>
+        <log>/var/log/clickhouse-server/clickhouse-server.log</log>
+        <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
+        <size>1000M</size>
+        <count>10</count>
+    </logger>
+    <http_port>8123</http_port>
+    <tcp_port>9000</tcp_port>
+    <mysql_port>9004</mysql_port>
+    <postgresql_port>9005</postgresql_port>
+    <interserver_http_port>9009</interserver_http_port>
+    <max_connections>4096</max_connections>
+    <keep_alive_timeout>3</keep_alive_timeout>
+    <grpc>
+        <enable_ssl>false</enable_ssl>
+        <ssl_cert_file>/path/to/ssl_cert_file</ssl_cert_file>
+        <ssl_key_file>/path/to/ssl_key_file</ssl_key_file>
+        <ssl_require_client_auth>false</ssl_require_client_auth>
+        <ssl_ca_cert_file>/path/to/ssl_ca_cert_file</ssl_ca_cert_file>
+        <compression>deflate</compression>
+        <compression_level>medium</compression_level>
+        <max_send_message_size>-1</max_send_message_size>
+        <max_receive_message_size>-1</max_receive_message_size>
+        <verbose_logs>false</verbose_logs>
+    </grpc>
+    <openSSL>
+        <server> 
+            <certificateFile>/etc/clickhouse-server/server.crt</certificateFile>
+            <privateKeyFile>/etc/clickhouse-server/server.key</privateKeyFile>
+            <dhParamsFile>/etc/clickhouse-server/dhparam.pem</dhParamsFile>
+            <verificationMode>none</verificationMode>
+            <loadDefaultCAFile>true</loadDefaultCAFile>
+            <cacheSessions>true</cacheSessions>
+            <disableProtocols>sslv2,sslv3</disableProtocols>
+            <preferServerCiphers>true</preferServerCiphers>
+        </server>
+
+        <client>
+            <loadDefaultCAFile>true</loadDefaultCAFile>
+            <cacheSessions>true</cacheSessions>
+            <disableProtocols>sslv2,sslv3</disableProtocols>
+            <preferServerCiphers>true</preferServerCiphers>
+            <invalidCertificateHandler>
+                <name>RejectCertificateHandler</name>
+            </invalidCertificateHandler>
+        </client>
+    </openSSL>
+    <max_concurrent_queries>100</max_concurrent_queries>
+    <max_server_memory_usage>0</max_server_memory_usage>
+    <max_thread_pool_size>10000</max_thread_pool_size>
+
+    <max_server_memory_usage_to_ram_ratio>0.9</max_server_memory_usage_to_ram_ratio>
+
+    <total_memory_profiler_step>4194304</total_memory_profiler_step>
+
+    <total_memory_tracker_sample_probability>0</total_memory_tracker_sample_probability>
+
+    <uncompressed_cache_size>8589934592</uncompressed_cache_size>
+
+    <mark_cache_size>5368709120</mark_cache_size>
+
+    <mmap_cache_size>1000</mmap_cache_size>
+
+    <path>/var/lib/clickhouse/</path>
+
+    <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
+    <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
+    <ldap_servers>
+    </ldap_servers>
+    <user_directories>
+        <users_xml>
+            <path>users.xml</path>
+        </users_xml>
+        <local_directory>
+            <path>/var/lib/clickhouse/access/</path>
+        </local_directory>
+    </user_directories>
+    <default_profile>default</default_profile>
+    <custom_settings_prefixes></custom_settings_prefixes>
+    <default_database>default</default_database>
+    <mlock_executable>true</mlock_executable>
+
+    <remap_executable>false</remap_executable>
+    <remote_servers>
+        <test_shard_localhost>
+            <shard>
+                <replica>
+                    <host>localhost</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </test_shard_localhost>
+        <test_cluster_two_shards_localhost>
+             <shard>
+                 <replica>
+                     <host>localhost</host>
+                     <port>9000</port>
+                 </replica>
+             </shard>
+             <shard>
+                 <replica>
+                     <host>localhost</host>
+                     <port>9000</port>
+                 </replica>
+             </shard>
+        </test_cluster_two_shards_localhost>
+        <test_cluster_two_shards>
+            <shard>
+                <replica>
+                    <host>127.0.0.1</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+            <shard>
+                <replica>
+                    <host>127.0.0.2</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </test_cluster_two_shards>
+        <test_cluster_two_shards_internal_replication>
+            <shard>
+                <internal_replication>true</internal_replication>
+                <replica>
+                    <host>127.0.0.1</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+            <shard>
+                <internal_replication>true</internal_replication>
+                <replica>
+                    <host>127.0.0.2</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </test_cluster_two_shards_internal_replication>
+        <test_shard_localhost_secure>
+            <shard>
+                <replica>
+                    <host>localhost</host>
+                    <port>9440</port>
+                    <secure>1</secure>
+                </replica>
+            </shard>
+        </test_shard_localhost_secure>
+        <test_unavailable_shard>
+            <shard>
+                <replica>
+                    <host>localhost</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+            <shard>
+                <replica>
+                    <host>localhost</host>
+                    <port>1</port>
+                </replica>
+            </shard>
+        </test_unavailable_shard>
+    </remote_servers>
+
+    <builtin_dictionaries_reload_interval>3600</builtin_dictionaries_reload_interval>
+
+    <max_session_timeout>3600</max_session_timeout>
+
+    <default_session_timeout>60</default_session_timeout>
+
+    <query_log>
+        <database>system</database>
+        <table>query_log</table>
+        
+        <partition_by>toYYYYMM(event_date)</partition_by>
+        
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </query_log>
+
+    <trace_log>
+        <database>system</database>
+        <table>trace_log</table>
+
+        <partition_by>toYYYYMM(event_date)</partition_by>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </trace_log>
+
+    <query_thread_log>
+        <database>system</database>
+        <table>query_thread_log</table>
+        <partition_by>toYYYYMM(event_date)</partition_by>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </query_thread_log>
+
+    <metric_log>
+        <database>system</database>
+        <table>metric_log</table>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+        <collect_interval_milliseconds>1000</collect_interval_milliseconds>
+    </metric_log>
+
+    <asynchronous_metric_log>
+        <database>system</database>
+        <table>asynchronous_metric_log</table>
+
+        <flush_interval_milliseconds>60000</flush_interval_milliseconds>
+    </asynchronous_metric_log>
+
+    <opentelemetry_span_log>
+        <engine>
+            engine MergeTree
+            partition by toYYYYMM(finish_date)
+            order by (finish_date, finish_time_us, trace_id)
+        </engine>
+        <database>system</database>
+        <table>opentelemetry_span_log</table>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </opentelemetry_span_log>
+
+    <crash_log>
+        <database>system</database>
+        <table>crash_log</table>
+
+        <partition_by />
+        <flush_interval_milliseconds>1000</flush_interval_milliseconds>
+    </crash_log>
+
+    <top_level_domains_lists>
+    </top_level_domains_lists>
+
+    <dictionaries_config>*_dictionary.xml</dictionaries_config>
+    <distributed_ddl>
+        <path>/clickhouse/task_queue/ddl</path>
+    </distributed_ddl>
+    <graphite_rollup_example>
+        <pattern>
+            <regexp>click_cost</regexp>
+            <function>any</function>
+            <retention>
+                <age>0</age>
+                <precision>3600</precision>
+            </retention>
+            <retention>
+                <age>86400</age>
+                <precision>60</precision>
+            </retention>
+        </pattern>
+        <default>
+            <function>max</function>
+            <retention>
+                <age>0</age>
+                <precision>60</precision>
+            </retention>
+            <retention>
+                <age>3600</age>
+                <precision>300</precision>
+            </retention>
+            <retention>
+                <age>86400</age>
+                <precision>3600</precision>
+            </retention>
+        </default>
+    </graphite_rollup_example>
+    <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
+    <query_masking_rules>
+        <rule>
+            <name>hide encrypt/decrypt arguments</name>
+            <regexp>((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:'(?:\\'|.)+'|.*?)\s*\)</regexp>
+            <replace>\1(???)</replace>
+        </rule>
+    </query_masking_rules>
+    <send_crash_reports>
+        <enabled>false</enabled>
+        <anonymize>false</anonymize>
+        <endpoint>https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277</endpoint>
+    </send_crash_reports>
+</yandex>
diff --git a/tests/integration/test_config_xml_main/configs/embedded.xml b/tests/integration/test_config_xml_main/configs/embedded.xml
new file mode 100644
index 00000000000..a66f57d1eb7
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/embedded.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0"?>
+<!-- Config that is used when server is run without config file. -->
+<yandex>
+    <logger>
+        <level>trace</level>
+        <console>true</console>
+    </logger>
+
+    <http_port>8123</http_port>
+    <tcp_port>9000</tcp_port>
+    <mysql_port>9004</mysql_port>
+
+    <path>./</path>
+
+    <uncompressed_cache_size>8589934592</uncompressed_cache_size>
+    <mark_cache_size>5368709120</mark_cache_size>
+    <mlock_executable>true</mlock_executable>
+
+    <users>
+        <default>
+            <password></password>
+
+            <networks>
+                <ip>::/0</ip>
+            </networks>
+
+            <profile>default</profile>
+            <quota>default</quota>
+            <access_management>1</access_management>
+        </default>
+    </users>
+
+    <profiles>
+        <default/>
+    </profiles>
+
+    <quotas>
+        <default />
+    </quotas>
+</yandex>
diff --git a/tests/integration/test_config_xml_main/configs/users.d/allow_introspection_functions.yaml b/tests/integration/test_config_xml_main/configs/users.d/allow_introspection_functions.yaml
new file mode 100644
index 00000000000..84612c198c9
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/users.d/allow_introspection_functions.yaml
@@ -0,0 +1,3 @@
+profiles:
+  default:
+    allow_introspection_functions: 1
diff --git a/tests/integration/test_config_xml_main/configs/users.d/log_queries.yaml b/tests/integration/test_config_xml_main/configs/users.d/log_queries.yaml
new file mode 100644
index 00000000000..88574e8f764
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/users.d/log_queries.yaml
@@ -0,0 +1,3 @@
+profiles:
+  default:
+    log_queries: 1
diff --git a/tests/integration/test_config_xml_main/configs/users.xml b/tests/integration/test_config_xml_main/configs/users.xml
new file mode 100644
index 00000000000..b473413bdfa
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/users.xml
@@ -0,0 +1,19 @@
+<?xml version="1.0"?>
+<yandex>
+    <profiles>
+        <default>
+            <max_memory_usage>10000000000</max_memory_usage>
+            <max_block_size>64999</max_block_size>
+        </default>
+    </profiles>
+
+    <users>
+        <default>
+            <password></password>
+            <networks replace="replace">
+                <ip>::/0</ip>
+            </networks>
+            <profile>default</profile>
+        </default>
+    </users>
+</yandex>
diff --git a/tests/integration/test_config_xml_main/test.py b/tests/integration/test_config_xml_main/test.py
new file mode 100644
index 00000000000..052f9adb01f
--- /dev/null
+++ b/tests/integration/test_config_xml_main/test.py
@@ -0,0 +1,43 @@
+
+
+import time
+import threading
+from os import path as p, unlink
+from tempfile import NamedTemporaryFile
+
+import helpers
+import pytest
+from helpers.cluster import ClickHouseCluster
+
+
+def test_xml_main_conf():
+    # main configs are in XML; config.d and users.d are in YAML
+    cluster = ClickHouseCluster(__file__, zookeeper_config_path='configs/config.d/zookeeper.yaml')
+
+    all_confd = ['configs/config.d/access_control.yaml',
+                'configs/config.d/keeper_port.yaml',
+                'configs/config.d/logging_no_rotate.yaml',
+                'configs/config.d/log_to_console.yaml',
+                'configs/config.d/macros.yaml',
+                'configs/config.d/metric_log.yaml',
+                'configs/config.d/more_clusters.yaml',
+                'configs/config.d/part_log.yaml',
+                'configs/config.d/path.yaml',
+                'configs/config.d/query_masking_rules.yaml',
+                'configs/config.d/tcp_with_proxy.yaml',
+                'configs/config.d/test_cluster_with_incorrect_pw.yaml',
+                'configs/config.d/text_log.yaml',
+                'configs/config.d/zookeeper.yaml']
+
+    all_userd = ['configs/users.d/allow_introspection_functions.yaml',
+                'configs/users.d/log_queries.yaml']
+
+    node = cluster.add_instance('node', base_config_dir='configs', main_configs=all_confd, user_configs=all_userd, with_zookeeper=False)
+
+    try:
+        cluster.start()
+        assert(node.query("select value from system.settings where name = 'max_memory_usage'") == "10000000000\n")
+        assert(node.query("select value from system.settings where name = 'max_block_size'") == "64999\n")
+
+    finally:
+        cluster.shutdown()
diff --git a/tests/integration/test_config_xml_yaml_mix/__init__.py b/tests/integration/test_config_xml_yaml_mix/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/0_common_instance_config.yaml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/0_common_instance_config.yaml
new file mode 100644
index 00000000000..62e4ba8c744
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/0_common_instance_config.yaml
@@ -0,0 +1,6 @@
+timezone: Europe/Moscow
+listen_host: 0.0.0.0
+custom_settings_prefixes: custom_
+path: /var/lib/clickhouse/
+tmp_path: /var/lib/clickhouse/tmp/
+users_config: users.yaml
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/access_control.yaml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/access_control.yaml
new file mode 100644
index 00000000000..ce2e23839ef
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/access_control.yaml
@@ -0,0 +1,7 @@
+user_directories:
+  users_xml:
+    path: users.yaml
+  local_directory:
+    path: access/
+  "@replace": replace
+
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/keeper_port.xml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/keeper_port.xml
new file mode 100644
index 00000000000..b21df47bc85
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/keeper_port.xml
@@ -0,0 +1,23 @@
+<yandex>
+    <keeper_server>
+        <tcp_port>9181</tcp_port>
+        <server_id>1</server_id>
+
+        <coordination_settings>
+            <operation_timeout_ms>10000</operation_timeout_ms>
+            <session_timeout_ms>30000</session_timeout_ms>
+            <force_sync>false</force_sync>
+            <startup_timeout>60000</startup_timeout>
+            <!-- we want all logs for complex problems investigation -->
+            <reserved_log_items>1000000000000000</reserved_log_items>
+        </coordination_settings>
+
+        <raft_configuration>
+            <server>
+                <id>1</id>
+                <hostname>localhost</hostname>
+                <port>44444</port>
+            </server>
+        </raft_configuration>
+    </keeper_server>
+</yandex>
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/log_to_console.yaml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/log_to_console.yaml
new file mode 100644
index 00000000000..7b59339800b
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/log_to_console.yaml
@@ -0,0 +1,7 @@
+logger:
+  console: true
+  log:
+    "@remove": remove
+  errorlog:
+    "@remove": remove
+
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/logging_no_rotate.xml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/logging_no_rotate.xml
new file mode 100644
index 00000000000..2c34585437b
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/logging_no_rotate.xml
@@ -0,0 +1,8 @@
+<yandex>
+    <logger>
+        <!-- Disable rotation
+             https://pocoproject.org/docs/Poco.FileChannel.html
+        -->
+        <size>never</size>
+    </logger>
+</yandex>
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/macros.yaml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/macros.yaml
new file mode 100644
index 00000000000..a9c61e270ab
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/macros.yaml
@@ -0,0 +1,7 @@
+macros:
+  test: 'Hello, world!'
+  shard: s1
+  replica: r1
+  default_path_test: '/clickhouse/tables/{database}/{shard}/'
+  default_name_test: 'table_{table}'
+
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/metric_log.xml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/metric_log.xml
new file mode 100644
index 00000000000..0ca9f162416
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/metric_log.xml
@@ -0,0 +1,8 @@
+<yandex>
+    <metric_log>
+        <database>system</database>
+        <table>metric_log</table>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+        <collect_interval_milliseconds>1000</collect_interval_milliseconds>
+    </metric_log>
+</yandex>
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/more_clusters.yaml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/more_clusters.yaml
new file mode 100644
index 00000000000..7da07190894
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/more_clusters.yaml
@@ -0,0 +1,23 @@
+remote_servers:
+  single_remote_shard_at_port_9001:
+    shard:
+      replica:
+        host: localhost
+        port: 9001
+  two_remote_shards_at_port_9001_9002:
+    shard:
+      - replica:
+          host: localhost
+          port: 9001
+      - replica:
+          host: localhost
+          port: 9002
+  two_shards_one_local_one_remote_at_port_9001:
+    shard:
+      - replica:
+          host: localhost
+          port: 9000
+      - replica:
+          host: localhost
+          port: 9001
+
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/part_log.xml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/part_log.xml
new file mode 100644
index 00000000000..6c6fc9c6982
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/part_log.xml
@@ -0,0 +1,8 @@
+<yandex>
+    <part_log>
+        <database>system</database>
+        <table>part_log</table>
+
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </part_log>
+</yandex>
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/path.yaml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/path.yaml
new file mode 100644
index 00000000000..3e26e8906ee
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/path.yaml
@@ -0,0 +1,18 @@
+path:
+  - ./
+  - "@replace": replace
+tmp_path:
+  - ./tmp/
+  - "@replace": replace
+user_files_path:
+  - ./user_files/
+  - "@replace": replace
+format_schema_path:
+  - ./format_schemas/
+  - "@replace": replace
+access_control_path:
+  - ./access/
+  - "@replace": replace
+top_level_domains_path:
+  - ./top_level_domains/
+  - "@replace": replace
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/query_masking_rules.xml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/query_masking_rules.xml
new file mode 100644
index 00000000000..5a854848f3d
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/query_masking_rules.xml
@@ -0,0 +1,10 @@
+<?xml version="1.0"?>
+<!-- Config for test server -->
+<yandex>
+    <query_masking_rules>
+        <rule>
+            <regexp>TOPSECRET.TOPSECRET</regexp>
+            <replace>[hidden]</replace>
+        </rule>
+    </query_masking_rules>
+</yandex>
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/tcp_with_proxy.yaml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/tcp_with_proxy.yaml
new file mode 100644
index 00000000000..b0349f5a9b9
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/tcp_with_proxy.yaml
@@ -0,0 +1 @@
+tcp_with_proxy_port: 9010
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/test_cluster_with_incorrect_pw.xml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/test_cluster_with_incorrect_pw.xml
new file mode 100644
index 00000000000..109e35afc37
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/test_cluster_with_incorrect_pw.xml
@@ -0,0 +1,21 @@
+<yandex>
+    <remote_servers>
+        <test_cluster_with_incorrect_pw>
+             <shard>
+                 <internal_replication>true</internal_replication>
+                 <replica>
+                     <host>127.0.0.1</host>
+                     <port>9000</port>
+                     <!-- password is incorrect -->
+                     <password>foo</password>
+                 </replica>
+                 <replica>
+                     <host>127.0.0.2</host>
+                     <port>9000</port>
+                     <!-- password is incorrect -->
+                     <password>foo</password>
+                 </replica>
+             </shard>
+         </test_cluster_with_incorrect_pw>
+    </remote_servers>
+</yandex>
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/text_log.yaml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/text_log.yaml
new file mode 100644
index 00000000000..4d188020e37
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/text_log.yaml
@@ -0,0 +1,4 @@
+text_log:
+  database: system
+  table: text_log
+  flush_interval_milliseconds: 7500
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/zookeeper.xml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/zookeeper.xml
new file mode 100644
index 00000000000..06ed7fcd39f
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/zookeeper.xml
@@ -0,0 +1,8 @@
+<yandex>
+    <zookeeper>
+        <node index="1">
+            <host>localhost</host>
+            <port>9181</port>
+        </node>
+    </zookeeper>
+</yandex>
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.xml b/tests/integration/test_config_xml_yaml_mix/configs/config.xml
new file mode 100644
index 00000000000..e6a2b6d5324
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/config.xml
@@ -0,0 +1,277 @@
+<?xml version="1.0"?>
+<yandex>
+    <logger>
+        <level>trace</level>
+        <log>/var/log/clickhouse-server/clickhouse-server.log</log>
+        <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
+        <size>1000M</size>
+        <count>10</count>
+    </logger>
+    <http_port>8123</http_port>
+    <tcp_port>9000</tcp_port>
+    <mysql_port>9004</mysql_port>
+    <postgresql_port>9005</postgresql_port>
+    <interserver_http_port>9009</interserver_http_port>
+    <max_connections>4096</max_connections>
+    <keep_alive_timeout>3</keep_alive_timeout>
+    <grpc>
+        <enable_ssl>false</enable_ssl>
+        <ssl_cert_file>/path/to/ssl_cert_file</ssl_cert_file>
+        <ssl_key_file>/path/to/ssl_key_file</ssl_key_file>
+        <ssl_require_client_auth>false</ssl_require_client_auth>
+        <ssl_ca_cert_file>/path/to/ssl_ca_cert_file</ssl_ca_cert_file>
+        <compression>deflate</compression>
+        <compression_level>medium</compression_level>
+        <max_send_message_size>-1</max_send_message_size>
+        <max_receive_message_size>-1</max_receive_message_size>
+        <verbose_logs>false</verbose_logs>
+    </grpc>
+    <openSSL>
+        <server> 
+            <certificateFile>/etc/clickhouse-server/server.crt</certificateFile>
+            <privateKeyFile>/etc/clickhouse-server/server.key</privateKeyFile>
+            <dhParamsFile>/etc/clickhouse-server/dhparam.pem</dhParamsFile>
+            <verificationMode>none</verificationMode>
+            <loadDefaultCAFile>true</loadDefaultCAFile>
+            <cacheSessions>true</cacheSessions>
+            <disableProtocols>sslv2,sslv3</disableProtocols>
+            <preferServerCiphers>true</preferServerCiphers>
+        </server>
+
+        <client>
+            <loadDefaultCAFile>true</loadDefaultCAFile>
+            <cacheSessions>true</cacheSessions>
+            <disableProtocols>sslv2,sslv3</disableProtocols>
+            <preferServerCiphers>true</preferServerCiphers>
+            <invalidCertificateHandler>
+                <name>RejectCertificateHandler</name>
+            </invalidCertificateHandler>
+        </client>
+    </openSSL>
+    <max_concurrent_queries>100</max_concurrent_queries>
+    <max_server_memory_usage>0</max_server_memory_usage>
+    <max_thread_pool_size>10000</max_thread_pool_size>
+
+    <max_server_memory_usage_to_ram_ratio>0.9</max_server_memory_usage_to_ram_ratio>
+
+    <total_memory_profiler_step>4194304</total_memory_profiler_step>
+
+    <total_memory_tracker_sample_probability>0</total_memory_tracker_sample_probability>
+
+    <uncompressed_cache_size>8589934592</uncompressed_cache_size>
+
+    <mark_cache_size>5368709120</mark_cache_size>
+
+    <mmap_cache_size>1000</mmap_cache_size>
+
+    <path>/var/lib/clickhouse/</path>
+
+    <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
+    <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
+    <ldap_servers>
+    </ldap_servers>
+    <user_directories>
+        <users_xml>
+            <path>users.yaml</path>
+        </users_xml>
+        <local_directory>
+            <path>/var/lib/clickhouse/access/</path>
+        </local_directory>
+    </user_directories>
+    <default_profile>default</default_profile>
+    <custom_settings_prefixes></custom_settings_prefixes>
+    <default_database>default</default_database>
+    <mlock_executable>true</mlock_executable>
+
+    <remap_executable>false</remap_executable>
+    <remote_servers>
+        <test_shard_localhost>
+            <shard>
+                <replica>
+                    <host>localhost</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </test_shard_localhost>
+        <test_cluster_two_shards_localhost>
+             <shard>
+                 <replica>
+                     <host>localhost</host>
+                     <port>9000</port>
+                 </replica>
+             </shard>
+             <shard>
+                 <replica>
+                     <host>localhost</host>
+                     <port>9000</port>
+                 </replica>
+             </shard>
+        </test_cluster_two_shards_localhost>
+        <test_cluster_two_shards>
+            <shard>
+                <replica>
+                    <host>127.0.0.1</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+            <shard>
+                <replica>
+                    <host>127.0.0.2</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </test_cluster_two_shards>
+        <test_cluster_two_shards_internal_replication>
+            <shard>
+                <internal_replication>true</internal_replication>
+                <replica>
+                    <host>127.0.0.1</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+            <shard>
+                <internal_replication>true</internal_replication>
+                <replica>
+                    <host>127.0.0.2</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </test_cluster_two_shards_internal_replication>
+        <test_shard_localhost_secure>
+            <shard>
+                <replica>
+                    <host>localhost</host>
+                    <port>9440</port>
+                    <secure>1</secure>
+                </replica>
+            </shard>
+        </test_shard_localhost_secure>
+        <test_unavailable_shard>
+            <shard>
+                <replica>
+                    <host>localhost</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+            <shard>
+                <replica>
+                    <host>localhost</host>
+                    <port>1</port>
+                </replica>
+            </shard>
+        </test_unavailable_shard>
+    </remote_servers>
+
+    <builtin_dictionaries_reload_interval>3600</builtin_dictionaries_reload_interval>
+
+    <max_session_timeout>3600</max_session_timeout>
+
+    <default_session_timeout>60</default_session_timeout>
+
+    <query_log>
+        <database>system</database>
+        <table>query_log</table>
+        
+        <partition_by>toYYYYMM(event_date)</partition_by>
+        
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </query_log>
+
+    <trace_log>
+        <database>system</database>
+        <table>trace_log</table>
+
+        <partition_by>toYYYYMM(event_date)</partition_by>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </trace_log>
+
+    <query_thread_log>
+        <database>system</database>
+        <table>query_thread_log</table>
+        <partition_by>toYYYYMM(event_date)</partition_by>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </query_thread_log>
+
+    <metric_log>
+        <database>system</database>
+        <table>metric_log</table>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+        <collect_interval_milliseconds>1000</collect_interval_milliseconds>
+    </metric_log>
+
+    <asynchronous_metric_log>
+        <database>system</database>
+        <table>asynchronous_metric_log</table>
+
+        <flush_interval_milliseconds>60000</flush_interval_milliseconds>
+    </asynchronous_metric_log>
+
+    <opentelemetry_span_log>
+        <engine>
+            engine MergeTree
+            partition by toYYYYMM(finish_date)
+            order by (finish_date, finish_time_us, trace_id)
+        </engine>
+        <database>system</database>
+        <table>opentelemetry_span_log</table>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </opentelemetry_span_log>
+
+    <crash_log>
+        <database>system</database>
+        <table>crash_log</table>
+
+        <partition_by />
+        <flush_interval_milliseconds>1000</flush_interval_milliseconds>
+    </crash_log>
+
+    <top_level_domains_lists>
+    </top_level_domains_lists>
+
+    <dictionaries_config>*_dictionary.xml</dictionaries_config>
+    <distributed_ddl>
+        <path>/clickhouse/task_queue/ddl</path>
+    </distributed_ddl>
+    <graphite_rollup_example>
+        <pattern>
+            <regexp>click_cost</regexp>
+            <function>any</function>
+            <retention>
+                <age>0</age>
+                <precision>3600</precision>
+            </retention>
+            <retention>
+                <age>86400</age>
+                <precision>60</precision>
+            </retention>
+        </pattern>
+        <default>
+            <function>max</function>
+            <retention>
+                <age>0</age>
+                <precision>60</precision>
+            </retention>
+            <retention>
+                <age>3600</age>
+                <precision>300</precision>
+            </retention>
+            <retention>
+                <age>86400</age>
+                <precision>3600</precision>
+            </retention>
+        </default>
+    </graphite_rollup_example>
+    <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
+    <query_masking_rules>
+        <rule>
+            <name>hide encrypt/decrypt arguments</name>
+            <regexp>((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:'(?:\\'|.)+'|.*?)\s*\)</regexp>
+            <replace>\1(???)</replace>
+        </rule>
+    </query_masking_rules>
+    <send_crash_reports>
+        <enabled>false</enabled>
+        <anonymize>false</anonymize>
+        <endpoint>https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277</endpoint>
+    </send_crash_reports>
+</yandex>
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/embedded.xml b/tests/integration/test_config_xml_yaml_mix/configs/embedded.xml
new file mode 100644
index 00000000000..a66f57d1eb7
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/embedded.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0"?>
+<!-- Config that is used when server is run without config file. -->
+<yandex>
+    <logger>
+        <level>trace</level>
+        <console>true</console>
+    </logger>
+
+    <http_port>8123</http_port>
+    <tcp_port>9000</tcp_port>
+    <mysql_port>9004</mysql_port>
+
+    <path>./</path>
+
+    <uncompressed_cache_size>8589934592</uncompressed_cache_size>
+    <mark_cache_size>5368709120</mark_cache_size>
+    <mlock_executable>true</mlock_executable>
+
+    <users>
+        <default>
+            <password></password>
+
+            <networks>
+                <ip>::/0</ip>
+            </networks>
+
+            <profile>default</profile>
+            <quota>default</quota>
+            <access_management>1</access_management>
+        </default>
+    </users>
+
+    <profiles>
+        <default/>
+    </profiles>
+
+    <quotas>
+        <default />
+    </quotas>
+</yandex>
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/users.d/allow_introspection_functions.xml b/tests/integration/test_config_xml_yaml_mix/configs/users.d/allow_introspection_functions.xml
new file mode 100644
index 00000000000..b94e95bc043
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/users.d/allow_introspection_functions.xml
@@ -0,0 +1,8 @@
+<?xml version="1.0"?>
+<yandex>
+    <profiles>
+        <default>
+            <allow_introspection_functions>1</allow_introspection_functions>
+        </default>
+    </profiles>
+</yandex>
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/users.d/log_queries.yaml b/tests/integration/test_config_xml_yaml_mix/configs/users.d/log_queries.yaml
new file mode 100644
index 00000000000..88574e8f764
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/users.d/log_queries.yaml
@@ -0,0 +1,3 @@
+profiles:
+  default:
+    log_queries: 1
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/users.yaml b/tests/integration/test_config_xml_yaml_mix/configs/users.yaml
new file mode 100644
index 00000000000..a87a8c82819
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/users.yaml
@@ -0,0 +1,12 @@
+profiles:
+  default:
+    max_memory_usage: 10000000000
+    max_block_size: 64999
+users:
+  default:
+    password: ''
+    networks:
+      "@replace": replace
+      ip: '::/0'
+    profile: default
+
diff --git a/tests/integration/test_config_xml_yaml_mix/test.py b/tests/integration/test_config_xml_yaml_mix/test.py
new file mode 100644
index 00000000000..90ee8a2dea5
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/test.py
@@ -0,0 +1,43 @@
+import time
+import threading
+from os import path as p, unlink
+from tempfile import NamedTemporaryFile
+
+import helpers
+import pytest
+from helpers.cluster import ClickHouseCluster
+
+
+def test_extra_yaml_mix():
+    # some configs are written in XML, others are written in YAML
+    cluster = ClickHouseCluster(__file__, zookeeper_config_path='configs/config.d/zookeeper.xml')
+
+    all_confd = ['configs/config.d/0_common_instance_config.yaml',
+                'configs/config.d/access_control.yaml',
+                'configs/config.d/keeper_port.xml',
+                'configs/config.d/logging_no_rotate.xml',
+                'configs/config.d/log_to_console.yaml',
+                'configs/config.d/macros.yaml',
+                'configs/config.d/metric_log.xml',
+                'configs/config.d/more_clusters.yaml',
+                'configs/config.d/part_log.xml',
+                'configs/config.d/path.yaml',
+                'configs/config.d/query_masking_rules.xml',
+                'configs/config.d/tcp_with_proxy.yaml',
+                'configs/config.d/test_cluster_with_incorrect_pw.xml',
+                'configs/config.d/text_log.yaml',
+                'configs/config.d/zookeeper.xml']
+
+    all_userd = ['configs/users.d/allow_introspection_functions.xml',
+                'configs/users.d/log_queries.yaml']
+
+    node = cluster.add_instance('node', base_config_dir='configs', main_configs=all_confd, user_configs=all_userd, with_zookeeper=False,
+    users_config_name="users.yaml", copy_common_configs=False)
+
+    try:
+        cluster.start()
+        assert(node.query("select value from system.settings where name = 'max_memory_usage'") == "10000000000\n")
+        assert(node.query("select value from system.settings where name = 'max_block_size'") == "64999\n")
+
+    finally:
+        cluster.shutdown()
diff --git a/tests/integration/test_config_yaml_full/__init__.py b/tests/integration/test_config_yaml_full/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/integration/test_config_yaml_full/configs/config.d/0_common_instance_config.yaml b/tests/integration/test_config_yaml_full/configs/config.d/0_common_instance_config.yaml
new file mode 100644
index 00000000000..62e4ba8c744
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/config.d/0_common_instance_config.yaml
@@ -0,0 +1,6 @@
+timezone: Europe/Moscow
+listen_host: 0.0.0.0
+custom_settings_prefixes: custom_
+path: /var/lib/clickhouse/
+tmp_path: /var/lib/clickhouse/tmp/
+users_config: users.yaml
diff --git a/tests/integration/test_config_yaml_full/configs/config.d/access_control.yaml b/tests/integration/test_config_yaml_full/configs/config.d/access_control.yaml
new file mode 100644
index 00000000000..ce2e23839ef
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/config.d/access_control.yaml
@@ -0,0 +1,7 @@
+user_directories:
+  users_xml:
+    path: users.yaml
+  local_directory:
+    path: access/
+  "@replace": replace
+
diff --git a/tests/integration/test_config_yaml_full/configs/config.d/keeper_port.yaml b/tests/integration/test_config_yaml_full/configs/config.d/keeper_port.yaml
new file mode 100644
index 00000000000..91723bc372f
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/config.d/keeper_port.yaml
@@ -0,0 +1,15 @@
+keeper_server:
+  tcp_port: 9181
+  server_id: 1
+  coordination_settings:
+    operation_timeout_ms: 10000
+    session_timeout_ms: 30000
+    force_sync: false
+    startup_timeout: 60000
+    reserved_log_items: 1000000000000000
+  raft_configuration:
+    server:
+      id: 1
+      hostname: localhost
+      port: 44444
+
diff --git a/tests/integration/test_config_yaml_full/configs/config.d/log_to_console.yaml b/tests/integration/test_config_yaml_full/configs/config.d/log_to_console.yaml
new file mode 100644
index 00000000000..7b59339800b
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/config.d/log_to_console.yaml
@@ -0,0 +1,7 @@
+logger:
+  console: true
+  log:
+    "@remove": remove
+  errorlog:
+    "@remove": remove
+
diff --git a/tests/integration/test_config_yaml_full/configs/config.d/logging_no_rotate.yaml b/tests/integration/test_config_yaml_full/configs/config.d/logging_no_rotate.yaml
new file mode 100644
index 00000000000..513cd3f7dc5
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/config.d/logging_no_rotate.yaml
@@ -0,0 +1,2 @@
+logger:
+  size: never
diff --git a/tests/integration/test_config_yaml_full/configs/config.d/macros.yaml b/tests/integration/test_config_yaml_full/configs/config.d/macros.yaml
new file mode 100644
index 00000000000..a9c61e270ab
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/config.d/macros.yaml
@@ -0,0 +1,7 @@
+macros:
+  test: 'Hello, world!'
+  shard: s1
+  replica: r1
+  default_path_test: '/clickhouse/tables/{database}/{shard}/'
+  default_name_test: 'table_{table}'
+
diff --git a/tests/integration/test_config_yaml_full/configs/config.d/metric_log.yaml b/tests/integration/test_config_yaml_full/configs/config.d/metric_log.yaml
new file mode 100644
index 00000000000..aafad939c37
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/config.d/metric_log.yaml
@@ -0,0 +1,6 @@
+metric_log:
+  database: system
+  table: metric_log
+  flush_interval_milliseconds: 7500
+  collect_interval_milliseconds: 1000
+
diff --git a/tests/integration/test_config_yaml_full/configs/config.d/more_clusters.yaml b/tests/integration/test_config_yaml_full/configs/config.d/more_clusters.yaml
new file mode 100644
index 00000000000..7da07190894
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/config.d/more_clusters.yaml
@@ -0,0 +1,23 @@
+remote_servers:
+  single_remote_shard_at_port_9001:
+    shard:
+      replica:
+        host: localhost
+        port: 9001
+  two_remote_shards_at_port_9001_9002:
+    shard:
+      - replica:
+          host: localhost
+          port: 9001
+      - replica:
+          host: localhost
+          port: 9002
+  two_shards_one_local_one_remote_at_port_9001:
+    shard:
+      - replica:
+          host: localhost
+          port: 9000
+      - replica:
+          host: localhost
+          port: 9001
+
diff --git a/tests/integration/test_config_yaml_full/configs/config.d/part_log.yaml b/tests/integration/test_config_yaml_full/configs/config.d/part_log.yaml
new file mode 100644
index 00000000000..43f60146dca
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/config.d/part_log.yaml
@@ -0,0 +1,5 @@
+part_log:
+  database: system
+  table: part_log
+  flush_interval_milliseconds: 7500
+
diff --git a/tests/integration/test_config_yaml_full/configs/config.d/path.yaml b/tests/integration/test_config_yaml_full/configs/config.d/path.yaml
new file mode 100644
index 00000000000..3e26e8906ee
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/config.d/path.yaml
@@ -0,0 +1,18 @@
+path:
+  - ./
+  - "@replace": replace
+tmp_path:
+  - ./tmp/
+  - "@replace": replace
+user_files_path:
+  - ./user_files/
+  - "@replace": replace
+format_schema_path:
+  - ./format_schemas/
+  - "@replace": replace
+access_control_path:
+  - ./access/
+  - "@replace": replace
+top_level_domains_path:
+  - ./top_level_domains/
+  - "@replace": replace
diff --git a/tests/integration/test_config_yaml_full/configs/config.d/query_masking_rules.yaml b/tests/integration/test_config_yaml_full/configs/config.d/query_masking_rules.yaml
new file mode 100644
index 00000000000..38163429d81
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/config.d/query_masking_rules.yaml
@@ -0,0 +1,4 @@
+query_masking_rules:
+  rule:
+    regexp: TOPSECRET.TOPSECRET
+    replace: '[hidden]'
diff --git a/tests/integration/test_config_yaml_full/configs/config.d/tcp_with_proxy.yaml b/tests/integration/test_config_yaml_full/configs/config.d/tcp_with_proxy.yaml
new file mode 100644
index 00000000000..b0349f5a9b9
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/config.d/tcp_with_proxy.yaml
@@ -0,0 +1 @@
+tcp_with_proxy_port: 9010
diff --git a/tests/integration/test_config_yaml_full/configs/config.d/test_cluster_with_incorrect_pw.yaml b/tests/integration/test_config_yaml_full/configs/config.d/test_cluster_with_incorrect_pw.yaml
new file mode 100644
index 00000000000..309a7daa81d
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/config.d/test_cluster_with_incorrect_pw.yaml
@@ -0,0 +1,11 @@
+remote_servers:
+  test_cluster_with_incorrect_pw:
+    shard:
+      internal_replication: true
+      replica:
+        - host: 127.0.0.1
+          port: 9000
+          password: foo
+        - host: 127.0.0.2
+          port: 9000
+          password: foo
diff --git a/tests/integration/test_config_yaml_full/configs/config.d/text_log.yaml b/tests/integration/test_config_yaml_full/configs/config.d/text_log.yaml
new file mode 100644
index 00000000000..4d188020e37
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/config.d/text_log.yaml
@@ -0,0 +1,4 @@
+text_log:
+  database: system
+  table: text_log
+  flush_interval_milliseconds: 7500
diff --git a/tests/integration/test_config_yaml_full/configs/config.d/zookeeper.yaml b/tests/integration/test_config_yaml_full/configs/config.d/zookeeper.yaml
new file mode 100644
index 00000000000..be02c516798
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/config.d/zookeeper.yaml
@@ -0,0 +1,5 @@
+zookeeper:
+  node:
+    host: localhost
+    port: 9181
+    "@index": 1
diff --git a/tests/integration/test_config_yaml_full/configs/config.yaml b/tests/integration/test_config_yaml_full/configs/config.yaml
new file mode 100644
index 00000000000..619a3735269
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/config.yaml
@@ -0,0 +1,183 @@
+logger:
+  level: trace
+  log: /var/log/clickhouse-server/clickhouse-server.log
+  errorlog: /var/log/clickhouse-server/clickhouse-server.err.log
+  size: 1000M
+  count: 10
+http_port: 8123
+tcp_port: 9000
+mysql_port: 9004
+postgresql_port: 9005
+interserver_http_port: 9009
+max_connections: 4096
+keep_alive_timeout: 3
+grpc:
+  enable_ssl: false
+  ssl_cert_file: /path/to/ssl_cert_file
+  ssl_key_file: /path/to/ssl_key_file
+  ssl_require_client_auth: false
+  ssl_ca_cert_file: /path/to/ssl_ca_cert_file
+  compression: deflate
+  compression_level: medium
+  max_send_message_size: -1
+  max_receive_message_size: -1
+  verbose_logs: false
+openSSL:
+  server:
+    certificateFile: /etc/clickhouse-server/server.crt
+    privateKeyFile: /etc/clickhouse-server/server.key
+    dhParamsFile: /etc/clickhouse-server/dhparam.pem
+    verificationMode: none
+    loadDefaultCAFile: true
+    cacheSessions: true
+    disableProtocols: 'sslv2,sslv3'
+    preferServerCiphers: true
+  client:
+    loadDefaultCAFile: true
+    cacheSessions: true
+    disableProtocols: 'sslv2,sslv3'
+    preferServerCiphers: true
+    invalidCertificateHandler:
+      name: RejectCertificateHandler
+max_concurrent_queries: 100
+max_server_memory_usage: 0
+max_thread_pool_size: 10000
+max_server_memory_usage_to_ram_ratio: 0.9
+total_memory_profiler_step: 4194304
+total_memory_tracker_sample_probability: 0
+uncompressed_cache_size: 8589934592
+mark_cache_size: 5368709120
+mmap_cache_size: 1000
+path: /var/lib/clickhouse/
+tmp_path: /var/lib/clickhouse/tmp/
+user_files_path: /var/lib/clickhouse/user_files/
+ldap_servers: ''
+user_directories:
+  users_xml:
+    path: users.yaml
+  local_directory:
+    path: /var/lib/clickhouse/access/
+default_profile: default
+custom_settings_prefixes: ''
+default_database: default
+mlock_executable: true
+remap_executable: false
+remote_servers:
+  test_shard_localhost:
+    shard:
+      replica:
+        host: localhost
+        port: 9000
+  test_cluster_two_shards_localhost:
+    shard:
+      - replica:
+          host: localhost
+          port: 9000
+      - replica:
+          host: localhost
+          port: 9000
+  test_cluster_two_shards:
+    shard:
+      - replica:
+          host: 127.0.0.1
+          port: 9000
+      - replica:
+          host: 127.0.0.2
+          port: 9000
+  test_cluster_two_shards_internal_replication:
+    shard:
+      - internal_replication: true
+        replica:
+          host: 127.0.0.1
+          port: 9000
+      - internal_replication: true
+        replica:
+          host: 127.0.0.2
+          port: 9000
+  test_shard_localhost_secure:
+    shard:
+      replica:
+        host: localhost
+        port: 9440
+        secure: 1
+  test_unavailable_shard:
+    shard:
+      - replica:
+          host: localhost
+          port: 9000
+      - replica:
+          host: localhost
+          port: 1
+builtin_dictionaries_reload_interval: 3600
+max_session_timeout: 3600
+default_session_timeout: 60
+query_log:
+  database: system
+  table: query_log
+  partition_by: toYYYYMM(event_date)
+  flush_interval_milliseconds: 7500
+trace_log:
+  database: system
+  table: trace_log
+  partition_by: toYYYYMM(event_date)
+  flush_interval_milliseconds: 7500
+query_thread_log:
+  database: system
+  table: query_thread_log
+  partition_by: toYYYYMM(event_date)
+  flush_interval_milliseconds: 7500
+metric_log:
+  database: system
+  table: metric_log
+  flush_interval_milliseconds: 7500
+  collect_interval_milliseconds: 1000
+asynchronous_metric_log:
+  database: system
+  table: asynchronous_metric_log
+  flush_interval_milliseconds: 60000
+opentelemetry_span_log:
+  engine: |-
+    engine MergeTree
+                partition by toYYYYMM(finish_date)
+                order by (finish_date, finish_time_us, trace_id)
+  database: system
+  table: opentelemetry_span_log
+  flush_interval_milliseconds: 7500
+crash_log:
+  database: system
+  table: crash_log
+  partition_by: ''
+  flush_interval_milliseconds: 1000
+top_level_domains_lists: ''
+dictionaries_config: '*_dictionary.xml'
+distributed_ddl:
+  path: /clickhouse/task_queue/ddl
+graphite_rollup_example:
+  pattern:
+    regexp: click_cost
+    function: any
+    retention:
+      - age: 0
+        precision: 3600
+      - age: 86400
+        precision: 60
+  default:
+    function: max
+    retention:
+      - age: 0
+        precision: 60
+      - age: 3600
+        precision: 300
+      - age: 86400
+        precision: 3600
+format_schema_path: /var/lib/clickhouse/format_schemas/
+query_masking_rules:
+  rule:
+    name: hide encrypt/decrypt arguments
+    regexp: '((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:''(?:\\''|.)+''|.*?)\s*\)'
+    replace: \1(???)
+send_crash_reports:
+  enabled: false
+  anonymize: false
+  endpoint: 'https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277'
+
diff --git a/tests/integration/test_config_yaml_full/configs/embedded.xml b/tests/integration/test_config_yaml_full/configs/embedded.xml
new file mode 100644
index 00000000000..a66f57d1eb7
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/embedded.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0"?>
+<!-- Config that is used when server is run without config file. -->
+<yandex>
+    <logger>
+        <level>trace</level>
+        <console>true</console>
+    </logger>
+
+    <http_port>8123</http_port>
+    <tcp_port>9000</tcp_port>
+    <mysql_port>9004</mysql_port>
+
+    <path>./</path>
+
+    <uncompressed_cache_size>8589934592</uncompressed_cache_size>
+    <mark_cache_size>5368709120</mark_cache_size>
+    <mlock_executable>true</mlock_executable>
+
+    <users>
+        <default>
+            <password></password>
+
+            <networks>
+                <ip>::/0</ip>
+            </networks>
+
+            <profile>default</profile>
+            <quota>default</quota>
+            <access_management>1</access_management>
+        </default>
+    </users>
+
+    <profiles>
+        <default/>
+    </profiles>
+
+    <quotas>
+        <default />
+    </quotas>
+</yandex>
diff --git a/tests/integration/test_config_yaml_full/configs/users.d/allow_introspection_functions.yaml b/tests/integration/test_config_yaml_full/configs/users.d/allow_introspection_functions.yaml
new file mode 100644
index 00000000000..84612c198c9
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/users.d/allow_introspection_functions.yaml
@@ -0,0 +1,3 @@
+profiles:
+  default:
+    allow_introspection_functions: 1
diff --git a/tests/integration/test_config_yaml_full/configs/users.d/log_queries.yaml b/tests/integration/test_config_yaml_full/configs/users.d/log_queries.yaml
new file mode 100644
index 00000000000..88574e8f764
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/users.d/log_queries.yaml
@@ -0,0 +1,3 @@
+profiles:
+  default:
+    log_queries: 1
diff --git a/tests/integration/test_config_yaml_full/configs/users.yaml b/tests/integration/test_config_yaml_full/configs/users.yaml
new file mode 100644
index 00000000000..a87a8c82819
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/users.yaml
@@ -0,0 +1,12 @@
+profiles:
+  default:
+    max_memory_usage: 10000000000
+    max_block_size: 64999
+users:
+  default:
+    password: ''
+    networks:
+      "@replace": replace
+      ip: '::/0'
+    profile: default
+
diff --git a/tests/integration/test_config_yaml_full/test.py b/tests/integration/test_config_yaml_full/test.py
new file mode 100644
index 00000000000..bc4fa40384c
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/test.py
@@ -0,0 +1,42 @@
+import time
+import threading
+from os import path as p, unlink
+from tempfile import NamedTemporaryFile
+
+import helpers
+import pytest
+from helpers.cluster import ClickHouseCluster
+
+def test_yaml_full_conf():
+    # all configs are in YAML
+    cluster = ClickHouseCluster(__file__, zookeeper_config_path='configs/config.d/zookeeper.yaml')
+
+    all_confd = ['configs/config.d/0_common_instance_config.yaml',
+                'configs/config.d/access_control.yaml',
+                'configs/config.d/keeper_port.yaml',
+                'configs/config.d/logging_no_rotate.yaml',
+                'configs/config.d/log_to_console.yaml',
+                'configs/config.d/macros.yaml',
+                'configs/config.d/metric_log.yaml',
+                'configs/config.d/more_clusters.yaml',
+                'configs/config.d/part_log.yaml',
+                'configs/config.d/path.yaml',
+                'configs/config.d/query_masking_rules.yaml',
+                'configs/config.d/tcp_with_proxy.yaml',
+                'configs/config.d/test_cluster_with_incorrect_pw.yaml',
+                'configs/config.d/text_log.yaml',
+                'configs/config.d/zookeeper.yaml']
+
+    all_userd = ['configs/users.d/allow_introspection_functions.yaml',
+                'configs/users.d/log_queries.yaml']
+
+    node = cluster.add_instance('node', base_config_dir='configs', main_configs=all_confd, user_configs=all_userd,
+                                with_zookeeper=False, main_config_name="config.yaml", users_config_name="users.yaml", copy_common_configs=False)
+
+    try:
+        cluster.start()
+        assert(node.query("select value from system.settings where name = 'max_memory_usage'") == "10000000000\n")
+        assert(node.query("select value from system.settings where name = 'max_block_size'") == "64999\n")
+
+    finally:
+        cluster.shutdown()
diff --git a/tests/integration/test_config_yaml_main/__init__.py b/tests/integration/test_config_yaml_main/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/integration/test_config_yaml_main/configs/config.d/0_common_instance_config.yaml b/tests/integration/test_config_yaml_main/configs/config.d/0_common_instance_config.yaml
new file mode 100644
index 00000000000..62e4ba8c744
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/config.d/0_common_instance_config.yaml
@@ -0,0 +1,6 @@
+timezone: Europe/Moscow
+listen_host: 0.0.0.0
+custom_settings_prefixes: custom_
+path: /var/lib/clickhouse/
+tmp_path: /var/lib/clickhouse/tmp/
+users_config: users.yaml
diff --git a/tests/integration/test_config_yaml_main/configs/config.d/access_control.xml b/tests/integration/test_config_yaml_main/configs/config.d/access_control.xml
new file mode 100644
index 00000000000..b61f89bd904
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/config.d/access_control.xml
@@ -0,0 +1,13 @@
+<yandex>
+    <!-- Sources to read users, roles, access rights, profiles of settings, quotas. -->
+    <user_directories replace="replace">
+        <users_xml>
+            <!-- Path to configuration file with predefined users. -->
+            <path>users.yaml</path>
+        </users_xml>
+        <local_directory>
+            <!-- Path to folder where users created by SQL commands are stored. -->
+            <path>access/</path>
+        </local_directory>
+    </user_directories>
+</yandex>
diff --git a/tests/integration/test_config_yaml_main/configs/config.d/keeper_port.xml b/tests/integration/test_config_yaml_main/configs/config.d/keeper_port.xml
new file mode 100644
index 00000000000..b21df47bc85
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/config.d/keeper_port.xml
@@ -0,0 +1,23 @@
+<yandex>
+    <keeper_server>
+        <tcp_port>9181</tcp_port>
+        <server_id>1</server_id>
+
+        <coordination_settings>
+            <operation_timeout_ms>10000</operation_timeout_ms>
+            <session_timeout_ms>30000</session_timeout_ms>
+            <force_sync>false</force_sync>
+            <startup_timeout>60000</startup_timeout>
+            <!-- we want all logs for complex problems investigation -->
+            <reserved_log_items>1000000000000000</reserved_log_items>
+        </coordination_settings>
+
+        <raft_configuration>
+            <server>
+                <id>1</id>
+                <hostname>localhost</hostname>
+                <port>44444</port>
+            </server>
+        </raft_configuration>
+    </keeper_server>
+</yandex>
diff --git a/tests/integration/test_config_yaml_main/configs/config.d/log_to_console.xml b/tests/integration/test_config_yaml_main/configs/config.d/log_to_console.xml
new file mode 100644
index 00000000000..227c53647f3
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/config.d/log_to_console.xml
@@ -0,0 +1,7 @@
+<yandex>
+    <logger>
+        <console>true</console>
+        <log remove="remove"/>
+        <errorlog remove="remove"/>
+    </logger>
+</yandex>
diff --git a/tests/integration/test_config_yaml_main/configs/config.d/logging_no_rotate.xml b/tests/integration/test_config_yaml_main/configs/config.d/logging_no_rotate.xml
new file mode 100644
index 00000000000..2c34585437b
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/config.d/logging_no_rotate.xml
@@ -0,0 +1,8 @@
+<yandex>
+    <logger>
+        <!-- Disable rotation
+             https://pocoproject.org/docs/Poco.FileChannel.html
+        -->
+        <size>never</size>
+    </logger>
+</yandex>
diff --git a/tests/integration/test_config_yaml_main/configs/config.d/macros.xml b/tests/integration/test_config_yaml_main/configs/config.d/macros.xml
new file mode 100644
index 00000000000..4902b12bc81
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/config.d/macros.xml
@@ -0,0 +1,9 @@
+<yandex>
+    <macros>
+        <test>Hello, world!</test>
+        <shard>s1</shard>
+        <replica>r1</replica>
+        <default_path_test>/clickhouse/tables/{database}/{shard}/</default_path_test>
+        <default_name_test>table_{table}</default_name_test>
+    </macros>
+</yandex>
diff --git a/tests/integration/test_config_yaml_main/configs/config.d/metric_log.xml b/tests/integration/test_config_yaml_main/configs/config.d/metric_log.xml
new file mode 100644
index 00000000000..0ca9f162416
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/config.d/metric_log.xml
@@ -0,0 +1,8 @@
+<yandex>
+    <metric_log>
+        <database>system</database>
+        <table>metric_log</table>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+        <collect_interval_milliseconds>1000</collect_interval_milliseconds>
+    </metric_log>
+</yandex>
diff --git a/tests/integration/test_config_yaml_main/configs/config.d/more_clusters.xml b/tests/integration/test_config_yaml_main/configs/config.d/more_clusters.xml
new file mode 100644
index 00000000000..aecbf9e0ba7
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/config.d/more_clusters.xml
@@ -0,0 +1,49 @@
+<yandex>
+    <remote_servers>
+
+        <![CDATA[
+            You can run additional servers simply as
+             ./clickhouse-server -- --path=9001 --tcp_port=9001
+        ]]>
+
+        <single_remote_shard_at_port_9001>
+            <shard>
+                <replica>
+                    <host>localhost</host>
+                    <port>9001</port>
+                </replica>
+            </shard>
+        </single_remote_shard_at_port_9001>
+
+        <two_remote_shards_at_port_9001_9002>
+            <shard>
+                <replica>
+                    <host>localhost</host>
+                    <port>9001</port>
+                </replica>
+            </shard>
+            <shard>
+                <replica>
+                    <host>localhost</host>
+                    <port>9002</port>
+                </replica>
+            </shard>
+        </two_remote_shards_at_port_9001_9002>
+
+        <two_shards_one_local_one_remote_at_port_9001>
+            <shard>
+                <replica>
+                    <host>localhost</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+            <shard>
+                <replica>
+                    <host>localhost</host>
+                    <port>9001</port>
+                </replica>
+            </shard>
+        </two_shards_one_local_one_remote_at_port_9001>
+
+    </remote_servers>
+</yandex>
diff --git a/tests/integration/test_config_yaml_main/configs/config.d/part_log.xml b/tests/integration/test_config_yaml_main/configs/config.d/part_log.xml
new file mode 100644
index 00000000000..6c6fc9c6982
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/config.d/part_log.xml
@@ -0,0 +1,8 @@
+<yandex>
+    <part_log>
+        <database>system</database>
+        <table>part_log</table>
+
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </part_log>
+</yandex>
diff --git a/tests/integration/test_config_yaml_main/configs/config.d/path.xml b/tests/integration/test_config_yaml_main/configs/config.d/path.xml
new file mode 100644
index 00000000000..466ed0d1663
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/config.d/path.xml
@@ -0,0 +1,8 @@
+<yandex>
+    <path replace="replace">./</path>
+    <tmp_path replace="replace">./tmp/</tmp_path>
+    <user_files_path replace="replace">./user_files/</user_files_path>
+    <format_schema_path replace="replace">./format_schemas/</format_schema_path>
+    <access_control_path replace="replace">./access/</access_control_path>
+    <top_level_domains_path replace="replace">./top_level_domains/</top_level_domains_path>
+</yandex>
diff --git a/tests/integration/test_config_yaml_main/configs/config.d/query_masking_rules.xml b/tests/integration/test_config_yaml_main/configs/config.d/query_masking_rules.xml
new file mode 100644
index 00000000000..5a854848f3d
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/config.d/query_masking_rules.xml
@@ -0,0 +1,10 @@
+<?xml version="1.0"?>
+<!-- Config for test server -->
+<yandex>
+    <query_masking_rules>
+        <rule>
+            <regexp>TOPSECRET.TOPSECRET</regexp>
+            <replace>[hidden]</replace>
+        </rule>
+    </query_masking_rules>
+</yandex>
diff --git a/tests/integration/test_config_yaml_main/configs/config.d/tcp_with_proxy.xml b/tests/integration/test_config_yaml_main/configs/config.d/tcp_with_proxy.xml
new file mode 100644
index 00000000000..19046054c16
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/config.d/tcp_with_proxy.xml
@@ -0,0 +1,3 @@
+<yandex>
+    <tcp_with_proxy_port>9010</tcp_with_proxy_port>
+</yandex>
diff --git a/tests/integration/test_config_yaml_main/configs/config.d/test_cluster_with_incorrect_pw.xml b/tests/integration/test_config_yaml_main/configs/config.d/test_cluster_with_incorrect_pw.xml
new file mode 100644
index 00000000000..109e35afc37
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/config.d/test_cluster_with_incorrect_pw.xml
@@ -0,0 +1,21 @@
+<yandex>
+    <remote_servers>
+        <test_cluster_with_incorrect_pw>
+             <shard>
+                 <internal_replication>true</internal_replication>
+                 <replica>
+                     <host>127.0.0.1</host>
+                     <port>9000</port>
+                     <!-- password is incorrect -->
+                     <password>foo</password>
+                 </replica>
+                 <replica>
+                     <host>127.0.0.2</host>
+                     <port>9000</port>
+                     <!-- password is incorrect -->
+                     <password>foo</password>
+                 </replica>
+             </shard>
+         </test_cluster_with_incorrect_pw>
+    </remote_servers>
+</yandex>
diff --git a/tests/integration/test_config_yaml_main/configs/config.d/text_log.xml b/tests/integration/test_config_yaml_main/configs/config.d/text_log.xml
new file mode 100644
index 00000000000..3699a23578c
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/config.d/text_log.xml
@@ -0,0 +1,7 @@
+<yandex>
+    <text_log>
+        <database>system</database>
+        <table>text_log</table>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </text_log>
+</yandex>
diff --git a/tests/integration/test_config_yaml_main/configs/config.d/zookeeper.xml b/tests/integration/test_config_yaml_main/configs/config.d/zookeeper.xml
new file mode 100644
index 00000000000..06ed7fcd39f
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/config.d/zookeeper.xml
@@ -0,0 +1,8 @@
+<yandex>
+    <zookeeper>
+        <node index="1">
+            <host>localhost</host>
+            <port>9181</port>
+        </node>
+    </zookeeper>
+</yandex>
diff --git a/tests/integration/test_config_yaml_main/configs/config.yaml b/tests/integration/test_config_yaml_main/configs/config.yaml
new file mode 100644
index 00000000000..e5a36b1e49b
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/config.yaml
@@ -0,0 +1,183 @@
+logger:
+  level: trace
+  log: /var/log/clickhouse-server/clickhouse-server.log
+  errorlog: /var/log/clickhouse-server/clickhouse-server.err.log
+  size: 1000M
+  count: 10
+http_port: 8123
+tcp_port: 9000
+mysql_port: 9004
+postgresql_port: 9005
+interserver_http_port: 9009
+max_connections: 4096
+keep_alive_timeout: 3
+grpc:
+  enable_ssl: false
+  ssl_cert_file: /path/to/ssl_cert_file
+  ssl_key_file: /path/to/ssl_key_file
+  ssl_require_client_auth: false
+  ssl_ca_cert_file: /path/to/ssl_ca_cert_file
+  compression: deflate
+  compression_level: medium
+  max_send_message_size: -1
+  max_receive_message_size: -1
+  verbose_logs: false
+openSSL:
+  server:
+    certificateFile: /etc/clickhouse-server/server.crt
+    privateKeyFile: /etc/clickhouse-server/server.key
+    dhParamsFile: /etc/clickhouse-server/dhparam.pem
+    verificationMode: none
+    loadDefaultCAFile: true
+    cacheSessions: true
+    disableProtocols: 'sslv2,sslv3'
+    preferServerCiphers: true
+  client:
+    loadDefaultCAFile: true
+    cacheSessions: true
+    disableProtocols: 'sslv2,sslv3'
+    preferServerCiphers: true
+    invalidCertificateHandler:
+      name: RejectCertificateHandler
+max_concurrent_queries: 100
+max_server_memory_usage: 0
+max_thread_pool_size: 10000
+max_server_memory_usage_to_ram_ratio: 0.9
+total_memory_profiler_step: 4194304
+total_memory_tracker_sample_probability: 0
+uncompressed_cache_size: 8589934592
+mark_cache_size: 5368709120
+mmap_cache_size: 1000
+path: /var/lib/clickhouse/
+tmp_path: /var/lib/clickhouse/tmp/
+user_files_path: /var/lib/clickhouse/user_files/
+ldap_servers: ''
+user_directories:
+  users_xml:
+    path: users.yml
+  local_directory:
+    path: /var/lib/clickhouse/access/
+default_profile: default
+custom_settings_prefixes: ''
+default_database: default
+mlock_executable: true
+remap_executable: false
+remote_servers:
+  test_shard_localhost:
+    shard:
+      replica:
+        host: localhost
+        port: 9000
+  test_cluster_two_shards_localhost:
+    shard:
+      - replica:
+          host: localhost
+          port: 9000
+      - replica:
+          host: localhost
+          port: 9000
+  test_cluster_two_shards:
+    shard:
+      - replica:
+          host: 127.0.0.1
+          port: 9000
+      - replica:
+          host: 127.0.0.2
+          port: 9000
+  test_cluster_two_shards_internal_replication:
+    shard:
+      - internal_replication: true
+        replica:
+          host: 127.0.0.1
+          port: 9000
+      - internal_replication: true
+        replica:
+          host: 127.0.0.2
+          port: 9000
+  test_shard_localhost_secure:
+    shard:
+      replica:
+        host: localhost
+        port: 9440
+        secure: 1
+  test_unavailable_shard:
+    shard:
+      - replica:
+          host: localhost
+          port: 9000
+      - replica:
+          host: localhost
+          port: 1
+builtin_dictionaries_reload_interval: 3600
+max_session_timeout: 3600
+default_session_timeout: 60
+query_log:
+  database: system
+  table: query_log
+  partition_by: toYYYYMM(event_date)
+  flush_interval_milliseconds: 7500
+trace_log:
+  database: system
+  table: trace_log
+  partition_by: toYYYYMM(event_date)
+  flush_interval_milliseconds: 7500
+query_thread_log:
+  database: system
+  table: query_thread_log
+  partition_by: toYYYYMM(event_date)
+  flush_interval_milliseconds: 7500
+metric_log:
+  database: system
+  table: metric_log
+  flush_interval_milliseconds: 7500
+  collect_interval_milliseconds: 1000
+asynchronous_metric_log:
+  database: system
+  table: asynchronous_metric_log
+  flush_interval_milliseconds: 60000
+opentelemetry_span_log:
+  engine: |-
+    engine MergeTree
+                partition by toYYYYMM(finish_date)
+                order by (finish_date, finish_time_us, trace_id)
+  database: system
+  table: opentelemetry_span_log
+  flush_interval_milliseconds: 7500
+crash_log:
+  database: system
+  table: crash_log
+  partition_by: ''
+  flush_interval_milliseconds: 1000
+top_level_domains_lists: ''
+dictionaries_config: '*_dictionary.xml'
+distributed_ddl:
+  path: /clickhouse/task_queue/ddl
+graphite_rollup_example:
+  pattern:
+    regexp: click_cost
+    function: any
+    retention:
+      - age: 0
+        precision: 3600
+      - age: 86400
+        precision: 60
+  default:
+    function: max
+    retention:
+      - age: 0
+        precision: 60
+      - age: 3600
+        precision: 300
+      - age: 86400
+        precision: 3600
+format_schema_path: /var/lib/clickhouse/format_schemas/
+query_masking_rules:
+  rule:
+    name: hide encrypt/decrypt arguments
+    regexp: '((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:''(?:\\''|.)+''|.*?)\s*\)'
+    replace: \1(???)
+send_crash_reports:
+  enabled: false
+  anonymize: false
+  endpoint: 'https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277'
+
diff --git a/tests/integration/test_config_yaml_main/configs/embedded.xml b/tests/integration/test_config_yaml_main/configs/embedded.xml
new file mode 100644
index 00000000000..a66f57d1eb7
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/embedded.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0"?>
+<!-- Config that is used when server is run without config file. -->
+<yandex>
+    <logger>
+        <level>trace</level>
+        <console>true</console>
+    </logger>
+
+    <http_port>8123</http_port>
+    <tcp_port>9000</tcp_port>
+    <mysql_port>9004</mysql_port>
+
+    <path>./</path>
+
+    <uncompressed_cache_size>8589934592</uncompressed_cache_size>
+    <mark_cache_size>5368709120</mark_cache_size>
+    <mlock_executable>true</mlock_executable>
+
+    <users>
+        <default>
+            <password></password>
+
+            <networks>
+                <ip>::/0</ip>
+            </networks>
+
+            <profile>default</profile>
+            <quota>default</quota>
+            <access_management>1</access_management>
+        </default>
+    </users>
+
+    <profiles>
+        <default/>
+    </profiles>
+
+    <quotas>
+        <default />
+    </quotas>
+</yandex>
diff --git a/tests/integration/test_config_yaml_main/configs/users.d/allow_introspection_functions.xml b/tests/integration/test_config_yaml_main/configs/users.d/allow_introspection_functions.xml
new file mode 100644
index 00000000000..b94e95bc043
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/users.d/allow_introspection_functions.xml
@@ -0,0 +1,8 @@
+<?xml version="1.0"?>
+<yandex>
+    <profiles>
+        <default>
+            <allow_introspection_functions>1</allow_introspection_functions>
+        </default>
+    </profiles>
+</yandex>
diff --git a/tests/integration/test_config_yaml_main/configs/users.d/log_queries.xml b/tests/integration/test_config_yaml_main/configs/users.d/log_queries.xml
new file mode 100644
index 00000000000..25261072ade
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/users.d/log_queries.xml
@@ -0,0 +1,7 @@
+<yandex>
+    <profiles>
+        <default>
+            <log_queries>1</log_queries>
+        </default>
+    </profiles>
+</yandex>
diff --git a/tests/integration/test_config_yaml_main/configs/users.yaml b/tests/integration/test_config_yaml_main/configs/users.yaml
new file mode 100644
index 00000000000..a87a8c82819
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/users.yaml
@@ -0,0 +1,12 @@
+profiles:
+  default:
+    max_memory_usage: 10000000000
+    max_block_size: 64999
+users:
+  default:
+    password: ''
+    networks:
+      "@replace": replace
+      ip: '::/0'
+    profile: default
+
diff --git a/tests/integration/test_config_yaml_main/test.py b/tests/integration/test_config_yaml_main/test.py
new file mode 100644
index 00000000000..f4de16c35a2
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/test.py
@@ -0,0 +1,43 @@
+import time
+import threading
+from os import path as p, unlink
+from tempfile import NamedTemporaryFile
+
+import helpers
+import pytest
+from helpers.cluster import ClickHouseCluster
+
+
+def test_yaml_main_conf():
+    # main configs are in YAML; config.d and users.d are in XML
+    cluster = ClickHouseCluster(__file__, zookeeper_config_path='configs/config.d/zookeeper.xml')
+
+    all_confd = ['configs/config.d/0_common_instance_config.yaml',
+                'configs/config.d/access_control.xml',
+                'configs/config.d/keeper_port.xml',
+                'configs/config.d/logging_no_rotate.xml',
+                'configs/config.d/log_to_console.xml',
+                'configs/config.d/macros.xml',
+                'configs/config.d/metric_log.xml',
+                'configs/config.d/more_clusters.xml',
+                'configs/config.d/part_log.xml',
+                'configs/config.d/path.xml',
+                'configs/config.d/query_masking_rules.xml',
+                'configs/config.d/tcp_with_proxy.xml',
+                'configs/config.d/test_cluster_with_incorrect_pw.xml',
+                'configs/config.d/text_log.xml',
+                'configs/config.d/zookeeper.xml']
+
+    all_userd = ['configs/users.d/allow_introspection_functions.xml',
+                'configs/users.d/log_queries.xml']
+
+    node = cluster.add_instance('node', base_config_dir='configs', main_configs=all_confd, user_configs=all_userd,
+    with_zookeeper=False, main_config_name="config.yaml", users_config_name="users.yaml", copy_common_configs=False)
+
+    try:
+        cluster.start()
+        assert(node.query("select value from system.settings where name = 'max_memory_usage'") == "10000000000\n")
+        assert(node.query("select value from system.settings where name = 'max_block_size'") == "64999\n")
+
+    finally:
+        cluster.shutdown()

From 0536db0ec304c7aa238fb70bcb440f5712a7cd4e Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Thu, 20 May 2021 08:13:17 +0300
Subject: [PATCH 71/76] Fix resolving of IPv6 addresses

v2: move the check from Cluster to DNSResolver
---
 src/Common/DNSResolver.cpp                      | 17 ++++++++++++++---
 .../0_stateless/01880_remote_ipv6.reference     |  0
 tests/queries/0_stateless/01880_remote_ipv6.sql |  8 ++++++++
 3 files changed, 22 insertions(+), 3 deletions(-)
 create mode 100644 tests/queries/0_stateless/01880_remote_ipv6.reference
 create mode 100644 tests/queries/0_stateless/01880_remote_ipv6.sql

diff --git a/src/Common/DNSResolver.cpp b/src/Common/DNSResolver.cpp
index f54d3e72df5..8b006bc550d 100644
--- a/src/Common/DNSResolver.cpp
+++ b/src/Common/DNSResolver.cpp
@@ -87,9 +87,20 @@ static DNSResolver::IPAddresses resolveIPAddressImpl(const std::string & host)
 {
     Poco::Net::IPAddress ip;
 
-    /// NOTE: Poco::Net::DNS::resolveOne(host) doesn't work for IP addresses like 127.0.0.2
-    if (Poco::Net::IPAddress::tryParse(host, ip))
-        return DNSResolver::IPAddresses(1, ip);
+    /// NOTE:
+    /// - Poco::Net::DNS::resolveOne(host) doesn't work for IP addresses like 127.0.0.2
+    /// - Poco::Net::IPAddress::tryParse() expect hex string for IPv6 (w/o brackets)
+    if (host.starts_with('['))
+    {
+        assert(host.ends_with(']'));
+        if (Poco::Net::IPAddress::tryParse(host.substr(1, host.size() - 2), ip))
+            return DNSResolver::IPAddresses(1, ip);
+    }
+    else
+    {
+        if (Poco::Net::IPAddress::tryParse(host, ip))
+            return DNSResolver::IPAddresses(1, ip);
+    }
 
     /// Family: AF_UNSPEC
     /// AI_ALL is required for checking if client is allowed to connect from an address
diff --git a/tests/queries/0_stateless/01880_remote_ipv6.reference b/tests/queries/0_stateless/01880_remote_ipv6.reference
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/queries/0_stateless/01880_remote_ipv6.sql b/tests/queries/0_stateless/01880_remote_ipv6.sql
new file mode 100644
index 00000000000..11bdf850ec0
--- /dev/null
+++ b/tests/queries/0_stateless/01880_remote_ipv6.sql
@@ -0,0 +1,8 @@
+SET connections_with_failover_max_tries=0;
+
+SELECT * FROM remote('[::1]', system.one) FORMAT Null;
+SELECT * FROM remote('[::1]:9000', system.one) FORMAT Null;
+
+SELECT * FROM remote('[::1', system.one) FORMAT Null; -- { serverError 36 }
+SELECT * FROM remote('::1]', system.one) FORMAT Null; -- { serverError 519 }
+SELECT * FROM remote('::1', system.one) FORMAT Null; -- { serverError 519 }

From 09020242b212f5066329cd572aae28267e45bf5f Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Thu, 20 May 2021 21:02:45 +0300
Subject: [PATCH 72/76] parseAddress: improve parsing of port (by using
 tryParse() over parse())

---
 src/Common/parseAddress.cpp | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/Common/parseAddress.cpp b/src/Common/parseAddress.cpp
index 664e0ca2f59..465c424166e 100644
--- a/src/Common/parseAddress.cpp
+++ b/src/Common/parseAddress.cpp
@@ -35,7 +35,12 @@ std::pair<std::string, UInt16> parseAddress(const std::string & str, UInt16 defa
 
     if (port != end)
     {
-        UInt16 port_number = parse<UInt16>(port + 1);
+        UInt16 port_number;
+        if (!tryParse<UInt16>(port_number, port + 1))
+        {
+            throw Exception(ErrorCodes::BAD_ARGUMENTS,
+                "Illegal port passed to function parseAddress: {}", port + 1);
+        }
         return { std::string(begin, port), port_number };
     }
     else if (default_port)

From 44c85edae043f136b9194845d97cb7903031e4be Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Thu, 20 May 2021 21:02:45 +0300
Subject: [PATCH 73/76] parseAddress: improve parsing of port for IPv6

---
 src/Common/parseAddress.cpp                     | 6 +++++-
 tests/queries/0_stateless/01880_remote_ipv6.sql | 4 ++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/Common/parseAddress.cpp b/src/Common/parseAddress.cpp
index 465c424166e..b398f5cff10 100644
--- a/src/Common/parseAddress.cpp
+++ b/src/Common/parseAddress.cpp
@@ -28,13 +28,17 @@ std::pair<std::string, UInt16> parseAddress(const std::string & str, UInt16 defa
             throw Exception("Illegal address passed to function parseAddress: "
                 "the address begins with opening square bracket, but no closing square bracket found", ErrorCodes::BAD_ARGUMENTS);
 
-        port = find_first_symbols<':'>(closing_square_bracket + 1, end);
+        port = closing_square_bracket + 1;
     }
     else
         port = find_first_symbols<':'>(begin, end);
 
     if (port != end)
     {
+        if (*port != ':')
+            throw Exception(ErrorCodes::BAD_ARGUMENTS,
+                "Illegal port prefix passed to function parseAddress: {}", port);
+
         UInt16 port_number;
         if (!tryParse<UInt16>(port_number, port + 1))
         {
diff --git a/tests/queries/0_stateless/01880_remote_ipv6.sql b/tests/queries/0_stateless/01880_remote_ipv6.sql
index 11bdf850ec0..9d9c25addbf 100644
--- a/tests/queries/0_stateless/01880_remote_ipv6.sql
+++ b/tests/queries/0_stateless/01880_remote_ipv6.sql
@@ -6,3 +6,7 @@ SELECT * FROM remote('[::1]:9000', system.one) FORMAT Null;
 SELECT * FROM remote('[::1', system.one) FORMAT Null; -- { serverError 36 }
 SELECT * FROM remote('::1]', system.one) FORMAT Null; -- { serverError 519 }
 SELECT * FROM remote('::1', system.one) FORMAT Null; -- { serverError 519 }
+
+SELECT * FROM remote('[::1][::1]', system.one) FORMAT Null; -- { serverError 36 }
+SELECT * FROM remote('[::1][::1', system.one) FORMAT Null; -- { serverError 36 }
+SELECT * FROM remote('[::1]::1]', system.one) FORMAT Null; -- { serverError 519 }

From 1eceb5a8d39067c5ef1926be8bdbdc4e4450a8f8 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Thu, 20 May 2021 21:02:45 +0300
Subject: [PATCH 74/76] parseAddress: improve port parsing (verify that port is
 a valid int)

---
 src/Common/parseAddress.cpp                     | 9 ++++++---
 tests/queries/0_stateless/01880_remote_ipv6.sql | 6 +++---
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/Common/parseAddress.cpp b/src/Common/parseAddress.cpp
index b398f5cff10..c99c08896ab 100644
--- a/src/Common/parseAddress.cpp
+++ b/src/Common/parseAddress.cpp
@@ -39,13 +39,16 @@ std::pair<std::string, UInt16> parseAddress(const std::string & str, UInt16 defa
             throw Exception(ErrorCodes::BAD_ARGUMENTS,
                 "Illegal port prefix passed to function parseAddress: {}", port);
 
+        ++port;
+
         UInt16 port_number;
-        if (!tryParse<UInt16>(port_number, port + 1))
+        ReadBufferFromMemory port_buf(port, end - port);
+        if (!tryReadText<UInt16>(port_number, port_buf) || !port_buf.eof())
         {
             throw Exception(ErrorCodes::BAD_ARGUMENTS,
-                "Illegal port passed to function parseAddress: {}", port + 1);
+                "Illegal port passed to function parseAddress: {}", port);
         }
-        return { std::string(begin, port), port_number };
+        return { std::string(begin, port - 1), port_number };
     }
     else if (default_port)
     {
diff --git a/tests/queries/0_stateless/01880_remote_ipv6.sql b/tests/queries/0_stateless/01880_remote_ipv6.sql
index 9d9c25addbf..057b3ad7ec6 100644
--- a/tests/queries/0_stateless/01880_remote_ipv6.sql
+++ b/tests/queries/0_stateless/01880_remote_ipv6.sql
@@ -4,9 +4,9 @@ SELECT * FROM remote('[::1]', system.one) FORMAT Null;
 SELECT * FROM remote('[::1]:9000', system.one) FORMAT Null;
 
 SELECT * FROM remote('[::1', system.one) FORMAT Null; -- { serverError 36 }
-SELECT * FROM remote('::1]', system.one) FORMAT Null; -- { serverError 519 }
-SELECT * FROM remote('::1', system.one) FORMAT Null; -- { serverError 519 }
+SELECT * FROM remote('::1]', system.one) FORMAT Null; -- { serverError 36 }
+SELECT * FROM remote('::1', system.one) FORMAT Null; -- { serverError 36 }
 
 SELECT * FROM remote('[::1][::1]', system.one) FORMAT Null; -- { serverError 36 }
 SELECT * FROM remote('[::1][::1', system.one) FORMAT Null; -- { serverError 36 }
-SELECT * FROM remote('[::1]::1]', system.one) FORMAT Null; -- { serverError 519 }
+SELECT * FROM remote('[::1]::1]', system.one) FORMAT Null; -- { serverError 36 }

From fbe8b37ff6c44931a3da35b2593a150e9b4dd5d6 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Fri, 21 May 2021 22:47:55 +0300
Subject: [PATCH 75/76] Skip 01880_remote_ipv6 in arcadia

---
 tests/queries/0_stateless/arcadia_skip_list.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/queries/0_stateless/arcadia_skip_list.txt b/tests/queries/0_stateless/arcadia_skip_list.txt
index 4e8ac456455..98c3a04e2ce 100644
--- a/tests/queries/0_stateless/arcadia_skip_list.txt
+++ b/tests/queries/0_stateless/arcadia_skip_list.txt
@@ -236,3 +236,4 @@
 01801_s3_distributed
 01833_test_collation_alvarotuso
 01850_dist_INSERT_preserve_error
+01880_remote_ipv6

From 3aa1dba6e1a01e0979e755aa19a5c10449d1d1f8 Mon Sep 17 00:00:00 2001
From: Maksim Kita <kitaetoya@gmail.com>
Date: Sat, 22 May 2021 00:13:27 +0300
Subject: [PATCH 76/76] Added AggregateFunctionSegmentLengthSum to ya.make

---
 src/AggregateFunctions/ya.make | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/AggregateFunctions/ya.make b/src/AggregateFunctions/ya.make
index 2091341887b..c31376219af 100644
--- a/src/AggregateFunctions/ya.make
+++ b/src/AggregateFunctions/ya.make
@@ -43,6 +43,7 @@ SRCS(
     AggregateFunctionRankCorrelation.cpp
     AggregateFunctionResample.cpp
     AggregateFunctionRetention.cpp
+    AggregateFunctionSegmentLengthSum.cpp
     AggregateFunctionSequenceMatch.cpp
     AggregateFunctionSimpleLinearRegression.cpp
     AggregateFunctionSimpleState.cpp