From 50df893dc63b6af266d809e2d82fd92a332c5c5d Mon Sep 17 00:00:00 2001
From: Alexander Kuzmenkov <akuzm@yandex-team.ru>
Date: Fri, 15 May 2020 19:23:31 +0300
Subject: [PATCH 001/184] Assert that allocator & container are consistent wrt.
 inline memory

There are lots of places where HashTable is used with
AllocatorWithStackMemory, but the size of allocator inline memory is set
incorrectly, and it's less than the initial HashTable buffer size.
Because of this, HashTable is always allocated on the heap, and the
inline memory becomes a useless dead weight.

For PODArray, we have previously added a helper template that makes sure
these values are in sync, so there was only one such discrepancy left,
in the unit test.
---
 .../AggregateFunctionEntropy.h                | 14 +++------
 .../AggregateFunctionGroupUniqArray.h         | 15 ++++------
 .../AggregateFunctionTopK.h                   | 17 ++---------
 .../QuantileExactWeighted.h                   |  8 ++---
 src/Common/Allocator.h                        | 28 ++++++++++++-----
 src/Common/HashTable/HashTable.h              | 30 +++++++++++++++++++
 src/Common/HashTable/HashTableAllocator.h     |  4 +--
 src/Common/PODArray.h                         |  5 ++++
 src/Common/SpaceSaving.h                      |  7 ++---
 src/Common/tests/pod_array.cpp                | 18 +++++------
 src/Functions/array/arrayDistinct.cpp         | 16 ++++------
 src/Functions/array/arrayEnumerateExtended.h  | 30 ++++++++++++-------
 src/Functions/array/arrayEnumerateRanked.h    | 10 +++----
 src/Functions/array/arrayIntersect.cpp        | 26 ++++++++--------
 src/Functions/array/arrayUniq.cpp             | 25 +++++++++-------
 src/Interpreters/tests/hash_map3.cpp          |  5 ++--
 16 files changed, 145 insertions(+), 113 deletions(-)

diff --git a/src/AggregateFunctions/AggregateFunctionEntropy.h b/src/AggregateFunctions/AggregateFunctionEntropy.h
index 7586cebd8ec..59cacd0e9ca 100644
--- a/src/AggregateFunctions/AggregateFunctionEntropy.h
+++ b/src/AggregateFunctions/AggregateFunctionEntropy.h
@@ -23,18 +23,12 @@ struct EntropyData
 {
     using Weight = UInt64;
 
-    using HashingMap = HashMap<
-        Value, Weight,
-        HashCRC32<Value>,
-        HashTableGrower<4>,
-        HashTableAllocatorWithStackMemory<sizeof(std::pair<Value, Weight>) * (1 << 3)>>;
+    using HashingMap = HASH_TABLE_WITH_STACK_MEMORY(HashMap, Value, Weight,
+        HashCRC32<Value>, HashTableGrower<4>);
 
     /// For the case of pre-hashed values.
-    using TrivialMap = HashMap<
-        Value, Weight,
-        UInt128TrivialHash,
-        HashTableGrower<4>,
-        HashTableAllocatorWithStackMemory<sizeof(std::pair<Value, Weight>) * (1 << 3)>>;
+    using TrivialMap = HASH_TABLE_WITH_STACK_MEMORY(HashMap, Value, Weight,
+        UInt128TrivialHash, HashTableGrower<4>);
 
     using Map = std::conditional_t<std::is_same_v<UInt128, Value>, TrivialMap, HashingMap>;
 
diff --git a/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h b/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h
index 9dbf2c921c2..af97df209f6 100644
--- a/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h
+++ b/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h
@@ -28,12 +28,8 @@ template <typename T>
 struct AggregateFunctionGroupUniqArrayData
 {
     /// When creating, the hash table must be small.
-    using Set = HashSet<
-        T,
-        DefaultHash<T>,
-        HashTableGrower<4>,
-        HashTableAllocatorWithStackMemory<sizeof(T) * (1 << 4)>
-    >;
+    using Set = HASH_TABLE_WITH_STACK_MEMORY(HashSet, T, DefaultHash<T>,
+        HashTableGrower<4>);
 
     Set value;
 };
@@ -126,9 +122,10 @@ public:
 /// Generic implementation, it uses serialized representation as object descriptor.
 struct AggregateFunctionGroupUniqArrayGenericData
 {
-    static constexpr size_t INIT_ELEMS = 2; /// adjustable
-    static constexpr size_t ELEM_SIZE = sizeof(HashSetCellWithSavedHash<StringRef, StringRefHash>);
-    using Set = HashSetWithSavedHash<StringRef, StringRefHash, HashTableGrower<INIT_ELEMS>, HashTableAllocatorWithStackMemory<INIT_ELEMS * ELEM_SIZE>>;
+    static constexpr size_t INITIAL_SIZE_DEGREE = 1; /// adjustable
+
+    using Set = HASH_TABLE_WITH_STACK_MEMORY(HashSetWithSavedHash,
+        StringRef, StringRefHash, HashTableGrower<INITIAL_SIZE_DEGREE>);
 
     Set value;
 };
diff --git a/src/AggregateFunctions/AggregateFunctionTopK.h b/src/AggregateFunctions/AggregateFunctionTopK.h
index 9c5e62bb6d7..23eb0e7ff09 100644
--- a/src/AggregateFunctions/AggregateFunctionTopK.h
+++ b/src/AggregateFunctions/AggregateFunctionTopK.h
@@ -23,13 +23,8 @@ namespace DB
 template <typename T>
 struct AggregateFunctionTopKData
 {
-    using Set = SpaceSaving
-    <
-        T,
-        HashCRC32<T>,
-        HashTableGrower<4>,
-        HashTableAllocatorWithStackMemory<sizeof(T) * (1 << 4)>
-    >;
+    using Set = SpaceSaving<T, HashCRC32<T>>;
+
     Set value;
 };
 
@@ -109,13 +104,7 @@ public:
 /// Generic implementation, it uses serialized representation as object descriptor.
 struct AggregateFunctionTopKGenericData
 {
-    using Set = SpaceSaving
-    <
-        StringRef,
-        StringRefHash,
-        HashTableGrower<4>,
-        HashTableAllocatorWithStackMemory<sizeof(StringRef) * (1 << 4)>
-    >;
+    using Set = SpaceSaving<StringRef, StringRefHash>;
 
     Set value;
 };
diff --git a/src/AggregateFunctions/QuantileExactWeighted.h b/src/AggregateFunctions/QuantileExactWeighted.h
index 6053bddc947..7731ff980fe 100644
--- a/src/AggregateFunctions/QuantileExactWeighted.h
+++ b/src/AggregateFunctions/QuantileExactWeighted.h
@@ -33,12 +33,8 @@ struct QuantileExactWeighted
     using Hasher = std::conditional_t<std::is_same_v<Value, Decimal128>, Int128Hash, HashCRC32<UnderlyingType>>;
 
     /// When creating, the hash table must be small.
-    using Map = HashMap<
-        UnderlyingType, Weight,
-        Hasher,
-        HashTableGrower<4>,
-        HashTableAllocatorWithStackMemory<sizeof(std::pair<Value, Weight>) * (1 << 3)>
-    >;
+    using Map = HASH_TABLE_WITH_STACK_MEMORY(HashMap, UnderlyingType, Weight,
+        Hasher, HashTableGrower<4>);
 
     Map map;
 
diff --git a/src/Common/Allocator.h b/src/Common/Allocator.h
index 9add9299430..43d7e67c4bb 100644
--- a/src/Common/Allocator.h
+++ b/src/Common/Allocator.h
@@ -278,13 +278,15 @@ private:
 
 /** Allocator with optimization to place small memory ranges in automatic memory.
   */
-template <typename Base, size_t N, size_t Alignment>
+template <typename Base, size_t _initial_bytes, size_t Alignment>
 class AllocatorWithStackMemory : private Base
 {
 private:
-    alignas(Alignment) char stack_memory[N];
+    alignas(Alignment) char stack_memory[_initial_bytes];
 
 public:
+    static constexpr size_t initial_bytes = _initial_bytes;
+
     /// Do not use boost::noncopyable to avoid the warning about direct base
     /// being inaccessible due to ambiguity, when derived classes are also
     /// noncopiable (-Winaccessible-base).
@@ -295,10 +297,10 @@ public:
 
     void * alloc(size_t size)
     {
-        if (size <= N)
+        if (size <= initial_bytes)
         {
             if constexpr (Base::clear_memory)
-                memset(stack_memory, 0, N);
+                memset(stack_memory, 0, initial_bytes);
             return stack_memory;
         }
 
@@ -307,18 +309,18 @@ public:
 
     void free(void * buf, size_t size)
     {
-        if (size > N)
+        if (size > initial_bytes)
             Base::free(buf, size);
     }
 
     void * realloc(void * buf, size_t old_size, size_t new_size)
     {
         /// Was in stack_memory, will remain there.
-        if (new_size <= N)
+        if (new_size <= initial_bytes)
             return buf;
 
         /// Already was big enough to not fit in stack_memory.
-        if (old_size > N)
+        if (old_size > initial_bytes)
             return Base::realloc(buf, old_size, new_size, Alignment);
 
         /// Was in stack memory, but now will not fit there.
@@ -330,10 +332,20 @@ public:
 protected:
     static constexpr size_t getStackThreshold()
     {
-        return N;
+        return initial_bytes;
     }
 };
 
+// A constant that gives the number of initially available bytes in
+// the allocator. Used to check that this number is in sync with the
+// initial size of array or hash table that uses the allocator.
+template<typename TAllocator>
+constexpr size_t allocatorInitialBytes = 0;
+
+template<typename Base, size_t initial_bytes, size_t Alignment>
+constexpr size_t allocatorInitialBytes<AllocatorWithStackMemory<
+    Base, initial_bytes, Alignment>> = initial_bytes;
+
 
 #if !__clang__
 #pragma GCC diagnostic pop
diff --git a/src/Common/HashTable/HashTable.h b/src/Common/HashTable/HashTable.h
index 58b7cd81901..2f7272ca8ec 100644
--- a/src/Common/HashTable/HashTable.h
+++ b/src/Common/HashTable/HashTable.h
@@ -308,7 +308,16 @@ struct ZeroValueStorage<false, Cell>
     const Cell * zeroValue() const { return nullptr; }
 };
 
+// These templates give the initial hash table size, so that we can check
+// that it is in sync with initial allocator size.
+template <typename GenericGrower>
+constexpr size_t growerInitialCount = 0;
 
+template <size_t initial_size_degree>
+constexpr size_t growerInitialCount<HashTableGrower<initial_size_degree>>
+    = 1ULL << initial_size_degree;
+
+// The HashTable
 template
 <
     typename Key,
@@ -324,6 +333,17 @@ class HashTable :
     protected Cell::State,
     protected ZeroValueStorage<Cell::need_zero_value_storage, Cell>     /// empty base optimization
 {
+public:
+    // Export the initial buffer sizes for the ease of using allocators with
+    // inline memory.
+    static constexpr size_t initial_buffer_bytes
+        = growerInitialCount<Grower> * sizeof(Cell);
+
+    // If we use an allocator with inline memory, check that the initial
+    // size of the hash table is in sync with the amount of this memory.
+    static_assert(allocatorInitialBytes<Allocator> == 0
+        || allocatorInitialBytes<Allocator> == initial_buffer_bytes);
+
 protected:
     friend class const_iterator;
     friend class iterator;
@@ -1075,3 +1095,13 @@ public:
     }
 #endif
 };
+
+// A helper macro that declares hash table with allocator with stack memory,
+// and the initial size of the allocator is in sync with initial size of the
+// hash table.
+#define HASH_TABLE_WITH_STACK_MEMORY(HASH_TABLE_VARIANT, ...) \
+    HASH_TABLE_VARIANT<__VA_ARGS__, \
+        HashTableAllocatorWithStackMemory< \
+            HASH_TABLE_VARIANT<__VA_ARGS__>::initial_buffer_bytes \
+        > \
+    >
diff --git a/src/Common/HashTable/HashTableAllocator.h b/src/Common/HashTable/HashTableAllocator.h
index 99f9c979685..47e3fdfc4b6 100644
--- a/src/Common/HashTable/HashTableAllocator.h
+++ b/src/Common/HashTable/HashTableAllocator.h
@@ -10,5 +10,5 @@
   */
 using HashTableAllocator = Allocator<true /* clear_memory */, true /* mmap_populate */>;
 
-template <size_t N = 64>
-using HashTableAllocatorWithStackMemory = AllocatorWithStackMemory<HashTableAllocator, N>;
+template <size_t initial_bytes = 64>
+using HashTableAllocatorWithStackMemory = AllocatorWithStackMemory<HashTableAllocator, initial_bytes>;
diff --git a/src/Common/PODArray.h b/src/Common/PODArray.h
index 8fe1f74484e..8b30b314e91 100644
--- a/src/Common/PODArray.h
+++ b/src/Common/PODArray.h
@@ -85,6 +85,11 @@ protected:
 
     static_assert(pad_left <= EmptyPODArraySize && "Left Padding exceeds EmptyPODArraySize. Is the element size too large?");
 
+    // If we are using allocator with inline memory, the minimal size of
+    // array must be in sync with the size of this memory.
+    static_assert(allocatorInitialBytes<TAllocator> == 0
+                  || allocatorInitialBytes<TAllocator> == initial_bytes);
+
     char * c_start          = null;    /// Does not include pad_left.
     char * c_end            = null;
     char * c_end_of_storage = null;    /// Does not include pad_right.
diff --git a/src/Common/SpaceSaving.h b/src/Common/SpaceSaving.h
index 9ad7f6275d6..47b49add060 100644
--- a/src/Common/SpaceSaving.h
+++ b/src/Common/SpaceSaving.h
@@ -67,9 +67,7 @@ private:
 template
 <
     typename TKey,
-    typename Hash = DefaultHash<TKey>,
-    typename Grower = HashTableGrower<>,
-    typename Allocator = HashTableAllocator
+    typename Hash = DefaultHash<TKey>
 >
 class SpaceSaving
 {
@@ -380,7 +378,8 @@ private:
             counter_map[counter->key] = counter;
     }
 
-    using CounterMap = HashMap<TKey, Counter *, Hash, Grower, Allocator>;
+    using CounterMap = HASH_TABLE_WITH_STACK_MEMORY(HashMap,
+        TKey, Counter *, Hash, HashTableGrower<4>);
 
     CounterMap counter_map;
     std::vector<Counter *> counter_list;
diff --git a/src/Common/tests/pod_array.cpp b/src/Common/tests/pod_array.cpp
index de15b485411..6e9634ba3cf 100644
--- a/src/Common/tests/pod_array.cpp
+++ b/src/Common/tests/pod_array.cpp
@@ -18,9 +18,9 @@ static void test1()
 {
     using namespace DB;
 
-    static constexpr size_t initial_size = 8;
-    static constexpr size_t stack_threshold = 32;
-    using Array = PODArray<UInt64, initial_size, AllocatorWithStackMemory<Allocator<false>, stack_threshold>>;
+    static constexpr size_t initial_bytes = 32;
+    using Array = PODArray<UInt64, initial_bytes,
+        AllocatorWithStackMemory<Allocator<false>, initial_bytes>>;
 
     bool res = true;
 
@@ -139,9 +139,9 @@ static void test2()
 {
     using namespace DB;
 
-    static constexpr size_t initial_size = 8;
-    static constexpr size_t stack_threshold = 32;
-    using Array = PODArray<UInt64, initial_size, AllocatorWithStackMemory<Allocator<false>, stack_threshold>>;
+    static constexpr size_t initial_bytes = 32;
+    using Array = PODArray<UInt64, initial_bytes,
+        AllocatorWithStackMemory<Allocator<false>, initial_bytes>>;
 
     bool res = true;
 
@@ -389,9 +389,9 @@ static void test3()
 {
     using namespace DB;
 
-    static constexpr size_t initial_size = 8;
-    static constexpr size_t stack_threshold = 32;
-    using Array = PODArray<UInt64, initial_size, AllocatorWithStackMemory<Allocator<false>, stack_threshold>>;
+    static constexpr size_t initial_bytes = 32;
+    using Array = PODArray<UInt64, initial_bytes,
+        AllocatorWithStackMemory<Allocator<false>, initial_bytes>>;
 
     bool res = true;
 
diff --git a/src/Functions/array/arrayDistinct.cpp b/src/Functions/array/arrayDistinct.cpp
index d24de638865..d3da77e43d2 100644
--- a/src/Functions/array/arrayDistinct.cpp
+++ b/src/Functions/array/arrayDistinct.cpp
@@ -153,10 +153,8 @@ bool FunctionArrayDistinct::executeNumber(
     if (nullable_col)
         src_null_map = &nullable_col->getNullMapData();
 
-    using Set = ClearableHashSet<T,
-        DefaultHash<T>,
-        HashTableGrower<INITIAL_SIZE_DEGREE>,
-        HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(T)>>;
+    using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashSet,
+        T, DefaultHash<T>, HashTableGrower<INITIAL_SIZE_DEGREE>);
 
     Set set;
 
@@ -201,10 +199,8 @@ bool FunctionArrayDistinct::executeString(
 
     ColumnString & res_data_column_string = typeid_cast<ColumnString &>(res_data_col);
 
-    using Set = ClearableHashSet<StringRef,
-        StringRefHash,
-        HashTableGrower<INITIAL_SIZE_DEGREE>,
-        HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(StringRef)>>;
+    using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashSet,
+        StringRef, StringRefHash, HashTableGrower<INITIAL_SIZE_DEGREE>);
 
     const PaddedPODArray<UInt8> * src_null_map = nullptr;
 
@@ -249,8 +245,8 @@ void FunctionArrayDistinct::executeHashed(
     ColumnArray::Offsets & res_offsets,
     const ColumnNullable * nullable_col)
 {
-    using Set = ClearableHashSet<UInt128, UInt128TrivialHash, HashTableGrower<INITIAL_SIZE_DEGREE>,
-        HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(UInt128)>>;
+    using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashSet,
+        UInt128, UInt128TrivialHash, HashTableGrower<INITIAL_SIZE_DEGREE>);
 
     const PaddedPODArray<UInt8> * src_null_map = nullptr;
 
diff --git a/src/Functions/array/arrayEnumerateExtended.h b/src/Functions/array/arrayEnumerateExtended.h
index 69db59e698e..e97766c5827 100644
--- a/src/Functions/array/arrayEnumerateExtended.h
+++ b/src/Functions/array/arrayEnumerateExtended.h
@@ -64,36 +64,46 @@ private:
     template <typename T>
     struct MethodOneNumber
     {
-        using Set = ClearableHashMap<T, UInt32, DefaultHash<T>, HashTableGrower<INITIAL_SIZE_DEGREE>,
-                HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(T)>>;
+        using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap,
+            T, UInt32, DefaultHash<T>,
+            HashTableGrower<INITIAL_SIZE_DEGREE>);
+
         using Method = ColumnsHashing::HashMethodOneNumber<typename Set::value_type, UInt32, T, false>;
     };
 
     struct MethodString
     {
-        using Set = ClearableHashMap<StringRef, UInt32, StringRefHash, HashTableGrower<INITIAL_SIZE_DEGREE>,
-                HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(StringRef)>>;
+        using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap,
+            StringRef, UInt32, StringRefHash,
+            HashTableGrower<INITIAL_SIZE_DEGREE>);
+
         using Method = ColumnsHashing::HashMethodString<typename Set::value_type, UInt32, false, false>;
     };
 
     struct MethodFixedString
     {
-        using Set = ClearableHashMap<StringRef, UInt32, StringRefHash, HashTableGrower<INITIAL_SIZE_DEGREE>,
-                HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(StringRef)>>;
+        using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap,
+            StringRef, UInt32, StringRefHash,
+            HashTableGrower<INITIAL_SIZE_DEGREE>);
+
         using Method = ColumnsHashing::HashMethodFixedString<typename Set::value_type, UInt32, false, false>;
     };
 
     struct MethodFixed
     {
-        using Set = ClearableHashMap<UInt128, UInt32, UInt128HashCRC32, HashTableGrower<INITIAL_SIZE_DEGREE>,
-                HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(UInt128)>>;
+        using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap,
+            UInt128, UInt32, UInt128HashCRC32,
+            HashTableGrower<INITIAL_SIZE_DEGREE>);
+
         using Method = ColumnsHashing::HashMethodKeysFixed<typename Set::value_type, UInt128, UInt32, false, false, false>;
     };
 
     struct MethodHashed
     {
-        using Set =  ClearableHashMap<UInt128, UInt32, UInt128TrivialHash, HashTableGrower<INITIAL_SIZE_DEGREE>,
-                HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(UInt128)>>;
+        using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap,
+            UInt128, UInt32, UInt128TrivialHash,
+            HashTableGrower<INITIAL_SIZE_DEGREE>);
+
         using Method = ColumnsHashing::HashMethodHashed<typename Set::value_type, UInt32, false>;
     };
 
diff --git a/src/Functions/array/arrayEnumerateRanked.h b/src/Functions/array/arrayEnumerateRanked.h
index 133f5d7cb81..1700260dd9f 100644
--- a/src/Functions/array/arrayEnumerateRanked.h
+++ b/src/Functions/array/arrayEnumerateRanked.h
@@ -308,12 +308,10 @@ void FunctionArrayEnumerateRankedExtended<Derived>::executeMethodImpl(
     const size_t depth_to_look = arrays_depths.max_array_depth;
     const auto & offsets = *offsets_by_depth[depth_to_look - 1];
 
-    using Map = ClearableHashMap<
-        UInt128,
-        UInt32,
-        UInt128TrivialHash,
-        HashTableGrower<INITIAL_SIZE_DEGREE>,
-        HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(UInt128)>>;
+    using Map = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap,
+        UInt128, UInt32, UInt128TrivialHash,
+        HashTableGrower<INITIAL_SIZE_DEGREE>);
+
     Map indices;
 
     std::vector<size_t> indices_by_depth(depth_to_look);
diff --git a/src/Functions/array/arrayIntersect.cpp b/src/Functions/array/arrayIntersect.cpp
index cccad7c7a03..acd162974c9 100644
--- a/src/Functions/array/arrayIntersect.cpp
+++ b/src/Functions/array/arrayIntersect.cpp
@@ -418,16 +418,18 @@ void FunctionArrayIntersect::executeImpl(Block & block, const ColumnNumbers & ar
     TypeListNativeNumbers::forEach(NumberExecutor(arrays, not_nullable_nested_return_type, result_column));
     TypeListDecimalNumbers::forEach(DecimalExecutor(arrays, not_nullable_nested_return_type, result_column));
 
-    using DateMap = ClearableHashMap<DataTypeDate::FieldType, size_t, DefaultHash<DataTypeDate::FieldType>,
-            HashTableGrower<INITIAL_SIZE_DEGREE>,
-            HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(DataTypeDate::FieldType)>>;
+    using DateMap = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap,
+        DataTypeDate::FieldType, size_t, DefaultHash<DataTypeDate::FieldType>,
+        HashTableGrower<INITIAL_SIZE_DEGREE>);
 
-    using DateTimeMap = ClearableHashMap<DataTypeDateTime::FieldType, size_t, DefaultHash<DataTypeDateTime::FieldType>,
-            HashTableGrower<INITIAL_SIZE_DEGREE>,
-            HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(DataTypeDateTime::FieldType)>>;
+    using DateTimeMap = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap,
+        DataTypeDateTime::FieldType, size_t,
+        DefaultHash<DataTypeDateTime::FieldType>,
+        HashTableGrower<INITIAL_SIZE_DEGREE>);
 
-    using StringMap = ClearableHashMap<StringRef, size_t, StringRefHash, HashTableGrower<INITIAL_SIZE_DEGREE>,
-            HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(StringRef)>>;
+    using StringMap = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap,
+        StringRef, size_t, StringRefHash,
+        HashTableGrower<INITIAL_SIZE_DEGREE>);
 
     if (!result_column)
     {
@@ -455,8 +457,8 @@ void FunctionArrayIntersect::executeImpl(Block & block, const ColumnNumbers & ar
 template <typename T, size_t>
 void FunctionArrayIntersect::NumberExecutor::operator()()
 {
-    using Map = ClearableHashMap<T, size_t, DefaultHash<T>, HashTableGrower<INITIAL_SIZE_DEGREE>,
-            HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(T)>>;
+    using Map = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap,
+        T, size_t, DefaultHash<T>, HashTableGrower<INITIAL_SIZE_DEGREE>);
 
     if (!result && typeid_cast<const DataTypeNumber<T> *>(data_type.get()))
         result = execute<Map, ColumnVector<T>, true>(arrays, ColumnVector<T>::create());
@@ -465,8 +467,8 @@ void FunctionArrayIntersect::NumberExecutor::operator()()
 template <typename T, size_t>
 void FunctionArrayIntersect::DecimalExecutor::operator()()
 {
-    using Map = ClearableHashMap<T, size_t, DefaultHash<T>, HashTableGrower<INITIAL_SIZE_DEGREE>,
-            HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(T)>>;
+    using Map = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap,
+        T, size_t, DefaultHash<T>, HashTableGrower<INITIAL_SIZE_DEGREE>);
 
     if (!result)
         if (auto * decimal = typeid_cast<const DataTypeDecimal<T> *>(data_type.get()))
diff --git a/src/Functions/array/arrayUniq.cpp b/src/Functions/array/arrayUniq.cpp
index 02129781c13..a83c535d88d 100644
--- a/src/Functions/array/arrayUniq.cpp
+++ b/src/Functions/array/arrayUniq.cpp
@@ -66,36 +66,41 @@ private:
     template <typename T>
     struct MethodOneNumber
     {
-        using Set = ClearableHashSet<T, DefaultHash<T>, HashTableGrower<INITIAL_SIZE_DEGREE>,
-                HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(T)>>;
+        using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashSet,
+            T, DefaultHash<T>, HashTableGrower<INITIAL_SIZE_DEGREE>);
+
         using Method = ColumnsHashing::HashMethodOneNumber<typename Set::value_type, void, T, false>;
     };
 
     struct MethodString
     {
-        using Set = ClearableHashSet<StringRef, StringRefHash, HashTableGrower<INITIAL_SIZE_DEGREE>,
-                HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(StringRef)>>;
+        using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashSet,
+            StringRef, StringRefHash, HashTableGrower<INITIAL_SIZE_DEGREE>);
+
         using Method = ColumnsHashing::HashMethodString<typename Set::value_type, void, false, false>;
     };
 
     struct MethodFixedString
     {
-        using Set = ClearableHashSet<StringRef, StringRefHash, HashTableGrower<INITIAL_SIZE_DEGREE>,
-                HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(StringRef)>>;
+        using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashSet,
+            StringRef, StringRefHash, HashTableGrower<INITIAL_SIZE_DEGREE>);
+
         using Method = ColumnsHashing::HashMethodFixedString<typename Set::value_type, void, false, false>;
     };
 
     struct MethodFixed
     {
-        using Set = ClearableHashSet<UInt128, UInt128HashCRC32, HashTableGrower<INITIAL_SIZE_DEGREE>,
-                HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(UInt128)>>;
+        using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashSet,
+            UInt128, UInt128HashCRC32, HashTableGrower<INITIAL_SIZE_DEGREE>);
+
         using Method = ColumnsHashing::HashMethodKeysFixed<typename Set::value_type, UInt128, void, false, false, false>;
     };
 
     struct MethodHashed
     {
-        using Set = ClearableHashSet<UInt128, UInt128TrivialHash, HashTableGrower<INITIAL_SIZE_DEGREE>,
-                HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(UInt128)>>;
+        using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashSet,
+            UInt128, UInt128TrivialHash, HashTableGrower<INITIAL_SIZE_DEGREE>);
+
         using Method = ColumnsHashing::HashMethodHashed<typename Set::value_type, void, false>;
     };
 
diff --git a/src/Interpreters/tests/hash_map3.cpp b/src/Interpreters/tests/hash_map3.cpp
index 2207edc6cc1..804b084b2a1 100644
--- a/src/Interpreters/tests/hash_map3.cpp
+++ b/src/Interpreters/tests/hash_map3.cpp
@@ -61,12 +61,11 @@ struct Grower : public HashTableGrower<2>
 
 int main(int, char **)
 {
-    using Map = HashMapWithDump<
+    using Map = HASH_TABLE_WITH_STACK_MEMORY(HashMapWithDump,
         StringRef,
         UInt64,
         SimpleHash,
-        Grower,
-        HashTableAllocatorWithStackMemory<4 * 24>>;
+        Grower);
 
     Map map;
 

From 09ea7253cd0466fa29fe4f02414472aaa9cc72b5 Mon Sep 17 00:00:00 2001
From: Alexander Kuzmenkov <akuzm@yandex-team.ru>
Date: Wed, 20 May 2020 05:19:02 +0300
Subject: [PATCH 002/184] performance test for some functions that use hash
 tables

---
 tests/performance/functions_with_hash_tables.xml | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 tests/performance/functions_with_hash_tables.xml

diff --git a/tests/performance/functions_with_hash_tables.xml b/tests/performance/functions_with_hash_tables.xml
new file mode 100644
index 00000000000..3ff2633c4e7
--- /dev/null
+++ b/tests/performance/functions_with_hash_tables.xml
@@ -0,0 +1,8 @@
+<test>
+    <query>select arrayUniq(range(1 + (number % 10) * 10)) from numbers(1000000) format Null</query>
+    <query>select arrayDistinct(range(1 + (number % 10) * 10)) from numbers(1000000) format Null</query>
+    <query>select arrayEnumerateUniq(range(1 + (number % 10) * 10)) from numbers(1000000) format Null</query>
+    <query>select arrayIntersect(range((1 + number % 10)), range(1, (1 + number % 10) + 1)) from numbers(1000000) format Null</query>
+    <query>select groupUniqArray(number / 10) from numbers(1000 * 1000) group by number / 1000 format Null</query>
+    <query>select entropy(number / 10) from numbers(1000 * 1000) group by number / 1000 format Null</query>
+</test>

From 7ad8228854edecce8e20bd8a8db00ea97580d786 Mon Sep 17 00:00:00 2001
From: Alexander Kuzmenkov <akuzm@yandex-team.ru>
Date: Wed, 20 May 2020 05:19:19 +0300
Subject: [PATCH 003/184] [wip] Add memory stats to performance test

---
 docker/test/performance-comparison/compare.sh | 311 +++++++++++-------
 .../test/performance-comparison/entrypoint.sh |   2 +-
 docker/test/performance-comparison/eqmed.sql  |  41 ++-
 docker/test/performance-comparison/perf.py    |  23 +-
 docker/test/performance-comparison/report.py  |  45 +--
 5 files changed, 256 insertions(+), 166 deletions(-)

diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh
index af88a3954d7..c748a09497d 100755
--- a/docker/test/performance-comparison/compare.sh
+++ b/docker/test/performance-comparison/compare.sh
@@ -133,7 +133,7 @@ function run_tests
     fi
 
     # Delete old report files.
-    for x in {test-times,skipped-tests,wall-clock-times,report-thresholds,client-times}.tsv
+    for x in {test-times,wall-clock-times}.tsv
     do
         rm -v "$x" ||:
         touch "$x"
@@ -220,66 +220,117 @@ function get_profiles
 # Build and analyze randomization distribution for all queries.
 function analyze_queries
 {
-rm -v analyze-commands.txt analyze-errors.log all-queries.tsv unstable-queries.tsv ./*-report.tsv raw-queries.tsv client-times.tsv report-thresholds.tsv ||:
+rm -v analyze-commands.txt analyze-errors.log all-queries.tsv unstable-queries.tsv ./*-report.tsv raw-queries.tsv ||:
+rm -rfv analyze ||:
+mkdir analyze ||:
+
+# FIXME This loop builds column definitons from TSVWithNamesAndTypes in an
+# absolutely atrocious way. This should be done by the file() function itself.
+for x in {right,left}-{addresses,{query,query-thread,trace,metric}-log}.tsv
+do
+    paste -d' ' \
+        <(sed -n '1{s/\t/\n/g;p;q}' "$x" | sed 's/\(^.*$\)/"\1"/') \
+        <(sed -n '2{s/\t/\n/g;p;q}' "$x" ) \
+        | tr '\n' ', ' | sed 's/,$//' > "$x.columns"
+done
 
 # Split the raw test output into files suitable for analysis.
 IFS=$'\n'
 for test_file in $(find . -maxdepth 1 -name "*-raw.tsv" -print)
 do
     test_name=$(basename "$test_file" "-raw.tsv")
-    sed -n "s/^query\t//p" < "$test_file" > "$test_name-queries.tsv"
-    sed -n "s/^client-time/$test_name/p" < "$test_file" >> "client-times.tsv"
-    sed -n "s/^report-threshold/$test_name/p" < "$test_file" >> "report-thresholds.tsv"
-    sed -n "s/^skipped/$test_name/p" < "$test_file" >> "skipped-tests.tsv"
+    sed -n "s/^query\t/$test_name\t/p" < "$test_file" >> "analyze/query-runs.tsv"
+    sed -n "s/^client-time/$test_name/p" < "$test_file" >> "analyze/client-times.tsv"
+    sed -n "s/^report-threshold/$test_name/p" < "$test_file" >> "analyze/report-thresholds.tsv"
+    sed -n "s/^skipped/$test_name/p" < "$test_file" >> "analyze/skipped-tests.tsv"
+    sed -n "s/^display-name/$test_name/p" < "$test_file" >> "analyze/query-display-names.tsv"
 done
 unset IFS
 
+# for each query run, prepare array of metrics from query log
+clickhouse-local --query "
+create view query_runs as select * from file('analyze/query-runs.tsv', TSV,
+    'test text, query_index int, query_id text, version UInt8, time float');
+
+create view left_query_log as select *
+    from file('left-query-log.tsv', TSVWithNamesAndTypes,
+        '$(cat "left-query-log.tsv.columns")');
+
+create view right_query_log as select *
+    from file('right-query-log.tsv', TSVWithNamesAndTypes,
+        '$(cat "right-query-log.tsv.columns")');
+
+create table query_metrics engine File(TSV, -- do not add header -- will parse with grep
+        'analyze/query-run-metrics.tsv')
+    as select
+        test, query_index, 0 run, version,
+        [
+            -- FIXME server-reported duration might be significantly less than
+            -- the client-reported one, when there is some slow cleanup that
+            -- happens after we reported the completion to the client (this did
+            -- happen with some queries).
+            -- To fix this, generate a unique query id for each run, that would
+            -- let us match the client and server times.
+            query_duration_ms / toFloat64(1000)
+            , toFloat64(memory_usage)
+            , query_runs.time
+        ] metrics
+    from (
+        select *, 0 version from left_query_log
+        union all
+        select *, 1 version from right_query_log
+    ) query_logs
+    right join query_runs
+    using (query_id, version)
+    ;
+"
+
 # This is a lateral join in bash... please forgive me.
-# We don't have arrayPermute(), so I have to make random permutations with 
+# We don't have arrayPermute(), so I have to make random permutations with
 # `order by rand`, and it becomes really slow if I do it for more than one
 # query. We also don't have lateral joins. So I just put all runs of each
 # query into a separate file, and then compute randomization distribution
 # for each file. I do this in parallel using GNU parallel.
+query_index=1
 IFS=$'\n'
-for test_file in $(find . -maxdepth 1 -name "*-queries.tsv" -print)
+for prefix in $(cut -f1,2 "analyze/query-run-metrics.tsv" | sort | uniq)
 do
-    test_name=$(basename "$test_file" "-queries.tsv")
-    query_index=1
-    for query in $(cut -d'	' -f1 "$test_file" | sort | uniq)
-    do
-        query_prefix="$test_name.q$query_index"
-        query_index=$((query_index + 1))
-        grep -F "$query	" "$test_file" > "$query_prefix.tmp"
-        printf "%s\0\n" \
-            "clickhouse-local \
-                --file \"$query_prefix.tmp\" \
-                --structure 'query text, run int, version UInt32, time float' \
-                --query \"$(cat "$script_dir/eqmed.sql")\" \
-                >> \"$test_name-report.tsv\"" \
-                2>> analyze-errors.log \
-            >> analyze-commands.txt
-    done
+    file="analyze/q$query_index.tmp"
+    grep -F "$prefix	" "analyze/query-run-metrics.tsv" > "$file" &
+    printf "%s\0\n" \
+        "clickhouse-local \
+            --file \"$file\" \
+            --structure 'test text, query text, run int, version UInt8, metrics Array(float)' \
+            --query \"$(cat "$script_dir/eqmed.sql")\" \
+            >> \"analyze/query-reports.tsv\"" \
+            2>> analyze/errors.log \
+        >> analyze/commands.txt
+
+    query_index=$((query_index + 1))
 done
 wait
 unset IFS
 
-parallel --null < analyze-commands.txt
+parallel --joblog analyze/parallel-log.txt --null < analyze/commands.txt
 }
 
 # Analyze results
 function report
 {
-
 rm -r report ||:
 mkdir report ||:
 
-
 rm ./*.{rep,svg} test-times.tsv test-dump.tsv unstable.tsv unstable-query-ids.tsv unstable-query-metrics.tsv changed-perf.tsv unstable-tests.tsv unstable-queries.tsv bad-tests.tsv slow-on-client.tsv all-queries.tsv ||:
 
-cat analyze-errors.log >> report/errors.log ||:
+cat analyze/errors.log >> report/errors.log ||:
 cat profile-errors.log >> report/errors.log ||:
 
 clickhouse-local --query "
+create view query_display_names as select * from
+    file('analyze/query-display-names.tsv', TSV,
+        'test text, query_index int, query_display_name text')
+    ;
+
 create table queries engine File(TSVWithNamesAndTypes, 'report/queries.tsv')
     as select
         -- FIXME Comparison mode doesn't make sense for queries that complete
@@ -296,21 +347,26 @@ create table queries engine File(TSVWithNamesAndTypes, 'report/queries.tsv')
         
         left, right, diff, stat_threshold,
         if(report_threshold > 0, report_threshold, 0.10) as report_threshold,
-        reports.test,
-        query
+        reports.test test, query_index, query_display_name
     from
         (
-            select *,
-                replaceAll(_file, '-report.tsv', '') test
-            from file('*-report.tsv', TSV, 'left float, right float, diff float, stat_threshold float, query text')
+            select left[1] as left, right[1] as right, diff[1] diff,
+                stat_threshold[1] stat_threshold, test, query_index
+            from file ('analyze/query-reports.tsv', TSV,
+                'left Array(float), right Array(float), diff Array(float),
+                    stat_threshold Array(float), test text, query_index int')
         ) reports
-        left join file('report-thresholds.tsv', TSV, 'test text, report_threshold float') thresholds
-        using test
+        left join file('analyze/report-thresholds.tsv', TSV, 'test text, report_threshold float') thresholds
+        on reports.test = thresholds.test
+        left join query_display_names
+        on reports.test = query_display_names.test
+            and reports.query_index = query_display_names.query_index
         ;
 
 -- keep the table in old format so that we can analyze new and old data together
 create table queries_old_format engine File(TSVWithNamesAndTypes, 'queries.rep')
-    as select short, changed_fail, unstable_fail, left, right, diff, stat_threshold, test, query
+    as select short, changed_fail, unstable_fail, left, right, diff,
+        stat_threshold, test, query_display_name query
     from queries
     ;
 
@@ -334,15 +390,16 @@ create table all_query_funs_json engine File(JSON, 'report/all-query-runs.json')
     ;
 
 create table changed_perf_tsv engine File(TSV, 'report/changed-perf.tsv') as
-    select left, right, diff, stat_threshold, changed_fail, test, query from queries where changed_show
-    order by abs(diff) desc;
+    select left, right, diff, stat_threshold, changed_fail, test, query_display_name
+    from queries where changed_show order by abs(diff) desc;
 
 create table unstable_queries_tsv engine File(TSV, 'report/unstable-queries.tsv') as
-    select left, right, diff, stat_threshold, unstable_fail, test, query from queries where unstable_show
-    order by stat_threshold desc;
+    select left, right, diff, stat_threshold, unstable_fail, test, query_display_name
+    from queries where unstable_show order by stat_threshold desc;
 
-create table queries_for_flamegraph engine File(TSVWithNamesAndTypes, 'report/queries-for-flamegraph.tsv') as
-    select query, test from queries where unstable_show or changed_show
+create table queries_for_flamegraph engine File(TSVWithNamesAndTypes,
+        'report/queries-for-flamegraph.tsv') as
+    select test, query_index from queries where unstable_show or changed_show
     ;
 
 create table unstable_tests_tsv engine File(TSV, 'report/bad-tests.tsv') as
@@ -350,23 +407,23 @@ create table unstable_tests_tsv engine File(TSV, 'report/bad-tests.tsv') as
     group by test having s > 0 order by s desc;
 
 create table query_time engine Memory as select *
-    from file('client-times.tsv', TSV, 'test text, query text, client float, server float');
+    from file('analyze/client-times.tsv', TSV,
+        'test text, query_index int, client float, server float');
 
 create table wall_clock engine Memory as select *
     from file('wall-clock-times.tsv', TSV, 'test text, real float, user float, system float');
 
 create table slow_on_client_tsv engine File(TSV, 'report/slow-on-client.tsv') as
-    select client, server, floor(client/server, 3) p, query
-    from query_time where p > 1.02 order by p desc;
+    select client, server, floor(client/server, 3) p, query_display_name
+    from query_time left join query_display_names using (test, query_index)
+    where p > 1.02 order by p desc;
 
 create table test_time engine Memory as
     select test, sum(client) total_client_time,
         maxIf(client, not short) query_max,
         minIf(client, not short) query_min,
-        count(*) queries,
-        sum(short) short_queries
-    from query_time full join queries
-    using test, query
+        count(*) queries, sum(short) short_queries
+    from query_time full join queries using (test, query_index)
     group by test;
 
 create table test_times_tsv engine File(TSV, 'report/test-times.tsv') as
@@ -378,40 +435,80 @@ create table test_times_tsv engine File(TSV, 'report/test-times.tsv') as
         floor(real / queries, 3) avg_real_per_query,
         floor(query_min, 3)
     from test_time
-        -- wall clock times are also measured for skipped tests, so don't
-        -- do full join
-        left join wall_clock using test
+    -- wall clock times are also measured for skipped tests, so don't
+    -- do full join
+    left join wall_clock using test
     order by avg_real_per_query desc;
 
 create table all_tests_tsv engine File(TSV, 'report/all-queries.tsv') as
     select changed_fail, unstable_fail,
         left, right, diff,
         floor(left > right ? left / right : right / left, 3),
-        stat_threshold, test, query
-    from queries order by test, query;
+        stat_threshold, test, query_display_name
+    from queries order by test, query_display_name;
 " 2> >(tee -a report/errors.log 1>&2)
 
-for x in {right,left}-{addresses,{query,query-thread,trace,metric}-log}.tsv
-do
-    # FIXME This loop builds column definitons from TSVWithNamesAndTypes in an
-    # absolutely atrocious way. This should be done by the file() function itself.
-    paste -d' ' \
-        <(sed -n '1{s/\t/\n/g;p;q}' "$x" | sed 's/\(^.*$\)/"\1"/') \
-        <(sed -n '2{s/\t/\n/g;p;q}' "$x" ) \
-        | tr '\n' ', ' | sed 's/,$//' > "$x.columns"
-done
-
+# Prepare source data for metrics and flamegraphs for unstable queries.
 for version in {right,left}
-do
-clickhouse-local --query "
+    do
+    rm -rf data
+    clickhouse-local --query "
 create view queries_for_flamegraph as
     select * from file('report/queries-for-flamegraph.tsv', TSVWithNamesAndTypes,
-        'query text, test text');
+        'test text, query_index int');
+
+create view query_runs as
+    with 0 as left, 1 as right
+    select * from file('analyze/query-runs.tsv', TSV,
+        'test text, query_index int, query_id text, version UInt8, time float')
+    where version = $version
+    ;
+
+create view query_display_names as select * from
+    file('analyze/query-display-names.tsv', TSV,
+        'test text, query_index int, query_display_name text')
+    ;
+
+create table unstable_query_runs engine File(TSVWithNamesAndTypes,
+        'unstable-query-runs.$version.rep') as
+    select test, query_index, query_display_name, query_id
+    from query_runs
+    join queries_for_flamegraph on
+        query_runs.test = queries_for_flamegraph.test
+        and query_runs.query_index = queries_for_flamegraph.query_index
+    left join query_display_names on
+        query_runs.test = query_display_names.test
+        and query_runs.query_index = query_display_names.query_index
+    ;
 
 create view query_log as select *
     from file('$version-query-log.tsv', TSVWithNamesAndTypes,
         '$(cat "$version-query-log.tsv.columns")');
 
+create table unstable_run_metrics engine File(TSVWithNamesAndTypes,
+        'unstable-run-metrics.$version.rep') as
+    select
+        test, query_index, query_id,
+        ProfileEvents.Values value, ProfileEvents.Names metric
+    from query_log array join ProfileEvents
+    join unstable_query_runs using (query_id)
+    ;
+
+create table unstable_run_metrics_2 engine File(TSVWithNamesAndTypes,
+        'unstable-run-metrics-2.$version.rep') as
+    select
+        test, query_index, query_id,
+        v, n
+    from (
+        select
+            test, query_index, query_id,
+            ['memory_usage', 'read_bytes', 'written_bytes', 'query_duration_ms'] n,
+            [memory_usage, read_bytes, written_bytes, query_duration_ms] v
+        from query_log
+        join unstable_query_runs using (query_id)
+    )
+    array join v, n;
+
 create view trace_log as select *
     from file('$version-trace-log.tsv', TSVWithNamesAndTypes,
         '$(cat "$version-trace-log.tsv.columns")');
@@ -423,88 +520,64 @@ create view addresses_src as select *
 create table addresses_join_$version engine Join(any, left, address) as
     select addr address, name from addresses_src;
 
-create table unstable_query_runs engine File(TSVWithNamesAndTypes,
-        'unstable-query-runs.$version.rep') as
-    select query, query_id from query_log
-    where query in (select query from queries_for_flamegraph)
-        and query_id not like 'prewarm %'
-    ;
-
-create table unstable_query_log engine File(Vertical,
-        'unstable-query-log.$version.rep') as
-    select * from query_log
-    where query_id in (select query_id from unstable_query_runs);
-
-create table unstable_run_metrics engine File(TSVWithNamesAndTypes,
-        'unstable-run-metrics.$version.rep') as
-    select ProfileEvents.Values value, ProfileEvents.Names metric, query_id, query
-    from query_log array join ProfileEvents
-    where query_id in (select query_id from unstable_query_runs)
-    ;
-
-create table unstable_run_metrics_2 engine File(TSVWithNamesAndTypes,
-        'unstable-run-metrics-2.$version.rep') as
-    select v, n, query_id, query
-    from
-        (select
-            ['memory_usage', 'read_bytes', 'written_bytes', 'query_duration_ms'] n,
-            [memory_usage, read_bytes, written_bytes, query_duration_ms] v,
-            query,
-            query_id
-        from query_log
-        where query_id in (select query_id from unstable_query_runs))
-    array join n, v;
-
 create table unstable_run_traces engine File(TSVWithNamesAndTypes,
         'unstable-run-traces.$version.rep') as
     select
+        test, query_index, query_id,
         count() value,
-        joinGet(addresses_join_$version, 'name', arrayJoin(trace)) metric,
-        unstable_query_runs.query_id,
-        any(unstable_query_runs.query) query
-    from unstable_query_runs
-    join trace_log on trace_log.query_id = unstable_query_runs.query_id
-    group by unstable_query_runs.query_id, metric
+        joinGet(addresses_join_$version, 'name', arrayJoin(trace)) metric
+    from trace_log
+    join unstable_query_runs using query_id
+    group by test, query_index, query_id, metric
     order by count() desc
     ;
 
 create table metric_devation engine File(TSVWithNamesAndTypes,
         'metric-deviation.$version.rep') as
-    select query, floor((q[3] - q[1])/q[2], 3) d,
-        quantilesExact(0, 0.5, 1)(value) q, metric
-    from (select * from unstable_run_metrics
-        union all select * from unstable_run_traces
-        union all select * from unstable_run_metrics_2) mm
-    join queries_for_flamegraph using query
-    group by query, metric
-    having d > 0.5
-    order by query desc, d desc
+    -- first goes the key used to split the file with grep
+    select test, query_index, query_display_name,
+        d, q, metric
+    from (
+        select
+            test, query_index,
+            floor((q[3] - q[1])/q[2], 3) d,
+            quantilesExact(0, 0.5, 1)(value) q, metric
+        from (select * from unstable_run_metrics
+            union all select * from unstable_run_traces
+            union all select * from unstable_run_metrics_2) mm
+        group by test, query_index, metric
+        having d > 0.5
+    ) metrics
+    left join unstable_query_runs using (test, query_index)
+    order by test, query_index, d desc
     ;
 
 create table stacks engine File(TSV, 'stacks.$version.rep') as
     select
-        query,
+        -- first goes the key used to split the file with grep
+        test, query_index, any(query_display_name),
         arrayStringConcat(
             arrayMap(x -> joinGet(addresses_join_$version, 'name', x),
                 arrayReverse(trace)
             ),
             ';'
         ) readable_trace,
-        count()
+        count() c
     from trace_log
     join unstable_query_runs using query_id
-    group by query, trace
+    group by test, query_index, trace
     ;
 " 2> >(tee -a report/errors.log 1>&2) # do not run in parallel because they use the same data dir for StorageJoins which leads to weird errors.
 done
 wait
 
+# Create per-query flamegraphs and files with metrics
 IFS=$'\n'
 for version in {right,left}
 do
-    for query in $(cut -d'	' -f1 "stacks.$version.rep" | sort | uniq)
+    for query in $(cut -d'	' -f1,2,3 "stacks.$version.rep" | sort | uniq)
     do
-        query_file=$(echo "$query" | cut -c-120 | sed 's/[/]/_/g')
+        query_file=$(echo "$query" | cut -c-120 | sed 's/[/	]/_/g')
 
         # Build separate .svg flamegraph for each query.
         grep -F "$query	" "stacks.$version.rep" \
diff --git a/docker/test/performance-comparison/entrypoint.sh b/docker/test/performance-comparison/entrypoint.sh
index 500dd872c8d..3b8a24ad5da 100755
--- a/docker/test/performance-comparison/entrypoint.sh
+++ b/docker/test/performance-comparison/entrypoint.sh
@@ -119,5 +119,5 @@ done
 
 dmesg -T > dmesg.log
 
-7z a /output/output.7z ./*.{log,tsv,html,txt,rep,svg} {right,left}/{performance,db/preprocessed_configs,scripts} ./report
+7z a /output/output.7z ./*.{log,tsv,html,txt,rep,svg} {right,left}/{performance,db/preprocessed_configs,scripts} report analyze
 cp compare.log /output
diff --git a/docker/test/performance-comparison/eqmed.sql b/docker/test/performance-comparison/eqmed.sql
index 4f8482ea74b..f7f8d6ac40d 100644
--- a/docker/test/performance-comparison/eqmed.sql
+++ b/docker/test/performance-comparison/eqmed.sql
@@ -1,32 +1,37 @@
--- input is table(query text, run UInt32, version int, time float)
+-- input is table(test text, query text, run UInt32, version int, metrics Array(float))
 select
-   floor(original_medians_array.time_by_version[1], 4) l,
-   floor(original_medians_array.time_by_version[2], 4) r,
-   floor((r - l) / l, 3) diff_percent,
-   floor(threshold / l, 3) threshold_percent,
-   query
+   arrayMap(x -> floor(x, 4), original_medians_array.medians_by_version[1] as l) l_rounded,
+   arrayMap(x -> floor(x, 4), original_medians_array.medians_by_version[2] as r) r_rounded,
+   arrayMap(x, y -> floor((y - x) / x, 3), l, r) diff_percent,
+   arrayMap(x, y -> floor(x / y, 3), threshold, l) threshold_percent,
+   test, query
 from
    (
       -- quantiles of randomization distributions
-      select quantileExact(0.999)(abs(time_by_label[1] - time_by_label[2]) as d) threshold
+      select quantileExactForEach(0.999)(
+        arrayMap(x, y -> abs(x - y), metrics_by_label[1], metrics_by_label[2]) as d
+      ) threshold
       ---- uncomment to see what the distribution is really like
-      --, uniqExact(d) u
+      --, uniqExact(d.1) u
       --, arraySort(x->x.1,
       --      arrayZip(
-      --          (sumMap([d], [1]) as f).1,
+      --          (sumMap([d.1], [1]) as f).1,
       --          f.2)) full_histogram
       from
          (
-            select virtual_run, groupArrayInsertAt(median_time, random_label) time_by_label -- make array 'random label' -> 'median time'
+            -- make array 'random label' -> '[median metric]'
+            select virtual_run, groupArrayInsertAt(median_metrics, random_label) metrics_by_label
             from (
-                  select medianExact(time) median_time, virtual_run, random_label -- get median times, grouping by random label
+                  -- get [median metric] arrays among virtual runs, grouping by random label
+                  select medianExactForEach(metrics) median_metrics, virtual_run, random_label
                   from (
-                        select *, toUInt32(rowNumberInAllBlocks() % 2) random_label -- randomly relabel measurements
+                        -- randomly relabel measurements
+                        select *, toUInt32(rowNumberInAllBlocks() % 2) random_label
                         from (
-                              select time, number virtual_run 
+                              select metrics, number virtual_run
                               from
                                 -- strip the query away before the join -- it might be several kB long;
-                                (select time, run, version from table) no_query,
+                                (select metrics, run, version from table) no_query,
                                 -- duplicate input measurements into many virtual runs
                                 numbers(1, 100000) nn
                               -- for each virtual run, randomly reorder measurements
@@ -40,19 +45,19 @@ from
       -- this select aggregates by virtual_run
    ) rd,
    (
-        select groupArrayInsertAt(median_time, version) time_by_version
+        select groupArrayInsertAt(median_metrics, version) medians_by_version
         from
         (
-            select medianExact(time) median_time, version
+            select medianExactForEach(metrics) median_metrics, version
             from table
             group by version
         ) original_medians
    ) original_medians_array,
    (
-        select any(query) query from table
+        select any(test) test, any(query) query from table
    ) any_query,
    (
-       select throwIf(uniq(query) != 1) from table
+       select throwIf(uniq((test, query)) != 1) from table
    ) check_single_query -- this subselect checks that there is only one query in the input table;
                         -- written this way so that it is not optimized away (#10523)
 ;
diff --git a/docker/test/performance-comparison/perf.py b/docker/test/performance-comparison/perf.py
index a749b158e83..be8155cdddb 100755
--- a/docker/test/performance-comparison/perf.py
+++ b/docker/test/performance-comparison/perf.py
@@ -29,6 +29,8 @@ parser.add_argument('--runs', type=int, default=int(os.environ.get('CHPC_RUNS',
 parser.add_argument('--no-long', type=bool, default=True, help='Skip the tests tagged as long.')
 args = parser.parse_args()
 
+test_name = os.path.splitext(os.path.basename(args.file[0].name))[0]
+
 tree = et.parse(args.file[0])
 root = tree.getroot()
 
@@ -141,19 +143,25 @@ test_queries = substitute_parameters(test_query_templates)
 
 report_stage_end('substitute2')
 
-for i, q in enumerate(test_queries):
+for query_index, q in enumerate(test_queries):
+    query_prefix = f'{test_name}.query{query_index}'
+
     # We have some crazy long queries (about 100kB), so trim them to a sane
-    # length.
+    # length. This means we can't use query text as an identifier and have to
+    # use the test name + the test-wide query index.
     query_display_name = q
     if len(query_display_name) > 1000:
         query_display_name = f'{query_display_name[:1000]}...({i})'
 
+    print(f'display-name\t{query_index}\t{tsv_escape(query_display_name)}')
+
     # Prewarm: run once on both servers. Helps to bring the data into memory,
     # precompile the queries, etc.
     try:
         for conn_index, c in enumerate(connections):
-            res = c.execute(q, query_id = f'prewarm {0} {query_display_name}')
-            print(f'prewarm\t{tsv_escape(query_display_name)}\t{conn_index}\t{c.last_query.elapsed}')
+            prewarm_id = f'{query_prefix}.prewarm0'
+            res = c.execute(q, query_id = prewarm_id)
+            print(f'prewarm\t{query_index}\t{prewarm_id}\t{conn_index}\t{c.last_query.elapsed}')
     except:
         # If prewarm fails for some query -- skip it, and try to test the others.
         # This might happen if the new test introduces some function that the
@@ -170,13 +178,14 @@ for i, q in enumerate(test_queries):
     start_seconds = time.perf_counter()
     server_seconds = 0
     for run in range(0, args.runs):
+        run_id = f'{query_prefix}.run{run}'
         for conn_index, c in enumerate(connections):
-            res = c.execute(q)
-            print(f'query\t{tsv_escape(query_display_name)}\t{run}\t{conn_index}\t{c.last_query.elapsed}')
+            res = c.execute(q, query_id = run_id)
+            print(f'query\t{query_index}\t{run_id}\t{conn_index}\t{c.last_query.elapsed}')
             server_seconds += c.last_query.elapsed
 
     client_seconds = time.perf_counter() - start_seconds
-    print(f'client-time\t{tsv_escape(query_display_name)}\t{client_seconds}\t{server_seconds}')
+    print(f'client-time\t{query_index}\t{client_seconds}\t{server_seconds}')
 
 report_stage_end('benchmark')
 
diff --git a/docker/test/performance-comparison/report.py b/docker/test/performance-comparison/report.py
index a6203ec40e6..c6b743a606e 100755
--- a/docker/test/performance-comparison/report.py
+++ b/docker/test/performance-comparison/report.py
@@ -25,6 +25,9 @@ very_unstable_queries = 0
 # max seconds to run one query by itself, not counting preparation
 allowed_single_run_time = 2
 
+color_bad='#ffb0c0'
+color_good='#b0d050'
+
 header_template = """
 <!DOCTYPE html>
 <html>
@@ -188,8 +191,8 @@ if args.report == 'main':
 
         print(tableStart('Changes in performance'))
         columns = [
-            'Old, s.',                                         # 0
-            'New, s.',                                         # 1
+            'Old, s',                                          # 0
+            'New, s',                                          # 1
             'Relative difference (new&nbsp;&minus;&nbsp;old) / old',   # 2
             'p&nbsp;<&nbsp;0.001 threshold',                   # 3
             # Failed                                           # 4
@@ -221,7 +224,7 @@ if args.report == 'main':
     slow_on_client_rows = tsvRows('report/slow-on-client.tsv')
     error_tests += len(slow_on_client_rows)
     printSimpleTable('Slow on client',
-        ['Client time, s.', 'Server time, s.', 'Ratio', 'Query'],
+        ['Client time, s', 'Server time, s', 'Ratio', 'Query'],
         slow_on_client_rows)
 
     def print_unstable_queries():
@@ -252,7 +255,7 @@ if args.report == 'main':
         for r in unstable_rows:
             if int(r[4]):
                 very_unstable_queries += 1
-                attrs[3] = 'style="background: #ffb0a0"'
+                attrs[3] = f'style="background: {color_bad}"'
             else:
                 attrs[3] = ''
 
@@ -266,7 +269,7 @@ if args.report == 'main':
     error_tests += len(run_error_rows)
     printSimpleTable('Run errors', ['Test', 'Error'], run_error_rows)
 
-    skipped_tests_rows = tsvRows('skipped-tests.tsv')
+    skipped_tests_rows = tsvRows('analyze/skipped-tests.tsv')
     printSimpleTable('Skipped tests', ['Test', 'Reason'], skipped_tests_rows)
 
     printSimpleTable('Tests with most unstable queries',
@@ -281,13 +284,13 @@ if args.report == 'main':
 
         columns = [
             'Test',                                          #0
-            'Wall clock time, s.',                           #1
-            'Total client time, s.',                         #2
+            'Wall clock time, s',                            #1
+            'Total client time, s',                          #2
             'Total queries',                                 #3
             'Ignored short queries',                         #4
-            'Longest query<br>(sum for all runs), s.',       #5
-            'Avg wall clock time<br>(sum for all runs), s.', #6
-            'Shortest query<br>(sum for all runs), s.',      #7
+            'Longest query<br>(sum for all runs), s',        #5
+            'Avg wall clock time<br>(sum for all runs), s',  #6
+            'Shortest query<br>(sum for all runs), s',       #7
             ]
 
         print(tableStart('Test times'))
@@ -300,13 +303,13 @@ if args.report == 'main':
             if float(r[6]) > 1.5 * total_runs:
                 # FIXME should be 15s max -- investigate parallel_insert
                 slow_average_tests += 1
-                attrs[6] = 'style="background: #ffb0a0"'
+                attrs[6] = f'style="background: {color_bad}"'
             else:
                 attrs[6] = ''
 
             if float(r[5]) > allowed_single_run_time * total_runs:
                 slow_average_tests += 1
-                attrs[5] = 'style="background: #ffb0a0"'
+                attrs[5] = f'style="background: {color_bad}"'
             else:
                 attrs[5] = ''
 
@@ -320,9 +323,9 @@ if args.report == 'main':
 
     print("""
     <p class="links">
-    <a href="output.7z">Test output</a>
     <a href="all-queries.html">All queries</a>
     <a href="compare.log">Log</a>
+    <a href="output.7z">Test output</a>
     </p>
     </body>
     </html>
@@ -382,8 +385,8 @@ elif args.report == 'all-queries':
         columns = [
             # Changed #0
             # Unstable #1
-            'Old, s.', #2
-            'New, s.', #3
+            'Old, s', #2
+            'New, s', #3
             'Relative difference (new&nbsp;&minus;&nbsp;old) / old', #4
             'Times speedup / slowdown',                 #5
             'p&nbsp;<&nbsp;0.001 threshold',          #6
@@ -399,21 +402,21 @@ elif args.report == 'all-queries':
         attrs[1] = None
         for r in rows:
             if int(r[1]):
-                attrs[6] = 'style="background: #ffb0a0"'
+                attrs[6] = f'style="background: {color_bad}"'
             else:
                 attrs[6] = ''
 
             if int(r[0]):
                 if float(r[4]) > 0.:
-                    attrs[4] = 'style="background: #ffb0a0"'
+                    attrs[4] = f'style="background: {color_bad}"'
                 else:
-                    attrs[4] = 'style="background: #adbdff"'
+                    attrs[4] = f'style="background: {color_good}"'
             else:
                 attrs[4] = ''
 
             if (float(r[2]) + float(r[3])) / 2 > allowed_single_run_time:
-                attrs[2] = 'style="background: #ffb0a0"'
-                attrs[3] = 'style="background: #ffb0a0"'
+                attrs[2] = f'style="background: {color_bad}"'
+                attrs[3] = f'style="background: {color_bad}"'
             else:
                 attrs[2] = ''
                 attrs[3] = ''
@@ -428,9 +431,9 @@ elif args.report == 'all-queries':
 
     print("""
     <p class="links">
-    <a href="output.7z">Test output</a>
     <a href="report.html">Main report</a>
     <a href="compare.log">Log</a>
+    <a href="output.7z">Test output</a>
     </p>
     </body>
     </html>

From e11fa03bddeb637255d19da1a2fbe5bf6f4a6a0e Mon Sep 17 00:00:00 2001
From: hcz <h.cz@qq.com>
Date: Thu, 21 May 2020 12:07:47 +0800
Subject: [PATCH 004/184] Add ArrowStream input and output format

---
 .../Formats/Impl/ArrowBlockInputFormat.cpp    | 70 ++++++++++++++-----
 .../Formats/Impl/ArrowBlockInputFormat.h      | 10 +--
 .../Formats/Impl/ArrowBlockOutputFormat.cpp   | 37 +++++++---
 .../Formats/Impl/ArrowBlockOutputFormat.h     |  6 +-
 4 files changed, 91 insertions(+), 32 deletions(-)

diff --git a/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp b/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp
index df73c93fd55..c8a098418b7 100644
--- a/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp
+++ b/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp
@@ -16,12 +16,12 @@ namespace DB
 
 namespace ErrorCodes
 {
-    extern const int BAD_ARGUMENTS;
+    extern const int UNKNOWN_EXCEPTION;
     extern const int CANNOT_READ_ALL_DATA;
 }
 
-ArrowBlockInputFormat::ArrowBlockInputFormat(ReadBuffer & in_, const Block & header_)
-    : IInputFormat(header_, in_)
+ArrowBlockInputFormat::ArrowBlockInputFormat(ReadBuffer & in_, const Block & header_, bool stream_)
+    : IInputFormat(header_, in_), stream{stream_}
 {
     prepareReader();
 }
@@ -31,11 +31,21 @@ Chunk ArrowBlockInputFormat::generate()
     Chunk res;
     const Block & header = getPort().getHeader();
 
-    if (record_batch_current >= record_batch_total)
+    if (!stream && record_batch_current >= record_batch_total)
         return res;
 
     std::vector<std::shared_ptr<arrow::RecordBatch>> single_batch(1);
-    arrow::Status read_status = file_reader->ReadRecordBatch(record_batch_current, &single_batch[0]);
+    arrow::Status read_status;
+    if (stream)
+    {
+        read_status = stream_reader->ReadNext(&single_batch[0]);
+    }
+    else
+    {
+        read_status = file_reader->ReadRecordBatch(record_batch_current, &single_batch[0]);
+        if (!single_batch[0])
+            return res;
+    }
     if (!read_status.ok())
         throw Exception{"Error while reading batch of Arrow data: " + read_status.ToString(),
                         ErrorCodes::CANNOT_READ_ALL_DATA};
@@ -57,30 +67,54 @@ void ArrowBlockInputFormat::resetParser()
 {
     IInputFormat::resetParser();
 
-    file_reader.reset();
+    if (stream)
+        stream_reader.reset();
+    else
+        file_reader.reset();
     prepareReader();
 }
 
 void ArrowBlockInputFormat::prepareReader()
 {
-    arrow::Status open_status = arrow::ipc::RecordBatchFileReader::Open(asArrowFile(in), &file_reader);
-    if (!open_status.ok())
-        throw Exception(open_status.ToString(), ErrorCodes::BAD_ARGUMENTS);
-    record_batch_total = file_reader->num_record_batches();
+    arrow::Status status;
+
+    if (stream)
+        status = arrow::ipc::RecordBatchStreamReader::Open(asArrowFile(in), &stream_reader);
+    else
+        status = arrow::ipc::RecordBatchFileReader::Open(asArrowFile(in), &file_reader);
+
+    if (!status.ok())
+        throw Exception{"Error while opening a table: " + status.ToString(), ErrorCodes::UNKNOWN_EXCEPTION};
+
+    if (stream)
+        record_batch_total = -1;
+    else
+        record_batch_total = file_reader->num_record_batches();
+
     record_batch_current = 0;
 }
 
 void registerInputFormatProcessorArrow(FormatFactory &factory)
 {
     factory.registerInputFormatProcessor(
-            "Arrow",
-            [](ReadBuffer & buf,
-               const Block & sample,
-               const RowInputFormatParams & /* params */,
-               const FormatSettings & /* format_settings */)
-            {
-                return std::make_shared<ArrowBlockInputFormat>(buf, sample);
-            });
+        "Arrow",
+        [](ReadBuffer & buf,
+           const Block & sample,
+           const RowInputFormatParams & /* params */,
+           const FormatSettings & /* format_settings */)
+        {
+            return std::make_shared<ArrowBlockInputFormat>(buf, sample, false);
+        });
+
+    factory.registerInputFormatProcessor(
+        "ArrowStream",
+        [](ReadBuffer & buf,
+           const Block & sample,
+           const RowInputFormatParams & /* params */,
+           const FormatSettings & /* format_settings */)
+        {
+            return std::make_shared<ArrowBlockInputFormat>(buf, sample, true);
+        });
 }
 
 }
diff --git a/src/Processors/Formats/Impl/ArrowBlockInputFormat.h b/src/Processors/Formats/Impl/ArrowBlockInputFormat.h
index 28ca033dd13..2e7f647b4f7 100644
--- a/src/Processors/Formats/Impl/ArrowBlockInputFormat.h
+++ b/src/Processors/Formats/Impl/ArrowBlockInputFormat.h
@@ -4,6 +4,7 @@
 
 #include <Processors/Formats/IInputFormat.h>
 
+namespace arrow { class RecordBatchReader; }
 namespace arrow::ipc { class RecordBatchFileReader; }
 
 namespace DB
@@ -14,7 +15,7 @@ class ReadBuffer;
 class ArrowBlockInputFormat : public IInputFormat
 {
 public:
-    ArrowBlockInputFormat(ReadBuffer & in_, const Block & header_);
+    ArrowBlockInputFormat(ReadBuffer & in_, const Block & header_, bool stream_);
 
     void resetParser() override;
 
@@ -24,12 +25,13 @@ protected:
     Chunk generate() override;
 
 private:
-    void prepareReader();
-
-private:
+    bool stream;
+    std::shared_ptr<arrow::RecordBatchReader> stream_reader;
     std::shared_ptr<arrow::ipc::RecordBatchFileReader> file_reader;
     int record_batch_total = 0;
     int record_batch_current = 0;
+
+    void prepareReader();
 };
 
 }
diff --git a/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp
index ed33eb08728..ef6bd315319 100644
--- a/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp
+++ b/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp
@@ -15,8 +15,8 @@ namespace ErrorCodes
     extern const int UNKNOWN_EXCEPTION;
 }
 
-ArrowBlockOutputFormat::ArrowBlockOutputFormat(WriteBuffer & out_, const Block & header_, const FormatSettings & format_settings_)
-    : IOutputFormat(header_, out_), format_settings{format_settings_}, arrow_ostream{std::make_shared<ArrowBufferedOutputStream>(out_)}
+ArrowBlockOutputFormat::ArrowBlockOutputFormat(WriteBuffer & out_, const Block & header_, bool stream_, const FormatSettings & format_settings_)
+    : IOutputFormat(header_, out_), stream{stream_}, format_settings{format_settings_}, arrow_ostream{std::make_shared<ArrowBufferedOutputStream>(out_)}
 {
 }
 
@@ -29,12 +29,7 @@ void ArrowBlockOutputFormat::consume(Chunk chunk)
     CHColumnToArrowColumn::chChunkToArrowTable(arrow_table, header, chunk, columns_num, "Arrow");
 
     if (!writer)
-    {
-        // TODO: should we use arrow::ipc::IpcOptions::alignment?
-        auto status = arrow::ipc::RecordBatchFileWriter::Open(arrow_ostream.get(), arrow_table->schema(), &writer);
-        if (!status.ok())
-            throw Exception{"Error while opening a table: " + status.ToString(), ErrorCodes::UNKNOWN_EXCEPTION};
-    }
+        prepareWriter(arrow_table->schema());
 
     // TODO: calculate row_group_size depending on a number of rows and table size
     auto status = writer->WriteTable(*arrow_table, format_settings.arrow.row_group_size);
@@ -53,6 +48,20 @@ void ArrowBlockOutputFormat::finalize()
     }
 }
 
+void ArrowBlockOutputFormat::prepareWriter(const std::shared_ptr<arrow::Schema> & schema)
+{
+    arrow::Status status;
+
+    // TODO: should we use arrow::ipc::IpcOptions::alignment?
+    if (stream)
+        status = arrow::ipc::RecordBatchStreamWriter::Open(arrow_ostream.get(), schema, &writer);
+    else
+        status = arrow::ipc::RecordBatchFileWriter::Open(arrow_ostream.get(), schema, &writer);
+
+    if (!status.ok())
+        throw Exception{"Error while opening a table writer: " + status.ToString(), ErrorCodes::UNKNOWN_EXCEPTION};
+}
+
 void registerOutputFormatProcessorArrow(FormatFactory & factory)
 {
     factory.registerOutputFormatProcessor(
@@ -62,7 +71,17 @@ void registerOutputFormatProcessorArrow(FormatFactory & factory)
            FormatFactory::WriteCallback,
            const FormatSettings & format_settings)
         {
-            return std::make_shared<ArrowBlockOutputFormat>(buf, sample, format_settings);
+            return std::make_shared<ArrowBlockOutputFormat>(buf, sample, false, format_settings);
+        });
+
+    factory.registerOutputFormatProcessor(
+        "ArrowStream",
+        [](WriteBuffer & buf,
+           const Block & sample,
+           FormatFactory::WriteCallback,
+           const FormatSettings & format_settings)
+        {
+            return std::make_shared<ArrowBlockOutputFormat>(buf, sample, true, format_settings);
         });
 }
 
diff --git a/src/Processors/Formats/Impl/ArrowBlockOutputFormat.h b/src/Processors/Formats/Impl/ArrowBlockOutputFormat.h
index 9ea84f83a0d..0cc6804705b 100644
--- a/src/Processors/Formats/Impl/ArrowBlockOutputFormat.h
+++ b/src/Processors/Formats/Impl/ArrowBlockOutputFormat.h
@@ -6,6 +6,7 @@
 #include <Processors/Formats/IOutputFormat.h>
 #include "ArrowBufferedStreams.h"
 
+namespace arrow { class Schema; }
 namespace arrow::ipc { class RecordBatchWriter; }
 
 namespace DB
@@ -14,7 +15,7 @@ namespace DB
 class ArrowBlockOutputFormat : public IOutputFormat
 {
 public:
-    ArrowBlockOutputFormat(WriteBuffer & out_, const Block & header_, const FormatSettings & format_settings_);
+    ArrowBlockOutputFormat(WriteBuffer & out_, const Block & header_, bool stream_, const FormatSettings & format_settings_);
 
     String getName() const override { return "ArrowBlockOutputFormat"; }
     void consume(Chunk) override;
@@ -23,9 +24,12 @@ public:
     String getContentType() const override { return "application/octet-stream"; }
 
 private:
+    bool stream;
     const FormatSettings format_settings;
     std::shared_ptr<ArrowBufferedOutputStream> arrow_ostream;
     std::shared_ptr<arrow::ipc::RecordBatchWriter> writer;
+
+    void prepareWriter(const std::shared_ptr<arrow::Schema> & schema);
 };
 
 }

From 4effdeaa0e2a07902a286d74a3d13e95d8a40689 Mon Sep 17 00:00:00 2001
From: hcz <h.cz@qq.com>
Date: Thu, 21 May 2020 12:13:49 +0800
Subject: [PATCH 005/184] Add tests

---
 .../0_stateless/01273_arrow_stream.reference  |  62 ++++++++++
 .../queries/0_stateless/01273_arrow_stream.sh | 115 ++++++++++++++++++
 2 files changed, 177 insertions(+)
 create mode 100644 tests/queries/0_stateless/01273_arrow_stream.reference
 create mode 100755 tests/queries/0_stateless/01273_arrow_stream.sh

diff --git a/tests/queries/0_stateless/01273_arrow_stream.reference b/tests/queries/0_stateless/01273_arrow_stream.reference
new file mode 100644
index 00000000000..0f4be2c74a0
--- /dev/null
+++ b/tests/queries/0_stateless/01273_arrow_stream.reference
@@ -0,0 +1,62 @@
+9999
+9998
+9997
+9996
+9995
+9994
+9993
+9992
+9991
+9990
+99999
+99998
+99997
+99996
+99995
+99994
+99993
+99992
+99991
+99990
+2
+1
+0
+999
+998
+997
+996
+995
+994
+993
+992
+991
+990
+ContextLock	Number of times the lock of Context was acquired or tried to acquire. This is global lock.
+Query	Number of queries to be interpreted and potentially executed. Does not include queries that failed to parse or were rejected due to AST size limits, quota limits or limits on the number of simultaneously running queries. May include internal queries initiated by ClickHouse itself. Does not count subqueries.
+original:
+-128	0	-32768	0	-2147483648	0	-9223372036854775808	0	-1.032	-1.064	string-1	fixedstring-1\0\0	2003-04-05	2003-02-03 04:05:06
+-108	108	-1016	1116	-1032	1132	-1064	1164	-1.032	-1.064	string-0	fixedstring\0\0\0\0	2001-02-03	2002-02-03 04:05:06
+127	255	32767	65535	2147483647	4294967295	9223372036854775807	9223372036854775807	-1.032	-1.064	string-2	fixedstring-2\0\0	2004-06-07	2004-02-03 04:05:06
+converted:
+-128	0	-32768	0	-2147483648	0	-9223372036854775808	0	-1.032	-1.064	string-1	fixedstring-1\0\0	2003-04-05	2003-02-03 04:05:06
+-108	108	-1016	1116	-1032	1132	-1064	1164	-1.032	-1.064	string-0	fixedstring\0\0\0\0	2001-02-03	2002-02-03 04:05:06
+127	255	32767	65535	2147483647	4294967295	9223372036854775807	9223372036854775807	-1.032	-1.064	string-2	fixedstring-2\0\0	2004-06-07	2004-02-03 04:05:06
+diff:
+dest:
+79	81	82	83	84	85	86	87	88	89	str01\0\0\0\0\0\0\0\0\0\0	fstr1\0\0\0\0\0\0\0\0\0\0	2003-03-04	1970-01-01 06:29:04
+80	81	82	83	84	85	86	87	88	89	str02	fstr2\0\0\0\0\0\0\0\0\0\0	2005-03-04	2006-08-09 10:11:12
+min:
+-128	0	0	0	0	0	0	0	-1	-1	string-1\0\0\0\0\0\0\0	fixedstring-1\0\0	2003-04-05	2003-02-03
+-108	108	8	92	-8	108	-40	-116	-1	-1	string-0\0\0\0\0\0\0\0	fixedstring\0\0\0\0	2001-02-03	2002-02-03
+79	81	82	83	84	85	86	87	88	89	str01\0\0\0\0\0\0\0\0\0\0	fstr1\0\0\0\0\0\0\0\0\0\0	2003-03-04	2004-05-06
+127	-1	-1	-1	-1	-1	-1	-1	-1	-1	string-2\0\0\0\0\0\0\0	fixedstring-2\0\0	2004-06-07	2004-02-03
+max:
+-128	0	-32768	0	-2147483648	0	-9223372036854775808	0	-1	-1	string-1	fixedstring-1\0\0	1970-01-01 06:22:27	2003-02-03 04:05:06
+-108	108	-1016	1116	-1032	1132	-1064	1164	-1	-1	string-0	fixedstring\0\0\0\0	1970-01-01 06:09:16	2002-02-03 04:05:06
+80	81	82	83	84	85	86	87	88	89	str02	fstr2	2005-03-04 05:06:07	2006-08-09 10:11:12
+127	255	32767	65535	2147483647	4294967295	9223372036854775807	9223372036854775807	-1	-1	string-2	fixedstring-2\0\0	1970-01-01 06:29:36	2004-02-03 04:05:06
+dest from null:
+-128	0	-32768	0	-2147483648	0	-9223372036854775808	0	-1.032	-1.064	string-1	fixedstring-1\0\0	2003-04-05	2003-02-03 04:05:06
+-108	108	-1016	1116	-1032	1132	-1064	1164	-1.032	-1.064	string-0	fixedstring\0\0\0\0	2001-02-03	2002-02-03 04:05:06
+127	255	32767	65535	2147483647	4294967295	9223372036854775807	9223372036854775807	-1.032	-1.064	string-2	fixedstring-2\0\0	2004-06-07	2004-02-03 04:05:06
+\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N
diff --git a/tests/queries/0_stateless/01273_arrow_stream.sh b/tests/queries/0_stateless/01273_arrow_stream.sh
new file mode 100755
index 00000000000..2fc630a3a8b
--- /dev/null
+++ b/tests/queries/0_stateless/01273_arrow_stream.sh
@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+
+set -e
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+. $CUR_DIR/../shell_config.sh
+
+
+${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS contributors"
+${CLICKHOUSE_CLIENT} --query="CREATE TABLE contributors (name String) ENGINE = Memory"
+${CLICKHOUSE_CLIENT} --query="SELECT * FROM system.contributors ORDER BY name DESC FORMAT ArrowStream" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO contributors FORMAT ArrowStream"
+# random results
+${CLICKHOUSE_CLIENT} --query="SELECT * FROM contributors LIMIT 10" > /dev/null
+${CLICKHOUSE_CLIENT} --query="DROP TABLE contributors"
+
+${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_numbers"
+${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_numbers (number UInt64) ENGINE = Memory"
+# less than default block size (65k)
+${CLICKHOUSE_CLIENT} --query="SELECT * FROM system.numbers LIMIT 10000 FORMAT ArrowStream" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_numbers FORMAT ArrowStream"
+${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_numbers ORDER BY number DESC LIMIT 10"
+${CLICKHOUSE_CLIENT} --query="TRUNCATE TABLE arrow_numbers"
+
+# More than default block size
+${CLICKHOUSE_CLIENT} --query="SELECT * FROM system.numbers LIMIT 100000 FORMAT ArrowStream" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_numbers FORMAT ArrowStream"
+${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_numbers ORDER BY number DESC LIMIT 10"
+${CLICKHOUSE_CLIENT} --query="TRUNCATE TABLE arrow_numbers"
+
+${CLICKHOUSE_CLIENT} --max_block_size=2 --query="SELECT * FROM system.numbers LIMIT 3 FORMAT ArrowStream" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_numbers FORMAT ArrowStream"
+${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_numbers ORDER BY number DESC LIMIT 10"
+
+${CLICKHOUSE_CLIENT} --query="TRUNCATE TABLE arrow_numbers"
+${CLICKHOUSE_CLIENT} --max_block_size=1 --query="SELECT * FROM system.numbers LIMIT 1000 FORMAT ArrowStream" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_numbers FORMAT ArrowStream"
+${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_numbers ORDER BY number DESC LIMIT 10"
+
+${CLICKHOUSE_CLIENT} --query="DROP TABLE arrow_numbers"
+
+${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_events"
+${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_events (event String, value UInt64, description String) ENGINE = Memory"
+${CLICKHOUSE_CLIENT} --query="SELECT * FROM system.events FORMAT ArrowStream" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_events FORMAT ArrowStream"
+${CLICKHOUSE_CLIENT} --query="SELECT event, description FROM arrow_events WHERE event IN ('ContextLock', 'Query') ORDER BY event"
+${CLICKHOUSE_CLIENT} --query="DROP TABLE arrow_events"
+
+
+
+${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_types1"
+${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_types2"
+${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_types3"
+${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_types4"
+${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types1       (int8 Int8, uint8 UInt8, int16 Int16, uint16 UInt16, int32 Int32, uint32 UInt32, int64 Int64, uint64 UInt64, float32 Float32, float64 Float64, string String, fixedstring FixedString(15), date Date, datetime DateTime) ENGINE = Memory"
+${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types2       (int8 Int8, uint8 UInt8, int16 Int16, uint16 UInt16, int32 Int32, uint32 UInt32, int64 Int64, uint64 UInt64, float32 Float32, float64 Float64, string String, fixedstring FixedString(15), date Date, datetime DateTime) ENGINE = Memory"
+# convert min type
+${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types3       (int8 Int8,  uint8 Int8,  int16 Int8,   uint16 Int8,  int32 Int8,   uint32 Int8,  int64 Int8,   uint64 Int8,    float32 Int8,    float64 Int8, string FixedString(15), fixedstring FixedString(15), date Date,    datetime Date) ENGINE = Memory"
+# convert max type
+${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types4       (int8 Int64, uint8 Int64, int16 Int64, uint16 Int64, int32 Int64,  uint32 Int64, int64 Int64,  uint64 Int64,   float32 Int64,   float64 Int64, string String,          fixedstring String, date DateTime, datetime DateTime) ENGINE = Memory"
+
+${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types1 values (     -108,         108,       -1016,          1116,       -1032,          1132,       -1064,          1164,          -1.032,          -1.064,    'string-0',               'fixedstring', '2001-02-03', '2002-02-03 04:05:06')"
+
+# min
+${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types1 values (     -128,           0,      -32768,             0, -2147483648,             0, -9223372036854775808, 0,             -1.032,          -1.064,    'string-1',             'fixedstring-1', '2003-04-05', '2003-02-03 04:05:06')"
+
+# max
+${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types1 values (      127,         255,       32767,         65535,  2147483647,    4294967295, 9223372036854775807, 9223372036854775807, -1.032,     -1.064,    'string-2',             'fixedstring-2', '2004-06-07', '2004-02-03 04:05:06')"
+
+# 'SELECT -127,-128,-129,126,127,128,255,256,257,-32767,-32768,-32769,32766,32767,32768,65535,65536,65537,  -2147483647,-2147483648,-2147483649,2147483646,2147483647,2147483648,4294967295,4294967296,4294967297,   -9223372036854775807,-9223372036854775808,9223372036854775806,9223372036854775807,9223372036854775808,18446744073709551615';
+
+${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types1 FORMAT ArrowStream" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types2 FORMAT ArrowStream"
+
+echo original:
+${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types1 ORDER BY int8" | tee ${CLICKHOUSE_TMP}/arrow_all_types_1.dump
+echo converted:
+${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types2 ORDER BY int8" | tee ${CLICKHOUSE_TMP}/arrow_all_types_2.dump
+${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types1 ORDER BY int8 FORMAT ArrowStream" > ${CLICKHOUSE_TMP}/arrow_all_types_1.arrow
+${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types2 ORDER BY int8 FORMAT ArrowStream" > ${CLICKHOUSE_TMP}/arrow_all_types_2.arrow
+echo diff:
+diff ${CLICKHOUSE_TMP}/arrow_all_types_1.dump ${CLICKHOUSE_TMP}/arrow_all_types_2.dump
+
+${CLICKHOUSE_CLIENT} --query="TRUNCATE TABLE arrow_types2"
+${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types3 values (       79,          81,          82,            83,          84,            85,          86,            87,              88,              89,         'str01',                  'fstr1', '2003-03-04', '2004-05-06')"
+${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types3 ORDER BY int8 FORMAT ArrowStream" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types2 FORMAT ArrowStream"
+${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types1 ORDER BY int8 FORMAT ArrowStream" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types3 FORMAT ArrowStream"
+
+${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types4 values (       80,          81,          82,            83,          84,            85,          86,            87,              88,              89,         'str02',                  'fstr2', '2005-03-04 05:06:07', '2006-08-09 10:11:12')"
+${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types4 ORDER BY int8 FORMAT ArrowStream" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types2 FORMAT ArrowStream"
+${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types1 ORDER BY int8 FORMAT ArrowStream" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types4 FORMAT ArrowStream"
+
+echo dest:
+${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types2 ORDER BY int8"
+echo min:
+${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types3 ORDER BY int8"
+echo max:
+${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types4 ORDER BY int8"
+
+
+${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_types5"
+${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_types6"
+${CLICKHOUSE_CLIENT} --query="TRUNCATE TABLE arrow_types2"
+${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types5       (int8 Nullable(Int8), uint8 Nullable(UInt8), int16 Nullable(Int16), uint16 Nullable(UInt16), int32 Nullable(Int32), uint32 Nullable(UInt32), int64 Nullable(Int64), uint64 Nullable(UInt64), float32 Nullable(Float32), float64 Nullable(Float64), string Nullable(String), fixedstring Nullable(FixedString(15)), date Nullable(Date), datetime Nullable(DateTime)) ENGINE = Memory"
+${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types6       (int8 Nullable(Int8), uint8 Nullable(UInt8), int16 Nullable(Int16), uint16 Nullable(UInt16), int32 Nullable(Int32), uint32 Nullable(UInt32), int64 Nullable(Int64), uint64 Nullable(UInt64), float32 Nullable(Float32), float64 Nullable(Float64), string Nullable(String), fixedstring Nullable(FixedString(15)), date Nullable(Date), datetime Nullable(DateTime)) ENGINE = Memory"
+${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types5 values (               NULL,                  NULL,                  NULL,                    NULL,                  NULL,                    NULL,                  NULL,                    NULL,                      NULL,                      NULL,                    NULL,                                  NULL,                NULL,                        NULL)"
+${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types5 ORDER BY int8 FORMAT ArrowStream" > ${CLICKHOUSE_TMP}/arrow_all_types_5.arrow
+#${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types5 FORMAT ArrowStream" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types6 FORMAT ArrowStream"
+${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types5 ORDER BY int8 FORMAT ArrowStream" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types6 FORMAT ArrowStream"
+${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types1 ORDER BY int8 FORMAT ArrowStream" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types6 FORMAT ArrowStream"
+echo dest from null:
+${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types6 ORDER BY int8"
+
+${CLICKHOUSE_CLIENT} --query="DROP TABLE arrow_types5"
+${CLICKHOUSE_CLIENT} --query="DROP TABLE arrow_types6"
+
+
+${CLICKHOUSE_CLIENT} --query="DROP TABLE arrow_types1"
+${CLICKHOUSE_CLIENT} --query="DROP TABLE arrow_types2"
+${CLICKHOUSE_CLIENT} --query="DROP TABLE arrow_types3"
+${CLICKHOUSE_CLIENT} --query="DROP TABLE arrow_types4"
+

From ab37791134b7317c99679ef5e4cb738c2ff9d9a4 Mon Sep 17 00:00:00 2001
From: hcz <h.cz@qq.com>
Date: Thu, 21 May 2020 15:28:10 +0800
Subject: [PATCH 006/184] Fix arrow stream termination

---
 src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp b/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp
index c8a098418b7..5468fbcac16 100644
--- a/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp
+++ b/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp
@@ -39,12 +39,12 @@ Chunk ArrowBlockInputFormat::generate()
     if (stream)
     {
         read_status = stream_reader->ReadNext(&single_batch[0]);
+        if (!single_batch[0])
+            return res;
     }
     else
     {
         read_status = file_reader->ReadRecordBatch(record_batch_current, &single_batch[0]);
-        if (!single_batch[0])
-            return res;
     }
     if (!read_status.ok())
         throw Exception{"Error while reading batch of Arrow data: " + read_status.ToString(),

From 95e2f0fa0769e87cbf77cb8d264c75a6fb198a8d Mon Sep 17 00:00:00 2001
From: hcz <h.cz@qq.com>
Date: Thu, 21 May 2020 18:20:41 +0800
Subject: [PATCH 007/184] Add brief docs about apache arrow

---
 docs/en/interfaces/formats.md | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md
index 4697fbca7e1..ed411a1b0d4 100644
--- a/docs/en/interfaces/formats.md
+++ b/docs/en/interfaces/formats.md
@@ -37,6 +37,8 @@ The supported formats are:
 | [Avro](#data-format-avro)                                       | ✔     | ✔      |
 | [AvroConfluent](#data-format-avro-confluent)                    | ✔     | ✗      |
 | [Parquet](#data-format-parquet)                                 | ✔     | ✔      |
+| [Arrow](#data-format-arrow)                                     | ✔     | ✔      |
+| [ArrowStream](#data-format-arrow-stream)                        | ✔     | ✔      |
 | [ORC](#data-format-orc)                                         | ✔     | ✗      |
 | [RowBinary](#rowbinary)                                         | ✔     | ✔      |
 | [RowBinaryWithNamesAndTypes](#rowbinarywithnamesandtypes)       | ✔     | ✔      |
@@ -985,9 +987,9 @@ See also [how to read/write length-delimited protobuf messages in popular langua
 
 ## Avro {#data-format-avro}
 
-[Apache Avro](http://avro.apache.org/) is a row-oriented data serialization framework developed within Apache’s Hadoop project.
+[Apache Avro](https://avro.apache.org/) is a row-oriented data serialization framework developed within Apache’s Hadoop project.
 
-ClickHouse Avro format supports reading and writing [Avro data files](http://avro.apache.org/docs/current/spec.html#Object+Container+Files).
+ClickHouse Avro format supports reading and writing [Avro data files](https://avro.apache.org/docs/current/spec.html#Object+Container+Files).
 
 ### Data Types Matching {#data_types-matching}
 
@@ -1009,7 +1011,7 @@ The table below shows supported data types and how they match ClickHouse [data t
 | `long (timestamp-millis)` \*                | [DateTime64(3)](../sql-reference/data-types/datetime.md)                                                              | `long (timestamp-millis)` \* |
 | `long (timestamp-micros)` \*                | [DateTime64(6)](../sql-reference/data-types/datetime.md)                                                              | `long (timestamp-micros)` \* |
 
-\* [Avro logical types](http://avro.apache.org/docs/current/spec.html#Logical+Types)
+\* [Avro logical types](https://avro.apache.org/docs/current/spec.html#Logical+Types)
 
 Unsupported Avro data types: `record` (non-root), `map`
 
@@ -1095,7 +1097,7 @@ SELECT * FROM topic1_stream;
 
 ## Parquet {#data-format-parquet}
 
-[Apache Parquet](http://parquet.apache.org/) is a columnar storage format widespread in the Hadoop ecosystem. ClickHouse supports read and write operations for this format.
+[Apache Parquet](https://parquet.apache.org/) is a columnar storage format widespread in the Hadoop ecosystem. ClickHouse supports read and write operations for this format.
 
 ### Data Types Matching {#data_types-matching-2}
 
@@ -1141,6 +1143,16 @@ $ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Parquet" > {some_
 
 To exchange data with Hadoop, you can use [HDFS table engine](../engines/table-engines/integrations/hdfs.md).
 
+## Arrow {data-format-arrow}
+
+[Apache Arrow](https://arrow.apache.org/) comes with two built-in columnar storage formats. ClickHouse supports read and write operations for these formats.
+
+`Arrow` is Apache Arrow's "file mode" format. It is designed for in-memory random access.
+
+## ArrowStream {data-format-arrow-stream}
+
+`ArrowStream` is Apache Arrow's "stream mode" format. It is designed for in-memory stream processing.
+
 ## ORC {#data-format-orc}
 
 [Apache ORC](https://orc.apache.org/) is a columnar storage format widespread in the Hadoop ecosystem. You can only insert data in this format to ClickHouse.

From f026fa7de1109c3c0c7e5ad2f643c9c3267f7b42 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Fri, 22 May 2020 16:29:33 +0300
Subject: [PATCH 008/184] Rename PartDestinationType to DataDestionationType,
 because there is no part-specific information

---
 src/Interpreters/InterpreterAlterQuery.cpp    |  6 ++---
 src/Parsers/ASTAlterQuery.cpp                 |  8 +++----
 src/Parsers/ASTAlterQuery.h                   |  2 +-
 src/Parsers/ASTTTLElement.cpp                 |  6 ++---
 src/Parsers/ASTTTLElement.h                   |  6 ++---
 src/Parsers/ExpressionElementParsers.cpp      |  8 +++----
 src/Parsers/ParserAlterQuery.cpp              | 16 +++++++-------
 ...estinationType.h => DataDestinationType.h} |  2 +-
 src/Storages/MergeTree/MergeTreeData.cpp      | 22 +++++++++----------
 src/Storages/MergeTree/MergeTreeData.h        |  4 ++--
 src/Storages/PartitionCommands.cpp            |  8 +++----
 src/Storages/StorageInMemoryMetadata.h        | 13 +++++++++++
 12 files changed, 57 insertions(+), 44 deletions(-)
 rename src/Storages/{MergeTree/PartDestinationType.h => DataDestinationType.h} (73%)

diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp
index c2ab8776c25..c3ee740a486 100644
--- a/src/Interpreters/InterpreterAlterQuery.cpp
+++ b/src/Interpreters/InterpreterAlterQuery.cpp
@@ -244,12 +244,12 @@ AccessRightsElements InterpreterAlterQuery::getRequiredAccessForCommand(const AS
         }
         case ASTAlterCommand::MOVE_PARTITION:
         {
-            if ((command.move_destination_type == PartDestinationType::DISK)
-                || (command.move_destination_type == PartDestinationType::VOLUME))
+            if ((command.move_destination_type == DataDestinationType::DISK)
+                || (command.move_destination_type == DataDestinationType::VOLUME))
             {
                 required_access.emplace_back(AccessType::ALTER_MOVE_PARTITION, database, table);
             }
-            else if (command.move_destination_type == PartDestinationType::TABLE)
+            else if (command.move_destination_type == DataDestinationType::TABLE)
             {
                 required_access.emplace_back(AccessType::SELECT | AccessType::ALTER_DELETE, database, table);
                 required_access.emplace_back(AccessType::INSERT, command.to_database, command.to_table);
diff --git a/src/Parsers/ASTAlterQuery.cpp b/src/Parsers/ASTAlterQuery.cpp
index 9ec2fad5768..f323f66be17 100644
--- a/src/Parsers/ASTAlterQuery.cpp
+++ b/src/Parsers/ASTAlterQuery.cpp
@@ -181,13 +181,13 @@ void ASTAlterCommand::formatImpl(
         settings.ostr << " TO ";
         switch (move_destination_type)
         {
-            case PartDestinationType::DISK:
+            case DataDestinationType::DISK:
                 settings.ostr << "DISK ";
                 break;
-            case PartDestinationType::VOLUME:
+            case DataDestinationType::VOLUME:
                 settings.ostr << "VOLUME ";
                 break;
-            case PartDestinationType::TABLE:
+            case DataDestinationType::TABLE:
                 settings.ostr << "TABLE ";
                 if (!to_database.empty())
                 {
@@ -201,7 +201,7 @@ void ASTAlterCommand::formatImpl(
             default:
                 break;
         }
-        if (move_destination_type != PartDestinationType::TABLE)
+        if (move_destination_type != DataDestinationType::TABLE)
         {
             settings.ostr << quoteString(move_destination_name);
         }
diff --git a/src/Parsers/ASTAlterQuery.h b/src/Parsers/ASTAlterQuery.h
index 85e9a4d7552..a9ae06863a9 100644
--- a/src/Parsers/ASTAlterQuery.h
+++ b/src/Parsers/ASTAlterQuery.h
@@ -136,7 +136,7 @@ public:
 
     bool if_exists = false;     /// option for DROP_COLUMN, MODIFY_COLUMN, COMMENT_COLUMN
 
-    PartDestinationType move_destination_type; /// option for MOVE PART/PARTITION
+    DataDestinationType move_destination_type; /// option for MOVE PART/PARTITION
 
     String move_destination_name;             /// option for MOVE PART/PARTITION
 
diff --git a/src/Parsers/ASTTTLElement.cpp b/src/Parsers/ASTTTLElement.cpp
index 7e03a73e36d..acc3546ddfb 100644
--- a/src/Parsers/ASTTTLElement.cpp
+++ b/src/Parsers/ASTTTLElement.cpp
@@ -10,15 +10,15 @@ namespace DB
 void ASTTTLElement::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
 {
     children.front()->formatImpl(settings, state, frame);
-    if (destination_type == PartDestinationType::DISK)
+    if (destination_type == DataDestinationType::DISK)
     {
         settings.ostr << " TO DISK " << quoteString(destination_name);
     }
-    else if (destination_type == PartDestinationType::VOLUME)
+    else if (destination_type == DataDestinationType::VOLUME)
     {
         settings.ostr << " TO VOLUME " << quoteString(destination_name);
     }
-    else if (destination_type == PartDestinationType::DELETE)
+    else if (destination_type == DataDestinationType::DELETE)
     {
         /// It would be better to output "DELETE" here but that will break compatibility with earlier versions.
     }
diff --git a/src/Parsers/ASTTTLElement.h b/src/Parsers/ASTTTLElement.h
index 02f70094e04..61f4a6e958f 100644
--- a/src/Parsers/ASTTTLElement.h
+++ b/src/Parsers/ASTTTLElement.h
@@ -1,7 +1,7 @@
 #pragma once
 
 #include <Parsers/IAST.h>
-#include <Storages/MergeTree/PartDestinationType.h>
+#include <Storages/DataDestinationType.h>
 
 
 namespace DB
@@ -11,10 +11,10 @@ namespace DB
 class ASTTTLElement : public IAST
 {
 public:
-    PartDestinationType destination_type;
+    DataDestinationType destination_type;
     String destination_name;
 
-    ASTTTLElement(PartDestinationType destination_type_, const String & destination_name_)
+    ASTTTLElement(DataDestinationType destination_type_, const String & destination_name_)
         : destination_type(destination_type_)
         , destination_name(destination_name_)
     {
diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp
index 70a8b282a72..c206db24626 100644
--- a/src/Parsers/ExpressionElementParsers.cpp
+++ b/src/Parsers/ExpressionElementParsers.cpp
@@ -1462,16 +1462,16 @@ bool ParserTTLElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
     if (!parser_exp.parse(pos, expr_elem, expected))
         return false;
 
-    PartDestinationType destination_type = PartDestinationType::DELETE;
+    DataDestinationType destination_type = DataDestinationType::DELETE;
     String destination_name;
     if (s_to_disk.ignore(pos))
-        destination_type = PartDestinationType::DISK;
+        destination_type = DataDestinationType::DISK;
     else if (s_to_volume.ignore(pos))
-        destination_type = PartDestinationType::VOLUME;
+        destination_type = DataDestinationType::VOLUME;
     else
         s_delete.ignore(pos);
 
-    if (destination_type == PartDestinationType::DISK || destination_type == PartDestinationType::VOLUME)
+    if (destination_type == DataDestinationType::DISK || destination_type == DataDestinationType::VOLUME)
     {
         ASTPtr ast_space_name;
         if (!parser_string_literal.parse(pos, ast_space_name, expected))
diff --git a/src/Parsers/ParserAlterQuery.cpp b/src/Parsers/ParserAlterQuery.cpp
index 623bca440bb..f90d010e9de 100644
--- a/src/Parsers/ParserAlterQuery.cpp
+++ b/src/Parsers/ParserAlterQuery.cpp
@@ -260,19 +260,19 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected
             command->part = true;
 
             if (s_to_disk.ignore(pos))
-                command->move_destination_type = PartDestinationType::DISK;
+                command->move_destination_type = DataDestinationType::DISK;
             else if (s_to_volume.ignore(pos))
-                command->move_destination_type = PartDestinationType::VOLUME;
+                command->move_destination_type = DataDestinationType::VOLUME;
             else if (s_to_table.ignore(pos))
             {
                 if (!parseDatabaseAndTableName(pos, expected, command->to_database, command->to_table))
                     return false;
-                command->move_destination_type = PartDestinationType::TABLE;
+                command->move_destination_type = DataDestinationType::TABLE;
             }
             else
                 return false;
 
-            if (command->move_destination_type != PartDestinationType::TABLE)
+            if (command->move_destination_type != DataDestinationType::TABLE)
             {
                 ASTPtr ast_space_name;
                 if (!parser_string_literal.parse(pos, ast_space_name, expected))
@@ -289,19 +289,19 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected
             command->type = ASTAlterCommand::MOVE_PARTITION;
 
             if (s_to_disk.ignore(pos))
-                command->move_destination_type = PartDestinationType::DISK;
+                command->move_destination_type = DataDestinationType::DISK;
             else if (s_to_volume.ignore(pos))
-                command->move_destination_type = PartDestinationType::VOLUME;
+                command->move_destination_type = DataDestinationType::VOLUME;
             else if (s_to_table.ignore(pos))
             {
                 if (!parseDatabaseAndTableName(pos, expected, command->to_database, command->to_table))
                     return false;
-                command->move_destination_type = PartDestinationType::TABLE;
+                command->move_destination_type = DataDestinationType::TABLE;
             }
             else
                 return false;
 
-            if (command->move_destination_type != PartDestinationType::TABLE)
+            if (command->move_destination_type != DataDestinationType::TABLE)
             {
                 ASTPtr ast_space_name;
                 if (!parser_string_literal.parse(pos, ast_space_name, expected))
diff --git a/src/Storages/MergeTree/PartDestinationType.h b/src/Storages/DataDestinationType.h
similarity index 73%
rename from src/Storages/MergeTree/PartDestinationType.h
rename to src/Storages/DataDestinationType.h
index 8f3e44537a6..05d1d89c2b5 100644
--- a/src/Storages/MergeTree/PartDestinationType.h
+++ b/src/Storages/DataDestinationType.h
@@ -4,7 +4,7 @@
 namespace DB
 {
 
-enum class PartDestinationType
+enum class DataDestinationType
 {
     DISK,
     VOLUME,
diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 728bfa18bd8..ef8e3789cb1 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -621,7 +621,7 @@ void MergeTreeData::setTTLExpressions(const ColumnsDescription & new_columns,
 
         auto syntax_result = SyntaxAnalyzer(global_context).analyze(ttl_ast, new_columns.getAllPhysical());
         result.expression = ExpressionAnalyzer(ttl_ast, syntax_result, global_context).getActions(false);
-        result.destination_type = PartDestinationType::DELETE;
+        result.destination_type = DataDestinationType::DELETE;
         result.result_column = ttl_ast->getColumnName();
 
         checkTTLExpression(result.expression, result.result_column);
@@ -665,7 +665,7 @@ void MergeTreeData::setTTLExpressions(const ColumnsDescription & new_columns,
             if (!ttl_element)
                 throw Exception("Unexpected AST element in TTL expression", ErrorCodes::UNEXPECTED_AST_STRUCTURE);
 
-            if (ttl_element->destination_type == PartDestinationType::DELETE)
+            if (ttl_element->destination_type == DataDestinationType::DELETE)
             {
                 if (seen_delete_ttl)
                 {
@@ -688,7 +688,7 @@ void MergeTreeData::setTTLExpressions(const ColumnsDescription & new_columns,
                 if (!new_ttl_entry.getDestination(getStoragePolicy()))
                 {
                     String message;
-                    if (new_ttl_entry.destination_type == PartDestinationType::DISK)
+                    if (new_ttl_entry.destination_type == DataDestinationType::DISK)
                         message = "No such disk " + backQuote(new_ttl_entry.destination_name) + " for given storage policy.";
                     else
                         message = "No such volume " + backQuote(new_ttl_entry.destination_name) + " for given storage policy.";
@@ -2921,11 +2921,11 @@ ReservationPtr MergeTreeData::tryReserveSpacePreferringTTLRules(UInt64 expected_
         SpacePtr destination_ptr = ttl_entry->getDestination(getStoragePolicy());
         if (!destination_ptr)
         {
-            if (ttl_entry->destination_type == PartDestinationType::VOLUME)
+            if (ttl_entry->destination_type == DataDestinationType::VOLUME)
                 LOG_WARNING(log, "Would like to reserve space on volume '"
                         << ttl_entry->destination_name << "' by TTL rule of table '"
                         << log_name << "' but volume was not found");
-            else if (ttl_entry->destination_type == PartDestinationType::DISK)
+            else if (ttl_entry->destination_type == DataDestinationType::DISK)
                 LOG_WARNING(log, "Would like to reserve space on disk '"
                         << ttl_entry->destination_name << "' by TTL rule of table '"
                         << log_name << "' but disk was not found");
@@ -2936,11 +2936,11 @@ ReservationPtr MergeTreeData::tryReserveSpacePreferringTTLRules(UInt64 expected_
             if (reservation)
                 return reservation;
             else
-                if (ttl_entry->destination_type == PartDestinationType::VOLUME)
+                if (ttl_entry->destination_type == DataDestinationType::VOLUME)
                     LOG_WARNING(log, "Would like to reserve space on volume '"
                             << ttl_entry->destination_name << "' by TTL rule of table '"
                             << log_name << "' but there is not enough space");
-                else if (ttl_entry->destination_type == PartDestinationType::DISK)
+                else if (ttl_entry->destination_type == DataDestinationType::DISK)
                     LOG_WARNING(log, "Would like to reserve space on disk '"
                             << ttl_entry->destination_name << "' by TTL rule of table '"
                             << log_name << "' but there is not enough space");
@@ -2954,9 +2954,9 @@ ReservationPtr MergeTreeData::tryReserveSpacePreferringTTLRules(UInt64 expected_
 
 SpacePtr MergeTreeData::TTLEntry::getDestination(StoragePolicyPtr policy) const
 {
-    if (destination_type == PartDestinationType::VOLUME)
+    if (destination_type == DataDestinationType::VOLUME)
         return policy->getVolumeByName(destination_name);
-    else if (destination_type == PartDestinationType::DISK)
+    else if (destination_type == DataDestinationType::DISK)
         return policy->getDiskByName(destination_name);
     else
         return {};
@@ -2964,13 +2964,13 @@ SpacePtr MergeTreeData::TTLEntry::getDestination(StoragePolicyPtr policy) const
 
 bool MergeTreeData::TTLEntry::isPartInDestination(StoragePolicyPtr policy, const IMergeTreeDataPart & part) const
 {
-    if (destination_type == PartDestinationType::VOLUME)
+    if (destination_type == DataDestinationType::VOLUME)
     {
         for (const auto & disk : policy->getVolumeByName(destination_name)->getDisks())
             if (disk->getName() == part.volume->getDisk()->getName())
                 return true;
     }
-    else if (destination_type == PartDestinationType::DISK)
+    else if (destination_type == DataDestinationType::DISK)
         return policy->getDiskByName(destination_name)->getName() == part.volume->getDisk()->getName();
     return false;
 }
diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h
index 70d3a1c3514..2b5b75110f1 100644
--- a/src/Storages/MergeTree/MergeTreeData.h
+++ b/src/Storages/MergeTree/MergeTreeData.h
@@ -8,7 +8,7 @@
 #include <Storages/MergeTree/MergeTreeSettings.h>
 #include <Storages/MergeTree/MergeTreeMutationStatus.h>
 #include <Storages/MergeTree/MergeList.h>
-#include <Storages/MergeTree/PartDestinationType.h>
+#include <Storages/DataDestinationType.h>
 #include <IO/ReadBufferFromString.h>
 #include <IO/WriteBufferFromFile.h>
 #include <IO/ReadBufferFromFile.h>
@@ -652,7 +652,7 @@ public:
         String result_column;
 
         /// Name and type of a destination are only valid in table-level context.
-        PartDestinationType destination_type;
+        DataDestinationType destination_type;
         String destination_name;
 
         ASTPtr entry_ast;
diff --git a/src/Storages/PartitionCommands.cpp b/src/Storages/PartitionCommands.cpp
index 9c8fc2cb598..c3bf00187af 100644
--- a/src/Storages/PartitionCommands.cpp
+++ b/src/Storages/PartitionCommands.cpp
@@ -1,6 +1,6 @@
 #include <Storages/PartitionCommands.h>
 #include <Storages/IStorage.h>
-#include <Storages/MergeTree/PartDestinationType.h>
+#include <Storages/DataDestinationType.h>
 #include <Parsers/ASTAlterQuery.h>
 #include <Parsers/ASTIdentifier.h>
 
@@ -42,13 +42,13 @@ std::optional<PartitionCommand> PartitionCommand::parse(const ASTAlterCommand *
         res.part = command_ast->part;
         switch (command_ast->move_destination_type)
         {
-            case PartDestinationType::DISK:
+            case DataDestinationType::DISK:
                 res.move_destination_type = PartitionCommand::MoveDestinationType::DISK;
                 break;
-            case PartDestinationType::VOLUME:
+            case DataDestinationType::VOLUME:
                 res.move_destination_type = PartitionCommand::MoveDestinationType::VOLUME;
                 break;
-            case PartDestinationType::TABLE:
+            case DataDestinationType::TABLE:
                 res.move_destination_type = PartitionCommand::MoveDestinationType::TABLE;
                 res.to_database = command_ast->to_database;
                 res.to_table = command_ast->to_table;
diff --git a/src/Storages/StorageInMemoryMetadata.h b/src/Storages/StorageInMemoryMetadata.h
index 37dce5a78b9..3faaed8d680 100644
--- a/src/Storages/StorageInMemoryMetadata.h
+++ b/src/Storages/StorageInMemoryMetadata.h
@@ -3,6 +3,7 @@
 #include <Storages/ColumnsDescription.h>
 #include <Storages/IndicesDescription.h>
 #include <Storages/ConstraintsDescription.h>
+#include <Storages/DataDestinationType.h>
 #include <Parsers/IAST_fwd.h>
 
 namespace DB
@@ -72,5 +73,17 @@ struct StorageMetadataKeyField
     static StorageMetadataKeyField getKeyFromAST(const ASTPtr & definition_ast, const ColumnsDescription & columns, const Context & context);
 };
 
+struct StorageMetadataTTLField
+{
+    ASTPtr definition_ast;
+
+    ExpressionActionsPtr expression;
+
+    String result_column;
+
+    DataDestinationType destination_type;
+
+    String destination_name;
+};
 
 }

From 409cff11df96b810f3dbe8adb75c62d71894d10f Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Fri, 22 May 2020 00:37:47 +0300
Subject: [PATCH 009/184] Add port() function

It uses domain() (getURLHost()) internally so it has the same
limitations:
- no ipv6 support
- no host-without-dots support (i.e. "foobar")
---
 .../sql-reference/functions/url-functions.md  |  4 +
 src/Functions/URL/port.cpp                    | 95 +++++++++++++++++++
 src/Functions/URL/registerFunctionsURL.cpp    |  2 +
 .../queries/0_stateless/01284_port.reference  | 21 ++++
 tests/queries/0_stateless/01284_port.sql      | 25 +++++
 5 files changed, 147 insertions(+)
 create mode 100644 src/Functions/URL/port.cpp
 create mode 100644 tests/queries/0_stateless/01284_port.reference
 create mode 100644 tests/queries/0_stateless/01284_port.sql

diff --git a/docs/en/sql-reference/functions/url-functions.md b/docs/en/sql-reference/functions/url-functions.md
index 143bd42b08e..f35c621a6e2 100644
--- a/docs/en/sql-reference/functions/url-functions.md
+++ b/docs/en/sql-reference/functions/url-functions.md
@@ -117,6 +117,10 @@ Returns the part of the domain that includes top-level subdomains up to the “f
 
 For example, `cutToFirstSignificantSubdomain('https://news.yandex.com.tr/') = 'yandex.com.tr'`.
 
+### port(URL) {#port}
+
+Returns the port or zero if there is no port in the URL (or in case of validation error).
+
 ### path {#path}
 
 Returns the path. Example: `/top/news.html` The path does not include the query string.
diff --git a/src/Functions/URL/port.cpp b/src/Functions/URL/port.cpp
new file mode 100644
index 00000000000..d4876e9f647
--- /dev/null
+++ b/src/Functions/URL/port.cpp
@@ -0,0 +1,95 @@
+#include <Functions/FunctionFactory.h>
+#include <Functions/FunctionStringOrArrayToT.h>
+#include <Common/StringUtils/StringUtils.h>
+#include "domain.h"
+
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+}
+
+struct ExtractPort
+{
+    static constexpr auto name = "port";
+    static constexpr auto is_fixed_to_constant = true;
+
+    static void vector(const ColumnString::Chars & data, const ColumnString::Offsets & offsets, PaddedPODArray<UInt16> & res)
+    {
+        size_t size = offsets.size();
+
+        ColumnString::Offset prev_offset = 0;
+        for (size_t i = 0; i < size; ++i)
+        {
+            res[i] = parse(data, prev_offset, offsets[i] - prev_offset - 1);
+            prev_offset = offsets[i];
+        }
+    }
+
+    static void vectorFixedToConstant(const ColumnString::Chars & data, size_t n, UInt16 & res) { res = parse(data, 0, n); }
+
+    static void vectorFixedToVector(const ColumnString::Chars & data, size_t n, PaddedPODArray<UInt16> & res)
+    {
+        size_t size = data.size() / n;
+
+        for (size_t i = 0; i < size; ++i)
+        {
+            res[i] = parse(data, i * n, n);
+        }
+    }
+
+    [[noreturn]] static void array(const ColumnString::Offsets &, PaddedPODArray<UInt16> &)
+    {
+        throw Exception("Cannot apply function port to Array argument", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+    }
+
+private:
+    static UInt16 parse(const ColumnString::Chars & buf, size_t offset, size_t size)
+    {
+        const char * p = reinterpret_cast<const char *>(&buf[0]) + offset;
+        const char * end = p + size;
+
+        StringRef host = getURLHost(p, size);
+        if (!host.size)
+            return 0;
+        if (host.size == size)
+            return 0;
+
+        p = host.data + host.size;
+        if (*p++ != ':')
+            return 0;
+
+        Int64 port = 0;
+        while (p < end)
+        {
+            if (*p == '/')
+                break;
+            if (!isNumericASCII(*p))
+                return 0;
+
+            port = (port * 10) + (*p - '0');
+            if (port < 0 || port > UInt16(-1))
+                return 0;
+            ++p;
+        }
+        return port;
+    }
+};
+
+struct NamePort
+{
+    static constexpr auto name = "port";
+};
+
+using FunctionPort = FunctionStringOrArrayToT<ExtractPort, NamePort, UInt16>;
+
+void registerFunctionPort(FunctionFactory & factory)
+{
+    factory.registerFunction<FunctionPort>();
+}
+
+}
+
diff --git a/src/Functions/URL/registerFunctionsURL.cpp b/src/Functions/URL/registerFunctionsURL.cpp
index 66a847185f3..9ba5261f728 100644
--- a/src/Functions/URL/registerFunctionsURL.cpp
+++ b/src/Functions/URL/registerFunctionsURL.cpp
@@ -8,6 +8,7 @@ void registerFunctionDomain(FunctionFactory & factory);
 void registerFunctionDomainWithoutWWW(FunctionFactory & factory);
 void registerFunctionFirstSignificantSubdomain(FunctionFactory & factory);
 void registerFunctionTopLevelDomain(FunctionFactory & factory);
+void registerFunctionPort(FunctionFactory & factory);
 void registerFunctionPath(FunctionFactory & factory);
 void registerFunctionPathFull(FunctionFactory & factory);
 void registerFunctionQueryString(FunctionFactory & factory);
@@ -33,6 +34,7 @@ void registerFunctionsURL(FunctionFactory & factory)
     registerFunctionDomainWithoutWWW(factory);
     registerFunctionFirstSignificantSubdomain(factory);
     registerFunctionTopLevelDomain(factory);
+    registerFunctionPort(factory);
     registerFunctionPath(factory);
     registerFunctionPathFull(factory);
     registerFunctionQueryString(factory);
diff --git a/tests/queries/0_stateless/01284_port.reference b/tests/queries/0_stateless/01284_port.reference
new file mode 100644
index 00000000000..5bf1c035ca8
--- /dev/null
+++ b/tests/queries/0_stateless/01284_port.reference
@@ -0,0 +1,21 @@
+ipv4
+0
+80
+80
+80
+80
+hostname
+0
+80
+80
+80
+80
+ipv6
+0
+0
+0
+0
+0
+0
+host-no-dot
+0
diff --git a/tests/queries/0_stateless/01284_port.sql b/tests/queries/0_stateless/01284_port.sql
new file mode 100644
index 00000000000..3ba5aecd691
--- /dev/null
+++ b/tests/queries/0_stateless/01284_port.sql
@@ -0,0 +1,25 @@
+select 'ipv4';
+select port('http://127.0.0.1/');
+select port('http://127.0.0.1:80');
+select port('http://127.0.0.1:80/');
+select port('//127.0.0.1:80/');
+select port('127.0.0.1:80');
+select 'hostname';
+select port('http://foobar.com/');
+select port('http://foobar.com:80');
+select port('http://foobar.com:80/');
+select port('//foobar.com:80/');
+select port('foobar.com:80');
+
+--
+-- Known limitations of domain() (getURLHost())
+--
+select 'ipv6';
+select port('http://[2001:db8::8a2e:370:7334]/');
+select port('http://[2001:db8::8a2e:370:7334]:80');
+select port('http://[2001:db8::8a2e:370:7334]:80/');
+select port('//[2001:db8::8a2e:370:7334]:80/');
+select port('[2001:db8::8a2e:370:7334]:80');
+select port('2001:db8::8a2e:370:7334:80');
+select 'host-no-dot';
+select port('//foobar:80/');

From aa5247f1fed85e64479985dc0de65621118a249d Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Thu, 21 May 2020 23:58:55 +0300
Subject: [PATCH 010/184] Get back 01281_group_by_limit_memory_tracking
 (without flackiness)

---
 ...1_group_by_limit_memory_tracking.reference |  0
 .../01281_group_by_limit_memory_tracking.sh   | 34 +++++++++++++++++++
 2 files changed, 34 insertions(+)
 create mode 100644 tests/queries/0_stateless/01281_group_by_limit_memory_tracking.reference
 create mode 100755 tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh

diff --git a/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.reference b/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.reference
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh b/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh
new file mode 100755
index 00000000000..79a731b9273
--- /dev/null
+++ b/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+# Regression for MemoryTracker that had been incorrectly accounted
+# (it was reseted before deallocation)
+#
+# For this will be used:
+# - two-level group by
+# - max_memory_usage_for_user
+# - one users' query in background (to avoid reseting max_memory_usage_for_user)
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+. $CURDIR/../shell_config.sh
+
+set -o pipefail
+
+function execute_null()
+{
+    ${CLICKHOUSE_CLIENT} --format Null -n "$@"
+}
+
+function execute_group_by()
+{
+    local opts=(
+        --max_memory_usage_for_user=$((150<<20))
+        --max_threads=2
+    )
+    execute_null "${opts[@]}" <<<'SELECT uniq(number) FROM numbers_mt(toUInt64(1e6)) GROUP BY number % 5e5'
+}
+
+execute_null <<<'SELECT sleep(3)' &
+execute_group_by
+# if memory accounting will be incorrect, the second query will be failed with MEMORY_LIMIT_EXCEEDED
+execute_group_by
+wait

From 9de1a231c9b003c306570cedbf380b4bff922294 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sat, 23 May 2020 21:14:52 +0300
Subject: [PATCH 011/184] Add optional default_port argument for the port()
 function

---
 .../sql-reference/functions/url-functions.md  |   4 +-
 src/Functions/URL/port.cpp                    | 110 ++++++++++++------
 .../queries/0_stateless/01284_port.reference  |   3 +
 tests/queries/0_stateless/01284_port.sql      |   9 ++
 4 files changed, 86 insertions(+), 40 deletions(-)

diff --git a/docs/en/sql-reference/functions/url-functions.md b/docs/en/sql-reference/functions/url-functions.md
index f35c621a6e2..34477529649 100644
--- a/docs/en/sql-reference/functions/url-functions.md
+++ b/docs/en/sql-reference/functions/url-functions.md
@@ -117,9 +117,9 @@ Returns the part of the domain that includes top-level subdomains up to the “f
 
 For example, `cutToFirstSignificantSubdomain('https://news.yandex.com.tr/') = 'yandex.com.tr'`.
 
-### port(URL) {#port}
+### port(URL[, default_port = 0]) {#port}
 
-Returns the port or zero if there is no port in the URL (or in case of validation error).
+Returns the port or `default_port` if there is no port in the URL (or in case of validation error).
 
 ### path {#path}
 
diff --git a/src/Functions/URL/port.cpp b/src/Functions/URL/port.cpp
index d4876e9f647..e703da2602b 100644
--- a/src/Functions/URL/port.cpp
+++ b/src/Functions/URL/port.cpp
@@ -1,6 +1,10 @@
 #include <Functions/FunctionFactory.h>
-#include <Functions/FunctionStringOrArrayToT.h>
+#include <Functions/IFunctionImpl.h>
 #include <Common/StringUtils/StringUtils.h>
+#include <DataTypes/DataTypesNumber.h>
+#include <Columns/ColumnsNumber.h>
+#include <Columns/ColumnArray.h>
+#include <Columns/ColumnConst.h>
 #include "domain.h"
 
 
@@ -9,83 +13,113 @@ namespace DB
 
 namespace ErrorCodes
 {
+    extern const int ILLEGAL_COLUMN;
     extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
 }
 
-struct ExtractPort
+struct FunctionPort : public IFunction
 {
     static constexpr auto name = "port";
-    static constexpr auto is_fixed_to_constant = true;
+    static FunctionPtr create(const Context &) { return std::make_shared<FunctionPort>(); }
 
-    static void vector(const ColumnString::Chars & data, const ColumnString::Offsets & offsets, PaddedPODArray<UInt16> & res)
+    String getName() const override { return name; }
+    bool isVariadic() const override { return true; }
+    size_t getNumberOfArguments() const override { return 0; }
+    bool useDefaultImplementationForConstants() const override { return true; }
+    ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; }
+
+    DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
+    {
+        if (arguments.size() != 1 && arguments.size() != 2)
+            throw Exception("Number of arguments for function " + getName() + " doesn't match: passed "
+                            + std::to_string(arguments.size()) + ", should be 1 or 2",
+                            ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
+
+        if (!WhichDataType(arguments[0].type).isString())
+            throw Exception("Illegal type " + arguments[0].type->getName() + " of first argument of function " + getName() + ". Must be String.",
+                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+
+        if (arguments.size() == 2 && !WhichDataType(arguments[1].type).isUInt16())
+            throw Exception("Illegal type " + arguments[1].type->getName() + " of second argument of function " + getName() + ". Must be UInt16.",
+                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+
+        return std::make_shared<DataTypeUInt16>();
+    }
+
+
+    void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t) override
+    {
+        UInt16 default_port = 0;
+        if (arguments.size() == 2)
+        {
+            const auto * port_column = checkAndGetColumn<ColumnConst>(block.getByPosition(arguments[1]).column.get());
+            if (!port_column)
+                throw Exception("Second argument for function " + getName() + " must be constant UInt16", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+            default_port = port_column->getValue<UInt16>();
+        }
+
+        const ColumnPtr url_column = block.getByPosition(arguments[0]).column;
+        if (const ColumnString * url_strs = checkAndGetColumn<ColumnString>(url_column.get()))
+        {
+            auto col_res = ColumnVector<UInt16>::create();
+            typename ColumnVector<UInt16>::Container & vec_res = col_res->getData();
+            vec_res.resize(url_column->size());
+
+            vector(default_port, url_strs->getChars(), url_strs->getOffsets(), vec_res);
+            block.getByPosition(result).column = std::move(col_res);
+        }
+        else
+            throw Exception(
+                "Illegal column " + block.getByPosition(arguments[0]).column->getName() + " of argument of function " + getName(),
+                ErrorCodes::ILLEGAL_COLUMN);
+}
+
+private:
+    static void vector(UInt16 default_port, const ColumnString::Chars & data, const ColumnString::Offsets & offsets, PaddedPODArray<UInt16> & res)
     {
         size_t size = offsets.size();
 
         ColumnString::Offset prev_offset = 0;
         for (size_t i = 0; i < size; ++i)
         {
-            res[i] = parse(data, prev_offset, offsets[i] - prev_offset - 1);
+            res[i] = extractPort(default_port, data, prev_offset, offsets[i] - prev_offset - 1);
             prev_offset = offsets[i];
         }
-    }
+}
 
-    static void vectorFixedToConstant(const ColumnString::Chars & data, size_t n, UInt16 & res) { res = parse(data, 0, n); }
-
-    static void vectorFixedToVector(const ColumnString::Chars & data, size_t n, PaddedPODArray<UInt16> & res)
-    {
-        size_t size = data.size() / n;
-
-        for (size_t i = 0; i < size; ++i)
-        {
-            res[i] = parse(data, i * n, n);
-        }
-    }
-
-    [[noreturn]] static void array(const ColumnString::Offsets &, PaddedPODArray<UInt16> &)
-    {
-        throw Exception("Cannot apply function port to Array argument", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
-    }
-
-private:
-    static UInt16 parse(const ColumnString::Chars & buf, size_t offset, size_t size)
+    static UInt16 extractPort(UInt16 default_port, const ColumnString::Chars & buf, size_t offset, size_t size)
     {
         const char * p = reinterpret_cast<const char *>(&buf[0]) + offset;
         const char * end = p + size;
 
         StringRef host = getURLHost(p, size);
         if (!host.size)
-            return 0;
+            return default_port;
         if (host.size == size)
-            return 0;
+            return default_port;
 
         p = host.data + host.size;
         if (*p++ != ':')
-            return 0;
+            return default_port;
 
-        Int64 port = 0;
+        Int64 port = default_port;
         while (p < end)
         {
             if (*p == '/')
                 break;
             if (!isNumericASCII(*p))
-                return 0;
+                return default_port;
 
             port = (port * 10) + (*p - '0');
             if (port < 0 || port > UInt16(-1))
-                return 0;
+                return default_port;
             ++p;
         }
         return port;
     }
 };
 
-struct NamePort
-{
-    static constexpr auto name = "port";
-};
-
-using FunctionPort = FunctionStringOrArrayToT<ExtractPort, NamePort, UInt16>;
-
 void registerFunctionPort(FunctionFactory & factory)
 {
     factory.registerFunction<FunctionPort>();
diff --git a/tests/queries/0_stateless/01284_port.reference b/tests/queries/0_stateless/01284_port.reference
index 5bf1c035ca8..7e776595065 100644
--- a/tests/queries/0_stateless/01284_port.reference
+++ b/tests/queries/0_stateless/01284_port.reference
@@ -10,6 +10,9 @@ hostname
 80
 80
 80
+default-port
+80
+80
 ipv6
 0
 0
diff --git a/tests/queries/0_stateless/01284_port.sql b/tests/queries/0_stateless/01284_port.sql
index 3ba5aecd691..9c31a5d42ad 100644
--- a/tests/queries/0_stateless/01284_port.sql
+++ b/tests/queries/0_stateless/01284_port.sql
@@ -11,6 +11,15 @@ select port('http://foobar.com:80/');
 select port('//foobar.com:80/');
 select port('foobar.com:80');
 
+select 'default-port';
+select port('http://127.0.0.1/', toUInt16(80));
+select port('http://foobar.com/', toUInt16(80));
+
+-- unsupported
+/* ILLEGAL_TYPE_OF_ARGUMENT */ select port(toFixedString('', 1)); -- { serverError 43; }
+/* ILLEGAL_TYPE_OF_ARGUMENT */ select port('', 1); -- { serverError 43; }
+/* NUMBER_OF_ARGUMENTS_DOESNT_MATCH */ select port('', 1, 1); -- { serverError 42; }
+
 --
 -- Known limitations of domain() (getURLHost())
 --

From 96260cf33a12e906e8db3303c057713db954f1b9 Mon Sep 17 00:00:00 2001
From: Albert Kidrachev <kidrachev@sandbox-410109003>
Date: Mon, 25 May 2020 02:55:00 +0300
Subject: [PATCH 012/184] init

---
 src/Common/SharedBlockRowRef.h                | 30 +++++++
 src/Interpreters/InterpreterSelectQuery.cpp   |  1 +
 .../OptimizedPartialSortingTransform.cpp      | 83 +++++++++++++++++++
 .../OptimizedPartialSortingTransform.h        | 34 ++++++++
 src/Processors/ya.make                        |  1 +
 5 files changed, 149 insertions(+)
 create mode 100644 src/Processors/Transforms/OptimizedPartialSortingTransform.cpp
 create mode 100644 src/Processors/Transforms/OptimizedPartialSortingTransform.h

diff --git a/src/Common/SharedBlockRowRef.h b/src/Common/SharedBlockRowRef.h
index 193f7e4dd05..e9fd076da07 100644
--- a/src/Common/SharedBlockRowRef.h
+++ b/src/Common/SharedBlockRowRef.h
@@ -2,6 +2,7 @@
 
 #include <algorithm>
 #include <Core/Block.h>
+#include <Core/SortDescription.h>
 #include <Columns/IColumn.h>
 #include <boost/smart_ptr/intrusive_ptr.hpp>
 
@@ -86,4 +87,33 @@ struct SharedBlockRowRef
     }
 };
 
+struct SharedBlockRowWithSortDescriptionRef : SharedBlockRowRef
+{
+    SortDescription * description = nullptr;
+
+    void set(SharedBlockPtr & shared_block_, ColumnRawPtrs * columns_, size_t row_num_) = delete;
+
+    bool operator< (const SharedBlockRowRef & other) const
+    {
+        size_t size = columns->size();
+        for (size_t i = 0; i < size; ++i)
+        {
+            int res = (*description)[i].direction * (*columns)[i]->compareAt(row_num, other.row_num, *(*other.columns)[i], 1);
+            if (res < 0)
+                return true;
+            else if (res > 0)
+                return false;
+        }
+        return false;
+    }
+
+    void set(SharedBlockPtr & shared_block_, ColumnRawPtrs * columns_, size_t row_num_, SortDescription * description_)
+    {
+        shared_block = shared_block_;
+        columns = columns_;
+        row_num = row_num_;
+        description = description_;
+    }
+};
+
 }
diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp
index c8d842dfa53..3d95265cb27 100644
--- a/src/Interpreters/InterpreterSelectQuery.cpp
+++ b/src/Interpreters/InterpreterSelectQuery.cpp
@@ -84,6 +84,7 @@
 #include <Processors/Transforms/MergingAggregatedMemoryEfficientTransform.h>
 #include <Processors/Transforms/TotalsHavingTransform.h>
 #include <Processors/Transforms/PartialSortingTransform.h>
+#include <Processors/Transforms/OptimizedPartialSortingTransform.h>
 #include <Processors/Transforms/LimitsCheckingTransform.h>
 #include <Processors/Transforms/MergeSortingTransform.h>
 #include <Processors/Transforms/DistinctTransform.h>
diff --git a/src/Processors/Transforms/OptimizedPartialSortingTransform.cpp b/src/Processors/Transforms/OptimizedPartialSortingTransform.cpp
new file mode 100644
index 00000000000..e753052a2b3
--- /dev/null
+++ b/src/Processors/Transforms/OptimizedPartialSortingTransform.cpp
@@ -0,0 +1,83 @@
+#include <Processors/Transforms/OptimizedPartialSortingTransform.h>
+#include <Interpreters/sortBlock.h>
+#include <Common/PODArray.h>
+
+namespace DB
+{
+
+OptimizedPartialSortingTransform::OptimizedPartialSortingTransform(
+    const Block & header_, SortDescription & description_, UInt64 limit_)
+    : ISimpleTransform(header_, header_, false)
+    , description(description_), limit(limit_)
+    , threshold_shared_block(nullptr)
+{
+}
+
+static ColumnRawPtrs extractColumns(const Block & block, const SortDescription& description)
+{
+    size_t size = description.size();
+    ColumnRawPtrs res;
+    res.reserve(size);
+
+    for (size_t i = 0; i < size; ++i)
+    {
+        const IColumn * column = !description[i].column_name.empty()
+            ? block.getByName(description[i].column_name).column.get()
+            : block.safeGetByPosition(description[i].column_number).column.get();
+        res.emplace_back(column);
+    }
+
+    return res;
+}
+
+void OptimizedPartialSortingTransform::transform(Chunk & chunk)
+{
+    if (read_rows)
+        read_rows->add(chunk.getNumRows());
+
+    auto block = getInputPort().getHeader().cloneWithColumns(chunk.detachColumns());
+    chunk.clear();
+
+    SharedBlockPtr shared_block = new detail::SharedBlock(std::move(block));
+    UInt64 rows_num = shared_block->rows();
+
+
+    if (threshold_shared_block) {
+        SharedBlockRowWithSortDescriptionRef row;
+        IColumn::Filter filter(rows_num);
+        ColumnRawPtrs shared_block_columns = extractColumns(*shared_block, description);
+        size_t filtered_count = 0;
+
+        for (UInt64 i = 0; i < rows_num; ++i) {
+            row.set(shared_block, &shared_block_columns, i, &description);
+
+            if (threshold_row < row)
+            {
+                ++filtered_count;
+                filter[i] = 1;
+            }
+        }
+
+        if (filtered_count)
+        {
+            for (auto & column : shared_block->getColumns())
+            {
+                column = column->filter(filter, filtered_count);
+            }
+        }
+    }
+
+    sortBlock(*shared_block, description, limit);
+
+    if (!threshold_shared_block && limit && limit < rows_num)
+    {
+        Block threshold_block = shared_block->cloneWithColumns(shared_block->getColumns());
+        threshold_shared_block = new detail::SharedBlock(std::move(threshold_block));
+        threshold_block_columns = extractColumns(*threshold_shared_block, description);
+        threshold_row.set(threshold_shared_block, &threshold_block_columns, limit - 1, &description);
+    }
+
+    chunk.setColumns(shared_block->getColumns(), shared_block->rows());
+}
+
+}
diff --git a/src/Processors/Transforms/OptimizedPartialSortingTransform.h b/src/Processors/Transforms/OptimizedPartialSortingTransform.h
new file mode 100644
index 00000000000..20e72bd836f
--- /dev/null
+++ b/src/Processors/Transforms/OptimizedPartialSortingTransform.h
@@ -0,0 +1,34 @@
+#pragma once
+#include <Processors/ISimpleTransform.h>
+#include <Processors/RowsBeforeLimitCounter.h>
+#include <Common/SharedBlockRowRef.h>
+#include <Core/SortDescription.h>
+
+
+namespace DB
+{
+class OptimizedPartialSortingTransform : public ISimpleTransform
+{
+public:
+    OptimizedPartialSortingTransform(
+        const Block & header_,
+        SortDescription & description_,
+        UInt64 limit_ = 0);
+
+    String getName() const override { return "OptimizedPartialSortingTransform"; }
+
+    void setRowsBeforeLimitCounter(RowsBeforeLimitCounterPtr counter) { read_rows.swap(counter); }
+
+protected:
+    void transform(Chunk & chunk) override;
+
+private:
+    SortDescription description;
+    UInt64 limit;
+    RowsBeforeLimitCounterPtr read_rows;
+    SharedBlockRowWithSortDescriptionRef threshold_row;
+    SharedBlockPtr threshold_shared_block;
+    ColumnRawPtrs threshold_block_columns;
+};
+
+}
diff --git a/src/Processors/ya.make b/src/Processors/ya.make
index 62320f1c147..5952341527b 100644
--- a/src/Processors/ya.make
+++ b/src/Processors/ya.make
@@ -129,6 +129,7 @@ SRCS(
     Transforms/MergeSortingTransform.cpp
     Transforms/MergingAggregatedMemoryEfficientTransform.cpp
     Transforms/MergingAggregatedTransform.cpp
+    Transforms/OptimizedPartialSortingTransform.cpp
     Transforms/PartialSortingTransform.cpp
     Transforms/ReverseTransform.cpp
     Transforms/RollupTransform.cpp

From 3abf7b7dcdc26acda9aae9663e9e026f1b5cb631 Mon Sep 17 00:00:00 2001
From: Albert Kidrachev <kidrachev@sandbox-410109003>
Date: Mon, 25 May 2020 03:32:08 +0300
Subject: [PATCH 013/184] fix

---
 src/Interpreters/InterpreterSelectQuery.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp
index 3d95265cb27..c3fce45ac61 100644
--- a/src/Interpreters/InterpreterSelectQuery.cpp
+++ b/src/Interpreters/InterpreterSelectQuery.cpp
@@ -2109,7 +2109,7 @@ void InterpreterSelectQuery::executeOrder(QueryPipeline & pipeline, InputSorting
         if (stream_type != QueryPipeline::StreamType::Main)
             return nullptr;
 
-        return std::make_shared<PartialSortingTransform>(header, output_order_descr, limit);
+        return std::make_shared<OptimizedPartialSortingTransform>(header, output_order_descr, limit);
     });
 
     pipeline.addSimpleTransform([&](const Block & header, QueryPipeline::StreamType stream_type) -> ProcessorPtr

From f2e3bcdd15d5436ea2c8866457964f6c3572978f Mon Sep 17 00:00:00 2001
From: hcz <h.cz@qq.com>
Date: Mon, 25 May 2020 10:50:55 +0800
Subject: [PATCH 014/184] Improve code style in src and test files of Arrow
 formats

---
 src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp |  9 +++++++--
 src/Processors/Formats/Impl/ArrowBlockInputFormat.h   |  4 ++++
 tests/queries/0_stateless/01273_arrow.reference       |  2 --
 tests/queries/0_stateless/01273_arrow.sh              | 11 -----------
 .../queries/0_stateless/01273_arrow_stream.reference  |  2 --
 tests/queries/0_stateless/01273_arrow_stream.sh       | 11 -----------
 6 files changed, 11 insertions(+), 28 deletions(-)

diff --git a/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp b/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp
index 5468fbcac16..e0eb0e21567 100644
--- a/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp
+++ b/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp
@@ -31,11 +31,15 @@ Chunk ArrowBlockInputFormat::generate()
     Chunk res;
     const Block & header = getPort().getHeader();
 
-    if (!stream && record_batch_current >= record_batch_total)
-        return res;
+    if (!stream)
+    {
+        if (record_batch_current >= record_batch_total)
+            return res;
+    }
 
     std::vector<std::shared_ptr<arrow::RecordBatch>> single_batch(1);
     arrow::Status read_status;
+
     if (stream)
     {
         read_status = stream_reader->ReadNext(&single_batch[0]);
@@ -46,6 +50,7 @@ Chunk ArrowBlockInputFormat::generate()
     {
         read_status = file_reader->ReadRecordBatch(record_batch_current, &single_batch[0]);
     }
+
     if (!read_status.ok())
         throw Exception{"Error while reading batch of Arrow data: " + read_status.ToString(),
                         ErrorCodes::CANNOT_READ_ALL_DATA};
diff --git a/src/Processors/Formats/Impl/ArrowBlockInputFormat.h b/src/Processors/Formats/Impl/ArrowBlockInputFormat.h
index 2e7f647b4f7..5ad112efde9 100644
--- a/src/Processors/Formats/Impl/ArrowBlockInputFormat.h
+++ b/src/Processors/Formats/Impl/ArrowBlockInputFormat.h
@@ -25,9 +25,13 @@ protected:
     Chunk generate() override;
 
 private:
+    // Whether to use ArrowStream format
     bool stream;
+    // This field is only used for ArrowStream format
     std::shared_ptr<arrow::RecordBatchReader> stream_reader;
+    // The following fields are used only for Arrow format
     std::shared_ptr<arrow::ipc::RecordBatchFileReader> file_reader;
+
     int record_batch_total = 0;
     int record_batch_current = 0;
 
diff --git a/tests/queries/0_stateless/01273_arrow.reference b/tests/queries/0_stateless/01273_arrow.reference
index 0f4be2c74a0..0dc503f65e4 100644
--- a/tests/queries/0_stateless/01273_arrow.reference
+++ b/tests/queries/0_stateless/01273_arrow.reference
@@ -31,8 +31,6 @@
 992
 991
 990
-ContextLock	Number of times the lock of Context was acquired or tried to acquire. This is global lock.
-Query	Number of queries to be interpreted and potentially executed. Does not include queries that failed to parse or were rejected due to AST size limits, quota limits or limits on the number of simultaneously running queries. May include internal queries initiated by ClickHouse itself. Does not count subqueries.
 original:
 -128	0	-32768	0	-2147483648	0	-9223372036854775808	0	-1.032	-1.064	string-1	fixedstring-1\0\0	2003-04-05	2003-02-03 04:05:06
 -108	108	-1016	1116	-1032	1132	-1064	1164	-1.032	-1.064	string-0	fixedstring\0\0\0\0	2001-02-03	2002-02-03 04:05:06
diff --git a/tests/queries/0_stateless/01273_arrow.sh b/tests/queries/0_stateless/01273_arrow.sh
index f659a81f118..deb0aa20ccc 100755
--- a/tests/queries/0_stateless/01273_arrow.sh
+++ b/tests/queries/0_stateless/01273_arrow.sh
@@ -34,14 +34,6 @@ ${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_numbers ORDER BY number DESC L
 
 ${CLICKHOUSE_CLIENT} --query="DROP TABLE arrow_numbers"
 
-${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_events"
-${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_events (event String, value UInt64, description String) ENGINE = Memory"
-${CLICKHOUSE_CLIENT} --query="SELECT * FROM system.events FORMAT Arrow" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_events FORMAT Arrow"
-${CLICKHOUSE_CLIENT} --query="SELECT event, description FROM arrow_events WHERE event IN ('ContextLock', 'Query') ORDER BY event"
-${CLICKHOUSE_CLIENT} --query="DROP TABLE arrow_events"
-
-
-
 ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_types1"
 ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_types2"
 ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_types3"
@@ -61,8 +53,6 @@ ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types1 values (     -128,
 # max
 ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types1 values (      127,         255,       32767,         65535,  2147483647,    4294967295, 9223372036854775807, 9223372036854775807, -1.032,     -1.064,    'string-2',             'fixedstring-2', '2004-06-07', '2004-02-03 04:05:06')"
 
-# 'SELECT -127,-128,-129,126,127,128,255,256,257,-32767,-32768,-32769,32766,32767,32768,65535,65536,65537,  -2147483647,-2147483648,-2147483649,2147483646,2147483647,2147483648,4294967295,4294967296,4294967297,   -9223372036854775807,-9223372036854775808,9223372036854775806,9223372036854775807,9223372036854775808,18446744073709551615';
-
 ${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types1 FORMAT Arrow" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types2 FORMAT Arrow"
 
 echo original:
@@ -98,7 +88,6 @@ ${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types5       (int8 Nullable(Int
 ${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types6       (int8 Nullable(Int8), uint8 Nullable(UInt8), int16 Nullable(Int16), uint16 Nullable(UInt16), int32 Nullable(Int32), uint32 Nullable(UInt32), int64 Nullable(Int64), uint64 Nullable(UInt64), float32 Nullable(Float32), float64 Nullable(Float64), string Nullable(String), fixedstring Nullable(FixedString(15)), date Nullable(Date), datetime Nullable(DateTime)) ENGINE = Memory"
 ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types5 values (               NULL,                  NULL,                  NULL,                    NULL,                  NULL,                    NULL,                  NULL,                    NULL,                      NULL,                      NULL,                    NULL,                                  NULL,                NULL,                        NULL)"
 ${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types5 ORDER BY int8 FORMAT Arrow" > ${CLICKHOUSE_TMP}/arrow_all_types_5.arrow
-#${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types5 FORMAT Arrow" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types6 FORMAT Arrow"
 ${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types5 ORDER BY int8 FORMAT Arrow" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types6 FORMAT Arrow"
 ${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types1 ORDER BY int8 FORMAT Arrow" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types6 FORMAT Arrow"
 echo dest from null:
diff --git a/tests/queries/0_stateless/01273_arrow_stream.reference b/tests/queries/0_stateless/01273_arrow_stream.reference
index 0f4be2c74a0..0dc503f65e4 100644
--- a/tests/queries/0_stateless/01273_arrow_stream.reference
+++ b/tests/queries/0_stateless/01273_arrow_stream.reference
@@ -31,8 +31,6 @@
 992
 991
 990
-ContextLock	Number of times the lock of Context was acquired or tried to acquire. This is global lock.
-Query	Number of queries to be interpreted and potentially executed. Does not include queries that failed to parse or were rejected due to AST size limits, quota limits or limits on the number of simultaneously running queries. May include internal queries initiated by ClickHouse itself. Does not count subqueries.
 original:
 -128	0	-32768	0	-2147483648	0	-9223372036854775808	0	-1.032	-1.064	string-1	fixedstring-1\0\0	2003-04-05	2003-02-03 04:05:06
 -108	108	-1016	1116	-1032	1132	-1064	1164	-1.032	-1.064	string-0	fixedstring\0\0\0\0	2001-02-03	2002-02-03 04:05:06
diff --git a/tests/queries/0_stateless/01273_arrow_stream.sh b/tests/queries/0_stateless/01273_arrow_stream.sh
index 2fc630a3a8b..a4bb403cb5a 100755
--- a/tests/queries/0_stateless/01273_arrow_stream.sh
+++ b/tests/queries/0_stateless/01273_arrow_stream.sh
@@ -34,14 +34,6 @@ ${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_numbers ORDER BY number DESC L
 
 ${CLICKHOUSE_CLIENT} --query="DROP TABLE arrow_numbers"
 
-${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_events"
-${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_events (event String, value UInt64, description String) ENGINE = Memory"
-${CLICKHOUSE_CLIENT} --query="SELECT * FROM system.events FORMAT ArrowStream" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_events FORMAT ArrowStream"
-${CLICKHOUSE_CLIENT} --query="SELECT event, description FROM arrow_events WHERE event IN ('ContextLock', 'Query') ORDER BY event"
-${CLICKHOUSE_CLIENT} --query="DROP TABLE arrow_events"
-
-
-
 ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_types1"
 ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_types2"
 ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_types3"
@@ -61,8 +53,6 @@ ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types1 values (     -128,
 # max
 ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types1 values (      127,         255,       32767,         65535,  2147483647,    4294967295, 9223372036854775807, 9223372036854775807, -1.032,     -1.064,    'string-2',             'fixedstring-2', '2004-06-07', '2004-02-03 04:05:06')"
 
-# 'SELECT -127,-128,-129,126,127,128,255,256,257,-32767,-32768,-32769,32766,32767,32768,65535,65536,65537,  -2147483647,-2147483648,-2147483649,2147483646,2147483647,2147483648,4294967295,4294967296,4294967297,   -9223372036854775807,-9223372036854775808,9223372036854775806,9223372036854775807,9223372036854775808,18446744073709551615';
-
 ${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types1 FORMAT ArrowStream" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types2 FORMAT ArrowStream"
 
 echo original:
@@ -98,7 +88,6 @@ ${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types5       (int8 Nullable(Int
 ${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types6       (int8 Nullable(Int8), uint8 Nullable(UInt8), int16 Nullable(Int16), uint16 Nullable(UInt16), int32 Nullable(Int32), uint32 Nullable(UInt32), int64 Nullable(Int64), uint64 Nullable(UInt64), float32 Nullable(Float32), float64 Nullable(Float64), string Nullable(String), fixedstring Nullable(FixedString(15)), date Nullable(Date), datetime Nullable(DateTime)) ENGINE = Memory"
 ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types5 values (               NULL,                  NULL,                  NULL,                    NULL,                  NULL,                    NULL,                  NULL,                    NULL,                      NULL,                      NULL,                    NULL,                                  NULL,                NULL,                        NULL)"
 ${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types5 ORDER BY int8 FORMAT ArrowStream" > ${CLICKHOUSE_TMP}/arrow_all_types_5.arrow
-#${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types5 FORMAT ArrowStream" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types6 FORMAT ArrowStream"
 ${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types5 ORDER BY int8 FORMAT ArrowStream" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types6 FORMAT ArrowStream"
 ${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types1 ORDER BY int8 FORMAT ArrowStream" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types6 FORMAT ArrowStream"
 echo dest from null:

From 9b3cc9e5252d6c3905338d1951b12720daffccdc Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Mon, 25 May 2020 20:07:14 +0300
Subject: [PATCH 015/184] Almost working ttl in IStorage

---
 src/DataStreams/TTLBlockInputStream.cpp       |  11 +-
 src/Storages/IStorage.cpp                     |  46 +++++++
 src/Storages/IStorage.h                       |  24 +++-
 src/Storages/MergeTree/MergeTreeData.cpp      | 124 +++++++-----------
 src/Storages/MergeTree/MergeTreeData.h        |  46 ++-----
 .../MergeTree/MergeTreeDataWriter.cpp         |  13 +-
 .../MergeTree/MergeTreePartsMover.cpp         |   8 +-
 .../ReplicatedMergeTreeTableMetadata.cpp      |   2 +-
 .../MergeTree/StorageFromMergeTreeDataPart.h  |   5 +-
 src/Storages/StorageInMemoryMetadata.cpp      |  70 ++++++++++
 src/Storages/StorageInMemoryMetadata.h        |  14 ++
 11 files changed, 228 insertions(+), 135 deletions(-)

diff --git a/src/DataStreams/TTLBlockInputStream.cpp b/src/DataStreams/TTLBlockInputStream.cpp
index 692d2c0d5cf..5064e2236f8 100644
--- a/src/DataStreams/TTLBlockInputStream.cpp
+++ b/src/DataStreams/TTLBlockInputStream.cpp
@@ -110,10 +110,11 @@ void TTLBlockInputStream::readSuffixImpl()
 
 void TTLBlockInputStream::removeRowsWithExpiredTableTTL(Block & block)
 {
-    storage.rows_ttl_entry.expression->execute(block);
+    const auto & rows_ttl = storage.getRowsTTL();
+    rows_ttl.expression->execute(block);
 
     const IColumn * ttl_column =
-        block.getByName(storage.rows_ttl_entry.result_column).column.get();
+        block.getByName(rows_ttl.result_column).column.get();
 
     const auto & column_names = header.getNames();
     MutableColumns result_columns;
@@ -152,7 +153,8 @@ void TTLBlockInputStream::removeValuesWithExpiredColumnTTL(Block & block)
     }
 
     std::vector<String> columns_to_remove;
-    for (const auto & [name, ttl_entry] : storage.column_ttl_entries_by_name)
+    const auto & column_ttl_entries_by_name = storage.getColumnTTLs();
+    for (const auto & [name, ttl_entry] : column_ttl_entries_by_name)
     {
         /// If we read not all table columns. E.g. while mutation.
         if (!block.has(name))
@@ -212,8 +214,9 @@ void TTLBlockInputStream::removeValuesWithExpiredColumnTTL(Block & block)
 
 void TTLBlockInputStream::updateMovesTTL(Block & block)
 {
+    const auto & move_ttl_entries = storage.getMoveTTLs();
     std::vector<String> columns_to_remove;
-    for (const auto & ttl_entry : storage.move_ttl_entries)
+    for (const auto & ttl_entry : move_ttl_entries)
     {
         auto & new_ttl_info = new_ttl_infos.moves_ttl[ttl_entry.result_column];
 
diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp
index 64e06a2534e..f8f2599184f 100644
--- a/src/Storages/IStorage.cpp
+++ b/src/Storages/IStorage.cpp
@@ -541,4 +541,50 @@ Names IStorage::getColumnsRequiredForSampling() const
     return {};
 }
 
+
+const StorageMetadataTTLField & IStorage::getRowsTTL() const
+{
+    return table_ttl.rows_ttl;
+}
+
+bool IStorage::hasRowsTTL() const
+{
+    return table_ttl.rows_ttl.expression != nullptr;
+}
+
+const StorageMetadataTTLFields & IStorage::getMoveTTLs() const
+{
+    return table_ttl.move_ttl;
+}
+
+bool IStorage::hasAnyMoveTTL() const
+{
+    return !table_ttl.move_ttl.empty();
+}
+
+const StorageMetadataTTLColumnEntries & IStorage::getColumnTTLs() const
+{
+    return column_ttls_by_name;
+}
+
+void IStorage::setColumnTTLs(const StorageMetadataTTLColumnEntries & column_ttls_by_name_)
+{
+    column_ttls_by_name = column_ttls_by_name_;
+}
+
+const StorageMetadataTableTTL & IStorage::getTableTTLs() const
+{
+    return table_ttl;
+}
+
+void IStorage::setTableTTLs(const StorageMetadataTableTTL & table_ttl_)
+{
+    table_ttl = table_ttl_;
+}
+
+bool IStorage::hasAnyTableTTL() const
+{
+    return hasAnyMoveTTL() || hasRowsTTL();
+}
+
 }
diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h
index f707b3a4b5c..416d8ffbbb1 100644
--- a/src/Storages/IStorage.h
+++ b/src/Storages/IStorage.h
@@ -130,10 +130,7 @@ public:
     virtual bool hasEvenlyDistributedRead() const { return false; }
 
     /// Returns true if there is set table TTL, any column TTL or any move TTL.
-    virtual bool hasAnyTTL() const { return false; }
-
-    /// Returns true if there is set TTL for rows.
-    virtual bool hasRowsTTL() const { return false; }
+    virtual bool hasAnyTTL() const { return hasRowsTTL() || hasAnyTableTTL(); }
 
     /// Optional size information of each physical column.
     /// Currently it's only used by the MergeTree family for query optimizations.
@@ -205,6 +202,9 @@ private:
     StorageMetadataKeyField sorting_key;
     StorageMetadataKeyField sampling_key;
 
+    StorageMetadataTTLColumnEntries column_ttls_by_name;
+    StorageMetadataTableTTL table_ttl;
+
 private:
     RWLockImpl::LockHolder tryLockTimed(
         const RWLock & rwlock, RWLockImpl::Type type, const String & query_id, const SettingSeconds & acquire_timeout) const;
@@ -514,6 +514,22 @@ public:
     /// Returns storage policy if storage supports it
     virtual StoragePolicyPtr getStoragePolicy() const { return {}; }
 
+    /// Returns true if there is set TTL for rows.
+    const StorageMetadataTTLField & getRowsTTL() const;
+    bool hasRowsTTL() const;
+
+    const StorageMetadataTTLFields & getMoveTTLs() const;
+    bool hasAnyMoveTTL() const;
+
+    const StorageMetadataTableTTL & getTableTTLs() const;
+    void setTableTTLs(const StorageMetadataTableTTL & table_ttl_);
+    bool hasAnyTableTTL() const;
+
+    const StorageMetadataTTLColumnEntries & getColumnTTLs() const;
+    void setColumnTTLs(const StorageMetadataTTLColumnEntries & column_ttls_by_name_);
+    bool hasAnyColumnTTL() const { return !column_ttls_by_name.empty(); }
+
+
     /// If it is possible to quickly determine exact number of rows in the table at this moment of time, then return it.
     /// Used for:
     /// - Simple count() opimization
diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 142d503569f..a9ba8c871b0 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -580,55 +580,17 @@ void MergeTreeData::initPartitionKey(ASTPtr partition_by_ast)
     setPartitionKey(new_partition_key);
 }
 
-namespace
-{
-
-void checkTTLExpression(const ExpressionActionsPtr & ttl_expression, const String & result_column_name)
-{
-    for (const auto & action : ttl_expression->getActions())
-    {
-        if (action.type == ExpressionAction::APPLY_FUNCTION)
-        {
-            IFunctionBase & func = *action.function_base;
-            if (!func.isDeterministic())
-                throw Exception("TTL expression cannot contain non-deterministic functions, "
-                    "but contains function " + func.getName(), ErrorCodes::BAD_ARGUMENTS);
-        }
-    }
-
-    const auto & result_column = ttl_expression->getSampleBlock().getByName(result_column_name);
-
-    if (!typeid_cast<const DataTypeDateTime *>(result_column.type.get())
-        && !typeid_cast<const DataTypeDate *>(result_column.type.get()))
-    {
-        throw Exception("TTL expression result column should have DateTime or Date type, but has "
-            + result_column.type->getName(), ErrorCodes::BAD_TTL_EXPRESSION);
-    }
-}
-
-}
 
 
 void MergeTreeData::setTTLExpressions(const ColumnsDescription & new_columns,
         const ASTPtr & new_ttl_table_ast, bool only_check)
 {
 
-    auto new_column_ttls = new_columns.getColumnTTLs();
+    auto new_column_ttl_asts = new_columns.getColumnTTLs();
 
-    auto create_ttl_entry = [this, &new_columns](ASTPtr ttl_ast)
-    {
-        TTLEntry result;
+    StorageMetadataTTLColumnEntries new_column_ttl_by_name = getColumnTTLs();
 
-        auto syntax_result = SyntaxAnalyzer(global_context).analyze(ttl_ast, new_columns.getAllPhysical());
-        result.expression = ExpressionAnalyzer(ttl_ast, syntax_result, global_context).getActions(false);
-        result.destination_type = DataDestinationType::DELETE;
-        result.result_column = ttl_ast->getColumnName();
-
-        checkTTLExpression(result.expression, result.result_column);
-        return result;
-    };
-
-    if (!new_column_ttls.empty())
+    if (!new_column_ttl_asts.empty())
     {
         NameSet columns_ttl_forbidden;
 
@@ -640,23 +602,24 @@ void MergeTreeData::setTTLExpressions(const ColumnsDescription & new_columns,
             for (const auto & col : getColumnsRequiredForSortingKey())
                 columns_ttl_forbidden.insert(col);
 
-        for (const auto & [name, ast] : new_column_ttls)
+        for (const auto & [name, ast] : new_column_ttl_asts)
         {
             if (columns_ttl_forbidden.count(name))
                 throw Exception("Trying to set TTL for key column " + name, ErrorCodes::ILLEGAL_COLUMN);
             else
             {
-                auto new_ttl_entry = create_ttl_entry(ast);
-                if (!only_check)
-                    column_ttl_entries_by_name[name] = new_ttl_entry;
+                auto new_ttl_entry = StorageMetadataTTLField::getTTLFromAST(ast, new_columns, global_context);
+                new_column_ttl_by_name[name] = new_ttl_entry;
             }
         }
+        if (!only_check)
+            setColumnTTLs(new_column_ttl_by_name);
     }
 
     if (new_ttl_table_ast)
     {
-        std::vector<TTLEntry> update_move_ttl_entries;
-        TTLEntry update_rows_ttl_entry;
+        StorageMetadataTTLFields update_move_ttl_entries;
+        StorageMetadataTTLField update_rows_ttl_entry;
 
         bool seen_delete_ttl = false;
         for (const auto & ttl_element_ptr : new_ttl_table_ast->children)
@@ -672,20 +635,16 @@ void MergeTreeData::setTTLExpressions(const ColumnsDescription & new_columns,
                     throw Exception("More than one DELETE TTL expression is not allowed", ErrorCodes::BAD_TTL_EXPRESSION);
                 }
 
-                auto new_rows_ttl_entry = create_ttl_entry(ttl_element->children[0]);
-                if (!only_check)
-                    update_rows_ttl_entry = new_rows_ttl_entry;
+                auto new_rows_ttl_entry = StorageMetadataTTLField::getTTLFromAST(ttl_element_ptr, new_columns, global_context);
+                update_rows_ttl_entry = new_rows_ttl_entry;
 
                 seen_delete_ttl = true;
             }
             else
             {
-                auto new_ttl_entry = create_ttl_entry(ttl_element->children[0]);
+                auto new_ttl_entry = StorageMetadataTTLField::getTTLFromAST(ttl_element_ptr, new_columns, global_context);
 
-                new_ttl_entry.entry_ast = ttl_element_ptr;
-                new_ttl_entry.destination_type = ttl_element->destination_type;
-                new_ttl_entry.destination_name = ttl_element->destination_name;
-                if (!new_ttl_entry.getDestination(getStoragePolicy()))
+                if (!getDestinationForTTL(new_ttl_entry))
                 {
                     String message;
                     if (new_ttl_entry.destination_type == DataDestinationType::DISK)
@@ -695,18 +654,21 @@ void MergeTreeData::setTTLExpressions(const ColumnsDescription & new_columns,
                     throw Exception(message, ErrorCodes::BAD_TTL_EXPRESSION);
                 }
 
-                if (!only_check)
-                    update_move_ttl_entries.emplace_back(std::move(new_ttl_entry));
+                update_move_ttl_entries.emplace_back(std::move(new_ttl_entry));
             }
         }
 
         if (!only_check)
         {
-            rows_ttl_entry = update_rows_ttl_entry;
-            ttl_table_ast = new_ttl_table_ast;
+            StorageMetadataTableTTL new_table_ttl
+            {
+                .definition_ast = new_ttl_table_ast,
+                .rows_ttl = update_rows_ttl_entry,
+                .move_ttl = update_move_ttl_entries,
+            };
 
             auto move_ttl_entries_lock = std::lock_guard<std::mutex>(move_ttl_entries_mutex);
-            move_ttl_entries = update_move_ttl_entries;
+            setTableTTLs(new_table_ttl);
         }
     }
 }
@@ -2918,7 +2880,7 @@ ReservationPtr MergeTreeData::tryReserveSpacePreferringTTLRules(UInt64 expected_
     auto ttl_entry = selectTTLEntryForTTLInfos(ttl_infos, time_of_move);
     if (ttl_entry)
     {
-        SpacePtr destination_ptr = ttl_entry->getDestination(getStoragePolicy());
+        SpacePtr destination_ptr = getDestinationForTTL(*ttl_entry);
         if (!destination_ptr)
         {
             if (ttl_entry->destination_type == DataDestinationType::VOLUME)
@@ -2952,37 +2914,39 @@ ReservationPtr MergeTreeData::tryReserveSpacePreferringTTLRules(UInt64 expected_
     return reservation;
 }
 
-SpacePtr MergeTreeData::TTLEntry::getDestination(StoragePolicyPtr policy) const
+SpacePtr MergeTreeData::getDestinationForTTL(const StorageMetadataTTLField & ttl) const
 {
-    if (destination_type == DataDestinationType::VOLUME)
-        return policy->getVolumeByName(destination_name);
-    else if (destination_type == DataDestinationType::DISK)
-        return policy->getDiskByName(destination_name);
+    auto policy = getStoragePolicy();
+    if (ttl.destination_type == DataDestinationType::VOLUME)
+        return policy->getVolumeByName(ttl.destination_name);
+    else if (ttl.destination_type == DataDestinationType::DISK)
+        return policy->getDiskByName(ttl.destination_name);
     else
         return {};
 }
 
-bool MergeTreeData::TTLEntry::isPartInDestination(StoragePolicyPtr policy, const IMergeTreeDataPart & part) const
+bool MergeTreeData::isPartInTTLDestination(const StorageMetadataTTLField & ttl, const IMergeTreeDataPart & part) const
 {
-    if (destination_type == DataDestinationType::VOLUME)
+    auto policy = getStoragePolicy();
+    if (ttl.destination_type == DataDestinationType::VOLUME)
     {
-        for (const auto & disk : policy->getVolumeByName(destination_name)->getDisks())
+        for (const auto & disk : policy->getVolumeByName(ttl.destination_name)->getDisks())
             if (disk->getName() == part.volume->getDisk()->getName())
                 return true;
     }
-    else if (destination_type == DataDestinationType::DISK)
-        return policy->getDiskByName(destination_name)->getName() == part.volume->getDisk()->getName();
+    else if (ttl.destination_type == DataDestinationType::DISK)
+        return policy->getDiskByName(ttl.destination_name)->getName() == part.volume->getDisk()->getName();
     return false;
 }
 
-std::optional<MergeTreeData::TTLEntry> MergeTreeData::selectTTLEntryForTTLInfos(
-        const IMergeTreeDataPart::TTLInfos & ttl_infos,
-        time_t time_of_move) const
+std::optional<StorageMetadataTTLField>
+MergeTreeData::selectTTLEntryForTTLInfos(const IMergeTreeDataPart::TTLInfos & ttl_infos, time_t time_of_move) const
 {
     time_t max_max_ttl = 0;
-    std::vector<DB::MergeTreeData::TTLEntry>::const_iterator best_entry_it;
+    std::vector<StorageMetadataTTLField>::const_iterator best_entry_it;
 
     auto lock = std::lock_guard(move_ttl_entries_mutex);
+    const auto & move_ttl_entries = getMoveTTLs();
     for (auto ttl_entry_it = move_ttl_entries.begin(); ttl_entry_it != move_ttl_entries.end(); ++ttl_entry_it)
     {
         auto ttl_info_it = ttl_infos.moves_ttl.find(ttl_entry_it->result_column);
@@ -2996,7 +2960,7 @@ std::optional<MergeTreeData::TTLEntry> MergeTreeData::selectTTLEntryForTTLInfos(
         }
     }
 
-    return max_max_ttl ? *best_entry_it : std::optional<MergeTreeData::TTLEntry>();
+    return max_max_ttl ? *best_entry_it : std::optional<StorageMetadataTTLField>();
 }
 
 MergeTreeData::DataParts MergeTreeData::getDataParts(const DataPartStates & affordable_states) const
@@ -3416,7 +3380,7 @@ bool MergeTreeData::areBackgroundMovesNeeded() const
     if (policy->getVolumes().size() > 1)
         return true;
 
-    return policy->getVolumes().size() == 1 && policy->getVolumes()[0]->getDisks().size() > 1 && !move_ttl_entries.empty();
+    return policy->getVolumes().size() == 1 && policy->getVolumes()[0]->getDisks().size() > 1 && hasAnyMoveTTL();
 }
 
 bool MergeTreeData::movePartsToSpace(const DataPartsVector & parts, SpacePtr space)
@@ -3555,7 +3519,7 @@ ColumnDependencies MergeTreeData::getColumnDependencies(const NameSet & updated_
 
     if (hasRowsTTL())
     {
-        if (add_dependent_columns(rows_ttl_entry.expression, required_ttl_columns))
+        if (add_dependent_columns(getRowsTTL().expression, required_ttl_columns))
         {
             /// Filter all columns, if rows TTL expression have to be recalculated.
             for (const auto & column : getColumns().getAllPhysical())
@@ -3563,13 +3527,13 @@ ColumnDependencies MergeTreeData::getColumnDependencies(const NameSet & updated_
         }
     }
 
-    for (const auto & [name, entry] : column_ttl_entries_by_name)
+    for (const auto & [name, entry] : getColumnTTLs())
     {
         if (add_dependent_columns(entry.expression, required_ttl_columns))
             updated_ttl_columns.insert(name);
     }
 
-    for (const auto & entry : move_ttl_entries)
+    for (const auto & entry : getMoveTTLs())
         add_dependent_columns(entry.expression, required_ttl_columns);
 
     for (const auto & column : indices_columns)
diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h
index 2b5b75110f1..b1e14188e21 100644
--- a/src/Storages/MergeTree/MergeTreeData.h
+++ b/src/Storages/MergeTree/MergeTreeData.h
@@ -518,11 +518,6 @@ public:
 
     bool hasSkipIndices() const { return !skip_indices.empty(); }
 
-    bool hasAnyColumnTTL() const { return !column_ttl_entries_by_name.empty(); }
-    bool hasAnyMoveTTL() const { return !move_ttl_entries.empty(); }
-    bool hasRowsTTL() const override { return !rows_ttl_entry.isEmpty(); }
-    bool hasAnyTTL() const override { return hasRowsTTL() || hasAnyMoveTTL() || hasAnyColumnTTL(); }
-
     /// Check that the part is not broken and calculate the checksums for it if they are not present.
     MutableDataPartPtr loadPartAndFixMetadata(const VolumePtr & volume, const String & relative_path) const;
 
@@ -624,6 +619,13 @@ public:
 
     /// Return alter conversions for part which must be applied on fly.
     AlterConversions getAlterConversionsForPart(const MergeTreeDataPartPtr part) const;
+    /// Returns destination disk or volume for the TTL rule according to current
+    /// storage policy
+    SpacePtr getDestinationForTTL(const StorageMetadataTTLField & ttl) const;
+
+    /// Checks if given part already belongs destination disk or volume for the
+    /// TTL rule.
+    bool isPartInTTLDestination(const StorageMetadataTTLField & ttl, const IMergeTreeDataPart & part) const;
 
     MergeTreeDataFormatVersion format_version;
 
@@ -646,39 +648,13 @@ public:
     ExpressionActionsPtr primary_key_and_skip_indices_expr;
     ExpressionActionsPtr sorting_key_and_skip_indices_expr;
 
-    struct TTLEntry
-    {
-        ExpressionActionsPtr expression;
-        String result_column;
+    std::optional<StorageMetadataTTLField> selectTTLEntryForTTLInfos(const IMergeTreeDataPart::TTLInfos & ttl_infos, time_t time_of_move) const;
 
-        /// Name and type of a destination are only valid in table-level context.
-        DataDestinationType destination_type;
-        String destination_name;
-
-        ASTPtr entry_ast;
-
-        /// Returns destination disk or volume for this rule.
-        SpacePtr getDestination(StoragePolicyPtr policy) const;
-
-        /// Checks if given part already belongs destination disk or volume for this rule.
-        bool isPartInDestination(StoragePolicyPtr policy, const IMergeTreeDataPart & part) const;
-
-        bool isEmpty() const { return expression == nullptr; }
-    };
-
-    std::optional<TTLEntry> selectTTLEntryForTTLInfos(const IMergeTreeDataPart::TTLInfos & ttl_infos, time_t time_of_move) const;
-
-    using TTLEntriesByName = std::unordered_map<String, TTLEntry>;
-    TTLEntriesByName column_ttl_entries_by_name;
-
-    TTLEntry rows_ttl_entry;
-
-    /// This mutex is required for background move operations which do not obtain global locks.
+    /// This mutex is required for background move operations which do not
+    /// obtain global locks.
+    /// TODO (alesap) It will be removed after metadata became atomic
     mutable std::mutex move_ttl_entries_mutex;
 
-    /// Vector rw operations have to be done under "move_ttl_entries_mutex".
-    std::vector<TTLEntry> move_ttl_entries;
-
     /// Limiting parallel sends per one table, used in DataPartsExchange
     std::atomic_uint current_table_sends {0};
 
diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp
index 48dbf66e72b..0209e6cc23d 100644
--- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp
+++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp
@@ -77,10 +77,12 @@ void buildScatterSelector(
 }
 
 /// Computes ttls and updates ttl infos
-void updateTTL(const MergeTreeData::TTLEntry & ttl_entry,
+void updateTTL(
+    const StorageMetadataTTLField & ttl_entry,
     IMergeTreeDataPart::TTLInfos & ttl_infos,
     DB::MergeTreeDataPartTTLInfo & ttl_info,
-    Block & block, bool update_part_min_max_ttls)
+    Block & block,
+    bool update_part_min_max_ttls)
 {
     bool remove_column = false;
     if (!block.has(ttl_entry.result_column))
@@ -228,7 +230,8 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa
     size_t expected_size = block.bytes();
 
     DB::IMergeTreeDataPart::TTLInfos move_ttl_infos;
-    for (const auto & ttl_entry : data.move_ttl_entries)
+    const auto & move_ttl_entries = data.getMoveTTLs();
+    for (const auto & ttl_entry : move_ttl_entries)
         updateTTL(ttl_entry, move_ttl_infos, move_ttl_infos.moves_ttl[ttl_entry.result_column], block, false);
 
     NamesAndTypesList columns = data.getColumns().getAllPhysical().filter(block.getNames());
@@ -287,9 +290,9 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa
     }
 
     if (data.hasRowsTTL())
-        updateTTL(data.rows_ttl_entry, new_data_part->ttl_infos, new_data_part->ttl_infos.table_ttl, block, true);
+        updateTTL(data.getRowsTTL(), new_data_part->ttl_infos, new_data_part->ttl_infos.table_ttl, block, true);
 
-    for (const auto & [name, ttl_entry] : data.column_ttl_entries_by_name)
+    for (const auto & [name, ttl_entry] : data.getColumnTTLs())
         updateTTL(ttl_entry, new_data_part->ttl_infos, new_data_part->ttl_infos.columns_ttl[name], block, true);
 
     new_data_part->ttl_infos.update(move_ttl_infos);
diff --git a/src/Storages/MergeTree/MergeTreePartsMover.cpp b/src/Storages/MergeTree/MergeTreePartsMover.cpp
index 1609e1e517e..a2c2099a5f5 100644
--- a/src/Storages/MergeTree/MergeTreePartsMover.cpp
+++ b/src/Storages/MergeTree/MergeTreePartsMover.cpp
@@ -128,14 +128,14 @@ bool MergeTreePartsMover::selectPartsForMove(
         if (!can_move(part, &reason))
             continue;
 
-        auto ttl_entry = part->storage.selectTTLEntryForTTLInfos(part->ttl_infos, time_of_move);
+        auto ttl_entry = data->selectTTLEntryForTTLInfos(part->ttl_infos, time_of_move);
         auto to_insert = need_to_move.find(part->volume->getDisk());
         ReservationPtr reservation;
         if (ttl_entry)
         {
-            auto destination = ttl_entry->getDestination(policy);
-            if (destination && !ttl_entry->isPartInDestination(policy, *part))
-                reservation = part->storage.tryReserveSpace(part->getBytesOnDisk(), ttl_entry->getDestination(policy));
+            auto destination = data->getDestinationForTTL(*ttl_entry);
+            if (destination && !data->isPartInTTLDestination(*ttl_entry, *part))
+                reservation = data->tryReserveSpace(part->getBytesOnDisk(), data->getDestinationForTTL(*ttl_entry));
         }
 
         if (reservation) /// Found reservation by TTL rule.
diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp
index 5925c4a6935..12bec501181 100644
--- a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp
+++ b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp
@@ -53,7 +53,7 @@ ReplicatedMergeTreeTableMetadata::ReplicatedMergeTreeTableMetadata(const MergeTr
     if (data.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING)
         partition_key = formattedAST(data.getPartitionKey().expression_list_ast);
 
-    ttl_table = formattedAST(data.ttl_table_ast);
+    ttl_table = formattedAST(data.getTableTTLs().definition_ast);
 
     skip_indices = data.getIndices().toString();
     if (data.canUseAdaptiveGranularity())
diff --git a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h
index a25a3a25337..77fdfd60b46 100644
--- a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h
+++ b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h
@@ -39,8 +39,7 @@ public:
         return part->storage.mayBenefitFromIndexForIn(left_in_operand, query_context);
     }
 
-    bool hasAnyTTL() const override { return part->storage.hasAnyTTL(); }
-    bool hasRowsTTL() const override { return part->storage.hasRowsTTL(); }
+    //bool hasAnyTTL() const override { return part->storage.hasAnyTTL(); }
 
     ColumnDependencies getColumnDependencies(const NameSet & updated_columns) const override
     {
@@ -65,6 +64,8 @@ protected:
         setColumns(part_->storage.getColumns());
         setIndices(part_->storage.getIndices());
         setSortingKey(part_->storage.getSortingKey());
+        setColumnTTLs(part->storage.getColumnTTLs());
+        setTableTTLs(part->storage.getTableTTLs());
     }
 
 private:
diff --git a/src/Storages/StorageInMemoryMetadata.cpp b/src/Storages/StorageInMemoryMetadata.cpp
index 6c13033da1e..61982dcf46f 100644
--- a/src/Storages/StorageInMemoryMetadata.cpp
+++ b/src/Storages/StorageInMemoryMetadata.cpp
@@ -6,10 +6,21 @@
 #include <Parsers/ASTExpressionList.h>
 #include <Parsers/ASTFunction.h>
 #include <Parsers/queryToString.h>
+#include <Parsers/ASTTTLElement.h>
+#include <Functions/IFunction.h>
+
+#include <DataTypes/DataTypeDateTime.h>
+#include <DataTypes/DataTypeDate.h>
 
 namespace DB
 {
 
+
+namespace ErrorCodes
+{
+    extern const int BAD_TTL_EXPRESSION;
+};
+
 StorageInMemoryMetadata::StorageInMemoryMetadata(
     const ColumnsDescription & columns_,
     const IndicesDescription & indices_,
@@ -138,4 +149,63 @@ StorageMetadataKeyField StorageMetadataKeyField::getKeyFromAST(const ASTPtr & de
     return result;
 }
 
+
+namespace
+{
+
+void checkTTLExpression(const ExpressionActionsPtr & ttl_expression, const String & result_column_name)
+{
+    for (const auto & action : ttl_expression->getActions())
+    {
+        if (action.type == ExpressionAction::APPLY_FUNCTION)
+        {
+            IFunctionBase & func = *action.function_base;
+            if (!func.isDeterministic())
+                throw Exception(
+                    "TTL expression cannot contain non-deterministic functions, "
+                    "but contains function "
+                        + func.getName(),
+                    ErrorCodes::BAD_ARGUMENTS);
+        }
+    }
+
+    const auto & result_column = ttl_expression->getSampleBlock().getByName(result_column_name);
+
+    if (!typeid_cast<const DataTypeDateTime *>(result_column.type.get())
+        && !typeid_cast<const DataTypeDate *>(result_column.type.get()))
+    {
+        throw Exception(
+            "TTL expression result column should have DateTime or Date type, but has " + result_column.type->getName(),
+            ErrorCodes::BAD_TTL_EXPRESSION);
+    }
+}
+
+}
+
+StorageMetadataTTLField StorageMetadataTTLField::getTTLFromAST(const ASTPtr & definition_ast, const ColumnsDescription & columns, const Context & context)
+{
+    StorageMetadataTTLField result;
+    const auto * ttl_element = definition_ast->as<ASTTTLElement>();
+
+    /// First child is expression, like `TTL expr TO DISK`
+    if (ttl_element != nullptr)
+        result.definition_ast = ttl_element->children.front()->clone();
+    else
+        result.definition_ast = definition_ast->clone();
+
+    auto ttl_ast = result.definition_ast->clone();
+    auto syntax_result = SyntaxAnalyzer(context).analyze(ttl_ast, columns.getAllPhysical());
+    result.expression = ExpressionAnalyzer(ttl_ast, syntax_result, context).getActions(false);
+
+    if (ttl_element != nullptr)
+    {
+        result.destination_type = ttl_element->destination_type;
+        result.destination_name = ttl_element->destination_name;
+    }
+
+    result.result_column = ttl_ast->getColumnName();
+
+    checkTTLExpression(result.expression, result.result_column);
+    return result;
+}
 }
diff --git a/src/Storages/StorageInMemoryMetadata.h b/src/Storages/StorageInMemoryMetadata.h
index 3faaed8d680..76720248fbd 100644
--- a/src/Storages/StorageInMemoryMetadata.h
+++ b/src/Storages/StorageInMemoryMetadata.h
@@ -84,6 +84,20 @@ struct StorageMetadataTTLField
     DataDestinationType destination_type;
 
     String destination_name;
+
+    static StorageMetadataTTLField getTTLFromAST(const ASTPtr & definition_ast, const ColumnsDescription & columns, const Context & context);
+};
+
+using StorageMetadataTTLColumnEntries = std::unordered_map<String, StorageMetadataTTLField>;
+using StorageMetadataTTLFields = std::vector<StorageMetadataTTLField>;
+
+struct StorageMetadataTableTTL
+{
+    ASTPtr definition_ast;
+
+    StorageMetadataTTLField rows_ttl;
+
+    StorageMetadataTTLFields move_ttl;
 };
 
 }

From 62e9f676391dd0088c4ad69f8b4a4f1c78cb48a0 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Mon, 25 May 2020 20:25:18 +0300
Subject: [PATCH 016/184] Better

---
 src/DataStreams/TTLBlockInputStream.cpp | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/DataStreams/TTLBlockInputStream.cpp b/src/DataStreams/TTLBlockInputStream.cpp
index 99b185d39b9..00020be36bf 100644
--- a/src/DataStreams/TTLBlockInputStream.cpp
+++ b/src/DataStreams/TTLBlockInputStream.cpp
@@ -156,8 +156,7 @@ void TTLBlockInputStream::removeValuesWithExpiredColumnTTL(Block & block)
     }
 
     std::vector<String> columns_to_remove;
-    const auto & column_ttl_entries_by_name = storage.getColumnTTLs();
-    for (const auto & [name, ttl_entry] : column_ttl_entries_by_name)
+    for (const auto & [name, ttl_entry] : storage.getColumnTTLs())
     {
         /// If we read not all table columns. E.g. while mutation.
         if (!block.has(name))
@@ -217,9 +216,8 @@ void TTLBlockInputStream::removeValuesWithExpiredColumnTTL(Block & block)
 
 void TTLBlockInputStream::updateMovesTTL(Block & block)
 {
-    const auto & move_ttl_entries = storage.getMoveTTLs();
     std::vector<String> columns_to_remove;
-    for (const auto & ttl_entry : move_ttl_entries)
+    for (const auto & ttl_entry : storage.getMoveTTLs())
     {
         auto & new_ttl_info = new_ttl_infos.moves_ttl[ttl_entry.result_column];
 

From adc7439a4ca0e96fd1f82de9be09554679eb50d2 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Mon, 25 May 2020 20:57:08 +0300
Subject: [PATCH 017/184] Comments and refactoring

---
 src/Storages/IStorage.cpp                     | 54 ++++++++++---------
 src/Storages/IStorage.h                       | 33 ++++++------
 src/Storages/MergeTree/MergeTreeData.cpp      |  2 +-
 .../MergeTree/StorageFromMergeTreeDataPart.h  |  2 -
 src/Storages/StorageInMemoryMetadata.cpp      | 11 ++--
 src/Storages/StorageInMemoryMetadata.h        | 22 +++++++-
 6 files changed, 74 insertions(+), 50 deletions(-)

diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp
index f8f2599184f..6a0b6b46d5a 100644
--- a/src/Storages/IStorage.cpp
+++ b/src/Storages/IStorage.cpp
@@ -541,6 +541,35 @@ Names IStorage::getColumnsRequiredForSampling() const
     return {};
 }
 
+const StorageMetadataTableTTL & IStorage::getTableTTLs() const
+{
+    return table_ttl;
+}
+
+void IStorage::setTableTTLs(const StorageMetadataTableTTL & table_ttl_)
+{
+    table_ttl = table_ttl_;
+}
+
+bool IStorage::hasAnyTableTTL() const
+{
+    return hasAnyMoveTTL() || hasRowsTTL();
+}
+
+const StorageMetadataTTLColumnFields & IStorage::getColumnTTLs() const
+{
+    return column_ttls_by_name;
+}
+
+void IStorage::setColumnTTLs(const StorageMetadataTTLColumnFields & column_ttls_by_name_)
+{
+    column_ttls_by_name = column_ttls_by_name_;
+}
+
+bool IStorage::hasAnyColumnTTL() const
+{
+    return !column_ttls_by_name.empty();
+}
 
 const StorageMetadataTTLField & IStorage::getRowsTTL() const
 {
@@ -562,29 +591,4 @@ bool IStorage::hasAnyMoveTTL() const
     return !table_ttl.move_ttl.empty();
 }
 
-const StorageMetadataTTLColumnEntries & IStorage::getColumnTTLs() const
-{
-    return column_ttls_by_name;
-}
-
-void IStorage::setColumnTTLs(const StorageMetadataTTLColumnEntries & column_ttls_by_name_)
-{
-    column_ttls_by_name = column_ttls_by_name_;
-}
-
-const StorageMetadataTableTTL & IStorage::getTableTTLs() const
-{
-    return table_ttl;
-}
-
-void IStorage::setTableTTLs(const StorageMetadataTableTTL & table_ttl_)
-{
-    table_ttl = table_ttl_;
-}
-
-bool IStorage::hasAnyTableTTL() const
-{
-    return hasAnyMoveTTL() || hasRowsTTL();
-}
-
 }
diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h
index 416d8ffbbb1..551117bd00a 100644
--- a/src/Storages/IStorage.h
+++ b/src/Storages/IStorage.h
@@ -130,7 +130,7 @@ public:
     virtual bool hasEvenlyDistributedRead() const { return false; }
 
     /// Returns true if there is set table TTL, any column TTL or any move TTL.
-    virtual bool hasAnyTTL() const { return hasRowsTTL() || hasAnyTableTTL(); }
+    virtual bool hasAnyTTL() const { return hasAnyColumnTTL() || hasAnyTableTTL(); }
 
     /// Optional size information of each physical column.
     /// Currently it's only used by the MergeTree family for query optimizations.
@@ -202,7 +202,7 @@ private:
     StorageMetadataKeyField sorting_key;
     StorageMetadataKeyField sampling_key;
 
-    StorageMetadataTTLColumnEntries column_ttls_by_name;
+    StorageMetadataTTLColumnFields column_ttls_by_name;
     StorageMetadataTableTTL table_ttl;
 
 private:
@@ -507,28 +507,31 @@ public:
     Names getColumnsRequiredForFinal() const { return getColumnsRequiredForSortingKey(); }
 
 
-    /// Returns columns, which will be needed to calculate dependencies
-    /// (skip indices, TTL expressions) if we update @updated_columns set of columns.
+    /// Returns columns, which will be needed to calculate dependencies (skip
+    /// indices, TTL expressions) if we update @updated_columns set of columns.
     virtual ColumnDependencies getColumnDependencies(const NameSet & /* updated_columns */) const { return {}; }
 
-    /// Returns storage policy if storage supports it
+    /// Returns storage policy if storage supports it.
     virtual StoragePolicyPtr getStoragePolicy() const { return {}; }
 
-    /// Returns true if there is set TTL for rows.
-    const StorageMetadataTTLField & getRowsTTL() const;
-    bool hasRowsTTL() const;
-
-    const StorageMetadataTTLFields & getMoveTTLs() const;
-    bool hasAnyMoveTTL() const;
-
+    /// Common tables TTLs (for rows and moves).
     const StorageMetadataTableTTL & getTableTTLs() const;
     void setTableTTLs(const StorageMetadataTableTTL & table_ttl_);
     bool hasAnyTableTTL() const;
 
-    const StorageMetadataTTLColumnEntries & getColumnTTLs() const;
-    void setColumnTTLs(const StorageMetadataTTLColumnEntries & column_ttls_by_name_);
-    bool hasAnyColumnTTL() const { return !column_ttls_by_name.empty(); }
+    /// Separate TTLs for columns.
+    const StorageMetadataTTLColumnFields & getColumnTTLs() const;
+    void setColumnTTLs(const StorageMetadataTTLColumnFields & column_ttls_by_name_);
+    bool hasAnyColumnTTL() const;
 
+    /// Just wrapper for table TTLs, return rows part of table TTLs.
+    const StorageMetadataTTLField & getRowsTTL() const;
+    bool hasRowsTTL() const;
+
+    /// Just wrapper for table TTLs, return moves (to disks or volumes) parts of
+    /// table TTL.
+    const StorageMetadataTTLFields & getMoveTTLs() const;
+    bool hasAnyMoveTTL() const;
 
     /// If it is possible to quickly determine exact number of rows in the table at this moment of time, then return it.
     /// Used for:
diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index fddd08716fd..737b90a56cf 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -588,7 +588,7 @@ void MergeTreeData::setTTLExpressions(const ColumnsDescription & new_columns,
 
     auto new_column_ttl_asts = new_columns.getColumnTTLs();
 
-    StorageMetadataTTLColumnEntries new_column_ttl_by_name = getColumnTTLs();
+    StorageMetadataTTLColumnFields new_column_ttl_by_name = getColumnTTLs();
 
     if (!new_column_ttl_asts.empty())
     {
diff --git a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h
index 77fdfd60b46..37acf0f0160 100644
--- a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h
+++ b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h
@@ -39,8 +39,6 @@ public:
         return part->storage.mayBenefitFromIndexForIn(left_in_operand, query_context);
     }
 
-    //bool hasAnyTTL() const override { return part->storage.hasAnyTTL(); }
-
     ColumnDependencies getColumnDependencies(const NameSet & updated_columns) const override
     {
         return part->storage.getColumnDependencies(updated_columns);
diff --git a/src/Storages/StorageInMemoryMetadata.cpp b/src/Storages/StorageInMemoryMetadata.cpp
index 61982dcf46f..7d9defef429 100644
--- a/src/Storages/StorageInMemoryMetadata.cpp
+++ b/src/Storages/StorageInMemoryMetadata.cpp
@@ -187,16 +187,17 @@ StorageMetadataTTLField StorageMetadataTTLField::getTTLFromAST(const ASTPtr & de
     StorageMetadataTTLField result;
     const auto * ttl_element = definition_ast->as<ASTTTLElement>();
 
-    /// First child is expression, like `TTL expr TO DISK`
+    /// First child is expression: `TTL expr TO DISK`
     if (ttl_element != nullptr)
-        result.definition_ast = ttl_element->children.front()->clone();
-    else
-        result.definition_ast = definition_ast->clone();
+        result.expression_ast = ttl_element->children.front()->clone();
+    else /// It's columns TTL without any additions, just copy it
+        result.expression_ast = definition_ast->clone();
 
-    auto ttl_ast = result.definition_ast->clone();
+    auto ttl_ast = result.expression_ast->clone();
     auto syntax_result = SyntaxAnalyzer(context).analyze(ttl_ast, columns.getAllPhysical());
     result.expression = ExpressionAnalyzer(ttl_ast, syntax_result, context).getActions(false);
 
+    /// Move TTL to disk or volume
     if (ttl_element != nullptr)
     {
         result.destination_type = ttl_element->destination_type;
diff --git a/src/Storages/StorageInMemoryMetadata.h b/src/Storages/StorageInMemoryMetadata.h
index 76720248fbd..40864d79aad 100644
--- a/src/Storages/StorageInMemoryMetadata.h
+++ b/src/Storages/StorageInMemoryMetadata.h
@@ -73,30 +73,48 @@ struct StorageMetadataKeyField
     static StorageMetadataKeyField getKeyFromAST(const ASTPtr & definition_ast, const ColumnsDescription & columns, const Context & context);
 };
 
+/// Common struct for TTL record in storage
 struct StorageMetadataTTLField
 {
-    ASTPtr definition_ast;
+    /// Expression part of TTL AST:
+    /// TTL d + INTERVAL 1 DAY
+    ///    ^~~~~expression~~~~^
+    ASTPtr expression_ast;
 
+    /// Expresion actions evaluated from AST
     ExpressionActionsPtr expression;
 
+    /// Result column of this TTL expression
     String result_column;
 
+    /// Destination type, only valid for table TTLs.
+    /// For example DISK or VOLUME
     DataDestinationType destination_type;
 
+    /// Name of destination disk or volume
     String destination_name;
 
+    /// Parse TTL structure from definition. Able to parse both column and table
+    /// TTLs.
     static StorageMetadataTTLField getTTLFromAST(const ASTPtr & definition_ast, const ColumnsDescription & columns, const Context & context);
 };
 
-using StorageMetadataTTLColumnEntries = std::unordered_map<String, StorageMetadataTTLField>;
+/// Mapping from column name to column TTL
+using StorageMetadataTTLColumnFields = std::unordered_map<String, StorageMetadataTTLField>;
 using StorageMetadataTTLFields = std::vector<StorageMetadataTTLField>;
 
+/// Common TTL for all table. Specified after defining the table columns.
 struct StorageMetadataTableTTL
 {
+    /// Definition. Include all parts of TTL:
+    /// TTL d + INTERVAL 1 day TO VOLUME 'disk1'
+    /// ^~~~~~~~~~~~~~~definition~~~~~~~~~~~~~~~^
     ASTPtr definition_ast;
 
+    /// Rows removing TTL
     StorageMetadataTTLField rows_ttl;
 
+    /// Moving data TTL (to other disks or volumes)
     StorageMetadataTTLFields move_ttl;
 };
 

From 211d7e229ff031ba4fa594afeb9d75e187c6054d Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Mon, 25 May 2020 21:03:04 +0300
Subject: [PATCH 018/184] Better using

---
 src/Storages/MergeTree/MergeTreeData.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 737b90a56cf..a58f176cffa 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -2930,7 +2930,7 @@ std::optional<StorageMetadataTTLField>
 MergeTreeData::selectTTLEntryForTTLInfos(const IMergeTreeDataPart::TTLInfos & ttl_infos, time_t time_of_move) const
 {
     time_t max_max_ttl = 0;
-    std::vector<StorageMetadataTTLField>::const_iterator best_entry_it;
+    StorageMetadataTTLFields::const_iterator best_entry_it;
 
     auto lock = std::lock_guard(move_ttl_entries_mutex);
     const auto & move_ttl_entries = getMoveTTLs();

From 548fc7141a47583ae33ab02e716cd338a0b7300e Mon Sep 17 00:00:00 2001
From: Ivan Blinkov <github@blinkov.ru>
Date: Mon, 25 May 2020 21:12:03 +0300
Subject: [PATCH 019/184] Enable log_queries by default

---
 src/Core/Settings.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index be7c09ec6c2..142e0872d72 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -156,7 +156,7 @@ struct Settings : public SettingsCollection<Settings>
     M(SettingUInt64, priority, 0, "Priority of the query. 1 - the highest, higher value - lower priority; 0 - do not use priorities.", 0) \
     M(SettingInt64, os_thread_priority, 0, "If non zero - set corresponding 'nice' value for query processing threads. Can be used to adjust query priority for OS scheduler.", 0) \
     \
-    M(SettingBool, log_queries, 0, "Log requests and write the log to the system table.", 0) \
+    M(SettingBool, log_queries, 1, "Log requests and write the log to the system table.", 0) \
     M(SettingLogQueriesType, log_queries_min_type, QueryLogElementType::QUERY_START, "query_log minimal type to log, possible values (from low to high): QUERY_START, QUERY_FINISH, EXCEPTION_BEFORE_START, EXCEPTION_WHILE_PROCESSING.", 0) \
     M(SettingUInt64, log_queries_cut_to_length, 100000, "If query length is greater than specified threshold (in bytes), then cut query when writing to query log. Also limit length of printed query in ordinary text log.", 0) \
     \

From 355fbadb316e2a652a6ac8d3b405831416a736ec Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Mon, 25 May 2020 22:18:01 +0300
Subject: [PATCH 020/184] Fix style

---
 src/Storages/MergeTree/MergeTreeData.cpp | 1 -
 src/Storages/StorageInMemoryMetadata.cpp | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index a58f176cffa..beeb163d5b5 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -581,7 +581,6 @@ void MergeTreeData::initPartitionKey(ASTPtr partition_by_ast)
 }
 
 
-
 void MergeTreeData::setTTLExpressions(const ColumnsDescription & new_columns,
         const ASTPtr & new_ttl_table_ast, bool only_check)
 {
diff --git a/src/Storages/StorageInMemoryMetadata.cpp b/src/Storages/StorageInMemoryMetadata.cpp
index 7d9defef429..cf319ce319b 100644
--- a/src/Storages/StorageInMemoryMetadata.cpp
+++ b/src/Storages/StorageInMemoryMetadata.cpp
@@ -19,6 +19,7 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int BAD_TTL_EXPRESSION;
+    extern const int BAD_ARGUMENTS;
 };
 
 StorageInMemoryMetadata::StorageInMemoryMetadata(

From 7142e68d7abab49f0d734e907f07ac139c6702ec Mon Sep 17 00:00:00 2001
From: Alexander Kuzmenkov <akuzm@yandex-team.ru>
Date: Tue, 26 May 2020 08:54:04 +0300
Subject: [PATCH 021/184] fixup

---
 .../AggregateFunctionEntropy.h                |  6 ++---
 .../AggregateFunctionGroupUniqArray.h         |  9 +++----
 .../QuantileExactWeighted.h                   |  3 +--
 src/Common/HashTable/ClearableHashMap.h       | 11 ++++++++
 src/Common/HashTable/ClearableHashSet.h       | 12 +++++++++
 src/Common/HashTable/HashMap.h                | 11 ++++++++
 src/Common/HashTable/HashSet.h                | 17 +++++++++++++
 src/Common/HashTable/HashTable.h              | 21 +++-------------
 src/Common/SpaceSaving.h                      |  3 +--
 src/Functions/array/arrayDistinct.cpp         | 12 ++++-----
 src/Functions/array/arrayEnumerateExtended.h  | 25 ++++++++-----------
 src/Functions/array/arrayEnumerateRanked.h    |  5 ++--
 src/Functions/array/arrayIntersect.cpp        | 23 ++++++++---------
 src/Functions/array/arrayUniq.cpp             | 20 +++++++--------
 src/Interpreters/tests/hash_map3.cpp          |  6 +++--
 .../functions_with_hash_tables.xml            |  8 +++---
 16 files changed, 108 insertions(+), 84 deletions(-)

diff --git a/src/AggregateFunctions/AggregateFunctionEntropy.h b/src/AggregateFunctions/AggregateFunctionEntropy.h
index 59cacd0e9ca..ff233a5ac93 100644
--- a/src/AggregateFunctions/AggregateFunctionEntropy.h
+++ b/src/AggregateFunctions/AggregateFunctionEntropy.h
@@ -23,12 +23,10 @@ struct EntropyData
 {
     using Weight = UInt64;
 
-    using HashingMap = HASH_TABLE_WITH_STACK_MEMORY(HashMap, Value, Weight,
-        HashCRC32<Value>, HashTableGrower<4>);
+    using HashingMap = HashMapWithStackMemory<Value, Weight, HashCRC32<Value>, 4>;
 
     /// For the case of pre-hashed values.
-    using TrivialMap = HASH_TABLE_WITH_STACK_MEMORY(HashMap, Value, Weight,
-        UInt128TrivialHash, HashTableGrower<4>);
+    using TrivialMap = HashMapWithStackMemory<Value, Weight, UInt128TrivialHash, 4>;
 
     using Map = std::conditional_t<std::is_same_v<UInt128, Value>, TrivialMap, HashingMap>;
 
diff --git a/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h b/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h
index af97df209f6..88b1c87f526 100644
--- a/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h
+++ b/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h
@@ -28,8 +28,7 @@ template <typename T>
 struct AggregateFunctionGroupUniqArrayData
 {
     /// When creating, the hash table must be small.
-    using Set = HASH_TABLE_WITH_STACK_MEMORY(HashSet, T, DefaultHash<T>,
-        HashTableGrower<4>);
+    using Set = HashSetWithStackMemory<T, DefaultHash<T>, 4>;
 
     Set value;
 };
@@ -122,10 +121,10 @@ public:
 /// Generic implementation, it uses serialized representation as object descriptor.
 struct AggregateFunctionGroupUniqArrayGenericData
 {
-    static constexpr size_t INITIAL_SIZE_DEGREE = 1; /// adjustable
+    static constexpr size_t INITIAL_SIZE_DEGREE = 3; /// adjustable
 
-    using Set = HASH_TABLE_WITH_STACK_MEMORY(HashSetWithSavedHash,
-        StringRef, StringRefHash, HashTableGrower<INITIAL_SIZE_DEGREE>);
+    using Set = HashSetWithSavedHashWithStackMemory<StringRef, StringRefHash,
+        INITIAL_SIZE_DEGREE>;
 
     Set value;
 };
diff --git a/src/AggregateFunctions/QuantileExactWeighted.h b/src/AggregateFunctions/QuantileExactWeighted.h
index 7731ff980fe..666dbd6b622 100644
--- a/src/AggregateFunctions/QuantileExactWeighted.h
+++ b/src/AggregateFunctions/QuantileExactWeighted.h
@@ -33,8 +33,7 @@ struct QuantileExactWeighted
     using Hasher = std::conditional_t<std::is_same_v<Value, Decimal128>, Int128Hash, HashCRC32<UnderlyingType>>;
 
     /// When creating, the hash table must be small.
-    using Map = HASH_TABLE_WITH_STACK_MEMORY(HashMap, UnderlyingType, Weight,
-        Hasher, HashTableGrower<4>);
+    using Map = HashMapWithStackMemory<UnderlyingType, Weight, Hasher, 4>;
 
     Map map;
 
diff --git a/src/Common/HashTable/ClearableHashMap.h b/src/Common/HashTable/ClearableHashMap.h
index 4370f6b6dc7..fda01dcf4bc 100644
--- a/src/Common/HashTable/ClearableHashMap.h
+++ b/src/Common/HashTable/ClearableHashMap.h
@@ -43,3 +43,14 @@ public:
         this->m_size = 0;
     }
 };
+
+template <typename Key, typename Mapped, typename Hash,
+    size_t initial_size_degree>
+using ClearableHashMapWithStackMemory = ClearableHashMap<
+    Key,
+    Mapped,
+    Hash,
+    HashTableGrower<initial_size_degree>,
+    HashTableAllocatorWithStackMemory<
+        (1ULL << initial_size_degree)
+        * sizeof(ClearableHashMapCell<Key, Mapped, Hash>)>>;
diff --git a/src/Common/HashTable/ClearableHashSet.h b/src/Common/HashTable/ClearableHashSet.h
index 824ec9d8e5f..dc057afacd8 100644
--- a/src/Common/HashTable/ClearableHashSet.h
+++ b/src/Common/HashTable/ClearableHashSet.h
@@ -84,3 +84,15 @@ public:
         this->m_size = 0;
     }
 };
+
+template <typename Key, typename Hash, size_t initial_size_degree>
+using ClearableHashSetWithStackMemory = ClearableHashSet<
+    Key,
+    Hash,
+    HashTableGrower<initial_size_degree>,
+    HashTableAllocatorWithStackMemory<
+        (1ULL << initial_size_degree)
+        * sizeof(
+            ClearableHashTableCell<
+                Key,
+                HashTableCell<Key, Hash, ClearableHashSetState>>)>>;
diff --git a/src/Common/HashTable/HashMap.h b/src/Common/HashTable/HashMap.h
index cdc4a003af8..3ecbd9f263c 100644
--- a/src/Common/HashTable/HashMap.h
+++ b/src/Common/HashTable/HashMap.h
@@ -239,3 +239,14 @@ template <
     typename Grower = HashTableGrower<>,
     typename Allocator = HashTableAllocator>
 using HashMapWithSavedHash = HashMapTable<Key, HashMapCellWithSavedHash<Key, Mapped, Hash>, Hash, Grower, Allocator>;
+
+template <typename Key, typename Mapped, typename Hash,
+    size_t initial_size_degree>
+using HashMapWithStackMemory = HashMapTable<
+    Key,
+    HashMapCellWithSavedHash<Key, Mapped, Hash>,
+    Hash,
+    HashTableGrower<initial_size_degree>,
+    HashTableAllocatorWithStackMemory<
+        (1ULL << initial_size_degree)
+        * sizeof(HashMapCellWithSavedHash<Key, Mapped, Hash>)>>;
diff --git a/src/Common/HashTable/HashSet.h b/src/Common/HashTable/HashSet.h
index c1970e898a1..c79e05073fc 100644
--- a/src/Common/HashTable/HashSet.h
+++ b/src/Common/HashTable/HashSet.h
@@ -93,6 +93,14 @@ template
 >
 using HashSet = HashSetTable<Key, HashTableCell<Key, Hash>, Hash, Grower, Allocator>;
 
+template <typename Key, typename Hash, size_t initial_size_degree>
+using HashSetWithStackMemory = HashSet<
+    Key,
+    Hash,
+    HashTableGrower<initial_size_degree>,
+    HashTableAllocatorWithStackMemory<
+        (1ULL << initial_size_degree)
+        * sizeof(HashTableCell<Key, Hash>)>>;
 
 template
 <
@@ -102,3 +110,12 @@ template
     typename Allocator = HashTableAllocator
 >
 using HashSetWithSavedHash = HashSetTable<Key, HashSetCellWithSavedHash<Key, Hash>, Hash, Grower, Allocator>;
+
+template <typename Key, typename Hash, size_t initial_size_degree>
+using HashSetWithSavedHashWithStackMemory = HashSetWithSavedHash<
+    Key,
+    Hash,
+    HashTableGrower<initial_size_degree>,
+    HashTableAllocatorWithStackMemory<
+        (1ULL << initial_size_degree)
+        * sizeof(HashSetCellWithSavedHash<Key, Hash>)>>;
diff --git a/src/Common/HashTable/HashTable.h b/src/Common/HashTable/HashTable.h
index 2f7272ca8ec..9d66c7b7a89 100644
--- a/src/Common/HashTable/HashTable.h
+++ b/src/Common/HashTable/HashTable.h
@@ -208,6 +208,7 @@ struct HashTableGrower
     /// The state of this structure is enough to get the buffer size of the hash table.
 
     UInt8 size_degree = initial_size_degree;
+    static constexpr auto initial_count = 1ULL << initial_size_degree;
 
     /// The size of the hash table in the cells.
     size_t bufSize() const               { return 1ULL << size_degree; }
@@ -255,6 +256,7 @@ struct HashTableGrower
 template <size_t key_bits>
 struct HashTableFixedGrower
 {
+    static constexpr auto initial_count = 1ULL << key_bits;
     size_t bufSize() const               { return 1ULL << key_bits; }
     size_t place(size_t x) const         { return x; }
     /// You could write __builtin_unreachable(), but the compiler does not optimize everything, and it turns out less efficiently.
@@ -308,14 +310,6 @@ struct ZeroValueStorage<false, Cell>
     const Cell * zeroValue() const { return nullptr; }
 };
 
-// These templates give the initial hash table size, so that we can check
-// that it is in sync with initial allocator size.
-template <typename GenericGrower>
-constexpr size_t growerInitialCount = 0;
-
-template <size_t initial_size_degree>
-constexpr size_t growerInitialCount<HashTableGrower<initial_size_degree>>
-    = 1ULL << initial_size_degree;
 
 // The HashTable
 template
@@ -337,7 +331,7 @@ public:
     // Export the initial buffer sizes for the ease of using allocators with
     // inline memory.
     static constexpr size_t initial_buffer_bytes
-        = growerInitialCount<Grower> * sizeof(Cell);
+        = Grower::initial_count * sizeof(Cell);
 
     // If we use an allocator with inline memory, check that the initial
     // size of the hash table is in sync with the amount of this memory.
@@ -1096,12 +1090,3 @@ public:
 #endif
 };
 
-// A helper macro that declares hash table with allocator with stack memory,
-// and the initial size of the allocator is in sync with initial size of the
-// hash table.
-#define HASH_TABLE_WITH_STACK_MEMORY(HASH_TABLE_VARIANT, ...) \
-    HASH_TABLE_VARIANT<__VA_ARGS__, \
-        HashTableAllocatorWithStackMemory< \
-            HASH_TABLE_VARIANT<__VA_ARGS__>::initial_buffer_bytes \
-        > \
-    >
diff --git a/src/Common/SpaceSaving.h b/src/Common/SpaceSaving.h
index 47b49add060..56063340240 100644
--- a/src/Common/SpaceSaving.h
+++ b/src/Common/SpaceSaving.h
@@ -378,8 +378,7 @@ private:
             counter_map[counter->key] = counter;
     }
 
-    using CounterMap = HASH_TABLE_WITH_STACK_MEMORY(HashMap,
-        TKey, Counter *, Hash, HashTableGrower<4>);
+    using CounterMap = HashMapWithStackMemory<TKey, Counter *, Hash, 4>;
 
     CounterMap counter_map;
     std::vector<Counter *> counter_list;
diff --git a/src/Functions/array/arrayDistinct.cpp b/src/Functions/array/arrayDistinct.cpp
index d3da77e43d2..bb26567e8a5 100644
--- a/src/Functions/array/arrayDistinct.cpp
+++ b/src/Functions/array/arrayDistinct.cpp
@@ -153,8 +153,8 @@ bool FunctionArrayDistinct::executeNumber(
     if (nullable_col)
         src_null_map = &nullable_col->getNullMapData();
 
-    using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashSet,
-        T, DefaultHash<T>, HashTableGrower<INITIAL_SIZE_DEGREE>);
+    using Set = ClearableHashSetWithStackMemory<T, DefaultHash<T>,
+        INITIAL_SIZE_DEGREE>;
 
     Set set;
 
@@ -199,8 +199,8 @@ bool FunctionArrayDistinct::executeString(
 
     ColumnString & res_data_column_string = typeid_cast<ColumnString &>(res_data_col);
 
-    using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashSet,
-        StringRef, StringRefHash, HashTableGrower<INITIAL_SIZE_DEGREE>);
+    using Set = ClearableHashSetWithStackMemory<StringRef, StringRefHash,
+        INITIAL_SIZE_DEGREE>;
 
     const PaddedPODArray<UInt8> * src_null_map = nullptr;
 
@@ -245,8 +245,8 @@ void FunctionArrayDistinct::executeHashed(
     ColumnArray::Offsets & res_offsets,
     const ColumnNullable * nullable_col)
 {
-    using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashSet,
-        UInt128, UInt128TrivialHash, HashTableGrower<INITIAL_SIZE_DEGREE>);
+    using Set = ClearableHashSetWithStackMemory<UInt128, UInt128TrivialHash,
+        INITIAL_SIZE_DEGREE>;
 
     const PaddedPODArray<UInt8> * src_null_map = nullptr;
 
diff --git a/src/Functions/array/arrayEnumerateExtended.h b/src/Functions/array/arrayEnumerateExtended.h
index e97766c5827..e2f83ae8826 100644
--- a/src/Functions/array/arrayEnumerateExtended.h
+++ b/src/Functions/array/arrayEnumerateExtended.h
@@ -64,45 +64,40 @@ private:
     template <typename T>
     struct MethodOneNumber
     {
-        using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap,
-            T, UInt32, DefaultHash<T>,
-            HashTableGrower<INITIAL_SIZE_DEGREE>);
+        using Set = ClearableHashMapWithStackMemory<T, UInt32, DefaultHash<T>,
+            INITIAL_SIZE_DEGREE>;
 
         using Method = ColumnsHashing::HashMethodOneNumber<typename Set::value_type, UInt32, T, false>;
     };
 
     struct MethodString
     {
-        using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap,
-            StringRef, UInt32, StringRefHash,
-            HashTableGrower<INITIAL_SIZE_DEGREE>);
+        using Set = ClearableHashMapWithStackMemory<StringRef, UInt32, StringRefHash,
+            INITIAL_SIZE_DEGREE>;
 
         using Method = ColumnsHashing::HashMethodString<typename Set::value_type, UInt32, false, false>;
     };
 
     struct MethodFixedString
     {
-        using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap,
-            StringRef, UInt32, StringRefHash,
-            HashTableGrower<INITIAL_SIZE_DEGREE>);
+        using Set = ClearableHashMapWithStackMemory<StringRef, UInt32, StringRefHash,
+            INITIAL_SIZE_DEGREE>;
 
         using Method = ColumnsHashing::HashMethodFixedString<typename Set::value_type, UInt32, false, false>;
     };
 
     struct MethodFixed
     {
-        using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap,
-            UInt128, UInt32, UInt128HashCRC32,
-            HashTableGrower<INITIAL_SIZE_DEGREE>);
+        using Set = ClearableHashMapWithStackMemory<UInt128, UInt32, UInt128HashCRC32,
+            INITIAL_SIZE_DEGREE>;
 
         using Method = ColumnsHashing::HashMethodKeysFixed<typename Set::value_type, UInt128, UInt32, false, false, false>;
     };
 
     struct MethodHashed
     {
-        using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap,
-            UInt128, UInt32, UInt128TrivialHash,
-            HashTableGrower<INITIAL_SIZE_DEGREE>);
+        using Set = ClearableHashMapWithStackMemory<UInt128, UInt32, UInt128TrivialHash,
+            INITIAL_SIZE_DEGREE>;
 
         using Method = ColumnsHashing::HashMethodHashed<typename Set::value_type, UInt32, false>;
     };
diff --git a/src/Functions/array/arrayEnumerateRanked.h b/src/Functions/array/arrayEnumerateRanked.h
index 1700260dd9f..c021b5fbfd1 100644
--- a/src/Functions/array/arrayEnumerateRanked.h
+++ b/src/Functions/array/arrayEnumerateRanked.h
@@ -308,9 +308,8 @@ void FunctionArrayEnumerateRankedExtended<Derived>::executeMethodImpl(
     const size_t depth_to_look = arrays_depths.max_array_depth;
     const auto & offsets = *offsets_by_depth[depth_to_look - 1];
 
-    using Map = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap,
-        UInt128, UInt32, UInt128TrivialHash,
-        HashTableGrower<INITIAL_SIZE_DEGREE>);
+    using Map = ClearableHashMapWithStackMemory<UInt128, UInt32,
+        UInt128TrivialHash, INITIAL_SIZE_DEGREE>;
 
     Map indices;
 
diff --git a/src/Functions/array/arrayIntersect.cpp b/src/Functions/array/arrayIntersect.cpp
index acd162974c9..3525914b0d3 100644
--- a/src/Functions/array/arrayIntersect.cpp
+++ b/src/Functions/array/arrayIntersect.cpp
@@ -418,18 +418,15 @@ void FunctionArrayIntersect::executeImpl(Block & block, const ColumnNumbers & ar
     TypeListNativeNumbers::forEach(NumberExecutor(arrays, not_nullable_nested_return_type, result_column));
     TypeListDecimalNumbers::forEach(DecimalExecutor(arrays, not_nullable_nested_return_type, result_column));
 
-    using DateMap = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap,
-        DataTypeDate::FieldType, size_t, DefaultHash<DataTypeDate::FieldType>,
-        HashTableGrower<INITIAL_SIZE_DEGREE>);
+    using DateMap = ClearableHashMapWithStackMemory<DataTypeDate::FieldType,
+        size_t, DefaultHash<DataTypeDate::FieldType>, INITIAL_SIZE_DEGREE>;
 
-    using DateTimeMap = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap,
+    using DateTimeMap = ClearableHashMapWithStackMemory<
         DataTypeDateTime::FieldType, size_t,
-        DefaultHash<DataTypeDateTime::FieldType>,
-        HashTableGrower<INITIAL_SIZE_DEGREE>);
+        DefaultHash<DataTypeDateTime::FieldType>, INITIAL_SIZE_DEGREE>;
 
-    using StringMap = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap,
-        StringRef, size_t, StringRefHash,
-        HashTableGrower<INITIAL_SIZE_DEGREE>);
+    using StringMap = ClearableHashMapWithStackMemory<StringRef, size_t,
+        StringRefHash, INITIAL_SIZE_DEGREE>;
 
     if (!result_column)
     {
@@ -457,8 +454,8 @@ void FunctionArrayIntersect::executeImpl(Block & block, const ColumnNumbers & ar
 template <typename T, size_t>
 void FunctionArrayIntersect::NumberExecutor::operator()()
 {
-    using Map = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap,
-        T, size_t, DefaultHash<T>, HashTableGrower<INITIAL_SIZE_DEGREE>);
+    using Map = ClearableHashMapWithStackMemory<T, size_t, DefaultHash<T>,
+        INITIAL_SIZE_DEGREE>;
 
     if (!result && typeid_cast<const DataTypeNumber<T> *>(data_type.get()))
         result = execute<Map, ColumnVector<T>, true>(arrays, ColumnVector<T>::create());
@@ -467,8 +464,8 @@ void FunctionArrayIntersect::NumberExecutor::operator()()
 template <typename T, size_t>
 void FunctionArrayIntersect::DecimalExecutor::operator()()
 {
-    using Map = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashMap,
-        T, size_t, DefaultHash<T>, HashTableGrower<INITIAL_SIZE_DEGREE>);
+    using Map = ClearableHashMapWithStackMemory<T, size_t, DefaultHash<T>,
+        INITIAL_SIZE_DEGREE);
 
     if (!result)
         if (auto * decimal = typeid_cast<const DataTypeDecimal<T> *>(data_type.get()))
diff --git a/src/Functions/array/arrayUniq.cpp b/src/Functions/array/arrayUniq.cpp
index a83c535d88d..d9a44f6d8d3 100644
--- a/src/Functions/array/arrayUniq.cpp
+++ b/src/Functions/array/arrayUniq.cpp
@@ -66,40 +66,40 @@ private:
     template <typename T>
     struct MethodOneNumber
     {
-        using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashSet,
-            T, DefaultHash<T>, HashTableGrower<INITIAL_SIZE_DEGREE>);
+        using Set = ClearableHashSetWithStackMemory<T, DefaultHash<T>,
+            INITIAL_SIZE_DEGREE>;
 
         using Method = ColumnsHashing::HashMethodOneNumber<typename Set::value_type, void, T, false>;
     };
 
     struct MethodString
     {
-        using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashSet,
-            StringRef, StringRefHash, HashTableGrower<INITIAL_SIZE_DEGREE>);
+        using Set = ClearableHashSetWithStackMemory<StringRef, StringRefHash,
+            INITIAL_SIZE_DEGREE>;
 
         using Method = ColumnsHashing::HashMethodString<typename Set::value_type, void, false, false>;
     };
 
     struct MethodFixedString
     {
-        using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashSet,
-            StringRef, StringRefHash, HashTableGrower<INITIAL_SIZE_DEGREE>);
+        using Set = ClearableHashSetWithStackMemory<StringRef, StringRefHash,
+            INITIAL_SIZE_DEGREE>;
 
         using Method = ColumnsHashing::HashMethodFixedString<typename Set::value_type, void, false, false>;
     };
 
     struct MethodFixed
     {
-        using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashSet,
-            UInt128, UInt128HashCRC32, HashTableGrower<INITIAL_SIZE_DEGREE>);
+        using Set = ClearableHashSetWithStackMemory<UInt128, UInt128HashCRC32,
+            INITIAL_SIZE_DEGREE>;
 
         using Method = ColumnsHashing::HashMethodKeysFixed<typename Set::value_type, UInt128, void, false, false, false>;
     };
 
     struct MethodHashed
     {
-        using Set = HASH_TABLE_WITH_STACK_MEMORY(ClearableHashSet,
-            UInt128, UInt128TrivialHash, HashTableGrower<INITIAL_SIZE_DEGREE>);
+        using Set = ClearableHashSetWithStackMemory<UInt128, UInt128TrivialHash,
+            INITIAL_SIZE_DEGREE>;
 
         using Method = ColumnsHashing::HashMethodHashed<typename Set::value_type, void, false>;
     };
diff --git a/src/Interpreters/tests/hash_map3.cpp b/src/Interpreters/tests/hash_map3.cpp
index 804b084b2a1..1b4ce8eac77 100644
--- a/src/Interpreters/tests/hash_map3.cpp
+++ b/src/Interpreters/tests/hash_map3.cpp
@@ -61,11 +61,13 @@ struct Grower : public HashTableGrower<2>
 
 int main(int, char **)
 {
-    using Map = HASH_TABLE_WITH_STACK_MEMORY(HashMapWithDump,
+    using Map = HashMapWithDump<
         StringRef,
         UInt64,
         SimpleHash,
-        Grower);
+        Grower,
+        HashTableAllocatorWithStackMemory<
+            4 * sizeof(HashMapCell<StringRef, UInt64, SimpleHash>)>>;
 
     Map map;
 
diff --git a/tests/performance/functions_with_hash_tables.xml b/tests/performance/functions_with_hash_tables.xml
index 3ff2633c4e7..b247f0e4f6f 100644
--- a/tests/performance/functions_with_hash_tables.xml
+++ b/tests/performance/functions_with_hash_tables.xml
@@ -1,8 +1,8 @@
 <test>
-    <query>select arrayUniq(range(1 + (number % 10) * 10)) from numbers(1000000) format Null</query>
-    <query>select arrayDistinct(range(1 + (number % 10) * 10)) from numbers(1000000) format Null</query>
-    <query>select arrayEnumerateUniq(range(1 + (number % 10) * 10)) from numbers(1000000) format Null</query>
+    <query>select arrayUniq(range(1 + (number % 100) * 10)) from numbers(1000000) format Null</query>
+    <query>select arrayDistinct(range(1 + (number % 100) * 10)) from numbers(1000000) format Null</query>
+    <query>select arrayEnumerateUniq(range(1 + (number % 100) * 10)) from numbers(1000000) format Null</query>
     <query>select arrayIntersect(range((1 + number % 10)), range(1, (1 + number % 10) + 1)) from numbers(1000000) format Null</query>
-    <query>select groupUniqArray(number / 10) from numbers(1000 * 1000) group by number / 1000 format Null</query>
+    <query>select groupUniqArray(rand() % 100) from numbers(1000 * 1000) group by number / 1000 format Null</query>
     <query>select entropy(number / 10) from numbers(1000 * 1000) group by number / 1000 format Null</query>
 </test>

From bf941755f2e776304d0c318d8d0aa6dc6dc068b3 Mon Sep 17 00:00:00 2001
From: Alexander Kuzmenkov <akuzm@yandex-team.ru>
Date: Tue, 26 May 2020 09:00:51 +0300
Subject: [PATCH 022/184] fixup

---
 src/Common/HashTable/HashTable.h       | 8 ++------
 src/Functions/array/arrayIntersect.cpp | 2 +-
 2 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/src/Common/HashTable/HashTable.h b/src/Common/HashTable/HashTable.h
index 9d66c7b7a89..528e719c05b 100644
--- a/src/Common/HashTable/HashTable.h
+++ b/src/Common/HashTable/HashTable.h
@@ -328,13 +328,10 @@ class HashTable :
     protected ZeroValueStorage<Cell::need_zero_value_storage, Cell>     /// empty base optimization
 {
 public:
-    // Export the initial buffer sizes for the ease of using allocators with
-    // inline memory.
-    static constexpr size_t initial_buffer_bytes
-        = Grower::initial_count * sizeof(Cell);
-
     // If we use an allocator with inline memory, check that the initial
     // size of the hash table is in sync with the amount of this memory.
+    static constexpr size_t initial_buffer_bytes
+        = Grower::initial_count * sizeof(Cell);
     static_assert(allocatorInitialBytes<Allocator> == 0
         || allocatorInitialBytes<Allocator> == initial_buffer_bytes);
 
@@ -1089,4 +1086,3 @@ public:
     }
 #endif
 };
-
diff --git a/src/Functions/array/arrayIntersect.cpp b/src/Functions/array/arrayIntersect.cpp
index 3525914b0d3..24db3c0cd08 100644
--- a/src/Functions/array/arrayIntersect.cpp
+++ b/src/Functions/array/arrayIntersect.cpp
@@ -465,7 +465,7 @@ template <typename T, size_t>
 void FunctionArrayIntersect::DecimalExecutor::operator()()
 {
     using Map = ClearableHashMapWithStackMemory<T, size_t, DefaultHash<T>,
-        INITIAL_SIZE_DEGREE);
+        INITIAL_SIZE_DEGREE>;
 
     if (!result)
         if (auto * decimal = typeid_cast<const DataTypeDecimal<T> *>(data_type.get()))

From 27df8d5aca9ee12a82a6f83337fd65bfd256db4d Mon Sep 17 00:00:00 2001
From: Alexander Kuzmenkov <akuzm@yandex-team.ru>
Date: Tue, 26 May 2020 09:06:46 +0300
Subject: [PATCH 023/184] fixup

---
 tests/performance/functions_with_hash_tables.xml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/performance/functions_with_hash_tables.xml b/tests/performance/functions_with_hash_tables.xml
index b247f0e4f6f..72570a5925c 100644
--- a/tests/performance/functions_with_hash_tables.xml
+++ b/tests/performance/functions_with_hash_tables.xml
@@ -1,8 +1,8 @@
 <test>
-    <query>select arrayUniq(range(1 + (number % 100) * 10)) from numbers(1000000) format Null</query>
-    <query>select arrayDistinct(range(1 + (number % 100) * 10)) from numbers(1000000) format Null</query>
-    <query>select arrayEnumerateUniq(range(1 + (number % 100) * 10)) from numbers(1000000) format Null</query>
-    <query>select arrayIntersect(range((1 + number % 10)), range(1, (1 + number % 10) + 1)) from numbers(1000000) format Null</query>
+    <query>select arrayUniq(range(1 + (number % 100) * 10)) from numbers(100000) format Null</query>
+    <query>select arrayDistinct(range(1 + (number % 100) * 10)) from numbers(10000) format Null</query>
+    <query>select arrayEnumerateUniq(range(1 + (number % 100) * 10)) from numbers(100000) format Null</query>
+    <query>select arrayIntersect(range((1 + number % 100)), range(1, (1 + number % 100) + 1)) from numbers(100000) format Null</query>
     <query>select groupUniqArray(rand() % 100) from numbers(1000 * 1000) group by number / 1000 format Null</query>
     <query>select entropy(number / 10) from numbers(1000 * 1000) group by number / 1000 format Null</query>
 </test>

From 7bbb064b34a9bab7a21d736ecbc00eea633cbf93 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Tue, 26 May 2020 11:58:07 +0300
Subject: [PATCH 024/184] Fix metadata update

---
 src/Storages/MergeTree/MergeTreeData.cpp | 4 ++--
 src/Storages/MergeTree/MergeTreeData.h   | 1 -
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index beeb163d5b5..cb7d8257f05 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -259,8 +259,8 @@ StorageInMemoryMetadata MergeTreeData::getInMemoryMetadata() const
     if (isPrimaryKeyDefined())
         metadata.primary_key_ast = getPrimaryKeyAST()->clone();
 
-    if (ttl_table_ast)
-        metadata.ttl_for_table_ast = ttl_table_ast->clone();
+    if (hasAnyTableTTL())
+        metadata.ttl_for_table_ast = getTableTTLs().definition_ast->clone();
 
     if (hasSamplingKey())
         metadata.sample_by_ast = getSamplingKeyAST()->clone();
diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h
index a94680658ff..c4ab18b6099 100644
--- a/src/Storages/MergeTree/MergeTreeData.h
+++ b/src/Storages/MergeTree/MergeTreeData.h
@@ -682,7 +682,6 @@ protected:
     friend struct ReplicatedMergeTreeTableMetadata;
     friend class StorageReplicatedMergeTree;
 
-    ASTPtr ttl_table_ast;
     ASTPtr settings_ast;
 
     bool require_part_metadata;

From 5780240e884c9c3915721f159b3c5fd33b40111d Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Tue, 26 May 2020 12:46:04 +0300
Subject: [PATCH 025/184] Fix primry key check

---
 src/Storages/MergeTree/IMergeTreeDataPart.cpp               | 4 ++--
 .../integration/test_version_update_after_mutation/test.py  | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp
index 6e048a956da..78548cdddb2 100644
--- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp
+++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp
@@ -841,7 +841,7 @@ void IMergeTreeDataPart::checkConsistencyBase() const
 
     if (!checksums.empty())
     {
-        if (storage.hasPrimaryKey() && !checksums.files.count("primary.idx"))
+        if (storage.isPrimaryKeyDefined() && !checksums.files.count("primary.idx"))
             throw Exception("No checksum for primary.idx", ErrorCodes::NO_FILE_IN_DATA_PART);
 
         if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING)
@@ -875,7 +875,7 @@ void IMergeTreeDataPart::checkConsistencyBase() const
         };
 
         /// Check that the primary key index is not empty.
-        if (storage.hasPrimaryKey())
+        if (storage.isPrimaryKeyDefined())
             check_file_not_empty(volume->getDisk(), path + "primary.idx");
 
         if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING)
diff --git a/tests/integration/test_version_update_after_mutation/test.py b/tests/integration/test_version_update_after_mutation/test.py
index 3f2063bdddb..9e3471175fa 100644
--- a/tests/integration/test_version_update_after_mutation/test.py
+++ b/tests/integration/test_version_update_after_mutation/test.py
@@ -6,9 +6,9 @@ from helpers.test_tools import assert_eq_with_retry
 
 cluster = ClickHouseCluster(__file__)
 
-node1 = cluster.add_instance('node1', with_zookeeper=True, image='yandex/clickhouse-server:20.1.6.3', with_installed_binary=True, stay_alive=True)
-node2 = cluster.add_instance('node2', with_zookeeper=True, image='yandex/clickhouse-server:20.1.6.3', with_installed_binary=True, stay_alive=True)
-node3 = cluster.add_instance('node3', with_zookeeper=True, image='yandex/clickhouse-server:20.1.6.3', with_installed_binary=True, stay_alive=True)
+node1 = cluster.add_instance('node1', with_zookeeper=True, image='yandex/clickhouse-server:20.1.10.70', with_installed_binary=True, stay_alive=True)
+node2 = cluster.add_instance('node2', with_zookeeper=True, image='yandex/clickhouse-server:20.1.10.70', with_installed_binary=True, stay_alive=True)
+node3 = cluster.add_instance('node3', with_zookeeper=True, image='yandex/clickhouse-server:20.1.10.70', with_installed_binary=True, stay_alive=True)
 
 @pytest.fixture(scope="module")
 def start_cluster():

From 54f50745ec40c3841862640aa6711cfab0df85fb Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Tue, 26 May 2020 13:19:56 +0300
Subject: [PATCH 026/184] Fix StrictResizeProcessor.

---
 src/Processors/ResizeProcessor.cpp | 26 ++++++++++++++++++++++----
 src/Processors/ResizeProcessor.h   |  3 +++
 2 files changed, 25 insertions(+), 4 deletions(-)

diff --git a/src/Processors/ResizeProcessor.cpp b/src/Processors/ResizeProcessor.cpp
index 3fcf4281105..d652a342150 100644
--- a/src/Processors/ResizeProcessor.cpp
+++ b/src/Processors/ResizeProcessor.cpp
@@ -347,11 +347,16 @@ IProcessor::Status StrictResizeProcessor::prepare(const PortNumbers & updated_in
 
         auto & waiting_output = output_ports[input_with_data.waiting_output];
 
-        if (waiting_output.status != OutputStatus::NeedData)
-            throw Exception("Invalid status for associated output.", ErrorCodes::LOGICAL_ERROR);
+        if (waiting_output.status == OutputStatus::NotActive)
+            throw Exception("Invalid status NotActive for associated output.", ErrorCodes::LOGICAL_ERROR);
 
-        waiting_output.port->pushData(input_with_data.port->pullData(/* set_not_needed = */ true));
-        waiting_output.status = OutputStatus::NotActive;
+        if (waiting_output.status != OutputStatus::Finished)
+        {
+            waiting_output.port->pushData(input_with_data.port->pullData(/* set_not_needed = */ true));
+            waiting_output.status = OutputStatus::NotActive;
+        }
+        else
+            abandoned_chunks.emplace_back(input_with_data.port->pullData(/* set_not_needed = */ true));
 
         if (input_with_data.port->isFinished())
         {
@@ -370,6 +375,18 @@ IProcessor::Status StrictResizeProcessor::prepare(const PortNumbers & updated_in
         return Status::Finished;
     }
 
+    /// Process abandoned chunks if any.
+    while (!abandoned_chunks.empty() && !waiting_outputs.empty())
+    {
+        auto & waiting_output = output_ports[waiting_outputs.front()];
+        waiting_outputs.pop();
+
+        waiting_output.port->pushData(std::move(abandoned_chunks.back()));
+        abandoned_chunks.pop_back();
+
+        waiting_output.status = OutputStatus::NotActive;
+    }
+
     /// Enable more inputs if needed.
     while (!disabled_input_ports.empty() && !waiting_outputs.empty())
     {
@@ -383,6 +400,7 @@ IProcessor::Status StrictResizeProcessor::prepare(const PortNumbers & updated_in
         waiting_outputs.pop();
     }
 
+    /// Close all other waiting for data outputs (there is no corresponding input for them).
     while (!waiting_outputs.empty())
     {
        auto & output = output_ports[waiting_outputs.front()];
diff --git a/src/Processors/ResizeProcessor.h b/src/Processors/ResizeProcessor.h
index 1f364ffcf15..5c9660af113 100644
--- a/src/Processors/ResizeProcessor.h
+++ b/src/Processors/ResizeProcessor.h
@@ -128,6 +128,9 @@ private:
 
     std::vector<InputPortWithStatus> input_ports;
     std::vector<OutputPortWithStatus> output_ports;
+    /// This field contained chunks which were read for output which had became finished while reading was happening.
+    /// They will be pushed to any next waiting output.
+    std::vector<Port::Data> abandoned_chunks;
 };
 
 }

From b16936ba9dfd480ef9fa42538dee6f1751c11685 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Tue, 26 May 2020 13:37:10 +0300
Subject: [PATCH 027/184] Added test.

---
 .../queries/0_stateless/001283_strict_resize_bug.reference | 1 +
 tests/queries/0_stateless/001283_strict_resize_bug.sql     | 7 +++++++
 2 files changed, 8 insertions(+)
 create mode 100644 tests/queries/0_stateless/001283_strict_resize_bug.reference
 create mode 100644 tests/queries/0_stateless/001283_strict_resize_bug.sql

diff --git a/tests/queries/0_stateless/001283_strict_resize_bug.reference b/tests/queries/0_stateless/001283_strict_resize_bug.reference
new file mode 100644
index 00000000000..e1fe7af425d
--- /dev/null
+++ b/tests/queries/0_stateless/001283_strict_resize_bug.reference
@@ -0,0 +1 @@
+49999995000000
diff --git a/tests/queries/0_stateless/001283_strict_resize_bug.sql b/tests/queries/0_stateless/001283_strict_resize_bug.sql
new file mode 100644
index 00000000000..f462f50c61f
--- /dev/null
+++ b/tests/queries/0_stateless/001283_strict_resize_bug.sql
@@ -0,0 +1,7 @@
+drop table if exists num_10m;
+create table num_10m (number UInt64) engine = MergeTree order by tuple();
+insert into num_10m select * from numbers(10000000);
+
+select * from (select sum(number) from num_10m union all select sum(number) from num_10m) limit 1 settings max_block_size = 1024;
+
+drop table if exists num_1m;

From 6626ef08e83c27ee6cb12eb6a15e90083cdbdbff Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Tue, 26 May 2020 15:14:50 +0300
Subject: [PATCH 028/184] Fix PVS studio errors

---
 src/Storages/IStorage.h                  | 4 +++-
 src/Storages/MergeTree/MergeTreeData.cpp | 3 ---
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h
index 551117bd00a..fc99c1a5f78 100644
--- a/src/Storages/IStorage.h
+++ b/src/Storages/IStorage.h
@@ -82,7 +82,9 @@ class IStorage : public std::enable_shared_from_this<IStorage>, public TypePromo
 {
 public:
     IStorage() = delete;
-    explicit IStorage(StorageID storage_id_) : storage_id(std::move(storage_id_)) {}
+    /// Storage fields should be initialized in separate methods like setColumns
+    /// or setTableTTLs.
+    explicit IStorage(StorageID storage_id_) : storage_id(std::move(storage_id_)) {} //-V730
 
     virtual ~IStorage() = default;
     IStorage(const IStorage &) = delete;
diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index cb7d8257f05..92a23e172a9 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -2481,9 +2481,6 @@ void MergeTreeData::movePartitionToDisk(const ASTPtr & partition, const String &
             return part_ptr->volume->getDisk()->getName() == disk->getName();
         }), parts.end());
 
-    if (parts.empty())
-        throw Exception("Nothing to move", ErrorCodes::NO_SUCH_DATA_PART);
-
     if (parts.empty())
     {
         String no_parts_to_move_message;

From 12384551da5fd4d280eb95c356da449c31761d01 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Tue, 26 May 2020 16:19:24 +0300
Subject: [PATCH 029/184] Fix totals and extremes header for views.

---
 src/Processors/Pipe.h            |  2 +-
 src/Processors/QueryPipeline.cpp | 13 ++++++++++-
 src/Processors/QueryPipeline.h   |  3 ++-
 src/Storages/StorageView.cpp     | 37 ++++++++++++++++++++++----------
 4 files changed, 41 insertions(+), 14 deletions(-)

diff --git a/src/Processors/Pipe.h b/src/Processors/Pipe.h
index 984fa7605c6..ec5514915a7 100644
--- a/src/Processors/Pipe.h
+++ b/src/Processors/Pipe.h
@@ -23,7 +23,7 @@ public:
     /// Will connect pipes outputs with transform inputs automatically.
     Pipe(Pipes && pipes, ProcessorPtr transform);
     /// Create pipe from output port. If pipe was created that way, it possibly will not have tree shape.
-    Pipe(OutputPort * port);
+    explicit Pipe(OutputPort * port);
 
     Pipe(const Pipe & other) = delete;
     Pipe(Pipe && other) = default;
diff --git a/src/Processors/QueryPipeline.cpp b/src/Processors/QueryPipeline.cpp
index 13787a3fd3b..26736672412 100644
--- a/src/Processors/QueryPipeline.cpp
+++ b/src/Processors/QueryPipeline.cpp
@@ -703,6 +703,11 @@ void QueryPipeline::initRowsBeforeLimit()
 Pipe QueryPipeline::getPipe() &&
 {
     resize(1);
+    return std::move(std::move(*this).getPipes()[0]);
+}
+
+Pipes QueryPipeline::getPipes() &&
+{
     Pipe pipe(std::move(processors), streams.at(0), totals_having_port, extremes_port);
     pipe.max_parallel_streams = streams.maxParallelStreams();
 
@@ -721,7 +726,13 @@ Pipe QueryPipeline::getPipe() &&
     if (extremes_port)
         pipe.setExtremesPort(extremes_port);
 
-    return pipe;
+    Pipes pipes;
+    pipes.emplace_back(std::move(pipe));
+
+    for (size_t i = 1; i < streams.size(); ++i)
+        pipes.emplace_back(Pipe(streams[i]));
+
+    return pipes;
 }
 
 PipelineExecutorPtr QueryPipeline::execute()
diff --git a/src/Processors/QueryPipeline.h b/src/Processors/QueryPipeline.h
index 45e38ffa715..d248853131c 100644
--- a/src/Processors/QueryPipeline.h
+++ b/src/Processors/QueryPipeline.h
@@ -155,8 +155,9 @@ public:
     /// Set upper limit for the recommend number of threads
     void setMaxThreads(size_t max_threads_) { max_threads = max_threads_; }
 
-    /// Convert query pipeline to single pipe.
+    /// Convert query pipeline to single or several pipes.
     Pipe getPipe() &&;
+    Pipes getPipes() &&;
 
 private:
     /// Destruction order: processors, header, locks, temporary storages, local contexts
diff --git a/src/Storages/StorageView.cpp b/src/Storages/StorageView.cpp
index 636c7f9d64d..01c85c5238a 100644
--- a/src/Storages/StorageView.cpp
+++ b/src/Storages/StorageView.cpp
@@ -19,6 +19,8 @@
 #include <Processors/Sources/SourceFromInputStream.h>
 #include <Processors/Transforms/MaterializingTransform.h>
 #include <Processors/Transforms/ConvertingTransform.h>
+#include <DataStreams/MaterializingBlockInputStream.h>
+#include <DataStreams/ConvertingBlockInputStream.h>
 
 
 namespace DB
@@ -62,29 +64,42 @@ Pipes StorageView::read(
     if (context.getSettings().enable_optimize_predicate_expression)
         current_inner_query = getRuntimeViewQuery(*query_info.query->as<const ASTSelectQuery>(), context);
 
-    QueryPipeline pipeline;
     InterpreterSelectWithUnionQuery interpreter(current_inner_query, context, {}, column_names);
     /// FIXME res may implicitly use some objects owned be pipeline, but them will be destructed after return
     if (query_info.force_tree_shaped_pipeline)
     {
+        QueryPipeline pipeline;
         BlockInputStreams streams = interpreter.executeWithMultipleStreams(pipeline);
+
+        for (auto & stream : streams)
+        {
+            stream = std::make_shared<MaterializingBlockInputStream>(stream);
+            stream = std::make_shared<ConvertingBlockInputStream>(stream, getSampleBlockForColumns(column_names),
+                                                                  ConvertingBlockInputStream::MatchColumnsMode::Name);
+        }
+
         for (auto & stream : streams)
             pipes.emplace_back(std::make_shared<SourceFromInputStream>(std::move(stream)));
     }
     else
-        /// TODO: support multiple streams here. Need more general interface than pipes.
-        pipes.emplace_back(interpreter.executeWithProcessors().getPipe());
-
-    /// It's expected that the columns read from storage are not constant.
-    /// Because method 'getSampleBlockForColumns' is used to obtain a structure of result in InterpreterSelectQuery.
-    for (auto & pipe : pipes)
     {
-        pipe.addSimpleTransform(std::make_shared<MaterializingTransform>(pipe.getHeader()));
+        auto pipeline = interpreter.executeWithProcessors();
+
+        /// It's expected that the columns read from storage are not constant.
+        /// Because method 'getSampleBlockForColumns' is used to obtain a structure of result in InterpreterSelectQuery.
+        pipeline.addSimpleTransform([](const Block & header)
+        {
+            return std::make_shared<MaterializingTransform>(header);
+        });
 
         /// And also convert to expected structure.
-        pipe.addSimpleTransform(std::make_shared<ConvertingTransform>(
-            pipe.getHeader(), getSampleBlockForColumns(column_names),
-            ConvertingTransform::MatchColumnsMode::Name));
+        pipeline.addSimpleTransform([&](const Block & header)
+        {
+            return std::make_shared<ConvertingTransform>(header, getSampleBlockForColumns(column_names),
+                                                         ConvertingTransform::MatchColumnsMode::Name);
+        });
+
+        pipes = std::move(pipeline).getPipes();
     }
 
     return pipes;

From 8074e2027fd76e03725598fbdf5e15565301441b Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Tue, 26 May 2020 16:32:44 +0300
Subject: [PATCH 030/184] Added test.

---
 .../queries/0_stateless/01284_view_and_extremes_bug.reference | 1 +
 tests/queries/0_stateless/01284_view_and_extremes_bug.sql     | 4 ++++
 2 files changed, 5 insertions(+)
 create mode 100644 tests/queries/0_stateless/01284_view_and_extremes_bug.reference
 create mode 100644 tests/queries/0_stateless/01284_view_and_extremes_bug.sql

diff --git a/tests/queries/0_stateless/01284_view_and_extremes_bug.reference b/tests/queries/0_stateless/01284_view_and_extremes_bug.reference
new file mode 100644
index 00000000000..216e97ce082
--- /dev/null
+++ b/tests/queries/0_stateless/01284_view_and_extremes_bug.reference
@@ -0,0 +1 @@
+World
diff --git a/tests/queries/0_stateless/01284_view_and_extremes_bug.sql b/tests/queries/0_stateless/01284_view_and_extremes_bug.sql
new file mode 100644
index 00000000000..c444441a258
--- /dev/null
+++ b/tests/queries/0_stateless/01284_view_and_extremes_bug.sql
@@ -0,0 +1,4 @@
+drop table if exists view_bug_const;
+CREATE VIEW view_bug_const AS SELECT 'World' AS hello FROM (SELECT number FROM system.numbers LIMIT 1) AS n1 JOIN (SELECT number FROM system.numbers LIMIT 1) AS n2 USING (number);
+select * from view_bug_const;
+drop table if exists view_bug_const;

From 8614487a5a44b833cf808bcd95cf312f2ab4eecf Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Tue, 26 May 2020 16:46:19 +0300
Subject: [PATCH 031/184] Fix primary key check

---
 src/Storages/MergeTree/IMergeTreeDataPart.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp
index 12baef4b7be..dcbda0a2c80 100644
--- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp
+++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp
@@ -837,9 +837,10 @@ void IMergeTreeDataPart::checkConsistencyBase() const
 {
     String path = getFullRelativePath();
 
+    const auto & pk = storage.getPrimaryKey();
     if (!checksums.empty())
     {
-        if (storage.hasPrimaryKey() && !checksums.files.count("primary.idx"))
+        if (!pk.column_names.empty() && !checksums.files.count("primary.idx"))
             throw Exception("No checksum for primary.idx", ErrorCodes::NO_FILE_IN_DATA_PART);
 
         if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING)
@@ -873,7 +874,7 @@ void IMergeTreeDataPart::checkConsistencyBase() const
         };
 
         /// Check that the primary key index is not empty.
-        if (storage.hasPrimaryKey())
+        if (!pk.column_names.empty())
             check_file_not_empty(volume->getDisk(), path + "primary.idx");
 
         if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING)

From e09bcb4290b374b654d2a71ddd9fbde5beeeaa1f Mon Sep 17 00:00:00 2001
From: Maxim Akhmedov <max42@yandex-team.ru>
Date: Tue, 26 May 2020 17:05:06 +0300
Subject: [PATCH 032/184] Make possible adding nested storages to
 MultipleAccessStorage in run-time.

---
 src/Access/MultipleAccessStorage.cpp | 8 ++++++++
 src/Access/MultipleAccessStorage.h   | 2 ++
 2 files changed, 10 insertions(+)

diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp
index 0dd1f142f31..9b80c16e487 100644
--- a/src/Access/MultipleAccessStorage.cpp
+++ b/src/Access/MultipleAccessStorage.cpp
@@ -143,6 +143,14 @@ const IAccessStorage & MultipleAccessStorage::getStorage(const UUID & id) const
     return const_cast<MultipleAccessStorage *>(this)->getStorage(id);
 }
 
+void MultipleAccessStorage::addStorage(std::unique_ptr<Storage> nested_storage)
+{
+    /// Note that IStorage::storage_name is not changed. It is ok as this method
+    /// is considered as a temporary solution allowing third-party Arcadia applications
+    /// using CH as a library to register their own access storages. Do not remove
+    /// this method without providing any alternative :)
+    nested_storages.emplace_back(std::move(nested_storage));
+}
 
 AccessEntityPtr MultipleAccessStorage::readImpl(const UUID & id) const
 {
diff --git a/src/Access/MultipleAccessStorage.h b/src/Access/MultipleAccessStorage.h
index ec8c8f2a101..06fb3d45c05 100644
--- a/src/Access/MultipleAccessStorage.h
+++ b/src/Access/MultipleAccessStorage.h
@@ -25,6 +25,8 @@ public:
     const Storage & getStorage(const UUID & id) const;
     Storage & getStorage(const UUID & id);
 
+    void addStorage(std::unique_ptr<Storage> nested_storage);
+
     Storage & getStorageByIndex(size_t i) { return *(nested_storages[i]); }
     const Storage & getStorageByIndex(size_t i) const { return *(nested_storages[i]); }
 

From 9d0c28d96c1987d4f4412a679354e22ad0f6304c Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Tue, 26 May 2020 17:14:08 +0300
Subject: [PATCH 033/184] Add missing methods

---
 src/Storages/IStorage.cpp                     | 24 +++++++++++++++----
 src/Storages/IStorage.h                       | 11 +++++++--
 src/Storages/MergeTree/IMergeTreeDataPart.cpp |  4 ++--
 src/Storages/MergeTree/MergeTreeData.cpp      |  6 ++---
 4 files changed, 34 insertions(+), 11 deletions(-)

diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp
index 64e06a2534e..6564fee827e 100644
--- a/src/Storages/IStorage.cpp
+++ b/src/Storages/IStorage.cpp
@@ -444,9 +444,14 @@ void IStorage::setPartitionKey(const StorageMetadataKeyField & partition_key_)
     partition_key = partition_key_;
 }
 
+bool IStorage::isPartitionKeyDefined() const
+{
+    return partition_key.definition_ast != nullptr;
+}
+
 bool IStorage::hasPartitionKey() const
 {
-    return partition_key.expression != nullptr;
+    return !partition_key.column_names.empty();
 }
 
 Names IStorage::getColumnsRequiredForPartitionKey() const
@@ -466,9 +471,14 @@ void IStorage::setSortingKey(const StorageMetadataKeyField & sorting_key_)
     sorting_key = sorting_key_;
 }
 
+bool IStorage::isSortingKeyDefined() const
+{
+    return sorting_key.definition_ast != nullptr;
+}
+
 bool IStorage::hasSortingKey() const
 {
-    return sorting_key.expression != nullptr;
+    return !sorting_key.column_names.empty();
 }
 
 Names IStorage::getColumnsRequiredForSortingKey() const
@@ -502,7 +512,7 @@ bool IStorage::isPrimaryKeyDefined() const
 
 bool IStorage::hasPrimaryKey() const
 {
-    return primary_key.expression != nullptr;
+    return !primary_key.column_names.empty();
 }
 
 Names IStorage::getColumnsRequiredForPrimaryKey() const
@@ -529,9 +539,15 @@ void IStorage::setSamplingKey(const StorageMetadataKeyField & sampling_key_)
     sampling_key = sampling_key_;
 }
 
+
+bool IStorage::isSamplingKeyDefined() const
+{
+    return sampling_key.definition_ast != nullptr;
+}
+
 bool IStorage::hasSamplingKey() const
 {
-    return sampling_key.expression != nullptr;
+    return !sampling_key.column_names.empty();
 }
 
 Names IStorage::getColumnsRequiredForSampling() const
diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h
index f707b3a4b5c..655ced0d8cf 100644
--- a/src/Storages/IStorage.h
+++ b/src/Storages/IStorage.h
@@ -453,6 +453,8 @@ public:
     void setPartitionKey(const StorageMetadataKeyField & partition_key_);
     /// Returns ASTExpressionList of partition key expression for storage or nullptr if there is none.
     ASTPtr getPartitionKeyAST() const { return partition_key.definition_ast; }
+    /// Storage has user-defined (in CREATE query) partition key.
+    bool isPartitionKeyDefined() const;
     /// Storage has partition key.
     bool hasPartitionKey() const;
     /// Returns column names that need to be read to calculate partition key.
@@ -466,7 +468,9 @@ public:
     void setSortingKey(const StorageMetadataKeyField & sorting_key_);
     /// Returns ASTExpressionList of sorting key expression for storage or nullptr if there is none.
     ASTPtr getSortingKeyAST() const { return sorting_key.definition_ast; }
-    /// Storage has sorting key.
+    /// Storage has user-defined (in CREATE query) sorting key.
+    bool isSortingKeyDefined() const;
+    /// Storage has sorting key. It means, that it contains at least one column.
     bool hasSortingKey() const;
     /// Returns column names that need to be read to calculate sorting key.
     Names getColumnsRequiredForSortingKey() const;
@@ -483,7 +487,8 @@ public:
     ASTPtr getPrimaryKeyAST() const { return primary_key.definition_ast; }
     /// Storage has user-defined (in CREATE query) sorting key.
     bool isPrimaryKeyDefined() const;
-    /// Storage has primary key (maybe part of some other key).
+    /// Storage has primary key (maybe part of some other key). It means, that
+    /// it contains at least one column.
     bool hasPrimaryKey() const;
     /// Returns column names that need to be read to calculate primary key.
     Names getColumnsRequiredForPrimaryKey() const;
@@ -498,6 +503,8 @@ public:
     void setSamplingKey(const StorageMetadataKeyField & sampling_key_);
     /// Returns sampling expression AST for storage or nullptr if there is none.
     ASTPtr getSamplingKeyAST() const { return sampling_key.definition_ast; }
+    /// Storage has user-defined (in CREATE query) sampling key.
+    bool isSamplingKeyDefined() const;
     /// Storage has sampling key.
     bool hasSamplingKey() const;
     /// Returns column names that need to be read to calculate sampling key.
diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp
index 78548cdddb2..6e048a956da 100644
--- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp
+++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp
@@ -841,7 +841,7 @@ void IMergeTreeDataPart::checkConsistencyBase() const
 
     if (!checksums.empty())
     {
-        if (storage.isPrimaryKeyDefined() && !checksums.files.count("primary.idx"))
+        if (storage.hasPrimaryKey() && !checksums.files.count("primary.idx"))
             throw Exception("No checksum for primary.idx", ErrorCodes::NO_FILE_IN_DATA_PART);
 
         if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING)
@@ -875,7 +875,7 @@ void IMergeTreeDataPart::checkConsistencyBase() const
         };
 
         /// Check that the primary key index is not empty.
-        if (storage.isPrimaryKeyDefined())
+        if (storage.hasPrimaryKey())
             check_file_not_empty(volume->getDisk(), path + "primary.idx");
 
         if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING)
diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index d80dce167b4..48cf3934820 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -250,10 +250,10 @@ StorageInMemoryMetadata MergeTreeData::getInMemoryMetadata() const
 {
     StorageInMemoryMetadata metadata(getColumns(), getIndices(), getConstraints());
 
-    if (hasPartitionKey())
+    if (isPartitionKeyDefined())
         metadata.partition_by_ast = getPartitionKeyAST()->clone();
 
-    if (hasSortingKey())
+    if (isSortingKeyDefined())
         metadata.order_by_ast = getSortingKeyAST()->clone();
 
     if (isPrimaryKeyDefined())
@@ -262,7 +262,7 @@ StorageInMemoryMetadata MergeTreeData::getInMemoryMetadata() const
     if (ttl_table_ast)
         metadata.ttl_for_table_ast = ttl_table_ast->clone();
 
-    if (hasSamplingKey())
+    if (isSamplingKeyDefined())
         metadata.sample_by_ast = getSamplingKeyAST()->clone();
 
     if (settings_ast)

From b2057159c1d12fb016b7918b7a00b11ee73967fc Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Tue, 26 May 2020 17:24:20 +0300
Subject: [PATCH 034/184] Experiment on parsing backslash escapes in a more
 convenient way #10922

---
 src/IO/ReadHelpers.cpp                        | 16 +++-
 src/IO/tests/CMakeLists.txt                   |  3 -
 src/IO/tests/mempbrk.cpp                      | 90 -------------------
 ...escape_sequences_php_mysql_style.reference |  3 +
 ...01284_escape_sequences_php_mysql_style.sql |  6 ++
 5 files changed, 22 insertions(+), 96 deletions(-)
 delete mode 100644 src/IO/tests/mempbrk.cpp
 create mode 100644 tests/queries/0_stateless/01284_escape_sequences_php_mysql_style.reference
 create mode 100644 tests/queries/0_stateless/01284_escape_sequences_php_mysql_style.sql

diff --git a/src/IO/ReadHelpers.cpp b/src/IO/ReadHelpers.cpp
index dccb413af2c..65c3cfab55a 100644
--- a/src/IO/ReadHelpers.cpp
+++ b/src/IO/ReadHelpers.cpp
@@ -283,7 +283,9 @@ static void parseComplexEscapeSequence(Vector & s, ReadBuffer & buf)
     if (buf.eof())
         throw Exception("Cannot parse escape sequence", ErrorCodes::CANNOT_PARSE_ESCAPE_SEQUENCE);
 
-    if (*buf.position() == 'x')
+    char char_after_backslash = *buf.position();
+
+    if (char_after_backslash == 'x')
     {
         ++buf.position();
         /// escape sequence of the form \xAA
@@ -291,7 +293,7 @@ static void parseComplexEscapeSequence(Vector & s, ReadBuffer & buf)
         readPODBinary(hex_code, buf);
         s.push_back(unhex2(hex_code));
     }
-    else if (*buf.position() == 'N')
+    else if (char_after_backslash == 'N')
     {
         /// Support for NULLs: \N sequence must be parsed as empty string.
         ++buf.position();
@@ -299,7 +301,15 @@ static void parseComplexEscapeSequence(Vector & s, ReadBuffer & buf)
     else
     {
         /// The usual escape sequence of a single character.
-        s.push_back(parseEscapeSequence(*buf.position()));
+        char decoded_char = parseEscapeSequence(char_after_backslash);
+
+        /// For convenience using LIKE and regular expressions,
+        /// we leave backslash when user write something like 'Hello 100\%':
+        /// it is parsed like Hello 100\% instead of Hello 100%
+        if (decoded_char != '\\' && !isControlASCII(decoded_char))
+            s.push_back('\\');
+
+        s.push_back(decoded_char);
         ++buf.position();
     }
 }
diff --git a/src/IO/tests/CMakeLists.txt b/src/IO/tests/CMakeLists.txt
index 2767ce6e271..dfbbfa77853 100644
--- a/src/IO/tests/CMakeLists.txt
+++ b/src/IO/tests/CMakeLists.txt
@@ -37,9 +37,6 @@ target_link_libraries (parse_int_perf2 PRIVATE clickhouse_common_io)
 add_executable (read_write_int read_write_int.cpp)
 target_link_libraries (read_write_int PRIVATE clickhouse_common_io)
 
-add_executable (mempbrk mempbrk.cpp)
-target_link_libraries (mempbrk PRIVATE clickhouse_common_io)
-
 add_executable (o_direct_and_dirty_pages o_direct_and_dirty_pages.cpp)
 target_link_libraries (o_direct_and_dirty_pages PRIVATE clickhouse_common_io)
 
diff --git a/src/IO/tests/mempbrk.cpp b/src/IO/tests/mempbrk.cpp
deleted file mode 100644
index 55097d989af..00000000000
--- a/src/IO/tests/mempbrk.cpp
+++ /dev/null
@@ -1,90 +0,0 @@
-#include <string>
-#include <iostream>
-#include <iomanip>
-
-#include <Common/Stopwatch.h>
-
-#include <Core/Types.h>
-#include <IO/ReadHelpers.h>
-#include <IO/WriteHelpers.h>
-#include <IO/ReadBufferFromFileDescriptor.h>
-#include <IO/WriteBufferFromFileDescriptor.h>
-
-#include <common/find_symbols.h>
-
-namespace DB
-{
-namespace ErrorCodes
-{
-    extern const int CANNOT_PARSE_ESCAPE_SEQUENCE;
-}
-}
-
-
-namespace test
-{
-static void readEscapedString(DB::String & s, DB::ReadBuffer & buf)
-    {
-        s = "";
-        while (!buf.eof())
-        {
-            const char * next_pos = find_first_symbols<'\b', '\f', '\n', '\r', '\t', '\0', '\\'>(buf.position(), buf.buffer().end());
-
-            s.append(buf.position(), next_pos - buf.position());
-            buf.position() += next_pos - buf.position();
-
-            if (!buf.hasPendingData())
-                continue;
-
-            if (*buf.position() == '\t' || *buf.position() == '\n')
-                return;
-
-            if (*buf.position() == '\\')
-            {
-                ++buf.position();
-                if (buf.eof())
-                    throw DB::Exception("Cannot parse escape sequence", DB::ErrorCodes::CANNOT_PARSE_ESCAPE_SEQUENCE);
-                s += DB::parseEscapeSequence(*buf.position());
-                ++buf.position();
-            }
-        }
-    }
-}
-
-
-int main(int, char **)
-{
-    try
-    {
-        DB::ReadBufferFromFileDescriptor in(STDIN_FILENO);
-//        DB::WriteBufferFromFileDescriptor out(STDOUT_FILENO);
-        std::string s;
-        size_t rows = 0;
-
-        Stopwatch watch;
-
-        while (!in.eof())
-        {
-            test::readEscapedString(s, in);
-            in.ignore();
-
-            ++rows;
-
-/*            DB::writeEscapedString(s, out);
-            DB::writeChar('\n', out);*/
-        }
-
-        watch.stop();
-        std::cerr << std::fixed << std::setprecision(2)
-            << "Read " << rows << " rows (" << in.count() / 1000000.0 << " MB) in " << watch.elapsedSeconds() << " sec., "
-            << rows / watch.elapsedSeconds() << " rows/sec. (" << in.count() / watch.elapsedSeconds() / 1000000 << " MB/s.)"
-            << std::endl;
-    }
-    catch (const DB::Exception & e)
-    {
-        std::cerr << e.what() << ", " << e.displayText() << std::endl;
-        return 1;
-    }
-
-    return 0;
-}
diff --git a/tests/queries/0_stateless/01284_escape_sequences_php_mysql_style.reference b/tests/queries/0_stateless/01284_escape_sequences_php_mysql_style.reference
new file mode 100644
index 00000000000..19a5fc680ff
--- /dev/null
+++ b/tests/queries/0_stateless/01284_escape_sequences_php_mysql_style.reference
@@ -0,0 +1,3 @@
+a\\_\\c\\l\\i\\c\\k\\h\\o\\u\\s	a\\_\\c\\l\\i\\c\\k\\h\\o\\u\\s\\e
+1	0	1	1
+1	1	0	0	0
diff --git a/tests/queries/0_stateless/01284_escape_sequences_php_mysql_style.sql b/tests/queries/0_stateless/01284_escape_sequences_php_mysql_style.sql
new file mode 100644
index 00000000000..36ad7b4506c
--- /dev/null
+++ b/tests/queries/0_stateless/01284_escape_sequences_php_mysql_style.sql
@@ -0,0 +1,6 @@
+SELECT 'a\_\c\l\i\c\k\h\o\u\s\e', 'a\\_\\c\\l\\i\\c\\k\\h\\o\\u\\s\\e';
+select 'aXb' like 'a_b', 'aXb' like 'a\_b', 'a_b' like 'a\_b', 'a_b' like 'a\\_b';
+SELECT match('Hello', '\w+'), match('Hello', '\\w+'), match('Hello', '\\\w+'), match('Hello', '\w\+'), match('Hello', 'w+');
+
+SELECT match('Hello', '\He\l\l\o'); -- { serverError 427 }
+SELECT match('Hello', '\H\e\l\l\o'); -- { serverError 427 }

From 009f57fc2fc84e4cffe08eea118fa0543ca4c66c Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Tue, 26 May 2020 21:26:17 +0300
Subject: [PATCH 035/184] Fix flacky test #11190

---
 .../01247_dist_on_dist_group_by_sharding_key_optimization.sql | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/queries/0_stateless/01247_dist_on_dist_group_by_sharding_key_optimization.sql b/tests/queries/0_stateless/01247_dist_on_dist_group_by_sharding_key_optimization.sql
index 7cf171c8c73..29e45ffdb80 100644
--- a/tests/queries/0_stateless/01247_dist_on_dist_group_by_sharding_key_optimization.sql
+++ b/tests/queries/0_stateless/01247_dist_on_dist_group_by_sharding_key_optimization.sql
@@ -1,5 +1,9 @@
 -- TODO: correct testing with real unique shards
 
+-- Avoid "Connection failed at try â„–1" messages.
+SET send_logs_level = 'none';
+SET connect_timeout_with_failover_ms = 5000;
+
 set optimize_distributed_group_by_sharding_key=1;
 
 drop table if exists dist_01247;

From 90f49c0c17ff5ce8182a5d45543c5c3aab9985f4 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Tue, 26 May 2020 21:37:20 +0300
Subject: [PATCH 036/184] Fix error in performance test

---
 tests/performance/agg_functions_min_max_any.xml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/tests/performance/agg_functions_min_max_any.xml b/tests/performance/agg_functions_min_max_any.xml
index 4591182cdfc..9c16cb88970 100644
--- a/tests/performance/agg_functions_min_max_any.xml
+++ b/tests/performance/agg_functions_min_max_any.xml
@@ -35,10 +35,6 @@
 <query>select max(PageCharset) from test.hits where PageCharset != '' group by intHash32(UserID) % 1000000 FORMAT Null</query>
 <query>select any(PageCharset) from test.hits where PageCharset != '' group by intHash32(UserID) % 1000000 FORMAT Null</query>
 <query>select anyHeavy(PageCharset) from test.hits where PageCharset != '' group by intHash32(UserID) % 1000000 FORMAT Null</query>
-<query>select min(OriginalURL) from test.hits where OriginalURL != '' group by intHash32(UserID) % 1000000 FORMAT Null</query>
-<query>select max(OriginalURL) from test.hits where OriginalURL != '' group by intHash32(UserID) % 1000000 FORMAT Null</query>
-<query>select any(OriginalURL) from test.hits where OriginalURL != '' group by intHash32(UserID) % 1000000 FORMAT Null</query>
-<query>select anyHeavy(OriginalURL) from test.hits where OriginalURL != '' group by intHash32(UserID) % 1000000 FORMAT Null</query>
 <query>select min(SocialNetwork) from test.hits where SocialNetwork != '' group by intHash32(UserID) % 1000000 FORMAT Null</query>
 <query>select max(SocialNetwork) from test.hits where SocialNetwork != '' group by intHash32(UserID) % 1000000 FORMAT Null</query>
 <query>select any(SocialNetwork) from test.hits where SocialNetwork != '' group by intHash32(UserID) % 1000000 FORMAT Null</query>

From e049290af7b311f60205684c9e2d5bd154e17459 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <avtokmakov@sandbox-407922969>
Date: Tue, 26 May 2020 22:14:54 +0300
Subject: [PATCH 037/184] add cassandra-driver to integration tests

---
 docker/test/integration/runner/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/test/integration/runner/Dockerfile b/docker/test/integration/runner/Dockerfile
index 53f41905f10..9c1fe66cf7b 100644
--- a/docker/test/integration/runner/Dockerfile
+++ b/docker/test/integration/runner/Dockerfile
@@ -36,7 +36,7 @@ RUN apt-get update \
 ENV TZ=Europe/Moscow
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
-RUN pip install urllib3==1.23 pytest docker-compose==1.22.0 docker dicttoxml kazoo PyMySQL psycopg2==2.7.5 pymongo tzlocal kafka-python protobuf redis aerospike pytest-timeout minio rpm-confluent-schemaregistry grpcio grpcio-tools
+RUN pip install urllib3==1.23 pytest docker-compose==1.22.0 docker dicttoxml kazoo PyMySQL psycopg2==2.7.5 pymongo tzlocal kafka-python protobuf redis aerospike pytest-timeout minio rpm-confluent-schemaregistry grpcio grpcio-tools cassandra-driver
 
 ENV DOCKER_CHANNEL stable
 ENV DOCKER_VERSION 17.09.1-ce

From 15e7c618bb5023ed2d63465e2e3eed7e99503799 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Tue, 26 May 2020 22:21:07 +0300
Subject: [PATCH 038/184] Fix cctz linking with dirty hack

---
 contrib/cctz-cmake/CMakeLists.txt | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/contrib/cctz-cmake/CMakeLists.txt b/contrib/cctz-cmake/CMakeLists.txt
index df9fd6aa61c..d353fc9d4de 100644
--- a/contrib/cctz-cmake/CMakeLists.txt
+++ b/contrib/cctz-cmake/CMakeLists.txt
@@ -618,7 +618,11 @@ if (USE_INTERNAL_CCTZ)
 
         add_library(tzdata STATIC ${TZ_OBJS})
         set_target_properties(tzdata PROPERTIES LINKER_LANGUAGE C)
-        target_link_libraries(cctz -Wl,--whole-archive tzdata -Wl,--no-whole-archive) # whole-archive prevents symbols from being discarded
+        # whole-archive prevents symbols from being discarded for unknown reason
+        # CMake can shuffle each of target_link_libraries arguments with other
+        # libraries in linker command. To avoid this we hardcode whole-archive
+        # library into single string.
+        target_link_libraries(cctz PUBLIC "-Wl,--whole-archive contrib/cctz-cmake/libtzdata.a -Wl,--no-whole-archive")
     endif ()
 
 else ()

From 4a92ad11ec02a4a94b7b3567cd02a62df4dd37ea Mon Sep 17 00:00:00 2001
From: Ivan Blinkov <github@blinkov.ru>
Date: Tue, 26 May 2020 23:27:31 +0300
Subject: [PATCH 039/184] Update test.py

---
 tests/integration/test_system_queries/test.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tests/integration/test_system_queries/test.py b/tests/integration/test_system_queries/test.py
index 1761017362a..9397cd20ec7 100644
--- a/tests/integration/test_system_queries/test.py
+++ b/tests/integration/test_system_queries/test.py
@@ -95,6 +95,10 @@ def test_RELOAD_CONFIG_AND_MACROS(started_cluster):
 
 def test_SYSTEM_FLUSH_LOGS(started_cluster):
     instance = cluster.instances['ch1']
+    instance.query('''
+        SET log_queries = 0;
+        TRUNCATE TABLE system.query_log;
+    ''')
     for i in range(4):
         # Sleep to execute flushing from background thread at first query
         # by expiration of flush_interval_millisecond and test probable race condition.
@@ -105,7 +109,10 @@ def test_SYSTEM_FLUSH_LOGS(started_cluster):
             SET log_queries = 0;
             SYSTEM FLUSH LOGS;
             SELECT count() FROM system.query_log;''')
-        instance.query('TRUNCATE TABLE system.query_log')
+        instance.query('''
+            SET log_queries = 0;
+            TRUNCATE TABLE system.query_log;
+        ''')
         assert TSV(result) == TSV('4')
 
 

From a5bde57acfd00acf1af44f794daa4ebb23583201 Mon Sep 17 00:00:00 2001
From: Saulius Valatka <saulius.valatka@adform.com>
Date: Tue, 26 May 2020 23:43:20 +0300
Subject: [PATCH 040/184] add reference to stream-loader (#11206)

---
 docs/en/interfaces/third-party/integrations.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/en/interfaces/third-party/integrations.md b/docs/en/interfaces/third-party/integrations.md
index 5e9e8f841bf..716e774871b 100644
--- a/docs/en/interfaces/third-party/integrations.md
+++ b/docs/en/interfaces/third-party/integrations.md
@@ -25,6 +25,7 @@ toc_title: Integrations
 -   Message queues
     -   [Kafka](https://kafka.apache.org)
         -   [clickhouse\_sinker](https://github.com/housepower/clickhouse_sinker) (uses [Go client](https://github.com/ClickHouse/clickhouse-go/))
+        -   [stream-loader-clickhouse](https://github.com/adform/stream-loader)
 -   Stream processing
     -   [Flink](https://flink.apache.org)
         -   [flink-clickhouse-sink](https://github.com/ivi-ru/flink-clickhouse-sink)

From fb4089ab8e28c2e66ada5556344a225ffc629579 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Wed, 27 May 2020 00:48:29 +0300
Subject: [PATCH 041/184] Fix error

---
 src/IO/ReadHelpers.cpp | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/IO/ReadHelpers.cpp b/src/IO/ReadHelpers.cpp
index 65c3cfab55a..0797e63cdd9 100644
--- a/src/IO/ReadHelpers.cpp
+++ b/src/IO/ReadHelpers.cpp
@@ -306,8 +306,15 @@ static void parseComplexEscapeSequence(Vector & s, ReadBuffer & buf)
         /// For convenience using LIKE and regular expressions,
         /// we leave backslash when user write something like 'Hello 100\%':
         /// it is parsed like Hello 100\% instead of Hello 100%
-        if (decoded_char != '\\' && !isControlASCII(decoded_char))
+        if (decoded_char != '\\'
+            && decoded_char != '\''
+            && decoded_char != '"'
+            && decoded_char != '`'  /// MySQL style identifiers
+            && decoded_char != '/'  /// JavaScript in HTML
+            && !isControlASCII(decoded_char))
+        {
             s.push_back('\\');
+        }
 
         s.push_back(decoded_char);
         ++buf.position();

From 734cc78e5838bafc35a3abc544fd7609c09902ca Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 24 May 2020 13:16:34 +0300
Subject: [PATCH 042/184] jemalloc: make include_linux_*/jemalloc/*.h common
 for aarch64 and x86_64

This patch can go upstream, regardless of the jemalloc update
eventually.

Also I noticed that header with jemalloc version differs (someone forgot
to update?)
---
 .../jemalloc/jemalloc_defs.h                  |   0
 .../jemalloc/jemalloc_macros.h                |   0
 .../jemalloc/jemalloc_protos.h                |   0
 .../jemalloc/jemalloc_typedefs.h              |  18 +--
 .../jemalloc/jemalloc_macros.h                | 123 ------------------
 .../jemalloc/jemalloc_defs.h                  |  43 ------
 .../jemalloc/jemalloc_protos.h                |  66 ----------
 .../jemalloc/jemalloc_typedefs.h              |  77 -----------
 8 files changed, 9 insertions(+), 318 deletions(-)
 rename contrib/jemalloc-cmake/{include_linux_aarch64 => include}/jemalloc/jemalloc_defs.h (100%)
 rename contrib/jemalloc-cmake/{include_linux_x86_64 => include}/jemalloc/jemalloc_macros.h (100%)
 rename contrib/jemalloc-cmake/{include_linux_aarch64 => include}/jemalloc/jemalloc_protos.h (100%)
 rename contrib/jemalloc-cmake/{include_linux_aarch64 => include}/jemalloc/jemalloc_typedefs.h (88%)
 delete mode 100644 contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/jemalloc_macros.h
 delete mode 100644 contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_defs.h
 delete mode 100644 contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_protos.h
 delete mode 100644 contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_typedefs.h

diff --git a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/jemalloc_defs.h b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_defs.h
similarity index 100%
rename from contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/jemalloc_defs.h
rename to contrib/jemalloc-cmake/include/jemalloc/jemalloc_defs.h
diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_macros.h b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_macros.h
similarity index 100%
rename from contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_macros.h
rename to contrib/jemalloc-cmake/include/jemalloc/jemalloc_macros.h
diff --git a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/jemalloc_protos.h b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_protos.h
similarity index 100%
rename from contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/jemalloc_protos.h
rename to contrib/jemalloc-cmake/include/jemalloc/jemalloc_protos.h
diff --git a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/jemalloc_typedefs.h b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_typedefs.h
similarity index 88%
rename from contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/jemalloc_typedefs.h
rename to contrib/jemalloc-cmake/include/jemalloc/jemalloc_typedefs.h
index 1a58874306e..5f94f16f937 100644
--- a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/jemalloc_typedefs.h
+++ b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_typedefs.h
@@ -65,13 +65,13 @@ typedef bool (extent_merge_t)(extent_hooks_t *, void *, size_t, void *, size_t,
     bool, unsigned);
 
 struct extent_hooks_s {
-	extent_alloc_t		*alloc;
-	extent_dalloc_t		*dalloc;
-	extent_destroy_t	*destroy;
-	extent_commit_t		*commit;
-	extent_decommit_t	*decommit;
-	extent_purge_t		*purge_lazy;
-	extent_purge_t		*purge_forced;
-	extent_split_t		*split;
-	extent_merge_t		*merge;
+    extent_alloc_t		*alloc;
+    extent_dalloc_t		*dalloc;
+    extent_destroy_t	*destroy;
+    extent_commit_t		*commit;
+    extent_decommit_t	*decommit;
+    extent_purge_t		*purge_lazy;
+    extent_purge_t		*purge_forced;
+    extent_split_t		*split;
+    extent_merge_t		*merge;
 };
diff --git a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/jemalloc_macros.h b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/jemalloc_macros.h
deleted file mode 100644
index 79b13337fbb..00000000000
--- a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/jemalloc_macros.h
+++ /dev/null
@@ -1,123 +0,0 @@
-#include <stdlib.h>
-#include <stdbool.h>
-#include <stdint.h>
-#include <limits.h>
-#include <strings.h>
-
-#define JEMALLOC_VERSION "5.1.0-97-gcd2931ad9bbd78208565716ab102e86d858c2fff"
-#define JEMALLOC_VERSION_MAJOR 5
-#define JEMALLOC_VERSION_MINOR 1
-#define JEMALLOC_VERSION_BUGFIX 0
-#define JEMALLOC_VERSION_NREV 97
-#define JEMALLOC_VERSION_GID "cd2931ad9bbd78208565716ab102e86d858c2fff"
-#define JEMALLOC_VERSION_GID_IDENT cd2931ad9bbd78208565716ab102e86d858c2fff
-
-#define MALLOCX_LG_ALIGN(la)	((int)(la))
-#if LG_SIZEOF_PTR == 2
-#  define MALLOCX_ALIGN(a)	((int)(ffs((int)(a))-1))
-#else
-#  define MALLOCX_ALIGN(a)						\
-     ((int)(((size_t)(a) < (size_t)INT_MAX) ? ffs((int)(a))-1 :	\
-     ffs((int)(((size_t)(a))>>32))+31))
-#endif
-#define MALLOCX_ZERO	((int)0x40)
-/*
- * Bias tcache index bits so that 0 encodes "automatic tcache management", and 1
- * encodes MALLOCX_TCACHE_NONE.
- */
-#define MALLOCX_TCACHE(tc)	((int)(((tc)+2) << 8))
-#define MALLOCX_TCACHE_NONE	MALLOCX_TCACHE(-1)
-/*
- * Bias arena index bits so that 0 encodes "use an automatically chosen arena".
- */
-#define MALLOCX_ARENA(a)	((((int)(a))+1) << 20)
-
-/*
- * Use as arena index in "arena.<i>.{purge,decay,dss}" and
- * "stats.arenas.<i>.*" mallctl interfaces to select all arenas.  This
- * definition is intentionally specified in raw decimal format to support
- * cpp-based string concatenation, e.g.
- *
- *   #define STRINGIFY_HELPER(x) #x
- *   #define STRINGIFY(x) STRINGIFY_HELPER(x)
- *
- *   mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge", NULL, NULL, NULL,
- *       0);
- */
-#define MALLCTL_ARENAS_ALL	4096
-/*
- * Use as arena index in "stats.arenas.<i>.*" mallctl interfaces to select
- * destroyed arenas.
- */
-#define MALLCTL_ARENAS_DESTROYED	4097
-
-#if defined(__cplusplus) && defined(JEMALLOC_USE_CXX_THROW)
-#  define JEMALLOC_CXX_THROW throw()
-#else
-#  define JEMALLOC_CXX_THROW
-#endif
-
-#if defined(_MSC_VER)
-#  define JEMALLOC_ATTR(s)
-#  define JEMALLOC_ALIGNED(s) __declspec(align(s))
-#  define JEMALLOC_ALLOC_SIZE(s)
-#  define JEMALLOC_ALLOC_SIZE2(s1, s2)
-#  ifndef JEMALLOC_EXPORT
-#    ifdef DLLEXPORT
-#      define JEMALLOC_EXPORT __declspec(dllexport)
-#    else
-#      define JEMALLOC_EXPORT __declspec(dllimport)
-#    endif
-#  endif
-#  define JEMALLOC_FORMAT_PRINTF(s, i)
-#  define JEMALLOC_NOINLINE __declspec(noinline)
-#  ifdef __cplusplus
-#    define JEMALLOC_NOTHROW __declspec(nothrow)
-#  else
-#    define JEMALLOC_NOTHROW
-#  endif
-#  define JEMALLOC_SECTION(s) __declspec(allocate(s))
-#  define JEMALLOC_RESTRICT_RETURN __declspec(restrict)
-#  if _MSC_VER >= 1900 && !defined(__EDG__)
-#    define JEMALLOC_ALLOCATOR __declspec(allocator)
-#  else
-#    define JEMALLOC_ALLOCATOR
-#  endif
-#elif defined(JEMALLOC_HAVE_ATTR)
-#  define JEMALLOC_ATTR(s) __attribute__((s))
-#  define JEMALLOC_ALIGNED(s) JEMALLOC_ATTR(aligned(s))
-#  ifdef JEMALLOC_HAVE_ATTR_ALLOC_SIZE
-#    define JEMALLOC_ALLOC_SIZE(s) JEMALLOC_ATTR(alloc_size(s))
-#    define JEMALLOC_ALLOC_SIZE2(s1, s2) JEMALLOC_ATTR(alloc_size(s1, s2))
-#  else
-#    define JEMALLOC_ALLOC_SIZE(s)
-#    define JEMALLOC_ALLOC_SIZE2(s1, s2)
-#  endif
-#  ifndef JEMALLOC_EXPORT
-#    define JEMALLOC_EXPORT JEMALLOC_ATTR(visibility("default"))
-#  endif
-#  ifdef JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF
-#    define JEMALLOC_FORMAT_PRINTF(s, i) JEMALLOC_ATTR(format(gnu_printf, s, i))
-#  elif defined(JEMALLOC_HAVE_ATTR_FORMAT_PRINTF)
-#    define JEMALLOC_FORMAT_PRINTF(s, i) JEMALLOC_ATTR(format(printf, s, i))
-#  else
-#    define JEMALLOC_FORMAT_PRINTF(s, i)
-#  endif
-#  define JEMALLOC_NOINLINE JEMALLOC_ATTR(noinline)
-#  define JEMALLOC_NOTHROW JEMALLOC_ATTR(nothrow)
-#  define JEMALLOC_SECTION(s) JEMALLOC_ATTR(section(s))
-#  define JEMALLOC_RESTRICT_RETURN
-#  define JEMALLOC_ALLOCATOR
-#else
-#  define JEMALLOC_ATTR(s)
-#  define JEMALLOC_ALIGNED(s)
-#  define JEMALLOC_ALLOC_SIZE(s)
-#  define JEMALLOC_ALLOC_SIZE2(s1, s2)
-#  define JEMALLOC_EXPORT
-#  define JEMALLOC_FORMAT_PRINTF(s, i)
-#  define JEMALLOC_NOINLINE
-#  define JEMALLOC_NOTHROW
-#  define JEMALLOC_SECTION(s)
-#  define JEMALLOC_RESTRICT_RETURN
-#  define JEMALLOC_ALLOCATOR
-#endif
diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_defs.h b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_defs.h
deleted file mode 100644
index d1389237a77..00000000000
--- a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_defs.h
+++ /dev/null
@@ -1,43 +0,0 @@
-/* include/jemalloc/jemalloc_defs.h.  Generated from jemalloc_defs.h.in by configure.  */
-/* Defined if __attribute__((...)) syntax is supported. */
-#define JEMALLOC_HAVE_ATTR
-
-/* Defined if alloc_size attribute is supported. */
-#define JEMALLOC_HAVE_ATTR_ALLOC_SIZE
-
-/* Defined if format(printf, ...) attribute is supported. */
-#define JEMALLOC_HAVE_ATTR_FORMAT_PRINTF
-
-/*
- * Define overrides for non-standard allocator-related functions if they are
- * present on the system.
- */
-#define JEMALLOC_OVERRIDE_MEMALIGN
-#define JEMALLOC_OVERRIDE_VALLOC
-
-/*
- * At least Linux omits the "const" in:
- *
- *   size_t malloc_usable_size(const void *ptr);
- *
- * Match the operating system's prototype.
- */
-#define JEMALLOC_USABLE_SIZE_CONST
-
-/*
- * If defined, specify throw() for the public function prototypes when compiling
- * with C++.  The only justification for this is to match the prototypes that
- * glibc defines.
- */
-#define JEMALLOC_USE_CXX_THROW
-
-#ifdef _MSC_VER
-#  ifdef _WIN64
-#    define LG_SIZEOF_PTR_WIN 3
-#  else
-#    define LG_SIZEOF_PTR_WIN 2
-#  endif
-#endif
-
-/* sizeof(void *) == 2^LG_SIZEOF_PTR. */
-#define LG_SIZEOF_PTR 3
diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_protos.h b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_protos.h
deleted file mode 100644
index ff025e30fa7..00000000000
--- a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_protos.h
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * The je_ prefix on the following public symbol declarations is an artifact
- * of namespace management, and should be omitted in application code unless
- * JEMALLOC_NO_DEMANGLE is defined (see jemalloc_mangle.h).
- */
-extern JEMALLOC_EXPORT const char	*je_malloc_conf;
-extern JEMALLOC_EXPORT void		(*je_malloc_message)(void *cbopaque,
-    const char *s);
-
-JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
-    void JEMALLOC_NOTHROW	*je_malloc(size_t size)
-    JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1);
-JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
-    void JEMALLOC_NOTHROW	*je_calloc(size_t num, size_t size)
-    JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE2(1, 2);
-JEMALLOC_EXPORT int JEMALLOC_NOTHROW	je_posix_memalign(void **memptr,
-    size_t alignment, size_t size) JEMALLOC_CXX_THROW JEMALLOC_ATTR(nonnull(1));
-JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
-    void JEMALLOC_NOTHROW	*je_aligned_alloc(size_t alignment,
-    size_t size) JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc)
-    JEMALLOC_ALLOC_SIZE(2);
-JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
-    void JEMALLOC_NOTHROW	*je_realloc(void *ptr, size_t size)
-    JEMALLOC_CXX_THROW JEMALLOC_ALLOC_SIZE(2);
-JEMALLOC_EXPORT void JEMALLOC_NOTHROW	je_free(void *ptr)
-    JEMALLOC_CXX_THROW;
-
-JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
-    void JEMALLOC_NOTHROW	*je_mallocx(size_t size, int flags)
-    JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1);
-JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
-    void JEMALLOC_NOTHROW	*je_rallocx(void *ptr, size_t size,
-    int flags) JEMALLOC_ALLOC_SIZE(2);
-JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW	je_xallocx(void *ptr, size_t size,
-    size_t extra, int flags);
-JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW	je_sallocx(const void *ptr,
-    int flags) JEMALLOC_ATTR(pure);
-JEMALLOC_EXPORT void JEMALLOC_NOTHROW	je_dallocx(void *ptr, int flags);
-JEMALLOC_EXPORT void JEMALLOC_NOTHROW	je_sdallocx(void *ptr, size_t size,
-    int flags);
-JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW	je_nallocx(size_t size, int flags)
-    JEMALLOC_ATTR(pure);
-
-JEMALLOC_EXPORT int JEMALLOC_NOTHROW	je_mallctl(const char *name,
-    void *oldp, size_t *oldlenp, void *newp, size_t newlen);
-JEMALLOC_EXPORT int JEMALLOC_NOTHROW	je_mallctlnametomib(const char *name,
-    size_t *mibp, size_t *miblenp);
-JEMALLOC_EXPORT int JEMALLOC_NOTHROW	je_mallctlbymib(const size_t *mib,
-    size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen);
-JEMALLOC_EXPORT void JEMALLOC_NOTHROW	je_malloc_stats_print(
-    void (*write_cb)(void *, const char *), void *je_cbopaque,
-    const char *opts);
-JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW	je_malloc_usable_size(
-    JEMALLOC_USABLE_SIZE_CONST void *ptr) JEMALLOC_CXX_THROW;
-
-#ifdef JEMALLOC_OVERRIDE_MEMALIGN
-JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
-    void JEMALLOC_NOTHROW	*je_memalign(size_t alignment, size_t size)
-    JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc);
-#endif
-
-#ifdef JEMALLOC_OVERRIDE_VALLOC
-JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
-    void JEMALLOC_NOTHROW	*je_valloc(size_t size) JEMALLOC_CXX_THROW
-    JEMALLOC_ATTR(malloc);
-#endif
diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_typedefs.h b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_typedefs.h
deleted file mode 100644
index 1a58874306e..00000000000
--- a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_typedefs.h
+++ /dev/null
@@ -1,77 +0,0 @@
-typedef struct extent_hooks_s extent_hooks_t;
-
-/*
- * void *
- * extent_alloc(extent_hooks_t *extent_hooks, void *new_addr, size_t size,
- *     size_t alignment, bool *zero, bool *commit, unsigned arena_ind);
- */
-typedef void *(extent_alloc_t)(extent_hooks_t *, void *, size_t, size_t, bool *,
-    bool *, unsigned);
-
-/*
- * bool
- * extent_dalloc(extent_hooks_t *extent_hooks, void *addr, size_t size,
- *     bool committed, unsigned arena_ind);
- */
-typedef bool (extent_dalloc_t)(extent_hooks_t *, void *, size_t, bool,
-    unsigned);
-
-/*
- * void
- * extent_destroy(extent_hooks_t *extent_hooks, void *addr, size_t size,
- *     bool committed, unsigned arena_ind);
- */
-typedef void (extent_destroy_t)(extent_hooks_t *, void *, size_t, bool,
-    unsigned);
-
-/*
- * bool
- * extent_commit(extent_hooks_t *extent_hooks, void *addr, size_t size,
- *     size_t offset, size_t length, unsigned arena_ind);
- */
-typedef bool (extent_commit_t)(extent_hooks_t *, void *, size_t, size_t, size_t,
-    unsigned);
-
-/*
- * bool
- * extent_decommit(extent_hooks_t *extent_hooks, void *addr, size_t size,
- *     size_t offset, size_t length, unsigned arena_ind);
- */
-typedef bool (extent_decommit_t)(extent_hooks_t *, void *, size_t, size_t,
-    size_t, unsigned);
-
-/*
- * bool
- * extent_purge(extent_hooks_t *extent_hooks, void *addr, size_t size,
- *     size_t offset, size_t length, unsigned arena_ind);
- */
-typedef bool (extent_purge_t)(extent_hooks_t *, void *, size_t, size_t, size_t,
-    unsigned);
-
-/*
- * bool
- * extent_split(extent_hooks_t *extent_hooks, void *addr, size_t size,
- *     size_t size_a, size_t size_b, bool committed, unsigned arena_ind);
- */
-typedef bool (extent_split_t)(extent_hooks_t *, void *, size_t, size_t, size_t,
-    bool, unsigned);
-
-/*
- * bool
- * extent_merge(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a,
- *     void *addr_b, size_t size_b, bool committed, unsigned arena_ind);
- */
-typedef bool (extent_merge_t)(extent_hooks_t *, void *, size_t, void *, size_t,
-    bool, unsigned);
-
-struct extent_hooks_s {
-	extent_alloc_t		*alloc;
-	extent_dalloc_t		*dalloc;
-	extent_destroy_t	*destroy;
-	extent_commit_t		*commit;
-	extent_decommit_t	*decommit;
-	extent_purge_t		*purge_lazy;
-	extent_purge_t		*purge_forced;
-	extent_split_t		*split;
-	extent_merge_t		*merge;
-};

From b53c6ddfd9a9960fc94486b85db1d1d07f86fbc7 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 24 May 2020 02:37:11 +0300
Subject: [PATCH 043/184] jemalloc: bump to 5.2.1

---
 contrib/jemalloc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/jemalloc b/contrib/jemalloc
index cd2931ad9bb..ea6b3e973b4 160000
--- a/contrib/jemalloc
+++ b/contrib/jemalloc
@@ -1 +1 @@
-Subproject commit cd2931ad9bbd78208565716ab102e86d858c2fff
+Subproject commit ea6b3e973b477b8061e0076bb257dbd7f3faa756

From bbb38664c10cbed66c82922595102ff28d8b3f71 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 24 May 2020 12:55:21 +0300
Subject: [PATCH 044/184] jemalloc: add missing safety_check.c

---
 contrib/jemalloc-cmake/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt
index 9081de593b8..e187ca2926d 100644
--- a/contrib/jemalloc-cmake/CMakeLists.txt
+++ b/contrib/jemalloc-cmake/CMakeLists.txt
@@ -55,6 +55,7 @@ if (ENABLE_JEMALLOC)
             ${LIBRARY_DIR}/src/ticker.c
             ${LIBRARY_DIR}/src/tsd.c
             ${LIBRARY_DIR}/src/witness.c
+            ${LIBRARY_DIR}/src/safety_check.c
         )
         if (OS_DARWIN)
             list(APPEND SRCS ${LIBRARY_DIR}/src/zone.c)

From b2ff1b3f20d15722e5097fd8c7622572eb809d01 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 24 May 2020 13:05:20 +0300
Subject: [PATCH 045/184] jemalloc: merge jemalloc_preamble.h for x86_64

---
 .../jemalloc/internal/jemalloc_preamble.h     | 23 +++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_preamble.h b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_preamble.h
index c150785fb4a..e5e34925b55 100644
--- a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_preamble.h
+++ b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_preamble.h
@@ -21,7 +21,7 @@
 #  include "jemalloc/jemalloc.h"
 #endif
 
-#if (defined(JEMALLOC_OSATOMIC) || defined(JEMALLOC_OSSPIN))
+#if defined(JEMALLOC_OSATOMIC)
 #include <libkern/OSAtomic.h>
 #endif
 
@@ -161,7 +161,26 @@ static const bool config_log =
     false
 #endif
     ;
-#ifdef JEMALLOC_HAVE_SCHED_GETCPU
+/*
+ * Are extra safety checks enabled; things like checking the size of sized
+ * deallocations, double-frees, etc.
+ */
+static const bool config_opt_safety_checks =
+#ifdef JEMALLOC_OPT_SAFETY_CHECKS
+    true
+#elif defined(JEMALLOC_DEBUG)
+    /*
+     * This lets us only guard safety checks by one flag instead of two; fast
+     * checks can guard solely by config_opt_safety_checks and run in debug mode
+     * too.
+     */
+    true
+#else
+    false
+#endif
+    ;
+
+#if defined(_WIN32) || defined(JEMALLOC_HAVE_SCHED_GETCPU)
 /* Currently percpu_arena depends on sched_getcpu. */
 #define JEMALLOC_PERCPU_ARENA
 #endif

From 2590ee64d902cf18be3ba5ad715803bfe94370b3 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 24 May 2020 12:55:55 +0300
Subject: [PATCH 046/184] jemalloc: merge jemalloc_preamble.h for aarch64

---
 .../jemalloc/internal/jemalloc_preamble.h     | 23 +++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_preamble.h b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_preamble.h
index c150785fb4a..e5e34925b55 100644
--- a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_preamble.h
+++ b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_preamble.h
@@ -21,7 +21,7 @@
 #  include "jemalloc/jemalloc.h"
 #endif
 
-#if (defined(JEMALLOC_OSATOMIC) || defined(JEMALLOC_OSSPIN))
+#if defined(JEMALLOC_OSATOMIC)
 #include <libkern/OSAtomic.h>
 #endif
 
@@ -161,7 +161,26 @@ static const bool config_log =
     false
 #endif
     ;
-#ifdef JEMALLOC_HAVE_SCHED_GETCPU
+/*
+ * Are extra safety checks enabled; things like checking the size of sized
+ * deallocations, double-frees, etc.
+ */
+static const bool config_opt_safety_checks =
+#ifdef JEMALLOC_OPT_SAFETY_CHECKS
+    true
+#elif defined(JEMALLOC_DEBUG)
+    /*
+     * This lets us only guard safety checks by one flag instead of two; fast
+     * checks can guard solely by config_opt_safety_checks and run in debug mode
+     * too.
+     */
+    true
+#else
+    false
+#endif
+    ;
+
+#if defined(_WIN32) || defined(JEMALLOC_HAVE_SCHED_GETCPU)
 /* Currently percpu_arena depends on sched_getcpu. */
 #define JEMALLOC_PERCPU_ARENA
 #endif

From 2f256b0d2de001d064efa5c755b5a10da4ddb013 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 24 May 2020 13:00:41 +0300
Subject: [PATCH 047/184] jemalloc: merge jemalloc_internal_defs.h.in for
 x86_64

- Use upstream JEMALLOC_PURGE_MADVISE_FREE
---
 .../internal/jemalloc_internal_defs.h.in      | 56 +++++++++----------
 1 file changed, 25 insertions(+), 31 deletions(-)

diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in
index 63f7f765023..71a93253441 100644
--- a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in
+++ b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in
@@ -1,11 +1,6 @@
 /* include/jemalloc/internal/jemalloc_internal_defs.h.  Generated from jemalloc_internal_defs.h.in by configure.  */
 #ifndef JEMALLOC_INTERNAL_DEFS_H_
 #define JEMALLOC_INTERNAL_DEFS_H_
-
-#ifndef _GNU_SOURCE
-    #define _GNU_SOURCE
-#endif
-
 /*
  * If JEMALLOC_PREFIX is defined via --with-jemalloc-prefix, it will cause all
  * public APIs to be prefixed.  This makes it possible, with some care, to use
@@ -24,7 +19,7 @@
 #define JEMALLOC_OVERRIDE___LIBC_MEMALIGN
 #define JEMALLOC_OVERRIDE___LIBC_REALLOC
 #define JEMALLOC_OVERRIDE___LIBC_VALLOC
-#define JEMALLOC_OVERRIDE___POSIX_MEMALIGN
+/* #undef JEMALLOC_OVERRIDE___POSIX_MEMALIGN */
 
 /*
  * JEMALLOC_PRIVATE_NAMESPACE is used as a prefix for all library-private APIs.
@@ -54,25 +49,13 @@
 
 /* Defined if GCC __atomic atomics are available. */
 #define JEMALLOC_GCC_ATOMIC_ATOMICS 1
+/* and the 8-bit variant support. */
+#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS 1
 
 /* Defined if GCC __sync atomics are available. */
 #define JEMALLOC_GCC_SYNC_ATOMICS 1
-
-/*
- * Defined if __sync_add_and_fetch(uint32_t *, uint32_t) and
- * __sync_sub_and_fetch(uint32_t *, uint32_t) are available, despite
- * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 not being defined (which means the
- * functions are defined in libgcc instead of being inlines).
- */
-/* #undef JE_FORCE_SYNC_COMPARE_AND_SWAP_4 */
-
-/*
- * Defined if __sync_add_and_fetch(uint64_t *, uint64_t) and
- * __sync_sub_and_fetch(uint64_t *, uint64_t) are available, despite
- * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 not being defined (which means the
- * functions are defined in libgcc instead of being inlines).
- */
-/* #undef JE_FORCE_SYNC_COMPARE_AND_SWAP_8 */
+/* and the 8-bit variant support. */
+#define JEMALLOC_GCC_U8_SYNC_ATOMICS 1
 
 /*
  * Defined if __builtin_clz() and __builtin_clzl() are available.
@@ -84,20 +67,13 @@
  */
 /* #undef JEMALLOC_OS_UNFAIR_LOCK */
 
-/*
- * Defined if OSSpin*() functions are available, as provided by Darwin, and
- * documented in the spinlock(3) manual page.
- */
-/* #undef JEMALLOC_OSSPIN */
-
 /* Defined if syscall(2) is usable. */
 #define JEMALLOC_USE_SYSCALL
 
 /*
  * Defined if secure_getenv(3) is available.
  */
-// Don't want dependency on newer GLIBC
-//#define JEMALLOC_HAVE_SECURE_GETENV
+#define JEMALLOC_HAVE_SECURE_GETENV
 
 /*
  * Defined if issetugid(2) is available.
@@ -160,6 +136,9 @@
 /* JEMALLOC_STATS enables statistics calculation. */
 #define JEMALLOC_STATS
 
+/* JEMALLOC_EXPERIMENTAL_SMALLOCX_API enables experimental smallocx API. */
+/* #undef JEMALLOC_EXPERIMENTAL_SMALLOCX_API */
+
 /* JEMALLOC_PROF enables allocation profiling. */
 /* #undef JEMALLOC_PROF */
 
@@ -240,6 +219,12 @@
 #define JEMALLOC_INTERNAL_FFSL __builtin_ffsl
 #define JEMALLOC_INTERNAL_FFS __builtin_ffs
 
+/*
+ * popcount*() functions to use for bitmapping.
+ */
+#define JEMALLOC_INTERNAL_POPCOUNTL __builtin_popcountl
+#define JEMALLOC_INTERNAL_POPCOUNT __builtin_popcount
+
 /*
  * If defined, explicitly attempt to more uniformly distribute large allocation
  * pointer alignments across all cache indices.
@@ -252,6 +237,12 @@
  */
 /* #undef JEMALLOC_LOG */
 
+/*
+ * If defined, use readlinkat() (instead of readlink()) to follow
+ * /etc/malloc_conf.
+ */
+/* #undef JEMALLOC_READLINKAT */
+
 /*
  * Darwin (OS X) uses zones to work around Mach-O symbol override shortcomings.
  */
@@ -288,7 +279,7 @@
  *                                 MADV_FREE, though typically with higher
  *                                 system overhead.
  */
-//#define JEMALLOC_PURGE_MADVISE_FREE
+#define JEMALLOC_PURGE_MADVISE_FREE
 #define JEMALLOC_PURGE_MADVISE_DONTNEED
 #define JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS
 
@@ -370,4 +361,7 @@
  */
 #define JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE
 
+/* Performs additional safety checks when defined. */
+/* #undef JEMALLOC_OPT_SAFETY_CHECKS */
+
 #endif /* JEMALLOC_INTERNAL_DEFS_H_ */

From 82f2b1a986965fc850188460f4873566101692a0 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 24 May 2020 13:02:23 +0300
Subject: [PATCH 048/184] jemalloc: merge jemalloc_internal_defs.h.in for
 aarch64

- keep LG_PAGE/LG_HUGEPAGE
- use upstream JEMALLOC_PURGE_MADVISE_FREE
---
 .../internal/jemalloc_internal_defs.h.in      | 49 +++++++------------
 1 file changed, 17 insertions(+), 32 deletions(-)

diff --git a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in
index 089f1a773aa..231d80ceb5c 100644
--- a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in
+++ b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in
@@ -1,12 +1,6 @@
 /* include/jemalloc/internal/jemalloc_internal_defs.h.  Generated from jemalloc_internal_defs.h.in by configure.  */
 #ifndef JEMALLOC_INTERNAL_DEFS_H_
 #define JEMALLOC_INTERNAL_DEFS_H_
-
-
-#ifndef _GNU_SOURCE
-    #define _GNU_SOURCE
-#endif
-
 /*
  * If JEMALLOC_PREFIX is defined via --with-jemalloc-prefix, it will cause all
  * public APIs to be prefixed.  This makes it possible, with some care, to use
@@ -25,7 +19,7 @@
 #define JEMALLOC_OVERRIDE___LIBC_MEMALIGN
 #define JEMALLOC_OVERRIDE___LIBC_REALLOC
 #define JEMALLOC_OVERRIDE___LIBC_VALLOC
-#define JEMALLOC_OVERRIDE___POSIX_MEMALIGN
+/* #undef JEMALLOC_OVERRIDE___POSIX_MEMALIGN */
 
 /*
  * JEMALLOC_PRIVATE_NAMESPACE is used as a prefix for all library-private APIs.
@@ -39,9 +33,9 @@
  * Hyper-threaded CPUs may need a special instruction inside spin loops in
  * order to yield to another virtual CPU.
  */
-#define CPU_SPINWAIT
+#define CPU_SPINWAIT __asm__ volatile("pause")
 /* 1 if CPU_SPINWAIT is defined, 0 otherwise. */
-#define HAVE_CPU_SPINWAIT 0
+#define HAVE_CPU_SPINWAIT 1
 
 /*
  * Number of significant bits in virtual addresses.  This may be less than the
@@ -55,25 +49,13 @@
 
 /* Defined if GCC __atomic atomics are available. */
 #define JEMALLOC_GCC_ATOMIC_ATOMICS 1
+/* and the 8-bit variant support. */
+#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS 1
 
 /* Defined if GCC __sync atomics are available. */
 #define JEMALLOC_GCC_SYNC_ATOMICS 1
-
-/*
- * Defined if __sync_add_and_fetch(uint32_t *, uint32_t) and
- * __sync_sub_and_fetch(uint32_t *, uint32_t) are available, despite
- * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 not being defined (which means the
- * functions are defined in libgcc instead of being inlines).
- */
-/* #undef JE_FORCE_SYNC_COMPARE_AND_SWAP_4 */
-
-/*
- * Defined if __sync_add_and_fetch(uint64_t *, uint64_t) and
- * __sync_sub_and_fetch(uint64_t *, uint64_t) are available, despite
- * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 not being defined (which means the
- * functions are defined in libgcc instead of being inlines).
- */
-/* #undef JE_FORCE_SYNC_COMPARE_AND_SWAP_8 */
+/* and the 8-bit variant support. */
+#define JEMALLOC_GCC_U8_SYNC_ATOMICS 1
 
 /*
  * Defined if __builtin_clz() and __builtin_clzl() are available.
@@ -85,12 +67,6 @@
  */
 /* #undef JEMALLOC_OS_UNFAIR_LOCK */
 
-/*
- * Defined if OSSpin*() functions are available, as provided by Darwin, and
- * documented in the spinlock(3) manual page.
- */
-/* #undef JEMALLOC_OSSPIN */
-
 /* Defined if syscall(2) is usable. */
 #define JEMALLOC_USE_SYSCALL
 
@@ -243,6 +219,12 @@
 #define JEMALLOC_INTERNAL_FFSL __builtin_ffsl
 #define JEMALLOC_INTERNAL_FFS __builtin_ffs
 
+/*
+ * popcount*() functions to use for bitmapping.
+ */
+#define JEMALLOC_INTERNAL_POPCOUNTL __builtin_popcountl
+#define JEMALLOC_INTERNAL_POPCOUNT __builtin_popcount
+
 /*
  * If defined, explicitly attempt to more uniformly distribute large allocation
  * pointer alignments across all cache indices.
@@ -297,7 +279,7 @@
  *                                 MADV_FREE, though typically with higher
  *                                 system overhead.
  */
-// #define JEMALLOC_PURGE_MADVISE_FREE
+#define JEMALLOC_PURGE_MADVISE_FREE
 #define JEMALLOC_PURGE_MADVISE_DONTNEED
 #define JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS
 
@@ -379,4 +361,7 @@
  */
 #define JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE
 
+/* Performs additional safety checks when defined. */
+/* #undef JEMALLOC_OPT_SAFETY_CHECKS */
+
 #endif /* JEMALLOC_INTERNAL_DEFS_H_ */

From 9f1cefd8dfea9e28b1d1b2a8ae7eb64dedb8ceaf Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 24 May 2020 13:09:10 +0300
Subject: [PATCH 049/184] jemalloc: disable secure_getenv()

---
 .../jemalloc/internal/jemalloc_internal_defs.h.in               | 2 +-
 .../jemalloc/internal/jemalloc_internal_defs.h.in               | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in
index 231d80ceb5c..3852f8b6382 100644
--- a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in
+++ b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in
@@ -73,7 +73,7 @@
 /*
  * Defined if secure_getenv(3) is available.
  */
-#define JEMALLOC_HAVE_SECURE_GETENV
+// #define JEMALLOC_HAVE_SECURE_GETENV
 
 /*
  * Defined if issetugid(2) is available.
diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in
index 71a93253441..d5cf0e719ef 100644
--- a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in
+++ b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in
@@ -73,7 +73,7 @@
 /*
  * Defined if secure_getenv(3) is available.
  */
-#define JEMALLOC_HAVE_SECURE_GETENV
+// #define JEMALLOC_HAVE_SECURE_GETENV
 
 /*
  * Defined if issetugid(2) is available.

From 5cd31dbf0deda64dadcf6a3f6b8f5ad33eb9726a Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 24 May 2020 13:12:00 +0300
Subject: [PATCH 050/184] jemalloc: add _GNU_SOURCE for RTLD_NEXT

---
 contrib/jemalloc-cmake/CMakeLists.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt
index e187ca2926d..e5f52f4dc8e 100644
--- a/contrib/jemalloc-cmake/CMakeLists.txt
+++ b/contrib/jemalloc-cmake/CMakeLists.txt
@@ -90,6 +90,8 @@ if (ENABLE_JEMALLOC)
         endif ()
 
         target_compile_options(jemalloc PRIVATE -Wno-redundant-decls)
+        # for RTLD_NEXT
+        target_compile_options(jemalloc PRIVATE -D_GNU_SOURCE)
     else ()
         find_library(LIBRARY_JEMALLOC jemalloc)
         find_path(INCLUDE_JEMALLOC jemalloc/jemalloc.h)

From bc572dc5f22b139a60a9a0dc188f2e38ed0ba9f9 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 24 May 2020 13:20:07 +0300
Subject: [PATCH 051/184] jemalloc: sync common (non per-arch) pre-generated
 headers

$ for f in include/jemalloc/jemalloc_*.h; do cp /src/oss/jemalloc/$f $f; done
---
 .../include/jemalloc/jemalloc_defs.h            |  6 ++++++
 .../include/jemalloc/jemalloc_macros.h          | 17 ++++++++++++-----
 .../include/jemalloc/jemalloc_rename.h          |  1 +
 3 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_defs.h b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_defs.h
index d1389237a77..6a03a231a0e 100644
--- a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_defs.h
+++ b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_defs.h
@@ -5,6 +5,12 @@
 /* Defined if alloc_size attribute is supported. */
 #define JEMALLOC_HAVE_ATTR_ALLOC_SIZE
 
+/* Defined if format_arg(...) attribute is supported. */
+#define JEMALLOC_HAVE_ATTR_FORMAT_ARG
+
+/* Defined if format(gnu_printf, ...) attribute is supported. */
+#define JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF
+
 /* Defined if format(printf, ...) attribute is supported. */
 #define JEMALLOC_HAVE_ATTR_FORMAT_PRINTF
 
diff --git a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_macros.h b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_macros.h
index 7432f1cda53..34235894285 100644
--- a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_macros.h
+++ b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_macros.h
@@ -4,12 +4,13 @@
 #include <limits.h>
 #include <strings.h>
 
-#define JEMALLOC_VERSION "5.1.0-56-g41b7372eadee941b9164751b8d4963f915d3ceae"
+#define JEMALLOC_VERSION "5.2.1-0-gea6b3e973b477b8061e0076bb257dbd7f3faa756"
 #define JEMALLOC_VERSION_MAJOR 5
-#define JEMALLOC_VERSION_MINOR 1
-#define JEMALLOC_VERSION_BUGFIX 0
-#define JEMALLOC_VERSION_NREV 56
-#define JEMALLOC_VERSION_GID "41b7372eadee941b9164751b8d4963f915d3ceae"
+#define JEMALLOC_VERSION_MINOR 2
+#define JEMALLOC_VERSION_BUGFIX 1
+#define JEMALLOC_VERSION_NREV 0
+#define JEMALLOC_VERSION_GID "ea6b3e973b477b8061e0076bb257dbd7f3faa756"
+#define JEMALLOC_VERSION_GID_IDENT ea6b3e973b477b8061e0076bb257dbd7f3faa756
 
 #define MALLOCX_LG_ALIGN(la)	((int)(la))
 #if LG_SIZEOF_PTR == 2
@@ -68,6 +69,7 @@
 #      define JEMALLOC_EXPORT __declspec(dllimport)
 #    endif
 #  endif
+#  define JEMALLOC_FORMAT_ARG(i)
 #  define JEMALLOC_FORMAT_PRINTF(s, i)
 #  define JEMALLOC_NOINLINE __declspec(noinline)
 #  ifdef __cplusplus
@@ -95,6 +97,11 @@
 #  ifndef JEMALLOC_EXPORT
 #    define JEMALLOC_EXPORT JEMALLOC_ATTR(visibility("default"))
 #  endif
+#  ifdef JEMALLOC_HAVE_ATTR_FORMAT_ARG
+#    define JEMALLOC_FORMAT_ARG(i) JEMALLOC_ATTR(__format_arg__(3))
+#  else
+#    define JEMALLOC_FORMAT_ARG(i)
+#  endif
 #  ifdef JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF
 #    define JEMALLOC_FORMAT_PRINTF(s, i) JEMALLOC_ATTR(format(gnu_printf, s, i))
 #  elif defined(JEMALLOC_HAVE_ATTR_FORMAT_PRINTF)
diff --git a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_rename.h b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_rename.h
index a2ea2dd3533..2e94f7a0cc3 100644
--- a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_rename.h
+++ b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_rename.h
@@ -17,6 +17,7 @@
 #  define je_malloc_stats_print malloc_stats_print
 #  define je_malloc_usable_size malloc_usable_size
 #  define je_mallocx mallocx
+#  define je_smallocx_ea6b3e973b477b8061e0076bb257dbd7f3faa756 smallocx_ea6b3e973b477b8061e0076bb257dbd7f3faa756
 #  define je_nallocx nallocx
 #  define je_posix_memalign posix_memalign
 #  define je_rallocx rallocx

From a8749c5d5c668a079239112f96a1fafe072a9ad5 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 24 May 2020 19:17:50 +0300
Subject: [PATCH 052/184] jemalloc: aarch64 does not have pause

---
 .../jemalloc/internal/jemalloc_internal_defs.h.in             | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in
index 3852f8b6382..cbd2740e1f1 100644
--- a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in
+++ b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in
@@ -33,9 +33,9 @@
  * Hyper-threaded CPUs may need a special instruction inside spin loops in
  * order to yield to another virtual CPU.
  */
-#define CPU_SPINWAIT __asm__ volatile("pause")
+#define CPU_SPINWAIT
 /* 1 if CPU_SPINWAIT is defined, 0 otherwise. */
-#define HAVE_CPU_SPINWAIT 1
+#define HAVE_CPU_SPINWAIT 9
 
 /*
  * Number of significant bits in virtual addresses.  This may be less than the

From b694d6ed8c78eacb248a0cd20255e81433f88d27 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Tue, 26 May 2020 21:45:51 +0300
Subject: [PATCH 053/184] jemalloc: disable oversize_threshold (slower)

---
 contrib/jemalloc-cmake/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt
index e5f52f4dc8e..a02a63ba999 100644
--- a/contrib/jemalloc-cmake/CMakeLists.txt
+++ b/contrib/jemalloc-cmake/CMakeLists.txt
@@ -17,7 +17,7 @@ if (ENABLE_JEMALLOC)
         #
         # By enabling percpu_arena number of arenas limited to number of CPUs and hence
         # this problem should go away.
-        set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu" CACHE STRING "Change default configuration string of JEMalloc" )
+        set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0" CACHE STRING "Change default configuration string of JEMalloc" )
         message (STATUS "jemalloc malloc_conf: ${JEMALLOC_CONFIG_MALLOC_CONF}")
 
         set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/jemalloc")

From 210e67d8317252252402aeba6ef6d05051f7fc7e Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Tue, 26 May 2020 21:51:06 +0300
Subject: [PATCH 054/184] jemalloc: allow changing default malloc_conf without
 purging cmake cache

---
 contrib/jemalloc-cmake/CMakeLists.txt | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt
index a02a63ba999..63c5a5e66ea 100644
--- a/contrib/jemalloc-cmake/CMakeLists.txt
+++ b/contrib/jemalloc-cmake/CMakeLists.txt
@@ -17,7 +17,13 @@ if (ENABLE_JEMALLOC)
         #
         # By enabling percpu_arena number of arenas limited to number of CPUs and hence
         # this problem should go away.
-        set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0" CACHE STRING "Change default configuration string of JEMalloc" )
+        set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0")
+        # CACHE variable is empty, to allow changing defaults without necessity
+        # to purge cache
+        set (JEMALLOC_CONFIG_MALLOC_CONF_OVERRIDE "" CACHE STRING "Change default configuration string of JEMalloc" )
+        if (JEMALLOC_CONFIG_MALLOC_CONF_OVERRIDE)
+            set (JEMALLOC_CONFIG_MALLOC_CONF "${JEMALLOC_CONFIG_MALLOC_CONF_OVERRIDE}")
+        endif()
         message (STATUS "jemalloc malloc_conf: ${JEMALLOC_CONFIG_MALLOC_CONF}")
 
         set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/jemalloc")

From ec894e7cb3e0e3f9f7c880b00d44ee3b83e028ad Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 24 May 2020 19:39:31 +0300
Subject: [PATCH 055/184] Check for POPCNT instruction

---
 programs/main.cpp                             | 19 ++++++++++++++++---
 ...heck_cpu_instructions_at_startup.reference |  1 +
 ...01103_check_cpu_instructions_at_startup.sh | 13 +++++++++----
 3 files changed, 26 insertions(+), 7 deletions(-)

diff --git a/programs/main.cpp b/programs/main.cpp
index 2eb226d3c00..b1bff1de1b0 100644
--- a/programs/main.cpp
+++ b/programs/main.cpp
@@ -127,9 +127,10 @@ enum class InstructionFail
     SSSE3 = 2,
     SSE4_1 = 3,
     SSE4_2 = 4,
-    AVX = 5,
-    AVX2 = 6,
-    AVX512 = 7
+    POPCNT = 5,
+    AVX = 6,
+    AVX2 = 7,
+    AVX512 = 8
 };
 
 const char * instructionFailToString(InstructionFail fail)
@@ -146,6 +147,8 @@ const char * instructionFailToString(InstructionFail fail)
             return "SSE4.1";
         case InstructionFail::SSE4_2:
             return "SSE4.2";
+        case InstructionFail::POPCNT:
+            return "POPCNT";
         case InstructionFail::AVX:
             return "AVX";
         case InstructionFail::AVX2:
@@ -189,6 +192,16 @@ void checkRequiredInstructionsImpl(volatile InstructionFail & fail)
     __asm__ volatile ("pcmpgtq %%xmm0, %%xmm0" : : : "xmm0");
 #endif
 
+    /// Defined by -msse4.2
+#if defined(__POPCNT__)
+    fail = InstructionFail::POPCNT;
+    {
+        uint64_t a = 0;
+        uint64_t b = 0;
+        __asm__ volatile ("popcnt %1, %0" : "=r"(a) :"r"(b) : );
+    }
+#endif
+
 #if defined(__AVX__)
     fail = InstructionFail::AVX;
     __asm__ volatile ("vaddpd %%ymm0, %%ymm0, %%ymm0" : : : "ymm0");
diff --git a/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.reference b/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.reference
index 59c88f013dd..03ed07cf1a4 100644
--- a/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.reference
+++ b/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.reference
@@ -1,4 +1,5 @@
 Instruction check fail. The CPU does not support SSSE3 instruction set.
 Instruction check fail. The CPU does not support SSE4.1 instruction set.
 Instruction check fail. The CPU does not support SSE4.2 instruction set.
+Instruction check fail. The CPU does not support POPCNT instruction set.
 1
diff --git a/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.sh b/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.sh
index 15466e1889a..5ae4f8b3dd2 100755
--- a/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.sh
+++ b/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.sh
@@ -9,8 +9,13 @@ ${CLICKHOUSE_LOCAL} --query "SELECT max(value LIKE '%sanitize%') FROM system.bui
 
 command=$(command -v ${CLICKHOUSE_LOCAL})
 
-qemu-x86_64-static -cpu qemu64                        $command --query "SELECT 1" 2>&1 | grep -v -F "warning: TCG doesn't support requested feature" ||:
-qemu-x86_64-static -cpu qemu64,+ssse3                 $command --query "SELECT 1" 2>&1 | grep -v -F "warning: TCG doesn't support requested feature" ||:
-qemu-x86_64-static -cpu qemu64,+ssse3,+sse4.1         $command --query "SELECT 1" 2>&1 | grep -v -F "warning: TCG doesn't support requested feature" ||:
-qemu-x86_64-static -cpu qemu64,+ssse3,+sse4.1,+sse4.2 $command --query "SELECT 1" 2>&1 | grep -v -F "warning: TCG doesn't support requested feature" ||:
+function run_with_cpu()
+{
+    qemu-x86_64-static -cpu "$@" $command --query "SELECT 1" 2>&1 | grep -v -F "warning: TCG doesn't support requested feature" ||:
+}
 
+run_with_cpu qemu64
+run_with_cpu qemu64,+ssse3
+run_with_cpu qemu64,+ssse3,+sse4.1
+run_with_cpu qemu64,+ssse3,+sse4.1,+sse4.2
+run_with_cpu qemu64,+ssse3,+sse4.1,+sse4.2,+popcnt

From efc0b977078af8a80a2afd6a2849efeb78148201 Mon Sep 17 00:00:00 2001
From: Mikhail Filimonov <mfilimonov@altinity.com>
Date: Fri, 22 May 2020 18:58:08 +0200
Subject: [PATCH 056/184] Fix Kafka performance issue related to reschedules
 based on limits, which were always applied.

---
 src/Storages/Kafka/KafkaBlockInputStream.h    |  1 +
 .../Kafka/ReadBufferFromKafkaConsumer.h       |  1 +
 src/Storages/Kafka/StorageKafka.cpp           | 31 ++++++++++----
 tests/integration/test_storage_kafka/test.py  | 40 ++++++++++++++++++-
 4 files changed, 64 insertions(+), 9 deletions(-)

diff --git a/src/Storages/Kafka/KafkaBlockInputStream.h b/src/Storages/Kafka/KafkaBlockInputStream.h
index 1f94ee332d3..e3052122894 100644
--- a/src/Storages/Kafka/KafkaBlockInputStream.h
+++ b/src/Storages/Kafka/KafkaBlockInputStream.h
@@ -25,6 +25,7 @@ public:
     void readSuffixImpl() override;
 
     void commit();
+    bool isStalled() const { return buffer->isStalled(); }
 
 private:
     StorageKafka & storage;
diff --git a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h
index 12da701a55d..e90e3b48881 100644
--- a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h
+++ b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h
@@ -38,6 +38,7 @@ public:
 
     bool hasMorePolledMessages() const;
     bool polledDataUnusable() const { return (was_stopped || rebalance_happened); }
+    bool isStalled() const { return stalled; }
 
     void storeLastReadMessageOffset();
     void resetToLastCommitted(const char * msg);
diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp
index d1dfbe9d875..7731cf3c06a 100644
--- a/src/Storages/Kafka/StorageKafka.cpp
+++ b/src/Storages/Kafka/StorageKafka.cpp
@@ -51,6 +51,7 @@ namespace
 {
     const auto RESCHEDULE_MS = 500;
     const auto CLEANUP_TIMEOUT_MS = 3000;
+    const auto MAX_THREAD_WORK_DURATION_MS = 60000;  // once per minute leave do reschedule (we can't lock threads in pool forever)
 
     /// Configuration prefix
     const String CONFIG_PREFIX = "kafka";
@@ -386,6 +387,8 @@ void StorageKafka::threadFunc()
         size_t dependencies_count = DatabaseCatalog::instance().getDependencies(table_id).size();
         if (dependencies_count)
         {
+            auto start_time = std::chrono::steady_clock::now();
+
             // Keep streaming as long as there are attached views and streaming is not cancelled
             while (!stream_cancelled && num_created_consumers > 0)
             {
@@ -394,9 +397,21 @@ void StorageKafka::threadFunc()
 
                 LOG_DEBUG(log, "Started streaming to {} attached views", dependencies_count);
 
-                // Reschedule if not limited
-                if (!streamToViews())
+                // Exit the loop & reschedule if some stream stalled
+                auto some_stream_is_stalled = streamToViews();
+                if (some_stream_is_stalled)
+                {
+                    LOG_TRACE(log, "Stream(s) stalled. Reschedule.");
                     break;
+                }
+
+                auto ts = std::chrono::steady_clock::now();
+                auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(ts-start_time);
+                if (duration.count() > MAX_THREAD_WORK_DURATION_MS)
+                {
+                    LOG_TRACE(log, "Thread work duration limit exceeded. Reschedule.");
+                    break;
+                }
             }
         }
     }
@@ -459,15 +474,15 @@ bool StorageKafka::streamToViews()
     // It will be cancelled on underlying layer (kafka buffer)
     std::atomic<bool> stub = {false};
     copyData(*in, *block_io.out, &stub);
+
+    bool some_stream_is_stalled = false;
     for (auto & stream : streams)
+    {
+        some_stream_is_stalled = some_stream_is_stalled || stream->as<KafkaBlockInputStream>()->isStalled();
         stream->as<KafkaBlockInputStream>()->commit();
+    }
 
-    // Check whether the limits were applied during query execution
-    bool limits_applied = false;
-    const BlockStreamProfileInfo & info = in->getProfileInfo();
-    limits_applied = info.hasAppliedLimit();
-
-    return limits_applied;
+    return some_stream_is_stalled;
 }
 
 void registerStorageKafka(StorageFactory & factory)
diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py
index 8e329a1cb60..d71347e83f1 100644
--- a/tests/integration/test_storage_kafka/test.py
+++ b/tests/integration/test_storage_kafka/test.py
@@ -1017,7 +1017,10 @@ def test_kafka_flush_by_block_size(kafka_cluster):
 
     time.sleep(1)
 
-    result = instance.query('SELECT count() FROM test.view')
+    # TODO: due to https://github.com/ClickHouse/ClickHouse/pull/11149
+    # second flush happens earlier than expected, so we have 2 parts here instead of one
+    # flush by block size works correctly, so the feature checked by the test is working correctly
+    result = instance.query("SELECT count() FROM test.view WHERE _part='all_1_1_0'")
     # print(result)
 
     # kafka_cluster.open_bash_shell('instance')
@@ -1390,6 +1393,41 @@ def test_commits_of_unprocessed_messages_on_drop(kafka_cluster):
     assert TSV(result) == TSV('{0}\t{0}\t{0}'.format(i[0]-1)), 'Missing data!'
 
 
+
+@pytest.mark.timeout(120)
+def test_bad_reschedule(kafka_cluster):
+    messages = [json.dumps({'key': j+1, 'value': j+1}) for j in range(20000)]
+    kafka_produce('test_bad_reschedule', messages)
+
+    instance.query('''
+        CREATE TABLE test.kafka (key UInt64, value UInt64)
+            ENGINE = Kafka
+            SETTINGS kafka_broker_list = 'kafka1:19092',
+                    kafka_topic_list = 'test_bad_reschedule',
+                    kafka_group_name = 'test_bad_reschedule',
+                    kafka_format = 'JSONEachRow',
+                    kafka_max_block_size = 1000;
+
+        CREATE MATERIALIZED VIEW test.destination Engine=Log AS
+        SELECT
+            key,
+            now() as consume_ts,
+            value,
+            _topic,
+            _key,
+            _offset,
+            _partition,
+            _timestamp
+        FROM test.kafka;
+    ''')
+
+    while int(instance.query("SELECT count() FROM test.destination")) < 20000:
+        print("Waiting for consume")
+        time.sleep(1)
+
+    assert int(instance.query("SELECT max(consume_ts) - min(consume_ts) FROM test.destination")) < 8
+
+
 @pytest.mark.timeout(1200)
 def test_kafka_duplicates_when_commit_failed(kafka_cluster):
     messages = [json.dumps({'key': j+1, 'value': 'x' * 300}) for j in range(22)]

From c73b8372e786ba6de958e8be222bf20081c97afd Mon Sep 17 00:00:00 2001
From: filimonov <1549571+filimonov@users.noreply.github.com>
Date: Wed, 27 May 2020 07:44:13 +0200
Subject: [PATCH 057/184] Update test.py

---
 tests/integration/test_storage_kafka/test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py
index d71347e83f1..13577864870 100644
--- a/tests/integration/test_storage_kafka/test.py
+++ b/tests/integration/test_storage_kafka/test.py
@@ -1017,7 +1017,7 @@ def test_kafka_flush_by_block_size(kafka_cluster):
 
     time.sleep(1)
 
-    # TODO: due to https://github.com/ClickHouse/ClickHouse/pull/11149
+    # TODO: due to https://github.com/ClickHouse/ClickHouse/issues/11216
     # second flush happens earlier than expected, so we have 2 parts here instead of one
     # flush by block size works correctly, so the feature checked by the test is working correctly
     result = instance.query("SELECT count() FROM test.view WHERE _part='all_1_1_0'")

From cd008f9b7fb427895ab3bcaffd163aa9746de166 Mon Sep 17 00:00:00 2001
From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com>
Date: Wed, 27 May 2020 10:58:29 +0300
Subject: [PATCH 058/184] Update functions_with_hash_tables.xml

---
 tests/performance/functions_with_hash_tables.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/performance/functions_with_hash_tables.xml b/tests/performance/functions_with_hash_tables.xml
index 72570a5925c..2e9a88107bd 100644
--- a/tests/performance/functions_with_hash_tables.xml
+++ b/tests/performance/functions_with_hash_tables.xml
@@ -1,6 +1,6 @@
 <test>
     <query>select arrayUniq(range(1 + (number % 100) * 10)) from numbers(100000) format Null</query>
-    <query>select arrayDistinct(range(1 + (number % 100) * 10)) from numbers(10000) format Null</query>
+    <query>select arrayDistinct(range(1 + (number % 100) * 10)) from numbers(100000) format Null</query>
     <query>select arrayEnumerateUniq(range(1 + (number % 100) * 10)) from numbers(100000) format Null</query>
     <query>select arrayIntersect(range((1 + number % 100)), range(1, (1 + number % 100) + 1)) from numbers(100000) format Null</query>
     <query>select groupUniqArray(rand() % 100) from numbers(1000 * 1000) group by number / 1000 format Null</query>

From d941c483b2923c378536631fc479c06f557a7e44 Mon Sep 17 00:00:00 2001
From: Ivan Blinkov <github@blinkov.ru>
Date: Wed, 27 May 2020 11:25:36 +0300
Subject: [PATCH 059/184] Update test.py

---
 tests/integration/test_system_queries/test.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/integration/test_system_queries/test.py b/tests/integration/test_system_queries/test.py
index 9397cd20ec7..6f36a13b184 100644
--- a/tests/integration/test_system_queries/test.py
+++ b/tests/integration/test_system_queries/test.py
@@ -97,6 +97,7 @@ def test_SYSTEM_FLUSH_LOGS(started_cluster):
     instance = cluster.instances['ch1']
     instance.query('''
         SET log_queries = 0;
+        SYSTEM FLUSH LOGS;
         TRUNCATE TABLE system.query_log;
     ''')
     for i in range(4):
@@ -104,13 +105,13 @@ def test_SYSTEM_FLUSH_LOGS(started_cluster):
         # by expiration of flush_interval_millisecond and test probable race condition.
         time.sleep(0.5)
         result = instance.query('''
-            SET log_queries = 1;
             SELECT 1 FORMAT Null;
             SET log_queries = 0;
             SYSTEM FLUSH LOGS;
             SELECT count() FROM system.query_log;''')
         instance.query('''
             SET log_queries = 0;
+            SYSTEM FLUSH LOGS;
             TRUNCATE TABLE system.query_log;
         ''')
         assert TSV(result) == TSV('4')

From f29d3b8cc916d9ff750186f89e4932b076481193 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Wed, 27 May 2020 11:58:11 +0300
Subject: [PATCH 060/184] Fix cmake one more time

---
 contrib/cctz-cmake/CMakeLists.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/contrib/cctz-cmake/CMakeLists.txt b/contrib/cctz-cmake/CMakeLists.txt
index d353fc9d4de..76a1fe6e8ac 100644
--- a/contrib/cctz-cmake/CMakeLists.txt
+++ b/contrib/cctz-cmake/CMakeLists.txt
@@ -622,7 +622,8 @@ if (USE_INTERNAL_CCTZ)
         # CMake can shuffle each of target_link_libraries arguments with other
         # libraries in linker command. To avoid this we hardcode whole-archive
         # library into single string.
-        target_link_libraries(cctz PUBLIC "-Wl,--whole-archive contrib/cctz-cmake/libtzdata.a -Wl,--no-whole-archive")
+        add_dependencies(cctz tzdata)
+        target_link_libraries(cctz PRIVATE "-Wl,--whole-archive contrib/cctz-cmake/libtzdata.a -Wl,--no-whole-archive")
     endif ()
 
 else ()

From f452c50d86f897e40bc167abf56674d41b77c7c5 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Wed, 27 May 2020 12:00:33 +0300
Subject: [PATCH 061/184] Replace private with public

---
 contrib/cctz-cmake/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/cctz-cmake/CMakeLists.txt b/contrib/cctz-cmake/CMakeLists.txt
index 76a1fe6e8ac..707ebe30a13 100644
--- a/contrib/cctz-cmake/CMakeLists.txt
+++ b/contrib/cctz-cmake/CMakeLists.txt
@@ -623,7 +623,7 @@ if (USE_INTERNAL_CCTZ)
         # libraries in linker command. To avoid this we hardcode whole-archive
         # library into single string.
         add_dependencies(cctz tzdata)
-        target_link_libraries(cctz PRIVATE "-Wl,--whole-archive contrib/cctz-cmake/libtzdata.a -Wl,--no-whole-archive")
+        target_link_libraries(cctz PUBLIC "-Wl,--whole-archive contrib/cctz-cmake/libtzdata.a -Wl,--no-whole-archive")
     endif ()
 
 else ()

From d04962808e1f0c4fe53db55aab765741f8b6b945 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Wed, 27 May 2020 12:14:40 +0300
Subject: [PATCH 062/184] Actually it's INTERFACE linking

---
 contrib/cctz-cmake/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/cctz-cmake/CMakeLists.txt b/contrib/cctz-cmake/CMakeLists.txt
index 707ebe30a13..2c44b25b17a 100644
--- a/contrib/cctz-cmake/CMakeLists.txt
+++ b/contrib/cctz-cmake/CMakeLists.txt
@@ -623,7 +623,7 @@ if (USE_INTERNAL_CCTZ)
         # libraries in linker command. To avoid this we hardcode whole-archive
         # library into single string.
         add_dependencies(cctz tzdata)
-        target_link_libraries(cctz PUBLIC "-Wl,--whole-archive contrib/cctz-cmake/libtzdata.a -Wl,--no-whole-archive")
+        target_link_libraries(cctz INTERFACE "-Wl,--whole-archive contrib/cctz-cmake/libtzdata.a -Wl,--no-whole-archive")
     endif ()
 
 else ()

From 61974e0047e9826ab84d5d4b07f93ec065b285dc Mon Sep 17 00:00:00 2001
From: Nikolai Sorokin <sonikint@gmail.com>
Date: Mon, 27 Apr 2020 17:47:59 +0300
Subject: [PATCH 063/184] TTL Expressions WHERE and GROUP BY draft

---
 src/DataStreams/TTLBlockInputStream.cpp  | 225 +++++++++++++++++++++--
 src/DataStreams/TTLBlockInputStream.h    |   8 +
 src/Interpreters/ExpressionAnalyzer.cpp  |   9 +-
 src/Interpreters/SyntaxAnalyzer.cpp      |  17 +-
 src/Interpreters/SyntaxAnalyzer.h        |   2 +-
 src/Parsers/ASTTTLElement.cpp            |  67 ++++++-
 src/Parsers/ASTTTLElement.h              |  38 +++-
 src/Parsers/ExpressionElementParsers.cpp |  84 ++++++++-
 src/Parsers/ExpressionListParsers.cpp    |   9 +
 src/Parsers/ExpressionListParsers.h      |  11 +-
 src/Storages/MergeTree/MergeTreeData.cpp |  44 ++++-
 src/Storages/MergeTree/MergeTreeData.h   |   9 +
 12 files changed, 479 insertions(+), 44 deletions(-)

diff --git a/src/DataStreams/TTLBlockInputStream.cpp b/src/DataStreams/TTLBlockInputStream.cpp
index d2d5d6a92f9..d816cba4b2f 100644
--- a/src/DataStreams/TTLBlockInputStream.cpp
+++ b/src/DataStreams/TTLBlockInputStream.cpp
@@ -67,6 +67,30 @@ TTLBlockInputStream::TTLBlockInputStream(
             default_expr_list, storage.getColumns().getAllPhysical());
         defaults_expression = ExpressionAnalyzer{default_expr_list, syntax_result, storage.global_context}.getActions(true);
     }
+
+    if (storage.hasRowsTTL() && !storage.rows_ttl_entry.group_by_keys.empty())
+    {
+        ColumnNumbers keys;
+        for (const auto & key : storage.rows_ttl_entry.group_by_keys)
+            keys.push_back(header.getPositionByName(key));
+        agg_key_columns.resize(storage.rows_ttl_entry.group_by_keys.size());
+
+        AggregateDescriptions aggregates = storage.rows_ttl_entry.aggregate_descriptions;
+        for (auto & descr : aggregates)
+            if (descr.arguments.empty())
+                for (const auto & name : descr.argument_names)
+                    descr.arguments.push_back(header.getPositionByName(name));
+        agg_aggregate_columns.resize(storage.rows_ttl_entry.aggregate_descriptions.size());
+
+        const Settings & settings = storage.global_context.getSettingsRef();
+
+        Aggregator::Params params(header, keys, aggregates,
+            false, settings.max_rows_to_group_by, settings.group_by_overflow_mode,
+            SettingUInt64(0), SettingUInt64(0),
+            settings.max_bytes_before_external_group_by, settings.empty_result_for_aggregation_by_empty_set,
+            storage.global_context.getTemporaryVolume(), settings.max_threads, settings.min_free_disk_space_for_temporary_data);
+        aggregator = std::make_unique<Aggregator>(params);
+    }
 }
 
 bool TTLBlockInputStream::isTTLExpired(time_t ttl) const
@@ -77,7 +101,8 @@ bool TTLBlockInputStream::isTTLExpired(time_t ttl) const
 Block TTLBlockInputStream::readImpl()
 {
     /// Skip all data if table ttl is expired for part
-    if (storage.hasRowsTTL() && isTTLExpired(old_ttl_infos.table_ttl.max))
+    if (storage.hasRowsTTL() && !storage.rows_ttl_entry.where_expression &&
+        storage.rows_ttl_entry.group_by_keys.empty() && isTTLExpired(old_ttl_infos.table_ttl.max))
     {
         rows_removed = data_part->rows_count;
         return {};
@@ -85,7 +110,43 @@ Block TTLBlockInputStream::readImpl()
 
     Block block = children.at(0)->read();
     if (!block)
+    {
+        if (aggregator && !agg_result.empty())
+        {
+            MutableColumns result_columns;
+            const auto & column_names = header.getNames();
+            for (const auto & column_name : column_names)
+            {
+                const IColumn * values_column = header.getByName(column_name).column.get();
+                MutableColumnPtr result_column = values_column->cloneEmpty();
+                result_columns.emplace_back(std::move(result_column));
+            }
+
+            auto aggregated_res = aggregator->convertToBlocks(agg_result, true, 1);
+            for (auto & agg_block : aggregated_res)
+            {
+                for (const auto & [name, actions] : storage.rows_ttl_entry.group_by_aggregations)
+                    actions->execute(agg_block);
+                for (const auto & name : storage.rows_ttl_entry.group_by_keys)
+                {
+                    const IColumn * values_column = agg_block.getByName(name).column.get();
+                    auto & result_column = result_columns[header.getPositionByName(name)];
+                    result_column->insertRangeFrom(*values_column, 0, agg_block.rows());
+                }
+                for (const auto & [name, res_column] : storage.rows_ttl_entry.group_by_aggregations_res_column)
+                {
+                    const IColumn * values_column = agg_block.getByName(res_column).column.get();
+                    auto & result_column = result_columns[header.getPositionByName(name)];
+                    result_column->insertRangeFrom(*values_column, 0, agg_block.rows());
+                }
+            }
+
+            block = header.cloneWithColumns(std::move(result_columns));
+            agg_result.invalidate();
+        }
+
         return block;
+    }
 
     if (storage.hasRowsTTL() && (force || isTTLExpired(old_ttl_infos.table_ttl.min)))
         removeRowsWithExpiredTableTTL(block);
@@ -114,35 +175,171 @@ void TTLBlockInputStream::readSuffixImpl()
 void TTLBlockInputStream::removeRowsWithExpiredTableTTL(Block & block)
 {
     storage.rows_ttl_entry.expression->execute(block);
+    if (storage.rows_ttl_entry.where_expression)
+        storage.rows_ttl_entry.where_expression->execute(block);
 
     const IColumn * ttl_column =
         block.getByName(storage.rows_ttl_entry.result_column).column.get();
 
+    const IColumn * where_filter_column = storage.rows_ttl_entry.where_expression ?
+        block.getByName(storage.rows_ttl_entry.where_filter_column).column.get() : nullptr;
+
     const auto & column_names = header.getNames();
-    MutableColumns result_columns;
-    result_columns.reserve(column_names.size());
 
-    for (auto it = column_names.begin(); it != column_names.end(); ++it)
+    if (!aggregator)
     {
-        const IColumn * values_column = block.getByName(*it).column.get();
-        MutableColumnPtr result_column = values_column->cloneEmpty();
-        result_column->reserve(block.rows());
+        MutableColumns result_columns;
+        result_columns.reserve(column_names.size());
+        for (auto it = column_names.begin(); it != column_names.end(); ++it)
+        {
+            const IColumn * values_column = block.getByName(*it).column.get();
+            MutableColumnPtr result_column = values_column->cloneEmpty();
+            result_column->reserve(block.rows());
 
+            for (size_t i = 0; i < block.rows(); ++i)
+            {
+                UInt32 cur_ttl = getTimestampByIndex(ttl_column, i);
+                bool where_filter_passed = !where_filter_column || where_filter_column->getBool(i);
+                if (!isTTLExpired(cur_ttl) || !where_filter_passed)
+                {
+                    new_ttl_infos.table_ttl.update(cur_ttl);
+                    result_column->insertFrom(*values_column, i);
+                }
+                else if (it == column_names.begin())
+                    ++rows_removed;
+            }
+            result_columns.emplace_back(std::move(result_column));
+        }
+        block = header.cloneWithColumns(std::move(result_columns));
+    }
+    else
+    {
+        MutableColumns result_columns;
+        MutableColumns aggregate_columns;
+
+        for (const auto & column_name : column_names)
+        {
+            const IColumn * values_column = block.getByName(column_name).column.get();
+            MutableColumnPtr result_column = values_column->cloneEmpty();
+            result_column->reserve(block.rows());
+            result_columns.emplace_back(std::move(result_column));
+
+            MutableColumnPtr aggregate_column = values_column->cloneEmpty();
+            aggregate_column->reserve(block.rows());
+            aggregate_columns.emplace_back(std::move(aggregate_column));
+        }
+
+        size_t rows_aggregated = 0;
+        size_t current_key_start = 0;
+        size_t rows_with_current_key = 0; 
         for (size_t i = 0; i < block.rows(); ++i)
         {
             UInt32 cur_ttl = getTimestampByIndex(ttl_column, i);
-            if (!isTTLExpired(cur_ttl))
+            bool where_filter_passed = !where_filter_column || where_filter_column->getBool(i);
+            bool ttl_expired = isTTLExpired(cur_ttl) && where_filter_passed;
+
+            bool same_as_current = true;
+            if (current_key_value.empty())
+            {
+                same_as_current = false;
+                current_key_value.resize(storage.rows_ttl_entry.group_by_keys.size());
+            }
+            for (size_t j = 0; j < storage.rows_ttl_entry.group_by_keys.size(); ++j)
+            {
+                const String & key_column = storage.rows_ttl_entry.group_by_keys[j];
+                const IColumn * values_column = block.getByName(key_column).column.get();
+                if (!same_as_current)
+                    values_column->get(i, current_key_value[j]);
+                else
+                {
+                    Field value;
+                    values_column->get(i, value);
+                    if (value != current_key_value[j])
+                    {
+                        current_key_value[j] = value;
+                        same_as_current = false;
+                    }
+                }
+            }
+            if (!same_as_current)
+            {
+                if (rows_with_current_key)
+                {
+                    Columns aggregate_chunk;
+                    aggregate_chunk.reserve(aggregate_columns.size());
+                    for (const auto & name : column_names)
+                    {
+                        const auto & column = aggregate_columns[header.getPositionByName(name)];
+                        ColumnPtr chunk_column = column->cut(current_key_start, rows_with_current_key);
+                        aggregate_chunk.emplace_back(std::move(chunk_column));
+                    }
+                    aggregator->executeOnBlock(aggregate_chunk, rows_with_current_key, agg_result, agg_key_columns,
+                                            agg_aggregate_columns, agg_no_more_keys);
+                }
+                if (!agg_result.empty())
+                {
+                    auto aggregated_res = aggregator->convertToBlocks(agg_result, true, 1);
+                    for (auto & agg_block : aggregated_res)
+                    {
+                        for (const auto & [name, actions] : storage.rows_ttl_entry.group_by_aggregations)
+                            actions->execute(agg_block);
+                        for (const auto & name : storage.rows_ttl_entry.group_by_keys)
+                        {
+                            const IColumn * values_column = agg_block.getByName(name).column.get();
+                            auto & result_column = result_columns[header.getPositionByName(name)];
+                            result_column->insertRangeFrom(*values_column, 0, agg_block.rows());
+                        }
+                        for (const auto & [name, res_column] : storage.rows_ttl_entry.group_by_aggregations_res_column)
+                        {
+                            const IColumn * values_column = agg_block.getByName(res_column).column.get();
+                            auto & result_column = result_columns[header.getPositionByName(name)];
+                            result_column->insertRangeFrom(*values_column, 0, agg_block.rows());
+                        }
+                    }
+                }
+                agg_result.invalidate();
+                current_key_start = rows_aggregated;
+                rows_with_current_key = 0;
+            }
+            if (ttl_expired)
+            {
+                ++rows_with_current_key;
+                ++rows_aggregated;
+                for (const auto & name : column_names)
+                {
+                    const IColumn * values_column = block.getByName(name).column.get();
+                    auto & column = aggregate_columns[header.getPositionByName(name)];
+                    column->insertFrom(*values_column, i);
+                }
+            }
+            else
             {
                 new_ttl_infos.table_ttl.update(cur_ttl);
-                result_column->insertFrom(*values_column, i);
+                for (const auto & name : column_names)
+                {
+                    const IColumn * values_column = block.getByName(name).column.get();
+                    auto & column = result_columns[header.getPositionByName(name)];
+                    column->insertFrom(*values_column, i);
+                }
             }
-            else if (it == column_names.begin())
-                ++rows_removed;
         }
-        result_columns.emplace_back(std::move(result_column));
-    }
 
-    block = header.cloneWithColumns(std::move(result_columns));
+        if (rows_with_current_key)
+        {
+            Columns aggregate_chunk;
+            aggregate_chunk.reserve(aggregate_columns.size());
+            for (const auto & name : column_names)
+            {
+                const auto & column = aggregate_columns[header.getPositionByName(name)];
+                ColumnPtr chunk_column = column->cut(current_key_start, rows_with_current_key);
+                aggregate_chunk.emplace_back(std::move(chunk_column));
+            }
+            aggregator->executeOnBlock(aggregate_chunk, rows_with_current_key, agg_result, agg_key_columns,
+                                       agg_aggregate_columns, agg_no_more_keys);
+        }
+
+        block = header.cloneWithColumns(std::move(result_columns));
+    }
 }
 
 void TTLBlockInputStream::removeValuesWithExpiredColumnTTL(Block & block)
diff --git a/src/DataStreams/TTLBlockInputStream.h b/src/DataStreams/TTLBlockInputStream.h
index 3896e5232f8..821f8dd9284 100644
--- a/src/DataStreams/TTLBlockInputStream.h
+++ b/src/DataStreams/TTLBlockInputStream.h
@@ -3,6 +3,7 @@
 #include <Storages/MergeTree/MergeTreeData.h>
 #include <Storages/MergeTree/IMergeTreeDataPart.h>
 #include <Core/Block.h>
+#include <Interpreters/Aggregator.h>
 
 #include <common/DateLUT.h>
 
@@ -39,6 +40,13 @@ private:
     time_t current_time;
     bool force;
 
+    std::unique_ptr<Aggregator> aggregator;
+    std::vector<Field> current_key_value;
+    AggregatedDataVariants agg_result;
+    ColumnRawPtrs agg_key_columns; 
+    Aggregator::AggregateColumns agg_aggregate_columns;
+    bool agg_no_more_keys;
+
     IMergeTreeDataPart::TTLInfos old_ttl_infos;
     IMergeTreeDataPart::TTLInfos new_ttl_infos;
     NameSet empty_columns;
diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp
index 3341855b8c6..c432b920a9f 100644
--- a/src/Interpreters/ExpressionAnalyzer.cpp
+++ b/src/Interpreters/ExpressionAnalyzer.cpp
@@ -192,10 +192,10 @@ void ExpressionAnalyzer::analyzeAggregation()
 
     if (has_aggregation)
     {
-        getSelectQuery(); /// assertSelect()
+        // getSelectQuery(); /// assertSelect()
 
         /// Find out aggregation keys.
-        if (select_query->groupBy())
+        if (select_query && select_query->groupBy())
         {
             NameSet unique_keys;
             ASTs & group_asts = select_query->groupBy()->children;
@@ -926,7 +926,10 @@ void ExpressionAnalyzer::appendExpression(ExpressionActionsChain & chain, const
 
 ExpressionActionsPtr ExpressionAnalyzer::getActions(bool add_aliases, bool project_result)
 {
-    ExpressionActionsPtr actions = std::make_shared<ExpressionActions>(sourceColumns(), context);
+    NamesAndTypesList columns(sourceColumns());
+    for (const auto & col : aggregated_columns)
+        columns.push_back(col);
+    ExpressionActionsPtr actions = std::make_shared<ExpressionActions>(columns, context);
     NamesWithAliases result_columns;
     Names result_names;
 
diff --git a/src/Interpreters/SyntaxAnalyzer.cpp b/src/Interpreters/SyntaxAnalyzer.cpp
index b3d566dbdc8..ec7f2154dad 100644
--- a/src/Interpreters/SyntaxAnalyzer.cpp
+++ b/src/Interpreters/SyntaxAnalyzer.cpp
@@ -839,7 +839,7 @@ SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyzeSelect(
     return std::make_shared<const SyntaxAnalyzerResult>(result);
 }
 
-SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyze(ASTPtr & query, const NamesAndTypesList & source_columns, ConstStoragePtr storage) const
+SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyze(ASTPtr & query, const NamesAndTypesList & source_columns, ConstStoragePtr storage, bool allow_aggregates) const
 {
     if (query->as<ASTSelectQuery>())
         throw Exception("Not select analyze for select asts.", ErrorCodes::LOGICAL_ERROR);
@@ -855,7 +855,20 @@ SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyze(ASTPtr & query, const NamesAndTy
 
     optimizeIf(query, result.aliases, settings.optimize_if_chain_to_miltiif);
 
-    assertNoAggregates(query, "in wrong place");
+    if (allow_aggregates) 
+    {
+        GetAggregatesVisitor::Data data;
+        GetAggregatesVisitor(data).visit(query);
+
+        /// There can not be other aggregate functions within the aggregate functions.
+        for (const ASTFunction * node : data.aggregates)
+            for (auto & arg : node->arguments->children)
+                assertNoAggregates(arg, "inside another aggregate function");
+        result.aggregates = data.aggregates;
+    }
+    else 
+        assertNoAggregates(query, "in wrong place");
+
     result.collectUsedColumns(query);
     return std::make_shared<const SyntaxAnalyzerResult>(result);
 }
diff --git a/src/Interpreters/SyntaxAnalyzer.h b/src/Interpreters/SyntaxAnalyzer.h
index dda0add38db..abacb25ac4d 100644
--- a/src/Interpreters/SyntaxAnalyzer.h
+++ b/src/Interpreters/SyntaxAnalyzer.h
@@ -86,7 +86,7 @@ public:
     {}
 
     /// Analyze and rewrite not select query
-    SyntaxAnalyzerResultPtr analyze(ASTPtr & query, const NamesAndTypesList & source_columns_, ConstStoragePtr storage = {}) const;
+    SyntaxAnalyzerResultPtr analyze(ASTPtr & query, const NamesAndTypesList & source_columns_, ConstStoragePtr storage = {}, bool allow_aggregations = false) const;
 
     /// Analyze and rewrite select query
     SyntaxAnalyzerResultPtr analyzeSelect(
diff --git a/src/Parsers/ASTTTLElement.cpp b/src/Parsers/ASTTTLElement.cpp
index 7e03a73e36d..fb35266465b 100644
--- a/src/Parsers/ASTTTLElement.cpp
+++ b/src/Parsers/ASTTTLElement.cpp
@@ -7,21 +7,80 @@
 namespace DB
 {
 
+ASTPtr ASTTTLElement::clone() const 
+{
+    auto clone = std::make_shared<ASTTTLElement>(*this);
+    clone->children.clear();
+    clone->positions.clear();
+    
+    for (auto expr : {Expression::TTL, Expression::WHERE})
+        clone->setExpression(expr, getExpression(expr, true));
+
+    for (auto & [name, expr] : clone->group_by_aggregations)
+        expr = expr->clone();
+
+    return clone;
+}
+
 void ASTTTLElement::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
 {
-    children.front()->formatImpl(settings, state, frame);
-    if (destination_type == PartDestinationType::DISK)
+    ttl()->formatImpl(settings, state, frame);
+    if (mode == Mode::MOVE && destination_type == PartDestinationType::DISK)
     {
         settings.ostr << " TO DISK " << quoteString(destination_name);
     }
-    else if (destination_type == PartDestinationType::VOLUME)
+    else if (mode == Mode::MOVE && destination_type == PartDestinationType::VOLUME)
     {
         settings.ostr << " TO VOLUME " << quoteString(destination_name);
     }
-    else if (destination_type == PartDestinationType::DELETE)
+    else if (mode == Mode::GROUP_BY)
+    {
+        settings.ostr << " GROUP BY ";
+        for (size_t i = 0; i < group_by_key_columns.size(); ++i)
+        {
+            settings.ostr << group_by_key_columns[i];
+            if (i + 1 != group_by_key_columns.size())
+                settings.ostr << ", ";
+        }
+        settings.ostr << " SET ";
+        for (size_t i =  0; i < group_by_aggregations.size(); ++i)
+        {
+            settings.ostr << group_by_aggregations[i].first << " = ";
+            group_by_aggregations[i].second->formatImpl(settings, state, frame);
+            if (i + 1 != group_by_aggregations.size())
+                settings.ostr << ", ";
+        }
+    }
+    else if (mode == Mode::DELETE)
     {
         /// It would be better to output "DELETE" here but that will break compatibility with earlier versions.
     }
+
+    if (where()) 
+    {
+        settings.ostr << " WHERE ";
+        where()->formatImpl(settings, state, frame);
+    }
+}
+
+void ASTTTLElement::setExpression(Expression expr, ASTPtr && ast) 
+{
+    auto it = positions.find(expr);
+    if (it == positions.end())
+    {
+        positions[expr] = children.size();
+        children.emplace_back(ast);
+    }
+    else
+        children[it->second] = ast;
+}
+
+ASTPtr ASTTTLElement::getExpression(Expression expr, bool clone) const
+{
+    auto it = positions.find(expr);
+    if (it != positions.end())
+        return clone ? children[it->second]->clone() : children[it->second];
+    return {};
 }
 
 }
diff --git a/src/Parsers/ASTTTLElement.h b/src/Parsers/ASTTTLElement.h
index 02f70094e04..1a2d7d3c723 100644
--- a/src/Parsers/ASTTTLElement.h
+++ b/src/Parsers/ASTTTLElement.h
@@ -6,31 +6,53 @@
 
 namespace DB
 {
+
 /** Element of TTL expression.
   */
 class ASTTTLElement : public IAST
 {
 public:
+    enum class Expression : uint8_t
+    {
+        TTL,
+        WHERE
+    };
+
+    enum class Mode : uint8_t
+    {
+        DELETE,
+        MOVE,
+        GROUP_BY
+    };
+
+    Mode mode;
     PartDestinationType destination_type;
     String destination_name;
+    std::vector<String> group_by_key_columns;
+    std::vector<std::pair<String, ASTPtr>> group_by_aggregations;
 
-    ASTTTLElement(PartDestinationType destination_type_, const String & destination_name_)
-        : destination_type(destination_type_)
+    ASTTTLElement(Mode mode_, PartDestinationType destination_type_, const String & destination_name_)
+        : mode(mode_) 
+        , destination_type(destination_type_)
         , destination_name(destination_name_)
     {
     }
 
     String getID(char) const override { return "TTLElement"; }
 
-    ASTPtr clone() const override
-    {
-        auto clone = std::make_shared<ASTTTLElement>(*this);
-        clone->cloneChildren();
-        return clone;
-    }
+    ASTPtr clone() const override;
+
+    const ASTPtr ttl() const { return getExpression(Expression::TTL); }
+    const ASTPtr where() const { return getExpression(Expression::WHERE); }
+
+    void setExpression(Expression expr, ASTPtr && ast);
+    ASTPtr getExpression(Expression expr, bool clone = false) const;
 
 protected:
     void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
+
+private:
+    std::unordered_map<Expression, size_t> positions;
 };
 
 }
diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp
index 70a8b282a72..2734aa67193 100644
--- a/src/Parsers/ExpressionElementParsers.cpp
+++ b/src/Parsers/ExpressionElementParsers.cpp
@@ -1455,23 +1455,50 @@ bool ParserTTLElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
     ParserKeyword s_to_disk("TO DISK");
     ParserKeyword s_to_volume("TO VOLUME");
     ParserKeyword s_delete("DELETE");
+    ParserKeyword s_where("WHERE");
+    ParserKeyword s_group_by("GROUP BY");
+    ParserKeyword s_set("SET");
+    ParserToken s_comma(TokenType::Comma);
+    ParserToken s_eq(TokenType::Equals);
+    ParserIdentifier parser_identifier;
     ParserStringLiteral parser_string_literal;
     ParserExpression parser_exp;
+    ParserIdentifierList parser_identifier_list;
 
-    ASTPtr expr_elem;
-    if (!parser_exp.parse(pos, expr_elem, expected))
+
+    ASTPtr ttl_expr;
+    if (!parser_exp.parse(pos, ttl_expr, expected))
         return false;
 
+    ASTPtr where_expr;
+
+    std::vector<String> group_by_key_columns;
+    std::vector<std::pair<String, ASTPtr>> group_by_aggregations;
+
+    ASTTTLElement::Mode mode;
     PartDestinationType destination_type = PartDestinationType::DELETE;
     String destination_name;
     if (s_to_disk.ignore(pos))
+    {
+        mode = ASTTTLElement::Mode::MOVE;
         destination_type = PartDestinationType::DISK;
+    }
     else if (s_to_volume.ignore(pos))
+    {
+        mode = ASTTTLElement::Mode::MOVE;
         destination_type = PartDestinationType::VOLUME;
+    }
+    else if (s_group_by.ignore(pos)) 
+    {
+        mode = ASTTTLElement::Mode::GROUP_BY;
+    }
     else
+    {
         s_delete.ignore(pos);
+        mode = ASTTTLElement::Mode::DELETE;
+    }
 
-    if (destination_type == PartDestinationType::DISK || destination_type == PartDestinationType::VOLUME)
+    if (mode == ASTTTLElement::Mode::MOVE)
     {
         ASTPtr ast_space_name;
         if (!parser_string_literal.parse(pos, ast_space_name, expected))
@@ -1479,10 +1506,57 @@ bool ParserTTLElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
 
         destination_name = ast_space_name->as<ASTLiteral &>().value.get<const String &>();
     }
+    else if (mode == ASTTTLElement::Mode::GROUP_BY)
+    {
+        ASTPtr ast_group_by_key_columns;
+        if (!parser_identifier_list.parse(pos, ast_group_by_key_columns, expected))
+            return false;
+        for (const auto identifier : ast_group_by_key_columns->children)
+        {
+            String identifier_str;
+            if (!tryGetIdentifierNameInto(identifier, identifier_str))
+                return false;
+            group_by_key_columns.emplace_back(std::move(identifier_str));
+        }        
 
-    node = std::make_shared<ASTTTLElement>(destination_type, destination_name);
-    node->children.push_back(expr_elem);
+        if (!s_set.ignore(pos))
+            return false;
+        while (true)
+        {
+            if (!group_by_aggregations.empty() && !s_comma.ignore(pos))
+                break;
+            
+            ASTPtr name;
+            ASTPtr value;
+            if (!parser_identifier.parse(pos, name, expected))
+                return false;
+            if (!s_eq.ignore(pos))
+                return false;
+            if (!parser_exp.parse(pos, value, expected))
+                return false;
 
+            String name_str;
+            if (!tryGetIdentifierNameInto(name, name_str))
+                return false;
+            group_by_aggregations.emplace_back(name_str, std::move(value));
+        }
+    }
+
+    if ((mode == ASTTTLElement::Mode::MOVE || mode == ASTTTLElement::Mode::DELETE) && s_where.ignore(pos))
+    {
+        if (!parser_exp.parse(pos, where_expr, expected))
+            return false;
+    }
+
+    auto ttl_element = std::make_shared<ASTTTLElement>(mode, destination_type, destination_name);
+    ttl_element->setExpression(ASTTTLElement::Expression::TTL, std::move(ttl_expr));
+    if (where_expr)
+        ttl_element->setExpression(ASTTTLElement::Expression::WHERE, std::move(where_expr));
+
+    ttl_element->group_by_key_columns = std::move(group_by_key_columns);
+    ttl_element->group_by_aggregations = std::move(group_by_aggregations);
+
+    node = ttl_element;
     return true;
 }
 
diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp
index a967ae19691..062d1dd4483 100644
--- a/src/Parsers/ExpressionListParsers.cpp
+++ b/src/Parsers/ExpressionListParsers.cpp
@@ -742,4 +742,13 @@ bool ParserKeyValuePairsList::parseImpl(Pos & pos, ASTPtr & node, Expected & exp
     return parser.parse(pos, node, expected);
 }
 
+
+bool ParserIdentifierList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
+{
+    return ParserList(
+        std::make_unique<ParserIdentifier>(),
+        std::make_unique<ParserToken>(TokenType::Comma))
+        .parse(pos, node, expected);
 }
+
+}
\ No newline at end of file
diff --git a/src/Parsers/ExpressionListParsers.h b/src/Parsers/ExpressionListParsers.h
index 0cef29b6d67..ba939d8d160 100644
--- a/src/Parsers/ExpressionListParsers.h
+++ b/src/Parsers/ExpressionListParsers.h
@@ -421,4 +421,13 @@ protected:
     bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
 };
 
-}
+
+/** A comma-separated list of identifiers, probably empty. */
+class ParserIdentifierList : public IParserBase
+{
+protected:
+    const char * getName() const override { return "list of identifiers"; }
+    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
+};
+
+}
\ No newline at end of file
diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 48cf3934820..9eb14ed5475 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -615,19 +615,50 @@ void MergeTreeData::setTTLExpressions(const ColumnsDescription & new_columns,
 
     auto new_column_ttls = new_columns.getColumnTTLs();
 
-    auto create_ttl_entry = [this, &new_columns](ASTPtr ttl_ast)
+    auto create_ttl_entry = [this, &new_columns](ASTPtr ttl_expr_ast)
     {
         TTLEntry result;
 
-        auto syntax_result = SyntaxAnalyzer(global_context).analyze(ttl_ast, new_columns.getAllPhysical());
-        result.expression = ExpressionAnalyzer(ttl_ast, syntax_result, global_context).getActions(false);
+        auto ttl_syntax_result = SyntaxAnalyzer(global_context).analyze(ttl_expr_ast, new_columns.getAllPhysical());
+        result.expression = ExpressionAnalyzer(ttl_expr_ast, ttl_syntax_result, global_context).getActions(false);
+        result.result_column = ttl_expr_ast->getColumnName();
+
         result.destination_type = PartDestinationType::DELETE;
-        result.result_column = ttl_ast->getColumnName();
 
         checkTTLExpression(result.expression, result.result_column);
         return result;
     };
 
+    auto create_rows_ttl_entry = [this, &new_columns, &create_ttl_entry](const ASTTTLElement * ttl_element)
+    {
+        auto result = create_ttl_entry(ttl_element->ttl());
+        if (ttl_element->mode == ASTTTLElement::Mode::DELETE || ttl_element->mode ==ASTTTLElement::Mode::MOVE)         
+        {
+            if (ASTPtr where_expr_ast = ttl_element->where()) 
+            {
+                auto where_syntax_result = SyntaxAnalyzer(global_context).analyze(where_expr_ast, new_columns.getAllPhysical());
+                result.where_expression = ExpressionAnalyzer(where_expr_ast, where_syntax_result, global_context).getActions(false);
+                result.where_filter_column = where_expr_ast->getColumnName();
+            }
+        }
+        else if (ttl_element->mode == ASTTTLElement::Mode::GROUP_BY) 
+        {
+            result.group_by_keys = ttl_element->group_by_key_columns;
+            for (auto [name, value] : ttl_element->group_by_aggregations)
+            {
+                auto syntax_result = SyntaxAnalyzer(global_context).analyze(value, new_columns.getAllPhysical(), {}, true);
+                auto expr_analyzer = ExpressionAnalyzer(value, syntax_result, global_context);
+
+                result.group_by_aggregations.emplace_back(name, expr_analyzer.getActions(false));
+                result.group_by_aggregations_res_column.emplace_back(name, value->getColumnName());
+
+                for (const auto descr : expr_analyzer.getAnalyzedData().aggregate_descriptions)
+                    result.aggregate_descriptions.push_back(descr);
+            }
+        }
+        return result;
+    };
+
     if (!new_column_ttls.empty())
     {
         NameSet columns_ttl_forbidden;
@@ -672,7 +703,8 @@ void MergeTreeData::setTTLExpressions(const ColumnsDescription & new_columns,
                     throw Exception("More than one DELETE TTL expression is not allowed", ErrorCodes::BAD_TTL_EXPRESSION);
                 }
 
-                auto new_rows_ttl_entry = create_ttl_entry(ttl_element->children[0]);
+                LOG_DEBUG(log, "ttl_element->size is " << ttl_element->size());
+                auto new_rows_ttl_entry = create_rows_ttl_entry(ttl_element);
                 if (!only_check)
                     update_rows_ttl_entry = new_rows_ttl_entry;
 
@@ -680,7 +712,7 @@ void MergeTreeData::setTTLExpressions(const ColumnsDescription & new_columns,
             }
             else
             {
-                auto new_ttl_entry = create_ttl_entry(ttl_element->children[0]);
+                auto new_ttl_entry = create_rows_ttl_entry(ttl_element);
 
                 new_ttl_entry.entry_ast = ttl_element_ptr;
                 new_ttl_entry.destination_type = ttl_element->destination_type;
diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h
index cd1d4912a29..dfcddb40f3c 100644
--- a/src/Storages/MergeTree/MergeTreeData.h
+++ b/src/Storages/MergeTree/MergeTreeData.h
@@ -20,6 +20,7 @@
 #include <Storages/MergeTree/MergeTreePartsMover.h>
 #include <Interpreters/PartLog.h>
 #include <Disks/StoragePolicy.h>
+#include <Interpreters/Aggregator.h>
 
 #include <boost/multi_index_container.hpp>
 #include <boost/multi_index/ordered_index.hpp>
@@ -652,6 +653,14 @@ public:
         ExpressionActionsPtr expression;
         String result_column;
 
+        ExpressionActionsPtr where_expression;
+        String where_filter_column;
+
+        Names group_by_keys;
+        std::vector<std::pair<String, ExpressionActionsPtr>> group_by_aggregations;
+        std::vector<std::pair<String, String>> group_by_aggregations_res_column;
+        AggregateDescriptions aggregate_descriptions;
+
         /// Name and type of a destination are only valid in table-level context.
         PartDestinationType destination_type;
         String destination_name;

From eeaf608aa912543540106ec5bc4363b88d3324f3 Mon Sep 17 00:00:00 2001
From: Nikolai Sorokin <sonikint@gmail.com>
Date: Tue, 12 May 2020 17:27:00 +0300
Subject: [PATCH 064/184] Refactor code; support prefix of PK as GROUP BY key

---
 src/DataStreams/TTLBlockInputStream.cpp  | 175 +++++++++--------------
 src/DataStreams/TTLBlockInputStream.h    |   6 +
 src/Parsers/ASTTTLElement.cpp            |  57 ++++----
 src/Parsers/ASTTTLElement.h              |  38 +++--
 src/Parsers/ExpressionElementParsers.cpp |  33 +++--
 src/Storages/MergeTree/MergeTreeData.cpp |  32 +++--
 src/Storages/MergeTree/MergeTreeData.h   |   8 +-
 src/Storages/MergeTree/TTLMode.h         |  14 ++
 8 files changed, 177 insertions(+), 186 deletions(-)
 create mode 100644 src/Storages/MergeTree/TTLMode.h

diff --git a/src/DataStreams/TTLBlockInputStream.cpp b/src/DataStreams/TTLBlockInputStream.cpp
index d816cba4b2f..d5e46a9929d 100644
--- a/src/DataStreams/TTLBlockInputStream.cpp
+++ b/src/DataStreams/TTLBlockInputStream.cpp
@@ -5,6 +5,7 @@
 #include <Interpreters/ExpressionAnalyzer.h>
 #include <Columns/ColumnConst.h>
 #include <Interpreters/addTypeConversionToAST.h>
+#include <Storages/MergeTree/TTLMode.h>
 
 namespace DB
 {
@@ -68,8 +69,10 @@ TTLBlockInputStream::TTLBlockInputStream(
         defaults_expression = ExpressionAnalyzer{default_expr_list, syntax_result, storage.global_context}.getActions(true);
     }
 
-    if (storage.hasRowsTTL() && !storage.rows_ttl_entry.group_by_keys.empty())
+    if (storage.hasRowsTTL() && storage.rows_ttl_entry.mode == TTLMode::GROUP_BY)
     {
+        current_key_value.resize(storage.rows_ttl_entry.group_by_keys.size());
+
         ColumnNumbers keys;
         for (const auto & key : storage.rows_ttl_entry.group_by_keys)
             keys.push_back(header.getPositionByName(key));
@@ -84,9 +87,11 @@ TTLBlockInputStream::TTLBlockInputStream(
 
         const Settings & settings = storage.global_context.getSettingsRef();
 
+        bool allow_to_use_two_level_group_by = false;  // settings.max_bytes_before_external_group_by != 0;
         Aggregator::Params params(header, keys, aggregates,
             false, settings.max_rows_to_group_by, settings.group_by_overflow_mode,
-            SettingUInt64(0), SettingUInt64(0),
+            allow_to_use_two_level_group_by ? settings.group_by_two_level_threshold : SettingUInt64(0),
+            allow_to_use_two_level_group_by ? settings.group_by_two_level_threshold_bytes : SettingUInt64(0),
             settings.max_bytes_before_external_group_by, settings.empty_result_for_aggregation_by_empty_set,
             storage.global_context.getTemporaryVolume(), settings.max_threads, settings.min_free_disk_space_for_temporary_data);
         aggregator = std::make_unique<Aggregator>(params);
@@ -102,7 +107,7 @@ Block TTLBlockInputStream::readImpl()
 {
     /// Skip all data if table ttl is expired for part
     if (storage.hasRowsTTL() && !storage.rows_ttl_entry.where_expression &&
-        storage.rows_ttl_entry.group_by_keys.empty() && isTTLExpired(old_ttl_infos.table_ttl.max))
+        storage.rows_ttl_entry.mode != TTLMode::GROUP_BY && isTTLExpired(old_ttl_infos.table_ttl.max))
     {
         rows_removed = data_part->rows_count;
         return {};
@@ -113,36 +118,9 @@ Block TTLBlockInputStream::readImpl()
     {
         if (aggregator && !agg_result.empty())
         {
-            MutableColumns result_columns;
-            const auto & column_names = header.getNames();
-            for (const auto & column_name : column_names)
-            {
-                const IColumn * values_column = header.getByName(column_name).column.get();
-                MutableColumnPtr result_column = values_column->cloneEmpty();
-                result_columns.emplace_back(std::move(result_column));
-            }
-
-            auto aggregated_res = aggregator->convertToBlocks(agg_result, true, 1);
-            for (auto & agg_block : aggregated_res)
-            {
-                for (const auto & [name, actions] : storage.rows_ttl_entry.group_by_aggregations)
-                    actions->execute(agg_block);
-                for (const auto & name : storage.rows_ttl_entry.group_by_keys)
-                {
-                    const IColumn * values_column = agg_block.getByName(name).column.get();
-                    auto & result_column = result_columns[header.getPositionByName(name)];
-                    result_column->insertRangeFrom(*values_column, 0, agg_block.rows());
-                }
-                for (const auto & [name, res_column] : storage.rows_ttl_entry.group_by_aggregations_res_column)
-                {
-                    const IColumn * values_column = agg_block.getByName(res_column).column.get();
-                    auto & result_column = result_columns[header.getPositionByName(name)];
-                    result_column->insertRangeFrom(*values_column, 0, agg_block.rows());
-                }
-            }
-
+            MutableColumns result_columns = header.cloneEmptyColumns();
+            finalizeAggregates(result_columns);
             block = header.cloneWithColumns(std::move(result_columns));
-            agg_result.invalidate();
         }
 
         return block;
@@ -181,8 +159,8 @@ void TTLBlockInputStream::removeRowsWithExpiredTableTTL(Block & block)
     const IColumn * ttl_column =
         block.getByName(storage.rows_ttl_entry.result_column).column.get();
 
-    const IColumn * where_filter_column = storage.rows_ttl_entry.where_expression ?
-        block.getByName(storage.rows_ttl_entry.where_filter_column).column.get() : nullptr;
+    const IColumn * where_result_column = storage.rows_ttl_entry.where_expression ?
+        block.getByName(storage.rows_ttl_entry.where_result_column).column.get() : nullptr;
 
     const auto & column_names = header.getNames();
 
@@ -199,7 +177,7 @@ void TTLBlockInputStream::removeRowsWithExpiredTableTTL(Block & block)
             for (size_t i = 0; i < block.rows(); ++i)
             {
                 UInt32 cur_ttl = getTimestampByIndex(ttl_column, i);
-                bool where_filter_passed = !where_filter_column || where_filter_column->getBool(i);
+                bool where_filter_passed = !where_result_column || where_result_column->getBool(i);
                 if (!isTTLExpired(cur_ttl) || !where_filter_passed)
                 {
                     new_ttl_infos.table_ttl.update(cur_ttl);
@@ -214,20 +192,8 @@ void TTLBlockInputStream::removeRowsWithExpiredTableTTL(Block & block)
     }
     else
     {
-        MutableColumns result_columns;
-        MutableColumns aggregate_columns;
-
-        for (const auto & column_name : column_names)
-        {
-            const IColumn * values_column = block.getByName(column_name).column.get();
-            MutableColumnPtr result_column = values_column->cloneEmpty();
-            result_column->reserve(block.rows());
-            result_columns.emplace_back(std::move(result_column));
-
-            MutableColumnPtr aggregate_column = values_column->cloneEmpty();
-            aggregate_column->reserve(block.rows());
-            aggregate_columns.emplace_back(std::move(aggregate_column));
-        }
+        MutableColumns result_columns = header.cloneEmptyColumns();
+        MutableColumns aggregate_columns = header.cloneEmptyColumns();
 
         size_t rows_aggregated = 0;
         size_t current_key_start = 0;
@@ -235,72 +201,30 @@ void TTLBlockInputStream::removeRowsWithExpiredTableTTL(Block & block)
         for (size_t i = 0; i < block.rows(); ++i)
         {
             UInt32 cur_ttl = getTimestampByIndex(ttl_column, i);
-            bool where_filter_passed = !where_filter_column || where_filter_column->getBool(i);
+            bool where_filter_passed = !where_result_column || where_result_column->getBool(i);
             bool ttl_expired = isTTLExpired(cur_ttl) && where_filter_passed;
 
             bool same_as_current = true;
-            if (current_key_value.empty())
-            {
-                same_as_current = false;
-                current_key_value.resize(storage.rows_ttl_entry.group_by_keys.size());
-            }
             for (size_t j = 0; j < storage.rows_ttl_entry.group_by_keys.size(); ++j)
             {
                 const String & key_column = storage.rows_ttl_entry.group_by_keys[j];
                 const IColumn * values_column = block.getByName(key_column).column.get();
-                if (!same_as_current)
-                    values_column->get(i, current_key_value[j]);
-                else
+                if (!same_as_current || (*values_column)[i] != current_key_value[j])
                 {
-                    Field value;
-                    values_column->get(i, value);
-                    if (value != current_key_value[j])
-                    {
-                        current_key_value[j] = value;
-                        same_as_current = false;
-                    }
+                    values_column->get(i, current_key_value[j]);
+                    same_as_current = false;
                 }
             }
             if (!same_as_current)
             {
                 if (rows_with_current_key)
-                {
-                    Columns aggregate_chunk;
-                    aggregate_chunk.reserve(aggregate_columns.size());
-                    for (const auto & name : column_names)
-                    {
-                        const auto & column = aggregate_columns[header.getPositionByName(name)];
-                        ColumnPtr chunk_column = column->cut(current_key_start, rows_with_current_key);
-                        aggregate_chunk.emplace_back(std::move(chunk_column));
-                    }
-                    aggregator->executeOnBlock(aggregate_chunk, rows_with_current_key, agg_result, agg_key_columns,
-                                            agg_aggregate_columns, agg_no_more_keys);
-                }
-                if (!agg_result.empty())
-                {
-                    auto aggregated_res = aggregator->convertToBlocks(agg_result, true, 1);
-                    for (auto & agg_block : aggregated_res)
-                    {
-                        for (const auto & [name, actions] : storage.rows_ttl_entry.group_by_aggregations)
-                            actions->execute(agg_block);
-                        for (const auto & name : storage.rows_ttl_entry.group_by_keys)
-                        {
-                            const IColumn * values_column = agg_block.getByName(name).column.get();
-                            auto & result_column = result_columns[header.getPositionByName(name)];
-                            result_column->insertRangeFrom(*values_column, 0, agg_block.rows());
-                        }
-                        for (const auto & [name, res_column] : storage.rows_ttl_entry.group_by_aggregations_res_column)
-                        {
-                            const IColumn * values_column = agg_block.getByName(res_column).column.get();
-                            auto & result_column = result_columns[header.getPositionByName(name)];
-                            result_column->insertRangeFrom(*values_column, 0, agg_block.rows());
-                        }
-                    }
-                }
-                agg_result.invalidate();
+                    calculateAggregates(aggregate_columns, current_key_start, rows_with_current_key);
+                finalizeAggregates(result_columns);
+
                 current_key_start = rows_aggregated;
                 rows_with_current_key = 0;
             }
+
             if (ttl_expired)
             {
                 ++rows_with_current_key;
@@ -325,23 +249,52 @@ void TTLBlockInputStream::removeRowsWithExpiredTableTTL(Block & block)
         }
 
         if (rows_with_current_key)
-        {
-            Columns aggregate_chunk;
-            aggregate_chunk.reserve(aggregate_columns.size());
-            for (const auto & name : column_names)
-            {
-                const auto & column = aggregate_columns[header.getPositionByName(name)];
-                ColumnPtr chunk_column = column->cut(current_key_start, rows_with_current_key);
-                aggregate_chunk.emplace_back(std::move(chunk_column));
-            }
-            aggregator->executeOnBlock(aggregate_chunk, rows_with_current_key, agg_result, agg_key_columns,
-                                       agg_aggregate_columns, agg_no_more_keys);
-        }
+            calculateAggregates(aggregate_columns, current_key_start, rows_with_current_key);
 
         block = header.cloneWithColumns(std::move(result_columns));
     }
 }
 
+void TTLBlockInputStream::calculateAggregates(const MutableColumns & aggregate_columns, size_t start_pos, size_t length)
+{
+    Columns aggregate_chunk;
+    aggregate_chunk.reserve(aggregate_columns.size());
+    for (const auto & name : header.getNames())
+    {
+        const auto & column = aggregate_columns[header.getPositionByName(name)];
+        ColumnPtr chunk_column = column->cut(start_pos, length);
+        aggregate_chunk.emplace_back(std::move(chunk_column));
+    }
+    aggregator->executeOnBlock(aggregate_chunk, length, agg_result, agg_key_columns,
+                               agg_aggregate_columns, agg_no_more_keys);
+}
+
+void TTLBlockInputStream::finalizeAggregates(MutableColumns & result_columns)
+{
+    if (!agg_result.empty())
+    {
+        auto aggregated_res = aggregator->convertToBlocks(agg_result, true, 1);
+        for (auto & agg_block : aggregated_res)
+        {
+            for (const auto & it : storage.rows_ttl_entry.group_by_aggregations)
+                std::get<2>(it)->execute(agg_block);
+            for (const auto & name : storage.rows_ttl_entry.group_by_keys)
+            {
+                const IColumn * values_column = agg_block.getByName(name).column.get();
+                auto & result_column = result_columns[header.getPositionByName(name)];
+                result_column->insertRangeFrom(*values_column, 0, agg_block.rows());
+            }
+            for (const auto & it : storage.rows_ttl_entry.group_by_aggregations)
+            {
+                const IColumn * values_column = agg_block.getByName(get<1>(it)).column.get();
+                auto & result_column = result_columns[header.getPositionByName(std::get<0>(it))];
+                result_column->insertRangeFrom(*values_column, 0, agg_block.rows());
+            }
+        }
+    }
+    agg_result.invalidate();
+}
+
 void TTLBlockInputStream::removeValuesWithExpiredColumnTTL(Block & block)
 {
     Block block_with_defaults;
diff --git a/src/DataStreams/TTLBlockInputStream.h b/src/DataStreams/TTLBlockInputStream.h
index 821f8dd9284..c5ec6596038 100644
--- a/src/DataStreams/TTLBlockInputStream.h
+++ b/src/DataStreams/TTLBlockInputStream.h
@@ -66,6 +66,12 @@ private:
 
     /// Removes rows with expired table ttl and computes new ttl_infos for part
     void removeRowsWithExpiredTableTTL(Block & block);
+    
+    // Calculate aggregates of aggregate_columns into agg_result
+    void calculateAggregates(const MutableColumns & aggregate_columns, size_t start_pos, size_t length);
+
+    /// Finalize agg_result into result_columns
+    void finalizeAggregates(MutableColumns & result_columns);
 
     /// Updates TTL for moves
     void updateMovesTTL(Block & block);
diff --git a/src/Parsers/ASTTTLElement.cpp b/src/Parsers/ASTTTLElement.cpp
index fb35266465b..35e9dc89a0d 100644
--- a/src/Parsers/ASTTTLElement.cpp
+++ b/src/Parsers/ASTTTLElement.cpp
@@ -11,10 +11,11 @@ ASTPtr ASTTTLElement::clone() const
 {
     auto clone = std::make_shared<ASTTTLElement>(*this);
     clone->children.clear();
-    clone->positions.clear();
+    clone->ttl_expr_pos = -1;
+    clone->where_expr_pos = -1;
     
-    for (auto expr : {Expression::TTL, Expression::WHERE})
-        clone->setExpression(expr, getExpression(expr, true));
+    clone->setExpression(clone->ttl_expr_pos, getExpression(ttl_expr_pos, true));
+    clone->setExpression(clone->where_expr_pos, getExpression(where_expr_pos, true));
 
     for (auto & [name, expr] : clone->group_by_aggregations)
         expr = expr->clone();
@@ -25,33 +26,33 @@ ASTPtr ASTTTLElement::clone() const
 void ASTTTLElement::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
 {
     ttl()->formatImpl(settings, state, frame);
-    if (mode == Mode::MOVE && destination_type == PartDestinationType::DISK)
+    if (mode == TTLMode::MOVE && destination_type == PartDestinationType::DISK)
     {
         settings.ostr << " TO DISK " << quoteString(destination_name);
     }
-    else if (mode == Mode::MOVE && destination_type == PartDestinationType::VOLUME)
+    else if (mode == TTLMode::MOVE && destination_type == PartDestinationType::VOLUME)
     {
         settings.ostr << " TO VOLUME " << quoteString(destination_name);
     }
-    else if (mode == Mode::GROUP_BY)
+    else if (mode == TTLMode::GROUP_BY)
     {
         settings.ostr << " GROUP BY ";
-        for (size_t i = 0; i < group_by_key_columns.size(); ++i)
+        for (auto it = group_by_key_columns.begin(); it != group_by_key_columns.end(); ++it)
         {
-            settings.ostr << group_by_key_columns[i];
-            if (i + 1 != group_by_key_columns.size())
+            if (it != group_by_key_columns.begin())
                 settings.ostr << ", ";
+            settings.ostr << *it;
         }
         settings.ostr << " SET ";
-        for (size_t i =  0; i < group_by_aggregations.size(); ++i)
+        for (auto it = group_by_aggregations.begin(); it != group_by_aggregations.end(); ++it)
         {
-            settings.ostr << group_by_aggregations[i].first << " = ";
-            group_by_aggregations[i].second->formatImpl(settings, state, frame);
-            if (i + 1 != group_by_aggregations.size())
+            if (it != group_by_aggregations.begin())
                 settings.ostr << ", ";
+            settings.ostr << it->first << " = ";
+            it->second->formatImpl(settings, state, frame);
         }
     }
-    else if (mode == Mode::DELETE)
+    else if (mode == TTLMode::DELETE)
     {
         /// It would be better to output "DELETE" here but that will break compatibility with earlier versions.
     }
@@ -63,24 +64,28 @@ void ASTTTLElement::formatImpl(const FormatSettings & settings, FormatState & st
     }
 }
 
-void ASTTTLElement::setExpression(Expression expr, ASTPtr && ast) 
+void ASTTTLElement::setExpression(int & pos, ASTPtr && ast) 
 {
-    auto it = positions.find(expr);
-    if (it == positions.end())
+    if (ast)
     {
-        positions[expr] = children.size();
-        children.emplace_back(ast);
+        if (pos == -1)
+        {
+            pos = children.size();
+            children.emplace_back(ast);
+        }
+        else
+            children[pos] = ast;
+    }
+    else if (pos != -1)
+    {
+        children[pos] = ASTPtr{};
+        pos = -1;
     }
-    else
-        children[it->second] = ast;
 }
 
-ASTPtr ASTTTLElement::getExpression(Expression expr, bool clone) const
+ASTPtr ASTTTLElement::getExpression(int  pos, bool clone) const 
 {
-    auto it = positions.find(expr);
-    if (it != positions.end())
-        return clone ? children[it->second]->clone() : children[it->second];
-    return {};
+    return pos != -1 ? (clone ? children[pos]->clone() : children[pos]) : ASTPtr{};
 }
 
 }
diff --git a/src/Parsers/ASTTTLElement.h b/src/Parsers/ASTTTLElement.h
index 1a2d7d3c723..24b02860f32 100644
--- a/src/Parsers/ASTTTLElement.h
+++ b/src/Parsers/ASTTTLElement.h
@@ -2,6 +2,7 @@
 
 #include <Parsers/IAST.h>
 #include <Storages/MergeTree/PartDestinationType.h>
+#include <Storages/MergeTree/TTLMode.h>
 
 
 namespace DB
@@ -12,29 +13,19 @@ namespace DB
 class ASTTTLElement : public IAST
 {
 public:
-    enum class Expression : uint8_t
-    {
-        TTL,
-        WHERE
-    };
-
-    enum class Mode : uint8_t
-    {
-        DELETE,
-        MOVE,
-        GROUP_BY
-    };
-
-    Mode mode;
+    TTLMode mode;
     PartDestinationType destination_type;
     String destination_name;
-    std::vector<String> group_by_key_columns;
+
+    Strings group_by_key_columns;
     std::vector<std::pair<String, ASTPtr>> group_by_aggregations;
 
-    ASTTTLElement(Mode mode_, PartDestinationType destination_type_, const String & destination_name_)
+    ASTTTLElement(TTLMode mode_, PartDestinationType destination_type_, const String & destination_name_)
         : mode(mode_) 
         , destination_type(destination_type_)
         , destination_name(destination_name_)
+        , ttl_expr_pos(-1)
+        , where_expr_pos(-1)
     {
     }
 
@@ -42,17 +33,22 @@ public:
 
     ASTPtr clone() const override;
 
-    const ASTPtr ttl() const { return getExpression(Expression::TTL); }
-    const ASTPtr where() const { return getExpression(Expression::WHERE); }
+    const ASTPtr ttl() const { return getExpression(ttl_expr_pos); }
+    const ASTPtr where() const { return getExpression(where_expr_pos); }
 
-    void setExpression(Expression expr, ASTPtr && ast);
-    ASTPtr getExpression(Expression expr, bool clone = false) const;
+    void setTTL(ASTPtr && ast) { setExpression(ttl_expr_pos, std::forward<ASTPtr>(ast)); }
+    void setWhere(ASTPtr && ast) { setExpression(where_expr_pos, std::forward<ASTPtr>(ast)); }
 
 protected:
     void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
 
+private: 
+    int ttl_expr_pos;
+    int where_expr_pos;
+
 private:
-    std::unordered_map<Expression, size_t> positions;
+    void setExpression(int & pos, ASTPtr && ast);    
+    ASTPtr getExpression(int pos, bool clone = false) const;
 };
 
 }
diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp
index 2734aa67193..afd3814d913 100644
--- a/src/Parsers/ExpressionElementParsers.cpp
+++ b/src/Parsers/ExpressionElementParsers.cpp
@@ -1460,45 +1460,45 @@ bool ParserTTLElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
     ParserKeyword s_set("SET");
     ParserToken s_comma(TokenType::Comma);
     ParserToken s_eq(TokenType::Equals);
+
     ParserIdentifier parser_identifier;
     ParserStringLiteral parser_string_literal;
     ParserExpression parser_exp;
     ParserIdentifierList parser_identifier_list;
 
-
     ASTPtr ttl_expr;
     if (!parser_exp.parse(pos, ttl_expr, expected))
         return false;
 
-    ASTPtr where_expr;
-
-    std::vector<String> group_by_key_columns;
-    std::vector<std::pair<String, ASTPtr>> group_by_aggregations;
-
-    ASTTTLElement::Mode mode;
+    TTLMode mode;
     PartDestinationType destination_type = PartDestinationType::DELETE;
     String destination_name;
+
     if (s_to_disk.ignore(pos))
     {
-        mode = ASTTTLElement::Mode::MOVE;
+        mode = TTLMode::MOVE;
         destination_type = PartDestinationType::DISK;
     }
     else if (s_to_volume.ignore(pos))
     {
-        mode = ASTTTLElement::Mode::MOVE;
+        mode = TTLMode::MOVE;
         destination_type = PartDestinationType::VOLUME;
     }
     else if (s_group_by.ignore(pos)) 
     {
-        mode = ASTTTLElement::Mode::GROUP_BY;
+        mode = TTLMode::GROUP_BY;
     }
     else
     {
         s_delete.ignore(pos);
-        mode = ASTTTLElement::Mode::DELETE;
+        mode = TTLMode::DELETE;
     }
 
-    if (mode == ASTTTLElement::Mode::MOVE)
+    ASTPtr where_expr;
+    std::vector<String> group_by_key_columns;
+    std::vector<std::pair<String, ASTPtr>> group_by_aggregations;
+
+    if (mode == TTLMode::MOVE)
     {
         ASTPtr ast_space_name;
         if (!parser_string_literal.parse(pos, ast_space_name, expected))
@@ -1506,7 +1506,7 @@ bool ParserTTLElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
 
         destination_name = ast_space_name->as<ASTLiteral &>().value.get<const String &>();
     }
-    else if (mode == ASTTTLElement::Mode::GROUP_BY)
+    else if (mode == TTLMode::GROUP_BY)
     {
         ASTPtr ast_group_by_key_columns;
         if (!parser_identifier_list.parse(pos, ast_group_by_key_columns, expected))
@@ -1541,17 +1541,16 @@ bool ParserTTLElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
             group_by_aggregations.emplace_back(name_str, std::move(value));
         }
     }
-
-    if ((mode == ASTTTLElement::Mode::MOVE || mode == ASTTTLElement::Mode::DELETE) && s_where.ignore(pos))
+    else if (mode == TTLMode::DELETE && s_where.ignore(pos))
     {
         if (!parser_exp.parse(pos, where_expr, expected))
             return false;
     }
 
     auto ttl_element = std::make_shared<ASTTTLElement>(mode, destination_type, destination_name);
-    ttl_element->setExpression(ASTTTLElement::Expression::TTL, std::move(ttl_expr));
+    ttl_element->setTTL(std::move(ttl_expr));
     if (where_expr)
-        ttl_element->setExpression(ASTTTLElement::Expression::WHERE, std::move(where_expr));
+        ttl_element->setWhere(std::move(where_expr));
 
     ttl_element->group_by_key_columns = std::move(group_by_key_columns);
     ttl_element->group_by_aggregations = std::move(group_by_aggregations);
diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 9eb14ed5475..1f55f5ecff5 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -624,6 +624,7 @@ void MergeTreeData::setTTLExpressions(const ColumnsDescription & new_columns,
         result.result_column = ttl_expr_ast->getColumnName();
 
         result.destination_type = PartDestinationType::DELETE;
+        result.mode = TTLMode::DELETE;
 
         checkTTLExpression(result.expression, result.result_column);
         return result;
@@ -632,27 +633,43 @@ void MergeTreeData::setTTLExpressions(const ColumnsDescription & new_columns,
     auto create_rows_ttl_entry = [this, &new_columns, &create_ttl_entry](const ASTTTLElement * ttl_element)
     {
         auto result = create_ttl_entry(ttl_element->ttl());
-        if (ttl_element->mode == ASTTTLElement::Mode::DELETE || ttl_element->mode ==ASTTTLElement::Mode::MOVE)         
+        result.mode = ttl_element->mode;
+
+        if (ttl_element->mode == TTLMode::DELETE)         
         {
             if (ASTPtr where_expr_ast = ttl_element->where()) 
             {
                 auto where_syntax_result = SyntaxAnalyzer(global_context).analyze(where_expr_ast, new_columns.getAllPhysical());
                 result.where_expression = ExpressionAnalyzer(where_expr_ast, where_syntax_result, global_context).getActions(false);
-                result.where_filter_column = where_expr_ast->getColumnName();
+                result.where_result_column = where_expr_ast->getColumnName();
             }
         }
-        else if (ttl_element->mode == ASTTTLElement::Mode::GROUP_BY) 
+        else if (ttl_element->mode == TTLMode::GROUP_BY) 
         {
+            if (ttl_element->group_by_key_columns.size() > this->primary_key_columns.size())
+                throw Exception("TTL Expression GROUP BY key should be a prefix of primary key", ErrorCodes::BAD_TTL_EXPRESSION);
+            for (size_t i = 0; i < ttl_element->group_by_key_columns.size(); ++i)
+            {
+                if (ttl_element->group_by_key_columns[i] != this->primary_key_columns[i])
+                    throw Exception("TTL Expression GROUP BY key should be a prefix of primary key", ErrorCodes::BAD_TTL_EXPRESSION);
+            }
+
             result.group_by_keys = ttl_element->group_by_key_columns;
-            for (auto [name, value] : ttl_element->group_by_aggregations)
+
+            auto aggregations = ttl_element->group_by_aggregations;
+            for (size_t i = ttl_element->group_by_key_columns.size(); i < this->primary_key_columns.size(); ++i)
+            {
+                ASTPtr expr = makeASTFunction("max", std::make_shared<ASTIdentifier>(this->primary_key_columns[i]));
+                aggregations.emplace_back(this->primary_key_columns[i], std::move(expr));
+            }
+            for (auto [name, value] : aggregations)
             {
                 auto syntax_result = SyntaxAnalyzer(global_context).analyze(value, new_columns.getAllPhysical(), {}, true);
                 auto expr_analyzer = ExpressionAnalyzer(value, syntax_result, global_context);
 
-                result.group_by_aggregations.emplace_back(name, expr_analyzer.getActions(false));
-                result.group_by_aggregations_res_column.emplace_back(name, value->getColumnName());
+                result.group_by_aggregations.emplace_back(name, value->getColumnName(), expr_analyzer.getActions(false));
 
-                for (const auto descr : expr_analyzer.getAnalyzedData().aggregate_descriptions)
+                for (const auto & descr : expr_analyzer.getAnalyzedData().aggregate_descriptions)
                     result.aggregate_descriptions.push_back(descr);
             }
         }
@@ -703,7 +720,6 @@ void MergeTreeData::setTTLExpressions(const ColumnsDescription & new_columns,
                     throw Exception("More than one DELETE TTL expression is not allowed", ErrorCodes::BAD_TTL_EXPRESSION);
                 }
 
-                LOG_DEBUG(log, "ttl_element->size is " << ttl_element->size());
                 auto new_rows_ttl_entry = create_rows_ttl_entry(ttl_element);
                 if (!only_check)
                     update_rows_ttl_entry = new_rows_ttl_entry;
diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h
index dfcddb40f3c..5272ee25314 100644
--- a/src/Storages/MergeTree/MergeTreeData.h
+++ b/src/Storages/MergeTree/MergeTreeData.h
@@ -21,6 +21,7 @@
 #include <Interpreters/PartLog.h>
 #include <Disks/StoragePolicy.h>
 #include <Interpreters/Aggregator.h>
+#include <Storages/MergeTree/TTLMode.h>
 
 #include <boost/multi_index_container.hpp>
 #include <boost/multi_index/ordered_index.hpp>
@@ -650,15 +651,16 @@ public:
 
     struct TTLEntry
     {
+        TTLMode mode;
+
         ExpressionActionsPtr expression;
         String result_column;
 
         ExpressionActionsPtr where_expression;
-        String where_filter_column;
+        String where_result_column;
 
         Names group_by_keys;
-        std::vector<std::pair<String, ExpressionActionsPtr>> group_by_aggregations;
-        std::vector<std::pair<String, String>> group_by_aggregations_res_column;
+        std::vector<std::tuple<String, String, ExpressionActionsPtr>> group_by_aggregations;
         AggregateDescriptions aggregate_descriptions;
 
         /// Name and type of a destination are only valid in table-level context.
diff --git a/src/Storages/MergeTree/TTLMode.h b/src/Storages/MergeTree/TTLMode.h
new file mode 100644
index 00000000000..99a57f46853
--- /dev/null
+++ b/src/Storages/MergeTree/TTLMode.h
@@ -0,0 +1,14 @@
+#pragma once
+
+
+namespace DB
+{
+
+enum class TTLMode 
+{
+    DELETE,
+    MOVE,
+    GROUP_BY
+};
+
+}

From 66496dc7e350971da8774f66517a4c3a1b62dbbe Mon Sep 17 00:00:00 2001
From: Nikolai Sorokin <sonikint@gmail.com>
Date: Tue, 12 May 2020 23:44:48 +0300
Subject: [PATCH 065/184] Fix trailing spaces

---
 src/DataStreams/TTLBlockInputStream.cpp  |  2 +-
 src/DataStreams/TTLBlockInputStream.h    |  4 ++--
 src/Interpreters/SyntaxAnalyzer.cpp      |  4 ++--
 src/Parsers/ASTTTLElement.cpp            | 10 +++++-----
 src/Parsers/ASTTTLElement.h              |  6 +++---
 src/Parsers/ExpressionElementParsers.cpp |  6 +++---
 src/Storages/MergeTree/MergeTreeData.cpp |  6 +++---
 src/Storages/MergeTree/TTLMode.h         |  2 +-
 8 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/src/DataStreams/TTLBlockInputStream.cpp b/src/DataStreams/TTLBlockInputStream.cpp
index d5e46a9929d..106ae1fd5a0 100644
--- a/src/DataStreams/TTLBlockInputStream.cpp
+++ b/src/DataStreams/TTLBlockInputStream.cpp
@@ -197,7 +197,7 @@ void TTLBlockInputStream::removeRowsWithExpiredTableTTL(Block & block)
 
         size_t rows_aggregated = 0;
         size_t current_key_start = 0;
-        size_t rows_with_current_key = 0; 
+        size_t rows_with_current_key = 0;
         for (size_t i = 0; i < block.rows(); ++i)
         {
             UInt32 cur_ttl = getTimestampByIndex(ttl_column, i);
diff --git a/src/DataStreams/TTLBlockInputStream.h b/src/DataStreams/TTLBlockInputStream.h
index c5ec6596038..a8f86477b88 100644
--- a/src/DataStreams/TTLBlockInputStream.h
+++ b/src/DataStreams/TTLBlockInputStream.h
@@ -43,7 +43,7 @@ private:
     std::unique_ptr<Aggregator> aggregator;
     std::vector<Field> current_key_value;
     AggregatedDataVariants agg_result;
-    ColumnRawPtrs agg_key_columns; 
+    ColumnRawPtrs agg_key_columns;
     Aggregator::AggregateColumns agg_aggregate_columns;
     bool agg_no_more_keys;
 
@@ -66,7 +66,7 @@ private:
 
     /// Removes rows with expired table ttl and computes new ttl_infos for part
     void removeRowsWithExpiredTableTTL(Block & block);
-    
+
     // Calculate aggregates of aggregate_columns into agg_result
     void calculateAggregates(const MutableColumns & aggregate_columns, size_t start_pos, size_t length);
 
diff --git a/src/Interpreters/SyntaxAnalyzer.cpp b/src/Interpreters/SyntaxAnalyzer.cpp
index ec7f2154dad..54f369c9495 100644
--- a/src/Interpreters/SyntaxAnalyzer.cpp
+++ b/src/Interpreters/SyntaxAnalyzer.cpp
@@ -855,7 +855,7 @@ SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyze(ASTPtr & query, const NamesAndTy
 
     optimizeIf(query, result.aliases, settings.optimize_if_chain_to_miltiif);
 
-    if (allow_aggregates) 
+    if (allow_aggregates)
     {
         GetAggregatesVisitor::Data data;
         GetAggregatesVisitor(data).visit(query);
@@ -866,7 +866,7 @@ SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyze(ASTPtr & query, const NamesAndTy
                 assertNoAggregates(arg, "inside another aggregate function");
         result.aggregates = data.aggregates;
     }
-    else 
+    else
         assertNoAggregates(query, "in wrong place");
 
     result.collectUsedColumns(query);
diff --git a/src/Parsers/ASTTTLElement.cpp b/src/Parsers/ASTTTLElement.cpp
index 35e9dc89a0d..991274e8350 100644
--- a/src/Parsers/ASTTTLElement.cpp
+++ b/src/Parsers/ASTTTLElement.cpp
@@ -7,13 +7,13 @@
 namespace DB
 {
 
-ASTPtr ASTTTLElement::clone() const 
+ASTPtr ASTTTLElement::clone() const
 {
     auto clone = std::make_shared<ASTTTLElement>(*this);
     clone->children.clear();
     clone->ttl_expr_pos = -1;
     clone->where_expr_pos = -1;
-    
+
     clone->setExpression(clone->ttl_expr_pos, getExpression(ttl_expr_pos, true));
     clone->setExpression(clone->where_expr_pos, getExpression(where_expr_pos, true));
 
@@ -57,14 +57,14 @@ void ASTTTLElement::formatImpl(const FormatSettings & settings, FormatState & st
         /// It would be better to output "DELETE" here but that will break compatibility with earlier versions.
     }
 
-    if (where()) 
+    if (where())
     {
         settings.ostr << " WHERE ";
         where()->formatImpl(settings, state, frame);
     }
 }
 
-void ASTTTLElement::setExpression(int & pos, ASTPtr && ast) 
+void ASTTTLElement::setExpression(int & pos, ASTPtr && ast)
 {
     if (ast)
     {
@@ -83,7 +83,7 @@ void ASTTTLElement::setExpression(int & pos, ASTPtr && ast)
     }
 }
 
-ASTPtr ASTTTLElement::getExpression(int  pos, bool clone) const 
+ASTPtr ASTTTLElement::getExpression(int  pos, bool clone) const
 {
     return pos != -1 ? (clone ? children[pos]->clone() : children[pos]) : ASTPtr{};
 }
diff --git a/src/Parsers/ASTTTLElement.h b/src/Parsers/ASTTTLElement.h
index 24b02860f32..9b40adf2be4 100644
--- a/src/Parsers/ASTTTLElement.h
+++ b/src/Parsers/ASTTTLElement.h
@@ -21,7 +21,7 @@ public:
     std::vector<std::pair<String, ASTPtr>> group_by_aggregations;
 
     ASTTTLElement(TTLMode mode_, PartDestinationType destination_type_, const String & destination_name_)
-        : mode(mode_) 
+        : mode(mode_)
         , destination_type(destination_type_)
         , destination_name(destination_name_)
         , ttl_expr_pos(-1)
@@ -42,12 +42,12 @@ public:
 protected:
     void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
 
-private: 
+private:
     int ttl_expr_pos;
     int where_expr_pos;
 
 private:
-    void setExpression(int & pos, ASTPtr && ast);    
+    void setExpression(int & pos, ASTPtr && ast);
     ASTPtr getExpression(int pos, bool clone = false) const;
 };
 
diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp
index afd3814d913..65ff9e1da94 100644
--- a/src/Parsers/ExpressionElementParsers.cpp
+++ b/src/Parsers/ExpressionElementParsers.cpp
@@ -1484,7 +1484,7 @@ bool ParserTTLElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
         mode = TTLMode::MOVE;
         destination_type = PartDestinationType::VOLUME;
     }
-    else if (s_group_by.ignore(pos)) 
+    else if (s_group_by.ignore(pos))
     {
         mode = TTLMode::GROUP_BY;
     }
@@ -1517,7 +1517,7 @@ bool ParserTTLElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
             if (!tryGetIdentifierNameInto(identifier, identifier_str))
                 return false;
             group_by_key_columns.emplace_back(std::move(identifier_str));
-        }        
+        }
 
         if (!s_set.ignore(pos))
             return false;
@@ -1525,7 +1525,7 @@ bool ParserTTLElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
         {
             if (!group_by_aggregations.empty() && !s_comma.ignore(pos))
                 break;
-            
+
             ASTPtr name;
             ASTPtr value;
             if (!parser_identifier.parse(pos, name, expected))
diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 1f55f5ecff5..bb9d8ce186e 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -635,16 +635,16 @@ void MergeTreeData::setTTLExpressions(const ColumnsDescription & new_columns,
         auto result = create_ttl_entry(ttl_element->ttl());
         result.mode = ttl_element->mode;
 
-        if (ttl_element->mode == TTLMode::DELETE)         
+        if (ttl_element->mode == TTLMode::DELETE)
         {
-            if (ASTPtr where_expr_ast = ttl_element->where()) 
+            if (ASTPtr where_expr_ast = ttl_element->where())
             {
                 auto where_syntax_result = SyntaxAnalyzer(global_context).analyze(where_expr_ast, new_columns.getAllPhysical());
                 result.where_expression = ExpressionAnalyzer(where_expr_ast, where_syntax_result, global_context).getActions(false);
                 result.where_result_column = where_expr_ast->getColumnName();
             }
         }
-        else if (ttl_element->mode == TTLMode::GROUP_BY) 
+        else if (ttl_element->mode == TTLMode::GROUP_BY)
         {
             if (ttl_element->group_by_key_columns.size() > this->primary_key_columns.size())
                 throw Exception("TTL Expression GROUP BY key should be a prefix of primary key", ErrorCodes::BAD_TTL_EXPRESSION);
diff --git a/src/Storages/MergeTree/TTLMode.h b/src/Storages/MergeTree/TTLMode.h
index 99a57f46853..0681f10fc17 100644
--- a/src/Storages/MergeTree/TTLMode.h
+++ b/src/Storages/MergeTree/TTLMode.h
@@ -4,7 +4,7 @@
 namespace DB
 {
 
-enum class TTLMode 
+enum class TTLMode
 {
     DELETE,
     MOVE,

From a991dcf9f4f7d5552b9406548e414432fd237c41 Mon Sep 17 00:00:00 2001
From: Nikolai Sorokin <sonikint@gmail.com>
Date: Thu, 14 May 2020 13:48:45 +0300
Subject: [PATCH 066/184] Add tests

---
 .../01246_ttl_where_group_by.reference        | 15 +++++++
 .../0_stateless/01246_ttl_where_group_by.sql  | 41 +++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 tests/queries/0_stateless/01246_ttl_where_group_by.reference
 create mode 100644 tests/queries/0_stateless/01246_ttl_where_group_by.sql

diff --git a/tests/queries/0_stateless/01246_ttl_where_group_by.reference b/tests/queries/0_stateless/01246_ttl_where_group_by.reference
new file mode 100644
index 00000000000..7e276e2681e
--- /dev/null
+++ b/tests/queries/0_stateless/01246_ttl_where_group_by.reference
@@ -0,0 +1,15 @@
+1	1	0	4
+1	2	3	7
+1	3	0	5
+2	1	20	1
+2	1	0	1
+1	1	0	4
+1	1	10	6
+1	2	3	7
+1	3	0	5
+2	1	10	1
+3	1	0	8
+1	1	0	4
+1	3	10	6
+2	1	20	1
+3	1	0	8
diff --git a/tests/queries/0_stateless/01246_ttl_where_group_by.sql b/tests/queries/0_stateless/01246_ttl_where_group_by.sql
new file mode 100644
index 00000000000..fbbd0336e1b
--- /dev/null
+++ b/tests/queries/0_stateless/01246_ttl_where_group_by.sql
@@ -0,0 +1,41 @@
+drop table if exists ttl_01246_1;
+
+create table ttl_01246_1 (a Int, b Int, x Int, y Int, d DateTime) engine = MergeTree order by (a, b) ttl d + interval 1 second delete where x % 10 == 0 and y > 5;
+insert into ttl_01246_1 values (1, 1, 0, 4, now() + 10);
+insert into ttl_01246_1 values (1, 1, 10, 6, now());
+insert into ttl_01246_1 values (1, 2, 3, 7, now());
+insert into ttl_01246_1 values (1, 3, 0, 5, now());
+insert into ttl_01246_1 values (2, 1, 20, 1, now());
+insert into ttl_01246_1 values (2, 1, 0, 1, now());
+insert into ttl_01246_1 values (3, 1, 0, 8, now());
+select sleep(1.1) format Null;
+optimize table ttl_01246_1 final;
+select a, b, x, y from ttl_01246_1;
+
+drop table if exists ttl_01246_1;
+
+create table ttl_01246_1 (a Int, b Int, x Int32, y Double, d DateTime) engine = MergeTree order by (a, b) ttl d + interval 1 second group by a, b set x = cast(median(x) as Int32), y = avg(y), d = max(d);
+insert into ttl_01246_1 values (1, 1, 0, 4, now() + 10);
+insert into ttl_01246_1 values (1, 1, 10, 6, now());
+insert into ttl_01246_1 values (1, 2, 3, 7, now());
+insert into ttl_01246_1 values (1, 3, 0, 5, now());
+insert into ttl_01246_1 values (2, 1, 20, 1, now());
+insert into ttl_01246_1 values (2, 1, 0, 1, now());
+insert into ttl_01246_1 values (3, 1, 0, 8, now());
+select sleep(1.1) format Null;
+optimize table ttl_01246_1 final;
+select a, b, x, y from ttl_01246_1;
+
+drop table if exists ttl_01246_1;
+
+create table ttl_01246_1 (a Int, b Int, x Int32, y Int, d DateTime) engine = MergeTree order by (a, b) ttl d + interval 1 second group by a set x = max(x), y = cast(round(avg(y)) as Int), d = max(d);
+insert into ttl_01246_1 values (1, 1, 0, 4, now() + 10);
+insert into ttl_01246_1 values (1, 1, 10, 6, now());
+insert into ttl_01246_1 values (1, 2, 3, 7, now());
+insert into ttl_01246_1 values (1, 3, 0, 5, now());
+insert into ttl_01246_1 values (2, 1, 20, 1, now());
+insert into ttl_01246_1 values (2, 1, 0, 1, now());
+insert into ttl_01246_1 values (3, 1, 0, 8, now());
+select sleep(1.1) format Null;
+optimize table ttl_01246_1 final;
+select a, b, x, y from ttl_01246_1;
\ No newline at end of file

From 9269db072af72d5f8c5dbf6483d21b50bdad4ac9 Mon Sep 17 00:00:00 2001
From: Nikolai Sorokin <sonikint@gmail.com>
Date: Thu, 14 May 2020 17:07:36 +0300
Subject: [PATCH 067/184] Better tests

---
 .../0_stateless/01246_ttl_where_group_by.reference   | 11 +++++------
 .../queries/0_stateless/01246_ttl_where_group_by.sql | 12 ++++++++----
 2 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/tests/queries/0_stateless/01246_ttl_where_group_by.reference b/tests/queries/0_stateless/01246_ttl_where_group_by.reference
index 7e276e2681e..69d3d77cd46 100644
--- a/tests/queries/0_stateless/01246_ttl_where_group_by.reference
+++ b/tests/queries/0_stateless/01246_ttl_where_group_by.reference
@@ -4,12 +4,11 @@
 2	1	20	1
 2	1	0	1
 1	1	0	4
-1	1	10	6
-1	2	3	7
-1	3	0	5
-2	1	10	1
+1	1	6	6.5
+1	3	4	5.666666666666667
+2	1	10	4
 3	1	0	8
 1	1	0	4
 1	3	10	6
-2	1	20	1
-3	1	0	8
+2	1	20	2
+3	5	8	4
diff --git a/tests/queries/0_stateless/01246_ttl_where_group_by.sql b/tests/queries/0_stateless/01246_ttl_where_group_by.sql
index fbbd0336e1b..ad969127eb1 100644
--- a/tests/queries/0_stateless/01246_ttl_where_group_by.sql
+++ b/tests/queries/0_stateless/01246_ttl_where_group_by.sql
@@ -17,9 +17,11 @@ drop table if exists ttl_01246_1;
 create table ttl_01246_1 (a Int, b Int, x Int32, y Double, d DateTime) engine = MergeTree order by (a, b) ttl d + interval 1 second group by a, b set x = cast(median(x) as Int32), y = avg(y), d = max(d);
 insert into ttl_01246_1 values (1, 1, 0, 4, now() + 10);
 insert into ttl_01246_1 values (1, 1, 10, 6, now());
-insert into ttl_01246_1 values (1, 2, 3, 7, now());
+insert into ttl_01246_1 values (1, 1, 3, 7, now());
 insert into ttl_01246_1 values (1, 3, 0, 5, now());
-insert into ttl_01246_1 values (2, 1, 20, 1, now());
+insert into ttl_01246_1 values (1, 3, 4, 9, now());
+insert into ttl_01246_1 values (1, 3, 6, 3, now());
+insert into ttl_01246_1 values (2, 1, 20, 7, now());
 insert into ttl_01246_1 values (2, 1, 0, 1, now());
 insert into ttl_01246_1 values (3, 1, 0, 8, now());
 select sleep(1.1) format Null;
@@ -34,8 +36,10 @@ insert into ttl_01246_1 values (1, 1, 10, 6, now());
 insert into ttl_01246_1 values (1, 2, 3, 7, now());
 insert into ttl_01246_1 values (1, 3, 0, 5, now());
 insert into ttl_01246_1 values (2, 1, 20, 1, now());
-insert into ttl_01246_1 values (2, 1, 0, 1, now());
-insert into ttl_01246_1 values (3, 1, 0, 8, now());
+insert into ttl_01246_1 values (2, 1, 0, 3, now());
+insert into ttl_01246_1 values (3, 1, 0, 3, now());
+insert into ttl_01246_1 values (3, 2, 8, 2, now());
+insert into ttl_01246_1 values (3, 5, 5, 8, now());
 select sleep(1.1) format Null;
 optimize table ttl_01246_1 final;
 select a, b, x, y from ttl_01246_1;
\ No newline at end of file

From 1f02ba9692b7152dc82c3ce8f0e6f62ec5aa120f Mon Sep 17 00:00:00 2001
From: Nikolai Sorokin <sonikint@gmail.com>
Date: Thu, 14 May 2020 21:46:15 +0300
Subject: [PATCH 068/184] Add eof

---
 src/Parsers/ExpressionListParsers.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Parsers/ExpressionListParsers.h b/src/Parsers/ExpressionListParsers.h
index ba939d8d160..9cc36967bd7 100644
--- a/src/Parsers/ExpressionListParsers.h
+++ b/src/Parsers/ExpressionListParsers.h
@@ -430,4 +430,4 @@ protected:
     bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
 };
 
-}
\ No newline at end of file
+}

From c6541b1dad1fe95644a9dc1c916dbd162be5d37e Mon Sep 17 00:00:00 2001
From: Nikolai Sorokin <sonikint@gmail.com>
Date: Fri, 15 May 2020 02:32:45 +0300
Subject: [PATCH 069/184] Up test id; Fix clang build

---
 src/Interpreters/SyntaxAnalyzer.cpp                           | 4 ++--
 src/Parsers/ExpressionElementParsers.cpp                      | 2 +-
 src/Parsers/ExpressionListParsers.cpp                         | 2 +-
 ..._group_by.reference => 01280_ttl_where_group_by.reference} | 0
 ...46_ttl_where_group_by.sql => 01280_ttl_where_group_by.sql} | 0
 5 files changed, 4 insertions(+), 4 deletions(-)
 rename tests/queries/0_stateless/{01246_ttl_where_group_by.reference => 01280_ttl_where_group_by.reference} (100%)
 rename tests/queries/0_stateless/{01246_ttl_where_group_by.sql => 01280_ttl_where_group_by.sql} (100%)

diff --git a/src/Interpreters/SyntaxAnalyzer.cpp b/src/Interpreters/SyntaxAnalyzer.cpp
index 54f369c9495..b5f86b87fdc 100644
--- a/src/Interpreters/SyntaxAnalyzer.cpp
+++ b/src/Interpreters/SyntaxAnalyzer.cpp
@@ -839,7 +839,7 @@ SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyzeSelect(
     return std::make_shared<const SyntaxAnalyzerResult>(result);
 }
 
-SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyze(ASTPtr & query, const NamesAndTypesList & source_columns, ConstStoragePtr storage, bool allow_aggregates) const
+SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyze(ASTPtr & query, const NamesAndTypesList & source_columns, ConstStoragePtr storage, bool allow_aggregations) const
 {
     if (query->as<ASTSelectQuery>())
         throw Exception("Not select analyze for select asts.", ErrorCodes::LOGICAL_ERROR);
@@ -855,7 +855,7 @@ SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyze(ASTPtr & query, const NamesAndTy
 
     optimizeIf(query, result.aliases, settings.optimize_if_chain_to_miltiif);
 
-    if (allow_aggregates)
+    if (allow_aggregations)
     {
         GetAggregatesVisitor::Data data;
         GetAggregatesVisitor(data).visit(query);
diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp
index 65ff9e1da94..5b5d9f05b22 100644
--- a/src/Parsers/ExpressionElementParsers.cpp
+++ b/src/Parsers/ExpressionElementParsers.cpp
@@ -1511,7 +1511,7 @@ bool ParserTTLElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
         ASTPtr ast_group_by_key_columns;
         if (!parser_identifier_list.parse(pos, ast_group_by_key_columns, expected))
             return false;
-        for (const auto identifier : ast_group_by_key_columns->children)
+        for (const auto & identifier : ast_group_by_key_columns->children)
         {
             String identifier_str;
             if (!tryGetIdentifierNameInto(identifier, identifier_str))
diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp
index 062d1dd4483..2561bf7ef83 100644
--- a/src/Parsers/ExpressionListParsers.cpp
+++ b/src/Parsers/ExpressionListParsers.cpp
@@ -751,4 +751,4 @@ bool ParserIdentifierList::parseImpl(Pos & pos, ASTPtr & node, Expected & expect
         .parse(pos, node, expected);
 }
 
-}
\ No newline at end of file
+}
diff --git a/tests/queries/0_stateless/01246_ttl_where_group_by.reference b/tests/queries/0_stateless/01280_ttl_where_group_by.reference
similarity index 100%
rename from tests/queries/0_stateless/01246_ttl_where_group_by.reference
rename to tests/queries/0_stateless/01280_ttl_where_group_by.reference
diff --git a/tests/queries/0_stateless/01246_ttl_where_group_by.sql b/tests/queries/0_stateless/01280_ttl_where_group_by.sql
similarity index 100%
rename from tests/queries/0_stateless/01246_ttl_where_group_by.sql
rename to tests/queries/0_stateless/01280_ttl_where_group_by.sql

From 6585571ee2e8d8b0a832e4b51f0e40762d096a4b Mon Sep 17 00:00:00 2001
From: Nikolai Sorokin <sonikint@gmail.com>
Date: Sat, 16 May 2020 02:53:47 +0300
Subject: [PATCH 070/184] Fix bug

---
 src/Interpreters/ExpressionAnalyzer.cpp | 89 +++++++++++++------------
 1 file changed, 45 insertions(+), 44 deletions(-)

diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp
index c432b920a9f..b38af6feef9 100644
--- a/src/Interpreters/ExpressionAnalyzer.cpp
+++ b/src/Interpreters/ExpressionAnalyzer.cpp
@@ -192,61 +192,65 @@ void ExpressionAnalyzer::analyzeAggregation()
 
     if (has_aggregation)
     {
-        // getSelectQuery(); /// assertSelect()
 
         /// Find out aggregation keys.
-        if (select_query && select_query->groupBy())
+        if (select_query)
         {
-            NameSet unique_keys;
-            ASTs & group_asts = select_query->groupBy()->children;
-            for (ssize_t i = 0; i < ssize_t(group_asts.size()); ++i)
+            if (select_query->groupBy())
             {
-                ssize_t size = group_asts.size();
-                getRootActionsNoMakeSet(group_asts[i], true, temp_actions, false);
-
-                const auto & column_name = group_asts[i]->getColumnName();
-                const auto & block = temp_actions->getSampleBlock();
-
-                if (!block.has(column_name))
-                    throw Exception("Unknown identifier (in GROUP BY): " + column_name, ErrorCodes::UNKNOWN_IDENTIFIER);
-
-                const auto & col = block.getByName(column_name);
-
-                /// Constant expressions have non-null column pointer at this stage.
-                if (col.column && isColumnConst(*col.column))
+                NameSet unique_keys;
+                ASTs & group_asts = select_query->groupBy()->children;
+                for (ssize_t i = 0; i < ssize_t(group_asts.size()); ++i)
                 {
-                    /// But don't remove last key column if no aggregate functions, otherwise aggregation will not work.
-                    if (!aggregate_descriptions.empty() || size > 1)
+                    ssize_t size = group_asts.size();
+                    getRootActionsNoMakeSet(group_asts[i], true, temp_actions, false);
+
+                    const auto & column_name = group_asts[i]->getColumnName();
+                    const auto & block = temp_actions->getSampleBlock();
+
+                    if (!block.has(column_name))
+                        throw Exception("Unknown identifier (in GROUP BY): " + column_name, ErrorCodes::UNKNOWN_IDENTIFIER);
+
+                    const auto & col = block.getByName(column_name);
+
+                    /// Constant expressions have non-null column pointer at this stage.
+                    if (col.column && isColumnConst(*col.column))
                     {
-                        if (i + 1 < static_cast<ssize_t>(size))
-                            group_asts[i] = std::move(group_asts.back());
+                        /// But don't remove last key column if no aggregate functions, otherwise aggregation will not work.
+                        if (!aggregate_descriptions.empty() || size > 1)
+                        {
+                            if (i + 1 < static_cast<ssize_t>(size))
+                                group_asts[i] = std::move(group_asts.back());
 
-                        group_asts.pop_back();
+                            group_asts.pop_back();
 
-                        --i;
-                        continue;
+                            --i;
+                            continue;
+                        }
+                    }
+
+                    NameAndTypePair key{column_name, col.type};
+
+                    /// Aggregation keys are uniqued.
+                    if (!unique_keys.count(key.name))
+                    {
+                        unique_keys.insert(key.name);
+                        aggregation_keys.push_back(key);
+
+                        /// Key is no longer needed, therefore we can save a little by moving it.
+                        aggregated_columns.push_back(std::move(key));
                     }
                 }
 
-                NameAndTypePair key{column_name, col.type};
-
-                /// Aggregation keys are uniqued.
-                if (!unique_keys.count(key.name))
+                if (group_asts.empty())
                 {
-                    unique_keys.insert(key.name);
-                    aggregation_keys.push_back(key);
-
-                    /// Key is no longer needed, therefore we can save a little by moving it.
-                    aggregated_columns.push_back(std::move(key));
+                    select_query->setExpression(ASTSelectQuery::Expression::GROUP_BY, {});
+                    has_aggregation = select_query->having() || !aggregate_descriptions.empty();
                 }
             }
-
-            if (group_asts.empty())
-            {
-                select_query->setExpression(ASTSelectQuery::Expression::GROUP_BY, {});
-                has_aggregation = select_query->having() || !aggregate_descriptions.empty();
-            }
         }
+        else
+            aggregated_columns = temp_actions->getSampleBlock().getNamesAndTypesList();
 
         for (const auto & desc : aggregate_descriptions)
             aggregated_columns.emplace_back(desc.column_name, desc.function->getReturnType());
@@ -926,10 +930,7 @@ void ExpressionAnalyzer::appendExpression(ExpressionActionsChain & chain, const
 
 ExpressionActionsPtr ExpressionAnalyzer::getActions(bool add_aliases, bool project_result)
 {
-    NamesAndTypesList columns(sourceColumns());
-    for (const auto & col : aggregated_columns)
-        columns.push_back(col);
-    ExpressionActionsPtr actions = std::make_shared<ExpressionActions>(columns, context);
+    ExpressionActionsPtr actions = std::make_shared<ExpressionActions>(aggregated_columns, context);
     NamesWithAliases result_columns;
     Names result_names;
 

From ed09f68b8dbead260ee3506ff3533616259dca04 Mon Sep 17 00:00:00 2001
From: Nikolai Sorokin <sonikint@gmail.com>
Date: Sat, 16 May 2020 19:31:29 +0300
Subject: [PATCH 071/184] Better tests

---
 .../01280_ttl_where_group_by.reference        | 14 ++--
 .../0_stateless/01280_ttl_where_group_by.sql  | 76 +++++++++----------
 2 files changed, 45 insertions(+), 45 deletions(-)

diff --git a/tests/queries/0_stateless/01280_ttl_where_group_by.reference b/tests/queries/0_stateless/01280_ttl_where_group_by.reference
index 69d3d77cd46..572fc7731d3 100644
--- a/tests/queries/0_stateless/01280_ttl_where_group_by.reference
+++ b/tests/queries/0_stateless/01280_ttl_where_group_by.reference
@@ -3,12 +3,12 @@
 1	3	0	5
 2	1	20	1
 2	1	0	1
-1	1	0	4
-1	1	6	6.5
-1	3	4	5.666666666666667
-2	1	10	4
-3	1	0	8
+1	1	[0,2,3]	4
+1	1	[5,4,1]	13
+1	3	[1,0,1,0]	17
+2	1	[3,1,0,3]	8
+3	1	[2,4,5]	8
 1	1	0	4
 1	3	10	6
-2	1	20	2
-3	5	8	4
+2	1	0	3
+3	5	8	2
diff --git a/tests/queries/0_stateless/01280_ttl_where_group_by.sql b/tests/queries/0_stateless/01280_ttl_where_group_by.sql
index ad969127eb1..0ca06e77001 100644
--- a/tests/queries/0_stateless/01280_ttl_where_group_by.sql
+++ b/tests/queries/0_stateless/01280_ttl_where_group_by.sql
@@ -1,45 +1,45 @@
-drop table if exists ttl_01246_1;
+drop table if exists ttl_01280_1;
 
-create table ttl_01246_1 (a Int, b Int, x Int, y Int, d DateTime) engine = MergeTree order by (a, b) ttl d + interval 1 second delete where x % 10 == 0 and y > 5;
-insert into ttl_01246_1 values (1, 1, 0, 4, now() + 10);
-insert into ttl_01246_1 values (1, 1, 10, 6, now());
-insert into ttl_01246_1 values (1, 2, 3, 7, now());
-insert into ttl_01246_1 values (1, 3, 0, 5, now());
-insert into ttl_01246_1 values (2, 1, 20, 1, now());
-insert into ttl_01246_1 values (2, 1, 0, 1, now());
-insert into ttl_01246_1 values (3, 1, 0, 8, now());
+create table ttl_01280_1 (a Int, b Int, x Int, y Int, d DateTime) engine = MergeTree order by (a, b) ttl d + interval 1 second delete where x % 10 == 0 and y > 5;
+insert into ttl_01280_1 values (1, 1, 0, 4, now() + 10);
+insert into ttl_01280_1 values (1, 1, 10, 6, now());
+insert into ttl_01280_1 values (1, 2, 3, 7, now());
+insert into ttl_01280_1 values (1, 3, 0, 5, now());
+insert into ttl_01280_1 values (2, 1, 20, 1, now());
+insert into ttl_01280_1 values (2, 1, 0, 1, now());
+insert into ttl_01280_1 values (3, 1, 0, 8, now());
 select sleep(1.1) format Null;
-optimize table ttl_01246_1 final;
-select a, b, x, y from ttl_01246_1;
+optimize table ttl_01280_1 final;
+select a, b, x, y from ttl_01280_1;
 
-drop table if exists ttl_01246_1;
+drop table if exists ttl_01280_2;
 
-create table ttl_01246_1 (a Int, b Int, x Int32, y Double, d DateTime) engine = MergeTree order by (a, b) ttl d + interval 1 second group by a, b set x = cast(median(x) as Int32), y = avg(y), d = max(d);
-insert into ttl_01246_1 values (1, 1, 0, 4, now() + 10);
-insert into ttl_01246_1 values (1, 1, 10, 6, now());
-insert into ttl_01246_1 values (1, 1, 3, 7, now());
-insert into ttl_01246_1 values (1, 3, 0, 5, now());
-insert into ttl_01246_1 values (1, 3, 4, 9, now());
-insert into ttl_01246_1 values (1, 3, 6, 3, now());
-insert into ttl_01246_1 values (2, 1, 20, 7, now());
-insert into ttl_01246_1 values (2, 1, 0, 1, now());
-insert into ttl_01246_1 values (3, 1, 0, 8, now());
+create table ttl_01280_2 (a Int, b Int, x Array(Int32), y Double, d DateTime) engine = MergeTree order by (a, b) ttl d + interval 1 second group by a, b set x = minForEach(x), y = sum(y), d = max(d);
+insert into ttl_01280_2 values (1, 1, array(0, 2, 3), 4, now() + 10);
+insert into ttl_01280_2 values (1, 1, array(5, 4, 3), 6, now());
+insert into ttl_01280_2 values (1, 1, array(5, 5, 1), 7, now());
+insert into ttl_01280_2 values (1, 3, array(3, 0, 4), 5, now());
+insert into ttl_01280_2 values (1, 3, array(1, 1, 2, 1), 9, now());
+insert into ttl_01280_2 values (1, 3, array(3, 2, 1, 0), 3, now());
+insert into ttl_01280_2 values (2, 1, array(3, 3, 3), 7, now());
+insert into ttl_01280_2 values (2, 1, array(11, 1, 0, 3), 1, now());
+insert into ttl_01280_2 values (3, 1, array(2, 4, 5), 8, now());
 select sleep(1.1) format Null;
-optimize table ttl_01246_1 final;
-select a, b, x, y from ttl_01246_1;
+optimize table ttl_01280_2 final;
+select a, b, x, y from ttl_01280_2;
 
-drop table if exists ttl_01246_1;
+drop table if exists ttl_01280_3;
 
-create table ttl_01246_1 (a Int, b Int, x Int32, y Int, d DateTime) engine = MergeTree order by (a, b) ttl d + interval 1 second group by a set x = max(x), y = cast(round(avg(y)) as Int), d = max(d);
-insert into ttl_01246_1 values (1, 1, 0, 4, now() + 10);
-insert into ttl_01246_1 values (1, 1, 10, 6, now());
-insert into ttl_01246_1 values (1, 2, 3, 7, now());
-insert into ttl_01246_1 values (1, 3, 0, 5, now());
-insert into ttl_01246_1 values (2, 1, 20, 1, now());
-insert into ttl_01246_1 values (2, 1, 0, 3, now());
-insert into ttl_01246_1 values (3, 1, 0, 3, now());
-insert into ttl_01246_1 values (3, 2, 8, 2, now());
-insert into ttl_01246_1 values (3, 5, 5, 8, now());
-select sleep(1.1) format Null;
-optimize table ttl_01246_1 final;
-select a, b, x, y from ttl_01246_1;
\ No newline at end of file
+create table ttl_01280_3 (a Int, b Int, x Int64, y Int, d DateTime) engine = MergeTree order by (a, b) ttl d + interval 1 second group by a set x = argMax(x, d), y = argMax(y, d), d = max(d);
+insert into ttl_01280_3 values (1, 1, 0, 4, now() + 10);
+insert into ttl_01280_3 values (1, 1, 10, 6, now() + 1);
+insert into ttl_01280_3 values (1, 2, 3, 7, now());
+insert into ttl_01280_3 values (1, 3, 0, 5, now());
+insert into ttl_01280_3 values (2, 1, 20, 1, now());
+insert into ttl_01280_3 values (2, 1, 0, 3, now() + 1);
+insert into ttl_01280_3 values (3, 1, 0, 3, now());
+insert into ttl_01280_3 values (3, 2, 8, 2, now() + 1);
+insert into ttl_01280_3 values (3, 5, 5, 8, now());
+select sleep(2.1) format Null;
+optimize table ttl_01280_3 final;
+select a, b, x, y from ttl_01280_3;
\ No newline at end of file

From 7d937b43c7f0248069bdb7b1686244ce9d0c0f2b Mon Sep 17 00:00:00 2001
From: Nikolai Sorokin <sonikint@gmail.com>
Date: Sun, 17 May 2020 16:01:38 +0300
Subject: [PATCH 072/184] Init with correct value

---
 src/DataStreams/TTLBlockInputStream.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/DataStreams/TTLBlockInputStream.h b/src/DataStreams/TTLBlockInputStream.h
index a8f86477b88..c6ffa95cd75 100644
--- a/src/DataStreams/TTLBlockInputStream.h
+++ b/src/DataStreams/TTLBlockInputStream.h
@@ -45,7 +45,7 @@ private:
     AggregatedDataVariants agg_result;
     ColumnRawPtrs agg_key_columns;
     Aggregator::AggregateColumns agg_aggregate_columns;
-    bool agg_no_more_keys;
+    bool agg_no_more_keys = false;
 
     IMergeTreeDataPart::TTLInfos old_ttl_infos;
     IMergeTreeDataPart::TTLInfos new_ttl_infos;

From 141ed887515e6523c192690baf66dd1c2ca3a3b8 Mon Sep 17 00:00:00 2001
From: Nikolai Sorokin <sonikint@gmail.com>
Date: Sun, 24 May 2020 23:21:23 +0300
Subject: [PATCH 073/184] Allow functions in group by keys; Add default
 aggregate function; Add more tests

---
 src/Parsers/ASTTTLElement.cpp                 | 23 ++++---
 src/Parsers/ASTTTLElement.h                   |  2 +-
 src/Parsers/ExpressionElementParsers.cpp      | 58 ++++++++---------
 src/Parsers/ExpressionListParsers.cpp         |  9 ---
 src/Parsers/ExpressionListParsers.h           |  9 ---
 src/Storages/MergeTree/MergeTreeData.cpp      | 62 ++++++++++++++++---
 .../01280_ttl_where_group_by.reference        |  6 ++
 .../0_stateless/01280_ttl_where_group_by.sql  | 44 ++++++++++++-
 8 files changed, 146 insertions(+), 67 deletions(-)

diff --git a/src/Parsers/ASTTTLElement.cpp b/src/Parsers/ASTTTLElement.cpp
index 991274e8350..33f8a3ac906 100644
--- a/src/Parsers/ASTTTLElement.cpp
+++ b/src/Parsers/ASTTTLElement.cpp
@@ -17,6 +17,8 @@ ASTPtr ASTTTLElement::clone() const
     clone->setExpression(clone->ttl_expr_pos, getExpression(ttl_expr_pos, true));
     clone->setExpression(clone->where_expr_pos, getExpression(where_expr_pos, true));
 
+    for (auto & expr : clone->group_by_key)
+        expr = expr->clone();
     for (auto & [name, expr] : clone->group_by_aggregations)
         expr = expr->clone();
 
@@ -37,19 +39,22 @@ void ASTTTLElement::formatImpl(const FormatSettings & settings, FormatState & st
     else if (mode == TTLMode::GROUP_BY)
     {
         settings.ostr << " GROUP BY ";
-        for (auto it = group_by_key_columns.begin(); it != group_by_key_columns.end(); ++it)
+        for (auto it = group_by_key.begin(); it != group_by_key.end(); ++it)
         {
-            if (it != group_by_key_columns.begin())
+            if (it != group_by_key.begin())
                 settings.ostr << ", ";
-            settings.ostr << *it;
+            (*it)->formatImpl(settings, state, frame);
         }
-        settings.ostr << " SET ";
-        for (auto it = group_by_aggregations.begin(); it != group_by_aggregations.end(); ++it)
+        if (!group_by_aggregations.empty())
         {
-            if (it != group_by_aggregations.begin())
-                settings.ostr << ", ";
-            settings.ostr << it->first << " = ";
-            it->second->formatImpl(settings, state, frame);
+            settings.ostr << " SET ";
+            for (auto it = group_by_aggregations.begin(); it != group_by_aggregations.end(); ++it)
+            {
+                if (it != group_by_aggregations.begin())
+                    settings.ostr << ", ";
+                settings.ostr << it->first << " = ";
+                it->second->formatImpl(settings, state, frame);
+            }
         }
     }
     else if (mode == TTLMode::DELETE)
diff --git a/src/Parsers/ASTTTLElement.h b/src/Parsers/ASTTTLElement.h
index 9b40adf2be4..2004a4f927b 100644
--- a/src/Parsers/ASTTTLElement.h
+++ b/src/Parsers/ASTTTLElement.h
@@ -17,7 +17,7 @@ public:
     PartDestinationType destination_type;
     String destination_name;
 
-    Strings group_by_key_columns;
+    ASTs group_by_key;
     std::vector<std::pair<String, ASTPtr>> group_by_aggregations;
 
     ASTTTLElement(TTLMode mode_, PartDestinationType destination_type_, const String & destination_name_)
diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp
index 5b5d9f05b22..2ea39946ff7 100644
--- a/src/Parsers/ExpressionElementParsers.cpp
+++ b/src/Parsers/ExpressionElementParsers.cpp
@@ -1464,7 +1464,7 @@ bool ParserTTLElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
     ParserIdentifier parser_identifier;
     ParserStringLiteral parser_string_literal;
     ParserExpression parser_exp;
-    ParserIdentifierList parser_identifier_list;
+    ParserExpressionList parser_expression_list(false);
 
     ASTPtr ttl_expr;
     if (!parser_exp.parse(pos, ttl_expr, expected))
@@ -1495,7 +1495,7 @@ bool ParserTTLElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
     }
 
     ASTPtr where_expr;
-    std::vector<String> group_by_key_columns;
+    ASTPtr ast_group_by_key;
     std::vector<std::pair<String, ASTPtr>> group_by_aggregations;
 
     if (mode == TTLMode::MOVE)
@@ -1508,37 +1508,30 @@ bool ParserTTLElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
     }
     else if (mode == TTLMode::GROUP_BY)
     {
-        ASTPtr ast_group_by_key_columns;
-        if (!parser_identifier_list.parse(pos, ast_group_by_key_columns, expected))
+        if (!parser_expression_list.parse(pos, ast_group_by_key, expected))
             return false;
-        for (const auto & identifier : ast_group_by_key_columns->children)
+
+        if (s_set.ignore(pos))
         {
-            String identifier_str;
-            if (!tryGetIdentifierNameInto(identifier, identifier_str))
-                return false;
-            group_by_key_columns.emplace_back(std::move(identifier_str));
-        }
+            while (true)
+            {
+                if (!group_by_aggregations.empty() && !s_comma.ignore(pos))
+                    break;
 
-        if (!s_set.ignore(pos))
-            return false;
-        while (true)
-        {
-            if (!group_by_aggregations.empty() && !s_comma.ignore(pos))
-                break;
+                ASTPtr name;
+                ASTPtr value;
+                if (!parser_identifier.parse(pos, name, expected))
+                    return false;
+                if (!s_eq.ignore(pos))
+                    return false;
+                if (!parser_exp.parse(pos, value, expected))
+                    return false;
 
-            ASTPtr name;
-            ASTPtr value;
-            if (!parser_identifier.parse(pos, name, expected))
-                return false;
-            if (!s_eq.ignore(pos))
-                return false;
-            if (!parser_exp.parse(pos, value, expected))
-                return false;
-
-            String name_str;
-            if (!tryGetIdentifierNameInto(name, name_str))
-                return false;
-            group_by_aggregations.emplace_back(name_str, std::move(value));
+                String name_str;
+                if (!tryGetIdentifierNameInto(name, name_str))
+                    return false;
+                group_by_aggregations.emplace_back(name_str, std::move(value));
+            }
         }
     }
     else if (mode == TTLMode::DELETE && s_where.ignore(pos))
@@ -1552,8 +1545,11 @@ bool ParserTTLElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
     if (where_expr)
         ttl_element->setWhere(std::move(where_expr));
 
-    ttl_element->group_by_key_columns = std::move(group_by_key_columns);
-    ttl_element->group_by_aggregations = std::move(group_by_aggregations);
+    if (mode == TTLMode::GROUP_BY)
+    {
+        ttl_element->group_by_key = std::move(ast_group_by_key->children);
+        ttl_element->group_by_aggregations = std::move(group_by_aggregations);
+    }
 
     node = ttl_element;
     return true;
diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp
index 2561bf7ef83..a967ae19691 100644
--- a/src/Parsers/ExpressionListParsers.cpp
+++ b/src/Parsers/ExpressionListParsers.cpp
@@ -742,13 +742,4 @@ bool ParserKeyValuePairsList::parseImpl(Pos & pos, ASTPtr & node, Expected & exp
     return parser.parse(pos, node, expected);
 }
 
-
-bool ParserIdentifierList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
-{
-    return ParserList(
-        std::make_unique<ParserIdentifier>(),
-        std::make_unique<ParserToken>(TokenType::Comma))
-        .parse(pos, node, expected);
-}
-
 }
diff --git a/src/Parsers/ExpressionListParsers.h b/src/Parsers/ExpressionListParsers.h
index 9cc36967bd7..0cef29b6d67 100644
--- a/src/Parsers/ExpressionListParsers.h
+++ b/src/Parsers/ExpressionListParsers.h
@@ -421,13 +421,4 @@ protected:
     bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
 };
 
-
-/** A comma-separated list of identifiers, probably empty. */
-class ParserIdentifierList : public IParserBase
-{
-protected:
-    const char * getName() const override { return "list of identifiers"; }
-    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
-};
-
 }
diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index bb9d8ce186e..5de3d204f16 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -646,22 +646,70 @@ void MergeTreeData::setTTLExpressions(const ColumnsDescription & new_columns,
         }
         else if (ttl_element->mode == TTLMode::GROUP_BY)
         {
-            if (ttl_element->group_by_key_columns.size() > this->primary_key_columns.size())
+            if (ttl_element->group_by_key.size() > this->primary_key_columns.size())
                 throw Exception("TTL Expression GROUP BY key should be a prefix of primary key", ErrorCodes::BAD_TTL_EXPRESSION);
-            for (size_t i = 0; i < ttl_element->group_by_key_columns.size(); ++i)
+
+            NameSet primary_key_columns_set(this->primary_key_columns.begin(), this->primary_key_columns.end());
+            NameSet aggregation_columns_set;
+
+            for (const auto & column : this->primary_key_expr->getRequiredColumns())
+                primary_key_columns_set.insert(column);
+
+            for (size_t i = 0; i < ttl_element->group_by_key.size(); ++i)
             {
-                if (ttl_element->group_by_key_columns[i] != this->primary_key_columns[i])
+                if (ttl_element->group_by_key[i]->getColumnName() != this->primary_key_columns[i])
                     throw Exception("TTL Expression GROUP BY key should be a prefix of primary key", ErrorCodes::BAD_TTL_EXPRESSION);
             }
+            for (const auto & [name, value] : ttl_element->group_by_aggregations)
+            {
+                if (primary_key_columns_set.contains(name))
+                    throw Exception("Can not set custom aggregation for column in primary key in TTL Expression", ErrorCodes::BAD_TTL_EXPRESSION);
+                aggregation_columns_set.insert(name);
+            }
+            if (aggregation_columns_set.size() != ttl_element->group_by_aggregations.size())
+                throw Exception("Multiple aggregations set for one column in TTL Expression", ErrorCodes::BAD_TTL_EXPRESSION);
 
-            result.group_by_keys = ttl_element->group_by_key_columns;
+            result.group_by_keys = Names(this->primary_key_columns.begin(), this->primary_key_columns.begin() + ttl_element->group_by_key.size());
 
             auto aggregations = ttl_element->group_by_aggregations;
-            for (size_t i = ttl_element->group_by_key_columns.size(); i < this->primary_key_columns.size(); ++i)
+            for (size_t i = 0; i < this->primary_key_columns.size(); ++i)
             {
-                ASTPtr expr = makeASTFunction("max", std::make_shared<ASTIdentifier>(this->primary_key_columns[i]));
-                aggregations.emplace_back(this->primary_key_columns[i], std::move(expr));
+                ASTPtr value = this->primary_key_expr_ast->children[i]->clone();
+
+                if (i >= ttl_element->group_by_key.size())
+                {
+                    ASTPtr value_max = makeASTFunction("max", value->clone());
+                    aggregations.emplace_back(value->getColumnName(), std::move(value_max));
+                }
+
+                if (value->as<ASTFunction>())
+                {
+                    auto syntax_result = SyntaxAnalyzer(global_context).analyze(value, new_columns.getAllPhysical(), {}, true);
+                    auto expr_actions = ExpressionAnalyzer(value, syntax_result, global_context).getActions(false);
+                    for (const auto & column : expr_actions->getRequiredColumns())
+                    {
+                        if (i < ttl_element->group_by_key.size())
+                        {
+                            ASTPtr expr = makeASTFunction("any", std::make_shared<ASTIdentifier>(column));
+                            aggregations.emplace_back(column, std::move(expr));
+                        }
+                        else
+                        {
+                            ASTPtr expr = makeASTFunction("argMax", std::make_shared<ASTIdentifier>(column), value->clone());
+                            aggregations.emplace_back(column, std::move(expr));
+                        }
+                    }
+                }
             }
+            for (const auto & column : new_columns.getAllPhysical())
+            {
+                if (!primary_key_columns_set.contains(column.name) && !aggregation_columns_set.contains(column.name))
+                {
+                    ASTPtr expr = makeASTFunction("any", std::make_shared<ASTIdentifier>(column.name));
+                    aggregations.emplace_back(column.name, std::move(expr));
+                }
+            }
+
             for (auto [name, value] : aggregations)
             {
                 auto syntax_result = SyntaxAnalyzer(global_context).analyze(value, new_columns.getAllPhysical(), {}, true);
diff --git a/tests/queries/0_stateless/01280_ttl_where_group_by.reference b/tests/queries/0_stateless/01280_ttl_where_group_by.reference
index 572fc7731d3..dead0a5aac3 100644
--- a/tests/queries/0_stateless/01280_ttl_where_group_by.reference
+++ b/tests/queries/0_stateless/01280_ttl_where_group_by.reference
@@ -12,3 +12,9 @@
 1	3	10	6
 2	1	0	3
 3	5	8	2
+1	1	0	4
+3	3	13	9
+1	2	7	5
+2	3	6	5
+1	2	3	5
+2	3	3	5
diff --git a/tests/queries/0_stateless/01280_ttl_where_group_by.sql b/tests/queries/0_stateless/01280_ttl_where_group_by.sql
index 0ca06e77001..e61716cfe81 100644
--- a/tests/queries/0_stateless/01280_ttl_where_group_by.sql
+++ b/tests/queries/0_stateless/01280_ttl_where_group_by.sql
@@ -42,4 +42,46 @@ insert into ttl_01280_3 values (3, 2, 8, 2, now() + 1);
 insert into ttl_01280_3 values (3, 5, 5, 8, now());
 select sleep(2.1) format Null;
 optimize table ttl_01280_3 final;
-select a, b, x, y from ttl_01280_3;
\ No newline at end of file
+select a, b, x, y from ttl_01280_3;
+
+drop table if exists ttl_01280_4;
+
+create table ttl_01280_4 (a Int, b Int, x Int64, y Int64, d DateTime) engine = MergeTree order by (toDate(d), -(a + b)) ttl d + interval 1 second group by toDate(d) set x = sum(x), y = max(y);
+insert into ttl_01280_4 values (1, 1, 0, 4, now() + 10);
+insert into ttl_01280_4 values (10, 2, 3, 3, now());
+insert into ttl_01280_4 values (2, 10, 1, 7, now());
+insert into ttl_01280_4 values (3, 3, 5, 2, now());
+insert into ttl_01280_4 values (1, 5, 4, 9, now());
+select sleep(1.1) format Null;
+optimize table ttl_01280_4 final;
+select a, b, x, y from ttl_01280_4;
+
+drop table if exists ttl_01280_5;
+
+create table ttl_01280_5 (a Int, b Int, x Int64, y Int64, d DateTime) engine = MergeTree order by (toDate(d), a, -b) ttl d + interval 1 second group by toDate(d), a set x = sum(x);
+insert into ttl_01280_5 values (1, 2, 3, 5, now());
+insert into ttl_01280_5 values (2, 10, 1, 5, now());
+insert into ttl_01280_5 values (2, 3, 5, 5, now());
+insert into ttl_01280_5 values (1, 5, 4, 5, now());
+select sleep(1.1) format Null;
+optimize table ttl_01280_5 final;
+select a, b, x, y from ttl_01280_5;
+
+drop table if exists ttl_01280_6;
+
+create table ttl_01280_6 (a Int, b Int, x Int64, y Int64, d DateTime) engine = MergeTree order by (toDate(d), a, -b) ttl d + interval 1 second group by toDate(d), a;
+insert into ttl_01280_6 values (1, 2, 3, 5, now());
+insert into ttl_01280_6 values (2, 10, 3, 5, now());
+insert into ttl_01280_6 values (2, 3, 3, 5, now());
+insert into ttl_01280_6 values (1, 5, 3, 5, now());
+select sleep(1.1) format Null;
+optimize table ttl_01280_6 final;
+select a, b, x, y from ttl_01280_6;
+
+create table ttl_01280_error (a Int, b Int, x Int64, y Int64, d DateTime) engine = MergeTree order by (a, b) ttl d + interval 1 second group by x set y = max(y); -- { serverError 450}
+create table ttl_01280_error (a Int, b Int, x Int64, y Int64, d DateTime) engine = MergeTree order by (a, b) ttl d + interval 1 second group by b set y = max(y); -- { serverError 450}
+create table ttl_01280_error (a Int, b Int, x Int64, y Int64, d DateTime) engine = MergeTree order by (a, b) ttl d + interval 1 second group by a, b, x set y = max(y); -- { serverError 450}
+create table ttl_01280_error (a Int, b Int, x Int64, y Int64, d DateTime) engine = MergeTree order by (a, b) ttl d + interval 1 second group by a set b = min(b), y = max(y); -- { serverError 450}
+create table ttl_01280_error (a Int, b Int, x Int64, y Int64, d DateTime) engine = MergeTree order by (a, b) ttl d + interval 1 second group by a, b set y = max(y), y = max(y); -- { serverError 450}
+create table ttl_01280_error (a Int, b Int, x Int64, y Int64, d DateTime) engine = MergeTree order by (toDate(d), a) ttl d + interval 1 second group by toDate(d), a set d = min(d), b = max(b); -- { serverError 450}
+create table ttl_01280_error (a Int, b Int, x Int64, y Int64, d DateTime) engine = MergeTree order by (d, -(a + b)) ttl d + interval 1 second group by d, -(a + b) set a = sum(a), b = min(b); -- { serverError 450}

From e182d4df2ef95c59527bb06b162d92c9af3fd341 Mon Sep 17 00:00:00 2001
From: Nikolai Sorokin <sonikint@gmail.com>
Date: Mon, 25 May 2020 01:52:18 +0300
Subject: [PATCH 074/184] Fix context usage

---
 src/DataStreams/TTLBlockInputStream.cpp | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/DataStreams/TTLBlockInputStream.cpp b/src/DataStreams/TTLBlockInputStream.cpp
index 106ae1fd5a0..f11f6ea38cd 100644
--- a/src/DataStreams/TTLBlockInputStream.cpp
+++ b/src/DataStreams/TTLBlockInputStream.cpp
@@ -6,6 +6,7 @@
 #include <Columns/ColumnConst.h>
 #include <Interpreters/addTypeConversionToAST.h>
 #include <Storages/MergeTree/TTLMode.h>
+#include <Interpreters/Context.h>
 
 namespace DB
 {
@@ -87,11 +88,9 @@ TTLBlockInputStream::TTLBlockInputStream(
 
         const Settings & settings = storage.global_context.getSettingsRef();
 
-        bool allow_to_use_two_level_group_by = false;  // settings.max_bytes_before_external_group_by != 0;
         Aggregator::Params params(header, keys, aggregates,
             false, settings.max_rows_to_group_by, settings.group_by_overflow_mode,
-            allow_to_use_two_level_group_by ? settings.group_by_two_level_threshold : SettingUInt64(0),
-            allow_to_use_two_level_group_by ? settings.group_by_two_level_threshold_bytes : SettingUInt64(0),
+            SettingUInt64(0), SettingUInt64(0),
             settings.max_bytes_before_external_group_by, settings.empty_result_for_aggregation_by_empty_set,
             storage.global_context.getTemporaryVolume(), settings.max_threads, settings.min_free_disk_space_for_temporary_data);
         aggregator = std::make_unique<Aggregator>(params);

From f62adfd588b579e51f76ac78fe19e66df250ff8b Mon Sep 17 00:00:00 2001
From: Anton Popov <pad11rus@gmail.com>
Date: Wed, 27 May 2020 04:52:02 +0300
Subject: [PATCH 075/184] Fix Arcadia build

---
 src/Storages/MergeTree/MergeTreeData.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 5de3d204f16..9a1b4f27872 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -662,7 +662,7 @@ void MergeTreeData::setTTLExpressions(const ColumnsDescription & new_columns,
             }
             for (const auto & [name, value] : ttl_element->group_by_aggregations)
             {
-                if (primary_key_columns_set.contains(name))
+                if (primary_key_columns_set.count(name))
                     throw Exception("Can not set custom aggregation for column in primary key in TTL Expression", ErrorCodes::BAD_TTL_EXPRESSION);
                 aggregation_columns_set.insert(name);
             }
@@ -703,7 +703,7 @@ void MergeTreeData::setTTLExpressions(const ColumnsDescription & new_columns,
             }
             for (const auto & column : new_columns.getAllPhysical())
             {
-                if (!primary_key_columns_set.contains(column.name) && !aggregation_columns_set.contains(column.name))
+                if (!primary_key_columns_set.count(column.name) && !aggregation_columns_set.count(column.name))
                 {
                     ASTPtr expr = makeASTFunction("any", std::make_shared<ASTIdentifier>(column.name));
                     aggregations.emplace_back(column.name, std::move(expr));

From 146370934a295759f342c310ab74eb660d675423 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Wed, 27 May 2020 13:07:38 +0300
Subject: [PATCH 076/184] Keep the value of DistributedFilesToInsert metric on
 exceptions

---
 src/Common/CurrentMetrics.h                   |  6 ++++
 src/Storages/Distributed/DirectoryMonitor.cpp | 32 +++++++++++++------
 src/Storages/Distributed/DirectoryMonitor.h   |  8 +++--
 3 files changed, 34 insertions(+), 12 deletions(-)

diff --git a/src/Common/CurrentMetrics.h b/src/Common/CurrentMetrics.h
index b87504ef49a..a3bac96a16c 100644
--- a/src/Common/CurrentMetrics.h
+++ b/src/Common/CurrentMetrics.h
@@ -94,6 +94,12 @@ namespace CurrentMetrics
             amount = new_amount;
         }
 
+        void sub(Value value = 1)
+        {
+            what->fetch_sub(value, std::memory_order_relaxed);
+            amount -= value;
+        }
+
         /// Subtract value before destructor.
         void destroy()
         {
diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp
index fdcbe8e69f9..4dd62db0965 100644
--- a/src/Storages/Distributed/DirectoryMonitor.cpp
+++ b/src/Storages/Distributed/DirectoryMonitor.cpp
@@ -110,8 +110,9 @@ void StorageDistributedDirectoryMonitor::flushAllData()
 {
     if (!quit)
     {
+        CurrentMetrics::Increment metric_pending_files{CurrentMetrics::DistributedFilesToInsert, 0};
         std::unique_lock lock{mutex};
-        processFiles();
+        processFiles(metric_pending_files);
     }
 }
 
@@ -131,6 +132,9 @@ void StorageDistributedDirectoryMonitor::run()
 {
     std::unique_lock lock{mutex};
 
+    /// This metric will be updated with the number of pending files later.
+    CurrentMetrics::Increment metric_pending_files{CurrentMetrics::DistributedFilesToInsert, 0};
+
     bool do_sleep = false;
     while (!quit)
     {
@@ -139,7 +143,7 @@ void StorageDistributedDirectoryMonitor::run()
         {
             try
             {
-                do_sleep = !processFiles();
+                do_sleep = !processFiles(metric_pending_files);
             }
             catch (...)
             {
@@ -222,7 +226,7 @@ ConnectionPoolPtr StorageDistributedDirectoryMonitor::createPool(const std::stri
 }
 
 
-bool StorageDistributedDirectoryMonitor::processFiles()
+bool StorageDistributedDirectoryMonitor::processFiles(CurrentMetrics::Increment & metric_pending_files)
 {
     std::map<UInt64, std::string> files;
 
@@ -236,14 +240,16 @@ bool StorageDistributedDirectoryMonitor::processFiles()
             files[parse<UInt64>(file_path.getBaseName())] = file_path_str;
     }
 
+    /// Note: the value of this metric will be kept if this function will throw an exception.
+    /// This is needed, because in case of exception, files still pending.
+    metric_pending_files.changeTo(files.size());
+
     if (files.empty())
         return false;
 
-    CurrentMetrics::Increment metric_increment{CurrentMetrics::DistributedFilesToInsert, CurrentMetrics::Value(files.size())};
-
     if (should_batch_inserts)
     {
-        processFilesWithBatching(files);
+        processFilesWithBatching(files, metric_pending_files);
     }
     else
     {
@@ -252,14 +258,14 @@ bool StorageDistributedDirectoryMonitor::processFiles()
             if (quit)
                 return true;
 
-            processFile(file.second);
+            processFile(file.second, metric_pending_files);
         }
     }
 
     return true;
 }
 
-void StorageDistributedDirectoryMonitor::processFile(const std::string & file_path)
+void StorageDistributedDirectoryMonitor::processFile(const std::string & file_path, CurrentMetrics::Increment & metric_pending_files)
 {
     LOG_TRACE(log, "Started processing `{}`", file_path);
     auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(storage.global_context->getSettingsRef());
@@ -289,6 +295,7 @@ void StorageDistributedDirectoryMonitor::processFile(const std::string & file_pa
     }
 
     Poco::File{file_path}.remove();
+    metric_pending_files.sub();
 
     LOG_TRACE(log, "Finished processing `{}`", file_path);
 }
@@ -584,7 +591,9 @@ bool StorageDistributedDirectoryMonitor::scheduleAfter(size_t ms)
     return task_handle->scheduleAfter(ms, false);
 }
 
-void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map<UInt64, std::string> & files)
+void StorageDistributedDirectoryMonitor::processFilesWithBatching(
+    const std::map<UInt64, std::string> & files,
+    CurrentMetrics::Increment & metric_pending_files)
 {
     std::unordered_set<UInt64> file_indices_to_skip;
 
@@ -596,6 +605,7 @@ void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map
         batch.readText(in);
         file_indices_to_skip.insert(batch.file_indices.begin(), batch.file_indices.end());
         batch.send();
+        metric_pending_files.sub(batch.file_indices.size());
     }
 
     std::unordered_map<BatchHeader, Batch, BatchHeader::Hash> header_to_batch;
@@ -656,13 +666,17 @@ void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map
         batch.total_bytes += total_bytes;
 
         if (batch.isEnoughSize())
+        {
             batch.send();
+            metric_pending_files.sub(batch.file_indices.size());
+        }
     }
 
     for (auto & kv : header_to_batch)
     {
         Batch & batch = kv.second;
         batch.send();
+        metric_pending_files.sub(batch.file_indices.size());
     }
 
     /// current_batch.txt will not exist if there was no send
diff --git a/src/Storages/Distributed/DirectoryMonitor.h b/src/Storages/Distributed/DirectoryMonitor.h
index 77abf35630c..e2a913ee1ef 100644
--- a/src/Storages/Distributed/DirectoryMonitor.h
+++ b/src/Storages/Distributed/DirectoryMonitor.h
@@ -9,6 +9,8 @@
 #include <IO/ReadBufferFromFile.h>
 
 
+namespace CurrentMetrics { class Increment; }
+
 namespace DB
 {
 
@@ -37,9 +39,9 @@ public:
     bool scheduleAfter(size_t ms);
 private:
     void run();
-    bool processFiles();
-    void processFile(const std::string & file_path);
-    void processFilesWithBatching(const std::map<UInt64, std::string> & files);
+    bool processFiles(CurrentMetrics::Increment & metric_pending_files);
+    void processFile(const std::string & file_path, CurrentMetrics::Increment & metric_pending_files);
+    void processFilesWithBatching(const std::map<UInt64, std::string> & files, CurrentMetrics::Increment & metric_pending_files);
 
     static bool isFileBrokenErrorCode(int code);
     void markAsBroken(const std::string & file_path) const;

From 57555dbabf11780ca751e8f89a8e8f3787294d90 Mon Sep 17 00:00:00 2001
From: Nikolai Sorokin <sonikint@gmail.com>
Date: Wed, 27 May 2020 14:00:17 +0300
Subject: [PATCH 077/184] Fix after rebase

---
 src/Storages/MergeTree/MergeTreeData.cpp | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 9a1b4f27872..da351fc7750 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -646,18 +646,18 @@ void MergeTreeData::setTTLExpressions(const ColumnsDescription & new_columns,
         }
         else if (ttl_element->mode == TTLMode::GROUP_BY)
         {
-            if (ttl_element->group_by_key.size() > this->primary_key_columns.size())
+            if (ttl_element->group_by_key.size() > this->getPrimaryKey().column_names.size())
                 throw Exception("TTL Expression GROUP BY key should be a prefix of primary key", ErrorCodes::BAD_TTL_EXPRESSION);
 
-            NameSet primary_key_columns_set(this->primary_key_columns.begin(), this->primary_key_columns.end());
+            NameSet primary_key_columns_set(this->getPrimaryKey().column_names.begin(), this->getPrimaryKey().column_names.end());
             NameSet aggregation_columns_set;
 
-            for (const auto & column : this->primary_key_expr->getRequiredColumns())
+            for (const auto & column : this->getPrimaryKey().expression->getRequiredColumns())
                 primary_key_columns_set.insert(column);
 
             for (size_t i = 0; i < ttl_element->group_by_key.size(); ++i)
             {
-                if (ttl_element->group_by_key[i]->getColumnName() != this->primary_key_columns[i])
+                if (ttl_element->group_by_key[i]->getColumnName() != this->getPrimaryKey().column_names[i])
                     throw Exception("TTL Expression GROUP BY key should be a prefix of primary key", ErrorCodes::BAD_TTL_EXPRESSION);
             }
             for (const auto & [name, value] : ttl_element->group_by_aggregations)
@@ -669,12 +669,12 @@ void MergeTreeData::setTTLExpressions(const ColumnsDescription & new_columns,
             if (aggregation_columns_set.size() != ttl_element->group_by_aggregations.size())
                 throw Exception("Multiple aggregations set for one column in TTL Expression", ErrorCodes::BAD_TTL_EXPRESSION);
 
-            result.group_by_keys = Names(this->primary_key_columns.begin(), this->primary_key_columns.begin() + ttl_element->group_by_key.size());
+            result.group_by_keys = Names(this->getPrimaryKey().column_names.begin(), this->getPrimaryKey().column_names.begin() + ttl_element->group_by_key.size());
 
             auto aggregations = ttl_element->group_by_aggregations;
-            for (size_t i = 0; i < this->primary_key_columns.size(); ++i)
+            for (size_t i = 0; i < this->getPrimaryKey().column_names.size(); ++i)
             {
-                ASTPtr value = this->primary_key_expr_ast->children[i]->clone();
+                ASTPtr value = this->getPrimaryKey().expression_list_ast->children[i]->clone();
 
                 if (i >= ttl_element->group_by_key.size())
                 {

From 8bfe21f10194ae05f4f1fa74409560622f5c0460 Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Wed, 27 May 2020 14:09:12 +0300
Subject: [PATCH 078/184] Update 01281_group_by_limit_memory_tracking.sh

---
 .../queries/0_stateless/01281_group_by_limit_memory_tracking.sh  | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh b/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh
index 79a731b9273..5922b8d74d2 100755
--- a/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh
+++ b/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh
@@ -27,6 +27,7 @@ function execute_group_by()
     execute_null "${opts[@]}" <<<'SELECT uniq(number) FROM numbers_mt(toUInt64(1e6)) GROUP BY number % 5e5'
 }
 
+# This is needed to keep at least one running query for user for the time of test.
 execute_null <<<'SELECT sleep(3)' &
 execute_group_by
 # if memory accounting will be incorrect, the second query will be failed with MEMORY_LIMIT_EXCEEDED

From d24286d5da2ce890872de6f0837b076032673339 Mon Sep 17 00:00:00 2001
From: Albert Kidrachev <kidrachev@sandbox-410109003>
Date: Wed, 27 May 2020 14:52:45 +0300
Subject: [PATCH 079/184] fix

---
 src/Common/SharedBlockRowRef.h                | 29 -------
 src/Interpreters/InterpreterSelectQuery.cpp   |  3 +-
 .../OptimizedPartialSortingTransform.cpp      | 83 -------------------
 .../OptimizedPartialSortingTransform.h        | 34 --------
 .../Transforms/PartialSortingTransform.cpp    | 66 +++++++++++++++
 .../Transforms/PartialSortingTransform.h      |  2 +
 6 files changed, 69 insertions(+), 148 deletions(-)
 delete mode 100644 src/Processors/Transforms/OptimizedPartialSortingTransform.cpp
 delete mode 100644 src/Processors/Transforms/OptimizedPartialSortingTransform.h

diff --git a/src/Common/SharedBlockRowRef.h b/src/Common/SharedBlockRowRef.h
index e9fd076da07..957d66243dd 100644
--- a/src/Common/SharedBlockRowRef.h
+++ b/src/Common/SharedBlockRowRef.h
@@ -87,33 +87,4 @@ struct SharedBlockRowRef
     }
 };
 
-struct SharedBlockRowWithSortDescriptionRef : SharedBlockRowRef
-{
-    SortDescription * description = nullptr;
-
-    void set(SharedBlockPtr & shared_block_, ColumnRawPtrs * columns_, size_t row_num_) = delete;
-
-    bool operator< (const SharedBlockRowRef & other) const
-    {
-        size_t size = columns->size();
-        for (size_t i = 0; i < size; ++i)
-        {
-            int res = (*description)[i].direction * (*columns)[i]->compareAt(row_num, other.row_num, *(*other.columns)[i], 1);
-            if (res < 0)
-                return true;
-            else if (res > 0)
-                return false;
-        }
-        return false;
-    }
-
-    void set(SharedBlockPtr & shared_block_, ColumnRawPtrs * columns_, size_t row_num_, SortDescription * description_)
-    {
-        shared_block = shared_block_;
-        columns = columns_;
-        row_num = row_num_;
-        description = description_;
-    }
-};
-
 }
diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp
index c3fce45ac61..c8d842dfa53 100644
--- a/src/Interpreters/InterpreterSelectQuery.cpp
+++ b/src/Interpreters/InterpreterSelectQuery.cpp
@@ -84,7 +84,6 @@
 #include <Processors/Transforms/MergingAggregatedMemoryEfficientTransform.h>
 #include <Processors/Transforms/TotalsHavingTransform.h>
 #include <Processors/Transforms/PartialSortingTransform.h>
-#include <Processors/Transforms/OptimizedPartialSortingTransform.h>
 #include <Processors/Transforms/LimitsCheckingTransform.h>
 #include <Processors/Transforms/MergeSortingTransform.h>
 #include <Processors/Transforms/DistinctTransform.h>
@@ -2109,7 +2108,7 @@ void InterpreterSelectQuery::executeOrder(QueryPipeline & pipeline, InputSorting
         if (stream_type != QueryPipeline::StreamType::Main)
             return nullptr;
 
-        return std::make_shared<OptimizedPartialSortingTransform>(header, output_order_descr, limit);
+        return std::make_shared<PartialSortingTransform>(header, output_order_descr, limit);
     });
 
     pipeline.addSimpleTransform([&](const Block & header, QueryPipeline::StreamType stream_type) -> ProcessorPtr
diff --git a/src/Processors/Transforms/OptimizedPartialSortingTransform.cpp b/src/Processors/Transforms/OptimizedPartialSortingTransform.cpp
deleted file mode 100644
index e753052a2b3..00000000000
--- a/src/Processors/Transforms/OptimizedPartialSortingTransform.cpp
+++ /dev/null
@@ -1,83 +0,0 @@
-#include <Processors/Transforms/OptimizedPartialSortingTransform.h>
-#include <Interpreters/sortBlock.h>
-#include <Common/PODArray.h>
-
-namespace DB
-{
-
-OptimizedPartialSortingTransform::OptimizedPartialSortingTransform(
-    const Block & header_, SortDescription & description_, UInt64 limit_)
-    : ISimpleTransform(header_, header_, false)
-    , description(description_), limit(limit_)
-    , threshold_shared_block(nullptr)
-{
-}
-
-static ColumnRawPtrs extractColumns(const Block & block, const SortDescription& description)
-{
-    size_t size = description.size();
-    ColumnRawPtrs res;
-    res.reserve(size);
-
-    for (size_t i = 0; i < size; ++i)
-    {
-        const IColumn * column = !description[i].column_name.empty()
-            ? block.getByName(description[i].column_name).column.get()
-            : block.safeGetByPosition(description[i].column_number).column.get();
-        res.emplace_back(column);
-    }
-
-    return res;
-}
-
-void OptimizedPartialSortingTransform::transform(Chunk & chunk)
-{
-    if (read_rows)
-        read_rows->add(chunk.getNumRows());
-
-    auto block = getInputPort().getHeader().cloneWithColumns(chunk.detachColumns());
-    chunk.clear();
-
-    SharedBlockPtr shared_block = new detail::SharedBlock(std::move(block));
-    UInt64 rows_num = shared_block->rows();
-
-
-    if (threshold_shared_block) {
-        SharedBlockRowWithSortDescriptionRef row;
-        IColumn::Filter filter(rows_num);
-        ColumnRawPtrs shared_block_columns = extractColumns(*shared_block, description);
-        size_t filtered_count = 0;
-
-        for (UInt64 i = 0; i < rows_num; ++i) {
-            row.set(shared_block, &shared_block_columns, i, &description);
-
-            if (threshold_row < row)
-            {
-                ++filtered_count;
-                filter[i] = 1;
-            }
-        }
-
-        if (filtered_count)
-        {
-            for (auto & column : shared_block->getColumns())
-            {
-                column = column->filter(filter, filtered_count);
-            }
-        }
-    }
-
-    sortBlock(*shared_block, description, limit);
-
-    if (!threshold_shared_block && limit && limit < rows_num)
-    {
-        Block threshold_block = shared_block->cloneWithColumns(shared_block->getColumns());
-        threshold_shared_block = new detail::SharedBlock(std::move(threshold_block));
-        threshold_block_columns = extractColumns(*threshold_shared_block, description);
-        threshold_row.set(threshold_shared_block, &threshold_block_columns, limit - 1, &description);
-    }
-
-    chunk.setColumns(shared_block->getColumns(), shared_block->rows());
-}
-
-}
diff --git a/src/Processors/Transforms/OptimizedPartialSortingTransform.h b/src/Processors/Transforms/OptimizedPartialSortingTransform.h
deleted file mode 100644
index 20e72bd836f..00000000000
--- a/src/Processors/Transforms/OptimizedPartialSortingTransform.h
+++ /dev/null
@@ -1,34 +0,0 @@
-#pragma once
-#include <Processors/ISimpleTransform.h>
-#include <Processors/RowsBeforeLimitCounter.h>
-#include <Common/SharedBlockRowRef.h>
-#include <Core/SortDescription.h>
-
-
-namespace DB
-{
-class OptimizedPartialSortingTransform : public ISimpleTransform
-{
-public:
-    OptimizedPartialSortingTransform(
-        const Block & header_,
-        SortDescription & description_,
-        UInt64 limit_ = 0);
-
-    String getName() const override { return "OptimizedPartialSortingTransform"; }
-
-    void setRowsBeforeLimitCounter(RowsBeforeLimitCounterPtr counter) { read_rows.swap(counter); }
-
-protected:
-    void transform(Chunk & chunk) override;
-
-private:
-    SortDescription description;
-    UInt64 limit;
-    RowsBeforeLimitCounterPtr read_rows;
-    SharedBlockRowWithSortDescriptionRef threshold_row;
-    SharedBlockPtr threshold_shared_block;
-    ColumnRawPtrs threshold_block_columns;
-};
-
-}
diff --git a/src/Processors/Transforms/PartialSortingTransform.cpp b/src/Processors/Transforms/PartialSortingTransform.cpp
index 018614f0165..062064c0fd6 100644
--- a/src/Processors/Transforms/PartialSortingTransform.cpp
+++ b/src/Processors/Transforms/PartialSortingTransform.cpp
@@ -1,5 +1,6 @@
 #include <Processors/Transforms/PartialSortingTransform.h>
 #include <Interpreters/sortBlock.h>
+#include <Common/PODArray.h>
 
 namespace DB
 {
@@ -11,6 +12,38 @@ PartialSortingTransform::PartialSortingTransform(
 {
 }
 
+static ColumnRawPtrs extractColumns(const Block & block, const SortDescription & description)
+{
+    size_t size = description.size();
+    ColumnRawPtrs res;
+    res.reserve(size);
+
+    for (size_t i = 0; i < size; ++i)
+    {
+        const IColumn * column = !description[i].column_name.empty()
+            ? block.getByName(description[i].column_name).column.get()
+            : block.safeGetByPosition(description[i].column_number).column.get();
+        res.emplace_back(column);
+    }
+
+    return res;
+}
+
+bool less(const ColumnRawPtrs & lhs, UInt64 lhs_row_num,
+          const ColumnRawPtrs & rhs, UInt64 rhs_row_num, const SortDescription & description)
+{
+    size_t size = description.size();
+    for (size_t i = 0; i < size; ++i)
+    {
+        int res = description[i].direction * lhs[i]->compareAt(lhs_row_num, rhs_row_num, *rhs[i], 1);
+        if (res < 0)
+            return true;
+        else if (res > 0)
+            return false;
+    }
+    return false;
+}
+
 void PartialSortingTransform::transform(Chunk & chunk)
 {
     if (read_rows)
@@ -19,7 +52,40 @@ void PartialSortingTransform::transform(Chunk & chunk)
     auto block = getInputPort().getHeader().cloneWithColumns(chunk.detachColumns());
     chunk.clear();
 
+    UInt64 rows_num = block.rows();
+
+    if (!threshold_block_columns.empty())
+    {
+        IColumn::Filter filter(rows_num, 0);
+        ColumnRawPtrs block_columns = extractColumns(block, description);
+        size_t filtered_count = 0;
+
+        for (UInt64 i = 0; i < rows_num; ++i)
+        {
+            if (less(threshold_block_columns, limit - 1, block_columns, i, description))
+            {
+                ++filtered_count;
+                filter[i] = 1;
+            }
+        }
+
+        if (filtered_count)
+        {
+            for (auto & column : block.getColumns())
+            {
+                column = column->filter(filter, filtered_count);
+            }
+        }
+    }
+
     sortBlock(block, description, limit);
+
+    if (threshold_block_columns.empty() && limit && limit < rows_num)
+    {
+        threshold_block = block.cloneWithColumns(block.getColumns());
+        threshold_block_columns = extractColumns(threshold_block, description);
+    }
+
     chunk.setColumns(block.getColumns(), block.rows());
 }
 
diff --git a/src/Processors/Transforms/PartialSortingTransform.h b/src/Processors/Transforms/PartialSortingTransform.h
index 47ac90c6904..d6749e4dfad 100644
--- a/src/Processors/Transforms/PartialSortingTransform.h
+++ b/src/Processors/Transforms/PartialSortingTransform.h
@@ -29,6 +29,8 @@ private:
     SortDescription description;
     UInt64 limit;
     RowsBeforeLimitCounterPtr read_rows;
+    Block threshold_block;
+    ColumnRawPtrs threshold_block_columns;
 };
 
 }

From a0f99fde700d702f29ad2411ee53571498900e41 Mon Sep 17 00:00:00 2001
From: Albert Kidrachev <kidrachev@sandbox-410109003>
Date: Wed, 27 May 2020 14:56:01 +0300
Subject: [PATCH 080/184] fix

---
 src/Common/SharedBlockRowRef.h | 1 -
 src/Processors/ya.make         | 1 -
 2 files changed, 2 deletions(-)

diff --git a/src/Common/SharedBlockRowRef.h b/src/Common/SharedBlockRowRef.h
index 957d66243dd..193f7e4dd05 100644
--- a/src/Common/SharedBlockRowRef.h
+++ b/src/Common/SharedBlockRowRef.h
@@ -2,7 +2,6 @@
 
 #include <algorithm>
 #include <Core/Block.h>
-#include <Core/SortDescription.h>
 #include <Columns/IColumn.h>
 #include <boost/smart_ptr/intrusive_ptr.hpp>
 
diff --git a/src/Processors/ya.make b/src/Processors/ya.make
index 5952341527b..62320f1c147 100644
--- a/src/Processors/ya.make
+++ b/src/Processors/ya.make
@@ -129,7 +129,6 @@ SRCS(
     Transforms/MergeSortingTransform.cpp
     Transforms/MergingAggregatedMemoryEfficientTransform.cpp
     Transforms/MergingAggregatedTransform.cpp
-    Transforms/OptimizedPartialSortingTransform.cpp
     Transforms/PartialSortingTransform.cpp
     Transforms/ReverseTransform.cpp
     Transforms/RollupTransform.cpp

From 81a5af1938afbd6168500b02579b259393b8a987 Mon Sep 17 00:00:00 2001
From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com>
Date: Wed, 27 May 2020 15:41:45 +0300
Subject: [PATCH 081/184] Update
 src/Processors/Formats/Impl/ArrowBlockInputFormat.h

---
 src/Processors/Formats/Impl/ArrowBlockInputFormat.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Processors/Formats/Impl/ArrowBlockInputFormat.h b/src/Processors/Formats/Impl/ArrowBlockInputFormat.h
index 5ad112efde9..7599182bb3a 100644
--- a/src/Processors/Formats/Impl/ArrowBlockInputFormat.h
+++ b/src/Processors/Formats/Impl/ArrowBlockInputFormat.h
@@ -25,6 +25,7 @@ protected:
     Chunk generate() override;
 
 private:
+    // Whether to use ArrowStream format
     // Whether to use ArrowStream format
     bool stream;
     // This field is only used for ArrowStream format

From 61f833efc11f6dfb109343e86721327bf83cd742 Mon Sep 17 00:00:00 2001
From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com>
Date: Wed, 27 May 2020 15:41:52 +0300
Subject: [PATCH 082/184] Update
 src/Processors/Formats/Impl/ArrowBlockInputFormat.h

---
 src/Processors/Formats/Impl/ArrowBlockInputFormat.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Processors/Formats/Impl/ArrowBlockInputFormat.h b/src/Processors/Formats/Impl/ArrowBlockInputFormat.h
index 7599182bb3a..92cced2ead8 100644
--- a/src/Processors/Formats/Impl/ArrowBlockInputFormat.h
+++ b/src/Processors/Formats/Impl/ArrowBlockInputFormat.h
@@ -31,6 +31,7 @@ private:
     // This field is only used for ArrowStream format
     std::shared_ptr<arrow::RecordBatchReader> stream_reader;
     // The following fields are used only for Arrow format
+    // The following fields are used only for Arrow format
     std::shared_ptr<arrow::ipc::RecordBatchFileReader> file_reader;
 
     int record_batch_total = 0;

From f54c892b49a3f1f52135d0ba9008cf2641f19898 Mon Sep 17 00:00:00 2001
From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com>
Date: Wed, 27 May 2020 15:41:59 +0300
Subject: [PATCH 083/184] Update
 src/Processors/Formats/Impl/ArrowBlockInputFormat.h

---
 src/Processors/Formats/Impl/ArrowBlockInputFormat.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Processors/Formats/Impl/ArrowBlockInputFormat.h b/src/Processors/Formats/Impl/ArrowBlockInputFormat.h
index 92cced2ead8..b1ee21a51e4 100644
--- a/src/Processors/Formats/Impl/ArrowBlockInputFormat.h
+++ b/src/Processors/Formats/Impl/ArrowBlockInputFormat.h
@@ -29,6 +29,7 @@ private:
     // Whether to use ArrowStream format
     bool stream;
     // This field is only used for ArrowStream format
+    // This field is only used for ArrowStream format
     std::shared_ptr<arrow::RecordBatchReader> stream_reader;
     // The following fields are used only for Arrow format
     // The following fields are used only for Arrow format

From 935524096545a1d0ed7dc7980609c2b98dddfc97 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Wed, 27 May 2020 15:43:25 +0300
Subject: [PATCH 084/184] Better path detection

---
 contrib/cctz-cmake/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/cctz-cmake/CMakeLists.txt b/contrib/cctz-cmake/CMakeLists.txt
index 2c44b25b17a..0837a366f20 100644
--- a/contrib/cctz-cmake/CMakeLists.txt
+++ b/contrib/cctz-cmake/CMakeLists.txt
@@ -623,7 +623,7 @@ if (USE_INTERNAL_CCTZ)
         # libraries in linker command. To avoid this we hardcode whole-archive
         # library into single string.
         add_dependencies(cctz tzdata)
-        target_link_libraries(cctz INTERFACE "-Wl,--whole-archive contrib/cctz-cmake/libtzdata.a -Wl,--no-whole-archive")
+        target_link_libraries(cctz INTERFACE "-Wl,--whole-archive $<TARGET_FILE:tzdata> -Wl,--no-whole-archive")
     endif ()
 
 else ()

From c1cfc68cd611bf5aa7f625185867aec18dfa02b4 Mon Sep 17 00:00:00 2001
From: Alexander Kuzmenkov <akuzm@yandex-team.ru>
Date: Wed, 27 May 2020 15:46:49 +0300
Subject: [PATCH 085/184] revert accidental changes

---
 src/Processors/Formats/Impl/ArrowBlockInputFormat.h | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/Processors/Formats/Impl/ArrowBlockInputFormat.h b/src/Processors/Formats/Impl/ArrowBlockInputFormat.h
index b1ee21a51e4..5ad112efde9 100644
--- a/src/Processors/Formats/Impl/ArrowBlockInputFormat.h
+++ b/src/Processors/Formats/Impl/ArrowBlockInputFormat.h
@@ -25,14 +25,11 @@ protected:
     Chunk generate() override;
 
 private:
-    // Whether to use ArrowStream format
     // Whether to use ArrowStream format
     bool stream;
     // This field is only used for ArrowStream format
-    // This field is only used for ArrowStream format
     std::shared_ptr<arrow::RecordBatchReader> stream_reader;
     // The following fields are used only for Arrow format
-    // The following fields are used only for Arrow format
     std::shared_ptr<arrow::ipc::RecordBatchFileReader> file_reader;
 
     int record_batch_total = 0;

From ad936442547d69ef3d7cf39c708c0f2f78072110 Mon Sep 17 00:00:00 2001
From: Alexander Kuzmenkov <akuzm@yandex-team.ru>
Date: Wed, 27 May 2020 15:50:12 +0300
Subject: [PATCH 086/184] make single if

---
 src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp b/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp
index e0eb0e21567..2873a5417ea 100644
--- a/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp
+++ b/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp
@@ -30,13 +30,6 @@ Chunk ArrowBlockInputFormat::generate()
 {
     Chunk res;
     const Block & header = getPort().getHeader();
-
-    if (!stream)
-    {
-        if (record_batch_current >= record_batch_total)
-            return res;
-    }
-
     std::vector<std::shared_ptr<arrow::RecordBatch>> single_batch(1);
     arrow::Status read_status;
 
@@ -48,6 +41,9 @@ Chunk ArrowBlockInputFormat::generate()
     }
     else
     {
+        if (record_batch_current >= record_batch_total)
+            return res;
+
         read_status = file_reader->ReadRecordBatch(record_batch_current, &single_batch[0]);
     }
 

From a727e1fb1b0a0d51caa2c578e0dfdd30dce18140 Mon Sep 17 00:00:00 2001
From: Albert Kidrachev <kidrachev@sandbox-410109003>
Date: Wed, 27 May 2020 15:57:14 +0300
Subject: [PATCH 087/184] more relaxation

---
 src/Processors/Transforms/PartialSortingTransform.cpp | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/Processors/Transforms/PartialSortingTransform.cpp b/src/Processors/Transforms/PartialSortingTransform.cpp
index 062064c0fd6..ad45159f77e 100644
--- a/src/Processors/Transforms/PartialSortingTransform.cpp
+++ b/src/Processors/Transforms/PartialSortingTransform.cpp
@@ -52,12 +52,13 @@ void PartialSortingTransform::transform(Chunk & chunk)
     auto block = getInputPort().getHeader().cloneWithColumns(chunk.detachColumns());
     chunk.clear();
 
+    ColumnRawPtrs block_columns;
     UInt64 rows_num = block.rows();
 
     if (!threshold_block_columns.empty())
     {
         IColumn::Filter filter(rows_num, 0);
-        ColumnRawPtrs block_columns = extractColumns(block, description);
+        block_columns = extractColumns(block, description);
         size_t filtered_count = 0;
 
         for (UInt64 i = 0; i < rows_num; ++i)
@@ -80,7 +81,8 @@ void PartialSortingTransform::transform(Chunk & chunk)
 
     sortBlock(block, description, limit);
 
-    if (threshold_block_columns.empty() && limit && limit < rows_num)
+    if (limit && limit < rows_num &&
+        (threshold_block_columns.empty() || less(block_columns, limit - 1, threshold_block_columns, limit - 1, description)))
     {
         threshold_block = block.cloneWithColumns(block.getColumns());
         threshold_block_columns = extractColumns(threshold_block, description);

From c34f1ed6fdbfaffca28aa2d61d8b3271fea9f0a8 Mon Sep 17 00:00:00 2001
From: BayoNet <da-daos@yandex.ru>
Date: Wed, 27 May 2020 17:13:37 +0300
Subject: [PATCH 088/184] DOCS-631: Settings partial_merge_join_optimizations,
 partial_merge_join_rows_in_right_blocks (#11130)

* CLICKHOUSEDOCS-631: Settings partial_merge_join_optimizations, partial_merge_join_rows_in_right_blocks

* CLICKHOUSEDOCS-631: Updated by comments.

* CLICKHOUSEDOCS-631: Fixed some grammar.

Co-authored-by: Sergei Shtykov <bayonet@yandex-team.ru>
---
 docs/en/operations/settings/settings.md | 29 +++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index 909ba7bf876..7103819e499 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -404,6 +404,35 @@ Possible values:
 
 Default value: 0.
 
+## partial_merge_join_optimizations {#partial_merge_join_optimizations}
+
+Disables optimizations in partial merge join algorithm for [JOIN](../../sql-reference/statements/select/join.md) queries.
+
+By default, this setting enables improvements that could lead to wrong results. If you see suspicious results in your queries, disable optimizations by this setting. Optimizations can be different in different versions of the ClickHouse server. 
+
+Possible values:
+
+-   0 — Optimizations disabled.
+-   1 — Optimizations enabled.
+
+Default value: 1.
+
+## partial_merge_join_rows_in_right_blocks {#partial_merge_join_rows_in_right_blocks}
+
+Limits sizes of right-hand join data blocks in partial merge join algorithm for [JOIN](../../sql-reference/statements/select/join.md) queries.
+
+ClickHouse server:
+
+1. Splits right-hand join data into blocks with up to the specified number of rows.
+2. Indexes each block with their minimum and maximum values
+3. Unloads prepared blocks to disk if possible.
+
+Possible values:
+
+- Any positive integer. Recommended range of values: [1000, 100000].
+
+Default value: 65536.
+
 ## any_join_distinct_right_table_keys {#any_join_distinct_right_table_keys}
 
 Enables legacy ClickHouse server behavior in `ANY INNER|LEFT JOIN` operations.

From 4fa3628097180dced023b35a4e83aef10beaa6f5 Mon Sep 17 00:00:00 2001
From: Artem Zuikov <chertus@gmail.com>
Date: Wed, 27 May 2020 17:25:11 +0300
Subject: [PATCH 089/184] test for probably fixed issue (#11221)

---
 .../01138_join_on_distributed_and_tmp.reference  |  0
 .../01138_join_on_distributed_and_tmp.sql        | 16 ++++++++++++++++
 2 files changed, 16 insertions(+)
 create mode 100644 tests/queries/0_stateless/01138_join_on_distributed_and_tmp.reference
 create mode 100644 tests/queries/0_stateless/01138_join_on_distributed_and_tmp.sql

diff --git a/tests/queries/0_stateless/01138_join_on_distributed_and_tmp.reference b/tests/queries/0_stateless/01138_join_on_distributed_and_tmp.reference
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/queries/0_stateless/01138_join_on_distributed_and_tmp.sql b/tests/queries/0_stateless/01138_join_on_distributed_and_tmp.sql
new file mode 100644
index 00000000000..67492e7c683
--- /dev/null
+++ b/tests/queries/0_stateless/01138_join_on_distributed_and_tmp.sql
@@ -0,0 +1,16 @@
+DROP TABLE IF EXISTS foo_local;
+DROP TABLE IF EXISTS foo_distributed;
+
+CREATE TABLE foo_local (bar UInt64)
+ENGINE = MergeTree()
+ORDER BY tuple();
+
+CREATE TABLE foo_distributed AS foo_local
+ENGINE = Distributed('test_cluster_two_shards_localhost', currentDatabase(), foo_local);
+
+CREATE TEMPORARY TABLE _tmp_baz (qux UInt64);
+
+SELECT * FROM foo_distributed JOIN _tmp_baz ON foo_distributed.bar = _tmp_baz.qux;
+
+DROP TABLE foo_local;
+DROP TABLE foo_distributed;

From 1658705f7c82839906b9d9b40afde08f2d37f428 Mon Sep 17 00:00:00 2001
From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com>
Date: Wed, 27 May 2020 18:29:22 +0300
Subject: [PATCH 090/184] Add URL/port.cpp to ya.make (a follow-up for #11120)

---
 src/Functions/ya.make | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Functions/ya.make b/src/Functions/ya.make
index ddb1d27b398..0b4776853e9 100644
--- a/src/Functions/ya.make
+++ b/src/Functions/ya.make
@@ -425,6 +425,7 @@ SRCS(
     URL/path.cpp
     URL/pathFull.cpp
     URL/protocol.cpp
+    URL/port.cpp
     URL/queryStringAndFragment.cpp
     URL/queryString.cpp
     URL/registerFunctionsURL.cpp

From 4c5324600b6c847009da083d185d97a9c425b0f6 Mon Sep 17 00:00:00 2001
From: Albert Kidrachev <kidrachev@sandbox-410109003>
Date: Wed, 27 May 2020 18:38:29 +0300
Subject: [PATCH 091/184] fix tests

---
 src/Processors/Transforms/PartialSortingTransform.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Processors/Transforms/PartialSortingTransform.cpp b/src/Processors/Transforms/PartialSortingTransform.cpp
index ad45159f77e..f68a415117b 100644
--- a/src/Processors/Transforms/PartialSortingTransform.cpp
+++ b/src/Processors/Transforms/PartialSortingTransform.cpp
@@ -81,7 +81,7 @@ void PartialSortingTransform::transform(Chunk & chunk)
 
     sortBlock(block, description, limit);
 
-    if (limit && limit < rows_num &&
+    if (limit && limit < block.rows() &&
         (threshold_block_columns.empty() || less(block_columns, limit - 1, threshold_block_columns, limit - 1, description)))
     {
         threshold_block = block.cloneWithColumns(block.getColumns());

From 788e32822adc97ebb4538d335049c25174011b2f Mon Sep 17 00:00:00 2001
From: Michael Smitasin <46496263+michaelsmitasin@users.noreply.github.com>
Date: Wed, 27 May 2020 09:29:34 -0700
Subject: [PATCH 092/184] Added LBNL (#11229)

---
 docs/en/introduction/adopters.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/en/introduction/adopters.md b/docs/en/introduction/adopters.md
index 8868522e977..4daadf32be6 100644
--- a/docs/en/introduction/adopters.md
+++ b/docs/en/introduction/adopters.md
@@ -41,6 +41,7 @@ toc_title: Adopters
 | [Integros](https://integros.com){.favicon}                                    | Platform for video services     | Analytics             | —                                                          | —                                                                            | [Slides in Russian, May 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf)                                                                                               |
 | [Kodiak Data](https://www.kodiakdata.com/){.favicon}                          | Clouds                          | Main product          | —                                                          | —                                                                            | [Slides in Engish, April 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup13/kodiak_data.pdf)                                                                                             |
 | [Kontur](https://kontur.ru){.favicon}                                         | Software Development            | Metrics               | —                                                          | —                                                                            | [Talk in Russian, November 2018](https://www.youtube.com/watch?v=U4u4Bd0FtrY)                                                                                                                                           |
+| [Lawrence Berkeley National Laboratory](https://www.lbl.gov){.favicon}        | Research                        | Traffic analysis      | 1 server                                                   | 11.8 TiB                                                                     | [Slides in English, April 2019](https://www.smitasin.com/presentations/2019-04-17_DOE-NSM.pdf)                                                                                                                          |
 | [LifeStreet](https://lifestreet.com/){.favicon}                               | Ad network                      | Main product          | 75 servers (3 replicas)                                    | 5.27 PiB                                                                     | [Blog post in Russian, February 2017](https://habr.com/en/post/322620/)                                                                                                                                                 |
 | [Mail.ru Cloud Solutions](https://mcs.mail.ru/){.favicon}                     | Cloud services                  | Main product          | —                                                          | —                                                                            | [Article in Russian](https://mcs.mail.ru/help/db-create/clickhouse#)                                                                                                                               |
 | [MessageBird](https://www.messagebird.com){.favicon}                          | Telecommunications              | Statistics            | —                                                          | —                                                                            | [Slides in English, November 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup20/messagebird.pdf)                                                                                         |

From 7b03e36c2a6c390344ce855eac7b6dd95a6b9e08 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitbar@yandex-team.ru>
Date: Wed, 27 May 2020 17:03:38 +0300
Subject: [PATCH 093/184] Improve build scripts related to protobuf and gRPC a
 little more.

---
 cmake/protobuf_generate_cpp.cmake | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/cmake/protobuf_generate_cpp.cmake b/cmake/protobuf_generate_cpp.cmake
index 40ec3c8e681..7ee9e8d7c81 100644
--- a/cmake/protobuf_generate_cpp.cmake
+++ b/cmake/protobuf_generate_cpp.cmake
@@ -112,16 +112,21 @@ if (PROTOBUF_GENERATE_CPP_SCRIPT_MODE)
     set (intermediate_dir ${DIR}/intermediate)
     set (intermediate_output "${intermediate_dir}/${FILENAME}")
 
-    if (COMPILER_ID STREQUAL "GNU")
+    if (COMPILER_ID STREQUAL "Clang")
+        set (pragma_push "#pragma clang diagnostic push\n")
+        set (pragma_pop "#pragma clang diagnostic pop\n")
+        set (pragma_disable_warnings "#pragma clang diagnostic ignored \"-Weverything\"\n")
+    elseif (COMPILER_ID MATCHES "GNU")
         set (pragma_push "#pragma GCC diagnostic push\n")
         set (pragma_pop "#pragma GCC diagnostic pop\n")
         set (pragma_disable_warnings "#pragma GCC diagnostic ignored \"-Wall\"\n"
                                      "#pragma GCC diagnostic ignored \"-Wextra\"\n"
-                                     "#pragma GCC diagnostic ignored \"-Warray-bounds\"\n")
-    elseif (COMPILER_ID MATCHES "Clang")
-        set (pragma_push "#pragma clang diagnostic push\n")
-        set (pragma_pop "#pragma clang diagnostic pop\n")
-        set (pragma_disable_warnings "#pragma clang diagnostic ignored \"-Weverything\"\n")
+                                     "#pragma GCC diagnostic ignored \"-Warray-bounds\"\n"
+                                     "#pragma GCC diagnostic ignored \"-Wold-style-cast\"\n"
+                                     "#pragma GCC diagnostic ignored \"-Wshadow\"\n"
+                                     "#pragma GCC diagnostic ignored \"-Wsuggest-override\"\n"
+                                     "#pragma GCC diagnostic ignored \"-Wcast-qual\"\n"
+                                     "#pragma GCC diagnostic ignored \"-Wunused-parameter\"\n")
     endif()
 
     if (${FILENAME} MATCHES ".*\\.h")

From fd64d391666a06fe737fde6be951185b5af0c74e Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Wed, 27 May 2020 20:23:09 +0300
Subject: [PATCH 094/184] Fix style check for POPCNT check

---
 programs/main.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/main.cpp b/programs/main.cpp
index b1bff1de1b0..746b624e065 100644
--- a/programs/main.cpp
+++ b/programs/main.cpp
@@ -198,7 +198,7 @@ void checkRequiredInstructionsImpl(volatile InstructionFail & fail)
     {
         uint64_t a = 0;
         uint64_t b = 0;
-        __asm__ volatile ("popcnt %1, %0" : "=r"(a) :"r"(b) : );
+        __asm__ volatile ("popcnt %1, %0" : "=r"(a) :"r"(b) :);
     }
 #endif
 

From bb0045a242e5f23908ddbaf84706fa10da6bcf0a Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitbar@yandex-team.ru>
Date: Wed, 27 May 2020 16:27:51 +0300
Subject: [PATCH 095/184] Add test for row policy that policy defined in
 users.xml affects only the user it assigned to.

---
 tests/integration/test_row_policy/test.py | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/tests/integration/test_row_policy/test.py b/tests/integration/test_row_policy/test.py
index a1884d059c7..71496c6dbf2 100644
--- a/tests/integration/test_row_policy/test.py
+++ b/tests/integration/test_row_policy/test.py
@@ -42,8 +42,13 @@ def started_cluster():
 
                 CREATE TABLE mydb.`.filtered_table4` (a UInt8, b UInt8, c UInt16 ALIAS a + b) ENGINE MergeTree ORDER BY a;
                 INSERT INTO mydb.`.filtered_table4` values (0, 0), (0, 1), (1, 0), (1, 1);
+                
+                CREATE TABLE mydb.local (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a;
             ''')
 
+        node.query("INSERT INTO mydb.local values (2, 0), (2, 1), (1, 0), (1, 1)")
+        node2.query("INSERT INTO mydb.local values (3, 0), (3, 1), (1, 0), (1, 1)")
+
         yield cluster
 
     finally:
@@ -122,6 +127,17 @@ def test_single_table_name():
     assert node.query("SELECT a + b = 1 FROM mydb.filtered_table3") == TSV([[1], [1]])
 
 
+def test_policy_from_users_xml_affects_only_user_assigned():
+    assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1,0], [1, 1]])
+    assert node.query("SELECT * FROM mydb.filtered_table1", user="another") == TSV([[0, 0], [0, 1], [1, 0], [1, 1]])
+
+    assert node.query("SELECT * FROM mydb.filtered_table2") == TSV([[0, 0, 0, 0], [0, 0, 6, 0]])
+    assert node.query("SELECT * FROM mydb.filtered_table2", user="another") == TSV([[0, 0, 0, 0], [0, 0, 6, 0], [1, 2, 3, 4], [4, 3, 2, 1]])
+
+    assert node.query("SELECT * FROM mydb.local") == TSV([[1,0], [1, 1], [2, 0], [2, 1]])
+    assert node.query("SELECT * FROM mydb.local", user="another") == TSV([[1, 0], [1, 1]])
+
+
 def test_custom_table_name():
     copy_policy_xml('multiple_tags_with_table_names.xml')
     assert node.query("SELECT * FROM mydb.table") == TSV([[1, 0], [1, 1]])
@@ -286,9 +302,5 @@ def test_miscellaneous_engines():
     # DistributedMergeTree
     node.query("DROP TABLE IF EXISTS mydb.not_filtered_table")
     node.query("CREATE TABLE mydb.not_filtered_table (a UInt8, b UInt8) ENGINE Distributed('test_local_cluster', mydb, local)")
-    node.query("CREATE TABLE mydb.local (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a")
-    node2.query("CREATE TABLE mydb.local (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a")
-    node.query("INSERT INTO mydb.local values (2, 0), (2, 1), (1, 0), (1, 1)")
-    node2.query("INSERT INTO mydb.local values (3, 0), (3, 1), (1, 0), (1, 1)")
     assert node.query("SELECT * FROM mydb.not_filtered_table", user="another") == TSV([[1, 0], [1, 1], [1, 0], [1, 1]])
     assert node.query("SELECT sum(a), b FROM mydb.not_filtered_table GROUP BY b ORDER BY b", user="another") == TSV([[2, 0], [2, 1]])

From 5b858de374b1de5d4c1a53d64142c70dbf4841ec Mon Sep 17 00:00:00 2001
From: Ivan <5627721+abyss7@users.noreply.github.com>
Date: Wed, 27 May 2020 20:52:52 +0300
Subject: [PATCH 096/184] Split programs/server into actual program and library
 (#11186)

* Split programs/server into actual program and library
---
 programs/server/CMakeLists.txt                | 19 ++--------------
 programs/server/Server.cpp                    |  6 ++---
 programs/server/Server.h                      |  2 +-
 programs/server/ya.make                       | 11 ----------
 src/CMakeLists.txt                            |  2 ++
 src/Server/CMakeLists.txt                     |  0
 .../server => src/Server}/HTTPHandler.cpp     |  0
 {programs/server => src/Server}/HTTPHandler.h |  3 ++-
 .../Server}/HTTPHandlerFactory.cpp            |  1 +
 .../Server}/HTTPHandlerFactory.h              |  0
 .../Server}/HTTPHandlerRequestFilter.h        |  1 +
 {programs/server => src/Server}/IServer.h     | 14 +++++++++---
 .../Server}/InterserverIOHTTPHandler.cpp      |  1 +
 .../Server}/InterserverIOHTTPHandler.h        |  0
 .../server => src/Server}/MySQLHandler.cpp    |  0
 .../server => src/Server}/MySQLHandler.h      |  0
 .../Server}/MySQLHandlerFactory.cpp           |  3 +--
 .../Server}/MySQLHandlerFactory.h             |  3 ++-
 .../server => src/Server}/NotFoundHandler.cpp |  0
 .../server => src/Server}/NotFoundHandler.h   |  0
 .../Server}/PrometheusMetricsWriter.cpp       |  0
 .../Server}/PrometheusMetricsWriter.h         |  0
 .../Server}/PrometheusRequestHandler.cpp      |  1 +
 .../Server}/PrometheusRequestHandler.h        |  0
 .../Server}/ReplicasStatusHandler.cpp         |  0
 .../Server}/ReplicasStatusHandler.h           |  0
 .../Server}/StaticRequestHandler.cpp          |  0
 .../Server}/StaticRequestHandler.h            |  2 ++
 .../server => src/Server}/TCPHandler.cpp      |  0
 {programs/server => src/Server}/TCPHandler.h  |  0
 .../server => src/Server}/TCPHandlerFactory.h |  4 ++--
 src/Server/ya.make                            | 22 +++++++++++++++++++
 src/ya.make                                   |  1 +
 33 files changed, 55 insertions(+), 41 deletions(-)
 create mode 100644 src/Server/CMakeLists.txt
 rename {programs/server => src/Server}/HTTPHandler.cpp (100%)
 rename {programs/server => src/Server}/HTTPHandler.h (97%)
 rename {programs/server => src/Server}/HTTPHandlerFactory.cpp (99%)
 rename {programs/server => src/Server}/HTTPHandlerFactory.h (100%)
 rename {programs/server => src/Server}/HTTPHandlerRequestFilter.h (99%)
 rename {programs/server => src/Server}/IServer.h (82%)
 rename {programs/server => src/Server}/InterserverIOHTTPHandler.cpp (99%)
 rename {programs/server => src/Server}/InterserverIOHTTPHandler.h (100%)
 rename {programs/server => src/Server}/MySQLHandler.cpp (100%)
 rename {programs/server => src/Server}/MySQLHandler.h (100%)
 rename {programs/server => src/Server}/MySQLHandlerFactory.cpp (99%)
 rename {programs/server => src/Server}/MySQLHandlerFactory.h (95%)
 rename {programs/server => src/Server}/NotFoundHandler.cpp (100%)
 rename {programs/server => src/Server}/NotFoundHandler.h (100%)
 rename {programs/server => src/Server}/PrometheusMetricsWriter.cpp (100%)
 rename {programs/server => src/Server}/PrometheusMetricsWriter.h (100%)
 rename {programs/server => src/Server}/PrometheusRequestHandler.cpp (95%)
 rename {programs/server => src/Server}/PrometheusRequestHandler.h (100%)
 rename {programs/server => src/Server}/ReplicasStatusHandler.cpp (100%)
 rename {programs/server => src/Server}/ReplicasStatusHandler.h (100%)
 rename {programs/server => src/Server}/StaticRequestHandler.cpp (100%)
 rename {programs/server => src/Server}/StaticRequestHandler.h (93%)
 rename {programs/server => src/Server}/TCPHandler.cpp (100%)
 rename {programs/server => src/Server}/TCPHandler.h (100%)
 rename {programs/server => src/Server}/TCPHandlerFactory.h (95%)
 create mode 100644 src/Server/ya.make

diff --git a/programs/server/CMakeLists.txt b/programs/server/CMakeLists.txt
index 026bb0bfeb2..1563f5ac51e 100644
--- a/programs/server/CMakeLists.txt
+++ b/programs/server/CMakeLists.txt
@@ -1,21 +1,6 @@
 set(CLICKHOUSE_SERVER_SOURCES
-    ${CMAKE_CURRENT_SOURCE_DIR}/HTTPHandler.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/HTTPHandlerFactory.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/InterserverIOHTTPHandler.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/MetricsTransmitter.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/NotFoundHandler.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/PrometheusMetricsWriter.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/PrometheusRequestHandler.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/ReplicasStatusHandler.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/StaticRequestHandler.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/Server.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/TCPHandler.cpp
-)
-
-set(CLICKHOUSE_SERVER_SOURCES
-    ${CLICKHOUSE_SERVER_SOURCES}
-    ${CMAKE_CURRENT_SOURCE_DIR}/MySQLHandler.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/MySQLHandlerFactory.cpp
+    MetricsTransmitter.cpp
+    Server.cpp
 )
 
 set (CLICKHOUSE_SERVER_LINK
diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp
index c1a520030f4..ce1d35e65d4 100644
--- a/programs/server/Server.cpp
+++ b/programs/server/Server.cpp
@@ -53,13 +53,13 @@
 #include <Dictionaries/registerDictionaries.h>
 #include <Disks/registerDisks.h>
 #include <Common/Config/ConfigReloader.h>
-#include "HTTPHandlerFactory.h"
+#include <Server/HTTPHandlerFactory.h>
 #include "MetricsTransmitter.h"
 #include <Common/StatusFile.h>
-#include "TCPHandlerFactory.h"
+#include <Server/TCPHandlerFactory.h>
 #include <Common/SensitiveDataMasker.h>
 #include <Common/ThreadFuzzer.h>
-#include "MySQLHandlerFactory.h"
+#include <Server/MySQLHandlerFactory.h>
 
 #if !defined(ARCADIA_BUILD)
 #   include "config_core.h"
diff --git a/programs/server/Server.h b/programs/server/Server.h
index ffd89df6af4..ad9e51c881c 100644
--- a/programs/server/Server.h
+++ b/programs/server/Server.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#include "IServer.h"
+#include <Server/IServer.h>
 
 #include <daemon/BaseDaemon.h>
 
diff --git a/programs/server/ya.make b/programs/server/ya.make
index 2c74c01c7cb..2e13267f715 100644
--- a/programs/server/ya.make
+++ b/programs/server/ya.make
@@ -11,19 +11,8 @@ PEERDIR(
 SRCS(
     clickhouse-server.cpp
 
-    HTTPHandler.cpp
-    HTTPHandlerFactory.cpp
-    InterserverIOHTTPHandler.cpp
     MetricsTransmitter.cpp
-    MySQLHandler.cpp
-    MySQLHandlerFactory.cpp
-    NotFoundHandler.cpp
-    PrometheusMetricsWriter.cpp
-    PrometheusRequestHandler.cpp
-    ReplicasStatusHandler.cpp
-    StaticRequestHandler.cpp
     Server.cpp
-    TCPHandler.cpp
 )
 
 END()
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 222a3e486f9..baa0fbcb883 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -58,6 +58,7 @@ add_subdirectory (TableFunctions)
 add_subdirectory (Processors)
 add_subdirectory (Formats)
 add_subdirectory (Compression)
+add_subdirectory (Server)
 
 
 set(dbms_headers)
@@ -145,6 +146,7 @@ add_object_library(clickhouse_storages_distributed Storages/Distributed)
 add_object_library(clickhouse_storages_mergetree Storages/MergeTree)
 add_object_library(clickhouse_storages_liveview Storages/LiveView)
 add_object_library(clickhouse_client Client)
+add_object_library(clickhouse_server Server)
 add_object_library(clickhouse_formats Formats)
 add_object_library(clickhouse_processors Processors)
 add_object_library(clickhouse_processors_executors Processors/Executors)
diff --git a/src/Server/CMakeLists.txt b/src/Server/CMakeLists.txt
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/programs/server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp
similarity index 100%
rename from programs/server/HTTPHandler.cpp
rename to src/Server/HTTPHandler.cpp
diff --git a/programs/server/HTTPHandler.h b/src/Server/HTTPHandler.h
similarity index 97%
rename from programs/server/HTTPHandler.h
rename to src/Server/HTTPHandler.h
index 6228523d343..b1a6355d281 100644
--- a/programs/server/HTTPHandler.h
+++ b/src/Server/HTTPHandler.h
@@ -6,6 +6,7 @@
 
 #include <Common/CurrentMetrics.h>
 #include <Common/HTMLForm.h>
+#include <Core/Names.h>
 
 #include <re2/re2.h>
 
@@ -21,7 +22,7 @@ namespace DB
 
 class WriteBufferFromHTTPServerResponse;
 
-typedef std::shared_ptr<const re2::RE2> CompiledRegexPtr;
+using CompiledRegexPtr = std::shared_ptr<const re2::RE2>;
 
 class HTTPHandler : public Poco::Net::HTTPRequestHandler
 {
diff --git a/programs/server/HTTPHandlerFactory.cpp b/src/Server/HTTPHandlerFactory.cpp
similarity index 99%
rename from programs/server/HTTPHandlerFactory.cpp
rename to src/Server/HTTPHandlerFactory.cpp
index f302216e22b..e916070be22 100644
--- a/programs/server/HTTPHandlerFactory.cpp
+++ b/src/Server/HTTPHandlerFactory.cpp
@@ -4,6 +4,7 @@
 #include <re2/stringpiece.h>
 #include <common/find_symbols.h>
 #include <Poco/StringTokenizer.h>
+#include <Poco/Util/LayeredConfiguration.h>
 
 #include "HTTPHandler.h"
 #include "NotFoundHandler.h"
diff --git a/programs/server/HTTPHandlerFactory.h b/src/Server/HTTPHandlerFactory.h
similarity index 100%
rename from programs/server/HTTPHandlerFactory.h
rename to src/Server/HTTPHandlerFactory.h
diff --git a/programs/server/HTTPHandlerRequestFilter.h b/src/Server/HTTPHandlerRequestFilter.h
similarity index 99%
rename from programs/server/HTTPHandlerRequestFilter.h
rename to src/Server/HTTPHandlerRequestFilter.h
index b0b748506e5..f952efd7653 100644
--- a/programs/server/HTTPHandlerRequestFilter.h
+++ b/src/Server/HTTPHandlerRequestFilter.h
@@ -6,6 +6,7 @@
 #include <re2/stringpiece.h>
 #include <Poco/StringTokenizer.h>
 #include <Poco/Net/HTTPServerRequest.h>
+#include <Poco/Util/LayeredConfiguration.h>
 
 #include <common/find_symbols.h>
 
diff --git a/programs/server/IServer.h b/src/Server/IServer.h
similarity index 82%
rename from programs/server/IServer.h
rename to src/Server/IServer.h
index 29e9bc16a75..131e7443646 100644
--- a/programs/server/IServer.h
+++ b/src/Server/IServer.h
@@ -1,14 +1,22 @@
 #pragma once
 
-#include <Poco/Logger.h>
-#include <Poco/Util/LayeredConfiguration.h>
+namespace Poco
+{
 
-#include <Interpreters/Context.h>
+namespace Util
+{
+class LayeredConfiguration;
+}
+class Logger;
+
+}
 
 
 namespace DB
 {
 
+class Context;
+
 class IServer
 {
 public:
diff --git a/programs/server/InterserverIOHTTPHandler.cpp b/src/Server/InterserverIOHTTPHandler.cpp
similarity index 99%
rename from programs/server/InterserverIOHTTPHandler.cpp
rename to src/Server/InterserverIOHTTPHandler.cpp
index 4b733c7f1fd..062721a01aa 100644
--- a/programs/server/InterserverIOHTTPHandler.cpp
+++ b/src/Server/InterserverIOHTTPHandler.cpp
@@ -3,6 +3,7 @@
 #include <Poco/Net/HTTPBasicCredentials.h>
 #include <Poco/Net/HTTPServerRequest.h>
 #include <Poco/Net/HTTPServerResponse.h>
+#include <Poco/Util/LayeredConfiguration.h>
 #include <common/logger_useful.h>
 #include <Common/HTMLForm.h>
 #include <Common/setThreadName.h>
diff --git a/programs/server/InterserverIOHTTPHandler.h b/src/Server/InterserverIOHTTPHandler.h
similarity index 100%
rename from programs/server/InterserverIOHTTPHandler.h
rename to src/Server/InterserverIOHTTPHandler.h
diff --git a/programs/server/MySQLHandler.cpp b/src/Server/MySQLHandler.cpp
similarity index 100%
rename from programs/server/MySQLHandler.cpp
rename to src/Server/MySQLHandler.cpp
diff --git a/programs/server/MySQLHandler.h b/src/Server/MySQLHandler.h
similarity index 100%
rename from programs/server/MySQLHandler.h
rename to src/Server/MySQLHandler.h
diff --git a/programs/server/MySQLHandlerFactory.cpp b/src/Server/MySQLHandlerFactory.cpp
similarity index 99%
rename from programs/server/MySQLHandlerFactory.cpp
rename to src/Server/MySQLHandlerFactory.cpp
index 022167fe766..5d78ed81068 100644
--- a/programs/server/MySQLHandlerFactory.cpp
+++ b/src/Server/MySQLHandlerFactory.cpp
@@ -4,8 +4,7 @@
 #include <Poco/Util/Application.h>
 #include <common/logger_useful.h>
 #include <ext/scope_guard.h>
-#include "IServer.h"
-#include "MySQLHandler.h"
+#include <Server/MySQLHandler.h>
 
 #if USE_SSL
 #    include <Poco/Net/SSLManager.h>
diff --git a/programs/server/MySQLHandlerFactory.h b/src/Server/MySQLHandlerFactory.h
similarity index 95%
rename from programs/server/MySQLHandlerFactory.h
rename to src/Server/MySQLHandlerFactory.h
index 74f0bb35a40..df7bd794b16 100644
--- a/programs/server/MySQLHandlerFactory.h
+++ b/src/Server/MySQLHandlerFactory.h
@@ -2,7 +2,8 @@
 
 #include <Poco/Net/TCPServerConnectionFactory.h>
 #include <atomic>
-#include "IServer.h"
+#include <memory>
+#include <Server/IServer.h>
 
 #if !defined(ARCADIA_BUILD)
 #    include <Common/config.h>
diff --git a/programs/server/NotFoundHandler.cpp b/src/Server/NotFoundHandler.cpp
similarity index 100%
rename from programs/server/NotFoundHandler.cpp
rename to src/Server/NotFoundHandler.cpp
diff --git a/programs/server/NotFoundHandler.h b/src/Server/NotFoundHandler.h
similarity index 100%
rename from programs/server/NotFoundHandler.h
rename to src/Server/NotFoundHandler.h
diff --git a/programs/server/PrometheusMetricsWriter.cpp b/src/Server/PrometheusMetricsWriter.cpp
similarity index 100%
rename from programs/server/PrometheusMetricsWriter.cpp
rename to src/Server/PrometheusMetricsWriter.cpp
diff --git a/programs/server/PrometheusMetricsWriter.h b/src/Server/PrometheusMetricsWriter.h
similarity index 100%
rename from programs/server/PrometheusMetricsWriter.h
rename to src/Server/PrometheusMetricsWriter.h
diff --git a/programs/server/PrometheusRequestHandler.cpp b/src/Server/PrometheusRequestHandler.cpp
similarity index 95%
rename from programs/server/PrometheusRequestHandler.cpp
rename to src/Server/PrometheusRequestHandler.cpp
index b5a48d13b64..43f39e36de8 100644
--- a/programs/server/PrometheusRequestHandler.cpp
+++ b/src/Server/PrometheusRequestHandler.cpp
@@ -6,6 +6,7 @@
 
 #include <Poco/Net/HTTPServerRequest.h>
 #include <Poco/Net/HTTPServerResponse.h>
+#include <Poco/Util/LayeredConfiguration.h>
 
 #include <Common/ProfileEvents.h>
 #include <Common/CurrentMetrics.h>
diff --git a/programs/server/PrometheusRequestHandler.h b/src/Server/PrometheusRequestHandler.h
similarity index 100%
rename from programs/server/PrometheusRequestHandler.h
rename to src/Server/PrometheusRequestHandler.h
diff --git a/programs/server/ReplicasStatusHandler.cpp b/src/Server/ReplicasStatusHandler.cpp
similarity index 100%
rename from programs/server/ReplicasStatusHandler.cpp
rename to src/Server/ReplicasStatusHandler.cpp
diff --git a/programs/server/ReplicasStatusHandler.h b/src/Server/ReplicasStatusHandler.h
similarity index 100%
rename from programs/server/ReplicasStatusHandler.h
rename to src/Server/ReplicasStatusHandler.h
diff --git a/programs/server/StaticRequestHandler.cpp b/src/Server/StaticRequestHandler.cpp
similarity index 100%
rename from programs/server/StaticRequestHandler.cpp
rename to src/Server/StaticRequestHandler.cpp
diff --git a/programs/server/StaticRequestHandler.h b/src/Server/StaticRequestHandler.h
similarity index 93%
rename from programs/server/StaticRequestHandler.h
rename to src/Server/StaticRequestHandler.h
index bdbc17f5e00..707087df24d 100644
--- a/programs/server/StaticRequestHandler.h
+++ b/src/Server/StaticRequestHandler.h
@@ -4,6 +4,8 @@
 
 #include <Poco/Net/HTTPRequestHandler.h>
 #include <Common/StringUtils/StringUtils.h>
+#include <Core/Types.h>
+#include <IO/WriteBuffer.h>
 
 
 namespace DB
diff --git a/programs/server/TCPHandler.cpp b/src/Server/TCPHandler.cpp
similarity index 100%
rename from programs/server/TCPHandler.cpp
rename to src/Server/TCPHandler.cpp
diff --git a/programs/server/TCPHandler.h b/src/Server/TCPHandler.h
similarity index 100%
rename from programs/server/TCPHandler.h
rename to src/Server/TCPHandler.h
diff --git a/programs/server/TCPHandlerFactory.h b/src/Server/TCPHandlerFactory.h
similarity index 95%
rename from programs/server/TCPHandlerFactory.h
rename to src/Server/TCPHandlerFactory.h
index 68652540192..a5532a8dc02 100644
--- a/programs/server/TCPHandlerFactory.h
+++ b/src/Server/TCPHandlerFactory.h
@@ -3,8 +3,8 @@
 #include <Poco/Net/TCPServerConnectionFactory.h>
 #include <Poco/Net/NetException.h>
 #include <common/logger_useful.h>
-#include "IServer.h"
-#include "TCPHandler.h"
+#include <Server/IServer.h>
+#include <Server/TCPHandler.h>
 
 namespace Poco { class Logger; }
 
diff --git a/src/Server/ya.make b/src/Server/ya.make
new file mode 100644
index 00000000000..1d689ee73b8
--- /dev/null
+++ b/src/Server/ya.make
@@ -0,0 +1,22 @@
+LIBRARY()
+
+PEERDIR(
+    clickhouse/src/Common
+    contrib/libs/poco/Util
+)
+
+SRCS(
+    HTTPHandler.cpp
+    HTTPHandlerFactory.cpp
+    InterserverIOHTTPHandler.cpp
+    MySQLHandler.cpp
+    MySQLHandlerFactory.cpp
+    NotFoundHandler.cpp
+    PrometheusMetricsWriter.cpp
+    PrometheusRequestHandler.cpp
+    ReplicasStatusHandler.cpp
+    StaticRequestHandler.cpp
+    TCPHandler.cpp
+)
+
+END()
diff --git a/src/ya.make b/src/ya.make
index 95e5914bbe7..eedd98cb178 100644
--- a/src/ya.make
+++ b/src/ya.make
@@ -19,6 +19,7 @@ PEERDIR(
     clickhouse/src/IO
     clickhouse/src/Parsers
     clickhouse/src/Processors
+    clickhouse/src/Server
     clickhouse/src/Storages
     clickhouse/src/TableFunctions
 )

From 0456302355d6eedc1c502ba60497b80941e75d91 Mon Sep 17 00:00:00 2001
From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com>
Date: Wed, 27 May 2020 20:58:05 +0300
Subject: [PATCH 097/184] Update formats.md

---
 docs/en/interfaces/formats.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md
index ed411a1b0d4..a08f2db7149 100644
--- a/docs/en/interfaces/formats.md
+++ b/docs/en/interfaces/formats.md
@@ -1143,13 +1143,13 @@ $ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Parquet" > {some_
 
 To exchange data with Hadoop, you can use [HDFS table engine](../engines/table-engines/integrations/hdfs.md).
 
-## Arrow {data-format-arrow}
+## Arrow {#data-format-arrow}
 
 [Apache Arrow](https://arrow.apache.org/) comes with two built-in columnar storage formats. ClickHouse supports read and write operations for these formats.
 
 `Arrow` is Apache Arrow's "file mode" format. It is designed for in-memory random access.
 
-## ArrowStream {data-format-arrow-stream}
+## ArrowStream {#data-format-arrow-stream}
 
 `ArrowStream` is Apache Arrow's "stream mode" format. It is designed for in-memory stream processing.
 

From e93882c977ba53486e6bd06eac2d4b16cb7d3ecc Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Wed, 27 May 2020 21:20:26 +0300
Subject: [PATCH 098/184] Insert select using PipelineExecutor.

---
 programs/server/TCPHandler.cpp                |   4 +-
 .../NullAndDoCopyBlockInputStream.h           |  24 +---
 src/DataStreams/copyData.cpp                  |  81 ------------
 src/DataStreams/copyData.h                    |   2 -
 src/Interpreters/InterpreterInsertQuery.cpp   |  78 +++++++-----
 src/Interpreters/executeQuery.cpp             |  48 ++++---
 .../PullingAsyncPipelineExecutor.cpp          |  51 ++++++--
 .../Executors/PullingPipelineExecutor.cpp     |   2 +-
 src/Processors/QueryPipeline.cpp              | 117 +++++++++++++-----
 src/Processors/QueryPipeline.h                |  16 ++-
 10 files changed, 222 insertions(+), 201 deletions(-)

diff --git a/programs/server/TCPHandler.cpp b/programs/server/TCPHandler.cpp
index e64d7799dcb..21a4a389b9e 100644
--- a/programs/server/TCPHandler.cpp
+++ b/programs/server/TCPHandler.cpp
@@ -263,8 +263,8 @@ void TCPHandler::runImpl()
             else if (state.need_receive_data_for_input)
             {
                 /// It is special case for input(), all works for reading data from client will be done in callbacks.
-                /// state.io.in is NullAndDoCopyBlockInputStream so read it once.
-                state.io.in->read();
+                auto executor = state.io.pipeline.execute();
+                executor->execute(state.io.pipeline.getNumThreads());
                 state.io.onFinish();
             }
             else if (state.io.pipeline.initialized())
diff --git a/src/DataStreams/NullAndDoCopyBlockInputStream.h b/src/DataStreams/NullAndDoCopyBlockInputStream.h
index 8fe05c387a3..8bfb3538f3a 100644
--- a/src/DataStreams/NullAndDoCopyBlockInputStream.h
+++ b/src/DataStreams/NullAndDoCopyBlockInputStream.h
@@ -21,19 +21,10 @@ class NullAndDoCopyBlockInputStream : public IBlockInputStream
 {
 public:
     NullAndDoCopyBlockInputStream(const BlockInputStreamPtr & input_, BlockOutputStreamPtr output_)
+        : input(std::move(input_))
+        , output(std::move(output_))
     {
-        input_streams.push_back(input_);
-        output_streams.push_back(output_);
-
-        for (auto & input_stream : input_streams)
-            children.push_back(input_stream);
-    }
-
-    NullAndDoCopyBlockInputStream(const BlockInputStreams & input_, BlockOutputStreams & output_)
-        : input_streams(input_), output_streams(output_)
-    {
-        for (auto & input_stream : input_)
-            children.push_back(input_stream);
+        children.push_back(input);
     }
 
     /// Suppress readPrefix and readSuffix, because they are called by copyData.
@@ -53,16 +44,13 @@ protected:
         /// If query was cancelled, it will be processed by child streams.
         /// Part of the data will be processed.
 
-        if (input_streams.size() == 1 && output_streams.size() == 1)
-            copyData(*input_streams.at(0), *output_streams.at(0));
-        else
-            copyData(input_streams, output_streams);
+        copyData(*input, *output);
         return Block();
     }
 
 private:
-    BlockInputStreams input_streams;
-    BlockOutputStreams output_streams;
+    BlockInputStreamPtr input;
+    BlockOutputStreamPtr output;
 };
 
 }
diff --git a/src/DataStreams/copyData.cpp b/src/DataStreams/copyData.cpp
index fd4bfab28d8..a0651999034 100644
--- a/src/DataStreams/copyData.cpp
+++ b/src/DataStreams/copyData.cpp
@@ -1,9 +1,6 @@
-#include <thread>
 #include <DataStreams/IBlockInputStream.h>
 #include <DataStreams/IBlockOutputStream.h>
 #include <DataStreams/copyData.h>
-#include <DataStreams/ParallelInputsProcessor.h>
-#include <Common/ConcurrentBoundedQueue.h>
 #include <Common/ThreadPool.h>
 
 
@@ -55,79 +52,6 @@ void copyDataImpl(IBlockInputStream & from, IBlockOutputStream & to, TCancelCall
 
 inline void doNothing(const Block &) {}
 
-namespace
-{
-
-
-struct ParallelInsertsHandler
-{
-    using CencellationHook = std::function<void()>;
-
-    explicit ParallelInsertsHandler(BlockOutputStreams & output_streams, CencellationHook cancellation_hook_, size_t num_threads)
-        : outputs(output_streams.size()), cancellation_hook(std::move(cancellation_hook_))
-    {
-        exceptions.resize(num_threads);
-
-        for (auto & output : output_streams)
-            outputs.push(output.get());
-    }
-
-    void onBlock(Block & block, size_t /*thread_num*/)
-    {
-        IBlockOutputStream * out = nullptr;
-
-        outputs.pop(out);
-        out->write(block);
-        outputs.push(out);
-    }
-
-    void onFinishThread(size_t /*thread_num*/) {}
-    void onFinish() {}
-
-    void onException(std::exception_ptr & exception, size_t thread_num)
-    {
-        exceptions[thread_num] = exception;
-        cancellation_hook();
-    }
-
-    void rethrowFirstException()
-    {
-        for (auto & exception : exceptions)
-            if (exception)
-                std::rethrow_exception(exception);
-    }
-
-    ConcurrentBoundedQueue<IBlockOutputStream *> outputs;
-    std::vector<std::exception_ptr> exceptions;
-    CencellationHook cancellation_hook;
-};
-
-}
-
-static void copyDataImpl(BlockInputStreams & inputs, BlockOutputStreams & outputs)
-{
-    for (auto & output : outputs)
-        output->writePrefix();
-
-    using Processor = ParallelInputsProcessor<ParallelInsertsHandler>;
-    Processor * processor_ptr = nullptr;
-
-    ParallelInsertsHandler handler(outputs, [&processor_ptr]() { processor_ptr->cancel(false); }, inputs.size());
-    ParallelInputsProcessor<ParallelInsertsHandler> processor(inputs, nullptr, inputs.size(), handler);
-    processor_ptr = &processor;
-
-    processor.process();
-    processor.wait();
-    handler.rethrowFirstException();
-
-    /// readPrefix is called in ParallelInputsProcessor.
-    for (auto & input : inputs)
-        input->readSuffix();
-
-    for (auto & output : outputs)
-        output->writeSuffix();
-}
-
 void copyData(IBlockInputStream & from, IBlockOutputStream & to, std::atomic<bool> * is_cancelled)
 {
     auto is_cancelled_pred = [is_cancelled] ()
@@ -138,11 +62,6 @@ void copyData(IBlockInputStream & from, IBlockOutputStream & to, std::atomic<boo
     copyDataImpl(from, to, is_cancelled_pred, doNothing);
 }
 
-void copyData(BlockInputStreams & inputs, BlockOutputStreams & outputs)
-{
-    copyDataImpl(inputs, outputs);
-}
-
 void copyData(IBlockInputStream & from, IBlockOutputStream & to, const std::function<bool()> & is_cancelled)
 {
     copyDataImpl(from, to, is_cancelled, doNothing);
diff --git a/src/DataStreams/copyData.h b/src/DataStreams/copyData.h
index ae72dbd2421..f2bce8f411b 100644
--- a/src/DataStreams/copyData.h
+++ b/src/DataStreams/copyData.h
@@ -16,8 +16,6 @@ class Block;
   */
 void copyData(IBlockInputStream & from, IBlockOutputStream & to, std::atomic<bool> * is_cancelled = nullptr);
 
-void copyData(BlockInputStreams & inputs, BlockOutputStreams & outputs);
-
 void copyData(IBlockInputStream & from, IBlockOutputStream & to, const std::function<bool()> & is_cancelled);
 
 void copyData(IBlockInputStream & from, IBlockOutputStream & to, const std::function<bool()> & is_cancelled,
diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp
index f3b116e490c..b6efa5d6d46 100644
--- a/src/Interpreters/InterpreterInsertQuery.cpp
+++ b/src/Interpreters/InterpreterInsertQuery.cpp
@@ -28,6 +28,11 @@
 #include <Storages/StorageDistributed.h>
 #include <TableFunctions/TableFunctionFactory.h>
 #include <Common/checkStackSize.h>
+#include <Processors/Sources/SourceFromInputStream.h>
+#include <Processors/NullSink.h>
+#include <Processors/Transforms/ConvertingTransform.h>
+#include <Processors/Sources/SinkToOutputStream.h>
+#include <Processors/ConcatProcessor.h>
 
 
 namespace DB
@@ -117,8 +122,6 @@ BlockIO InterpreterInsertQuery::execute()
     if (!query.table_function)
         context.checkAccess(AccessType::INSERT, query.table_id, query_sample_block.getNames());
 
-    BlockInputStreams in_streams;
-    BlockOutputStreams out_streams;
     bool is_distributed_insert_select = false;
 
     if (query.select && table->isRemote() && settings.parallel_distributed_insert_select)
@@ -159,6 +162,8 @@ BlockIO InterpreterInsertQuery::execute()
             const auto & cluster = storage_src->getCluster();
             const auto & shards_info = cluster->getShardsInfo();
 
+            std::vector<QueryPipeline> pipelines;
+
             String new_query_str = queryToString(new_query);
             for (size_t shard_index : ext::range(0, shards_info.size()))
             {
@@ -166,8 +171,7 @@ BlockIO InterpreterInsertQuery::execute()
                 if (shard_info.isLocal())
                 {
                     InterpreterInsertQuery interpreter(new_query, context);
-                    auto block_io = interpreter.execute();
-                    in_streams.push_back(block_io.in);
+                    pipelines.emplace_back(interpreter.execute().pipeline);
                 }
                 else
                 {
@@ -179,13 +183,20 @@ BlockIO InterpreterInsertQuery::execute()
 
                     ///  INSERT SELECT query returns empty block
                     auto in_stream = std::make_shared<RemoteBlockInputStream>(std::move(connections), new_query_str, Block{}, context);
-                    in_streams.push_back(in_stream);
+                    pipelines.emplace_back();
+                    pipelines.back().init(Pipe(std::make_shared<SourceFromInputStream>(std::move(in_stream))));
+                    pipelines.back().setSinks([](const Block & header, QueryPipeline::StreamType) -> ProcessorPtr
+                    {
+                        return std::make_shared<EmptySink>(header);
+                    });
                 }
-                out_streams.push_back(std::make_shared<NullBlockOutputStream>(Block()));
             }
+
+            res.pipeline.unitePipelines(std::move(pipelines), {});
         }
     }
 
+    BlockOutputStreams out_streams;
     if (!is_distributed_insert_select || query.watch)
     {
         size_t out_streams_size = 1;
@@ -193,27 +204,21 @@ BlockIO InterpreterInsertQuery::execute()
         {
             /// Passing 1 as subquery_depth will disable limiting size of intermediate result.
             InterpreterSelectWithUnionQuery interpreter_select{ query.select, context, SelectQueryOptions(QueryProcessingStage::Complete, 1)};
+            res.pipeline = interpreter_select.executeWithProcessors();
 
             if (table->supportsParallelInsert() && settings.max_insert_threads > 1)
-            {
-                in_streams = interpreter_select.executeWithMultipleStreams(res.pipeline);
-                out_streams_size = std::min(size_t(settings.max_insert_threads), in_streams.size());
-            }
+                out_streams_size = std::min(size_t(settings.max_insert_threads), res.pipeline.getNumStreams());
+
+            if (out_streams_size == 1)
+                res.pipeline.addPipe({std::make_shared<ConcatProcessor>(res.pipeline.getHeader(), res.pipeline.getNumStreams())});
             else
-            {
-                res = interpreter_select.execute();
-                in_streams.emplace_back(res.in);
-                res.in = nullptr;
-                res.out = nullptr;
-            }
+                res.pipeline.resize(out_streams_size);
         }
         else if (query.watch)
         {
             InterpreterWatchQuery interpreter_watch{ query.watch, context };
             res = interpreter_watch.execute();
-            in_streams.emplace_back(res.in);
-            res.in = nullptr;
-            res.out = nullptr;
+            res.pipeline.init(Pipe(std::make_shared<SourceFromInputStream>(std::move(res.in))));
         }
 
         for (size_t i = 0; i < out_streams_size; i++)
@@ -256,27 +261,35 @@ BlockIO InterpreterInsertQuery::execute()
     }
 
     /// What type of query: INSERT or INSERT SELECT or INSERT WATCH?
-    if (query.select || query.watch)
+    if (is_distributed_insert_select)
     {
-        for (auto & in_stream : in_streams)
-        {
-            in_stream = std::make_shared<ConvertingBlockInputStream>(
-                in_stream, out_streams.at(0)->getHeader(), ConvertingBlockInputStream::MatchColumnsMode::Position);
-        }
+        /// Pipeline was already built.
+    }
+    else if (query.select || query.watch)
+    {
+        const auto & header = out_streams.at(0)->getHeader();
 
-        Block in_header = in_streams.at(0)->getHeader();
-        if (in_streams.size() > 1)
+        res.pipeline.addSimpleTransform([&](const Block & in_header) -> ProcessorPtr
         {
-            for (size_t i = 1; i < in_streams.size(); ++i)
-                assertBlocksHaveEqualStructure(in_streams[i]->getHeader(), in_header, query.select ? "INSERT SELECT" : "INSERT WATCH");
-        }
+            return std::make_shared<ConvertingTransform>(in_header, header,
+                    ConvertingTransform::MatchColumnsMode::Position);
+        });
 
-        res.in = std::make_shared<NullAndDoCopyBlockInputStream>(in_streams, out_streams);
+        res.pipeline.setSinks([&](const Block &, QueryPipeline::StreamType type) -> ProcessorPtr
+        {
+            if (type != QueryPipeline::StreamType::Main)
+                return nullptr;
+
+            auto stream = std::move(out_streams.back());
+            out_streams.pop_back();
+
+            return std::make_shared<SinkToOutputStream>(std::move(stream));
+        });
 
         if (!allow_materialized)
         {
             for (const auto & column : table->getColumns())
-                if (column.default_desc.kind == ColumnDefaultKind::Materialized && in_header.has(column.name))
+                if (column.default_desc.kind == ColumnDefaultKind::Materialized && header.has(column.name))
                     throw Exception("Cannot insert column " + column.name + ", because it is MATERIALIZED column.", ErrorCodes::ILLEGAL_COLUMN);
         }
     }
@@ -288,6 +301,7 @@ BlockIO InterpreterInsertQuery::execute()
     }
     else
         res.out = std::move(out_streams.at(0));
+
     res.pipeline.addStorageHolder(table);
 
     return res;
diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp
index 2d8f901d3bb..f90f37b88c3 100644
--- a/src/Interpreters/executeQuery.cpp
+++ b/src/Interpreters/executeQuery.cpp
@@ -342,6 +342,9 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
         else
             res = interpreter->execute();
 
+        if (res.pipeline.initialized())
+            use_processors = true;
+
         if (const auto * insert_interpreter = typeid_cast<const InterpreterInsertQuery *>(&*interpreter))
         {
             /// Save insertion table (not table function). TODO: support remote() table function.
@@ -369,7 +372,7 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
             /// Limits apply only to the final result.
             pipeline.setProgressCallback(context.getProgressCallback());
             pipeline.setProcessListElement(context.getProcessListElement());
-            if (stage == QueryProcessingStage::Complete)
+            if (stage == QueryProcessingStage::Complete && !pipeline.isCompleted())
             {
                 pipeline.resize(1);
                 pipeline.addSimpleTransform([&](const Block & header)
@@ -740,29 +743,36 @@ void executeQuery(
             if (ast_query_with_output && ast_query_with_output->settings_ast)
                 InterpreterSetQuery(ast_query_with_output->settings_ast, context).executeForCurrentContext();
 
-            pipeline.addSimpleTransform([](const Block & header)
+            if (!pipeline.isCompleted())
             {
-                return std::make_shared<MaterializingTransform>(header);
-            });
+                pipeline.addSimpleTransform([](const Block & header)
+                {
+                    return std::make_shared<MaterializingTransform>(header);
+                });
 
-            auto out = context.getOutputFormatProcessor(format_name, *out_buf, pipeline.getHeader());
-            out->setAutoFlush();
+                auto out = context.getOutputFormatProcessor(format_name, *out_buf, pipeline.getHeader());
+                out->setAutoFlush();
 
-            /// Save previous progress callback if any. TODO Do it more conveniently.
-            auto previous_progress_callback = context.getProgressCallback();
+                /// Save previous progress callback if any. TODO Do it more conveniently.
+                auto previous_progress_callback = context.getProgressCallback();
 
-            /// NOTE Progress callback takes shared ownership of 'out'.
-            pipeline.setProgressCallback([out, previous_progress_callback] (const Progress & progress)
+                /// NOTE Progress callback takes shared ownership of 'out'.
+                pipeline.setProgressCallback([out, previous_progress_callback] (const Progress & progress)
+                {
+                    if (previous_progress_callback)
+                        previous_progress_callback(progress);
+                    out->onProgress(progress);
+                });
+
+                if (set_result_details)
+                    set_result_details(context.getClientInfo().current_query_id, out->getContentType(), format_name, DateLUT::instance().getTimeZone());
+
+                pipeline.setOutputFormat(std::move(out));
+            }
+            else
             {
-                if (previous_progress_callback)
-                    previous_progress_callback(progress);
-                out->onProgress(progress);
-            });
-
-            if (set_result_details)
-                set_result_details(context.getClientInfo().current_query_id, out->getContentType(), format_name, DateLUT::instance().getTimeZone());
-
-            pipeline.setOutput(std::move(out));
+                pipeline.setProgressCallback(context.getProgressCallback());
+            }
 
             {
                 auto executor = pipeline.execute();
diff --git a/src/Processors/Executors/PullingAsyncPipelineExecutor.cpp b/src/Processors/Executors/PullingAsyncPipelineExecutor.cpp
index 003508ab86f..49e85fcc6d3 100644
--- a/src/Processors/Executors/PullingAsyncPipelineExecutor.cpp
+++ b/src/Processors/Executors/PullingAsyncPipelineExecutor.cpp
@@ -14,9 +14,10 @@ struct PullingAsyncPipelineExecutor::Data
 {
     PipelineExecutorPtr executor;
     std::exception_ptr exception;
-    std::atomic_bool is_executed = false;
+    std::atomic_bool is_finished = false;
     std::atomic_bool has_exception = false;
     ThreadFromGlobalPool thread;
+    Poco::Event finish_event;
 
     ~Data()
     {
@@ -36,8 +37,11 @@ struct PullingAsyncPipelineExecutor::Data
 
 PullingAsyncPipelineExecutor::PullingAsyncPipelineExecutor(QueryPipeline & pipeline_) : pipeline(pipeline_)
 {
-    lazy_format = std::make_shared<LazyOutputFormat>(pipeline.getHeader());
-    pipeline.setOutput(lazy_format);
+    if (!pipeline.isCompleted())
+    {
+        lazy_format = std::make_shared<LazyOutputFormat>(pipeline.getHeader());
+        pipeline.setOutputFormat(lazy_format);
+    }
 }
 
 PullingAsyncPipelineExecutor::~PullingAsyncPipelineExecutor()
@@ -54,7 +58,8 @@ PullingAsyncPipelineExecutor::~PullingAsyncPipelineExecutor()
 
 const Block & PullingAsyncPipelineExecutor::getHeader() const
 {
-    return lazy_format->getPort(IOutputFormat::PortKind::Main).getHeader();
+    return lazy_format ? lazy_format->getPort(IOutputFormat::PortKind::Main).getHeader()
+                       : pipeline.getHeader(); /// Empty.
 }
 
 static void threadFunction(PullingAsyncPipelineExecutor::Data & data, ThreadGroupStatusPtr thread_group, size_t num_threads)
@@ -78,6 +83,9 @@ static void threadFunction(PullingAsyncPipelineExecutor::Data & data, ThreadGrou
         data.exception = std::current_exception();
         data.has_exception = true;
     }
+
+    data.is_finished = true;
+    data.finish_event.set();
 }
 
 
@@ -99,20 +107,33 @@ bool PullingAsyncPipelineExecutor::pull(Chunk & chunk, uint64_t milliseconds)
     if (data->has_exception)
     {
         /// Finish lazy format in case of exception. Otherwise thread.join() may hung.
-        lazy_format->finish();
+        if (lazy_format)
+            lazy_format->finish();
+
         data->has_exception = false;
         std::rethrow_exception(std::move(data->exception));
     }
 
-    if (lazy_format->isFinished())
+    bool is_execution_finished = lazy_format ? lazy_format->isFinished()
+                                             : data->is_finished.load();
+
+    if (is_execution_finished)
     {
-        data->is_executed = true;
+        /// If lazy format is finished, we don't cancel pipeline but wait for main thread to be finished.
+        data->is_finished = true;
         /// Wait thread ant rethrow exception if any.
         cancel();
         return false;
     }
 
-    chunk = lazy_format->getChunk(milliseconds);
+    if (lazy_format)
+    {
+        chunk = lazy_format->getChunk(milliseconds);
+        return true;
+    }
+
+    chunk.clear();
+    data->finish_event.tryWait(milliseconds);
     return true;
 }
 
@@ -147,11 +168,11 @@ bool PullingAsyncPipelineExecutor::pull(Block & block, uint64_t milliseconds)
 void PullingAsyncPipelineExecutor::cancel()
 {
     /// Cancel execution if it wasn't finished.
-    if (data && !data->is_executed && data->executor)
+    if (data && !data->is_finished && data->executor)
         data->executor->cancel();
 
     /// Finish lazy format. Otherwise thread.join() may hung.
-    if (!lazy_format->isFinished())
+    if (lazy_format && !lazy_format->isFinished())
         lazy_format->finish();
 
     /// Join thread here to wait for possible exception.
@@ -165,12 +186,14 @@ void PullingAsyncPipelineExecutor::cancel()
 
 Chunk PullingAsyncPipelineExecutor::getTotals()
 {
-    return lazy_format->getTotals();
+    return lazy_format ? lazy_format->getTotals()
+                       : Chunk();
 }
 
 Chunk PullingAsyncPipelineExecutor::getExtremes()
 {
-    return lazy_format->getExtremes();
+    return lazy_format ? lazy_format->getExtremes()
+                       : Chunk();
 }
 
 Block PullingAsyncPipelineExecutor::getTotalsBlock()
@@ -197,7 +220,9 @@ Block PullingAsyncPipelineExecutor::getExtremesBlock()
 
 BlockStreamProfileInfo & PullingAsyncPipelineExecutor::getProfileInfo()
 {
-    return lazy_format->getProfileInfo();
+    static BlockStreamProfileInfo profile_info;
+    return lazy_format ? lazy_format->getProfileInfo()
+                       : profile_info;
 }
 
 }
diff --git a/src/Processors/Executors/PullingPipelineExecutor.cpp b/src/Processors/Executors/PullingPipelineExecutor.cpp
index 375f6c9ed0e..af061a373cc 100644
--- a/src/Processors/Executors/PullingPipelineExecutor.cpp
+++ b/src/Processors/Executors/PullingPipelineExecutor.cpp
@@ -9,7 +9,7 @@ namespace DB
 PullingPipelineExecutor::PullingPipelineExecutor(QueryPipeline & pipeline_) : pipeline(pipeline_)
 {
     pulling_format = std::make_shared<PullingOutputFormat>(pipeline.getHeader(), has_data_flag);
-    pipeline.setOutput(pulling_format);
+    pipeline.setOutputFormat(pulling_format);
 }
 
 PullingPipelineExecutor::~PullingPipelineExecutor()
diff --git a/src/Processors/QueryPipeline.cpp b/src/Processors/QueryPipeline.cpp
index 13787a3fd3b..146061e8a47 100644
--- a/src/Processors/QueryPipeline.cpp
+++ b/src/Processors/QueryPipeline.cpp
@@ -34,6 +34,14 @@ void QueryPipeline::checkInitialized()
         throw Exception("QueryPipeline wasn't initialized.", ErrorCodes::LOGICAL_ERROR);
 }
 
+void QueryPipeline::checkInitializedAndNotCompleted()
+{
+    checkInitialized();
+
+    if (streams.empty())
+        throw Exception("QueryPipeline was already completed.", ErrorCodes::LOGICAL_ERROR);
+}
+
 void QueryPipeline::checkSource(const ProcessorPtr & source, bool can_have_totals)
 {
     if (!source->getInputs().empty())
@@ -194,11 +202,11 @@ static ProcessorPtr callProcessorGetter(
 template <typename TProcessorGetter>
 void QueryPipeline::addSimpleTransformImpl(const TProcessorGetter & getter)
 {
-    checkInitialized();
+    checkInitializedAndNotCompleted();
 
     Block header;
 
-    auto add_transform = [&](OutputPort *& stream, StreamType stream_type, size_t stream_num [[maybe_unused]] = IProcessor::NO_STREAM)
+    auto add_transform = [&](OutputPort *& stream, StreamType stream_type)
     {
         if (!stream)
             return;
@@ -231,17 +239,14 @@ void QueryPipeline::addSimpleTransformImpl(const TProcessorGetter & getter)
 
         if (transform)
         {
-//            if (stream_type == StreamType::Main)
-//                transform->setStream(stream_num);
-
             connect(*stream, transform->getInputs().front());
             stream = &transform->getOutputs().front();
             processors.emplace_back(std::move(transform));
         }
     };
 
-    for (size_t stream_num = 0; stream_num < streams.size(); ++stream_num)
-        add_transform(streams[stream_num], StreamType::Main, stream_num);
+    for (auto & stream : streams)
+        add_transform(stream, StreamType::Main);
 
     add_transform(totals_having_port, StreamType::Totals);
     add_transform(extremes_port, StreamType::Extremes);
@@ -259,9 +264,50 @@ void QueryPipeline::addSimpleTransform(const ProcessorGetterWithStreamKind & get
     addSimpleTransformImpl(getter);
 }
 
+void QueryPipeline::setSinks(const ProcessorGetterWithStreamKind & getter)
+{
+    checkInitializedAndNotCompleted();
+
+    auto add_transform = [&](OutputPort *& stream, StreamType stream_type)
+    {
+        if (!stream)
+            return;
+
+        auto transform = getter(stream->getHeader(), stream_type);
+
+        if (transform)
+        {
+            if (transform->getInputs().size() != 1)
+                throw Exception("Sink for query pipeline transform should have single input, "
+                                "but " + transform->getName() + " has " +
+                                toString(transform->getInputs().size()) + " inputs.", ErrorCodes::LOGICAL_ERROR);
+
+            if (!transform->getOutputs().empty())
+                throw Exception("Sink for query pipeline transform should have no outputs, "
+                                "but " + transform->getName() + " has " +
+                                toString(transform->getOutputs().size()) + " outputs.", ErrorCodes::LOGICAL_ERROR);
+        }
+
+        if (!transform)
+            transform = std::make_shared<NullSink>(stream->getHeader());
+
+        connect(*stream, transform->getInputs().front());
+        processors.emplace_back(std::move(transform));
+    };
+
+    for (auto & stream : streams)
+        add_transform(stream, StreamType::Main);
+
+    add_transform(totals_having_port, StreamType::Totals);
+    add_transform(extremes_port, StreamType::Extremes);
+
+    streams.clear();
+    current_header.clear();
+}
+
 void QueryPipeline::addPipe(Processors pipe)
 {
-    checkInitialized();
+    checkInitializedAndNotCompleted();
 
     if (pipe.empty())
         throw Exception("Can't add empty processors list to QueryPipeline.", ErrorCodes::LOGICAL_ERROR);
@@ -298,7 +344,7 @@ void QueryPipeline::addPipe(Processors pipe)
 
 void QueryPipeline::addDelayedStream(ProcessorPtr source)
 {
-    checkInitialized();
+    checkInitializedAndNotCompleted();
 
     checkSource(source, false);
     assertBlocksHaveEqualStructure(current_header, source->getOutputs().front().getHeader(), "QueryPipeline");
@@ -313,7 +359,7 @@ void QueryPipeline::addDelayedStream(ProcessorPtr source)
 
 void QueryPipeline::resize(size_t num_streams, bool force, bool strict)
 {
-    checkInitialized();
+    checkInitializedAndNotCompleted();
 
     if (!force && num_streams == getNumStreams())
         return;
@@ -347,7 +393,7 @@ void QueryPipeline::enableQuotaForCurrentStreams()
 
 void QueryPipeline::addTotalsHavingTransform(ProcessorPtr transform)
 {
-    checkInitialized();
+    checkInitializedAndNotCompleted();
 
     if (!typeid_cast<const TotalsHavingTransform *>(transform.get()))
         throw Exception("TotalsHavingTransform expected for QueryPipeline::addTotalsHavingTransform.",
@@ -370,7 +416,7 @@ void QueryPipeline::addTotalsHavingTransform(ProcessorPtr transform)
 
 void QueryPipeline::addDefaultTotals()
 {
-    checkInitialized();
+    checkInitializedAndNotCompleted();
 
     if (totals_having_port)
         throw Exception("Totals having transform was already added to pipeline.", ErrorCodes::LOGICAL_ERROR);
@@ -392,7 +438,7 @@ void QueryPipeline::addDefaultTotals()
 
 void QueryPipeline::addTotals(ProcessorPtr source)
 {
-    checkInitialized();
+    checkInitializedAndNotCompleted();
 
     if (totals_having_port)
         throw Exception("Totals having transform was already added to pipeline.", ErrorCodes::LOGICAL_ERROR);
@@ -423,7 +469,7 @@ void QueryPipeline::dropTotalsAndExtremes()
 
 void QueryPipeline::addExtremesTransform()
 {
-    checkInitialized();
+    checkInitializedAndNotCompleted();
 
     if (extremes_port)
         throw Exception("Extremes transform was already added to pipeline.", ErrorCodes::LOGICAL_ERROR);
@@ -450,7 +496,7 @@ void QueryPipeline::addExtremesTransform()
 
 void QueryPipeline::addCreatingSetsTransform(ProcessorPtr transform)
 {
-    checkInitialized();
+    checkInitializedAndNotCompleted();
 
     if (!typeid_cast<const CreatingSetsTransform *>(transform.get()))
         throw Exception("CreatingSetsTransform expected for QueryPipeline::addExtremesTransform.",
@@ -467,14 +513,14 @@ void QueryPipeline::addCreatingSetsTransform(ProcessorPtr transform)
     processors.emplace_back(std::move(concat));
 }
 
-void QueryPipeline::setOutput(ProcessorPtr output)
+void QueryPipeline::setOutputFormat(ProcessorPtr output)
 {
-    checkInitialized();
+    checkInitializedAndNotCompleted();
 
     auto * format = dynamic_cast<IOutputFormat * >(output.get());
 
     if (!format)
-        throw Exception("IOutputFormat processor expected for QueryPipeline::setOutput.", ErrorCodes::LOGICAL_ERROR);
+        throw Exception("IOutputFormat processor expected for QueryPipeline::setOutputFormat.", ErrorCodes::LOGICAL_ERROR);
 
     if (output_format)
         throw Exception("QueryPipeline already has output.", ErrorCodes::LOGICAL_ERROR);
@@ -507,19 +553,25 @@ void QueryPipeline::setOutput(ProcessorPtr output)
     connect(*totals_having_port, totals);
     connect(*extremes_port, extremes);
 
+    streams.clear();
+    current_header.clear();
+    extremes_port = nullptr;
+    totals_having_port = nullptr;
+
     initRowsBeforeLimit();
 }
 
 void QueryPipeline::unitePipelines(
     std::vector<QueryPipeline> && pipelines, const Block & common_header)
 {
-    checkInitialized();
-
-    addSimpleTransform([&](const Block & header)
+    if (initialized())
     {
-        return std::make_shared<ConvertingTransform>(
-                header, common_header, ConvertingTransform::MatchColumnsMode::Position);
-    });
+        addSimpleTransform([&](const Block & header)
+        {
+            return std::make_shared<ConvertingTransform>(
+                    header, common_header, ConvertingTransform::MatchColumnsMode::Position);
+        });
+    }
 
     std::vector<OutputPort *> extremes;
     std::vector<OutputPort *> totals;
@@ -534,11 +586,14 @@ void QueryPipeline::unitePipelines(
     {
         pipeline.checkInitialized();
 
-        pipeline.addSimpleTransform([&](const Block & header)
+        if (!pipeline.isCompleted())
         {
-           return std::make_shared<ConvertingTransform>(
-                   header, common_header, ConvertingTransform::MatchColumnsMode::Position);
-        });
+            pipeline.addSimpleTransform([&](const Block & header)
+            {
+               return std::make_shared<ConvertingTransform>(
+                       header, common_header, ConvertingTransform::MatchColumnsMode::Position);
+            });
+        }
 
         if (pipeline.extremes_port)
         {
@@ -726,10 +781,8 @@ Pipe QueryPipeline::getPipe() &&
 
 PipelineExecutorPtr QueryPipeline::execute()
 {
-    checkInitialized();
-
-    if (!output_format)
-        throw Exception("Cannot execute pipeline because it doesn't have output.", ErrorCodes::LOGICAL_ERROR);
+    if (!isCompleted())
+        throw Exception("Cannot execute pipeline because it is not completed.", ErrorCodes::LOGICAL_ERROR);
 
     return std::make_shared<PipelineExecutor>(processors, process_list_element);
 }
diff --git a/src/Processors/QueryPipeline.h b/src/Processors/QueryPipeline.h
index 45e38ffa715..4b2b32c3258 100644
--- a/src/Processors/QueryPipeline.h
+++ b/src/Processors/QueryPipeline.h
@@ -28,6 +28,7 @@ private:
     {
     public:
         auto size() const { return data.size(); }
+        bool empty() const { return size() == 0; }
         auto begin() { return data.begin(); }
         auto end() { return data.end(); }
         auto & front() { return data.front(); }
@@ -81,6 +82,7 @@ public:
     void init(Pipes pipes);
     void init(Pipe pipe); /// Simple init for single pipe
     bool initialized() { return !processors.empty(); }
+    bool isCompleted() { return initialized() && streams.empty(); }
 
     /// Type of logical data stream for simple transform.
     /// Sometimes it's important to know which part of pipeline we are working for.
@@ -95,13 +97,23 @@ public:
     using ProcessorGetter = std::function<ProcessorPtr(const Block & header)>;
     using ProcessorGetterWithStreamKind = std::function<ProcessorPtr(const Block & header, StreamType stream_type)>;
 
+    /// Add transform with simple input and simple output for each port.
     void addSimpleTransform(const ProcessorGetter & getter);
     void addSimpleTransform(const ProcessorGetterWithStreamKind & getter);
+    /// Add several processors. They must have same header for inputs and same for outputs.
+    /// Total number of inputs must be the same as the number of streams. Output ports will become new streams.
     void addPipe(Processors pipe);
+    /// Add TotalsHavingTransform. Resize pipeline to single input. Adds totals port.
     void addTotalsHavingTransform(ProcessorPtr transform);
+    /// Add transform which calculates extremes. This transform adds extremes port and doesn't change inputs number.
     void addExtremesTransform();
+    /// Adds transform which creates sets. It will be executed before reading any data from input ports.
     void addCreatingSetsTransform(ProcessorPtr transform);
-    void setOutput(ProcessorPtr output);
+    /// Resize pipeline to single output and add IOutputFormat. Pipeline will be completed after this transformation.
+    void setOutputFormat(ProcessorPtr output);
+    /// Sink is a processor with single input port and no output ports. Creates sink for each output port.
+    /// Pipeline will be completed after this transformation.
+    void setSinks(const ProcessorGetterWithStreamKind & getter);
 
     /// Add totals which returns one chunk with single row with defaults.
     void addDefaultTotals();
@@ -118,6 +130,7 @@ public:
     /// Check if resize transform was used. (In that case another distinct transform will be added).
     bool hasMixedStreams() const { return has_resize || hasMoreThanOneStream(); }
 
+    /// Changes the number of input ports if needed. Adds ResizeTransform.
     void resize(size_t num_streams, bool force = false, bool strict = false);
 
     void enableQuotaForCurrentStreams();
@@ -193,6 +206,7 @@ private:
     QueryStatus * process_list_element = nullptr;
 
     void checkInitialized();
+    void checkInitializedAndNotCompleted();
     static void checkSource(const ProcessorPtr & source, bool can_have_totals);
 
     template <typename TProcessorGetter>

From ec479839d93919f6f604cf5ca1fa4546353f7154 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <mikhaylovnikitka@gmail.com>
Date: Wed, 27 May 2020 22:44:22 +0300
Subject: [PATCH 099/184] fixed

---
 src/Dictionaries/CacheDictionary.inc.h        | 48 +++++++--------
 .../cache_strings_default_settings.xml        | 35 +++++++++++
 .../test_default_string.py                    | 61 +++++++++++++++++++
 3 files changed, 118 insertions(+), 26 deletions(-)
 create mode 100644 tests/integration/test_dictionary_allow_read_expired_keys/configs/dictionaries/cache_strings_default_settings.xml
 create mode 100644 tests/integration/test_dictionary_allow_read_expired_keys/test_default_string.py

diff --git a/src/Dictionaries/CacheDictionary.inc.h b/src/Dictionaries/CacheDictionary.inc.h
index 71f3f1857ce..27064d113e6 100644
--- a/src/Dictionaries/CacheDictionary.inc.h
+++ b/src/Dictionaries/CacheDictionary.inc.h
@@ -302,37 +302,33 @@ void CacheDictionary::getItemsString(
 
     /// Request new values sync.
     /// We have request both cache_not_found_ids and cache_expired_ids.
-    if (!cache_not_found_ids.empty())
+    std::vector<Key> required_ids;
+    required_ids.reserve(cache_not_found_ids.size() + cache_expired_ids.size());
+    std::transform(
+        std::begin(cache_not_found_ids), std::end(cache_not_found_ids),
+        std::back_inserter(required_ids), [](auto & pair) { return pair.first; });
+    std::transform(
+        std::begin(cache_expired_ids), std::end(cache_expired_ids),
+        std::back_inserter(required_ids), [](auto & pair) { return pair.first; });
+
+    auto on_cell_updated = [&] (const auto id, const auto cell_idx)
     {
-        std::vector<Key> required_ids;
-        required_ids.reserve(cache_not_found_ids.size() + cache_expired_ids.size());
-        std::transform(
-                std::begin(cache_not_found_ids), std::end(cache_not_found_ids),
-                std::back_inserter(required_ids), [](auto & pair) { return pair.first; });
-        std::transform(
-                std::begin(cache_expired_ids), std::end(cache_expired_ids),
-                std::back_inserter(required_ids), [](auto & pair) { return pair.first; });
+        const auto attribute_value = attribute_array[cell_idx];
 
-        auto on_cell_updated = [&] (const auto id, const auto cell_idx)
-        {
-            const auto attribute_value = attribute_array[cell_idx];
+        map[id] = String{attribute_value};
+        total_length += (attribute_value.size + 1) * cache_not_found_ids[id].size();
+    };
 
-            map[id] = String{attribute_value};
-            total_length += (attribute_value.size + 1) * cache_not_found_ids[id].size();
-        };
+    auto on_id_not_found = [&] (const auto id, const auto)
+    {
+        for (const auto row : cache_not_found_ids[id])
+            total_length += get_default(row).size + 1;
+    };
 
-        auto on_id_not_found = [&] (const auto id, const auto)
-        {
-            for (const auto row : cache_not_found_ids[id])
-                total_length += get_default(row).size + 1;
-        };
-
-        auto update_unit_ptr = std::make_shared<UpdateUnit>(required_ids, on_cell_updated, on_id_not_found);
-
-        tryPushToUpdateQueueOrThrow(update_unit_ptr);
-        waitForCurrentUpdateFinish(update_unit_ptr);
-    }
+    auto update_unit_ptr = std::make_shared<UpdateUnit>(required_ids, on_cell_updated, on_id_not_found);
 
+    tryPushToUpdateQueueOrThrow(update_unit_ptr);
+    waitForCurrentUpdateFinish(update_unit_ptr);
     out->getChars().reserve(total_length);
 
     for (const auto row : ext::range(0, ext::size(ids)))
diff --git a/tests/integration/test_dictionary_allow_read_expired_keys/configs/dictionaries/cache_strings_default_settings.xml b/tests/integration/test_dictionary_allow_read_expired_keys/configs/dictionaries/cache_strings_default_settings.xml
new file mode 100644
index 00000000000..11807bc1ad6
--- /dev/null
+++ b/tests/integration/test_dictionary_allow_read_expired_keys/configs/dictionaries/cache_strings_default_settings.xml
@@ -0,0 +1,35 @@
+<yandex>
+<dictionary>
+    <name>default_string</name>
+    <source>
+        <clickhouse>
+            <host>dictionary_node</host>
+            <port>9000</port>
+            <user>default</user>
+            <password></password>
+            <db>test</db>
+            <table>strings</table>
+        </clickhouse>
+    </source>
+    <lifetime>
+        <max>2</max>
+        <min>1</min>
+    </lifetime>
+    <layout>
+        <cache>
+            <size_in_cells>1000</size_in_cells>
+            <max_update_queue_size>10000</max_update_queue_size>
+        </cache>
+    </layout>
+    <structure>
+        <id>
+            <name>key</name>
+        </id>
+        <attribute>
+            <name>value</name>
+            <type>String</type>
+            <null_value></null_value>>
+        </attribute>
+    </structure>
+</dictionary>
+</yandex>
\ No newline at end of file
diff --git a/tests/integration/test_dictionary_allow_read_expired_keys/test_default_string.py b/tests/integration/test_dictionary_allow_read_expired_keys/test_default_string.py
new file mode 100644
index 00000000000..7d762db2a6d
--- /dev/null
+++ b/tests/integration/test_dictionary_allow_read_expired_keys/test_default_string.py
@@ -0,0 +1,61 @@
+from __future__ import print_function
+import pytest
+import os
+import random
+import string
+import time
+
+from helpers.cluster import ClickHouseCluster
+from helpers.test_tools import TSV
+
+SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
+cluster = ClickHouseCluster(__file__, base_configs_dir=os.path.join(SCRIPT_DIR, 'configs'))
+
+dictionary_node = cluster.add_instance('dictionary_node', stay_alive=True)
+main_node = cluster.add_instance('main_node', main_configs=['configs/dictionaries/cache_strings_default_settings.xml'])
+
+
+def get_random_string(string_length=8):
+    alphabet = string.ascii_letters + string.digits
+    return ''.join((random.choice(alphabet) for _ in range(string_length)))
+
+@pytest.fixture(scope="module")
+def started_cluster():
+    try:
+        cluster.start()
+        dictionary_node.query("CREATE DATABASE IF NOT EXISTS test;")
+        dictionary_node.query("DROP TABLE IF EXISTS test.strings;")
+        dictionary_node.query("""
+                             CREATE TABLE test.strings 
+                             (key UInt64, value String)
+                             ENGINE = Memory;
+                             """)
+
+        values_to_insert = ", ".join(["({}, '{}')".format(1000000 + number, get_random_string()) for number in range(100)])
+        dictionary_node.query("INSERT INTO test.strings VALUES {}".format(values_to_insert))
+
+        yield cluster
+    finally:
+        cluster.shutdown()
+
+# @pytest.mark.skip(reason="debugging")
+def test_return_real_values(started_cluster):
+    assert None != dictionary_node.get_process_pid("clickhouse"), "ClickHouse must be alive"
+
+    first_batch = """
+    SELECT count(*)
+    FROM
+    (
+    SELECT
+        arrayJoin(arrayMap(x -> (x + 1000000), range(100))) AS id,
+        dictGetString('default_string', 'value', toUInt64(id)) AS value
+    )
+    WHERE value = '';
+    """
+
+    assert TSV("0") == TSV(main_node.query(first_batch))
+
+    # Waiting for cache to become expired
+    time.sleep(5)
+
+    assert TSV("0") == TSV(main_node.query(first_batch))

From 2432ea2c4b5a0307c175f0a5dcc7e95639e1da68 Mon Sep 17 00:00:00 2001
From: Ivan Lezhankin <ilezhankin@yandex-team.ru>
Date: Thu, 28 May 2020 00:58:46 +0300
Subject: [PATCH 100/184] Build universal binary using ya make

---
 programs/client/CMakeLists.txt                | 11 ++---
 programs/client/Client.cpp                    |  5 ++-
 programs/client/ConnectionParameters.cpp      |  2 +-
 programs/client/config_client.h.in            |  3 --
 programs/client/readpassphrase/CMakeLists.txt | 12 ++----
 .../{includes.h.in => includes.h}             |  2 +-
 .../client/readpassphrase/readpassphrase.c    | 16 +------
 .../client/readpassphrase/readpassphrase.h    | 39 +++++------------
 programs/client/readpassphrase/ya.make        |  7 +++
 programs/main.cpp                             | 43 +++++++++----------
 programs/ya.make                              | 28 +++++++++++-
 src/Common/ya.make                            |  3 ++
 12 files changed, 80 insertions(+), 91 deletions(-)
 delete mode 100644 programs/client/config_client.h.in
 rename programs/client/readpassphrase/{includes.h.in => includes.h} (79%)
 create mode 100644 programs/client/readpassphrase/ya.make

diff --git a/programs/client/CMakeLists.txt b/programs/client/CMakeLists.txt
index 11ade559a8d..e273123afe0 100644
--- a/programs/client/CMakeLists.txt
+++ b/programs/client/CMakeLists.txt
@@ -6,14 +6,9 @@ set(CLICKHOUSE_CLIENT_SOURCES
 
 set(CLICKHOUSE_CLIENT_LINK PRIVATE clickhouse_common_config clickhouse_functions clickhouse_aggregate_functions clickhouse_common_io clickhouse_parsers string_utils ${Boost_PROGRAM_OPTIONS_LIBRARY})
 
-include(CheckSymbolExists)
-check_symbol_exists(readpassphrase readpassphrase.h HAVE_READPASSPHRASE)
-configure_file(config_client.h.in ${ConfigIncludePath}/config_client.h)
-
-if(NOT HAVE_READPASSPHRASE)
-    add_subdirectory(readpassphrase)
-    list(APPEND CLICKHOUSE_CLIENT_LINK PRIVATE readpassphrase)
-endif()
+# Always use internal readpassphrase
+add_subdirectory(readpassphrase)
+list(APPEND CLICKHOUSE_CLIENT_LINK PRIVATE readpassphrase)
 
 clickhouse_program_add(client)
 
diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp
index d6cac7a7b02..afc8f9a72b1 100644
--- a/programs/client/Client.cpp
+++ b/programs/client/Client.cpp
@@ -39,7 +39,6 @@
 #include <Common/StringUtils/StringUtils.h>
 #include <Common/typeid_cast.h>
 #include <Common/Config/ConfigProcessor.h>
-#include <Common/config_version.h>
 #include <Core/Types.h>
 #include <Core/QueryProcessingStage.h>
 #include <Core/ExternalTable.h>
@@ -77,6 +76,10 @@
 #include <common/argsToConfig.h>
 #include <Common/TerminalSize.h>
 
+#if !defined(ARCADIA_BUILD)
+#    include <Common/config_version.h>
+#endif
+
 #ifndef __clang__
 #pragma GCC optimize("-fno-var-tracking-assignments")
 #endif
diff --git a/programs/client/ConnectionParameters.cpp b/programs/client/ConnectionParameters.cpp
index f0ef3ae5694..d8b4d0f1add 100644
--- a/programs/client/ConnectionParameters.cpp
+++ b/programs/client/ConnectionParameters.cpp
@@ -9,7 +9,7 @@
 #include <Common/Exception.h>
 #include <common/setTerminalEcho.h>
 #include <ext/scope_guard.h>
-#include <readpassphrase.h>
+#include "readpassphrase/readpassphrase.h"
 
 namespace DB
 {
diff --git a/programs/client/config_client.h.in b/programs/client/config_client.h.in
deleted file mode 100644
index 5ad788ff54c..00000000000
--- a/programs/client/config_client.h.in
+++ /dev/null
@@ -1,3 +0,0 @@
-#pragma once
-
-#cmakedefine HAVE_READPASSPHRASE
diff --git a/programs/client/readpassphrase/CMakeLists.txt b/programs/client/readpassphrase/CMakeLists.txt
index a10b54c377d..dd1bf2c91b9 100644
--- a/programs/client/readpassphrase/CMakeLists.txt
+++ b/programs/client/readpassphrase/CMakeLists.txt
@@ -1,13 +1,7 @@
-
 # wget https://raw.githubusercontent.com/openssh/openssh-portable/master/openbsd-compat/readpassphrase.c
 # wget https://raw.githubusercontent.com/openssh/openssh-portable/master/openbsd-compat/readpassphrase.h
 
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-result -Wno-reserved-id-macro")
+add_library(readpassphrase readpassphrase.c)
 
-configure_file(includes.h.in ${CMAKE_CURRENT_BINARY_DIR}/include/includes.h)
-add_library(readpassphrase ${CMAKE_CURRENT_SOURCE_DIR}/readpassphrase.c)
-set_target_properties(readpassphrase
-                      PROPERTIES LINKER_LANGUAGE C
-                      )
-# . to allow #include <readpassphrase.h>
-target_include_directories(readpassphrase PUBLIC . ${CMAKE_CURRENT_BINARY_DIR}/include)
+set_target_properties(readpassphrase PROPERTIES LINKER_LANGUAGE C)
+target_compile_options(readpassphrase PRIVATE -Wno-unused-result -Wno-reserved-id-macro)
diff --git a/programs/client/readpassphrase/includes.h.in b/programs/client/readpassphrase/includes.h
similarity index 79%
rename from programs/client/readpassphrase/includes.h.in
rename to programs/client/readpassphrase/includes.h
index 44580d1ed95..3ca5eb2bff8 100644
--- a/programs/client/readpassphrase/includes.h.in
+++ b/programs/client/readpassphrase/includes.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#cmakedefine HAVE_READPASSPHRASE
+/* #undef HAVE_READPASSPHRASE */
 
 #if !defined(HAVE_READPASSPHRASE)
 #    ifndef _PATH_TTY
diff --git a/programs/client/readpassphrase/readpassphrase.c b/programs/client/readpassphrase/readpassphrase.c
index 243701239bf..21b48e7efc3 100644
--- a/programs/client/readpassphrase/readpassphrase.c
+++ b/programs/client/readpassphrase/readpassphrase.c
@@ -25,13 +25,11 @@
 
 #include "includes.h"
 
-#ifndef HAVE_READPASSPHRASE
-
 #include <termios.h>
 #include <signal.h>
 #include <ctype.h>
 #include <fcntl.h>
-#include <readpassphrase.h>
+#include "readpassphrase.h"
 #include <errno.h>
 #include <string.h>
 #include <unistd.h>
@@ -193,19 +191,7 @@ restart:
 }
 //DEF_WEAK(readpassphrase);
 
-#if 0
-char *
-getpass(const char *prompt)
-{
-    static char buf[_PASSWORD_LEN + 1];
-
-    return(readpassphrase(prompt, buf, sizeof(buf), RPP_ECHO_OFF));
-}
-#endif
-
 static void handler(int s)
 {
-
     signo[s] = 1;
 }
-#endif /* HAVE_READPASSPHRASE */
diff --git a/programs/client/readpassphrase/readpassphrase.h b/programs/client/readpassphrase/readpassphrase.h
index 0782a1773ea..399eb7b062c 100644
--- a/programs/client/readpassphrase/readpassphrase.h
+++ b/programs/client/readpassphrase/readpassphrase.h
@@ -23,39 +23,22 @@
 /* OPENBSD ORIGINAL: include/readpassphrase.h */
 
 #pragma once
-// #ifndef _READPASSPHRASE_H_
-// #define _READPASSPHRASE_H_
 
-//#include "includes.h"
-#include "config_client.h"
-
-// Should not be included on BSD systems, but if it happen...
-#ifdef HAVE_READPASSPHRASE
-#   include_next <readpassphrase.h>
+#if defined(__cplusplus)
+extern "C" {
 #endif
 
-#ifndef HAVE_READPASSPHRASE
 
-#    ifdef __cplusplus
-extern "C" {
-#    endif
-
-
-#    define RPP_ECHO_OFF 0x00 /* Turn off echo (default). */
-#    define RPP_ECHO_ON 0x01 /* Leave echo on. */
-#    define RPP_REQUIRE_TTY 0x02 /* Fail if there is no tty. */
-#    define RPP_FORCELOWER 0x04 /* Force input to lower case. */
-#    define RPP_FORCEUPPER 0x08 /* Force input to upper case. */
-#    define RPP_SEVENBIT 0x10 /* Strip the high bit from input. */
-#    define RPP_STDIN 0x20 /* Read from stdin, not /dev/tty */
+#define RPP_ECHO_OFF 0x00 /* Turn off echo (default). */
+#define RPP_ECHO_ON 0x01 /* Leave echo on. */
+#define RPP_REQUIRE_TTY 0x02 /* Fail if there is no tty. */
+#define RPP_FORCELOWER 0x04 /* Force input to lower case. */
+#define RPP_FORCEUPPER 0x08 /* Force input to upper case. */
+#define RPP_SEVENBIT 0x10 /* Strip the high bit from input. */
+#define RPP_STDIN 0x20 /* Read from stdin, not /dev/tty */
 
 char * readpassphrase(const char *, char *, size_t, int);
 
-#    ifdef __cplusplus
+#if defined(__cplusplus)
 }
-#    endif
-
-
-#endif /* HAVE_READPASSPHRASE */
-
-// #endif /* !_READPASSPHRASE_H_ */
+#endif
diff --git a/programs/client/readpassphrase/ya.make b/programs/client/readpassphrase/ya.make
new file mode 100644
index 00000000000..80ad197e5d4
--- /dev/null
+++ b/programs/client/readpassphrase/ya.make
@@ -0,0 +1,7 @@
+LIBRARY()
+
+SRCS(
+    readpassphrase.c
+)
+
+END()
diff --git a/programs/main.cpp b/programs/main.cpp
index 2eb226d3c00..6b34504ce37 100644
--- a/programs/main.cpp
+++ b/programs/main.cpp
@@ -8,11 +8,8 @@
 #include <string>
 #include <utility> /// pair
 
-#if __has_include("config_tools.h")
-#include "config_tools.h"
-#endif
-#if __has_include("config_core.h")
-#include "config_core.h"
+#if !defined(ARCADIA_BUILD)
+#    include "config_tools.h"
 #endif
 
 #include <Common/StringUtils/StringUtils.h>
@@ -22,31 +19,31 @@
 
 
 /// Universal executable for various clickhouse applications
-#if ENABLE_CLICKHOUSE_SERVER || !defined(ENABLE_CLICKHOUSE_SERVER)
+#if ENABLE_CLICKHOUSE_SERVER
 int mainEntryClickHouseServer(int argc, char ** argv);
 #endif
-#if ENABLE_CLICKHOUSE_CLIENT || !defined(ENABLE_CLICKHOUSE_CLIENT)
+#if ENABLE_CLICKHOUSE_CLIENT
 int mainEntryClickHouseClient(int argc, char ** argv);
 #endif
-#if ENABLE_CLICKHOUSE_LOCAL || !defined(ENABLE_CLICKHOUSE_LOCAL)
+#if ENABLE_CLICKHOUSE_LOCAL
 int mainEntryClickHouseLocal(int argc, char ** argv);
 #endif
-#if ENABLE_CLICKHOUSE_BENCHMARK || !defined(ENABLE_CLICKHOUSE_BENCHMARK)
+#if ENABLE_CLICKHOUSE_BENCHMARK
 int mainEntryClickHouseBenchmark(int argc, char ** argv);
 #endif
-#if ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG || !defined(ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG)
+#if ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG
 int mainEntryClickHouseExtractFromConfig(int argc, char ** argv);
 #endif
-#if ENABLE_CLICKHOUSE_COMPRESSOR || !defined(ENABLE_CLICKHOUSE_COMPRESSOR)
+#if ENABLE_CLICKHOUSE_COMPRESSOR
 int mainEntryClickHouseCompressor(int argc, char ** argv);
 #endif
-#if ENABLE_CLICKHOUSE_FORMAT || !defined(ENABLE_CLICKHOUSE_FORMAT)
+#if ENABLE_CLICKHOUSE_FORMAT
 int mainEntryClickHouseFormat(int argc, char ** argv);
 #endif
-#if ENABLE_CLICKHOUSE_COPIER || !defined(ENABLE_CLICKHOUSE_COPIER)
+#if ENABLE_CLICKHOUSE_COPIER
 int mainEntryClickHouseClusterCopier(int argc, char ** argv);
 #endif
-#if ENABLE_CLICKHOUSE_OBFUSCATOR || !defined(ENABLE_CLICKHOUSE_OBFUSCATOR)
+#if ENABLE_CLICKHOUSE_OBFUSCATOR
 int mainEntryClickHouseObfuscator(int argc, char ** argv);
 #endif
 
@@ -60,31 +57,31 @@ using MainFunc = int (*)(int, char**);
 /// Add an item here to register new application
 std::pair<const char *, MainFunc> clickhouse_applications[] =
 {
-#if ENABLE_CLICKHOUSE_LOCAL || !defined(ENABLE_CLICKHOUSE_LOCAL)
+#if ENABLE_CLICKHOUSE_LOCAL
     {"local", mainEntryClickHouseLocal},
 #endif
-#if ENABLE_CLICKHOUSE_CLIENT || !defined(ENABLE_CLICKHOUSE_CLIENT)
+#if ENABLE_CLICKHOUSE_CLIENT
     {"client", mainEntryClickHouseClient},
 #endif
-#if ENABLE_CLICKHOUSE_BENCHMARK || !defined(ENABLE_CLICKHOUSE_BENCHMARK)
+#if ENABLE_CLICKHOUSE_BENCHMARK
     {"benchmark", mainEntryClickHouseBenchmark},
 #endif
-#if ENABLE_CLICKHOUSE_SERVER || !defined(ENABLE_CLICKHOUSE_SERVER)
+#if ENABLE_CLICKHOUSE_SERVER
     {"server", mainEntryClickHouseServer},
 #endif
-#if ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG || !defined(ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG)
+#if ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG
     {"extract-from-config", mainEntryClickHouseExtractFromConfig},
 #endif
-#if ENABLE_CLICKHOUSE_COMPRESSOR || !defined(ENABLE_CLICKHOUSE_COMPRESSOR)
+#if ENABLE_CLICKHOUSE_COMPRESSOR
     {"compressor", mainEntryClickHouseCompressor},
 #endif
-#if ENABLE_CLICKHOUSE_FORMAT || !defined(ENABLE_CLICKHOUSE_FORMAT)
+#if ENABLE_CLICKHOUSE_FORMAT
     {"format", mainEntryClickHouseFormat},
 #endif
-#if ENABLE_CLICKHOUSE_COPIER || !defined(ENABLE_CLICKHOUSE_COPIER)
+#if ENABLE_CLICKHOUSE_COPIER
     {"copier", mainEntryClickHouseClusterCopier},
 #endif
-#if ENABLE_CLICKHOUSE_OBFUSCATOR || !defined(ENABLE_CLICKHOUSE_OBFUSCATOR)
+#if ENABLE_CLICKHOUSE_OBFUSCATOR
     {"obfuscator", mainEntryClickHouseObfuscator},
 #endif
 };
diff --git a/programs/ya.make b/programs/ya.make
index 6c773c312b8..f4a61850212 100644
--- a/programs/ya.make
+++ b/programs/ya.make
@@ -1,3 +1,27 @@
-RECURSE(
-    server
+PROGRAM(clickhouse)
+
+CFLAGS(
+    -DENABLE_CLICKHOUSE_CLIENT
+    -DENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG
+    -DENABLE_CLICKHOUSE_SERVER
 )
+
+PEERDIR(
+    clickhouse/base/daemon
+    clickhouse/base/loggers
+    clickhouse/programs/client/readpassphrase
+    clickhouse/src
+)
+
+SRCS(
+    main.cpp
+
+    client/Client.cpp
+    client/ConnectionParameters.cpp
+    client/Suggest.cpp
+    extract-from-config/ExtractFromConfig.cpp
+    server/Server.cpp
+    server/MetricsTransmitter.cpp
+)
+
+END()
diff --git a/src/Common/ya.make b/src/Common/ya.make
index c0178f3d310..99659aa2aa0 100644
--- a/src/Common/ya.make
+++ b/src/Common/ya.make
@@ -25,6 +25,9 @@ CFLAGS (GLOBAL -DDBMS_VERSION_MINOR=0)
 CFLAGS (GLOBAL -DDBMS_VERSION_PATCH=0)
 CFLAGS (GLOBAL -DVERSION_FULL=\"ClickHouse\")
 CFLAGS (GLOBAL -DVERSION_INTEGER=0)
+CFLAGS (GLOBAL -DVERSION_MAJOR=0)
+CFLAGS (GLOBAL -DVERSION_MINOR=0)
+CFLAGS (GLOBAL -DVERSION_PATCH=0)
 CFLAGS (GLOBAL -DVERSION_NAME=\"ClickHouse\")
 CFLAGS (GLOBAL -DVERSION_OFFICIAL=\"\\\(arcadia\\\)\")
 CFLAGS (GLOBAL -DVERSION_REVISION=0)

From 98172deffc81b1bd962ae5213aa3e62dbf031e07 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitbar@yandex-team.ru>
Date: Wed, 20 May 2020 23:31:55 +0300
Subject: [PATCH 101/184] Show authentication type in table system.users and
 while executing SHOW CREATE USER query.

---
 src/Access/Authentication.cpp                 |  12 +-
 src/Access/Authentication.h                   |  61 +++++++-
 ...InterpreterShowCreateAccessEntityQuery.cpp |   4 +-
 src/Parsers/ASTCreateUserQuery.cpp            |  68 ++++++---
 src/Parsers/ASTCreateUserQuery.h              |   5 +-
 src/Parsers/ParserCreateUserQuery.cpp         | 138 +++++++-----------
 src/Parsers/ParserCreateUserQuery.h           |   4 +-
 src/Storages/System/StorageSystemUsers.cpp    |  23 ++-
 .../test_disk_access_storage/test.py          |   4 +-
 .../integration/test_grant_and_revoke/test.py |   6 +-
 10 files changed, 207 insertions(+), 118 deletions(-)

diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp
index f435d6e6336..cb2c9e7a256 100644
--- a/src/Access/Authentication.cpp
+++ b/src/Access/Authentication.cpp
@@ -7,8 +7,8 @@ namespace DB
 {
 namespace ErrorCodes
 {
-    extern const int LOGICAL_ERROR;
     extern const int BAD_ARGUMENTS;
+    extern const int NOT_IMPLEMENTED;
 }
 
 
@@ -36,8 +36,11 @@ Authentication::Digest Authentication::getPasswordDoubleSHA1() const
 
         case DOUBLE_SHA1_PASSWORD:
             return password_hash;
+
+        case MAX_TYPE:
+            break;
     }
-    throw Exception("Unknown authentication type: " + std::to_string(static_cast<int>(type)), ErrorCodes::LOGICAL_ERROR);
+    throw Exception("getPasswordDoubleSHA1(): authentication type " + toString(type) + " not supported", ErrorCodes::NOT_IMPLEMENTED);
 }
 
 
@@ -71,8 +74,11 @@ bool Authentication::isCorrectPassword(const String & password_) const
 
             return encodeSHA1(first_sha1) == password_hash;
         }
+
+        case MAX_TYPE:
+            break;
     }
-    throw Exception("Unknown authentication type: " + std::to_string(static_cast<int>(type)), ErrorCodes::LOGICAL_ERROR);
+    throw Exception("Cannot check if the password is correct for authentication type " + toString(type), ErrorCodes::NOT_IMPLEMENTED);
 }
 
 }
diff --git a/src/Access/Authentication.h b/src/Access/Authentication.h
index 3f16dc56de3..c410a101cdd 100644
--- a/src/Access/Authentication.h
+++ b/src/Access/Authentication.h
@@ -5,6 +5,7 @@
 #include <Common/OpenSSLHelpers.h>
 #include <Poco/SHA1Engine.h>
 #include <boost/algorithm/hex.hpp>
+#include <boost/algorithm/string/case_conv.hpp>
 
 
 namespace DB
@@ -14,6 +15,7 @@ namespace ErrorCodes
     extern const int SUPPORT_IS_DISABLED;
     extern const int BAD_ARGUMENTS;
     extern const int LOGICAL_ERROR;
+    extern const int NOT_IMPLEMENTED;
 }
 
 
@@ -35,6 +37,15 @@ public:
         /// SHA1(SHA1(password)).
         /// This kind of hash is used by the `mysql_native_password` authentication plugin.
         DOUBLE_SHA1_PASSWORD,
+
+        MAX_TYPE,
+    };
+
+    struct TypeInfo
+    {
+        const char * const raw_name;
+        const String name; /// Lowercased with underscores, e.g. "sha256_password".
+        static const TypeInfo & get(Type type_);
     };
 
     using Digest = std::vector<uint8_t>;
@@ -85,6 +96,48 @@ private:
 };
 
 
+inline const Authentication::TypeInfo & Authentication::TypeInfo::get(Type type_)
+{
+    static constexpr auto make_info = [](const char * raw_name_)
+    {
+        String init_name = raw_name_;
+        boost::to_lower(init_name);
+        return TypeInfo{raw_name_, std::move(init_name)};
+    };
+
+    switch (type_)
+    {
+        case NO_PASSWORD:
+        {
+            static const auto info = make_info("NO_PASSWORD");
+            return info;
+        }
+        case PLAINTEXT_PASSWORD:
+        {
+            static const auto info = make_info("PLAINTEXT_PASSWORD");
+            return info;
+        }
+        case SHA256_PASSWORD:
+        {
+            static const auto info = make_info("SHA256_PASSWORD");
+            return info;
+        }
+        case DOUBLE_SHA1_PASSWORD:
+        {
+            static const auto info = make_info("DOUBLE_SHA1_PASSWORD");
+            return info;
+        }
+        case MAX_TYPE: break;
+    }
+    throw Exception("Unknown authentication type: " + std::to_string(static_cast<int>(type_)), ErrorCodes::LOGICAL_ERROR);
+}
+
+inline String toString(Authentication::Type type_)
+{
+    return Authentication::TypeInfo::get(type_).raw_name;
+}
+
+
 inline Authentication::Digest Authentication::encodeSHA256(const std::string_view & text [[maybe_unused]])
 {
 #if USE_SSL
@@ -122,8 +175,10 @@ inline void Authentication::setPassword(const String & password_)
 
         case DOUBLE_SHA1_PASSWORD:
             return setPasswordHashBinary(encodeDoubleSHA1(password_));
+
+        case MAX_TYPE: break;
     }
-    throw Exception("Unknown authentication type: " + std::to_string(static_cast<int>(type)), ErrorCodes::LOGICAL_ERROR);
+    throw Exception("setPassword(): authentication type " + toString(type) + " not supported", ErrorCodes::NOT_IMPLEMENTED);
 }
 
 
@@ -186,8 +241,10 @@ inline void Authentication::setPasswordHashBinary(const Digest & hash)
             password_hash = hash;
             return;
         }
+
+        case MAX_TYPE: break;
     }
-    throw Exception("Unknown authentication type: " + std::to_string(static_cast<int>(type)), ErrorCodes::LOGICAL_ERROR);
+    throw Exception("setPasswordHashBinary(): authentication type " + toString(type) + " not supported", ErrorCodes::NOT_IMPLEMENTED);
 }
 
 }
diff --git a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp
index 87e1265f793..e37c31aab22 100644
--- a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp
+++ b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp
@@ -56,10 +56,10 @@ namespace
                 query->default_roles = user.default_roles.toASTWithNames(*manager);
         }
 
-        if (attach_mode && (user.authentication.getType() != Authentication::NO_PASSWORD))
+        if (user.authentication.getType() != Authentication::NO_PASSWORD)
         {
-            /// We don't show password unless it's an ATTACH statement.
             query->authentication = user.authentication;
+            query->show_password = attach_mode; /// We don't show password unless it's an ATTACH statement.
         }
 
         if (!user.settings.empty())
diff --git a/src/Parsers/ASTCreateUserQuery.cpp b/src/Parsers/ASTCreateUserQuery.cpp
index c8e2a76dfa2..e5c1178285b 100644
--- a/src/Parsers/ASTCreateUserQuery.cpp
+++ b/src/Parsers/ASTCreateUserQuery.cpp
@@ -6,6 +6,12 @@
 
 namespace DB
 {
+namespace ErrorCodes
+{
+    extern const int LOGICAL_ERROR;
+}
+
+
 namespace
 {
     void formatRenameTo(const String & new_name, const IAST::FormatSettings & settings)
@@ -15,27 +21,51 @@ namespace
     }
 
 
-    void formatAuthentication(const Authentication & authentication, const IAST::FormatSettings & settings)
+    void formatAuthentication(const Authentication & authentication, bool show_password, const IAST::FormatSettings & settings)
     {
-        settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " IDENTIFIED WITH " << (settings.hilite ? IAST::hilite_none : "");
-        switch (authentication.getType())
+        auto authentication_type = authentication.getType();
+        if (authentication_type == Authentication::NO_PASSWORD)
         {
-            case Authentication::Type::NO_PASSWORD:
-                settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "no_password" << (settings.hilite ? IAST::hilite_none : "");
-                break;
-            case Authentication::Type::PLAINTEXT_PASSWORD:
-                settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "plaintext_password BY " << (settings.hilite ? IAST::hilite_none : "")
-                              << quoteString(authentication.getPassword());
-                break;
-            case Authentication::Type::SHA256_PASSWORD:
-                settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "sha256_hash BY " << (settings.hilite ? IAST::hilite_none : "")
-                              << quoteString(authentication.getPasswordHashHex());
-                break;
-            case Authentication::Type::DOUBLE_SHA1_PASSWORD:
-                settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "double_sha1_hash BY " << (settings.hilite ? IAST::hilite_none : "")
-                              << quoteString(authentication.getPasswordHashHex());
-                break;
+            settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " NOT IDENTIFIED"
+                          << (settings.hilite ? IAST::hilite_none : "");
+            return;
         }
+
+        String authentication_type_name = Authentication::TypeInfo::get(authentication_type).name;
+        std::optional<String> password;
+
+        if (show_password)
+        {
+            switch (authentication_type)
+            {
+                case Authentication::PLAINTEXT_PASSWORD:
+                {
+                    password = authentication.getPassword();
+                    break;
+                }
+                case Authentication::SHA256_PASSWORD:
+                {
+                    authentication_type_name = "sha256_hash";
+                    password = authentication.getPasswordHashHex();
+                    break;
+                }
+                case Authentication::DOUBLE_SHA1_PASSWORD:
+                {
+                    authentication_type_name = "double_sha1_hash";
+                    password = authentication.getPasswordHashHex();
+                    break;
+                }
+
+                case Authentication::NO_PASSWORD: [[fallthrough]];
+                case Authentication::MAX_TYPE:
+                    throw Exception("AST: Unexpected authentication type " + toString(authentication_type), ErrorCodes::LOGICAL_ERROR);
+            }
+        }
+
+        settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " IDENTIFIED WITH " << authentication_type_name
+                      << (settings.hilite ? IAST::hilite_none : "");
+        if (password)
+            settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " BY " << quoteString(*password);
     }
 
 
@@ -190,7 +220,7 @@ void ASTCreateUserQuery::formatImpl(const FormatSettings & format, FormatState &
         formatRenameTo(new_name, format);
 
     if (authentication)
-        formatAuthentication(*authentication, format);
+        formatAuthentication(*authentication, show_password, format);
 
     if (hosts)
         formatHosts(nullptr, *hosts, format);
diff --git a/src/Parsers/ASTCreateUserQuery.h b/src/Parsers/ASTCreateUserQuery.h
index 54dc51d783b..28ef6c059da 100644
--- a/src/Parsers/ASTCreateUserQuery.h
+++ b/src/Parsers/ASTCreateUserQuery.h
@@ -12,14 +12,14 @@ class ASTExtendedRoleSet;
 class ASTSettingsProfileElements;
 
 /** CREATE USER [IF NOT EXISTS | OR REPLACE] name
-  *     [IDENTIFIED [WITH {NO_PASSWORD|PLAINTEXT_PASSWORD|SHA256_PASSWORD|SHA256_HASH|DOUBLE_SHA1_PASSWORD|DOUBLE_SHA1_HASH}] BY {'password'|'hash'}]
+  *     [NOT IDENTIFIED | IDENTIFIED [WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}]
   *     [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
   *     [DEFAULT ROLE role [,...]]
   *     [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...]
   *
   * ALTER USER [IF EXISTS] name
   *      [RENAME TO new_name]
-  *      [IDENTIFIED [WITH {PLAINTEXT_PASSWORD|SHA256_PASSWORD|DOUBLE_SHA1_PASSWORD}] BY {'password'|'hash'}]
+  *      [NOT IDENTIFIED | IDENTIFIED [WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}]
   *      [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
   *      [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ]
   *      [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...]
@@ -38,6 +38,7 @@ public:
     String new_name;
 
     std::optional<Authentication> authentication;
+    bool show_password = true; /// formatImpl() will show the password or hash.
 
     std::optional<AllowedClientHosts> hosts;
     std::optional<AllowedClientHosts> add_hosts;
diff --git a/src/Parsers/ParserCreateUserQuery.cpp b/src/Parsers/ParserCreateUserQuery.cpp
index 76a06a0282f..2f73e7fa75a 100644
--- a/src/Parsers/ParserCreateUserQuery.cpp
+++ b/src/Parsers/ParserCreateUserQuery.cpp
@@ -35,100 +35,74 @@ namespace
     }
 
 
-    bool parseByPassword(IParserBase::Pos & pos, Expected & expected, String & password)
-    {
-        return IParserBase::wrapParseImpl(pos, [&]
-        {
-            if (!ParserKeyword{"BY"}.ignore(pos, expected))
-                return false;
-
-            ASTPtr ast;
-            if (!ParserStringLiteral{}.parse(pos, ast, expected))
-                return false;
-
-            password = ast->as<const ASTLiteral &>().value.safeGet<String>();
-            return true;
-        });
-    }
-
-
     bool parseAuthentication(IParserBase::Pos & pos, Expected & expected, std::optional<Authentication> & authentication)
     {
         return IParserBase::wrapParseImpl(pos, [&]
         {
+            if (ParserKeyword{"NOT IDENTIFIED"}.ignore(pos, expected))
+            {
+                authentication = Authentication{Authentication::NO_PASSWORD};
+                return true;
+            }
+
             if (!ParserKeyword{"IDENTIFIED"}.ignore(pos, expected))
                 return false;
 
-            if (!ParserKeyword{"WITH"}.ignore(pos, expected))
+            std::optional<Authentication::Type> type;
+            bool expect_password = false;
+            bool expect_hash = false;
+
+            if (ParserKeyword{"WITH"}.ignore(pos, expected))
             {
-                String password;
-                if (!parseByPassword(pos, expected, password))
+                for (auto check_type : ext::range(Authentication::MAX_TYPE))
+                {
+                    if (ParserKeyword{Authentication::TypeInfo::get(check_type).raw_name}.ignore(pos, expected))
+                    {
+                        type = check_type;
+                        expect_password = (check_type != Authentication::NO_PASSWORD);
+                        break;
+                    }
+                }
+
+                if (!type)
+                {
+                    if (ParserKeyword{"SHA256_HASH"}.ignore(pos, expected))
+                    {
+                        type = Authentication::SHA256_PASSWORD;
+                        expect_hash = true;
+                    }
+                    else if (ParserKeyword{"DOUBLE_SHA1_HASH"}.ignore(pos, expected))
+                    {
+                        type = Authentication::DOUBLE_SHA1_PASSWORD;
+                        expect_hash = true;
+                    }
+                    else
+                        return false;
+                }
+            }
+
+            if (!type)
+            {
+                type = Authentication::SHA256_PASSWORD;
+                expect_password = true;
+            }
+
+            String password;
+            if (expect_password || expect_hash)
+            {
+                ASTPtr ast;
+                if (!ParserKeyword{"BY"}.ignore(pos, expected) || !ParserStringLiteral{}.parse(pos, ast, expected))
                     return false;
 
-                authentication = Authentication{Authentication::SHA256_PASSWORD};
+                password = ast->as<const ASTLiteral &>().value.safeGet<String>();
+            }
+
+            authentication = Authentication{*type};
+            if (expect_password)
                 authentication->setPassword(password);
-                return true;
-            }
+            else if (expect_hash)
+                authentication->setPasswordHashHex(password);
 
-            if (ParserKeyword{"PLAINTEXT_PASSWORD"}.ignore(pos, expected))
-            {
-                String password;
-                if (!parseByPassword(pos, expected, password))
-                    return false;
-
-                authentication = Authentication{Authentication::PLAINTEXT_PASSWORD};
-                authentication->setPassword(password);
-                return true;
-            }
-
-            if (ParserKeyword{"SHA256_PASSWORD"}.ignore(pos, expected))
-            {
-                String password;
-                if (!parseByPassword(pos, expected, password))
-                    return false;
-
-                authentication = Authentication{Authentication::SHA256_PASSWORD};
-                authentication->setPassword(password);
-                return true;
-            }
-
-            if (ParserKeyword{"SHA256_HASH"}.ignore(pos, expected))
-            {
-                String hash;
-                if (!parseByPassword(pos, expected, hash))
-                    return false;
-
-                authentication = Authentication{Authentication::SHA256_PASSWORD};
-                authentication->setPasswordHashHex(hash);
-                return true;
-            }
-
-            if (ParserKeyword{"DOUBLE_SHA1_PASSWORD"}.ignore(pos, expected))
-            {
-                String password;
-                if (!parseByPassword(pos, expected, password))
-                    return false;
-
-                authentication = Authentication{Authentication::DOUBLE_SHA1_PASSWORD};
-                authentication->setPassword(password);
-                return true;
-            }
-
-            if (ParserKeyword{"DOUBLE_SHA1_HASH"}.ignore(pos, expected))
-            {
-                String hash;
-                if (!parseByPassword(pos, expected, hash))
-                    return false;
-
-                authentication = Authentication{Authentication::DOUBLE_SHA1_PASSWORD};
-                authentication->setPasswordHashHex(hash);
-                return true;
-            }
-
-            if (!ParserKeyword{"NO_PASSWORD"}.ignore(pos, expected))
-                return false;
-
-            authentication = Authentication{Authentication::NO_PASSWORD};
             return true;
         });
     }
diff --git a/src/Parsers/ParserCreateUserQuery.h b/src/Parsers/ParserCreateUserQuery.h
index d609894a7ec..2a890f41060 100644
--- a/src/Parsers/ParserCreateUserQuery.h
+++ b/src/Parsers/ParserCreateUserQuery.h
@@ -7,13 +7,13 @@ namespace DB
 {
 /** Parses queries like
   * CREATE USER [IF NOT EXISTS | OR REPLACE] name
-  *     [IDENTIFIED [WITH {NO_PASSWORD|PLAINTEXT_PASSWORD|SHA256_PASSWORD|SHA256_HASH|DOUBLE_SHA1_PASSWORD|DOUBLE_SHA1_HASH}] BY {'password'|'hash'}]
+  *     [NOT IDENTIFIED | IDENTIFIED [WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}]
   *     [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
   *     [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...]
   *
   * ALTER USER [IF EXISTS] name
   *     [RENAME TO new_name]
-  *     [IDENTIFIED [WITH {PLAINTEXT_PASSWORD|SHA256_PASSWORD|DOUBLE_SHA1_PASSWORD}] BY {'password'|'hash'}]
+  *     [NOT IDENTIFIED | IDENTIFIED [WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}]
   *     [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
   *     [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...]
   */
diff --git a/src/Storages/System/StorageSystemUsers.cpp b/src/Storages/System/StorageSystemUsers.cpp
index d0e042d054f..e0755fe59ab 100644
--- a/src/Storages/System/StorageSystemUsers.cpp
+++ b/src/Storages/System/StorageSystemUsers.cpp
@@ -3,6 +3,7 @@
 #include <DataTypes/DataTypesNumber.h>
 #include <DataTypes/DataTypeUUID.h>
 #include <DataTypes/DataTypeArray.h>
+#include <DataTypes/DataTypeEnum.h>
 #include <Columns/ColumnArray.h>
 #include <Columns/ColumnString.h>
 #include <Columns/ColumnsNumber.h>
@@ -15,12 +16,26 @@
 
 namespace DB
 {
+namespace
+{
+    DataTypeEnum8::Values getAuthenticationTypeEnumValues()
+    {
+        DataTypeEnum8::Values enum_values;
+        for (auto type : ext::range(Authentication::MAX_TYPE))
+            enum_values.emplace_back(Authentication::TypeInfo::get(type).name, static_cast<Int8>(type));
+        return enum_values;
+    }
+}
+
+
 NamesAndTypesList StorageSystemUsers::getNamesAndTypes()
 {
     NamesAndTypesList names_and_types{
         {"name", std::make_shared<DataTypeString>()},
         {"id", std::make_shared<DataTypeUUID>()},
         {"storage", std::make_shared<DataTypeString>()},
+        {"auth_type", std::make_shared<DataTypeEnum8>(getAuthenticationTypeEnumValues())},
+        {"auth_params", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())},
         {"host_ip", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())},
         {"host_names", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())},
         {"host_names_regexp", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())},
@@ -43,6 +58,9 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, const Context &
     auto & column_name = assert_cast<ColumnString &>(*res_columns[column_index++]);
     auto & column_id = assert_cast<ColumnUInt128 &>(*res_columns[column_index++]).getData();
     auto & column_storage = assert_cast<ColumnString &>(*res_columns[column_index++]);
+    auto & column_auth_type = assert_cast<ColumnInt8 &>(*res_columns[column_index++]).getData();
+    auto & column_auth_params = assert_cast<ColumnString &>(assert_cast<ColumnArray &>(*res_columns[column_index]).getData());
+    auto & column_auth_params_offsets = assert_cast<ColumnArray &>(*res_columns[column_index++]).getOffsets();
     auto & column_host_ip = assert_cast<ColumnString &>(assert_cast<ColumnArray &>(*res_columns[column_index]).getData());
     auto & column_host_ip_offsets = assert_cast<ColumnArray &>(*res_columns[column_index++]).getOffsets();
     auto & column_host_names = assert_cast<ColumnString &>(assert_cast<ColumnArray &>(*res_columns[column_index]).getData());
@@ -60,12 +78,15 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, const Context &
     auto add_row = [&](const String & name,
                        const UUID & id,
                        const String & storage_name,
+                       const Authentication & authentication,
                        const AllowedClientHosts & allowed_hosts,
                        const ExtendedRoleSet & default_roles)
     {
         column_name.insertData(name.data(), name.length());
         column_id.push_back(id);
         column_storage.insertData(storage_name.data(), storage_name.length());
+        column_auth_type.push_back(static_cast<Int8>(authentication.getType()));
+        column_auth_params_offsets.push_back(column_auth_params.size());
 
         if (allowed_hosts.containsAnyHost())
         {
@@ -128,7 +149,7 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, const Context &
         if (!storage)
             continue;
 
-        add_row(user->getName(), id, storage->getStorageName(), user->allowed_client_hosts, user->default_roles);
+        add_row(user->getName(), id, storage->getStorageName(), user->authentication, user->allowed_client_hosts, user->default_roles);
     }
 }
 
diff --git a/tests/integration/test_disk_access_storage/test.py b/tests/integration/test_disk_access_storage/test.py
index 315440b4358..a47af5ad5b8 100644
--- a/tests/integration/test_disk_access_storage/test.py
+++ b/tests/integration/test_disk_access_storage/test.py
@@ -39,7 +39,7 @@ def test_create():
 
     def check():
         assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1 SETTINGS PROFILE s1\n"
-        assert instance.query("SHOW CREATE USER u2") == "CREATE USER u2 HOST LOCAL DEFAULT ROLE rx\n"
+        assert instance.query("SHOW CREATE USER u2") == "CREATE USER u2 IDENTIFIED WITH sha256_password HOST LOCAL DEFAULT ROLE rx\n"
         assert instance.query("SHOW CREATE ROW POLICY p ON mydb.mytable") == "CREATE ROW POLICY p ON mydb.mytable FOR SELECT USING a < 1000 TO u1, u2\n"
         assert instance.query("SHOW CREATE QUOTA q") == "CREATE QUOTA q KEYED BY \\'none\\' FOR INTERVAL 1 HOUR MAX QUERIES 100 TO ALL EXCEPT rx\n"
         assert instance.query("SHOW GRANTS FOR u1") == ""
@@ -69,7 +69,7 @@ def test_alter():
 
     def check():
         assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1 SETTINGS PROFILE s1\n"
-        assert instance.query("SHOW CREATE USER u2") == "CREATE USER u2 HOST LOCAL DEFAULT ROLE ry\n"
+        assert instance.query("SHOW CREATE USER u2") == "CREATE USER u2 IDENTIFIED WITH sha256_password HOST LOCAL DEFAULT ROLE ry\n"
         assert instance.query("SHOW GRANTS FOR u1") == "GRANT SELECT ON mydb.mytable TO u1\n"
         assert instance.query("SHOW GRANTS FOR u2") == "GRANT rx, ry TO u2\n"
         assert instance.query("SHOW CREATE ROLE rx") == "CREATE ROLE rx SETTINGS PROFILE s2\n"
diff --git a/tests/integration/test_grant_and_revoke/test.py b/tests/integration/test_grant_and_revoke/test.py
index 8df68547f38..7054ce28e59 100644
--- a/tests/integration/test_grant_and_revoke/test.py
+++ b/tests/integration/test_grant_and_revoke/test.py
@@ -155,9 +155,9 @@ def test_introspection():
     assert instance.query("SHOW ENABLED ROLES", user='A') == TSV([[ "R1", 0, 1, 1 ]])
     assert instance.query("SHOW ENABLED ROLES", user='B') == TSV([[ "R2", 1, 1, 1 ]])
 
-    assert instance.query("SELECT name, storage, host_ip, host_names, host_names_regexp, host_names_like, default_roles_all, default_roles_list, default_roles_except from system.users WHERE name IN ('A', 'B') ORDER BY name") ==\
-           TSV([[ "A", "disk", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ],
-                [ "B", "disk", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ]])
+    assert instance.query("SELECT name, storage, auth_type, auth_params, host_ip, host_names, host_names_regexp, host_names_like, default_roles_all, default_roles_list, default_roles_except from system.users WHERE name IN ('A', 'B') ORDER BY name") ==\
+           TSV([[ "A", "disk", "no_password", "[]", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ],
+                [ "B", "disk", "no_password", "[]", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ]])
     
     assert instance.query("SELECT name, storage from system.roles WHERE name IN ('R1', 'R2') ORDER BY name") ==\
            TSV([[ "R1", "disk" ],

From aabf8d479eba263860b814437237a1c77fa34262 Mon Sep 17 00:00:00 2001
From: "dependabot-preview[bot]"
 <27856297+dependabot-preview[bot]@users.noreply.github.com>
Date: Thu, 28 May 2020 07:33:06 +0000
Subject: [PATCH 102/184] Bump protobuf from 3.12.1 to 3.12.2 in /docs/tools

Bumps [protobuf](https://github.com/protocolbuffers/protobuf) from 3.12.1 to 3.12.2.
- [Release notes](https://github.com/protocolbuffers/protobuf/releases)
- [Changelog](https://github.com/protocolbuffers/protobuf/blob/master/generate_changelog.py)
- [Commits](https://github.com/protocolbuffers/protobuf/compare/v3.12.1...v3.12.2)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>
---
 docs/tools/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/tools/requirements.txt b/docs/tools/requirements.txt
index 4d4e9f98780..d9ea19ff389 100644
--- a/docs/tools/requirements.txt
+++ b/docs/tools/requirements.txt
@@ -21,7 +21,7 @@ mkdocs-htmlproofer-plugin==0.0.3
 mkdocs-macros-plugin==0.4.9
 nltk==3.5
 nose==1.3.7
-protobuf==3.12.1
+protobuf==3.12.2
 numpy==1.18.4
 Pygments==2.5.2
 pymdown-extensions==7.1

From ca5d7cf5bb0d06224015881dcf4171be8adaffae Mon Sep 17 00:00:00 2001
From: Alexander Kuzmenkov <akuzm@yandex-team.ru>
Date: Thu, 28 May 2020 10:45:03 +0300
Subject: [PATCH 103/184] fix some broken performance tests

---
 docker/test/performance-comparison/compare.sh | 4 ++--
 docker/test/performance-comparison/report.py  | 2 +-
 tests/performance/leftpad.xml                 | 1 -
 tests/performance/sort_radix_trivial.xml      | 6 +++---
 4 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh
index bf4599acb9a..209b36f59af 100755
--- a/docker/test/performance-comparison/compare.sh
+++ b/docker/test/performance-comparison/compare.sh
@@ -317,7 +317,7 @@ function report
 rm -r report ||:
 mkdir report report/tmp ||:
 
-rm ./*.{rep,svg} test-times.tsv test-dump.tsv unstable.tsv unstable-query-ids.tsv unstable-query-metrics.tsv changed-perf.tsv unstable-tests.tsv unstable-queries.tsv bad-tests.tsv slow-on-client.tsv all-queries.tsv ||:
+rm ./*.{rep,svg} test-times.tsv test-dump.tsv unstable.tsv unstable-query-ids.tsv unstable-query-metrics.tsv changed-perf.tsv unstable-tests.tsv unstable-queries.tsv bad-tests.tsv slow-on-client.tsv all-queries.tsv run-errors.tsv ||:
 
 build_log_column_definitions
 
@@ -434,7 +434,7 @@ create table wall_clock engine Memory as select *
     from file('wall-clock-times.tsv', TSV, 'test text, real float, user float, system float');
 
 create table slow_on_client_tsv engine File(TSV, 'report/slow-on-client.tsv') as
-    select client, server, floor(client/server, 3) p, query_display_name
+    select client, server, floor(client/server, 3) p, test, query_display_name
     from query_time left join query_display_names using (test, query_index)
     where p > 1.02 order by p desc;
 
diff --git a/docker/test/performance-comparison/report.py b/docker/test/performance-comparison/report.py
index b171603700d..866e78da098 100755
--- a/docker/test/performance-comparison/report.py
+++ b/docker/test/performance-comparison/report.py
@@ -189,7 +189,7 @@ if args.report == 'main':
     slow_on_client_rows = tsvRows('report/slow-on-client.tsv')
     error_tests += len(slow_on_client_rows)
     printSimpleTable('Slow on client',
-                     ['Client time, s', 'Server time, s', 'Ratio', 'Query'],
+                     ['Client time, s', 'Server time, s', 'Ratio', 'Test', 'Query'],
                      slow_on_client_rows)
 
     def print_changes():
diff --git a/tests/performance/leftpad.xml b/tests/performance/leftpad.xml
index 4349c6ac0e5..199f9224b89 100644
--- a/tests/performance/leftpad.xml
+++ b/tests/performance/leftpad.xml
@@ -6,7 +6,6 @@
     <query><![CDATA[SELECT max(length(MobilePhoneModel)) FROM hits_100m_single]]></query>
     <query><![CDATA[SELECT max(length(Params)) FROM hits_100m_single]]></query>
     <query><![CDATA[SELECT max(length(Title)) FROM hits_100m_single]]></query>
-    <query><![CDATA[SELECT max(length(URLDomain)) FROM hits_100m_single]]></query>
     <query><![CDATA[SELECT max(length(PageCharset)) FROM hits_100m_single]]></query>
     <query><![CDATA[SELECT max(length(Referer)) FROM hits_100m_single]]></query>
     <query><![CDATA[SELECT max(length(URL)) FROM hits_100m_single]]></query>
diff --git a/tests/performance/sort_radix_trivial.xml b/tests/performance/sort_radix_trivial.xml
index fb3dc838a9c..096c3dd0854 100644
--- a/tests/performance/sort_radix_trivial.xml
+++ b/tests/performance/sort_radix_trivial.xml
@@ -1,5 +1,5 @@
 <test>
-    <query>SELECT rand32() AS x FROM numbers(1000000) ORDER BY x</query>
-    <query>SELECT rand64() AS x FROM numbers(1000000) ORDER BY x</query>
-    <query>SELECT 1 / rand64() AS x FROM numbers(1000000) ORDER BY x</query>
+    <query>SELECT rand32() AS x FROM numbers(1000000) ORDER BY x FORMAT Null</query>
+    <query>SELECT rand64() AS x FROM numbers(1000000) ORDER BY x FORMAT Null</query>
+    <query>SELECT 1 / rand64() AS x FROM numbers(1000000) ORDER BY x FORMAT Null</query>
 </test>

From 1fa795988faea4a8f76383f67252ccca373be5ab Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Thu, 28 May 2020 11:24:59 +0300
Subject: [PATCH 104/184] Remove some code.

---
 src/Interpreters/IInterpreter.h               |   4 -
 src/Interpreters/InterpreterSelectQuery.cpp   | 878 ++----------------
 src/Interpreters/InterpreterSelectQuery.h     |  95 +-
 .../InterpreterSelectWithUnionQuery.cpp       |  73 +-
 .../InterpreterSelectWithUnionQuery.h         |   6 -
 src/Interpreters/executeQuery.cpp             |   9 +-
 src/Storages/IStorage.cpp                     |  19 -
 src/Storages/IStorage.h                       |  10 -
 src/Storages/SelectQueryInfo.h                |  22 -
 src/Storages/StorageView.cpp                  |  12 +-
 10 files changed, 100 insertions(+), 1028 deletions(-)

diff --git a/src/Interpreters/IInterpreter.h b/src/Interpreters/IInterpreter.h
index 2425d015e56..63f36f1b18a 100644
--- a/src/Interpreters/IInterpreter.h
+++ b/src/Interpreters/IInterpreter.h
@@ -22,10 +22,6 @@ public:
       */
     virtual BlockIO execute() = 0;
 
-    virtual QueryPipeline executeWithProcessors() { throw Exception("executeWithProcessors not implemented", ErrorCodes::NOT_IMPLEMENTED); }
-
-    virtual bool canExecuteWithProcessors() const { return false; }
-
     virtual bool ignoreQuota() const { return false; }
     virtual bool ignoreLimits() const { return false; }
 
diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp
index 4f717eda706..3af128ffb00 100644
--- a/src/Interpreters/InterpreterSelectQuery.cpp
+++ b/src/Interpreters/InterpreterSelectQuery.cpp
@@ -1,39 +1,15 @@
 #include <DataStreams/ExpressionBlockInputStream.h>
-#include <DataStreams/FilterBlockInputStream.h>
-#include <DataStreams/FinishSortingBlockInputStream.h>
-#include <DataStreams/LimitBlockInputStream.h>
-#include <DataStreams/LimitByBlockInputStream.h>
-#include <DataStreams/PartialSortingBlockInputStream.h>
-#include <DataStreams/MergeSortingBlockInputStream.h>
-#include <DataStreams/MergingSortedBlockInputStream.h>
 #include <DataStreams/AggregatingBlockInputStream.h>
 #include <DataStreams/MergingAggregatedBlockInputStream.h>
-#include <DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.h>
-#include <DataStreams/AsynchronousBlockInputStream.h>
-#include <DataStreams/UnionBlockInputStream.h>
-#include <DataStreams/ParallelAggregatingBlockInputStream.h>
-#include <DataStreams/DistinctBlockInputStream.h>
-#include <DataStreams/NullBlockInputStream.h>
-#include <DataStreams/TotalsHavingBlockInputStream.h>
 #include <DataStreams/OneBlockInputStream.h>
 #include <DataStreams/copyData.h>
-#include <DataStreams/CreatingSetsBlockInputStream.h>
-#include <DataStreams/MaterializingBlockInputStream.h>
-#include <DataStreams/ConcatBlockInputStream.h>
-#include <DataStreams/RollupBlockInputStream.h>
-#include <DataStreams/CubeBlockInputStream.h>
 #include <DataStreams/ConvertColumnLowCardinalityToFullBlockInputStream.h>
-#include <DataStreams/ConvertingBlockInputStream.h>
-#include <DataStreams/ReverseBlockInputStream.h>
-#include <DataStreams/FillingBlockInputStream.h>
-
 #include <Parsers/ASTFunction.h>
 #include <Parsers/ASTIdentifier.h>
 #include <Parsers/ASTLiteral.h>
 #include <Parsers/ASTOrderByElement.h>
 #include <Parsers/ASTSelectWithUnionQuery.h>
 #include <Parsers/ASTTablesInSelectQuery.h>
-#include <Parsers/ParserSelectQuery.h>
 #include <Parsers/ExpressionListParsers.h>
 #include <Parsers/parseQuery.h>
 
@@ -59,7 +35,6 @@
 #include <Storages/IStorage.h>
 
 #include <TableFunctions/ITableFunction.h>
-#include <TableFunctions/TableFunctionFactory.h>
 
 #include <Functions/IFunction.h>
 #include <Core/Field.h>
@@ -68,7 +43,6 @@
 #include <Common/FieldVisitors.h>
 #include <Common/typeid_cast.h>
 #include <Common/checkStackSize.h>
-#include <Parsers/queryToString.h>
 #include <ext/map.h>
 #include <ext/scope_guard.h>
 #include <memory>
@@ -98,7 +72,7 @@
 #include <DataTypes/DataTypeAggregateFunction.h>
 #include <DataStreams/materializeBlock.h>
 #include <Processors/Pipe.h>
-#include <Processors/Executors/TreeExecutorBlockInputStream.h>
+#include <Processors/Sources/SourceFromSingleChunk.h>
 
 
 namespace DB
@@ -465,38 +439,13 @@ Block InterpreterSelectQuery::getSampleBlock()
 
 BlockIO InterpreterSelectQuery::execute()
 {
-    Pipeline pipeline;
     BlockIO res;
-    executeImpl(pipeline, input, std::move(input_pipe), res.pipeline);
-    executeUnion(pipeline, getSampleBlock());
-
-    res.in = pipeline.firstStream();
+    executeImpl(res.pipeline, input, std::move(input_pipe));
     res.pipeline.addInterpreterContext(context);
     res.pipeline.addStorageHolder(storage);
     return res;
 }
 
-BlockInputStreams InterpreterSelectQuery::executeWithMultipleStreams(QueryPipeline & parent_pipeline)
-{
-    ///FIXME pipeline must be alive until query is finished
-    Pipeline pipeline;
-    executeImpl(pipeline, input, std::move(input_pipe), parent_pipeline);
-    unifyStreams(pipeline, getSampleBlock());
-    parent_pipeline.addInterpreterContext(context);
-    parent_pipeline.addStorageHolder(storage);
-    return pipeline.streams;
-}
-
-QueryPipeline InterpreterSelectQuery::executeWithProcessors()
-{
-    QueryPipeline query_pipeline;
-    executeImpl(query_pipeline, input, std::move(input_pipe), query_pipeline);
-    query_pipeline.addInterpreterContext(context);
-    query_pipeline.addStorageHolder(storage);
-    return query_pipeline;
-}
-
-
 Block InterpreterSelectQuery::getSampleBlockImpl()
 {
     if (storage && !options.only_analyze)
@@ -687,9 +636,7 @@ static UInt64 getLimitForSorting(const ASTSelectQuery & query, const Context & c
     return 0;
 }
 
-
-template <typename TPipeline>
-void InterpreterSelectQuery::executeImpl(TPipeline & pipeline, const BlockInputStreamPtr & prepared_input, std::optional<Pipe> prepared_pipe, QueryPipeline & save_context_and_storage)
+void InterpreterSelectQuery::executeImpl(QueryPipeline & pipeline, const BlockInputStreamPtr & prepared_input, std::optional<Pipe> prepared_pipe)
 {
     /** Streams of data. When the query is executed in parallel, we have several data streams.
      *  If there is no GROUP BY, then perform all operations before ORDER BY and LIMIT in parallel, then
@@ -701,8 +648,6 @@ void InterpreterSelectQuery::executeImpl(TPipeline & pipeline, const BlockInputS
      *  then perform the remaining operations with one resulting stream.
      */
 
-    constexpr bool pipeline_with_processors = std::is_same<TPipeline, QueryPipeline>::value;
-
     /// Now we will compose block streams that perform the necessary actions.
     auto & query = getSelectQuery();
     const Settings & settings = context->getSettingsRef();
@@ -712,40 +657,27 @@ void InterpreterSelectQuery::executeImpl(TPipeline & pipeline, const BlockInputS
 
     if (options.only_analyze)
     {
-        if constexpr (pipeline_with_processors)
-            pipeline.init(Pipe(std::make_shared<NullSource>(source_header)));
-        else
-            pipeline.streams.emplace_back(std::make_shared<NullBlockInputStream>(source_header));
+        pipeline.init(Pipe(std::make_shared<NullSource>(source_header)));
 
         if (expressions.prewhere_info)
         {
-            if constexpr (pipeline_with_processors)
-                pipeline.addSimpleTransform([&](const Block & header)
-                {
-                    return std::make_shared<FilterTransform>(
-                            header,
-                            expressions.prewhere_info->prewhere_actions,
-                            expressions.prewhere_info->prewhere_column_name,
-                            expressions.prewhere_info->remove_prewhere_column);
-                });
-            else
-                pipeline.streams.back() = std::make_shared<FilterBlockInputStream>(
-                    pipeline.streams.back(), expressions.prewhere_info->prewhere_actions,
-                    expressions.prewhere_info->prewhere_column_name, expressions.prewhere_info->remove_prewhere_column);
+            pipeline.addSimpleTransform([&](const Block & header)
+            {
+                return std::make_shared<FilterTransform>(
+                        header,
+                        expressions.prewhere_info->prewhere_actions,
+                        expressions.prewhere_info->prewhere_column_name,
+                        expressions.prewhere_info->remove_prewhere_column);
+            });
 
             // To remove additional columns in dry run
             // For example, sample column which can be removed in this stage
             if (expressions.prewhere_info->remove_columns_actions)
             {
-                if constexpr (pipeline_with_processors)
+                pipeline.addSimpleTransform([&](const Block & header)
                 {
-                    pipeline.addSimpleTransform([&](const Block & header)
-                    {
-                        return std::make_shared<ExpressionTransform>(header, expressions.prewhere_info->remove_columns_actions);
-                    });
-                }
-                else
-                    pipeline.streams.back() = std::make_shared<ExpressionBlockInputStream>(pipeline.streams.back(), expressions.prewhere_info->remove_columns_actions);
+                    return std::make_shared<ExpressionTransform>(header, expressions.prewhere_info->remove_columns_actions);
+                });
             }
         }
     }
@@ -753,17 +685,11 @@ void InterpreterSelectQuery::executeImpl(TPipeline & pipeline, const BlockInputS
     {
         if (prepared_input)
         {
-            if constexpr (pipeline_with_processors)
-                pipeline.init(Pipe(std::make_shared<SourceFromInputStream>(prepared_input)));
-            else
-                pipeline.streams.push_back(prepared_input);
+            pipeline.init(Pipe(std::make_shared<SourceFromInputStream>(prepared_input)));
         }
         else if (prepared_pipe)
         {
-            if constexpr (pipeline_with_processors)
-                pipeline.init(std::move(*prepared_pipe));
-            else
-                pipeline.streams.push_back(std::make_shared<TreeExecutorBlockInputStream>(std::move(*prepared_pipe)));
+            pipeline.init(std::move(*prepared_pipe));
         }
 
         if (from_stage == QueryProcessingStage::WithMergeableState &&
@@ -774,7 +700,7 @@ void InterpreterSelectQuery::executeImpl(TPipeline & pipeline, const BlockInputS
             throw Exception("PREWHERE is not supported if the table is filtered by row-level security expression", ErrorCodes::ILLEGAL_PREWHERE);
 
         /** Read the data from Storage. from_stage - to what stage the request was completed in Storage. */
-        executeFetchColumns(from_stage, pipeline, expressions.prewhere_info, expressions.columns_to_remove_after_prewhere, save_context_and_storage);
+        executeFetchColumns(from_stage, pipeline, expressions.prewhere_info, expressions.columns_to_remove_after_prewhere);
 
         LOG_TRACE(log, "{} -> {}", QueryProcessingStage::toString(from_stage), QueryProcessingStage::toString(options.to_stage));
     }
@@ -817,12 +743,7 @@ void InterpreterSelectQuery::executeImpl(TPipeline & pipeline, const BlockInputS
                 }
 
                 if (query.limitLength())
-                {
-                    if constexpr (pipeline_with_processors)
-                        executePreLimit(pipeline, true);
-                    else
-                        executePreLimit(pipeline);
-                }
+                    executePreLimit(pipeline, true);
             }
         };
 
@@ -840,31 +761,17 @@ void InterpreterSelectQuery::executeImpl(TPipeline & pipeline, const BlockInputS
         {
             if (expressions.hasFilter())
             {
-                if constexpr (pipeline_with_processors)
+                pipeline.addSimpleTransform([&](const Block & block, QueryPipeline::StreamType stream_type) -> ProcessorPtr
                 {
-                    pipeline.addSimpleTransform([&](const Block & block, QueryPipeline::StreamType stream_type) -> ProcessorPtr
-                    {
-                        bool on_totals = stream_type == QueryPipeline::StreamType::Totals;
+                    bool on_totals = stream_type == QueryPipeline::StreamType::Totals;
 
-                        return std::make_shared<FilterTransform>(
-                            block,
-                            expressions.filter_info->actions,
-                            expressions.filter_info->column_name,
-                            expressions.filter_info->do_remove_column,
-                            on_totals);
-                    });
-                }
-                else
-                {
-                    pipeline.transform([&](auto & stream)
-                    {
-                        stream = std::make_shared<FilterBlockInputStream>(
-                            stream,
-                            expressions.filter_info->actions,
-                            expressions.filter_info->column_name,
-                            expressions.filter_info->do_remove_column);
-                    });
-                }
+                    return std::make_shared<FilterTransform>(
+                        block,
+                        expressions.filter_info->actions,
+                        expressions.filter_info->column_name,
+                        expressions.filter_info->do_remove_column,
+                        on_totals);
+                });
             }
 
             if (expressions.hasJoin())
@@ -872,59 +779,43 @@ void InterpreterSelectQuery::executeImpl(TPipeline & pipeline, const BlockInputS
                 Block join_result_sample;
                 JoinPtr join = expressions.before_join->getTableJoinAlgo();
 
-                if constexpr (pipeline_with_processors)
+                join_result_sample = ExpressionBlockInputStream(
+                    std::make_shared<OneBlockInputStream>(pipeline.getHeader()), expressions.before_join).getHeader();
+
+                /// In case joined subquery has totals, and we don't, add default chunk to totals.
+                bool default_totals = false;
+                if (!pipeline.hasTotals())
                 {
-                    join_result_sample = ExpressionBlockInputStream(
-                        std::make_shared<OneBlockInputStream>(pipeline.getHeader()), expressions.before_join).getHeader();
-
-                    /// In case joined subquery has totals, and we don't, add default chunk to totals.
-                    bool default_totals = false;
-                    if (!pipeline.hasTotals())
-                    {
-                        pipeline.addDefaultTotals();
-                        default_totals = true;
-                    }
-
-                    bool inflating_join = false;
-                    if (join)
-                    {
-                        inflating_join = true;
-                        if (auto * hash_join = typeid_cast<HashJoin *>(join.get()))
-                            inflating_join = isCross(hash_join->getKind());
-                    }
-
-                    pipeline.addSimpleTransform([&](const Block & header, QueryPipeline::StreamType type)
-                    {
-                        bool on_totals = type == QueryPipeline::StreamType::Totals;
-                        std::shared_ptr<IProcessor> ret;
-                        if (inflating_join)
-                            ret = std::make_shared<InflatingExpressionTransform>(header, expressions.before_join, on_totals, default_totals);
-                        else
-                            ret = std::make_shared<ExpressionTransform>(header, expressions.before_join, on_totals, default_totals);
-
-                        return ret;
-                    });
+                    pipeline.addDefaultTotals();
+                    default_totals = true;
                 }
-                else
+
+                bool inflating_join = false;
+                if (join)
                 {
-                    /// Applies to all sources except stream_with_non_joined_data.
-                    for (auto & stream : pipeline.streams)
-                        stream = std::make_shared<InflatingExpressionBlockInputStream>(stream, expressions.before_join);
-
-                    join_result_sample = pipeline.firstStream()->getHeader();
+                    inflating_join = true;
+                    if (auto * hash_join = typeid_cast<HashJoin *>(join.get()))
+                        inflating_join = isCross(hash_join->getKind());
                 }
 
+                pipeline.addSimpleTransform([&](const Block & header, QueryPipeline::StreamType type)
+                {
+                    bool on_totals = type == QueryPipeline::StreamType::Totals;
+                    std::shared_ptr<IProcessor> ret;
+                    if (inflating_join)
+                        ret = std::make_shared<InflatingExpressionTransform>(header, expressions.before_join, on_totals, default_totals);
+                    else
+                        ret = std::make_shared<ExpressionTransform>(header, expressions.before_join, on_totals, default_totals);
+
+                    return ret;
+                });
+
                 if (join)
                 {
                     if (auto stream = join->createStreamWithNonJoinedRows(join_result_sample, settings.max_block_size))
                     {
-                        if constexpr (pipeline_with_processors)
-                        {
-                            auto source = std::make_shared<SourceFromInputStream>(std::move(stream));
-                            pipeline.addDelayedStream(source);
-                        }
-                        else
-                            pipeline.stream_with_non_joined_data = std::move(stream);
+                        auto source = std::make_shared<SourceFromInputStream>(std::move(stream));
+                        pipeline.addDelayedStream(source);
                     }
                 }
             }
@@ -1009,28 +900,14 @@ void InterpreterSelectQuery::executeImpl(TPipeline & pipeline, const BlockInputS
             if (query.limitLength() && !query.limit_with_ties && pipeline.hasMoreThanOneStream() &&
                 !query.distinct && !expressions.hasLimitBy() && !settings.extremes)
             {
-                if constexpr (pipeline_with_processors)
-                    executePreLimit(pipeline, false);
-                else
-                    executePreLimit(pipeline);
-
+                executePreLimit(pipeline, false);
                 has_prelimit = true;
             }
 
-            bool need_merge_streams = need_second_distinct_pass || query.limitBy()
-                || (!pipeline_with_processors && query.limitLength()); /// Don't merge streams for pre-limit more.
-
-            if constexpr (!pipeline_with_processors)
-                if (pipeline.hasDelayedStream())
-                    need_merge_streams = true;
+            bool need_merge_streams = need_second_distinct_pass || query.limitBy();
 
             if (need_merge_streams)
-            {
-                if constexpr (pipeline_with_processors)
-                    pipeline.resize(1);
-                else
-                    executeUnion(pipeline, {});
-            }
+                pipeline.resize(1);
 
             /** If there was more than one stream,
               * then DISTINCT needs to be performed once again after merging all streams.
@@ -1054,7 +931,7 @@ void InterpreterSelectQuery::executeImpl(TPipeline & pipeline, const BlockInputS
               */
             executeExtremes(pipeline);
 
-            if (!(pipeline_with_processors && has_prelimit))  /// Limit is no longer needed if there is prelimit.
+            if (!has_prelimit)  /// Limit is no longer needed if there is prelimit.
                 executeLimit(pipeline);
 
             executeOffset(pipeline);
@@ -1065,14 +942,10 @@ void InterpreterSelectQuery::executeImpl(TPipeline & pipeline, const BlockInputS
         executeSubqueriesInSetsAndJoins(pipeline, subqueries_for_sets);
 }
 
-template <typename TPipeline>
 void InterpreterSelectQuery::executeFetchColumns(
-        QueryProcessingStage::Enum processing_stage, TPipeline & pipeline,
-        const PrewhereInfoPtr & prewhere_info, const Names & columns_to_remove_after_prewhere,
-        QueryPipeline & save_context_and_storage)
+    QueryProcessingStage::Enum processing_stage, QueryPipeline & pipeline,
+    const PrewhereInfoPtr & prewhere_info, const Names & columns_to_remove_after_prewhere)
 {
-    constexpr bool pipeline_with_processors = std::is_same<TPipeline, QueryPipeline>::value;
-
     auto & query = getSelectQuery();
     const Settings & settings = context->getSettingsRef();
 
@@ -1123,13 +996,13 @@ void InterpreterSelectQuery::executeFetchColumns(
                 argument_types[j] = header.getByName(desc->argument_names[j]).type;
 
             Block block_with_count{
-                {std::move(column), std::make_shared<DataTypeAggregateFunction>(func, argument_types, desc->parameters), desc->column_name}};
+                {nullptr, std::make_shared<DataTypeAggregateFunction>(func, argument_types, desc->parameters), desc->column_name}};
 
-            auto istream = std::make_shared<OneBlockInputStream>(block_with_count);
-            if constexpr (pipeline_with_processors)
-                pipeline.init(Pipe(std::make_shared<SourceFromInputStream>(istream)));
-            else
-                pipeline.streams.emplace_back(istream);
+            Chunk chunk(Columns(), column->size());
+            chunk.addColumn(std::move(column));
+
+            auto source = std::make_shared<SourceFromSingleChunk>(std::move(block_with_count), std::move(chunk));
+            pipeline.init(Pipe(std::move(source)));
             from_stage = QueryProcessingStage::WithMergeableState;
             analysis_result.first_stage = false;
             return;
@@ -1266,7 +1139,7 @@ void InterpreterSelectQuery::executeFetchColumns(
             /// Remove columns which will be added by prewhere.
             required_columns.erase(std::remove_if(required_columns.begin(), required_columns.end(), [&](const String & name)
             {
-                return !!required_columns_after_prewhere_set.count(name);
+                return required_columns_after_prewhere_set.count(name) != 0;
             }), required_columns.end());
 
             if (prewhere_info)
@@ -1377,11 +1250,7 @@ void InterpreterSelectQuery::executeFetchColumns(
                 interpreter_subquery->ignoreWithTotals();
         }
 
-        if constexpr (pipeline_with_processors)
-            /// Just use pipeline from subquery.
-            pipeline = interpreter_subquery->executeWithProcessors();
-        else
-            pipeline.streams = interpreter_subquery->executeWithMultipleStreams(save_context_and_storage);
+        pipeline = interpreter_subquery->execute().pipeline;
     }
     else if (storage)
     {
@@ -1411,48 +1280,9 @@ void InterpreterSelectQuery::executeFetchColumns(
             query_info.input_sorting_info = query_info.order_by_optimizer->getInputOrder(storage);
         }
 
+        Pipes pipes = storage->read(required_columns, query_info, *context, processing_stage, max_block_size, max_streams);
 
-        BlockInputStreams streams;
-        Pipes pipes;
-
-        if (pipeline_with_processors)
-            pipes = storage->read(required_columns, query_info, *context, processing_stage, max_block_size, max_streams);
-        else
-            streams = storage->readStreams(required_columns, query_info, *context, processing_stage, max_block_size, max_streams);
-
-        if (streams.empty() && !pipeline_with_processors)
-        {
-            streams = {std::make_shared<NullBlockInputStream>(storage->getSampleBlockForColumns(required_columns))};
-
-            if (query_info.prewhere_info)
-            {
-                if (query_info.prewhere_info->alias_actions)
-                {
-                    streams.back() = std::make_shared<ExpressionBlockInputStream>(
-                        streams.back(),
-                        query_info.prewhere_info->alias_actions);
-                }
-
-                streams.back() = std::make_shared<FilterBlockInputStream>(
-                    streams.back(),
-                    prewhere_info->prewhere_actions,
-                    prewhere_info->prewhere_column_name,
-                    prewhere_info->remove_prewhere_column);
-
-                // To remove additional columns
-                // In some cases, we did not read any marks so that the pipeline.streams is empty
-                // Thus, some columns in prewhere are not removed as expected
-                // This leads to mismatched header in distributed table
-                if (query_info.prewhere_info->remove_columns_actions)
-                {
-                    streams.back() = std::make_shared<ExpressionBlockInputStream>(streams.back(), query_info.prewhere_info->remove_columns_actions);
-                }
-            }
-        }
-
-        /// Copy-paste from prev if.
-        /// Code is temporarily copy-pasted while moving to new pipeline.
-        if (pipes.empty() && pipeline_with_processors)
+        if (pipes.empty())
         {
             Pipe pipe(std::make_shared<NullSource>(storage->getSampleBlockForColumns(required_columns)));
 
@@ -1468,6 +1298,10 @@ void InterpreterSelectQuery::executeFetchColumns(
                         prewhere_info->prewhere_column_name,
                         prewhere_info->remove_prewhere_column));
 
+                // To remove additional columns
+                // In some cases, we did not read any marks so that the pipeline.streams is empty
+                // Thus, some columns in prewhere are not removed as expected
+                // This leads to mismatched header in distributed table
                 if (query_info.prewhere_info->remove_columns_actions)
                     pipe.addSimpleTransform(std::make_shared<ExpressionTransform>(pipe.getHeader(), query_info.prewhere_info->remove_columns_actions));
             }
@@ -1475,14 +1309,8 @@ void InterpreterSelectQuery::executeFetchColumns(
             pipes.emplace_back(std::move(pipe));
         }
 
-        for (auto & stream : streams)
-            stream->addTableLock(table_lock);
-
-        if constexpr (pipeline_with_processors)
-        {
-            /// Table lock is stored inside pipeline here.
-            pipeline.addTableLock(table_lock);
-        }
+        /// Table lock is stored inside pipeline here.
+        pipeline.addTableLock(table_lock);
 
         /// Set the limits and quota for reading data, the speed and time of the query.
         {
@@ -1510,16 +1338,6 @@ void InterpreterSelectQuery::executeFetchColumns(
 
             auto quota = context->getQuota();
 
-            for (auto & stream : streams)
-            {
-                if (!options.ignore_limits)
-                    stream->setLimits(limits);
-
-                if (!options.ignore_quota && (options.to_stage == QueryProcessingStage::Complete))
-                    stream->setQuota(quota);
-            }
-
-            /// Copy-paste
             for (auto & pipe : pipes)
             {
                 if (!options.ignore_limits)
@@ -1530,47 +1348,13 @@ void InterpreterSelectQuery::executeFetchColumns(
             }
         }
 
-        if constexpr (pipeline_with_processors)
-        {
-            if (streams.size() == 1 || pipes.size() == 1)
-                pipeline.setMaxThreads(1);
+        if (pipes.size() == 1)
+            pipeline.setMaxThreads(1);
 
-            /// Unify streams. They must have same headers.
-            if (streams.size() > 1)
-            {
-                /// Unify streams in case they have different headers.
-                auto first_header = streams.at(0)->getHeader();
+        for (auto & pipe : pipes)
+            pipe.enableQuota();
 
-                if (first_header.columns() > 1 && first_header.has("_dummy"))
-                    first_header.erase("_dummy");
-
-                for (auto & stream : streams)
-                {
-                    auto header = stream->getHeader();
-                    auto mode = ConvertingBlockInputStream::MatchColumnsMode::Name;
-                    if (!blocksHaveEqualStructure(first_header, header))
-                        stream = std::make_shared<ConvertingBlockInputStream>(stream, first_header, mode);
-                }
-            }
-
-            for (auto & stream : streams)
-            {
-                bool force_add_agg_info = processing_stage == QueryProcessingStage::WithMergeableState;
-                auto source = std::make_shared<SourceFromInputStream>(stream, force_add_agg_info);
-
-                if (processing_stage == QueryProcessingStage::Complete)
-                    source->addTotalsPort();
-
-                pipes.emplace_back(std::move(source));
-            }
-
-            for (auto & pipe : pipes)
-                pipe.enableQuota();
-
-            pipeline.init(std::move(pipes));
-        }
-        else
-            pipeline.streams = std::move(streams);
+        pipeline.init(std::move(pipes));
     }
     else
         throw Exception("Logical error in InterpreterSelectQuery: nowhere to read", ErrorCodes::LOGICAL_ERROR);
@@ -1578,32 +1362,14 @@ void InterpreterSelectQuery::executeFetchColumns(
     /// Aliases in table declaration.
     if (processing_stage == QueryProcessingStage::FetchColumns && alias_actions)
     {
-        if constexpr (pipeline_with_processors)
+        pipeline.addSimpleTransform([&](const Block & header)
         {
-            pipeline.addSimpleTransform([&](const Block & header)
-            {
-                return std::make_shared<ExpressionTransform>(header, alias_actions);
-            });
-        }
-        else
-        {
-            pipeline.transform([&](auto & stream)
-            {
-                stream = std::make_shared<ExpressionBlockInputStream>(stream, alias_actions);
-            });
-        }
+            return std::make_shared<ExpressionTransform>(header, alias_actions);
+        });
     }
 }
 
 
-void InterpreterSelectQuery::executeWhere(Pipeline & pipeline, const ExpressionActionsPtr & expression, bool remove_filter)
-{
-    pipeline.transform([&](auto & stream)
-    {
-        stream = std::make_shared<FilterBlockInputStream>(stream, expression, getSelectQuery().where()->getColumnName(), remove_filter);
-    });
-}
-
 void InterpreterSelectQuery::executeWhere(QueryPipeline & pipeline, const ExpressionActionsPtr & expression, bool remove_filter)
 {
     pipeline.addSimpleTransform([&](const Block & block, QueryPipeline::StreamType stream_type)
@@ -1613,69 +1379,6 @@ void InterpreterSelectQuery::executeWhere(QueryPipeline & pipeline, const Expres
     });
 }
 
-void InterpreterSelectQuery::executeAggregation(Pipeline & pipeline, const ExpressionActionsPtr & expression, bool overflow_row, bool final)
-{
-    pipeline.transform([&](auto & stream)
-    {
-        stream = std::make_shared<ExpressionBlockInputStream>(stream, expression);
-    });
-
-    Block header = pipeline.firstStream()->getHeader();
-    ColumnNumbers keys;
-    for (const auto & key : query_analyzer->aggregationKeys())
-        keys.push_back(header.getPositionByName(key.name));
-
-    AggregateDescriptions aggregates = query_analyzer->aggregates();
-    for (auto & descr : aggregates)
-        if (descr.arguments.empty())
-            for (const auto & name : descr.argument_names)
-                descr.arguments.push_back(header.getPositionByName(name));
-
-    const Settings & settings = context->getSettingsRef();
-
-    /** Two-level aggregation is useful in two cases:
-      * 1. Parallel aggregation is done, and the results should be merged in parallel.
-      * 2. An aggregation is done with store of temporary data on the disk, and they need to be merged in a memory efficient way.
-      */
-    bool allow_to_use_two_level_group_by = pipeline.streams.size() > 1 || settings.max_bytes_before_external_group_by != 0;
-
-    Aggregator::Params params(header, keys, aggregates,
-        overflow_row, settings.max_rows_to_group_by, settings.group_by_overflow_mode,
-        allow_to_use_two_level_group_by ? settings.group_by_two_level_threshold : SettingUInt64(0),
-        allow_to_use_two_level_group_by ? settings.group_by_two_level_threshold_bytes : SettingUInt64(0),
-        settings.max_bytes_before_external_group_by, settings.empty_result_for_aggregation_by_empty_set,
-        context->getTemporaryVolume(), settings.max_threads, settings.min_free_disk_space_for_temporary_data);
-
-    /// If there are several sources, then we perform parallel aggregation
-    if (pipeline.streams.size() > 1)
-    {
-        pipeline.firstStream() = std::make_shared<ParallelAggregatingBlockInputStream>(
-            pipeline.streams, pipeline.stream_with_non_joined_data, params, final,
-            max_streams,
-            settings.aggregation_memory_efficient_merge_threads
-                ? static_cast<size_t>(settings.aggregation_memory_efficient_merge_threads)
-                : static_cast<size_t>(settings.max_threads));
-
-        pipeline.stream_with_non_joined_data = nullptr;
-        pipeline.streams.resize(1);
-    }
-    else
-    {
-        BlockInputStreams inputs;
-        if (!pipeline.streams.empty())
-            inputs.push_back(pipeline.firstStream());
-        else
-            pipeline.streams.resize(1);
-
-        if (pipeline.stream_with_non_joined_data)
-            inputs.push_back(pipeline.stream_with_non_joined_data);
-
-        pipeline.firstStream() = std::make_shared<AggregatingBlockInputStream>(std::make_shared<ConcatBlockInputStream>(inputs), params, final);
-
-        pipeline.stream_with_non_joined_data = nullptr;
-    }
-}
-
 
 void InterpreterSelectQuery::executeAggregation(QueryPipeline & pipeline, const ExpressionActionsPtr & expression, bool overflow_row, bool final)
 {
@@ -1749,53 +1452,6 @@ void InterpreterSelectQuery::executeAggregation(QueryPipeline & pipeline, const
 }
 
 
-void InterpreterSelectQuery::executeMergeAggregated(Pipeline & pipeline, bool overflow_row, bool final)
-{
-    Block header = pipeline.firstStream()->getHeader();
-
-    ColumnNumbers keys;
-    for (const auto & key : query_analyzer->aggregationKeys())
-        keys.push_back(header.getPositionByName(key.name));
-
-    /** There are two modes of distributed aggregation.
-      *
-      * 1. In different threads read from the remote servers blocks.
-      * Save all the blocks in the RAM. Merge blocks.
-      * If the aggregation is two-level - parallelize to the number of buckets.
-      *
-      * 2. In one thread, read blocks from different servers in order.
-      * RAM stores only one block from each server.
-      * If the aggregation is a two-level aggregation, we consistently merge the blocks of each next level.
-      *
-      * The second option consumes less memory (up to 256 times less)
-      *  in the case of two-level aggregation, which is used for large results after GROUP BY,
-      *  but it can work more slowly.
-      */
-
-    const Settings & settings = context->getSettingsRef();
-
-    Aggregator::Params params(header, keys, query_analyzer->aggregates(), overflow_row, settings.max_threads);
-
-    if (!settings.distributed_aggregation_memory_efficient)
-    {
-        /// We union several sources into one, parallelizing the work.
-        executeUnion(pipeline, {});
-
-        /// Now merge the aggregated blocks
-        pipeline.firstStream() = std::make_shared<MergingAggregatedBlockInputStream>(pipeline.firstStream(), params, final, settings.max_threads);
-    }
-    else
-    {
-        pipeline.firstStream() = std::make_shared<MergingAggregatedMemoryEfficientBlockInputStream>(pipeline.streams, params, final,
-            max_streams,
-            settings.aggregation_memory_efficient_merge_threads
-                ? static_cast<size_t>(settings.aggregation_memory_efficient_merge_threads)
-                : static_cast<size_t>(settings.max_threads));
-
-        pipeline.streams.resize(1);
-    }
-}
-
 void InterpreterSelectQuery::executeMergeAggregated(QueryPipeline & pipeline, bool overflow_row, bool final)
 {
     Block header_before_merge = pipeline.getHeader();
@@ -1856,14 +1512,6 @@ void InterpreterSelectQuery::executeMergeAggregated(QueryPipeline & pipeline, bo
 }
 
 
-void InterpreterSelectQuery::executeHaving(Pipeline & pipeline, const ExpressionActionsPtr & expression)
-{
-    pipeline.transform([&](auto & stream)
-    {
-        stream = std::make_shared<FilterBlockInputStream>(stream, expression, getSelectQuery().having()->getColumnName());
-    });
-}
-
 void InterpreterSelectQuery::executeHaving(QueryPipeline & pipeline, const ExpressionActionsPtr & expression)
 {
     pipeline.addSimpleTransform([&](const Block & header, QueryPipeline::StreamType stream_type) -> ProcessorPtr
@@ -1876,22 +1524,6 @@ void InterpreterSelectQuery::executeHaving(QueryPipeline & pipeline, const Expre
 }
 
 
-void InterpreterSelectQuery::executeTotalsAndHaving(Pipeline & pipeline, bool has_having, const ExpressionActionsPtr & expression, bool overflow_row, bool final)
-{
-    executeUnion(pipeline, {});
-
-    const Settings & settings = context->getSettingsRef();
-
-    pipeline.firstStream() = std::make_shared<TotalsHavingBlockInputStream>(
-        pipeline.firstStream(),
-        overflow_row,
-        expression,
-        has_having ? getSelectQuery().having()->getColumnName() : "",
-        settings.totals_mode,
-        settings.totals_auto_threshold,
-        final);
-}
-
 void InterpreterSelectQuery::executeTotalsAndHaving(QueryPipeline & pipeline, bool has_having, const ExpressionActionsPtr & expression, bool overflow_row, bool final)
 {
     const Settings & settings = context->getSettingsRef();
@@ -1905,31 +1537,6 @@ void InterpreterSelectQuery::executeTotalsAndHaving(QueryPipeline & pipeline, bo
 }
 
 
-void InterpreterSelectQuery::executeRollupOrCube(Pipeline & pipeline, Modificator modificator)
-{
-    executeUnion(pipeline, {});
-
-    Block header = pipeline.firstStream()->getHeader();
-
-    ColumnNumbers keys;
-
-    for (const auto & key : query_analyzer->aggregationKeys())
-        keys.push_back(header.getPositionByName(key.name));
-
-    const Settings & settings = context->getSettingsRef();
-
-    Aggregator::Params params(header, keys, query_analyzer->aggregates(),
-        false, settings.max_rows_to_group_by, settings.group_by_overflow_mode,
-        SettingUInt64(0), SettingUInt64(0),
-        settings.max_bytes_before_external_group_by, settings.empty_result_for_aggregation_by_empty_set,
-        context->getTemporaryVolume(), settings.max_threads, settings.min_free_disk_space_for_temporary_data);
-
-    if (modificator == Modificator::ROLLUP)
-        pipeline.firstStream() = std::make_shared<RollupBlockInputStream>(pipeline.firstStream(), params);
-    else
-        pipeline.firstStream() = std::make_shared<CubeBlockInputStream>(pipeline.firstStream(), params);
-}
-
 void InterpreterSelectQuery::executeRollupOrCube(QueryPipeline & pipeline, Modificator modificator)
 {
     pipeline.resize(1);
@@ -1964,14 +1571,6 @@ void InterpreterSelectQuery::executeRollupOrCube(QueryPipeline & pipeline, Modif
 }
 
 
-void InterpreterSelectQuery::executeExpression(Pipeline & pipeline, const ExpressionActionsPtr & expression)
-{
-    pipeline.transform([&](auto & stream)
-    {
-        stream = std::make_shared<ExpressionBlockInputStream>(stream, expression);
-    });
-}
-
 void InterpreterSelectQuery::executeExpression(QueryPipeline & pipeline, const ExpressionActionsPtr & expression)
 {
     pipeline.addSimpleTransform([&](const Block & header) -> ProcessorPtr
@@ -1980,68 +1579,6 @@ void InterpreterSelectQuery::executeExpression(QueryPipeline & pipeline, const E
     });
 }
 
-void InterpreterSelectQuery::executeOrder(Pipeline & pipeline, InputSortingInfoPtr input_sorting_info)
-{
-    auto & query = getSelectQuery();
-    SortDescription output_order_descr = getSortDescription(query, *context);
-    const Settings & settings = context->getSettingsRef();
-    UInt64 limit = getLimitForSorting(query, *context);
-
-    if (input_sorting_info)
-    {
-        /* Case of sorting with optimization using sorting key.
-         * We have several threads, each of them reads batch of parts in direct
-         *  or reverse order of sorting key using one input stream per part
-         *  and then merge them into one sorted stream.
-         * At this stage we merge per-thread streams into one.
-         * If the input is sorted by some prefix of the sorting key required for output,
-         * we have to finish sorting after the merge.
-         */
-
-        bool need_finish_sorting = (input_sorting_info->order_key_prefix_descr.size() < output_order_descr.size());
-
-        UInt64 limit_for_merging = (need_finish_sorting ? 0 : limit);
-        executeMergeSorted(pipeline, input_sorting_info->order_key_prefix_descr, limit_for_merging);
-
-        if (need_finish_sorting)
-        {
-            pipeline.transform([&](auto & stream)
-            {
-                stream = std::make_shared<PartialSortingBlockInputStream>(stream, output_order_descr, limit);
-            });
-
-            pipeline.firstStream() = std::make_shared<FinishSortingBlockInputStream>(
-                pipeline.firstStream(), input_sorting_info->order_key_prefix_descr,
-                output_order_descr, settings.max_block_size, limit);
-        }
-    }
-    else
-    {
-        pipeline.transform([&](auto & stream)
-        {
-            auto sorting_stream = std::make_shared<PartialSortingBlockInputStream>(stream, output_order_descr, limit);
-
-            /// Limits on sorting
-            IBlockInputStream::LocalLimits limits;
-            limits.mode = IBlockInputStream::LIMITS_TOTAL;
-            limits.size_limits = SizeLimits(settings.max_rows_to_sort, settings.max_bytes_to_sort, settings.sort_overflow_mode);
-            sorting_stream->setLimits(limits);
-
-            auto merging_stream = std::make_shared<MergeSortingBlockInputStream>(
-                sorting_stream, output_order_descr, settings.max_block_size, limit,
-                settings.max_bytes_before_remerge_sort,
-                settings.max_bytes_before_external_sort / pipeline.streams.size(),
-                context->getTemporaryVolume(),
-                settings.temporary_files_codec,
-                settings.min_free_disk_space_for_temporary_data);
-
-            stream = merging_stream;
-        });
-
-        /// If there are several streams, we merge them into one
-        executeMergeSorted(pipeline, output_order_descr, limit);
-    }
-}
 
 void InterpreterSelectQuery::executeOrder(QueryPipeline & pipeline, InputSortingInfoPtr input_sorting_info)
 {
@@ -2138,41 +1675,6 @@ void InterpreterSelectQuery::executeOrder(QueryPipeline & pipeline, InputSorting
 }
 
 
-void InterpreterSelectQuery::executeMergeSorted(Pipeline & pipeline)
-{
-    auto & query = getSelectQuery();
-    SortDescription order_descr = getSortDescription(query, *context);
-    UInt64 limit = getLimitForSorting(query, *context);
-
-    /// If there are several streams, then we merge them into one
-    if (pipeline.hasMoreThanOneStream())
-    {
-        unifyStreams(pipeline, pipeline.firstStream()->getHeader());
-        executeMergeSorted(pipeline, order_descr, limit);
-    }
-}
-
-
-void InterpreterSelectQuery::executeMergeSorted(Pipeline & pipeline, const SortDescription & sort_description, UInt64 limit)
-{
-    if (pipeline.hasMoreThanOneStream())
-    {
-        const Settings & settings = context->getSettingsRef();
-
-        /** MergingSortedBlockInputStream reads the sources sequentially.
-          * To make the data on the remote servers prepared in parallel, we wrap it in AsynchronousBlockInputStream.
-          */
-        pipeline.transform([&](auto & stream)
-        {
-            stream = std::make_shared<AsynchronousBlockInputStream>(stream);
-        });
-
-        pipeline.firstStream() = std::make_shared<MergingSortedBlockInputStream>(
-            pipeline.streams, sort_description, settings.max_block_size, limit);
-        pipeline.streams.resize(1);
-    }
-}
-
 void InterpreterSelectQuery::executeMergeSorted(QueryPipeline & pipeline)
 {
     auto & query = getSelectQuery();
@@ -2202,14 +1704,6 @@ void InterpreterSelectQuery::executeMergeSorted(QueryPipeline & pipeline, const
 }
 
 
-void InterpreterSelectQuery::executeProjection(Pipeline & pipeline, const ExpressionActionsPtr & expression)
-{
-    pipeline.transform([&](auto & stream)
-    {
-        stream = std::make_shared<ExpressionBlockInputStream>(stream, expression);
-    });
-}
-
 void InterpreterSelectQuery::executeProjection(QueryPipeline & pipeline, const ExpressionActionsPtr & expression)
 {
     pipeline.addSimpleTransform([&](const Block & header) -> ProcessorPtr
@@ -2219,28 +1713,6 @@ void InterpreterSelectQuery::executeProjection(QueryPipeline & pipeline, const E
 }
 
 
-void InterpreterSelectQuery::executeDistinct(Pipeline & pipeline, bool before_order, Names columns)
-{
-    auto & query = getSelectQuery();
-    if (query.distinct)
-    {
-        const Settings & settings = context->getSettingsRef();
-
-        auto [limit_length, limit_offset] = getLimitLengthAndOffset(query, *context);
-        UInt64 limit_for_distinct = 0;
-
-        /// If after this stage of DISTINCT ORDER BY is not executed, then you can get no more than limit_length + limit_offset of different rows.
-        if ((!query.orderBy() || !before_order) && !query.limit_with_ties)
-            limit_for_distinct = limit_length + limit_offset;
-
-        pipeline.transform([&](auto & stream)
-        {
-            SizeLimits limits(settings.max_rows_in_distinct, settings.max_bytes_in_distinct, settings.distinct_overflow_mode);
-            stream = std::make_shared<DistinctBlockInputStream>(stream, limits, limit_for_distinct, columns);
-        });
-    }
-}
-
 void InterpreterSelectQuery::executeDistinct(QueryPipeline & pipeline, bool before_order, Names columns)
 {
     auto & query = getSelectQuery();
@@ -2268,51 +1740,6 @@ void InterpreterSelectQuery::executeDistinct(QueryPipeline & pipeline, bool befo
 }
 
 
-void InterpreterSelectQuery::executeUnion(Pipeline & pipeline, Block header)
-{
-    /// If there are still several streams, then we combine them into one
-    if (pipeline.hasMoreThanOneStream())
-    {
-        if (!header)
-            header = pipeline.firstStream()->getHeader();
-
-        unifyStreams(pipeline, std::move(header));
-
-        pipeline.firstStream() = std::make_shared<UnionBlockInputStream>(pipeline.streams, pipeline.stream_with_non_joined_data, max_streams);
-        pipeline.stream_with_non_joined_data = nullptr;
-        pipeline.streams.resize(1);
-        pipeline.union_stream = true;
-    }
-    else if (pipeline.stream_with_non_joined_data)
-    {
-        pipeline.streams.push_back(pipeline.stream_with_non_joined_data);
-        pipeline.stream_with_non_joined_data = nullptr;
-    }
-}
-
-
-/// Preliminary LIMIT - is used in every source, if there are several sources, before they are combined.
-void InterpreterSelectQuery::executePreLimit(Pipeline & pipeline)
-{
-    auto & query = getSelectQuery();
-    /// If there is LIMIT
-    if (query.limitLength())
-    {
-        auto [limit_length, limit_offset] = getLimitLengthAndOffset(query, *context);
-        SortDescription sort_descr;
-        if (query.limit_with_ties)
-        {
-            if (!query.orderBy())
-                throw Exception("LIMIT WITH TIES without ORDER BY", ErrorCodes::LOGICAL_ERROR);
-            sort_descr = getSortDescription(query, *context);
-        }
-        pipeline.transform([&, limit = limit_length + limit_offset](auto & stream)
-        {
-            stream = std::make_shared<LimitBlockInputStream>(stream, limit, 0, false, false, query.limit_with_ties, sort_descr);
-        });
-    }
-}
-
 /// Preliminary LIMIT - is used in every source, if there are several sources, before they are combined.
 void InterpreterSelectQuery::executePreLimit(QueryPipeline & pipeline, bool do_not_skip_offset)
 {
@@ -2334,24 +1761,6 @@ void InterpreterSelectQuery::executePreLimit(QueryPipeline & pipeline, bool do_n
 }
 
 
-void InterpreterSelectQuery::executeLimitBy(Pipeline & pipeline)
-{
-    auto & query = getSelectQuery();
-    if (!query.limitByLength() || !query.limitBy())
-        return;
-
-    Names columns;
-    for (const auto & elem : query.limitBy()->children)
-        columns.emplace_back(elem->getColumnName());
-    UInt64 length = getLimitUIntValue(query.limitByLength(), *context, "LIMIT");
-    UInt64 offset = (query.limitByOffset() ? getLimitUIntValue(query.limitByOffset(), *context, "OFFSET") : 0);
-
-    pipeline.transform([&](auto & stream)
-    {
-        stream = std::make_shared<LimitByBlockInputStream>(stream, length, offset, columns);
-    });
-}
-
 void InterpreterSelectQuery::executeLimitBy(QueryPipeline & pipeline)
 {
     auto & query = getSelectQuery();
@@ -2400,73 +1809,6 @@ namespace
     }
 }
 
-void InterpreterSelectQuery::executeLimit(Pipeline & pipeline)
-{
-    auto & query = getSelectQuery();
-    /// If there is LIMIT
-    if (query.limitLength())
-    {
-        /** Rare case:
-          *  if there is no WITH TOTALS and there is a subquery in FROM, and there is WITH TOTALS on one of the levels,
-          *  then when using LIMIT, you should read the data to the end, rather than cancel the query earlier,
-          *  because if you cancel the query, we will not get `totals` data from the remote server.
-          *
-          * Another case:
-          *  if there is WITH TOTALS and there is no ORDER BY, then read the data to the end,
-          *  otherwise TOTALS is counted according to incomplete data.
-          */
-        bool always_read_till_end = false;
-
-        if (query.group_by_with_totals && !query.orderBy())
-            always_read_till_end = true;
-
-        if (!query.group_by_with_totals && hasWithTotalsInAnySubqueryInFromClause(query))
-            always_read_till_end = true;
-
-        SortDescription order_descr;
-        if (query.limit_with_ties)
-        {
-            if (!query.orderBy())
-                throw Exception("LIMIT WITH TIES without ORDER BY", ErrorCodes::LOGICAL_ERROR);
-            order_descr = getSortDescription(query, *context);
-        }
-
-        UInt64 limit_length;
-        UInt64 limit_offset;
-        std::tie(limit_length, limit_offset) = getLimitLengthAndOffset(query, *context);
-
-        pipeline.transform([&](auto & stream)
-        {
-            stream = std::make_shared<LimitBlockInputStream>(stream, limit_length, limit_offset, always_read_till_end, false, query.limit_with_ties, order_descr);
-        });
-    }
-}
-
-void InterpreterSelectQuery::executeOffset(Pipeline & /* pipeline */) {}
-
-void InterpreterSelectQuery::executeWithFill(Pipeline & pipeline)
-{
-    auto & query = getSelectQuery();
-    if (query.orderBy())
-    {
-        SortDescription order_descr = getSortDescription(query, *context);
-        SortDescription fill_descr;
-        for (auto & desc : order_descr)
-        {
-            if (desc.with_fill)
-                fill_descr.push_back(desc);
-        }
-
-        if (fill_descr.empty())
-            return;
-
-        pipeline.transform([&](auto & stream)
-        {
-            stream = std::make_shared<FillingBlockInputStream>(stream, fill_descr);
-        });
-    }
-}
-
 void InterpreterSelectQuery::executeWithFill(QueryPipeline & pipeline)
 {
     auto & query = getSelectQuery();
@@ -2557,18 +1899,6 @@ void InterpreterSelectQuery::executeOffset(QueryPipeline & pipeline)
     }
 }
 
-
-void InterpreterSelectQuery::executeExtremes(Pipeline & pipeline)
-{
-    if (!context->getSettingsRef().extremes)
-        return;
-
-    pipeline.transform([&](auto & stream)
-    {
-        stream->enableExtremes();
-    });
-}
-
 void InterpreterSelectQuery::executeExtremes(QueryPipeline & pipeline)
 {
     if (!context->getSettingsRef().extremes)
@@ -2577,23 +1907,6 @@ void InterpreterSelectQuery::executeExtremes(QueryPipeline & pipeline)
     pipeline.addExtremesTransform();
 }
 
-
-void InterpreterSelectQuery::executeSubqueriesInSetsAndJoins(Pipeline & pipeline, const SubqueriesForSets & subqueries_for_sets)
-{
-    /// Merge streams to one. Use MergeSorting if data was read in sorted order, Union otherwise.
-    if (query_info.input_sorting_info)
-    {
-        if (pipeline.stream_with_non_joined_data)
-            throw Exception("Using read in order optimization, but has stream with non-joined data in pipeline", ErrorCodes::LOGICAL_ERROR);
-        executeMergeSorted(pipeline, query_info.input_sorting_info->order_key_prefix_descr, 0);
-    }
-    else
-        executeUnion(pipeline, {});
-
-    pipeline.firstStream() = std::make_shared<CreatingSetsBlockInputStream>(
-        pipeline.firstStream(), subqueries_for_sets, *context);
-}
-
 void InterpreterSelectQuery::executeSubqueriesInSetsAndJoins(QueryPipeline & pipeline, const SubqueriesForSets & subqueries_for_sets)
 {
     if (query_info.input_sorting_info)
@@ -2610,25 +1923,6 @@ void InterpreterSelectQuery::executeSubqueriesInSetsAndJoins(QueryPipeline & pip
 }
 
 
-void InterpreterSelectQuery::unifyStreams(Pipeline & pipeline, Block header)
-{
-    /// Unify streams in case they have different headers.
-
-    /// TODO: remove previous addition of _dummy column.
-    if (header.columns() > 1 && header.has("_dummy"))
-        header.erase("_dummy");
-
-    for (auto & stream : pipeline.streams)
-    {
-        auto stream_header = stream->getHeader();
-        auto mode = ConvertingBlockInputStream::MatchColumnsMode::Name;
-
-        if (!blocksHaveEqualStructure(header, stream_header))
-            stream = std::make_shared<ConvertingBlockInputStream>(stream, header, mode);
-    }
-}
-
-
 void InterpreterSelectQuery::ignoreWithTotals()
 {
     getSelectQuery().group_by_with_totals = false;
diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h
index b97ff65e988..ca7fb4c72ba 100644
--- a/src/Interpreters/InterpreterSelectQuery.h
+++ b/src/Interpreters/InterpreterSelectQuery.h
@@ -77,12 +77,6 @@ public:
     /// Execute a query. Get the stream of blocks to read.
     BlockIO execute() override;
 
-    /// Execute the query and return multuple streams for parallel processing.
-    BlockInputStreams executeWithMultipleStreams(QueryPipeline & parent_pipeline);
-
-    QueryPipeline executeWithProcessors() override;
-    bool canExecuteWithProcessors() const override { return true; }
-
     bool ignoreLimits() const override { return options.ignore_limits; }
     bool ignoreQuota() const override { return options.ignore_quota; }
 
@@ -108,89 +102,15 @@ private:
 
     Block getSampleBlockImpl();
 
-    struct Pipeline
-    {
-        /** Streams of data.
-          * The source data streams are produced in the executeFetchColumns function.
-          * Then they are converted (wrapped in other streams) using the `execute*` functions,
-          *  to get the whole pipeline running the query.
-          */
-        BlockInputStreams streams;
-
-        /** When executing FULL or RIGHT JOIN, there will be a data stream from which you can read "not joined" rows.
-          * It has a special meaning, since reading from it should be done after reading from the main streams.
-          * It is appended to the main streams in UnionBlockInputStream or ParallelAggregatingBlockInputStream.
-          */
-        BlockInputStreamPtr stream_with_non_joined_data;
-        bool union_stream = false;
-
-        /// Cache value of InterpreterSelectQuery::max_streams
-        size_t max_threads = 1;
-
-        BlockInputStreamPtr & firstStream() { return streams.at(0); }
-
-        template <typename Transform>
-        void transform(Transform && transformation)
-        {
-            for (auto & stream : streams)
-                transformation(stream);
-
-            if (stream_with_non_joined_data)
-                transformation(stream_with_non_joined_data);
-        }
-
-        bool hasMoreThanOneStream() const
-        {
-            return streams.size() + (stream_with_non_joined_data ? 1 : 0) > 1;
-        }
-
-        /// Resulting stream is mix of other streams data. Distinct and/or order guaranties are broken.
-        bool hasMixedStreams() const
-        {
-            return hasMoreThanOneStream() || union_stream;
-        }
-
-        bool hasDelayedStream() const { return stream_with_non_joined_data != nullptr; }
-        bool initialized() const { return !streams.empty(); }
-
-        /// Compatibility with QueryPipeline (Processors)
-        void   setMaxThreads(size_t max_threads_) { max_threads = max_threads_; }
-        size_t getNumThreads() const { return max_threads; }
-    };
-
-    template <typename TPipeline>
-    void executeImpl(TPipeline & pipeline, const BlockInputStreamPtr & prepared_input, std::optional<Pipe> prepared_pipe, QueryPipeline & save_context_and_storage);
+    void executeImpl(QueryPipeline & pipeline, const BlockInputStreamPtr & prepared_input, std::optional<Pipe> prepared_pipe);
 
     /// Different stages of query execution.
 
-    /// dry_run - don't read from table, use empty header block instead.
-    void executeWithMultipleStreamsImpl(Pipeline & pipeline, const BlockInputStreamPtr & input, bool dry_run);
-
-    template <typename TPipeline>
-    void executeFetchColumns(QueryProcessingStage::Enum processing_stage, TPipeline & pipeline,
+    void executeFetchColumns(
+        QueryProcessingStage::Enum processing_stage,
+        QueryPipeline & pipeline,
         const PrewhereInfoPtr & prewhere_info,
-        const Names & columns_to_remove_after_prewhere,
-        QueryPipeline & save_context_and_storage);
-
-    void executeWhere(Pipeline & pipeline, const ExpressionActionsPtr & expression, bool remove_filter);
-    void executeAggregation(Pipeline & pipeline, const ExpressionActionsPtr & expression, bool overflow_row, bool final);
-    void executeMergeAggregated(Pipeline & pipeline, bool overflow_row, bool final);
-    void executeTotalsAndHaving(Pipeline & pipeline, bool has_having, const ExpressionActionsPtr & expression, bool overflow_row, bool final);
-    void executeHaving(Pipeline & pipeline, const ExpressionActionsPtr & expression);
-    static void executeExpression(Pipeline & pipeline, const ExpressionActionsPtr & expression);
-    void executeOrder(Pipeline & pipeline, InputSortingInfoPtr sorting_info);
-    void executeWithFill(Pipeline & pipeline);
-    void executeMergeSorted(Pipeline & pipeline);
-    void executePreLimit(Pipeline & pipeline);
-    void executeUnion(Pipeline & pipeline, Block header);
-    void executeLimitBy(Pipeline & pipeline);
-    void executeLimit(Pipeline & pipeline);
-    void executeOffset(Pipeline & pipeline);
-    static void executeProjection(Pipeline & pipeline, const ExpressionActionsPtr & expression);
-    void executeDistinct(Pipeline & pipeline, bool before_order, Names columns);
-    void executeExtremes(Pipeline & pipeline);
-    void executeSubqueriesInSetsAndJoins(Pipeline & pipeline, const std::unordered_map<String, SubqueryForSet> & subqueries_for_sets);
-    void executeMergeSorted(Pipeline & pipeline, const SortDescription & sort_description, UInt64 limit);
+        const Names & columns_to_remove_after_prewhere);
 
     void executeWhere(QueryPipeline & pipeline, const ExpressionActionsPtr & expression, bool remove_filter);
     void executeAggregation(QueryPipeline & pipeline, const ExpressionActionsPtr & expression, bool overflow_row, bool final);
@@ -213,17 +133,12 @@ private:
 
     String generateFilterActions(ExpressionActionsPtr & actions, const ASTPtr & row_policy_filter, const Names & prerequisite_columns = {}) const;
 
-    /// Add ConvertingBlockInputStream to specified header.
-    static void unifyStreams(Pipeline & pipeline, Block header);
-
     enum class Modificator
     {
         ROLLUP = 0,
         CUBE = 1
     };
 
-    void executeRollupOrCube(Pipeline & pipeline, Modificator modificator);
-
     void executeRollupOrCube(QueryPipeline & pipeline, Modificator modificator);
 
     /** If there is a SETTINGS section in the SELECT query, then apply settings from it.
diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp
index 7fe124b31e6..7b86616555a 100644
--- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp
+++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp
@@ -3,15 +3,9 @@
 #include <Interpreters/Context.h>
 #include <Parsers/ASTSelectWithUnionQuery.h>
 #include <Parsers/ASTSelectQuery.h>
-#include <DataStreams/UnionBlockInputStream.h>
-#include <DataStreams/NullBlockInputStream.h>
-#include <DataStreams/ConcatBlockInputStream.h>
-#include <DataStreams/ConvertingBlockInputStream.h>
 #include <Columns/getLeastSuperColumn.h>
-#include <Columns/ColumnConst.h>
 #include <Common/typeid_cast.h>
 #include <Parsers/queryToString.h>
-#include <Parsers/ASTExpressionList.h>
 
 #include <Processors/Sources/NullSource.h>
 #include <Processors/QueryPipeline.h>
@@ -180,69 +174,10 @@ Block InterpreterSelectWithUnionQuery::getSampleBlock(
 }
 
 
-BlockInputStreams InterpreterSelectWithUnionQuery::executeWithMultipleStreams(QueryPipeline & parent_pipeline)
-{
-    BlockInputStreams nested_streams;
-
-    for (auto & interpreter : nested_interpreters)
-    {
-        BlockInputStreams streams = interpreter->executeWithMultipleStreams(parent_pipeline);
-        nested_streams.insert(nested_streams.end(), streams.begin(), streams.end());
-    }
-
-    /// Unify data structure.
-    if (nested_interpreters.size() > 1)
-    {
-        for (auto & stream : nested_streams)
-            stream = std::make_shared<ConvertingBlockInputStream>(stream, result_header,ConvertingBlockInputStream::MatchColumnsMode::Position);
-        parent_pipeline.addInterpreterContext(context);
-    }
-
-    /// Update max_streams due to:
-    /// - max_distributed_connections for Distributed() engine
-    /// - max_streams_to_max_threads_ratio
-    ///
-    /// XXX: res.pipeline.getMaxThreads() cannot be used since it is capped to
-    ///      number of streams, which is empty for non-Processors case.
-    max_streams = (*std::min_element(nested_interpreters.begin(), nested_interpreters.end(), [](const auto &a, const auto &b)
-    {
-        return a->getMaxStreams() < b->getMaxStreams();
-    }))->getMaxStreams();
-
-    return nested_streams;
-}
-
-
 BlockIO InterpreterSelectWithUnionQuery::execute()
 {
     BlockIO res;
-    BlockInputStreams nested_streams = executeWithMultipleStreams(res.pipeline);
-    BlockInputStreamPtr result_stream;
-
-    if (nested_streams.empty())
-    {
-        result_stream = std::make_shared<NullBlockInputStream>(getSampleBlock());
-    }
-    else if (nested_streams.size() == 1)
-    {
-        result_stream = nested_streams.front();
-        nested_streams.clear();
-    }
-    else
-    {
-        result_stream = std::make_shared<UnionBlockInputStream>(nested_streams, nullptr, max_streams);
-        nested_streams.clear();
-    }
-
-    res.in = result_stream;
-    res.pipeline.addInterpreterContext(context);
-    return res;
-}
-
-
-QueryPipeline InterpreterSelectWithUnionQuery::executeWithProcessors()
-{
-    QueryPipeline main_pipeline;
+    QueryPipeline & main_pipeline = res.pipeline;
     std::vector<QueryPipeline> pipelines;
     bool has_main_pipeline = false;
 
@@ -254,12 +189,12 @@ QueryPipeline InterpreterSelectWithUnionQuery::executeWithProcessors()
         if (!has_main_pipeline)
         {
             has_main_pipeline = true;
-            main_pipeline = interpreter->executeWithProcessors();
+            main_pipeline = interpreter->execute().pipeline;
             headers.emplace_back(main_pipeline.getHeader());
         }
         else
         {
-            pipelines.emplace_back(interpreter->executeWithProcessors());
+            pipelines.emplace_back(interpreter->execute().pipeline);
             headers.emplace_back(pipelines.back().getHeader());
         }
     }
@@ -280,7 +215,7 @@ QueryPipeline InterpreterSelectWithUnionQuery::executeWithProcessors()
 
     main_pipeline.addInterpreterContext(context);
 
-    return main_pipeline;
+    return res;
 }
 
 
diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.h b/src/Interpreters/InterpreterSelectWithUnionQuery.h
index c7a8e09578b..3b5fe533a84 100644
--- a/src/Interpreters/InterpreterSelectWithUnionQuery.h
+++ b/src/Interpreters/InterpreterSelectWithUnionQuery.h
@@ -29,12 +29,6 @@ public:
 
     BlockIO execute() override;
 
-    /// Execute the query without union of streams.
-    BlockInputStreams executeWithMultipleStreams(QueryPipeline & parent_pipeline);
-
-    QueryPipeline executeWithProcessors() override;
-    bool canExecuteWithProcessors() const override { return true; }
-
     bool ignoreLimits() const override { return options.ignore_limits; }
     bool ignoreQuota() const override { return options.ignore_quota; }
 
diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp
index 2cc6730b90d..8fc799d0b48 100644
--- a/src/Interpreters/executeQuery.cpp
+++ b/src/Interpreters/executeQuery.cpp
@@ -280,7 +280,6 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
     /// Copy query into string. It will be written to log and presented in processlist. If an INSERT query, string will not include data to insertion.
     String query(begin, query_end);
     BlockIO res;
-    QueryPipeline & pipeline = res.pipeline;
 
     String query_for_logging;
 
@@ -338,7 +337,6 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
             context.resetInputCallbacks();
 
         auto interpreter = InterpreterFactory::get(ast, context, stage);
-        bool use_processors = interpreter->canExecuteWithProcessors();
 
         std::shared_ptr<const EnabledQuota> quota;
         if (!interpreter->ignoreQuota())
@@ -358,10 +356,9 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
             limits.size_limits = SizeLimits(settings.max_result_rows, settings.max_result_bytes, settings.result_overflow_mode);
         }
 
-        if (use_processors)
-            pipeline = interpreter->executeWithProcessors();
-        else
-            res = interpreter->execute();
+        res = interpreter->execute();
+        QueryPipeline & pipeline = res.pipeline;
+        bool use_processors = pipeline.initialized();
 
         if (res.pipeline.initialized())
             use_processors = true;
diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp
index 6564fee827e..f9666ed0548 100644
--- a/src/Storages/IStorage.cpp
+++ b/src/Storages/IStorage.cpp
@@ -397,25 +397,6 @@ void IStorage::checkAlterIsPossible(const AlterCommands & commands, const Settin
     }
 }
 
-BlockInputStreams IStorage::readStreams(
-    const Names & column_names,
-    const SelectQueryInfo & query_info,
-    const Context & context,
-    QueryProcessingStage::Enum processed_stage,
-    size_t max_block_size,
-    unsigned num_streams)
-{
-    ForceTreeShapedPipeline enable_tree_shape(query_info);
-    auto pipes = read(column_names, query_info, context, processed_stage, max_block_size, num_streams);
-
-    BlockInputStreams res;
-    res.reserve(pipes.size());
-
-    for (auto & pipe : pipes)
-        res.emplace_back(std::make_shared<TreeExecutorBlockInputStream>(std::move(pipe)));
-
-    return res;
-}
 
 StorageID IStorage::getStorageID() const
 {
diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h
index 655ced0d8cf..76a5d72262a 100644
--- a/src/Storages/IStorage.h
+++ b/src/Storages/IStorage.h
@@ -303,16 +303,6 @@ public:
         throw Exception("Method read is not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED);
     }
 
-    /** The same as read, but returns BlockInputStreams.
-     */
-    BlockInputStreams readStreams(
-            const Names & /*column_names*/,
-            const SelectQueryInfo & /*query_info*/,
-            const Context & /*context*/,
-            QueryProcessingStage::Enum /*processed_stage*/,
-            size_t /*max_block_size*/,
-            unsigned /*num_streams*/);
-
     /** Writes the data to a table.
       * Receives a description of the query, which can contain information about the data write method.
       * Returns an object by which you can write data sequentially.
diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h
index 3d7e26d3817..84cf3a32aa1 100644
--- a/src/Storages/SelectQueryInfo.h
+++ b/src/Storages/SelectQueryInfo.h
@@ -80,28 +80,6 @@ struct SelectQueryInfo
     /// Prepared sets are used for indices by storage engine.
     /// Example: x IN (1, 2, 3)
     PreparedSets sets;
-
-    /// Temporary flag is needed to support old pipeline with input streams.
-    /// If enabled, then pipeline returned by storage must be a tree.
-    /// Processors from the tree can't return ExpandPipeline status.
-    mutable bool force_tree_shaped_pipeline = false;
-};
-
-/// RAII class to enable force_tree_shaped_pipeline for SelectQueryInfo.
-/// Looks awful, but I hope it's temporary.
-struct ForceTreeShapedPipeline
-{
-    explicit ForceTreeShapedPipeline(const SelectQueryInfo & info_) : info(info_)
-    {
-        force_tree_shaped_pipeline = info.force_tree_shaped_pipeline;
-        info.force_tree_shaped_pipeline = true;
-    }
-
-    ~ForceTreeShapedPipeline() { info.force_tree_shaped_pipeline = force_tree_shaped_pipeline; }
-
-private:
-    bool force_tree_shaped_pipeline;
-    const SelectQueryInfo & info;
 };
 
 }
diff --git a/src/Storages/StorageView.cpp b/src/Storages/StorageView.cpp
index 636c7f9d64d..06c0bcacb4c 100644
--- a/src/Storages/StorageView.cpp
+++ b/src/Storages/StorageView.cpp
@@ -64,16 +64,8 @@ Pipes StorageView::read(
 
     QueryPipeline pipeline;
     InterpreterSelectWithUnionQuery interpreter(current_inner_query, context, {}, column_names);
-    /// FIXME res may implicitly use some objects owned be pipeline, but them will be destructed after return
-    if (query_info.force_tree_shaped_pipeline)
-    {
-        BlockInputStreams streams = interpreter.executeWithMultipleStreams(pipeline);
-        for (auto & stream : streams)
-            pipes.emplace_back(std::make_shared<SourceFromInputStream>(std::move(stream)));
-    }
-    else
-        /// TODO: support multiple streams here. Need more general interface than pipes.
-        pipes.emplace_back(interpreter.executeWithProcessors().getPipe());
+    /// TODO: support multiple streams here. Need more general interface than pipes.
+    pipes.emplace_back(interpreter.execute().pipeline.getPipe());
 
     /// It's expected that the columns read from storage are not constant.
     /// Because method 'getSampleBlockForColumns' is used to obtain a structure of result in InterpreterSelectQuery.

From 5572f6c27666a0bc5cb11997d472c317da8dfb68 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Thu, 28 May 2020 12:07:40 +0300
Subject: [PATCH 105/184] Fix style.

---
 src/Interpreters/IInterpreter.h | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/src/Interpreters/IInterpreter.h b/src/Interpreters/IInterpreter.h
index 63f36f1b18a..41613d30e87 100644
--- a/src/Interpreters/IInterpreter.h
+++ b/src/Interpreters/IInterpreter.h
@@ -2,14 +2,8 @@
 
 #include <DataStreams/BlockIO.h>
 
-#include <Processors/QueryPipeline.h>
-
 namespace DB
 {
-namespace ErrorCodes
-{
-    extern const int NOT_IMPLEMENTED;
-}
 
 /** Interpreters interface for different queries.
   */
@@ -25,7 +19,7 @@ public:
     virtual bool ignoreQuota() const { return false; }
     virtual bool ignoreLimits() const { return false; }
 
-    virtual ~IInterpreter() {}
+    virtual ~IInterpreter() = default;
 };
 
 }

From bc773be0e4dab18c45e67e053f82457e5c107534 Mon Sep 17 00:00:00 2001
From: Ilya <is.homutov@gmail.com>
Date: Thu, 28 May 2020 13:30:29 +0300
Subject: [PATCH 106/184] Fix package name in docker file: tzata -> tzdata

---
 docker/server/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile
index 3ad20a9479c..93f192c3f3c 100644
--- a/docker/server/Dockerfile
+++ b/docker/server/Dockerfile
@@ -21,7 +21,7 @@ RUN apt-get update \
             locales \
             ca-certificates \
             wget \
-            tzata \
+            tzdata \
     && rm -rf \
         /var/lib/apt/lists/* \
         /var/cache/debconf \

From 6efa4d1d4f3457cb5cc90832ebb9e8c90c9cab0c Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Thu, 28 May 2020 13:41:59 +0300
Subject: [PATCH 107/184] Fix style.

---
 src/Interpreters/InterpreterInsertQuery.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp
index b6efa5d6d46..300ea63af84 100644
--- a/src/Interpreters/InterpreterInsertQuery.cpp
+++ b/src/Interpreters/InterpreterInsertQuery.cpp
@@ -204,7 +204,7 @@ BlockIO InterpreterInsertQuery::execute()
         {
             /// Passing 1 as subquery_depth will disable limiting size of intermediate result.
             InterpreterSelectWithUnionQuery interpreter_select{ query.select, context, SelectQueryOptions(QueryProcessingStage::Complete, 1)};
-            res.pipeline = interpreter_select.executeWithProcessors();
+            res = interpreter_select.execute();
 
             if (table->supportsParallelInsert() && settings.max_insert_threads > 1)
                 out_streams_size = std::min(size_t(settings.max_insert_threads), res.pipeline.getNumStreams());

From 4f6b12457b8b390823765912d43fad98846e9506 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <mikhaylovnikitka@gmail.com>
Date: Thu, 28 May 2020 13:53:40 +0300
Subject: [PATCH 108/184] trigger


From da0052858dcaa1171d1e8af0635914d6a1764eb0 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Thu, 28 May 2020 13:57:04 +0300
Subject: [PATCH 109/184] Fix build.

---
 .../ClusterProxy/SelectStreamFactory.cpp      | 38 +++++--------------
 .../MergeTree/MergeTreeDataSelectExecutor.cpp |  2 +-
 src/Storages/StorageBuffer.cpp                |  2 +-
 src/Storages/StorageMerge.cpp                 | 22 +----------
 4 files changed, 13 insertions(+), 51 deletions(-)

diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp
index 53c61bea60e..4b35a982049 100644
--- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp
+++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp
@@ -15,6 +15,7 @@
 #include <Processors/Transforms/ConvertingTransform.h>
 #include <Processors/Sources/SourceFromInputStream.h>
 #include <Processors/Executors/TreeExecutorBlockInputStream.h>
+#include <Processors/Executors/PipelineExecutingBlockInputStream.h>
 
 namespace ProfileEvents
 {
@@ -70,35 +71,14 @@ SelectStreamFactory::SelectStreamFactory(
 namespace
 {
 
-Pipe createLocalStream(
-    const ASTPtr & query_ast, const Block & header, const Context & context, QueryProcessingStage::Enum processed_stage,
-    bool add_totals_port, bool add_extremes_port, bool force_tree_shaped_pipeline)
+QueryPipeline createLocalStream(
+    const ASTPtr & query_ast, const Block & header, const Context & context, QueryProcessingStage::Enum processed_stage)
 {
     checkStackSize();
 
     InterpreterSelectQuery interpreter{query_ast, context, SelectQueryOptions(processed_stage)};
 
-    if (force_tree_shaped_pipeline)
-    {
-        /// This flag means that pipeline must be tree-shaped,
-        /// so we can't enable processors for InterpreterSelectQuery here.
-        auto stream = interpreter.execute().in;
-        auto source = std::make_shared<SourceFromInputStream>(std::move(stream));
-
-        if (add_totals_port)
-            source->addTotalsPort();
-        if (add_extremes_port)
-            source->addExtremesPort();
-
-        Pipe pipe(std::move(source));
-
-        pipe.addSimpleTransform(std::make_shared<ConvertingTransform>(
-                pipe.getHeader(), header, ConvertingTransform::MatchColumnsMode::Name));
-
-        return pipe;
-    }
-
-    auto pipeline = interpreter.executeWithProcessors();
+    auto pipeline = interpreter.execute().pipeline;
 
     pipeline.addSimpleTransform([&](const Block & source_header)
     {
@@ -116,7 +96,8 @@ Pipe createLocalStream(
      */
     /// return std::make_shared<MaterializingBlockInputStream>(stream);
 
-    return std::move(pipeline).getPipe();
+    pipeline.setMaxThreads(1);
+    return pipeline;
 }
 
 String formattedAST(const ASTPtr & ast)
@@ -152,8 +133,7 @@ void SelectStreamFactory::createForShard(
 
     auto emplace_local_stream = [&]()
     {
-        res.emplace_back(createLocalStream(modified_query_ast, header, context, processed_stage,
-                                           add_totals_port, add_extremes_port, query_info.force_tree_shaped_pipeline));
+        res.emplace_back(createLocalStream(modified_query_ast, header, context, processed_stage).getPipe());
     };
 
     String modified_query = formattedAST(modified_query_ast);
@@ -297,8 +277,8 @@ void SelectStreamFactory::createForShard(
             }
 
             if (try_results.empty() || local_delay < max_remote_delay)
-                return std::make_shared<TreeExecutorBlockInputStream>(
-                        createLocalStream(modified_query_ast, header, context, stage, add_totals_port, add_extremes_port, true));
+                return std::make_shared<PipelineExecutingBlockInputStream>(
+                        createLocalStream(modified_query_ast, header, context, stage));
             else
             {
                 std::vector<IConnectionPool::Entry> connections;
diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
index f90add64732..24e3e3ac69d 100644
--- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
+++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
@@ -1151,7 +1151,7 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal(
     if (num_streams > settings.max_final_threads)
         num_streams = settings.max_final_threads;
 
-    if (num_streams <= 1 || sort_description.empty() || query_info.force_tree_shaped_pipeline)
+    if (num_streams <= 1 || sort_description.empty())
     {
 
         Pipe pipe(std::move(pipes), get_merging_processor());
diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp
index 95be1275d3d..ac423c001ac 100644
--- a/src/Storages/StorageBuffer.cpp
+++ b/src/Storages/StorageBuffer.cpp
@@ -234,7 +234,7 @@ Pipes StorageBuffer::read(
       */
     if (processed_stage > QueryProcessingStage::FetchColumns)
         for (auto & pipe : pipes_from_buffers)
-            pipe = InterpreterSelectQuery(query_info.query, context, std::move(pipe), SelectQueryOptions(processed_stage)).executeWithProcessors().getPipe();
+            pipe = InterpreterSelectQuery(query_info.query, context, std::move(pipe), SelectQueryOptions(processed_stage)).execute().pipeline.getPipe();
 
     if (query_info.prewhere_info)
     {
diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp
index fb6d88c8d33..d19f8b7b16d 100644
--- a/src/Storages/StorageMerge.cpp
+++ b/src/Storages/StorageMerge.cpp
@@ -238,20 +238,9 @@ Pipes StorageMerge::createSources(const SelectQueryInfo & query_info, const Quer
 
     if (!storage)
     {
-        if (query_info.force_tree_shaped_pipeline)
-        {
-            /// This flag means that pipeline must be tree-shaped,
-            /// so we can't enable processors for InterpreterSelectQuery here.
-            auto stream = InterpreterSelectQuery(modified_query_info.query, *modified_context, std::make_shared<OneBlockInputStream>(header),
-                                                 SelectQueryOptions(processed_stage).analyze()).execute().in;
-
-            pipes.emplace_back(std::make_shared<SourceFromInputStream>(std::move(stream)));
-            return pipes;
-        }
-
         auto pipe = InterpreterSelectQuery(modified_query_info.query, *modified_context,
                                              std::make_shared<OneBlockInputStream>(header),
-                                             SelectQueryOptions(processed_stage).analyze()).executeWithProcessors().getPipe();
+                                             SelectQueryOptions(processed_stage).analyze()).execute().pipeline.getPipe();
         pipe.addInterpreterContext(modified_context);
         pipes.emplace_back(std::move(pipe));
         return pipes;
@@ -276,15 +265,8 @@ Pipes StorageMerge::createSources(const SelectQueryInfo & query_info, const Quer
 
         InterpreterSelectQuery interpreter{modified_query_info.query, *modified_context, SelectQueryOptions(processed_stage)};
 
-        if (query_info.force_tree_shaped_pipeline)
         {
-            BlockInputStreamPtr stream = interpreter.execute().in;
-            Pipe pipe(std::make_shared<SourceFromInputStream>(std::move(stream)));
-            pipes.emplace_back(std::move(pipe));
-        }
-        else
-        {
-            Pipe pipe = interpreter.executeWithProcessors().getPipe();
+            Pipe pipe = interpreter.execute().pipeline.getPipe();
             pipes.emplace_back(std::move(pipe));
         }
 

From 2a9bcfbcd1f3acba8004ec5fe522dc695e4a0bc9 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Thu, 28 May 2020 14:00:12 +0300
Subject: [PATCH 110/184] Fix build.

---
 src/Interpreters/ClusterProxy/SelectStreamFactory.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp
index 4b35a982049..a97719427b0 100644
--- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp
+++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp
@@ -115,7 +115,7 @@ void SelectStreamFactory::createForShard(
     const Cluster::ShardInfo & shard_info,
     const String &, const ASTPtr & query_ast,
     const Context & context, const ThrottlerPtr & throttler,
-    const SelectQueryInfo & query_info,
+    const SelectQueryInfo &,
     Pipes & res)
 {
     bool force_add_agg_info = processed_stage == QueryProcessingStage::WithMergeableState;

From 557e531ff3ac333ef04444346c86b39d4e6ab7ab Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Thu, 28 May 2020 15:06:31 +0300
Subject: [PATCH 111/184] Fix tests.

---
 src/Interpreters/InterpreterSelectQuery.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp
index 3af128ffb00..eeee2df200c 100644
--- a/src/Interpreters/InterpreterSelectQuery.cpp
+++ b/src/Interpreters/InterpreterSelectQuery.cpp
@@ -996,7 +996,7 @@ void InterpreterSelectQuery::executeFetchColumns(
                 argument_types[j] = header.getByName(desc->argument_names[j]).type;
 
             Block block_with_count{
-                {nullptr, std::make_shared<DataTypeAggregateFunction>(func, argument_types, desc->parameters), desc->column_name}};
+                {ColumnPtr(column->cloneEmpty()), std::make_shared<DataTypeAggregateFunction>(func, argument_types, desc->parameters), desc->column_name}};
 
             Chunk chunk(Columns(), column->size());
             chunk.addColumn(std::move(column));

From b71d520a3f85f1d6474d5cec88b57404f03521c2 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Thu, 28 May 2020 15:30:42 +0300
Subject: [PATCH 112/184] Fix race for description in MemoryTracker.

---
 src/Common/MemoryTracker.cpp | 6 ++++--
 src/Common/MemoryTracker.h   | 6 +++---
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp
index 082b78682a2..16d166c191b 100644
--- a/src/Common/MemoryTracker.cpp
+++ b/src/Common/MemoryTracker.cpp
@@ -49,11 +49,13 @@ MemoryTracker::~MemoryTracker()
 
 void MemoryTracker::logPeakMemoryUsage() const
 {
+    const auto * description = description_ptr.load(std::memory_order_relaxed);
     LOG_DEBUG(&Logger::get("MemoryTracker"), "Peak memory usage{}: {}.", (description ? " " + std::string(description) : ""), formatReadableSizeWithBinarySuffix(peak));
 }
 
 void MemoryTracker::logMemoryUsage(Int64 current) const
 {
+    const auto * description = description_ptr.load(std::memory_order_relaxed);
     LOG_DEBUG(&Logger::get("MemoryTracker"), "Current memory usage{}: {}.", (description ? " " + std::string(description) : ""), formatReadableSizeWithBinarySuffix(current));
 }
 
@@ -85,7 +87,7 @@ void MemoryTracker::alloc(Int64 size)
 
         std::stringstream message;
         message << "Memory tracker";
-        if (description)
+        if (const auto * description = description_ptr.load(std::memory_order_relaxed))
             message << " " << description;
         message << ": fault injected. Would use " << formatReadableSizeWithBinarySuffix(will_be)
             << " (attempt to allocate chunk of " << size << " bytes)"
@@ -117,7 +119,7 @@ void MemoryTracker::alloc(Int64 size)
 
         std::stringstream message;
         message << "Memory limit";
-        if (description)
+        if (const auto * description = description_ptr.load(std::memory_order_relaxed))
             message << " " << description;
         message << " exceeded: would use " << formatReadableSizeWithBinarySuffix(will_be)
             << " (attempt to allocate chunk of " << size << " bytes)"
diff --git a/src/Common/MemoryTracker.h b/src/Common/MemoryTracker.h
index 23f32b051b2..8af683ae790 100644
--- a/src/Common/MemoryTracker.h
+++ b/src/Common/MemoryTracker.h
@@ -35,7 +35,7 @@ private:
     CurrentMetrics::Metric metric = CurrentMetrics::end();
 
     /// This description will be used as prefix into log messages (if isn't nullptr)
-    const char * description = nullptr;
+    std::atomic<const char *> description_ptr = nullptr;
 
     void updatePeak(Int64 will_be);
     void logMemoryUsage(Int64 current) const;
@@ -114,9 +114,9 @@ public:
         metric = metric_;
     }
 
-    void setDescription(const char * description_)
+    void setDescription(const char * description)
     {
-        description = description_;
+        description_ptr.store(description, std::memory_order_relaxed);
     }
 
     /// Reset the accumulated data

From f5b1ab8e4a7556644f3d2598ef108ecd635abe37 Mon Sep 17 00:00:00 2001
From: Maxim Akhmedov <max42@yandex-team.ru>
Date: Thu, 28 May 2020 16:46:56 +0300
Subject: [PATCH 113/184] Use proper versions in Arcadia build.

---
 cmake/autogenerated_versions.txt  |  9 +++++++++
 cmake/version.cmake               | 10 +---------
 cmake/yandex/ya.make.versions.inc | 22 ++++++++++++++++++++++
 src/Common/ya.make                | 15 +--------------
 utils/release/release_lib.sh      | 28 ++++++++++++++--------------
 5 files changed, 47 insertions(+), 37 deletions(-)
 create mode 100644 cmake/autogenerated_versions.txt
 create mode 100644 cmake/yandex/ya.make.versions.inc

diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt
new file mode 100644
index 00000000000..72959d3c1d8
--- /dev/null
+++ b/cmake/autogenerated_versions.txt
@@ -0,0 +1,9 @@
+# This strings autochanged from release_lib.sh:
+SET(VERSION_REVISION 54435)
+SET(VERSION_MAJOR 20)
+SET(VERSION_MINOR 5)
+SET(VERSION_PATCH 1)
+SET(VERSION_GITHASH 91df18a906dcffdbee6816e5389df6c65f86e35f)
+SET(VERSION_DESCRIBE v20.5.1.1-prestable)
+SET(VERSION_STRING 20.5.1.1)
+# end of autochange
diff --git a/cmake/version.cmake b/cmake/version.cmake
index 3f51f59cf0f..eea17f68c47 100644
--- a/cmake/version.cmake
+++ b/cmake/version.cmake
@@ -1,12 +1,4 @@
-# This strings autochanged from release_lib.sh:
-set(VERSION_REVISION 54435)
-set(VERSION_MAJOR 20)
-set(VERSION_MINOR 5)
-set(VERSION_PATCH 1)
-set(VERSION_GITHASH 91df18a906dcffdbee6816e5389df6c65f86e35f)
-set(VERSION_DESCRIBE v20.5.1.1-prestable)
-set(VERSION_STRING 20.5.1.1)
-# end of autochange
+include(${CMAKE_SOURCE_DIR}/cmake/autogenerated_versions.txt)
 
 set(VERSION_EXTRA "" CACHE STRING "")
 set(VERSION_TWEAK "" CACHE STRING "")
diff --git a/cmake/yandex/ya.make.versions.inc b/cmake/yandex/ya.make.versions.inc
new file mode 100644
index 00000000000..5c3c5eabb60
--- /dev/null
+++ b/cmake/yandex/ya.make.versions.inc
@@ -0,0 +1,22 @@
+INCLUDE(${ARCADIA_ROOT}/clickhouse/cmake/autogenerated_versions.txt)
+
+# TODO: not sure if this is customizable per-binary
+SET(VERSION_NAME "ClickHouse")
+
+# TODO: not quite sure how to replace dash with space in ya.make
+SET(VERSION_FULL "${VERSION_NAME}-${VERSION_STRING}")
+
+CFLAGS (GLOBAL -DDBMS_NAME=\"ClickHouse\")
+CFLAGS (GLOBAL -DDBMS_VERSION_MAJOR=${VERSION_MAJOR})
+CFLAGS (GLOBAL -DDBMS_VERSION_MINOR=${VERSION_MINOR})
+CFLAGS (GLOBAL -DDBMS_VERSION_PATCH=${VERSION_PATCH})
+CFLAGS (GLOBAL -DVERSION_FULL=\"\\\"${VERSION_FULL}\\\"\")
+
+# TODO: not supported yet, not sure if ya.make supports arithmetics.
+CFLAGS (GLOBAL -DVERSION_INTEGER=0)
+
+CFLAGS (GLOBAL -DVERSION_NAME=\"\\\"${VERSION_NAME}\\\"\")
+CFLAGS (GLOBAL -DVERSION_OFFICIAL=\"-arcadia\")
+CFLAGS (GLOBAL -DVERSION_REVISION=${VERSION_REVISION})
+CFLAGS (GLOBAL -DVERSION_STRING=\"\\\"${VERSION_STRING}\\\"\")
+
diff --git a/src/Common/ya.make b/src/Common/ya.make
index 99659aa2aa0..83a419212bd 100644
--- a/src/Common/ya.make
+++ b/src/Common/ya.make
@@ -18,20 +18,7 @@ PEERDIR(
     contrib/restricted/ryu
 )
 
-# TODO: stub for config_version.h
-CFLAGS (GLOBAL -DDBMS_NAME=\"ClickHouse\")
-CFLAGS (GLOBAL -DDBMS_VERSION_MAJOR=0)
-CFLAGS (GLOBAL -DDBMS_VERSION_MINOR=0)
-CFLAGS (GLOBAL -DDBMS_VERSION_PATCH=0)
-CFLAGS (GLOBAL -DVERSION_FULL=\"ClickHouse\")
-CFLAGS (GLOBAL -DVERSION_INTEGER=0)
-CFLAGS (GLOBAL -DVERSION_MAJOR=0)
-CFLAGS (GLOBAL -DVERSION_MINOR=0)
-CFLAGS (GLOBAL -DVERSION_PATCH=0)
-CFLAGS (GLOBAL -DVERSION_NAME=\"ClickHouse\")
-CFLAGS (GLOBAL -DVERSION_OFFICIAL=\"\\\(arcadia\\\)\")
-CFLAGS (GLOBAL -DVERSION_REVISION=0)
-CFLAGS (GLOBAL -DVERSION_STRING=\"Unknown\")
+INCLUDE(${ARCADIA_ROOT}/clickhouse/cmake/yandex/ya.make.versions.inc)
 
 SRCS(
     ActionLock.cpp
diff --git a/utils/release/release_lib.sh b/utils/release/release_lib.sh
index 2e700593523..569e1dfb9f2 100644
--- a/utils/release/release_lib.sh
+++ b/utils/release/release_lib.sh
@@ -12,10 +12,10 @@ function gen_version_string {
 function get_version {
     if [ -z "$VERSION_MAJOR" ] && [ -z "$VERSION_MINOR" ] && [ -z "$VERSION_PATCH" ]; then
         BASEDIR=$(dirname "${BASH_SOURCE[0]}")/../../
-        VERSION_REVISION=`grep "set(VERSION_REVISION" ${BASEDIR}/cmake/version.cmake | sed 's/^.*VERSION_REVISION \(.*\)$/\1/' | sed 's/[) ].*//'`
-        VERSION_MAJOR=`grep "set(VERSION_MAJOR" ${BASEDIR}/cmake/version.cmake | sed 's/^.*VERSION_MAJOR \(.*\)/\1/' | sed 's/[) ].*//'`
-        VERSION_MINOR=`grep "set(VERSION_MINOR" ${BASEDIR}/cmake/version.cmake | sed 's/^.*VERSION_MINOR \(.*\)/\1/' | sed 's/[) ].*//'`
-        VERSION_PATCH=`grep "set(VERSION_PATCH" ${BASEDIR}/cmake/version.cmake | sed 's/^.*VERSION_PATCH \(.*\)/\1/' | sed 's/[) ].*//'`
+        VERSION_REVISION=`grep "SET(VERSION_REVISION" ${BASEDIR}/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_REVISION \(.*\)$/\1/' | sed 's/[) ].*//'`
+        VERSION_MAJOR=`grep "SET(VERSION_MAJOR" ${BASEDIR}/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_MAJOR \(.*\)/\1/' | sed 's/[) ].*//'`
+        VERSION_MINOR=`grep "SET(VERSION_MINOR" ${BASEDIR}/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_MINOR \(.*\)/\1/' | sed 's/[) ].*//'`
+        VERSION_PATCH=`grep "SET(VERSION_PATCH" ${BASEDIR}/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_PATCH \(.*\)/\1/' | sed 's/[) ].*//'`
     fi
     VERSION_PREFIX="${VERSION_PREFIX:-v}"
     VERSION_POSTFIX_TAG="${VERSION_POSTFIX:--testing}"
@@ -90,28 +90,28 @@ function gen_revision_author {
 
             git_describe=`git describe`
             git_hash=`git rev-parse HEAD`
-            sed -i -e "s/set(VERSION_REVISION [^) ]*/set(VERSION_REVISION $VERSION_REVISION/g;" \
-                -e "s/set(VERSION_DESCRIBE [^) ]*/set(VERSION_DESCRIBE $git_describe/g;" \
-                -e "s/set(VERSION_GITHASH [^) ]*/set(VERSION_GITHASH $git_hash/g;" \
-                -e "s/set(VERSION_MAJOR [^) ]*/set(VERSION_MAJOR $VERSION_MAJOR/g;" \
-                -e "s/set(VERSION_MINOR [^) ]*/set(VERSION_MINOR $VERSION_MINOR/g;" \
-                -e "s/set(VERSION_PATCH [^) ]*/set(VERSION_PATCH $VERSION_PATCH/g;" \
-                -e "s/set(VERSION_STRING [^) ]*/set(VERSION_STRING $VERSION_STRING/g;" \
-                cmake/version.cmake
+            sed -i -e "s/SET(VERSION_REVISION [^) ]*/SET(VERSION_REVISION $VERSION_REVISION/g;" \
+                -e "s/SET(VERSION_DESCRIBE [^) ]*/SET(VERSION_DESCRIBE $git_describe/g;" \
+                -e "s/SET(VERSION_GITHASH [^) ]*/SET(VERSION_GITHASH $git_hash/g;" \
+                -e "s/SET(VERSION_MAJOR [^) ]*/SET(VERSION_MAJOR $VERSION_MAJOR/g;" \
+                -e "s/SET(VERSION_MINOR [^) ]*/SET(VERSION_MINOR $VERSION_MINOR/g;" \
+                -e "s/SET(VERSION_PATCH [^) ]*/SET(VERSION_PATCH $VERSION_PATCH/g;" \
+                -e "s/SET(VERSION_STRING [^) ]*/SET(VERSION_STRING $VERSION_STRING/g;" \
+                cmake/autogenerated_versions.txt
 
             gen_changelog "$VERSION_STRING" "" "$AUTHOR" ""
             gen_dockerfiles "$VERSION_STRING"
             src/Storages/System/StorageSystemContributors.sh ||:
             utils/list-versions/list-versions.sh > utils/list-versions/version_date.tsv
 
-            git commit -m "$auto_message [$VERSION_STRING] [$VERSION_REVISION]" cmake/version.cmake debian/changelog docker/*/Dockerfile src/Storages/System/StorageSystemContributors.generated.cpp utils/list-versions/version_date.tsv
+            git commit -m "$auto_message [$VERSION_STRING] [$VERSION_REVISION]" cmake/autogenerated_versions.txt debian/changelog docker/*/Dockerfile src/Storages/System/StorageSystemContributors.generated.cpp utils/list-versions/version_date.tsv
             if [ -z $NO_PUSH ]; then
                 git push
             fi
 
             echo "Generated version: ${VERSION_STRING}, revision: ${VERSION_REVISION}."
 
-            # Second tag for correct version information in version.cmake inside tag
+            # Second tag for correct version information in autogenerated_versions.txt inside tag
             if git tag --force -a "$tag" -m "$tag"
             then
                 if [ -z $NO_PUSH ]; then

From 030aa6eac4301d9671afac9bfbaa5de61080946c Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Thu, 28 May 2020 16:57:01 +0300
Subject: [PATCH 114/184] Fix build.

---
 src/Interpreters/ClusterProxy/SelectStreamFactory.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp
index a97719427b0..f8269c789ba 100644
--- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp
+++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp
@@ -246,7 +246,7 @@ void SelectStreamFactory::createForShard(
         auto lazily_create_stream = [
                 pool = shard_info.pool, shard_num = shard_info.shard_num, modified_query, header = header, modified_query_ast, context, throttler,
                 main_table = main_table, table_func_ptr = table_func_ptr, scalars = scalars, external_tables = external_tables,
-                stage = processed_stage, local_delay, add_totals_port, add_extremes_port]()
+                stage = processed_stage, local_delay]()
             -> BlockInputStreamPtr
         {
             auto current_settings = context.getSettingsRef();

From 28cc6d9c6b7628a86c4ca0eee46ed41f364ddcb9 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Thu, 28 May 2020 17:03:32 +0300
Subject: [PATCH 115/184] Try fix tests.

---
 src/Interpreters/MutationsInterpreter.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp
index 6962885534c..224c3cab02b 100644
--- a/src/Interpreters/MutationsInterpreter.cpp
+++ b/src/Interpreters/MutationsInterpreter.cpp
@@ -184,7 +184,7 @@ bool isStorageTouchedByMutations(
     /// For some reason it may copy context and and give it into ExpressionBlockInputStream
     /// after that we will use context from destroyed stack frame in our stream.
     InterpreterSelectQuery interpreter(select_query, context_copy, storage, SelectQueryOptions().ignoreLimits());
-    BlockInputStreamPtr in = interpreter.execute().in;
+    BlockInputStreamPtr in = interpreter.execute().getInputStream();
 
     Block block = in->read();
     if (!block.rows())
@@ -687,7 +687,7 @@ void MutationsInterpreter::validate(TableStructureReadLockHolder &)
     }
 
     /// Do not use getSampleBlock in order to check the whole pipeline.
-    Block first_stage_header = select_interpreter->execute().in->getHeader();
+    Block first_stage_header = select_interpreter->execute().getInputStream()->getHeader();
     BlockInputStreamPtr in = std::make_shared<NullBlockInputStream>(first_stage_header);
     addStreamsForLaterStages(stages, in)->getHeader();
 }
@@ -697,7 +697,7 @@ BlockInputStreamPtr MutationsInterpreter::execute(TableStructureReadLockHolder &
     if (!can_execute)
         throw Exception("Cannot execute mutations interpreter because can_execute flag set to false", ErrorCodes::LOGICAL_ERROR);
 
-    BlockInputStreamPtr in = select_interpreter->execute().in;
+    BlockInputStreamPtr in = select_interpreter->execute().getInputStream();
 
     auto result_stream = addStreamsForLaterStages(stages, in);
 

From ea3f638b30a3c6199a9f11e5a53dfcca327c8253 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Thu, 28 May 2020 17:11:36 +0300
Subject: [PATCH 116/184] Try fix tests.

---
 src/DataStreams/PushingToViewsBlockOutputStream.cpp |  2 +-
 src/Interpreters/ActionsVisitor.cpp                 |  2 +-
 src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp |  6 +++---
 src/Interpreters/ExpressionAnalyzer.cpp             | 10 +++++-----
 src/Interpreters/GlobalSubqueriesVisitor.h          |  2 +-
 src/Interpreters/SubqueryForSet.cpp                 |  2 +-
 src/Storages/LiveView/StorageLiveView.cpp           |  6 +++---
 7 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/src/DataStreams/PushingToViewsBlockOutputStream.cpp b/src/DataStreams/PushingToViewsBlockOutputStream.cpp
index 7f730b5fd3f..414be847cdd 100644
--- a/src/DataStreams/PushingToViewsBlockOutputStream.cpp
+++ b/src/DataStreams/PushingToViewsBlockOutputStream.cpp
@@ -274,7 +274,7 @@ void PushingToViewsBlockOutputStream::process(const Block & block, size_t view_n
                 StorageValues::create(
                     storage->getStorageID(), storage->getColumns(), block, storage->getVirtuals()));
             select.emplace(view.query, local_context, SelectQueryOptions());
-            in = std::make_shared<MaterializingBlockInputStream>(select->execute().in);
+            in = std::make_shared<MaterializingBlockInputStream>(select->execute().getInputStream());
 
             /// Squashing is needed here because the materialized view query can generate a lot of blocks
             /// even when only one block is inserted into the parent table (e.g. if the query is a GROUP BY
diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp
index 7c2133e629f..cd5934b32b2 100644
--- a/src/Interpreters/ActionsVisitor.cpp
+++ b/src/Interpreters/ActionsVisitor.cpp
@@ -706,7 +706,7 @@ SetPtr ActionsMatcher::makeSet(const ASTFunction & node, Data & data, bool no_su
         {
             auto interpreter = interpretSubquery(right_in_operand, data.context, data.subquery_depth, {});
             subquery_for_set.source = std::make_shared<LazyBlockInputStream>(
-                interpreter->getSampleBlock(), [interpreter]() mutable { return interpreter->execute().in; });
+                interpreter->getSampleBlock(), [interpreter]() mutable { return interpreter->execute().getInputStream(); });
 
             /** Why is LazyBlockInputStream used?
               *
diff --git a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp
index 0e9be3c7a96..e2884d99516 100644
--- a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp
+++ b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp
@@ -111,11 +111,11 @@ void ExecuteScalarSubqueriesMatcher::visit(const ASTSubquery & subquery, ASTPtr
         }
         else
         {
-            BlockIO res = interpreter.execute();
+            auto stream = interpreter.execute().getInputStream();
 
             try
             {
-                block = res.in->read();
+                block = stream->read();
 
                 if (!block)
                 {
@@ -126,7 +126,7 @@ void ExecuteScalarSubqueriesMatcher::visit(const ASTSubquery & subquery, ASTPtr
                     return;
                 }
 
-                if (block.rows() != 1 || res.in->read())
+                if (block.rows() != 1 || stream->read())
                     throw Exception("Scalar subquery returned more than one row", ErrorCodes::INCORRECT_RESULT_OF_SCALAR_SUBQUERY);
             }
             catch (const Exception & e)
diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp
index b38af6feef9..27d6daf7292 100644
--- a/src/Interpreters/ExpressionAnalyzer.cpp
+++ b/src/Interpreters/ExpressionAnalyzer.cpp
@@ -297,13 +297,13 @@ void SelectQueryExpressionAnalyzer::tryMakeSetForIndexFromSubquery(const ASTPtr
     }
 
     auto interpreter_subquery = interpretSubquery(subquery_or_table_name, context, {}, query_options);
-    BlockIO res = interpreter_subquery->execute();
+    auto stream = interpreter_subquery->execute().getInputStream();
 
     SetPtr set = std::make_shared<Set>(settings.size_limits_for_set, true, context.getSettingsRef().transform_null_in);
-    set->setHeader(res.in->getHeader());
+    set->setHeader(stream->getHeader());
 
-    res.in->readPrefix();
-    while (Block block = res.in->read())
+    stream->readPrefix();
+    while (Block block = stream->read())
     {
         /// If the limits have been exceeded, give up and let the default subquery processing actions take place.
         if (!set->insertFromBlock(block))
@@ -311,7 +311,7 @@ void SelectQueryExpressionAnalyzer::tryMakeSetForIndexFromSubquery(const ASTPtr
     }
 
     set->finishInsert();
-    res.in->readSuffix();
+    stream->readSuffix();
 
     prepared_sets[set_key] = std::move(set);
 }
diff --git a/src/Interpreters/GlobalSubqueriesVisitor.h b/src/Interpreters/GlobalSubqueriesVisitor.h
index 37a358c3d28..65dd3cf2667 100644
--- a/src/Interpreters/GlobalSubqueriesVisitor.h
+++ b/src/Interpreters/GlobalSubqueriesVisitor.h
@@ -134,7 +134,7 @@ public:
                 ast = database_and_table_name;
 
             external_tables[external_table_name] = external_storage_holder;
-            subqueries_for_sets[external_table_name].source = interpreter->execute().in;
+            subqueries_for_sets[external_table_name].source = interpreter->execute().getInputStream();
             subqueries_for_sets[external_table_name].table = external_storage;
 
             /** NOTE If it was written IN tmp_table - the existing temporary (but not external) table,
diff --git a/src/Interpreters/SubqueryForSet.cpp b/src/Interpreters/SubqueryForSet.cpp
index 72831970de7..ac5c1e3d9eb 100644
--- a/src/Interpreters/SubqueryForSet.cpp
+++ b/src/Interpreters/SubqueryForSet.cpp
@@ -13,7 +13,7 @@ void SubqueryForSet::makeSource(std::shared_ptr<InterpreterSelectWithUnionQuery>
 {
     joined_block_aliases = std::move(joined_block_aliases_);
     source = std::make_shared<LazyBlockInputStream>(interpreter->getSampleBlock(),
-                                                    [interpreter]() mutable { return interpreter->execute().in; });
+                                                    [interpreter]() mutable { return interpreter->execute().getInputStream(); });
 
     sample_block = source->getHeader();
     renameColumns(sample_block);
diff --git a/src/Storages/LiveView/StorageLiveView.cpp b/src/Storages/LiveView/StorageLiveView.cpp
index cd660407c89..c1372e2c8c4 100644
--- a/src/Storages/LiveView/StorageLiveView.cpp
+++ b/src/Storages/LiveView/StorageLiveView.cpp
@@ -111,7 +111,7 @@ MergeableBlocksPtr StorageLiveView::collectMergeableBlocks(const Context & conte
 
     InterpreterSelectQuery interpreter(mergeable_query->clone(), context, SelectQueryOptions(QueryProcessingStage::WithMergeableState), Names());
 
-    auto view_mergeable_stream = std::make_shared<MaterializingBlockInputStream>(interpreter.execute().in);
+    auto view_mergeable_stream = std::make_shared<MaterializingBlockInputStream>(interpreter.execute().getInputStream());
 
     while (Block this_block = view_mergeable_stream->read())
         base_blocks->push_back(this_block);
@@ -148,7 +148,7 @@ BlockInputStreamPtr StorageLiveView::completeQuery(Pipes pipes)
     block_context->addExternalTable(getBlocksTableName(), TemporaryTableHolder(global_context, creator));
 
     InterpreterSelectQuery select(getInnerBlocksQuery(), *block_context, StoragePtr(), SelectQueryOptions(QueryProcessingStage::Complete));
-    BlockInputStreamPtr data = std::make_shared<MaterializingBlockInputStream>(select.execute().in);
+    BlockInputStreamPtr data = std::make_shared<MaterializingBlockInputStream>(select.execute().getInputStream());
 
     /// Squashing is needed here because the view query can generate a lot of blocks
     /// even when only one block is inserted into the parent table (e.g. if the query is a GROUP BY
@@ -218,7 +218,7 @@ void StorageLiveView::writeIntoLiveView(
             QueryProcessingStage::WithMergeableState);
 
         auto data_mergeable_stream = std::make_shared<MaterializingBlockInputStream>(
-            select_block.execute().in);
+            select_block.execute().getInputStream());
 
         while (Block this_block = data_mergeable_stream->read())
             new_mergeable_blocks->push_back(this_block);

From 7853722519cf594a6a940e61151689fea967b706 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Thu, 28 May 2020 17:16:56 +0300
Subject: [PATCH 117/184] Update tests

---
 .../queries/0_stateless/00342_escape_sequences.reference  | 1 -
 tests/queries/0_stateless/00926_multimatch.sql            | 2 +-
 .../0_stateless/01075_allowed_client_hosts.reference      | 8 ++++----
 3 files changed, 5 insertions(+), 6 deletions(-)
 delete mode 100644 tests/queries/0_stateless/00342_escape_sequences.reference

diff --git a/tests/queries/0_stateless/00342_escape_sequences.reference b/tests/queries/0_stateless/00342_escape_sequences.reference
deleted file mode 100644
index b7ecd06be78..00000000000
--- a/tests/queries/0_stateless/00342_escape_sequences.reference
+++ /dev/null
@@ -1 +0,0 @@
-07080C0A0D090B5C27223FAA
diff --git a/tests/queries/0_stateless/00926_multimatch.sql b/tests/queries/0_stateless/00926_multimatch.sql
index d54e4fd2280..bf67533bb6c 100644
--- a/tests/queries/0_stateless/00926_multimatch.sql
+++ b/tests/queries/0_stateless/00926_multimatch.sql
@@ -89,4 +89,4 @@ SELECT [1, 2, 3, 11] = arraySort(multiMatchAllIndices('фабрикант', ['',
 SELECT [1] = multiMatchAllIndices(materialize('/odezhda-dlya-bega/'), ['/odezhda-dlya-bega/', 'kurtki-i-vetrovki-dlya-bega', 'futbolki-i-mayki-dlya-bega']);
 SELECT [] = multiMatchAllIndices(materialize('aaaa'), ['.*aa.*aaa.*', 'aaaaaa{2}', '\(aa\){3}']);
 SELECT 'All tests above must return 1, all tests below return something.';
-SELECT arraySort(multiMatchAllIndices(arrayJoin(['aaaa', 'aaaaaa', 'bbbb', 'aaaaaaaaaaaaaa']), ['.*aa.*aaa.*', 'aaaaaa{2}', '\(aa\){3}']));
+SELECT arraySort(multiMatchAllIndices(arrayJoin(['aaaa', 'aaaaaa', 'bbbb', 'aaaaaaaaaaaaaa']), ['.*aa.*aaa.*', 'aaaaaa{2}', '(aa){3}']));
diff --git a/tests/queries/0_stateless/01075_allowed_client_hosts.reference b/tests/queries/0_stateless/01075_allowed_client_hosts.reference
index 73f54c6027a..3fdea9d1cda 100644
--- a/tests/queries/0_stateless/01075_allowed_client_hosts.reference
+++ b/tests/queries/0_stateless/01075_allowed_client_hosts.reference
@@ -8,10 +8,10 @@ CREATE USER test_user_01075 HOST LOCAL, IP \'2001:db8:11a3:9d7:1f34:8a2e:7a0:765
 CREATE USER test_user_01075 HOST LOCAL
 CREATE USER test_user_01075 HOST NONE
 CREATE USER test_user_01075 HOST LIKE \'@.somesite.com\'
-CREATE USER test_user_01075 HOST REGEXP \'.*.anothersite.com\'
-CREATE USER test_user_01075 HOST REGEXP \'.*.anothersite.com\', \'.*.anothersite.org\'
-CREATE USER test_user_01075 HOST REGEXP \'.*.anothersite2.com\', \'.*.anothersite2.org\'
-CREATE USER test_user_01075 HOST REGEXP \'.*.anothersite3.com\', \'.*.anothersite3.org\'
+CREATE USER test_user_01075 HOST REGEXP \'.*\\\\.anothersite\\\\.com\'
+CREATE USER test_user_01075 HOST REGEXP \'.*\\\\.anothersite\\\\.com\', \'.*\\\\.anothersite\\\\.org\'
+CREATE USER test_user_01075 HOST REGEXP \'.*\\\\.anothersite2\\\\.com\', \'.*\\\\.anothersite2\\\\.org\'
+CREATE USER test_user_01075 HOST REGEXP \'.*\\\\.anothersite3\\\\.com\', \'.*\\\\.anothersite3\\\\.org\'
 CREATE USER `test_user_01075_x@localhost` HOST LOCAL
 CREATE USER test_user_01075_x
 CREATE USER `test_user_01075_x@192.168.23.15` HOST LIKE \'192.168.23.15\'

From c6f82590ad8a0a4cd56fd7458748d30af1a18fea Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Thu, 28 May 2020 18:34:33 +0300
Subject: [PATCH 118/184] Add missed files

---
 src/Storages/TTLDescription.cpp | 197 ++++++++++++++++++++++++++++++++
 src/Storages/TTLDescription.h   |  69 +++++++++++
 2 files changed, 266 insertions(+)
 create mode 100644 src/Storages/TTLDescription.cpp
 create mode 100644 src/Storages/TTLDescription.h

diff --git a/src/Storages/TTLDescription.cpp b/src/Storages/TTLDescription.cpp
new file mode 100644
index 00000000000..0efe6076174
--- /dev/null
+++ b/src/Storages/TTLDescription.cpp
@@ -0,0 +1,197 @@
+#include <Storages/TTLDescription.h>
+
+#include <Functions/IFunction.h>
+#include <Interpreters/ExpressionAnalyzer.h>
+#include <Interpreters/SyntaxAnalyzer.h>
+#include <Parsers/ASTExpressionList.h>
+#include <Parsers/ASTFunction.h>
+#include <Parsers/ASTTTLElement.h>
+#include <Parsers/ASTIdentifier.h>
+#include <Storages/ColumnsDescription.h>
+
+
+#include <DataTypes/DataTypeDate.h>
+#include <DataTypes/DataTypeDateTime.h>
+
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+extern const int BAD_ARGUMENTS;
+extern const int BAD_TTL_EXPRESSION;
+}
+
+namespace
+{
+
+void checkTTLExpression(const ExpressionActionsPtr & ttl_expression, const String & result_column_name)
+{
+    for (const auto & action : ttl_expression->getActions())
+    {
+        if (action.type == ExpressionAction::APPLY_FUNCTION)
+        {
+            IFunctionBase & func = *action.function_base;
+            if (!func.isDeterministic())
+                throw Exception(
+                    "TTL expression cannot contain non-deterministic functions, "
+                    "but contains function "
+                        + func.getName(),
+                    ErrorCodes::BAD_ARGUMENTS);
+        }
+    }
+
+    const auto & result_column = ttl_expression->getSampleBlock().getByName(result_column_name);
+
+    if (!typeid_cast<const DataTypeDateTime *>(result_column.type.get())
+        && !typeid_cast<const DataTypeDate *>(result_column.type.get()))
+    {
+        throw Exception(
+            "TTL expression result column should have DateTime or Date type, but has " + result_column.type->getName(),
+            ErrorCodes::BAD_TTL_EXPRESSION);
+    }
+}
+
+}
+
+TTLDescription TTLDescription::getTTLFromAST(
+    const ASTPtr & definition_ast,
+    const ColumnsDescription & columns,
+    const Context & context,
+    const StorageMetadataKeyField & primary_key)
+{
+    TTLDescription result;
+    const auto * ttl_element = definition_ast->as<ASTTTLElement>();
+
+    /// First child is expression: `TTL expr TO DISK`
+    if (ttl_element != nullptr)
+        result.expression_ast = ttl_element->children.front()->clone();
+    else /// It's columns TTL without any additions, just copy it
+        result.expression_ast = definition_ast->clone();
+
+    auto ttl_ast = result.expression_ast->clone();
+    auto syntax_analyzer_result = SyntaxAnalyzer(context).analyze(ttl_ast, columns.getAllPhysical());
+    result.expression = ExpressionAnalyzer(ttl_ast, syntax_analyzer_result, context).getActions(false);
+    result.result_column = ttl_ast->getColumnName();
+
+    if (ttl_element == nullptr) /// columns TTL
+    {
+        result.destination_type = DataDestinationType::DELETE;
+        result.mode = TTLMode::DELETE;
+    }
+    else /// rows TTL
+    {
+        result.destination_type = ttl_element->destination_type;
+        result.destination_name = ttl_element->destination_name;
+        result.mode = ttl_element->mode;
+
+        if (ttl_element->mode == TTLMode::DELETE)
+        {
+            if (ASTPtr where_expr_ast = ttl_element->where())
+            {
+                auto where_syntax_result = SyntaxAnalyzer(context).analyze(where_expr_ast, columns.getAllPhysical());
+                result.where_expression = ExpressionAnalyzer(where_expr_ast, where_syntax_result, context).getActions(false);
+                result.where_result_column = where_expr_ast->getColumnName();
+            }
+        }
+        else if (ttl_element->mode == TTLMode::GROUP_BY)
+        {
+            const auto & pk_columns = primary_key.column_names;
+
+            if (ttl_element->group_by_key.size() > pk_columns.size())
+                throw Exception("TTL Expression GROUP BY key should be a prefix of primary key", ErrorCodes::BAD_TTL_EXPRESSION);
+
+            NameSet primary_key_columns_set(pk_columns.begin(), pk_columns.end());
+            NameSet aggregation_columns_set;
+
+            for (const auto & column : primary_key.expression->getRequiredColumns())
+                primary_key_columns_set.insert(column);
+
+            for (size_t i = 0; i < ttl_element->group_by_key.size(); ++i)
+            {
+                if (ttl_element->group_by_key[i]->getColumnName() != pk_columns[i])
+                    throw Exception(
+                        "TTL Expression GROUP BY key should be a prefix of primary key",
+                        ErrorCodes::BAD_TTL_EXPRESSION);
+            }
+
+            for (const auto & [name, value] : ttl_element->group_by_aggregations)
+            {
+                if (primary_key_columns_set.count(name))
+                    throw Exception(
+                        "Can not set custom aggregation for column in primary key in TTL Expression",
+                        ErrorCodes::BAD_TTL_EXPRESSION);
+
+                aggregation_columns_set.insert(name);
+            }
+
+            if (aggregation_columns_set.size() != ttl_element->group_by_aggregations.size())
+                throw Exception(
+                    "Multiple aggregations set for one column in TTL Expression",
+                    ErrorCodes::BAD_TTL_EXPRESSION);
+
+
+            result.group_by_keys = Names(pk_columns.begin(), pk_columns.begin() + ttl_element->group_by_key.size());
+
+            auto aggregations = ttl_element->group_by_aggregations;
+
+            for (size_t i = 0; i < pk_columns.size(); ++i)
+            {
+                ASTPtr value = primary_key.expression_list_ast->children[i]->clone();
+
+                if (i >= ttl_element->group_by_key.size())
+                {
+                    ASTPtr value_max = makeASTFunction("max", value->clone());
+                    aggregations.emplace_back(value->getColumnName(), std::move(value_max));
+                }
+
+                if (value->as<ASTFunction>())
+                {
+                    auto syntax_result = SyntaxAnalyzer(context).analyze(value, columns.getAllPhysical(), {}, true);
+                    auto expr_actions = ExpressionAnalyzer(value, syntax_result, context).getActions(false);
+                    for (const auto & column : expr_actions->getRequiredColumns())
+                    {
+                        if (i < ttl_element->group_by_key.size())
+                        {
+                            ASTPtr expr = makeASTFunction("any", std::make_shared<ASTIdentifier>(column));
+                            aggregations.emplace_back(column, std::move(expr));
+                        }
+                        else
+                        {
+                            ASTPtr expr = makeASTFunction("argMax", std::make_shared<ASTIdentifier>(column), value->clone());
+                            aggregations.emplace_back(column, std::move(expr));
+                        }
+                    }
+                }
+            }
+
+            for (const auto & column : columns.getAllPhysical())
+            {
+                if (!primary_key_columns_set.count(column.name) && !aggregation_columns_set.count(column.name))
+                {
+                    ASTPtr expr = makeASTFunction("any", std::make_shared<ASTIdentifier>(column.name));
+                    aggregations.emplace_back(column.name, std::move(expr));
+                }
+            }
+
+            for (auto [name, value] : aggregations)
+            {
+                auto syntax_result = SyntaxAnalyzer(context).analyze(value, columns.getAllPhysical(), {}, true);
+                auto expr_analyzer = ExpressionAnalyzer(value, syntax_result, context);
+
+                result.group_by_aggregations.emplace_back(name, value->getColumnName(), expr_analyzer.getActions(false));
+
+                for (const auto & descr : expr_analyzer.getAnalyzedData().aggregate_descriptions)
+                    result.aggregate_descriptions.push_back(descr);
+            }
+        }
+    }
+
+    checkTTLExpression(result.expression, result.result_column);
+
+
+    return result;
+}
+
+}
diff --git a/src/Storages/TTLDescription.h b/src/Storages/TTLDescription.h
new file mode 100644
index 00000000000..59829dbecfd
--- /dev/null
+++ b/src/Storages/TTLDescription.h
@@ -0,0 +1,69 @@
+#pragma once
+#include <Parsers/IAST_fwd.h>
+#include <Storages/DataDestinationType.h>
+#include <Storages/ColumnsDescription.h>
+#include <Interpreters/ExpressionActions.h>
+#include <Interpreters/AggregateDescription.h>
+#include <Storages/StorageInMemoryMetadata.h>
+#include <Storages/TTLMode.h>
+
+namespace DB
+{
+
+/// Common struct for TTL record in storage
+struct TTLDescription
+{
+    TTLMode mode;
+
+    /// Expression part of TTL AST:
+    /// TTL d + INTERVAL 1 DAY
+    ///    ^~~~~expression~~~~^
+    ASTPtr expression_ast;
+
+    /// Expresion actions evaluated from AST
+    ExpressionActionsPtr expression;
+
+    /// Result column of this TTL expression
+    String result_column;
+
+    ExpressionActionsPtr where_expression;
+
+    String where_result_column;
+
+    Names group_by_keys;
+
+    std::vector<std::tuple<String, String, ExpressionActionsPtr>> group_by_aggregations;
+    AggregateDescriptions aggregate_descriptions;
+
+    /// Destination type, only valid for table TTLs.
+    /// For example DISK or VOLUME
+    DataDestinationType destination_type;
+
+    /// Name of destination disk or volume
+    String destination_name;
+
+    /// Parse TTL structure from definition. Able to parse both column and table
+    /// TTLs.
+    static TTLDescription getTTLFromAST(const ASTPtr & definition_ast, const ColumnsDescription & columns, const Context & context, const StorageMetadataKeyField & primary_key);
+};
+
+/// Mapping from column name to column TTL
+using TTLColumnsDescription = std::unordered_map<String, TTLDescription>;
+using TTLDescriptions = std::vector<TTLDescription>;
+
+/// Common TTL for all table. Specified after defining the table columns.
+struct TTLTableDescription
+{
+    /// Definition. Include all parts of TTL:
+    /// TTL d + INTERVAL 1 day TO VOLUME 'disk1'
+    /// ^~~~~~~~~~~~~~~definition~~~~~~~~~~~~~~~^
+    ASTPtr definition_ast;
+
+    /// Rows removing TTL
+    TTLDescription rows_ttl;
+
+    /// Moving data TTL (to other disks or volumes)
+    TTLDescriptions move_ttl;
+};
+
+}

From a452d037efe71a7b9d200ad42c06267279176d99 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitbar@yandex-team.ru>
Date: Thu, 28 May 2020 18:35:35 +0300
Subject: [PATCH 119/184] Fix handling bad data while reading in Protobuf
 format.

---
 src/Formats/ProtobufReader.cpp | 29 ++++++++++++++++++++---------
 src/Formats/ProtobufReader.h   | 10 +++++-----
 2 files changed, 25 insertions(+), 14 deletions(-)

diff --git a/src/Formats/ProtobufReader.cpp b/src/Formats/ProtobufReader.cpp
index d8302771002..d76b048c1e9 100644
--- a/src/Formats/ProtobufReader.cpp
+++ b/src/Formats/ProtobufReader.cpp
@@ -32,11 +32,12 @@ namespace
         BITS32 = 5,
     };
 
-    // The following condition must always be true:
-    // any_cursor_position < min(END_OF_VARINT, END_OF_GROUP)
-    // This inequation helps to check conditions in SimpleReader.
-    constexpr UInt64 END_OF_VARINT = static_cast<UInt64>(-1);
-    constexpr UInt64 END_OF_GROUP = static_cast<UInt64>(-2);
+    // The following conditions must always be true:
+    // any_cursor_position > END_OF_VARINT
+    // any_cursor_position > END_OF_GROUP
+    // Those inequations helps checking conditions in ProtobufReader::SimpleReader.
+    constexpr Int64 END_OF_VARINT = -1;
+    constexpr Int64 END_OF_GROUP = -2;
 
     Int64 decodeZigZag(UInt64 n) { return static_cast<Int64>((n >> 1) ^ (~(n & 1) + 1)); }
 
@@ -77,7 +78,7 @@ void ProtobufReader::SimpleReader::endMessage(bool ignore_errors)
     if (!current_message_level)
         return;
 
-    UInt64 root_message_end = (current_message_level == 1) ? current_message_end : parent_message_ends.front();
+    Int64 root_message_end = (current_message_level == 1) ? current_message_end : parent_message_ends.front();
     if (cursor != root_message_end)
     {
         if (cursor < root_message_end)
@@ -95,6 +96,9 @@ void ProtobufReader::SimpleReader::endMessage(bool ignore_errors)
 void ProtobufReader::SimpleReader::startNestedMessage()
 {
     assert(current_message_level >= 1);
+    if ((cursor > field_end) && (field_end != END_OF_GROUP))
+        throwUnknownFormat();
+
     // Start reading a nested message which is located inside a length-delimited field
     // of another message.
     parent_message_ends.emplace_back(current_message_end);
@@ -146,7 +150,7 @@ bool ProtobufReader::SimpleReader::readFieldNumber(UInt32 & field_number)
             throwUnknownFormat();
     }
 
-    if (cursor >= current_message_end)
+    if ((cursor >= current_message_end) && (current_message_end != END_OF_GROUP))
         return false;
 
     UInt64 varint = readVarint();
@@ -196,11 +200,17 @@ bool ProtobufReader::SimpleReader::readFieldNumber(UInt32 & field_number)
 
 bool ProtobufReader::SimpleReader::readUInt(UInt64 & value)
 {
+    if (field_end == END_OF_VARINT)
+    {
+        value = readVarint();
+        field_end = cursor;
+        return true;
+    }
+
     if (unlikely(cursor >= field_end))
         return false;
+
     value = readVarint();
-    if (field_end == END_OF_VARINT)
-        field_end = cursor;
     return true;
 }
 
@@ -227,6 +237,7 @@ bool ProtobufReader::SimpleReader::readFixed(T & value)
 {
     if (unlikely(cursor >= field_end))
         return false;
+
     readBinary(&value, sizeof(T));
     return true;
 }
diff --git a/src/Formats/ProtobufReader.h b/src/Formats/ProtobufReader.h
index a50c2f202f0..7c3c19ba376 100644
--- a/src/Formats/ProtobufReader.h
+++ b/src/Formats/ProtobufReader.h
@@ -124,12 +124,12 @@ private:
         void ignoreGroup();
 
         ReadBuffer & in;
-        UInt64 cursor;
+        Int64 cursor;
         size_t current_message_level;
-        UInt64 current_message_end;
-        std::vector<UInt64> parent_message_ends;
-        UInt64 field_end;
-        UInt64 last_string_pos;
+        Int64 current_message_end;
+        std::vector<Int64> parent_message_ends;
+        Int64 field_end;
+        Int64 last_string_pos;
     };
 
     class IConverter

From 8e63fd34b91c608feccc9ea2f803043deb73d72e Mon Sep 17 00:00:00 2001
From: Maxim Akhmedov <max42@yandex-team.ru>
Date: Thu, 28 May 2020 18:59:19 +0300
Subject: [PATCH 120/184] Bump build.


From 534e55f5b54f19be03cc8338a832d91bdd968c02 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Thu, 28 May 2020 19:01:12 +0300
Subject: [PATCH 121/184] try fix tests.

---
 src/Interpreters/InterpreterSelectQuery.cpp | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp
index eeee2df200c..9b161bc1883 100644
--- a/src/Interpreters/InterpreterSelectQuery.cpp
+++ b/src/Interpreters/InterpreterSelectQuery.cpp
@@ -996,13 +996,10 @@ void InterpreterSelectQuery::executeFetchColumns(
                 argument_types[j] = header.getByName(desc->argument_names[j]).type;
 
             Block block_with_count{
-                {ColumnPtr(column->cloneEmpty()), std::make_shared<DataTypeAggregateFunction>(func, argument_types, desc->parameters), desc->column_name}};
+                {std::move(column), std::make_shared<DataTypeAggregateFunction>(func, argument_types, desc->parameters), desc->column_name}};
 
-            Chunk chunk(Columns(), column->size());
-            chunk.addColumn(std::move(column));
-
-            auto source = std::make_shared<SourceFromSingleChunk>(std::move(block_with_count), std::move(chunk));
-            pipeline.init(Pipe(std::move(source)));
+            auto istream = std::make_shared<OneBlockInputStream>(block_with_count);
+            pipeline.init(Pipe(std::make_shared<SourceFromInputStream>(istream)));
             from_stage = QueryProcessingStage::WithMergeableState;
             analysis_result.first_stage = false;
             return;

From acc3e6b15a82c1ca93eb7e2061c00ae5a5ca72a8 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Thu, 28 May 2020 19:31:27 +0300
Subject: [PATCH 122/184] Better structure

---
 src/DataStreams/TTLBlockInputStream.cpp | 10 ++++----
 src/Storages/TTLDescription.cpp         |  3 ++-
 src/Storages/TTLDescription.h           | 32 +++++++++++++++++++++++--
 src/Storages/ya.make                    |  1 +
 4 files changed, 38 insertions(+), 8 deletions(-)

diff --git a/src/DataStreams/TTLBlockInputStream.cpp b/src/DataStreams/TTLBlockInputStream.cpp
index 87cab76ce8c..c6542763533 100644
--- a/src/DataStreams/TTLBlockInputStream.cpp
+++ b/src/DataStreams/TTLBlockInputStream.cpp
@@ -277,18 +277,18 @@ void TTLBlockInputStream::finalizeAggregates(MutableColumns & result_columns)
         auto aggregated_res = aggregator->convertToBlocks(agg_result, true, 1);
         for (auto & agg_block : aggregated_res)
         {
-            for (const auto & it : storage.getRowsTTL().group_by_aggregations)
-                std::get<2>(it)->execute(agg_block);
+            for (const auto & it : storage.getRowsTTL().set_parts)
+                it.expression->execute(agg_block);
             for (const auto & name : storage.getRowsTTL().group_by_keys)
             {
                 const IColumn * values_column = agg_block.getByName(name).column.get();
                 auto & result_column = result_columns[header.getPositionByName(name)];
                 result_column->insertRangeFrom(*values_column, 0, agg_block.rows());
             }
-            for (const auto & it : storage.getRowsTTL().group_by_aggregations)
+            for (const auto & it : storage.getRowsTTL().set_parts)
             {
-                const IColumn * values_column = agg_block.getByName(get<1>(it)).column.get();
-                auto & result_column = result_columns[header.getPositionByName(std::get<0>(it))];
+                const IColumn * values_column = agg_block.getByName(it.expression_result_column_name).column.get();
+                auto & result_column = result_columns[header.getPositionByName(it.column_name)];
                 result_column->insertRangeFrom(*values_column, 0, agg_block.rows());
             }
         }
diff --git a/src/Storages/TTLDescription.cpp b/src/Storages/TTLDescription.cpp
index 0efe6076174..92fcf400dd3 100644
--- a/src/Storages/TTLDescription.cpp
+++ b/src/Storages/TTLDescription.cpp
@@ -180,7 +180,8 @@ TTLDescription TTLDescription::getTTLFromAST(
                 auto syntax_result = SyntaxAnalyzer(context).analyze(value, columns.getAllPhysical(), {}, true);
                 auto expr_analyzer = ExpressionAnalyzer(value, syntax_result, context);
 
-                result.group_by_aggregations.emplace_back(name, value->getColumnName(), expr_analyzer.getActions(false));
+                result.set_parts.emplace_back(TTLSetPartDescription{
+                    name, value->getColumnName(), expr_analyzer.getActions(false)});
 
                 for (const auto & descr : expr_analyzer.getAnalyzedData().aggregate_descriptions)
                     result.aggregate_descriptions.push_back(descr);
diff --git a/src/Storages/TTLDescription.h b/src/Storages/TTLDescription.h
index 59829dbecfd..d0e669ef4cf 100644
--- a/src/Storages/TTLDescription.h
+++ b/src/Storages/TTLDescription.h
@@ -10,6 +10,24 @@
 namespace DB
 {
 
+struct TTLSetPartDescription
+{
+    /// Name of column in set part of ttl expression
+    /// x = sum(y)
+    /// ^
+    String column_name;
+
+    /// Name of column on the right hand of the set part of TTL expression
+    /// x = sum(y)
+    ///    ^~~~~~^
+    String expression_result_column_name;
+
+    /// Expressions to calculate the value of set expression
+    ExpressionActionsPtr expression;
+};
+
+using TTLSetPartDescriptions = std::vector<TTLSetPartDescription>;
+
 /// Common struct for TTL record in storage
 struct TTLDescription
 {
@@ -17,7 +35,7 @@ struct TTLDescription
 
     /// Expression part of TTL AST:
     /// TTL d + INTERVAL 1 DAY
-    ///    ^~~~~expression~~~~^
+    ///    ^~~~~~~~~~~~~~~~~~~^
     ASTPtr expression_ast;
 
     /// Expresion actions evaluated from AST
@@ -26,13 +44,23 @@ struct TTLDescription
     /// Result column of this TTL expression
     String result_column;
 
+    /// WHERE part in TTL expression
+    /// TTL ... WHERE x % 10 == 0 and y > 5
+    ///              ^~~~~~~~~~~~~~~~~~~~~~^
     ExpressionActionsPtr where_expression;
 
+    /// Name of result column from WHERE expression
     String where_result_column;
 
+    /// Names of key columns in GROUP BY expression
+    /// TTL ... GROUP BY toDate(d), x SET ...
+    ///                  ^~~~~~~~~~~~^
     Names group_by_keys;
 
-    std::vector<std::tuple<String, String, ExpressionActionsPtr>> group_by_aggregations;
+    /// SET parts of TTL expression
+    TTLSetPartDescriptions set_parts;
+
+    /// Aggregate descriptions for GROUP BY in TTL
     AggregateDescriptions aggregate_descriptions;
 
     /// Destination type, only valid for table TTLs.
diff --git a/src/Storages/ya.make b/src/Storages/ya.make
index ffa3924d11a..3e6943bac69 100644
--- a/src/Storages/ya.make
+++ b/src/Storages/ya.make
@@ -173,6 +173,7 @@ SRCS(
     StorageXDBC.cpp
     transformQueryForExternalDatabase.cpp
     VirtualColumnUtils.cpp
+    TTLDescriptions.cpp
 )
 
 END()

From c5fe87e123dca8a5fdcaace0f349209e51c9c11b Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Thu, 28 May 2020 19:32:35 +0300
Subject: [PATCH 123/184] Less includes

---
 src/Storages/StorageInMemoryMetadata.cpp | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/src/Storages/StorageInMemoryMetadata.cpp b/src/Storages/StorageInMemoryMetadata.cpp
index 0111c0632c8..6713519151f 100644
--- a/src/Storages/StorageInMemoryMetadata.cpp
+++ b/src/Storages/StorageInMemoryMetadata.cpp
@@ -5,22 +5,11 @@
 #include <Interpreters/SyntaxAnalyzer.h>
 #include <Parsers/ASTExpressionList.h>
 #include <Parsers/ASTFunction.h>
-#include <Parsers/queryToString.h>
-#include <Functions/IFunction.h>
-
-#include <DataTypes/DataTypeDateTime.h>
-#include <DataTypes/DataTypeDate.h>
 
 namespace DB
 {
 
 
-namespace ErrorCodes
-{
-    extern const int BAD_TTL_EXPRESSION;
-    extern const int BAD_ARGUMENTS;
-};
-
 StorageInMemoryMetadata::StorageInMemoryMetadata(
     const ColumnsDescription & columns_,
     const IndicesDescription & indices_,

From ce52a2c820b60d8a44476b42d261248e809c7c62 Mon Sep 17 00:00:00 2001
From: Maxim Akhmedov <max42@yandex-team.ru>
Date: Thu, 28 May 2020 19:44:30 +0300
Subject: [PATCH 124/184] Add lost VERSION_* defines to ya.make.versions.inc

---
 cmake/yandex/ya.make.versions.inc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/cmake/yandex/ya.make.versions.inc b/cmake/yandex/ya.make.versions.inc
index 5c3c5eabb60..3ac401cb108 100644
--- a/cmake/yandex/ya.make.versions.inc
+++ b/cmake/yandex/ya.make.versions.inc
@@ -11,6 +11,9 @@ CFLAGS (GLOBAL -DDBMS_VERSION_MAJOR=${VERSION_MAJOR})
 CFLAGS (GLOBAL -DDBMS_VERSION_MINOR=${VERSION_MINOR})
 CFLAGS (GLOBAL -DDBMS_VERSION_PATCH=${VERSION_PATCH})
 CFLAGS (GLOBAL -DVERSION_FULL=\"\\\"${VERSION_FULL}\\\"\")
+CFLAGS (GLOBAL -DVERSION_MAJOR=${VERSION_MAJOR})	
+CFLAGS (GLOBAL -DVERSION_MINOR=${VERSION_MINOR})	
+CFLAGS (GLOBAL -DVERSION_PATCH=${VERSION_PATCH})
 
 # TODO: not supported yet, not sure if ya.make supports arithmetics.
 CFLAGS (GLOBAL -DVERSION_INTEGER=0)

From 99626a5bd3fa3ba0470142793ce93faec1a0efc5 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitbar@yandex-team.ru>
Date: Thu, 28 May 2020 20:07:47 +0300
Subject: [PATCH 125/184] Add test for handling malformed protobuf data.

---
 .../0_stateless/00825_protobuf_format_input.reference    | 1 +
 tests/queries/0_stateless/00825_protobuf_format_input.sh | 9 ++++++++-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/00825_protobuf_format_input.reference b/tests/queries/0_stateless/00825_protobuf_format_input.reference
index 884cc74c4e5..0c56bc4ebf0 100644
--- a/tests/queries/0_stateless/00825_protobuf_format_input.reference
+++ b/tests/queries/0_stateless/00825_protobuf_format_input.reference
@@ -8,3 +8,4 @@ a7522158-3d41-4b77-ad69-6c598ee55c49	Ivan	Petrov	male	1980-12-29	png	+7495123456
 0	0
 2	4
 3	9
+ok
diff --git a/tests/queries/0_stateless/00825_protobuf_format_input.sh b/tests/queries/0_stateless/00825_protobuf_format_input.sh
index d28b70bb002..1c915bc3f24 100755
--- a/tests/queries/0_stateless/00825_protobuf_format_input.sh
+++ b/tests/queries/0_stateless/00825_protobuf_format_input.sh
@@ -3,7 +3,7 @@
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . $CURDIR/../shell_config.sh
 
-set -e -o pipefail
+set -eo pipefail
 
 # Run the client.
 $CLICKHOUSE_CLIENT --multiquery <<'EOF'
@@ -48,5 +48,12 @@ source $CURDIR/00825_protobuf_format_input.insh
 $CLICKHOUSE_CLIENT --query "SELECT * FROM in_persons_00825 ORDER BY uuid;"
 $CLICKHOUSE_CLIENT --query "SELECT * FROM in_squares_00825 ORDER BY number;"
 
+# Try to input malformed data.
+set +eo pipefail
+echo -ne '\xe0\x80\x3f\x0b' \
+    | $CLICKHOUSE_CLIENT --query="INSERT INTO in_persons_00825 FORMAT Protobuf SETTINGS format_schema = '$CURDIR/00825_protobuf_format:Person'" 2>&1 \
+    | grep -qF "Protobuf messages are corrupted" && echo "ok" || echo "fail"
+set -eo pipefail
+
 $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS in_persons_00825;"
 $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS in_squares_00825;"

From eb5a8dccfa3c090bbb45ccff787aa9383e1341a9 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Thu, 28 May 2020 20:32:22 +0300
Subject: [PATCH 126/184] Fix ya.make

---
 src/Storages/ya.make | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Storages/ya.make b/src/Storages/ya.make
index 3e6943bac69..8a36fad696f 100644
--- a/src/Storages/ya.make
+++ b/src/Storages/ya.make
@@ -173,7 +173,7 @@ SRCS(
     StorageXDBC.cpp
     transformQueryForExternalDatabase.cpp
     VirtualColumnUtils.cpp
-    TTLDescriptions.cpp
+    TTLDescription.cpp
 )
 
 END()

From 16de65a6f0c1111d8e8cf8402d0d937552c111d1 Mon Sep 17 00:00:00 2001
From: Mikhail Filimonov <mfilimonov@altinity.com>
Date: Thu, 28 May 2020 19:52:59 +0200
Subject: [PATCH 127/184] Add prefix & facility to librdkafka logs

---
 src/Storages/Kafka/StorageKafka.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp
index 7731cf3c06a..16c424eb7be 100644
--- a/src/Storages/Kafka/StorageKafka.cpp
+++ b/src/Storages/Kafka/StorageKafka.cpp
@@ -324,10 +324,10 @@ void StorageKafka::updateConfiguration(cppkafka::Configuration & conf)
     }
 
     // No need to add any prefix, messages can be distinguished
-    conf.set_log_callback([this](cppkafka::KafkaHandleBase &, int level, const std::string & /* facility */, const std::string & message)
+    conf.set_log_callback([this](cppkafka::KafkaHandleBase &, int level, const std::string & facility, const std::string & message)
     {
         auto [poco_level, client_logs_level] = parseSyslogLevel(level);
-        LOG_IMPL(log, client_logs_level, poco_level, message);
+        LOG_IMPL(log, client_logs_level, poco_level, "[rdk:{}] {}", facility, message);
     });
 
     // Configure interceptor to change thread name

From 87b0ea26d38f89df2bdac938c2ecb7f91bdd2177 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Thu, 28 May 2020 20:59:09 +0300
Subject: [PATCH 128/184] Try fix tests.

---
 src/DataStreams/BlocksBlockInputStream.h | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/DataStreams/BlocksBlockInputStream.h b/src/DataStreams/BlocksBlockInputStream.h
index 6301a92b6a4..3062a24a44c 100644
--- a/src/DataStreams/BlocksBlockInputStream.h
+++ b/src/DataStreams/BlocksBlockInputStream.h
@@ -13,6 +13,7 @@ limitations under the License. */
 
 #include <DataStreams/IBlockInputStream.h>
 #include <Processors/Sources/SourceWithProgress.h>
+#include <Processors/Transforms/AggregatingTransform.h>
 
 
 namespace DB
@@ -38,7 +39,12 @@ protected:
 
         Block res = *it;
         ++it;
-        return Chunk(res.getColumns(), res.rows());
+
+        auto info = std::make_shared<AggregatedChunkInfo>();
+        info->bucket_num = res.info.bucket_num;
+        info->is_overflows = res.info.is_overflows;
+
+        return Chunk(res.getColumns(), res.rows(), std::move(info));
     }
 
 private:

From a95469020adf75ca6e07e2294113539588a81989 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Thu, 28 May 2020 21:02:41 +0300
Subject: [PATCH 129/184] Better error codes

---
 src/Common/ErrorCodes.cpp                       | 1 +
 src/DataStreams/CheckSortedBlockInputStream.cpp | 6 +++---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp
index cb4c591041c..5898e956400 100644
--- a/src/Common/ErrorCodes.cpp
+++ b/src/Common/ErrorCodes.cpp
@@ -498,6 +498,7 @@ namespace ErrorCodes
     extern const int ALTER_OF_COLUMN_IS_FORBIDDEN = 524;
     extern const int INCORRECT_DISK_INDEX = 525;
     extern const int UNKNOWN_VOLUME_TYPE = 526;
+    extern const int SORT_ORDER_VIOLATED = 527;
 
     extern const int KEEPER_EXCEPTION = 999;
     extern const int POCO_EXCEPTION = 1000;
diff --git a/src/DataStreams/CheckSortedBlockInputStream.cpp b/src/DataStreams/CheckSortedBlockInputStream.cpp
index a56738a9a9f..db4794b917f 100644
--- a/src/DataStreams/CheckSortedBlockInputStream.cpp
+++ b/src/DataStreams/CheckSortedBlockInputStream.cpp
@@ -6,7 +6,7 @@ namespace DB
 
 namespace ErrorCodes
 {
-extern const int LOGICAL_ERROR;
+extern const int SORT_ORDER_VIOLATED;
 }
 
 CheckSortedBlockInputStream::CheckSortedBlockInputStream(
@@ -72,12 +72,12 @@ Block CheckSortedBlockInputStream::readImpl()
 
     auto block_columns = block.getColumns();
     if (!last_row.empty() && !less(last_row, 0, block_columns, 0))
-        throw Exception("Sort order of blocks violated", ErrorCodes::LOGICAL_ERROR);
+        throw Exception("Sort order of blocks violated", ErrorCodes::SORT_ORDER_VIOLATED);
 
     size_t rows = block.rows();
     for (size_t i = 1; i < rows; ++i)
         if (!less(block_columns, i - 1, block_columns, i))
-            throw Exception("Sort order of blocks violated", ErrorCodes::LOGICAL_ERROR);
+            throw Exception("Sort order of blocks violated", ErrorCodes::SORT_ORDER_VIOLATED);
 
     last_row.clear();
     for (size_t i = 0; i < block.columns(); ++i)

From 93801a1a9dcfc0c8f801d465ce9611b30ace5976 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Thu, 28 May 2020 21:52:08 +0300
Subject: [PATCH 130/184] Fix unit tests

---
 src/Common/tests/gtest_global_context.h       |  9 +++---
 src/Storages/tests/gtest_storage_log.cpp      |  6 ++--
 ..._transform_query_for_external_database.cpp | 28 +++++++++----------
 3 files changed, 22 insertions(+), 21 deletions(-)

diff --git a/src/Common/tests/gtest_global_context.h b/src/Common/tests/gtest_global_context.h
index 05f60e01774..959d51c6e0b 100644
--- a/src/Common/tests/gtest_global_context.h
+++ b/src/Common/tests/gtest_global_context.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <Interpreters/Context.h>
+#include <Functions/registerFunctions.h>
 
 struct ContextHolder
 {
@@ -11,15 +12,15 @@ struct ContextHolder
         : shared_context(DB::Context::createShared())
         , context(DB::Context::createGlobal(shared_context.get()))
     {
+        context.makeGlobalContext();
+        context.setPath("./");
     }
 
     ContextHolder(ContextHolder &&) = default;
 };
 
-inline ContextHolder getContext()
+inline const ContextHolder & getContext()
 {
-    ContextHolder holder;
-    holder.context.makeGlobalContext();
-    holder.context.setPath("./");
+    static ContextHolder holder;
     return holder;
 }
diff --git a/src/Storages/tests/gtest_storage_log.cpp b/src/Storages/tests/gtest_storage_log.cpp
index 203104801aa..fff352210e7 100644
--- a/src/Storages/tests/gtest_storage_log.cpp
+++ b/src/Storages/tests/gtest_storage_log.cpp
@@ -68,7 +68,7 @@ using DiskImplementations = testing::Types<DB::DiskMemory, DB::DiskLocal>;
 TYPED_TEST_SUITE(StorageLogTest, DiskImplementations);
 
 // Returns data written to table in Values format.
-std::string writeData(int rows, DB::StoragePtr & table, DB::Context & context)
+std::string writeData(int rows, DB::StoragePtr & table, const DB::Context & context)
 {
     using namespace DB;
 
@@ -104,7 +104,7 @@ std::string writeData(int rows, DB::StoragePtr & table, DB::Context & context)
 }
 
 // Returns all table data in Values format.
-std::string readData(DB::StoragePtr & table, DB::Context & context)
+std::string readData(DB::StoragePtr & table, const DB::Context & context)
 {
     using namespace DB;
 
@@ -136,7 +136,7 @@ std::string readData(DB::StoragePtr & table, DB::Context & context)
 TYPED_TEST(StorageLogTest, testReadWrite)
 {
     using namespace DB;
-    auto context_holder = getContext();
+    const auto & context_holder = getContext();
 
     std::string data;
 
diff --git a/src/Storages/tests/gtest_transform_query_for_external_database.cpp b/src/Storages/tests/gtest_transform_query_for_external_database.cpp
index df6ee2bd0a5..f51e38d32a9 100644
--- a/src/Storages/tests/gtest_transform_query_for_external_database.cpp
+++ b/src/Storages/tests/gtest_transform_query_for_external_database.cpp
@@ -8,7 +8,6 @@
 #include <Interpreters/Context.h>
 #include <Databases/DatabaseMemory.h>
 #include <Storages/StorageMemory.h>
-#include <Functions/registerFunctions.h>
 #include <Common/tests/gtest_global_context.h>
 
 
@@ -18,7 +17,7 @@ using namespace DB;
 /// NOTE How to do better?
 struct State
 {
-    Context & context;
+    Context context;
     NamesAndTypesList columns{
         {"column", std::make_shared<DataTypeUInt8>()},
         {"apply_id", std::make_shared<DataTypeUInt64>()},
@@ -27,9 +26,10 @@ struct State
         {"create_time", std::make_shared<DataTypeDateTime>()},
     };
 
-    explicit State(Context & context_) : context(context_)
+    explicit State()
+        : context(getContext().context)
     {
-        registerFunctions();
+        DB::registerFunctions();
         DatabasePtr database = std::make_shared<DatabaseMemory>("test");
         database->attachTable("table", StorageMemory::create(StorageID("test", "table"), ColumnsDescription{columns}, ConstraintsDescription{}));
         context.makeGlobalContext();
@@ -38,6 +38,11 @@ struct State
     }
 };
 
+State getState()
+{
+    static State state;
+    return state;
+}
 
 static void check(const std::string & query, const std::string & expected, const Context & context, const NamesAndTypesList & columns)
 {
@@ -54,8 +59,7 @@ static void check(const std::string & query, const std::string & expected, const
 
 TEST(TransformQueryForExternalDatabase, InWithSingleElement)
 {
-    auto context_holder = getContext();
-    State state(context_holder.context);
+    const State & state = getState();
 
     check("SELECT column FROM test.table WHERE 1 IN (1)",
           R"(SELECT "column" FROM "test"."table" WHERE 1)",
@@ -70,8 +74,7 @@ TEST(TransformQueryForExternalDatabase, InWithSingleElement)
 
 TEST(TransformQueryForExternalDatabase, Like)
 {
-    auto context_holder = getContext();
-    State state(context_holder.context);
+    const State & state = getState();
 
     check("SELECT column FROM test.table WHERE column LIKE '%hello%'",
           R"(SELECT "column" FROM "test"."table" WHERE "column" LIKE '%hello%')",
@@ -83,8 +86,7 @@ TEST(TransformQueryForExternalDatabase, Like)
 
 TEST(TransformQueryForExternalDatabase, Substring)
 {
-    auto context_holder = getContext();
-    State state(context_holder.context);
+    const State & state = getState();
 
     check("SELECT column FROM test.table WHERE left(column, 10) = RIGHT(column, 10) AND SUBSTRING(column FROM 1 FOR 2) = 'Hello'",
           R"(SELECT "column" FROM "test"."table")",
@@ -93,8 +95,7 @@ TEST(TransformQueryForExternalDatabase, Substring)
 
 TEST(TransformQueryForExternalDatabase, MultipleAndSubqueries)
 {
-    auto context_holder = getContext();
-    State state(context_holder.context);
+    const State & state = getState();
 
     check("SELECT column FROM test.table WHERE 1 = 1 AND toString(column) = '42' AND column = 42 AND left(column, 10) = RIGHT(column, 10) AND column IN (1, 42) AND SUBSTRING(column FROM 1 FOR 2) = 'Hello' AND column != 4",
           R"(SELECT "column" FROM "test"."table" WHERE 1 AND ("column" = 42) AND ("column" IN (1, 42)) AND ("column" != 4))",
@@ -106,8 +107,7 @@ TEST(TransformQueryForExternalDatabase, MultipleAndSubqueries)
 
 TEST(TransformQueryForExternalDatabase, Issue7245)
 {
-    auto context_holder = getContext();
-    State state(context_holder.context);
+    const State & state = getState();
 
     check("select apply_id from test.table where apply_type = 2 and create_time > addDays(toDateTime('2019-01-01 01:02:03'),-7) and apply_status in (3,4)",
           R"(SELECT "apply_id", "apply_type", "apply_status", "create_time" FROM "test"."table" WHERE ("apply_type" = 2) AND ("create_time" > '2018-12-25 01:02:03') AND ("apply_status" IN (3, 4)))",

From e9c1e1a7e4913fafb1164b8633b79bf76a4fd48a Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Thu, 28 May 2020 21:54:42 +0300
Subject: [PATCH 131/184] Remove redundant header

---
 src/Common/tests/gtest_global_context.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/Common/tests/gtest_global_context.h b/src/Common/tests/gtest_global_context.h
index 959d51c6e0b..b6529f09b46 100644
--- a/src/Common/tests/gtest_global_context.h
+++ b/src/Common/tests/gtest_global_context.h
@@ -1,7 +1,6 @@
 #pragma once
 
 #include <Interpreters/Context.h>
-#include <Functions/registerFunctions.h>
 
 struct ContextHolder
 {

From 026244b07f796a5cd36c80ea20f3484b629b68ee Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Thu, 28 May 2020 21:57:09 +0300
Subject: [PATCH 132/184] Better

---
 .../tests/gtest_transform_query_for_external_database.cpp      | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/Storages/tests/gtest_transform_query_for_external_database.cpp b/src/Storages/tests/gtest_transform_query_for_external_database.cpp
index f51e38d32a9..6caf759da42 100644
--- a/src/Storages/tests/gtest_transform_query_for_external_database.cpp
+++ b/src/Storages/tests/gtest_transform_query_for_external_database.cpp
@@ -8,6 +8,7 @@
 #include <Interpreters/Context.h>
 #include <Databases/DatabaseMemory.h>
 #include <Storages/StorageMemory.h>
+#include <Functions/registerFunctions.h>
 #include <Common/tests/gtest_global_context.h>
 
 
@@ -29,7 +30,7 @@ struct State
     explicit State()
         : context(getContext().context)
     {
-        DB::registerFunctions();
+        registerFunctions();
         DatabasePtr database = std::make_shared<DatabaseMemory>("test");
         database->attachTable("table", StorageMemory::create(StorageID("test", "table"), ColumnsDescription{columns}, ConstraintsDescription{}));
         context.makeGlobalContext();

From db72f3959489ef54261071390e20703dea5caf51 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Thu, 28 May 2020 22:07:20 +0300
Subject: [PATCH 133/184] Remove error code

---
 src/Common/ErrorCodes.cpp                       | 1 -
 src/DataStreams/CheckSortedBlockInputStream.cpp | 6 +++---
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp
index 5898e956400..cb4c591041c 100644
--- a/src/Common/ErrorCodes.cpp
+++ b/src/Common/ErrorCodes.cpp
@@ -498,7 +498,6 @@ namespace ErrorCodes
     extern const int ALTER_OF_COLUMN_IS_FORBIDDEN = 524;
     extern const int INCORRECT_DISK_INDEX = 525;
     extern const int UNKNOWN_VOLUME_TYPE = 526;
-    extern const int SORT_ORDER_VIOLATED = 527;
 
     extern const int KEEPER_EXCEPTION = 999;
     extern const int POCO_EXCEPTION = 1000;
diff --git a/src/DataStreams/CheckSortedBlockInputStream.cpp b/src/DataStreams/CheckSortedBlockInputStream.cpp
index db4794b917f..a56738a9a9f 100644
--- a/src/DataStreams/CheckSortedBlockInputStream.cpp
+++ b/src/DataStreams/CheckSortedBlockInputStream.cpp
@@ -6,7 +6,7 @@ namespace DB
 
 namespace ErrorCodes
 {
-extern const int SORT_ORDER_VIOLATED;
+extern const int LOGICAL_ERROR;
 }
 
 CheckSortedBlockInputStream::CheckSortedBlockInputStream(
@@ -72,12 +72,12 @@ Block CheckSortedBlockInputStream::readImpl()
 
     auto block_columns = block.getColumns();
     if (!last_row.empty() && !less(last_row, 0, block_columns, 0))
-        throw Exception("Sort order of blocks violated", ErrorCodes::SORT_ORDER_VIOLATED);
+        throw Exception("Sort order of blocks violated", ErrorCodes::LOGICAL_ERROR);
 
     size_t rows = block.rows();
     for (size_t i = 1; i < rows; ++i)
         if (!less(block_columns, i - 1, block_columns, i))
-            throw Exception("Sort order of blocks violated", ErrorCodes::SORT_ORDER_VIOLATED);
+            throw Exception("Sort order of blocks violated", ErrorCodes::LOGICAL_ERROR);
 
     last_row.clear();
     for (size_t i = 0; i < block.columns(); ++i)

From d7ec509f337a88b0125b5af48d981bb4b98ba604 Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Thu, 28 May 2020 22:26:20 +0300
Subject: [PATCH 134/184] Update extended-roadmap.md

---
 docs/ru/whats-new/extended-roadmap.md | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/docs/ru/whats-new/extended-roadmap.md b/docs/ru/whats-new/extended-roadmap.md
index b7756f7fb79..9b92dbfa717 100644
--- a/docs/ru/whats-new/extended-roadmap.md
+++ b/docs/ru/whats-new/extended-roadmap.md
@@ -203,10 +203,11 @@ Upd. SharedContext выне�ен из Context.
 
 Upd. В очереди. Иван Лежанкин.
 
-### 2.9. Логгировние в format-�тиле {#loggirovnie-v-format-stile}
+### 2.9. + Логгировние в format-�тиле {#loggirovnie-v-format-stile}
 
-Делает [Иван Лежанкин](https://github.com/abyss7). �изкий приоритет.
-[\#6049](https://github.com/ClickHouse/ClickHouse/issues/6049#issuecomment-570836998)
+[#6049](https://github.com/ClickHouse/ClickHouse/issues/6049#issuecomment-570836998)
+
+Сделано.
 
 ### 2.10. Запрашивать у таблиц не �толбцы, а �резы {#zaprashivat-u-tablits-ne-stolbtsy-a-srezy}
 

From d8049be204ed7325c5725bdb0b2a49fef9cd8464 Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Thu, 28 May 2020 22:27:31 +0300
Subject: [PATCH 135/184] Update extended-roadmap.md

---
 docs/ru/whats-new/extended-roadmap.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/ru/whats-new/extended-roadmap.md b/docs/ru/whats-new/extended-roadmap.md
index 9b92dbfa717..0bedfda57b1 100644
--- a/docs/ru/whats-new/extended-roadmap.md
+++ b/docs/ru/whats-new/extended-roadmap.md
@@ -73,10 +73,10 @@ Upd. Включено дл� �и�темных таблиц.
 Q1. Закоммичено, но е�ть техниче�кий долг, который и�правл�ет�� �ейча�.
 Готово. �ет, не готово - там в�ё ещё техниче�кий долг.
 
-### 1.9. И�пользование TTL дл� прореживани� данных {#ispolzovanie-ttl-dlia-prorezhivaniia-dannykh}
+### 1.9. + И�пользование TTL дл� прореживани� данных {#ispolzovanie-ttl-dlia-prorezhivaniia-dannykh}
 
 Будет делать Сорокин �иколай, ВШЭ и Яндек�.
-Upd. Е�ть pull request.
+Upd. Е�ть pull request. Upd. Сделано.
 
 Сейча� пользователь может задать в таблице выражение, которое определ�ет, �колько времени хран�т�� данные. Обычно �то выражение задаёт�� отно�ительно значени� �толбца � датой - например: удал�ть данные через три ме��ца. https://clickhouse.tech/docs/ru/operations/table_engines/mergetree/\#table_engine-mergetree-ttl
 

From 24fb6266daa54d4aa8a7ad9e7bcfa98da9afe34b Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Thu, 28 May 2020 22:28:44 +0300
Subject: [PATCH 136/184] Fix wrong markup

---
 website/css/docs.css | 1 -
 1 file changed, 1 deletion(-)

diff --git a/website/css/docs.css b/website/css/docs.css
index 18bf40de13b..2c175b18db6 100644
--- a/website/css/docs.css
+++ b/website/css/docs.css
@@ -69,7 +69,6 @@ summary {
 #content code {
     color: #111;
     background: #eee;
-    padding: 2px;
 }
 
 

From a9c6fa1ca73dd2d35ee3d1152edd56d20b6db4e5 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Thu, 28 May 2020 23:05:07 +0300
Subject: [PATCH 137/184] Try fix tests.

---
 src/Processors/Executors/PullingPipelineExecutor.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Processors/Executors/PullingPipelineExecutor.cpp b/src/Processors/Executors/PullingPipelineExecutor.cpp
index af061a373cc..18722dfdc2f 100644
--- a/src/Processors/Executors/PullingPipelineExecutor.cpp
+++ b/src/Processors/Executors/PullingPipelineExecutor.cpp
@@ -76,8 +76,8 @@ void PullingPipelineExecutor::cancel()
         executor->cancel();
 
     /// Read all data and finish execution.
-    Chunk chunk;
-    while (pull(chunk));
+    /// Chunk chunk;
+    /// while (pull(chunk));
 }
 
 Chunk PullingPipelineExecutor::getTotals()

From 3cff070f897e812e8eb3f5f2dcc1d1a654f31771 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <avtokmakov@yandex-team.ru>
Date: Thu, 28 May 2020 23:10:45 +0300
Subject: [PATCH 138/184] add context to IDatabase methods

---
 programs/copier/ClusterCopierApp.cpp             |  2 +-
 programs/local/LocalServer.cpp                   | 10 +++++-----
 src/Databases/DatabaseAtomic.cpp                 |  6 +++---
 src/Databases/DatabaseAtomic.h                   |  2 +-
 src/Databases/DatabaseDictionary.cpp             |  8 ++++----
 src/Databases/DatabaseDictionary.h               |  8 ++++----
 src/Databases/DatabaseFactory.cpp                |  2 +-
 src/Databases/DatabaseLazy.cpp                   |  2 +-
 src/Databases/DatabaseLazy.h                     |  8 +++++---
 src/Databases/DatabaseMemory.cpp                 |  8 ++++----
 src/Databases/DatabaseMemory.h                   |  4 ++--
 src/Databases/DatabaseMySQL.cpp                  | 12 ++++++------
 src/Databases/DatabaseMySQL.h                    |  8 ++++----
 src/Databases/DatabaseOnDisk.cpp                 | 11 +++++------
 src/Databases/DatabaseOnDisk.h                   |  2 +-
 src/Databases/DatabaseWithDictionaries.cpp       |  2 +-
 src/Databases/DatabasesCommon.cpp                | 11 ++++++-----
 src/Databases/DatabasesCommon.h                  | 11 ++++++-----
 src/Databases/IDatabase.h                        | 16 ++++++++--------
 src/Interpreters/ActionLocksManager.cpp          |  8 ++++----
 src/Interpreters/ActionLocksManager.h            |  2 +-
 src/Interpreters/AsynchronousMetrics.cpp         |  2 +-
 src/Interpreters/DatabaseCatalog.cpp             |  8 ++++----
 src/Interpreters/InterpreterCreateQuery.cpp      |  6 +++---
 src/Interpreters/InterpreterDropQuery.cpp        |  2 +-
 src/Interpreters/InterpreterShowCreateQuery.cpp  |  2 +-
 src/Interpreters/InterpreterSystemQuery.cpp      |  6 +++---
 src/Server/ReplicasStatusHandler.cpp             |  2 +-
 src/Storages/StorageMerge.cpp                    | 12 ++++++------
 src/Storages/StorageMerge.h                      |  2 +-
 src/Storages/System/StorageSystemColumns.cpp     |  2 +-
 src/Storages/System/StorageSystemGraphite.cpp    |  8 ++++----
 src/Storages/System/StorageSystemMutations.cpp   |  2 +-
 src/Storages/System/StorageSystemPartsBase.cpp   |  2 +-
 src/Storages/System/StorageSystemReplicas.cpp    |  2 +-
 .../System/StorageSystemReplicationQueue.cpp     |  2 +-
 src/Storages/System/StorageSystemTables.cpp      |  4 ++--
 ...est_transform_query_for_external_database.cpp |  2 +-
 src/TableFunctions/TableFunctionMerge.cpp        |  6 +++---
 39 files changed, 109 insertions(+), 106 deletions(-)

diff --git a/programs/copier/ClusterCopierApp.cpp b/programs/copier/ClusterCopierApp.cpp
index 9b1393204d4..ce4bf94589e 100644
--- a/programs/copier/ClusterCopierApp.cpp
+++ b/programs/copier/ClusterCopierApp.cpp
@@ -114,7 +114,7 @@ void ClusterCopierApp::mainImpl()
     registerDisks();
 
     static const std::string default_database = "_local";
-    DatabaseCatalog::instance().attachDatabase(default_database, std::make_shared<DatabaseMemory>(default_database));
+    DatabaseCatalog::instance().attachDatabase(default_database, std::make_shared<DatabaseMemory>(default_database, *context));
     context->setCurrentDatabase(default_database);
 
     /// Initialize query scope just in case.
diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp
index 01acf250b1b..eb78c049825 100644
--- a/programs/local/LocalServer.cpp
+++ b/programs/local/LocalServer.cpp
@@ -118,13 +118,13 @@ void LocalServer::tryInitPath()
 }
 
 
-static void attachSystemTables()
+static void attachSystemTables(const Context & context)
 {
     DatabasePtr system_database = DatabaseCatalog::instance().tryGetDatabase(DatabaseCatalog::SYSTEM_DATABASE);
     if (!system_database)
     {
         /// TODO: add attachTableDelayed into DatabaseMemory to speedup loading
-        system_database = std::make_shared<DatabaseMemory>(DatabaseCatalog::SYSTEM_DATABASE);
+        system_database = std::make_shared<DatabaseMemory>(DatabaseCatalog::SYSTEM_DATABASE, context);
         DatabaseCatalog::instance().attachDatabase(DatabaseCatalog::SYSTEM_DATABASE, system_database);
     }
 
@@ -202,7 +202,7 @@ try
       *  if such tables will not be dropped, clickhouse-server will not be able to load them due to security reasons.
       */
     std::string default_database = config().getString("default_database", "_local");
-    DatabaseCatalog::instance().attachDatabase(default_database, std::make_shared<DatabaseMemory>(default_database));
+    DatabaseCatalog::instance().attachDatabase(default_database, std::make_shared<DatabaseMemory>(default_database, *context));
     context->setCurrentDatabase(default_database);
     applyCmdOptions();
 
@@ -213,14 +213,14 @@ try
 
         LOG_DEBUG(log, "Loading metadata from {}", context->getPath());
         loadMetadataSystem(*context);
-        attachSystemTables();
+        attachSystemTables(*context);
         loadMetadata(*context);
         DatabaseCatalog::instance().loadDatabases();
         LOG_DEBUG(log, "Loaded metadata.");
     }
     else
     {
-        attachSystemTables();
+        attachSystemTables(*context);
     }
 
     processQueries();
diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp
index 27a2441cec6..ed875f5fff4 100644
--- a/src/Databases/DatabaseAtomic.cpp
+++ b/src/Databases/DatabaseAtomic.cpp
@@ -288,15 +288,15 @@ void DatabaseAtomic::assertCanBeDetached(bool cleenup)
                         "because some tables are still in use. Retry later.", ErrorCodes::DATABASE_NOT_EMPTY);
 }
 
-DatabaseTablesIteratorPtr DatabaseAtomic::getTablesIterator(const IDatabase::FilterByNameFunction & filter_by_table_name)
+DatabaseTablesIteratorPtr DatabaseAtomic::getTablesIterator(const Context & context, const IDatabase::FilterByNameFunction & filter_by_table_name)
 {
-    auto base_iter = DatabaseWithOwnTablesBase::getTablesIterator(filter_by_table_name);
+    auto base_iter = DatabaseWithOwnTablesBase::getTablesIterator(context, filter_by_table_name);
     return std::make_unique<AtomicDatabaseTablesSnapshotIterator>(std::move(typeid_cast<DatabaseTablesSnapshotIterator &>(*base_iter)));
 }
 
 UUID DatabaseAtomic::tryGetTableUUID(const String & table_name) const
 {
-    if (auto table = tryGetTable(table_name))
+    if (auto table = tryGetTable(table_name, global_context))
         return table->getStorageID().uuid;
     return UUIDHelpers::Nil;
 }
diff --git a/src/Databases/DatabaseAtomic.h b/src/Databases/DatabaseAtomic.h
index 84297a42843..71428fdb420 100644
--- a/src/Databases/DatabaseAtomic.h
+++ b/src/Databases/DatabaseAtomic.h
@@ -42,7 +42,7 @@ public:
 
     void drop(const Context & /*context*/) override;
 
-    DatabaseTablesIteratorPtr getTablesIterator(const FilterByNameFunction & filter_by_table_name) override;
+    DatabaseTablesIteratorPtr getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name) override;
 
     void loadStoredObjects(Context & context, bool has_force_restore_data_flag) override;
 
diff --git a/src/Databases/DatabaseDictionary.cpp b/src/Databases/DatabaseDictionary.cpp
index 81bcbf7bc6b..c4c74f8e70c 100644
--- a/src/Databases/DatabaseDictionary.cpp
+++ b/src/Databases/DatabaseDictionary.cpp
@@ -50,18 +50,18 @@ Tables DatabaseDictionary::listTables(const FilterByNameFunction & filter_by_nam
     return tables;
 }
 
-bool DatabaseDictionary::isTableExist(const String & table_name) const
+bool DatabaseDictionary::isTableExist(const String & table_name, const Context &) const
 {
     return global_context.getExternalDictionariesLoader().getCurrentStatus(table_name) != ExternalLoader::Status::NOT_EXIST;
 }
 
-StoragePtr DatabaseDictionary::tryGetTable(const String & table_name) const
+StoragePtr DatabaseDictionary::tryGetTable(const String & table_name, const Context &) const
 {
     auto load_result = global_context.getExternalDictionariesLoader().getLoadResult(table_name);
     return createStorageDictionary(getDatabaseName(), load_result);
 }
 
-DatabaseTablesIteratorPtr DatabaseDictionary::getTablesIterator(const FilterByNameFunction & filter_by_table_name)
+DatabaseTablesIteratorPtr DatabaseDictionary::getTablesIterator(const Context &, const FilterByNameFunction & filter_by_table_name)
 {
     return std::make_unique<DatabaseTablesSnapshotIterator>(listTables(filter_by_table_name));
 }
@@ -71,7 +71,7 @@ bool DatabaseDictionary::empty() const
     return !global_context.getExternalDictionariesLoader().hasObjects();
 }
 
-ASTPtr DatabaseDictionary::getCreateTableQueryImpl(const String & table_name, bool throw_on_error) const
+ASTPtr DatabaseDictionary::getCreateTableQueryImpl(const String & table_name, const Context &, bool throw_on_error) const
 {
     String query;
     {
diff --git a/src/Databases/DatabaseDictionary.h b/src/Databases/DatabaseDictionary.h
index bf10dc134bd..b61c85033e8 100644
--- a/src/Databases/DatabaseDictionary.h
+++ b/src/Databases/DatabaseDictionary.h
@@ -29,11 +29,11 @@ public:
         return "Dictionary";
     }
 
-    bool isTableExist(const String & table_name) const override;
+    bool isTableExist(const String & table_name, const Context & context) const override;
 
-    StoragePtr tryGetTable(const String & table_name) const override;
+    StoragePtr tryGetTable(const String & table_name, const Context & context) const override;
 
-    DatabaseTablesIteratorPtr getTablesIterator(const FilterByNameFunction & filter_by_table_name) override;
+    DatabaseTablesIteratorPtr getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name) override;
 
     bool empty() const override;
 
@@ -44,7 +44,7 @@ public:
     void shutdown() override;
 
 protected:
-    ASTPtr getCreateTableQueryImpl(const String & table_name, bool throw_on_error) const override;
+    ASTPtr getCreateTableQueryImpl(const String & table_name, const Context & context, bool throw_on_error) const override;
 
 private:
     mutable std::mutex mutex;
diff --git a/src/Databases/DatabaseFactory.cpp b/src/Databases/DatabaseFactory.cpp
index 21feb122da1..f27bc509ebe 100644
--- a/src/Databases/DatabaseFactory.cpp
+++ b/src/Databases/DatabaseFactory.cpp
@@ -82,7 +82,7 @@ DatabasePtr DatabaseFactory::getImpl(
     else if (engine_name == "Atomic")
         return std::make_shared<DatabaseAtomic>(database_name, metadata_path, context);
     else if (engine_name == "Memory")
-        return std::make_shared<DatabaseMemory>(database_name);
+        return std::make_shared<DatabaseMemory>(database_name, context);
     else if (engine_name == "Dictionary")
         return std::make_shared<DatabaseDictionary>(database_name, context);
 
diff --git a/src/Databases/DatabaseLazy.cpp b/src/Databases/DatabaseLazy.cpp
index 51ee664a42a..11e5272110e 100644
--- a/src/Databases/DatabaseLazy.cpp
+++ b/src/Databases/DatabaseLazy.cpp
@@ -132,7 +132,7 @@ StoragePtr DatabaseLazy::tryGetTable(const String & table_name) const
     return loadTable(table_name);
 }
 
-DatabaseTablesIteratorPtr DatabaseLazy::getTablesIterator(const FilterByNameFunction & filter_by_table_name)
+DatabaseTablesIteratorPtr DatabaseLazy::getTablesIterator(const Context &, const FilterByNameFunction & filter_by_table_name)
 {
     std::lock_guard lock(mutex);
     Strings filtered_tables;
diff --git a/src/Databases/DatabaseLazy.h b/src/Databases/DatabaseLazy.h
index 4306e61b37b..2e24b687be5 100644
--- a/src/Databases/DatabaseLazy.h
+++ b/src/Databases/DatabaseLazy.h
@@ -51,13 +51,15 @@ public:
 
     time_t getObjectMetadataModificationTime(const String & table_name) const override;
 
-    bool isTableExist(const String & table_name) const override;
+    bool isTableExist(const String & table_name, const Context &) const override { return isTableExist(table_name); }
+    bool isTableExist(const String & table_name) const;
 
-    StoragePtr tryGetTable(const String & table_name) const override;
+    StoragePtr tryGetTable(const String & table_name, const Context &) const override { return tryGetTable(table_name); }
+    StoragePtr tryGetTable(const String & table_name) const;
 
     bool empty() const override;
 
-    DatabaseTablesIteratorPtr getTablesIterator(const FilterByNameFunction & filter_by_table_name) override;
+    DatabaseTablesIteratorPtr getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name) override;
 
     void attachTable(const String & table_name, const StoragePtr & table, const String & relative_table_path) override;
 
diff --git a/src/Databases/DatabaseMemory.cpp b/src/Databases/DatabaseMemory.cpp
index 84fec6bcc22..52b1f889943 100644
--- a/src/Databases/DatabaseMemory.cpp
+++ b/src/Databases/DatabaseMemory.cpp
@@ -16,8 +16,8 @@ namespace ErrorCodes
     extern const int UNKNOWN_TABLE;
 }
 
-DatabaseMemory::DatabaseMemory(const String & name_)
-    : DatabaseWithOwnTablesBase(name_, "DatabaseMemory(" + name_ + ")")
+DatabaseMemory::DatabaseMemory(const String & name_, const Context & context)
+    : DatabaseWithOwnTablesBase(name_, "DatabaseMemory(" + name_ + ")", context)
     , data_path("data/" + escapeForFileName(database_name) + "/")
 {}
 
@@ -64,7 +64,7 @@ ASTPtr DatabaseMemory::getCreateDatabaseQuery() const
     return create_query;
 }
 
-ASTPtr DatabaseMemory::getCreateTableQueryImpl(const String & table_name, bool throw_on_error) const
+ASTPtr DatabaseMemory::getCreateTableQueryImpl(const String & table_name, const Context &, bool throw_on_error) const
 {
     std::lock_guard lock{mutex};
     auto it = create_queries.find(table_name);
@@ -80,7 +80,7 @@ ASTPtr DatabaseMemory::getCreateTableQueryImpl(const String & table_name, bool t
 
 UUID DatabaseMemory::tryGetTableUUID(const String & table_name) const
 {
-    if (auto table = tryGetTable(table_name))
+    if (auto table = tryGetTable(table_name, global_context))
         return table->getStorageID().uuid;
     return UUIDHelpers::Nil;
 }
diff --git a/src/Databases/DatabaseMemory.h b/src/Databases/DatabaseMemory.h
index ad34c4d9097..40cc808e42b 100644
--- a/src/Databases/DatabaseMemory.h
+++ b/src/Databases/DatabaseMemory.h
@@ -19,7 +19,7 @@ namespace DB
 class DatabaseMemory final : public DatabaseWithOwnTablesBase
 {
 public:
-    DatabaseMemory(const String & name_);
+    DatabaseMemory(const String & name_, const Context & context);
 
     String getEngineName() const override { return "Memory"; }
 
@@ -34,7 +34,7 @@ public:
         const String & table_name,
         bool no_delay) override;
 
-    ASTPtr getCreateTableQueryImpl(const String & name, bool throw_on_error) const override;
+    ASTPtr getCreateTableQueryImpl(const String & name, const Context & context, bool throw_on_error) const override;
     ASTPtr getCreateDatabaseQuery() const override;
 
     /// DatabaseMemory allows to create tables, which store data on disk.
diff --git a/src/Databases/DatabaseMySQL.cpp b/src/Databases/DatabaseMySQL.cpp
index cbd3261975a..5d4b81014f9 100644
--- a/src/Databases/DatabaseMySQL.cpp
+++ b/src/Databases/DatabaseMySQL.cpp
@@ -89,7 +89,7 @@ bool DatabaseMySQL::empty() const
     return true;
 }
 
-DatabaseTablesIteratorPtr DatabaseMySQL::getTablesIterator(const FilterByNameFunction & filter_by_table_name)
+DatabaseTablesIteratorPtr DatabaseMySQL::getTablesIterator(const Context &, const FilterByNameFunction & filter_by_table_name)
 {
     Tables tables;
     std::lock_guard<std::mutex> lock(mutex);
@@ -103,12 +103,12 @@ DatabaseTablesIteratorPtr DatabaseMySQL::getTablesIterator(const FilterByNameFun
     return std::make_unique<DatabaseTablesSnapshotIterator>(tables);
 }
 
-bool DatabaseMySQL::isTableExist(const String & name) const
+bool DatabaseMySQL::isTableExist(const String & name, const Context &) const
 {
-    return bool(tryGetTable(name));
+    return bool(tryGetTable(name, global_context));
 }
 
-StoragePtr DatabaseMySQL::tryGetTable(const String & mysql_table_name) const
+StoragePtr DatabaseMySQL::tryGetTable(const String & mysql_table_name, const Context &) const
 {
     std::lock_guard<std::mutex> lock(mutex);
 
@@ -155,7 +155,7 @@ static ASTPtr getCreateQueryFromStorage(const StoragePtr & storage, const ASTPtr
     return create_table_query;
 }
 
-ASTPtr DatabaseMySQL::getCreateTableQueryImpl(const String & table_name, bool throw_on_error) const
+ASTPtr DatabaseMySQL::getCreateTableQueryImpl(const String & table_name, const Context &, bool throw_on_error) const
 {
     std::lock_guard<std::mutex> lock(mutex);
 
@@ -501,7 +501,7 @@ void DatabaseMySQL::createTable(const Context &, const String & table_name, cons
     /// XXX: hack
     /// In order to prevent users from broken the table structure by executing attach table database_name.table_name (...)
     /// we should compare the old and new create_query to make them completely consistent
-    const auto & origin_create_query = getCreateTableQuery(table_name);
+    const auto & origin_create_query = getCreateTableQuery(table_name, global_context);
     origin_create_query->as<ASTCreateQuery>()->attach = true;
 
     if (queryToString(origin_create_query) != queryToString(create_query))
diff --git a/src/Databases/DatabaseMySQL.h b/src/Databases/DatabaseMySQL.h
index a43da5d1762..70d8dc81e0f 100644
--- a/src/Databases/DatabaseMySQL.h
+++ b/src/Databases/DatabaseMySQL.h
@@ -32,13 +32,13 @@ public:
 
     bool empty() const override;
 
-    DatabaseTablesIteratorPtr getTablesIterator(const FilterByNameFunction & filter_by_table_name) override;
+    DatabaseTablesIteratorPtr getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name) override;
 
     ASTPtr getCreateDatabaseQuery() const override;
 
-    bool isTableExist(const String & name) const override;
+    bool isTableExist(const String & name, const Context & context) const override;
 
-    StoragePtr tryGetTable(const String & name) const override;
+    StoragePtr tryGetTable(const String & name, const Context & context) const override;
 
     time_t getObjectMetadataModificationTime(const String & name) const override;
 
@@ -59,7 +59,7 @@ public:
     void attachTable(const String & table_name, const StoragePtr & storage, const String & relative_table_path) override;
 
 protected:
-    ASTPtr getCreateTableQueryImpl(const String & name, bool throw_on_error) const override;
+    ASTPtr getCreateTableQueryImpl(const String & name, const Context & context, bool throw_on_error) const override;
 
 private:
     const Context & global_context;
diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp
index 70f86b19f1f..1b542c7a1ff 100644
--- a/src/Databases/DatabaseOnDisk.cpp
+++ b/src/Databases/DatabaseOnDisk.cpp
@@ -123,10 +123,9 @@ String getObjectDefinitionFromCreateQuery(const ASTPtr & query)
 }
 
 DatabaseOnDisk::DatabaseOnDisk(const String & name, const String & metadata_path_, const String & data_path_, const String & logger, const Context & context)
-    : DatabaseWithOwnTablesBase(name, logger)
+    : DatabaseWithOwnTablesBase(name, logger, context)
     , metadata_path(metadata_path_)
     , data_path(data_path_)
-    , global_context(context.getGlobalContext())
 {
     Poco::File(context.getPath() + data_path).createDirectories();
     Poco::File(metadata_path).createDirectories();
@@ -160,7 +159,7 @@ void DatabaseOnDisk::createTable(
         throw Exception("Dictionary " + backQuote(getDatabaseName()) + "." + backQuote(table_name) + " already exists.",
             ErrorCodes::DICTIONARY_ALREADY_EXISTS);
 
-    if (isTableExist(table_name))
+    if (isTableExist(table_name, global_context))
         throw Exception("Table " + backQuote(getDatabaseName()) + "." + backQuote(table_name) + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS);
 
     if (create.attach_short_syntax)
@@ -267,7 +266,7 @@ void DatabaseOnDisk::renameTable(
     String table_metadata_path;
     ASTPtr attach_query;
     /// DatabaseLazy::detachTable may return nullptr even if table exists, so we need tryGetTable for this case.
-    StoragePtr table = tryGetTable(table_name);
+    StoragePtr table = tryGetTable(table_name, global_context);
     detachTable(table_name);
     try
     {
@@ -304,10 +303,10 @@ void DatabaseOnDisk::renameTable(
     Poco::File(table_metadata_path).remove();
 }
 
-ASTPtr DatabaseOnDisk::getCreateTableQueryImpl(const String & table_name, bool throw_on_error) const
+ASTPtr DatabaseOnDisk::getCreateTableQueryImpl(const String & table_name, const Context &, bool throw_on_error) const
 {
     ASTPtr ast;
-    bool has_table = tryGetTable(table_name) != nullptr;
+    bool has_table = tryGetTable(table_name, global_context) != nullptr;
     auto table_metadata_path = getObjectMetadataPath(table_name);
     try
     {
diff --git a/src/Databases/DatabaseOnDisk.h b/src/Databases/DatabaseOnDisk.h
index 75609e231af..d4fb9b2aa17 100644
--- a/src/Databases/DatabaseOnDisk.h
+++ b/src/Databases/DatabaseOnDisk.h
@@ -76,6 +76,7 @@ protected:
 
     ASTPtr getCreateTableQueryImpl(
         const String & table_name,
+        const Context & context,
         bool throw_on_error) const override;
 
     ASTPtr getCreateQueryFromMetadata(const String & metadata_path, bool throw_on_error) const;
@@ -85,7 +86,6 @@ protected:
 
     const String metadata_path;
     const String data_path;
-    const Context & global_context;
 };
 
 }
diff --git a/src/Databases/DatabaseWithDictionaries.cpp b/src/Databases/DatabaseWithDictionaries.cpp
index 0d49078bd08..e0f2aa9286b 100644
--- a/src/Databases/DatabaseWithDictionaries.cpp
+++ b/src/Databases/DatabaseWithDictionaries.cpp
@@ -127,7 +127,7 @@ void DatabaseWithDictionaries::createDictionary(const Context & context, const S
                 "Dictionary " + backQuote(getDatabaseName()) + "." + backQuote(dictionary_name) + " already exists.",
                 ErrorCodes::DICTIONARY_ALREADY_EXISTS);
 
-    if (isTableExist(dictionary_name))
+    if (isTableExist(dictionary_name, global_context))
         throw Exception("Table " + backQuote(getDatabaseName()) + "." + backQuote(dictionary_name) + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS);
 
 
diff --git a/src/Databases/DatabasesCommon.cpp b/src/Databases/DatabasesCommon.cpp
index 5ad4fa20690..2a2ca1841cf 100644
--- a/src/Databases/DatabasesCommon.cpp
+++ b/src/Databases/DatabasesCommon.cpp
@@ -1,5 +1,6 @@
 #include <Databases/DatabasesCommon.h>
 #include <Interpreters/InterpreterCreateQuery.h>
+#include <Interpreters/Context.h>
 #include <Parsers/ParserCreateQuery.h>
 #include <Parsers/formatAST.h>
 #include <Storages/StorageDictionary.h>
@@ -18,18 +19,18 @@ namespace ErrorCodes
     extern const int UNKNOWN_TABLE;
 }
 
-DatabaseWithOwnTablesBase::DatabaseWithOwnTablesBase(const String & name_, const String & logger)
-        : IDatabase(name_), log(&Logger::get(logger))
+DatabaseWithOwnTablesBase::DatabaseWithOwnTablesBase(const String & name_, const String & logger, const Context & context)
+        : IDatabase(name_), log(&Logger::get(logger)), global_context(context.getGlobalContext())
 {
 }
 
-bool DatabaseWithOwnTablesBase::isTableExist(const String & table_name) const
+bool DatabaseWithOwnTablesBase::isTableExist(const String & table_name, const Context &) const
 {
     std::lock_guard lock(mutex);
     return tables.find(table_name) != tables.end();
 }
 
-StoragePtr DatabaseWithOwnTablesBase::tryGetTable(const String & table_name) const
+StoragePtr DatabaseWithOwnTablesBase::tryGetTable(const String & table_name, const Context &) const
 {
     std::lock_guard lock(mutex);
     auto it = tables.find(table_name);
@@ -38,7 +39,7 @@ StoragePtr DatabaseWithOwnTablesBase::tryGetTable(const String & table_name) con
     return {};
 }
 
-DatabaseTablesIteratorPtr DatabaseWithOwnTablesBase::getTablesIterator(const FilterByNameFunction & filter_by_table_name)
+DatabaseTablesIteratorPtr DatabaseWithOwnTablesBase::getTablesIterator(const Context &, const FilterByNameFunction & filter_by_table_name)
 {
     std::lock_guard lock(mutex);
     if (!filter_by_table_name)
diff --git a/src/Databases/DatabasesCommon.h b/src/Databases/DatabasesCommon.h
index e195fc59ce9..4238fd30137 100644
--- a/src/Databases/DatabasesCommon.h
+++ b/src/Databases/DatabasesCommon.h
@@ -19,9 +19,9 @@ class Context;
 class DatabaseWithOwnTablesBase : public IDatabase
 {
 public:
-    bool isTableExist(const String & table_name) const override;
+    bool isTableExist(const String & table_name, const Context & context) const override;
 
-    StoragePtr tryGetTable(const String & table_name) const override;
+    StoragePtr tryGetTable(const String & table_name, const Context & context) const override;
 
     bool empty() const override;
 
@@ -29,18 +29,19 @@ public:
 
     StoragePtr detachTable(const String & table_name) override;
 
-    DatabaseTablesIteratorPtr getTablesIterator(const FilterByNameFunction & filter_by_table_name) override;
+    DatabaseTablesIteratorPtr getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name) override;
 
     void shutdown() override;
 
-    virtual ~DatabaseWithOwnTablesBase() override;
+    ~DatabaseWithOwnTablesBase() override;
 
 protected:
     mutable std::mutex mutex;
     Tables tables;
     Poco::Logger * log;
+    const Context & global_context;
 
-    DatabaseWithOwnTablesBase(const String & name_, const String & logger);
+    DatabaseWithOwnTablesBase(const String & name_, const String & logger, const Context & context);
 
     void attachTableUnlocked(const String & table_name, const StoragePtr & table, std::unique_lock<std::mutex> & lock);
     StoragePtr detachTableUnlocked(const String & table_name, std::unique_lock<std::mutex> & lock);
diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h
index 24147a0e7c5..3d108cfd91c 100644
--- a/src/Databases/IDatabase.h
+++ b/src/Databases/IDatabase.h
@@ -130,7 +130,7 @@ public:
     virtual void loadStoredObjects(Context & /*context*/, bool /*has_force_restore_data_flag*/) {}
 
     /// Check the existence of the table.
-    virtual bool isTableExist(const String & name) const = 0;
+    virtual bool isTableExist(const String & name, const Context & context) const = 0;
 
     /// Check the existence of the dictionary
     virtual bool isDictionaryExist(const String & /*name*/) const
@@ -139,7 +139,7 @@ public:
     }
 
     /// Get the table for work. Return nullptr if there is no table.
-    virtual StoragePtr tryGetTable(const String & name) const = 0;
+    virtual StoragePtr tryGetTable(const String & name, const Context & context) const = 0;
 
     virtual UUID tryGetTableUUID(const String & /*table_name*/) const { return UUIDHelpers::Nil; }
 
@@ -147,7 +147,7 @@ public:
 
     /// Get an iterator that allows you to pass through all the tables.
     /// It is possible to have "hidden" tables that are not visible when passing through, but are visible if you get them by name using the functions above.
-    virtual DatabaseTablesIteratorPtr getTablesIterator(const FilterByNameFunction & filter_by_table_name = {}) = 0;
+    virtual DatabaseTablesIteratorPtr getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name = {}) = 0;
 
     /// Get an iterator to pass through all the dictionaries.
     virtual DatabaseDictionariesIteratorPtr getDictionariesIterator([[maybe_unused]] const FilterByNameFunction & filter_by_dictionary_name = {})
@@ -249,14 +249,14 @@ public:
     }
 
     /// Get the CREATE TABLE query for the table. It can also provide information for detached tables for which there is metadata.
-    ASTPtr tryGetCreateTableQuery(const String & name) const noexcept
+    ASTPtr tryGetCreateTableQuery(const String & name, const Context & context) const noexcept
     {
-        return getCreateTableQueryImpl(name, false);
+        return getCreateTableQueryImpl(name, context, false);
     }
 
-    ASTPtr getCreateTableQuery(const String & name) const
+    ASTPtr getCreateTableQuery(const String & name, const Context & context) const
     {
-        return getCreateTableQueryImpl(name, true);
+        return getCreateTableQueryImpl(name, context, true);
     }
 
     /// Get the CREATE DICTIONARY query for the dictionary. Returns nullptr if dictionary doesn't exists.
@@ -304,7 +304,7 @@ public:
     virtual ~IDatabase() {}
 
 protected:
-    virtual ASTPtr getCreateTableQueryImpl(const String & /*name*/, bool throw_on_error) const
+    virtual ASTPtr getCreateTableQueryImpl(const String & /*name*/, const Context & /*context*/, bool throw_on_error) const
     {
         if (throw_on_error)
             throw Exception("There is no SHOW CREATE TABLE query for Database" + getEngineName(), ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY);
diff --git a/src/Interpreters/ActionLocksManager.cpp b/src/Interpreters/ActionLocksManager.cpp
index fe5b3def658..8bc8542da3a 100644
--- a/src/Interpreters/ActionLocksManager.cpp
+++ b/src/Interpreters/ActionLocksManager.cpp
@@ -20,17 +20,17 @@ namespace ActionLocks
 
 
 template <typename F>
-inline void forEachTable(F && f)
+inline void forEachTable(F && f, const Context & context)
 {
     for (auto & elem : DatabaseCatalog::instance().getDatabases())
-        for (auto iterator = elem.second->getTablesIterator(); iterator->isValid(); iterator->next())
+        for (auto iterator = elem.second->getTablesIterator(context); iterator->isValid(); iterator->next())
             f(iterator->table());
 
 }
 
-void ActionLocksManager::add(StorageActionBlockType action_type)
+void ActionLocksManager::add(StorageActionBlockType action_type, const Context & context)
 {
-    forEachTable([&](const StoragePtr & table) { add(table, action_type); });
+    forEachTable([&](const StoragePtr & table) { add(table, action_type); }, context);
 }
 
 void ActionLocksManager::add(const StorageID & table_id, StorageActionBlockType action_type)
diff --git a/src/Interpreters/ActionLocksManager.h b/src/Interpreters/ActionLocksManager.h
index 039a95ce218..50855676fd4 100644
--- a/src/Interpreters/ActionLocksManager.h
+++ b/src/Interpreters/ActionLocksManager.h
@@ -20,7 +20,7 @@ class ActionLocksManager
 {
 public:
     /// Adds new locks for each table
-    void add(StorageActionBlockType action_type);
+    void add(StorageActionBlockType action_type, const Context & context);
     /// Add new lock for a table if it has not been already added
     void add(const StorageID & table_id, StorageActionBlockType action_type);
     void add(const StoragePtr & table, StorageActionBlockType action_type);
diff --git a/src/Interpreters/AsynchronousMetrics.cpp b/src/Interpreters/AsynchronousMetrics.cpp
index 59ff01bf972..a4e03d35d42 100644
--- a/src/Interpreters/AsynchronousMetrics.cpp
+++ b/src/Interpreters/AsynchronousMetrics.cpp
@@ -181,7 +181,7 @@ void AsynchronousMetrics::update()
             /// Lazy database can not contain MergeTree tables
             if (db.second->getEngineName() == "Lazy")
                 continue;
-            for (auto iterator = db.second->getTablesIterator(); iterator->isValid(); iterator->next())
+            for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next())
             {
                 ++total_number_of_tables;
                 const auto & table = iterator->table();
diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp
index 7606fdb255a..f292aaaf049 100644
--- a/src/Interpreters/DatabaseCatalog.cpp
+++ b/src/Interpreters/DatabaseCatalog.cpp
@@ -97,7 +97,7 @@ StorageID TemporaryTableHolder::getGlobalTableID() const
 
 StoragePtr TemporaryTableHolder::getTable() const
 {
-    auto table = temporary_tables->tryGetTable("_tmp_" + toString(id));
+    auto table = temporary_tables->tryGetTable("_tmp_" + toString(id), *global_context);
     if (!table)
         throw Exception("Temporary table " + getGlobalTableID().getNameForLogs() + " not found", ErrorCodes::LOGICAL_ERROR);
     return table;
@@ -108,7 +108,7 @@ void DatabaseCatalog::loadDatabases()
 {
     drop_delay_sec = global_context->getConfigRef().getInt("database_atomic_delay_before_drop_table_sec", default_drop_delay_sec);
 
-    auto db_for_temporary_and_external_tables = std::make_shared<DatabaseMemory>(TEMPORARY_DATABASE);
+    auto db_for_temporary_and_external_tables = std::make_shared<DatabaseMemory>(TEMPORARY_DATABASE, *global_context);
     attachDatabase(TEMPORARY_DATABASE, db_for_temporary_and_external_tables);
 
     loadMarkedAsDroppedTables();
@@ -206,7 +206,7 @@ DatabaseAndTable DatabaseCatalog::getTableImpl(
         database = it->second;
     }
 
-    auto table = database->tryGetTable(table_id.table_name);
+    auto table = database->tryGetTable(table_id.table_name, *global_context);
     if (!table && exception)
             exception->emplace("Table " + table_id.getNameForLogs() + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
 
@@ -331,7 +331,7 @@ bool DatabaseCatalog::isTableExist(const DB::StorageID & table_id) const
         if (iter != databases.end())
             db = iter->second;
     }
-    return db && db->isTableExist(table_id.table_name);
+    return db && db->isTableExist(table_id.table_name, *global_context);
 }
 
 void DatabaseCatalog::assertTableDoesntExist(const StorageID & table_id) const
diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp
index 147ef7d739b..bce15434418 100644
--- a/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/src/Interpreters/InterpreterCreateQuery.cpp
@@ -504,7 +504,7 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const
         String as_database_name = context.resolveDatabase(create.as_database);
         String as_table_name = create.as_table;
 
-        ASTPtr as_create_ptr = DatabaseCatalog::instance().getDatabase(as_database_name)->getCreateTableQuery(as_table_name);
+        ASTPtr as_create_ptr = DatabaseCatalog::instance().getDatabase(as_database_name)->getCreateTableQuery(as_table_name, context);
         const auto & as_create = as_create_ptr->as<ASTCreateQuery &>();
 
         const String qualified_name = backQuoteIfNeed(as_database_name) + "." + backQuoteIfNeed(as_table_name);
@@ -546,7 +546,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create)
         bool if_not_exists = create.if_not_exists;
 
         // Table SQL definition is available even if the table is detached
-        auto query = database->getCreateTableQuery(create.table);
+        auto query = database->getCreateTableQuery(create.table, context);
         create = query->as<ASTCreateQuery &>(); // Copy the saved create query, but use ATTACH instead of CREATE
         create.attach = true;
         create.attach_short_syntax = true;
@@ -608,7 +608,7 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create,
         guard = DatabaseCatalog::instance().getDDLGuard(create.database, table_name);
 
         /// Table can be created before or it can be created concurrently in another thread, while we were waiting in DDLGuard.
-        if (database->isTableExist(table_name))
+        if (database->isTableExist(table_name, context))
         {
             /// TODO Check structure of table
             if (create.if_not_exists)
diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp
index ecdad0168f0..8015e28d467 100644
--- a/src/Interpreters/InterpreterDropQuery.cpp
+++ b/src/Interpreters/InterpreterDropQuery.cpp
@@ -234,7 +234,7 @@ BlockIO InterpreterDropQuery::executeToDatabase(const String & database_name, AS
                 ASTDropQuery query;
                 query.kind = kind;
                 query.database = database_name;
-                for (auto iterator = database->getTablesIterator(); iterator->isValid(); iterator->next())
+                for (auto iterator = database->getTablesIterator(context); iterator->isValid(); iterator->next())
                 {
                     query.table = iterator->name();
                     executeToTable({query.database, query.table}, query);
diff --git a/src/Interpreters/InterpreterShowCreateQuery.cpp b/src/Interpreters/InterpreterShowCreateQuery.cpp
index 9938910cf1e..30005c7b169 100644
--- a/src/Interpreters/InterpreterShowCreateQuery.cpp
+++ b/src/Interpreters/InterpreterShowCreateQuery.cpp
@@ -50,7 +50,7 @@ BlockInputStreamPtr InterpreterShowCreateQuery::executeImpl()
         auto resolve_table_type = show_query->temporary ? Context::ResolveExternal : Context::ResolveOrdinary;
         auto table_id = context.resolveStorageID(*show_query, resolve_table_type);
         context.checkAccess(AccessType::SHOW_COLUMNS, table_id);
-        create_query = DatabaseCatalog::instance().getDatabase(table_id.database_name)->getCreateTableQuery(table_id.table_name);
+        create_query = DatabaseCatalog::instance().getDatabase(table_id.database_name)->getCreateTableQuery(table_id.table_name, context);
     }
     else if ((show_query = query_ptr->as<ASTShowCreateDatabaseQuery>()))
     {
diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp
index f3d48651148..557fb4c71e8 100644
--- a/src/Interpreters/InterpreterSystemQuery.cpp
+++ b/src/Interpreters/InterpreterSystemQuery.cpp
@@ -144,7 +144,7 @@ void InterpreterSystemQuery::startStopAction(StorageActionBlockType action_type,
         auto access = context.getAccess();
         for (auto & elem : DatabaseCatalog::instance().getDatabases())
         {
-            for (auto iterator = elem.second->getTablesIterator(); iterator->isValid(); iterator->next())
+            for (auto iterator = elem.second->getTablesIterator(context); iterator->isValid(); iterator->next())
             {
                 if (!access->isGranted(log, getRequiredAccessType(action_type), elem.first, iterator->name()))
                     continue;
@@ -332,7 +332,7 @@ StoragePtr InterpreterSystemQuery::tryRestartReplica(const StorageID & replica,
     {
         /// If table was already dropped by anyone, an exception will be thrown
         auto table_lock = table->lockExclusively(context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout);
-        create_ast = database->getCreateTableQuery(replica.table_name);
+        create_ast = database->getCreateTableQuery(replica.table_name, context);
 
         database->detachTable(replica.table_name);
     }
@@ -369,7 +369,7 @@ void InterpreterSystemQuery::restartReplicas(Context & system_context)
     for (auto & elem : catalog.getDatabases())
     {
         DatabasePtr & database = elem.second;
-        for (auto iterator = database->getTablesIterator(); iterator->isValid(); iterator->next())
+        for (auto iterator = database->getTablesIterator(context); iterator->isValid(); iterator->next())
         {
             if (dynamic_cast<const StorageReplicatedMergeTree *>(iterator->table().get()))
                 replica_names.emplace_back(StorageID{database->getDatabaseName(), iterator->name()});
diff --git a/src/Server/ReplicasStatusHandler.cpp b/src/Server/ReplicasStatusHandler.cpp
index f2d1ffe2ee5..986af59d3a4 100644
--- a/src/Server/ReplicasStatusHandler.cpp
+++ b/src/Server/ReplicasStatusHandler.cpp
@@ -44,7 +44,7 @@ void ReplicasStatusHandler::handleRequest(Poco::Net::HTTPServerRequest & request
             if (db.second->getEngineName() == "Lazy")
                 continue;
 
-            for (auto iterator = db.second->getTablesIterator(); iterator->isValid(); iterator->next())
+            for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next())
             {
                 const auto & table = iterator->table();
                 StorageReplicatedMergeTree * table_replicated = dynamic_cast<StorageReplicatedMergeTree *>(table.get());
diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp
index fb6d88c8d33..6f76ae4b7b3 100644
--- a/src/Storages/StorageMerge.cpp
+++ b/src/Storages/StorageMerge.cpp
@@ -62,7 +62,7 @@ StorageMerge::StorageMerge(
 template <typename F>
 StoragePtr StorageMerge::getFirstTable(F && predicate) const
 {
-    auto iterator = getDatabaseIterator();
+    auto iterator = getDatabaseIterator(global_context);
 
     while (iterator->isValid())
     {
@@ -110,7 +110,7 @@ QueryProcessingStage::Enum StorageMerge::getQueryProcessingStage(const Context &
 {
     auto stage_in_source_tables = QueryProcessingStage::FetchColumns;
 
-    DatabaseTablesIteratorPtr iterator = getDatabaseIterator();
+    DatabaseTablesIteratorPtr iterator = getDatabaseIterator(context);
 
     size_t selected_table_size = 0;
 
@@ -329,7 +329,7 @@ Pipes StorageMerge::createSources(const SelectQueryInfo & query_info, const Quer
 StorageMerge::StorageListWithLocks StorageMerge::getSelectedTables(const String & query_id, const Settings & settings) const
 {
     StorageListWithLocks selected_tables;
-    auto iterator = getDatabaseIterator();
+    auto iterator = getDatabaseIterator(global_context);
 
     while (iterator->isValid())
     {
@@ -349,7 +349,7 @@ StorageMerge::StorageListWithLocks StorageMerge::getSelectedTables(
         const ASTPtr & query, bool has_virtual_column, const String & query_id, const Settings & settings) const
 {
     StorageListWithLocks selected_tables;
-    DatabaseTablesIteratorPtr iterator = getDatabaseIterator();
+    DatabaseTablesIteratorPtr iterator = getDatabaseIterator(global_context);
 
     auto virtual_column = ColumnString::create();
 
@@ -384,12 +384,12 @@ StorageMerge::StorageListWithLocks StorageMerge::getSelectedTables(
 }
 
 
-DatabaseTablesIteratorPtr StorageMerge::getDatabaseIterator() const
+DatabaseTablesIteratorPtr StorageMerge::getDatabaseIterator(const Context & context) const
 {
     checkStackSize();
     auto database = DatabaseCatalog::instance().getDatabase(source_database);
     auto table_name_match = [this](const String & table_name_) { return table_name_regexp.match(table_name_); };
-    return database->getTablesIterator(table_name_match);
+    return database->getTablesIterator(context, table_name_match);
 }
 
 
diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h
index ebe40109d84..401c5facf0c 100644
--- a/src/Storages/StorageMerge.h
+++ b/src/Storages/StorageMerge.h
@@ -61,7 +61,7 @@ private:
     template <typename F>
     StoragePtr getFirstTable(F && predicate) const;
 
-    DatabaseTablesIteratorPtr getDatabaseIterator() const;
+    DatabaseTablesIteratorPtr getDatabaseIterator(const Context & context) const;
 
     NamesAndTypesList getVirtuals() const override;
 
diff --git a/src/Storages/System/StorageSystemColumns.cpp b/src/Storages/System/StorageSystemColumns.cpp
index ab824fc8bdc..8eb8856512e 100644
--- a/src/Storages/System/StorageSystemColumns.cpp
+++ b/src/Storages/System/StorageSystemColumns.cpp
@@ -301,7 +301,7 @@ Pipes StorageSystemColumns::read(
             const DatabasePtr database = databases.at(database_name);
             offsets[i] = i ? offsets[i - 1] : 0;
 
-            for (auto iterator = database->getTablesIterator(); iterator->isValid(); iterator->next())
+            for (auto iterator = database->getTablesIterator(context); iterator->isValid(); iterator->next())
             {
                 const String & table_name = iterator->name();
                 storages.emplace(std::piecewise_construct,
diff --git a/src/Storages/System/StorageSystemGraphite.cpp b/src/Storages/System/StorageSystemGraphite.cpp
index cde75aa5550..bfa25a99838 100644
--- a/src/Storages/System/StorageSystemGraphite.cpp
+++ b/src/Storages/System/StorageSystemGraphite.cpp
@@ -25,7 +25,7 @@ NamesAndTypesList StorageSystemGraphite::getNamesAndTypes()
 /*
  * Looking for (Replicated)*GraphiteMergeTree and get all configuration parameters for them
  */
-static StorageSystemGraphite::Configs getConfigs()
+static StorageSystemGraphite::Configs getConfigs(const Context & context)
 {
     const Databases databases = DatabaseCatalog::instance().getDatabases();
     StorageSystemGraphite::Configs graphite_configs;
@@ -36,7 +36,7 @@ static StorageSystemGraphite::Configs getConfigs()
         if (db.second->getEngineName() == "Lazy")
             continue;
 
-        for (auto iterator = db.second->getTablesIterator(); iterator->isValid(); iterator->next())
+        for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next())
         {
             const auto & table = iterator->table();
 
@@ -71,9 +71,9 @@ static StorageSystemGraphite::Configs getConfigs()
     return graphite_configs;
 }
 
-void StorageSystemGraphite::fillData(MutableColumns & res_columns, const Context &, const SelectQueryInfo &) const
+void StorageSystemGraphite::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const
 {
-    Configs graphite_configs = getConfigs();
+    Configs graphite_configs = getConfigs(context);
 
     for (const auto & config : graphite_configs)
     {
diff --git a/src/Storages/System/StorageSystemMutations.cpp b/src/Storages/System/StorageSystemMutations.cpp
index d4a262860dc..685565d82e1 100644
--- a/src/Storages/System/StorageSystemMutations.cpp
+++ b/src/Storages/System/StorageSystemMutations.cpp
@@ -51,7 +51,7 @@ void StorageSystemMutations::fillData(MutableColumns & res_columns, const Contex
 
         const bool check_access_for_tables = check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, db.first);
 
-        for (auto iterator = db.second->getTablesIterator(); iterator->isValid(); iterator->next())
+        for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next())
         {
             if (!dynamic_cast<const MergeTreeData *>(iterator->table().get()))
                 continue;
diff --git a/src/Storages/System/StorageSystemPartsBase.cpp b/src/Storages/System/StorageSystemPartsBase.cpp
index 6356e6d699e..3b97cbb2d9b 100644
--- a/src/Storages/System/StorageSystemPartsBase.cpp
+++ b/src/Storages/System/StorageSystemPartsBase.cpp
@@ -111,7 +111,7 @@ StoragesInfoStream::StoragesInfoStream(const SelectQueryInfo & query_info, const
                 const DatabasePtr database = databases.at(database_name);
 
                 offsets[i] = i ? offsets[i - 1] : 0;
-                for (auto iterator = database->getTablesIterator(); iterator->isValid(); iterator->next())
+                for (auto iterator = database->getTablesIterator(context); iterator->isValid(); iterator->next())
                 {
                     String table_name = iterator->name();
                     StoragePtr storage = iterator->table();
diff --git a/src/Storages/System/StorageSystemReplicas.cpp b/src/Storages/System/StorageSystemReplicas.cpp
index 734d0098d8d..251b45e44b6 100644
--- a/src/Storages/System/StorageSystemReplicas.cpp
+++ b/src/Storages/System/StorageSystemReplicas.cpp
@@ -76,7 +76,7 @@ Pipes StorageSystemReplicas::read(
         if (db.second->getEngineName() == "Lazy")
             continue;
         const bool check_access_for_tables = check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, db.first);
-        for (auto iterator = db.second->getTablesIterator(); iterator->isValid(); iterator->next())
+        for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next())
         {
             if (!dynamic_cast<const StorageReplicatedMergeTree *>(iterator->table().get()))
                 continue;
diff --git a/src/Storages/System/StorageSystemReplicationQueue.cpp b/src/Storages/System/StorageSystemReplicationQueue.cpp
index a6f0af8cc6c..2c188cf3734 100644
--- a/src/Storages/System/StorageSystemReplicationQueue.cpp
+++ b/src/Storages/System/StorageSystemReplicationQueue.cpp
@@ -60,7 +60,7 @@ void StorageSystemReplicationQueue::fillData(MutableColumns & res_columns, const
 
         const bool check_access_for_tables = check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, db.first);
 
-        for (auto iterator = db.second->getTablesIterator(); iterator->isValid(); iterator->next())
+        for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next())
         {
             if (!dynamic_cast<const StorageReplicatedMergeTree *>(iterator->table().get()))
                 continue;
diff --git a/src/Storages/System/StorageSystemTables.cpp b/src/Storages/System/StorageSystemTables.cpp
index 2caf599fe51..2bf6595bf53 100644
--- a/src/Storages/System/StorageSystemTables.cpp
+++ b/src/Storages/System/StorageSystemTables.cpp
@@ -232,7 +232,7 @@ protected:
             const bool check_access_for_tables = check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, database_name);
 
             if (!tables_it || !tables_it->isValid())
-                tables_it = database->getTablesIterator();
+                tables_it = database->getTablesIterator(context);
 
             const bool need_lock_structure = needLockStructure(database, getPort().getHeader());
 
@@ -331,7 +331,7 @@ protected:
 
                 if (columns_mask[src_index] || columns_mask[src_index + 1])
                 {
-                    ASTPtr ast = database->tryGetCreateTableQuery(table_name);
+                    ASTPtr ast = database->tryGetCreateTableQuery(table_name, context);
 
                     if (columns_mask[src_index++])
                         res_columns[res_index++]->insert(ast ? queryToString(ast) : "");
diff --git a/src/Storages/tests/gtest_transform_query_for_external_database.cpp b/src/Storages/tests/gtest_transform_query_for_external_database.cpp
index df6ee2bd0a5..480852f9b6b 100644
--- a/src/Storages/tests/gtest_transform_query_for_external_database.cpp
+++ b/src/Storages/tests/gtest_transform_query_for_external_database.cpp
@@ -30,7 +30,7 @@ struct State
     explicit State(Context & context_) : context(context_)
     {
         registerFunctions();
-        DatabasePtr database = std::make_shared<DatabaseMemory>("test");
+        DatabasePtr database = std::make_shared<DatabaseMemory>("test", context);
         database->attachTable("table", StorageMemory::create(StorageID("test", "table"), ColumnsDescription{columns}, ConstraintsDescription{}));
         context.makeGlobalContext();
         DatabaseCatalog::instance().attachDatabase("test", database);
diff --git a/src/TableFunctions/TableFunctionMerge.cpp b/src/TableFunctions/TableFunctionMerge.cpp
index 7a161c13bdc..cd924270f7c 100644
--- a/src/TableFunctions/TableFunctionMerge.cpp
+++ b/src/TableFunctions/TableFunctionMerge.cpp
@@ -22,7 +22,7 @@ namespace ErrorCodes
 }
 
 
-static NamesAndTypesList chooseColumns(const String & source_database, const String & table_name_regexp_)
+static NamesAndTypesList chooseColumns(const String & source_database, const String & table_name_regexp_, const Context & context)
 {
     OptimizedRegularExpression table_name_regexp(table_name_regexp_);
     auto table_name_match = [&](const String & table_name) { return table_name_regexp.match(table_name); };
@@ -31,7 +31,7 @@ static NamesAndTypesList chooseColumns(const String & source_database, const Str
 
     {
         auto database = DatabaseCatalog::instance().getDatabase(source_database);
-        auto iterator = database->getTablesIterator(table_name_match);
+        auto iterator = database->getTablesIterator(context, table_name_match);
 
         if (iterator->isValid())
             any_table = iterator->table();
@@ -69,7 +69,7 @@ StoragePtr TableFunctionMerge::executeImpl(const ASTPtr & ast_function, const Co
 
     auto res = StorageMerge::create(
         StorageID(getDatabaseName(), table_name),
-        ColumnsDescription{chooseColumns(source_database, table_name_regexp)},
+        ColumnsDescription{chooseColumns(source_database, table_name_regexp, context)},
         source_database,
         table_name_regexp,
         context);

From 05642d2a84bae35313be9050070425158e8b596e Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Thu, 28 May 2020 23:53:13 +0300
Subject: [PATCH 139/184] Update extended-roadmap.md

---
 docs/ru/whats-new/extended-roadmap.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/ru/whats-new/extended-roadmap.md b/docs/ru/whats-new/extended-roadmap.md
index 0bedfda57b1..091e7f37c66 100644
--- a/docs/ru/whats-new/extended-roadmap.md
+++ b/docs/ru/whats-new/extended-roadmap.md
@@ -124,7 +124,7 @@ Q2.
 Upd. Олег будет делать только ча�ть про HDFS.
 Upd. Реализаци� поверх S3 �вл�ет�� рабочей на уровне PoC.
 
-### 1.13. У�корение запро�ов � FINAL {#uskorenie-zaprosov-s-final}
+### 1.13. + У�корение запро�ов � FINAL {#uskorenie-zaprosov-s-final}
 
 Требует 2.1. Делает [�иколай Кочетов](https://github.com/KochetovNicolai). �ужно дл� Яндек�.Метрики. Q2.
 Upd: PR [#10463](https://github.com/ClickHouse/ClickHouse/pull/10463)

From 2918677a9e3ece71b5403e4d72b9275bbcba71ff Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Thu, 28 May 2020 23:58:25 +0300
Subject: [PATCH 140/184] Update extended-roadmap.md

---
 docs/ru/whats-new/extended-roadmap.md | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/docs/ru/whats-new/extended-roadmap.md b/docs/ru/whats-new/extended-roadmap.md
index 091e7f37c66..895138e1a33 100644
--- a/docs/ru/whats-new/extended-roadmap.md
+++ b/docs/ru/whats-new/extended-roadmap.md
@@ -283,24 +283,20 @@ Upd. Сейча� об�уждает��, как �делать другую з
 
 ### 4.3. Ограничение чи�ла одновременных �качиваний � реплик {#ogranichenie-chisla-odnovremennykh-skachivanii-s-replik}
 
-Дмитрий Григорьев, ВШЭ.
 Изначально делал Олег �лек�еенков, но пока решение не готово, хот� там не так уж много доделывать.
 
 ### 4.4. Ограничение �етевой поло�ы при репликации {#ogranichenie-setevoi-polosy-pri-replikatsii}
 
-Дмитрий Григорьев, ВШЭ. �ужно дл� Метрики.
+�ужно дл� Метрики.
 
 ### 4.5. Возможно�ть продолжить передачу ку�ка данных при репликации по�ле �бо� {#vozmozhnost-prodolzhit-peredachu-kuska-dannykh-pri-replikatsii-posle-sboia}
 
-Дмитрий Григорьев, ВШЭ.
-
 ### 4.6. p2p передача дл� GLOBAL подзапро�ов {#p2p-peredacha-dlia-global-podzaprosov}
 
 ### 4.7. Ленива� загрузка множе�тв дл� IN и JOIN � помощью k/v запро�ов {#lenivaia-zagruzka-mnozhestv-dlia-in-i-join-s-pomoshchiu-kv-zaprosov}
 
 ### 4.8. Разделить background pool дл� fetch и merge {#razdelit-background-pool-dlia-fetch-i-merge}
 
-Дмитрий Григорьев, ВШЭ.
 В очереди. И�править проблему, что во��танавливающа��� реплика пере�таёт мержить. Ча�тично компен�ирует�� 4.3.
 
 
@@ -330,6 +326,7 @@ Upd. Сделано. Эффективно�ть работы под вопро�
 Метрика, БК, Маркет, Altinity уже и�пользуют более �вежие вер�ии чем LTS.
 Upd. По�вила�ь втора� вер�и� LTS - 20.3.
 
+
 ## 6. Ин�трументирование {#instrumentirovanie}
 
 ### 6.1. + И�правлени� ��мплирующего профайлера запро�ов {#ispravleniia-sempliruiushchego-profailera-zaprosov}
@@ -426,11 +423,11 @@ Upd. Ра��мотрели в�е проверки подр�д.
 
 UBSan включен в функциональных те�тах, но не включен в интеграционных те�тах. Требует 7.7.
 
-### 7.11. Включение \*San в unit те�тах {#vkliuchenie-san-v-unit-testakh}
+### 7.11. + Включение \*San в unit те�тах {#vkliuchenie-san-v-unit-testakh}
 
 У на� мало unit те�тов по �равнению � функциональными те�тами и их и�пользование не об�зательно. �о они в�ё-равно важны и нет причин не запу�кать их под в�еми видами sanitizers.
 
-Иль� Яцишин.
+Иль� Яцишин. Сделано.
 
 ### 7.12. Показывать те�товое покрытие нового кода в PR {#pokazyvat-testovoe-pokrytie-novogo-koda-v-pr}
 
@@ -529,6 +526,8 @@ Upd. Е�ть �борки, [пример](https://clickhouse-builds.s3.yandex.n
 
 Дарь� Петрова, УрФУ.
 
+Рабочий прототип: https://pulls-dashboard-demo.herokuapp.com/dashboard/ClickHouse/ClickHouse
+
 �ад ClickHouse одновременно работает большое количе�тво разработчиков, которые оформл�ют �вои изменени� в виде pull requests. Когда непомерженных pull requests много, то возникает �ложно�ть � организацией работы - непон�тно, на какой pull request �мотреть в первую очередь.
 
 Предлагает�� реализовать про�тое одно�траничное веб-приложение, в котором отображает�� �пи�ок pull requests �о �ледующей информацией:

From f0f7b7cc7cafba1376e2ac33f142f973686ea136 Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Fri, 29 May 2020 00:21:03 +0300
Subject: [PATCH 141/184] Update extended-roadmap.md

---
 docs/ru/whats-new/extended-roadmap.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/ru/whats-new/extended-roadmap.md b/docs/ru/whats-new/extended-roadmap.md
index 895138e1a33..df4bbb4ca57 100644
--- a/docs/ru/whats-new/extended-roadmap.md
+++ b/docs/ru/whats-new/extended-roadmap.md
@@ -627,6 +627,7 @@ Upd. Готово (в�е директории кроме contrib).
 ### 7.32. Обфу�каци� продакшен запро�ов {#obfuskatsiia-prodakshen-zaprosov}
 
 Роман Ильгов�кий. �ужно дл� Яндек�.Метрики.
+Е�ть pull request, почти готово: https://github.com/ClickHouse/ClickHouse/pull/10973
 
 Име� SQL запро�, требует�� выве�ти �труктуру таблиц, на которых �тот запро� будет выполнен, и заполнить �ти таблицы �лучайными данными, такими, что результат �того запро�а зави�ит от выбора подмноже�тва данных.
 

From 9d5b1b69f7c3d6279e09ab593c22f6b1ec3ec7bf Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Fri, 29 May 2020 00:22:18 +0300
Subject: [PATCH 142/184] Update extended-roadmap.md

---
 docs/ru/whats-new/extended-roadmap.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/ru/whats-new/extended-roadmap.md b/docs/ru/whats-new/extended-roadmap.md
index df4bbb4ca57..20ebe28fe5b 100644
--- a/docs/ru/whats-new/extended-roadmap.md
+++ b/docs/ru/whats-new/extended-roadmap.md
@@ -1398,11 +1398,11 @@ Constraints позвол�ют задать выражение, и�тинно�
 Ва�илий Морозов, �р�лан Гумеров, �льберт Кидрачев, ВШЭ.
 В прошлом году задачу начинал делать другой человек, но не добил�� до�таточного прогре��а.
 
-1.  Оптимизаци� top sort.
++ 1.  Оптимизаци� top sort.
 
 В ClickHouse и�пользует�� неоптимальный вариант top sort. Суть его в том, что из каждого блока до�таёт�� top N запи�ей, а затем, в�е блоки мержат��. �о до�тавание top N запи�ей у каждого �ледующего блока бе��мы�ленно, е�ли мы знаем, что из них в глобальный top N войдёт меньше. Конечно нужно реализовать вариацию на тему priority queue (heap) � бы�трым пропу�ком целых блоков, е�ли ни одна �трока не попадёт в накопленный top.
 
-2.  Рекур�ивный вариант �ортировки по кортежам.
++ 2.  Рекур�ивный вариант �ортировки по кортежам.
 
 Дл� �ортировки по кортежам и�пользует�� обычна� �ортировка � компаратором, который в цикле по �лементам кортежа делает виртуальные вызовы `IColumn::compareAt`. Это неоптимально - как из-за короткого цикла по неизве�тному в compile-time количе�тву �лементов, так и из-за виртуальных вызовов. Чтобы обойти�ь без виртуальных вызовов, е�ть метод `IColumn::getPermutation`. Он и�пользует�� в �лучае �ортировки по одному �толбцу. Е�ть вариант, что в �лучае �ортировки по кортежу, что-то похожее тоже можно применить… например, �делать метод `updatePermutation`, принимающий аргументы offset и limit, и допере�тавл�ющий пере�тановку в диапазоне значений, в которых предыдущий �толбец имел равные значени�.
 

From 2b4d9a6349a6c530d1574577254bde216ccced84 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 29 May 2020 01:00:01 +0300
Subject: [PATCH 143/184] Add a test from @donmikel

---
 .../01285_engine_join_donmikel.reference      |   1 +
 .../0_stateless/01285_engine_join_donmikel.sh |  65 ++++++++++++
 .../01285_engine_join_donmikel.tsv            | 100 ++++++++++++++++++
 3 files changed, 166 insertions(+)
 create mode 100644 tests/queries/0_stateless/01285_engine_join_donmikel.reference
 create mode 100755 tests/queries/0_stateless/01285_engine_join_donmikel.sh
 create mode 100644 tests/queries/0_stateless/01285_engine_join_donmikel.tsv

diff --git a/tests/queries/0_stateless/01285_engine_join_donmikel.reference b/tests/queries/0_stateless/01285_engine_join_donmikel.reference
new file mode 100644
index 00000000000..573541ac970
--- /dev/null
+++ b/tests/queries/0_stateless/01285_engine_join_donmikel.reference
@@ -0,0 +1 @@
+0
diff --git a/tests/queries/0_stateless/01285_engine_join_donmikel.sh b/tests/queries/0_stateless/01285_engine_join_donmikel.sh
new file mode 100755
index 00000000000..8a867921dbe
--- /dev/null
+++ b/tests/queries/0_stateless/01285_engine_join_donmikel.sh
@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+. $CURDIR/../shell_config.sh
+
+$CLICKHOUSE_CLIENT --multiquery --query "
+DROP TABLE IF EXISTS NmSubj;
+DROP TABLE IF EXISTS events;
+
+create table NmSubj
+(
+    NmId      UInt32,
+    SubjectId UInt32
+)
+    engine = Join(All, inner, NmId);
+
+create table events
+(
+    EventDate       Date,
+    EventDateTime   DateTime,
+    EventId         String,
+    SessionId       FixedString(36),
+    PageViewId      FixedString(36),
+    UserId          UInt64,
+    UniqUserId      FixedString(36),
+    UrlReferrer     String,
+    Param1          String,
+    Param2          String,
+    Param3          String,
+    Param4          String,
+    Param5          String,
+    Param6          String,
+    Param7          String,
+    Param8          String,
+    Param9          String,
+    Param10         String,
+    ApplicationType UInt8,
+    Locale          String,
+    Lang            String,
+    Version         String,
+    Path            String,
+    QueryString     String,
+    UserHostAddress UInt32
+)
+    engine = MergeTree()
+        PARTITION BY (toYYYYMM(EventDate), EventId)
+        ORDER BY (EventId, EventDate, Locale, ApplicationType, intHash64(UserId))
+        SAMPLE BY intHash64(UserId)
+        SETTINGS index_granularity = 8192;
+
+insert into NmSubj values (1, 1), (2, 2), (3, 3);
+"
+
+$CLICKHOUSE_CLIENT --query "INSERT INTO events FORMAT TSV" < ${CURDIR}/01285_engine_join_donmikel.tsv
+
+$CLICKHOUSE_CLIENT --query "
+SELECT toInt32(count() / 24) as Count
+FROM events as e INNER JOIN NmSubj as ns
+ON ns.NmId = toUInt32(e.Param1)
+WHERE e.EventDate = today() - 7 AND e.EventId = 'GCO' AND ns.SubjectId = 2073"
+
+$CLICKHOUSE_CLIENT --multiquery --query "
+DROP TABLE NmSubj;
+DROP TABLE events;
+"
diff --git a/tests/queries/0_stateless/01285_engine_join_donmikel.tsv b/tests/queries/0_stateless/01285_engine_join_donmikel.tsv
new file mode 100644
index 00000000000..a6f47144ff2
--- /dev/null
+++ b/tests/queries/0_stateless/01285_engine_join_donmikel.tsv
@@ -0,0 +1,100 @@
+2020-02-17	2020-02-17 19:40:09	EMC	wFxiX8iZ3Vns8pO6bøx0Wv¶YftGEOM9WMff	Ku5ofnKw„BaQ0CtcdZ÷a6sAänSQbyufu0zfd	0	X16qWFb_‹h8JOÈNLGxs7oiXFZTdBFfTniRMG		mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=33&userId=274574634&email=Z21haWwucnU%3D&source=mn	657583482
+2020-02-17	2020-02-17 20:23:53	EMC	foco1fj5Li6ey>MSfn?jKHBBdiwKXiBmy7Ni	ICZBjN7en5snnszHKbTVUU*xyOhuZEgI1EK	0	I7XAKnCMsD9tdFHmmècCrI~KqiKQdo7Gxd5		mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=E17727463464LnJ1&source=mn	2992671877
+2020-02-17	2020-02-17 19:49:04	EMC	q3AxEdMQhsr1NewVSK€jHzYvh63XdxtGcBZX	GVGMPZheŸnJd8Ý3JarÙI7Qicq5BKRfijT0ls	18536997	D_myvD3U4bmy5anwdNRFaxm KNoDfcCDL9U6	http&o=mail.ru/redir?u=http&o=mail.ru/redir?u=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6userId	mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=AD0581918974621haWwuY29t&source=mn	1656424571
+2020-02-17	2020-02-17 21:55:32	EMC	ef7tm7k7‘Yvpb4XIfvf7ON-s4Xv3ztD7SkB	KIDkUzEg�cAMu%ObAQÍAkI3·jmm3638bSnX0	22341123	6NcB26yJrLhWgíldC_HvV2fçUOHNLRFvkLbA		mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=E943634627468871D581940274695&userId=2746887723D&source=mn	1429586160
+2020-02-17	2020-02-17 21:37:47	EMC	3Y_KZLEJgn4bw?omv4ÀEizIs1lUMz0Li0Ws0	I3QaVG9UÏvXWvÚWReLSHuDCœs3DdgTbX8av6	28213800	Knoofunr÷qFKdIta8öpMuR!FNX2yu1I6gNo	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https://click.mail&v=2&s=3745772	mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=5YWxpbC5jb20%3D&source=mn	872136829
+2020-02-17	2020-02-17 19:15:34	EMC	JcHEtSzMžb70K}04FW#ShpZÔw1ot_fb7wxj8	KG9bmRo6øyweNß2EX4–8CUBQcsOqzbHn6g3u	30047107	UKb5rJ5D\bFncBHxCioYswtJKAlv6Ua5MVSrj		mn										48	Ru	ru	7.0.32	/confirm/vipstatus	?key=75ydQ%3D%3D&source=mn	626057004
+2020-02-17	2020-02-17 18:15:04	EMC	_sjptcVUô5CF4øsTnXÿh0C6ÆDCYcOo0rNowf	0a0AcabOŽllf7\rN9X9Àaa7r€VmJ0J1oZGJY2	30648090	Z2K_4kffíe21iW1hl2¶jby1RXAAh5XtPLP3		mn										48	Ru	ru	7.0.32	/confirm/vipstatus	?key=5&ts=15819420%3D&source=mn	3682305993
+2020-02-17	2020-02-17 16:10:20	EMC	WM44OUvEüZse_9hQ0NJsn0«k47pX6_UnaS7	LgCWwTMZ+GwiQEbNT95zUnÍkMCXYqPs1Xt1	0	QUuSVpqq#SgMdveJDW¾0bQ6�s8jn8MLXzL1w		mn										187	Ru	ru	7.0.32	/confirm/vipstatus	?key=CCD&source=mn	726391852
+2020-02-17	2020-02-17 17:30:44	EMC	PGfQVZHuüAY0A�_kUA\04ceY�RjWqgPJtfhCn	4TR9oRKCNrouy:zJBLBi8J4=u7Mkt9jgzgmB	0	xbCoBX9pTDaWÉiRxW@F3fO€gKD9lX58oWsc		mn										187	Ru	ru	7.0.32	/confirm/vipstatus	?key=DB3D&source=pcap	2498835913
+2020-02-17	2020-02-17 19:10:51	EMC	gnd66xWl»Kf2HÔ2tiZ®dbiv•joz4ILmJ4lp_	ysgw6WLd‘bNWS•HQryÈbWjhãH9uJqnlZMWAv	0	nDQDqsnm/dlpbFMPJE93b6ŸEETHzFxUSzVv		pcap										187	Ru	ru	7.0.32	/confirm/vipstatus	?key=4&userId=274634688&userId=274688&userId=27465D85CD4F634627468877266792YXJ1&source=mn	816668334
+2020-02-17	2020-02-17 20:52:00	EMC	pBzkcTpn50WpO]BZH7^js6TpPGp7Uid7mFg	jQVdKDl4+Sp_jØ5CZcPtVo€MGTIGOfYO6Q2	0	aQGyrF9s3qUMPq_dUnÅOiM6àQVMX5PNQ_eiV		pcap										187	Ru	ru	7.0.32	/confirm/vipstatus	?key=50&email=YWlsLmNvbQ%3D%3D&source=pcap	1770779506
+2020-02-17	2020-02-17 21:31:14	EMC	cyTu3wNR†CI1TzkxCy¸bFNz\rCkrdDZQxYtlW	gmtJF_hWE8M79bWSaK·kn8UT5Xdwfb0uK8T	0	RSJUU0SB�DIXj€pYOR¬na59‹cWouc20W5VIP		mn										187	Ru	ru	7.0.32	/confirm/vipstatus	?key=4CD%3D&source=mn	3448881839
+2020-02-17	2020-02-17 17:15:46	EMC	I3bvr8ceé2SU_&UuDXƒ4J6KÐRJl5uipnL2Or	myxD8nQXÊFYnKçqBoPáGGo4Hy6hBTXRL04K	0	ntHjLMvhÉnb7b‰09Tu’7jRkM8KgS5ni0w4oH		mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=36274695&userId=2560&userId=27463465&userId=27468872EDB908&ts=158193819382	2447621965
+2020-02-17	2020-02-17 18:21:21	EMC	_468tOVLPpIk�gKvP’GX8y�pjfTZV09k8w5	0KakGIWkÅ9uaL»0Rv3]1T5wPeHPklVWzhytg	0	QAJxAuQA©DoEQêqY0H†UT56ãP0O4K5y55K_L	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https://mail&v=2&s=92Ymail&v=2&s=3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D66email&v=2&s=93D158192YWlsLnJ1%26source%3Dmn&c=swm&r=https%3A%2F%2Fwww.wildberries.ru	mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=5819392158193634&userId=274688&ts=15819408BD1haWwucnU%3D&source=mn	1547300784
+2020-02-17	2020-02-17 19:07:33	EMC	Y8yBoZ7Cÿp2GUÆTQ8srLMZJáFIZ85WgiBeau	wmIgwW88aSGvvjKwV,8EooßQonI_90taLrh	0	KXVqerLQyaMaMtuT1FzUjr‘Lev8Fw11mkfO		pcap										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=72846239707&userId=274688&ts=1581921haWwucnU%3D&source=pcap	2777789095
+2020-02-17	2020-02-17 11:27:36	EMC	n66D1ZN9·C6y7ëLASQ›ucHNdkw_QseVRnt32	yeqFeZFa,7cQ_JyTO3yJJKSQaCKEWERPx4Z	24388614	6h91HeiPŠyIKw8yV0p\tcaK99w_zmnmXU77RI	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https://click.mail.ru/redir?u=https://click.mail&v=2&s=3D158192YWlsLnJ1%26source%3Dmn&c=swm&r=http&o=mail.ru/redir?u=https://click.mail.ru/redir?u=https://mail%3DYWlsLnJ1%26source	mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=6167462E4CD570A0312D6BF1BtYWlsLnJ1&source=mn	662531688
+2020-02-17	2020-02-17 14:11:06	EMC	jamANU2z“ei_lBkfEk_Im5ow0gWB5G0gL1S	So7Jc6gheYgclfoq8Þl0hqòsPExrMNSXPLV	20685668	nwRXATVEY2PaYr34qw•tZX0©Q6ah_sceXjbL	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=http&o=mail&v=2&s=9444%26email.ru/redir?u=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69103Dmn&c=swm&r=http&o=mail&v=2&s=92YWlsLnJ1%26source%3Dpc&c=swm&r=https	mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=420%3D&source=mn	3233174651
+2020-02-17	2020-02-17 14:32:04	EMC	lsruHZ0Ywg1yDdrio.j5pi€NMTnqNxeaOQJ	G6K62xDbêm9aJ_kNlLqEo4AhVf46wDtWi	32046518	wgpf1yqV»tVhLL7yT9sp1RbqaXGF81GmSiQ6		mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=1581943D&source=pcap	3183159832
+2020-02-17	2020-02-17 15:11:37	EMC	5EW8o4vD¥850H>TQamÑ3NA2@egQm32jpASIF	KJKN5QAHdoqw_˜RGolâ5hLvìSq6753cKLswM	32971244	8Vh3DVlC­jfXs[5hqV“MGDDP2iBjoiqhoACB	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https://mail&v=2&s=3D27457AD3D274576userId%3D27461haWwucnU%253D%26source%3Dmn&c=swm&r=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6ts%3D158193Fkey%3D669%26email&v=2&s=3Fkey%3D69D01	mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=5634&userId=274695&ts=15819440446274695&userId=27465&email=Z21haWwucnU%3D&source=pcap	574798405
+2020-02-17	2020-02-17 13:59:25	EMC	i6LAlVL3U0GUF”Ftji¡RgfPyjyattjzds6y2	IYv_jZ2k+bNO3¿3nbLëKRRG„cJvTHVIueEDI	21779069	eUgOCUGbEZM15\'Of0VÜz6zQmAT0T0zX3XSO	http&o=mail.ru/redir?u=http&o=mail.ru/redir?u=https%3A%2F%2Fwww.wildberries	mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=D1F9sail=YWxpbC5ydQ%3D%3D&source=mn	3475274476
+2020-02-17	2020-02-17 06:40:47	EMC	A_3372QT5xhTmüjNrokk3yçZCtuOm52FfZT	oOD7es9hÅGUs7ôjB3_)jVKhŠ3Oo10IQvAu01	27168564	UYEHX0it¨bLKZêjChWˆxLsP÷4a7LUXQLX4iB		mn										48	Ru	ru	7.0.32	/confirm/vipstatus	?key=467468874634&email=dG92&userId=274695&email=email=a2B7AD60&email=bWFpbC5ydQ%3D%3D&source=mn	3084609784
+2020-02-17	2020-02-17 14:10:13	EMC	WSlfONgzîmGj8ákylbªItxLW8MX4u60oapxy	U8TMovlAcJTcA~u3KoOFG7öpchP3CBPbcB4	28904171	nBfRyelBAkOfRó70tIzBusu½S0M8_KbRsOGI		pcap										187	Ru	ru	7.0.32	/confirm/vipstatus	?key=4462F5B37274695&email=Ymail=Z21haWwucnU%3D&source=mn	2020344465
+2020-02-17	2020-02-17 01:43:57	EMC	khjsNWq7Iq56¦4D1u\'vWuK…ZHN4kfq0Kyfv	hcAK5RGO jZHYíbFJwôufN7\tZeXCRWAEKoT2	0	G0Zi9TAk`t0aUÄcHRi…Ocb1|RgoT8xMOH4KV		mn										187	Ru	ru	7.0.32	/confirm/vipstatus	?key=1581940095&ts=158193819382&userId=27465D8B8F1936256C35B4B466&userId=2746887C1A97&userId=274695&userId=274634&userId	902363005
+2020-02-17	2020-02-17 02:17:35	EMC	H56Q4qwa\'eYiF‡CmGR40Pp7,m6fDiE2nJSmB	Efk7W3ZrgNsEi.anvkrteuN>lF2UlX0RZFzr	0	QhyRjVGEsD_4YaQiaL�orQ6þy0fbCxkEVzQ5		pcap										187	Ru	ru	7.0.32	/confirm/vipstatus	?key=44695&email=YW5hcnU%3D&source=mn	2161496531
+2020-02-17	2020-02-17 03:20:48	EMC	1wBv9Q7r¬DQuImuif>0v1t¬8OXWSIXymKrI	sNujW7iaÜtZLPxEOpI1fZdT4p0QRaotsXPZ	0	LSKNxAcrªTMsADK9VI7AMqm­05blzLWXCJ05		pcap										187	Ru	ru	7.0.32	/confirm/vipstatus	?key=AE1FAC2BFA1E75hce=mn	985607445
+2020-02-17	2020-02-17 06:18:16	EMC	6dqrbyyKHWPBEöyPZG«bJRKjxtmt0Aq8IsON	k57VHYX59Ws9u[L3nN±EruOÝGCVgGH_l27J1	0	_lwuZQycÍqGeyMxkRSM7pR1³lHNU6_V0nVWG		pcap										187	Ru	ru	7.0.32	/confirm/vipstatus	?key=B27465D88&userId=274572BE3274577273AF0&email=a274578&userId=274692&userId=274688&userId=27465D8192	130983489
+2020-02-17	2020-02-17 09:20:13	EMC	UCLTP56_±YcQ9„rWVJa8rif2Aq5NNqVThlwH	7Rx9OsBsª7BemòVxuAˆrwZ7N5BqcFwK7CSI	0	JOIbkOPdŽPPbPÐCtqRV6wQH¨sdWYNnmarCxm		mn										187	Ru	ru	7.0.32	/confirm/vipstatus	?key=A6E819420%3D&source=pcap	2567294448
+2020-02-17	2020-02-17 13:05:00	EMC	28AZ4n0zpsVuwzenZ6*gd0VÒNMjnT8Kmfncv	PFiGZRLT÷jK1CxfLJajtGVFOATfWnTE00	0	QYPUzvwV=w8ev0E77B|RK6Z<66ZX5y1YIK1J		mn										187	Ru	ru	7.0.32	/confirm/vipstatus	?key=C19213C0060312E9E28A82&email=bmFA2095&userId=2746346&userId=27457455266&ts=15819213D&source=pcap	2881908356
+2020-02-17	2020-02-17 13:40:53	EMC	uDOkWKoTÆLCbB2SFYoepda0itJJ9zJgyZVY	d50Y87RQCO70èci0u{QDZc°A8dU31qm0IS1	0	0MuqyXoT‚tKYpØmheIçD1Ai™XWz9sZ0zroDj		mn										187	Ru	ru	7.0.32	/confirm/vipstatus	?key=A3274695&userId=2746346470&userId=274634&userId=256AEA56926&ts=158194407&userId=27463467720%3D&source=pcap	700256219
+2020-02-17	2020-02-17 13:56:10	EMC	rOXnSk45QcoEkw03yMYbGUhD_YZ_yCx9OU	DyGXuVg8;P6Tqï8CDp2_0XI`zPz4wuG8iJJh	0	3IM74UyhDyv2Jamik02pDQJøu_L10vfPCKTm		mn										187	Ru	ru	7.0.32	/confirm/vipstatus	?key=459&email=bmFA81942095&userId=27468872095&userId=27457F1944672Y29t&source=pcap	607085190
+2020-02-17	2020-02-17 13:57:55	EMC	xo09Zqxj”uPBo“_2aY±OgnG\\sH3HDRtPH7xW	1I3ceRxYüeGTFgNy30ÇojqJ]qrOgqu1Ag8yO	0	PwQkOkG2‰N1_p*ZFjOwmM6FviVATMGY5_rbK		mn										187	Ru	ru	7.0.32	/confirm/vipstatus	?key=158193979967F628BBA48E274634&email=dGV4LnJ1&source=mn	3851759707
+2020-02-17	2020-02-17 14:09:55	EMC	vmlDYHpCèg270cyeLJÜZBJTÝwBBnnJHH99Av	l9FPMTHyyKlHijgeO]2oJK90OO8hWvntXS8	0	0Xu7Ha8e³qQdwcG3pm#GNMw6HzEsxoAszdK		mn										187	Ru	ru	7.0.32	/confirm/vipstatus	?key=BtYWlsLnJ1&source=mn	2783694582
+2020-02-17	2020-02-17 15:31:52	EMC	WIzjCT0nä1XqAlOuRi2YskE’x619ZjBaFe4g	z_NGqjbLB0IhQöV1Sx‰DRMNG0DlMUxPlqaQ6	0	bv75_7V5êB4pBB2aNK\fRwnB·DQm0ngrcFPLN		mn										187	Ru	ru	7.0.32	/confirm/vipstatus	?key=158193628FA603193692&email=bmFA934&userId=265&ts=158192YW5haWwucnU%3D&source=pcap	722494088
+2020-02-17	2020-02-17 06:28:37	EMC	434gE67EâZuzQ L3ss<YSbsÝlSKkc3m3x_yv	z1Pq6pVwzBbbooT7uGVDX_}tTYKMpBkc4sR	0	yDB73mVPÖ4_s4¶5wcAÜFidijmoz9Z8Z42YgR		mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=EAC5jb20%3D&source=mn	1715283655
+2020-02-17	2020-02-17 13:00:54	EMC	YtB5N7m6èfG2TÄYx7FdUcwmexJvYphKtmqW	hAd5dqRmE0nQI4zKofhKFll6AVX08xJOYk	0	2nUgIwJhýwYTtŸeNxh1H_zOJcbKA6y_ECSO		mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=BtYWlsLnJ1&source=mn	3600915089
+2020-02-17	2020-02-17 11:52:28	EMC	zTqvvbns™qeoO�JoDNõbX6d…C4WFm5s0aN0J	62Nwk85BŒaoJb
T5QSÓa16QQGhmItxS9DiK	0	LAdBkCNA®3XVYEazFQlM65aƒHvJLa2Uwcbou		mn										48	Ru	ru	7.0.32	/confirm/vipstatus	?key=AA4D&source=pcap	3757909189
+2020-02-17	2020-02-17 10:47:40	EMC	j72d0hbmaSx8z£DgG3tQJ4I2GADb2ODpcjr	OXHTehM1qzfV¹kDM3L820Dj9W9jM1w6M8rn	0	c_q9hJgP¶HrdU¤lhlHhiA4uycDO_boRaNMT_		mn										187	Ru	ru	7.0.32	/confirm/vipstatus	?key=462DF2921355274634&userId=274692&ts=15819397A7C5ydQ%3D%3D&source=mn	1740722989
+2020-02-17	2020-02-17 11:04:34	EMC	ym0RU_o2Ô2n0hkvvZOÿOvxP4LYfXtzgmsjF	sEUZx6WdxQKfQ#p8YgåX07I[kfwWziyMCQ0A	0	JsbtDnKA†5QYPêwgZoÖnSmRUk4td1meycR2y		mn										187	Ru	ru	7.0.32	/confirm/vipstatus	?key=666&ts=15819382&email=YW5hce=pcap	3685826953
+2020-02-17	2020-02-17 15:35:55	EMC	cvPUJNgQÿb6omlvnDp/jZQtbpAmtLBDbrkvl	Uof6vXhjçCJ2rWMT7P¡SR1in0v1NOLW9fBhf	0	noQAt7IALqk4WfBYOâspAiÝOqvCOMDBX0It		mn										187	Ru	ru	7.0.32	/confirm/vipstatus	?key=EF31940BE3974634&userId=27457095&ts=158193634634&email=dGdtYWlsLnJ1&source=mn	2225930849
+2020-02-17	2020-02-17 15:36:22	EMC	bAiQYrkn¨NTsK˜x2Xzz3Y0Y|htnTleAO6ykM	6g6IaOBb2ySmØBtlcY29nªFc0tkHBDefJ5	0	dyVlnui8k6b7eWHdwçg8q9ÁAHZaMvK_nvcJ		mn										187	Ru	ru	7.0.32	/confirm/vipstatus	?key=E760&userId=27465&userId=274688&email=Z21haWwuY29t&source=mn	1995672768
+2020-02-17	2020-02-17 12:06:44	EMC	hK6YNczKhnANLb_v1B�Ac02ÚmmJ4v3eHlP1v	2zBNHZW3Žbhf0Æ26Se±hB6b¿Ns3ibmJjnHZq	30807093	a7HUuGsA\rHDZoÿ0qB6\\eEiI.mhFXMDkuWVok	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https://click.mail%3DYWlsLnJ1%26source%3Dmn&c=swm&r=https://click.mail%3DYWlsLnJ1%26source%3Dmn&c=swm&r=http&o=mail%3DZG1haWwucnU%253D%26source%3Dmn&c=swm&r=http&o=mail.ru/redir	mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=EA1942095&userId=274552095&email=Z21haWwuY29t&source=mn	1131763241
+2020-02-17	2020-02-17 13:29:00	EMC	yL4jFOc_2M63V‹zEuQyNNN:l3yf54Az0gBM	y8Mk0a0v²xOHa)7mp0)npYs§JTlKq73zNSH0	32458442	_Q8lQMngíPyH2qGZvp_yjqÂF2x5K0Mm_Pxe		mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=C6346887465&ts=1581940C5ydQ%3D%3D&source=mn	3476824011
+2020-02-17	2020-02-17 13:35:20	EMC	nQ8H1CE0ºzu5ZËdAVjsr16YBwMvcC2RrTSE3	xxeQBboi\fChONZrrlz¯LwJvÂbKcTM0BhtRT0	23232160	GbEGq80sÏBaiAóQc0vëjpypÕ88CmkMJjyHVq		mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=4E9BC8&email=a2B634&email=YWlsLnJ1&source=mn	1803353690
+2020-02-17	2020-02-17 14:35:33	EMC	6q7D5tuGBcVx4«971wšeVnBïR2wSldXMEi9i	ws0bhhM0!yYYb OD33iTffOtXR7Z3SQyWu5	26234412	diKZMgVo•uQYP\'GVQybabDRGHgyfohtzfmK		mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=5D82&ts=1581921haWwuY29t&source=pcap	1534869198
+2020-02-17	2020-02-17 14:56:20	EMC	6LPYg86GÉ0SWz�c45KEYeM0�35bisDLk9BVq	fQqF75EIUhQxVRWgBkvGeqIkEQ_ev_2ko3	32251365	HrU8iJjt|_h3iÉnvMb7psYVŸsJJVsYLvLZs4	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=http&o=mail&v=2&s=3c	mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=5ydQ%3D%3D&source=mn	1107116720
+2020-02-17	2020-02-17 15:57:00	EMC	8bVyLB64õZ_fCUL3R„68gI\t818cRZJwctr7	4ANTHKEQeWVL´Cyl8¦bT3s�pOxYtr4uS0FA	26326848	hjLpN8IV²D8gubNzOKAc4ÎUltehFJvxrN6	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https%3A%2F%2Fwwwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6email&v=2&s=3D2746126source%3Dmn&c=swm&r=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6ACA7746194441581944461%26source%3Dmn&c=swm&r=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6userId%3D27457461581937BBE5ydQ	pcap										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=758193634&email=bWFpbC5ydQ%3D%3D&source=pcap	3192390078
+2020-02-17	2020-02-17 06:37:18	EMC	vkWez1lPåPK87£iyEyvFzKRærsZ6gXCXwsPh	Cvezv2mmySmX6DKK[EIL7µuccdaX1Z6rGG	18846619	2ChPFtaO+dQB7´JLRaÌvUOtÛdtB1gaeAJYK7	http&o=mail.ru/redir?u=http&o=mail.ru/redir?u=https://click.mail&v=2	pcap										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=42095&email=bW95&ts=1581940FCCAD&source=pcap	1825815078
+2020-02-17	2020-02-17 17:28:02	EMC	rXmItx0Za8A6z½3nJxûqyAwÙFYgntazShou0	k3f6e_5EhvLY`rFcr0LiOiªCjjo_MxbCOhq	26234806	CKG16UI8Qxp3ÿjVz2Ûr7mS\\_Ka0_r838J0r	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https://click.mail.ru/redir?u=http&o=mail.ru/redir?u=https://click.mail%3DZG1haWwucnU%253D%26source%3Dmn&c=swm&r=https://click.mail.ru/redir?u=http&o=mail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6186	mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=EBE3D&source=mn	3849914033
+2020-02-17	2020-02-17 17:04:50	EMC	0y0AXkG7ãFyv_)diHn3nL6pxxlxN3YIbDCX	QuY2xKikýHKCO¨O37jÄdJQ8ƒtIY0xSj1P9xU	27942721	rcOVMm5600B3ðjju3Ímsnk›mTmNXN8pc37H		mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=15819189&email=Z21haWwucnU%3D&source=mn	108142745
+2020-02-17	2020-02-17 23:25:00	EMC	A8tzIc0jñyNY2ûmplQ|hXDFÌFLKSpHo5Qs0w	Mb_hLrLH‰v1WdÙW02eƒzAxfFHOy5zxu0dE2	31886688	a7ISoIAkêQuUy¡_jr7ÖB0TObOs7oy4iBp00d		mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=158194406&userId=27457B2566&ts=158193629t&source=mn	132315033
+2020-02-17	2020-02-17 09:59:38	EMC	znVyLmnCxmBmPwbI1žQRlwˆNepu6ZgHbcLz	pTD62K17àAHgoJvdK‚Hoe4á0d1ciQ8lZSVj	18096770	I_lmWWYsâT2AX°KjMOµDb72Hh2hqzjbS4t6	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6userId%3D27461005A%2F%2Fwwwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D68FDDberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D65	mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=EBE327455274634&email=c2Vsa2CAZ21haWwuY29t&source=mn	1574091904
+2020-02-17	2020-02-17 07:48:43	EMC	X_kKbaxzœj5D1c5eaaÂl9Mpr79AQyZnhsOL	Lu_EhlFvºevSAøn38B©PcyP UVBjxhkjLpuf	28018695	XWs5eS1bÔRTB2ÞKjgK.MABzòURMSDAc6NXLn	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=http&o=mail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6703D27457079D1581938194461106ts%3D158194446158193DbW94457D91haWwucnU%253D%253D%253D%26source%3Dmn&c=swm&r=https%3A%2F	mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=C70&email=Ymail=Z21haWwucnU%3D&source=pcap	1963271726
+2020-02-17	2020-02-17 04:41:36	EMC	G4IywqjHÂANSgABFcxÖyX92JkyRRaypfrhNb	QKpsJQlYÂdfkN�plibƒb7yxuxWXsqoG6PWz	22669596	A4B8xM1JXGrwV¬YZdlvdgMt0itiUFqoiE15	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https://click.mail%3DZG1haWwucnU%253D%26source%3Dmn&c=swm&r=http&o=mail.ru/redir?u=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6ts%3D1581944E6email.ru/redir?u=https%3A%2F%2Fwwwwwwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6A%2F%2Fwwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6203D158192Ymail&v=2&s=9BBE4D3D27461haWwucnU%253D%253D%26source%3Dmn&c=swm&r=https%3A%2F%2Fwww	mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=7465&userId=274695&userId=27465D86887B6&userId=274552416&ts=15819266	2384793579
+2020-02-17	2020-02-17 07:58:50	EMC	kWh5m_gAíYWjwmGYpp0yKohX3jXUWnK0ZMg	iyE6SVGecMrS9i00FßpqUG÷TAcsinxktMxS	32418178	kHtD_Sbaíx6eseNNnK´VTgHžtGA3wUEpO_vJ	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https%3A%2F%2Fwwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6source%3Dmn&c=swm&r=https%3A%2F%2Fwwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D620Fkey%3D6792YWlsLnJ1%26source%3Dmn&c=swm&r=https%3A%2F%2Fwwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6901	mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=15819392YWlsLnJ1&source=pcap	1365119965
+2020-02-17	2020-02-17 08:36:56	EMC	ChsfAa0kí2S4f÷4gG4nolqíUEHuuF9DnjUL	xXit4wGEobkT7ebgNO§CqAWiSkw5zvcQr1k	23623655	PgaTdfnrµE2LeãagJlƒUWlbŽD_tSsAastO4l	http&o=mail.ru/redir?u=https	pcap										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=15819363468874695&ts=1581944095&userId=27465&email=bWFpbC5ydQ%3D%3D&source=mn	1139408171
+2020-02-17	2020-02-17 00:25:53	EMC	ExIWQbTfŸaZ2j»Ojla_vwzCe0CKomREUDEex	FWdWk09I•PqPWiqJh5Ÿbi5oš6xjTF0D7N9cL	23644610	ZzTIlkXB>U31NIt0eo¢eEpm8YplE7RGkXuG	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https%3A%2F%2Fwwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6source%3Dpc&c=swm&r=http&o=mail.ru/redir?u=https://click.mail.ru/redir?u=http&o=mail&v=2&s=974578Fvipstatus%3Fkey	mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=1581938C8193634&email=a29t&source=mn	4267472950
+2020-02-17	2020-02-17 03:43:33	EMC	TAl7tSOQîaATLÝUUMZTx90iv0sqRoADhEn2r	zvCR7ost–HdMiQuVI8uxOyYGf0mm6k3ckHSL	32488481	9kFERwTF"AWKx•n0YP‚OUv_tA8c5X2cdy4bA		pcap										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=1581936292&email=a256F095&ts=158192&userId=274695&userId=27465D84D65D8B8E7EA692YW5haWwucnU%3D&source	1722142578
+2020-02-17	2020-02-17 04:06:30	EMC	kL0p32eE J8hjî51lG¡JQdDpzRytjcc2_AnA	eO7MVT5A47viQüfb3A01Jap�oD001J0l_UAt	23078127	KDZSswsocsumoçOtSx\\phTv�uffYOTHGNoUa		mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=7724D&source=mn	2669624134
+2020-02-17	2020-02-17 04:23:38	EMC	q3PyeDUQPYw7ÝQN9BwE1eP\nDzWsZDAv3GTN	u3GnfRRy>bOqp–qFvSîDike
B8V0zHacc0ur	26965394	CxxiDtfVM9T6WÞuleLÛ1P4m«MbzF9k0TLWLF		pcap										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=58193695&ts=1581942095&email=bWFpbC5ydQ%3D%3D&source=mn	2552851257
+2020-02-17	2020-02-17 05:36:35	EMC	ipti_lG9ê4H8Pª5RgF¬wKzaÄoxUwWNEV9beK	zolMT7vtµ_ttY|BmlDù2PSeújtg6O7WzJ2S_	31107154	ha6rDRxktOKP¹jl2gŠzz7R£2TpQu2KKta9Q		mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=4942095&ts=15819369266&userId=274692158193634&userId=2745523552D695	1459132007
+2020-02-17	2020-02-17 00:45:31	EMC	hXw0ETb7piXc>PKMzÓqxhVí_F_UdWZnFB5O	VXB5E4w2é8Orr±ZT0XÜ1Sn9SrTWuJhjff7I	19656120	v06fJehG\'_m05þ7sFduiQBx(mS5JnuSD6jph		mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=B193695&ts=15819346165D85jb20%3D&source=pcap	1300966980
+2020-02-17	2020-02-17 03:44:40	EMC	0Q0EMlWcxIACIHUsZ–X4mEêhJnxPpCJV6Qe	ZqVPZ1Yb>rFSB\tzEMNdTEjoWIHLaQ2o5vI_E	17361913	AuRaBJOGvceRXWiMb9\'wolU¬nAGR0hxXQgOS		pcap										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=725634&email=bWFpbC5ydQ%3D%3D&source=mn	1552491257
+2020-02-17	2020-02-17 02:35:40	EMC	mSk0tnq0soJQsºlU0K´lkSmßfnXFu6fqRw0A	hjlyHsr0&gybjsAJ0WRqdLJ‰z32l3lmiEgDN	32664890	YAoxYQxl"Ce8H¸vY2jï90VWzKQxlGCTNRAyJ		mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=6459&ts=158194464089&userId=27465D82&userId=27455229t&source=mn	3051884197
+2020-02-17	2020-02-17 13:44:36	EMC	znOwy_e4…iHG8(NdChÑ2qgzÜ1tjBR0ZjsFus	eNSFOPi9
oax3Ru7x0‚1JA4TWeX66Ev8T9U	18979240	LosZ3fZjëVY8h•XjH8AJgKx:O3FNAMVDRGlZ		mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=71581944027465D819384019215819397327457AbWFpbC5ydQ%3D%3D&source=mn	1828619156
+2020-02-17	2020-02-17 02:07:55	EMC	yvDmRiaYÁkm43úiWRNnnW3Y×DPBqq55XWVXr	7gpmg2PiÏrkSp]TYhcY7i6]DawnwV9tqC0b	22858229	yDybPZi3Oc7m,JVxfltuRt„Wm8TmiHQJOjY	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https://click.mail%3DZG1haWwucnU%253D%26source%3Dmn&c=swm&r=http&o=mail%3DZG1haWwucnU%253D%26source%3Dmn&c=swm&r=http&o=mail&v=2&s=9158192YWlsLnJ1%26source%3Dmn&c=swm&r=https://click.mail&v=2&s=3Dmn&c=swm&r=https://click.mail.ru/redir?u=http&o=mail%3DYm_ce%3Dmn	mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=158189721581939265D89&email=dG1haWwuY29t&source=pcap	1735264694
+2020-02-17	2020-02-17 00:39:49	EMC	BOA0kaQben9xa\fOjQ8YZt2L9UV_FYiMKYa3n	0BL0DaCp›3weXÉj7aIq4ZBØhaKzr0Uwpdpm	21820865	xATcz14a½2hYJcGJb2gRsòIIQ9K2A5fOGo	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https://mail.ru/redir?u=https://mail&v=2&s=90BtYWlsLnJ1%26source%3Dmn&c=swm&r=https://click.mail.ru/redir?u=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6ts%3D158192YWlsLnJ1%26source%3Dmn&c=swm&r=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6userId%3D27457%26ts%3D158192YWlsLnJ1%26source%3Dmn&c=swm&r=http&o=mail.ru/redir?u=http&o=mail.ru/redir?u=http	mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=77256160&userId=274634&email=bWFpbC5ydQ%3D%3D&source=mn	1855925025
+2020-02-17	2020-02-17 01:38:01	EMC	uNpwiDVWø2_p6†HEMlâ4YIHw0z_vOpEqPyir	napvpdbMVeSjy KJz5spT94‚Yu6wGCm_xgkX	27148457	R8wZY_PRôklNk¦xX0Dÿn6uA{sGHEPaMT3kc8		mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=3C3274695&userId=2745528194401908193467BD9189&ts=1581926&ts=1581939772B67B6AE4CD2DBD910B392YW5haWwucnU	2249892028
+2020-02-17	2020-02-17 23:32:22	EMC	uKrr0IYxƒ5Kt1\'QPtv@toS9ØnQ4ZcrvAD0l8	4dWFNMPK50HO0z6ZPSoX6g0�2Ceu35kLzjtB	25390020	CXq72VsbŽolj2.WJUoÅa8Uo³TZlw2wHUHxWX	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=http&o=mail%3DYWlsLnJ1%26source%3Dmn&c=swm&r=https%3A%2F%2Fwwwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6email&v=2&s=3A%2F%2Fwwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D67972	mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=BE304F9CDBC6&ts=1581934&email=YW5haWwuY29t&source=mn	3489237198
+2020-02-17	2020-02-17 22:36:42	EMC	hQkEQSur§ISp_óuy9O×Rd05yjXxg7QByQH9	FURnnJ51„xl2NiQHcN\r6hpI£kfYa17e26Mt8	25551409	NBIMBrwi mVoYÙYuxJq_8Df0CM8CZkrZezFE		mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=158194463465&userId=25616C3D&source=mn	2913713687
+2020-02-17	2020-02-17 20:19:16	EMC	Xahxt0V3`JlMo+tyHm8MxJFÒT9NEywdtAZeD	YV0_62kSEM9X0GW7‚eHjX“xF95KfKoarfR	27982291	iuEEgZrUãYV04*QSEQg_uWbYo8cewVWPriGO		mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=DFD92Y29t&source=mn	1136043092
+2020-02-17	2020-02-17 20:05:44	EMC	gAraBOGN
zDEdëkz8qßxSB6¼8sQXrgyTFpX6	KC9GBYXC´c2sL;fDL4#6DMT%2ELzdoqHtjMp	27849879	hmvHfcpQ7YcXyûk6byq3rSq«EUfDuei6M62h		mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=71&source=mn	795793408
+2020-02-17	2020-02-17 01:03:41	EMC	Sujwqk4kâhZF0Ôdd3dRrn0zésOYRdkd9ynAR	NaKyF0O3rR3r®0wXN‚Lf_3KSANWona7Ftry	30475228	calDaEsaÆ910bkYOkR\bbKjQ¹eUvAOoR7CQJH	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=http&o=mail.ru/redir?u=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6source%3Dmn&c=swm&r=http&o=mail%3DYWlsLnJ1%26source%3Dmn&c=swm&r=https://click.mail&v	mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=B1581918970&userId=274634&userId=27457&email=YW5hce=mn	3165615449
+2020-02-17	2020-02-17 18:40:14	EMC	zBHwSrKIßMfAFÆFJxuvWtIs¢Z2DA_nAX98Aa	NQiRPz123luXCH0fvH?59tbøjHdGAlVgiHhc	26251034	0KbAptV7Ëu8SZ?V4N7ÂWHvx[0vY5D_saLU2G	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https://click.mail%3DYWlsLnJ1%26source%3Dmn&c=swm&r=https://click.mail&v=2&s=9158192Ymail&v=2&s=936source%3Dmn&c=swm&r=http&o=mail%3DbW9%26userId%3D2746158192YWlsLnJ1%26source	mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=5&userId=274552746887BF4LnJ1&source=mn	115808044
+2020-02-17	2020-02-17 22:03:35	EMC	dvrVwEyo£cDVpJ3YEoÍzYAH1XuC51DQcby59	wcCGF71w¥VXRjÅ0pz2h131p\\1c7jnQYzejlC	24153343	mHlXjDveàl9Abë8kZxtR6ELtFOSq7X1Q8_7	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=http&o=mail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D65	mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=A63468874692&email=ce=pcap	1125992029
+2020-02-17	2020-02-17 13:11:31	EMC	ELifUEthŒNP3xZTZ5RZd2tIgUWpctZJNKkdp	3qxE4qStGm1jI›RPc9,JCHKxNAJqeRRgVaJ5	32523658	opEAQkCl*DGc0œVozrkcYlaUmWvfN0IAYa	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=http&o=mail%3DYWlsLnJ1%26source%3Dmn&c=swm&r=https://click.mail&v=2&s=3%26userId%3D274570%26userId%3D274615819383%26userId%3D2745746193%26userId%3D2745745776source%3Dmn&c=swm&r=http&o=mail&v=2&s=34Fww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6source%3Dmn&c=swm&r=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6126source%3Dmn&c=swm&r=https://click.mail%3DbWFpbC5ydQ%253D%253D	mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=616&userId=27465&email=YWlsLmNvbQ%3D%3D&source=pcap	660510007
+2020-02-17	2020-02-17 23:08:20	EMC	Ww8aun1EXuPT3stlfv12_A4nVx4RfDfGaK	Gj6ixLDK{FkfiIQ0ZqHK9IsJMWys80zowLQy	28692974	nB20Riv63uMc]E9s0µBv7x1XcxILAKq2zwc		pcap										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=3D&source=pcap	458549627
+2020-02-17	2020-02-17 15:45:11	EMC	_nkKyF0NV20B2ÑAAxI@oOj8§uSGus_lOrqBS	0vXB63kV�oAv97D8cU»CQSR\bmet_h2UhiQ4X	31753811	zA7KKsesŽYWPrÉI1KD"FFmd°emMIJ3AcxwZr	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https://click.mail%3DZG1haWwucnU%253D%26source%3Dmn&c=swm&r=http&o=mail&v=2&s=9%26ts%3D1581944577457EC6source%3Dmn&c=swm&r=https://click.mail.ru/redir?u=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D67D492YWlsLnJ1%26source%3Dmn&c=swm&r=https%3A%2F%2Fwwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D685764%26email&v=2&s=9%26email.ru/redir?u=https%3A%2F%2Fwwww.wildberries.ru%2Fconfirm	mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=DA8189&email=a2DC9serId=274634&userId=25695&userId=274688&userId=27465	4039013194
+2020-02-17	2020-02-17 14:54:17	EMC	wHTfVQKqšlnQp!PXzWG80qÅc0oWU1pvtKeL	EaNVlRXuýzZ33\fLB0Fmjet2¹9f3xaJplvaiX	30992376	vJnOh3mp�Ob0h!xPw1slS0e[Ehj1AIx920Jo	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https://click.mail.ru/redir?u=https://click.mail&v=2&s=3c	pcap										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=7462B60&email=cnU%3D&source=mn	2569826295
+2020-02-17	2020-02-17 22:51:48	EMC	C1a6zgETp1v0mSMHJJgMxp8w7uJ7zN14J9Z	5cn50cUip60T6ënRihVzsSáRarErtllzgH6	26715992	0D4M2m041DlURNuFG¹9QJtµIcjHVkp5_TEZ	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https://click.mail&v=2&s=9446158194419447	mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=DEBE34&email=Z21haWwucnU%3D&source=mn	1504060974
+2020-02-17	2020-02-17 13:53:54	EMC	5a2rniMpúduiEÎUzYFsgZjKHqgoZWbDYts	ziqQkEkDÿ3tYgùLGhe&QwROPlb3hkAIZW_UI	32696355	0rSWK9Um™GDPuªRvPwXqs3mÖK9cF0balOqbS	http&o=mail.ru/redir?u=http&o=mail.ru/redir?u=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6email	pcap										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=77274552E&ts=158192158192YXNvbQ%3D%3D&source=pcap	1675780127
+2020-02-17	2020-02-17 20:08:49	EMC	0IBozTjy/SMIkäY5tY\\_GAd\'0VIzFgFK2VSY	V0fFnEjk¡PzK3.y_ng3Nfs1oJtBLCWPq35z	33102601	NQhlj1k9¹bLy234WJi\rKtOdž2239rJDYJSKh	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=http&o=mail.ru/redir?u=http&o=mail.ru/redir?u=https%3A%2F%2Fwwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D61haWwucnU%253D%253D%26source%3Dmn&c=swm&r=https://click.mail%3DYWlsLnJ1%26source%3Dmn&c=swm&r=http&o=mail.ru/redir?u=https://click.mail.ru/redir?u=https%3A%2F%2Fwwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69%26userId%3D2746193A%2F%2Fwwwwwww.wildberries.ru%2Fconfirm%2Fvipstatus	mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=D8BD7B082&ts=15819392YW5haWwuY29t&source=mn	832387252
+2020-02-17	2020-02-17 21:39:56	EMC	3aiRtPZF[Sjelo0e2EVnauT�wl0tsW9SRi7f	i8OkH4au\rfMiBýV0E4ôfrxZ·u9WJpRG5bhp4	29554136	gzPJTobNÀMGgG–1kUY�Ve9s„OwfpHmKiF1p0	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=http&o=mail%3DbW97B365	mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=AE&ts=158194466&ts=15819213CD274688&ts=1581934688&email=Ymn	74272104
+2020-02-17	2020-02-17 17:21:43	EMC	yqM2AqiUÛS2LmŠo0S5�KNYR-RKGxDrME0w5U	T1Q2fyBqs8qhiWB_JƒBE3KÒo_Ro0VD79pT0	18406884	mVoGr7MMðzfVXÐ0HtU»KAYeØCVX68_NOaiQY	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https%3A%2F%2Fwwwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6email%3DbWFpbC5ydQ%253D%253D%26source%3Dmn&c=swm&r=http&o=mail&v=2&s=9158192YWlsLnJ1%26source%3Dmn&c=swm&r=https	mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=5819392&userId=27465D8B446887AF95&email=bWFpbC5ydQ%3D%3D&source=pcap	1422459332
+2020-02-17	2020-02-17 15:06:14	EMC	NXf_EKmdwAkZZIKr0NÏ0w4HuVTpNqZ30Ldk0	PpN_0d7yÃwY45öY_Wm#L2S0¼1pN5klv0Wbsa	21385681	JH3MQSLtÞn6gUIe92ilJPT.EFDSucrCuGHZ		mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=A4627465D819397D96AE65&email=dG1haWwuY29t&source=pcap	959722117
+2020-02-17	2020-02-17 19:52:47	EMC	OY90Zcjc�8ebLcK_uULQHDNvpwO1hN3MwIat	ASdCffii½d6DweXoVjžmpL8~5DHOzhFnPRuW	33422761	4T1zxu2F4evId\rETTryZBYsßCBkTxsXLeuC9		mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=E97265D8A70F&ts=15819403193634&userId=2746887&email=ce=mn	997133189
+2020-02-17	2020-02-17 17:55:20	EMC	ManEZ_qtÈzd0v\bdxdBë07LpVWqoZUfpDzZW	MnNRUoUE2ASWa´g8ClûKK5l¦0yPRafHb0gsw	29428950	clyoRyW5\00F6wDd5P1¹88B6îk_OdKCbUFQqT		mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=52666799692&email=dGF7470E634&email=ce=mn	3015790084
+2020-02-17	2020-02-17 20:07:34	EMC	pr7D2hoMay0b‰wJjvH6UaPÌLZpeDXtq0Njs	bDlkkbc3OnwzdcNZHDücBkR|V0ApitIVgPOv	22120090	TC2aVmxXNfOhSùL0y2»oqzb=drQElfeo0_OQ		mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=581943D&source=mn	2666495492
+2020-02-17	2020-02-17 18:20:19	EMC	a_EpwBGWôRXNSÂTY_FñyAhRB25SpX8Gw8JaW	0XDllB5u!HDENidqN/CRrkxwnjTXLeEKIyb	32607759	OKByJYVHá3pLb:BWhWœwvLnµnpw0urPjP04i	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6userId%3D27461203Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6email&v=2&s=3D158192YWlsLnJ1%26source%3Dpc&c=swm&r=https	mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=BC66&ts=158193926&userId=256BFC35FD2AEAZ21haWwucnU%3D&source=mn	590291581
+2020-02-17	2020-02-17 10:27:11	EMC	bn8x3TAHKlMOs‡dlmzybH0S*iDc0hYull7Zt	7PY8hsBEŠrbST£npoU7iPxD8CFjIMxK8eN8F	30559777	lgByHEG2ffme xKgJXv5r“C6CW78KV4Kia		mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=6667&email=bWFpbC5ydQ%3D%3D&source=mn	1898373016
+2020-02-17	2020-02-17 14:09:12	EMC	jYFR0DCEŠ2eTk´EFCuAIyAHVYLUy9ENh9Cqc	dhe0lvmT§Dchj´8XhgtsZ3mCPzv5hTGPM8Q	24813187	Y7MmPfU2”YhtaY85K"2xIfqNraWWqBp5QvK	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https%3A%2F%2Fwwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6source%3Dmn&c=swm&r=http&o=mail&v=2&s=96email%3DZG1haWwucnU%253D%26source%3Dmn&c=swm&r=http&o=mail&v=2&s=944576userId%3D27461819443Fkey%3D646192YWlsLnJ1%26source%3Dmn&c=swm&r=https://click.mail.ru/redir?u=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6userId%3D274615819360	mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=AD9397158192&userId=25616AEBD63465D88&email=bmRleC5ydQ%3D%3D&source=mn	1726149098
+2020-02-17	2020-02-17 18:27:00	EMC	VnzEdMooö2lhW?Yr6AQyNVwyFnUj_l0pBiG	EnQ5CpnJŒt3LM1hiuèC7ls‘qJChM0i4JMZ0	28169388	zBJ8p4pS
lPo_îbkk4\t2ObECW2ReYnHoWrQH	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6userId%3D27461203Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6email&v=2&s=3D158192YWlsLnJ1%26source%3Dmn&c=swm&r=https	mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=397729t&source=mn	1102313974
+2020-02-17	2020-02-17 19:49:07	EMC	fxpHcK20ouDlcëjYFRÞhFT4DMtbMML0qb8ds	hBF63Vf_ìqZrpÁsfD4âEOfd›6YGUBvflIJqF	22835697	TkBqOH2Jjp0wHÇ35uDnjAxqgyb2P4HqtznK	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6userId%3D2746158192YWlsLnJ1%26source%3Dmn&c=swm&r=https://click.mail&v=2&s=9158192YWlsLnJ1%26source%3Dmn&c=swm&r=https://click.mail&v=2&s=92YWlsLnJ1%26source%3Dmn&c=swm&r=https://click.mail%3DbWFpbC5ydQ%253D%253D%26source%3Dpc&c=swm&r=http&o=mail%3DZG1haWwucnU%253D%26source%3Dmn&c=swm&r=https://click.mail&v=2&s=9%26source%3Dpc&c	pcap										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=ADDABAA48E95&userId=2746346887D9420%3D&source=mn	3497823576
+2020-02-17	2020-02-17 13:28:56	EMC	hZbZqj2vRV2F¦pCXlvxIEyùqdeQrwceKg35	8fpYiVLEÂ9gfSIzfP_$dzlj1H8tMzVbcg59	20129530	8bY0GHO5KMpHW9cIAk”e4yQQ94oQfvoHBxY		mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=C5ydQ%3D%3D&source=mn	3102735519
+2020-02-17	2020-02-17 21:37:19	EMC	TAlSFJzIsXzTWbuEymOTU3¾20iUcNXmLT5j	J2Vf06b6¥CMLF“AvwHÉ_wsUÊNOv4O9nnCy20	17587521	nEczrivzˆdrNaBYYvdnQRUT2pBBo8hu3kCvI		mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=66&email=YWlsLnJ1&source=mn	1613554206
+2020-02-17	2020-02-17 21:58:32	EMC	XPXxpWq0ÕwHcK2FB4$JaOaå9xIGhEaajLeS	l59HKsiwgQuXg‘AweGZRZrE¸kN0Pqk_plZ4c	24296347	3s8mV_3W&hisrcwCm6QXCg_ÿ0zgLyr6nnqPB	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6userId%3D274616	pcap										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=467105695&email=a2YWlsLnJ1&source=pcap	2033387311
+2020-02-17	2020-02-17 16:54:16	EMC	2beVLC_ZYMmZuZbJZ�1mjVLQeoXV0BsZKq	J4Y22CvJx36nq6KND¼MXBg!H5lUx1Z7BoFf	31434491	dabaBlCY¤351pÐXUW0ÑHdErŸDTW_XGkqwmta	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6userId%3D27461203Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6Fconfirm%2Fvipstatus%3Fkey%3D661076userId%3D27457CD15819444613	mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=76031938874634&userId=2564BFA93626&userId=256&email=ce=mn	1613641575
+2020-02-17	2020-02-17 00:27:36	EMC	Au3PRMFC´W0Ij8iL0Z¦tsrryAvKJowqXJ8k	ntqK0ayP±8ltR0twuGraE`Vwgcmux6jFwg	21307924	ruixkHLK�jmENcmpbvvbrrLU8FDLIpWOFtU	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https://click.mail.ru/redir?u=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6email.ru/redir?u=https%3A%2F%2Fwwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6CD2745785dQ	mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=A41E772FFC5ydQ%3D%3D&source=pcap	2395792653
+2020-02-17	2020-02-17 22:03:46	EMC	Tc9EmNpb5Xy0edNdTC‹Lg7Xò2eLmGuTdEkIt	YsCMp8oiUBPyÁhHBsçDmEZs0xvixA2L6atY	23431515	Bh67PWI6ãjKZ6«Q102þnTeN(YP6e8kLoqiVA	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https://mail&v=2&s=96userId%3D274616	pcap										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=5D819446&userId=27465D8191897C7462F400&userId=27463465D8192&email=Z21haWwuY29t&source=pcap	2211382541
+2020-02-17	2020-02-17 06:34:15	EMC	S13SmJGaHVUBXqd3bs\tRpB3ÑG_hXZBJjxOt6	FP_GWMlf²w2REi3Up5Cqu0÷qWcA99YlCduy	27883082	xJ1eD6ixãour7M0K_¿l6Ziv6awoxMpnzsaW	https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https://mail%3Dby%3D6%26ts%3D158193DBDB3A%2F%2Fwwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D63158192YWlsLnJ1%26source%3Dmn&c=swm&r=https://click.mail&v=2&s=92YWlsLnJ1%26source	mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=15819362hQG1haWwuY29t&source=pcap	2045856196
+2020-02-17	2020-02-17 00:36:06	EMC	2mjWiSh70SN_â1xOBÂst7vbYq1z5EL26Z0i	7e5aIak4ôPca2ãwoYGâZIJi!P_gRQdnPpHDP	28784173	b5hw0LlyæfMCy‚YstJ9XbB6Pq7VN5hO9kIy		mn										1	Ru	ru	7.0.32	/confirm/vipstatus	?key=15819387AE3634&email=cnU%3D&source=mn	1255281910

From 5548c9d1ad8b752cac133193d774e6a84a03321c Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 29 May 2020 01:53:15 +0300
Subject: [PATCH 144/184] Repeat test on curl timeout

---
 tests/clickhouse-test | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index 1a2735296e8..86a9ecebd96 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -30,6 +30,7 @@ from contextlib import closing
 MESSAGES_TO_RETRY = [
     "DB::Exception: ZooKeeper session has been expired",
     "Coordination::Exception: Connection loss",
+    "Operation timed out",
 ]
 
 

From c071e0a473c39d8e48eac154037f96ad56557095 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <avtokmakov@yandex-team.ru>
Date: Fri, 29 May 2020 02:01:18 +0300
Subject: [PATCH 145/184] add context to DatabaseCatalog methods

---
 .../InputStreamFromASTInsertQuery.cpp         |  2 +-
 .../PushingToViewsBlockOutputStream.cpp       |  2 +-
 src/DataStreams/tests/union_stream2.cpp       |  2 +-
 src/Functions/FunctionJoinGet.cpp             |  2 +-
 src/Functions/hasColumnInTable.cpp            |  2 +-
 src/Interpreters/ActionLocksManager.cpp       |  9 +++++--
 src/Interpreters/ActionLocksManager.h         |  3 +++
 src/Interpreters/ActionsVisitor.cpp           |  2 +-
 .../ClusterProxy/SelectStreamFactory.cpp      |  2 +-
 src/Interpreters/Context.cpp                  |  2 +-
 src/Interpreters/DDLWorker.cpp                |  2 +-
 src/Interpreters/DatabaseCatalog.cpp          | 27 ++++++++++---------
 src/Interpreters/DatabaseCatalog.h            | 16 ++++++-----
 src/Interpreters/ExpressionAnalyzer.cpp       |  2 +-
 .../InJoinSubqueriesPreprocessor.cpp          |  2 +-
 src/Interpreters/InterpreterAlterQuery.cpp    |  2 +-
 src/Interpreters/InterpreterCheckQuery.cpp    |  2 +-
 src/Interpreters/InterpreterCreateQuery.cpp   |  2 +-
 src/Interpreters/InterpreterDescribeQuery.cpp |  2 +-
 src/Interpreters/InterpreterDropQuery.cpp     |  6 ++---
 src/Interpreters/InterpreterExistsQuery.cpp   |  2 +-
 src/Interpreters/InterpreterExplainQuery.cpp  |  2 +-
 src/Interpreters/InterpreterInsertQuery.cpp   |  2 +-
 .../InterpreterKillQueryQuery.cpp             |  2 +-
 src/Interpreters/InterpreterOptimizeQuery.cpp |  2 +-
 src/Interpreters/InterpreterRenameQuery.cpp   |  2 +-
 src/Interpreters/InterpreterSystemQuery.cpp   |  6 ++---
 src/Interpreters/InterpreterWatchQuery.cpp    |  2 +-
 src/Interpreters/JoinedTables.cpp             |  4 +--
 src/Interpreters/SystemLog.h                  |  6 ++---
 src/Interpreters/getTableExpressions.cpp      |  2 +-
 src/Interpreters/interpretSubquery.cpp        |  2 +-
 src/Server/MySQLHandler.cpp                   |  2 +-
 src/Server/TCPHandler.cpp                     |  6 ++---
 src/Storages/Kafka/StorageKafka.cpp           |  4 +--
 src/Storages/LiveView/StorageLiveView.cpp     |  2 +-
 src/Storages/LiveView/StorageLiveView.h       |  2 +-
 src/Storages/StorageBuffer.cpp                | 12 ++++-----
 src/Storages/StorageBuffer.h                  |  2 +-
 src/Storages/StorageMaterializedView.cpp      |  8 +++---
 src/Storages/StorageMergeTree.cpp             |  4 +--
 src/Storages/StorageReplicatedMergeTree.cpp   |  6 ++---
 src/Storages/getStructureOfRemoteTable.cpp    |  2 +-
 43 files changed, 93 insertions(+), 82 deletions(-)

diff --git a/src/DataStreams/InputStreamFromASTInsertQuery.cpp b/src/DataStreams/InputStreamFromASTInsertQuery.cpp
index 3b4946e4bc9..47b61294da3 100644
--- a/src/DataStreams/InputStreamFromASTInsertQuery.cpp
+++ b/src/DataStreams/InputStreamFromASTInsertQuery.cpp
@@ -58,7 +58,7 @@ InputStreamFromASTInsertQuery::InputStreamFromASTInsertQuery(
 
     if (context.getSettingsRef().input_format_defaults_for_omitted_fields && ast_insert_query->table_id && !input_function)
     {
-        StoragePtr storage = DatabaseCatalog::instance().getTable(ast_insert_query->table_id);
+        StoragePtr storage = DatabaseCatalog::instance().getTable(ast_insert_query->table_id, context);
         auto column_defaults = storage->getColumns().getDefaults();
         if (!column_defaults.empty())
             res_stream = std::make_shared<AddingDefaultsBlockInputStream>(res_stream, column_defaults, context);
diff --git a/src/DataStreams/PushingToViewsBlockOutputStream.cpp b/src/DataStreams/PushingToViewsBlockOutputStream.cpp
index 7f730b5fd3f..f68e0d81c03 100644
--- a/src/DataStreams/PushingToViewsBlockOutputStream.cpp
+++ b/src/DataStreams/PushingToViewsBlockOutputStream.cpp
@@ -59,7 +59,7 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream(
 
     for (const auto & database_table : dependencies)
     {
-        auto dependent_table = DatabaseCatalog::instance().getTable(database_table);
+        auto dependent_table = DatabaseCatalog::instance().getTable(database_table, context);
 
         ASTPtr query;
         BlockOutputStreamPtr out;
diff --git a/src/DataStreams/tests/union_stream2.cpp b/src/DataStreams/tests/union_stream2.cpp
index 6e0997e7f72..5b84d89a435 100644
--- a/src/DataStreams/tests/union_stream2.cpp
+++ b/src/DataStreams/tests/union_stream2.cpp
@@ -35,7 +35,7 @@ try
     Names column_names;
     column_names.push_back("WatchID");
 
-    StoragePtr table = DatabaseCatalog::instance().getTable({"default", "hits6"});
+    StoragePtr table = DatabaseCatalog::instance().getTable({"default", "hits6"}, context);
 
     QueryProcessingStage::Enum stage = table->getQueryProcessingStage(context);
     auto pipes = table->read(column_names, {}, context, stage, settings.max_block_size, settings.max_threads);
diff --git a/src/Functions/FunctionJoinGet.cpp b/src/Functions/FunctionJoinGet.cpp
index d6be2469759..a4569684e7f 100644
--- a/src/Functions/FunctionJoinGet.cpp
+++ b/src/Functions/FunctionJoinGet.cpp
@@ -43,7 +43,7 @@ static auto getJoin(const ColumnsWithTypeAndName & arguments, const Context & co
         ++dot;
     }
     String table_name = join_name.substr(dot);
-    auto table = DatabaseCatalog::instance().getTable({database_name, table_name});
+    auto table = DatabaseCatalog::instance().getTable({database_name, table_name}, context);
     auto storage_join = std::dynamic_pointer_cast<StorageJoin>(table);
     if (!storage_join)
         throw Exception{"Table " + join_name + " should have engine StorageJoin", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
diff --git a/src/Functions/hasColumnInTable.cpp b/src/Functions/hasColumnInTable.cpp
index 03e5a4513db..b9ec2b84837 100644
--- a/src/Functions/hasColumnInTable.cpp
+++ b/src/Functions/hasColumnInTable.cpp
@@ -113,7 +113,7 @@ void FunctionHasColumnInTable::executeImpl(Block & block, const ColumnNumbers &
     bool has_column;
     if (host_name.empty())
     {
-        const StoragePtr & table = DatabaseCatalog::instance().getTable({database_name, table_name});
+        const StoragePtr & table = DatabaseCatalog::instance().getTable({database_name, table_name}, global_context);
         has_column = table->getColumns().hasPhysical(column_name);
     }
     else
diff --git a/src/Interpreters/ActionLocksManager.cpp b/src/Interpreters/ActionLocksManager.cpp
index 8bc8542da3a..e8887f44a45 100644
--- a/src/Interpreters/ActionLocksManager.cpp
+++ b/src/Interpreters/ActionLocksManager.cpp
@@ -19,6 +19,11 @@ namespace ActionLocks
 }
 
 
+ActionLocksManager::ActionLocksManager(const Context & context)
+        : global_context(context.getGlobalContext())
+{
+}
+
 template <typename F>
 inline void forEachTable(F && f, const Context & context)
 {
@@ -35,7 +40,7 @@ void ActionLocksManager::add(StorageActionBlockType action_type, const Context &
 
 void ActionLocksManager::add(const StorageID & table_id, StorageActionBlockType action_type)
 {
-    if (auto table = DatabaseCatalog::instance().tryGetTable(table_id))
+    if (auto table = DatabaseCatalog::instance().tryGetTable(table_id, global_context))
         add(table, action_type);
 }
 
@@ -60,7 +65,7 @@ void ActionLocksManager::remove(StorageActionBlockType action_type)
 
 void ActionLocksManager::remove(const StorageID & table_id, StorageActionBlockType action_type)
 {
-    if (auto table = DatabaseCatalog::instance().tryGetTable(table_id))
+    if (auto table = DatabaseCatalog::instance().tryGetTable(table_id, global_context))
         remove(table, action_type);
 }
 
diff --git a/src/Interpreters/ActionLocksManager.h b/src/Interpreters/ActionLocksManager.h
index 50855676fd4..ea5d96ad8ce 100644
--- a/src/Interpreters/ActionLocksManager.h
+++ b/src/Interpreters/ActionLocksManager.h
@@ -19,6 +19,8 @@ class Context;
 class ActionLocksManager
 {
 public:
+    ActionLocksManager(const Context & context);
+
     /// Adds new locks for each table
     void add(StorageActionBlockType action_type, const Context & context);
     /// Add new lock for a table if it has not been already added
@@ -41,6 +43,7 @@ private:
 
     mutable std::mutex mutex;
     StorageLocks storage_locks;
+    const Context & global_context;
 };
 
 }
diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp
index 7c2133e629f..6b227657ba5 100644
--- a/src/Interpreters/ActionsVisitor.cpp
+++ b/src/Interpreters/ActionsVisitor.cpp
@@ -670,7 +670,7 @@ SetPtr ActionsMatcher::makeSet(const ASTFunction & node, Data & data, bool no_su
         if (identifier)
         {
             auto table_id = data.context.resolveStorageID(right_in_operand);
-            StoragePtr table = DatabaseCatalog::instance().tryGetTable(table_id);
+            StoragePtr table = DatabaseCatalog::instance().tryGetTable(table_id, data.context);
 
             if (table)
             {
diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp
index 53c61bea60e..d601d460be9 100644
--- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp
+++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp
@@ -191,7 +191,7 @@ void SelectStreamFactory::createForShard(
         else
         {
             auto resolved_id = context.resolveStorageID(main_table);
-            main_table_storage = DatabaseCatalog::instance().tryGetTable(resolved_id);
+            main_table_storage = DatabaseCatalog::instance().tryGetTable(resolved_id, context);
         }
 
 
diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp
index 4605c606594..9c78fe59da1 100644
--- a/src/Interpreters/Context.cpp
+++ b/src/Interpreters/Context.cpp
@@ -2017,7 +2017,7 @@ std::shared_ptr<ActionLocksManager> Context::getActionLocksManager()
     auto lock = getLock();
 
     if (!shared->action_locks_manager)
-        shared->action_locks_manager = std::make_shared<ActionLocksManager>();
+        shared->action_locks_manager = std::make_shared<ActionLocksManager>(*this);
 
     return shared->action_locks_manager;
 }
diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp
index fefeeeed988..ee91d903416 100644
--- a/src/Interpreters/DDLWorker.cpp
+++ b/src/Interpreters/DDLWorker.cpp
@@ -634,7 +634,7 @@ void DDLWorker::processTask(DDLTask & task, const ZooKeeperPtr & zookeeper)
                 {
                     /// It's not CREATE DATABASE
                     auto table_id = context.tryResolveStorageID(*query_with_table, Context::ResolveOrdinary);
-                    storage = DatabaseCatalog::instance().tryGetTable(table_id);
+                    storage = DatabaseCatalog::instance().tryGetTable(table_id, context);
                 }
 
                 /// For some reason we check consistency of cluster definition only
diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp
index f292aaaf049..a2157641d4d 100644
--- a/src/Interpreters/DatabaseCatalog.cpp
+++ b/src/Interpreters/DatabaseCatalog.cpp
@@ -159,6 +159,7 @@ DatabaseAndTable DatabaseCatalog::tryGetByUUID(const UUID & uuid) const
 
 DatabaseAndTable DatabaseCatalog::getTableImpl(
     const StorageID & table_id,
+    const Context & context,
     std::optional<Exception> * exception) const
 {
     if (!table_id)
@@ -206,7 +207,7 @@ DatabaseAndTable DatabaseCatalog::getTableImpl(
         database = it->second;
     }
 
-    auto table = database->tryGetTable(table_id.table_name, *global_context);
+    auto table = database->tryGetTable(table_id.table_name, context);
     if (!table && exception)
             exception->emplace("Table " + table_id.getNameForLogs() + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
 
@@ -319,7 +320,7 @@ Databases DatabaseCatalog::getDatabases() const
     return databases;
 }
 
-bool DatabaseCatalog::isTableExist(const DB::StorageID & table_id) const
+bool DatabaseCatalog::isTableExist(const DB::StorageID & table_id, const Context & context) const
 {
     if (table_id.hasUUID())
         return tryGetByUUID(table_id.uuid).second != nullptr;
@@ -331,12 +332,12 @@ bool DatabaseCatalog::isTableExist(const DB::StorageID & table_id) const
         if (iter != databases.end())
             db = iter->second;
     }
-    return db && db->isTableExist(table_id.table_name, *global_context);
+    return db && db->isTableExist(table_id.table_name, context);
 }
 
-void DatabaseCatalog::assertTableDoesntExist(const StorageID & table_id) const
+void DatabaseCatalog::assertTableDoesntExist(const StorageID & table_id, const Context & context) const
 {
-    if (isTableExist(table_id))
+    if (isTableExist(table_id, context))
         throw Exception("Table " + table_id.getNameForLogs() + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS);
 }
 
@@ -468,32 +469,32 @@ bool DatabaseCatalog::isDictionaryExist(const StorageID & table_id) const
     return db && db->isDictionaryExist(table_id.getTableName());
 }
 
-StoragePtr DatabaseCatalog::getTable(const StorageID & table_id) const
+StoragePtr DatabaseCatalog::getTable(const StorageID & table_id, const Context & context) const
 {
     std::optional<Exception> exc;
-    auto res = getTableImpl(table_id, &exc);
+    auto res = getTableImpl(table_id, context, &exc);
     if (!res.second)
         throw Exception(*exc);
     return res.second;
 }
 
-StoragePtr DatabaseCatalog::tryGetTable(const StorageID & table_id) const
+StoragePtr DatabaseCatalog::tryGetTable(const StorageID & table_id, const Context & context) const
 {
-    return getTableImpl(table_id, nullptr).second;
+    return getTableImpl(table_id, context, nullptr).second;
 }
 
-DatabaseAndTable DatabaseCatalog::getDatabaseAndTable(const StorageID & table_id) const
+DatabaseAndTable DatabaseCatalog::getDatabaseAndTable(const StorageID & table_id, const Context & context) const
 {
     std::optional<Exception> exc;
-    auto res = getTableImpl(table_id, &exc);
+    auto res = getTableImpl(table_id, context, &exc);
     if (!res.second)
         throw Exception(*exc);
     return res;
 }
 
-DatabaseAndTable DatabaseCatalog::tryGetDatabaseAndTable(const StorageID & table_id) const
+DatabaseAndTable DatabaseCatalog::tryGetDatabaseAndTable(const StorageID & table_id, const Context & context) const
 {
-    return getTableImpl(table_id, nullptr);
+    return getTableImpl(table_id, context, nullptr);
 }
 
 void DatabaseCatalog::loadMarkedAsDroppedTables()
diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h
index a481e3d7e5e..ac8a99c58d1 100644
--- a/src/Interpreters/DatabaseCatalog.h
+++ b/src/Interpreters/DatabaseCatalog.h
@@ -129,15 +129,17 @@ public:
     DatabasePtr getDatabase(const String & database_name, const Context & local_context) const;
 
     /// For all of the following methods database_name in table_id must be not empty (even for temporary tables).
-    void assertTableDoesntExist(const StorageID & table_id) const;
-    bool isTableExist(const StorageID & table_id) const;
+    void assertTableDoesntExist(const StorageID & table_id, const Context & context) const;
+    bool isTableExist(const StorageID & table_id, const Context & context) const;
     bool isDictionaryExist(const StorageID & table_id) const;
 
-    StoragePtr getTable(const StorageID & table_id) const;
-    StoragePtr tryGetTable(const StorageID & table_id) const;
-    DatabaseAndTable getDatabaseAndTable(const StorageID & table_id) const;
-    DatabaseAndTable tryGetDatabaseAndTable(const StorageID & table_id) const;
-    DatabaseAndTable getTableImpl(const StorageID & table_id, std::optional<Exception> * exception = nullptr) const;
+    StoragePtr getTable(const StorageID & table_id, const Context & context) const;
+    StoragePtr tryGetTable(const StorageID & table_id, const Context & context) const;
+    DatabaseAndTable getDatabaseAndTable(const StorageID & table_id, const Context & context) const;
+    DatabaseAndTable tryGetDatabaseAndTable(const StorageID & table_id, const Context & context) const;
+    DatabaseAndTable getTableImpl(const StorageID & table_id,
+                                  const Context & context,
+                                  std::optional<Exception> * exception = nullptr) const;
 
     void addDependency(const StorageID & from, const StorageID & where);
     void removeDependency(const StorageID & from, const StorageID & where);
diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp
index b38af6feef9..6f846cc0277 100644
--- a/src/Interpreters/ExpressionAnalyzer.cpp
+++ b/src/Interpreters/ExpressionAnalyzer.cpp
@@ -322,7 +322,7 @@ SetPtr SelectQueryExpressionAnalyzer::isPlainStorageSetInSubquery(const ASTPtr &
     if (!table)
         return nullptr;
     auto table_id = context.resolveStorageID(subquery_or_table_name);
-    const auto storage = DatabaseCatalog::instance().getTable(table_id);
+    const auto storage = DatabaseCatalog::instance().getTable(table_id, context);
     if (storage->getName() != "Set")
         return nullptr;
     const auto storage_set = std::dynamic_pointer_cast<StorageSet>(storage);
diff --git a/src/Interpreters/InJoinSubqueriesPreprocessor.cpp b/src/Interpreters/InJoinSubqueriesPreprocessor.cpp
index f40e91e7dcd..432a7f40b12 100644
--- a/src/Interpreters/InJoinSubqueriesPreprocessor.cpp
+++ b/src/Interpreters/InJoinSubqueriesPreprocessor.cpp
@@ -27,7 +27,7 @@ namespace
 StoragePtr tryGetTable(const ASTPtr & database_and_table, const Context & context)
 {
     auto table_id = context.resolveStorageID(database_and_table);
-    return DatabaseCatalog::instance().tryGetTable(table_id);
+    return DatabaseCatalog::instance().tryGetTable(table_id, context);
 }
 
 using CheckShardsAndTables = InJoinSubqueriesPreprocessor::CheckShardsAndTables;
diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp
index c2ab8776c25..205f4afb793 100644
--- a/src/Interpreters/InterpreterAlterQuery.cpp
+++ b/src/Interpreters/InterpreterAlterQuery.cpp
@@ -42,7 +42,7 @@ BlockIO InterpreterAlterQuery::execute()
 
     context.checkAccess(getRequiredAccess());
     auto table_id = context.resolveStorageID(alter, Context::ResolveOrdinary);
-    StoragePtr table = DatabaseCatalog::instance().getTable(table_id);
+    StoragePtr table = DatabaseCatalog::instance().getTable(table_id, context);
 
     /// Add default database to table identifiers that we can encounter in e.g. default expressions,
     /// mutation expression, etc.
diff --git a/src/Interpreters/InterpreterCheckQuery.cpp b/src/Interpreters/InterpreterCheckQuery.cpp
index b8f7203e607..b3cd807abe5 100644
--- a/src/Interpreters/InterpreterCheckQuery.cpp
+++ b/src/Interpreters/InterpreterCheckQuery.cpp
@@ -41,7 +41,7 @@ BlockIO InterpreterCheckQuery::execute()
     auto table_id = context.resolveStorageID(check, Context::ResolveOrdinary);
 
     context.checkAccess(AccessType::SHOW_TABLES, table_id);
-    StoragePtr table = DatabaseCatalog::instance().getTable(table_id);
+    StoragePtr table = DatabaseCatalog::instance().getTable(table_id, context);
     auto check_results = table->checkData(query_ptr, context);
 
     Block block;
diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp
index bce15434418..488fd79199d 100644
--- a/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/src/Interpreters/InterpreterCreateQuery.cpp
@@ -406,7 +406,7 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::setProperties(AS
     else if (!create.as_table.empty())
     {
         String as_database_name = context.resolveDatabase(create.as_database);
-        StoragePtr as_storage = DatabaseCatalog::instance().getTable({as_database_name, create.as_table});
+        StoragePtr as_storage = DatabaseCatalog::instance().getTable({as_database_name, create.as_table}, context);
 
         /// as_storage->getColumns() and setEngine(...) must be called under structure lock of other_table for CREATE ... AS other_table.
         as_storage_lock = as_storage->lockStructureForShare(
diff --git a/src/Interpreters/InterpreterDescribeQuery.cpp b/src/Interpreters/InterpreterDescribeQuery.cpp
index 1a6dbbcfc4d..d457fefed6a 100644
--- a/src/Interpreters/InterpreterDescribeQuery.cpp
+++ b/src/Interpreters/InterpreterDescribeQuery.cpp
@@ -86,7 +86,7 @@ BlockInputStreamPtr InterpreterDescribeQuery::executeImpl()
         {
             auto table_id = context.resolveStorageID(table_expression.database_and_table_name);
             context.checkAccess(AccessType::SHOW_COLUMNS, table_id);
-            table = DatabaseCatalog::instance().getTable(table_id);
+            table = DatabaseCatalog::instance().getTable(table_id, context);
         }
 
         auto table_lock = table->lockStructureForShare(
diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp
index 8015e28d467..5ffce2fc3ec 100644
--- a/src/Interpreters/InterpreterDropQuery.cpp
+++ b/src/Interpreters/InterpreterDropQuery.cpp
@@ -81,8 +81,8 @@ BlockIO InterpreterDropQuery::executeToTable(
     auto ddl_guard = (!query.no_ddl_lock ? DatabaseCatalog::instance().getDDLGuard(table_id.database_name, table_id.table_name) : nullptr);
 
     /// If table was already dropped by anyone, an exception will be thrown
-    auto [database, table] = query.if_exists ? DatabaseCatalog::instance().tryGetDatabaseAndTable(table_id)
-                                             : DatabaseCatalog::instance().getDatabaseAndTable(table_id);
+    auto [database, table] = query.if_exists ? DatabaseCatalog::instance().tryGetDatabaseAndTable(table_id, context)
+                                             : DatabaseCatalog::instance().getDatabaseAndTable(table_id, context);
 
     if (database && table)
     {
@@ -182,7 +182,7 @@ BlockIO InterpreterDropQuery::executeToTemporaryTable(const String & table_name,
         auto resolved_id = context_handle.tryResolveStorageID(StorageID("", table_name), Context::ResolveExternal);
         if (resolved_id)
         {
-            StoragePtr table = DatabaseCatalog::instance().getTable(resolved_id);
+            StoragePtr table = DatabaseCatalog::instance().getTable(resolved_id, context);
             if (kind == ASTDropQuery::Kind::Truncate)
             {
                 auto table_lock = table->lockExclusively(context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout);
diff --git a/src/Interpreters/InterpreterExistsQuery.cpp b/src/Interpreters/InterpreterExistsQuery.cpp
index 993b3631e06..8539cb6816e 100644
--- a/src/Interpreters/InterpreterExistsQuery.cpp
+++ b/src/Interpreters/InterpreterExistsQuery.cpp
@@ -50,7 +50,7 @@ BlockInputStreamPtr InterpreterExistsQuery::executeImpl()
         {
             String database = context.resolveDatabase(exists_query->database);
             context.checkAccess(AccessType::SHOW_TABLES, database, exists_query->table);
-            result = DatabaseCatalog::instance().isTableExist({database, exists_query->table});
+            result = DatabaseCatalog::instance().isTableExist({database, exists_query->table}, context);
         }
     }
     else if ((exists_query = query_ptr->as<ASTExistsDictionaryQuery>()))
diff --git a/src/Interpreters/InterpreterExplainQuery.cpp b/src/Interpreters/InterpreterExplainQuery.cpp
index 1c1e21fc32c..dacd7ca5f20 100644
--- a/src/Interpreters/InterpreterExplainQuery.cpp
+++ b/src/Interpreters/InterpreterExplainQuery.cpp
@@ -76,7 +76,7 @@ namespace
                 if (const auto * identifier = expression.database_and_table_name->as<ASTIdentifier>())
                 {
                     auto table_id = data.context.resolveStorageID(*identifier);
-                    const auto & storage = DatabaseCatalog::instance().getTable(table_id);
+                    const auto & storage = DatabaseCatalog::instance().getTable(table_id, data.context);
 
                     if (auto * storage_view = dynamic_cast<StorageView *>(storage.get()))
                         storage_view->getRuntimeViewQuery(&select_query, data.context, true);
diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp
index b6efa5d6d46..069dfbf006f 100644
--- a/src/Interpreters/InterpreterInsertQuery.cpp
+++ b/src/Interpreters/InterpreterInsertQuery.cpp
@@ -70,7 +70,7 @@ StoragePtr InterpreterInsertQuery::getTable(ASTInsertQuery & query)
     }
 
     query.table_id = context.resolveStorageID(query.table_id);
-    return DatabaseCatalog::instance().getTable(query.table_id);
+    return DatabaseCatalog::instance().getTable(query.table_id, context);
 }
 
 Block InterpreterInsertQuery::getSampleBlock(const ASTInsertQuery & query, const StoragePtr & table) const
diff --git a/src/Interpreters/InterpreterKillQueryQuery.cpp b/src/Interpreters/InterpreterKillQueryQuery.cpp
index 42afd0ef477..82c134aeba6 100644
--- a/src/Interpreters/InterpreterKillQueryQuery.cpp
+++ b/src/Interpreters/InterpreterKillQueryQuery.cpp
@@ -261,7 +261,7 @@ BlockIO InterpreterKillQueryQuery::execute()
             CancellationCode code = CancellationCode::Unknown;
             if (!query.test)
             {
-                auto storage = DatabaseCatalog::instance().tryGetTable(table_id);
+                auto storage = DatabaseCatalog::instance().tryGetTable(table_id, context);
                 if (!storage)
                     code = CancellationCode::NotFound;
                 else
diff --git a/src/Interpreters/InterpreterOptimizeQuery.cpp b/src/Interpreters/InterpreterOptimizeQuery.cpp
index b6c50e59cc3..c47fe1160cf 100644
--- a/src/Interpreters/InterpreterOptimizeQuery.cpp
+++ b/src/Interpreters/InterpreterOptimizeQuery.cpp
@@ -25,7 +25,7 @@ BlockIO InterpreterOptimizeQuery::execute()
     context.checkAccess(getRequiredAccess());
 
     auto table_id = context.resolveStorageID(ast, Context::ResolveOrdinary);
-    StoragePtr table = DatabaseCatalog::instance().getTable(table_id);
+    StoragePtr table = DatabaseCatalog::instance().getTable(table_id, context);
     table->optimize(query_ptr, ast.partition, ast.final, ast.deduplicate, context);
     return {};
 }
diff --git a/src/Interpreters/InterpreterRenameQuery.cpp b/src/Interpreters/InterpreterRenameQuery.cpp
index 78d1c7ee486..de2b6bb0c1c 100644
--- a/src/Interpreters/InterpreterRenameQuery.cpp
+++ b/src/Interpreters/InterpreterRenameQuery.cpp
@@ -78,7 +78,7 @@ BlockIO InterpreterRenameQuery::execute()
     for (auto & elem : descriptions)
     {
         if (!rename.exchange)
-            database_catalog.assertTableDoesntExist(StorageID(elem.to_database_name, elem.to_table_name));
+            database_catalog.assertTableDoesntExist(StorageID(elem.to_database_name, elem.to_table_name), context);
 
         database_catalog.getDatabase(elem.from_database_name)->renameTable(
             context,
diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp
index 557fb4c71e8..fedda7cab50 100644
--- a/src/Interpreters/InterpreterSystemQuery.cpp
+++ b/src/Interpreters/InterpreterSystemQuery.cpp
@@ -321,7 +321,7 @@ StoragePtr InterpreterSystemQuery::tryRestartReplica(const StorageID & replica,
     context.checkAccess(AccessType::SYSTEM_RESTART_REPLICA, replica);
 
     auto table_ddl_guard = need_ddl_guard ? DatabaseCatalog::instance().getDDLGuard(replica.getDatabaseName(), replica.getTableName()) : nullptr;
-    auto [database, table] = DatabaseCatalog::instance().tryGetDatabaseAndTable(replica);
+    auto [database, table] = DatabaseCatalog::instance().tryGetDatabaseAndTable(replica, context);
     ASTPtr create_ast;
 
     /// Detach actions
@@ -394,7 +394,7 @@ void InterpreterSystemQuery::restartReplicas(Context & system_context)
 void InterpreterSystemQuery::syncReplica(ASTSystemQuery &)
 {
     context.checkAccess(AccessType::SYSTEM_SYNC_REPLICA, table_id);
-    StoragePtr table = DatabaseCatalog::instance().getTable(table_id);
+    StoragePtr table = DatabaseCatalog::instance().getTable(table_id, context);
 
     if (auto * storage_replicated = dynamic_cast<StorageReplicatedMergeTree *>(table.get()))
     {
@@ -416,7 +416,7 @@ void InterpreterSystemQuery::flushDistributed(ASTSystemQuery &)
 {
     context.checkAccess(AccessType::SYSTEM_FLUSH_DISTRIBUTED, table_id);
 
-    if (auto * storage_distributed = dynamic_cast<StorageDistributed *>(DatabaseCatalog::instance().getTable(table_id).get()))
+    if (auto * storage_distributed = dynamic_cast<StorageDistributed *>(DatabaseCatalog::instance().getTable(table_id, context).get()))
         storage_distributed->flushClusterNodesAllData();
     else
         throw Exception("Table " + table_id.getNameForLogs() + " is not distributed", ErrorCodes::BAD_ARGUMENTS);
diff --git a/src/Interpreters/InterpreterWatchQuery.cpp b/src/Interpreters/InterpreterWatchQuery.cpp
index dc1ae6a7cad..489be488b4d 100644
--- a/src/Interpreters/InterpreterWatchQuery.cpp
+++ b/src/Interpreters/InterpreterWatchQuery.cpp
@@ -40,7 +40,7 @@ BlockIO InterpreterWatchQuery::execute()
     auto table_id = context.resolveStorageID(query, Context::ResolveOrdinary);
 
     /// Get storage
-    storage = DatabaseCatalog::instance().tryGetTable(table_id);
+    storage = DatabaseCatalog::instance().tryGetTable(table_id, context);
 
     if (!storage)
         throw Exception("Table " + table_id.getNameForLogs() + " doesn't exist.",
diff --git a/src/Interpreters/JoinedTables.cpp b/src/Interpreters/JoinedTables.cpp
index 3bd883dcce8..7450890952a 100644
--- a/src/Interpreters/JoinedTables.cpp
+++ b/src/Interpreters/JoinedTables.cpp
@@ -181,7 +181,7 @@ StoragePtr JoinedTables::getLeftTableStorage()
     }
 
     /// Read from table. Even without table expression (implicit SELECT ... FROM system.one).
-    return DatabaseCatalog::instance().getTable(table_id);
+    return DatabaseCatalog::instance().getTable(table_id, context);
 }
 
 bool JoinedTables::resolveTables()
@@ -261,7 +261,7 @@ std::shared_ptr<TableJoin> JoinedTables::makeTableJoin(const ASTSelectQuery & se
     if (table_to_join.database_and_table_name)
     {
         auto joined_table_id = context.resolveStorageID(table_to_join.database_and_table_name);
-        StoragePtr table = DatabaseCatalog::instance().tryGetTable(joined_table_id);
+        StoragePtr table = DatabaseCatalog::instance().tryGetTable(joined_table_id, context);
         if (table)
         {
             if (dynamic_cast<StorageJoin *>(table.get()) ||
diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h
index 3c0d2159ef9..649cfa28e6e 100644
--- a/src/Interpreters/SystemLog.h
+++ b/src/Interpreters/SystemLog.h
@@ -431,7 +431,7 @@ void SystemLog<LogElement>::prepareTable()
 {
     String description = table_id.getNameForLogs();
 
-    table = DatabaseCatalog::instance().tryGetTable(table_id);
+    table = DatabaseCatalog::instance().tryGetTable(table_id, context);
 
     if (table)
     {
@@ -442,7 +442,7 @@ void SystemLog<LogElement>::prepareTable()
         {
             /// Rename the existing table.
             int suffix = 0;
-            while (DatabaseCatalog::instance().isTableExist({table_id.database_name, table_id.table_name + "_" + toString(suffix)}))
+            while (DatabaseCatalog::instance().isTableExist({table_id.database_name, table_id.table_name + "_" + toString(suffix)}, context))
                 ++suffix;
 
             auto rename = std::make_shared<ASTRenameQuery>();
@@ -483,7 +483,7 @@ void SystemLog<LogElement>::prepareTable()
         interpreter.setInternal(true);
         interpreter.execute();
 
-        table = DatabaseCatalog::instance().getTable(table_id);
+        table = DatabaseCatalog::instance().getTable(table_id, context);
     }
 
     is_prepared = true;
diff --git a/src/Interpreters/getTableExpressions.cpp b/src/Interpreters/getTableExpressions.cpp
index b5444f73b35..8467a98685d 100644
--- a/src/Interpreters/getTableExpressions.cpp
+++ b/src/Interpreters/getTableExpressions.cpp
@@ -96,7 +96,7 @@ static NamesAndTypesList getColumnsFromTableExpression(const ASTTableExpression
     else if (table_expression.database_and_table_name)
     {
         auto table_id = context.resolveStorageID(table_expression.database_and_table_name);
-        const auto & table = DatabaseCatalog::instance().getTable(table_id);
+        const auto & table = DatabaseCatalog::instance().getTable(table_id, context);
         const auto & columns = table->getColumns();
         names_and_type_list = columns.getOrdinary();
         materialized = columns.getMaterialized();
diff --git a/src/Interpreters/interpretSubquery.cpp b/src/Interpreters/interpretSubquery.cpp
index e108db1af30..c94759897f5 100644
--- a/src/Interpreters/interpretSubquery.cpp
+++ b/src/Interpreters/interpretSubquery.cpp
@@ -96,7 +96,7 @@ std::shared_ptr<InterpreterSelectWithUnionQuery> interpretSubquery(
         else
         {
             auto table_id = context.resolveStorageID(table_expression);
-            const auto & storage = DatabaseCatalog::instance().getTable(table_id);
+            const auto & storage = DatabaseCatalog::instance().getTable(table_id, context);
             columns = storage->getColumns().getOrdinary();
             select_query->replaceDatabaseAndTable(table_id);
         }
diff --git a/src/Server/MySQLHandler.cpp b/src/Server/MySQLHandler.cpp
index dabb0c12208..51b3d7eaef5 100644
--- a/src/Server/MySQLHandler.cpp
+++ b/src/Server/MySQLHandler.cpp
@@ -253,7 +253,7 @@ void MySQLHandler::comFieldList(ReadBuffer & payload)
     ComFieldList packet;
     packet.readPayload(payload);
     String database = connection_context.getCurrentDatabase();
-    StoragePtr table_ptr = DatabaseCatalog::instance().getTable({database, packet.table});
+    StoragePtr table_ptr = DatabaseCatalog::instance().getTable({database, packet.table}, connection_context);
     for (const NameAndTypePair & column: table_ptr->getColumns().getAll())
     {
         ColumnDefinition column_definition(
diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp
index b8f3660179d..561634e11d7 100644
--- a/src/Server/TCPHandler.cpp
+++ b/src/Server/TCPHandler.cpp
@@ -474,7 +474,7 @@ void TCPHandler::processInsertQuery(const Settings & connection_settings)
         if (query_context->getSettingsRef().input_format_defaults_for_omitted_fields)
         {
             if (!table_id.empty())
-                sendTableColumns(DatabaseCatalog::instance().getTable(table_id)->getColumns());
+                sendTableColumns(DatabaseCatalog::instance().getTable(table_id, *query_context)->getColumns());
         }
     }
 
@@ -627,7 +627,7 @@ void TCPHandler::processTablesStatusRequest()
     for (const QualifiedTableName & table_name: request.tables)
     {
         auto resolved_id = connection_context.tryResolveStorageID({table_name.database, table_name.table});
-        StoragePtr table = DatabaseCatalog::instance().tryGetTable(resolved_id);
+        StoragePtr table = DatabaseCatalog::instance().tryGetTable(resolved_id, connection_context);
         if (!table)
             continue;
 
@@ -944,7 +944,7 @@ bool TCPHandler::receiveData(bool scalar)
                 StoragePtr storage;
                 /// If such a table does not exist, create it.
                 if (resolved)
-                    storage = DatabaseCatalog::instance().getTable(resolved);
+                    storage = DatabaseCatalog::instance().getTable(resolved, *query_context);
                 else
                 {
                     NamesAndTypesList columns = block.getNamesAndTypesList();
diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp
index 7731cf3c06a..259d3d68eb0 100644
--- a/src/Storages/Kafka/StorageKafka.cpp
+++ b/src/Storages/Kafka/StorageKafka.cpp
@@ -361,7 +361,7 @@ bool StorageKafka::checkDependencies(const StorageID & table_id)
     // Check the dependencies are ready?
     for (const auto & db_tab : dependencies)
     {
-        auto table = DatabaseCatalog::instance().tryGetTable(db_tab);
+        auto table = DatabaseCatalog::instance().tryGetTable(db_tab, global_context);
         if (!table)
             return false;
 
@@ -429,7 +429,7 @@ void StorageKafka::threadFunc()
 bool StorageKafka::streamToViews()
 {
     auto table_id = getStorageID();
-    auto table = DatabaseCatalog::instance().getTable(table_id);
+    auto table = DatabaseCatalog::instance().getTable(table_id, global_context);
     if (!table)
         throw Exception("Engine table " + table_id.getNameForLogs() + " doesn't exist.", ErrorCodes::LOGICAL_ERROR);
 
diff --git a/src/Storages/LiveView/StorageLiveView.cpp b/src/Storages/LiveView/StorageLiveView.cpp
index cd660407c89..633669cbf1b 100644
--- a/src/Storages/LiveView/StorageLiveView.cpp
+++ b/src/Storages/LiveView/StorageLiveView.cpp
@@ -401,7 +401,7 @@ void StorageLiveView::noUsersThread(std::shared_ptr<StorageLiveView> storage, co
 
     if (drop_table)
     {
-        if (DatabaseCatalog::instance().tryGetTable(table_id))
+        if (DatabaseCatalog::instance().tryGetTable(table_id, storage->global_context))
         {
             try
             {
diff --git a/src/Storages/LiveView/StorageLiveView.h b/src/Storages/LiveView/StorageLiveView.h
index 801b0b42ec4..fe62de224da 100644
--- a/src/Storages/LiveView/StorageLiveView.h
+++ b/src/Storages/LiveView/StorageLiveView.h
@@ -53,7 +53,7 @@ public:
     {
         return getStorageID().table_name + "_blocks";
     }
-    StoragePtr getParentStorage() const { return DatabaseCatalog::instance().getTable(select_table_id); }
+    StoragePtr getParentStorage() const { return DatabaseCatalog::instance().getTable(select_table_id, global_context); }
 
     ASTPtr getInnerQuery() const { return inner_query->clone(); }
     ASTPtr getInnerSubQuery() const
diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp
index 95be1275d3d..47beeec196f 100644
--- a/src/Storages/StorageBuffer.cpp
+++ b/src/Storages/StorageBuffer.cpp
@@ -129,7 +129,7 @@ QueryProcessingStage::Enum StorageBuffer::getQueryProcessingStage(const Context
 {
     if (destination_id)
     {
-        auto destination = DatabaseCatalog::instance().getTable(destination_id);
+        auto destination = DatabaseCatalog::instance().getTable(destination_id, context);
 
         if (destination.get() == this)
             throw Exception("Destination table is myself. Read will cause infinite loop.", ErrorCodes::INFINITE_LOOP);
@@ -153,7 +153,7 @@ Pipes StorageBuffer::read(
 
     if (destination_id)
     {
-        auto destination = DatabaseCatalog::instance().getTable(destination_id);
+        auto destination = DatabaseCatalog::instance().getTable(destination_id, context);
 
         if (destination.get() == this)
             throw Exception("Destination table is myself. Read will cause infinite loop.", ErrorCodes::INFINITE_LOOP);
@@ -334,7 +334,7 @@ public:
         StoragePtr destination;
         if (storage.destination_id)
         {
-            destination = DatabaseCatalog::instance().tryGetTable(storage.destination_id);
+            destination = DatabaseCatalog::instance().tryGetTable(storage.destination_id, storage.global_context);
             if (destination.get() == &storage)
                 throw Exception("Destination table is myself. Write will cause infinite loop.", ErrorCodes::INFINITE_LOOP);
         }
@@ -434,7 +434,7 @@ bool StorageBuffer::mayBenefitFromIndexForIn(const ASTPtr & left_in_operand, con
     if (!destination_id)
         return false;
 
-    auto destination = DatabaseCatalog::instance().getTable(destination_id);
+    auto destination = DatabaseCatalog::instance().getTable(destination_id, query_context);
 
     if (destination.get() == this)
         throw Exception("Destination table is myself. Read will cause infinite loop.", ErrorCodes::INFINITE_LOOP);
@@ -602,7 +602,7 @@ void StorageBuffer::flushBuffer(Buffer & buffer, bool check_thresholds, bool loc
         */
     try
     {
-        writeBlockToDestination(block_to_write, DatabaseCatalog::instance().tryGetTable(destination_id));
+        writeBlockToDestination(block_to_write, DatabaseCatalog::instance().tryGetTable(destination_id, global_context));
     }
     catch (...)
     {
@@ -739,7 +739,7 @@ void StorageBuffer::checkAlterIsPossible(const AlterCommands & commands, const S
 std::optional<UInt64> StorageBuffer::totalRows() const
 {
     std::optional<UInt64> underlying_rows;
-    auto underlying = DatabaseCatalog::instance().tryGetTable(destination_id);
+    auto underlying = DatabaseCatalog::instance().tryGetTable(destination_id, global_context);
 
     if (underlying)
         underlying_rows = underlying->totalRows();
diff --git a/src/Storages/StorageBuffer.h b/src/Storages/StorageBuffer.h
index 6d6c1f66569..10a4482c801 100644
--- a/src/Storages/StorageBuffer.h
+++ b/src/Storages/StorageBuffer.h
@@ -75,7 +75,7 @@ public:
     {
         if (!destination_id)
             return false;
-        auto dest = DatabaseCatalog::instance().tryGetTable(destination_id);
+        auto dest = DatabaseCatalog::instance().tryGetTable(destination_id, global_context);
         if (dest && dest.get() != this)
             return dest->supportsPrewhere();
         return false;
diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp
index a565c8c6260..2afa7f7d713 100644
--- a/src/Storages/StorageMaterializedView.cpp
+++ b/src/Storages/StorageMaterializedView.cpp
@@ -149,7 +149,7 @@ StorageMaterializedView::StorageMaterializedView(
         create_interpreter.setInternal(true);
         create_interpreter.execute();
 
-        target_table_id = DatabaseCatalog::instance().getTable({manual_create_query->database, manual_create_query->table})->getStorageID();
+        target_table_id = DatabaseCatalog::instance().getTable({manual_create_query->database, manual_create_query->table}, global_context)->getStorageID();
     }
 
     if (!select_table_id.empty())
@@ -204,7 +204,7 @@ BlockOutputStreamPtr StorageMaterializedView::write(const ASTPtr & query, const
 
 static void executeDropQuery(ASTDropQuery::Kind kind, Context & global_context, const StorageID & target_table_id)
 {
-    if (DatabaseCatalog::instance().tryGetTable(target_table_id))
+    if (DatabaseCatalog::instance().tryGetTable(target_table_id, global_context))
     {
         /// We create and execute `drop` query for internal table.
         auto drop_query = std::make_shared<ASTDropQuery>();
@@ -362,12 +362,12 @@ void StorageMaterializedView::shutdown()
 
 StoragePtr StorageMaterializedView::getTargetTable() const
 {
-    return DatabaseCatalog::instance().getTable(target_table_id);
+    return DatabaseCatalog::instance().getTable(target_table_id, global_context);
 }
 
 StoragePtr StorageMaterializedView::tryGetTargetTable() const
 {
-    return DatabaseCatalog::instance().tryGetTable(target_table_id);
+    return DatabaseCatalog::instance().tryGetTable(target_table_id, global_context);
 }
 
 Strings StorageMaterializedView::getDataPaths() const
diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp
index f6114d709b6..c6af41cc163 100644
--- a/src/Storages/StorageMergeTree.cpp
+++ b/src/Storages/StorageMergeTree.cpp
@@ -990,7 +990,7 @@ void StorageMergeTree::alterPartition(const ASTPtr & query, const PartitionComma
                     case PartitionCommand::MoveDestinationType::TABLE:
                         checkPartitionCanBeDropped(command.partition);
                         String dest_database = context.resolveDatabase(command.to_database);
-                        auto dest_storage = DatabaseCatalog::instance().getTable({dest_database, command.to_table});
+                        auto dest_storage = DatabaseCatalog::instance().getTable({dest_database, command.to_table}, context);
                         movePartitionToTable(dest_storage, command.partition, context);
                         break;
                 }
@@ -1002,7 +1002,7 @@ void StorageMergeTree::alterPartition(const ASTPtr & query, const PartitionComma
             {
                 checkPartitionCanBeDropped(command.partition);
                 String from_database = context.resolveDatabase(command.from_database);
-                auto from_storage = DatabaseCatalog::instance().getTable({from_database, command.from_table});
+                auto from_storage = DatabaseCatalog::instance().getTable({from_database, command.from_table}, context);
                 replacePartitionFrom(from_storage, command.partition, command.replace, context);
             }
             break;
diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index b4ca97602c4..00565e777ae 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -1570,7 +1570,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry)
 
     auto clone_data_parts_from_source_table = [&] () -> size_t
     {
-        source_table = DatabaseCatalog::instance().tryGetTable(source_table_id);
+        source_table = DatabaseCatalog::instance().tryGetTable(source_table_id, global_context);
         if (!source_table)
         {
             LOG_DEBUG(log, "Can't use {} as source table for REPLACE PARTITION command. It does not exist.", source_table_id.getNameForLogs());
@@ -3485,7 +3485,7 @@ void StorageReplicatedMergeTree::alterPartition(const ASTPtr & query, const Part
                     case PartitionCommand::MoveDestinationType::TABLE:
                         checkPartitionCanBeDropped(command.partition);
                         String dest_database = query_context.resolveDatabase(command.to_database);
-                        auto dest_storage = DatabaseCatalog::instance().getTable({dest_database, command.to_table});
+                        auto dest_storage = DatabaseCatalog::instance().getTable({dest_database, command.to_table}, query_context);
                         movePartitionToTable(dest_storage, command.partition, query_context);
                         break;
                 }
@@ -3496,7 +3496,7 @@ void StorageReplicatedMergeTree::alterPartition(const ASTPtr & query, const Part
             {
                 checkPartitionCanBeDropped(command.partition);
                 String from_database = query_context.resolveDatabase(command.from_database);
-                auto from_storage = DatabaseCatalog::instance().getTable({from_database, command.from_table});
+                auto from_storage = DatabaseCatalog::instance().getTable({from_database, command.from_table}, query_context);
                 replacePartitionFrom(from_storage, command.partition, command.replace, query_context);
             }
             break;
diff --git a/src/Storages/getStructureOfRemoteTable.cpp b/src/Storages/getStructureOfRemoteTable.cpp
index 711612cf9e5..19d1172f1ff 100644
--- a/src/Storages/getStructureOfRemoteTable.cpp
+++ b/src/Storages/getStructureOfRemoteTable.cpp
@@ -84,7 +84,7 @@ ColumnsDescription getStructureOfRemoteTableInShard(
     else
     {
         if (shard_info.isLocal())
-            return DatabaseCatalog::instance().getTable(table_id)->getColumns();
+            return DatabaseCatalog::instance().getTable(table_id, context)->getColumns();
 
         /// Request for a table description
         query = "DESC TABLE " + table_id.getFullTableName();

From 7df5280060cd05942b255710acb0815136d67887 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 29 May 2020 02:09:11 +0300
Subject: [PATCH 146/184] Fix potentially flacky test

---
 .../0_stateless/00731_long_merge_tree_select_opened_files.sh  | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh b/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh
index 5177cb6087c..5ae9dcf67ec 100755
--- a/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh
+++ b/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh
@@ -14,9 +14,11 @@ $CLICKHOUSE_CLIENT $settings -n -q "
 DROP TABLE IF EXISTS merge_tree_table;
 CREATE TABLE merge_tree_table (id UInt64, date Date, uid UInt32) ENGINE = MergeTree(date, id, 8192);"
 
+
 $CLICKHOUSE_CLIENT $settings -q "INSERT INTO merge_tree_table SELECT (intHash64(number)) % 10000, toDate('2018-08-01'), rand() FROM system.numbers LIMIT 10000000;"
 
-$CLICKHOUSE_CLIENT $settings -q "OPTIMIZE TABLE merge_tree_table FINAL;"
+# If merge is already happening, OPTIMIZE will be noop. But we have to ensure that the data is merged.
+for i in {1..100}; do $CLICKHOUSE_CLIENT $settings --optimize_throw_if_noop=1 -q "OPTIMIZE TABLE merge_tree_table FINAL;" && break; sleep 1; done
 
 # The query may open more files if query log will be flushed during the query.
 # To lower this chance, we also flush logs before the query.

From e8f52337aacb261f55d72716a09bc85490464286 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 29 May 2020 03:09:12 +0300
Subject: [PATCH 147/184] Fix incorrect usage of fmtlib

---
 base/common/logger_useful.h    | 18 +++++++++++++++---
 src/Common/tests/gtest_log.cpp | 19 +++++++++++++++++++
 2 files changed, 34 insertions(+), 3 deletions(-)
 create mode 100644 src/Common/tests/gtest_log.cpp

diff --git a/base/common/logger_useful.h b/base/common/logger_useful.h
index c89ec63ef22..ad56ef7e857 100644
--- a/base/common/logger_useful.h
+++ b/base/common/logger_useful.h
@@ -15,15 +15,27 @@ using Poco::Message;
 using DB::LogsLevel;
 using DB::CurrentThread;
 
-/// Logs a message to a specified logger with that level.
 
-#define LOG_IMPL(logger, priority, PRIORITY, ...) do                    \
+namespace
+{
+    template <typename... Ts> constexpr size_t numArgs(Ts &&...) { return sizeof...(Ts); }
+    template <typename T, typename... Ts> constexpr auto firstArg(T && x, Ts &&...) { return std::forward<T>(x); }
+}
+
+
+/// Logs a message to a specified logger with that level.
+/// If more than one argument is provided,
+///  the first argument is interpreted as template with {}-substitutions
+///  and the latter arguments treat as values to substitute.
+/// If only one argument is provided, it is threat as message without substitutions.
+
+#define LOG_IMPL(logger, priority, PRIORITY, ...) do                              \
 {                                                                                 \
     const bool is_clients_log = (CurrentThread::getGroup() != nullptr) &&         \
             (CurrentThread::getGroup()->client_logs_level >= (priority));         \
     if ((logger)->is((PRIORITY)) || is_clients_log)                               \
     {                                                                             \
-        std::string formatted_message = fmt::format(__VA_ARGS__);                 \
+        std::string formatted_message = numArgs(__VA_ARGS__) > 1 ? fmt::format(__VA_ARGS__) : firstArg(__VA_ARGS__); \
         if (auto channel = (logger)->getChannel())                                \
         {                                                                         \
             std::string file_function;                                            \
diff --git a/src/Common/tests/gtest_log.cpp b/src/Common/tests/gtest_log.cpp
new file mode 100644
index 00000000000..a1e532f92e6
--- /dev/null
+++ b/src/Common/tests/gtest_log.cpp
@@ -0,0 +1,19 @@
+#include <string>
+#include <vector>
+#include <common/logger_useful.h>
+#include <gtest/gtest.h>
+
+#include <Poco/Logger.h>
+#include <Poco/AutoPtr.h>
+#include <Poco/NullChannel.h>
+
+
+TEST(Logger, Log)
+{
+    Poco::Logger::root().setLevel("none");
+    Poco::Logger::root().setChannel(Poco::AutoPtr<Poco::NullChannel>(new Poco::NullChannel()));
+    Logger * log = &Logger::get("Log");
+
+    /// This test checks that we don't pass this string to fmtlib, because it is the only argument.
+    EXPECT_NO_THROW(LOG_INFO(log, "Hello {} World"));
+}

From 263bc6e097a271df52af743caa6339db2a95abfb Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 29 May 2020 03:26:56 +0300
Subject: [PATCH 148/184] Added missing file

---
 tests/queries/0_stateless/00342_escape_sequences.reference | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 tests/queries/0_stateless/00342_escape_sequences.reference

diff --git a/tests/queries/0_stateless/00342_escape_sequences.reference b/tests/queries/0_stateless/00342_escape_sequences.reference
new file mode 100644
index 00000000000..50e0756d16f
--- /dev/null
+++ b/tests/queries/0_stateless/00342_escape_sequences.reference
@@ -0,0 +1 @@
+07080C0A0D090B5C27225C3FAA

From 13b0acdc9f44e474a0c02fa186dc2372d33130d6 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 29 May 2020 03:42:25 +0300
Subject: [PATCH 149/184] Fix strange code

---
 docker/test/test_runner.sh          | 76 ++++++++++++++---------------
 utils/ci/docker-multiarch/update.sh | 60 +++++++++++------------
 2 files changed, 68 insertions(+), 68 deletions(-)

diff --git a/docker/test/test_runner.sh b/docker/test/test_runner.sh
index 76e142e61f9..561117492b0 100755
--- a/docker/test/test_runner.sh
+++ b/docker/test/test_runner.sh
@@ -12,7 +12,7 @@ readonly CLICKHOUSE_PACKAGES_ARG="${2}"
 CLICKHOUSE_SERVER_IMAGE="${3}"
 
 if [ ${CLICKHOUSE_PACKAGES_ARG} != ${NO_REBUILD_FLAG} ]; then
-	readonly CLICKHOUSE_PACKAGES_DIR="$(realpath ${2})" # or --no-rebuild
+    readonly CLICKHOUSE_PACKAGES_DIR="$(realpath ${2})" # or --no-rebuild
 fi
 
 
@@ -26,19 +26,19 @@ fi
 # TODO: optionally mount most recent clickhouse-test and queries directory from local machine
 
 if [ ${CLICKHOUSE_PACKAGES_ARG} != ${NO_REBUILD_FLAG} ]; then
-	docker build \
-		-f "${CLICKHOUSE_DOCKER_DIR}/test/stateless/clickhouse-statelest-test-runner.Dockerfile" \
-		--target clickhouse-test-runner-base \
-		-t clickhouse-test-runner-base:preinstall \
-		"${CLICKHOUSE_DOCKER_DIR}/test/stateless"
+    docker build \
+        -f "${CLICKHOUSE_DOCKER_DIR}/test/stateless/clickhouse-statelest-test-runner.Dockerfile" \
+        --target clickhouse-test-runner-base \
+        -t clickhouse-test-runner-base:preinstall \
+        "${CLICKHOUSE_DOCKER_DIR}/test/stateless"
 
-	docker rm -f clickhouse-test-runner-installing-packages || true
-	docker run \
-		-v "${CLICKHOUSE_PACKAGES_DIR}:/packages" \
-		--name clickhouse-test-runner-installing-packages \
-		clickhouse-test-runner-base:preinstall
-	docker commit clickhouse-test-runner-installing-packages clickhouse-statelest-test-runner:local
-	docker rm -f clickhouse-test-runner-installing-packages || true
+    docker rm -f clickhouse-test-runner-installing-packages || true
+    docker run \
+        -v "${CLICKHOUSE_PACKAGES_DIR}:/packages" \
+        --name clickhouse-test-runner-installing-packages \
+        clickhouse-test-runner-base:preinstall
+    docker commit clickhouse-test-runner-installing-packages clickhouse-statelest-test-runner:local
+    docker rm -f clickhouse-test-runner-installing-packages || true
 fi
 
 # # Create a bind-volume to the clickhouse-test script file
@@ -47,38 +47,38 @@ fi
 
 # Build server image (optional) from local packages
 if [ -z "${CLICKHOUSE_SERVER_IMAGE}" ]; then
-	CLICKHOUSE_SERVER_IMAGE="yandex/clickhouse-server:local"
+    CLICKHOUSE_SERVER_IMAGE="yandex/clickhouse-server:local"
 
-	if [ ${CLICKHOUSE_PACKAGES_ARG} != ${NO_REBUILD_FLAG} ]; then
-		docker build \
-			-f "${CLICKHOUSE_DOCKER_DIR}/server/local.Dockerfile" \
-			--target clickhouse-server-base \
-			-t clickhouse-server-base:preinstall \
-			"${CLICKHOUSE_DOCKER_DIR}/server"
+    if [ ${CLICKHOUSE_PACKAGES_ARG} != ${NO_REBUILD_FLAG} ]; then
+        docker build \
+            -f "${CLICKHOUSE_DOCKER_DIR}/server/local.Dockerfile" \
+            --target clickhouse-server-base \
+            -t clickhouse-server-base:preinstall \
+            "${CLICKHOUSE_DOCKER_DIR}/server"
 
-		docker rm -f clickhouse_server_base_installing_server || true
-		docker run -v "${CLICKHOUSE_PACKAGES_DIR}:/packages" \
-			--name clickhouse_server_base_installing_server \
-			clickhouse-server-base:preinstall
-		docker commit clickhouse_server_base_installing_server clickhouse-server-base:postinstall
+        docker rm -f clickhouse_server_base_installing_server || true
+        docker run -v "${CLICKHOUSE_PACKAGES_DIR}:/packages" \
+            --name clickhouse_server_base_installing_server \
+            clickhouse-server-base:preinstall
+        docker commit clickhouse_server_base_installing_server clickhouse-server-base:postinstall
 
-		docker build \
-			-f "${CLICKHOUSE_DOCKER_DIR}/server/local.Dockerfile" \
-			--target clickhouse-server \
-			-t "${CLICKHOUSE_SERVER_IMAGE}" \
-			"${CLICKHOUSE_DOCKER_DIR}/server"
-	fi
+        docker build \
+            -f "${CLICKHOUSE_DOCKER_DIR}/server/local.Dockerfile" \
+            --target clickhouse-server \
+            -t "${CLICKHOUSE_SERVER_IMAGE}" \
+            "${CLICKHOUSE_DOCKER_DIR}/server"
+    fi
 fi
 
 docker rm -f test-runner || true
 docker-compose down
 CLICKHOUSE_SERVER_IMAGE="${CLICKHOUSE_SERVER_IMAGE}" \
-	docker-compose -f "${CLICKHOUSE_DOCKER_DIR}/test/test_runner_docker_compose.yaml" \
-	create \
-	--build --force-recreate
+    docker-compose -f "${CLICKHOUSE_DOCKER_DIR}/test/test_runner_docker_compose.yaml" \
+    create \
+    --build --force-recreate
 
 CLICKHOUSE_SERVER_IMAGE="${CLICKHOUSE_SERVER_IMAGE}" \
-	docker-compose -f "${CLICKHOUSE_DOCKER_DIR}/test/test_runner_docker_compose.yaml" \
-	run \
-	--name test-runner \
-	test-runner
+    docker-compose -f "${CLICKHOUSE_DOCKER_DIR}/test/test_runner_docker_compose.yaml" \
+    run \
+    --name test-runner \
+    test-runner
diff --git a/utils/ci/docker-multiarch/update.sh b/utils/ci/docker-multiarch/update.sh
index 6e3e18047de..6abcf339607 100755
--- a/utils/ci/docker-multiarch/update.sh
+++ b/utils/ci/docker-multiarch/update.sh
@@ -37,27 +37,27 @@ fi
 
 # get the image
 if \
-	wget -q --spider "$baseUrl/current" \
-	&& wget -q --spider "$baseUrl/current/$thisTar" \
-	; then
-		baseUrl+='/current'
+    wget -q --spider "$baseUrl/current" \
+    && wget -q --spider "$baseUrl/current/$thisTar" \
+    ; then
+        baseUrl+='/current'
 fi
 wget -qN "$baseUrl/"{{MD5,SHA{1,256}}SUMS{,.gpg},"$thisTarBase.manifest",'unpacked/build-info.txt'} || true
 wget -N "$baseUrl/$thisTar"
 
 # check checksum
 if [ -f SHA256SUMS ]; then
-	sha256sum="$(sha256sum "$thisTar" | cut -d' ' -f1)"
-	if ! grep -q "$sha256sum" SHA256SUMS; then
-		echo >&2 "error: '$thisTar' has invalid SHA256"
-		exit 1
-	fi
+    sha256sum="$(sha256sum "$thisTar" | cut -d' ' -f1)"
+    if ! grep -q "$sha256sum" SHA256SUMS; then
+        echo >&2 "error: '$thisTar' has invalid SHA256"
+        exit 1
+    fi
 fi
 
 cat > Dockerfile <<-EOF
-	FROM scratch
-	ADD $thisTar /
-	ENV ARCH=${ARCH} UBUNTU_SUITE=${VERSION} DOCKER_REPO=${DOCKER_REPO}
+    FROM scratch
+    ADD $thisTar /
+    ENV ARCH=${ARCH} UBUNTU_SUITE=${VERSION} DOCKER_REPO=${DOCKER_REPO}
 EOF
 
 # add qemu-user-static binary
@@ -70,26 +70,26 @@ EOF
 fi
 
 cat >> Dockerfile <<-EOF
-	# a few minor docker-specific tweaks
-	# see https://github.com/docker/docker/blob/master/contrib/mkimage/debootstrap
-	RUN echo '#!/bin/sh' > /usr/sbin/policy-rc.d \\
-		&& echo 'exit 101' >> /usr/sbin/policy-rc.d \\
-		&& chmod +x /usr/sbin/policy-rc.d \\
-		&& dpkg-divert --local --rename --add /sbin/initctl \\
-		&& cp -a /usr/sbin/policy-rc.d /sbin/initctl \\
-		&& sed -i 's/^exit.*/exit 0/' /sbin/initctl \\
-		&& echo 'force-unsafe-io' > /etc/dpkg/dpkg.cfg.d/docker-apt-speedup \\
-		&& echo 'DPkg::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };' > /etc/apt/apt.conf.d/docker-clean \\
-		&& echo 'APT::Update::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };' >> /etc/apt/apt.conf.d/docker-clean \\
-		&& echo 'Dir::Cache::pkgcache ""; Dir::Cache::srcpkgcache "";' >> /etc/apt/apt.conf.d/docker-clean \\
-		&& echo 'Acquire::Languages "none";' > /etc/apt/apt.conf.d/docker-no-languages \\
-		&& echo 'Acquire::GzipIndexes "true"; Acquire::CompressionTypes::Order:: "gz";' > /etc/apt/apt.conf.d/docker-gzip-indexes
+    # a few minor docker-specific tweaks
+    # see https://github.com/docker/docker/blob/master/contrib/mkimage/debootstrap
+    RUN echo '#!/bin/sh' > /usr/sbin/policy-rc.d \\
+        && echo 'exit 101' >> /usr/sbin/policy-rc.d \\
+        && chmod +x /usr/sbin/policy-rc.d \\
+        && dpkg-divert --local --rename --add /sbin/initctl \\
+        && cp -a /usr/sbin/policy-rc.d /sbin/initctl \\
+        && sed -i 's/^exit.*/exit 0/' /sbin/initctl \\
+        && echo 'force-unsafe-io' > /etc/dpkg/dpkg.cfg.d/docker-apt-speedup \\
+        && echo 'DPkg::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };' > /etc/apt/apt.conf.d/docker-clean \\
+        && echo 'APT::Update::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };' >> /etc/apt/apt.conf.d/docker-clean \\
+        && echo 'Dir::Cache::pkgcache ""; Dir::Cache::srcpkgcache "";' >> /etc/apt/apt.conf.d/docker-clean \\
+        && echo 'Acquire::Languages "none";' > /etc/apt/apt.conf.d/docker-no-languages \\
+        && echo 'Acquire::GzipIndexes "true"; Acquire::CompressionTypes::Order:: "gz";' > /etc/apt/apt.conf.d/docker-gzip-indexes
 
-	# enable the universe
-	RUN sed -i 's/^#\s*\(deb.*universe\)$/\1/g' /etc/apt/sources.list
+    # enable the universe
+    RUN sed -i 's/^#\s*\(deb.*universe\)$/\1/g' /etc/apt/sources.list
 
-	# overwrite this with 'CMD []' in a dependent Dockerfile
-	CMD ["/bin/bash"]
+    # overwrite this with 'CMD []' in a dependent Dockerfile
+    CMD ["/bin/bash"]
 EOF
 
 docker build -t "${DOCKER_REPO}:${TAG_ARCH}-${VERSION}" .

From 025c1a8830ff3fcd7a2b42557ca5942fb38fdd49 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 29 May 2020 03:46:42 +0300
Subject: [PATCH 150/184] Added another check #11170

---
 tests/clickhouse-test | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index 86a9ecebd96..2d155313b06 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -31,6 +31,7 @@ MESSAGES_TO_RETRY = [
     "DB::Exception: ZooKeeper session has been expired",
     "Coordination::Exception: Connection loss",
     "Operation timed out",
+    "ConnectionPoolWithFailover: Connection failed at try",
 ]
 
 

From 23d7947549cb44b45bdcf053af29aa45e9a2a211 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 29 May 2020 05:08:48 +0300
Subject: [PATCH 151/184] Fix constraints for DEFAULT fields; fix constraints
 for temporary tables; improve performance of INSERT for small blocks

---
 src/Core/ExternalTable.cpp                    |  2 +-
 src/Interpreters/DatabaseCatalog.cpp          |  8 +++--
 src/Interpreters/DatabaseCatalog.h            |  7 ++++-
 src/Interpreters/GlobalSubqueriesVisitor.h    |  2 +-
 src/Interpreters/InterpreterCreateQuery.cpp   |  2 +-
 src/Interpreters/InterpreterInsertQuery.cpp   | 24 +++++++++------
 src/Server/TCPHandler.cpp                     |  2 +-
 .../01286_constraints_on_default.reference    |  2 ++
 .../01286_constraints_on_default.sql          | 29 +++++++++++++++++++
 9 files changed, 62 insertions(+), 16 deletions(-)
 create mode 100644 tests/queries/0_stateless/01286_constraints_on_default.reference
 create mode 100644 tests/queries/0_stateless/01286_constraints_on_default.sql

diff --git a/src/Core/ExternalTable.cpp b/src/Core/ExternalTable.cpp
index 62a99cea97e..5ec6980dbfa 100644
--- a/src/Core/ExternalTable.cpp
+++ b/src/Core/ExternalTable.cpp
@@ -164,7 +164,7 @@ void ExternalTablesHandler::handlePart(const Poco::Net::MessageHeader & header,
 
     /// Create table
     NamesAndTypesList columns = sample_block.getNamesAndTypesList();
-    auto temporary_table = TemporaryTableHolder(context, ColumnsDescription{columns});
+    auto temporary_table = TemporaryTableHolder(context, ColumnsDescription{columns}, {});
     auto storage = temporary_table.getTable();
     context.addExternalTable(data->table_name, std::move(temporary_table));
     BlockOutputStreamPtr output = storage->write(ASTPtr(), context);
diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp
index 7606fdb255a..c55e60898b9 100644
--- a/src/Interpreters/DatabaseCatalog.cpp
+++ b/src/Interpreters/DatabaseCatalog.cpp
@@ -58,13 +58,17 @@ TemporaryTableHolder::TemporaryTableHolder(const Context & context_,
 }
 
 
-TemporaryTableHolder::TemporaryTableHolder(const Context & context_, const ColumnsDescription & columns, const ASTPtr & query)
+TemporaryTableHolder::TemporaryTableHolder(
+    const Context & context_,
+    const ColumnsDescription & columns,
+    const ConstraintsDescription & constraints,
+    const ASTPtr & query)
     : TemporaryTableHolder
       (
           context_,
           [&](const StorageID & table_id)
           {
-              return StorageMemory::create(table_id, ColumnsDescription{columns}, ConstraintsDescription{});
+              return StorageMemory::create(table_id, ColumnsDescription{columns}, ConstraintsDescription{constraints});
           },
           query
       )
diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h
index a481e3d7e5e..a274d294af0 100644
--- a/src/Interpreters/DatabaseCatalog.h
+++ b/src/Interpreters/DatabaseCatalog.h
@@ -21,6 +21,7 @@ class Context;
 class IDatabase;
 class Exception;
 class ColumnsDescription;
+struct ConstraintsDescription;
 
 using DatabasePtr = std::shared_ptr<IDatabase>;
 using DatabaseAndTable = std::pair<DatabasePtr, StoragePtr>;
@@ -71,7 +72,11 @@ struct TemporaryTableHolder : boost::noncopyable
     TemporaryTableHolder(const Context & context, const Creator & creator, const ASTPtr & query = {});
 
     /// Creates temporary table with Engine=Memory
-    TemporaryTableHolder(const Context & context, const ColumnsDescription & columns, const ASTPtr & query = {});
+    TemporaryTableHolder(
+        const Context & context,
+        const ColumnsDescription & columns,
+        const ConstraintsDescription & constraints,
+        const ASTPtr & query = {});
 
     TemporaryTableHolder(TemporaryTableHolder && rhs);
     TemporaryTableHolder & operator = (TemporaryTableHolder && rhs);
diff --git a/src/Interpreters/GlobalSubqueriesVisitor.h b/src/Interpreters/GlobalSubqueriesVisitor.h
index 37a358c3d28..9e616b04dab 100644
--- a/src/Interpreters/GlobalSubqueriesVisitor.h
+++ b/src/Interpreters/GlobalSubqueriesVisitor.h
@@ -103,7 +103,7 @@ public:
             Block sample = interpreter->getSampleBlock();
             NamesAndTypesList columns = sample.getNamesAndTypesList();
 
-            auto external_storage_holder = std::make_shared<TemporaryTableHolder>(context, ColumnsDescription{columns});
+            auto external_storage_holder = std::make_shared<TemporaryTableHolder>(context, ColumnsDescription{columns}, ConstraintsDescription{});
             StoragePtr external_storage = external_storage_holder->getTable();
 
             /** We replace the subquery with the name of the temporary table.
diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp
index 147ef7d739b..6d35e25ba44 100644
--- a/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/src/Interpreters/InterpreterCreateQuery.cpp
@@ -637,7 +637,7 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create,
         if (create.if_not_exists && context.tryResolveStorageID({"", table_name}, Context::ResolveExternal))
             return false;
 
-        auto temporary_table = TemporaryTableHolder(context, properties.columns, query_ptr);
+        auto temporary_table = TemporaryTableHolder(context, properties.columns, properties.constraints, query_ptr);
         context.getSessionContext().addExternalTable(table_name, std::move(temporary_table));
         return true;
     }
diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp
index b6efa5d6d46..149a78c3803 100644
--- a/src/Interpreters/InterpreterInsertQuery.cpp
+++ b/src/Interpreters/InterpreterInsertQuery.cpp
@@ -233,6 +233,21 @@ BlockIO InterpreterInsertQuery::execute()
             else
                 out = std::make_shared<PushingToViewsBlockOutputStream>(table, context, query_ptr, no_destination);
 
+            /// Note that we wrap transforms one on top of another, so we write them in reverse of data processing order.
+
+            /// Checking constraints. It must be done after calculation of all defaults, so we can check them on calculated columns.
+            if (const auto & constraints = table->getConstraints(); !constraints.empty())
+                out = std::make_shared<CheckConstraintsBlockOutputStream>(
+                    query.table_id, out, out->getHeader(), table->getConstraints(), context);
+
+            /// Actually we don't know structure of input blocks from query/table,
+            /// because some clients break insertion protocol (columns != header)
+            out = std::make_shared<AddingDefaultBlockOutputStream>(
+                out, query_sample_block, out->getHeader(), table->getColumns().getDefaults(), context);
+
+            /// It's important to squash blocks as early as possible (before other transforms),
+            ///  because other transforms may work inefficient if block size is small.
+
             /// Do not squash blocks if it is a sync INSERT into Distributed, since it lead to double bufferization on client and server side.
             /// Client-side bufferization might cause excessive timeouts (especially in case of big blocks).
             if (!(context.getSettingsRef().insert_distributed_sync && table->isRemote()) && !no_squash && !query.watch)
@@ -244,15 +259,6 @@ BlockIO InterpreterInsertQuery::execute()
                     context.getSettingsRef().min_insert_block_size_bytes);
             }
 
-            /// Actually we don't know structure of input blocks from query/table,
-            /// because some clients break insertion protocol (columns != header)
-            out = std::make_shared<AddingDefaultBlockOutputStream>(
-                out, query_sample_block, out->getHeader(), table->getColumns().getDefaults(), context);
-
-            if (const auto & constraints = table->getConstraints(); !constraints.empty())
-                out = std::make_shared<CheckConstraintsBlockOutputStream>(
-                    query.table_id, out, query_sample_block, table->getConstraints(), context);
-
             auto out_wrapper = std::make_shared<CountingBlockOutputStream>(out);
             out_wrapper->setProcessListElement(context.getProcessListElement());
             out = std::move(out_wrapper);
diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp
index b8f3660179d..65ed1f5eb90 100644
--- a/src/Server/TCPHandler.cpp
+++ b/src/Server/TCPHandler.cpp
@@ -948,7 +948,7 @@ bool TCPHandler::receiveData(bool scalar)
                 else
                 {
                     NamesAndTypesList columns = block.getNamesAndTypesList();
-                    auto temporary_table = TemporaryTableHolder(*query_context, ColumnsDescription{columns});
+                    auto temporary_table = TemporaryTableHolder(*query_context, ColumnsDescription{columns}, {});
                     storage = temporary_table.getTable();
                     query_context->addExternalTable(temporary_id.table_name, std::move(temporary_table));
                 }
diff --git a/tests/queries/0_stateless/01286_constraints_on_default.reference b/tests/queries/0_stateless/01286_constraints_on_default.reference
new file mode 100644
index 00000000000..aa47d0d46d4
--- /dev/null
+++ b/tests/queries/0_stateless/01286_constraints_on_default.reference
@@ -0,0 +1,2 @@
+0
+0
diff --git a/tests/queries/0_stateless/01286_constraints_on_default.sql b/tests/queries/0_stateless/01286_constraints_on_default.sql
new file mode 100644
index 00000000000..d150bac15b5
--- /dev/null
+++ b/tests/queries/0_stateless/01286_constraints_on_default.sql
@@ -0,0 +1,29 @@
+DROP TABLE IF EXISTS default_constraints;
+CREATE TABLE default_constraints
+(
+    x UInt8,
+    y UInt8 DEFAULT x + 1,
+    CONSTRAINT c CHECK y < 5
+) ENGINE = Memory;
+
+INSERT INTO default_constraints (x) SELECT number FROM system.numbers LIMIT 5; -- { serverError 469 }
+INSERT INTO default_constraints (x) VALUES (0),(1),(2),(3),(4); -- { serverError 469 }
+
+SELECT y, throwIf(NOT y < 5) FROM default_constraints;
+SELECT count() FROM default_constraints;
+
+DROP TABLE default_constraints;
+
+
+CREATE TEMPORARY TABLE default_constraints
+(
+    x UInt8,
+    y UInt8 DEFAULT x + 1,
+    CONSTRAINT c CHECK y < 5
+);
+
+INSERT INTO default_constraints (x) SELECT number FROM system.numbers LIMIT 5; -- { serverError 469 }
+INSERT INTO default_constraints (x) VALUES (0),(1),(2),(3),(4); -- { serverError 469 }
+
+SELECT y, throwIf(NOT y < 5) FROM default_constraints;
+SELECT count() FROM default_constraints;

From b7b63801411c64bdeffc7439cb6aeaea509ddc4f Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 29 May 2020 05:30:49 +0300
Subject: [PATCH 152/184] Add performance test

---
 .../insert_select_default_small_block.xml     | 38 +++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100644 tests/performance/insert_select_default_small_block.xml

diff --git a/tests/performance/insert_select_default_small_block.xml b/tests/performance/insert_select_default_small_block.xml
new file mode 100644
index 00000000000..12e67b09d2f
--- /dev/null
+++ b/tests/performance/insert_select_default_small_block.xml
@@ -0,0 +1,38 @@
+<test>
+    <settings><max_block_size>1</max_block_size></settings>
+
+    <create_query>
+CREATE TABLE insert_small_block_performance
+(
+    `x` String, 
+    `a` DEFAULT SHA256(x), 
+    `b` DEFAULT SHA256(toString(a)), 
+    `c` DEFAULT SHA256(toString(b)), 
+    `d` DEFAULT SHA256(toString(c)), 
+    `e` DEFAULT SHA256(toString(d)), 
+    `f` DEFAULT SHA256(toString(e)), 
+    `g` DEFAULT SHA256(toString(f)), 
+    `h` DEFAULT SHA256(toString(g)), 
+    `i` DEFAULT SHA256(toString(h)), 
+    `j` DEFAULT SHA256(toString(i)), 
+    `k` DEFAULT SHA256(toString(j)), 
+    `l` DEFAULT SHA256(toString(k)), 
+    `m` DEFAULT SHA256(toString(l)), 
+    `n` DEFAULT SHA256(toString(m)), 
+    `o` DEFAULT SHA256(toString(n)), 
+    `p` DEFAULT SHA256(toString(o)), 
+    `q` DEFAULT SHA256(toString(p)), 
+    `r` DEFAULT SHA256(toString(q)), 
+    `s` DEFAULT SHA256(toString(r)), 
+    `t` DEFAULT SHA256(toString(s)), 
+    `u` DEFAULT SHA256(toString(t)), 
+    `v` DEFAULT SHA256(toString(u)), 
+    `w` DEFAULT SHA256(toString(v))
+)
+ENGINE = Null;
+    </create_query>
+
+    <query>INSERT INTO insert_small_block_performance (x) SELECT toString(number) FROM numbers(10000);</query>
+
+    <drop_query>DROP TABLE IF EXISTS insert_small_block_performance</drop_query>
+</test>

From 9120495c4861aaf71a693ec0ad06c0333e68c4d5 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 29 May 2020 05:31:45 +0300
Subject: [PATCH 153/184] Increase chunk size for parallel parsing just in case

---
 src/Core/Settings.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index 142e0872d72..8cf90fa146a 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -126,7 +126,7 @@ struct Settings : public SettingsCollection<Settings>
     M(SettingBool, force_optimize_skip_unused_shards_no_nested, false, "Do not apply force_optimize_skip_unused_shards for nested Distributed tables.", 0) \
     \
     M(SettingBool, input_format_parallel_parsing, true, "Enable parallel parsing for some data formats.", 0) \
-    M(SettingUInt64, min_chunk_bytes_for_parallel_parsing, (1024 * 1024), "The minimum chunk size in bytes, which each thread will parse in parallel.", 0) \
+    M(SettingUInt64, min_chunk_bytes_for_parallel_parsing, (10 * 1024 * 1024), "The minimum chunk size in bytes, which each thread will parse in parallel.", 0) \
     \
     M(SettingUInt64, merge_tree_min_rows_for_concurrent_read, (20 * 8192), "If at least as many lines are read from one file, the reading can be parallelized.", 0) \
     M(SettingUInt64, merge_tree_min_bytes_for_concurrent_read, (24 * 10 * 1024 * 1024), "If at least as many bytes are read from one file, the reading can be parallelized.", 0) \

From 2d12b4d3ac606296675496c90fafcd62d7b17d30 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitbar@yandex-team.ru>
Date: Fri, 29 May 2020 09:54:27 +0300
Subject: [PATCH 154/184] Fix crash when SET DEFAULT ROLE is called with wrong
 arguments.

---
 src/Access/ExtendedRoleSet.cpp                | 24 ++++--
 src/Interpreters/InterpreterSetRoleQuery.cpp  |  2 +-
 src/Parsers/ASTExtendedRoleSet.h              |  5 +-
 src/Parsers/ParserCreateUserQuery.cpp         |  1 +
 src/Parsers/ParserSetRoleQuery.cpp            |  2 +
 .../integration/test_default_role/__init__.py |  0
 tests/integration/test_default_role/test.py   | 77 +++++++++++++++++++
 7 files changed, 103 insertions(+), 8 deletions(-)
 create mode 100644 tests/integration/test_default_role/__init__.py
 create mode 100644 tests/integration/test_default_role/test.py

diff --git a/src/Access/ExtendedRoleSet.cpp b/src/Access/ExtendedRoleSet.cpp
index a29ee40380c..a8e674b3722 100644
--- a/src/Access/ExtendedRoleSet.cpp
+++ b/src/Access/ExtendedRoleSet.cpp
@@ -68,15 +68,27 @@ void ExtendedRoleSet::init(const ASTExtendedRoleSet & ast, const AccessControlMa
 {
     all = ast.all;
 
-    auto name_to_id = [id_mode{ast.id_mode}, manager](const String & name) -> UUID
+    auto name_to_id = [&ast, manager](const String & name) -> UUID
     {
-        if (id_mode)
+        if (ast.id_mode)
             return parse<UUID>(name);
         assert(manager);
-        auto id = manager->find<User>(name);
-        if (id)
-            return *id;
-        return manager->getID<Role>(name);
+        if (ast.can_contain_users && ast.can_contain_roles)
+        {
+            auto id = manager->find<User>(name);
+            if (id)
+                return *id;
+            return manager->getID<Role>(name);
+        }
+        else if (ast.can_contain_users)
+        {
+            return manager->getID<User>(name);
+        }
+        else
+        {
+            assert(ast.can_contain_roles);
+            return manager->getID<Role>(name);
+        }
     };
 
     if (!ast.names.empty() && !all)
diff --git a/src/Interpreters/InterpreterSetRoleQuery.cpp b/src/Interpreters/InterpreterSetRoleQuery.cpp
index f8e0167d748..c627061dd51 100644
--- a/src/Interpreters/InterpreterSetRoleQuery.cpp
+++ b/src/Interpreters/InterpreterSetRoleQuery.cpp
@@ -62,7 +62,7 @@ void InterpreterSetRoleQuery::setRole(const ASTSetRoleQuery & query)
 
 void InterpreterSetRoleQuery::setDefaultRole(const ASTSetRoleQuery & query)
 {
-    context.checkAccess(AccessType::CREATE_USER | AccessType::DROP_USER);
+    context.checkAccess(AccessType::ALTER_USER);
 
     auto & access_control = context.getAccessControlManager();
     std::vector<UUID> to_users = ExtendedRoleSet{*query.to_users, access_control, context.getUserID()}.getMatchingIDs(access_control);
diff --git a/src/Parsers/ASTExtendedRoleSet.h b/src/Parsers/ASTExtendedRoleSet.h
index 8d619e5d6a0..656f563bd9a 100644
--- a/src/Parsers/ASTExtendedRoleSet.h
+++ b/src/Parsers/ASTExtendedRoleSet.h
@@ -15,7 +15,10 @@ public:
     bool all = false;
     Strings except_names;
     bool except_current_user = false;
-    bool id_mode = false;  /// If true then `names` and `except_names` keeps UUIDs, not names.
+
+    bool id_mode = false;          /// true if `names` and `except_names` keep UUIDs, not names.
+    bool can_contain_roles = true; /// true if this set can contain names of roles.
+    bool can_contain_users = true; /// true if this set can contain names of users.
 
     bool empty() const { return names.empty() && !current_user && !all; }
     void replaceCurrentUserTagWithName(const String & current_user_name);
diff --git a/src/Parsers/ParserCreateUserQuery.cpp b/src/Parsers/ParserCreateUserQuery.cpp
index 76a06a0282f..a6a007855e2 100644
--- a/src/Parsers/ParserCreateUserQuery.cpp
+++ b/src/Parsers/ParserCreateUserQuery.cpp
@@ -227,6 +227,7 @@ namespace
                 return false;
 
             default_roles = typeid_cast<std::shared_ptr<ASTExtendedRoleSet>>(ast);
+            default_roles->can_contain_users = false;
             return true;
         });
     }
diff --git a/src/Parsers/ParserSetRoleQuery.cpp b/src/Parsers/ParserSetRoleQuery.cpp
index e6ff7893891..a69480f89eb 100644
--- a/src/Parsers/ParserSetRoleQuery.cpp
+++ b/src/Parsers/ParserSetRoleQuery.cpp
@@ -18,6 +18,7 @@ namespace
                 return false;
 
             roles = typeid_cast<std::shared_ptr<ASTExtendedRoleSet>>(ast);
+            roles->can_contain_users = false;
             return true;
         });
     }
@@ -34,6 +35,7 @@ namespace
                 return false;
 
             to_users = typeid_cast<std::shared_ptr<ASTExtendedRoleSet>>(ast);
+            to_users->can_contain_roles = false;
             return true;
         });
     }
diff --git a/tests/integration/test_default_role/__init__.py b/tests/integration/test_default_role/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/integration/test_default_role/test.py b/tests/integration/test_default_role/test.py
new file mode 100644
index 00000000000..2b6b4698b20
--- /dev/null
+++ b/tests/integration/test_default_role/test.py
@@ -0,0 +1,77 @@
+import pytest
+from helpers.cluster import ClickHouseCluster
+from helpers.test_tools import TSV
+import re
+
+cluster = ClickHouseCluster(__file__)
+instance = cluster.add_instance('instance')
+
+
+@pytest.fixture(scope="module", autouse=True)
+def started_cluster():
+    try:
+        cluster.start()
+        
+        instance.query("CREATE USER john")
+        instance.query("CREATE ROLE rx")
+        instance.query("CREATE ROLE ry")
+
+        yield cluster
+
+    finally:
+        cluster.shutdown()
+
+
+@pytest.fixture(autouse=True)
+def reset_users_and_roles():
+    instance.query("CREATE USER OR REPLACE john")
+    yield
+
+
+def test_set_default_roles():
+    assert instance.query("SHOW CURRENT ROLES", user="john") == ""
+
+    instance.query("GRANT rx, ry TO john")
+    assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['rx', 0, 1], ['ry', 0, 1]] )
+
+    instance.query("SET DEFAULT ROLE NONE TO john")
+    assert instance.query("SHOW CURRENT ROLES", user="john") == ""
+
+    instance.query("SET DEFAULT ROLE rx TO john")
+    assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['rx', 0, 1]] )
+
+    instance.query("SET DEFAULT ROLE ry TO john")
+    assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['ry', 0, 1]] )
+
+    instance.query("SET DEFAULT ROLE ALL TO john")
+    assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['rx', 0, 1], ['ry', 0, 1]] )
+
+    instance.query("SET DEFAULT ROLE ALL EXCEPT rx TO john")
+    assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['ry', 0, 1]] )
+
+
+def test_alter_user():
+    assert instance.query("SHOW CURRENT ROLES", user="john") == ""
+
+    instance.query("GRANT rx, ry TO john")
+    assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['rx', 0, 1], ['ry', 0, 1]] )
+
+    instance.query("ALTER USER john DEFAULT ROLE NONE")
+    assert instance.query("SHOW CURRENT ROLES", user="john") == ""
+
+    instance.query("ALTER USER john DEFAULT ROLE rx")
+    assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['rx', 0, 1]] )
+
+    instance.query("ALTER USER john DEFAULT ROLE ALL")
+    assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['rx', 0, 1], ['ry', 0, 1]] )
+
+    instance.query("ALTER USER john DEFAULT ROLE ALL EXCEPT rx")
+    assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['ry', 0, 1]] )
+
+
+def test_wrong_set_default_role():
+    assert "There is no user `rx`" in instance.query_and_get_error("SET DEFAULT ROLE NONE TO rx")
+    assert "There is no user `ry`" in instance.query_and_get_error("SET DEFAULT ROLE rx TO ry")
+    assert "There is no role `john`" in instance.query_and_get_error("SET DEFAULT ROLE john TO john")
+    assert "There is no role `john`" in instance.query_and_get_error("ALTER USER john DEFAULT ROLE john")
+    assert "There is no role `john`" in instance.query_and_get_error("ALTER USER john DEFAULT ROLE ALL EXCEPT john")

From 0e27f476323ca8c17ae8ea0b87159128825f0c2d Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Fri, 29 May 2020 11:42:55 +0300
Subject: [PATCH 155/184] Try update test.

---
 ...in_insert_block_size_rows_for_materialized_views.reference | 4 ++--
 ...01278_min_insert_block_size_rows_for_materialized_views.sh | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/queries/0_stateless/01278_min_insert_block_size_rows_for_materialized_views.reference b/tests/queries/0_stateless/01278_min_insert_block_size_rows_for_materialized_views.reference
index ed22b7e1e35..e4872ddeddd 100644
--- a/tests/queries/0_stateless/01278_min_insert_block_size_rows_for_materialized_views.reference
+++ b/tests/queries/0_stateless/01278_min_insert_block_size_rows_for_materialized_views.reference
@@ -1,4 +1,4 @@
 0
 0
-100000
-200000
+800000
+1600000
diff --git a/tests/queries/0_stateless/01278_min_insert_block_size_rows_for_materialized_views.sh b/tests/queries/0_stateless/01278_min_insert_block_size_rows_for_materialized_views.sh
index 7e08c930f67..c9a4de647b9 100755
--- a/tests/queries/0_stateless/01278_min_insert_block_size_rows_for_materialized_views.sh
+++ b/tests/queries/0_stateless/01278_min_insert_block_size_rows_for_materialized_views.sh
@@ -54,7 +54,7 @@ EOL
 echo "create table null_01278 as data_01278 Engine=Null();" | execute
 for i in $(seq 1 $TEST_01278_PARTS); do
     echo "create table part_01278_$i as data_01278 Engine=Buffer(currentDatabase(), null_01278, 1, 86400, 86400, 1e5, 1e6, 10e6, 100e6);"
-    echo "create materialized view mv_01278_$i to part_01278_$i as select * from data_01278 where key%$TEST_01278_PARTS+1 == $i;"
+    echo "create materialized view mv_01278_$i to part_01278_$i as select * from data_01278 where key%$TEST_01278_PARTS+1 != $i;"
 done | execute
 echo "create table out_01278 as data_01278 Engine=Merge(currentDatabase(), 'part_01278_');" | execute
 

From 09369c259c1cd343ef6a0324b0b93e110170cc10 Mon Sep 17 00:00:00 2001
From: BayoNet <da-daos@yandex.ru>
Date: Fri, 29 May 2020 12:43:49 +0300
Subject: [PATCH 156/184] DOCS-527: system.query_log (#11197)

* DOCSUP-824: query log (#115)

* Draft changes.

* Next step.

* Structure was changed. Missed information was added.

* More edits.

* Update docs/en/operations/system-tables.md

Co-authored-by: BayoNet <da-daos@yandex.ru>

* Update docs/en/operations/system-tables.md

Co-authored-by: BayoNet <da-daos@yandex.ru>

* Update docs/en/operations/system-tables.md

Co-authored-by: BayoNet <da-daos@yandex.ru>

* Comments editing was done.

* Translated to russian.

Co-authored-by: Anastasia Uvarova <ana-uvarova@yandex-team.ru>
Co-authored-by: AnaUvarova <64017504+AnaUvarova@users.noreply.github.com>

* CLICKHOUSEDOCS-527: query_log description.

* CLICKHOUSEDOCS-527: Updated structure.

* CLICKHOUSEDOCS-527: Fixed links.

* Update docs/en/operations/system-tables.md

Co-authored-by: Ivan Blinkov <github@blinkov.ru>

* Update docs/en/operations/system-tables.md

Co-authored-by: Ivan Blinkov <github@blinkov.ru>

* Update docs/en/operations/system-tables.md

Co-authored-by: Ivan Blinkov <github@blinkov.ru>

* Update docs/en/operations/system-tables.md

Co-authored-by: Ivan Blinkov <github@blinkov.ru>

* Update docs/en/operations/system-tables.md

Co-authored-by: Ivan Blinkov <github@blinkov.ru>

* Updated by comments.

* CLICKHOUSEDOCS-527: Updated by comments
.

* CLICKHOUSEDOCS-527: Syncronized EN->RU

Co-authored-by: Anastasia Uvarova <ana-uvarova@yandex-team.ru>
Co-authored-by: AnaUvarova <64017504+AnaUvarova@users.noreply.github.com>
Co-authored-by: Sergei Shtykov <bayonet@yandex-team.ru>
Co-authored-by: Ivan Blinkov <github@blinkov.ru>
---
 .../settings.md                               |   4 +-
 docs/en/operations/settings/settings.md       |   2 +-
 docs/en/operations/system-tables.md           | 180 ++++++++----
 .../settings.md                               |   4 +-
 docs/ru/operations/settings/settings.md       |   2 +-
 docs/ru/operations/system-tables.md           | 271 ++++++++----------
 6 files changed, 248 insertions(+), 215 deletions(-)

diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md
index 93acf3cae7a..5961c701283 100644
--- a/docs/en/operations/server-configuration-parameters/settings.md
+++ b/docs/en/operations/server-configuration-parameters/settings.md
@@ -586,11 +586,11 @@ If the table doesn’t exist, ClickHouse will create it. If the structure of the
 </query_log>
 ```
 
-## query\_thread\_log {#server_configuration_parameters-query-thread-log}
+## query\_thread\_log {#server_configuration_parameters-query_thread_log}
 
 Setting for logging threads of queries received with the [log\_query\_threads=1](../settings/settings.md#settings-log-query-threads) setting.
 
-Queries are logged in the [system.query\_thread\_log](../../operations/system-tables.md#system_tables-query-thread-log) table, not in a separate file. You can change the name of the table in the `table` parameter (see below).
+Queries are logged in the [system.query\_thread\_log](../../operations/system-tables.md#system_tables-query_thread_log) table, not in a separate file. You can change the name of the table in the `table` parameter (see below).
 
 Use the following parameters to configure logging:
 
diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index 7103819e499..880f0ffedb1 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -598,7 +598,7 @@ log_queries_min_type='EXCEPTION_WHILE_PROCESSING'
 
 Setting up query threads logging.
 
-Queries’ threads runned by ClickHouse with this setup are logged according to the rules in the [query\_thread\_log](../server-configuration-parameters/settings.md#server_configuration_parameters-query-thread-log) server configuration parameter.
+Queries’ threads runned by ClickHouse with this setup are logged according to the rules in the [query\_thread\_log](../server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) server configuration parameter.
 
 Example:
 
diff --git a/docs/en/operations/system-tables.md b/docs/en/operations/system-tables.md
index cc5456f5324..f364d4e3068 100644
--- a/docs/en/operations/system-tables.md
+++ b/docs/en/operations/system-tables.md
@@ -5,7 +5,7 @@ toc_title: System Tables
 
 # System Tables {#system-tables}
 
-## Introduction
+## Introduction {#system-tables-introduction}
 
 System tables provide information about:
 
@@ -18,9 +18,12 @@ System tables:
 - Available only for reading data.
 - Can't be dropped or altered, but can be detached.
 
-The `metric_log`, `query_log`, `query_thread_log`, `trace_log` system tables store data in a storage filesystem. Other system tables store their data in RAM. ClickHouse server creates such system tables at the start.
+Most of system tables store their data in RAM. ClickHouse server creates such system tables at the start.
 
-### Sources of System Metrics
+The [metric_log](#system_tables-metric_log), [query_log](#system_tables-query_log), [query_thread_log](#system_tables-query_thread_log), [trace_log](#system_tables-trace_log) system tables store data in a storage filesystem. You can alter them or remove from a disk manually. If you remove one of that tables from a disk, the ClickHouse server creates the table again at the time of the next recording. A storage period for these tables is not limited, and ClickHouse server doesn't delete their data automatically. You need to organize removing of outdated logs by yourself. For example, you can use [TTL](../sql-reference/statements/alter.md#manipulations-with-table-ttl) settings for removing outdated log records. 
+
+
+### Sources of System Metrics {#system-tables-sources-of-system-metrics}
 
 For collecting system metrics ClickHouse server uses:
 
@@ -587,97 +590,150 @@ Columns:
 -   `source_file` (LowCardinality(String)) — Source file from which the logging was done.
 -   `source_line` (UInt64) — Source line from which the logging was done.
 
-## system.query\_log {#system_tables-query_log}
+## system.query_log {#system_tables-query_log}
 
-Contains information about execution of queries. For each query, you can see processing start time, duration of processing, error messages and other information.
+Contains information about executed queries, for example, start time, duration of processing, error messages.
 
 !!! note "Note"
     The table doesn’t contain input data for `INSERT` queries.
 
-ClickHouse creates this table only if the [query\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) server parameter is specified. This parameter sets the logging rules, such as the logging interval or the name of the table the queries will be logged in.
+You can change settings of queries logging in the [query_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) section of the server configuration.
 
-To enable query logging, set the [log\_queries](settings/settings.md#settings-log-queries) parameter to 1. For details, see the [Settings](settings/settings.md) section.
+You can disable queries logging by setting [log_queries = 0](settings/settings.md#settings-log-queries). We don't recommend to turn off logging because information in this table is important for solving issues.
+
+The flushing period of logs is set in `flush_interval_milliseconds` parameter of the [query_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) server settings section. To force flushing logs, use the [SYSTEM FLUSH LOGS](../sql-reference/statements/system.md#query_language-system-flush_logs) query.
+
+ClickHouse doesn't delete logs from the table automatically. See [Introduction](#system-tables-introduction) for more details.
 
 The `system.query_log` table registers two kinds of queries:
 
 1.  Initial queries that were run directly by the client.
 2.  Child queries that were initiated by other queries (for distributed query execution). For these types of queries, information about the parent queries is shown in the `initial_*` columns.
 
+Each query creates one or two rows in the `query_log` table, depending on the status (see the `type` column) of the query:
+
+1.  If the query execution was successful, two rows with the `QueryStart` and `QueryFinish` types are created .
+2.  If an error occurred during query processing, two events with the `QueryStart` and `ExceptionWhileProcessing` types are created .
+3.  If an error occurred before launching the query, a single event with the `ExceptionBeforeStart` type is created.
+
 Columns:
 
--   `type` (`Enum8`) — Type of event that occurred when executing the query. Values:
+-   `type` ([Enum8](../sql-reference/data-types/enum.md)) — Type of an event that occurred when executing the query. Values:
     -   `'QueryStart' = 1` — Successful start of query execution.
     -   `'QueryFinish' = 2` — Successful end of query execution.
     -   `'ExceptionBeforeStart' = 3` — Exception before the start of query execution.
     -   `'ExceptionWhileProcessing' = 4` — Exception during the query execution.
--   `event_date` (Date) — Query starting date.
--   `event_time` (DateTime) — Query starting time.
--   `query_start_time` (DateTime) — Start time of query execution.
--   `query_duration_ms` (UInt64) — Duration of query execution.
--   `read_rows` (UInt64) — Number of read rows.
--   `read_bytes` (UInt64) — Number of read bytes.
--   `written_rows` (UInt64) — For `INSERT` queries, the number of written rows. For other queries, the column value is 0.
--   `written_bytes` (UInt64) — For `INSERT` queries, the number of written bytes. For other queries, the column value is 0.
--   `result_rows` (UInt64) — Number of rows in the result.
--   `result_bytes` (UInt64) — Number of bytes in the result.
--   `memory_usage` (UInt64) — Memory consumption by the query.
--   `query` (String) — Query string.
--   `exception` (String) — Exception message.
--   `stack_trace` (String) — Stack trace (a list of methods called before the error occurred). An empty string, if the query is completed successfully.
--   `is_initial_query` (UInt8) — Query type. Possible values:
+-   `event_date` ([Date](../sql-reference/data-types/date.md)) — Query starting date.
+-   `event_time` ([DateTime](../sql-reference/data-types/datetime.md)) — Query starting time.
+-   `query_start_time` ([DateTime](../sql-reference/data-types/datetime.md)) — Start time of query execution.
+-   `query_duration_ms` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Duration of query execution in milliseconds.
+-   `read_rows` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Total number or rows read from all tables and table functions participated in query. It includes usual subqueries, subqueries for `IN` and `JOIN`. For distributed queries `read_rows` includes the total number of rows read at all replicas. Each replica sends it's `read_rows` value, and the server-initiator of the query summarize all received and local values. The cache volumes doesn't affect this value.
+-   `read_bytes` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Total number or bytes read from all tables and table functions participated in query. It includes usual subqueries, subqueries for `IN` and `JOIN`. For distributed queries `read_bytes` includes the total number of rows read at all replicas. Each replica sends it's `read_bytes` value, and the server-initiator of the query summarize all received and local values. The cache volumes doesn't affect this value.
+-   `written_rows` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — For `INSERT` queries, the number of written rows. For other queries, the column value is 0.
+-   `written_bytes` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — For `INSERT` queries, the number of written bytes. For other queries, the column value is 0.
+-   `result_rows` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Number of rows in a result of the `SELECT` query, or a number of rows in the `INSERT` query.
+-   `result_bytes` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — RAM volume in bytes used to store a query result.
+-   `memory_usage` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Memory consumption by the query.
+-   `query` ([String](../sql-reference/data-types/string.md)) — Query string.
+-   `exception` ([String](../sql-reference/data-types/string.md)) — Exception message.
+-   `exception_code` ([Int32](../sql-reference/data-types/int-uint.md)) — Code of an exception. 
+-   `stack_trace` ([String](../sql-reference/data-types/string.md)) — [Stack trace](https://en.wikipedia.org/wiki/Stack_trace). An empty string, if the query was completed successfully.
+-   `is_initial_query` ([UInt8](../sql-reference/data-types/int-uint.md)) — Query type. Possible values:
     -   1 — Query was initiated by the client.
-    -   0 — Query was initiated by another query for distributed query execution.
--   `user` (String) — Name of the user who initiated the current query.
--   `query_id` (String) — ID of the query.
--   `address` (IPv6) — IP address that was used to make the query.
--   `port` (UInt16) — The client port that was used to make the query.
--   `initial_user` (String) — Name of the user who ran the initial query (for distributed query execution).
--   `initial_query_id` (String) — ID of the initial query (for distributed query execution).
--   `initial_address` (IPv6) — IP address that the parent query was launched from.
--   `initial_port` (UInt16) — The client port that was used to make the parent query.
--   `interface` (UInt8) — Interface that the query was initiated from. Possible values:
+    -   0 — Query was initiated by another query as part of distributed query execution.
+-   `user` ([String](../sql-reference/data-types/string.md)) — Name of the user who initiated the current query.
+-   `query_id` ([String](../sql-reference/data-types/string.md)) — ID of the query.
+-   `address` ([IPv6](../sql-reference/data-types/domains/ipv6.md)) — IP address that was used to make the query.
+-   `port` ([UInt16](../sql-reference/data-types/int-uint.md)) — The client port that was used to make the query.
+-   `initial_user` ([String](../sql-reference/data-types/string.md)) — Name of the user who ran the initial query (for distributed query execution).
+-   `initial_query_id` ([String](../sql-reference/data-types/string.md)) — ID of the initial query (for distributed query execution).
+-   `initial_address` ([IPv6](../sql-reference/data-types/domains/ipv6.md)) — IP address that the parent query was launched from.
+-   `initial_port` ([UInt16](../sql-reference/data-types/int-uint.md)) — The client port that was used to make the parent query.
+-   `interface` ([UInt8](../sql-reference/data-types/int-uint.md)) — Interface that the query was initiated from. Possible values:
     -   1 — TCP.
     -   2 — HTTP.
--   `os_user` (String) — OS’s username who runs [clickhouse-client](../interfaces/cli.md).
--   `client_hostname` (String) — Hostname of the client machine where the [clickhouse-client](../interfaces/cli.md) or another TCP client is run.
--   `client_name` (String) — The [clickhouse-client](../interfaces/cli.md) or another TCP client name.
--   `client_revision` (UInt32) — Revision of the [clickhouse-client](../interfaces/cli.md) or another TCP client.
--   `client_version_major` (UInt32) — Major version of the [clickhouse-client](../interfaces/cli.md) or another TCP client.
--   `client_version_minor` (UInt32) — Minor version of the [clickhouse-client](../interfaces/cli.md) or another TCP client.
--   `client_version_patch` (UInt32) — Patch component of the [clickhouse-client](../interfaces/cli.md) or another TCP client version.
+-   `os_user` ([String](../sql-reference/data-types/string.md)) — Operating system username who runs [clickhouse-client](../interfaces/cli.md).
+-   `client_hostname` ([String](../sql-reference/data-types/string.md)) — Hostname of the client machine where the [clickhouse-client](../interfaces/cli.md) or another TCP client is run.
+-   `client_name` ([String](../sql-reference/data-types/string.md)) — The [clickhouse-client](../interfaces/cli.md) or another TCP client name.
+-   `client_revision` ([UInt32](../sql-reference/data-types/int-uint.md)) — Revision of the [clickhouse-client](../interfaces/cli.md) or another TCP client.
+-   `client_version_major` ([UInt32](../sql-reference/data-types/int-uint.md)) — Major version of the [clickhouse-client](../interfaces/cli.md) or another TCP client.
+-   `client_version_minor` ([UInt32](../sql-reference/data-types/int-uint.md)) — Minor version of the [clickhouse-client](../interfaces/cli.md) or another TCP client.
+-   `client_version_patch` ([UInt32](../sql-reference/data-types/int-uint.md)) — Patch component of the [clickhouse-client](../interfaces/cli.md) or another TCP client version.
 -   `http_method` (UInt8) — HTTP method that initiated the query. Possible values:
     -   0 — The query was launched from the TCP interface.
     -   1 — `GET` method was used.
     -   2 — `POST` method was used.
--   `http_user_agent` (String) — The `UserAgent` header passed in the HTTP request.
--   `quota_key` (String) — The “quota key� specified in the [quotas](quotas.md) setting (see `keyed`).
--   `revision` (UInt32) — ClickHouse revision.
--   `thread_numbers` (Array(UInt32)) — Number of threads that are participating in query execution.
--   `ProfileEvents.Names` (Array(String)) — Counters that measure different metrics. The description of them could be found in the table [system.events](#system_tables-events)
--   `ProfileEvents.Values` (Array(UInt64)) — Values of metrics that are listed in the `ProfileEvents.Names` column.
--   `Settings.Names` (Array(String)) — Names of settings that were changed when the client ran the query. To enable logging changes to settings, set the `log_query_settings` parameter to 1.
--   `Settings.Values` (Array(String)) — Values of settings that are listed in the `Settings.Names` column.
+-   `http_user_agent` ([String](../sql-reference/data-types/string.md)) — The `UserAgent` header passed in the HTTP request.
+-   `quota_key` ([String](../sql-reference/data-types/string.md)) — The “quota key� specified in the [quotas](quotas.md) setting (see `keyed`).
+-   `revision` ([UInt32](../sql-reference/data-types/int-uint.md)) — ClickHouse revision.
+-   `thread_numbers` ([Array(UInt32)](../sql-reference/data-types/array.md)) — Number of threads that are participating in query execution.
+-   `ProfileEvents.Names` ([Array(String)](../sql-reference/data-types/array.md)) — Counters that measure different metrics. The description of them could be found in the table [system.events](#system_tables-events)
+-   `ProfileEvents.Values` ([Array(UInt64)](../sql-reference/data-types/array.md)) — Values of metrics that are listed in the `ProfileEvents.Names` column.
+-   `Settings.Names` ([Array(String)](../sql-reference/data-types/array.md)) — Names of settings that were changed when the client ran the query. To enable logging changes to settings, set the `log_query_settings` parameter to 1.
+-   `Settings.Values` ([Array(String)](../sql-reference/data-types/array.md)) — Values of settings that are listed in the `Settings.Names` column.
 
-Each query creates one or two rows in the `query_log` table, depending on the status of the query:
+**Example**
 
-1.  If the query execution is successful, two events with types 1 and 2 are created (see the `type` column).
-2.  If an error occurred during query processing, two events with types 1 and 4 are created.
-3.  If an error occurred before launching the query, a single event with type 3 is created.
+``` sql
+SELECT * FROM system.query_log LIMIT 1 FORMAT Vertical;
+```
 
-By default, logs are added to the table at intervals of 7.5 seconds. You can set this interval in the [query\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) server setting (see the `flush_interval_milliseconds` parameter). To flush the logs forcibly from the memory buffer into the table, use the `SYSTEM FLUSH LOGS` query.
+``` text
+Row 1:
+──────
+type:                 QueryStart
+event_date:           2020-05-13
+event_time:           2020-05-13 14:02:28
+query_start_time:     2020-05-13 14:02:28
+query_duration_ms:    0
+read_rows:            0
+read_bytes:           0
+written_rows:         0
+written_bytes:        0
+result_rows:          0
+result_bytes:         0
+memory_usage:         0
+query:                SELECT 1
+exception_code:       0
+exception:
+stack_trace:
+is_initial_query:     1
+user:                 default
+query_id:             5e834082-6f6d-4e34-b47b-cd1934f4002a
+address:              ::ffff:127.0.0.1
+port:                 57720
+initial_user:         default
+initial_query_id:     5e834082-6f6d-4e34-b47b-cd1934f4002a
+initial_address:      ::ffff:127.0.0.1
+initial_port:         57720
+interface:            1
+os_user:              bayonet
+client_hostname:      clickhouse.ru-central1.internal
+client_name:          ClickHouse client
+client_revision:      54434
+client_version_major: 20
+client_version_minor: 4
+client_version_patch: 1
+http_method:          0
+http_user_agent:
+quota_key:
+revision:             54434
+thread_ids:           []
+ProfileEvents.Names:  []
+ProfileEvents.Values: []
+Settings.Names:       ['use_uncompressed_cache','load_balancing','log_queries','max_memory_usage']
+Settings.Values:      ['0','random','1','10000000000']
 
-When the table is deleted manually, it will be automatically created on the fly. Note that all the previous logs will be deleted.
+```
+**See Also**
 
-!!! note "Note"
-    The storage period for logs is unlimited. Logs aren’t automatically deleted from the table. You need to organize the removal of outdated logs yourself.
+-   [system.query_thread_log](#system_tables-query_thread_log) — This table contains information about each query execution thread.
 
-You can specify an arbitrary partitioning key for the `system.query_log` table in the [query\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) server setting (see the `partition_by` parameter).
-
-## system.query\_thread\_log {#system_tables-query-thread-log}
+## system.query_thread_log {#system_tables-query_thread_log}
 
 The table contains information about each query execution thread.
 
-ClickHouse creates this table only if the [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-thread-log) server parameter is specified. This parameter sets the logging rules, such as the logging interval or the name of the table the queries will be logged in.
+ClickHouse creates this table only if the [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) server parameter is specified. This parameter sets the logging rules, such as the logging interval or the name of the table the queries will be logged in.
 
 To enable query logging, set the [log\_query\_threads](settings/settings.md#settings-log-query-threads) parameter to 1. For details, see the [Settings](settings/settings.md) section.
 
@@ -729,14 +785,14 @@ Columns:
 -   `ProfileEvents.Names` (Array(String)) — Counters that measure different metrics for this thread. The description of them could be found in the table [system.events](#system_tables-events)
 -   `ProfileEvents.Values` (Array(UInt64)) — Values of metrics for this thread that are listed in the `ProfileEvents.Names` column.
 
-By default, logs are added to the table at intervals of 7.5 seconds. You can set this interval in the [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-thread-log) server setting (see the `flush_interval_milliseconds` parameter). To flush the logs forcibly from the memory buffer into the table, use the `SYSTEM FLUSH LOGS` query.
+By default, logs are added to the table at intervals of 7.5 seconds. You can set this interval in the [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) server setting (see the `flush_interval_milliseconds` parameter). To flush the logs forcibly from the memory buffer into the table, use the `SYSTEM FLUSH LOGS` query.
 
 When the table is deleted manually, it will be automatically created on the fly. Note that all the previous logs will be deleted.
 
 !!! note "Note"
     The storage period for logs is unlimited. Logs aren’t automatically deleted from the table. You need to organize the removal of outdated logs yourself.
 
-You can specify an arbitrary partitioning key for the `system.query_thread_log` table in the [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-thread-log) server setting (see the `partition_by` parameter).
+You can specify an arbitrary partitioning key for the `system.query_thread_log` table in the [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) server setting (see the `partition_by` parameter).
 
 ## system.trace\_log {#system_tables-trace_log}
 
diff --git a/docs/ru/operations/server-configuration-parameters/settings.md b/docs/ru/operations/server-configuration-parameters/settings.md
index c664580c659..ccde7a945ac 100644
--- a/docs/ru/operations/server-configuration-parameters/settings.md
+++ b/docs/ru/operations/server-configuration-parameters/settings.md
@@ -574,11 +574,11 @@ ClickHouse проверит у�лови� `min_part_size` и `min_part_size_rat
 </query_log>
 ```
 
-## query\_thread\_log {#server_configuration_parameters-query-thread-log}
+## query\_thread\_log {#server_configuration_parameters-query_thread_log}
 
 �а�тройка логировани� потоков выполнени� запро�ов, прин�тых � на�тройкой [log\_query\_threads=1](../settings/settings.md#settings-log-query-threads).
 
-Запро�ы логируют�� не в отдельный файл, а в �и�темную таблицу [system.query\_thread\_log](../../operations/server-configuration-parameters/settings.md#system_tables-query-thread-log). Вы можете изменить название �той таблицы в параметре `table` (�м. ниже).
+Запро�ы логируют�� не в отдельный файл, а в �и�темную таблицу [system.query\_thread\_log](../../operations/server-configuration-parameters/settings.md#system_tables-query_thread_log). Вы можете изменить название �той таблицы в параметре `table` (�м. ниже).
 
 При на�тройке логировани� и�пользуют�� �ледующие параметры:
 
diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md
index 56c3042bfa3..4dd43e9607b 100644
--- a/docs/ru/operations/settings/settings.md
+++ b/docs/ru/operations/settings/settings.md
@@ -536,7 +536,7 @@ log_queries=1
 
 У�тановка логировани� информации о потоках выполнени� запро�а.
 
-Лог информации о потоках выполнени� запро�ов, переданных в ClickHouse � �той у�тановкой, запи�ывает�� �огла�но правилам конфигурационного параметра �ервера [query\_thread\_log](../server-configuration-parameters/settings.md#server_configuration_parameters-query-thread-log).
+Лог информации о потоках выполнени� запро�ов, переданных в ClickHouse � �той у�тановкой, запи�ывает�� �огла�но правилам конфигурационного параметра �ервера [query\_thread\_log](../server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log).
 
 Пример:
 
diff --git a/docs/ru/operations/system-tables.md b/docs/ru/operations/system-tables.md
index ae0e67a4515..8954a484560 100644
--- a/docs/ru/operations/system-tables.md
+++ b/docs/ru/operations/system-tables.md
@@ -1,4 +1,7 @@
-# Си�темные таблицы {#sistemnye-tablitsy}
+# Си�темные таблицы {#system-tables}
+
+
+## Введение {#system-tables-introduction}
 
 Си�темные таблицы и�пользуют�� дл� реализации ча�ти функционально�ти �и�темы, а также предо�тавл�ют до�туп к информации о работе �и�темы.
 Вы не можете удалить �и�темную таблицу (хот� можете �делать DETACH).
@@ -544,182 +547,156 @@ CurrentMetric_ReplicatedChecks:                             0
 -   `source_file` (LowCardinality(String)) — И�ходный файл, из которого была �делана запи�ь.
 -   `source_line` (UInt64) — И�ходна� �трока, из которой была �делана запи�ь.
 
-## system.query\_log {#system_tables-query_log}
+## system.query_log {#system_tables-query_log}
 
-Содержит информацию о выполнении запро�ов. Дл� каждого запро�а вы можете увидеть врем� начала обработки, продолжительно�ть обработки, �ообщени� об ошибках и другую информацию.
+Содержит информацию о выполн�емых запро�ах, например, врем� начала обработки, продолжительно�ть обработки, �ообщени� об ошибках.
 
 !!! note "Внимание"
     Таблица не �одержит входных данных дл� запро�ов `INSERT`.
 
-ClickHouse �оздаёт таблицу только в том �лучае, когда у�тановлен конфигурационный параметр �ервера [query\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log). Параметр задаёт правила ведени� лога, такие как интервал логировани� или им� таблицы, в которую будут логгировать�� запро�ы.
+�а�тойки логгировани� можно изменить в �екции �ерверной конфигурации [query_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log).
 
-Чтобы включить логирование, задайте значение параметра [log\_queries](settings/settings.md#settings-log-queries) равным 1. Подробно�ти �мотрите в разделе [�а�тройки](settings/settings.md#settings).
+Можно отключить логгирование на�тройкой [log_queries = 0](settings/settings.md#settings-log-queries). По-возможно�ти, не отключайте логгирование, по�кольку информаци� из таблицы важна при решении проблем.
+
+Период �бро�а логов в таблицу задаёт�� параметром `flush_interval_milliseconds` в конфигурационной �екции [query_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log). Чтобы принудительно запи�ать логи из буффера пам�ти в таблицу, и�пользуйте запро� [SYSTEM FLUSH LOGS](../sql-reference/statements/system.md#query_language-system-flush_logs).
+
+ClickHouse не удал�ет логи из таблица автоматиче�ки. Смотрите [Введение](#system-tables-introduction).
+
+Можно указать произвольный ключ партиционировани� дл� таблицы `system.query_log` в конфигурации [query\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) (параметр `partition_by`).
+
+
+
+Е�ли таблицу удалить вручную, она �оздает�� заново автоматиче�ки «на лету». При �том в�е логи на момент удалени� таблицы будут убраны.
 
 Таблица `system.query_log` �одержит информацию о двух видах запро�ов:
 
 1.  Первоначальные запро�ы, которые были выполнены непо�ред�твенно клиентом.
 2.  Дочерние запро�ы, инициированные другими запро�ами (дл� выполнени� ра�пределенных запро�ов). Дл� дочерних запро�ов информаци� о первоначальном запро�е �одержит�� в �толбцах `initial_*`.
 
+В зави�имо�ти от �тату�а (�толбец `type`) каждый запро� �оздаёт одну или две �троки в таблице `query_log`:
+
+1.  Е�ли запро� выполнен у�пешно, �оздают�� два �обыти� типа `QueryStart` и `QueryFinish`.
+2.  Е�ли во врем� обработки запро�а возникла ошибка, �оздают�� два �обыти� � типами `QueryStart` и `ExceptionWhileProcessing`.
+3.  Е�ли ошибка произошла ещё до запу�ка запро�а, �оздает�� одно �обытие � типом `ExceptionBeforeStart`.
+
 Столбцы:
 
--   `type` (`Enum8`) — тип �обыти�, произошедшего при выполнении запро�а. Значени�:
+-   `type` ([Enum8](../sql-reference/data-types/enum.md)) — тип �обыти�, произошедшего при выполнении запро�а. Значени�:
     -   `'QueryStart' = 1` — у�пешное начало выполнени� запро�а.
     -   `'QueryFinish' = 2` — у�пешное завершение выполнени� запро�а.
     -   `'ExceptionBeforeStart' = 3` — и�ключение перед началом обработки запро�а.
     -   `'ExceptionWhileProcessing' = 4` — и�ключение во врем� обработки запро�а.
--   `event_date` (Date) — дата начала запро�а.
--   `event_time` (DateTime) — врем� начала запро�а.
--   `query_start_time` (DateTime) — врем� начала обработки запро�а.
--   `query_duration_ms` (UInt64) — длительно�ть обработки запро�а.
--   `read_rows` (UInt64) — количе�тво прочитанных �трок.
--   `read_bytes` (UInt64) — количе�тво прочитанных байтов.
--   `written_rows` (UInt64) — количе�тво запи�анных �трок дл� запро�ов `INSERT`. Дл� других запро�ов, значение �толбца 0.
--   `written_bytes` (UInt64) — объём запи�анных данных в байтах дл� запро�ов `INSERT`. Дл� других запро�ов, значение �толбца 0.
--   `result_rows` (UInt64) — количе�тво �трок в результате.
--   `result_bytes` (UInt64) — объём результата в байтах.
--   `memory_usage` (UInt64) — потребление RAM запро�ом.
--   `query` (String) — тек�т запро�а.
--   `exception` (String) — �ообщение и�ключени�, е�ли запро� завершил�� по и�ключению.
--   `stack_trace` (String) — тра��ировка (�пи�ок функций, по�ледовательно вызванных перед ошибкой). Пу�та� �трока, е�ли запро� у�пешно завершен.
--   `is_initial_query` (UInt8) — вид запро�а. Возможные значени�:
+-   `event_date` ([Date](../sql-reference/data-types/date.md)) — дата начала запро�а.
+-   `event_time` ([DateTime](../sql-reference/data-types/datetime.md)) — врем� начала запро�а.
+-   `query_start_time` ([DateTime](../sql-reference/data-types/datetime.md)) — врем� начала обработки запро�а.
+-   `query_duration_ms` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — длительно�ть выполнени� запро�а в милли�екундах.
+-   `read_rows` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Общее количе�тво �трок, �читанных из в�ех таблиц и табличных функций, уча�твующих в запро�е. Включает в �еб� обычные подзапро�ы, подзапро�ы дл� `IN` и `JOIN`. Дл� ра�пределенных запро�ов `read_rows` включает в �еб� общее количе�тво �трок, прочитанных на в�ех репликах. Кажда� реплика передает �об�твенное значение `read_rows`, а �ервер-инициатор запро�а �уммирует в�е полученные и локальные значени�. Объемы к�ша не учитываю��.
+-   `read_bytes` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Общее количе�тво байтов, �читанных из в�ех таблиц и табличных функций, уча�твующих в запро�е. Включает в �еб� обычные подзапро�ы, подзапро�ы дл� `IN` и `JOIN`. Дл� ра�пределенных запро�ов `read_bytes` включает в �еб� общее количе�тво байтов, прочитанных на в�ех репликах. Кажда� реплика передает �об�твенное значение `read_bytes`, а �ервер-инициатор запро�а �уммирует в�е полученные и локальные значени�. Объемы к�ша не учитываю��.
+-   `written_rows` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — количе�тво запи�анных �трок дл� запро�ов `INSERT`. Дл� других запро�ов, значение �толбца 0.
+-   `written_bytes` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — объём запи�анных данных в байтах дл� запро�ов `INSERT`. Дл� других запро�ов, значение �толбца 0.
+-   `result_rows` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — количе�тво �трок в результате запро�а `SELECT` или количе�тво �трок в запро�е `INSERT`.
+-   `result_bytes` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — объём RAM в байтах, и�пользованный дл� хранени� результата запро�а.
+-   `memory_usage` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — потребление RAM запро�ом.
+-   `query` ([String](../sql-reference/data-types/string.md)) — тек�т запро�а.
+-   `exception` ([String](../sql-reference/data-types/string.md)) — �ообщение и�ключени�, е�ли запро� завершил�� по и�ключению.
+-   `exception_code` ([Int32](../sql-reference/data-types/int-uint.md)) — код и�ключени�. 
+-   `stack_trace` ([String](../sql-reference/data-types/string.md)) — [stack trace](https://en.wikipedia.org/wiki/Stack_trace). Пу�та� �трока, е�ли запро� у�пешно завершен.
+-   `is_initial_query` ([UInt8](../sql-reference/data-types/int-uint.md)) — вид запро�а. Возможные значени�:
     -   1 — запро� был инициирован клиентом.
-    -   0 — запро� был инициирован другим запро�ом при ра�пределенном запро�е.
--   `user` (String) — пользователь, запу�тивший текущий запро�.
--   `query_id` (String) — ID запро�а.
--   `address` (IPv6) — IP адре�, � которого пришел запро�.
--   `port` (UInt16) — порт, � которого клиент �делал запро�
--   `initial_user` (String) — пользователь, запу�тивший первоначальный запро� (дл� ра�пределенных запро�ов).
--   `initial_query_id` (String) — ID родитель�кого запро�а.
--   `initial_address` (IPv6) — IP адре�, � которого пришел родитель�кий запро�.
--   `initial_port` (UInt16) — порт, � которого клиент �делал родитель�кий запро�.
--   `interface` (UInt8) — интерфей�, � которого ушёл запро�. Возможные значени�:
+    -   0 — запро� был инициирован другим запро�ом при выполнении ра�пределенного запро�а.
+-   `user` ([String](../sql-reference/data-types/string.md)) — пользователь, запу�тивший текущий запро�.
+-   `query_id` ([String](../sql-reference/data-types/string.md)) — ID запро�а.
+-   `address` ([IPv6](../sql-reference/data-types/domains/ipv6.md)) — IP адре�, � которого пришел запро�.
+-   `port` ([UInt16](../sql-reference/data-types/int-uint.md)) — порт, � которого клиент �делал запро�
+-   `initial_user` ([String](../sql-reference/data-types/string.md)) — пользователь, запу�тивший первоначальный запро� (дл� ра�пределенных запро�ов).
+-   `initial_query_id` ([String](../sql-reference/data-types/string.md)) — ID родитель�кого запро�а.
+-   `initial_address` ([IPv6](../sql-reference/data-types/domains/ipv6.md)) — IP адре�, � которого пришел родитель�кий запро�.
+-   `initial_port` ([UInt16](../sql-reference/data-types/int-uint.md)) — порт, � которого клиент �делал родитель�кий запро�.
+-   `interface` ([UInt8](../sql-reference/data-types/int-uint.md)) — интерфей�, � которого ушёл запро�. Возможные значени�:
     -   1 — TCP.
     -   2 — HTTP.
--   `os_user` (String) — им� пользовател� в OS, который запу�тил [clickhouse-client](../interfaces/cli.md).
--   `client_hostname` (String) — им� �ервера, � которого при�оединил�� [clickhouse-client](../interfaces/cli.md) или другой TCP клиент.
--   `client_name` (String) — [clickhouse-client](../interfaces/cli.md) или другой TCP клиент.
--   `client_revision` (UInt32) — ревизи� [clickhouse-client](../interfaces/cli.md) или другого TCP клиента.
--   `client_version_major` (UInt32) — �тарша� вер�и� [clickhouse-client](../interfaces/cli.md) или другого TCP клиента.
--   `client_version_minor` (UInt32) — младша� вер�и� [clickhouse-client](../interfaces/cli.md) или другого TCP клиента.
--   `client_version_patch` (UInt32) — патч [clickhouse-client](../interfaces/cli.md) или другого TCP клиента.
--   `http_method` (UInt8) — HTTP метод, инициировавший запро�. Возможные значени�:
+-   `os_user` ([String](../sql-reference/data-types/string.md)) — им� пользовател� операционной �и�темы, который запу�тил [clickhouse-client](../interfaces/cli.md).
+-   `client_hostname` ([String](../sql-reference/data-types/string.md)) — им� �ервера, � которого при�оединил�� [clickhouse-client](../interfaces/cli.md) или другой TCP клиент.
+-   `client_name` ([String](../sql-reference/data-types/string.md)) — [clickhouse-client](../interfaces/cli.md) или другой TCP клиент.
+-   `client_revision` ([UInt32](../sql-reference/data-types/int-uint.md)) — ревизи� [clickhouse-client](../interfaces/cli.md) или другого TCP клиента.
+-   `client_version_major` ([UInt32](../sql-reference/data-types/int-uint.md)) — �тарша� вер�и� [clickhouse-client](../interfaces/cli.md) или другого TCP клиента.
+-   `client_version_minor` ([UInt32](../sql-reference/data-types/int-uint.md)) — младша� вер�и� [clickhouse-client](../interfaces/cli.md) или другого TCP клиента.
+-   `client_version_patch` ([UInt32](../sql-reference/data-types/int-uint.md)) — патч [clickhouse-client](../interfaces/cli.md) или другого TCP клиента.
+-   `http_method` ([UInt8](../sql-reference/data-types/int-uint.md)) — HTTP метод, инициировавший запро�. Возможные значени�:
     -   0 — запро� запущен � интерфей�а TCP.
     -   1 — `GET`.
     -   2 — `POST`.
--   `http_user_agent` (String) — HTTP заголовок `UserAgent`.
--   `quota_key` (String) — «ключ квоты» из на�троек [квот](quotas.md) (�м. `keyed`).
--   `revision` (UInt32) — ревизи� ClickHouse.
--   `thread_numbers` (Array(UInt32)) — количе�тво потоков, уча�твующих в обработке запро�ов.
--   `ProfileEvents.Names` (Array(String)) — Счетчики дл� изменени� различных метрик. Опи�ание метрик можно получить из таблицы [system.events](#system_tables-events)(\#system\_tables-events
--   `ProfileEvents.Values` (Array(UInt64)) — метрики, перечи�ленные в �толбце `ProfileEvents.Names`.
--   `Settings.Names` (Array(String)) — имена на�троек, которые мен�ют��, когда клиент выполн�ет запро�. Чтобы разрешить логирование изменений на�троек, у�тановите параметр `log_query_settings` равным 1.
--   `Settings.Values` (Array(String)) — Значени� на�троек, которые перечи�лены в �толбце `Settings.Names`.
+-   `http_user_agent` ([String](../sql-reference/data-types/string.md)) — HTTP заголовок `UserAgent`.
+-   `quota_key` ([String](../sql-reference/data-types/string.md)) — «ключ квоты» из на�троек [квот](quotas.md) (�м. `keyed`).
+-   `revision` ([UInt32](../sql-reference/data-types/int-uint.md)) — ревизи� ClickHouse.
+-   `thread_numbers` ([Array(UInt32)](../sql-reference/data-types/array.md)) — количе�тво потоков, уча�твующих в обработке запро�ов.
+-   `ProfileEvents.Names` ([Array(String)](../sql-reference/data-types/array.md)) — Счетчики дл� изменени� различных метрик. Опи�ание метрик можно получить из таблицы [system.events](#system_tables-events)(\#system\_tables-events
+-   `ProfileEvents.Values` ([Array(UInt64)](../sql-reference/data-types/array.md)) — метрики, перечи�ленные в �толбце `ProfileEvents.Names`.
+-   `Settings.Names` ([Array(String)](../sql-reference/data-types/array.md)) — имена на�троек, которые мен�ют��, когда клиент выполн�ет запро�. Чтобы разрешить логирование изменений на�троек, у�тановите параметр `log_query_settings` равным 1.
+-   `Settings.Values` ([Array(String)](../sql-reference/data-types/array.md)) — Значени� на�троек, которые перечи�лены в �толбце `Settings.Names`.
 
-Каждый запро� �оздаёт одну или две �троки в таблице `query_log`, в зави�имо�ти от �тату�а запро�а:
+**Пример**
 
-1.  Е�ли запро� выполнен у�пешно, �оздают�� два �обыти� типа 1 и 2 (�мотрите �толбец `type`).
-2.  Е�ли во врем� обработки запро�а произошла ошибка, �оздают�� два �обыти� � типами 1 и 4.
-3.  Е�ли ошибка произошла до запу�ка запро�а, �оздает�� одно �обытие � типом 3.
+``` sql
+SELECT * FROM system.query_log LIMIT 1 FORMAT Vertical;
+```
 
-По умолчанию, �троки добавл�ют�� в таблицу логировани� � интервалом в 7,5 �екунд. Можно задать интервал в конфигурационном параметре �ервера [query\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) (�мотрите параметр `flush_interval_milliseconds`). Чтобы принудительно запи�ать логи из буффера пам�ти в таблицу, и�пользуйте запро� `SYSTEM FLUSH LOGS`.
+``` text
+Row 1:
+──────
+type:                 QueryStart
+event_date:           2020-05-13
+event_time:           2020-05-13 14:02:28
+query_start_time:     2020-05-13 14:02:28
+query_duration_ms:    0
+read_rows:            0
+read_bytes:           0
+written_rows:         0
+written_bytes:        0
+result_rows:          0
+result_bytes:         0
+memory_usage:         0
+query:                SELECT 1
+exception_code:       0
+exception:
+stack_trace:
+is_initial_query:     1
+user:                 default
+query_id:             5e834082-6f6d-4e34-b47b-cd1934f4002a
+address:              ::ffff:127.0.0.1
+port:                 57720
+initial_user:         default
+initial_query_id:     5e834082-6f6d-4e34-b47b-cd1934f4002a
+initial_address:      ::ffff:127.0.0.1
+initial_port:         57720
+interface:            1
+os_user:              bayonet
+client_hostname:      clickhouse.ru-central1.internal
+client_name:          ClickHouse client
+client_revision:      54434
+client_version_major: 20
+client_version_minor: 4
+client_version_patch: 1
+http_method:          0
+http_user_agent:
+quota_key:
+revision:             54434
+thread_ids:           []
+ProfileEvents.Names:  []
+ProfileEvents.Values: []
+Settings.Names:       ['use_uncompressed_cache','load_balancing','log_queries','max_memory_usage']
+Settings.Values:      ['0','random','1','10000000000']
 
-Е�ли таблицу удалить вручную, она пере�озда�т�� автоматиче�ки «на лету». При �том в�е логи на момент удалени� таблицы будут удалены.
+```
+**Смотрите также**
 
-!!! note "Примечание"
-    Срок хранени� логов не ограничен. Логи не удал�ют�� из таблицы автоматиче�ки. Вам необходимо �амо�то�тельно организовать удаление у�таревших логов.
+-   [system.query_thread_log](#system_tables-query_thread_log) — в �той таблице �одержит�� информаци� о цепочке каждого выполненного запро�а.
 
-Можно указать произвольный ключ партиционировани� дл� таблицы `system.query_log` в конфигурации [query\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) (параметр `partition_by`).
-
-## system.query\_log {#system_tables-query_log}
-
-Contains information about execution of queries. For each query, you can see processing start time, duration of processing, error messages and other information.
-
-!!! note "Note"
-    The table doesn’t contain input data for `INSERT` queries.
-
-ClickHouse creates this table only if the [query\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) server parameter is specified. This parameter sets the logging rules, such as the logging interval or the name of the table the queries will be logged in.
-
-To enable query logging, set the [log\_queries](settings/settings.md#settings-log-queries) parameter to 1. For details, see the [Settings](settings/settings.md) section.
-
-The `system.query_log` table registers two kinds of queries:
-
-1.  Initial queries that were run directly by the client.
-2.  Child queries that were initiated by other queries (for distributed query execution). For these types of queries, information about the parent queries is shown in the `initial_*` columns.
-
-Columns:
-
--   `type` (`Enum8`) — Type of event that occurred when executing the query. Values:
-    -   `'QueryStart' = 1` — Successful start of query execution.
-    -   `'QueryFinish' = 2` — Successful end of query execution.
-    -   `'ExceptionBeforeStart' = 3` — Exception before the start of query execution.
-    -   `'ExceptionWhileProcessing' = 4` — Exception during the query execution.
--   `event_date` (Date) — Query starting date.
--   `event_time` (DateTime) — Query starting time.
--   `query_start_time` (DateTime) — Start time of query execution.
--   `query_duration_ms` (UInt64) — Duration of query execution.
--   `read_rows` (UInt64) — Number of read rows.
--   `read_bytes` (UInt64) — Number of read bytes.
--   `written_rows` (UInt64) — For `INSERT` queries, the number of written rows. For other queries, the column value is 0.
--   `written_bytes` (UInt64) — For `INSERT` queries, the number of written bytes. For other queries, the column value is 0.
--   `result_rows` (UInt64) — Number of rows in the result.
--   `result_bytes` (UInt64) — Number of bytes in the result.
--   `memory_usage` (UInt64) — Memory consumption by the query.
--   `query` (String) — Query string.
--   `exception` (String) — Exception message.
--   `stack_trace` (String) — Stack trace (a list of methods called before the error occurred). An empty string, if the query is completed successfully.
--   `is_initial_query` (UInt8) — Query type. Possible values:
-    -   1 — Query was initiated by the client.
-    -   0 — Query was initiated by another query for distributed query execution.
--   `user` (String) — Name of the user who initiated the current query.
--   `query_id` (String) — ID of the query.
--   `address` (IPv6) — IP address that was used to make the query.
--   `port` (UInt16) — The client port that was used to make the query.
--   `initial_user` (String) — Name of the user who ran the initial query (for distributed query execution).
--   `initial_query_id` (String) — ID of the initial query (for distributed query execution).
--   `initial_address` (IPv6) — IP address that the parent query was launched from.
--   `initial_port` (UInt16) — The client port that was used to make the parent query.
--   `interface` (UInt8) — Interface that the query was initiated from. Possible values:
-    -   1 — TCP.
-    -   2 — HTTP.
--   `os_user` (String) — OS’s username who runs [clickhouse-client](../interfaces/cli.md).
--   `client_hostname` (String) — Hostname of the client machine where the [clickhouse-client](../interfaces/cli.md) or another TCP client is run.
--   `client_name` (String) — The [clickhouse-client](../interfaces/cli.md) or another TCP client name.
--   `client_revision` (UInt32) — Revision of the [clickhouse-client](../interfaces/cli.md) or another TCP client.
--   `client_version_major` (UInt32) — Major version of the [clickhouse-client](../interfaces/cli.md) or another TCP client.
--   `client_version_minor` (UInt32) — Minor version of the [clickhouse-client](../interfaces/cli.md) or another TCP client.
--   `client_version_patch` (UInt32) — Patch component of the [clickhouse-client](../interfaces/cli.md) or another TCP client version.
--   `http_method` (UInt8) — HTTP method that initiated the query. Possible values:
-    -   0 — The query was launched from the TCP interface.
-    -   1 — `GET` method was used.
-    -   2 — `POST` method was used.
--   `http_user_agent` (String) — The `UserAgent` header passed in the HTTP request.
--   `quota_key` (String) — The «quota key» specified in the [quotas](quotas.md) setting (see `keyed`).
--   `revision` (UInt32) — ClickHouse revision.
--   `thread_numbers` (Array(UInt32)) — Number of threads that are participating in query execution.
--   `ProfileEvents.Names` (Array(String)) — Counters that measure different metrics. The description of them could be found in the table [system.events](#system_tables-events)
--   `ProfileEvents.Values` (Array(UInt64)) — Values of metrics that are listed in the `ProfileEvents.Names` column.
--   `Settings.Names` (Array(String)) — Names of settings that were changed when the client ran the query. To enable logging changes to settings, set the `log_query_settings` parameter to 1.
--   `Settings.Values` (Array(String)) — Values of settings that are listed in the `Settings.Names` column.
-
-Each query creates one or two rows in the `query_log` table, depending on the status of the query:
-
-1.  If the query execution is successful, two events with types 1 and 2 are created (see the `type` column).
-2.  If an error occurred during query processing, two events with types 1 and 4 are created.
-3.  If an error occurred before launching the query, a single event with type 3 is created.
-
-By default, logs are added to the table at intervals of 7.5 seconds. You can set this interval in the [query\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) server setting (see the `flush_interval_milliseconds` parameter). To flush the logs forcibly from the memory buffer into the table, use the `SYSTEM FLUSH LOGS` query.
-
-When the table is deleted manually, it will be automatically created on the fly. Note that all the previous logs will be deleted.
-
-!!! note "Note"
-    The storage period for logs is unlimited. Logs aren’t automatically deleted from the table. You need to organize the removal of outdated logs yourself.
-
-You can specify an arbitrary partitioning key for the `system.query_log` table in the [query\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) server setting (see the `partition_by` parameter).
-\#\# system.query\_thread\_log {\#system\_tables-query-thread-log}
+## system.query_thread_log {#system_tables-query_thread_log}
 
 Содержит информацию о каждом потоке выполн�емых запро�ов.
 
-ClickHouse �оздаёт таблицу только в том �лучае, когда у�тановлен конфигурационный параметр �ервера [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-thread-log). Параметр задаёт правила ведени� лога, такие как интервал логировани� или им� таблицы, в которую будут логгировать�� запро�ы.
+ClickHouse �оздаёт таблицу только в том �лучае, когда у�тановлен конфигурационный параметр �ервера [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log). Параметр задаёт правила ведени� лога, такие как интервал логировани� или им� таблицы, в которую будут логгировать�� запро�ы.
 
 Чтобы включить логирование, задайте значение параметра [log\_query\_threads](settings/settings.md#settings-log-query-threads) равным 1. Подробно�ти �мотрите в разделе [�а�тройки](settings/settings.md#settings).
 
@@ -770,16 +747,16 @@ ClickHouse �оздаёт таблицу только в том �лучае, к
 -   `ProfileEvents.Names` (Array(String)) — Счетчики дл� изменени� различных метрик дл� данного потока. Опи�ание метрик можно получить из таблицы [system.events](#system_tables-events)(\#system\_tables-events
 -   `ProfileEvents.Values` (Array(UInt64)) — метрики дл� данного потока, перечи�ленные в �толбце `ProfileEvents.Names`.
 
-По умолчанию, �троки добавл�ют�� в таблицу логировани� � интервалом в 7,5 �екунд. Можно задать интервал в конфигурационном параметре �ервера [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-thread-log) (�мотрите параметр `flush_interval_milliseconds`). Чтобы принудительно запи�ать логи из буффера пам�ти в таблицу, и�пользуйте запро� `SYSTEM FLUSH LOGS`.
+По умолчанию, �троки добавл�ют�� в таблицу логировани� � интервалом в 7,5 �екунд. Можно задать интервал в конфигурационном параметре �ервера [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) (�мотрите параметр `flush_interval_milliseconds`). Чтобы принудительно запи�ать логи из буффера пам�ти в таблицу, и�пользуйте запро� `SYSTEM FLUSH LOGS`.
 
 Е�ли таблицу удалить вручную, она пере�озда�т�� автоматиче�ки «на лету». При �том в�е логи на момент удалени� таблицы будут удалены.
 
 !!! note "Примечание"
     Срок хранени� логов не ограничен. Логи не удал�ют�� из таблицы автоматиче�ки. Вам необходимо �амо�то�тельно организовать удаление у�таревших логов.
 
-Можно указать произвольный ключ партиционировани� дл� таблицы `system.query_log` в конфигурации [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-thread-log) (параметр `partition_by`).
+Можно указать произвольный ключ партиционировани� дл� таблицы `system.query_log` в конфигурации [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) (параметр `partition_by`).
 
-## system.query_thread_log {#system_tables-query-thread-log}
+## system.query_thread_log {#system_tables-query_thread_log}
 
 Содержит информацию о каждом потоке и�полнени� запро�а.
 

From 3f53b5b335c9993c092809253f736af406d8ade1 Mon Sep 17 00:00:00 2001
From: Alexander Kazakov <Akazz@users.noreply.github.com>
Date: Fri, 29 May 2020 12:45:55 +0300
Subject: [PATCH 157/184] Fixed improper logging of failures in destructors of
 certain metrics (#11264)

---
 src/Common/MemoryStatisticsOS.cpp    | 15 ++++++++++++++-
 src/Common/ProcfsMetricsProvider.cpp | 22 +++++++++++++++++++---
 2 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/src/Common/MemoryStatisticsOS.cpp b/src/Common/MemoryStatisticsOS.cpp
index adc5bf5d904..6082d23cbd0 100644
--- a/src/Common/MemoryStatisticsOS.cpp
+++ b/src/Common/MemoryStatisticsOS.cpp
@@ -9,6 +9,7 @@
 #include <Common/Exception.h>
 #include <IO/ReadBufferFromMemory.h>
 #include <IO/ReadHelpers.h>
+#include <common/logger_useful.h>
 
 
 namespace DB
@@ -19,6 +20,7 @@ namespace ErrorCodes
     extern const int FILE_DOESNT_EXIST;
     extern const int CANNOT_OPEN_FILE;
     extern const int CANNOT_READ_FROM_FILE_DESCRIPTOR;
+    extern const int CANNOT_CLOSE_FILE;
 }
 
 static constexpr auto filename = "/proc/self/statm";
@@ -35,7 +37,18 @@ MemoryStatisticsOS::MemoryStatisticsOS()
 MemoryStatisticsOS::~MemoryStatisticsOS()
 {
     if (0 != ::close(fd))
-        tryLogCurrentException(__PRETTY_FUNCTION__);
+    {
+        try
+        {
+            throwFromErrno(
+                    "File descriptor for \"" + std::string(filename) + "\" could not be closed. "
+                    "Something seems to have gone wrong. Inspect errno.", ErrorCodes::CANNOT_CLOSE_FILE);
+        }
+        catch (const ErrnoException &)
+        {
+            DB::tryLogCurrentException(__PRETTY_FUNCTION__);
+        }
+    }
 }
 
 MemoryStatisticsOS::Data MemoryStatisticsOS::get() const
diff --git a/src/Common/ProcfsMetricsProvider.cpp b/src/Common/ProcfsMetricsProvider.cpp
index 633558a7b67..fcc4124aa85 100644
--- a/src/Common/ProcfsMetricsProvider.cpp
+++ b/src/Common/ProcfsMetricsProvider.cpp
@@ -7,6 +7,7 @@
 #include <IO/ReadHelpers.h>
 
 #include <common/find_symbols.h>
+#include <common/logger_useful.h>
 
 #include <cassert>
 #include <sys/types.h>
@@ -22,6 +23,7 @@ namespace ErrorCodes
 {
     extern const int FILE_DOESNT_EXIST;
     extern const int CANNOT_OPEN_FILE;
+    extern const int CANNOT_CLOSE_FILE;
     extern const int CANNOT_READ_FROM_FILE_DESCRIPTOR;
 }
 
@@ -39,6 +41,20 @@ namespace
             errno == ENOENT ? ErrorCodes::FILE_DOESNT_EXIST : ErrorCodes::CANNOT_OPEN_FILE);
 }
 
+inline void emitErrorMsgWithFailedToCloseFile(const std::string & filename)
+{
+    try
+    {
+        throwFromErrno(
+                "File descriptor for \"" + filename + "\" could not be closed. "
+                "Something seems to have gone wrong. Inspect errno.", ErrorCodes::CANNOT_CLOSE_FILE);
+    }
+    catch (const ErrnoException &)
+    {
+        DB::tryLogCurrentException(__PRETTY_FUNCTION__);
+    }
+}
+
 ssize_t readFromFD(const int fd, const char * filename, char * buf, size_t buf_size)
 {
     ssize_t res = 0;
@@ -100,11 +116,11 @@ ProcfsMetricsProvider::ProcfsMetricsProvider(const pid_t /*tid*/)
 ProcfsMetricsProvider::~ProcfsMetricsProvider()
 {
     if (stats_version >= 3 && 0 != ::close(thread_io_fd))
-        tryLogCurrentException(__PRETTY_FUNCTION__);
+        emitErrorMsgWithFailedToCloseFile(thread_io);
     if (0 != ::close(thread_stat_fd))
-        tryLogCurrentException(__PRETTY_FUNCTION__);
+        emitErrorMsgWithFailedToCloseFile(thread_stat);
     if (0 != ::close(thread_schedstat_fd))
-        tryLogCurrentException(__PRETTY_FUNCTION__);
+        emitErrorMsgWithFailedToCloseFile(thread_schedstat);
 }
 
 

From 998a9f4c0b7bb099e55f25a96bd79c1a42fb9db7 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Fri, 29 May 2020 12:52:30 +0300
Subject: [PATCH 158/184] Try fix copier tests.

---
 programs/copier/ClusterCopier.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/copier/ClusterCopier.cpp b/programs/copier/ClusterCopier.cpp
index d50b89738aa..d6a2fe103d1 100644
--- a/programs/copier/ClusterCopier.cpp
+++ b/programs/copier/ClusterCopier.cpp
@@ -1690,7 +1690,7 @@ std::set<String> ClusterCopier::getShardPartitions(const ConnectionTimeouts & ti
 
     Context local_context = context;
     local_context.setSettings(task_cluster->settings_pull);
-    Block block = getBlockWithAllStreamData(InterpreterFactory::get(query_ast, local_context)->execute().in);
+    Block block = getBlockWithAllStreamData(InterpreterFactory::get(query_ast, local_context)->execute().getInputStream());
 
     std::set<String> res;
     if (block)

From bd03d1acb964978638d57e98d8d675b857ddc289 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Fri, 29 May 2020 13:08:11 +0300
Subject: [PATCH 159/184] Remove hardcoded database from some tests and add
 retries to clickhouse-test

---
 tests/clickhouse-test                         |  19 +-
 .../00365_statistics_in_formats.sh            |  20 +-
 ...00563_insert_into_remote_and_zookeeper.sql |  16 +-
 ..._replace_partition_from_table_zookeeper.sh | 181 +++++++++---------
 .../queries/0_stateless/00763_lock_buffer.sh  |  14 +-
 ...ong_lock_buffer_alter_destination_table.sh |  18 +-
 ...ted_minimalistic_part_header_zookeeper.sql |  48 ++---
 ...ndex_granularity_replicated_merge_tree.sql | 164 ++++++++--------
 .../0_stateless/00937_test_use_header_csv.sh  |  10 +-
 .../queries/0_stateless/00942_mutate_index.sh |  20 +-
 .../0_stateless/00943_materialize_index.sh    |  46 ++---
 .../00944_clear_index_in_partition.sh         |  26 +--
 ...5_indices_mutation_replicated_zookeeper.sh |  30 +--
 .../01019_parallel_parsing_cancel.sh          |  16 +-
 ...1122_totals_rollup_having_block_header.sql |  16 +-
 .../0_stateless/01124_view_bad_types.sql      |  16 +-
 ...285_date_datetime_key_condition.reference} |   0
 .../01285_date_datetime_key_condition.sql     |  22 +++
 .../00155_date_datetime_key_condition.sql     |  22 ---
 19 files changed, 360 insertions(+), 344 deletions(-)
 rename tests/queries/{1_stateful/00155_date_datetime_key_condition.reference => 0_stateless/01285_date_datetime_key_condition.reference} (100%)
 create mode 100644 tests/queries/0_stateless/01285_date_datetime_key_condition.sql
 delete mode 100644 tests/queries/1_stateful/00155_date_datetime_key_condition.sql

diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index 1a2735296e8..1b45e4bb5d3 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -331,6 +331,19 @@ def run_tests_array(all_tests_with_params):
 server_logs_level = "warning"
 
 
+def check_server_started(client, retry_count):
+    while retry_count > 0:
+        clickhouse_proc = Popen(shlex.split(client), stdin=PIPE, stdout=PIPE, stderr=PIPE)
+        (stdout, stderr) = clickhouse_proc.communicate("SELECT 1")
+        if clickhouse_proc.returncode != 0 or not stdout.startswith("1"):
+            retry_count -= 1
+            sleep(0.5)
+        else:
+            return True
+
+    return False
+
+
 def main(args):
     global SERVER_DIED
     global exit_code
@@ -344,6 +357,9 @@ def main(args):
 
         return stdout.startswith('1')
 
+    if not check_server_started(args.client, args.server_check_retries):
+        raise Exception("clickhouse-server is not responding. Cannot execute 'SELECT 1' query.")
+
     base_dir = os.path.abspath(args.queries)
     tmp_dir = os.path.abspath(args.tmp)
 
@@ -360,7 +376,7 @@ def main(args):
     os.environ.setdefault("CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL", server_logs_level)
 
     if args.zookeeper is None:
-        code, out = commands.getstatusoutput(args.extract_from_config +" --try --config " + args.configserver + ' --key zookeeper | grep . | wc -l')
+        code, out = commands.getstatusoutput(args.extract_from_config + " --try --config " + args.configserver + ' --key zookeeper | grep . | wc -l')
         try:
             if int(out) > 0:
                 args.zookeeper = True
@@ -570,6 +586,7 @@ if __name__ == '__main__':
     parser.add_argument('--parallel', default='1/1', help='One parallel test run number/total')
     parser.add_argument('-j', '--jobs', default=1, nargs='?', type=int, help='Run all tests in parallel')
     parser.add_argument('-U', '--unified', default=3, type=int, help='output NUM lines of unified context')
+    parser.add_argument('-r', '--server-check-retries', default=30, type=int, help='Num of tries to execute SELECT 1 before tests started')
 
     parser.add_argument('--no-stateless', action='store_true', help='Disable all stateless tests')
     parser.add_argument('--no-stateful', action='store_true', help='Disable all stateful tests')
diff --git a/tests/queries/0_stateless/00365_statistics_in_formats.sh b/tests/queries/0_stateless/00365_statistics_in_formats.sh
index f0e23337806..fb79b23f78e 100755
--- a/tests/queries/0_stateless/00365_statistics_in_formats.sh
+++ b/tests/queries/0_stateless/00365_statistics_in_formats.sh
@@ -3,16 +3,16 @@
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . $CURDIR/../shell_config.sh
 
-$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.numbers";
-$CLICKHOUSE_CLIENT --query="CREATE TABLE test.numbers (number UInt64) engine = MergeTree order by number";
-$CLICKHOUSE_CLIENT --query="INSERT INTO test.numbers select * from system.numbers limit 10";
+$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS numbers";
+$CLICKHOUSE_CLIENT --query="CREATE TABLE numbers (number UInt64) engine = MergeTree order by number";
+$CLICKHOUSE_CLIENT --query="INSERT INTO numbers select * from system.numbers limit 10";
 
-$CLICKHOUSE_CLIENT --query="SELECT number FROM test.numbers LIMIT 10 FORMAT JSON" | grep 'rows_read';
-$CLICKHOUSE_CLIENT --query="SELECT number FROM test.numbers LIMIT 10 FORMAT JSONCompact" | grep 'rows_read';
-$CLICKHOUSE_CLIENT --query="SELECT number FROM test.numbers LIMIT 10 FORMAT XML" | grep 'rows_read';
+$CLICKHOUSE_CLIENT --query="SELECT number FROM numbers LIMIT 10 FORMAT JSON" | grep 'rows_read';
+$CLICKHOUSE_CLIENT --query="SELECT number FROM numbers LIMIT 10 FORMAT JSONCompact" | grep 'rows_read';
+$CLICKHOUSE_CLIENT --query="SELECT number FROM numbers LIMIT 10 FORMAT XML" | grep 'rows_read';
 
-${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "SELECT number FROM test.numbers LIMIT 10 FORMAT JSON" | grep 'rows_read';
-${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "SELECT number FROM test.numbers LIMIT 10 FORMAT JSONCompact" | grep 'rows_read';
-${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "SELECT number FROM test.numbers LIMIT 10 FORMAT XML" | grep 'rows_read';
+${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "SELECT number FROM numbers LIMIT 10 FORMAT JSON" | grep 'rows_read';
+${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "SELECT number FROM numbers LIMIT 10 FORMAT JSONCompact" | grep 'rows_read';
+${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "SELECT number FROM numbers LIMIT 10 FORMAT XML" | grep 'rows_read';
 
-$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.numbers";
\ No newline at end of file
+$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS numbers";
diff --git a/tests/queries/0_stateless/00563_insert_into_remote_and_zookeeper.sql b/tests/queries/0_stateless/00563_insert_into_remote_and_zookeeper.sql
index d26e947b54f..b4579e6aac3 100644
--- a/tests/queries/0_stateless/00563_insert_into_remote_and_zookeeper.sql
+++ b/tests/queries/0_stateless/00563_insert_into_remote_and_zookeeper.sql
@@ -1,14 +1,14 @@
 -- Check that settings are correctly passed through Distributed table
-DROP TABLE IF EXISTS test.simple;
-CREATE TABLE test.simple (d Int8) ENGINE = ReplicatedMergeTree('/clickhouse/test/tables/test/simple', '1') ORDER BY d;
+DROP TABLE IF EXISTS simple;
+CREATE TABLE simple (d Int8) ENGINE = ReplicatedMergeTree('/clickhouse/test/tables/test/simple', '1') ORDER BY d;
 
 -- TODO: replace '127.0.0.2' -> '127.0.0.1' after a fix
-INSERT INTO TABLE FUNCTION remote('127.0.0.2', 'test', 'simple') VALUES (1);
-INSERT INTO TABLE FUNCTION remote('127.0.0.2', 'test', 'simple') VALUES (1);
+INSERT INTO TABLE FUNCTION remote('127.0.0.2', currentDatabase(), 'simple') VALUES (1);
+INSERT INTO TABLE FUNCTION remote('127.0.0.2', currentDatabase(), 'simple') VALUES (1);
 
 SET insert_deduplicate=0;
-INSERT INTO TABLE FUNCTION remote('127.0.0.2', 'test', 'simple') VALUES (2);
-INSERT INTO TABLE FUNCTION remote('127.0.0.2', 'test', 'simple') VALUES (2);
+INSERT INTO TABLE FUNCTION remote('127.0.0.2', currentDatabase(), 'simple') VALUES (2);
+INSERT INTO TABLE FUNCTION remote('127.0.0.2', currentDatabase(), 'simple') VALUES (2);
 
-SELECT * FROM remote('127.0.0.2', 'test', 'simple') ORDER BY d;
-DROP TABLE test.simple;
\ No newline at end of file
+SELECT * FROM remote('127.0.0.2', currentDatabase(), 'simple') ORDER BY d;
+DROP TABLE simple;
diff --git a/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh b/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh
index 44ab712bcf8..2f3514ae2f5 100755
--- a/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh
+++ b/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh
@@ -26,144 +26,143 @@ function query_with_retry
     echo "Query '$1' failed with '$result'"
 }
 
-$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.src;"
-$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.dst_r1;"
-$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.dst_r2;"
+$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS src;"
+$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS dst_r1;"
+$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS dst_r2;"
 
-$CLICKHOUSE_CLIENT --query="CREATE TABLE test.src (p UInt64, k String, d UInt64) ENGINE = MergeTree PARTITION BY p ORDER BY k;"
-$CLICKHOUSE_CLIENT --query="CREATE TABLE test.dst_r1 (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/test/dst_1', '1') PARTITION BY p ORDER BY k SETTINGS old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=0;"
-$CLICKHOUSE_CLIENT --query="CREATE TABLE test.dst_r2 (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/test/dst_1', '2') PARTITION BY p ORDER BY k SETTINGS old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=0;"
+$CLICKHOUSE_CLIENT --query="CREATE TABLE src (p UInt64, k String, d UInt64) ENGINE = MergeTree PARTITION BY p ORDER BY k;"
+$CLICKHOUSE_CLIENT --query="CREATE TABLE dst_r1 (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/test/dst_1', '1') PARTITION BY p ORDER BY k SETTINGS old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=0;"
+$CLICKHOUSE_CLIENT --query="CREATE TABLE dst_r2 (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/test/dst_1', '2') PARTITION BY p ORDER BY k SETTINGS old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=0;"
 
-$CLICKHOUSE_CLIENT --query="INSERT INTO test.src VALUES (0, '0', 1);"
-$CLICKHOUSE_CLIENT --query="INSERT INTO test.src VALUES (1, '0', 1);"
-$CLICKHOUSE_CLIENT --query="INSERT INTO test.src VALUES (1, '1', 1);"
-$CLICKHOUSE_CLIENT --query="INSERT INTO test.src VALUES (2, '0', 1);"
+$CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (0, '0', 1);"
+$CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (1, '0', 1);"
+$CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (1, '1', 1);"
+$CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (2, '0', 1);"
 
 $CLICKHOUSE_CLIENT --query="SELECT 'Initial';"
-$CLICKHOUSE_CLIENT --query="INSERT INTO test.dst_r1 VALUES (0, '1', 2);"
-$CLICKHOUSE_CLIENT --query="INSERT INTO test.dst_r1 VALUES (1, '1', 2), (1, '2', 2);"
-$CLICKHOUSE_CLIENT --query="INSERT INTO test.dst_r1 VALUES (2, '1', 2);"
+$CLICKHOUSE_CLIENT --query="INSERT INTO dst_r1 VALUES (0, '1', 2);"
+$CLICKHOUSE_CLIENT --query="INSERT INTO dst_r1 VALUES (1, '1', 2), (1, '2', 2);"
+$CLICKHOUSE_CLIENT --query="INSERT INTO dst_r1 VALUES (2, '1', 2);"
 
-$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA test.dst_r2;"
-$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM test.src;"
-$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM test.dst_r1;"
-$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM test.dst_r2;"
+$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA dst_r2;"
+$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM src;"
+$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM dst_r1;"
+$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM dst_r2;"
 
 
 $CLICKHOUSE_CLIENT --query="SELECT 'REPLACE simple';"
-query_with_retry "ALTER TABLE test.dst_r1 REPLACE PARTITION 1 FROM test.src;"
-query_with_retry "ALTER TABLE test.src DROP PARTITION 1;"
+query_with_retry "ALTER TABLE dst_r1 REPLACE PARTITION 1 FROM src;"
+query_with_retry "ALTER TABLE src DROP PARTITION 1;"
 
-$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA test.dst_r2;"
-$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM test.src;"
-$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM test.dst_r1;"
-$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM test.dst_r2;"
+$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA dst_r2;"
+$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM src;"
+$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM dst_r1;"
+$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM dst_r2;"
 
 
 $CLICKHOUSE_CLIENT --query="SELECT 'REPLACE empty';"
-query_with_retry "ALTER TABLE test.src DROP PARTITION 1;"
-query_with_retry "ALTER TABLE test.dst_r1 REPLACE PARTITION 1 FROM test.src;"
+query_with_retry "ALTER TABLE src DROP PARTITION 1;"
+query_with_retry "ALTER TABLE dst_r1 REPLACE PARTITION 1 FROM src;"
 
-$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA test.dst_r2;"
-$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM test.dst_r1;"
-$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM test.dst_r2;"
+$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA dst_r2;"
+$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM dst_r1;"
+$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM dst_r2;"
 
 
 $CLICKHOUSE_CLIENT --query="SELECT 'REPLACE recursive';"
-query_with_retry "ALTER TABLE test.dst_r1 DROP PARTITION 1;"
-$CLICKHOUSE_CLIENT --query="INSERT INTO test.dst_r1 VALUES (1, '1', 2), (1, '2', 2);"
+query_with_retry "ALTER TABLE dst_r1 DROP PARTITION 1;"
+$CLICKHOUSE_CLIENT --query="INSERT INTO dst_r1 VALUES (1, '1', 2), (1, '2', 2);"
 
 $CLICKHOUSE_CLIENT --query="CREATE table test_block_numbers (m UInt64) ENGINE MergeTree() ORDER BY tuple();"
-$CLICKHOUSE_CLIENT --query="INSERT INTO test_block_numbers SELECT max(max_block_number) AS m FROM system.parts WHERE database='test' AND  table='dst_r1' AND active AND name LIKE '1_%';"
+$CLICKHOUSE_CLIENT --query="INSERT INTO test_block_numbers SELECT max(max_block_number) AS m FROM system.parts WHERE database=currentDatabase() AND  table='dst_r1' AND active AND name LIKE '1_%';"
 
-query_with_retry "ALTER TABLE test.dst_r1 REPLACE PARTITION 1 FROM test.dst_r1;"
-$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA test.dst_r2;"
-$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM test.dst_r1;"
-$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM test.dst_r2;"
+query_with_retry "ALTER TABLE dst_r1 REPLACE PARTITION 1 FROM dst_r1;"
+$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA dst_r2;"
+$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM dst_r1;"
+$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM dst_r2;"
 
-$CLICKHOUSE_CLIENT --query="INSERT INTO test_block_numbers SELECT max(max_block_number) AS m FROM system.parts WHERE database='test' AND  table='dst_r1' AND active AND name LIKE '1_%';"
+$CLICKHOUSE_CLIENT --query="INSERT INTO test_block_numbers SELECT max(max_block_number) AS m FROM system.parts WHERE database=currentDatabase() AND  table='dst_r1' AND active AND name LIKE '1_%';"
 $CLICKHOUSE_CLIENT --query="SELECT (max(m) - min(m) > 1) AS new_block_is_generated FROM test_block_numbers;"
 $CLICKHOUSE_CLIENT --query="DROP TABLE test_block_numbers;"
 
 
 $CLICKHOUSE_CLIENT --query="SELECT 'ATTACH FROM';"
-query_with_retry "ALTER TABLE test.dst_r1 DROP PARTITION 1;"
-$CLICKHOUSE_CLIENT --query="DROP TABLE test.src;"
+query_with_retry "ALTER TABLE dst_r1 DROP PARTITION 1;"
+$CLICKHOUSE_CLIENT --query="DROP TABLE src;"
 
-$CLICKHOUSE_CLIENT --query="CREATE TABLE test.src (p UInt64, k String, d UInt64) ENGINE = MergeTree PARTITION BY p ORDER BY k;"
-$CLICKHOUSE_CLIENT --query="INSERT INTO test.src VALUES (1, '0', 1);"
-$CLICKHOUSE_CLIENT --query="INSERT INTO test.src VALUES (1, '1', 1);"
+$CLICKHOUSE_CLIENT --query="CREATE TABLE src (p UInt64, k String, d UInt64) ENGINE = MergeTree PARTITION BY p ORDER BY k;"
+$CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (1, '0', 1);"
+$CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (1, '1', 1);"
 
-$CLICKHOUSE_CLIENT --query="INSERT INTO test.dst_r2 VALUES (1, '1', 2);"
-query_with_retry "ALTER TABLE test.dst_r2 ATTACH PARTITION 1 FROM test.src;"
+$CLICKHOUSE_CLIENT --query="INSERT INTO dst_r2 VALUES (1, '1', 2);"
+query_with_retry "ALTER TABLE dst_r2 ATTACH PARTITION 1 FROM src;"
 
-$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA test.dst_r1;"
-$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM test.dst_r1;"
-$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM test.dst_r2;"
+$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA dst_r1;"
+$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM dst_r1;"
+$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM dst_r2;"
 
 
 $CLICKHOUSE_CLIENT --query="SELECT 'REPLACE with fetch';"
-$CLICKHOUSE_CLIENT --query="DROP TABLE test.src;"
-$CLICKHOUSE_CLIENT --query="CREATE TABLE test.src (p UInt64, k String, d UInt64) ENGINE = MergeTree PARTITION BY p ORDER BY k;"
-$CLICKHOUSE_CLIENT --query="INSERT INTO test.src VALUES (1, '0', 1);"
-$CLICKHOUSE_CLIENT --query="INSERT INTO test.src VALUES (1, '1', 1);"
-$CLICKHOUSE_CLIENT --query="INSERT INTO test.dst_r1 VALUES (1, '1', 2); -- trash part to be deleted"
+$CLICKHOUSE_CLIENT --query="DROP TABLE src;"
+$CLICKHOUSE_CLIENT --query="CREATE TABLE src (p UInt64, k String, d UInt64) ENGINE = MergeTree PARTITION BY p ORDER BY k;"
+$CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (1, '0', 1);"
+$CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (1, '1', 1);"
+$CLICKHOUSE_CLIENT --query="INSERT INTO dst_r1 VALUES (1, '1', 2); -- trash part to be deleted"
 
 # Stop replication at the second replica and remove source table to use fetch instead of copying
-$CLICKHOUSE_CLIENT --query="SYSTEM STOP REPLICATION QUEUES test.dst_r2;"
-query_with_retry "ALTER TABLE test.dst_r1 REPLACE PARTITION 1 FROM test.src;"
-$CLICKHOUSE_CLIENT --query="DROP TABLE test.src;"
-$CLICKHOUSE_CLIENT --query="SYSTEM START REPLICATION QUEUES test.dst_r2;"
+$CLICKHOUSE_CLIENT --query="SYSTEM STOP REPLICATION QUEUES dst_r2;"
+query_with_retry "ALTER TABLE dst_r1 REPLACE PARTITION 1 FROM src;"
+$CLICKHOUSE_CLIENT --query="DROP TABLE src;"
+$CLICKHOUSE_CLIENT --query="SYSTEM START REPLICATION QUEUES dst_r2;"
 
-$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA test.dst_r2;"
-$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM test.dst_r1;"
-$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM test.dst_r2;"
+$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA dst_r2;"
+$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM dst_r1;"
+$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM dst_r2;"
 
 
 $CLICKHOUSE_CLIENT --query="SELECT 'REPLACE with fetch of merged';"
-$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.src;"
-query_with_retry "ALTER TABLE test.dst_r1 DROP PARTITION 1;"
+$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS src;"
+query_with_retry "ALTER TABLE dst_r1 DROP PARTITION 1;"
 
-$CLICKHOUSE_CLIENT --query="CREATE TABLE test.src (p UInt64, k String, d UInt64) ENGINE = MergeTree PARTITION BY p ORDER BY k;"
-$CLICKHOUSE_CLIENT --query="INSERT INTO test.src VALUES (1, '0', 1);"
-$CLICKHOUSE_CLIENT --query="INSERT INTO test.src VALUES (1, '1', 1);"
-$CLICKHOUSE_CLIENT --query="INSERT INTO test.dst_r1 VALUES (1, '1', 2); -- trash part to be deleted"
+$CLICKHOUSE_CLIENT --query="CREATE TABLE src (p UInt64, k String, d UInt64) ENGINE = MergeTree PARTITION BY p ORDER BY k;"
+$CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (1, '0', 1);"
+$CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (1, '1', 1);"
+$CLICKHOUSE_CLIENT --query="INSERT INTO dst_r1 VALUES (1, '1', 2); -- trash part to be deleted"
 
-$CLICKHOUSE_CLIENT --query="SYSTEM STOP MERGES test.dst_r2;"
-$CLICKHOUSE_CLIENT --query="SYSTEM STOP REPLICATION QUEUES test.dst_r2;"
-query_with_retry "ALTER TABLE test.dst_r1 REPLACE PARTITION 1 FROM test.src;"
-$CLICKHOUSE_CLIENT --query="DROP TABLE test.src;"
+$CLICKHOUSE_CLIENT --query="SYSTEM STOP MERGES dst_r2;"
+$CLICKHOUSE_CLIENT --query="SYSTEM STOP REPLICATION QUEUES dst_r2;"
+query_with_retry "ALTER TABLE dst_r1 REPLACE PARTITION 1 FROM src;"
+$CLICKHOUSE_CLIENT --query="DROP TABLE src;"
 
 # do not wait other replicas to execute OPTIMIZE
 
-$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d), uniqExact(_part) FROM test.dst_r1;"
-$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA test.dst_r1;"
-query_with_retry "OPTIMIZE TABLE test.dst_r1 PARTITION 1;" "--replication_alter_partitions_sync=0 --optimize_throw_if_noop=1"
+$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d), uniqExact(_part) FROM dst_r1;"
+$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA dst_r1;"
+query_with_retry "OPTIMIZE TABLE dst_r1 PARTITION 1;" "--replication_alter_partitions_sync=0 --optimize_throw_if_noop=1"
 
-$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA test.dst_r1;"
-$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d), uniqExact(_part) FROM test.dst_r1;"
+$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA dst_r1;"
+$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d), uniqExact(_part) FROM dst_r1;"
 
-$CLICKHOUSE_CLIENT --query="SYSTEM START REPLICATION QUEUES test.dst_r2;"
-$CLICKHOUSE_CLIENT --query="SYSTEM START MERGES test.dst_r2;"
-$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA test.dst_r2;"
-$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d), uniqExact(_part) FROM test.dst_r2;"
+$CLICKHOUSE_CLIENT --query="SYSTEM START REPLICATION QUEUES dst_r2;"
+$CLICKHOUSE_CLIENT --query="SYSTEM START MERGES dst_r2;"
+$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA dst_r2;"
+$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d), uniqExact(_part) FROM dst_r2;"
 
 $CLICKHOUSE_CLIENT --query="SELECT 'After restart';"
-$CLICKHOUSE_CLIENT --query="USE test;"
-$CLICKHOUSE_CLIENT --query="SYSTEM RESTART REPLICA test.dst_r1;"
+$CLICKHOUSE_CLIENT --query="SYSTEM RESTART REPLICA dst_r1;"
 $CLICKHOUSE_CLIENT --query="SYSTEM RESTART REPLICAS;"
-$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM test.dst_r1;"
-$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM test.dst_r2;"
+$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM dst_r1;"
+$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM dst_r2;"
 
 $CLICKHOUSE_CLIENT --query="SELECT 'DETACH+ATTACH PARTITION';"
-query_with_retry "ALTER TABLE test.dst_r1 DETACH PARTITION 0;"
-query_with_retry "ALTER TABLE test.dst_r1 DETACH PARTITION 1;"
-query_with_retry "ALTER TABLE test.dst_r1 DETACH PARTITION 2;"
-query_with_retry "ALTER TABLE test.dst_r1 ATTACH PARTITION 1;"
-$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM test.dst_r1;"
-$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA test.dst_r2;"
-$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM test.dst_r2;"
+query_with_retry "ALTER TABLE dst_r1 DETACH PARTITION 0;"
+query_with_retry "ALTER TABLE dst_r1 DETACH PARTITION 1;"
+query_with_retry "ALTER TABLE dst_r1 DETACH PARTITION 2;"
+query_with_retry "ALTER TABLE dst_r1 ATTACH PARTITION 1;"
+$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM dst_r1;"
+$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA dst_r2;"
+$CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM dst_r2;"
 
-$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.src;"
-$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.dst_r1;"
-$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.dst_r2;"
+$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS src;"
+$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS dst_r1;"
+$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS dst_r2;"
diff --git a/tests/queries/0_stateless/00763_lock_buffer.sh b/tests/queries/0_stateless/00763_lock_buffer.sh
index 3146ce96132..d320ae51a37 100755
--- a/tests/queries/0_stateless/00763_lock_buffer.sh
+++ b/tests/queries/0_stateless/00763_lock_buffer.sh
@@ -4,20 +4,20 @@ set -e
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . $CURDIR/../shell_config.sh
 
-${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS test.mt_00763_2"
-${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS test.buffer_00763_2"
+${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS mt_00763_2"
+${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS buffer_00763_2"
 
-${CLICKHOUSE_CLIENT} --query="CREATE TABLE test.buffer_00763_2 (s String) ENGINE = Buffer(test, mt_00763_2, 1, 1, 1, 1, 1, 1, 1)"
+${CLICKHOUSE_CLIENT} --query="CREATE TABLE buffer_00763_2 (s String) ENGINE = Buffer('$CLICKHOUSE_DATABASE', mt_00763_2, 1, 1, 1, 1, 1, 1, 1)"
 
 
 function thread1()
 {
-    seq 1 500 | sed -r -e 's/.+/DROP TABLE IF EXISTS test.mt_00763_2; CREATE TABLE test.mt_00763_2 (s String) ENGINE = MergeTree ORDER BY s; INSERT INTO test.mt_00763_2 SELECT toString(number) FROM numbers(10);/' | ${CLICKHOUSE_CLIENT} --multiquery --ignore-error ||:
+    seq 1 500 | sed -r -e 's/.+/DROP TABLE IF EXISTS mt_00763_2; CREATE TABLE mt_00763_2 (s String) ENGINE = MergeTree ORDER BY s; INSERT INTO mt_00763_2 SELECT toString(number) FROM numbers(10);/' | ${CLICKHOUSE_CLIENT} --multiquery --ignore-error ||:
 }
 
 function thread2()
 {
-    seq 1 1000 | sed -r -e 's/.+/SELECT count() FROM test.buffer_00763_2;/' | ${CLICKHOUSE_CLIENT} --multiquery --server_logs_file='/dev/null' --ignore-error 2>&1 | grep -vP '^0$|^10$|^Received exception|^Code: 60|^Code: 218|^Code: 473'
+    seq 1 1000 | sed -r -e 's/.+/SELECT count() FROM buffer_00763_2;/' | ${CLICKHOUSE_CLIENT} --multiquery --server_logs_file='/dev/null' --ignore-error 2>&1 | grep -vP '^0$|^10$|^Received exception|^Code: 60|^Code: 218|^Code: 473'
 }
 
 thread1 &
@@ -25,5 +25,5 @@ thread2 &
 
 wait
 
-${CLICKHOUSE_CLIENT} --query="DROP TABLE test.mt_00763_2"
-${CLICKHOUSE_CLIENT} --query="DROP TABLE test.buffer_00763_2"
+${CLICKHOUSE_CLIENT} --query="DROP TABLE mt_00763_2"
+${CLICKHOUSE_CLIENT} --query="DROP TABLE buffer_00763_2"
diff --git a/tests/queries/0_stateless/00763_long_lock_buffer_alter_destination_table.sh b/tests/queries/0_stateless/00763_long_lock_buffer_alter_destination_table.sh
index 059d70253b2..26bdb712ec5 100755
--- a/tests/queries/0_stateless/00763_long_lock_buffer_alter_destination_table.sh
+++ b/tests/queries/0_stateless/00763_long_lock_buffer_alter_destination_table.sh
@@ -6,21 +6,21 @@ CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=none
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . $CURDIR/../shell_config.sh
 
-${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS test.mt_00763_1"
-${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS test.buffer_00763_1"
+${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS mt_00763_1"
+${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS buffer_00763_1"
 
-${CLICKHOUSE_CLIENT} --query="CREATE TABLE test.buffer_00763_1 (s String) ENGINE = Buffer(test, mt_00763_1, 1, 1, 1, 1, 1, 1, 1)"
-${CLICKHOUSE_CLIENT} --query="CREATE TABLE test.mt_00763_1 (x UInt32, s String) ENGINE = MergeTree ORDER BY x"
-${CLICKHOUSE_CLIENT} --query="INSERT INTO test.mt_00763_1 VALUES (1, '1'), (2, '2'), (3, '3')"
+${CLICKHOUSE_CLIENT} --query="CREATE TABLE buffer_00763_1 (s String) ENGINE = Buffer($CLICKHOUSE_DATABASE, mt_00763_1, 1, 1, 1, 1, 1, 1, 1)"
+${CLICKHOUSE_CLIENT} --query="CREATE TABLE mt_00763_1 (x UInt32, s String) ENGINE = MergeTree ORDER BY x"
+${CLICKHOUSE_CLIENT} --query="INSERT INTO mt_00763_1 VALUES (1, '1'), (2, '2'), (3, '3')"
 
 function thread1()
 {
-    seq 1 300 | sed -r -e 's/.+/ALTER TABLE test.mt_00763_1 MODIFY column s UInt32; ALTER TABLE test.mt_00763_1 MODIFY column s String;/' | ${CLICKHOUSE_CLIENT} --multiquery --ignore-error ||:
+    seq 1 300 | sed -r -e 's/.+/ALTER TABLE mt_00763_1 MODIFY column s UInt32; ALTER TABLE mt_00763_1 MODIFY column s String;/' | ${CLICKHOUSE_CLIENT} --multiquery --ignore-error ||:
 }
 
 function thread2()
 {
-    seq 1 2000 | sed -r -e 's/.+/SELECT sum(length(s)) FROM test.buffer_00763_1;/' | ${CLICKHOUSE_CLIENT} --multiquery --ignore-error 2>&1 | grep -vP '(^3$|^Received exception from server|^Code: 473)'
+    seq 1 2000 | sed -r -e 's/.+/SELECT sum(length(s)) FROM buffer_00763_1;/' | ${CLICKHOUSE_CLIENT} --multiquery --ignore-error 2>&1 | grep -vP '(^3$|^Received exception from server|^Code: 473)'
 }
 
 thread1 &
@@ -28,5 +28,5 @@ thread2 &
 
 wait
 
-${CLICKHOUSE_CLIENT} --query="DROP TABLE test.mt_00763_1"
-${CLICKHOUSE_CLIENT} --query="DROP TABLE test.buffer_00763_1"
+${CLICKHOUSE_CLIENT} --query="DROP TABLE mt_00763_1"
+${CLICKHOUSE_CLIENT} --query="DROP TABLE buffer_00763_1"
diff --git a/tests/queries/0_stateless/00814_replicated_minimalistic_part_header_zookeeper.sql b/tests/queries/0_stateless/00814_replicated_minimalistic_part_header_zookeeper.sql
index 2e1df541f14..9f0850ff7e5 100644
--- a/tests/queries/0_stateless/00814_replicated_minimalistic_part_header_zookeeper.sql
+++ b/tests/queries/0_stateless/00814_replicated_minimalistic_part_header_zookeeper.sql
@@ -1,15 +1,15 @@
-DROP TABLE IF EXISTS test.part_header_r1;
-DROP TABLE IF EXISTS test.part_header_r2;
+DROP TABLE IF EXISTS part_header_r1;
+DROP TABLE IF EXISTS part_header_r2;
 
 SET replication_alter_partitions_sync = 2;
 
-CREATE TABLE test.part_header_r1(x UInt32, y UInt32)
+CREATE TABLE part_header_r1(x UInt32, y UInt32)
     ENGINE ReplicatedMergeTree('/clickhouse/tables/test/part_header', '1') ORDER BY x
     SETTINGS use_minimalistic_part_header_in_zookeeper = 0,
              old_parts_lifetime = 1,
              cleanup_delay_period = 0,
              cleanup_delay_period_random_add = 0;
-CREATE TABLE test.part_header_r2(x UInt32, y UInt32)
+CREATE TABLE part_header_r2(x UInt32, y UInt32)
     ENGINE ReplicatedMergeTree('/clickhouse/tables/test/part_header', '2') ORDER BY x
     SETTINGS use_minimalistic_part_header_in_zookeeper = 1,
              old_parts_lifetime = 1,
@@ -17,47 +17,47 @@ CREATE TABLE test.part_header_r2(x UInt32, y UInt32)
              cleanup_delay_period_random_add = 0;
 
 SELECT '*** Test fetches ***';
-INSERT INTO test.part_header_r1 VALUES (1, 1);
-INSERT INTO test.part_header_r2 VALUES (2, 2);
-SYSTEM SYNC REPLICA test.part_header_r1;
-SYSTEM SYNC REPLICA test.part_header_r2;
+INSERT INTO part_header_r1 VALUES (1, 1);
+INSERT INTO part_header_r2 VALUES (2, 2);
+SYSTEM SYNC REPLICA part_header_r1;
+SYSTEM SYNC REPLICA part_header_r2;
 SELECT '*** replica 1 ***';
-SELECT x, y FROM test.part_header_r1 ORDER BY x;
+SELECT x, y FROM part_header_r1 ORDER BY x;
 SELECT '*** replica 2 ***';
-SELECT x, y FROM test.part_header_r2 ORDER BY x;
+SELECT x, y FROM part_header_r2 ORDER BY x;
 
 SELECT '*** Test merges ***';
-OPTIMIZE TABLE test.part_header_r1;
-SYSTEM SYNC REPLICA test.part_header_r2;
+OPTIMIZE TABLE part_header_r1;
+SYSTEM SYNC REPLICA part_header_r2;
 SELECT '*** replica 1 ***';
-SELECT _part, x FROM test.part_header_r1 ORDER BY x;
+SELECT _part, x FROM part_header_r1 ORDER BY x;
 SELECT '*** replica 2 ***';
-SELECT _part, x FROM test.part_header_r2 ORDER BY x;
+SELECT _part, x FROM part_header_r2 ORDER BY x;
 
 SELECT sleep(3) FORMAT Null;
 
 SELECT '*** Test part removal ***';
 SELECT '*** replica 1 ***';
-SELECT name FROM system.parts WHERE active AND database = 'test' AND table = 'part_header_r1';
+SELECT name FROM system.parts WHERE active AND database = currentDatabase() AND table = 'part_header_r1';
 SELECT name FROM system.zookeeper WHERE path = '/clickhouse/tables/test/part_header/replicas/1/parts';
 SELECT '*** replica 2 ***';
-SELECT name FROM system.parts WHERE active AND database = 'test' AND table = 'part_header_r2';
+SELECT name FROM system.parts WHERE active AND database = currentDatabase() AND table = 'part_header_r2';
 SELECT name FROM system.zookeeper WHERE path = '/clickhouse/tables/test/part_header/replicas/1/parts';
 
 SELECT '*** Test ALTER ***';
-ALTER TABLE test.part_header_r1 MODIFY COLUMN y String;
+ALTER TABLE part_header_r1 MODIFY COLUMN y String;
 SELECT '*** replica 1 ***';
-SELECT x, length(y) FROM test.part_header_r1 ORDER BY x;
+SELECT x, length(y) FROM part_header_r1 ORDER BY x;
 SELECT '*** replica 2 ***';
-SELECT x, length(y) FROM test.part_header_r2 ORDER BY x;
+SELECT x, length(y) FROM part_header_r2 ORDER BY x;
 
 SELECT '*** Test CLEAR COLUMN ***';
 SET replication_alter_partitions_sync = 2;
-ALTER TABLE test.part_header_r1 CLEAR COLUMN y IN PARTITION tuple();
+ALTER TABLE part_header_r1 CLEAR COLUMN y IN PARTITION tuple();
 SELECT '*** replica 1 ***';
-SELECT x, length(y) FROM test.part_header_r1 ORDER BY x;
+SELECT x, length(y) FROM part_header_r1 ORDER BY x;
 SELECT '*** replica 2 ***';
-SELECT x, length(y) FROM test.part_header_r2 ORDER BY x;
+SELECT x, length(y) FROM part_header_r2 ORDER BY x;
 
-DROP TABLE test.part_header_r1;
-DROP TABLE test.part_header_r2;
+DROP TABLE part_header_r1;
+DROP TABLE part_header_r2;
diff --git a/tests/queries/0_stateless/00926_zookeeper_adaptive_index_granularity_replicated_merge_tree.sql b/tests/queries/0_stateless/00926_zookeeper_adaptive_index_granularity_replicated_merge_tree.sql
index 5b785d31497..a7ae0402b5a 100644
--- a/tests/queries/0_stateless/00926_zookeeper_adaptive_index_granularity_replicated_merge_tree.sql
+++ b/tests/queries/0_stateless/00926_zookeeper_adaptive_index_granularity_replicated_merge_tree.sql
@@ -1,214 +1,214 @@
 ----- Group of very similar simple tests ------
 select '----HORIZONTAL MERGE TESTS----';
-DROP TABLE IF EXISTS test.zero_rows_per_granule1;
-DROP TABLE IF EXISTS test.zero_rows_per_granule2;
+DROP TABLE IF EXISTS zero_rows_per_granule1;
+DROP TABLE IF EXISTS zero_rows_per_granule2;
 
-CREATE TABLE test.zero_rows_per_granule1 (
+CREATE TABLE zero_rows_per_granule1 (
   p Date,
   k UInt64,
   v1 UInt64,
   v2 Int64
 ) ENGINE ReplicatedMergeTree('/clickhouse/tables/test/zero_rows_in_granule', '1') PARTITION BY toYYYYMM(p) ORDER BY k SETTINGS index_granularity_bytes = 20, write_final_mark = 0;
 
-CREATE TABLE test.zero_rows_per_granule2 (
+CREATE TABLE zero_rows_per_granule2 (
   p Date,
   k UInt64,
   v1 UInt64,
   v2 Int64
 ) ENGINE ReplicatedMergeTree('/clickhouse/tables/test/zero_rows_in_granule', '2') PARTITION BY toYYYYMM(p) ORDER BY k SETTINGS index_granularity_bytes = 20, write_final_mark = 0;
 
-INSERT INTO test.zero_rows_per_granule1 (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000);
+INSERT INTO zero_rows_per_granule1 (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000);
 
-SYSTEM SYNC REPLICA test.zero_rows_per_granule2;
+SYSTEM SYNC REPLICA zero_rows_per_granule2;
 
 SELECT 'Replica synced';
 
-SELECT COUNT(*) FROM test.zero_rows_per_granule1;
+SELECT COUNT(*) FROM zero_rows_per_granule1;
 
-SELECT distinct(marks) from system.parts WHERE table = 'zero_rows_per_granule1' and database='test' and active=1;
+SELECT distinct(marks) from system.parts WHERE table = 'zero_rows_per_granule1' and database=currentDatabase() and active=1;
 
-SELECT COUNT(*) FROM test.zero_rows_per_granule2;
+SELECT COUNT(*) FROM zero_rows_per_granule2;
 
-SELECT distinct(marks) from system.parts WHERE table = 'zero_rows_per_granule2' and database='test' and active=1;
+SELECT distinct(marks) from system.parts WHERE table = 'zero_rows_per_granule2' and database=currentDatabase() and active=1;
 
-INSERT INTO test.zero_rows_per_granule2 (p, k, v1, v2) VALUES ('2018-05-15', 5, 1000, 2000), ('2018-05-16', 6, 3000, 4000), ('2018-05-17', 7, 5000, 6000), ('2018-05-19', 8, 7000, 8000);
+INSERT INTO zero_rows_per_granule2 (p, k, v1, v2) VALUES ('2018-05-15', 5, 1000, 2000), ('2018-05-16', 6, 3000, 4000), ('2018-05-17', 7, 5000, 6000), ('2018-05-19', 8, 7000, 8000);
 
-SELECT distinct(marks) from system.parts WHERE table = 'zero_rows_per_granule2' and database='test' and active=1;
+SELECT distinct(marks) from system.parts WHERE table = 'zero_rows_per_granule2' and database=currentDatabase() and active=1;
 
-SELECT distinct(marks) from system.parts WHERE table = 'zero_rows_per_granule1' and database='test' and active=1;
+SELECT distinct(marks) from system.parts WHERE table = 'zero_rows_per_granule1' and database=currentDatabase() and active=1;
 
 SELECT sleep(0.7) Format Null;
 
-OPTIMIZE TABLE test.zero_rows_per_granule2 FINAL;
+OPTIMIZE TABLE zero_rows_per_granule2 FINAL;
 
 SELECT 'Parts optimized';
 
-SYSTEM SYNC REPLICA test.zero_rows_per_granule1;
+SYSTEM SYNC REPLICA zero_rows_per_granule1;
 
 SELECT 'Replica synced';
 
-SELECT COUNT(*) FROM test.zero_rows_per_granule2;
+SELECT COUNT(*) FROM zero_rows_per_granule2;
 
-SELECT distinct(marks) from system.parts WHERE table = 'zero_rows_per_granule2' and database='test' and active=1;
+SELECT distinct(marks) from system.parts WHERE table = 'zero_rows_per_granule2' and database=currentDatabase() and active=1;
 
-SELECT COUNT(*) FROM test.zero_rows_per_granule1;
+SELECT COUNT(*) FROM zero_rows_per_granule1;
 
-SELECT distinct(marks) from system.parts WHERE table = 'zero_rows_per_granule1' and database='test' and active=1;
+SELECT distinct(marks) from system.parts WHERE table = 'zero_rows_per_granule1' and database=currentDatabase() and active=1;
 
-DROP TABLE IF EXISTS test.zero_rows_per_granule1;
-DROP TABLE IF EXISTS test.zero_rows_per_granule2;
+DROP TABLE IF EXISTS zero_rows_per_granule1;
+DROP TABLE IF EXISTS zero_rows_per_granule2;
 
 SELECT '-----';
 
-DROP TABLE IF EXISTS test.four_rows_per_granule1;
-DROP TABLE IF EXISTS test.four_rows_per_granule2;
+DROP TABLE IF EXISTS four_rows_per_granule1;
+DROP TABLE IF EXISTS four_rows_per_granule2;
 
-CREATE TABLE test.four_rows_per_granule1 (
+CREATE TABLE four_rows_per_granule1 (
   p Date,
   k UInt64,
   v1 UInt64,
   v2 Int64
 ) ENGINE ReplicatedMergeTree('/clickhouse/tables/test/four_rows_in_granule', '1') PARTITION BY toYYYYMM(p) ORDER BY k SETTINGS index_granularity_bytes = 110, write_final_mark = 0;
 
-CREATE TABLE test.four_rows_per_granule2 (
+CREATE TABLE four_rows_per_granule2 (
   p Date,
   k UInt64,
   v1 UInt64,
   v2 Int64
 ) ENGINE ReplicatedMergeTree('/clickhouse/tables/test/four_rows_in_granule', '2') PARTITION BY toYYYYMM(p) ORDER BY k SETTINGS index_granularity_bytes = 110, write_final_mark = 0;
 
-INSERT INTO test.four_rows_per_granule1 (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000);
+INSERT INTO four_rows_per_granule1 (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000);
 
-SELECT COUNT(*) FROM test.four_rows_per_granule1;
+SELECT COUNT(*) FROM four_rows_per_granule1;
 
-SELECT distinct(marks) from system.parts WHERE table = 'four_rows_per_granule1' and database='test' and active=1;
+SELECT distinct(marks) from system.parts WHERE table = 'four_rows_per_granule1' and database=currentDatabase() and active=1;
 
-SYSTEM SYNC REPLICA test.four_rows_per_granule2;
+SYSTEM SYNC REPLICA four_rows_per_granule2;
 
 SELECT 'Replica synced';
 
-SELECT COUNT(*) FROM test.four_rows_per_granule2;
+SELECT COUNT(*) FROM four_rows_per_granule2;
 
-SELECT distinct(marks) from system.parts WHERE table = 'four_rows_per_granule2' and database='test' and active=1;
+SELECT distinct(marks) from system.parts WHERE table = 'four_rows_per_granule2' and database=currentDatabase() and active=1;
 
-DETACH TABLE test.four_rows_per_granule2;
-ATTACH TABLE test.four_rows_per_granule2;
+DETACH TABLE four_rows_per_granule2;
+ATTACH TABLE four_rows_per_granule2;
 
 SELECT 'Table attached';
 
-INSERT INTO test.four_rows_per_granule2 (p, k, v1, v2) VALUES ('2018-05-15', 5, 1000, 2000), ('2018-05-16', 6, 3000, 4000), ('2018-05-17', 7, 5000, 6000), ('2018-05-19', 8, 7000, 8000);
+INSERT INTO four_rows_per_granule2 (p, k, v1, v2) VALUES ('2018-05-15', 5, 1000, 2000), ('2018-05-16', 6, 3000, 4000), ('2018-05-17', 7, 5000, 6000), ('2018-05-19', 8, 7000, 8000);
 
-SELECT distinct(marks) from system.parts WHERE table = 'four_rows_per_granule2' and database='test' and active=1;
+SELECT distinct(marks) from system.parts WHERE table = 'four_rows_per_granule2' and database=currentDatabase() and active=1;
 
-SELECT distinct(marks) from system.parts WHERE table = 'four_rows_per_granule1' and database='test' and active=1;
+SELECT distinct(marks) from system.parts WHERE table = 'four_rows_per_granule1' and database=currentDatabase() and active=1;
 
 SELECT sleep(0.7) Format Null;
 
-OPTIMIZE TABLE test.four_rows_per_granule2 FINAL;
+OPTIMIZE TABLE four_rows_per_granule2 FINAL;
 SELECT 'Parts optimized';
 
-DETACH TABLE test.four_rows_per_granule2;
+DETACH TABLE four_rows_per_granule2;
 
-ATTACH TABLE test.four_rows_per_granule2;
+ATTACH TABLE four_rows_per_granule2;
 
-SELECT COUNT(*) FROM test.four_rows_per_granule2;
+SELECT COUNT(*) FROM four_rows_per_granule2;
 
---SELECT distinct(marks) from system.parts WHERE table = 'four_rows_per_granule2' and database='test' and active=1;
+--SELECT distinct(marks) from system.parts WHERE table = 'four_rows_per_granule2' and database=currentDatabase() and active=1;
 
-SYSTEM SYNC REPLICA test.four_rows_per_granule1;
+SYSTEM SYNC REPLICA four_rows_per_granule1;
 SELECT 'Replica synced';
 
-SELECT COUNT(*) FROM test.four_rows_per_granule1;
+SELECT COUNT(*) FROM four_rows_per_granule1;
 
---SELECT distinct(marks) from system.parts WHERE table = 'four_rows_per_granule1' and database='test' and active=1;
+--SELECT distinct(marks) from system.parts WHERE table = 'four_rows_per_granule1' and database=currentDatabase() and active=1;
 
-DROP TABLE IF EXISTS test.four_rows_per_granule1;
-DROP TABLE IF EXISTS test.four_rows_per_granule2;
+DROP TABLE IF EXISTS four_rows_per_granule1;
+DROP TABLE IF EXISTS four_rows_per_granule2;
 
 SELECT '-----';
 
-DROP TABLE IF EXISTS test.adaptive_granularity_alter1;
-DROP TABLE IF EXISTS test.adaptive_granularity_alter2;
+DROP TABLE IF EXISTS adaptive_granularity_alter1;
+DROP TABLE IF EXISTS adaptive_granularity_alter2;
 
-CREATE TABLE test.adaptive_granularity_alter1 (
+CREATE TABLE adaptive_granularity_alter1 (
   p Date,
   k UInt64,
   v1 UInt64,
   v2 Int64
 ) ENGINE ReplicatedMergeTree('/clickhouse/tables/test/adaptive_granularity_alter', '1') PARTITION BY toYYYYMM(p) ORDER BY k SETTINGS index_granularity_bytes = 110, write_final_mark = 0;
 
-CREATE TABLE test.adaptive_granularity_alter2 (
+CREATE TABLE adaptive_granularity_alter2 (
   p Date,
   k UInt64,
   v1 UInt64,
   v2 Int64
 ) ENGINE ReplicatedMergeTree('/clickhouse/tables/test/adaptive_granularity_alter', '2') PARTITION BY toYYYYMM(p) ORDER BY k SETTINGS index_granularity_bytes = 110, write_final_mark = 0;
 
-INSERT INTO test.adaptive_granularity_alter1 (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000);
+INSERT INTO adaptive_granularity_alter1 (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000);
 
-SELECT COUNT(*) FROM test.adaptive_granularity_alter1;
+SELECT COUNT(*) FROM adaptive_granularity_alter1;
 
-SELECT distinct(marks) from system.parts WHERE table = 'adaptive_granularity_alter1' and database='test' and active=1;
+SELECT distinct(marks) from system.parts WHERE table = 'adaptive_granularity_alter1' and database=currentDatabase() and active=1;
 
-SYSTEM SYNC REPLICA test.adaptive_granularity_alter2;
+SYSTEM SYNC REPLICA adaptive_granularity_alter2;
 SELECT 'Replica synced';
 
-SELECT COUNT(*) FROM test.adaptive_granularity_alter2;
+SELECT COUNT(*) FROM adaptive_granularity_alter2;
 
-SELECT distinct(marks) from system.parts WHERE table = 'adaptive_granularity_alter2' and database='test' and active=1;
+SELECT distinct(marks) from system.parts WHERE table = 'adaptive_granularity_alter2' and database=currentDatabase() and active=1;
 
-ALTER TABLE test.adaptive_granularity_alter2 MODIFY COLUMN v1 Int16;
+ALTER TABLE adaptive_granularity_alter2 MODIFY COLUMN v1 Int16;
 
-DETACH TABLE test.adaptive_granularity_alter2;
+DETACH TABLE adaptive_granularity_alter2;
 
-ATTACH TABLE test.adaptive_granularity_alter2;
+ATTACH TABLE adaptive_granularity_alter2;
 
-SELECT COUNT(*) FROM test.adaptive_granularity_alter2;
+SELECT COUNT(*) FROM adaptive_granularity_alter2;
 
-SELECT distinct(marks) from system.parts WHERE table = 'adaptive_granularity_alter2' and database='test' and active=1;
+SELECT distinct(marks) from system.parts WHERE table = 'adaptive_granularity_alter2' and database=currentDatabase() and active=1;
 
-SYSTEM SYNC REPLICA test.adaptive_granularity_alter1;
+SYSTEM SYNC REPLICA adaptive_granularity_alter1;
 SELECT 'Replica synced';
 
-SELECT COUNT(*) FROM test.adaptive_granularity_alter1;
+SELECT COUNT(*) FROM adaptive_granularity_alter1;
 
-SELECT distinct(marks) from system.parts WHERE table = 'adaptive_granularity_alter1' and database='test' and active=1;
+SELECT distinct(marks) from system.parts WHERE table = 'adaptive_granularity_alter1' and database=currentDatabase() and active=1;
 
-INSERT INTO test.adaptive_granularity_alter1 (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 5, 3000, 4000), ('2018-05-17', 6, 5000, 6000), ('2018-05-19', 42, 42, 42);
+INSERT INTO adaptive_granularity_alter1 (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 5, 3000, 4000), ('2018-05-17', 6, 5000, 6000), ('2018-05-19', 42, 42, 42);
 
-SELECT COUNT(*) FROM test.adaptive_granularity_alter1;
+SELECT COUNT(*) FROM adaptive_granularity_alter1;
 
-SELECT distinct(marks) from system.parts WHERE table = 'adaptive_granularity_alter1' and database='test' and active=1;
+SELECT distinct(marks) from system.parts WHERE table = 'adaptive_granularity_alter1' and database=currentDatabase() and active=1;
 
-SYSTEM SYNC REPLICA test.adaptive_granularity_alter2;
+SYSTEM SYNC REPLICA adaptive_granularity_alter2;
 
-SELECT COUNT(*) FROM test.adaptive_granularity_alter2;
+SELECT COUNT(*) FROM adaptive_granularity_alter2;
 
-SELECT distinct(marks) from system.parts WHERE table = 'adaptive_granularity_alter2' and database='test' and active=1;
+SELECT distinct(marks) from system.parts WHERE table = 'adaptive_granularity_alter2' and database=currentDatabase() and active=1;
 
-ALTER TABLE test.adaptive_granularity_alter1 MODIFY COLUMN v2 String;
+ALTER TABLE adaptive_granularity_alter1 MODIFY COLUMN v2 String;
 
-DETACH TABLE test.adaptive_granularity_alter1;
+DETACH TABLE adaptive_granularity_alter1;
 
-ATTACH TABLE test.adaptive_granularity_alter1;
+ATTACH TABLE adaptive_granularity_alter1;
 
-INSERT INTO test.adaptive_granularity_alter1 (p, k, v1, v2) VALUES ('2018-05-15', 100, 1000, 'aaaa'), ('2018-05-16', 101, 3000, 'bbbb'), ('2018-05-17', 102, 5000, 'cccc'), ('2018-05-19', 103, 7000, 'dddd');
+INSERT INTO adaptive_granularity_alter1 (p, k, v1, v2) VALUES ('2018-05-15', 100, 1000, 'aaaa'), ('2018-05-16', 101, 3000, 'bbbb'), ('2018-05-17', 102, 5000, 'cccc'), ('2018-05-19', 103, 7000, 'dddd');
 
 SELECT sleep(0.7) Format Null;
 
-OPTIMIZE TABLE test.adaptive_granularity_alter1 FINAL;
+OPTIMIZE TABLE adaptive_granularity_alter1 FINAL;
 SELECT 'Parts optimized';
 
-SELECT k, v2 FROM test.adaptive_granularity_alter1 WHERE k >= 100 OR k = 42 ORDER BY k;
+SELECT k, v2 FROM adaptive_granularity_alter1 WHERE k >= 100 OR k = 42 ORDER BY k;
 
-SELECT sum(marks) from system.parts WHERE table = 'adaptive_granularity_alter1' and database='test' and active=1;
+SELECT sum(marks) from system.parts WHERE table = 'adaptive_granularity_alter1' and database=currentDatabase() and active=1;
 
-SYSTEM SYNC REPLICA test.adaptive_granularity_alter2;
+SYSTEM SYNC REPLICA adaptive_granularity_alter2;
 SELECT 'Replica synced';
 
-SELECT k, v2 FROM test.adaptive_granularity_alter2 WHERE k >= 100 OR k = 42 ORDER BY k;
+SELECT k, v2 FROM adaptive_granularity_alter2 WHERE k >= 100 OR k = 42 ORDER BY k;
 
-SELECT sum(marks) from system.parts WHERE table = 'adaptive_granularity_alter2' and database='test' and active=1;
+SELECT sum(marks) from system.parts WHERE table = 'adaptive_granularity_alter2' and database=currentDatabase() and active=1;
 
-DROP TABLE IF EXISTS test.adaptive_granularity_alter1;
-DROP TABLE IF EXISTS test.adaptive_granularity_alter2;
+DROP TABLE IF EXISTS adaptive_granularity_alter1;
+DROP TABLE IF EXISTS adaptive_granularity_alter2;
diff --git a/tests/queries/0_stateless/00937_test_use_header_csv.sh b/tests/queries/0_stateless/00937_test_use_header_csv.sh
index bd9204ffdcc..c0d4f2be522 100755
--- a/tests/queries/0_stateless/00937_test_use_header_csv.sh
+++ b/tests/queries/0_stateless/00937_test_use_header_csv.sh
@@ -3,10 +3,10 @@
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . $CURDIR/../shell_config.sh
 
-$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.csv"
-$CLICKHOUSE_CLIENT --query="CREATE TABLE test.csv (d Date, u UInt8, str String) ENGINE = TinyLog"
+$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS csv"
+$CLICKHOUSE_CLIENT --query="CREATE TABLE csv (d Date, u UInt8, str String) ENGINE = TinyLog"
 
-INSERT_QUERY='$CLICKHOUSE_CLIENT --query="INSERT INTO test.csv FORMAT CSVWithNames"'
+INSERT_QUERY='$CLICKHOUSE_CLIENT --query="INSERT INTO csv FORMAT CSVWithNames"'
 USE_HEADER='--input_format_with_names_use_header=1'
 SKIP_UNKNOWN='--input_format_skip_unknown_fields=1'
 
@@ -32,5 +32,5 @@ echo -ne 'str,u\nLine16,1\nLine17,2\n'                  | eval $INSERT_QUERY $US
 echo -ne 'd,str\n2019-04-18,Line18\n2019-04-18,Line19\n'| eval $INSERT_QUERY $USE_HEADER
 echo -ne 'unknown\n\n\n'                                | eval $INSERT_QUERY $USE_HEADER $SKIP_UNKNOWN
 
-$CLICKHOUSE_CLIENT --query="SELECT * FROM test.csv"
-$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.csv"
+$CLICKHOUSE_CLIENT --query="SELECT * FROM csv"
+$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS csv"
diff --git a/tests/queries/0_stateless/00942_mutate_index.sh b/tests/queries/0_stateless/00942_mutate_index.sh
index 30ac7e8821b..d14c5659fc9 100755
--- a/tests/queries/0_stateless/00942_mutate_index.sh
+++ b/tests/queries/0_stateless/00942_mutate_index.sh
@@ -4,11 +4,11 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . $CURDIR/../shell_config.sh
 . $CURDIR/mergetree_mutations.lib
 
-$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.minmax_idx;"
+$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS minmax_idx;"
 
 
 $CLICKHOUSE_CLIENT -n --query="
-CREATE TABLE test.minmax_idx
+CREATE TABLE minmax_idx
 (
     u64 UInt64,
     i64 Int64,
@@ -19,7 +19,7 @@ ORDER BY u64
 SETTINGS index_granularity = 2;"
 
 
-$CLICKHOUSE_CLIENT --query="INSERT INTO test.minmax_idx VALUES
+$CLICKHOUSE_CLIENT --query="INSERT INTO minmax_idx VALUES
 (0, 1, 1),
 (1, 1, 2),
 (2, 1, 3),
@@ -31,13 +31,13 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO test.minmax_idx VALUES
 (8, 1, 9),
 (9, 1, 10)"
 
-$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 1;"
-$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 5;"
+$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 1;"
+$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 5;"
 
-$CLICKHOUSE_CLIENT --query="ALTER TABLE test.minmax_idx UPDATE i64 = 5 WHERE i64 = 1;"
-wait_for_mutation "minmax_idx" "mutation_2.txt" "test"
+$CLICKHOUSE_CLIENT --query="ALTER TABLE minmax_idx UPDATE i64 = 5 WHERE i64 = 1;"
+wait_for_mutation "minmax_idx" "mutation_2.txt" "$CLICKHOUSE_DATABASE"
 
-$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 1;"
-$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 5;"
+$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 1;"
+$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 5;"
 
-$CLICKHOUSE_CLIENT --query="DROP TABLE test.minmax_idx"
+$CLICKHOUSE_CLIENT --query="DROP TABLE minmax_idx"
diff --git a/tests/queries/0_stateless/00943_materialize_index.sh b/tests/queries/0_stateless/00943_materialize_index.sh
index b406f3894eb..c3d5a6bb219 100755
--- a/tests/queries/0_stateless/00943_materialize_index.sh
+++ b/tests/queries/0_stateless/00943_materialize_index.sh
@@ -4,11 +4,11 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . $CURDIR/../shell_config.sh
 . $CURDIR/mergetree_mutations.lib
 
-$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.minmax_idx;"
+$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS minmax_idx;"
 
 
 $CLICKHOUSE_CLIENT -n --query="
-CREATE TABLE test.minmax_idx
+CREATE TABLE minmax_idx
 (
     u64 UInt64,
     i64 Int64,
@@ -19,7 +19,7 @@ ORDER BY u64
 SETTINGS index_granularity = 2;"
 
 
-$CLICKHOUSE_CLIENT --query="INSERT INTO test.minmax_idx VALUES
+$CLICKHOUSE_CLIENT --query="INSERT INTO minmax_idx VALUES
 (0, 2, 1),
 (1, 1, 1),
 (2, 1, 1),
@@ -31,35 +31,35 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO test.minmax_idx VALUES
 (8, 1, 2),
 (9, 1, 2)"
 
-$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2;"
-$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read"
+$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2;"
+$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read"
 
 $CLICKHOUSE_CLIENT -n --query="
-ALTER TABLE test.minmax_idx ADD INDEX idx (i64, u64 * i64) TYPE minmax GRANULARITY 1;"
+ALTER TABLE minmax_idx ADD INDEX idx (i64, u64 * i64) TYPE minmax GRANULARITY 1;"
 
-$CLICKHOUSE_CLIENT --query="ALTER TABLE test.minmax_idx MATERIALIZE INDEX idx IN PARTITION 1;"
-wait_for_mutation "minmax_idx" "mutation_3.txt" "test"
+$CLICKHOUSE_CLIENT --query="ALTER TABLE minmax_idx MATERIALIZE INDEX idx IN PARTITION 1;"
+wait_for_mutation "minmax_idx" "mutation_3.txt" "$CLICKHOUSE_DATABASE"
 
-$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2;"
-$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read"
+$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2;"
+$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read"
 
-$CLICKHOUSE_CLIENT --query="ALTER TABLE test.minmax_idx MATERIALIZE INDEX idx IN PARTITION 2;"
-wait_for_mutation "minmax_idx" "mutation_4.txt" "test"
+$CLICKHOUSE_CLIENT --query="ALTER TABLE minmax_idx MATERIALIZE INDEX idx IN PARTITION 2;"
+wait_for_mutation "minmax_idx" "mutation_4.txt" "$CLICKHOUSE_DATABASE"
 
-$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2;"
-$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read"
+$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2;"
+$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read"
 
-$CLICKHOUSE_CLIENT --query="ALTER TABLE test.minmax_idx CLEAR INDEX idx IN PARTITION 1;"
-$CLICKHOUSE_CLIENT --query="ALTER TABLE test.minmax_idx CLEAR INDEX idx IN PARTITION 2;"
+$CLICKHOUSE_CLIENT --query="ALTER TABLE minmax_idx CLEAR INDEX idx IN PARTITION 1;"
+$CLICKHOUSE_CLIENT --query="ALTER TABLE minmax_idx CLEAR INDEX idx IN PARTITION 2;"
 sleep 0.5
 
-$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2;"
-$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read"
+$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2;"
+$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read"
 
-$CLICKHOUSE_CLIENT --query="ALTER TABLE test.minmax_idx MATERIALIZE INDEX idx;"
-wait_for_mutation "minmax_idx" "mutation_5.txt" "test"
+$CLICKHOUSE_CLIENT --query="ALTER TABLE minmax_idx MATERIALIZE INDEX idx;"
+wait_for_mutation "minmax_idx" "mutation_5.txt" "$CLICKHOUSE_DATABASE"
 
-$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2;"
-$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read"
+$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2;"
+$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read"
 
-$CLICKHOUSE_CLIENT --query="DROP TABLE test.minmax_idx"
+$CLICKHOUSE_CLIENT --query="DROP TABLE minmax_idx"
diff --git a/tests/queries/0_stateless/00944_clear_index_in_partition.sh b/tests/queries/0_stateless/00944_clear_index_in_partition.sh
index 9f53ec16c35..d6ca4106a0e 100755
--- a/tests/queries/0_stateless/00944_clear_index_in_partition.sh
+++ b/tests/queries/0_stateless/00944_clear_index_in_partition.sh
@@ -4,11 +4,11 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . $CURDIR/../shell_config.sh
 . $CURDIR/mergetree_mutations.lib
 
-$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.minmax_idx;"
+$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS minmax_idx;"
 
 
 $CLICKHOUSE_CLIENT -n --query="
-CREATE TABLE test.minmax_idx
+CREATE TABLE minmax_idx
 (
     u64 UInt64,
     i64 Int64,
@@ -20,7 +20,7 @@ ORDER BY u64
 SETTINGS index_granularity = 2;"
 
 
-$CLICKHOUSE_CLIENT --query="INSERT INTO test.minmax_idx VALUES
+$CLICKHOUSE_CLIENT --query="INSERT INTO minmax_idx VALUES
 (0, 2, 1),
 (1, 1, 1),
 (2, 1, 1),
@@ -32,19 +32,19 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO test.minmax_idx VALUES
 (8, 1, 2),
 (9, 1, 2)"
 
-$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2;"
-$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read"
+$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2;"
+$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read"
 
-$CLICKHOUSE_CLIENT --query="ALTER TABLE test.minmax_idx CLEAR INDEX idx IN PARTITION 1;" --replication_alter_partitions_sync=2
+$CLICKHOUSE_CLIENT --query="ALTER TABLE minmax_idx CLEAR INDEX idx IN PARTITION 1;" --replication_alter_partitions_sync=2
 
-$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2;"
-$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read"
+$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2;"
+$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read"
 
-$CLICKHOUSE_CLIENT --query="ALTER TABLE test.minmax_idx MATERIALIZE INDEX idx IN PARTITION 1;"
-wait_for_mutation "minmax_idx" "mutation_3.txt" "test"
+$CLICKHOUSE_CLIENT --query="ALTER TABLE minmax_idx MATERIALIZE INDEX idx IN PARTITION 1;"
+wait_for_mutation "minmax_idx" "mutation_3.txt" "$CLICKHOUSE_DATABASE"
 
-$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2;"
-$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read"
+$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2;"
+$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read"
 
 
-$CLICKHOUSE_CLIENT --query="DROP TABLE test.minmax_idx"
+$CLICKHOUSE_CLIENT --query="DROP TABLE minmax_idx"
diff --git a/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh b/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh
index 28e7bd0b94e..b8b133cde3d 100755
--- a/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh
+++ b/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh
@@ -4,12 +4,12 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . $CURDIR/../shell_config.sh
 . $CURDIR/mergetree_mutations.lib
 
-$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.indices_mutaions1;"
-$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.indices_mutaions2;"
+$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS indices_mutaions1;"
+$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS indices_mutaions2;"
 
 
 $CLICKHOUSE_CLIENT -n --query="
-CREATE TABLE test.indices_mutaions1
+CREATE TABLE indices_mutaions1
 (
     u64 UInt64,
     i64 Int64,
@@ -19,7 +19,7 @@ CREATE TABLE test.indices_mutaions1
 PARTITION BY i32
 ORDER BY u64
 SETTINGS index_granularity = 2;
-CREATE TABLE test.indices_mutaions2
+CREATE TABLE indices_mutaions2
 (
     u64 UInt64,
     i64 Int64,
@@ -31,7 +31,7 @@ ORDER BY u64
 SETTINGS index_granularity = 2;"
 
 
-$CLICKHOUSE_CLIENT --query="INSERT INTO test.indices_mutaions1 VALUES
+$CLICKHOUSE_CLIENT --query="INSERT INTO indices_mutaions1 VALUES
 (0, 2, 1),
 (1, 1, 1),
 (2, 1, 1),
@@ -43,18 +43,18 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO test.indices_mutaions1 VALUES
 (8, 1, 2),
 (9, 1, 2)"
 
-$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2;"
-$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2 FORMAT JSON;" | grep "rows_read"
+$CLICKHOUSE_CLIENT --query="SELECT count() FROM indices_mutaions2 WHERE i64 = 2;"
+$CLICKHOUSE_CLIENT --query="SELECT count() FROM indices_mutaions2 WHERE i64 = 2 FORMAT JSON;" | grep "rows_read"
 
-$CLICKHOUSE_CLIENT --query="ALTER TABLE test.indices_mutaions1 CLEAR INDEX idx IN PARTITION 1;" --replication_alter_partitions_sync=2 --mutations_sync=2
+$CLICKHOUSE_CLIENT --query="ALTER TABLE indices_mutaions1 CLEAR INDEX idx IN PARTITION 1;" --replication_alter_partitions_sync=2 --mutations_sync=2
 
-$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2;"
-$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2 FORMAT JSON;" | grep "rows_read"
+$CLICKHOUSE_CLIENT --query="SELECT count() FROM indices_mutaions2 WHERE i64 = 2;"
+$CLICKHOUSE_CLIENT --query="SELECT count() FROM indices_mutaions2 WHERE i64 = 2 FORMAT JSON;" | grep "rows_read"
 
-$CLICKHOUSE_CLIENT --query="ALTER TABLE test.indices_mutaions1 MATERIALIZE INDEX idx IN PARTITION 1;" --replication_alter_partitions_sync=2 --mutations_sync=2
+$CLICKHOUSE_CLIENT --query="ALTER TABLE indices_mutaions1 MATERIALIZE INDEX idx IN PARTITION 1;" --replication_alter_partitions_sync=2 --mutations_sync=2
 
-$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2;"
-$CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2 FORMAT JSON;" | grep "rows_read"
+$CLICKHOUSE_CLIENT --query="SELECT count() FROM indices_mutaions2 WHERE i64 = 2;"
+$CLICKHOUSE_CLIENT --query="SELECT count() FROM indices_mutaions2 WHERE i64 = 2 FORMAT JSON;" | grep "rows_read"
 
-$CLICKHOUSE_CLIENT --query="DROP TABLE test.indices_mutaions1"
-$CLICKHOUSE_CLIENT --query="DROP TABLE test.indices_mutaions2"
+$CLICKHOUSE_CLIENT --query="DROP TABLE indices_mutaions1"
+$CLICKHOUSE_CLIENT --query="DROP TABLE indices_mutaions2"
diff --git a/tests/queries/0_stateless/01019_parallel_parsing_cancel.sh b/tests/queries/0_stateless/01019_parallel_parsing_cancel.sh
index 19d2b487955..067c19658ab 100755
--- a/tests/queries/0_stateless/01019_parallel_parsing_cancel.sh
+++ b/tests/queries/0_stateless/01019_parallel_parsing_cancel.sh
@@ -3,17 +3,17 @@
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . $CURDIR/../shell_config.sh
 
-$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.a;"
-$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.b;"
+$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS a;"
+$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS b;"
 
-$CLICKHOUSE_CLIENT --query="CREATE TABLE test.a (x UInt64) ENGINE = Memory;"
-$CLICKHOUSE_CLIENT --query="CREATE TABLE test.b (x UInt64) ENGINE = Memory;"
+$CLICKHOUSE_CLIENT --query="CREATE TABLE a (x UInt64) ENGINE = Memory;"
+$CLICKHOUSE_CLIENT --query="CREATE TABLE b (x UInt64) ENGINE = Memory;"
 
 function thread1()
 {
         for attempt_thread1 in {1..10}
         do
-                seq 1 500000 | $CLICKHOUSE_CLIENT --query_id=11 --query="INSERT INTO test.a FORMAT TSV" &
+                seq 1 500000 | $CLICKHOUSE_CLIENT --query_id=11 --query="INSERT INTO a FORMAT TSV" &
                 while true; do
                         $CLICKHOUSE_CLIENT --query="KILL QUERY WHERE query_id='11' SYNC" | grep -q "cant_cancel" && sleep .1 || break ||:
                 done
@@ -27,7 +27,7 @@ function thread2()
 {
         for attempt_thread2 in {1..10}
         do
-                seq 1 500000 | $CLICKHOUSE_CLIENT --query_id=22 --query="INSERT INTO test.b FORMAT TSV" &
+                seq 1 500000 | $CLICKHOUSE_CLIENT --query_id=22 --query="INSERT INTO b FORMAT TSV" &
                 while true; do
                         $CLICKHOUSE_CLIENT --query="KILL QUERY WHERE query_id='22' SYNC" | grep -q "cant_cancel" && sleep .1 || break ||:
                 done
@@ -46,5 +46,5 @@ bash -c thread2 > /dev/null 2>&1 &
 wait
 echo OK
 
-$CLICKHOUSE_CLIENT --query "DROP TABLE test.a"
-$CLICKHOUSE_CLIENT --query "DROP TABLE test.b"
+$CLICKHOUSE_CLIENT --query "DROP TABLE a"
+$CLICKHOUSE_CLIENT --query "DROP TABLE b"
diff --git a/tests/queries/0_stateless/01122_totals_rollup_having_block_header.sql b/tests/queries/0_stateless/01122_totals_rollup_having_block_header.sql
index 4f4f3355912..6fb877c350a 100644
--- a/tests/queries/0_stateless/01122_totals_rollup_having_block_header.sql
+++ b/tests/queries/0_stateless/01122_totals_rollup_having_block_header.sql
@@ -1,14 +1,14 @@
-DROP TABLE IF EXISTS test.rollup_having;
-CREATE TABLE test.rollup_having (
+DROP TABLE IF EXISTS rollup_having;
+CREATE TABLE rollup_having (
   a Nullable(String),
   b Nullable(String)
 ) ENGINE = Memory;
 
-INSERT INTO test.rollup_having VALUES (NULL, NULL);
-INSERT INTO test.rollup_having VALUES ('a', NULL);
-INSERT INTO test.rollup_having VALUES ('a', 'b');
+INSERT INTO rollup_having VALUES (NULL, NULL);
+INSERT INTO rollup_having VALUES ('a', NULL);
+INSERT INTO rollup_having VALUES ('a', 'b');
 
-SELECT a, b, count(*) FROM test.rollup_having GROUP BY a, b WITH ROLLUP WITH TOTALS HAVING a IS NOT NULL; -- { serverError 48 }
-SELECT a, b, count(*) FROM test.rollup_having GROUP BY a, b WITH ROLLUP WITH TOTALS HAVING a IS NOT NULL and b IS NOT NULL; -- { serverError 48 }
+SELECT a, b, count(*) FROM rollup_having GROUP BY a, b WITH ROLLUP WITH TOTALS HAVING a IS NOT NULL; -- { serverError 48 }
+SELECT a, b, count(*) FROM rollup_having GROUP BY a, b WITH ROLLUP WITH TOTALS HAVING a IS NOT NULL and b IS NOT NULL; -- { serverError 48 }
 
-DROP TABLE test.rollup_having;
+DROP TABLE rollup_having;
diff --git a/tests/queries/0_stateless/01124_view_bad_types.sql b/tests/queries/0_stateless/01124_view_bad_types.sql
index 81fc53930c1..715f7b375f1 100644
--- a/tests/queries/0_stateless/01124_view_bad_types.sql
+++ b/tests/queries/0_stateless/01124_view_bad_types.sql
@@ -1,11 +1,11 @@
-DROP TABLE IF EXISTS test.table;
-CREATE TABLE test.table (x UInt16) ENGINE = TinyLog;
-INSERT INTO test.table SELECT * FROM system.numbers LIMIT 10;
+DROP TABLE IF EXISTS source_table;
+CREATE TABLE source_table (x UInt16) ENGINE = TinyLog;
+INSERT INTO source_table SELECT * FROM system.numbers LIMIT 10;
 
-DROP TABLE IF EXISTS test.view;
-CREATE VIEW test.view (x UInt64) AS SELECT * FROM test.table;
+DROP TABLE IF EXISTS dest_view;
+CREATE VIEW dest_view (x UInt64) AS SELECT * FROM source_table;
 
-SELECT x, any(x) FROM test.view GROUP BY x ORDER BY x;
+SELECT x, any(x) FROM dest_view GROUP BY x ORDER BY x;
 
-DROP TABLE test.view;
-DROP TABLE test.table;
+DROP TABLE dest_view;
+DROP TABLE source_table;
diff --git a/tests/queries/1_stateful/00155_date_datetime_key_condition.reference b/tests/queries/0_stateless/01285_date_datetime_key_condition.reference
similarity index 100%
rename from tests/queries/1_stateful/00155_date_datetime_key_condition.reference
rename to tests/queries/0_stateless/01285_date_datetime_key_condition.reference
diff --git a/tests/queries/0_stateless/01285_date_datetime_key_condition.sql b/tests/queries/0_stateless/01285_date_datetime_key_condition.sql
new file mode 100644
index 00000000000..fe1454cd7c4
--- /dev/null
+++ b/tests/queries/0_stateless/01285_date_datetime_key_condition.sql
@@ -0,0 +1,22 @@
+DROP TABLE IF EXISTS date_datetime_key_condition;
+
+CREATE TABLE date_datetime_key_condition (dt DateTime) ENGINE = MergeTree() ORDER BY dt;
+INSERT INTO date_datetime_key_condition VALUES ('2020-01-01 00:00:00'), ('2020-01-01 10:00:00'), ('2020-01-02 00:00:00');
+
+-- partial
+SELECT groupArray(dt) from date_datetime_key_condition WHERE dt > toDate('2020-01-01') AND dt < toDate('2020-01-02');
+SELECT groupArray(dt) from date_datetime_key_condition WHERE dt >= toDate('2020-01-02');
+SELECT groupArray(dt) from date_datetime_key_condition WHERE dt < toDate('2020-01-02');
+
+-- inside
+SELECT groupArray(dt) from date_datetime_key_condition WHERE dt > toDate('2019-01-02');
+SELECT groupArray(dt) from date_datetime_key_condition WHERE dt < toDate('2021-01-02');
+SELECT groupArray(dt) from date_datetime_key_condition WHERE dt >= toDate('2019-01-02') AND dt < toDate('2021-01-02');
+SELECT groupArray(dt) from date_datetime_key_condition WHERE dt > toDate('2019-01-02') OR dt <= toDate('2021-01-02');
+
+-- outside
+SELECT groupArray(dt) from date_datetime_key_condition WHERE dt < toDate('2019-01-02');
+SELECT groupArray(dt) from date_datetime_key_condition WHERE dt > toDate('2021-01-02');
+SELECT groupArray(dt) from date_datetime_key_condition WHERE dt < toDate('2019-01-02') OR dt > toDate('2021-01-02');
+
+DROP TABLE date_datetime_key_condition;
diff --git a/tests/queries/1_stateful/00155_date_datetime_key_condition.sql b/tests/queries/1_stateful/00155_date_datetime_key_condition.sql
deleted file mode 100644
index cee1c7cfb46..00000000000
--- a/tests/queries/1_stateful/00155_date_datetime_key_condition.sql
+++ /dev/null
@@ -1,22 +0,0 @@
-DROP TABLE IF EXISTS test.date_datetime_key_condition;
-
-CREATE TABLE test.date_datetime_key_condition (dt DateTime) ENGINE = MergeTree() ORDER BY dt;
-INSERT INTO test.date_datetime_key_condition VALUES ('2020-01-01 00:00:00'), ('2020-01-01 10:00:00'), ('2020-01-02 00:00:00');
-
--- partial
-SELECT groupArray(dt) from test.date_datetime_key_condition WHERE dt > toDate('2020-01-01') AND dt < toDate('2020-01-02');
-SELECT groupArray(dt) from test.date_datetime_key_condition WHERE dt >= toDate('2020-01-02');
-SELECT groupArray(dt) from test.date_datetime_key_condition WHERE dt < toDate('2020-01-02');
-
--- inside
-SELECT groupArray(dt) from test.date_datetime_key_condition WHERE dt > toDate('2019-01-02');
-SELECT groupArray(dt) from test.date_datetime_key_condition WHERE dt < toDate('2021-01-02');
-SELECT groupArray(dt) from test.date_datetime_key_condition WHERE dt >= toDate('2019-01-02') AND dt < toDate('2021-01-02');
-SELECT groupArray(dt) from test.date_datetime_key_condition WHERE dt > toDate('2019-01-02') OR dt <= toDate('2021-01-02');
-
--- outside 
-SELECT groupArray(dt) from test.date_datetime_key_condition WHERE dt < toDate('2019-01-02');
-SELECT groupArray(dt) from test.date_datetime_key_condition WHERE dt > toDate('2021-01-02');
-SELECT groupArray(dt) from test.date_datetime_key_condition WHERE dt < toDate('2019-01-02') OR dt > toDate('2021-01-02');
-
-DROP TABLE test.date_datetime_key_condition;
\ No newline at end of file

From bbb57b77893a38f363707d1bdf9540214a839dd0 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Fri, 29 May 2020 13:31:19 +0300
Subject: [PATCH 160/184] Try fix tests.

---
 src/Interpreters/InterpreterSelectQuery.cpp | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp
index 9b161bc1883..56532e532d1 100644
--- a/src/Interpreters/InterpreterSelectQuery.cpp
+++ b/src/Interpreters/InterpreterSelectQuery.cpp
@@ -73,6 +73,7 @@
 #include <DataStreams/materializeBlock.h>
 #include <Processors/Pipe.h>
 #include <Processors/Sources/SourceFromSingleChunk.h>
+#include <Processors/Transforms/ConvertingTransform.h>
 
 
 namespace DB
@@ -443,6 +444,16 @@ BlockIO InterpreterSelectQuery::execute()
     executeImpl(res.pipeline, input, std::move(input_pipe));
     res.pipeline.addInterpreterContext(context);
     res.pipeline.addStorageHolder(storage);
+
+    /// We must guarantee that result structure is the same as in getSampleBlock()
+    if (!blocksHaveEqualStructure(res.pipeline.getHeader(), result_header))
+    {
+        res.pipeline.addSimpleTransform([&](const Block & header)
+        {
+            return std::make_shared<ConvertingTransform>(header, result_header, ConvertingTransform::MatchColumnsMode::Name);
+        });
+    }
+
     return res;
 }
 

From 2ff7bbb5115270864beadf2fce71e497858903a1 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Fri, 29 May 2020 13:56:04 +0300
Subject: [PATCH 161/184] Better names

---
 src/Storages/TTLDescription.cpp |  2 +-
 src/Storages/TTLDescription.h   | 13 +++++++------
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/Storages/TTLDescription.cpp b/src/Storages/TTLDescription.cpp
index 92fcf400dd3..da9691aab4a 100644
--- a/src/Storages/TTLDescription.cpp
+++ b/src/Storages/TTLDescription.cpp
@@ -180,7 +180,7 @@ TTLDescription TTLDescription::getTTLFromAST(
                 auto syntax_result = SyntaxAnalyzer(context).analyze(value, columns.getAllPhysical(), {}, true);
                 auto expr_analyzer = ExpressionAnalyzer(value, syntax_result, context);
 
-                result.set_parts.emplace_back(TTLSetPartDescription{
+                result.set_parts.emplace_back(TTLAggregateDescription{
                     name, value->getColumnName(), expr_analyzer.getActions(false)});
 
                 for (const auto & descr : expr_analyzer.getAnalyzedData().aggregate_descriptions)
diff --git a/src/Storages/TTLDescription.h b/src/Storages/TTLDescription.h
index d0e669ef4cf..99a145b8acc 100644
--- a/src/Storages/TTLDescription.h
+++ b/src/Storages/TTLDescription.h
@@ -10,23 +10,24 @@
 namespace DB
 {
 
-struct TTLSetPartDescription
+/// Assignment expression in TTL with GROUP BY
+struct TTLAggregateDescription
 {
-    /// Name of column in set part of ttl expression
+    /// Name of column in assignment
     /// x = sum(y)
     /// ^
     String column_name;
 
-    /// Name of column on the right hand of the set part of TTL expression
+    /// Name of column on the right hand of the assignment
     /// x = sum(y)
     ///    ^~~~~~^
     String expression_result_column_name;
 
-    /// Expressions to calculate the value of set expression
+    /// Expressions to calculate the value of assignment expression
     ExpressionActionsPtr expression;
 };
 
-using TTLSetPartDescriptions = std::vector<TTLSetPartDescription>;
+using TTLAggregateDescriptions = std::vector<TTLAggregateDescription>;
 
 /// Common struct for TTL record in storage
 struct TTLDescription
@@ -58,7 +59,7 @@ struct TTLDescription
     Names group_by_keys;
 
     /// SET parts of TTL expression
-    TTLSetPartDescriptions set_parts;
+    TTLAggregateDescriptions set_parts;
 
     /// Aggregate descriptions for GROUP BY in TTL
     AggregateDescriptions aggregate_descriptions;

From bd814d83de02abdb6b7bda79bde173cc54835f92 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Fri, 29 May 2020 16:04:03 +0300
Subject: [PATCH 162/184] Try fix tests.

---
 src/Interpreters/InterpreterSelectQuery.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp
index 56532e532d1..fb3188545f1 100644
--- a/src/Interpreters/InterpreterSelectQuery.cpp
+++ b/src/Interpreters/InterpreterSelectQuery.cpp
@@ -450,7 +450,7 @@ BlockIO InterpreterSelectQuery::execute()
     {
         res.pipeline.addSimpleTransform([&](const Block & header)
         {
-            return std::make_shared<ConvertingTransform>(header, result_header, ConvertingTransform::MatchColumnsMode::Name);
+            return std::make_shared<ConvertingTransform>(header, result_header, ConvertingTransform::MatchColumnsMode::Position);
         });
     }
 

From 080b2ec891dbe8de1fc0ff7ddfe99d465e0e7438 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 29 May 2020 17:04:16 +0300
Subject: [PATCH 163/184] Attempt to fix flacky test

---
 .../01108_restart_replicas_rename_deadlock.reference        | 4 ++++
 .../0_stateless/01108_restart_replicas_rename_deadlock.sh   | 6 ++++++
 2 files changed, 10 insertions(+)

diff --git a/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock.reference b/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock.reference
index 337ab7fe2e3..5e89d51914c 100644
--- a/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock.reference
+++ b/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock.reference
@@ -1 +1,5 @@
+replica_01108_1_tmp
+replica_01108_2_tmp
+replica_01108_3_tmp
+replica_01108_4_tmp
 1180	40
diff --git a/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock.sh b/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock.sh
index 01c9ef0a277..d338c8692a8 100755
--- a/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock.sh
+++ b/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock.sh
@@ -63,6 +63,12 @@ timeout $TIMEOUT bash -c restart_thread_2 2> /dev/null &
 wait
 sleep 3
 
+for i in `seq 4`; do
+    $CLICKHOUSE_CLIENT -q "SYSTEM SYNC REPLICA replica_01108_$i" >/dev/null 2>&1
+    $CLICKHOUSE_CLIENT -q "SYSTEM SYNC REPLICA replica_01108_${i}_tmp" >/dev/null 2>&1
+done
+
+$CLICKHOUSE_CLIENT -q "SHOW TABLES LIKE 'replica\\_01108\\_%'"
 $CLICKHOUSE_CLIENT -q "SELECT sum(n), count(n) FROM merge(currentDatabase(), '^replica_01108_') GROUP BY position(_table, 'tmp')"
 
 

From e940229e7124d4f0e9c7fcd1691480e12e0f1c6c Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 29 May 2020 17:09:28 +0300
Subject: [PATCH 164/184] Attempt to fix flacky test

---
 .../01108_restart_replicas_rename_deadlock.reference      | 8 ++++----
 .../0_stateless/01108_restart_replicas_rename_deadlock.sh | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock.reference b/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock.reference
index 5e89d51914c..89a55127234 100644
--- a/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock.reference
+++ b/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock.reference
@@ -1,5 +1,5 @@
-replica_01108_1_tmp
-replica_01108_2_tmp
-replica_01108_3_tmp
-replica_01108_4_tmp
+replica_01108_1
+replica_01108_2
+replica_01108_3
+replica_01108_4
 1180	40
diff --git a/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock.sh b/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock.sh
index d338c8692a8..65b738aed8e 100755
--- a/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock.sh
+++ b/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock.sh
@@ -53,7 +53,7 @@ export -f rename_thread_2;
 export -f restart_thread_1;
 export -f restart_thread_2;
 
-TIMEOUT=30
+TIMEOUT=10
 
 timeout $TIMEOUT bash -c rename_thread_1 2> /dev/null &
 timeout $TIMEOUT bash -c rename_thread_2 2> /dev/null &
@@ -68,7 +68,7 @@ for i in `seq 4`; do
     $CLICKHOUSE_CLIENT -q "SYSTEM SYNC REPLICA replica_01108_${i}_tmp" >/dev/null 2>&1
 done
 
-$CLICKHOUSE_CLIENT -q "SHOW TABLES LIKE 'replica\\_01108\\_%'"
+$CLICKHOUSE_CLIENT -q "SELECT replaceOne(name, '_tmp', '') FROM system.tables WHERE database = currentDatabase() AND match(name, '^replica_01108_')"
 $CLICKHOUSE_CLIENT -q "SELECT sum(n), count(n) FROM merge(currentDatabase(), '^replica_01108_') GROUP BY position(_table, 'tmp')"
 
 

From f7e0e87c7df29bbb8e9def7c0c2bfb6c459465cd Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Fri, 29 May 2020 19:35:09 +0300
Subject: [PATCH 165/184] Try fix tests.

---
 src/Interpreters/InterpreterSelectQuery.cpp       | 2 +-
 src/Processors/Transforms/ConvertingTransform.cpp | 6 +++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp
index fb3188545f1..56532e532d1 100644
--- a/src/Interpreters/InterpreterSelectQuery.cpp
+++ b/src/Interpreters/InterpreterSelectQuery.cpp
@@ -450,7 +450,7 @@ BlockIO InterpreterSelectQuery::execute()
     {
         res.pipeline.addSimpleTransform([&](const Block & header)
         {
-            return std::make_shared<ConvertingTransform>(header, result_header, ConvertingTransform::MatchColumnsMode::Position);
+            return std::make_shared<ConvertingTransform>(header, result_header, ConvertingTransform::MatchColumnsMode::Name);
         });
     }
 
diff --git a/src/Processors/Transforms/ConvertingTransform.cpp b/src/Processors/Transforms/ConvertingTransform.cpp
index c5bf5ad4d70..a0e9626b302 100644
--- a/src/Processors/Transforms/ConvertingTransform.cpp
+++ b/src/Processors/Transforms/ConvertingTransform.cpp
@@ -59,7 +59,11 @@ ConvertingTransform::ConvertingTransform(
                 break;
 
             case MatchColumnsMode::Name:
-                if (source.has(res_elem.name))
+                /// It may seem strange, but sometimes block may have columns with the same name.
+                /// For this specific case, try to get column from the same position if it has correct name first.
+                if (result_col_num < source.columns() && source.getByPosition(result_col_num).name == res_elem.name)
+                    conversion[result_col_num] = result_col_num;
+                else if (source.has(res_elem.name))
                     conversion[result_col_num] = source.getPositionByName(res_elem.name);
                 else
                     throw Exception("Cannot find column " + backQuoteIfNeed(res_elem.name) + " in source stream",

From a5f60e9cce5d27190b997e83866c78b736a35214 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <avtokmakov@yandex-team.ru>
Date: Fri, 29 May 2020 20:11:08 +0300
Subject: [PATCH 166/184] trigger sync check


From 36b1a3792795b60e49ed8be6fbe121b3c6c79d39 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Fri, 29 May 2020 21:23:11 +0300
Subject: [PATCH 167/184] Revert setting.

---
 src/Core/Settings.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index 142e0872d72..e07f624408f 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -437,6 +437,7 @@ struct Settings : public SettingsCollection<Settings>
     M(SettingUInt64, mark_cache_min_lifetime, 0, "Obsolete setting, does nothing. Will be removed after 2020-05-31", 0) \
     M(SettingBool, partial_merge_join, false, "Obsolete. Use join_algorithm='prefer_partial_merge' instead.", 0) \
     M(SettingUInt64, max_memory_usage_for_all_queries, 0, "Obsolete. Will be removed after 2020-10-20", 0) \
+    M(SettingBool, experimental_use_processors, true, "Obsolete setting, does nothing. Will be removed after 2020-11-29.", 0) \
 
     DECLARE_SETTINGS_COLLECTION(LIST_OF_SETTINGS)
 

From 69afc13d55d3db72c4a85e4b683e5994e0500213 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitbar@yandex-team.ru>
Date: Fri, 29 May 2020 23:09:01 +0300
Subject: [PATCH 168/184] Fix build scripts related to protobuf and gRPC for
 Mac.

---
 cmake/protobuf_generate_cpp.cmake | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cmake/protobuf_generate_cpp.cmake b/cmake/protobuf_generate_cpp.cmake
index 7ee9e8d7c81..cc2502e5eeb 100644
--- a/cmake/protobuf_generate_cpp.cmake
+++ b/cmake/protobuf_generate_cpp.cmake
@@ -55,6 +55,7 @@ function(protobuf_generate_cpp_impl SRCS HDRS MODES OUTPUT_FILE_EXTS PLUGIN)
     endif()
 
     set (intermediate_dir ${CMAKE_CURRENT_BINARY_DIR}/intermediate)
+    file (MAKE_DIRECTORY ${intermediate_dir})
 
     set (protoc_args)
     foreach (mode ${MODES})
@@ -112,7 +113,7 @@ if (PROTOBUF_GENERATE_CPP_SCRIPT_MODE)
     set (intermediate_dir ${DIR}/intermediate)
     set (intermediate_output "${intermediate_dir}/${FILENAME}")
 
-    if (COMPILER_ID STREQUAL "Clang")
+    if (COMPILER_ID MATCHES "Clang")
         set (pragma_push "#pragma clang diagnostic push\n")
         set (pragma_pop "#pragma clang diagnostic pop\n")
         set (pragma_disable_warnings "#pragma clang diagnostic ignored \"-Weverything\"\n")

From ff12b9c54677214e90e2bd32b7724350ec4a1d93 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Sat, 30 May 2020 00:00:48 +0300
Subject: [PATCH 169/184] Fix bad sh script in Dockerfile

---
 docker/test/stateful/Dockerfile  | 4 ++--
 docker/test/stateless/Dockerfile | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/docker/test/stateful/Dockerfile b/docker/test/stateful/Dockerfile
index e51efadf653..3aff49bf5a1 100644
--- a/docker/test/stateful/Dockerfile
+++ b/docker/test/stateful/Dockerfile
@@ -32,8 +32,8 @@ CMD dpkg -i package_folder/clickhouse-common-static_*.deb; \
     ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/; \
     ln -s /usr/share/clickhouse-test/config/macros.xml /etc/clickhouse-server/config.d/; \
     ln -s /usr/lib/llvm-9/bin/llvm-symbolizer /usr/bin/llvm-symbolizer; \
-    if [ -n $USE_DATABASE_ATOMIC ] && [ $USE_DATABASE_ATOMIC -eq 1 ]; then ln -s /usr/share/clickhouse-test/config/database_atomic_configd.xml /etc/clickhouse-server/config.d/; fi; \
-    if [ -n $USE_DATABASE_ATOMIC ] && [ $USE_DATABASE_ATOMIC -eq 1 ]; then ln -s /usr/share/clickhouse-test/config/database_atomic_usersd.xml /etc/clickhouse-server/users.d/; fi; \
+    if [[ -n "$USE_DATABASE_ATOMIC" ]] && [[ "$USE_DATABASE_ATOMIC" -eq 1 ]]; then ln -s /usr/share/clickhouse-test/config/database_atomic_configd.xml /etc/clickhouse-server/config.d/; fi; \
+    if [[ -n "$USE_DATABASE_ATOMIC" ]] && [[ "$USE_DATABASE_ATOMIC" -eq 1 ]]; then ln -s /usr/share/clickhouse-test/config/database_atomic_usersd.xml /etc/clickhouse-server/users.d/; fi; \
     echo "TSAN_OPTIONS='verbosity=1000 halt_on_error=1 history_size=7'" >> /etc/environment; \
     echo "TSAN_SYMBOLIZER_PATH=/usr/lib/llvm-8/bin/llvm-symbolizer" >> /etc/environment; \
     echo "UBSAN_OPTIONS='print_stacktrace=1'" >> /etc/environment; \
diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile
index 35a8a5a9d3d..41a53f8a3f5 100644
--- a/docker/test/stateless/Dockerfile
+++ b/docker/test/stateless/Dockerfile
@@ -78,9 +78,9 @@ CMD dpkg -i package_folder/clickhouse-common-static_*.deb; \
     ln -s /usr/share/clickhouse-test/config/server.key /etc/clickhouse-server/; \
     ln -s /usr/share/clickhouse-test/config/server.crt /etc/clickhouse-server/; \
     ln -s /usr/share/clickhouse-test/config/dhparam.pem /etc/clickhouse-server/; \
-    if [ -n $USE_POLYMORPHIC_PARTS ] && [ $USE_POLYMORPHIC_PARTS -eq 1 ]; then ln -s /usr/share/clickhouse-test/config/polymorphic_parts.xml /etc/clickhouse-server/config.d/; fi; \
-    if [ -n $USE_DATABASE_ATOMIC ] && [ $USE_DATABASE_ATOMIC -eq 1 ]; then ln -s /usr/share/clickhouse-test/config/database_atomic_configd.xml /etc/clickhouse-server/config.d/; fi; \
-    if [ -n $USE_DATABASE_ATOMIC ] && [ $USE_DATABASE_ATOMIC -eq 1 ]; then ln -s /usr/share/clickhouse-test/config/database_atomic_usersd.xml /etc/clickhouse-server/users.d/; fi; \
+    if [[ -n "$USE_POLYMORPHIC_PARTS" ]] && [[ "$USE_POLYMORPHIC_PARTS" -eq 1 ]]; then ln -s /usr/share/clickhouse-test/config/polymorphic_parts.xml /etc/clickhouse-server/config.d/; fi; \
+    if [[ -n "$USE_DATABASE_ATOMIC" ]] && [[ "$USE_DATABASE_ATOMIC" -eq 1 ]]; then ln -s /usr/share/clickhouse-test/config/database_atomic_configd.xml /etc/clickhouse-server/config.d/; fi; \
+    if [[ -n "$USE_DATABASE_ATOMIC" ]] && [[ "$USE_DATABASE_ATOMIC" -eq 1 ]]; then ln -s /usr/share/clickhouse-test/config/database_atomic_usersd.xml /etc/clickhouse-server/users.d/; fi; \
     ln -sf /usr/share/clickhouse-test/config/client_config.xml /etc/clickhouse-client/config.xml; \
     service zookeeper start; sleep 5; \
     service clickhouse-server start && sleep 5 && clickhouse-test --testname --shard --zookeeper $ADDITIONAL_OPTIONS $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt

From 8c8821475cb25d221780f2fc1bda240278fe860a Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Sat, 30 May 2020 00:32:35 +0300
Subject: [PATCH 170/184] Fix issue #11286; add a test

---
 src/Storages/MergeTree/KeyCondition.cpp       | 10 ++-
 ...01290_empty_array_index_analysis.reference | 50 ++++++++++++++
 .../01290_empty_array_index_analysis.sql      | 66 +++++++++++++++++++
 3 files changed, 124 insertions(+), 2 deletions(-)
 create mode 100644 tests/queries/0_stateless/01290_empty_array_index_analysis.reference
 create mode 100644 tests/queries/0_stateless/01290_empty_array_index_analysis.sql

diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp
index 6ae22885dfd..ee381709dd4 100644
--- a/src/Storages/MergeTree/KeyCondition.cpp
+++ b/src/Storages/MergeTree/KeyCondition.cpp
@@ -181,8 +181,11 @@ const KeyCondition::AtomMap KeyCondition::atom_map
     },
     {
         "empty",
-        [] (RPNElement & out, const Field &)
+        [] (RPNElement & out, const Field & value)
         {
+            if (value.getType() != Field::Types::String)
+                return false;
+
             out.function = RPNElement::FUNCTION_IN_RANGE;
             out.range = Range("");
             return true;
@@ -190,8 +193,11 @@ const KeyCondition::AtomMap KeyCondition::atom_map
     },
     {
         "notEmpty",
-        [] (RPNElement & out, const Field &)
+        [] (RPNElement & out, const Field & value)
         {
+            if (value.getType() != Field::Types::String)
+                return false;
+
             out.function = RPNElement::FUNCTION_NOT_IN_RANGE;
             out.range = Range("");
             return true;
diff --git a/tests/queries/0_stateless/01290_empty_array_index_analysis.reference b/tests/queries/0_stateless/01290_empty_array_index_analysis.reference
new file mode 100644
index 00000000000..5037a64c0f0
--- /dev/null
+++ b/tests/queries/0_stateless/01290_empty_array_index_analysis.reference
@@ -0,0 +1,50 @@
+--- notEmpty
+	['a']	2
+	['a','b','c']	3
+	['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa']	4
+--- empty
+	[]	1
+--- = []
+	[]	1
+--- != []
+	['a']	2
+	['a','b','c']	3
+	['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa']	4
+--- > []
+	['a']	2
+	['a','b','c']	3
+	['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa']	4
+--- < []
+--- >= []
+	[]	1
+	['a']	2
+	['a','b','c']	3
+	['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa']	4
+--- <= []
+	[]	1
+---
+--- notEmpty
+	['a']	2
+	['a','b','c']	3
+	['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa']	4
+--- empty
+	[]	1
+--- = []
+	[]	1
+--- != []
+	['a']	2
+	['a','b','c']	3
+	['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa']	4
+--- > []
+	['a']	2
+	['a','b','c']	3
+	['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa']	4
+--- < []
+--- >= []
+	[]	1
+	['a']	2
+	['a','b','c']	3
+	['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa']	4
+--- <= []
+	[]	1
+---
diff --git a/tests/queries/0_stateless/01290_empty_array_index_analysis.sql b/tests/queries/0_stateless/01290_empty_array_index_analysis.sql
new file mode 100644
index 00000000000..b1b6067945d
--- /dev/null
+++ b/tests/queries/0_stateless/01290_empty_array_index_analysis.sql
@@ -0,0 +1,66 @@
+drop table if exists count_lc_test;
+
+CREATE TABLE count_lc_test
+(
+    `s` LowCardinality(String),
+    `arr` Array(LowCardinality(String)),
+    `num` UInt64
+)
+ENGINE = MergeTree
+ORDER BY (s, arr);
+
+INSERT INTO count_lc_test(num, arr) VALUES (1,[]),(2,['a']),(3,['a','b','c']),(4,['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa']);
+
+SELECT '--- notEmpty';
+select * from count_lc_test where notEmpty(arr);
+SELECT '--- empty';
+select * from count_lc_test where empty(arr);
+SELECT '--- = []';
+select * from count_lc_test where arr = [];
+SELECT '--- != []';
+select * from count_lc_test where arr != [];
+SELECT '--- > []';
+select * from count_lc_test where arr > [];
+SELECT '--- < []';
+select * from count_lc_test where arr < [];
+SELECT '--- >= []';
+select * from count_lc_test where arr >= [];
+SELECT '--- <= []';
+select * from count_lc_test where arr <= [];
+SELECT '---';
+
+DROP TABLE count_lc_test;
+
+
+drop table if exists count_lc_test;
+
+CREATE TABLE count_lc_test
+(
+    `s` LowCardinality(String),
+    `arr` Array(String),
+    `num` UInt64
+)
+ENGINE = MergeTree
+ORDER BY (s, arr);
+
+INSERT INTO count_lc_test(num, arr) VALUES (1,[]),(2,['a']),(3,['a','b','c']),(4,['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa']);
+
+SELECT '--- notEmpty';
+select * from count_lc_test where notEmpty(arr);
+SELECT '--- empty';
+select * from count_lc_test where empty(arr);
+SELECT '--- = []';
+select * from count_lc_test where arr = [];
+SELECT '--- != []';
+select * from count_lc_test where arr != [];
+SELECT '--- > []';
+select * from count_lc_test where arr > [];
+SELECT '--- < []';
+select * from count_lc_test where arr < [];
+SELECT '--- >= []';
+select * from count_lc_test where arr >= [];
+SELECT '--- <= []';
+select * from count_lc_test where arr <= [];
+SELECT '---';
+
+DROP TABLE count_lc_test;

From b79020d0a81796e38ce3ac3dfa10eab4c8869430 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Sat, 30 May 2020 00:41:33 +0300
Subject: [PATCH 171/184] Emit a warning if server was build in debug or with
 sanitizers

---
 programs/server/Server.cpp | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp
index ce1d35e65d4..29096327a71 100644
--- a/programs/server/Server.cpp
+++ b/programs/server/Server.cpp
@@ -236,6 +236,14 @@ int Server::main(const std::vector<std::string> & /*args*/)
     if (ThreadFuzzer::instance().isEffective())
         LOG_WARNING(log, "ThreadFuzzer is enabled. Application will run slowly and unstable.");
 
+#if !defined(NDEBUG) || !defined(__OPTIMIZE__)
+    LOG_WARNING(log, "Server was built in debug mode. It will work slowly.");
+#endif
+
+#if defined(ADDRESS_SANITIZER) || defined(THREAD_SANITIZER) || defined(MEMORY_SANITIZER)
+    LOG_WARNING(log, "Server was built with sanitizer. It will work slowly.");
+#endif
+
     /** Context contains all that query execution is dependent:
       *  settings, available functions, data types, aggregate functions, databases...
       */

From b78c48310bb7bb3267cdc70acbc8ce2135f8fcc1 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Sat, 30 May 2020 01:37:44 +0300
Subject: [PATCH 172/184] Whitespaces

---
 src/IO/ReadHelpers.cpp                        | 1 -
 src/Storages/MergeTree/IMergeTreeDataPart.cpp | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/IO/ReadHelpers.cpp b/src/IO/ReadHelpers.cpp
index 0797e63cdd9..9d1d27611b5 100644
--- a/src/IO/ReadHelpers.cpp
+++ b/src/IO/ReadHelpers.cpp
@@ -650,7 +650,6 @@ void readCSVStringInto(Vector & s, ReadBuffer & buf, const FormatSettings::CSV &
                     ++next_pos;
             }();
 
-
             appendToStringOrVector(s, buf, next_pos);
             buf.position() = next_pos;
 
diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp
index 21768644940..287bf916c19 100644
--- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp
+++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp
@@ -775,6 +775,7 @@ void IMergeTreeDataPart::remove() const
     }
 }
 
+
 String IMergeTreeDataPart::getRelativePathForDetachedPart(const String & prefix) const
 {
     /// Do not allow underscores in the prefix because they are used as separators.

From ffdec23d08d011b9a0c38adfc1ba380495b30346 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Sat, 30 May 2020 01:43:25 +0300
Subject: [PATCH 173/184] Remove wrong code #8441

---
 src/Server/HTTPHandler.cpp | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp
index e5c784fd409..82068496159 100644
--- a/src/Server/HTTPHandler.cpp
+++ b/src/Server/HTTPHandler.cpp
@@ -404,8 +404,6 @@ void HTTPHandler::processQuery(
         used_output.out_maybe_delayed_and_compressed = used_output.out_maybe_compressed;
     }
 
-    std::unique_ptr<ReadBuffer> in_post_raw = std::make_unique<ReadBufferFromIStream>(istr);
-
     /// Request body can be compressed using algorithm specified in the Content-Encoding header.
     String http_request_compression_method_str = request.get("Content-Encoding", "");
     std::unique_ptr<ReadBuffer> in_post = wrapReadBufferWithCompressionMethod(

From 9a3d449da03fce35ffd857bdb9216fe658167f6d Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Sat, 30 May 2020 10:36:00 +0300
Subject: [PATCH 174/184] Fix copier.

---
 programs/copier/ClusterCopier.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/copier/ClusterCopier.cpp b/programs/copier/ClusterCopier.cpp
index d6a2fe103d1..600f110e857 100644
--- a/programs/copier/ClusterCopier.cpp
+++ b/programs/copier/ClusterCopier.cpp
@@ -1774,7 +1774,7 @@ bool ClusterCopier::checkPresentPartitionPiecesOnCurrentShard(const ConnectionTi
 
     Context local_context = context;
     local_context.setSettings(task_cluster->settings_pull);
-    auto result = InterpreterFactory::get(query_ast, local_context)->execute().in->read().rows();
+    auto result = InterpreterFactory::get(query_ast, local_context)->execute().getInputStream()->read().rows();
     if (result != 0)
         LOG_DEBUG(log, "Partition {} piece number {} is PRESENT on shard {}", partition_quoted_name, std::to_string(current_piece_number), task_shard.getDescription());
     else

From 96568566eed8e4560f749b6fcc7c22d498a9975e Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Sat, 30 May 2020 10:50:45 +0300
Subject: [PATCH 175/184] Fix copier.

---
 programs/copier/ClusterCopier.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/copier/ClusterCopier.cpp b/programs/copier/ClusterCopier.cpp
index 600f110e857..77ec1421391 100644
--- a/programs/copier/ClusterCopier.cpp
+++ b/programs/copier/ClusterCopier.cpp
@@ -1403,7 +1403,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl(
                 BlockIO io_select = InterpreterFactory::get(query_select_ast, context_select)->execute();
                 BlockIO io_insert = InterpreterFactory::get(query_insert_ast, context_insert)->execute();
 
-                input = io_select.in;
+                input = io_select.getInputStream();
                 output = io_insert.out;
             }
 

From 134f9f2b5bc8ca3cf67d87a52b9959b0f94aa9be Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Sat, 30 May 2020 11:07:12 +0300
Subject: [PATCH 176/184] Fix copier.

---
 programs/copier/ClusterCopier.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/programs/copier/ClusterCopier.cpp b/programs/copier/ClusterCopier.cpp
index 77ec1421391..227ce9eb715 100644
--- a/programs/copier/ClusterCopier.cpp
+++ b/programs/copier/ClusterCopier.cpp
@@ -1293,7 +1293,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl(
             local_context.setSettings(task_cluster->settings_pull);
             local_context.setSetting("skip_unavailable_shards", true);
 
-            Block block = getBlockWithAllStreamData(InterpreterFactory::get(query_select_ast, local_context)->execute().in);
+            Block block = getBlockWithAllStreamData(InterpreterFactory::get(query_select_ast, local_context)->execute().getInputStream());
             count = (block) ? block.safeGetByPosition(0).column->getUInt(0) : 0;
         }
 
@@ -1735,7 +1735,7 @@ const auto & settings = context.getSettingsRef();
 
     Context local_context = context;
     local_context.setSettings(task_cluster->settings_pull);
-    return InterpreterFactory::get(query_ast, local_context)->execute().in->read().rows() != 0;
+    return InterpreterFactory::get(query_ast, local_context)->execute().getInputStream()->read().rows() != 0;
 }
 
 bool ClusterCopier::checkPresentPartitionPiecesOnCurrentShard(const ConnectionTimeouts & timeouts,

From c43976b6c581d699c5caf9632f53e77e84600399 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Sat, 30 May 2020 11:31:07 +0300
Subject: [PATCH 177/184] Update test.

---
 tests/integration/test_ttl_move/test.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/integration/test_ttl_move/test.py b/tests/integration/test_ttl_move/test.py
index c6453beea6a..fc5c6725148 100644
--- a/tests/integration/test_ttl_move/test.py
+++ b/tests/integration/test_ttl_move/test.py
@@ -864,6 +864,8 @@ def test_double_move_while_select(started_cluster, name, positive):
         thread = threading.Thread(target=long_select)
         thread.start()
 
+        time.sleep(1)
+
         node1.query("ALTER TABLE {name} MOVE PART '{part}' TO DISK 'jbod1'".format(name=name, part=parts[0]))
 
         # Fill jbod1 to force ClickHouse to make move of partition 1 to external.

From 808351048f89d75f59ed23f743de3e7356e67415 Mon Sep 17 00:00:00 2001
From: Denis Glazachev <d.glazachev@gmail.com>
Date: Sat, 30 May 2020 19:44:51 +0400
Subject: [PATCH 178/184] Add libsasl2-dev and heimdal-multidev required by
 static system openldap libraries

---
 docker/packager/deb/Dockerfile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docker/packager/deb/Dockerfile b/docker/packager/deb/Dockerfile
index ae70bc8c594..786e6620eac 100644
--- a/docker/packager/deb/Dockerfile
+++ b/docker/packager/deb/Dockerfile
@@ -80,7 +80,9 @@ RUN apt-get --allow-unauthenticated update -y \
             pigz \
             moreutils \
             libcctz-dev \
-            libldap2-dev
+            libldap2-dev \
+            libsasl2-dev \
+            heimdal-multidev
 
 
 

From 6468d7af8c6d54a7bcf0cfd4ad481d11446222d6 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Sat, 30 May 2020 19:20:19 +0300
Subject: [PATCH 179/184] Regenerate ya.make

---
 src/Dictionaries/ya.make | 2 +-
 src/Functions/ya.make    | 9 ++++-----
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/src/Dictionaries/ya.make b/src/Dictionaries/ya.make
index e47b55d5254..e30a22a372c 100644
--- a/src/Dictionaries/ya.make
+++ b/src/Dictionaries/ya.make
@@ -24,8 +24,8 @@ SRCS(
     ComplexKeyCacheDictionary_generate3.cpp
     ComplexKeyCacheDictionary_setAttributeValue.cpp
     ComplexKeyCacheDictionary_setDefaultAttributeValue.cpp
-    ComplexKeyHashedDictionary.cpp
     ComplexKeyDirectDictionary.cpp
+    ComplexKeyHashedDictionary.cpp
     DictionaryBlockInputStreamBase.cpp
     DictionaryFactory.cpp
     DictionarySourceFactory.cpp
diff --git a/src/Functions/ya.make b/src/Functions/ya.make
index 0b4776853e9..bbbf13d360b 100644
--- a/src/Functions/ya.make
+++ b/src/Functions/ya.make
@@ -171,7 +171,6 @@ SRCS(
     FunctionsRound.cpp
     FunctionsStringArray.cpp
     FunctionsStringSimilarity.cpp
-    FunctionUnixTimestamp64.h
     GatherUtils/concat.cpp
     GatherUtils/createArraySink.cpp
     GatherUtils/createArraySource.cpp
@@ -285,10 +284,10 @@ SRCS(
     rand64.cpp
     randConstant.cpp
     rand.cpp
+    randomFixedString.cpp
     randomPrintableASCII.cpp
     randomString.cpp
     randomStringUTF8.cpp
-    randomFixedString.cpp
     regexpQuoteMeta.cpp
     registerFunctionsArithmetic.cpp
     registerFunctionsComparison.cpp
@@ -308,8 +307,8 @@ SRCS(
     registerFunctionsStringRegexp.cpp
     registerFunctionsStringSearch.cpp
     registerFunctionsTuple.cpp
-    registerFunctionsVisitParam.cpp
     registerFunctionsUnixTimestamp64.cpp
+    registerFunctionsVisitParam.cpp
     reinterpretAsFixedString.cpp
     reinterpretAsString.cpp
     reinterpretStringAs.cpp
@@ -390,10 +389,10 @@ SRCS(
     toTime.cpp
     toTimeZone.cpp
     toTypeName.cpp
-    toValidUTF8.cpp
     toUnixTimestamp64Micro.cpp
     toUnixTimestamp64Milli.cpp
     toUnixTimestamp64Nano.cpp
+    toValidUTF8.cpp
     toYear.cpp
     toYYYYMM.cpp
     toYYYYMMDD.cpp
@@ -424,8 +423,8 @@ SRCS(
     URL/fragment.cpp
     URL/path.cpp
     URL/pathFull.cpp
-    URL/protocol.cpp
     URL/port.cpp
+    URL/protocol.cpp
     URL/queryStringAndFragment.cpp
     URL/queryString.cpp
     URL/registerFunctionsURL.cpp

From a5770fe041548d36b95b45802a543c1bda7e62af Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Sat, 30 May 2020 21:05:05 +0300
Subject: [PATCH 180/184] Update PullingPipelineExecutor.cpp

---
 src/Processors/Executors/PullingPipelineExecutor.cpp | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/src/Processors/Executors/PullingPipelineExecutor.cpp b/src/Processors/Executors/PullingPipelineExecutor.cpp
index 18722dfdc2f..dc59e0a2f5a 100644
--- a/src/Processors/Executors/PullingPipelineExecutor.cpp
+++ b/src/Processors/Executors/PullingPipelineExecutor.cpp
@@ -74,10 +74,6 @@ void PullingPipelineExecutor::cancel()
     /// Cancel execution if it wasn't finished.
     if (executor)
         executor->cancel();
-
-    /// Read all data and finish execution.
-    /// Chunk chunk;
-    /// while (pull(chunk));
 }
 
 Chunk PullingPipelineExecutor::getTotals()

From 24a81f3d139dcd7a7dbc6a6af84cdddf00fd7c34 Mon Sep 17 00:00:00 2001
From: Ivan Blinkov <github@blinkov.ru>
Date: Sun, 31 May 2020 11:23:25 +0300
Subject: [PATCH 181/184] Update common_meta.html

---
 website/templates/common_meta.html | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/website/templates/common_meta.html b/website/templates/common_meta.html
index 8b65e67d780..212e0b089d5 100644
--- a/website/templates/common_meta.html
+++ b/website/templates/common_meta.html
@@ -22,6 +22,9 @@
 <meta name="description" content="{{ description }}" />
 <meta name="keywords"
       content="ClickHouse, DBMS, OLAP, SQL, {{ _('open-source') }}, {{ _('relational') }}, {{ _('analytics') }}, {{ _('analytical') }}, {{ _('Big Data') }}, {{ _('web-analytics') }}" />
+{% if config.extra.single_page or config.extra.version_prefix %}
+<meta name="robots" content="noindex,follow">
+{% endif %}
 
 {% if config and page %}
     {% for code, name in config.extra.languages.items() %}

From dc6638838dc664f3ed34ab6dd7ffd997e026fe3c Mon Sep 17 00:00:00 2001
From: Ivan Blinkov <github@blinkov.ru>
Date: Sun, 31 May 2020 11:23:41 +0300
Subject: [PATCH 182/184] Update common_meta.html

---
 website/templates/common_meta.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/website/templates/common_meta.html b/website/templates/common_meta.html
index 212e0b089d5..9eda30a14d9 100644
--- a/website/templates/common_meta.html
+++ b/website/templates/common_meta.html
@@ -23,7 +23,7 @@
 <meta name="keywords"
       content="ClickHouse, DBMS, OLAP, SQL, {{ _('open-source') }}, {{ _('relational') }}, {{ _('analytics') }}, {{ _('analytical') }}, {{ _('Big Data') }}, {{ _('web-analytics') }}" />
 {% if config.extra.single_page or config.extra.version_prefix %}
-<meta name="robots" content="noindex,follow">
+<meta name="robots" content="noindex,follow" />
 {% endif %}
 
 {% if config and page %}

From 4b4d3b559271b30dcced9f7cc1a1d9e74839b56c Mon Sep 17 00:00:00 2001
From: Ivan Blinkov <github@blinkov.ru>
Date: Sun, 31 May 2020 11:25:28 +0300
Subject: [PATCH 183/184] Update amp.html

---
 website/templates/docs/amp.html | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/website/templates/docs/amp.html b/website/templates/docs/amp.html
index a60270d23d0..0cff614035f 100644
--- a/website/templates/docs/amp.html
+++ b/website/templates/docs/amp.html
@@ -7,6 +7,9 @@
     <link rel="canonical" href="{{ canonical_url or 'https://clickhouse.tech/' }}" />
     <meta name="viewport" content="width=device-width,minimum-scale=1,initial-scale=1">
     {% include "templates/docs/ld_json.html" %}
+    {% if config.extra.single_page or config.extra.version_prefix %}
+      <meta name="robots" content="noindex,follow" />
+    {% endif %}
     <style amp-boilerplate>body{-webkit-animation:-amp-start 8s steps(1,end) 0s 1 normal both;-moz-animation:-amp-start 8s steps(1,end) 0s 1 normal both;-ms-animation:-amp-start 8s steps(1,end) 0s 1 normal both;animation:-amp-start 8s steps(1,end) 0s 1 normal both}@-webkit-keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}@-moz-keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}@-ms-keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}@-o-keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}@keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}</style><noscript><style amp-boilerplate>body{-webkit-animation:none;-moz-animation:none;-ms-animation:none;animation:none}</style></noscript>
     <style amp-custom>CUSTOM_CSS_PLACEHOLDER</style>
     <script async custom-element="amp-analytics" src="https://cdn.ampproject.org/v0/amp-analytics-0.1.js"></script>

From 154d045c1b0ec0a2ac4d490e75630f7edb0fffe0 Mon Sep 17 00:00:00 2001
From: Ivan Blinkov <github@blinkov.ru>
Date: Sun, 31 May 2020 11:57:04 +0300
Subject: [PATCH 184/184] Update common_meta.html

---
 website/templates/common_meta.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/website/templates/common_meta.html b/website/templates/common_meta.html
index 9eda30a14d9..84bd93d5175 100644
--- a/website/templates/common_meta.html
+++ b/website/templates/common_meta.html
@@ -22,7 +22,7 @@
 <meta name="description" content="{{ description }}" />
 <meta name="keywords"
       content="ClickHouse, DBMS, OLAP, SQL, {{ _('open-source') }}, {{ _('relational') }}, {{ _('analytics') }}, {{ _('analytical') }}, {{ _('Big Data') }}, {{ _('web-analytics') }}" />
-{% if config.extra.single_page or config.extra.version_prefix %}
+{% if config and (config.extra.single_page or config.extra.version_prefix) %}
 <meta name="robots" content="noindex,follow" />
 {% endif %}