From 857bccc9813b02a80bdf319fbe05bd59ac79d8b0 Mon Sep 17 00:00:00 2001
From: Guillaume Tassery <tassery.guillaume@gmail.com>
Date: Mon, 15 Apr 2019 13:36:24 +0700
Subject: [PATCH 001/118] Add a way to set a size parameter for groupUniqArray

---
 .../AggregateFunctionGroupUniqArray.cpp       | 84 +++++++++++++++----
 .../AggregateFunctionGroupUniqArray.h         | 29 +++++--
 2 files changed, 86 insertions(+), 27 deletions(-)

diff --git a/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.cpp b/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.cpp
index 7a99709c33e..13c9b328938 100644
--- a/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.cpp
+++ b/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.cpp
@@ -9,55 +9,103 @@
 namespace DB
 {
 
+namespace ErrorCodes
+{
+    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+    extern const int BAD_ARGUMENTS;
+}
+
 namespace
 {
 
 /// Substitute return type for Date and DateTime
-class AggregateFunctionGroupUniqArrayDate : public AggregateFunctionGroupUniqArray<DataTypeDate::FieldType>
+template <typename has_limit>
+class AggregateFunctionGroupUniqArrayDate : public AggregateFunctionGroupUniqArray<DataTypeDate::FieldType, has_limit>
 {
 public:
-    AggregateFunctionGroupUniqArrayDate(const DataTypePtr & argument_type) : AggregateFunctionGroupUniqArray<DataTypeDate::FieldType>(argument_type) {}
+    AggregateFunctionGroupUniqArrayDate(const DataTypePtr & argument_type, UInt64 max_elems_ = std::numeric_limits<UInt64>::max()) : AggregateFunctionGroupUniqArray<DataTypeDate::FieldType, has_limit>(argument_type, max_elems_) {}
     DataTypePtr getReturnType() const override { return std::make_shared<DataTypeArray>(std::make_shared<DataTypeDate>()); }
 };
 
-class AggregateFunctionGroupUniqArrayDateTime : public AggregateFunctionGroupUniqArray<DataTypeDateTime::FieldType>
+template <typename has_limit>
+class AggregateFunctionGroupUniqArrayDateTime : public AggregateFunctionGroupUniqArray<DataTypeDateTime::FieldType, has_limit>
 {
 public:
-    AggregateFunctionGroupUniqArrayDateTime(const DataTypePtr & argument_type) : AggregateFunctionGroupUniqArray<DataTypeDateTime::FieldType>(argument_type) {}
+    AggregateFunctionGroupUniqArrayDateTime(const DataTypePtr & argument_type, UInt64 max_elems_ = std::numeric_limits<UInt64>::max()) : AggregateFunctionGroupUniqArray<DataTypeDateTime::FieldType, has_limit>(argument_type, max_elems_) {}
     DataTypePtr getReturnType() const override { return std::make_shared<DataTypeArray>(std::make_shared<DataTypeDateTime>()); }
 };
 
-
-static IAggregateFunction * createWithExtraTypes(const DataTypePtr & argument_type)
+template <typename has_limit, typename ... TArgs>
+static IAggregateFunction * createWithExtraTypes(const DataTypePtr & argument_type, TArgs && ... args)
 {
     WhichDataType which(argument_type);
-         if (which.idx == TypeIndex::Date) return new AggregateFunctionGroupUniqArrayDate(argument_type);
-    else if (which.idx == TypeIndex::DateTime) return new AggregateFunctionGroupUniqArrayDateTime(argument_type);
+    if (which.idx == TypeIndex::Date) return new AggregateFunctionGroupUniqArrayDate<has_limit>(argument_type, std::forward<TArgs>(args)...);
+    else if (which.idx == TypeIndex::DateTime) return new AggregateFunctionGroupUniqArrayDateTime<has_limit>(argument_type, std::forward<TArgs>(args)...);
     else
     {
         /// Check that we can use plain version of AggreagteFunctionGroupUniqArrayGeneric
         if (argument_type->isValueUnambiguouslyRepresentedInContiguousMemoryRegion())
-            return new AggreagteFunctionGroupUniqArrayGeneric<true>(argument_type);
+            return new AggreagteFunctionGroupUniqArrayGeneric<true, has_limit>(argument_type, std::forward<TArgs>(args)...);
         else
-            return new AggreagteFunctionGroupUniqArrayGeneric<false>(argument_type);
+            return new AggreagteFunctionGroupUniqArrayGeneric<false, has_limit>(argument_type, std::forward<TArgs>(args)...);
     }
 }
 
+template <template <typename, typename> class AggregateFunctionTemplate, typename Data, typename ... TArgs>
+static IAggregateFunction * createWithNumericOrTimeType(const IDataType & argument_type, TArgs && ... args)
+{
+    return createWithNumericType<AggregateFunctionTemplate, Data, TArgs...>(argument_type, std::forward<TArgs>(args)...);
+}
+
+template <typename has_limit, typename ... TArgs>
+inline AggregateFunctionPtr createAggregateFunctionGroupUniqArrayImpl(const std::string & name, const DataTypePtr & argument_type, TArgs ... args)
+{
+
+    AggregateFunctionPtr res(createWithNumericOrTimeType<AggregateFunctionGroupUniqArray, has_limit>(*argument_type, argument_type, std::forward<TArgs>(args)...));
+
+    if (!res)
+        res = AggregateFunctionPtr(createWithExtraTypes<has_limit>(argument_type, std::forward<TArgs>(args)...));
+
+    if (!res)
+        throw Exception("Illegal type " + argument_type->getName() +
+                        " of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+
+    return res;
+
+}
+
 AggregateFunctionPtr createAggregateFunctionGroupUniqArray(const std::string & name, const DataTypes & argument_types, const Array & parameters)
 {
-    assertNoParameters(name, parameters);
     assertUnary(name, argument_types);
 
-    AggregateFunctionPtr res(createWithNumericType<AggregateFunctionGroupUniqArray>(*argument_types[0], argument_types[0]));
+    bool limit_size = false;
+    UInt64 max_elems = std::numeric_limits<UInt64>::max();
 
-    if (!res)
-        res = AggregateFunctionPtr(createWithExtraTypes(argument_types[0]));
+    if (parameters.empty())
+    {
+        // no limit
+    }
+    else if (parameters.size() == 1)
+    {
+        auto type = parameters[0].getType();
+        if (type != Field::Types::Int64 && type != Field::Types::UInt64)
+            throw Exception("Parameter for aggregate function " + name + " should be positive number", ErrorCodes::BAD_ARGUMENTS);
 
-    if (!res)
-        throw Exception("Illegal type " + argument_types[0]->getName() +
-            " of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+        if ((type == Field::Types::Int64 && parameters[0].get<Int64>() < 0) ||
+            (type == Field::Types::UInt64 && parameters[0].get<UInt64>() == 0))
+            throw Exception("Parameter for aggregate function " + name + " should be positive number", ErrorCodes::BAD_ARGUMENTS);
 
-    return res;
+        limit_size = true;
+        max_elems = parameters[0].get<UInt64>();
+    }
+    else
+        throw Exception("Incorrect number of parameters for aggregate function " + name + ", should be 0 or 1",
+                        ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
+
+    if (!limit_size)
+        return createAggregateFunctionGroupUniqArrayImpl<std::false_type>(name, argument_types[0]);
+    else
+        return createAggregateFunctionGroupUniqArrayImpl<std::true_type>(name, argument_types[0], max_elems);
 }
 
 }
diff --git a/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h b/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h
index e51feedc53c..13b22001d1f 100644
--- a/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h
+++ b/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h
@@ -36,16 +36,21 @@ struct AggregateFunctionGroupUniqArrayData
 
 
 /// Puts all values to the hash set. Returns an array of unique values. Implemented for numeric types.
-template <typename T>
+template <typename T, typename Tlimit_num_elem>
 class AggregateFunctionGroupUniqArray
-    : public IAggregateFunctionDataHelper<AggregateFunctionGroupUniqArrayData<T>, AggregateFunctionGroupUniqArray<T>>
+    : public IAggregateFunctionDataHelper<AggregateFunctionGroupUniqArrayData<T>, AggregateFunctionGroupUniqArray<T, Tlimit_num_elem>>
 {
+    static constexpr bool limit_num_elems = Tlimit_num_elem::value;
+    UInt64 max_elems;
+
 private:
     using State = AggregateFunctionGroupUniqArrayData<T>;
 
 public:
-    AggregateFunctionGroupUniqArray(const DataTypePtr & argument_type)
-        : IAggregateFunctionDataHelper<AggregateFunctionGroupUniqArrayData<T>, AggregateFunctionGroupUniqArray<T>>({argument_type}, {}) {}
+    AggregateFunctionGroupUniqArray(const DataTypePtr & argument_type, UInt64 max_elems_ = std::numeric_limits<UInt64>::max())
+        : IAggregateFunctionDataHelper<AggregateFunctionGroupUniqArrayData<T>,
+          AggregateFunctionGroupUniqArray<T, Tlimit_num_elem>>({argument_type}, {}),
+          max_elems(max_elems_) {}
 
     String getName() const override { return "groupUniqArray"; }
 
@@ -56,6 +61,8 @@ public:
 
     void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override
     {
+        if (limit_num_elems && this->data(place).value.size() >= max_elems)
+            return;
         this->data(place).value.insert(static_cast<const ColumnVector<T> &>(*columns[0]).getData()[row_num]);
     }
 
@@ -114,12 +121,15 @@ struct AggreagteFunctionGroupUniqArrayGenericData
 /** Template parameter with true value should be used for columns that store their elements in memory continuously.
  *  For such columns groupUniqArray() can be implemented more efficiently (especially for small numeric arrays).
  */
-template <bool is_plain_column = false>
+template <bool is_plain_column = false, typename Tlimit_num_elem = std::false_type>
 class AggreagteFunctionGroupUniqArrayGeneric
-    : public IAggregateFunctionDataHelper<AggreagteFunctionGroupUniqArrayGenericData, AggreagteFunctionGroupUniqArrayGeneric<is_plain_column>>
+    : public IAggregateFunctionDataHelper<AggreagteFunctionGroupUniqArrayGenericData, AggreagteFunctionGroupUniqArrayGeneric<is_plain_column, Tlimit_num_elem>>
 {
     DataTypePtr & input_data_type;
 
+    static constexpr bool limit_num_elems = Tlimit_num_elem::value;
+    UInt64 max_elems;
+
     using State = AggreagteFunctionGroupUniqArrayGenericData;
 
     static StringRef getSerialization(const IColumn & column, size_t row_num, Arena & arena);
@@ -127,9 +137,10 @@ class AggreagteFunctionGroupUniqArrayGeneric
     static void deserializeAndInsert(StringRef str, IColumn & data_to);
 
 public:
-    AggreagteFunctionGroupUniqArrayGeneric(const DataTypePtr & input_data_type)
-        : IAggregateFunctionDataHelper<AggreagteFunctionGroupUniqArrayGenericData, AggreagteFunctionGroupUniqArrayGeneric<is_plain_column>>({input_data_type}, {})
-        , input_data_type(this->argument_types[0]) {}
+    AggreagteFunctionGroupUniqArrayGeneric(const DataTypePtr & input_data_type, UInt64 max_elems_ = std::numeric_limits<UInt64>::max())
+        : IAggregateFunctionDataHelper<AggreagteFunctionGroupUniqArrayGenericData, AggreagteFunctionGroupUniqArrayGeneric<is_plain_column, Tlimit_num_elem>>({input_data_type}, {})
+        , input_data_type(this->argument_types[0])
+        , max_elems(max_elems_) {}
 
     String getName() const override { return "groupUniqArray"; }
 

From 803675892b46363fac6bff54e556e45c880d1efd Mon Sep 17 00:00:00 2001
From: Guillaume Tassery <tassery.guillaume@gmail.com>
Date: Mon, 15 Apr 2019 16:09:52 +0700
Subject: [PATCH 002/118] Handle merge operation for size parameter on
 groupUniqArray aggregation function

---
 .../AggregateFunctionGroupUniqArray.h         | 53 +++++++++++++++++--
 1 file changed, 48 insertions(+), 5 deletions(-)

diff --git a/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h b/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h
index 13b22001d1f..bba64b4d786 100644
--- a/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h
+++ b/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h
@@ -68,7 +68,20 @@ public:
 
     void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override
     {
-        this->data(place).value.merge(this->data(rhs).value);
+        if (!limit_num_elems)
+            this->data(place).value.merge(this->data(rhs).value);
+        else
+        {
+            auto & cur_set = this->data(place).value;
+            auto & rhs_set = this->data(rhs).value;
+
+            for (auto & rhs_elem : rhs_set)
+            {
+                if (cur_set.size() >= max_elems)
+                    return ;
+                cur_set.insert(rhs_elem.getValue());
+            }
+        }
     }
 
     void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override
@@ -185,7 +198,10 @@ public:
         bool inserted;
         State::Set::iterator it;
 
+        if (limit_num_elems && set.size() >= max_elems)
+            return;
         StringRef str_serialized = getSerialization(*columns[0], row_num, *arena);
+
         set.emplace(str_serialized, it, inserted);
 
         if constexpr (!is_plain_column)
@@ -209,6 +225,8 @@ public:
         State::Set::iterator it;
         for (auto & rhs_elem : rhs_set)
         {
+            if (limit_num_elems && cur_set.size() >= max_elems)
+                return ;
             cur_set.emplace(rhs_elem.getValue(), it, inserted);
             if (inserted)
             {
@@ -240,30 +258,55 @@ public:
 
 
 template <>
-inline StringRef AggreagteFunctionGroupUniqArrayGeneric<false>::getSerialization(const IColumn & column, size_t row_num, Arena & arena)
+inline StringRef AggreagteFunctionGroupUniqArrayGeneric<false, std::false_type>::getSerialization(const IColumn & column, size_t row_num, Arena & arena)
 {
     const char * begin = nullptr;
     return column.serializeValueIntoArena(row_num, arena, begin);
 }
 
 template <>
-inline StringRef AggreagteFunctionGroupUniqArrayGeneric<true>::getSerialization(const IColumn & column, size_t row_num, Arena &)
+inline StringRef AggreagteFunctionGroupUniqArrayGeneric<false, std::true_type>::getSerialization(const IColumn & column, size_t row_num, Arena & arena)
+{
+    const char * begin = nullptr;
+    return column.serializeValueIntoArena(row_num, arena, begin);
+}
+
+template <>
+inline StringRef AggreagteFunctionGroupUniqArrayGeneric<true, std::false_type>::getSerialization(const IColumn & column, size_t row_num, Arena &)
 {
     return column.getDataAt(row_num);
 }
 
 template <>
-inline void AggreagteFunctionGroupUniqArrayGeneric<false>::deserializeAndInsert(StringRef str, IColumn & data_to)
+inline StringRef AggreagteFunctionGroupUniqArrayGeneric<true, std::true_type>::getSerialization(const IColumn & column, size_t row_num, Arena &)
+{
+    return column.getDataAt(row_num);
+}
+
+
+template <>
+inline void AggreagteFunctionGroupUniqArrayGeneric<false, std::false_type>::deserializeAndInsert(StringRef str, IColumn & data_to)
 {
     data_to.deserializeAndInsertFromArena(str.data);
 }
 
 template <>
-inline void AggreagteFunctionGroupUniqArrayGeneric<true>::deserializeAndInsert(StringRef str, IColumn & data_to)
+inline void AggreagteFunctionGroupUniqArrayGeneric<false, std::true_type>::deserializeAndInsert(StringRef str, IColumn & data_to)
+{
+    data_to.deserializeAndInsertFromArena(str.data);
+}
+
+template <>
+inline void AggreagteFunctionGroupUniqArrayGeneric<true, std::true_type>::deserializeAndInsert(StringRef str, IColumn & data_to)
 {
     data_to.insertData(str.data, str.size);
 }
 
+template <>
+inline void AggreagteFunctionGroupUniqArrayGeneric<true, std::false_type>::deserializeAndInsert(StringRef str, IColumn & data_to)
+{
+    data_to.insertData(str.data, str.size);
+}
 
 #undef AGGREGATE_FUNCTION_GROUP_ARRAY_UNIQ_MAX_SIZE
 

From bd5c2cb1d8979356153850eabbe8f6e179da7699 Mon Sep 17 00:00:00 2001
From: Ivan Blinkov <github@blinkov.ru>
Date: Mon, 15 Apr 2019 12:26:23 +0300
Subject: [PATCH 003/118] Add link to Limassol meetup

---
 website/index.html | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/website/index.html b/website/index.html
index f89d622566e..bfbfe2b1829 100644
--- a/website/index.html
+++ b/website/index.html
@@ -92,6 +92,11 @@
         <div class="clear"></div>
     </div>
 </div>
+<div id="announcement" class="colored-block">
+     <div class="page">
+         Upcoming ClickHouse Community Meetup: <a class="announcement-link" href="https://www.facebook.com/events/386638262181785/" rel="external nofollow" target="_blank">Limassol</a> on May 7
+     </div>
+ </div>
 <div class="page">
     <h2 id="slogan">ClickHouse. Just makes you think faster.</h2>
 

From 76e44259465ce02e7aa63d59115475d4c4e3aca1 Mon Sep 17 00:00:00 2001
From: Ivan Blinkov <github@blinkov.ru>
Date: Mon, 15 Apr 2019 12:26:39 +0300
Subject: [PATCH 004/118] Add link to MaxMind integration guide

---
 docs/en/interfaces/third-party/integrations.md | 3 +++
 docs/fa/interfaces/third-party/integrations.md | 3 +++
 docs/ru/interfaces/third-party/integrations.md | 3 +++
 docs/zh/interfaces/third-party/integrations.md | 4 +++-
 4 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/docs/en/interfaces/third-party/integrations.md b/docs/en/interfaces/third-party/integrations.md
index 97468341529..6533145c866 100644
--- a/docs/en/interfaces/third-party/integrations.md
+++ b/docs/en/interfaces/third-party/integrations.md
@@ -51,6 +51,9 @@
         - [loghouse](https://github.com/flant/loghouse) (for [Kubernetes](https://kubernetes.io))
     - [logagent](https://www.sematext.com/logagent)
         - [logagent output-plugin-clickhouse](https://sematext.com/docs/logagent/output-plugin-clickhouse/)
+- Geo
+    - [MaxMind](https://dev.maxmind.com/geoip/)
+        - [clickhouse-maxmind-geoip](https://github.com/AlexeyKupershtokh/clickhouse-maxmind-geoip)
 
 ## Programming Language Ecosystems
 
diff --git a/docs/fa/interfaces/third-party/integrations.md b/docs/fa/interfaces/third-party/integrations.md
index c9aebd9b235..b1d806f776e 100644
--- a/docs/fa/interfaces/third-party/integrations.md
+++ b/docs/fa/interfaces/third-party/integrations.md
@@ -52,6 +52,9 @@
         - [loghouse](https://github.com/flant/loghouse) (برای [Kubernetes](https://kubernetes.io))
     - [logagent](https://www.sematext.com/logagent)
         - [logagent output-plugin-clickhouse](https://sematext.com/docs/logagent/output-plugin-clickhouse/)
+- جغرافیایی
+    - [MaxMind](https://dev.maxmind.com/geoip/)
+        - [clickhouse-maxmind-geoip](https://github.com/AlexeyKupershtokh/clickhouse-maxmind-geoip)
 
 ## اکوسیستم زبان برنامه نویسی
 
diff --git a/docs/ru/interfaces/third-party/integrations.md b/docs/ru/interfaces/third-party/integrations.md
index 2f619d353a7..43d736c8e7c 100644
--- a/docs/ru/interfaces/third-party/integrations.md
+++ b/docs/ru/interfaces/third-party/integrations.md
@@ -50,6 +50,9 @@
         - [loghouse](https://github.com/flant/loghouse) (для [Kubernetes](https://kubernetes.io))
     - [logagent](https://www.sematext.com/logagent)
         - [logagent output-plugin-clickhouse](https://sematext.com/docs/logagent/output-plugin-clickhouse/)
+- Гео
+    - [MaxMind](https://dev.maxmind.com/geoip/)
+        - [clickhouse-maxmind-geoip](https://github.com/AlexeyKupershtokh/clickhouse-maxmind-geoip)
 
 ## Экосистемы вокруг языков программирования
 
diff --git a/docs/zh/interfaces/third-party/integrations.md b/docs/zh/interfaces/third-party/integrations.md
index 971c393d193..fab2bdd07b0 100644
--- a/docs/zh/interfaces/third-party/integrations.md
+++ b/docs/zh/interfaces/third-party/integrations.md
@@ -49,6 +49,9 @@
         - [loghouse](https://github.com/flant/loghouse) (对于 [Kubernetes](https://kubernetes.io))
     - [logagent](https://www.sematext.com/logagent)
         - [logagent output-plugin-clickhouse](https://sematext.com/docs/logagent/output-plugin-clickhouse/)
+- 地理
+    - [MaxMind](https://dev.maxmind.com/geoip/)
+        - [clickhouse-maxmind-geoip](https://github.com/AlexeyKupershtokh/clickhouse-maxmind-geoip)
 
 ## 编程语言生态系统
 
@@ -75,5 +78,4 @@
     - [Ecto](https://github.com/elixir-ecto/ecto)
         - [clickhouse_ecto](https://github.com/appodeal/clickhouse_ecto)
 
-
 [来源文章](https://clickhouse.yandex/docs/zh/interfaces/third-party/integrations/) <!--hide-->

From 3a23360e02a50fd22b1a006f11fb264f7d1a80ca Mon Sep 17 00:00:00 2001
From: Guillaume Tassery <tassery.guillaume@gmail.com>
Date: Mon, 15 Apr 2019 17:09:01 +0700
Subject: [PATCH 005/118] Add functional test for size parameter on
 groupUniqArray

---
 ...shard_group_uniq_array_of_string.reference | 20 +++++++++++++++++++
 ...00375_shard_group_uniq_array_of_string.sql |  2 ++
 ...rd_group_uniq_array_of_int_array.reference | 20 +++++++++++++++++++
 ...76_shard_group_uniq_array_of_int_array.sql |  6 +++++-
 ...9_group_uniq_array_date_datetime.reference |  4 ++++
 .../00399_group_uniq_array_date_datetime.sql  | 11 ++++++++--
 6 files changed, 60 insertions(+), 3 deletions(-)

diff --git a/dbms/tests/queries/0_stateless/00375_shard_group_uniq_array_of_string.reference b/dbms/tests/queries/0_stateless/00375_shard_group_uniq_array_of_string.reference
index 44d059430cd..bf0606cfc5b 100644
--- a/dbms/tests/queries/0_stateless/00375_shard_group_uniq_array_of_string.reference
+++ b/dbms/tests/queries/0_stateless/00375_shard_group_uniq_array_of_string.reference
@@ -18,3 +18,23 @@
 1000
 1000
 1000
+10
+10
+10
+10
+10
+10
+10
+10
+10
+10
+1000
+1000
+1000
+1000
+1000
+1000
+1000
+1000
+1000
+1000
diff --git a/dbms/tests/queries/0_stateless/00375_shard_group_uniq_array_of_string.sql b/dbms/tests/queries/0_stateless/00375_shard_group_uniq_array_of_string.sql
index 210c5f4a0fd..ab2dbd730cc 100644
--- a/dbms/tests/queries/0_stateless/00375_shard_group_uniq_array_of_string.sql
+++ b/dbms/tests/queries/0_stateless/00375_shard_group_uniq_array_of_string.sql
@@ -6,5 +6,7 @@ INSERT INTO test.group_uniq_str SELECT 5 as id, toString(number % 100) as v FROM
 
 SELECT length(groupUniqArray(v)) FROM test.group_uniq_str GROUP BY id ORDER BY id;
 SELECT length(groupUniqArray(v)) FROM remote('127.0.0.{2,3,4,5}', 'test', 'group_uniq_str') GROUP BY id ORDER BY id;
+SELECT length(groupUniqArray(10)(v)) FROM test.group_uniq_str GROUP BY id ORDER BY id;
+SELECT length(groupUniqArray(10000)(v)) FROM test.group_uniq_str GROUP BY id ORDER BY id;
 
 DROP TABLE IF EXISTS test.group_uniq_str;
diff --git a/dbms/tests/queries/0_stateless/00376_shard_group_uniq_array_of_int_array.reference b/dbms/tests/queries/0_stateless/00376_shard_group_uniq_array_of_int_array.reference
index 932499cb209..4f4b18847b4 100644
--- a/dbms/tests/queries/0_stateless/00376_shard_group_uniq_array_of_int_array.reference
+++ b/dbms/tests/queries/0_stateless/00376_shard_group_uniq_array_of_int_array.reference
@@ -18,3 +18,23 @@
 20001
 20001
 20001
+10
+10
+10
+10
+10
+10
+10
+10
+10
+10
+20001
+20001
+20001
+20001
+20001
+20001
+20001
+20001
+20001
+20001
diff --git a/dbms/tests/queries/0_stateless/00376_shard_group_uniq_array_of_int_array.sql b/dbms/tests/queries/0_stateless/00376_shard_group_uniq_array_of_int_array.sql
index fd8d4acbf02..4674da3c03c 100644
--- a/dbms/tests/queries/0_stateless/00376_shard_group_uniq_array_of_int_array.sql
+++ b/dbms/tests/queries/0_stateless/00376_shard_group_uniq_array_of_int_array.sql
@@ -4,6 +4,10 @@ CREATE TABLE test.group_uniq_arr_int ENGINE = Memory AS
 		(SELECT intDiv(number%1000000, 100) as v, intDiv(number%100, 10) as g, number%10 as c FROM system.numbers WHERE c < 3 LIMIT 10000000);
 
 SELECT length(groupUniqArray(v)) FROM test.group_uniq_arr_int GROUP BY id ORDER BY id;
-SELECT length(groupUniqArray(v)) FROM remote('127.0.0.{2,3,4,5}', 'test', 'group_uniq_arr_int') GROUP BY id ORDER BY id;
+SELECT length(groupUniqArray(v)) FROM test.group_uniq_arr_int GROUP BY id ORDER BY id;
+--SELECT length(groupUniqArray(v)) FROM remote('127.0.0.{2,3,4,5}', 'test', 'group_uniq_arr_int') GROUP BY id ORDER BY id;
+SELECT length(groupUniqArray(10)(v)) FROM test.group_uniq_arr_int GROUP BY id ORDER BY id;
+SELECT length(groupUniqArray(100000)(v)) FROM test.group_uniq_arr_int GROUP BY id ORDER BY id;
+
 
 DROP TABLE IF EXISTS test.group_uniq_arr_int;
diff --git a/dbms/tests/queries/0_stateless/00399_group_uniq_array_date_datetime.reference b/dbms/tests/queries/0_stateless/00399_group_uniq_array_date_datetime.reference
index 2ce0b460bb5..6f2b4335230 100644
--- a/dbms/tests/queries/0_stateless/00399_group_uniq_array_date_datetime.reference
+++ b/dbms/tests/queries/0_stateless/00399_group_uniq_array_date_datetime.reference
@@ -1 +1,5 @@
 ['2016-12-16']	['2016-12-16 12:00:00']
+2	2	3	3
+1	1	1	1
+3	3
+1	1
diff --git a/dbms/tests/queries/0_stateless/00399_group_uniq_array_date_datetime.sql b/dbms/tests/queries/0_stateless/00399_group_uniq_array_date_datetime.sql
index 8f760b5cac6..2ed2db11d4c 100644
--- a/dbms/tests/queries/0_stateless/00399_group_uniq_array_date_datetime.sql
+++ b/dbms/tests/queries/0_stateless/00399_group_uniq_array_date_datetime.sql
@@ -1,5 +1,12 @@
 DROP TABLE IF EXISTS test.grop_uniq_array_date;
-CREATE TABLE test.grop_uniq_array_date (d Date, dt DateTime) ENGINE = Memory;
-INSERT INTO test.grop_uniq_array_date VALUES (toDate('2016-12-16'), toDateTime('2016-12-16 12:00:00')) (toDate('2016-12-16'), toDateTime('2016-12-16 12:00:00'));
+CREATE TABLE test.grop_uniq_array_date (d Date, dt DateTime, id Integer) ENGINE = Memory;
+
+INSERT INTO test.grop_uniq_array_date VALUES (toDate('2016-12-16'), toDateTime('2016-12-16 12:00:00'), 1) (toDate('2016-12-16'), toDateTime('2016-12-16 12:00:00'), 1);
 SELECT groupUniqArray(d), groupUniqArray(dt) FROM test.grop_uniq_array_date;
+
+INSERT INTO test.grop_uniq_array_date VALUES (toDate('2016-12-17'), toDateTime('2016-12-17 12:00:00'), 1), (toDate('2016-12-18'), toDateTime('2016-12-18 12:00:00'), 1), (toDate('2016-12-16'), toDateTime('2016-12-16 12:00:00'), 2);
+
+SELECT length(groupUniqArray(2)(d)), length(groupUniqArray(2)(dt)), length(groupUniqArray(d)), length(groupUniqArray(dt)) FROM test.grop_uniq_array_date GROUP BY id ORDER BY id;
+SELECT length(groupUniqArray(10000)(d)), length(groupUniqArray(10000)(dt)) FROM test.grop_uniq_array_date GROUP BY id ORDER BY id;
+
 DROP TABLE IF EXISTS test.grop_uniq_array_date;

From 2a6ff2f240d0ae17b28782d1f50b07bba3e47103 Mon Sep 17 00:00:00 2001
From: Guillaume Tassery <tassery.guillaume@gmail.com>
Date: Mon, 15 Apr 2019 18:10:56 +0700
Subject: [PATCH 006/118] Code refactoring for the new size parameter on the
 GroupUniqArray aggregation function

---
 .../AggregateFunctionGroupUniqArray.cpp       |  8 +--
 .../AggregateFunctionGroupUniqArray.h         | 50 +++++++------------
 ...76_shard_group_uniq_array_of_int_array.sql |  3 +-
 3 files changed, 20 insertions(+), 41 deletions(-)

diff --git a/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.cpp b/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.cpp
index 13c9b328938..49cfe0271d8 100644
--- a/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.cpp
+++ b/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.cpp
@@ -51,17 +51,11 @@ static IAggregateFunction * createWithExtraTypes(const DataTypePtr & argument_ty
     }
 }
 
-template <template <typename, typename> class AggregateFunctionTemplate, typename Data, typename ... TArgs>
-static IAggregateFunction * createWithNumericOrTimeType(const IDataType & argument_type, TArgs && ... args)
-{
-    return createWithNumericType<AggregateFunctionTemplate, Data, TArgs...>(argument_type, std::forward<TArgs>(args)...);
-}
-
 template <typename has_limit, typename ... TArgs>
 inline AggregateFunctionPtr createAggregateFunctionGroupUniqArrayImpl(const std::string & name, const DataTypePtr & argument_type, TArgs ... args)
 {
 
-    AggregateFunctionPtr res(createWithNumericOrTimeType<AggregateFunctionGroupUniqArray, has_limit>(*argument_type, argument_type, std::forward<TArgs>(args)...));
+    AggregateFunctionPtr res(createWithNumericType<AggregateFunctionGroupUniqArray, has_limit, const DataTypePtr &, TArgs...>(*argument_type, argument_type, std::forward<TArgs>(args)...));
 
     if (!res)
         res = AggregateFunctionPtr(createWithExtraTypes<has_limit>(argument_type, std::forward<TArgs>(args)...));
diff --git a/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h b/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h
index bba64b4d786..22d85f5215e 100644
--- a/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h
+++ b/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h
@@ -131,6 +131,14 @@ struct AggreagteFunctionGroupUniqArrayGenericData
     Set value;
 };
 
+
+/// Helper function for deserialize and insert for the class AggreagteFunctionGroupUniqArrayGeneric
+template <bool is_plain_column>
+static StringRef getSerializationImpl(const IColumn & column, size_t row_num, Arena & arena);
+
+template <bool is_plain_column>
+static void deserializeAndInsertImpl(StringRef str, IColumn & data_to);
+
 /** Template parameter with true value should be used for columns that store their elements in memory continuously.
  *  For such columns groupUniqArray() can be implemented more efficiently (especially for small numeric arrays).
  */
@@ -145,9 +153,13 @@ class AggreagteFunctionGroupUniqArrayGeneric
 
     using State = AggreagteFunctionGroupUniqArrayGenericData;
 
-    static StringRef getSerialization(const IColumn & column, size_t row_num, Arena & arena);
+    static StringRef getSerialization(const IColumn & column, size_t row_num, Arena & arena) {
+        return getSerializationImpl<is_plain_column>(column, row_num, arena);
+    }
 
-    static void deserializeAndInsert(StringRef str, IColumn & data_to);
+    static void deserializeAndInsert(StringRef str, IColumn & data_to) {
+        return deserializeAndInsertImpl<is_plain_column>(str, data_to);
+    }
 
 public:
     AggreagteFunctionGroupUniqArrayGeneric(const DataTypePtr & input_data_type, UInt64 max_elems_ = std::numeric_limits<UInt64>::max())
@@ -258,52 +270,26 @@ public:
 
 
 template <>
-inline StringRef AggreagteFunctionGroupUniqArrayGeneric<false, std::false_type>::getSerialization(const IColumn & column, size_t row_num, Arena & arena)
+inline StringRef getSerializationImpl<false>(const IColumn & column, size_t row_num, Arena & arena)
 {
     const char * begin = nullptr;
     return column.serializeValueIntoArena(row_num, arena, begin);
 }
 
 template <>
-inline StringRef AggreagteFunctionGroupUniqArrayGeneric<false, std::true_type>::getSerialization(const IColumn & column, size_t row_num, Arena & arena)
-{
-    const char * begin = nullptr;
-    return column.serializeValueIntoArena(row_num, arena, begin);
-}
-
-template <>
-inline StringRef AggreagteFunctionGroupUniqArrayGeneric<true, std::false_type>::getSerialization(const IColumn & column, size_t row_num, Arena &)
+inline StringRef getSerializationImpl<true>(const IColumn & column, size_t row_num, Arena &)
 {
     return column.getDataAt(row_num);
 }
 
 template <>
-inline StringRef AggreagteFunctionGroupUniqArrayGeneric<true, std::true_type>::getSerialization(const IColumn & column, size_t row_num, Arena &)
-{
-    return column.getDataAt(row_num);
-}
-
-
-template <>
-inline void AggreagteFunctionGroupUniqArrayGeneric<false, std::false_type>::deserializeAndInsert(StringRef str, IColumn & data_to)
+inline void deserializeAndInsertImpl<false>(StringRef str, IColumn & data_to)
 {
     data_to.deserializeAndInsertFromArena(str.data);
 }
 
 template <>
-inline void AggreagteFunctionGroupUniqArrayGeneric<false, std::true_type>::deserializeAndInsert(StringRef str, IColumn & data_to)
-{
-    data_to.deserializeAndInsertFromArena(str.data);
-}
-
-template <>
-inline void AggreagteFunctionGroupUniqArrayGeneric<true, std::true_type>::deserializeAndInsert(StringRef str, IColumn & data_to)
-{
-    data_to.insertData(str.data, str.size);
-}
-
-template <>
-inline void AggreagteFunctionGroupUniqArrayGeneric<true, std::false_type>::deserializeAndInsert(StringRef str, IColumn & data_to)
+inline void deserializeAndInsertImpl<true>(StringRef str, IColumn & data_to)
 {
     data_to.insertData(str.data, str.size);
 }
diff --git a/dbms/tests/queries/0_stateless/00376_shard_group_uniq_array_of_int_array.sql b/dbms/tests/queries/0_stateless/00376_shard_group_uniq_array_of_int_array.sql
index 4674da3c03c..ef5dfc6c489 100644
--- a/dbms/tests/queries/0_stateless/00376_shard_group_uniq_array_of_int_array.sql
+++ b/dbms/tests/queries/0_stateless/00376_shard_group_uniq_array_of_int_array.sql
@@ -4,8 +4,7 @@ CREATE TABLE test.group_uniq_arr_int ENGINE = Memory AS
 		(SELECT intDiv(number%1000000, 100) as v, intDiv(number%100, 10) as g, number%10 as c FROM system.numbers WHERE c < 3 LIMIT 10000000);
 
 SELECT length(groupUniqArray(v)) FROM test.group_uniq_arr_int GROUP BY id ORDER BY id;
-SELECT length(groupUniqArray(v)) FROM test.group_uniq_arr_int GROUP BY id ORDER BY id;
---SELECT length(groupUniqArray(v)) FROM remote('127.0.0.{2,3,4,5}', 'test', 'group_uniq_arr_int') GROUP BY id ORDER BY id;
+SELECT length(groupUniqArray(v)) FROM remote('127.0.0.{2,3,4,5}', 'test', 'group_uniq_arr_int') GROUP BY id ORDER BY id;
 SELECT length(groupUniqArray(10)(v)) FROM test.group_uniq_arr_int GROUP BY id ORDER BY id;
 SELECT length(groupUniqArray(100000)(v)) FROM test.group_uniq_arr_int GROUP BY id ORDER BY id;
 

From 3816b4057901f8c0d3d64b2e16b31157a97b01c4 Mon Sep 17 00:00:00 2001
From: Guillaume Tassery <tassery.guillaume@gmail.com>
Date: Wed, 17 Apr 2019 14:15:59 +0700
Subject: [PATCH 007/118] Add documentation about the max_size argument for
 groupUniqArray

---
 docs/en/query_language/agg_functions/reference.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/docs/en/query_language/agg_functions/reference.md b/docs/en/query_language/agg_functions/reference.md
index 811d23c7b01..033cea705d3 100644
--- a/docs/en/query_language/agg_functions/reference.md
+++ b/docs/en/query_language/agg_functions/reference.md
@@ -369,10 +369,13 @@ Optional parameters:
 - The default value for substituting in empty positions.
 - The length of the resulting array. This allows you to receive arrays of the same size for all the aggregate keys. When using this parameter, the default value must be specified.
 
-## groupUniqArray(x)
+## groupUniqArray(x), groupUniqArray(max_size)(x)
 
 Creates an array from different argument values. Memory consumption is the same as for the `uniqExact` function.
 
+The second version (with the `max_size` parameter) limits the size of the resulting array to `max_size` elements.
+For example, `groupUniqArray (1) (x)` is equivalent to `[any (x)]`.
+
 ## quantile(level)(x)
 
 Approximates the `level` quantile. `level` is a constant, a floating-point number from 0 to 1.

From 915401eb089042911355daa3fc2b9bd647b05c21 Mon Sep 17 00:00:00 2001
From: Ivan Blinkov <github@blinkov.ru>
Date: Wed, 17 Apr 2019 11:42:31 +0300
Subject: [PATCH 008/118] Merge two requirements.txt after #5021

---
 docs/tools/README.md            |  3 +--
 docs/tools/requirements-py3.txt | 35 ---------------------------------
 docs/tools/requirements.txt     |  2 +-
 3 files changed, 2 insertions(+), 38 deletions(-)
 delete mode 100644 docs/tools/requirements-py3.txt

diff --git a/docs/tools/README.md b/docs/tools/README.md
index 627ae20e07f..6332a22f164 100644
--- a/docs/tools/README.md
+++ b/docs/tools/README.md
@@ -23,8 +23,7 @@ $ cd ClickHouse/docs/tools
 $ mkdir venv
 $ virtualenv venv
 $ source venv/bin/activate
-$ pip install -r requirements.txt     # for python2
-$ pip install -r requirements-py3.txt # for python3
+$ pip install -r requirements.txt
 ```
 
 Then running `build.py` without args (there are some, check `build.py --help`) will generate `ClickHouse/docs/build` folder with complete static html website.
diff --git a/docs/tools/requirements-py3.txt b/docs/tools/requirements-py3.txt
deleted file mode 100644
index 6ac9a4a034c..00000000000
--- a/docs/tools/requirements-py3.txt
+++ /dev/null
@@ -1,35 +0,0 @@
-alabaster==0.7.10
-Babel==2.5.1
-backports-abc==0.5
-beautifulsoup4==4.6.3
-certifi==2017.11.5
-chardet==3.0.4
-click==6.7
-CommonMark==0.5.4
-cssmin==0.2.0
-docutils==0.14
-futures==3.1.1
-htmlmin==0.1.12
-idna==2.6
-imagesize==0.7.1
-Jinja2==2.10
-jsmin==2.2.2
-livereload==2.5.1
-Markdown==2.6.11
-MarkupSafe==1.0
-mkdocs==1.0.4
-Pygments==2.2.0
-python-slugify==1.2.6
-pytz==2017.3
-PyYAML==3.12
-recommonmark==0.4.0
-requests==2.21.0
-singledispatch==3.4.0.3
-six==1.11.0
-snowballstemmer==1.2.1
-Sphinx==1.6.5
-sphinxcontrib-websupport==1.0.1
-tornado==5.1
-typing==3.6.2
-Unidecode==1.0.23
-urllib3==1.23
diff --git a/docs/tools/requirements.txt b/docs/tools/requirements.txt
index f0df3b8ff36..6ac9a4a034c 100644
--- a/docs/tools/requirements.txt
+++ b/docs/tools/requirements.txt
@@ -8,7 +8,7 @@ click==6.7
 CommonMark==0.5.4
 cssmin==0.2.0
 docutils==0.14
-futures==3.2.0
+futures==3.1.1
 htmlmin==0.1.12
 idna==2.6
 imagesize==0.7.1

From f46a18a492db270ef75cc0e0078a7b75baa9607f Mon Sep 17 00:00:00 2001
From: Ivan Blinkov <github@blinkov.ru>
Date: Wed, 17 Apr 2019 12:02:04 +0300
Subject: [PATCH 009/118] Add Beijing Meetup link to front page

---
 website/index.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/website/index.html b/website/index.html
index bfbfe2b1829..cbb8e60efe8 100644
--- a/website/index.html
+++ b/website/index.html
@@ -94,7 +94,7 @@
 </div>
 <div id="announcement" class="colored-block">
      <div class="page">
-         Upcoming ClickHouse Community Meetup: <a class="announcement-link" href="https://www.facebook.com/events/386638262181785/" rel="external nofollow" target="_blank">Limassol</a> on May 7
+         Upcoming ClickHouse Community Meetups: <a class="announcement-link" href="https://www.facebook.com/events/386638262181785/" rel="external nofollow" target="_blank">Limassol</a> on May 7 and <a class="announcement-link" href="https://www.huodongxing.com/event/2483759276200" rel="external nofollow" target="_blank">Beijing</a> on June 8
      </div>
  </div>
 <div class="page">

From a96e3c470e4fce76d5b3720ddeaf07d9c711eefa Mon Sep 17 00:00:00 2001
From: Guillaume Tassery <tassery.guillaume@gmail.com>
Date: Wed, 17 Apr 2019 17:24:02 +0700
Subject: [PATCH 010/118] Reformate code for optional parameter for
 groupUniqArray

---
 .../AggregateFunctions/AggregateFunctionGroupUniqArray.h    | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h b/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h
index 22d85f5215e..7bc462f7f3f 100644
--- a/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h
+++ b/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h
@@ -153,11 +153,13 @@ class AggreagteFunctionGroupUniqArrayGeneric
 
     using State = AggreagteFunctionGroupUniqArrayGenericData;
 
-    static StringRef getSerialization(const IColumn & column, size_t row_num, Arena & arena) {
+    static StringRef getSerialization(const IColumn & column, size_t row_num, Arena & arena)
+    {
         return getSerializationImpl<is_plain_column>(column, row_num, arena);
     }
 
-    static void deserializeAndInsert(StringRef str, IColumn & data_to) {
+    static void deserializeAndInsert(StringRef str, IColumn & data_to)
+    {
         return deserializeAndInsertImpl<is_plain_column>(str, data_to);
     }
 

From f5dfd7f9280dae8bc8a9d71ea7274b58a88fd569 Mon Sep 17 00:00:00 2001
From: ogorbacheva <ogorbacheva@yandex-team.ru>
Date: Fri, 19 Apr 2019 18:18:57 +0300
Subject: [PATCH 011/118] Added info about arraySort

---
 .../functions/array_functions.md              | 46 +++++++++++++++++++
 .../functions/array_functions.md              | 46 +++++++++++++++++++
 2 files changed, 92 insertions(+)

diff --git a/docs/en/query_language/functions/array_functions.md b/docs/en/query_language/functions/array_functions.md
index 05fbeeb76af..5a937c52f96 100644
--- a/docs/en/query_language/functions/array_functions.md
+++ b/docs/en/query_language/functions/array_functions.md
@@ -458,6 +458,52 @@ SELECT arraySlice([1, 2, NULL, 4, 5], 2, 3) AS res
 
 Array elements set to `NULL` are handled as normal values.
 
+## arraySort(arr)
+
+Returns an array as result of sorting the elements of `arr1` in ascending order. If the `func` function is specified, sorting order is determined by the result of the function `func` applied to the elements of array (arrays)  
+
+The [Schwartzian transform](https://en.wikipedia.org/wiki/Schwartzian_transform) is used to impove sorting efficiency.
+
+Example:
+
+``` sql
+SELECT arraySort([1, 3, 3, 0])
+```
+```
+┌─arraySort([1, 3, 3, 0])─┐
+│ [0,1,3,3]               │
+└─────────────────────────┘
+```
+
+Note that NULLs and NaNs go last (NaNs go before NULLs). For example:
+ 
+``` sql
+SELECT arraySort([1, nan, 2, NULL, 3, nan, 4, NULL])
+```
+```
+┌─arraySort([1, nan, 2, NULL, 3, nan, 4, NULL])─┐
+│ [1,2,3,4,nan,nan,NULL,NULL]                   │
+└───────────────────────────────────────────────┘
+```
+You can use lambda functions with the `arraySort` function. For details, see the [Higher-order Functions](higher_order_functions.md) section. 
+
+## arrayReverseSort(arr1)
+
+Returns an array as result of sorting the elements of `arr1` in descending order. If the `func` function is specified, sorting order is determined by the result of the function `func` applied to the elements of array (arrays)  
+
+Note that NULLs and NaNs go last (NaNs go before NULLs). For example:
+ 
+``` sql
+SELECT arrayReverseSort([1, nan, 2, NULL, 3, nan, 4, NULL])
+```
+```
+┌─arrayReverseSort([1, nan, 2, NULL, 3, nan, 4, NULL])─┐
+│ [4,3,2,1,nan,nan,NULL,NULL]                          │
+└──────────────────────────────────────────────────────┘
+```
+
+You can use lambda functions with the `arrayReverseSort` function. For details, see the [Higher-order Functions](higher_order_functions.md) section. 
+
 ## arrayUniq(arr, ...)
 
 If one argument is passed, it counts the number of different elements in the array.
diff --git a/docs/ru/query_language/functions/array_functions.md b/docs/ru/query_language/functions/array_functions.md
index 406f8e01509..81e76492dd1 100644
--- a/docs/ru/query_language/functions/array_functions.md
+++ b/docs/ru/query_language/functions/array_functions.md
@@ -458,6 +458,52 @@ SELECT arraySlice([1, 2, NULL, 4, 5], 2, 3) AS res
 
 Элементы массива равные `NULL` обрабатываются как обычные значения.
 
+## arraySort(arr)
+
+Возвращает отсортированный в восходящем порядке массив `arr1`. Если задана функция `func`, то порядок сортировки определяется результатом применения функции `func` на элементы массива (массивов).  
+
+Для улучшения эффективности сортировки применяется [Преобразование Шварца](https://ru.wikipedia.org/wiki/%D0%9F%D1%80%D0%B5%D0%BE%D0%B1%D1%80%D0%B0%D0%B7%D0%BE%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5_%D0%A8%D0%B2%D0%B0%D1%80%D1%86%D0%B0).
+
+Пример:
+
+``` sql
+SELECT arraySort([1, 3, 3, 0])
+```
+```
+┌─arraySort([1, 3, 3, 0])─┐
+│ [0,1,3,3]               │
+└─────────────────────────┘
+```
+
+`NULL` и `NaN` будут последними в массиве (при этом `NaN` будет перед `NULL`). Например:
+
+``` sql
+SELECT arraySort([1, nan, 2, NULL, 3, nan, 4, NULL])
+```
+```
+┌─arraySort([1, nan, 2, NULL, 3, nan, 4, NULL])─┐
+│ [1,2,3,4,nan,nan,NULL,NULL]                   │
+└───────────────────────────────────────────────┘
+```
+
+Функция `arraySort` поддерживает работу с лямбда-функциями. Подробнее см. в разделе [Функции высшего порядка](higher_order_functions.md).
+
+## arrayReverseSort(arr)
+
+Возвращает отсортированный в нисходящем порядке массив `arr1`. Если задана функция `func`, то порядок сортировки определяется результатом применения функции `func` на элементы массива (массивов).  
+
+`NULL` и `NaN` будут последними в массиве (при этом `NaN` будет перед `NULL`). Например:
+ 
+``` sql
+SELECT arrayReverseSort([1, nan, 2, NULL, 3, nan, 4, NULL])
+```
+```
+┌─arrayReverseSort([1, nan, 2, NULL, 3, nan, 4, NULL])─┐
+│ [4,3,2,1,nan,nan,NULL,NULL]                          │
+└──────────────────────────────────────────────────────┘
+```
+Функция `arrayReverseSort` поддерживает работу с лямбда-функциями.  Подробнее см. в разделе [Функции высшего порядка](higher_order_functions.md).
+
 ## arrayUniq(arr, ...)
 
 Если передан один аргумент, считает количество разных элементов в массиве.

From 60e35ae2dd960365ddae78df9d97f491871a455f Mon Sep 17 00:00:00 2001
From: ogorbacheva <ogorbacheva@yandex-team.ru>
Date: Fri, 19 Apr 2019 18:55:23 +0300
Subject: [PATCH 012/118] Added info about arraySort

---
 .../functions/array_functions.md              | 42 ++++++++++++++++---
 .../functions/higher_order_functions.md       |  2 +-
 .../functions/array_functions.md              | 39 +++++++++++++++--
 3 files changed, 74 insertions(+), 9 deletions(-)

diff --git a/docs/en/query_language/functions/array_functions.md b/docs/en/query_language/functions/array_functions.md
index 5a937c52f96..98467232325 100644
--- a/docs/en/query_language/functions/array_functions.md
+++ b/docs/en/query_language/functions/array_functions.md
@@ -460,11 +460,11 @@ Array elements set to `NULL` are handled as normal values.
 
 ## arraySort(arr)
 
-Returns an array as result of sorting the elements of `arr1` in ascending order. If the `func` function is specified, sorting order is determined by the result of the function `func` applied to the elements of array (arrays)  
+Returns an array as result of sorting the elements of `arr` in ascending order.  
 
-The [Schwartzian transform](https://en.wikipedia.org/wiki/Schwartzian_transform) is used to impove sorting efficiency.
+The [Schwartzian transform](https://en.wikipedia.org/wiki/Schwartzian_transform) is used to improve sorting efficiency.
 
-Example:
+Example 1
 
 ``` sql
 SELECT arraySort([1, 3, 3, 0])
@@ -475,6 +475,17 @@ SELECT arraySort([1, 3, 3, 0])
 └─────────────────────────┘
 ```
 
+Example 2
+
+``` sql
+SELECT arraySort(['hello', 'world', '!'])
+```
+```
+┌─arraySort(['hello', 'world', '!'])─┐
+│ ['!','hello','world']              │
+└────────────────────────────────────┘
+```
+
 Note that NULLs and NaNs go last (NaNs go before NULLs). For example:
  
 ``` sql
@@ -487,9 +498,30 @@ SELECT arraySort([1, nan, 2, NULL, 3, nan, 4, NULL])
 ```
 You can use lambda functions with the `arraySort` function. For details, see the [Higher-order Functions](higher_order_functions.md) section. 
 
-## arrayReverseSort(arr1)
+## arrayReverseSort(arr)
 
-Returns an array as result of sorting the elements of `arr1` in descending order. If the `func` function is specified, sorting order is determined by the result of the function `func` applied to the elements of array (arrays)  
+Returns an array as result of sorting the elements of `arr` in descending order.  
+
+Example 1
+
+``` sql
+SELECT arrayReverseSort([1, 3, 3, 0])
+```
+```
+┌─arrayReverseSort([1, 3, 3, 0])─┐
+│ [3,3,1,0]                      │
+└────────────────────────────────┘
+```
+
+Example 2
+``` sql
+SELECT arrayReverseSort(['hello', 'world', '!'])
+```
+```
+┌─arrayReverseSort(['hello', 'world', '!'])─┐
+│ ['world','hello','!']                     │
+└───────────────────────────────────────────┘
+```
 
 Note that NULLs and NaNs go last (NaNs go before NULLs). For example:
  
diff --git a/docs/en/query_language/functions/higher_order_functions.md b/docs/en/query_language/functions/higher_order_functions.md
index ca8612ddab5..f2bffa1aef0 100644
--- a/docs/en/query_language/functions/higher_order_functions.md
+++ b/docs/en/query_language/functions/higher_order_functions.md
@@ -105,7 +105,7 @@ SELECT arrayCumSumNonNegative([1, 1, -4, 1]) AS res
 
 Returns an array as result of sorting the elements of `arr1` in ascending order. If the `func` function is specified, sorting order is determined by the result of the function `func` applied to the elements of array (arrays)  
 
-The [Schwartzian transform](https://en.wikipedia.org/wiki/Schwartzian_transform) is used to impove sorting efficiency.
+The [Schwartzian transform](https://en.wikipedia.org/wiki/Schwartzian_transform) is used to improve sorting efficiency.
 
 Example:
 
diff --git a/docs/ru/query_language/functions/array_functions.md b/docs/ru/query_language/functions/array_functions.md
index 81e76492dd1..03f0176e5f8 100644
--- a/docs/ru/query_language/functions/array_functions.md
+++ b/docs/ru/query_language/functions/array_functions.md
@@ -460,11 +460,11 @@ SELECT arraySlice([1, 2, NULL, 4, 5], 2, 3) AS res
 
 ## arraySort(arr)
 
-Возвращает отсортированный в восходящем порядке массив `arr1`. Если задана функция `func`, то порядок сортировки определяется результатом применения функции `func` на элементы массива (массивов).  
+Возвращает массив `arr`, отсортированный в восходящем порядке.  
 
 Для улучшения эффективности сортировки применяется [Преобразование Шварца](https://ru.wikipedia.org/wiki/%D0%9F%D1%80%D0%B5%D0%BE%D0%B1%D1%80%D0%B0%D0%B7%D0%BE%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5_%D0%A8%D0%B2%D0%B0%D1%80%D1%86%D0%B0).
 
-Пример:
+Пример 1
 
 ``` sql
 SELECT arraySort([1, 3, 3, 0])
@@ -475,6 +475,17 @@ SELECT arraySort([1, 3, 3, 0])
 └─────────────────────────┘
 ```
 
+Пример 2
+
+``` sql
+SELECT arraySort(['hello', 'world', '!'])
+```
+```
+┌─arraySort(['hello', 'world', '!'])─┐
+│ ['!','hello','world']              │
+└────────────────────────────────────┘
+```
+
 `NULL` и `NaN` будут последними в массиве (при этом `NaN` будет перед `NULL`). Например:
 
 ``` sql
@@ -490,7 +501,29 @@ SELECT arraySort([1, nan, 2, NULL, 3, nan, 4, NULL])
 
 ## arrayReverseSort(arr)
 
-Возвращает отсортированный в нисходящем порядке массив `arr1`. Если задана функция `func`, то порядок сортировки определяется результатом применения функции `func` на элементы массива (массивов).  
+Возвращает массив `arr`, отсортированный в нисходящем порядке. Примеры:
+ 
+Пример 1
+  
+``` sql
+SELECT arrayReverseSort([1, 3, 3, 0])
+```
+```
+┌─arrayReverseSort([1, 3, 3, 0])─┐
+│ [3,3,1,0]                      │
+└────────────────────────────────┘
+```
+
+Пример 2
+
+``` sql
+SELECT arrayReverseSort(['hello', 'world', '!'])
+```
+```
+┌─arrayReverseSort(['hello', 'world', '!'])─┐
+│ ['world','hello','!']                     │
+└───────────────────────────────────────────┘
+```  
 
 `NULL` и `NaN` будут последними в массиве (при этом `NaN` будет перед `NULL`). Например:
  

From 35c41cf05749ac92cd7a626039d2bf8f381708f3 Mon Sep 17 00:00:00 2001
From: ogorbacheva <ogorbacheva@yandex-team.ru>
Date: Fri, 19 Apr 2019 19:04:45 +0300
Subject: [PATCH 013/118] Added info about arraySort

---
 docs/en/query_language/functions/array_functions.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/en/query_language/functions/array_functions.md b/docs/en/query_language/functions/array_functions.md
index 98467232325..27d9a847777 100644
--- a/docs/en/query_language/functions/array_functions.md
+++ b/docs/en/query_language/functions/array_functions.md
@@ -514,6 +514,7 @@ SELECT arrayReverseSort([1, 3, 3, 0])
 ```
 
 Example 2
+
 ``` sql
 SELECT arrayReverseSort(['hello', 'world', '!'])
 ```

From cf5265f1bee99c354e4c5c76dd6efa2aa3741810 Mon Sep 17 00:00:00 2001
From: Evgenii Pravda <evgeniipravda@gmail.com>
Date: Mon, 18 Feb 2019 19:28:58 +0300
Subject: [PATCH 014/118] Add a performance test for sorting

---
 dbms/tests/performance/sort/sort.xml | 71 ++++++++++++++++++++++++++++
 1 file changed, 71 insertions(+)
 create mode 100644 dbms/tests/performance/sort/sort.xml

diff --git a/dbms/tests/performance/sort/sort.xml b/dbms/tests/performance/sort/sort.xml
new file mode 100644
index 00000000000..1eba812d9bb
--- /dev/null
+++ b/dbms/tests/performance/sort/sort.xml
@@ -0,0 +1,71 @@
+<test>
+    <name>sort</name>
+
+    <type>loop</type>
+
+    <stop_conditions>
+        <all_of>
+            <iterations>10</iterations>
+        </all_of>
+    </stop_conditions>
+
+
+    <create_query>CREATE TABLE rand_unlimited_10m_8 (key UInt8) Engine = Memory</create_query>
+
+    <create_query>CREATE TABLE rand_1k_10m_16 (key UInt16) Engine = Memory</create_query>
+    <create_query>CREATE TABLE rand_100k_10m_16 (key UInt16) Engine = Memory</create_query>
+    <create_query>CREATE TABLE rand_unlimited_10m_16 (key UInt16) Engine = Memory</create_query>
+
+    <create_query>CREATE TABLE rand_1k_10m_32 (key UInt32) Engine = Memory</create_query>
+    <create_query>CREATE TABLE rand_100k_10m_32 (key UInt32) Engine = Memory</create_query>
+    <create_query>CREATE TABLE rand_unlimited_10m_32 (key UInt32) Engine = Memory</create_query>
+
+    <create_query>CREATE TABLE rand_1k_10m_64 (key UInt64) Engine = Memory</create_query>
+    <create_query>CREATE TABLE rand_100k_10m_64 (key UInt64) Engine = Memory</create_query>
+    <create_query>CREATE TABLE rand_unlimited_10m_64 (key UInt64) Engine = Memory</create_query>
+
+
+    <fill_query>INSERT INTO rand_unlimited_10m_8 SELECT rand() AS x FROM numbers(10000000)</fill_query>
+
+    <fill_query>INSERT INTO rand_1k_10m_16 SELECT rand()%1000 AS x FROM numbers(10000000)</fill_query>
+    <fill_query>INSERT INTO rand_100k_10m_16 SELECT rand()%100000 AS x FROM numbers(10000000)</fill_query>
+    <fill_query>INSERT INTO rand_unlimited_10m_16 SELECT rand() AS x FROM numbers(10000000)</fill_query>
+
+    <fill_query>INSERT INTO rand_1k_10m_32 SELECT rand()%1000 AS x FROM numbers(10000000)</fill_query>
+    <fill_query>INSERT INTO rand_100k_10m_32 SELECT rand()%100000 AS x FROM numbers(10000000)</fill_query>
+    <fill_query>INSERT INTO rand_unlimited_10m_32 SELECT rand() AS x FROM numbers(10000000)</fill_query>
+
+    <fill_query>INSERT INTO rand_1k_10m_64 SELECT rand()%1000 AS x FROM numbers(10000000)</fill_query>
+    <fill_query>INSERT INTO rand_100k_10m_64 SELECT rand()%100000 AS x FROM numbers(10000000)</fill_query>
+    <fill_query>INSERT INTO rand_unlimited_10m_64 SELECT rand() AS x FROM numbers(10000000)</fill_query>
+
+
+    <query>SELECT sum(key) FROM (SELECT * FROM rand_unlimited_10m_8 ORDER BY key)</query>
+
+    <query>SELECT sum(key) FROM (SELECT * FROM rand_1k_10m_16 ORDER BY key)</query>
+    <query>SELECT sum(key) FROM (SELECT * FROM rand_100k_10m_16 ORDER BY key)</query>
+    <query>SELECT sum(key) FROM (SELECT * FROM rand_unlimited_10m_16 ORDER BY key)</query>
+
+    <query>SELECT sum(key) FROM (SELECT * FROM rand_1k_10m_32 ORDER BY key)</query>
+    <query>SELECT sum(key) FROM (SELECT * FROM rand_100k_10m_32 ORDER BY key)</query>
+    <query>SELECT sum(key) FROM (SELECT * FROM rand_unlimited_10m_32 ORDER BY key)</query>
+
+    <query>SELECT sum(key) FROM (SELECT * FROM rand_1k_10m_64 ORDER BY key)</query>
+    <query>SELECT sum(key) FROM (SELECT * FROM rand_100k_10m_64 ORDER BY key)</query>
+    <query>SELECT sum(key) FROM (SELECT * FROM rand_unlimited_10m_64 ORDER BY key)</query>
+
+
+    <drop_query>DROP TABLE IF EXISTS rand_unlimited_10m_8</drop_query>
+
+    <drop_query>DROP TABLE IF EXISTS rand_1k_10m_16</drop_query>
+    <drop_query>DROP TABLE IF EXISTS rand_100k_10m_16</drop_query>
+    <drop_query>DROP TABLE IF EXISTS rand_unlimited_10m_16</drop_query>
+
+    <drop_query>DROP TABLE IF EXISTS rand_1k_10m_32</drop_query>
+    <drop_query>DROP TABLE IF EXISTS rand_100k_10m_32</drop_query>
+    <drop_query>DROP TABLE IF EXISTS rand_unlimited_10m_32</drop_query>
+
+    <drop_query>DROP TABLE IF EXISTS rand_1k_10m_64</drop_query>
+    <drop_query>DROP TABLE IF EXISTS rand_100k_10m_64</drop_query>
+    <drop_query>DROP TABLE IF EXISTS rand_unlimited_10m_64</drop_query>
+</test>

From f1e842cdf613722f7dad2c982a70edadd98c5041 Mon Sep 17 00:00:00 2001
From: Evgenii Pravda <evgeniipravda@gmail.com>
Date: Mon, 18 Feb 2019 19:29:37 +0300
Subject: [PATCH 015/118] Use radixSort when possible

---
 dbms/src/Columns/ColumnVector.cpp | 33 ++++++++++++++++++++++++++-----
 dbms/src/Common/RadixSort.h       | 33 +++++++++++++++++++++++++++++++
 2 files changed, 61 insertions(+), 5 deletions(-)

diff --git a/dbms/src/Columns/ColumnVector.cpp b/dbms/src/Columns/ColumnVector.cpp
index 8322e3f74eb..54842602d42 100644
--- a/dbms/src/Columns/ColumnVector.cpp
+++ b/dbms/src/Columns/ColumnVector.cpp
@@ -7,6 +7,7 @@
 #include <Common/Arena.h>
 #include <Common/SipHash.h>
 #include <Common/NaNUtils.h>
+#include <Common/RadixSort.h>
 #include <IO/WriteBuffer.h>
 #include <IO/WriteHelpers.h>
 #include <Columns/ColumnsCommon.h>
@@ -73,14 +74,15 @@ void ColumnVector<T>::getPermutation(bool reverse, size_t limit, int nan_directi
 {
     size_t s = data.size();
     res.resize(s);
-    for (size_t i = 0; i < s; ++i)
-        res[i] = i;
 
     if (limit >= s)
         limit = 0;
 
     if (limit)
     {
+        for (size_t i = 0; i < s; ++i)
+            res[i] = i;
+
         if (reverse)
             std::partial_sort(res.begin(), res.begin() + limit, res.end(), greater(*this, nan_direction_hint));
         else
@@ -88,10 +90,31 @@ void ColumnVector<T>::getPermutation(bool reverse, size_t limit, int nan_directi
     }
     else
     {
-        if (reverse)
-            pdqsort(res.begin(), res.end(), greater(*this, nan_direction_hint));
+        if constexpr ((std::is_signed_v<T> || std::is_unsigned_v<T>) && !std::is_same_v<T, UInt128>)
+        {
+            PaddedPODArray<std::pair<T, size_t>> pairs(s);
+            for (size_t i = 0; i < s; ++i)
+                pairs.data()[i] = {data[i], i};
+
+            radixSort(pairs.data(), s);
+
+            if (reverse)
+                for (size_t i = 0; i < s; ++i)
+                    res[s - 1 - i] = pairs[i].second;
+            else
+                for (size_t i = 0; i < s; ++i)
+                    res[i] = pairs[i].second;
+        }
         else
-            pdqsort(res.begin(), res.end(), less(*this, nan_direction_hint));
+        {
+            for (size_t i = 0; i < s; ++i)
+                res[i] = i;
+
+            if (reverse)
+                pdqsort(res.begin(), res.end(), greater(*this, nan_direction_hint));
+            else
+                pdqsort(res.begin(), res.end(), less(*this, nan_direction_hint));
+        }
     }
 }
 
diff --git a/dbms/src/Common/RadixSort.h b/dbms/src/Common/RadixSort.h
index f95c6237d74..fdb9cfda044 100644
--- a/dbms/src/Common/RadixSort.h
+++ b/dbms/src/Common/RadixSort.h
@@ -94,6 +94,22 @@ struct RadixSortFloatTraits
     }
 };
 
+template <typename Float>
+struct RadixSortFloatPTraits
+{
+    using Element = std::pair<Float, size_t>;
+    using Key = Float;
+    using CountType = uint32_t;
+    using KeyBits = std::conditional_t<sizeof(Float) == 8, uint64_t, uint32_t>;
+
+    static constexpr size_t PART_SIZE_BITS = 8;
+
+    using Transform = RadixSortFloatTransform<KeyBits>;
+    using Allocator = RadixSortMallocAllocator;
+
+    /// The function to get the key from an array element.
+    static Key & extractKey(Element & elem) { return elem.first; }
+};
 
 template <typename KeyBits>
 struct RadixSortIdentityTransform
@@ -161,6 +177,23 @@ struct RadixSortIntTraits
     }
 };
 
+template <typename Int>
+struct RadixSortIntPTraits
+{
+    using Element = std::pair<Int, size_t>;
+    using Key = Int;
+    using CountType = uint32_t;
+    using KeyBits = std::make_unsigned_t<Int>;
+
+    static constexpr size_t PART_SIZE_BITS = 8;
+
+    using Transform = RadixSortSignedTransform<KeyBits>;
+    using Allocator = RadixSortMallocAllocator;
+
+    /// The function to get the key from an array element.
+    static Key & extractKey(Element & elem) { return elem.first; }
+};
+
 
 template <typename Traits>
 struct RadixSort

From fecf0a36305ef2d476260a9abe44c6eef3d18d09 Mon Sep 17 00:00:00 2001
From: Evgenii Pravda <evgeniipravda@gmail.com>
Date: Thu, 7 Mar 2019 00:58:40 +0300
Subject: [PATCH 016/118] Some fixes

---
 dbms/src/Columns/ColumnVector.cpp | 9 ++++++++-
 dbms/src/Common/RadixSort.h       | 4 ++--
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/dbms/src/Columns/ColumnVector.cpp b/dbms/src/Columns/ColumnVector.cpp
index 54842602d42..1c29bdc10bc 100644
--- a/dbms/src/Columns/ColumnVector.cpp
+++ b/dbms/src/Columns/ColumnVector.cpp
@@ -94,7 +94,7 @@ void ColumnVector<T>::getPermutation(bool reverse, size_t limit, int nan_directi
         {
             PaddedPODArray<std::pair<T, size_t>> pairs(s);
             for (size_t i = 0; i < s; ++i)
-                pairs.data()[i] = {data[i], i};
+                pairs[i] = {data[i], i};
 
             radixSort(pairs.data(), s);
 
@@ -338,6 +338,10 @@ void ColumnVector<T>::getExtremes(Field & min, Field & max) const
 }
 
 /// Explicit template instantiations - to avoid code bloat in headers.
+#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ >= 8)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wclass-memaccess"
+#endif
 template class ColumnVector<UInt8>;
 template class ColumnVector<UInt16>;
 template class ColumnVector<UInt32>;
@@ -350,4 +354,7 @@ template class ColumnVector<Int64>;
 template class ColumnVector<Int128>;
 template class ColumnVector<Float32>;
 template class ColumnVector<Float64>;
+#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ >= 8)
+#pragma GCC diagnostic pop
+#endif
 }
diff --git a/dbms/src/Common/RadixSort.h b/dbms/src/Common/RadixSort.h
index fdb9cfda044..48fbdce8096 100644
--- a/dbms/src/Common/RadixSort.h
+++ b/dbms/src/Common/RadixSort.h
@@ -95,7 +95,7 @@ struct RadixSortFloatTraits
 };
 
 template <typename Float>
-struct RadixSortFloatPTraits
+struct RadixSortPairFloatKeyTraits
 {
     using Element = std::pair<Float, size_t>;
     using Key = Float;
@@ -178,7 +178,7 @@ struct RadixSortIntTraits
 };
 
 template <typename Int>
-struct RadixSortIntPTraits
+struct RadixSortPairIntKeyTraits
 {
     using Element = std::pair<Int, size_t>;
     using Key = Int;

From 6157ddba0de2a5d2de42621876ba684284885946 Mon Sep 17 00:00:00 2001
From: Evgenii Pravda <evgeniipravda@gmail.com>
Date: Tue, 26 Mar 2019 23:30:13 +0300
Subject: [PATCH 017/118] Trying to defeat gcc 8

---
 dbms/src/Columns/ColumnVector.cpp | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/dbms/src/Columns/ColumnVector.cpp b/dbms/src/Columns/ColumnVector.cpp
index 1c29bdc10bc..7d053f51796 100644
--- a/dbms/src/Columns/ColumnVector.cpp
+++ b/dbms/src/Columns/ColumnVector.cpp
@@ -19,6 +19,11 @@
     #include <emmintrin.h>
 #endif
 
+// Allow std::pair copying
+#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ >= 8)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wclass-memaccess"
+#endif
 
 namespace DB
 {
@@ -338,10 +343,6 @@ void ColumnVector<T>::getExtremes(Field & min, Field & max) const
 }
 
 /// Explicit template instantiations - to avoid code bloat in headers.
-#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ >= 8)
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wclass-memaccess"
-#endif
 template class ColumnVector<UInt8>;
 template class ColumnVector<UInt16>;
 template class ColumnVector<UInt32>;

From bc4eb309e223b45b26c22758a3ade2ac1c2413f1 Mon Sep 17 00:00:00 2001
From: Evgenii Pravda <evgeniipravda@gmail.com>
Date: Wed, 27 Mar 2019 00:55:37 +0300
Subject: [PATCH 018/118] Trying to defeat gcc 8 (2)

---
 dbms/src/Columns/ColumnVector.cpp | 9 ---------
 dbms/src/Common/RadixSort.h       | 9 ++++++++-
 2 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/dbms/src/Columns/ColumnVector.cpp b/dbms/src/Columns/ColumnVector.cpp
index 7d053f51796..3fca0266299 100644
--- a/dbms/src/Columns/ColumnVector.cpp
+++ b/dbms/src/Columns/ColumnVector.cpp
@@ -19,12 +19,6 @@
     #include <emmintrin.h>
 #endif
 
-// Allow std::pair copying
-#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ >= 8)
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wclass-memaccess"
-#endif
-
 namespace DB
 {
 
@@ -355,7 +349,4 @@ template class ColumnVector<Int64>;
 template class ColumnVector<Int128>;
 template class ColumnVector<Float32>;
 template class ColumnVector<Float64>;
-#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ >= 8)
-#pragma GCC diagnostic pop
-#endif
 }
diff --git a/dbms/src/Common/RadixSort.h b/dbms/src/Common/RadixSort.h
index 48fbdce8096..c3083e088da 100644
--- a/dbms/src/Common/RadixSort.h
+++ b/dbms/src/Common/RadixSort.h
@@ -195,6 +195,11 @@ struct RadixSortPairIntKeyTraits
 };
 
 
+// Allow std::pair copying
+#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ >= 8)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wclass-memaccess"
+#endif
 template <typename Traits>
 struct RadixSort
 {
@@ -289,7 +294,9 @@ public:
         allocator.deallocate(swap_buffer, size * sizeof(Element));
     }
 };
-
+#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ >= 8)
+#pragma GCC diagnostic pop
+#endif
 
 template <typename T>
 std::enable_if_t<std::is_unsigned_v<T> && std::is_integral_v<T>, void>

From 5a773ed67c8b2c4c619e54c237f382dfe9ffaa6f Mon Sep 17 00:00:00 2001
From: Evgenii Pravda <evgeniipravda@gmail.com>
Date: Tue, 2 Apr 2019 23:15:02 +0300
Subject: [PATCH 019/118] Fix nulls order

---
 dbms/src/Columns/ColumnVector.cpp |  2 +-
 dbms/src/Common/RadixSort.h       | 71 ++++++++++++++++++++++++++++---
 2 files changed, 66 insertions(+), 7 deletions(-)

diff --git a/dbms/src/Columns/ColumnVector.cpp b/dbms/src/Columns/ColumnVector.cpp
index 3fca0266299..b8ad5ddba8f 100644
--- a/dbms/src/Columns/ColumnVector.cpp
+++ b/dbms/src/Columns/ColumnVector.cpp
@@ -95,7 +95,7 @@ void ColumnVector<T>::getPermutation(bool reverse, size_t limit, int nan_directi
             for (size_t i = 0; i < s; ++i)
                 pairs[i] = {data[i], i};
 
-            radixSort(pairs.data(), s);
+            radixSort(pairs.data(), s, nan_direction_hint);
 
             if (reverse)
                 for (size_t i = 0; i < s; ++i)
diff --git a/dbms/src/Common/RadixSort.h b/dbms/src/Common/RadixSort.h
index c3083e088da..2903ba64af5 100644
--- a/dbms/src/Common/RadixSort.h
+++ b/dbms/src/Common/RadixSort.h
@@ -1,12 +1,16 @@
 #pragma once
 
+
 #include <string.h>
 #if !defined(__APPLE__) && !defined(__FreeBSD__)
 #include <malloc.h>
 #endif
+#include <algorithm>
+#include <cmath>
 #include <cstdlib>
 #include <cstdint>
 #include <type_traits>
+#include <vector>
 
 #include <ext/bit_cast.h>
 #include <Core/Types.h>
@@ -154,6 +158,23 @@ struct RadixSortUIntTraits
     }
 };
 
+template <typename UInt>
+struct RadixSortPairUIntKeyTraits
+{
+    using Element = std::pair<UInt, size_t>;
+    using Key = UInt;
+    using CountType = uint32_t;
+    using KeyBits = UInt;
+
+    static constexpr size_t PART_SIZE_BITS = 8;
+
+    using Transform = RadixSortIdentityTransform<KeyBits>;
+    using Allocator = RadixSortMallocAllocator;
+
+    /// The function to get the key from an array element.
+    static Key & extractKey(Element & elem) { return elem.first; }
+};
+
 template <typename _Element, typename _Key = _Element>
 struct RadixSortIntTraits
 {
@@ -293,6 +314,21 @@ public:
 
         allocator.deallocate(swap_buffer, size * sizeof(Element));
     }
+
+    // For floating point types
+    // Radix sort sometimes incorrectly handles NaNs
+    // Will move them to the right place
+    static void fixNanOrder(Element * arr, size_t size, int nan_direction_hint)
+    {
+        if (nan_direction_hint < 0)
+        {
+            size_t nans_count = std::count_if(arr, arr + size, [](Element d) {return std::isnan(Traits::extractKey(d));});
+            std::vector<Element> nans(nans_count);
+            std::copy(arr + size - nans_count, arr + size, nans.data());
+            std::copy_backward(arr, arr + size - nans_count, arr + size);
+            std::copy(nans.data(), nans.data() + nans_count, arr);
+        }
+    }
 };
 #if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ >= 8)
 #pragma GCC diagnostic pop
@@ -300,23 +336,24 @@ public:
 
 template <typename T>
 std::enable_if_t<std::is_unsigned_v<T> && std::is_integral_v<T>, void>
-radixSort(T * arr, size_t size)
+radixSort(T * arr, size_t size, int /*nan_direction_hint*/=1)
 {
-    return RadixSort<RadixSortUIntTraits<T>>::execute(arr, size);
+    RadixSort<RadixSortUIntTraits<T>>::execute(arr, size);
 }
 
 template <typename T>
 std::enable_if_t<std::is_signed_v<T> && std::is_integral_v<T>, void>
-radixSort(T * arr, size_t size)
+radixSort(T * arr, size_t size, int /*nan_direction_hint*/=1)
 {
-    return RadixSort<RadixSortIntTraits<T>>::execute(arr, size);
+    RadixSort<RadixSortIntTraits<T>>::execute(arr, size);
 }
 
 template <typename T>
 std::enable_if_t<std::is_floating_point_v<T>, void>
-radixSort(T * arr, size_t size)
+radixSort(T * arr, size_t size, int nan_direction_hint=1)
 {
-    return RadixSort<RadixSortFloatTraits<T>>::execute(arr, size);
+    RadixSort<RadixSortFloatTraits<T>>::execute(arr, size);
+    RadixSort<RadixSortFloatTraits<T>>::fixNanOrder(arr, size, nan_direction_hint);
 }
 
 template <typename _Element, typename _Key>
@@ -332,3 +369,25 @@ radixSort(_Element * arr, size_t size)
 {
     return RadixSort<RadixSortFloatTraits<_Element, _Key>>::execute(arr, size);
 }
+
+template <typename T>
+std::enable_if_t<std::is_unsigned_v<T> && !std::is_floating_point_v<T>, void>
+radixSort(std::pair<T, size_t> * arr, size_t size, int /*nan_direction_hint*/=1)
+{
+    RadixSort<RadixSortPairUIntKeyTraits<T>>::execute(arr, size);
+}
+
+template <typename T>
+std::enable_if_t<std::is_signed_v<T> && !std::is_floating_point_v<T>, void>
+radixSort(std::pair<T, size_t> * arr, size_t size, int /*nan_direction_hint*/=1)
+{
+    RadixSort<RadixSortPairIntKeyTraits<T>>::execute(arr, size);
+}
+
+template <typename T>
+std::enable_if_t<std::is_floating_point_v<T>, void>
+radixSort(std::pair<T, size_t> * arr, size_t size, int nan_direction_hint=1)
+{
+    RadixSort<RadixSortPairFloatKeyTraits<T>>::execute(arr, size);
+    RadixSort<RadixSortPairFloatKeyTraits<T>>::fixNanOrder(arr, size, nan_direction_hint);
+}

From a436247a150e9c6fd83a7ff59ca2cb079ea92434 Mon Sep 17 00:00:00 2001
From: Evgenii Pravda <evgeniipravda@gmail.com>
Date: Sat, 20 Apr 2019 02:06:23 +0300
Subject: [PATCH 020/118] Rebase, fix test

---
 .../tests/queries/0_stateless/00853_join_with_nulls_crash.sql | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dbms/tests/queries/0_stateless/00853_join_with_nulls_crash.sql b/dbms/tests/queries/0_stateless/00853_join_with_nulls_crash.sql
index 00de5b7d89f..cbef2f709ac 100644
--- a/dbms/tests/queries/0_stateless/00853_join_with_nulls_crash.sql
+++ b/dbms/tests/queries/0_stateless/00853_join_with_nulls_crash.sql
@@ -21,14 +21,14 @@ SELECT s1.other, s2.other, count_a, count_b, toTypeName(s1.other), toTypeName(s2
 ALL FULL JOIN
     ( SELECT other, count() AS count_b FROM table_b GROUP BY other ) s2
 ON s1.other = s2.other
-ORDER BY count_a DESC;
+ORDER BY s2.other DESC;
 
 SELECT s1.other, s2.other, count_a, count_b, toTypeName(s1.other), toTypeName(s2.other) FROM
     ( SELECT other, count() AS count_a FROM table_a GROUP BY other ) s1
 ALL FULL JOIN
     ( SELECT other, count() AS count_b FROM table_b GROUP BY other ) s2
 USING other
-ORDER BY count_a DESC;
+ORDER BY s2.other DESC;
 
 SELECT s1.something, s2.something, count_a, count_b, toTypeName(s1.something), toTypeName(s2.something) FROM
     ( SELECT something, count() AS count_a FROM table_a GROUP BY something ) s1

From 813d857b6231311832f991d06f907e295d9c1f60 Mon Sep 17 00:00:00 2001
From: Evgenii Pravda <evgeniipravda@gmail.com>
Date: Sat, 20 Apr 2019 17:06:36 +0300
Subject: [PATCH 021/118] Fix reference

---
 .../0_stateless/00853_join_with_nulls_crash.reference       | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/dbms/tests/queries/0_stateless/00853_join_with_nulls_crash.reference b/dbms/tests/queries/0_stateless/00853_join_with_nulls_crash.reference
index 7ee48046f3a..b6168923b09 100644
--- a/dbms/tests/queries/0_stateless/00853_join_with_nulls_crash.reference
+++ b/dbms/tests/queries/0_stateless/00853_join_with_nulls_crash.reference
@@ -1,11 +1,11 @@
-foo	\N	2	0	Nullable(String)	Nullable(String)
+\N	test	0	1	Nullable(String)	Nullable(String)
 bar	bar	1	2	Nullable(String)	Nullable(String)
 \N		0	1	Nullable(String)	Nullable(String)
-\N	test	0	1	Nullable(String)	Nullable(String)
 foo	\N	2	0	Nullable(String)	Nullable(String)
+\N	test	0	1	Nullable(String)	Nullable(String)
 bar	bar	1	2	Nullable(String)	Nullable(String)
 \N		0	1	Nullable(String)	Nullable(String)
-\N	test	0	1	Nullable(String)	Nullable(String)
+foo	\N	2	0	Nullable(String)	Nullable(String)
 foo		2	0	String	Nullable(String)
 bar	bar	1	2	String	Nullable(String)
 	test	0	1	String	Nullable(String)

From 2caf3d39ffa469d28d8d1e6c52950f25426707f8 Mon Sep 17 00:00:00 2001
From: Danila Kutenin <danlark@sandbox-286092131>
Date: Sun, 21 Apr 2019 20:31:15 +0300
Subject: [PATCH 022/118] Faster LZ4 decompression because of new release

---
 contrib/lz4                                   |   2 +-
 .../src/Compression/LZ4_decompress_faster.cpp | 212 +++++++++++++++---
 dbms/src/Compression/LZ4_decompress_faster.h  |   2 +-
 3 files changed, 180 insertions(+), 36 deletions(-)

diff --git a/contrib/lz4 b/contrib/lz4
index 780aac520b6..7a4e3b1fac5 160000
--- a/contrib/lz4
+++ b/contrib/lz4
@@ -1 +1 @@
-Subproject commit 780aac520b69d6369f4e3995624c37e56d75498d
+Subproject commit 7a4e3b1fac5cd9d4ec7c8d0091329ba107ec2131
diff --git a/dbms/src/Compression/LZ4_decompress_faster.cpp b/dbms/src/Compression/LZ4_decompress_faster.cpp
index 6c2f072649c..18b034540fc 100644
--- a/dbms/src/Compression/LZ4_decompress_faster.cpp
+++ b/dbms/src/Compression/LZ4_decompress_faster.cpp
@@ -14,6 +14,10 @@
 #include <emmintrin.h>
 #endif
 
+#ifdef __AVX__
+#include <immintrin.h>
+#endif
+
 #ifdef __SSSE3__
 #include <tmmintrin.h>
 #endif
@@ -341,45 +345,188 @@ template <> void inline wildCopy<16>(UInt8 * dst, const UInt8 * src, UInt8 * dst
 template <> void inline copyOverlap<16, false>(UInt8 * op, const UInt8 *& match, const size_t offset) { copyOverlap16(op, match, offset); }
 template <> void inline copyOverlap<16, true>(UInt8 * op, const UInt8 *& match, const size_t offset) { copyOverlap16Shuffle(op, match, offset); }
 
+inline void copy32(UInt8 * dst, const UInt8 * src)
+{
+#if defined(__AVX__)
+    _mm256_storeu_si256(reinterpret_cast<__m256i *>(dst),
+        _mm256_loadu_si256(reinterpret_cast<const __m256i *>(src)));
+#elif defined(__SSE2__)
+    _mm_storeu_si128(reinterpret_cast<__m128i *>(dst),
+        _mm_loadu_si128(reinterpret_cast<const __m128i *>(src)));
+    _mm_storeu_si128(reinterpret_cast<__m128i *>(dst + 16),
+        _mm_loadu_si128(reinterpret_cast<const __m128i *>(src + 16)));
+#else
+    memcpy(dst, src, 16);
+#endif
+}
+
+inline void wildCopy32(UInt8 * dst, const UInt8 * src, UInt8 * dst_end)
+{
+    do
+    {
+        copy32(dst, src);
+        dst += 32;
+        src += 32;
+    } while (dst < dst_end);
+}
+
+template <> void inline copy<32>(UInt8 * dst, const UInt8 * src) { copy32(dst, src); }
+template <> void inline wildCopy<32>(UInt8 * dst, const UInt8 * src, UInt8 * dst_end) { wildCopy32(dst, src, dst_end); }
+
+inline void copyUsingOffset(UInt8 * dst, const UInt8 * src, UInt8 * dst_end, const size_t offset)
+{
+    UInt8 v[8];
+    switch (offset)
+    {
+        case 1:
+            memset(v, *src, 8);
+            goto copy_loop;
+        case 2:
+            memcpy(v, src, 2);
+            memcpy(&v[2], src, 2);
+            memcpy(&v[4], &v[0], 4);
+            goto copy_loop;
+        case 4:
+            memcpy(v, src, 4);
+            memcpy(&v[4], src, 4);
+            goto copy_loop;
+        default:
+            if (offset < 8)
+            {
+                copyOverlap<8, false>(dst, src, offset);
+                dst += 8;
+            }
+            else
+            {
+                memcpy(dst, src, 8);
+                dst += 8;
+                src += 8;
+            }
+            wildCopy<8>(dst, src, dst_end);
+            return;
+    }
+copy_loop:
+    memcpy(dst, v, 8);
+    dst += 8;
+    while (dst < dst_end)
+    {
+        memcpy(dst, v, 8);
+        dst += 8;
+    }
+}
 
 /// See also https://stackoverflow.com/a/30669632
 
 
-template <size_t copy_amount, bool use_shuffle>
+/// use_optimized_new_lz4_version is for the new optimized LZ4 decompression that was introduced in LZ4 1.9.0
+/// With experiments it turned out to be faster for big columns, so we decided to add it to the statistics choice.
+
+template <size_t copy_amount, size_t guaranteed_minimal_dest_size, bool use_shuffle, bool use_optimized_new_lz4_version>
 void NO_INLINE decompressImpl(
      const char * const source,
      char * const dest,
+     [[maybe_unused]] size_t source_size,
      size_t dest_size)
 {
     const UInt8 * ip = reinterpret_cast<const UInt8 *>(source);
+    [[maybe_unused]] const UInt8 * const input_end = reinterpret_cast<const UInt8 *>(source) + source_size;
     UInt8 * op = reinterpret_cast<UInt8 *>(dest);
     UInt8 * const output_end = op + dest_size;
+    UInt8 * copy_end;
+    unsigned token;
+    size_t length;
+    size_t offset;
+    const UInt8 * match;
+
+    auto continue_read_length = [&]
+    {
+        unsigned s;
+        do
+        {
+            s = *ip++;
+            length += s;
+        } while (unlikely(s == 255));
+    };
+
+    if (use_optimized_new_lz4_version)
+    {
+        if (guaranteed_minimal_dest_size >= 64 || dest_size >= 64)
+        {
+            while (1)
+            {
+                /// Get literal length.
+                token = *ip++;
+                length = token >> 4;
+                if (length == 0x0F)
+                {
+                    continue_read_length();
+                    copy_end = op + length;
+                    if (copy_end > output_end - 32 || ip + length > input_end - 32)
+                        goto literal_copy;
+                    wildCopy<32>(op, ip, copy_end);
+                    ip += length;
+                    op = copy_end;
+                }
+                else
+                {
+                    copy_end = op + length;
+                    if (ip > input_end - 17)
+                        goto literal_copy;
+                    copy<16>(op, ip);
+                    ip += length;
+                    op = copy_end;
+                }
+
+                offset = unalignedLoad<UInt16>(ip);
+                ip += 2;
+                match = op - offset;
+
+                /// Get match length.
+                length = token & 0x0F;
+                if (length == 0x0F)
+                {
+                    continue_read_length();
+                    length += 4;
+                    if (op + length >= output_end - 64)
+                        goto match_copy;
+                }
+                else
+                {
+                    length += 4;
+                    if (op + length >= output_end - 64)
+                        goto match_copy;
+                    if (offset >= 8)
+                    {
+                        memcpy(op, match, 8);
+                        memcpy(op + 8, match + 8, 8);
+                        memcpy(op + 16, match + 16, 2);
+                        op += length;
+                        continue;
+                    }
+                }
+
+                /// Copy match within block, that produce overlapping pattern. Match may replicate itself.
+                copy_end = op + length;
+                if (unlikely(offset < 16))
+                    copyUsingOffset(op, match, copy_end, offset);
+                else
+                    wildCopy<32>(op, match, copy_end);
+
+                op = copy_end;
+            }
+        }
+    }
 
     while (1)
     {
-        size_t length;
-
-        auto continue_read_length = [&]
-        {
-            unsigned s;
-            do
-            {
-                s = *ip++;
-                length += s;
-            } while (unlikely(s == 255));
-        };
-
         /// Get literal length.
-
-        const unsigned token = *ip++;
+        token = *ip++;
         length = token >> 4;
         if (length == 0x0F)
             continue_read_length();
-
         /// Copy literals.
-
-        UInt8 * copy_end = op + length;
-
+        copy_end = op + length;
+literal_copy:
         /// input: Hello, world
         ///        ^-ip
         /// output: xyz
@@ -390,8 +537,7 @@ void NO_INLINE decompressImpl(
         ///              ^-ip
         /// output: xyzHello, w
         ///                  ^-op (we will overwrite excessive bytes on next iteration)
-
-        wildCopy<copy_amount>(op, ip, copy_end);    /// Here we can write up to copy_amount - 1 bytes after buffer.
+        wildCopy<copy_amount>(op, ip, copy_end);
 
         ip += length;
         op = copy_end;
@@ -400,20 +546,18 @@ void NO_INLINE decompressImpl(
             return;
 
         /// Get match offset.
-
-        size_t offset = unalignedLoad<UInt16>(ip);
+        offset = unalignedLoad<UInt16>(ip);
         ip += 2;
-        const UInt8 * match = op - offset;
+        match = op - offset;
 
         /// Get match length.
-
         length = token & 0x0F;
         if (length == 0x0F)
             continue_read_length();
         length += 4;
 
+match_copy:
         /// Copy match within block, that produce overlapping pattern. Match may replicate itself.
-
         copy_end = op + length;
 
         /** Here we can write up to copy_amount - 1 - 4 * 2 bytes after buffer.
@@ -432,7 +576,6 @@ void NO_INLINE decompressImpl(
             ///
             /// output: HelloHelloHel
             ///            ^-match   ^-op
-
             copyOverlap<copy_amount, use_shuffle>(op, match, offset);
         }
         else
@@ -453,7 +596,6 @@ void NO_INLINE decompressImpl(
 
 }
 
-
 void decompress(
     const char * const source,
     char * const dest,
@@ -463,9 +605,9 @@ void decompress(
 {
     if (source_size == 0 || dest_size == 0)
         return;
-
+    static constexpr size_t small_block_threshold = 32768;
     /// Don't run timer if the block is too small.
-    if (dest_size >= 32768)
+    if (dest_size >= small_block_threshold)
     {
         size_t best_variant = statistics.select();
 
@@ -474,11 +616,13 @@ void decompress(
         Stopwatch watch;
 
         if (best_variant == 0)
-            decompressImpl<16, true>(source, dest, dest_size);
+            decompressImpl<16, small_block_threshold, true, false>(source, dest, source_size, dest_size);
         if (best_variant == 1)
-            decompressImpl<16, false>(source, dest, dest_size);
+            decompressImpl<16, small_block_threshold, false, false>(source, dest, source_size, dest_size);
         if (best_variant == 2)
-            decompressImpl<8, true>(source, dest, dest_size);
+            decompressImpl<8, small_block_threshold, true, false>(source, dest, source_size, dest_size);
+        if (best_variant == 3)
+            decompressImpl<8, small_block_threshold, false, true>(source, dest, source_size, dest_size);
 
         watch.stop();
 
@@ -488,7 +632,7 @@ void decompress(
     }
     else
     {
-        decompressImpl<8, false>(source, dest, dest_size);
+        decompressImpl<8, 0, false, false>(source, dest, source_size, dest_size);
     }
 }
 
diff --git a/dbms/src/Compression/LZ4_decompress_faster.h b/dbms/src/Compression/LZ4_decompress_faster.h
index 1d5ee92ced2..3b48106c583 100644
--- a/dbms/src/Compression/LZ4_decompress_faster.h
+++ b/dbms/src/Compression/LZ4_decompress_faster.h
@@ -88,7 +88,7 @@ struct PerformanceStatistics
     };
 
     /// Number of different algorithms to select from.
-    static constexpr size_t NUM_ELEMENTS = 3;
+    static constexpr size_t NUM_ELEMENTS = 4;
 
     /// Cold invocations may be affected by additional memory latencies. Don't take first invocations into account.
     static constexpr double NUM_INVOCATIONS_TO_THROW_OFF = 2;

From cabcaf673ad224bf58bccb46eff180ccae9af33f Mon Sep 17 00:00:00 2001
From: Danila Kutenin <danlark@sandbox-286092131>
Date: Sun, 21 Apr 2019 20:41:45 +0300
Subject: [PATCH 023/118] Comments

---
 dbms/src/Compression/LZ4_decompress_faster.cpp | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/dbms/src/Compression/LZ4_decompress_faster.cpp b/dbms/src/Compression/LZ4_decompress_faster.cpp
index 18b034540fc..5bfab224538 100644
--- a/dbms/src/Compression/LZ4_decompress_faster.cpp
+++ b/dbms/src/Compression/LZ4_decompress_faster.cpp
@@ -356,7 +356,9 @@ inline void copy32(UInt8 * dst, const UInt8 * src)
     _mm_storeu_si128(reinterpret_cast<__m128i *>(dst + 16),
         _mm_loadu_si128(reinterpret_cast<const __m128i *>(src + 16)));
 #else
+    /// This is from reference implementation, actually there can be issues if we write 32 for ARM processor
     memcpy(dst, src, 16);
+    memcpy(dst + 16, src + 16, 16);
 #endif
 }
 
@@ -448,6 +450,9 @@ void NO_INLINE decompressImpl(
         } while (unlikely(s == 255));
     };
 
+    /// Don't use `if constexpr` here because compilers start warning about unused labels
+    /// And I don't want to insert pragma push etc. Codegen is the same because of dead code elimination.
+    /// TODO(danlark) Generalize this branch with copy_amount etc.
     if (use_optimized_new_lz4_version)
     {
         if (guaranteed_minimal_dest_size >= 64 || dest_size >= 64)

From d90bab5e33f86d8e4004d5095e1daf1b6cb9740a Mon Sep 17 00:00:00 2001
From: Alexander Kazakov <akazz@yandex-team.ru>
Date: Mon, 22 Apr 2019 16:31:17 +0300
Subject: [PATCH 024/118] In [T|C]SVWithNames formats start using column
 information from header

---
 dbms/src/Core/Settings.h                      |   3 +-
 dbms/src/Formats/CSVRowInputStream.cpp        | 284 ++++++++++-----
 dbms/src/Formats/CSVRowInputStream.h          |  15 +-
 dbms/src/Formats/FormatFactory.cpp            |   1 +
 dbms/src/Formats/FormatSettings.h             |   1 +
 .../Formats/TabSeparatedRowInputStream.cpp    | 333 ++++++++++++------
 dbms/src/Formats/TabSeparatedRowInputStream.h |  18 +-
 .../00937_test_use_header_csv.reference       |  21 ++
 .../0_stateless/00937_test_use_header_csv.sh  |  38 ++
 .../00937_test_use_header_tsv.reference       |  21 ++
 .../0_stateless/00937_test_use_header_tsv.sh  |  38 ++
 11 files changed, 577 insertions(+), 196 deletions(-)
 create mode 100644 dbms/tests/queries/0_stateless/00937_test_use_header_csv.reference
 create mode 100644 dbms/tests/queries/0_stateless/00937_test_use_header_csv.sh
 create mode 100644 dbms/tests/queries/0_stateless/00937_test_use_header_tsv.reference
 create mode 100644 dbms/tests/queries/0_stateless/00937_test_use_header_tsv.sh

diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h
index 2f571d82f22..19278e76515 100644
--- a/dbms/src/Core/Settings.h
+++ b/dbms/src/Core/Settings.h
@@ -153,7 +153,8 @@ struct Settings
     \
     M(SettingBool, add_http_cors_header, false, "Write add http CORS header.") \
     \
-    M(SettingBool, input_format_skip_unknown_fields, false, "Skip columns with unknown names from input data (it works for JSONEachRow and TSKV formats).") \
+    M(SettingBool, input_format_skip_unknown_fields, false, "Skip columns with unknown names from input data (it works for JSONEachRow, CSVWithNames, TSVWithNames and TSKV formats).") \
+    M(SettingBool, input_format_with_names_use_header, false, "For TSVWithNames and CSVWithNames input formats this controls whether format parser is to assume that column data appear in the input exactly as they are specified in the header.") \
     M(SettingBool, input_format_import_nested_json, false, "Map nested JSON data to nested tables (it works for JSONEachRow format).") \
     M(SettingBool, input_format_defaults_for_omitted_fields, false, "For input data calculate default expressions for omitted fields (it works for JSONEachRow format).") \
     \
diff --git a/dbms/src/Formats/CSVRowInputStream.cpp b/dbms/src/Formats/CSVRowInputStream.cpp
index 1c9f32095a0..22e0c28206c 100644
--- a/dbms/src/Formats/CSVRowInputStream.cpp
+++ b/dbms/src/Formats/CSVRowInputStream.cpp
@@ -19,17 +19,7 @@ namespace ErrorCodes
 }
 
 
-CSVRowInputStream::CSVRowInputStream(ReadBuffer & istr_, const Block & header_, bool with_names_, const FormatSettings & format_settings)
-    : istr(istr_), header(header_), with_names(with_names_), format_settings(format_settings)
-{
-    size_t num_columns = header.columns();
-    data_types.resize(num_columns);
-    for (size_t i = 0; i < num_columns; ++i)
-        data_types[i] = header.safeGetByPosition(i).type;
-}
-
-
-static void skipEndOfLine(ReadBuffer & istr)
+static inline void skipEndOfLine(ReadBuffer & istr)
 {
     /// \n (Unix) or \r\n (DOS/Windows) or \n\r (Mac OS Classic)
 
@@ -53,7 +43,7 @@ static void skipEndOfLine(ReadBuffer & istr)
 }
 
 
-static void skipDelimiter(ReadBuffer & istr, const char delimiter, bool is_last_column)
+static inline void skipDelimiter(ReadBuffer & istr, const char delimiter, bool is_last_column)
 {
     if (is_last_column)
     {
@@ -99,38 +89,149 @@ static void skipRow(ReadBuffer & istr, const FormatSettings::CSV & settings, siz
 }
 
 
+CSVRowInputStream::CSVRowInputStream(ReadBuffer & istr_, const Block & header_, bool with_names_, const FormatSettings & format_settings)
+    : istr(istr_), header(header_), with_names(with_names_), format_settings(format_settings)
+{
+    const auto num_columns = header.columns();
+
+    data_types.resize(num_columns);
+    column_indexes_by_names.reserve(num_columns);
+
+    for (size_t i = 0; i < num_columns; ++i)
+    {
+        const auto & column_info = header.getByPosition(i);
+
+        data_types[i] = column_info.type;
+        column_indexes_by_names.emplace(column_info.name, i);
+    }
+
+    input_fields_with_indexes.reserve(num_columns);
+    read_columns.assign(num_columns, false);
+}
+
+
+void CSVRowInputStream::setupAllColumnsByTableSchema()
+{
+    read_columns.assign(header.columns(), true);
+    input_fields_with_indexes.resize(header.columns());
+
+    for (size_t i = 0; i < input_fields_with_indexes.size(); ++i)
+    {
+        input_fields_with_indexes[i] = i;
+    }
+}
+
+
+void CSVRowInputStream::addInputColumn(const String & column_name) {
+    const auto column_it = column_indexes_by_names.find(column_name);
+    if (column_it == column_indexes_by_names.end())
+    {
+        if (format_settings.skip_unknown_fields)
+        {
+            input_fields_with_indexes.push_back(std::nullopt);
+            return;
+        }
+
+        throw Exception(
+            "Unknown field found in CSV header: '" + column_name + "' " +
+            "at position " + std::to_string(input_fields_with_indexes.size()) +
+            "\nSet the 'input_format_skip_unknown_fields' parameter explicitly to ignore and proceed",
+            ErrorCodes::INCORRECT_DATA
+        );
+    }
+
+    const auto column_index = column_it->second;
+
+    if (read_columns[column_index])
+        throw Exception("Duplicate field found while parsing CSV header: " + column_name, ErrorCodes::INCORRECT_DATA);
+
+    read_columns[column_index] = true;
+    input_fields_with_indexes.emplace_back(column_index);
+}
+
+
+void CSVRowInputStream::fillUnreadColumnsWithDefaults(MutableColumns & columns, RowReadExtension & row_read_extension)
+{
+    /// It is safe to memorize this on the first run - the format guarantees this does not change
+    if (unlikely(row_num == 1))
+    {
+        columns_to_fill_with_default_values.clear();
+        for (size_t index = 0; index < read_columns.size(); ++index)
+            if (read_columns[index] == 0)
+                columns_to_fill_with_default_values.push_back(index);
+    }
+
+    for (const auto column_index : columns_to_fill_with_default_values)
+        data_types[column_index]->insertDefaultInto(*columns[column_index]);
+
+    row_read_extension.read_columns = read_columns;
+}
+
+
 void CSVRowInputStream::readPrefix()
 {
     /// In this format, we assume, that if first string field contain BOM as value, it will be written in quotes,
     ///  so BOM at beginning of stream cannot be confused with BOM in first string value, and it is safe to skip it.
     skipBOMIfExists(istr);
 
-    size_t num_columns = data_types.size();
-    String tmp;
-
     if (with_names)
-        skipRow(istr, format_settings.csv, num_columns);
+    {
+        if (format_settings.with_names_use_header)
+        {
+            do
+            {
+                String column_name;
+
+                skipWhitespacesAndTabs(istr);
+                readCSVString(column_name, istr, format_settings.csv);
+                skipWhitespacesAndTabs(istr);
+
+                addInputColumn(column_name);
+            }
+            while(checkChar(format_settings.csv.delimiter, istr));
+
+            skipDelimiter(istr, format_settings.csv.delimiter, true);
+        }
+        else
+        {
+            setupAllColumnsByTableSchema();
+            skipRow(istr, format_settings.csv, input_fields_with_indexes.size());
+        }
+    }
+    else
+    {
+        setupAllColumnsByTableSchema();
+    }
 }
 
 
-bool CSVRowInputStream::read(MutableColumns & columns, RowReadExtension &)
+bool CSVRowInputStream::read(MutableColumns & columns, RowReadExtension & ext)
 {
     if (istr.eof())
         return false;
 
     updateDiagnosticInfo();
 
-    size_t size = data_types.size();
-
-    for (size_t i = 0; i < size; ++i)
+    String tmp;
+    for (size_t input_position = 0; input_position < input_fields_with_indexes.size(); ++input_position)
     {
-        skipWhitespacesAndTabs(istr);
-        data_types[i]->deserializeAsTextCSV(*columns[i], istr, format_settings);
-        skipWhitespacesAndTabs(istr);
+        const auto & column_index = input_fields_with_indexes[input_position];
+        if (column_index)
+        {
+            skipWhitespacesAndTabs(istr);
+            data_types[*column_index]->deserializeAsTextCSV(*columns[*column_index], istr, format_settings);
+            skipWhitespacesAndTabs(istr);
+        }
+        else
+        {
+            readCSVString(tmp, istr, format_settings.csv);
+        }
 
-        skipDelimiter(istr, format_settings.csv.delimiter, i + 1 == size);
+        skipDelimiter(istr, format_settings.csv.delimiter, input_position + 1 == input_fields_with_indexes.size());
     }
 
+    fillUnreadColumnsWithDefaults(columns, ext);
+
     return true;
 }
 
@@ -202,86 +303,101 @@ bool OPTIMIZE(1) CSVRowInputStream::parseRowAndPrintDiagnosticInfo(MutableColumn
 {
     const char delimiter = format_settings.csv.delimiter;
 
-    size_t size = data_types.size();
-    for (size_t i = 0; i < size; ++i)
+    for (size_t input_position = 0; input_position < input_fields_with_indexes.size(); ++input_position)
     {
-        if (i == 0 && istr.eof())
+        if (input_position == 0 && istr.eof())
         {
             out << "<End of stream>\n";
             return false;
         }
 
-        out << "Column " << i << ", " << std::string((i < 10 ? 2 : i < 100 ? 1 : 0), ' ')
-            << "name: " << header.safeGetByPosition(i).name << ", " << std::string(max_length_of_column_name - header.safeGetByPosition(i).name.size(), ' ')
-            << "type: " << data_types[i]->getName() << ", " << std::string(max_length_of_data_type_name - data_types[i]->getName().size(), ' ');
-
-        BufferBase::Position prev_position = istr.position();
-        BufferBase::Position curr_position = istr.position();
-        std::exception_ptr exception;
-
-        try
+        if (input_fields_with_indexes[input_position].has_value())
         {
-            skipWhitespacesAndTabs(istr);
-            prev_position = istr.position();
-            data_types[i]->deserializeAsTextCSV(*columns[i], istr, format_settings);
-            curr_position = istr.position();
-            skipWhitespacesAndTabs(istr);
-        }
-        catch (...)
-        {
-            exception = std::current_exception();
-        }
+            const auto & column_index = *input_fields_with_indexes[input_position];
+            const auto & current_column_type = data_types[column_index];
 
-        if (curr_position < prev_position)
-            throw Exception("Logical error: parsing is non-deterministic.", ErrorCodes::LOGICAL_ERROR);
+            out << "Column " << input_position << ", " << std::string((input_position < 10 ? 2 : input_position < 100 ? 1 : 0), ' ')
+                << "name: " << header.safeGetByPosition(column_index).name << ", " << std::string(max_length_of_column_name - header.safeGetByPosition(column_index).name.size(), ' ')
+                << "type: " << current_column_type->getName() << ", " << std::string(max_length_of_data_type_name - current_column_type->getName().size(), ' ');
 
-        if (isNumber(data_types[i]) || isDateOrDateTime(data_types[i]))
-        {
-            /// An empty string instead of a value.
-            if (curr_position == prev_position)
+            BufferBase::Position prev_position = istr.position();
+            BufferBase::Position curr_position = istr.position();
+            std::exception_ptr exception;
+
+            try
             {
-                out << "ERROR: text ";
-                verbosePrintString(prev_position, std::min(prev_position + 10, istr.buffer().end()), out);
-                out << " is not like " << data_types[i]->getName() << "\n";
-                return false;
+                skipWhitespacesAndTabs(istr);
+                prev_position = istr.position();
+                current_column_type->deserializeAsTextCSV(*columns[column_index], istr, format_settings);
+                curr_position = istr.position();
+                skipWhitespacesAndTabs(istr);
             }
-        }
-
-        out << "parsed text: ";
-        verbosePrintString(prev_position, curr_position, out);
-
-        if (exception)
-        {
-            if (data_types[i]->getName() == "DateTime")
-                out << "ERROR: DateTime must be in YYYY-MM-DD hh:mm:ss or NNNNNNNNNN (unix timestamp, exactly 10 digits) format.\n";
-            else if (data_types[i]->getName() == "Date")
-                out << "ERROR: Date must be in YYYY-MM-DD format.\n";
-            else
-                out << "ERROR\n";
-            return false;
-        }
-
-        out << "\n";
-
-        if (data_types[i]->haveMaximumSizeOfValue())
-        {
-            if (*curr_position != '\n' && *curr_position != '\r' && *curr_position != delimiter)
+            catch (...)
             {
-                out << "ERROR: garbage after " << data_types[i]->getName() << ": ";
-                verbosePrintString(curr_position, std::min(curr_position + 10, istr.buffer().end()), out);
-                out << "\n";
+                exception = std::current_exception();
+            }
 
-                if (data_types[i]->getName() == "DateTime")
+            if (curr_position < prev_position)
+                throw Exception("Logical error: parsing is non-deterministic.", ErrorCodes::LOGICAL_ERROR);
+
+            if (isNumber(current_column_type) || isDateOrDateTime(current_column_type))
+            {
+                /// An empty string instead of a value.
+                if (curr_position == prev_position)
+                {
+                    out << "ERROR: text ";
+                    verbosePrintString(prev_position, std::min(prev_position + 10, istr.buffer().end()), out);
+                    out << " is not like " << current_column_type->getName() << "\n";
+                    return false;
+                }
+            }
+
+            out << "parsed text: ";
+            verbosePrintString(prev_position, curr_position, out);
+
+            if (exception)
+            {
+                if (current_column_type->getName() == "DateTime")
                     out << "ERROR: DateTime must be in YYYY-MM-DD hh:mm:ss or NNNNNNNNNN (unix timestamp, exactly 10 digits) format.\n";
-                else if (data_types[i]->getName() == "Date")
+                else if (current_column_type->getName() == "Date")
                     out << "ERROR: Date must be in YYYY-MM-DD format.\n";
-
+                else
+                    out << "ERROR\n";
                 return false;
             }
+
+            out << "\n";
+
+            if (current_column_type->haveMaximumSizeOfValue())
+            {
+                if (*curr_position != '\n' && *curr_position != '\r' && *curr_position != delimiter)
+                {
+                    out << "ERROR: garbage after " << current_column_type->getName() << ": ";
+                    verbosePrintString(curr_position, std::min(curr_position + 10, istr.buffer().end()), out);
+                    out << "\n";
+
+                    if (current_column_type->getName() == "DateTime")
+                        out << "ERROR: DateTime must be in YYYY-MM-DD hh:mm:ss or NNNNNNNNNN (unix timestamp, exactly 10 digits) format.\n";
+                    else if (current_column_type->getName() == "Date")
+                        out << "ERROR: Date must be in YYYY-MM-DD format.\n";
+
+                    return false;
+                }
+            }
+        }
+        else
+        {
+            static const String skipped_column_str = "<SKIPPED COLUMN>";
+            out << "Column " << input_position << ", " << std::string((input_position < 10 ? 2 : input_position < 100 ? 1 : 0), ' ')
+                << "name: " << skipped_column_str << ", " << std::string(max_length_of_column_name - skipped_column_str.length(), ' ')
+                << "type: " << skipped_column_str << ", " << std::string(max_length_of_data_type_name - skipped_column_str.length(), ' ');
+
+            String tmp;
+            readCSVString(tmp, istr, format_settings.csv);
         }
 
         /// Delimiters
-        if (i + 1 == size)
+        if (input_position + 1 == input_fields_with_indexes.size())
         {
             if (istr.eof())
                 return false;
diff --git a/dbms/src/Formats/CSVRowInputStream.h b/dbms/src/Formats/CSVRowInputStream.h
index c04bda57008..a4748e7954a 100644
--- a/dbms/src/Formats/CSVRowInputStream.h
+++ b/dbms/src/Formats/CSVRowInputStream.h
@@ -21,7 +21,7 @@ public:
       */
     CSVRowInputStream(ReadBuffer & istr_, const Block & header_, bool with_names_, const FormatSettings & format_settings);
 
-    bool read(MutableColumns & columns, RowReadExtension &) override;
+    bool read(MutableColumns & columns, RowReadExtension & ext) override;
     void readPrefix() override;
     bool allowSyncAfterError() const override { return true; }
     void syncAfterError() override;
@@ -36,6 +36,19 @@ private:
 
     const FormatSettings format_settings;
 
+    using IndexesMap = std::unordered_map<String, size_t>;
+    IndexesMap column_indexes_by_names;
+
+    using OptionalIndexes = std::vector<std::optional<size_t>>;
+    OptionalIndexes input_fields_with_indexes;
+
+    std::vector<UInt8> read_columns;
+    std::vector<size_t> columns_to_fill_with_default_values;
+
+    void addInputColumn(const String & column_name);
+    void setupAllColumnsByTableSchema();
+    void fillUnreadColumnsWithDefaults(MutableColumns & columns, RowReadExtension& ext);
+
     /// For convenient diagnostics in case of an error.
 
     size_t row_num = 0;
diff --git a/dbms/src/Formats/FormatFactory.cpp b/dbms/src/Formats/FormatFactory.cpp
index 8086236db53..faf3fd7157c 100644
--- a/dbms/src/Formats/FormatFactory.cpp
+++ b/dbms/src/Formats/FormatFactory.cpp
@@ -40,6 +40,7 @@ BlockInputStreamPtr FormatFactory::getInput(const String & name, ReadBuffer & bu
     format_settings.csv.allow_single_quotes = settings.format_csv_allow_single_quotes;
     format_settings.csv.allow_double_quotes = settings.format_csv_allow_double_quotes;
     format_settings.values.interpret_expressions = settings.input_format_values_interpret_expressions;
+    format_settings.with_names_use_header = settings.input_format_with_names_use_header;
     format_settings.skip_unknown_fields = settings.input_format_skip_unknown_fields;
     format_settings.import_nested_json = settings.input_format_import_nested_json;
     format_settings.date_time_input_format = settings.date_time_input_format;
diff --git a/dbms/src/Formats/FormatSettings.h b/dbms/src/Formats/FormatSettings.h
index a2d787193f5..afddd047123 100644
--- a/dbms/src/Formats/FormatSettings.h
+++ b/dbms/src/Formats/FormatSettings.h
@@ -48,6 +48,7 @@ struct FormatSettings
     Values values;
 
     bool skip_unknown_fields = false;
+    bool with_names_use_header = false;
     bool write_statistics = true;
     bool import_nested_json = false;
 
diff --git a/dbms/src/Formats/TabSeparatedRowInputStream.cpp b/dbms/src/Formats/TabSeparatedRowInputStream.cpp
index f63461df3c4..70644b8c0bf 100644
--- a/dbms/src/Formats/TabSeparatedRowInputStream.cpp
+++ b/dbms/src/Formats/TabSeparatedRowInputStream.cpp
@@ -1,3 +1,5 @@
+#include <string>
+
 #include <Core/Defines.h>
 
 #include <IO/ReadHelpers.h>
@@ -20,46 +22,14 @@ namespace ErrorCodes
 }
 
 
-TabSeparatedRowInputStream::TabSeparatedRowInputStream(
-    ReadBuffer & istr_, const Block & header_, bool with_names_, bool with_types_, const FormatSettings & format_settings)
-    : istr(istr_), header(header_), with_names(with_names_), with_types(with_types_), format_settings(format_settings)
+static void skipTSVRow(ReadBuffer & istr, const size_t num_columns)
 {
-    size_t num_columns = header.columns();
-    data_types.resize(num_columns);
+    NullSink null_sink;
+
     for (size_t i = 0; i < num_columns; ++i)
-        data_types[i] = header.safeGetByPosition(i).type;
-}
-
-
-void TabSeparatedRowInputStream::readPrefix()
-{
-    size_t num_columns = header.columns();
-    String tmp;
-
-    if (with_names || with_types)
     {
-        /// In this format, we assume that column name or type cannot contain BOM,
-        ///  so, if format has header,
-        ///  then BOM at beginning of stream cannot be confused with name or type of field, and it is safe to skip it.
-        skipBOMIfExists(istr);
-    }
-
-    if (with_names)
-    {
-        for (size_t i = 0; i < num_columns; ++i)
-        {
-            readEscapedString(tmp, istr);
-            assertChar(i == num_columns - 1 ? '\n' : '\t', istr);
-        }
-    }
-
-    if (with_types)
-    {
-        for (size_t i = 0; i < num_columns; ++i)
-        {
-            readEscapedString(tmp, istr);
-            assertChar(i == num_columns - 1 ? '\n' : '\t', istr);
-        }
+        readEscapedStringInto(null_sink, istr);
+        assertChar(i == num_columns - 1 ? '\n' : '\t', istr);
     }
 }
 
@@ -77,34 +47,164 @@ static void checkForCarriageReturn(ReadBuffer & istr)
 }
 
 
-bool TabSeparatedRowInputStream::read(MutableColumns & columns, RowReadExtension &)
+TabSeparatedRowInputStream::TabSeparatedRowInputStream(
+    ReadBuffer & istr_, const Block & header_, bool with_names_, bool with_types_, const FormatSettings & format_settings)
+    : istr(istr_), header(header_), with_names(with_names_), with_types(with_types_), format_settings(format_settings)
+{
+    const auto num_columns = header.columns();
+
+    data_types.resize(num_columns);
+    column_indexes_by_names.reserve(num_columns);
+
+    for (size_t i = 0; i < num_columns; ++i)
+    {
+        const auto & column_info = header.getByPosition(i);
+
+        data_types[i] = column_info.type;
+        column_indexes_by_names.emplace(column_info.name, i);
+    }
+
+    input_fields_with_indexes.reserve(num_columns);
+    read_columns.assign(num_columns, false);
+}
+
+
+void TabSeparatedRowInputStream::setupAllColumnsByTableSchema()
+{
+    read_columns.assign(header.columns(), true);
+    input_fields_with_indexes.resize(header.columns());
+
+    for (size_t i = 0; i < input_fields_with_indexes.size(); ++i)
+        input_fields_with_indexes[i] = i;
+}
+
+
+void TabSeparatedRowInputStream::addInputColumn(const String & column_name) {
+    const auto column_it = column_indexes_by_names.find(column_name);
+    if (column_it == column_indexes_by_names.end())
+    {
+        if (format_settings.skip_unknown_fields)
+        {
+            input_fields_with_indexes.push_back(std::nullopt);
+            return;
+        }
+
+        throw Exception(
+            "Unknown field found in TSV header: '" + column_name + "' " +
+            "at position " + std::to_string(input_fields_with_indexes.size()) +
+            "\nSet the 'input_format_skip_unknown_fields' parameter explicitly to ignore and proceed",
+            ErrorCodes::INCORRECT_DATA
+        );
+    }
+
+    const auto column_index = column_it->second;
+
+    if (read_columns[column_index])
+        throw Exception("Duplicate field found while parsing TSV header: " + column_name, ErrorCodes::INCORRECT_DATA);
+
+    read_columns[column_index] = true;
+    input_fields_with_indexes.emplace_back(column_index);
+}
+
+
+void TabSeparatedRowInputStream::fillUnreadColumnsWithDefaults(MutableColumns & columns, RowReadExtension & row_read_extension)
+{
+    /// It is safe to memorize this on the first run - the format guarantees this does not change
+    if (unlikely(row_num == 1))
+    {
+        columns_to_fill_with_default_values.clear();
+        for (size_t index = 0; index < read_columns.size(); ++index)
+            if (read_columns[index] == 0)
+                columns_to_fill_with_default_values.push_back(index);
+    }
+
+    for (const auto column_index : columns_to_fill_with_default_values)
+        data_types[column_index]->insertDefaultInto(*columns[column_index]);
+
+    row_read_extension.read_columns = read_columns;
+}
+
+
+void TabSeparatedRowInputStream::readPrefix()
+{
+    if (with_names || with_types)
+    {
+        /// In this format, we assume that column name or type cannot contain BOM,
+        ///  so, if format has header,
+        ///  then BOM at beginning of stream cannot be confused with name or type of field, and it is safe to skip it.
+        skipBOMIfExists(istr);
+    }
+
+    if (with_names)
+    {
+        if (format_settings.with_names_use_header)
+        {
+            do
+            {
+                String column_name;
+                readEscapedString(column_name, istr);
+                addInputColumn(column_name);
+            }
+            while(checkChar('\t', istr));
+
+            if (!istr.eof())
+            {
+                checkForCarriageReturn(istr);
+                assertChar('\n', istr);
+            }
+        }
+        else
+        {
+            setupAllColumnsByTableSchema();
+            skipTSVRow(istr, input_fields_with_indexes.size());
+        }
+    }
+    else
+        setupAllColumnsByTableSchema();
+
+    if (with_types)
+    {
+        skipTSVRow(istr, input_fields_with_indexes.size());
+    }
+}
+
+
+bool TabSeparatedRowInputStream::read(MutableColumns & columns, RowReadExtension & ext)
 {
     if (istr.eof())
         return false;
 
     updateDiagnosticInfo();
 
-    size_t size = data_types.size();
-
-    for (size_t i = 0; i < size; ++i)
+    for (size_t input_position = 0; input_position < input_fields_with_indexes.size(); ++input_position)
     {
-        data_types[i]->deserializeAsTextEscaped(*columns[i], istr, format_settings);
-
-        /// skip separators
-        if (i + 1 == size)
+        const auto & column_index = input_fields_with_indexes[input_position];
+        if (column_index)
         {
-            if (!istr.eof())
-            {
-                if (unlikely(row_num == 1))
-                    checkForCarriageReturn(istr);
-
-                assertChar('\n', istr);
-            }
+            data_types[*column_index]->deserializeAsTextEscaped(*columns[*column_index], istr, format_settings);
         }
         else
+        {
+            NullSink null_sink;
+            readEscapedStringInto(null_sink, istr);
+        }
+
+        /// skip separators
+        if (input_position + 1 < input_fields_with_indexes.size())
+        {
             assertChar('\t', istr);
+        }
+        else if (!istr.eof())
+        {
+            if (unlikely(row_num == 1))
+                checkForCarriageReturn(istr);
+
+            assertChar('\n', istr);
+        }
     }
 
+    fillUnreadColumnsWithDefaults(columns, ext);
+
     return true;
 }
 
@@ -135,7 +235,7 @@ String TabSeparatedRowInputStream::getDiagnosticInfo()
         if (header.safeGetByPosition(i).type->getName().size() > max_length_of_data_type_name)
             max_length_of_data_type_name = header.safeGetByPosition(i).type->getName().size();
 
-    /// Roll back the cursor to the beginning of the previous or current line and pars all over again. But now we derive detailed information.
+    /// Roll back the cursor to the beginning of the previous or current line and parse all over again. But now we derive detailed information.
 
     if (pos_of_prev_row)
     {
@@ -173,83 +273,98 @@ String TabSeparatedRowInputStream::getDiagnosticInfo()
 bool OPTIMIZE(1) TabSeparatedRowInputStream::parseRowAndPrintDiagnosticInfo(
     MutableColumns & columns, WriteBuffer & out, size_t max_length_of_column_name, size_t max_length_of_data_type_name)
 {
-    size_t size = data_types.size();
-    for (size_t i = 0; i < size; ++i)
+    for (size_t input_position = 0; input_position < input_fields_with_indexes.size(); ++input_position)
     {
-        if (i == 0 && istr.eof())
+        if (input_position == 0 && istr.eof())
         {
             out << "<End of stream>\n";
             return false;
         }
 
-        out << "Column " << i << ", " << std::string((i < 10 ? 2 : i < 100 ? 1 : 0), ' ')
-            << "name: " << header.safeGetByPosition(i).name << ", " << std::string(max_length_of_column_name - header.safeGetByPosition(i).name.size(), ' ')
-            << "type: " << data_types[i]->getName() << ", " << std::string(max_length_of_data_type_name - data_types[i]->getName().size(), ' ');
-
-        auto prev_position = istr.position();
-        std::exception_ptr exception;
-
-        try
+        if (input_fields_with_indexes[input_position].has_value())
         {
-            data_types[i]->deserializeAsTextEscaped(*columns[i], istr, format_settings);
-        }
-        catch (...)
-        {
-            exception = std::current_exception();
-        }
+            const auto & column_index = *input_fields_with_indexes[input_position];
+            const auto & current_column_type = data_types[column_index];
 
-        auto curr_position = istr.position();
+            out << "Column " << input_position << ", " << std::string((input_position < 10 ? 2 : input_position < 100 ? 1 : 0), ' ')
+                << "name: " << header.safeGetByPosition(column_index).name << ", " << std::string(max_length_of_column_name - header.safeGetByPosition(column_index).name.size(), ' ')
+                << "type: " << current_column_type->getName() << ", " << std::string(max_length_of_data_type_name - current_column_type->getName().size(), ' ');
 
-        if (curr_position < prev_position)
-            throw Exception("Logical error: parsing is non-deterministic.", ErrorCodes::LOGICAL_ERROR);
+            auto prev_position = istr.position();
+            std::exception_ptr exception;
 
-        if (isNumber(data_types[i]) || isDateOrDateTime(data_types[i]))
-        {
-            /// An empty string instead of a value.
-            if (curr_position == prev_position)
+            try
             {
-                out << "ERROR: text ";
-                verbosePrintString(prev_position, std::min(prev_position + 10, istr.buffer().end()), out);
-                out << " is not like " << data_types[i]->getName() << "\n";
-                return false;
+                current_column_type->deserializeAsTextEscaped(*columns[column_index], istr, format_settings);
             }
-        }
-
-        out << "parsed text: ";
-        verbosePrintString(prev_position, curr_position, out);
-
-        if (exception)
-        {
-            if (data_types[i]->getName() == "DateTime")
-                out << "ERROR: DateTime must be in YYYY-MM-DD hh:mm:ss or NNNNNNNNNN (unix timestamp, exactly 10 digits) format.\n";
-            else if (data_types[i]->getName() == "Date")
-                out << "ERROR: Date must be in YYYY-MM-DD format.\n";
-            else
-                out << "ERROR\n";
-            return false;
-        }
-
-        out << "\n";
-
-        if (data_types[i]->haveMaximumSizeOfValue())
-        {
-            if (*curr_position != '\n' && *curr_position != '\t')
+            catch (...)
             {
-                out << "ERROR: garbage after " << data_types[i]->getName() << ": ";
-                verbosePrintString(curr_position, std::min(curr_position + 10, istr.buffer().end()), out);
-                out << "\n";
+                exception = std::current_exception();
+            }
 
-                if (data_types[i]->getName() == "DateTime")
+            auto curr_position = istr.position();
+
+            if (curr_position < prev_position)
+                throw Exception("Logical error: parsing is non-deterministic.", ErrorCodes::LOGICAL_ERROR);
+
+            if (isNumber(current_column_type) || isDateOrDateTime(current_column_type))
+            {
+                /// An empty string instead of a value.
+                if (curr_position == prev_position)
+                {
+                    out << "ERROR: text ";
+                    verbosePrintString(prev_position, std::min(prev_position + 10, istr.buffer().end()), out);
+                    out << " is not like " << current_column_type->getName() << "\n";
+                    return false;
+                }
+            }
+
+            out << "parsed text: ";
+            verbosePrintString(prev_position, curr_position, out);
+
+            if (exception)
+            {
+                if (current_column_type->getName() == "DateTime")
                     out << "ERROR: DateTime must be in YYYY-MM-DD hh:mm:ss or NNNNNNNNNN (unix timestamp, exactly 10 digits) format.\n";
-                else if (data_types[i]->getName() == "Date")
+                else if (current_column_type->getName() == "Date")
                     out << "ERROR: Date must be in YYYY-MM-DD format.\n";
-
+                else
+                    out << "ERROR\n";
                 return false;
             }
+
+            out << "\n";
+
+            if (current_column_type->haveMaximumSizeOfValue())
+            {
+                if (*curr_position != '\n' && *curr_position != '\t')
+                {
+                    out << "ERROR: garbage after " << current_column_type->getName() << ": ";
+                    verbosePrintString(curr_position, std::min(curr_position + 10, istr.buffer().end()), out);
+                    out << "\n";
+
+                    if (current_column_type->getName() == "DateTime")
+                        out << "ERROR: DateTime must be in YYYY-MM-DD hh:mm:ss or NNNNNNNNNN (unix timestamp, exactly 10 digits) format.\n";
+                    else if (current_column_type->getName() == "Date")
+                        out << "ERROR: Date must be in YYYY-MM-DD format.\n";
+
+                    return false;
+                }
+            }
+        }
+        else
+        {
+            static const String skipped_column_str = "<SKIPPED COLUMN>";
+            out << "Column " << input_position << ", " << std::string((input_position < 10 ? 2 : input_position < 100 ? 1 : 0), ' ')
+                << "name: " << skipped_column_str << ", " << std::string(max_length_of_column_name - skipped_column_str.length(), ' ')
+                << "type: " << skipped_column_str << ", " << std::string(max_length_of_data_type_name - skipped_column_str.length(), ' ');
+
+            NullSink null_sink;
+            readEscapedStringInto(null_sink, istr);
         }
 
         /// Delimiters
-        if (i + 1 == size)
+        if (input_position + 1 == input_fields_with_indexes.size())
         {
             if (!istr.eof())
             {
diff --git a/dbms/src/Formats/TabSeparatedRowInputStream.h b/dbms/src/Formats/TabSeparatedRowInputStream.h
index 2435d58d703..f8cfba4e331 100644
--- a/dbms/src/Formats/TabSeparatedRowInputStream.h
+++ b/dbms/src/Formats/TabSeparatedRowInputStream.h
@@ -1,5 +1,8 @@
 #pragma once
 
+#include <optional>
+#include <unordered_map>
+
 #include <Core/Block.h>
 #include <Formats/FormatSettings.h>
 #include <Formats/IRowInputStream.h>
@@ -22,7 +25,7 @@ public:
     TabSeparatedRowInputStream(
         ReadBuffer & istr_, const Block & header_, bool with_names_, bool with_types_, const FormatSettings & format_settings);
 
-    bool read(MutableColumns & columns, RowReadExtension &) override;
+    bool read(MutableColumns & columns, RowReadExtension & ext) override;
     void readPrefix() override;
     bool allowSyncAfterError() const override { return true; }
     void syncAfterError() override;
@@ -37,6 +40,19 @@ private:
     const FormatSettings format_settings;
     DataTypes data_types;
 
+    using IndexesMap = std::unordered_map<String, size_t>;
+    IndexesMap column_indexes_by_names;
+
+    using OptionalIndexes = std::vector<std::optional<size_t>>;
+    OptionalIndexes input_fields_with_indexes;
+
+    std::vector<UInt8> read_columns;
+    std::vector<size_t> columns_to_fill_with_default_values;
+
+    void addInputColumn(const String & column_name);
+    void setupAllColumnsByTableSchema();
+    void fillUnreadColumnsWithDefaults(MutableColumns & columns, RowReadExtension& ext);
+
     /// For convenient diagnostics in case of an error.
 
     size_t row_num = 0;
diff --git a/dbms/tests/queries/0_stateless/00937_test_use_header_csv.reference b/dbms/tests/queries/0_stateless/00937_test_use_header_csv.reference
new file mode 100644
index 00000000000..8a8b605ad41
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00937_test_use_header_csv.reference
@@ -0,0 +1,21 @@
+2019-04-18	42	Line1
+2019-04-18	42	Line2
+2019-04-18	42	Line3
+2019-04-18	42	Line4
+2019-04-18	42	Line5
+2019-04-18	42	Line6
+2019-04-18	42	Line7
+2019-04-18	42	Line8
+2019-04-18	42	Line9
+2019-04-18	1	Line10
+2019-04-18	2	Line11
+2019-04-18	1	Line12
+2019-04-18	2	Line13
+2019-04-18	1	
+2019-04-18	2	
+0000-00-00	1	Line16
+0000-00-00	2	Line17
+2019-04-18	0	Line18
+2019-04-18	0	Line19
+0000-00-00	0	
+0000-00-00	0	
diff --git a/dbms/tests/queries/0_stateless/00937_test_use_header_csv.sh b/dbms/tests/queries/0_stateless/00937_test_use_header_csv.sh
new file mode 100644
index 00000000000..28d8282e291
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00937_test_use_header_csv.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+#CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+#. $CURDIR/../shell_config.sh
+
+CLICKHOUSE_CLIENT=./clickhouse-client
+
+$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.csv"
+$CLICKHOUSE_CLIENT --query="CREATE TABLE test.csv (d Date, u UInt8, str String) ENGINE = TinyLog"
+
+INSERT_QUERY='$CLICKHOUSE_CLIENT --query="INSERT INTO test.csv FORMAT CSVWithNames"'
+USE_HEADER='--input_format_with_names_use_header=1'
+SKIP_UNKNOWN='--input_format_skip_unknown_fields=1'
+
+# - Simple check for parsing
+echo -ne 'd,u,str\n2019-04-18,42,Line1\n2019-04-18,42,Line2' | eval $INSERT_QUERY
+echo -ne 'd,u,str\n2019-04-18,42,Line3\n2019-04-18,42,Line4' | eval $INSERT_QUERY $USE_HEADER
+echo -ne 'd,u,str\n2019-04-18,42,Line5\n2019-04-18,42,Line6' | eval $INSERT_QUERY $USE_HEADER $SKIP_UNKNOWN
+
+# - Check random order of fields
+echo -ne 'u,d,str\n42,2019-04-18,Line7\n' | eval $INSERT_QUERY $USE_HEADER
+echo -ne 'u,str,d\n42,Line8,2019-04-18\n' | eval $INSERT_QUERY $USE_HEADER
+echo -ne 'str,u,d\nLine9,42,2019-04-18\n' | eval $INSERT_QUERY $USE_HEADER
+
+# - Check excessive fields
+echo -ne 'd,u,str,more,unknown,fields\n2019-04-18,1,Line10,,,\n2019-04-18,2,Line11,,,\n' \
+| eval $INSERT_QUERY $USE_HEADER $SKIP_UNKNOWN
+echo -ne 'd,unknown,str,more,u,fields\n2019-04-18,blahblah,Line12,,1,\n2019-04-18,,Line13,blahblah,2,\n' \
+| eval $INSERT_QUERY $USE_HEADER $SKIP_UNKNOWN
+
+# - Check missing fields (defaults)
+echo -ne 'd,u\n2019-04-18,1\n2019-04-18,2\n'            | eval $INSERT_QUERY $USE_HEADER
+echo -ne 'str,u\nLine16,1\nLine17,2\n'                  | eval $INSERT_QUERY $USE_HEADER
+echo -ne 'd,str\n2019-04-18,Line18\n2019-04-18,Line19\n'| eval $INSERT_QUERY $USE_HEADER
+echo -ne 'unknown\n\n\n'                                | eval $INSERT_QUERY $USE_HEADER $SKIP_UNKNOWN
+
+$CLICKHOUSE_CLIENT --query="SELECT * FROM test.csv"
+$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.csv"
diff --git a/dbms/tests/queries/0_stateless/00937_test_use_header_tsv.reference b/dbms/tests/queries/0_stateless/00937_test_use_header_tsv.reference
new file mode 100644
index 00000000000..8a8b605ad41
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00937_test_use_header_tsv.reference
@@ -0,0 +1,21 @@
+2019-04-18	42	Line1
+2019-04-18	42	Line2
+2019-04-18	42	Line3
+2019-04-18	42	Line4
+2019-04-18	42	Line5
+2019-04-18	42	Line6
+2019-04-18	42	Line7
+2019-04-18	42	Line8
+2019-04-18	42	Line9
+2019-04-18	1	Line10
+2019-04-18	2	Line11
+2019-04-18	1	Line12
+2019-04-18	2	Line13
+2019-04-18	1	
+2019-04-18	2	
+0000-00-00	1	Line16
+0000-00-00	2	Line17
+2019-04-18	0	Line18
+2019-04-18	0	Line19
+0000-00-00	0	
+0000-00-00	0	
diff --git a/dbms/tests/queries/0_stateless/00937_test_use_header_tsv.sh b/dbms/tests/queries/0_stateless/00937_test_use_header_tsv.sh
new file mode 100644
index 00000000000..5d7b6f41c70
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00937_test_use_header_tsv.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+#CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+#. $CURDIR/../shell_config.sh
+
+CLICKHOUSE_CLIENT=./clickhouse-client
+
+$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.tsv"
+$CLICKHOUSE_CLIENT --query="CREATE TABLE test.tsv (d Date, u UInt8, str String) ENGINE = TinyLog"
+
+INSERT_QUERY='$CLICKHOUSE_CLIENT --query="INSERT INTO test.tsv FORMAT TSVWithNames"'
+USE_HEADER='--input_format_with_names_use_header=1'
+SKIP_UNKNOWN='--input_format_skip_unknown_fields=1'
+
+# - Simple check for parsing
+echo -ne 'd\tu\tstr\n2019-04-18\t42\tLine1\n2019-04-18\t42\tLine2' | eval $INSERT_QUERY
+echo -ne 'd\tu\tstr\n2019-04-18\t42\tLine3\n2019-04-18\t42\tLine4' | eval $INSERT_QUERY $USE_HEADER
+echo -ne 'd\tu\tstr\n2019-04-18\t42\tLine5\n2019-04-18\t42\tLine6' | eval $INSERT_QUERY $USE_HEADER $SKIP_UNKNOWN
+
+# - Check random order of fields
+echo -ne 'u\td\tstr\n42\t2019-04-18\tLine7\n' | eval $INSERT_QUERY $USE_HEADER
+echo -ne 'u\tstr\td\n42\tLine8\t2019-04-18\n' | eval $INSERT_QUERY $USE_HEADER
+echo -ne 'str\tu\td\nLine9\t42\t2019-04-18\n' | eval $INSERT_QUERY $USE_HEADER
+
+# - Check excessive fields
+echo -ne 'd\tu\tstr\tmore\tunknown\tfields\n2019-04-18\t1\tLine10\t\t\t\n2019-04-18\t2\tLine11\t\t\t\n' \
+| eval $INSERT_QUERY $USE_HEADER $SKIP_UNKNOWN
+echo -ne 'd\tunknown\tstr\tmore\tu\tfields\n2019-04-18\tblahblah\tLine12\t\t1\t\n2019-04-18\t\tLine13\tblahblah\t2\t\n' \
+| eval $INSERT_QUERY $USE_HEADER $SKIP_UNKNOWN
+
+# - Check missing fields (defaults)
+echo -ne 'd\tu\n2019-04-18\t1\n2019-04-18\t2\n'            | eval $INSERT_QUERY $USE_HEADER
+echo -ne 'str\tu\nLine16\t1\nLine17\t2\n'                  | eval $INSERT_QUERY $USE_HEADER
+echo -ne 'd\tstr\n2019-04-18\tLine18\n2019-04-18\tLine19\n'| eval $INSERT_QUERY $USE_HEADER
+echo -ne 'unknown\n\n\n'                                   | eval $INSERT_QUERY $USE_HEADER $SKIP_UNKNOWN
+
+$CLICKHOUSE_CLIENT --query="SELECT * FROM test.tsv"
+$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.tsv"

From bd62651ea39c5c0cf303cdad6f2cafefad7460ba Mon Sep 17 00:00:00 2001
From: Alexander Kazakov <akazz@yandex-team.ru>
Date: Mon, 22 Apr 2019 16:42:52 +0300
Subject: [PATCH 025/118] Dropped a few debug leftovers

---
 .../0_stateless/00937_test_use_header_csv.sh       | 14 ++++++--------
 .../0_stateless/00937_test_use_header_tsv.sh       | 14 ++++++--------
 2 files changed, 12 insertions(+), 16 deletions(-)

diff --git a/dbms/tests/queries/0_stateless/00937_test_use_header_csv.sh b/dbms/tests/queries/0_stateless/00937_test_use_header_csv.sh
index 28d8282e291..bd9204ffdcc 100644
--- a/dbms/tests/queries/0_stateless/00937_test_use_header_csv.sh
+++ b/dbms/tests/queries/0_stateless/00937_test_use_header_csv.sh
@@ -1,9 +1,7 @@
 #!/usr/bin/env bash
 
-#CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-#. $CURDIR/../shell_config.sh
-
-CLICKHOUSE_CLIENT=./clickhouse-client
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+. $CURDIR/../shell_config.sh
 
 $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.csv"
 $CLICKHOUSE_CLIENT --query="CREATE TABLE test.csv (d Date, u UInt8, str String) ENGINE = TinyLog"
@@ -12,23 +10,23 @@ INSERT_QUERY='$CLICKHOUSE_CLIENT --query="INSERT INTO test.csv FORMAT CSVWithNam
 USE_HEADER='--input_format_with_names_use_header=1'
 SKIP_UNKNOWN='--input_format_skip_unknown_fields=1'
 
-# - Simple check for parsing
+# Simple check for parsing
 echo -ne 'd,u,str\n2019-04-18,42,Line1\n2019-04-18,42,Line2' | eval $INSERT_QUERY
 echo -ne 'd,u,str\n2019-04-18,42,Line3\n2019-04-18,42,Line4' | eval $INSERT_QUERY $USE_HEADER
 echo -ne 'd,u,str\n2019-04-18,42,Line5\n2019-04-18,42,Line6' | eval $INSERT_QUERY $USE_HEADER $SKIP_UNKNOWN
 
-# - Check random order of fields
+# Random order of fields
 echo -ne 'u,d,str\n42,2019-04-18,Line7\n' | eval $INSERT_QUERY $USE_HEADER
 echo -ne 'u,str,d\n42,Line8,2019-04-18\n' | eval $INSERT_QUERY $USE_HEADER
 echo -ne 'str,u,d\nLine9,42,2019-04-18\n' | eval $INSERT_QUERY $USE_HEADER
 
-# - Check excessive fields
+# Excessive fields
 echo -ne 'd,u,str,more,unknown,fields\n2019-04-18,1,Line10,,,\n2019-04-18,2,Line11,,,\n' \
 | eval $INSERT_QUERY $USE_HEADER $SKIP_UNKNOWN
 echo -ne 'd,unknown,str,more,u,fields\n2019-04-18,blahblah,Line12,,1,\n2019-04-18,,Line13,blahblah,2,\n' \
 | eval $INSERT_QUERY $USE_HEADER $SKIP_UNKNOWN
 
-# - Check missing fields (defaults)
+# Missing fields (defaults)
 echo -ne 'd,u\n2019-04-18,1\n2019-04-18,2\n'            | eval $INSERT_QUERY $USE_HEADER
 echo -ne 'str,u\nLine16,1\nLine17,2\n'                  | eval $INSERT_QUERY $USE_HEADER
 echo -ne 'd,str\n2019-04-18,Line18\n2019-04-18,Line19\n'| eval $INSERT_QUERY $USE_HEADER
diff --git a/dbms/tests/queries/0_stateless/00937_test_use_header_tsv.sh b/dbms/tests/queries/0_stateless/00937_test_use_header_tsv.sh
index 5d7b6f41c70..ae0d0f264f2 100644
--- a/dbms/tests/queries/0_stateless/00937_test_use_header_tsv.sh
+++ b/dbms/tests/queries/0_stateless/00937_test_use_header_tsv.sh
@@ -1,9 +1,7 @@
 #!/usr/bin/env bash
 
-#CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-#. $CURDIR/../shell_config.sh
-
-CLICKHOUSE_CLIENT=./clickhouse-client
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+. $CURDIR/../shell_config.sh
 
 $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.tsv"
 $CLICKHOUSE_CLIENT --query="CREATE TABLE test.tsv (d Date, u UInt8, str String) ENGINE = TinyLog"
@@ -12,23 +10,23 @@ INSERT_QUERY='$CLICKHOUSE_CLIENT --query="INSERT INTO test.tsv FORMAT TSVWithNam
 USE_HEADER='--input_format_with_names_use_header=1'
 SKIP_UNKNOWN='--input_format_skip_unknown_fields=1'
 
-# - Simple check for parsing
+# Simple check for parsing
 echo -ne 'd\tu\tstr\n2019-04-18\t42\tLine1\n2019-04-18\t42\tLine2' | eval $INSERT_QUERY
 echo -ne 'd\tu\tstr\n2019-04-18\t42\tLine3\n2019-04-18\t42\tLine4' | eval $INSERT_QUERY $USE_HEADER
 echo -ne 'd\tu\tstr\n2019-04-18\t42\tLine5\n2019-04-18\t42\tLine6' | eval $INSERT_QUERY $USE_HEADER $SKIP_UNKNOWN
 
-# - Check random order of fields
+# Random order of fields
 echo -ne 'u\td\tstr\n42\t2019-04-18\tLine7\n' | eval $INSERT_QUERY $USE_HEADER
 echo -ne 'u\tstr\td\n42\tLine8\t2019-04-18\n' | eval $INSERT_QUERY $USE_HEADER
 echo -ne 'str\tu\td\nLine9\t42\t2019-04-18\n' | eval $INSERT_QUERY $USE_HEADER
 
-# - Check excessive fields
+# Excessive fields
 echo -ne 'd\tu\tstr\tmore\tunknown\tfields\n2019-04-18\t1\tLine10\t\t\t\n2019-04-18\t2\tLine11\t\t\t\n' \
 | eval $INSERT_QUERY $USE_HEADER $SKIP_UNKNOWN
 echo -ne 'd\tunknown\tstr\tmore\tu\tfields\n2019-04-18\tblahblah\tLine12\t\t1\t\n2019-04-18\t\tLine13\tblahblah\t2\t\n' \
 | eval $INSERT_QUERY $USE_HEADER $SKIP_UNKNOWN
 
-# - Check missing fields (defaults)
+# Missing fields (defaults)
 echo -ne 'd\tu\n2019-04-18\t1\n2019-04-18\t2\n'            | eval $INSERT_QUERY $USE_HEADER
 echo -ne 'str\tu\nLine16\t1\nLine17\t2\n'                  | eval $INSERT_QUERY $USE_HEADER
 echo -ne 'd\tstr\n2019-04-18\tLine18\n2019-04-18\tLine19\n'| eval $INSERT_QUERY $USE_HEADER

From bde1845bdcb0e20c263870140a4fcad4c11e9acd Mon Sep 17 00:00:00 2001
From: Alexander Kazakov <akazz@yandex-team.ru>
Date: Mon, 22 Apr 2019 17:00:51 +0300
Subject: [PATCH 026/118] Better naming

---
 dbms/src/Formats/CSVRowInputStream.cpp        | 33 +++++++++---------
 dbms/src/Formats/CSVRowInputStream.h          |  2 +-
 .../Formats/TabSeparatedRowInputStream.cpp    | 34 +++++++++----------
 dbms/src/Formats/TabSeparatedRowInputStream.h |  2 +-
 4 files changed, 35 insertions(+), 36 deletions(-)

diff --git a/dbms/src/Formats/CSVRowInputStream.cpp b/dbms/src/Formats/CSVRowInputStream.cpp
index 22e0c28206c..5fa404a8e4d 100644
--- a/dbms/src/Formats/CSVRowInputStream.cpp
+++ b/dbms/src/Formats/CSVRowInputStream.cpp
@@ -105,7 +105,7 @@ CSVRowInputStream::CSVRowInputStream(ReadBuffer & istr_, const Block & header_,
         column_indexes_by_names.emplace(column_info.name, i);
     }
 
-    input_fields_with_indexes.reserve(num_columns);
+    column_indexes_for_input_fields.reserve(num_columns);
     read_columns.assign(num_columns, false);
 }
 
@@ -113,11 +113,11 @@ CSVRowInputStream::CSVRowInputStream(ReadBuffer & istr_, const Block & header_,
 void CSVRowInputStream::setupAllColumnsByTableSchema()
 {
     read_columns.assign(header.columns(), true);
-    input_fields_with_indexes.resize(header.columns());
+    column_indexes_for_input_fields.resize(header.columns());
 
-    for (size_t i = 0; i < input_fields_with_indexes.size(); ++i)
+    for (size_t i = 0; i < column_indexes_for_input_fields.size(); ++i)
     {
-        input_fields_with_indexes[i] = i;
+        column_indexes_for_input_fields[i] = i;
     }
 }
 
@@ -128,13 +128,13 @@ void CSVRowInputStream::addInputColumn(const String & column_name) {
     {
         if (format_settings.skip_unknown_fields)
         {
-            input_fields_with_indexes.push_back(std::nullopt);
+            column_indexes_for_input_fields.push_back(std::nullopt);
             return;
         }
 
         throw Exception(
             "Unknown field found in CSV header: '" + column_name + "' " +
-            "at position " + std::to_string(input_fields_with_indexes.size()) +
+            "at position " + std::to_string(column_indexes_for_input_fields.size()) +
             "\nSet the 'input_format_skip_unknown_fields' parameter explicitly to ignore and proceed",
             ErrorCodes::INCORRECT_DATA
         );
@@ -146,7 +146,7 @@ void CSVRowInputStream::addInputColumn(const String & column_name) {
         throw Exception("Duplicate field found while parsing CSV header: " + column_name, ErrorCodes::INCORRECT_DATA);
 
     read_columns[column_index] = true;
-    input_fields_with_indexes.emplace_back(column_index);
+    column_indexes_for_input_fields.emplace_back(column_index);
 }
 
 
@@ -178,10 +178,9 @@ void CSVRowInputStream::readPrefix()
     {
         if (format_settings.with_names_use_header)
         {
+            String column_name;
             do
             {
-                String column_name;
-
                 skipWhitespacesAndTabs(istr);
                 readCSVString(column_name, istr, format_settings.csv);
                 skipWhitespacesAndTabs(istr);
@@ -195,7 +194,7 @@ void CSVRowInputStream::readPrefix()
         else
         {
             setupAllColumnsByTableSchema();
-            skipRow(istr, format_settings.csv, input_fields_with_indexes.size());
+            skipRow(istr, format_settings.csv, column_indexes_for_input_fields.size());
         }
     }
     else
@@ -213,9 +212,9 @@ bool CSVRowInputStream::read(MutableColumns & columns, RowReadExtension & ext)
     updateDiagnosticInfo();
 
     String tmp;
-    for (size_t input_position = 0; input_position < input_fields_with_indexes.size(); ++input_position)
+    for (size_t input_position = 0; input_position < column_indexes_for_input_fields.size(); ++input_position)
     {
-        const auto & column_index = input_fields_with_indexes[input_position];
+        const auto & column_index = column_indexes_for_input_fields[input_position];
         if (column_index)
         {
             skipWhitespacesAndTabs(istr);
@@ -227,7 +226,7 @@ bool CSVRowInputStream::read(MutableColumns & columns, RowReadExtension & ext)
             readCSVString(tmp, istr, format_settings.csv);
         }
 
-        skipDelimiter(istr, format_settings.csv.delimiter, input_position + 1 == input_fields_with_indexes.size());
+        skipDelimiter(istr, format_settings.csv.delimiter, input_position + 1 == column_indexes_for_input_fields.size());
     }
 
     fillUnreadColumnsWithDefaults(columns, ext);
@@ -303,7 +302,7 @@ bool OPTIMIZE(1) CSVRowInputStream::parseRowAndPrintDiagnosticInfo(MutableColumn
 {
     const char delimiter = format_settings.csv.delimiter;
 
-    for (size_t input_position = 0; input_position < input_fields_with_indexes.size(); ++input_position)
+    for (size_t input_position = 0; input_position < column_indexes_for_input_fields.size(); ++input_position)
     {
         if (input_position == 0 && istr.eof())
         {
@@ -311,9 +310,9 @@ bool OPTIMIZE(1) CSVRowInputStream::parseRowAndPrintDiagnosticInfo(MutableColumn
             return false;
         }
 
-        if (input_fields_with_indexes[input_position].has_value())
+        if (column_indexes_for_input_fields[input_position].has_value())
         {
-            const auto & column_index = *input_fields_with_indexes[input_position];
+            const auto & column_index = *column_indexes_for_input_fields[input_position];
             const auto & current_column_type = data_types[column_index];
 
             out << "Column " << input_position << ", " << std::string((input_position < 10 ? 2 : input_position < 100 ? 1 : 0), ' ')
@@ -397,7 +396,7 @@ bool OPTIMIZE(1) CSVRowInputStream::parseRowAndPrintDiagnosticInfo(MutableColumn
         }
 
         /// Delimiters
-        if (input_position + 1 == input_fields_with_indexes.size())
+        if (input_position + 1 == column_indexes_for_input_fields.size())
         {
             if (istr.eof())
                 return false;
diff --git a/dbms/src/Formats/CSVRowInputStream.h b/dbms/src/Formats/CSVRowInputStream.h
index a4748e7954a..990deed406a 100644
--- a/dbms/src/Formats/CSVRowInputStream.h
+++ b/dbms/src/Formats/CSVRowInputStream.h
@@ -40,7 +40,7 @@ private:
     IndexesMap column_indexes_by_names;
 
     using OptionalIndexes = std::vector<std::optional<size_t>>;
-    OptionalIndexes input_fields_with_indexes;
+    OptionalIndexes column_indexes_for_input_fields;
 
     std::vector<UInt8> read_columns;
     std::vector<size_t> columns_to_fill_with_default_values;
diff --git a/dbms/src/Formats/TabSeparatedRowInputStream.cpp b/dbms/src/Formats/TabSeparatedRowInputStream.cpp
index 70644b8c0bf..0a3334090d2 100644
--- a/dbms/src/Formats/TabSeparatedRowInputStream.cpp
+++ b/dbms/src/Formats/TabSeparatedRowInputStream.cpp
@@ -64,7 +64,7 @@ TabSeparatedRowInputStream::TabSeparatedRowInputStream(
         column_indexes_by_names.emplace(column_info.name, i);
     }
 
-    input_fields_with_indexes.reserve(num_columns);
+    column_indexes_for_input_fields.reserve(num_columns);
     read_columns.assign(num_columns, false);
 }
 
@@ -72,10 +72,10 @@ TabSeparatedRowInputStream::TabSeparatedRowInputStream(
 void TabSeparatedRowInputStream::setupAllColumnsByTableSchema()
 {
     read_columns.assign(header.columns(), true);
-    input_fields_with_indexes.resize(header.columns());
+    column_indexes_for_input_fields.resize(header.columns());
 
-    for (size_t i = 0; i < input_fields_with_indexes.size(); ++i)
-        input_fields_with_indexes[i] = i;
+    for (size_t i = 0; i < column_indexes_for_input_fields.size(); ++i)
+        column_indexes_for_input_fields[i] = i;
 }
 
 
@@ -85,13 +85,13 @@ void TabSeparatedRowInputStream::addInputColumn(const String & column_name) {
     {
         if (format_settings.skip_unknown_fields)
         {
-            input_fields_with_indexes.push_back(std::nullopt);
+            column_indexes_for_input_fields.push_back(std::nullopt);
             return;
         }
 
         throw Exception(
             "Unknown field found in TSV header: '" + column_name + "' " +
-            "at position " + std::to_string(input_fields_with_indexes.size()) +
+            "at position " + std::to_string(column_indexes_for_input_fields.size()) +
             "\nSet the 'input_format_skip_unknown_fields' parameter explicitly to ignore and proceed",
             ErrorCodes::INCORRECT_DATA
         );
@@ -103,7 +103,7 @@ void TabSeparatedRowInputStream::addInputColumn(const String & column_name) {
         throw Exception("Duplicate field found while parsing TSV header: " + column_name, ErrorCodes::INCORRECT_DATA);
 
     read_columns[column_index] = true;
-    input_fields_with_indexes.emplace_back(column_index);
+    column_indexes_for_input_fields.emplace_back(column_index);
 }
 
 
@@ -139,9 +139,9 @@ void TabSeparatedRowInputStream::readPrefix()
     {
         if (format_settings.with_names_use_header)
         {
+            String column_name;
             do
             {
-                String column_name;
                 readEscapedString(column_name, istr);
                 addInputColumn(column_name);
             }
@@ -156,7 +156,7 @@ void TabSeparatedRowInputStream::readPrefix()
         else
         {
             setupAllColumnsByTableSchema();
-            skipTSVRow(istr, input_fields_with_indexes.size());
+            skipTSVRow(istr, column_indexes_for_input_fields.size());
         }
     }
     else
@@ -164,7 +164,7 @@ void TabSeparatedRowInputStream::readPrefix()
 
     if (with_types)
     {
-        skipTSVRow(istr, input_fields_with_indexes.size());
+        skipTSVRow(istr, column_indexes_for_input_fields.size());
     }
 }
 
@@ -176,9 +176,9 @@ bool TabSeparatedRowInputStream::read(MutableColumns & columns, RowReadExtension
 
     updateDiagnosticInfo();
 
-    for (size_t input_position = 0; input_position < input_fields_with_indexes.size(); ++input_position)
+    for (size_t input_position = 0; input_position < column_indexes_for_input_fields.size(); ++input_position)
     {
-        const auto & column_index = input_fields_with_indexes[input_position];
+        const auto & column_index = column_indexes_for_input_fields[input_position];
         if (column_index)
         {
             data_types[*column_index]->deserializeAsTextEscaped(*columns[*column_index], istr, format_settings);
@@ -190,7 +190,7 @@ bool TabSeparatedRowInputStream::read(MutableColumns & columns, RowReadExtension
         }
 
         /// skip separators
-        if (input_position + 1 < input_fields_with_indexes.size())
+        if (input_position + 1 < column_indexes_for_input_fields.size())
         {
             assertChar('\t', istr);
         }
@@ -273,7 +273,7 @@ String TabSeparatedRowInputStream::getDiagnosticInfo()
 bool OPTIMIZE(1) TabSeparatedRowInputStream::parseRowAndPrintDiagnosticInfo(
     MutableColumns & columns, WriteBuffer & out, size_t max_length_of_column_name, size_t max_length_of_data_type_name)
 {
-    for (size_t input_position = 0; input_position < input_fields_with_indexes.size(); ++input_position)
+    for (size_t input_position = 0; input_position < column_indexes_for_input_fields.size(); ++input_position)
     {
         if (input_position == 0 && istr.eof())
         {
@@ -281,9 +281,9 @@ bool OPTIMIZE(1) TabSeparatedRowInputStream::parseRowAndPrintDiagnosticInfo(
             return false;
         }
 
-        if (input_fields_with_indexes[input_position].has_value())
+        if (column_indexes_for_input_fields[input_position].has_value())
         {
-            const auto & column_index = *input_fields_with_indexes[input_position];
+            const auto & column_index = *column_indexes_for_input_fields[input_position];
             const auto & current_column_type = data_types[column_index];
 
             out << "Column " << input_position << ", " << std::string((input_position < 10 ? 2 : input_position < 100 ? 1 : 0), ' ')
@@ -364,7 +364,7 @@ bool OPTIMIZE(1) TabSeparatedRowInputStream::parseRowAndPrintDiagnosticInfo(
         }
 
         /// Delimiters
-        if (input_position + 1 == input_fields_with_indexes.size())
+        if (input_position + 1 == column_indexes_for_input_fields.size())
         {
             if (!istr.eof())
             {
diff --git a/dbms/src/Formats/TabSeparatedRowInputStream.h b/dbms/src/Formats/TabSeparatedRowInputStream.h
index f8cfba4e331..3a0ed13c1bd 100644
--- a/dbms/src/Formats/TabSeparatedRowInputStream.h
+++ b/dbms/src/Formats/TabSeparatedRowInputStream.h
@@ -44,7 +44,7 @@ private:
     IndexesMap column_indexes_by_names;
 
     using OptionalIndexes = std::vector<std::optional<size_t>>;
-    OptionalIndexes input_fields_with_indexes;
+    OptionalIndexes column_indexes_for_input_fields;
 
     std::vector<UInt8> read_columns;
     std::vector<size_t> columns_to_fill_with_default_values;

From f93fe77fd59a69002f155dd286c781f320058686 Mon Sep 17 00:00:00 2001
From: Alexander Kazakov <akazz@yandex-team.ru>
Date: Mon, 22 Apr 2019 19:37:22 +0300
Subject: [PATCH 027/118] Added chmod +x for test scripts

---
 dbms/tests/queries/0_stateless/00937_test_use_header_csv.sh | 0
 dbms/tests/queries/0_stateless/00937_test_use_header_tsv.sh | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 dbms/tests/queries/0_stateless/00937_test_use_header_csv.sh
 mode change 100644 => 100755 dbms/tests/queries/0_stateless/00937_test_use_header_tsv.sh

diff --git a/dbms/tests/queries/0_stateless/00937_test_use_header_csv.sh b/dbms/tests/queries/0_stateless/00937_test_use_header_csv.sh
old mode 100644
new mode 100755
diff --git a/dbms/tests/queries/0_stateless/00937_test_use_header_tsv.sh b/dbms/tests/queries/0_stateless/00937_test_use_header_tsv.sh
old mode 100644
new mode 100755

From c7a4b2ec75f84e358511a646092c8b6d13b9abf3 Mon Sep 17 00:00:00 2001
From: Alexander Kazakov <akazz@yandex-team.ru>
Date: Mon, 22 Apr 2019 19:53:54 +0300
Subject: [PATCH 028/118] Fixed style

---
 dbms/src/Formats/CSVRowInputStream.cpp          | 5 +++--
 dbms/src/Formats/TabSeparatedRowInputStream.cpp | 5 +++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/dbms/src/Formats/CSVRowInputStream.cpp b/dbms/src/Formats/CSVRowInputStream.cpp
index 5fa404a8e4d..9b3fd6c2524 100644
--- a/dbms/src/Formats/CSVRowInputStream.cpp
+++ b/dbms/src/Formats/CSVRowInputStream.cpp
@@ -122,7 +122,8 @@ void CSVRowInputStream::setupAllColumnsByTableSchema()
 }
 
 
-void CSVRowInputStream::addInputColumn(const String & column_name) {
+void CSVRowInputStream::addInputColumn(const String & column_name)
+{
     const auto column_it = column_indexes_by_names.find(column_name);
     if (column_it == column_indexes_by_names.end())
     {
@@ -187,7 +188,7 @@ void CSVRowInputStream::readPrefix()
 
                 addInputColumn(column_name);
             }
-            while(checkChar(format_settings.csv.delimiter, istr));
+            while (checkChar(format_settings.csv.delimiter, istr));
 
             skipDelimiter(istr, format_settings.csv.delimiter, true);
         }
diff --git a/dbms/src/Formats/TabSeparatedRowInputStream.cpp b/dbms/src/Formats/TabSeparatedRowInputStream.cpp
index 0a3334090d2..c922ab8453e 100644
--- a/dbms/src/Formats/TabSeparatedRowInputStream.cpp
+++ b/dbms/src/Formats/TabSeparatedRowInputStream.cpp
@@ -79,7 +79,8 @@ void TabSeparatedRowInputStream::setupAllColumnsByTableSchema()
 }
 
 
-void TabSeparatedRowInputStream::addInputColumn(const String & column_name) {
+void TabSeparatedRowInputStream::addInputColumn(const String & column_name)
+{
     const auto column_it = column_indexes_by_names.find(column_name);
     if (column_it == column_indexes_by_names.end())
     {
@@ -145,7 +146,7 @@ void TabSeparatedRowInputStream::readPrefix()
                 readEscapedString(column_name, istr);
                 addInputColumn(column_name);
             }
-            while(checkChar('\t', istr));
+            while (checkChar('\t', istr));
 
             if (!istr.eof())
             {

From ecc6da1429ff8a8d30a381f00500fe9c0cfde96b Mon Sep 17 00:00:00 2001
From: Alexander Kazakov <akazz@yandex-team.ru>
Date: Mon, 22 Apr 2019 20:02:34 +0300
Subject: [PATCH 029/118] Fixed missing includes

---
 dbms/src/Formats/CSVRowInputStream.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/dbms/src/Formats/CSVRowInputStream.h b/dbms/src/Formats/CSVRowInputStream.h
index 990deed406a..cd56a2f5ff7 100644
--- a/dbms/src/Formats/CSVRowInputStream.h
+++ b/dbms/src/Formats/CSVRowInputStream.h
@@ -1,5 +1,8 @@
 #pragma once
 
+#include <optional>
+#include <unordered_map>
+
 #include <Core/Block.h>
 #include <Formats/IRowInputStream.h>
 #include <Formats/FormatSettings.h>

From f8561d295092e64cedee0545c9227613513a40dc Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Mon, 22 Apr 2019 22:58:34 +0300
Subject: [PATCH 030/118] Removed XML output in clickhouse-test; removed
 trashing of UTF-8 characters

---
 dbms/tests/clickhouse-test | 69 +-------------------------------------
 1 file changed, 1 insertion(+), 68 deletions(-)

diff --git a/dbms/tests/clickhouse-test b/dbms/tests/clickhouse-test
index 6b11331f451..f78027c5547 100755
--- a/dbms/tests/clickhouse-test
+++ b/dbms/tests/clickhouse-test
@@ -3,7 +3,6 @@ import sys
 import os
 import os.path
 import re
-import lxml.etree as et
 
 from argparse import ArgumentParser
 from argparse import FileType
@@ -114,19 +113,6 @@ def main(args):
         return stdout.startswith('1')
 
 
-    def dump_report(destination, suite, test_case, report):
-        if destination is not None:
-            destination_file = os.path.join(destination, suite, test_case + ".xml")
-            destination_dir = os.path.dirname(destination_file)
-            if not os.path.exists(destination_dir):
-                os.makedirs(destination_dir)
-            with open(destination_file, 'w') as report_file:
-                report_root = et.Element("testsuites", attrib = {'name': 'ClickHouse Tests'})
-                report_suite = et.Element("testsuite", attrib = {"name": suite})
-                report_suite.append(report)
-                report_root.append(report_suite)
-                report_file.write(et.tostring(report_root, encoding = "UTF-8", xml_declaration=True, pretty_print=True))
-
     base_dir = os.path.abspath(args.queries)
     tmp_dir = os.path.abspath(args.tmp)
 
@@ -258,27 +244,21 @@ def main(args):
                 case_file = os.path.join(suite_dir, case)
                 (name, ext) = os.path.splitext(case)
 
-                report_testcase = et.Element("testcase", attrib = {"name": name})
-
                 try:
                     sys.stdout.write("{0:72}".format(name + ": "))
                     if run_total == 1:
                         sys.stdout.flush()
 
                     if args.skip and any(s in name for s in args.skip):
-                        report_testcase.append(et.Element("skipped", attrib = {"message": "skip"}))
                         print(MSG_SKIPPED + " - skip")
                         skipped_total += 1
                     elif not args.zookeeper and 'zookeeper' in name:
-                        report_testcase.append(et.Element("skipped", attrib = {"message": "no zookeeper"}))
                         print(MSG_SKIPPED + " - no zookeeper")
                         skipped_total += 1
                     elif not args.shard and 'shard' in name:
-                        report_testcase.append(et.Element("skipped", attrib = {"message": "no shard"}))
                         print(MSG_SKIPPED + " - no shard")
                         skipped_total += 1
                     elif not args.no_long and 'long' in name:
-                        report_testcase.append(et.Element("skipped", attrib = {"message": "no long"}))
                         print(MSG_SKIPPED + " - no long")
                         skipped_total += 1
                     else:
@@ -286,7 +266,6 @@ def main(args):
 
                         if os.path.exists(disabled_file) and not args.disabled:
                             message = open(disabled_file, 'r').read()
-                            report_testcase.append(et.Element("skipped", attrib = {"message": message}))
                             print(MSG_SKIPPED + " - " + message)
                         else:
 
@@ -306,9 +285,6 @@ def main(args):
                                     if e.errno != ESRCH:
                                         raise
 
-                                failure = et.Element("failure", attrib = {"message": "Timeout"})
-                                report_testcase.append(failure)
-
                                 failures += 1
                                 print("{0} - Timeout!".format(MSG_FAIL))
                             else:
@@ -321,72 +297,34 @@ def main(args):
                                         break
 
                                 if proc.returncode != 0:
-                                    failure = et.Element("failure", attrib = {"message": "return code {}".format(proc.returncode)})
-                                    report_testcase.append(failure)
-
-                                    stdout_element = et.Element("system-out")
-                                    stdout_element.text = et.CDATA(stdout)
-                                    report_testcase.append(stdout_element)
-
                                     failures += 1
                                     failures_chain += 1
                                     print("{0} - return code {1}".format(MSG_FAIL, proc.returncode))
 
                                     if stderr:
-                                        stderr_element = et.Element("system-err")
-                                        stderr_element.text = et.CDATA(stderr)
-                                        report_testcase.append(stderr_element)
                                         print(stderr.encode('utf-8'))
 
                                     if args.stop and ('Connection refused' in stderr or 'Attempt to read after eof' in stderr) and not 'Received exception from server' in stderr:
                                         SERVER_DIED = True
 
                                 elif stderr:
-                                    failure = et.Element("failure", attrib = {"message": "having stderror"})
-                                    report_testcase.append(failure)
-
-                                    stderr_element = et.Element("system-err")
-                                    stderr_element.text = et.CDATA(stderr)
-                                    report_testcase.append(stderr_element)
-
                                     failures += 1
                                     failures_chain += 1
                                     print("{0} - having stderror:\n{1}".format(MSG_FAIL, stderr.encode('utf-8')))
                                 elif 'Exception' in stdout:
-                                    failure = et.Element("error", attrib = {"message": "having exception"})
-                                    report_testcase.append(failure)
-
-                                    stdout_element = et.Element("system-out")
-                                    stdout_element.text = et.CDATA(stdout)
-                                    report_testcase.append(stdout_element)
-
                                     failures += 1
                                     failures_chain += 1
                                     print("{0} - having exception:\n{1}".format(MSG_FAIL, stdout.encode('utf-8')))
                                 elif not os.path.isfile(reference_file):
-                                    skipped = et.Element("skipped", attrib = {"message": "no reference file"})
-                                    report_testcase.append(skipped)
                                     print("{0} - no reference file".format(MSG_UNKNOWN))
                                 else:
                                     result_is_different = subprocess.call(['diff', '-q', reference_file, stdout_file], stdout = PIPE)
 
                                     if result_is_different:
                                         diff = Popen(['diff', '--unified', reference_file, stdout_file], stdout = PIPE).communicate()[0]
-                                        diff = unicode(diff, errors='replace', encoding='utf-8')
-                                        cat = Popen(['cat', '-vet'], stdin=PIPE, stdout=PIPE).communicate(input=diff.encode(encoding='utf-8', errors='replace'))[0]
 
-                                        failure = et.Element("failure", attrib = {"message": "result differs with reference"})
-                                        report_testcase.append(failure)
-
-                                        stdout_element = et.Element("system-out")
-                                        try:
-                                            stdout_element.text = et.CDATA(diff)
-                                        except:
-                                            stdout_element.text = et.CDATA(remove_control_characters(diff))
-
-                                        report_testcase.append(stdout_element)
                                         failures += 1
-                                        print("{0} - result differs with reference:\n{1}".format(MSG_FAIL, cat.encode('utf-8')))
+                                        print("{0} - result differs with reference:\n{1}".format(MSG_FAIL, diff))
                                     else:
                                         passed_total += 1
                                         failures_chain = 0
@@ -401,13 +339,8 @@ def main(args):
                 except:
                     import traceback
                     exc_type, exc_value, tb = sys.exc_info()
-                    error = et.Element("error", attrib = {"type": exc_type.__name__, "message": str(exc_value)})
-                    report_testcase.append(error)
-
                     failures += 1
                     print("{0} - Test internal error: {1}\n{2}\n{3}".format(MSG_FAIL, exc_type.__name__, exc_value, "\n".join(traceback.format_tb(tb, 10))))
-                finally:
-                    dump_report(args.output, suite, name, report_testcase)
 
                 if failures_chain >= 20:
                     break

From 13da9463bf963a42459def186fa210d217c88d8a Mon Sep 17 00:00:00 2001
From: ogorbacheva <ogorbacheva@yandex-team.ru>
Date: Tue, 23 Apr 2019 15:56:22 +0300
Subject: [PATCH 031/118] Fixes

---
 .../functions/array_functions.md              | 129 +++++++++++++-----
 .../functions/array_functions.md              | 108 +++++++++++----
 2 files changed, 178 insertions(+), 59 deletions(-)

diff --git a/docs/en/query_language/functions/array_functions.md b/docs/en/query_language/functions/array_functions.md
index 27d9a847777..f38728cdad2 100644
--- a/docs/en/query_language/functions/array_functions.md
+++ b/docs/en/query_language/functions/array_functions.md
@@ -458,16 +458,14 @@ SELECT arraySlice([1, 2, NULL, 4, 5], 2, 3) AS res
 
 Array elements set to `NULL` are handled as normal values.
 
-## arraySort(arr)
+## arraySort([func,] arr, ...)
 
-Returns an array as result of sorting the elements of `arr` in ascending order.  
+Sorts the elements of the `arr` array in ascending order.  
 
-The [Schwartzian transform](https://en.wikipedia.org/wiki/Schwartzian_transform) is used to improve sorting efficiency.
-
-Example 1
+Example of integer values sorting:
 
 ``` sql
-SELECT arraySort([1, 3, 3, 0])
+SELECT arraySort([1, 3, 3, 0]);
 ```
 ```
 ┌─arraySort([1, 3, 3, 0])─┐
@@ -475,10 +473,10 @@ SELECT arraySort([1, 3, 3, 0])
 └─────────────────────────┘
 ```
 
-Example 2
+Example of string values sorting:
 
 ``` sql
-SELECT arraySort(['hello', 'world', '!'])
+SELECT arraySort(['hello', 'world', '!']);
 ```
 ```
 ┌─arraySort(['hello', 'world', '!'])─┐
@@ -486,26 +484,57 @@ SELECT arraySort(['hello', 'world', '!'])
 └────────────────────────────────────┘
 ```
 
-Note that NULLs and NaNs go last (NaNs go before NULLs). For example:
- 
-``` sql
-SELECT arraySort([1, nan, 2, NULL, 3, nan, 4, NULL])
-```
-```
-┌─arraySort([1, nan, 2, NULL, 3, nan, 4, NULL])─┐
-│ [1,2,3,4,nan,nan,NULL,NULL]                   │
-└───────────────────────────────────────────────┘
-```
-You can use lambda functions with the `arraySort` function. For details, see the [Higher-order Functions](higher_order_functions.md) section. 
-
-## arrayReverseSort(arr)
-
-Returns an array as result of sorting the elements of `arr` in descending order.  
-
-Example 1
+Consider the following sorting order for the `NULL`, `NaN` and `Inf` values:
 
 ``` sql
-SELECT arrayReverseSort([1, 3, 3, 0])
+SELECT arraySort([1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf]);
+```
+```
+┌─arraySort([1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf])─┐
+│ [-inf,-4,1,2,3,inf,nan,nan,NULL,NULL]                     │
+└───────────────────────────────────────────────────────────┘
+```
+
+- `-Inf` values are first in the array.
+- `NULL` values are last in the array.
+- `NaN` values are right before `NULL`.
+- `Inf` values are right before `NaN`.
+
+Note that `arraySort` is a [high-order function](higher_order_functions.md). You can pass a lambda function to it as the first argument. In this case, sorting order is determined by the result of the lambda function applied to the elements of the array. Example is shown below.
+
+``` sql
+SELECT arraySort(lambda(tuple(x, y), y), ['hello', 'world'], [2, 1]) as res;
+```
+
+```
+┌─res────────────────┐
+│ ['world', 'hello'] │
+└────────────────────┘
+```
+
+The elements that are passed in the second array ([2, 1]), define a new position of each corresponding element from the source array (['hello', 'world']). So, 'hello' will be the second element in a result, and 'world' will be the first.
+
+Below is shown another example. It uses the arrow operator (`->`) to describe the lambda function:
+
+``` sql
+SELECT arraySort((x, y) -> y, [0, 1, 2], [2, 1, 3]) as res;
+```
+``` sql
+┌─res─────┐
+│ [1,0,2] │
+└─────────┘
+```
+
+To improve sorting efficiency, the [Schwartzian transform](https://en.wikipedia.org/wiki/Schwartzian_transform) is used.
+
+## arrayReverseSort([func,] arr, ..)
+
+Sorts the elements of the `arr` array in descending order.  
+
+Example of integer values sorting:
+
+``` sql
+SELECT arrayReverseSort([1, 3, 3, 0]);
 ```
 ```
 ┌─arrayReverseSort([1, 3, 3, 0])─┐
@@ -513,10 +542,10 @@ SELECT arrayReverseSort([1, 3, 3, 0])
 └────────────────────────────────┘
 ```
 
-Example 2
+Example of string values sorting:
 
 ``` sql
-SELECT arrayReverseSort(['hello', 'world', '!'])
+SELECT arrayReverseSort(['hello', 'world', '!']);
 ```
 ```
 ┌─arrayReverseSort(['hello', 'world', '!'])─┐
@@ -524,18 +553,48 @@ SELECT arrayReverseSort(['hello', 'world', '!'])
 └───────────────────────────────────────────┘
 ```
 
-Note that NULLs and NaNs go last (NaNs go before NULLs). For example:
+Consider the following sorting order for the `NULL`, `NaN` and `Inf` values:
  
 ``` sql
-SELECT arrayReverseSort([1, nan, 2, NULL, 3, nan, 4, NULL])
+SELECT arrayReverseSort([1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf]) as res;
 ```
-```
-┌─arrayReverseSort([1, nan, 2, NULL, 3, nan, 4, NULL])─┐
-│ [4,3,2,1,nan,nan,NULL,NULL]                          │
-└──────────────────────────────────────────────────────┘
+``` sql
+┌─res───────────────────────────────────┐
+│ [inf,3,2,1,-4,-inf,nan,nan,NULL,NULL] │
+└───────────────────────────────────────┘
 ```
 
-You can use lambda functions with the `arrayReverseSort` function. For details, see the [Higher-order Functions](higher_order_functions.md) section. 
+- `Inf` values are first in the array.
+- `NULL` values are last in the array.
+- `NaN` values are right before `NULL`.
+- `-Inf` values are right before `NaN`.
+
+Note that the `arrayReverseSort` is a [high-order function](higher_order_functions.md). You can pass a lambda function to it as the first argument. For example:
+
+``` sql
+SELECT arrayReverseSort(x, y) -> y, ['hello', 'world'], [2, 1]) as res;
+```
+``` sql
+┌─res───────────────┐
+│ ['hello','world'] │
+└───────────────────┘
+```
+
+In this case, the array is sorted in the following way:
+
+1. At first, the source array is sorted according to the result of the lambda function applied to the elements of the array. The elements that are passed in the second array ([2, 1]), define a new position of each corresponding element from the source array (['hello', 'world']). So, the intermediate result is an array ['world', 'hello'].
+2. The array that was sorted on the previous step is reversed. So, the result is ['hello', 'world'].
+                      
+Below is shown another example. 
+
+``` sql
+SELECT arrayReverseSort((x, y) -> y, [4, 3, 5], [3, 1, 2]) AS res;
+```
+``` sql
+┌─res─────┐
+│ [4,5,3] │
+└─────────┘
+```
 
 ## arrayUniq(arr, ...)
 
diff --git a/docs/ru/query_language/functions/array_functions.md b/docs/ru/query_language/functions/array_functions.md
index e4c9cc80265..26797e5d485 100644
--- a/docs/ru/query_language/functions/array_functions.md
+++ b/docs/ru/query_language/functions/array_functions.md
@@ -458,13 +458,11 @@ SELECT arraySlice([1, 2, NULL, 4, 5], 2, 3) AS res
 
 Элементы массива равные `NULL` обрабатываются как обычные значения.
 
-## arraySort(arr)
+## arraySort([func,] arr, ...)
 
 Возвращает массив `arr`, отсортированный в восходящем порядке.  
 
-Для улучшения эффективности сортировки применяется [Преобразование Шварца](https://ru.wikipedia.org/wiki/%D0%9F%D1%80%D0%B5%D0%BE%D0%B1%D1%80%D0%B0%D0%B7%D0%BE%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5_%D0%A8%D0%B2%D0%B0%D1%80%D1%86%D0%B0).
-
-Пример 1
+Пример сортировки целочисленных значений:
 
 ``` sql
 SELECT arraySort([1, 3, 3, 0])
@@ -475,7 +473,7 @@ SELECT arraySort([1, 3, 3, 0])
 └─────────────────────────┘
 ```
 
-Пример 2
+Пример сортировки строковых значений:
 
 ``` sql
 SELECT arraySort(['hello', 'world', '!'])
@@ -486,27 +484,57 @@ SELECT arraySort(['hello', 'world', '!'])
 └────────────────────────────────────┘
 ```
 
-`NULL` и `NaN` будут последними в массиве (при этом `NaN` будет перед `NULL`). Например:
+Значения `NULL`, `NaN` и `Inf` сортируются по следующему принципу:
 
 ``` sql
-SELECT arraySort([1, nan, 2, NULL, 3, nan, 4, NULL])
+SELECT arraySort([1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf]);
 ```
 ```
-┌─arraySort([1, nan, 2, NULL, 3, nan, 4, NULL])─┐
-│ [1,2,3,4,nan,nan,NULL,NULL]                   │
-└───────────────────────────────────────────────┘
+┌─arraySort([1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf])─┐
+│ [-inf,-4,1,2,3,inf,nan,nan,NULL,NULL]                     │
+└───────────────────────────────────────────────────────────┘
 ```
 
-Функция `arraySort` поддерживает работу с лямбда-функциями. Подробнее см. в разделе [Функции высшего порядка](higher_order_functions.md).
+- Значения `-Inf` идут в начале массива.
+- Значения `NULL` идут в конце массива.
+- Значения  `NaN` идут перед `NULL`.
+- Значения `Inf` идут перед `NaN`.
 
-## arrayReverseSort(arr)
+Функция `arraySort` является [функцией высшего порядка](higher_order_functions.md) — в качестве первого аргумента ей можно передать лямбда-функцию. В этом случае порядок сортировки определяется результатом применения лямбда-функции на элементы массива. Ниже приведён пример.
 
-Возвращает массив `arr`, отсортированный в нисходящем порядке. Примеры:
+``` sql
+SELECT arraySort(lambda(tuple(x, y), y), ['hello', 'world'], [2, 1]) as res;
+```
+
+```
+┌─res────────────────┐
+│ ['world', 'hello'] │
+└────────────────────┘
+```
+
+Элементы, указанные во втором массиве ([2,1]), определяют позицию соответствующих элементов из исходного массива (['hello', 'world']). То есть, 'hello' будет вторым элементом в отсортированном массиве, а 'world' — первым.
+
+Ниже приведен ещё один пример. В нём для описания лямбда-функции используется стрелочка (оператор `->`):
+
+``` sql
+SELECT arraySort((x, y) -> y, [0, 1, 2], [2, 1, 3]) as res;
+```
+``` sql
+┌─res─────┐
+│ [1,0,2] │
+└─────────┘
+```
+
+Для улучшения эффективности сортировки применяется [Преобразование Шварца](https://ru.wikipedia.org/wiki/%D0%9F%D1%80%D0%B5%D0%BE%D0%B1%D1%80%D0%B0%D0%B7%D0%BE%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5_%D0%A8%D0%B2%D0%B0%D1%80%D1%86%D0%B0).
+
+## arrayReverseSort([func,] arr, ...)
+
+Возвращает массив `arr`, отсортированный в нисходящем порядке.
  
-Пример 1
+Пример сортировки целочисленных значений:
   
 ``` sql
-SELECT arrayReverseSort([1, 3, 3, 0])
+SELECT arrayReverseSort([1, 3, 3, 0]);
 ```
 ```
 ┌─arrayReverseSort([1, 3, 3, 0])─┐
@@ -514,10 +542,10 @@ SELECT arrayReverseSort([1, 3, 3, 0])
 └────────────────────────────────┘
 ```
 
-Пример 2
+Пример сортировки строковых значений:
 
 ``` sql
-SELECT arrayReverseSort(['hello', 'world', '!'])
+SELECT arrayReverseSort(['hello', 'world', '!']);
 ```
 ```
 ┌─arrayReverseSort(['hello', 'world', '!'])─┐
@@ -525,17 +553,49 @@ SELECT arrayReverseSort(['hello', 'world', '!'])
 └───────────────────────────────────────────┘
 ```  
 
-`NULL` и `NaN` будут последними в массиве (при этом `NaN` будет перед `NULL`). Например:
- 
+Значения `NULL`, `NaN` и `Inf` сортируются в следующем порядке:
+
 ``` sql
-SELECT arrayReverseSort([1, nan, 2, NULL, 3, nan, 4, NULL])
+SELECT arrayReverseSort([1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf]) as res;
 ```
+``` sql
+┌─res───────────────────────────────────┐
+│ [inf,3,2,1,-4,-inf,nan,nan,NULL,NULL] │
+└───────────────────────────────────────┘
 ```
-┌─arrayReverseSort([1, nan, 2, NULL, 3, nan, 4, NULL])─┐
-│ [4,3,2,1,nan,nan,NULL,NULL]                          │
-└──────────────────────────────────────────────────────┘
+
+- Значения `Inf` идут в начале массива.
+- Значения `NULL` идут в конце массива.
+- Значения `NaN` идут перед `NULL`.
+- Значения `-Inf` идут перед `NaN`.
+
+Функция `arrayReverseSort` является [функцией высшего порядка](higher_order_functions.md). Вы можете передать ей в качестве первого аргумента лямбда-функцию. Например:
+
+``` sql
+SELECT arrayReverseSort((x, y) -> y, ['hello', 'world'], [2, 1]) as res;
+```
+``` sql
+┌─res───────────────┐
+│ ['hello','world'] │
+└───────────────────┘
+```
+
+В этом случае, массив сортируется следующим образом:
+
+1. Сначала массив сортируется в том порядке, который определяется лямбда-функцией. Элементы, указанные во втором массиве ([2,1]), определяют позицию соответствующих элементов из исходного массива (['hello', 'world']). То есть, промежуточным результатом будет массив ['world', 'hello'].
+2. Массив, который был отсортирован на предыдущем шаге, переворачивается. Получается массив ['hello', 'world'].
+
+Ниже приведён еще один пример.
+
+
+``` sql
+SELECT arrayReverseSort((x, y) -> y, [4, 3, 5], [3, 1, 2]) AS res;
+```
+``` sql
+┌─res─────┐
+│ [4,5,3] │
+└─────────┘
 ```
-Функция `arrayReverseSort` поддерживает работу с лямбда-функциями.  Подробнее см. в разделе [Функции высшего порядка](higher_order_functions.md).
 
 ## arrayUniq(arr, ...)
 

From 28396b04220c448f09629356fce315da871995ef Mon Sep 17 00:00:00 2001
From: ogorbacheva <ogorbacheva@yandex-team.ru>
Date: Tue, 23 Apr 2019 16:03:41 +0300
Subject: [PATCH 032/118] Fixes

---
 docs/en/query_language/functions/array_functions.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/en/query_language/functions/array_functions.md b/docs/en/query_language/functions/array_functions.md
index f38728cdad2..d821130cab4 100644
--- a/docs/en/query_language/functions/array_functions.md
+++ b/docs/en/query_language/functions/array_functions.md
@@ -512,9 +512,9 @@ SELECT arraySort(lambda(tuple(x, y), y), ['hello', 'world'], [2, 1]) as res;
 └────────────────────┘
 ```
 
-The elements that are passed in the second array ([2, 1]), define a new position of each corresponding element from the source array (['hello', 'world']). So, 'hello' will be the second element in a result, and 'world' will be the first.
+The elements that are passed in the second array ([2, 1]), define a new position of each corresponding element from the source array (['hello', 'world']). So, 'hello' will be second element in a result, and 'world' will be first.
 
-Below is shown another example. It uses the arrow operator (`->`) to describe the lambda function:
+Another example is shown below. It uses the arrow operator (`->`) to describe the lambda function:
 
 ``` sql
 SELECT arraySort((x, y) -> y, [0, 1, 2], [2, 1, 3]) as res;
@@ -585,7 +585,7 @@ In this case, the array is sorted in the following way:
 1. At first, the source array is sorted according to the result of the lambda function applied to the elements of the array. The elements that are passed in the second array ([2, 1]), define a new position of each corresponding element from the source array (['hello', 'world']). So, the intermediate result is an array ['world', 'hello'].
 2. The array that was sorted on the previous step is reversed. So, the result is ['hello', 'world'].
                       
-Below is shown another example. 
+Another example is shown below. 
 
 ``` sql
 SELECT arrayReverseSort((x, y) -> y, [4, 3, 5], [3, 1, 2]) AS res;

From fe7114282704951250ab033a8a18a548cff68709 Mon Sep 17 00:00:00 2001
From: proller <proller@github.com>
Date: Tue, 23 Apr 2019 17:02:26 +0300
Subject: [PATCH 033/118] Fix segfault in performance-test when no options
 specified

---
 dbms/programs/performance-test/PerformanceTestSuite.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/dbms/programs/performance-test/PerformanceTestSuite.cpp b/dbms/programs/performance-test/PerformanceTestSuite.cpp
index 7a373d7bfba..9daf1b86ce7 100644
--- a/dbms/programs/performance-test/PerformanceTestSuite.cpp
+++ b/dbms/programs/performance-test/PerformanceTestSuite.cpp
@@ -298,6 +298,8 @@ std::unordered_map<std::string, std::vector<std::size_t>> getTestQueryIndexes(co
 {
     std::unordered_map<std::string, std::vector<std::size_t>> result;
     const auto & options = parsed_opts.options;
+    if (options.empty())
+        return result;
     for (size_t i = 0; i < options.size() - 1; ++i)
     {
         const auto & opt = options[i];

From b52f1257a9e768925e808413eddd684b083d8bcb Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Tue, 23 Apr 2019 19:48:46 +0300
Subject: [PATCH 034/118] Removed useless include

---
 dbms/src/Interpreters/ExpressionAnalyzer.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp
index 075a2e3c21f..06a8b93f8e2 100644
--- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp
+++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp
@@ -36,7 +36,6 @@
 #include <AggregateFunctions/parseAggregateFunctionParameters.h>
 
 #include <Storages/StorageDistributed.h>
-#include <Storages/StorageMemory.h>
 #include <Storages/StorageJoin.h>
 
 #include <DataStreams/copyData.h>

From 585a27c4f40fdf7c8e8fcdc591fd694df5e6192a Mon Sep 17 00:00:00 2001
From: ogorbacheva <ogorbacheva@yandex-team.ru>
Date: Tue, 23 Apr 2019 22:43:20 +0300
Subject: [PATCH 035/118] Review

---
 .../functions/array_functions.md              | 70 ++++++++++++++-----
 .../functions/array_functions.md              | 45 ++++++++----
 2 files changed, 81 insertions(+), 34 deletions(-)

diff --git a/docs/en/query_language/functions/array_functions.md b/docs/en/query_language/functions/array_functions.md
index d821130cab4..fc51339acf5 100644
--- a/docs/en/query_language/functions/array_functions.md
+++ b/docs/en/query_language/functions/array_functions.md
@@ -458,9 +458,9 @@ SELECT arraySlice([1, 2, NULL, 4, 5], 2, 3) AS res
 
 Array elements set to `NULL` are handled as normal values.
 
-## arraySort([func,] arr, ...)
+## arraySort(\[func,\] arr, ...) {#array_functions-reverse-sort}
 
-Sorts the elements of the `arr` array in ascending order.  
+Sorts the elements of the `arr` array in ascending order. If the `func` function is specified, sorting order is determined by the result of the `func` function applied to the elements of the array. If `func` accepts multiple arguments, the `arraySort` function is passed several arrays that the arguments of `func` will correspond to. Detailed examples are shown at the end of `arraySort` description.
 
 Example of integer values sorting:
 
@@ -500,10 +500,23 @@ SELECT arraySort([1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf]);
 - `NaN` values are right before `NULL`.
 - `Inf` values are right before `NaN`.
 
-Note that `arraySort` is a [high-order function](higher_order_functions.md). You can pass a lambda function to it as the first argument. In this case, sorting order is determined by the result of the lambda function applied to the elements of the array. Example is shown below.
+Note that `arraySort` is a [higher-order function](higher_order_functions.md). You can pass a lambda function to it as the first argument. In this case, sorting order is determined by the result of the lambda function applied to the elements of the array. Example is shown below.
 
 ``` sql
-SELECT arraySort(lambda(tuple(x, y), y), ['hello', 'world'], [2, 1]) as res;
+SELECT arraySort((x) -> -x, [1, 2, 3]) as res;
+```
+```
+┌─res─────┐
+│ [3,2,1] │
+└─────────┘
+```
+
+In this example, for each element of the source array, the lambda function returns the "weight" of the element that will be applied for sorting, that is, [1 –> -1, 2 –> -2, 3 –> -3]. Since the elements are sorted by increasing of their "weight", the result is [3, 2, 1].
+
+The lambda function can accept multiple arguments. In this case, you need to pass the `arraySort` function several arrays of identical length that the arguments of lambda function will correspond to. For example:
+
+``` sql
+SELECT arraySort((x, y) -> y, ['hello', 'world'], [2, 1]) as res;
 ```
 
 ```
@@ -512,24 +525,25 @@ SELECT arraySort(lambda(tuple(x, y), y), ['hello', 'world'], [2, 1]) as res;
 └────────────────────┘
 ```
 
-The elements that are passed in the second array ([2, 1]), define a new position of each corresponding element from the source array (['hello', 'world']). So, 'hello' will be second element in a result, and 'world' will be first.
+Here, the elements that are passed in the second array ([2, 1]) define a new position of each corresponding element from the source array (['hello', 'world']), that is, ['hello' –> 2, 'world' –> 1]. So, 'hello' will be the second element in a result, and 'world' will be the first.
 
-Another example is shown below. It uses the arrow operator (`->`) to describe the lambda function:
+Another example is shown below. In this example, sorting order is specified by the string values:
 
 ``` sql
-SELECT arraySort((x, y) -> y, [0, 1, 2], [2, 1, 3]) as res;
+SELECT arraySort((x, y) -> y, [0, 1, 2], ['c', 'b', 'a']) as res;
 ```
 ``` sql
 ┌─res─────┐
-│ [1,0,2] │
+│ [2,1,0] │
 └─────────┘
 ```
 
-To improve sorting efficiency, the [Schwartzian transform](https://en.wikipedia.org/wiki/Schwartzian_transform) is used.
+!!! note
+    To improve sorting efficiency, the [Schwartzian transform](https://en.wikipedia.org/wiki/Schwartzian_transform) is used.
 
-## arrayReverseSort([func,] arr, ..)
+## arrayReverseSort([func,] arr, ...) {#array_functions-reverse-sort}
 
-Sorts the elements of the `arr` array in descending order.  
+Sorts the elements of the `arr` array in descending order. If the `func` function is specified, `arr` is sorted according to the result of the `func` function applied to the elements of the array, and then the sorted array is reversed. If `func` accepts multiple arguments, the `arrayReverseSort` function is passed several arrays that the arguments of `func` will correspond to. Detailed examples are shown at the end of `arrayReverseSort` description.
 
 Example of integer values sorting:
 
@@ -569,10 +583,28 @@ SELECT arrayReverseSort([1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf]) as res;
 - `NaN` values are right before `NULL`.
 - `-Inf` values are right before `NaN`.
 
-Note that the `arrayReverseSort` is a [high-order function](higher_order_functions.md). You can pass a lambda function to it as the first argument. For example:
+Note that the `arrayReverseSort` is a [higher-order function](higher_order_functions.md). You can pass a lambda function to it as the first argument.
+
+Let's consider the following example:
 
 ``` sql
-SELECT arrayReverseSort(x, y) -> y, ['hello', 'world'], [2, 1]) as res;
+SELECT arrayReverseSort((x) -> -x, [1, 2, 3]) as res;
+```
+```
+┌─res─────┐
+│ [1,2,3] │
+└─────────┘
+```
+
+The array is sorted in the following way:
+
+1. At first, the source array ([1, 2, 3]) is sorted according to the result of the lambda function applied to the elements of the array. The result is an array [3, 2, 1].
+2. Array that was sorted on the previous step, is reversed. So, the final result is [1, 2, 3].
+  
+The lambda function can accept multiple arguments. In this case, you need to pass the `arraySort` function several arrays of identical length that the arguments of lambda function will correspond to. For example:
+
+``` sql
+SELECT arrayReverseSort((x, y) -> y, ['hello', 'world'], [2, 1]) as res;
 ```
 ``` sql
 ┌─res───────────────┐
@@ -580,19 +612,19 @@ SELECT arrayReverseSort(x, y) -> y, ['hello', 'world'], [2, 1]) as res;
 └───────────────────┘
 ```
 
-In this case, the array is sorted in the following way:
+In this example, the array is sorted in the following way:
 
-1. At first, the source array is sorted according to the result of the lambda function applied to the elements of the array. The elements that are passed in the second array ([2, 1]), define a new position of each corresponding element from the source array (['hello', 'world']). So, the intermediate result is an array ['world', 'hello'].
-2. The array that was sorted on the previous step is reversed. So, the result is ['hello', 'world'].
+1. At first, the source array (['hello', 'world']) is sorted according to the result of the lambda function applied to the elements of the arrays. The elements that are passed in the second array ([2, 1]), define a new position of corresponding elements from the source array. The result is an array ['world', 'hello'].
+2. Array that was sorted on the previous step, is reversed. So, the final result is ['hello', 'world'].
                       
-Another example is shown below. 
+Another example is shown below. In this example, sorting order is specified by the string values: 
 
 ``` sql
-SELECT arrayReverseSort((x, y) -> y, [4, 3, 5], [3, 1, 2]) AS res;
+SELECT arrayReverseSort((x, y) -> y, [4, 3, 5], ['a', 'b', 'c']) AS res;
 ```
 ``` sql
 ┌─res─────┐
-│ [4,5,3] │
+│ [5,3,4] │
 └─────────┘
 ```
 
diff --git a/docs/ru/query_language/functions/array_functions.md b/docs/ru/query_language/functions/array_functions.md
index 26797e5d485..e3a0eaa411a 100644
--- a/docs/ru/query_language/functions/array_functions.md
+++ b/docs/ru/query_language/functions/array_functions.md
@@ -458,9 +458,9 @@ SELECT arraySlice([1, 2, NULL, 4, 5], 2, 3) AS res
 
 Элементы массива равные `NULL` обрабатываются как обычные значения.
 
-## arraySort([func,] arr, ...)
+## arraySort([func,] arr, ...) {#array_functions-sort}
 
-Возвращает массив `arr`, отсортированный в восходящем порядке.  
+Возвращает массив `arr`, отсортированный в восходящем порядке. Если задана функция `func`, то порядок сортировки определяется результатом применения этой функции на элементы массива `arr`. Если `func` принимает несколько аргументов, то в функцию `arraySort` нужно передавать несколько массивов, которые будут соответствовать аргументам функции `func`. Подробные примеры рассмотрены в конце описания `arraySort`.
 
 Пример сортировки целочисленных значений:
 
@@ -500,10 +500,25 @@ SELECT arraySort([1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf]);
 - Значения  `NaN` идут перед `NULL`.
 - Значения `Inf` идут перед `NaN`.
 
-Функция `arraySort` является [функцией высшего порядка](higher_order_functions.md) — в качестве первого аргумента ей можно передать лямбда-функцию. В этом случае порядок сортировки определяется результатом применения лямбда-функции на элементы массива. Ниже приведён пример.
+Функция `arraySort` является [функцией высшего порядка](higher_order_functions.md) — в качестве первого аргумента ей можно передать лямбда-функцию. В этом случае порядок сортировки определяется результатом применения лямбда-функции на элементы массива.
+
+Рассмотрим пример: 
 
 ``` sql
-SELECT arraySort(lambda(tuple(x, y), y), ['hello', 'world'], [2, 1]) as res;
+SELECT arraySort((x) -> -x, [1, 2, 3]) as res;
+```
+```
+┌─res─────┐
+│ [3,2,1] │
+└─────────┘
+```
+
+Для каждого элемента исходного массива лямбда-функция возвращает "вес" этого элемента — он будет использоваться при сортировке. То есть [1 –> -1, 2 –> -2, 3 –> -3]. Так как элементы сортируются по возрастанию их "веса", результат будет [3, 2, 1].
+
+Лямбда-функция может принимать несколько аргументов. В этом случае, в функцию `arraySort` нужно передавать несколько массивов, которые будут соответствовать аргументам лямбда-функции. Обратите внимание, массивы должны быть одинаковой длины. Например:
+
+``` sql
+SELECT arraySort((x, y) -> y, ['hello', 'world'], [2, 1]) as res;
 ```
 
 ```
@@ -512,24 +527,25 @@ SELECT arraySort(lambda(tuple(x, y), y), ['hello', 'world'], [2, 1]) as res;
 └────────────────────┘
 ```
 
-Элементы, указанные во втором массиве ([2,1]), определяют позицию соответствующих элементов из исходного массива (['hello', 'world']). То есть, 'hello' будет вторым элементом в отсортированном массиве, а 'world' — первым.
+Элементы, указанные во втором массиве ([2,1]), определяют позицию соответствующих элементов из исходного массива (['hello', 'world']), то есть ['hello' –> 2, 'world' –> 1]. Таким образом, 'hello' будет вторым элементом в отсортированном массиве, а 'world' — первым.
 
-Ниже приведен ещё один пример. В нём для описания лямбда-функции используется стрелочка (оператор `->`):
+Ниже приведен ещё один пример, в котором порядок сортировки устанавливается строковыми значениями:
 
 ``` sql
-SELECT arraySort((x, y) -> y, [0, 1, 2], [2, 1, 3]) as res;
+SELECT arraySort((x, y) -> y, [0, 1, 2], ['c', 'b', 'a']) as res;
 ```
 ``` sql
 ┌─res─────┐
-│ [1,0,2] │
+│ [2,1,0] │
 └─────────┘
 ```
 
-Для улучшения эффективности сортировки применяется [Преобразование Шварца](https://ru.wikipedia.org/wiki/%D0%9F%D1%80%D0%B5%D0%BE%D0%B1%D1%80%D0%B0%D0%B7%D0%BE%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5_%D0%A8%D0%B2%D0%B0%D1%80%D1%86%D0%B0).
+!!! note "Примечание"
+    Для улучшения эффективности сортировки применяется [Преобразование Шварца](https://ru.wikipedia.org/wiki/%D0%9F%D1%80%D0%B5%D0%BE%D0%B1%D1%80%D0%B0%D0%B7%D0%BE%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5_%D0%A8%D0%B2%D0%B0%D1%80%D1%86%D0%B0).
 
-## arrayReverseSort([func,] arr, ...)
+## arrayReverseSort([func,] arr, ...) {#array_functions-reverse-sort}
 
-Возвращает массив `arr`, отсортированный в нисходящем порядке.
+Возвращает массив `arr`, отсортированный в нисходящем порядке. Если указана функция `func`, то массив `arr` сначала сортируется в порядке, который определяется функцией `func`, а затем отсортированный массив переворачивается. Если функция `func` принимает несколько аргументов, то в функцию `arrayReverseSort` необходимо передавать несколько массивов, которые будут соответствовать аргументам функции `func`. Подробные примеры рассмотрены в конце описания функции `arrayReverseSort`.
  
 Пример сортировки целочисленных значений:
   
@@ -585,15 +601,14 @@ SELECT arrayReverseSort((x, y) -> y, ['hello', 'world'], [2, 1]) as res;
 1. Сначала массив сортируется в том порядке, который определяется лямбда-функцией. Элементы, указанные во втором массиве ([2,1]), определяют позицию соответствующих элементов из исходного массива (['hello', 'world']). То есть, промежуточным результатом будет массив ['world', 'hello'].
 2. Массив, который был отсортирован на предыдущем шаге, переворачивается. Получается массив ['hello', 'world'].
 
-Ниже приведён еще один пример.
-
+Ниже приведён еще один пример, в котором порядок сортировки устанавливается строковыми значениями:
 
 ``` sql
-SELECT arrayReverseSort((x, y) -> y, [4, 3, 5], [3, 1, 2]) AS res;
+SELECT arrayReverseSort((x, y) -> y, [0, 1, 2], ['c', 'b', 'a']) as res;
 ```
 ``` sql
 ┌─res─────┐
-│ [4,5,3] │
+│ [0,1,2] │
 └─────────┘
 ```
 

From c5a93dc52c635e455a114b1d8623bd1ca40bcb8b Mon Sep 17 00:00:00 2001
From: ogorbacheva <ogorbacheva@yandex-team.ru>
Date: Tue, 23 Apr 2019 22:54:32 +0300
Subject: [PATCH 036/118] Review

---
 .../functions/array_functions.md              | 29 +++++++++++++++----
 .../functions/array_functions.md              | 20 +++++++++++--
 2 files changed, 41 insertions(+), 8 deletions(-)

diff --git a/docs/en/query_language/functions/array_functions.md b/docs/en/query_language/functions/array_functions.md
index fc51339acf5..dd6a4392e58 100644
--- a/docs/en/query_language/functions/array_functions.md
+++ b/docs/en/query_language/functions/array_functions.md
@@ -500,7 +500,9 @@ SELECT arraySort([1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf]);
 - `NaN` values are right before `NULL`.
 - `Inf` values are right before `NaN`.
 
-Note that `arraySort` is a [higher-order function](higher_order_functions.md). You can pass a lambda function to it as the first argument. In this case, sorting order is determined by the result of the lambda function applied to the elements of the array. Example is shown below.
+Note that `arraySort` is a [higher-order function](higher_order_functions.md). You can pass a lambda function to it as the first argument. In this case, sorting order is determined by the result of the lambda function applied to the elements of the array.
+
+Let's consider the following example:
 
 ``` sql
 SELECT arraySort((x) -> -x, [1, 2, 3]) as res;
@@ -527,7 +529,7 @@ SELECT arraySort((x, y) -> y, ['hello', 'world'], [2, 1]) as res;
 
 Here, the elements that are passed in the second array ([2, 1]) define a new position of each corresponding element from the source array (['hello', 'world']), that is, ['hello' –> 2, 'world' –> 1]. So, 'hello' will be the second element in a result, and 'world' will be the first.
 
-Another example is shown below. In this example, sorting order is specified by the string values:
+Another examples are shown below.
 
 ``` sql
 SELECT arraySort((x, y) -> y, [0, 1, 2], ['c', 'b', 'a']) as res;
@@ -538,6 +540,15 @@ SELECT arraySort((x, y) -> y, [0, 1, 2], ['c', 'b', 'a']) as res;
 └─────────┘
 ```
 
+``` sql
+SELECT arraySort((x, y) -> -y, [0, 1, 2], [1, 2, 3]) as res;
+```
+``` sql
+┌─res─────┐
+│ [2,1,0] │
+└─────────┘
+```
+
 !!! note
     To improve sorting efficiency, the [Schwartzian transform](https://en.wikipedia.org/wiki/Schwartzian_transform) is used.
 
@@ -583,9 +594,7 @@ SELECT arrayReverseSort([1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf]) as res;
 - `NaN` values are right before `NULL`.
 - `-Inf` values are right before `NaN`.
 
-Note that the `arrayReverseSort` is a [higher-order function](higher_order_functions.md). You can pass a lambda function to it as the first argument.
-
-Let's consider the following example:
+Note that the `arrayReverseSort` is a [higher-order function](higher_order_functions.md). You can pass a lambda function to it as the first argument. Example is shown below.
 
 ``` sql
 SELECT arrayReverseSort((x) -> -x, [1, 2, 3]) as res;
@@ -617,7 +626,7 @@ In this example, the array is sorted in the following way:
 1. At first, the source array (['hello', 'world']) is sorted according to the result of the lambda function applied to the elements of the arrays. The elements that are passed in the second array ([2, 1]), define a new position of corresponding elements from the source array. The result is an array ['world', 'hello'].
 2. Array that was sorted on the previous step, is reversed. So, the final result is ['hello', 'world'].
                       
-Another example is shown below. In this example, sorting order is specified by the string values: 
+Another examples are shown below. 
 
 ``` sql
 SELECT arrayReverseSort((x, y) -> y, [4, 3, 5], ['a', 'b', 'c']) AS res;
@@ -627,6 +636,14 @@ SELECT arrayReverseSort((x, y) -> y, [4, 3, 5], ['a', 'b', 'c']) AS res;
 │ [5,3,4] │
 └─────────┘
 ```
+``` sql
+SELECT arrayReverseSort((x, y) -> -y, [4, 3, 5], [1, 2, 3]) AS res;
+```
+``` sql
+┌─res─────┐
+│ [4,3,5] │
+└─────────┘
+```
 
 ## arrayUniq(arr, ...)
 
diff --git a/docs/ru/query_language/functions/array_functions.md b/docs/ru/query_language/functions/array_functions.md
index e3a0eaa411a..e6f4b337dc5 100644
--- a/docs/ru/query_language/functions/array_functions.md
+++ b/docs/ru/query_language/functions/array_functions.md
@@ -529,7 +529,7 @@ SELECT arraySort((x, y) -> y, ['hello', 'world'], [2, 1]) as res;
 
 Элементы, указанные во втором массиве ([2,1]), определяют позицию соответствующих элементов из исходного массива (['hello', 'world']), то есть ['hello' –> 2, 'world' –> 1]. Таким образом, 'hello' будет вторым элементом в отсортированном массиве, а 'world' — первым.
 
-Ниже приведен ещё один пример, в котором порядок сортировки устанавливается строковыми значениями:
+Ниже приведены другие примеры.
 
 ``` sql
 SELECT arraySort((x, y) -> y, [0, 1, 2], ['c', 'b', 'a']) as res;
@@ -539,6 +539,14 @@ SELECT arraySort((x, y) -> y, [0, 1, 2], ['c', 'b', 'a']) as res;
 │ [2,1,0] │
 └─────────┘
 ```
+``` sql
+SELECT arraySort((x, y) -> -y, [0, 1, 2], [1, 2, 3]) as res;
+```
+``` sql
+┌─res─────┐
+│ [2,1,0] │
+└─────────┘
+```
 
 !!! note "Примечание"
     Для улучшения эффективности сортировки применяется [Преобразование Шварца](https://ru.wikipedia.org/wiki/%D0%9F%D1%80%D0%B5%D0%BE%D0%B1%D1%80%D0%B0%D0%B7%D0%BE%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5_%D0%A8%D0%B2%D0%B0%D1%80%D1%86%D0%B0).
@@ -601,7 +609,7 @@ SELECT arrayReverseSort((x, y) -> y, ['hello', 'world'], [2, 1]) as res;
 1. Сначала массив сортируется в том порядке, который определяется лямбда-функцией. Элементы, указанные во втором массиве ([2,1]), определяют позицию соответствующих элементов из исходного массива (['hello', 'world']). То есть, промежуточным результатом будет массив ['world', 'hello'].
 2. Массив, который был отсортирован на предыдущем шаге, переворачивается. Получается массив ['hello', 'world'].
 
-Ниже приведён еще один пример, в котором порядок сортировки устанавливается строковыми значениями:
+Ниже приведены ещё примеры.
 
 ``` sql
 SELECT arrayReverseSort((x, y) -> y, [0, 1, 2], ['c', 'b', 'a']) as res;
@@ -611,6 +619,14 @@ SELECT arrayReverseSort((x, y) -> y, [0, 1, 2], ['c', 'b', 'a']) as res;
 │ [0,1,2] │
 └─────────┘
 ```
+``` sql
+SELECT arrayReverseSort((x, y) -> -y, [4, 3, 5], [1, 2, 3]) AS res;
+```
+``` sql
+┌─res─────┐
+│ [4,3,5] │
+└─────────┘
+```
 
 ## arrayUniq(arr, ...)
 

From 5aacb7505ded126d9aaa94d3a084a092766c2167 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=BC=A0=E9=A3=8E=E5=95=B8?= <z1079911968@163.com>
Date: Tue, 23 Apr 2019 15:25:24 -0500
Subject: [PATCH 037/118] Zhdocs: SummingMergeTree and Dictionary (#5087)

---
 .../table_engines/summingmergetree.md         |  2 +-
 .../zh/operations/table_engines/dictionary.md | 18 ++---
 .../table_engines/summingmergetree.md         | 78 +++++++++----------
 3 files changed, 49 insertions(+), 49 deletions(-)

diff --git a/docs/en/operations/table_engines/summingmergetree.md b/docs/en/operations/table_engines/summingmergetree.md
index a5458a5a1e0..807476861f2 100644
--- a/docs/en/operations/table_engines/summingmergetree.md
+++ b/docs/en/operations/table_engines/summingmergetree.md
@@ -86,7 +86,7 @@ SELECT key, sum(value) FROM summtt GROUP BY key
 ```
 
 
-## Data Processing
+## Data Processing {#data-processing}
 
 When data are inserted into a table, they are saved as-is. Clickhouse merges the inserted parts of data periodically and this is when rows with the same primary key are summed and replaced with one for each resulting part of data.
 
diff --git a/docs/zh/operations/table_engines/dictionary.md b/docs/zh/operations/table_engines/dictionary.md
index fa2d6cbc7ce..951019161bc 100644
--- a/docs/zh/operations/table_engines/dictionary.md
+++ b/docs/zh/operations/table_engines/dictionary.md
@@ -1,9 +1,9 @@
 
 # Dictionary
 
-The `Dictionary` engine displays the dictionary data as a ClickHouse table.
+`Dictionary` 引擎将字典数据展示为一个ClickHouse的表。
 
-As an example, consider a dictionary of `products` with the following configuration:
+例如，考虑使用一个具有以下配置的 `products` 字典：
 
 ```xml
 <dictionaries>
@@ -36,7 +36,7 @@ As an example, consider a dictionary of `products` with the following configurat
 </dictionaries>
 ```
 
-Query the dictionary data:
+查询字典中的数据：
 
 ``` sql
 select name, type, key, attribute.names, attribute.types, bytes_allocated, element_count,source from system.dictionaries where name = 'products';                     
@@ -60,17 +60,17 @@ WHERE name = 'products'
 └──────────┴──────┴────────┴─────────────────┴─────────────────┴─────────────────┴───────────────┴─────────────────┘
 ```
 
-You can use the [dictGet*](../../query_language/functions/ext_dict_functions.md#ext_dict_functions) function to get the dictionary data in this format.
+你可以使用 [dictGet*](../../query_language/functions/ext_dict_functions.md#ext_dict_functions) 函数来获取这种格式的字典数据。
 
-This view isn't helpful when you need to get raw data, or when performing a `JOIN` operation. For these cases, you can use the `Dictionary` engine, which displays the dictionary data in a table.
+当你需要获取原始数据，或者是想要使用 `JOIN` 操作的时候，这种视图并没有什么帮助。对于这些情况，你可以使用 `Dictionary` 引擎，它可以将字典数据展示在表中。
 
-Syntax:
+语法：
 
 ```
 CREATE TABLE %table_name% (%fields%) engine = Dictionary(%dictionary_name%)`
 ```
 
-Usage example:
+示例：
 
 ``` sql
 create table products (product_id UInt64, title String) Engine = Dictionary(products);
@@ -89,7 +89,7 @@ Ok.
 0 rows in set. Elapsed: 0.004 sec.
 ```
 
-Take a look at what's in the table.
+看一看表中的内容。
 
 ``` sql
 select * from products limit 1;
@@ -108,4 +108,4 @@ LIMIT 1
 ```
 
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/dictionary/) <!--hide-->
+[来源文章](https://clickhouse.yandex/docs/en/operations/table_engines/dictionary/) <!--hide-->
diff --git a/docs/zh/operations/table_engines/summingmergetree.md b/docs/zh/operations/table_engines/summingmergetree.md
index 82664cc2d9b..5f529718ae6 100644
--- a/docs/zh/operations/table_engines/summingmergetree.md
+++ b/docs/zh/operations/table_engines/summingmergetree.md
@@ -1,11 +1,11 @@
 
 # SummingMergeTree
 
-The engine inherits from [MergeTree](mergetree.md). The difference is that when merging data parts for `SummingMergeTree` tables ClickHouse replaces all the rows with the same primary key with one row which contains summarized values for the columns with the numeric data type. If the primary key is composed in a way that a single key value corresponds to large number of rows, this significantly reduces storage volume and speeds up data selection.
+该引擎继承自 [MergeTree](mergetree.md)。区别在于，当合并 `SummingMergeTree` 表的数据片段时，ClickHouse 会把所有具有相同主键的行合并为一行，该行包含了被合并的行中具有数值数据类型的列的汇总值。如果主键的组合方式使得单个键值对应于大量的行，则可以显著的减少存储空间并加快数据查询的速度。
 
-We recommend to use the engine together with `MergeTree`. Store complete data in `MergeTree` table, and use `SummingMergeTree` for aggregated data storing, for example, when preparing reports. Such an approach will prevent you from losing valuable data due to an incorrectly composed primary key.
+我们推荐将该引擎和 `MergeTree` 一起使用。例如，在准备做报告的时候，将完整的数据存储在 `MergeTree` 表中，并且使用 `SummingMergeTree` 来存储聚合数据。这种方法可以使你避免因为使用不正确的主键组合方式而丢失有价值的数据。
 
-## Creating a Table
+## 建表
 
 ```
 CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
@@ -20,23 +20,23 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
 [SETTINGS name=value, ...]
 ```
 
-For a description of request parameters, see [request description](../../query_language/create.md).
+请求参数的描述，参考 [请求描述](../../query_language/create.md)。
 
-**Parameters of SummingMergeTree**
+**SummingMergeTree 的参数**
 
-- `columns` - a tuple with the names of columns where values will be summarized. Optional parameter.
-The columns must be of a numeric type and must not be in the primary key.
+- `columns` - 包含了将要被汇总的列的列名的元组。可选参数。
+所选的列必须是数值类型，并且不可位于主键中。
 
-    If `columns` not specified, ClickHouse summarizes the values in all columns with a numeric data type that are not in the primary key.
+    如果没有指定 `columns`，ClickHouse 会把所有不在主键中的数值类型的列都进行汇总。
 
-**Query clauses**
+**子句**
 
-When creating a `SummingMergeTree` table the same [clauses](mergetree.md) are required, as when creating a `MergeTree` table.
+创建 `SummingMergeTree` 表时，需要与创建 `MergeTree` 表时相同的子句。
 
-<details markdown="1"><summary>Deprecated Method for Creating a Table</summary>
+<details markdown="1"><summary>已弃用的建表方法</summary>
 
-!!! attention
-    Do not use this method in new projects and, if possible, switch the old projects to the method described above.
+!!! 注意
+    不要在新项目中使用该方法，可能的话，请将旧项目切换到上述方法。
 
 ```
 CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
@@ -47,14 +47,14 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
 ) ENGINE [=] SummingMergeTree(date-column [, sampling_expression], (primary, key), index_granularity, [columns])
 ```
 
-All of the parameters excepting `columns` have the same meaning as in `MergeTree`.
+除 `columns` 外的所有参数都与 `MergeTree` 中的含义相同。
 
-- `columns` — tuple with names of columns values of which will be summarized. Optional parameter. For a description, see the text above.
+- `columns` — 包含将要被汇总的列的列名的元组。可选参数。有关说明，请参阅上面的文字。
 </details>
 
-## Usage Example
+## 用法示例
 
-Consider the following table:
+考虑如下的表：
 
 ```sql
 CREATE TABLE summtt
@@ -66,13 +66,13 @@ ENGINE = SummingMergeTree()
 ORDER BY key
 ```
 
-Insert data to it:
+向其中插入数据：
 
 ```
 :) INSERT INTO summtt Values(1,1),(1,2),(2,1)
 ```
 
-ClickHouse may sum all the rows not completely ([see below](#data-processing)), so we use an aggregate function `sum` and `GROUP BY` clause in the query.
+ClickHouse可能不会完整的汇总所有行（[见下文](#data-processing)）,因此我们在查询中使用了聚合函数 `sum` 和 `GROUP BY` 子句。
 
 ```sql
 SELECT key, sum(value) FROM summtt GROUP BY key
@@ -86,38 +86,38 @@ SELECT key, sum(value) FROM summtt GROUP BY key
 ```
 
 
-## Data Processing
+## 数据处理 {#data-processing}
 
-When data are inserted into a table, they are saved as-is. Clickhouse merges the inserted parts of data periodically and this is when rows with the same primary key are summed and replaced with one for each resulting part of data.
+当数据被插入到表中时，他们将被原样保存。ClickHouse 定期合并插入的数据片段，并在这个时候对所有具有相同主键的行中的列进行汇总，将这些行替换为包含汇总数据的一行记录。
 
-ClickHouse can merge the data parts so that different resulting parts of data cat consist rows with the same primary key, i.e. the summation will be incomplete. Therefore (`SELECT`) an aggregate function [sum()](../../query_language/agg_functions/reference.md#agg_function-sum) and `GROUP BY` clause should be used in a query as described in the example above.
+ClickHouse 会按片段合并数据，以至于不同的数据片段中会包含具有相同主键的行，即单个汇总片段将会是不完整的。因此，聚合函数 [sum()](../../query_language/agg_functions/reference.md#agg_function-sum) 和 `GROUP BY` 子句应该在（`SELECT`）查询语句中被使用，如上文中的例子所述。
 
-### Common rules for summation
+### 汇总的通用规则
 
-The values in the columns with the numeric data type are summarized. The set of columns is defined by the parameter `columns`.
+列中数值类型的值会被汇总。这些列的集合在参数 `columns` 中被定义。
 
-If the values were 0 in all of the columns for summation, the row is deleted.
+如果用于汇总的所有列中的值均为0，则该行会被删除。
 
-If column is not in the primary key and is not summarized, an arbitrary value is selected from the existing ones.
+如果列不在主键中且无法被汇总，则会在现有的值中任选一个。
 
-The values are not summarized for columns in the primary key.
+主键所在的列中的值不会被汇总。
 
-### The Summation in the AggregateFunction Columns
+### AggregateFunction 列中的汇总
 
-For columns of [AggregateFunction type](../../data_types/nested_data_structures/aggregatefunction.md) ClickHouse behaves as [AggregatingMergeTree](aggregatingmergetree.md) engine aggregating according to the function.
+对于 [AggregateFunction 类型](../../data_types/nested_data_structures/aggregatefunction.md)的列，ClickHouse 根据对应函数表现为 [AggregatingMergeTree](aggregatingmergetree.md) 引擎的聚合。
 
-### Nested Structures
+### 嵌套结构
 
-Table can have nested data structures that are processed in a special way.
+表中可以具有以特殊方式处理的嵌套数据结构。
 
-If the name of a nested table ends with `Map` and it contains at least two columns that meet the following criteria:
+如果嵌套表的名称以 `Map` 结尾，并且包含至少两个符合以下条件的列：
 
-- the first column is numeric `(*Int*, Date, DateTime)`, let's call it `key`,
-- the other columns are arithmetic `(*Int*, Float32/64)`, let's call it `(values...)`,
+- 第一列是数值类型 `(*Int*, Date, DateTime)`，我们称之为 `key`,
+- 其他的列是可计算的 `(*Int*, Float32/64)`，我们称之为 `(values...)`,
 
-then this nested table is interpreted as a mapping of `key => (values...)`, and when merging its rows, the elements of two data sets are merged by `key` with a summation of the corresponding `(values...)`.
+然后这个嵌套表会被解释为一个 `key => (values...)` 的映射，当合并它们的行时，两个数据集中的元素会被根据 `key` 合并为相应的 `(values...)` 的汇总值。
 
-Examples:
+示例：
 
 ```
 [(1, 100)] + [(2, 150)] -> [(1, 100), (2, 150)]
@@ -126,8 +126,8 @@ Examples:
 [(1, 100), (2, 150)] + [(1, -100)] -> [(2, 150)]
 ```
 
-When requesting data, use the [sumMap(key, value)](../../query_language/agg_functions/reference.md) function for aggregation of `Map`.
+请求数据时，使用 [sumMap(key, value)](../../query_language/agg_functions/reference.md) 函数来对 `Map` 进行聚合。
 
-For nested data structure, you do not need to specify its columns in the tuple of columns for summation.
+对于嵌套数据结构，你无需在列的元组中指定列以进行汇总。
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/summingmergetree/) <!--hide-->
+[来源文章](https://clickhouse.yandex/docs/en/operations/table_engines/summingmergetree/) <!--hide-->

From 3db650861d168a97310e09fd355996d3af77419f Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Wed, 24 Apr 2019 01:40:16 +0300
Subject: [PATCH 038/118] Update AggregateFunctionGroupUniqArray.h

---
 dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h b/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h
index 7bc462f7f3f..f2b339d2729 100644
--- a/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h
+++ b/dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h
@@ -78,7 +78,7 @@ public:
             for (auto & rhs_elem : rhs_set)
             {
                 if (cur_set.size() >= max_elems)
-                    return ;
+                    return;
                 cur_set.insert(rhs_elem.getValue());
             }
         }

From 450e5974b73b0e3dbe60a471f325ef3906d25ee4 Mon Sep 17 00:00:00 2001
From: Ivan Blinkov <github@blinkov.ru>
Date: Wed, 24 Apr 2019 09:42:42 +0300
Subject: [PATCH 039/118] Add link to GitHub issues to Contacts section

---
 website/index.html | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/website/index.html b/website/index.html
index 90e25bb725d..134eb957d46 100644
--- a/website/index.html
+++ b/website/index.html
@@ -444,6 +444,8 @@ clickhouse-client
                rel="external nofollow" target="_blank">English</a> or in
             <a href="https://telegram.me/clickhouse_ru"
                rel="external nofollow" target="_blank">Russian</a>.</li>
+        <li>Open <a href="https://github.com/yandex/ClickHouse/issues/new/choose"
+                    rel="external nofollow" target="_blank">GitHub issue</a> if you have a bug report or feature request.<li>
         <li>Follow official <a
                href="https://twitter.com/ClickHouseDB"
                rel="external nofollow" target="_blank">Twitter account</a>.</li>

From 312f8cb6376ed91fd369c09f553c2bf2063caf1a Mon Sep 17 00:00:00 2001
From: Ivan Blinkov <github@blinkov.ru>
Date: Wed, 24 Apr 2019 09:43:51 +0300
Subject: [PATCH 040/118] Swap list items at website front page

---
 website/index.html | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/website/index.html b/website/index.html
index 134eb957d46..e6ebb8e6edd 100644
--- a/website/index.html
+++ b/website/index.html
@@ -444,11 +444,11 @@ clickhouse-client
                rel="external nofollow" target="_blank">English</a> or in
             <a href="https://telegram.me/clickhouse_ru"
                rel="external nofollow" target="_blank">Russian</a>.</li>
-        <li>Open <a href="https://github.com/yandex/ClickHouse/issues/new/choose"
-                    rel="external nofollow" target="_blank">GitHub issue</a> if you have a bug report or feature request.<li>
         <li>Follow official <a
                href="https://twitter.com/ClickHouseDB"
                rel="external nofollow" target="_blank">Twitter account</a>.</li>
+        <li>Open <a href="https://github.com/yandex/ClickHouse/issues/new/choose"
+                    rel="external nofollow" target="_blank">GitHub issue</a> if you have a bug report or feature request.<li>
         <li>Or email Yandex ClickHouse team directly at
             <a id="feedback_email" href="">turn on JavaScript to see email address</a>.
             You can also <a href="https://forms.yandex.com/surveys/meet-yandex-clickhouse-team/" target="_blank" rel="external nofollow">fill this form</a> to meet us in person.</li>

From 9440e1e61a850fa93a157baed0ecae74fc98eea1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=BC=A0=E9=A3=8E=E5=95=B8?= <z1079911968@163.com>
Date: Wed, 24 Apr 2019 01:51:01 -0500
Subject: [PATCH 041/118] translate ReplacingMergeTree and optimize zh
 attention format (#5094)

---
 docs/zh/operations/table_engines/mergetree.md |  2 +-
 .../table_engines/replacingmergetree.md       | 34 +++++++++----------
 .../table_engines/summingmergetree.md         |  6 ++--
 3 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/docs/zh/operations/table_engines/mergetree.md b/docs/zh/operations/table_engines/mergetree.md
index c8febd0a21b..5ddf837708a 100644
--- a/docs/zh/operations/table_engines/mergetree.md
+++ b/docs/zh/operations/table_engines/mergetree.md
@@ -87,7 +87,7 @@ ENGINE MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDa
 
 <details markdown="1"><summary>已弃用的建表方法</summary>
 
-!!! 注意
+!!! attention "注意"
     不要在新版项目中使用该方法，可能的话，请将旧项目切换到上述方法。
 
 ```
diff --git a/docs/zh/operations/table_engines/replacingmergetree.md b/docs/zh/operations/table_engines/replacingmergetree.md
index 568f94822dd..a9bb9d65b6d 100644
--- a/docs/zh/operations/table_engines/replacingmergetree.md
+++ b/docs/zh/operations/table_engines/replacingmergetree.md
@@ -1,12 +1,12 @@
 # ReplacingMergeTree
 
-The engine differs from [MergeTree](mergetree.md) in that it removes duplicate entries with the same primary key value.
+该引擎和[MergeTree](mergetree.md)的不同之处在于它会删除具有相同主键的重复项。
 
-Data deduplication occurs only during a merge. Merging occurs in the background at an unknown time, so you can't plan for it. Some of the data may remain unprocessed. Although you can run an unscheduled merge using the `OPTIMIZE` query, don't count on using it, because the `OPTIMIZE` query will read and write a large amount of data.
+数据的去重只会在合并的过程中出现。合并会在未知的时间在后台进行，因此你无法预先作出计划。有一些数据可能仍未被处理。尽管你可以调用 `OPTIMIZE` 语句发起计划外的合并，但请不要指望使用它，因为 `OPTIMIZE` 语句会引发对大量数据的读和写。
 
-Thus, `ReplacingMergeTree` is suitable for clearing out duplicate data in the background in order to save space, but it doesn't guarantee the absence of duplicates.
+因此，`ReplacingMergeTree` 适用于在后台清除重复的数据以节省空间，但是它不保证没有重复的数据出现。
 
-## Creating a Table
+## 建表
 
 ```sql
 CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
@@ -21,24 +21,24 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
 [SETTINGS name=value, ...]
 ```
 
-For a description of request parameters, see [request description](../../query_language/create.md).
+请求参数的描述，参考[请求参数](../../query_language/create.md)。
 
 **ReplacingMergeTree Parameters**
 
-- `ver` — column with version. Type `UInt*`, `Date` or `DateTime`. Optional parameter.
+- `ver` — 版本列。类型为 `UInt*`, `Date` 或 `DateTime`。可选参数。
 
-    When merging, `ReplacingMergeTree` from all the rows with the same primary key leaves only one:
-    - Last in the selection, if `ver` not set.
-    - With the maximum version, if `ver` specified.
+    合并的时候，`ReplacingMergeTree` 从所有具有相同主键的行中选择一行留下：
+    - 如果 `ver` 列未指定，选择最后一条。
+    - 如果 `ver` 列已指定，选择 `ver` 值最大的版本。
 
-**Query clauses**
+**子句**
 
-When creating a `ReplacingMergeTree` table the same [clauses](mergetree.md) are required, as when creating a `MergeTree` table.
+创建 `ReplacingMergeTree` 表时，需要与创建 `MergeTree` 表时相同的[子句](mergetree.md)。
 
-<details markdown="1"><summary>Deprecated Method for Creating a Table</summary>
+<details markdown="1"><summary>已弃用的建表方法</summary>
 
-!!! attention
-    Do not use this method in new projects and, if possible, switch the old projects to the method described above.
+!!! attention "注意"
+    不要在新项目中使用该方法，可能的话，请将旧项目切换到上述方法。
 
 ```sql
 CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
@@ -49,10 +49,10 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
 ) ENGINE [=] ReplacingMergeTree(date-column [, sampling_expression], (primary, key), index_granularity, [ver])
 ```
 
-All of the parameters excepting `ver` have the same meaning as in `MergeTree`.
+除了 `ver` 的所有参数都与 `MergeTree` 中的含义相同。
 
 
-- `ver` - column with the version. Optional parameter. For a description, see the text above.
+- `ver` - 版本列。可选参数，有关说明，请参阅上文。
 </details>
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/replacingmergetree/) <!--hide-->
+[来源文章](https://clickhouse.yandex/docs/en/operations/table_engines/replacingmergetree/) <!--hide-->
diff --git a/docs/zh/operations/table_engines/summingmergetree.md b/docs/zh/operations/table_engines/summingmergetree.md
index 5f529718ae6..ab3a50765e2 100644
--- a/docs/zh/operations/table_engines/summingmergetree.md
+++ b/docs/zh/operations/table_engines/summingmergetree.md
@@ -31,11 +31,11 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
 
 **子句**
 
-创建 `SummingMergeTree` 表时，需要与创建 `MergeTree` 表时相同的子句。
+创建 `SummingMergeTree` 表时，需要与创建 `MergeTree` 表时相同的[子句](mergetree.md)。
 
 <details markdown="1"><summary>已弃用的建表方法</summary>
 
-!!! 注意
+!!! attention "注意"
     不要在新项目中使用该方法，可能的话，请将旧项目切换到上述方法。
 
 ```
@@ -49,7 +49,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
 
 除 `columns` 外的所有参数都与 `MergeTree` 中的含义相同。
 
-- `columns` — 包含将要被汇总的列的列名的元组。可选参数。有关说明，请参阅上面的文字。
+- `columns` — 包含将要被汇总的列的列名的元组。可选参数。有关说明，请参阅上文。
 </details>
 
 ## 用法示例

From 566bf557b8c35049e8a85df1ddf1d05df39e3cd0 Mon Sep 17 00:00:00 2001
From: Ivan Blinkov <github@blinkov.ru>
Date: Wed, 24 Apr 2019 11:08:21 +0300
Subject: [PATCH 042/118] fix mistype

---
 website/index.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/website/index.html b/website/index.html
index e6ebb8e6edd..d0a95c73120 100644
--- a/website/index.html
+++ b/website/index.html
@@ -448,7 +448,7 @@ clickhouse-client
                href="https://twitter.com/ClickHouseDB"
                rel="external nofollow" target="_blank">Twitter account</a>.</li>
         <li>Open <a href="https://github.com/yandex/ClickHouse/issues/new/choose"
-                    rel="external nofollow" target="_blank">GitHub issue</a> if you have a bug report or feature request.<li>
+                    rel="external nofollow" target="_blank">GitHub issue</a> if you have a bug report or feature request.</li>
         <li>Or email Yandex ClickHouse team directly at
             <a id="feedback_email" href="">turn on JavaScript to see email address</a>.
             You can also <a href="https://forms.yandex.com/surveys/meet-yandex-clickhouse-team/" target="_blank" rel="external nofollow">fill this form</a> to meet us in person.</li>

From e28cca096f3ac1f91e187908e60892ca036b23ba Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Wed, 24 Apr 2019 11:33:24 +0300
Subject: [PATCH 043/118] Add images for compatibility check

---
 docker/test/compatibility/centos/Dockerfile | 4 ++++
 docker/test/compatibility/ubuntu/Dockerfile | 4 ++++
 2 files changed, 8 insertions(+)
 create mode 100644 docker/test/compatibility/centos/Dockerfile
 create mode 100644 docker/test/compatibility/ubuntu/Dockerfile

diff --git a/docker/test/compatibility/centos/Dockerfile b/docker/test/compatibility/centos/Dockerfile
new file mode 100644
index 00000000000..5586f6a0d7e
--- /dev/null
+++ b/docker/test/compatibility/centos/Dockerfile
@@ -0,0 +1,4 @@
+FROM centos:5
+
+CMD /bin/sh -c "/clickhouse server --config /config/config.xml > /var/log/clickhouse-server/stderr.log 2>&1 & \
+    sleep 5 && /clickhouse client --query \"select 'OK'\" 2> /var/log/clickhouse-server/clientstderr.log || echo 'FAIL'"
diff --git a/docker/test/compatibility/ubuntu/Dockerfile b/docker/test/compatibility/ubuntu/Dockerfile
new file mode 100644
index 00000000000..a84d5bb1720
--- /dev/null
+++ b/docker/test/compatibility/ubuntu/Dockerfile
@@ -0,0 +1,4 @@
+FROM ubuntu:12.04
+
+CMD /bin/sh -c "/clickhouse server --config /config/config.xml > /var/log/clickhouse-server/stderr.log 2>&1 & \
+    sleep 5 && /clickhouse client --query \"select 'OK'\" 2> /var/log/clickhouse-server/clientstderr.log || echo 'FAIL'"

From ba4687e110cd0028fc422c5324a4432acfa9effa Mon Sep 17 00:00:00 2001
From: proller <proller@users.noreply.github.com>
Date: Wed, 24 Apr 2019 12:06:28 +0300
Subject: [PATCH 044/118]  Cmake: dont use LINK_MODE (but use
 BUILD_SHARED_LIBS) (#5091)

---
 CMakeLists.txt                                         |  2 +-
 contrib/arrow-cmake/CMakeLists.txt                     |  6 +++---
 contrib/base64-cmake/CMakeLists.txt                    |  2 +-
 contrib/boost-cmake/CMakeLists.txt                     |  2 +-
 contrib/brotli-cmake/CMakeLists.txt                    |  2 +-
 contrib/cctz-cmake/CMakeLists.txt                      |  2 +-
 contrib/cppkafka-cmake/CMakeLists.txt                  |  2 +-
 contrib/libbtrie/CMakeLists.txt                        |  2 +-
 contrib/libhdfs3-cmake/CMakeLists.txt                  |  2 +-
 contrib/librdkafka-cmake/CMakeLists.txt                |  2 +-
 contrib/libxml2-cmake/CMakeLists.txt                   | 10 +++++-----
 contrib/mariadb-connector-c-cmake/CMakeLists.txt       |  6 +++---
 contrib/re2_st/CMakeLists.txt                          |  2 +-
 contrib/unixodbc-cmake/CMakeLists.txt                  |  4 ++--
 contrib/zstd-cmake/CMakeLists.txt                      |  2 +-
 dbms/CMakeLists.txt                                    |  2 +-
 dbms/programs/CMakeLists.txt                           |  2 +-
 dbms/programs/clang/Compiler-5.0.0/CMakeLists.txt      |  2 +-
 dbms/programs/clang/Compiler-6.0.0/CMakeLists.txt      |  3 ++-
 dbms/programs/clang/Compiler-7.0.0/CMakeLists.txt      |  2 +-
 .../clang/Compiler-7.0.0bundled/CMakeLists.txt         |  2 +-
 dbms/src/AggregateFunctions/CMakeLists.txt             |  2 +-
 dbms/src/Client/CMakeLists.txt                         |  2 +-
 dbms/src/Common/Config/CMakeLists.txt                  |  2 +-
 dbms/src/Common/StringUtils/CMakeLists.txt             |  2 +-
 dbms/src/Common/ZooKeeper/CMakeLists.txt               |  2 +-
 dbms/src/Compression/CMakeLists.txt                    |  2 +-
 dbms/src/Dictionaries/CMakeLists.txt                   |  2 +-
 dbms/src/Dictionaries/Embedded/CMakeLists.txt          |  2 +-
 dbms/src/Functions/CMakeLists.txt                      |  2 +-
 dbms/src/Parsers/CMakeLists.txt                        |  2 +-
 dbms/src/Storages/Kafka/CMakeLists.txt                 |  2 +-
 dbms/src/Storages/System/CMakeLists.txt                |  2 +-
 dbms/src/TableFunctions/CMakeLists.txt                 |  2 +-
 libs/libcommon/CMakeLists.txt                          |  4 ++--
 libs/libdaemon/CMakeLists.txt                          |  2 +-
 libs/libmemcpy/CMakeLists.txt                          |  2 +-
 libs/libmysqlxx/CMakeLists.txt                         |  2 +-
 38 files changed, 49 insertions(+), 48 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 45ee1dfbb41..4b45792ef76 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -59,7 +59,7 @@ if (NOT MAKE_STATIC_LIBRARIES)
 endif ()
 
 if (SPLIT_SHARED_LIBRARIES)
-    set (LINK_MODE SHARED)
+    set(BUILD_SHARED_LIBS 1 CACHE INTERNAL "")
 endif ()
 
 if (USE_STATIC_LIBRARIES)
diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt
index bec827354cd..c0b87efc63e 100644
--- a/contrib/arrow-cmake/CMakeLists.txt
+++ b/contrib/arrow-cmake/CMakeLists.txt
@@ -41,7 +41,7 @@ set( thriftcpp_threads_SOURCES
     ${LIBRARY_DIR}/src/thrift/concurrency/Monitor.cpp
     ${LIBRARY_DIR}/src/thrift/concurrency/Mutex.cpp
 )
-add_library(${THRIFT_LIBRARY} ${LINK_MODE} ${thriftcpp_SOURCES} ${thriftcpp_threads_SOURCES})
+add_library(${THRIFT_LIBRARY} ${thriftcpp_SOURCES} ${thriftcpp_threads_SOURCES})
 set_target_properties(${THRIFT_LIBRARY} PROPERTIES CXX_STANDARD 14) # REMOVE after https://github.com/apache/thrift/pull/1641
 target_include_directories(${THRIFT_LIBRARY} SYSTEM PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/thrift/lib/cpp/src PRIVATE ${Boost_INCLUDE_DIRS})
 
@@ -149,7 +149,7 @@ if (ARROW_WITH_ZSTD)
 endif()
 
 
-add_library(${ARROW_LIBRARY} ${LINK_MODE} ${ARROW_SRCS})
+add_library(${ARROW_LIBRARY} ${ARROW_SRCS})
 target_include_directories(${ARROW_LIBRARY} SYSTEM PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/cpp/src ${Boost_INCLUDE_DIRS})
 target_link_libraries(${ARROW_LIBRARY} PRIVATE ${DOUBLE_CONVERSION_LIBRARIES} Threads::Threads)
 if (ARROW_WITH_LZ4)
@@ -195,7 +195,7 @@ list(APPEND PARQUET_SRCS
   ${CMAKE_CURRENT_SOURCE_DIR}/cpp/src/parquet/parquet_constants.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/cpp/src/parquet/parquet_types.cpp
 )
-add_library(${PARQUET_LIBRARY} ${LINK_MODE} ${PARQUET_SRCS})
+add_library(${PARQUET_LIBRARY} ${PARQUET_SRCS})
 target_include_directories(${PARQUET_LIBRARY} SYSTEM PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src ${CMAKE_CURRENT_SOURCE_DIR}/cpp/src)
 include(${ClickHouse_SOURCE_DIR}/contrib/thrift/build/cmake/ConfigureChecks.cmake) # makes config.h
 target_link_libraries(${PARQUET_LIBRARY} PUBLIC ${ARROW_LIBRARY} PRIVATE ${THRIFT_LIBRARY} ${Boost_REGEX_LIBRARY})
diff --git a/contrib/base64-cmake/CMakeLists.txt b/contrib/base64-cmake/CMakeLists.txt
index 12379c190e1..0fdeef311f5 100644
--- a/contrib/base64-cmake/CMakeLists.txt
+++ b/contrib/base64-cmake/CMakeLists.txt
@@ -24,7 +24,7 @@ endif ()
 
 configure_file(config.h.in ${CMAKE_CURRENT_BINARY_DIR}/config.h)
 
-add_library(base64 ${LINK_MODE}
+add_library(base64
         ${LIBRARY_DIR}/lib/lib.c
         ${LIBRARY_DIR}/lib/codec_choose.c
         ${LIBRARY_DIR}/lib/arch/avx/codec.c
diff --git a/contrib/boost-cmake/CMakeLists.txt b/contrib/boost-cmake/CMakeLists.txt
index ee6721616ae..d9a8a70ef17 100644
--- a/contrib/boost-cmake/CMakeLists.txt
+++ b/contrib/boost-cmake/CMakeLists.txt
@@ -20,7 +20,7 @@ endif()
 
 macro(add_boost_lib lib_name)
     add_headers_and_sources(boost_${lib_name} ${LIBRARY_DIR}/libs/${lib_name}/src)
-    add_library(boost_${lib_name}_internal ${LINK_MODE} ${boost_${lib_name}_sources})
+    add_library(boost_${lib_name}_internal ${boost_${lib_name}_sources})
     target_include_directories(boost_${lib_name}_internal SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
     target_compile_definitions(boost_${lib_name}_internal PUBLIC BOOST_SYSTEM_NO_DEPRECATED)
 endmacro()
diff --git a/contrib/brotli-cmake/CMakeLists.txt b/contrib/brotli-cmake/CMakeLists.txt
index 6e5fb2d22ae..cbc290d9e67 100644
--- a/contrib/brotli-cmake/CMakeLists.txt
+++ b/contrib/brotli-cmake/CMakeLists.txt
@@ -28,6 +28,6 @@ set(SRCS
     ${BROTLI_SOURCE_DIR}/common/transform.c
 )
 
-add_library(brotli ${LINK_MODE} ${SRCS})
+add_library(brotli ${SRCS})
 
 target_include_directories(brotli PUBLIC ${BROTLI_SOURCE_DIR}/include)
diff --git a/contrib/cctz-cmake/CMakeLists.txt b/contrib/cctz-cmake/CMakeLists.txt
index 5057c2594be..9c2f6d9a658 100644
--- a/contrib/cctz-cmake/CMakeLists.txt
+++ b/contrib/cctz-cmake/CMakeLists.txt
@@ -1,6 +1,6 @@
 SET(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/cctz)
 
-add_library(cctz ${LINK_MODE}
+add_library(cctz
     ${LIBRARY_DIR}/src/civil_time_detail.cc
     ${LIBRARY_DIR}/src/time_zone_fixed.cc
     ${LIBRARY_DIR}/src/time_zone_format.cc
diff --git a/contrib/cppkafka-cmake/CMakeLists.txt b/contrib/cppkafka-cmake/CMakeLists.txt
index 9fb98e35b47..064eb0d04ad 100644
--- a/contrib/cppkafka-cmake/CMakeLists.txt
+++ b/contrib/cppkafka-cmake/CMakeLists.txt
@@ -23,7 +23,7 @@ set(SRCS
   ${CPPKAFKA_DIR}/src/consumer.cpp
 )
 
-add_library(cppkafka ${LINK_MODE} ${SRCS})
+add_library(cppkafka ${SRCS})
 
 target_link_libraries(cppkafka PRIVATE ${RDKAFKA_LIBRARY})
 target_include_directories(cppkafka PRIVATE ${CPPKAFKA_DIR}/include/cppkafka)
diff --git a/contrib/libbtrie/CMakeLists.txt b/contrib/libbtrie/CMakeLists.txt
index 279d678b001..f590520c416 100644
--- a/contrib/libbtrie/CMakeLists.txt
+++ b/contrib/libbtrie/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_library (btrie
+add_library(btrie
     src/btrie.c
     include/btrie.h
 )
diff --git a/contrib/libhdfs3-cmake/CMakeLists.txt b/contrib/libhdfs3-cmake/CMakeLists.txt
index 0d075922f07..398b415e307 100644
--- a/contrib/libhdfs3-cmake/CMakeLists.txt
+++ b/contrib/libhdfs3-cmake/CMakeLists.txt
@@ -183,7 +183,7 @@ set(SRCS
     )
 
 # target
-add_library(hdfs3 STATIC ${SRCS} ${PROTO_SOURCES} ${PROTO_HEADERS})
+add_library(hdfs3 ${SRCS} ${PROTO_SOURCES} ${PROTO_HEADERS})
 
 if (USE_INTERNAL_PROTOBUF_LIBRARY)
     add_dependencies(hdfs3 protoc)
diff --git a/contrib/librdkafka-cmake/CMakeLists.txt b/contrib/librdkafka-cmake/CMakeLists.txt
index 27e3a8cec05..6dc86bf3219 100644
--- a/contrib/librdkafka-cmake/CMakeLists.txt
+++ b/contrib/librdkafka-cmake/CMakeLists.txt
@@ -54,7 +54,7 @@ set(SRCS
   ${RDKAFKA_SOURCE_DIR}/rdgz.c
 )
 
-add_library(rdkafka ${LINK_MODE} ${SRCS})
+add_library(rdkafka ${SRCS})
 target_include_directories(rdkafka SYSTEM PUBLIC include)
 target_include_directories(rdkafka SYSTEM PUBLIC ${RDKAFKA_SOURCE_DIR})         # Because weird logic with "include_next" is used.
 target_include_directories(rdkafka SYSTEM PRIVATE ${ZSTD_INCLUDE_DIR}/common)   # Because wrong path to "zstd_errors.h" is used.
diff --git a/contrib/libxml2-cmake/CMakeLists.txt b/contrib/libxml2-cmake/CMakeLists.txt
index 85b5a364faa..326aa5c2d45 100644
--- a/contrib/libxml2-cmake/CMakeLists.txt
+++ b/contrib/libxml2-cmake/CMakeLists.txt
@@ -16,7 +16,7 @@ set(SRCS
     ${LIBXML2_SOURCE_DIR}/debugXML.c
     ${LIBXML2_SOURCE_DIR}/xpointer.c
     ${LIBXML2_SOURCE_DIR}/DOCBparser.c
-    ${LIBXML2_SOURCE_DIR}/xmlcatalog.c
+    #${LIBXML2_SOURCE_DIR}/xmlcatalog.c
     ${LIBXML2_SOURCE_DIR}/c14n.c
     ${LIBXML2_SOURCE_DIR}/xmlreader.c
     ${LIBXML2_SOURCE_DIR}/xmlstring.c
@@ -26,7 +26,7 @@ set(SRCS
     ${LIBXML2_SOURCE_DIR}/trionan.c
     ${LIBXML2_SOURCE_DIR}/pattern.c
     ${LIBXML2_SOURCE_DIR}/globals.c
-    ${LIBXML2_SOURCE_DIR}/xmllint.c
+    #${LIBXML2_SOURCE_DIR}/xmllint.c
     ${LIBXML2_SOURCE_DIR}/chvalid.c
     ${LIBXML2_SOURCE_DIR}/relaxng.c
     ${LIBXML2_SOURCE_DIR}/list.c
@@ -40,7 +40,7 @@ set(SRCS
     ${LIBXML2_SOURCE_DIR}/xmlschemas.c
     ${LIBXML2_SOURCE_DIR}/SAX2.c
     ${LIBXML2_SOURCE_DIR}/threads.c
-    ${LIBXML2_SOURCE_DIR}/runsuite.c
+    #${LIBXML2_SOURCE_DIR}/runsuite.c
     ${LIBXML2_SOURCE_DIR}/catalog.c
     ${LIBXML2_SOURCE_DIR}/uri.c
     ${LIBXML2_SOURCE_DIR}/xmlmodule.c
@@ -48,7 +48,7 @@ set(SRCS
     ${LIBXML2_SOURCE_DIR}/parserInternals.c
     ${LIBXML2_SOURCE_DIR}/xmlwriter.c
     ${LIBXML2_SOURCE_DIR}/xmlunicode.c
-    ${LIBXML2_SOURCE_DIR}/runxmlconf.c
+    #${LIBXML2_SOURCE_DIR}/runxmlconf.c
     ${LIBXML2_SOURCE_DIR}/xmlmemory.c
     ${LIBXML2_SOURCE_DIR}/nanoftp.c
     ${LIBXML2_SOURCE_DIR}/xmlschemastypes.c
@@ -56,7 +56,7 @@ set(SRCS
     ${LIBXML2_SOURCE_DIR}/nanohttp.c
     ${LIBXML2_SOURCE_DIR}/schematron.c
 )
-add_library(libxml2 STATIC ${SRCS})
+add_library(libxml2 ${SRCS})
 
 target_link_libraries(libxml2 ${ZLIB_LIBRARIES})
 
diff --git a/contrib/mariadb-connector-c-cmake/CMakeLists.txt b/contrib/mariadb-connector-c-cmake/CMakeLists.txt
index 9b095a2e15b..ced97b828cc 100644
--- a/contrib/mariadb-connector-c-cmake/CMakeLists.txt
+++ b/contrib/mariadb-connector-c-cmake/CMakeLists.txt
@@ -2,7 +2,7 @@ set(MARIADB_CLIENT_SOURCE_DIR ${CMAKE_SOURCE_DIR}/contrib/mariadb-connector-c)
 set(MARIADB_CLIENT_BINARY_DIR ${CMAKE_BINARY_DIR}/contrib/mariadb-connector-c)
 
 set(SRCS
-${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/bmove_upp.c
+#${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/bmove_upp.c
 ${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/get_password.c
 ${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_alloc.c
 ${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_array.c
@@ -58,10 +58,10 @@ if(OPENSSL_LIBRARIES)
     list(APPEND SRCS ${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/secure/openssl.c)
 endif()
 
-add_library(mysqlclient STATIC ${SRCS})
+add_library(mysqlclient ${SRCS})
 
 if(OPENSSL_LIBRARIES)
-    target_link_libraries(mysqlclient ${OPENSSL_LIBRARIES})
+    target_link_libraries(mysqlclient PRIVATE ${OPENSSL_LIBRARIES})
     target_compile_definitions(mysqlclient PRIVATE -D HAVE_OPENSSL -D HAVE_TLS)
 endif()
 
diff --git a/contrib/re2_st/CMakeLists.txt b/contrib/re2_st/CMakeLists.txt
index 6bc7fd8f343..c7243346988 100644
--- a/contrib/re2_st/CMakeLists.txt
+++ b/contrib/re2_st/CMakeLists.txt
@@ -10,7 +10,7 @@ foreach (src ${RE2_SOURCES_})
     list(APPEND RE2_ST_SOURCES ${RE2_SOURCE_DIR}/${src})
 endforeach ()
 
-add_library (re2_st ${RE2_ST_SOURCES})
+add_library(re2_st ${RE2_ST_SOURCES})
 target_compile_definitions (re2_st PRIVATE NDEBUG NO_THREADS re2=re2_st)
 target_include_directories (re2_st PRIVATE .)
 target_include_directories (re2_st SYSTEM PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${RE2_SOURCE_DIR})
diff --git a/contrib/unixodbc-cmake/CMakeLists.txt b/contrib/unixodbc-cmake/CMakeLists.txt
index 33c6e115065..e15f263d096 100644
--- a/contrib/unixodbc-cmake/CMakeLists.txt
+++ b/contrib/unixodbc-cmake/CMakeLists.txt
@@ -23,7 +23,7 @@ ${ODBC_SOURCE_DIR}/libltdl/loaders/preopen.c
 ${CMAKE_CURRENT_SOURCE_DIR}/linux_x86_64/libltdl/libltdlcS.c
 )
 
-add_library(ltdl ${LINK_MODE} ${SRCS})
+add_library(ltdl ${SRCS})
 
 target_include_directories(ltdl PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/linux_x86_64/libltdl)
 target_include_directories(ltdl PUBLIC ${ODBC_SOURCE_DIR}/libltdl)
@@ -273,7 +273,7 @@ ${ODBC_SOURCE_DIR}/lst/lstSetFreeFunc.c
 ${ODBC_SOURCE_DIR}/lst/_lstVisible.c
 )
 
-add_library(unixodbc ${LINK_MODE} ${SRCS})
+add_library(unixodbc ${SRCS})
 
 target_link_libraries(unixodbc ltdl)
 
diff --git a/contrib/zstd-cmake/CMakeLists.txt b/contrib/zstd-cmake/CMakeLists.txt
index 3686e9ae1c0..6d8ecce2a44 100644
--- a/contrib/zstd-cmake/CMakeLists.txt
+++ b/contrib/zstd-cmake/CMakeLists.txt
@@ -125,6 +125,6 @@ IF (ZSTD_LEGACY_SUPPORT)
             ${LIBRARY_LEGACY_DIR}/zstd_v07.h)
 ENDIF (ZSTD_LEGACY_SUPPORT)
 
-ADD_LIBRARY(zstd ${LINK_MODE} ${Sources} ${Headers})
+ADD_LIBRARY(zstd ${Sources} ${Headers})
 
 target_include_directories (zstd PUBLIC ${LIBRARY_DIR})
diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt
index 63e97f4e061..a04af1039de 100644
--- a/dbms/CMakeLists.txt
+++ b/dbms/CMakeLists.txt
@@ -134,7 +134,7 @@ list (APPEND dbms_headers src/TableFunctions/ITableFunction.h   src/TableFunctio
 list (APPEND dbms_sources src/Dictionaries/DictionaryFactory.cpp src/Dictionaries/DictionarySourceFactory.cpp src/Dictionaries/DictionaryStructure.cpp)
 list (APPEND dbms_headers src/Dictionaries/DictionaryFactory.h   src/Dictionaries/DictionarySourceFactory.h   src/Dictionaries/DictionaryStructure.h)
 
-add_library(clickhouse_common_io ${LINK_MODE} ${clickhouse_common_io_headers} ${clickhouse_common_io_sources})
+add_library(clickhouse_common_io ${clickhouse_common_io_headers} ${clickhouse_common_io_sources})
 
 if (OS_FREEBSD)
     target_compile_definitions (clickhouse_common_io PUBLIC CLOCK_MONOTONIC_COARSE=CLOCK_MONOTONIC_FAST)
diff --git a/dbms/programs/CMakeLists.txt b/dbms/programs/CMakeLists.txt
index b6928652801..52b58ad0df8 100644
--- a/dbms/programs/CMakeLists.txt
+++ b/dbms/programs/CMakeLists.txt
@@ -44,7 +44,7 @@ macro(clickhouse_program_add_library name)
     set(CLICKHOUSE_${name_uc}_INCLUDE ${CLICKHOUSE_${name_uc}_INCLUDE} PARENT_SCOPE)
 
     if(NOT CLICKHOUSE_ONE_SHARED)
-        add_library(clickhouse-${name}-lib ${LINK_MODE} ${CLICKHOUSE_${name_uc}_SOURCES})
+        add_library(clickhouse-${name}-lib ${CLICKHOUSE_${name_uc}_SOURCES})
 
         set(_link ${CLICKHOUSE_${name_uc}_LINK}) # can't use ${} in if()
         if(_link)
diff --git a/dbms/programs/clang/Compiler-5.0.0/CMakeLists.txt b/dbms/programs/clang/Compiler-5.0.0/CMakeLists.txt
index e0171630bf2..83e38cea257 100644
--- a/dbms/programs/clang/Compiler-5.0.0/CMakeLists.txt
+++ b/dbms/programs/clang/Compiler-5.0.0/CMakeLists.txt
@@ -2,7 +2,7 @@ add_definitions(-Wno-error -Wno-unused-parameter -Wno-non-virtual-dtor -U_LIBCPP
 
 link_directories(${LLVM_LIBRARY_DIRS})
 
-add_library(clickhouse-compiler-lib ${LINK_MODE}
+add_library(clickhouse-compiler-lib
     driver.cpp
     cc1_main.cpp
     cc1as_main.cpp
diff --git a/dbms/programs/clang/Compiler-6.0.0/CMakeLists.txt b/dbms/programs/clang/Compiler-6.0.0/CMakeLists.txt
index bac622ab09e..4a046674afc 100644
--- a/dbms/programs/clang/Compiler-6.0.0/CMakeLists.txt
+++ b/dbms/programs/clang/Compiler-6.0.0/CMakeLists.txt
@@ -1,8 +1,9 @@
+
 add_definitions(-Wno-error -Wno-unused-parameter -Wno-non-virtual-dtor -U_LIBCPP_DEBUG)
 
 link_directories(${LLVM_LIBRARY_DIRS})
 
-add_library(clickhouse-compiler-lib ${LINK_MODE}
+add_library(clickhouse-compiler-lib
     driver.cpp
     cc1_main.cpp
     cc1as_main.cpp
diff --git a/dbms/programs/clang/Compiler-7.0.0/CMakeLists.txt b/dbms/programs/clang/Compiler-7.0.0/CMakeLists.txt
index 35e23cc6b46..a042c821ec4 100644
--- a/dbms/programs/clang/Compiler-7.0.0/CMakeLists.txt
+++ b/dbms/programs/clang/Compiler-7.0.0/CMakeLists.txt
@@ -2,7 +2,7 @@ add_definitions(-Wno-error -Wno-unused-parameter -Wno-non-virtual-dtor -U_LIBCPP
 
 link_directories(${LLVM_LIBRARY_DIRS})
 
-add_library(clickhouse-compiler-lib ${LINK_MODE}
+add_library(clickhouse-compiler-lib
     driver.cpp
     cc1_main.cpp
     cc1gen_reproducer_main.cpp
diff --git a/dbms/programs/clang/Compiler-7.0.0bundled/CMakeLists.txt b/dbms/programs/clang/Compiler-7.0.0bundled/CMakeLists.txt
index d03052ffc28..a5f8314b862 100644
--- a/dbms/programs/clang/Compiler-7.0.0bundled/CMakeLists.txt
+++ b/dbms/programs/clang/Compiler-7.0.0bundled/CMakeLists.txt
@@ -2,7 +2,7 @@ add_definitions(-Wno-error -Wno-unused-parameter -Wno-non-virtual-dtor -U_LIBCPP
 
 link_directories(${LLVM_LIBRARY_DIRS})
 
-add_library(clickhouse-compiler-lib ${LINK_MODE}
+add_library(clickhouse-compiler-lib
     driver.cpp
     cc1_main.cpp
     cc1as_main.cpp
diff --git a/dbms/src/AggregateFunctions/CMakeLists.txt b/dbms/src/AggregateFunctions/CMakeLists.txt
index 148bf90446c..c458675922a 100644
--- a/dbms/src/AggregateFunctions/CMakeLists.txt
+++ b/dbms/src/AggregateFunctions/CMakeLists.txt
@@ -19,7 +19,7 @@ list(REMOVE_ITEM clickhouse_aggregate_functions_headers
     FactoryHelpers.h
 )
 
-add_library(clickhouse_aggregate_functions ${LINK_MODE} ${clickhouse_aggregate_functions_sources})
+add_library(clickhouse_aggregate_functions ${clickhouse_aggregate_functions_sources})
 target_link_libraries(clickhouse_aggregate_functions PRIVATE dbms PUBLIC ${CITYHASH_LIBRARIES})
 target_include_directories (clickhouse_aggregate_functions BEFORE PRIVATE ${COMMON_INCLUDE_DIR})
 
diff --git a/dbms/src/Client/CMakeLists.txt b/dbms/src/Client/CMakeLists.txt
index cc8b3f8eda8..9b9ec442a3c 100644
--- a/dbms/src/Client/CMakeLists.txt
+++ b/dbms/src/Client/CMakeLists.txt
@@ -1,7 +1,7 @@
 # TODO: make separate lib datastream, block, ...
 #include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
 #add_headers_and_sources(clickhouse_client .)
-#add_library(clickhouse_client ${LINK_MODE} ${clickhouse_client_headers} ${clickhouse_client_sources})
+#add_library(clickhouse_client ${clickhouse_client_headers} ${clickhouse_client_sources})
 #target_link_libraries (clickhouse_client clickhouse_common_io ${Poco_Net_LIBRARY})
 #target_include_directories (clickhouse_client PRIVATE ${DBMS_INCLUDE_DIR})
 
diff --git a/dbms/src/Common/Config/CMakeLists.txt b/dbms/src/Common/Config/CMakeLists.txt
index 298f1ed241b..e1f78e2c213 100644
--- a/dbms/src/Common/Config/CMakeLists.txt
+++ b/dbms/src/Common/Config/CMakeLists.txt
@@ -2,7 +2,7 @@ include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
 
 add_headers_and_sources(clickhouse_common_config .)
 
-add_library(clickhouse_common_config ${LINK_MODE} ${clickhouse_common_config_headers} ${clickhouse_common_config_sources})
+add_library(clickhouse_common_config ${clickhouse_common_config_headers} ${clickhouse_common_config_sources})
 
 target_link_libraries(clickhouse_common_config PUBLIC common PRIVATE clickhouse_common_zookeeper string_utils PUBLIC ${Poco_XML_LIBRARY} ${Poco_Util_LIBRARY} Threads::Threads)
 target_include_directories(clickhouse_common_config PUBLIC ${DBMS_INCLUDE_DIR})
diff --git a/dbms/src/Common/StringUtils/CMakeLists.txt b/dbms/src/Common/StringUtils/CMakeLists.txt
index 3e2b4102be4..c63e0f260ba 100644
--- a/dbms/src/Common/StringUtils/CMakeLists.txt
+++ b/dbms/src/Common/StringUtils/CMakeLists.txt
@@ -5,5 +5,5 @@ include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
 
 add_headers_and_sources(clickhouse_common_stringutils .)
 
-add_library(string_utils ${LINK_MODE} ${clickhouse_common_stringutils_headers} ${clickhouse_common_stringutils_sources})
+add_library(string_utils ${clickhouse_common_stringutils_headers} ${clickhouse_common_stringutils_sources})
 target_include_directories (string_utils PRIVATE ${DBMS_INCLUDE_DIR})
diff --git a/dbms/src/Common/ZooKeeper/CMakeLists.txt b/dbms/src/Common/ZooKeeper/CMakeLists.txt
index 41fd406faae..6a611886e65 100644
--- a/dbms/src/Common/ZooKeeper/CMakeLists.txt
+++ b/dbms/src/Common/ZooKeeper/CMakeLists.txt
@@ -2,7 +2,7 @@ include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
 
 add_headers_and_sources(clickhouse_common_zookeeper .)
 
-add_library(clickhouse_common_zookeeper ${LINK_MODE} ${clickhouse_common_zookeeper_headers} ${clickhouse_common_zookeeper_sources})
+add_library(clickhouse_common_zookeeper ${clickhouse_common_zookeeper_headers} ${clickhouse_common_zookeeper_sources})
 
 target_link_libraries (clickhouse_common_zookeeper PUBLIC clickhouse_common_io common PRIVATE string_utils PUBLIC ${Poco_Util_LIBRARY} Threads::Threads)
 target_include_directories(clickhouse_common_zookeeper PUBLIC ${DBMS_INCLUDE_DIR})
diff --git a/dbms/src/Compression/CMakeLists.txt b/dbms/src/Compression/CMakeLists.txt
index 1369493cb7a..2ca2a043978 100644
--- a/dbms/src/Compression/CMakeLists.txt
+++ b/dbms/src/Compression/CMakeLists.txt
@@ -1,6 +1,6 @@
 include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
 add_headers_and_sources(clickhouse_compression .)
-add_library(clickhouse_compression ${LINK_MODE} ${clickhouse_compression_headers} ${clickhouse_compression_sources})
+add_library(clickhouse_compression ${clickhouse_compression_headers} ${clickhouse_compression_sources})
 target_link_libraries(clickhouse_compression PRIVATE clickhouse_parsers clickhouse_common_io ${ZSTD_LIBRARY} ${LZ4_LIBRARY} ${CITYHASH_LIBRARIES})
 target_include_directories(clickhouse_compression PUBLIC ${DBMS_INCLUDE_DIR})
 target_include_directories(clickhouse_compression SYSTEM PUBLIC ${PCG_RANDOM_INCLUDE_DIR})
diff --git a/dbms/src/Dictionaries/CMakeLists.txt b/dbms/src/Dictionaries/CMakeLists.txt
index 59999bcb8e5..29b9649e661 100644
--- a/dbms/src/Dictionaries/CMakeLists.txt
+++ b/dbms/src/Dictionaries/CMakeLists.txt
@@ -14,7 +14,7 @@ add_headers_and_sources(clickhouse_dictionaries ${CMAKE_CURRENT_BINARY_DIR}/gene
 list(REMOVE_ITEM clickhouse_dictionaries_sources DictionaryFactory.cpp DictionarySourceFactory.cpp DictionaryStructure.cpp)
 list(REMOVE_ITEM clickhouse_dictionaries_headers DictionaryFactory.h DictionarySourceFactory.h DictionaryStructure.h)
 
-add_library(clickhouse_dictionaries ${LINK_MODE} ${clickhouse_dictionaries_sources})
+add_library(clickhouse_dictionaries ${clickhouse_dictionaries_sources})
 target_link_libraries(clickhouse_dictionaries PRIVATE dbms clickhouse_common_io ${BTRIE_LIBRARIES} PUBLIC Threads::Threads)
 
 if(Poco_SQL_FOUND AND NOT USE_INTERNAL_POCO_LIBRARY)
diff --git a/dbms/src/Dictionaries/Embedded/CMakeLists.txt b/dbms/src/Dictionaries/Embedded/CMakeLists.txt
index 2af439c9677..9324753808f 100644
--- a/dbms/src/Dictionaries/Embedded/CMakeLists.txt
+++ b/dbms/src/Dictionaries/Embedded/CMakeLists.txt
@@ -1,5 +1,5 @@
 include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
 add_headers_and_sources(clickhouse_dictionaries_embedded .)
 add_headers_and_sources(clickhouse_dictionaries_embedded GeodataProviders)
-add_library(clickhouse_dictionaries_embedded ${LINK_MODE} ${clickhouse_dictionaries_embedded_sources})
+add_library(clickhouse_dictionaries_embedded ${clickhouse_dictionaries_embedded_sources})
 target_link_libraries(clickhouse_dictionaries_embedded PRIVATE clickhouse_common_io ${MYSQLXX_LIBRARY})
diff --git a/dbms/src/Functions/CMakeLists.txt b/dbms/src/Functions/CMakeLists.txt
index 47d86190562..513cb2002a1 100644
--- a/dbms/src/Functions/CMakeLists.txt
+++ b/dbms/src/Functions/CMakeLists.txt
@@ -7,7 +7,7 @@ add_headers_and_sources(clickhouse_functions .)
 list(REMOVE_ITEM clickhouse_functions_sources IFunction.cpp FunctionFactory.cpp FunctionHelpers.cpp)
 list(REMOVE_ITEM clickhouse_functions_headers IFunction.h FunctionFactory.h FunctionHelpers.h)
 
-add_library(clickhouse_functions ${LINK_MODE} ${clickhouse_functions_sources})
+add_library(clickhouse_functions ${clickhouse_functions_sources})
 
 target_link_libraries(clickhouse_functions
     PUBLIC
diff --git a/dbms/src/Parsers/CMakeLists.txt b/dbms/src/Parsers/CMakeLists.txt
index 790e151dab3..176be236658 100644
--- a/dbms/src/Parsers/CMakeLists.txt
+++ b/dbms/src/Parsers/CMakeLists.txt
@@ -1,6 +1,6 @@
 include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
 add_headers_and_sources(clickhouse_parsers .)
-add_library(clickhouse_parsers ${LINK_MODE} ${clickhouse_parsers_headers} ${clickhouse_parsers_sources})
+add_library(clickhouse_parsers ${clickhouse_parsers_headers} ${clickhouse_parsers_sources})
 target_link_libraries(clickhouse_parsers PUBLIC clickhouse_common_io)
 target_include_directories(clickhouse_parsers PUBLIC ${DBMS_INCLUDE_DIR})
 
diff --git a/dbms/src/Storages/Kafka/CMakeLists.txt b/dbms/src/Storages/Kafka/CMakeLists.txt
index 520f1311d1d..e581b379322 100644
--- a/dbms/src/Storages/Kafka/CMakeLists.txt
+++ b/dbms/src/Storages/Kafka/CMakeLists.txt
@@ -1,7 +1,7 @@
 if(USE_RDKAFKA)
     include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
     add_headers_and_sources(clickhouse_storage_kafka .)
-    add_library(clickhouse_storage_kafka ${LINK_MODE} ${clickhouse_storage_kafka_sources})
+    add_library(clickhouse_storage_kafka ${clickhouse_storage_kafka_sources})
     target_link_libraries(clickhouse_storage_kafka PRIVATE clickhouse_common_io ${CPPKAFKA_LIBRARY} ${RDKAFKA_LIBRARY})
     if(NOT USE_INTERNAL_RDKAFKA_LIBRARY)
         target_include_directories(clickhouse_storage_kafka SYSTEM BEFORE PRIVATE ${RDKAFKA_INCLUDE_DIR})
diff --git a/dbms/src/Storages/System/CMakeLists.txt b/dbms/src/Storages/System/CMakeLists.txt
index 418c25de5fd..d59befa56b4 100644
--- a/dbms/src/Storages/System/CMakeLists.txt
+++ b/dbms/src/Storages/System/CMakeLists.txt
@@ -11,5 +11,5 @@ configure_file (StorageSystemBuildOptions.generated.cpp.in ${CONFIG_BUILD})
 include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
 add_headers_and_sources(storages_system .)
 list (APPEND storages_system_sources ${CONFIG_BUILD})
-add_library(clickhouse_storages_system ${LINK_MODE} ${storages_system_headers} ${storages_system_sources})
+add_library(clickhouse_storages_system ${storages_system_headers} ${storages_system_sources})
 target_link_libraries(clickhouse_storages_system PRIVATE dbms common string_utils clickhouse_common_zookeeper)
diff --git a/dbms/src/TableFunctions/CMakeLists.txt b/dbms/src/TableFunctions/CMakeLists.txt
index 0ab5acec19c..9bd1374cd4d 100644
--- a/dbms/src/TableFunctions/CMakeLists.txt
+++ b/dbms/src/TableFunctions/CMakeLists.txt
@@ -4,5 +4,5 @@ add_headers_and_sources(clickhouse_table_functions .)
 list(REMOVE_ITEM clickhouse_table_functions_sources ITableFunction.cpp TableFunctionFactory.cpp)
 list(REMOVE_ITEM clickhouse_table_functions_headers ITableFunction.h TableFunctionFactory.h)
 
-add_library(clickhouse_table_functions ${LINK_MODE} ${clickhouse_table_functions_sources})
+add_library(clickhouse_table_functions ${clickhouse_table_functions_sources})
 target_link_libraries(clickhouse_table_functions PRIVATE clickhouse_storages_system dbms ${Poco_Foundation_LIBRARY})
diff --git a/libs/libcommon/CMakeLists.txt b/libs/libcommon/CMakeLists.txt
index 02199c21a4f..6b4d1327cef 100644
--- a/libs/libcommon/CMakeLists.txt
+++ b/libs/libcommon/CMakeLists.txt
@@ -1,7 +1,7 @@
 set (CONFIG_COMMON ${CMAKE_CURRENT_BINARY_DIR}/include/common/config_common.h)
 configure_file (${CMAKE_CURRENT_SOURCE_DIR}/include/common/config_common.h.in ${CONFIG_COMMON})
 
-add_library (apple_rt
+add_library(apple_rt
     src/apple_rt.cpp
     include/port/clock.h
 )
@@ -10,7 +10,7 @@ if (DEFINED APPLE_HAVE_CLOCK_GETTIME)
     target_compile_definitions(apple_rt PUBLIC -DAPPLE_HAVE_CLOCK_GETTIME=${APPLE_HAVE_CLOCK_GETTIME})
 endif ()
 
-add_library (common ${LINK_MODE}
+add_library(common
     src/DateLUT.cpp
     src/DateLUTImpl.cpp
     src/preciseExp10.c
diff --git a/libs/libdaemon/CMakeLists.txt b/libs/libdaemon/CMakeLists.txt
index 181030009b0..a7839dc158f 100644
--- a/libs/libdaemon/CMakeLists.txt
+++ b/libs/libdaemon/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_library (daemon ${LINK_MODE}
+add_library (daemon
     src/BaseDaemon.cpp
     src/GraphiteWriter.cpp
     src/ExtendedLogChannel.cpp
diff --git a/libs/libmemcpy/CMakeLists.txt b/libs/libmemcpy/CMakeLists.txt
index c06085aacca..2f000d85e3c 100644
--- a/libs/libmemcpy/CMakeLists.txt
+++ b/libs/libmemcpy/CMakeLists.txt
@@ -1 +1 @@
-add_library (memcpy memcpy.c)
+add_library(memcpy memcpy.c)
diff --git a/libs/libmysqlxx/CMakeLists.txt b/libs/libmysqlxx/CMakeLists.txt
index d0c95312122..1e3aedfa888 100644
--- a/libs/libmysqlxx/CMakeLists.txt
+++ b/libs/libmysqlxx/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_library (mysqlxx ${LINK_MODE}
+add_library (mysqlxx
     src/Connection.cpp
     src/Exception.cpp
     src/Query.cpp

From 37414490173f464d198bc963f8d47c075e9edb0d Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Wed, 24 Apr 2019 19:45:21 +0300
Subject: [PATCH 045/118] Added option "always_load_suggestion_data" to
 clickhouse-client #5066

---
 dbms/programs/client/Client.cpp | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp
index 54271996290..e322eccee98 100644
--- a/dbms/programs/client/Client.cpp
+++ b/dbms/programs/client/Client.cpp
@@ -479,6 +479,15 @@ private:
         }
         else
         {
+            if (config().getBool("always_load_suggestion_data", false))
+            {
+#if USE_READLINE
+                Suggest::instance().load(connection_parameters, config().getInt("suggestion_limit"));
+#else
+                throw Exception("Command line suggestions cannot work without readline", ErrorCodes::BAD_ARGUMENTS);
+#endif
+            }
+
             query_id = config().getString("query_id", "");
             nonInteractive();
 
@@ -1629,6 +1638,7 @@ public:
             ("database,d", po::value<std::string>(), "database")
             ("pager", po::value<std::string>(), "pager")
             ("disable_suggestion,A", "Disable loading suggestion data. Note that suggestion data is loaded asynchronously through a second connection to ClickHouse server. Also it is reasonable to disable suggestion if you want to paste a query with TAB characters. Shorthand option -A is for those who get used to mysql client.")
+            ("always_load_suggestion_data", "Load suggestion data even if clickhouse-client is run in non-interactive mode. Used for testing.")
             ("suggestion_limit", po::value<int>()->default_value(10000),
                 "Suggestion limit for how many databases, tables and columns to fetch.")
             ("multiline,m", "multiline")
@@ -1782,6 +1792,13 @@ public:
             server_logs_file = options["server_logs_file"].as<std::string>();
         if (options.count("disable_suggestion"))
             config().setBool("disable_suggestion", true);
+        if (options.count("always_load_suggestion_data"))
+        {
+            if (options.count("disable_suggestion"))
+                throw Exception("Command line parameters disable_suggestion (-A) and always_load_suggestion_data cannot be specified simultaneously",
+                    ErrorCodes::BAD_ARGUMENTS);
+            config().setBool("always_load_suggestion_data", true);
+        }
         if (options.count("suggestion_limit"))
             config().setInt("suggestion_limit", options["suggestion_limit"].as<int>());
     }

From 2428ba70f53ead119cc535770572c97602249ccf Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Wed, 24 Apr 2019 19:47:04 +0300
Subject: [PATCH 046/118] Added option "always_load_suggestion_data" to
 clickhouse-client #5066

---
 dbms/programs/client/Client.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp
index e322eccee98..6e006c24648 100644
--- a/dbms/programs/client/Client.cpp
+++ b/dbms/programs/client/Client.cpp
@@ -482,6 +482,7 @@ private:
             if (config().getBool("always_load_suggestion_data", false))
             {
 #if USE_READLINE
+                SCOPE_EXIT({ Suggest::instance().finalize(); });
                 Suggest::instance().load(connection_parameters, config().getInt("suggestion_limit"));
 #else
                 throw Exception("Command line suggestions cannot work without readline", ErrorCodes::BAD_ARGUMENTS);

From ce9b5a4341f966c9c310779040e38c547f4aa747 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Wed, 24 Apr 2019 19:52:44 +0300
Subject: [PATCH 047/118] Added bug #5066

---
 dbms/tests/queries/bugs/00938_client_suggestions.sh | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100755 dbms/tests/queries/bugs/00938_client_suggestions.sh

diff --git a/dbms/tests/queries/bugs/00938_client_suggestions.sh b/dbms/tests/queries/bugs/00938_client_suggestions.sh
new file mode 100755
index 00000000000..b4bd9e4480d
--- /dev/null
+++ b/dbms/tests/queries/bugs/00938_client_suggestions.sh
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+. $CURDIR/../shell_config.sh
+
+for i in {1..100}; do $CLICKHOUSE_CLIENT --always_load_suggestion_data --query="SELECT 1 FORMAT Null"; done

From 33252941f9efa5d13f28fe95c3668e3afb838615 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Wed, 24 Apr 2019 21:40:26 +0300
Subject: [PATCH 048/118] Added a comment

---
 dbms/programs/client/Client.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp
index 6e006c24648..805e582e637 100644
--- a/dbms/programs/client/Client.cpp
+++ b/dbms/programs/client/Client.cpp
@@ -479,6 +479,7 @@ private:
         }
         else
         {
+            /// This is intended for testing purposes.
             if (config().getBool("always_load_suggestion_data", false))
             {
 #if USE_READLINE

From 352d1a8dcf27088db8f0ead4d53a018a3dee0384 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Wed, 24 Apr 2019 22:38:11 +0300
Subject: [PATCH 049/118] Fixed inconsistent indentation

---
 dbms/programs/server/ReplicasStatusHandler.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dbms/programs/server/ReplicasStatusHandler.cpp b/dbms/programs/server/ReplicasStatusHandler.cpp
index de916eddc7e..a5cd368c4c5 100644
--- a/dbms/programs/server/ReplicasStatusHandler.cpp
+++ b/dbms/programs/server/ReplicasStatusHandler.cpp
@@ -85,7 +85,7 @@ void ReplicasStatusHandler::handleRequest(Poco::Net::HTTPServerRequest & request
 
             if (!response.sent())
             {
-        /// We have not sent anything yet and we don't even know if we need to compress response.
+                /// We have not sent anything yet and we don't even know if we need to compress response.
                 response.send() << getCurrentExceptionMessage(false) << std::endl;
             }
         }

From 25fafde574585bdd968065505b5fecb8b0a2c033 Mon Sep 17 00:00:00 2001
From: ogorbacheva <ogorbacheva@yandex-team.ru>
Date: Thu, 25 Apr 2019 00:08:49 +0300
Subject: [PATCH 050/118] Review2

---
 .../functions/array_functions.md              | 16 +++++------
 .../functions/array_functions.md              | 28 +++++++++++++++----
 2 files changed, 30 insertions(+), 14 deletions(-)

diff --git a/docs/en/query_language/functions/array_functions.md b/docs/en/query_language/functions/array_functions.md
index dd6a4392e58..1292f55ca41 100644
--- a/docs/en/query_language/functions/array_functions.md
+++ b/docs/en/query_language/functions/array_functions.md
@@ -513,9 +513,9 @@ SELECT arraySort((x) -> -x, [1, 2, 3]) as res;
 └─────────┘
 ```
 
-In this example, for each element of the source array, the lambda function returns the "weight" of the element that will be applied for sorting, that is, [1 –> -1, 2 –> -2, 3 –> -3]. Since the elements are sorted by increasing of their "weight", the result is [3, 2, 1].
+For each element of the source array, the lambda function returns the sorting key, that is, [1 –> -1, 2 –> -2, 3 –> -3]. Since the `arraySort` function sorts the keys in ascending order, the result is [3, 2, 1]. Thus, the `(x) –> -x` lambda function sets the [descending order](#array_functions-reverse-sort) in a sorting. 
 
-The lambda function can accept multiple arguments. In this case, you need to pass the `arraySort` function several arrays of identical length that the arguments of lambda function will correspond to. For example:
+The lambda function can accept multiple arguments. In this case, you need to pass the `arraySort` function several arrays of identical length that the arguments of lambda function will correspond to. The resulting array will consist of elements from the first input array; elements from the next input array(s) specify the sorting keys. For example:
 
 ``` sql
 SELECT arraySort((x, y) -> y, ['hello', 'world'], [2, 1]) as res;
@@ -527,9 +527,9 @@ SELECT arraySort((x, y) -> y, ['hello', 'world'], [2, 1]) as res;
 └────────────────────┘
 ```
 
-Here, the elements that are passed in the second array ([2, 1]) define a new position of each corresponding element from the source array (['hello', 'world']), that is, ['hello' –> 2, 'world' –> 1]. So, 'hello' will be the second element in a result, and 'world' will be the first.
+Here, the elements that are passed in the second array ([2, 1]) define a sorting key for the corresponding element from the source array (['hello', 'world']), that is, ['hello' –> 2, 'world' –> 1]. Since the lambda function doesn't use `x`, actual values of the source array don't affect the order in the result. So, 'hello' will be the second element in the result, and 'world' will be the first.
 
-Another examples are shown below.
+Other examples are shown below.
 
 ``` sql
 SELECT arraySort((x, y) -> y, [0, 1, 2], ['c', 'b', 'a']) as res;
@@ -608,9 +608,9 @@ SELECT arrayReverseSort((x) -> -x, [1, 2, 3]) as res;
 The array is sorted in the following way:
 
 1. At first, the source array ([1, 2, 3]) is sorted according to the result of the lambda function applied to the elements of the array. The result is an array [3, 2, 1].
-2. Array that was sorted on the previous step, is reversed. So, the final result is [1, 2, 3].
+2. Array that is obtained on the previous step, is reversed. So, the final result is [1, 2, 3].
   
-The lambda function can accept multiple arguments. In this case, you need to pass the `arraySort` function several arrays of identical length that the arguments of lambda function will correspond to. For example:
+The lambda function can accept multiple arguments. In this case, you need to pass the `arrayReverseSort` function several arrays of identical length that the arguments of lambda function will correspond to. The resulting array will consist of elements from the first input array; elements from the next input array(s) specify the sorting keys. For example:
 
 ``` sql
 SELECT arrayReverseSort((x, y) -> y, ['hello', 'world'], [2, 1]) as res;
@@ -623,10 +623,10 @@ SELECT arrayReverseSort((x, y) -> y, ['hello', 'world'], [2, 1]) as res;
 
 In this example, the array is sorted in the following way:
 
-1. At first, the source array (['hello', 'world']) is sorted according to the result of the lambda function applied to the elements of the arrays. The elements that are passed in the second array ([2, 1]), define a new position of corresponding elements from the source array. The result is an array ['world', 'hello'].
+1. At first, the source array (['hello', 'world']) is sorted according to the result of the lambda function applied to the elements of the arrays. The elements that are passed in the second array ([2, 1]), define the sorting keys for corresponding elements from the source array. The result is an array ['world', 'hello'].
 2. Array that was sorted on the previous step, is reversed. So, the final result is ['hello', 'world'].
                       
-Another examples are shown below. 
+Other examples are shown below. 
 
 ``` sql
 SELECT arrayReverseSort((x, y) -> y, [4, 3, 5], ['a', 'b', 'c']) AS res;
diff --git a/docs/ru/query_language/functions/array_functions.md b/docs/ru/query_language/functions/array_functions.md
index e6f4b337dc5..7945276434f 100644
--- a/docs/ru/query_language/functions/array_functions.md
+++ b/docs/ru/query_language/functions/array_functions.md
@@ -513,9 +513,9 @@ SELECT arraySort((x) -> -x, [1, 2, 3]) as res;
 └─────────┘
 ```
 
-Для каждого элемента исходного массива лямбда-функция возвращает "вес" этого элемента — он будет использоваться при сортировке. То есть [1 –> -1, 2 –> -2, 3 –> -3]. Так как элементы сортируются по возрастанию их "веса", результат будет [3, 2, 1].
+Для каждого элемента исходного массива лямбда-функция возвращает ключ сортировки, то есть [1 –> -1, 2 –> -2, 3 –> -3]. Так как `arraySort` сортирует элементы в порядке возрастания ключей, результат будет [3, 2, 1]. Как можно заметить, функция `x –> -x` устанавливает [обратный порядок сортировки](#array_functions-reverse-sort).
 
-Лямбда-функция может принимать несколько аргументов. В этом случае, в функцию `arraySort` нужно передавать несколько массивов, которые будут соответствовать аргументам лямбда-функции. Обратите внимание, массивы должны быть одинаковой длины. Например:
+Лямбда-функция может принимать несколько аргументов. В этом случае, в функцию `arraySort` нужно передавать несколько массивов, которые будут соответствовать аргументам лямбда-функции (массивы должны быть одинаковой длины). Следует иметь в виду, что результат будет содержать элементы только из первого массива; элементы из всех последующих массивов будут задавать ключи сортировки. Например:
 
 ``` sql
 SELECT arraySort((x, y) -> y, ['hello', 'world'], [2, 1]) as res;
@@ -527,7 +527,7 @@ SELECT arraySort((x, y) -> y, ['hello', 'world'], [2, 1]) as res;
 └────────────────────┘
 ```
 
-Элементы, указанные во втором массиве ([2,1]), определяют позицию соответствующих элементов из исходного массива (['hello', 'world']), то есть ['hello' –> 2, 'world' –> 1]. Таким образом, 'hello' будет вторым элементом в отсортированном массиве, а 'world' — первым.
+Элементы, указанные во втором массиве ([2,1]), определяют ключ сортировки для элементов из исходного массива (['hello', 'world']), то есть ['hello' –> 2, 'world' –> 1]. Так как лямбда-функция не использует `x`, элементы исходного массива не влияют на порядок сортировки. Таким образом, 'hello' будет вторым элементом в отсортированном массиве, а 'world' — первым.
 
 Ниже приведены другие примеры.
 
@@ -549,7 +549,7 @@ SELECT arraySort((x, y) -> -y, [0, 1, 2], [1, 2, 3]) as res;
 ```
 
 !!! note "Примечание"
-    Для улучшения эффективности сортировки применяется [Преобразование Шварца](https://ru.wikipedia.org/wiki/%D0%9F%D1%80%D0%B5%D0%BE%D0%B1%D1%80%D0%B0%D0%B7%D0%BE%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5_%D0%A8%D0%B2%D0%B0%D1%80%D1%86%D0%B0).
+    Для улучшения эффективности сортировки применяется [преобразование Шварца](https://ru.wikipedia.org/wiki/%D0%9F%D1%80%D0%B5%D0%BE%D0%B1%D1%80%D0%B0%D0%B7%D0%BE%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5_%D0%A8%D0%B2%D0%B0%D1%80%D1%86%D0%B0).
 
 ## arrayReverseSort([func,] arr, ...) {#array_functions-reverse-sort}
 
@@ -595,6 +595,22 @@ SELECT arrayReverseSort([1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf]) as res;
 
 Функция `arrayReverseSort` является [функцией высшего порядка](higher_order_functions.md). Вы можете передать ей в качестве первого аргумента лямбда-функцию. Например:
 
+``` sql
+SELECT arrayReverseSort((x) -> -x, [1, 2, 3]) as res;
+```
+```
+┌─res─────┐
+│ [1,2,3] │
+└─────────┘
+```
+
+В этом примере, порядок сортировки устанавливается следующим образом:
+
+1. Сначала исходный массив ([1, 2, 3]) сортируется в том порядке, который определяется лямбда-функцией. Результатом будет массив [3, 2, 1].
+2. Массив, который был получен на предыдущем шаге, переворачивается. То есть, получается массив [1, 2, 3].
+  
+Лямбда-функция может принимать на вход несколько аргументов. В этом случае, в функцию `arrayReverseSort` нужно передавать несколько массивов, которые будут соответствовать аргументам лямбда-функции (массивы должны быть одинаковой длины). Следует иметь в виду, что результат будет содержать элементы только из первого массива; элементы из всех последующих массивов будут определять ключи сортировки. Например:
+  
 ``` sql
 SELECT arrayReverseSort((x, y) -> y, ['hello', 'world'], [2, 1]) as res;
 ```
@@ -604,9 +620,9 @@ SELECT arrayReverseSort((x, y) -> y, ['hello', 'world'], [2, 1]) as res;
 └───────────────────┘
 ```
 
-В этом случае, массив сортируется следующим образом:
+В этом примере, массив сортируется следующим образом:
 
-1. Сначала массив сортируется в том порядке, который определяется лямбда-функцией. Элементы, указанные во втором массиве ([2,1]), определяют позицию соответствующих элементов из исходного массива (['hello', 'world']). То есть, промежуточным результатом будет массив ['world', 'hello'].
+1. Сначала массив сортируется в том порядке, который определяется лямбда-функцией. Элементы, указанные во втором массиве ([2,1]), определяют ключи сортировки соответствующих элементов из исходного массива (['hello', 'world']). То есть, будет массив ['world', 'hello'].
 2. Массив, который был отсортирован на предыдущем шаге, переворачивается. Получается массив ['hello', 'world'].
 
 Ниже приведены ещё примеры.

From c6d0e095e0bcf31613fffb5fc6b26a10fb039cfa Mon Sep 17 00:00:00 2001
From: proller <proller@users.noreply.github.com>
Date: Thu, 25 Apr 2019 00:33:04 +0300
Subject: [PATCH 051/118] Prepare cmake for new contrib/poco (#5062)

---
 cmake/Modules/FindODBC.cmake             | 211 +++++++++++++++--------
 cmake/Modules/FindPoco.cmake             |   4 +-
 cmake/find_ccache.cmake                  |   3 +-
 cmake/find_odbc.cmake                    | 111 +++---------
 cmake/find_poco.cmake                    |   4 +-
 cmake/find_ssl.cmake                     |  66 +++++++
 contrib/CMakeLists.txt                   |   4 +
 contrib/unixodbc-cmake/CMakeLists.txt    |   8 +-
 dbms/CMakeLists.txt                      |   6 +-
 dbms/programs/odbc-bridge/CMakeLists.txt |   6 +-
 dbms/src/Dictionaries/CMakeLists.txt     |   4 +-
 libs/libmysqlxx/CMakeLists.txt           |   2 +-
 12 files changed, 250 insertions(+), 179 deletions(-)

diff --git a/cmake/Modules/FindODBC.cmake b/cmake/Modules/FindODBC.cmake
index 66d43e93d2d..55318b52061 100644
--- a/cmake/Modules/FindODBC.cmake
+++ b/cmake/Modules/FindODBC.cmake
@@ -1,88 +1,147 @@
-# This file copied from contrib/poco/cmake/FindODBC.cmake to allow build without submodules
+# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
+# file Copyright.txt or https://cmake.org/licensing for details.
 
+#.rst:
+# FindMySQL
+# -------
 #
-# Find the ODBC driver manager includes and library.
+# Find ODBC Runtime
 #
-# ODBC is an open standard for connecting to different databases in a
-# semi-vendor-independent fashion.  First you install the ODBC driver
-# manager.  Then you need a driver for each separate database you want
-# to connect to (unless a generic one works).  VTK includes neither
-# the driver manager nor the vendor-specific drivers: you have to find
-# those yourself.
+# This will define the following variables::
 #
-# This module defines
-# ODBC_INCLUDE_DIRECTORIES, where to find sql.h
-# ODBC_LIBRARIES, the libraries to link against to use ODBC
-# ODBC_FOUND.  If false, you cannot build anything that requires ODBC.
+#   ODBC_FOUND           - True if the system has the libraries
+#   ODBC_INCLUDE_DIRS    - where to find the headers
+#   ODBC_LIBRARIES       - where to find the libraries
+#   ODBC_DEFINITIONS     - compile definitons
+#
+# Hints:
+# Set ``ODBC_ROOT_DIR`` to the root directory of an installation.
+#
+include(FindPackageHandleStandardArgs)
 
-option (ENABLE_ODBC "Enable ODBC" ${OS_LINUX})
-if (OS_LINUX)
-    option (USE_INTERNAL_ODBC_LIBRARY "Set to FALSE to use system odbc library instead of bundled" ${NOT_UNBUNDLED})
-else ()
-    option (USE_INTERNAL_ODBC_LIBRARY "Set to FALSE to use system odbc library instead of bundled" OFF)
-endif ()
+find_package(PkgConfig QUIET)
+pkg_check_modules(PC_ODBC QUIET odbc)
 
-if (USE_INTERNAL_ODBC_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/unixodbc/README")
-    message (WARNING "submodule contrib/unixodbc is missing. to fix try run: \n git submodule update --init --recursive")
-   set (USE_INTERNAL_ODBC_LIBRARY 0)
-endif ()
+if(WIN32)
+	get_filename_component(kit_dir "[HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Windows Kits\\Installed Roots;KitsRoot]" REALPATH)
+	get_filename_component(kit81_dir "[HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Windows Kits\\Installed Roots;KitsRoot81]" REALPATH)
+endif()
 
-if (ENABLE_ODBC)
-    if (USE_INTERNAL_ODBC_LIBRARY)
-        set (ODBC_LIBRARIES unixodbc)
-        set (ODBC_INCLUDE_DIRECTORIES ${CMAKE_SOURCE_DIR}/contrib/unixodbc/include)
-        set (ODBC_FOUND 1)
-        set (USE_ODBC 1)
-    else ()
-        find_path(ODBC_INCLUDE_DIRECTORIES
-            NAMES sql.h
-            HINTS
-            /usr/include
-            /usr/include/iodbc
-            /usr/include/odbc
-            /usr/local/include
-            /usr/local/include/iodbc
-            /usr/local/include/odbc
-            /usr/local/iodbc/include
-            /usr/local/odbc/include
-            "C:/Program Files/ODBC/include"
-            "C:/Program Files/Microsoft SDKs/Windows/v7.0/include"
-            "C:/Program Files/Microsoft SDKs/Windows/v6.0a/include"
-            "C:/ODBC/include"
-            DOC "Specify the directory containing sql.h."
-        )
+find_path(ODBC_INCLUDE_DIR
+	NAMES sql.h
+	HINTS
+		${ODBC_ROOT_DIR}/include
+		${ODBC_ROOT_INCLUDE_DIRS}
+	PATHS
+		${PC_ODBC_INCLUDE_DIRS}
+		/usr/include
+		/usr/local/include
+		/usr/local/odbc/include
+		/usr/local/iodbc/include
+		"C:/Program Files/ODBC/include"
+		"C:/Program Files/Microsoft SDKs/Windows/v7.0/include"
+		"C:/Program Files/Microsoft SDKs/Windows/v6.0a/include"
+		"C:/ODBC/include"
+		"${kit_dir}/Include/um"
+		"${kit81_dir}/Include/um"
+	PATH_SUFFIXES
+		odbc
+		iodbc
+	DOC "Specify the directory containing sql.h."
+)
 
-        find_library(ODBC_LIBRARIES
-            NAMES iodbc odbc iodbcinst odbcinst odbc32
-            HINTS
-            /usr/lib
-            /usr/lib/iodbc
-            /usr/lib/odbc
-            /usr/local/lib
-            /usr/local/lib/iodbc
-            /usr/local/lib/odbc
-            /usr/local/iodbc/lib
-            /usr/local/odbc/lib
-            "C:/Program Files/ODBC/lib"
-            "C:/ODBC/lib/debug"
-            "C:/Program Files (x86)/Microsoft SDKs/Windows/v7.0A/Lib"
-            DOC "Specify the ODBC driver manager library here."
-        )
+if(NOT ODBC_INCLUDE_DIR AND WIN32)
+  set(ODBC_INCLUDE_DIR "")
+else()
+  set(REQUIRED_INCLUDE_DIR ODBC_INCLUDE_DIR)
+endif()
 
-        # MinGW find usually fails
-        if(MINGW)
-            set(ODBC_INCLUDE_DIRECTORIES ".")
-            set(ODBC_LIBRARIES odbc32)
-        endif()
+if(WIN32 AND CMAKE_SIZEOF_VOID_P EQUAL 8)
+  set(WIN_ARCH x64)
+elseif(WIN32 AND CMAKE_SIZEOF_VOID_P EQUAL 4)
+  set(WIN_ARCH x86)
+endif()
 
-        include(FindPackageHandleStandardArgs)
-        find_package_handle_standard_args(ODBC
-            DEFAULT_MSG
-            ODBC_INCLUDE_DIRECTORIES
-            ODBC_LIBRARIES)
+find_library(ODBC_LIBRARY
+	NAMES unixodbc iodbc odbc odbc32
+	HINTS
+		${ODBC_ROOT_DIR}/lib
+		${ODBC_ROOT_LIBRARY_DIRS}
+	PATHS
+		${PC_ODBC_LIBRARY_DIRS}
+		/usr/lib
+		/usr/local/lib
+		/usr/local/odbc/lib
+		/usr/local/iodbc/lib
+		"C:/Program Files/ODBC/lib"
+		"C:/ODBC/lib/debug"
+		"C:/Program Files (x86)/Microsoft SDKs/Windows/v7.0A/Lib"
+		"${kit81_dir}/Lib/winv6.3/um"
+		"${kit_dir}/Lib/win8/um"
+	PATH_SUFIXES
+		odbc
+		${WIN_ARCH}
+	DOC "Specify the ODBC driver manager library here."
+)
 
-        mark_as_advanced(ODBC_FOUND ODBC_LIBRARIES ODBC_INCLUDE_DIRECTORIES)
-    endif ()
-endif ()
+if(NOT ODBC_LIBRARY AND WIN32)
+  # List names of ODBC libraries on Windows
+  set(ODBC_LIBRARY odbc32.lib)
+endif()
 
-message (STATUS "Using odbc: ${ODBC_INCLUDE_DIRECTORIES} : ${ODBC_LIBRARIES}")
+# List additional libraries required to use ODBC library
+if(WIN32 AND MSVC OR CMAKE_CXX_COMPILER_ID MATCHES "Intel")
+  set(_odbc_required_libs_names odbccp32;ws2_32)
+endif()
+foreach(_lib_name IN LISTS _odbc_required_libs_names)
+	find_library(_lib_path
+		NAMES ${_lib_name}
+		HINTS
+			${ODBC_ROOT_DIR}/lib
+			${ODBC_ROOT_LIBRARY_DIRS}
+		PATHS
+			${PC_ODBC_LIBRARY_DIRS}
+			/usr/lib
+			/usr/local/lib
+			/usr/local/odbc/lib
+			/usr/local/iodbc/lib
+			"C:/Program Files/ODBC/lib"
+			"C:/ODBC/lib/debug"
+			"C:/Program Files (x86)/Microsoft SDKs/Windows/v7.0A/Lib"
+		PATH_SUFFIXES
+			odbc
+	)
+	if (_lib_path)
+		list(APPEND _odbc_required_libs_paths ${_lib_path})
+	endif()
+	unset(_lib_path CACHE)
+endforeach()
+unset(_odbc_lib_paths)
+unset(_odbc_required_libs_names)
+
+
+find_package_handle_standard_args(ODBC
+	FOUND_VAR ODBC_FOUND
+	REQUIRED_VARS
+		ODBC_LIBRARY
+		${REQUIRED_INCLUDE_DIR}
+	VERSION_VAR ODBC_VERSION
+)
+
+if(ODBC_FOUND)
+  set(ODBC_LIBRARIES ${ODBC_LIBRARY} ${_odbc_required_libs_paths})
+  set(ODBC_INCLUDE_DIRS ${ODBC_INCLUDE_DIR})
+  set(ODBC_DEFINITIONS ${PC_ODBC_CFLAGS_OTHER})
+endif()
+
+if(ODBC_FOUND AND NOT TARGET ODBC::ODBC)
+  add_library(ODBC::ODBC UNKNOWN IMPORTED)
+  set_target_properties(ODBC::ODBC PROPERTIES
+	IMPORTED_LOCATION "${ODBC_LIBRARY}"
+	INTERFACE_LINK_LIBRARIES "${_odbc_required_libs_paths}"
+	INTERFACE_COMPILE_OPTIONS "${PC_ODBC_CFLAGS_OTHER}"
+	INTERFACE_INCLUDE_DIRECTORIES "${ODBC_INCLUDE_DIR}"
+  )
+endif()
+
+mark_as_advanced(ODBC_LIBRARY ODBC_INCLUDE_DIR)
diff --git a/cmake/Modules/FindPoco.cmake b/cmake/Modules/FindPoco.cmake
index e56e2f70fb5..cad7e9afc8f 100644
--- a/cmake/Modules/FindPoco.cmake
+++ b/cmake/Modules/FindPoco.cmake
@@ -203,12 +203,12 @@ endforeach()
 
 if(Poco_DataODBC_LIBRARY)
     list(APPEND Poco_DataODBC_LIBRARY ${ODBC_LIBRARIES} ${LTDL_LIBRARY})
-    list(APPEND Poco_INCLUDE_DIRS ${ODBC_INCLUDE_DIRECTORIES})
+    list(APPEND Poco_INCLUDE_DIRS ${ODBC_INCLUDE_DIRS})
 endif()
 
 if(Poco_SQLODBC_LIBRARY)
     list(APPEND Poco_SQLODBC_LIBRARY ${ODBC_LIBRARIES} ${LTDL_LIBRARY})
-    list(APPEND Poco_INCLUDE_DIRS ${ODBC_INCLUDE_DIRECTORIES})
+    list(APPEND Poco_INCLUDE_DIRS ${ODBC_INCLUDE_DIRS})
 endif()
 
 if(Poco_NetSSL_LIBRARY)
diff --git a/cmake/find_ccache.cmake b/cmake/find_ccache.cmake
index 22372c3cbfe..c2d467ca6ba 100644
--- a/cmake/find_ccache.cmake
+++ b/cmake/find_ccache.cmake
@@ -1,8 +1,7 @@
-
 find_program (CCACHE_FOUND ccache)
 if (CCACHE_FOUND AND NOT CMAKE_CXX_COMPILER_LAUNCHER MATCHES "ccache" AND NOT CMAKE_CXX_COMPILER MATCHES "ccache")
     execute_process(COMMAND ${CCACHE_FOUND} "-V" OUTPUT_VARIABLE CCACHE_VERSION)
-    string(REGEX REPLACE "ccache version ([0-9\\.]+).*" "\\1" CCACHE_VERSION ${CCACHE_VERSION} )
+    string(REGEX REPLACE "ccache version ([0-9\\.]+).*" "\\1" CCACHE_VERSION ${CCACHE_VERSION})
 
     if (CCACHE_VERSION VERSION_GREATER "3.2.0" OR NOT CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
        #message(STATUS "Using ${CCACHE_FOUND} ${CCACHE_VERSION}")
diff --git a/cmake/find_odbc.cmake b/cmake/find_odbc.cmake
index 32a410c6f11..4992306cabf 100644
--- a/cmake/find_odbc.cmake
+++ b/cmake/find_odbc.cmake
@@ -1,93 +1,34 @@
 # This file copied from contrib/poco/cmake/FindODBC.cmake to allow build without submodules
 
-#
-# Find the ODBC driver manager includes and library.
-#
-# ODBC is an open standard for connecting to different databases in a
-# semi-vendor-independent fashion.  First you install the ODBC driver
-# manager.  Then you need a driver for each separate database you want
-# to connect to (unless a generic one works).  VTK includes neither
-# the driver manager nor the vendor-specific drivers: you have to find
-# those yourself.
-#
-# This module defines
-# ODBC_INCLUDE_DIRECTORIES, where to find sql.h
-# ODBC_LIBRARIES, the libraries to link against to use ODBC
-# ODBC_FOUND.  If false, you cannot build anything that requires ODBC.
-
 option (ENABLE_ODBC "Enable ODBC" ${OS_LINUX})
-if (OS_LINUX)
-    option (USE_INTERNAL_ODBC_LIBRARY "Set to FALSE to use system odbc library instead of bundled" ${NOT_UNBUNDLED})
-else ()
-    option (USE_INTERNAL_ODBC_LIBRARY "Set to FALSE to use system odbc library instead of bundled" OFF)
-endif ()
-
-if (USE_INTERNAL_ODBC_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/unixodbc/README")
-    message (WARNING "submodule contrib/unixodbc is missing. to fix try run: \n git submodule update --init --recursive")
-   set (USE_INTERNAL_ODBC_LIBRARY 0)
-endif ()
-
-set (ODBC_INCLUDE_DIRECTORIES ) # Include directories will be either used automatically by target_include_directories or set later.
-
-if (ENABLE_ODBC)
-    if (USE_INTERNAL_ODBC_LIBRARY)
-        set (ODBC_LIBRARIES unixodbc)
-        set (ODBC_FOUND 1)
-        set (USE_ODBC 1)
+if(ENABLE_ODBC)
+    if (OS_LINUX)
+        option(USE_INTERNAL_ODBC_LIBRARY "Set to FALSE to use system odbc library instead of bundled" ${NOT_UNBUNDLED})
     else ()
-        find_path(ODBC_INCLUDE_DIRECTORIES
-            NAMES sql.h
-            HINTS
-            /usr/include
-            /usr/include/iodbc
-            /usr/include/odbc
-            /usr/local/include
-            /usr/local/include/iodbc
-            /usr/local/include/odbc
-            /usr/local/iodbc/include
-            /usr/local/odbc/include
-            "C:/Program Files/ODBC/include"
-            "C:/Program Files/Microsoft SDKs/Windows/v7.0/include"
-            "C:/Program Files/Microsoft SDKs/Windows/v6.0a/include"
-            "C:/ODBC/include"
-            DOC "Specify the directory containing sql.h."
-        )
+        option(USE_INTERNAL_ODBC_LIBRARY "Set to FALSE to use system odbc library instead of bundled" OFF)
+    endif()
 
-        find_library(ODBC_LIBRARIES
-            NAMES iodbc odbc iodbcinst odbcinst odbc32
-            HINTS
-            /usr/lib
-            /usr/lib/iodbc
-            /usr/lib/odbc
-            /usr/local/lib
-            /usr/local/lib/iodbc
-            /usr/local/lib/odbc
-            /usr/local/iodbc/lib
-            /usr/local/odbc/lib
-            "C:/Program Files/ODBC/lib"
-            "C:/ODBC/lib/debug"
-            "C:/Program Files (x86)/Microsoft SDKs/Windows/v7.0A/Lib"
-            DOC "Specify the ODBC driver manager library here."
-        )
+    if(USE_INTERNAL_ODBC_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/unixodbc/README")
+        message(WARNING "submodule contrib/unixodbc is missing. to fix try run: \n git submodule update --init --recursive")
+        set(USE_INTERNAL_ODBC_LIBRARY 0)
+        set(MISSING_INTERNAL_ODBC_LIBRARY 1)
+    endif()
 
-        # MinGW find usually fails
-        if (MINGW)
-            set(ODBC_INCLUDE_DIRECTORIES ".")
-            set(ODBC_LIBRARIES odbc32)
-        endif ()
-
-        include(FindPackageHandleStandardArgs)
-        find_package_handle_standard_args(ODBC
-            DEFAULT_MSG
-            ODBC_INCLUDE_DIRECTORIES
-            ODBC_LIBRARIES)
-
-        if (USE_STATIC_LIBRARIES)
-            list(APPEND ODBC_LIBRARIES ${LTDL_LIBRARY})
-        endif ()
-
-        mark_as_advanced(ODBC_FOUND ODBC_LIBRARIES ODBC_INCLUDE_DIRECTORIES)
+    set(ODBC_INCLUDE_DIRS ) # Include directories will be either used automatically by target_include_directories or set later.
+    if(USE_INTERNAL_ODBC_LIBRARY AND NOT MISSING_INTERNAL_ODBC_LIBRARY)
+        set(ODBC_LIBRARY unixodbc)
+        set(ODBC_LIBRARIES ${ODBC_LIBRARY})
+        set(ODBC_INCLUDE_DIRS "${ClickHouse_SOURCE_DIR}/contrib/unixodbc/include")
+        set(ODBC_FOUND 1)
+    else()
+        find_package(ODBC)
     endif ()
-endif ()
 
-message (STATUS "Using odbc=${ODBC_FOUND}: ${ODBC_INCLUDE_DIRECTORIES} : ${ODBC_LIBRARIES}")
+    if(ODBC_FOUND)
+        set(USE_ODBC 1)
+        set(ODBC_INCLUDE_DIRECTORIES ${ODBC_INCLUDE_DIRS}) # for old poco
+        set(ODBC_INCLUDE_DIR ${ODBC_INCLUDE_DIRS}) # for old poco
+    endif()
+
+    message(STATUS "Using odbc=${USE_ODBC}: ${ODBC_INCLUDE_DIRS} : ${ODBC_LIBRARIES}")
+endif()
diff --git a/cmake/find_poco.cmake b/cmake/find_poco.cmake
index 07d81c1bbe8..c4523ecfdf6 100644
--- a/cmake/find_poco.cmake
+++ b/cmake/find_poco.cmake
@@ -76,7 +76,7 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY)
             set (Poco_SQLODBC_INCLUDE_DIR
                 "${ClickHouse_SOURCE_DIR}/contrib/poco/SQL/ODBC/include/"
                 "${ClickHouse_SOURCE_DIR}/contrib/poco/Data/ODBC/include/"
-                ${ODBC_INCLUDE_DIRECTORIES}
+                ${ODBC_INCLUDE_DIRS}
                 )
             set (Poco_SQLODBC_LIBRARY PocoSQLODBC ${ODBC_LIBRARIES} ${LTDL_LIBRARY})
         endif ()
@@ -88,7 +88,7 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY)
             set (USE_POCO_DATAODBC 1)
             set (Poco_DataODBC_INCLUDE_DIR
                 "${ClickHouse_SOURCE_DIR}/contrib/poco/Data/ODBC/include/"
-                ${ODBC_INCLUDE_DIRECTORIES}
+                ${ODBC_INCLUDE_DIRS}
             )
             set (Poco_DataODBC_LIBRARY PocoDataODBC ${ODBC_LIBRARIES} ${LTDL_LIBRARY})
         endif ()
diff --git a/cmake/find_ssl.cmake b/cmake/find_ssl.cmake
index 6d65ccc66bf..58027276c5c 100644
--- a/cmake/find_ssl.cmake
+++ b/cmake/find_ssl.cmake
@@ -60,6 +60,72 @@ if(OPENSSL_FOUND)
     set(USE_SSL 1)
 endif()
 
+# used by new poco
+# part from /usr/share/cmake-*/Modules/FindOpenSSL.cmake, with removed all "EXISTS "
+if(OPENSSL_FOUND AND NOT USE_INTERNAL_SSL_LIBRARY)
+  if(NOT TARGET OpenSSL::Crypto AND
+      (OPENSSL_CRYPTO_LIBRARY OR
+        LIB_EAY_LIBRARY_DEBUG OR
+        LIB_EAY_LIBRARY_RELEASE)
+      )
+    add_library(OpenSSL::Crypto UNKNOWN IMPORTED)
+    set_target_properties(OpenSSL::Crypto PROPERTIES
+      INTERFACE_INCLUDE_DIRECTORIES "${OPENSSL_INCLUDE_DIR}")
+    if(OPENSSL_CRYPTO_LIBRARY)
+      set_target_properties(OpenSSL::Crypto PROPERTIES
+        IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+        IMPORTED_LOCATION "${OPENSSL_CRYPTO_LIBRARY}")
+    endif()
+    if(LIB_EAY_LIBRARY_RELEASE)
+      set_property(TARGET OpenSSL::Crypto APPEND PROPERTY
+        IMPORTED_CONFIGURATIONS RELEASE)
+      set_target_properties(OpenSSL::Crypto PROPERTIES
+        IMPORTED_LINK_INTERFACE_LANGUAGES_RELEASE "C"
+        IMPORTED_LOCATION_RELEASE "${LIB_EAY_LIBRARY_RELEASE}")
+    endif()
+    if(LIB_EAY_LIBRARY_DEBUG)
+      set_property(TARGET OpenSSL::Crypto APPEND PROPERTY
+        IMPORTED_CONFIGURATIONS DEBUG)
+      set_target_properties(OpenSSL::Crypto PROPERTIES
+        IMPORTED_LINK_INTERFACE_LANGUAGES_DEBUG "C"
+        IMPORTED_LOCATION_DEBUG "${LIB_EAY_LIBRARY_DEBUG}")
+    endif()
+  endif()
+  if(NOT TARGET OpenSSL::SSL AND
+      (OPENSSL_SSL_LIBRARY OR
+        SSL_EAY_LIBRARY_DEBUG OR
+        SSL_EAY_LIBRARY_RELEASE)
+      )
+    add_library(OpenSSL::SSL UNKNOWN IMPORTED)
+    set_target_properties(OpenSSL::SSL PROPERTIES
+      INTERFACE_INCLUDE_DIRECTORIES "${OPENSSL_INCLUDE_DIR}")
+    if(OPENSSL_SSL_LIBRARY)
+      set_target_properties(OpenSSL::SSL PROPERTIES
+        IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+        IMPORTED_LOCATION "${OPENSSL_SSL_LIBRARY}")
+    endif()
+    if(SSL_EAY_LIBRARY_RELEASE)
+      set_property(TARGET OpenSSL::SSL APPEND PROPERTY
+        IMPORTED_CONFIGURATIONS RELEASE)
+      set_target_properties(OpenSSL::SSL PROPERTIES
+        IMPORTED_LINK_INTERFACE_LANGUAGES_RELEASE "C"
+        IMPORTED_LOCATION_RELEASE "${SSL_EAY_LIBRARY_RELEASE}")
+    endif()
+    if(SSL_EAY_LIBRARY_DEBUG)
+      set_property(TARGET OpenSSL::SSL APPEND PROPERTY
+        IMPORTED_CONFIGURATIONS DEBUG)
+      set_target_properties(OpenSSL::SSL PROPERTIES
+        IMPORTED_LINK_INTERFACE_LANGUAGES_DEBUG "C"
+        IMPORTED_LOCATION_DEBUG "${SSL_EAY_LIBRARY_DEBUG}")
+    endif()
+    if(TARGET OpenSSL::Crypto)
+      set_target_properties(OpenSSL::SSL PROPERTIES
+        INTERFACE_LINK_LIBRARIES OpenSSL::Crypto)
+    endif()
+  endif()
+endif()
+
+
 endif ()
 
 message (STATUS "Using ssl=${USE_SSL}: ${OPENSSL_INCLUDE_DIR} : ${OPENSSL_LIBRARIES}")
diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
index 447ff6fb880..abae6f7deec 100644
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@@ -120,6 +120,9 @@ if (USE_INTERNAL_SSL_LIBRARY)
     target_include_directories(${OPENSSL_CRYPTO_LIBRARY} SYSTEM PUBLIC ${OPENSSL_INCLUDE_DIR})
     target_include_directories(${OPENSSL_SSL_LIBRARY} SYSTEM PUBLIC ${OPENSSL_INCLUDE_DIR})
     set (POCO_SKIP_OPENSSL_FIND 1)
+
+    add_library(OpenSSL::Crypto ALIAS ${OPENSSL_CRYPTO_LIBRARY})
+    add_library(OpenSSL::SSL ALIAS ${OPENSSL_SSL_LIBRARY})
 endif ()
 
 if (ENABLE_MYSQL AND USE_INTERNAL_MYSQL_LIBRARY)
@@ -144,6 +147,7 @@ endif()
 
 if (ENABLE_ODBC AND USE_INTERNAL_ODBC_LIBRARY)
     add_subdirectory (unixodbc-cmake)
+    add_library(ODBC::ODBC ALIAS ${ODBC_LIBRARIES})
 endif ()
 
 if (USE_INTERNAL_CAPNP_LIBRARY)
diff --git a/contrib/unixodbc-cmake/CMakeLists.txt b/contrib/unixodbc-cmake/CMakeLists.txt
index e15f263d096..4fa54a55a3a 100644
--- a/contrib/unixodbc-cmake/CMakeLists.txt
+++ b/contrib/unixodbc-cmake/CMakeLists.txt
@@ -275,13 +275,13 @@ ${ODBC_SOURCE_DIR}/lst/_lstVisible.c
 
 add_library(unixodbc ${SRCS})
 
-target_link_libraries(unixodbc ltdl)
+target_link_libraries(unixodbc PRIVATE ltdl)
 
 # SYSTEM_FILE_PATH was changed to /etc
 
-target_include_directories(unixodbc PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/linux_x86_64/private)
-target_include_directories(unixodbc PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/linux_x86_64)
-target_include_directories(unixodbc PUBLIC ${ODBC_SOURCE_DIR}/include)
+target_include_directories(unixodbc SYSTEM PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/linux_x86_64/private)
+target_include_directories(unixodbc SYSTEM PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/linux_x86_64)
+target_include_directories(unixodbc SYSTEM PUBLIC ${ODBC_SOURCE_DIR}/include)
 
 target_compile_definitions(unixodbc PRIVATE -DHAVE_CONFIG_H)
 
diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt
index a04af1039de..615dc666500 100644
--- a/dbms/CMakeLists.txt
+++ b/dbms/CMakeLists.txt
@@ -257,8 +257,8 @@ if (USE_POCO_SQLODBC)
     target_link_libraries (clickhouse_common_io PRIVATE ${Poco_SQL_LIBRARY})
     target_link_libraries (dbms PRIVATE ${Poco_SQLODBC_LIBRARY} ${Poco_SQL_LIBRARY})
     if (NOT USE_INTERNAL_POCO_LIBRARY)
-        target_include_directories (clickhouse_common_io SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQL_INCLUDE_DIR})
-        target_include_directories (dbms SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIR} PUBLIC ${Poco_SQL_INCLUDE_DIR})
+        target_include_directories (clickhouse_common_io SYSTEM PRIVATE ${ODBC_INCLUDE_DIRS} ${Poco_SQL_INCLUDE_DIR})
+        target_include_directories (dbms SYSTEM PRIVATE ${ODBC_INCLUDE_DIRS} ${Poco_SQLODBC_INCLUDE_DIR} SYSTEM PUBLIC ${Poco_SQL_INCLUDE_DIR})
     endif()
 endif()
 
@@ -271,7 +271,7 @@ if (USE_POCO_DATAODBC)
     target_link_libraries (clickhouse_common_io PRIVATE ${Poco_Data_LIBRARY})
     target_link_libraries (dbms PRIVATE ${Poco_DataODBC_LIBRARY})
     if (NOT USE_INTERNAL_POCO_LIBRARY)
-        target_include_directories (dbms SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIR})
+        target_include_directories (dbms SYSTEM PRIVATE ${ODBC_INCLUDE_DIRS} ${Poco_DataODBC_INCLUDE_DIR})
     endif()
 endif()
 
diff --git a/dbms/programs/odbc-bridge/CMakeLists.txt b/dbms/programs/odbc-bridge/CMakeLists.txt
index 143d7edf738..38f2bc60b2d 100644
--- a/dbms/programs/odbc-bridge/CMakeLists.txt
+++ b/dbms/programs/odbc-bridge/CMakeLists.txt
@@ -16,18 +16,20 @@ set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemo
 
 if (USE_POCO_SQLODBC)
     set(CLICKHOUSE_ODBC_BRIDGE_LINK ${CLICKHOUSE_ODBC_BRIDGE_LINK} PRIVATE ${Poco_SQLODBC_LIBRARY})
-    set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIR})
+    set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${ODBC_INCLUDE_DIRS} ${Poco_SQLODBC_INCLUDE_DIR})
 endif ()
 if (Poco_SQL_FOUND)
     set(CLICKHOUSE_ODBC_BRIDGE_LINK ${CLICKHOUSE_ODBC_BRIDGE_LINK} PRIVATE ${Poco_SQL_LIBRARY})
+    set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${Poco_SQL_INCLUDE_DIR})
 endif ()
 
 if (USE_POCO_DATAODBC)
     set(CLICKHOUSE_ODBC_BRIDGE_LINK ${CLICKHOUSE_ODBC_BRIDGE_LINK} PRIVATE ${Poco_DataODBC_LIBRARY})
-    set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIR})
+    set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${ODBC_INCLUDE_DIRS} ${Poco_DataODBC_INCLUDE_DIR})
 endif()
 if (Poco_Data_FOUND)
     set(CLICKHOUSE_ODBC_BRIDGE_LINK ${CLICKHOUSE_ODBC_BRIDGE_LINK} PRIVATE ${Poco_Data_LIBRARY})
+    set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${Poco_Data_INCLUDE_DIR})
 endif ()
 
 clickhouse_program_add_library(odbc-bridge)
diff --git a/dbms/src/Dictionaries/CMakeLists.txt b/dbms/src/Dictionaries/CMakeLists.txt
index 29b9649e661..af858e6b26d 100644
--- a/dbms/src/Dictionaries/CMakeLists.txt
+++ b/dbms/src/Dictionaries/CMakeLists.txt
@@ -23,7 +23,7 @@ endif()
 
 if(USE_POCO_SQLODBC)
     target_link_libraries(clickhouse_dictionaries PRIVATE ${Poco_SQLODBC_LIBRARY} ${Poco_SQL_LIBRARY})
-    target_include_directories(clickhouse_dictionaries SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIR} ${Poco_SQL_INCLUDE_DIR})
+    target_include_directories(clickhouse_dictionaries SYSTEM PRIVATE ${ODBC_INCLUDE_DIRS} ${Poco_SQLODBC_INCLUDE_DIR} ${Poco_SQL_INCLUDE_DIR})
 endif()
 
 if(Poco_Data_FOUND)
@@ -32,7 +32,7 @@ endif()
 
 if(USE_POCO_DATAODBC)
     target_link_libraries(clickhouse_dictionaries PRIVATE ${Poco_DataODBC_LIBRARY} ${Poco_Data_LIBRARY})
-    target_include_directories(clickhouse_dictionaries SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIR})
+    target_include_directories(clickhouse_dictionaries SYSTEM PRIVATE ${ODBC_INCLUDE_DIRS} ${Poco_DataODBC_INCLUDE_DIR})
 endif()
 
 if(USE_POCO_MONGODB)
diff --git a/libs/libmysqlxx/CMakeLists.txt b/libs/libmysqlxx/CMakeLists.txt
index 1e3aedfa888..166c10e69f7 100644
--- a/libs/libmysqlxx/CMakeLists.txt
+++ b/libs/libmysqlxx/CMakeLists.txt
@@ -46,7 +46,7 @@ else ()
     endif ()
 endif ()
 
-target_link_libraries(mysqlxx PUBLIC common ${Poco_Util_LIBRARY} PRIVATE ${MYSQLCLIENT_LIBRARIES} PUBLIC ${Boost_SYSTEM_LIBRARY} PRIVATE ${ZLIB_LIBRARIES})
+target_link_libraries(mysqlxx PUBLIC common ${Poco_Util_LIBRARY} ${Poco_Foundation_LIBRARY} PRIVATE ${MYSQLCLIENT_LIBRARIES} PUBLIC ${Boost_SYSTEM_LIBRARY} PRIVATE ${ZLIB_LIBRARIES})
 if(OPENSSL_LIBRARIES)
     target_link_libraries(mysqlxx PRIVATE ${OPENSSL_LIBRARIES})
 endif()

From 661c840fbe5333f3622dc9095c9b7aef448e41fd Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Thu, 25 Apr 2019 04:16:26 +0300
Subject: [PATCH 052/118] Code cleanups after #4439

---
 dbms/src/Columns/ColumnVector.cpp | 106 ++++++++++++---
 dbms/src/Common/RadixSort.h       | 217 ++++++------------------------
 dbms/src/Interpreters/RowRefs.h   |  24 ++--
 3 files changed, 143 insertions(+), 204 deletions(-)

diff --git a/dbms/src/Columns/ColumnVector.cpp b/dbms/src/Columns/ColumnVector.cpp
index b8ad5ddba8f..ea2ce1d7af1 100644
--- a/dbms/src/Columns/ColumnVector.cpp
+++ b/dbms/src/Columns/ColumnVector.cpp
@@ -68,12 +68,33 @@ struct ColumnVector<T>::greater
     bool operator()(size_t lhs, size_t rhs) const { return CompareHelper<T>::greater(parent.data[lhs], parent.data[rhs], nan_direction_hint); }
 };
 
+
+namespace
+{
+    template <typename T>
+    struct ValueWithIndex
+    {
+        T value;
+        UInt32 index;
+    };
+
+    template <typename T>
+    struct RadixSortTraits : RadixSortNumTraits<T>
+    {
+        using Element = ValueWithIndex<T>;
+        static T & extractKey(Element & elem) { return elem.value; }
+    };
+}
+
 template <typename T>
 void ColumnVector<T>::getPermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res) const
 {
     size_t s = data.size();
     res.resize(s);
 
+    if (s == 0)
+        return;
+
     if (limit >= s)
         limit = 0;
 
@@ -89,34 +110,79 @@ void ColumnVector<T>::getPermutation(bool reverse, size_t limit, int nan_directi
     }
     else
     {
-        if constexpr ((std::is_signed_v<T> || std::is_unsigned_v<T>) && !std::is_same_v<T, UInt128>)
+        /// A case for radix sort
+        if constexpr (std::is_arithmetic_v<T> && !std::is_same_v<T, UInt128>)
         {
-            PaddedPODArray<std::pair<T, size_t>> pairs(s);
-            for (size_t i = 0; i < s; ++i)
-                pairs[i] = {data[i], i};
+            /// Thresholds on size. Lower threshold is arbitrary. Upper threshold is chosen by the type for histogram counters.
+            if (s >= 256 && s <= std::numeric_limits<UInt32>::max())
+            {
+                PaddedPODArray<ValueWithIndex<T>> pairs(s);
+                for (UInt32 i = 0; i < s; ++i)
+                    pairs[i] = {data[i], i};
 
-            radixSort(pairs.data(), s, nan_direction_hint);
+                RadixSort<RadixSortTraits<T>>::execute(pairs.data(), s);
 
-            if (reverse)
-                for (size_t i = 0; i < s; ++i)
-                    res[s - 1 - i] = pairs[i].second;
-            else
-                for (size_t i = 0; i < s; ++i)
-                    res[i] = pairs[i].second;
+                /// Radix sort treats all NaNs to be greater than all numbers.
+                /// If the user needs the opposite, we must move them accordingly.
+                size_t nans_to_move = 0;
+                if (std::is_floating_point_v<T> && nan_direction_hint < 0)
+                {
+                    for (ssize_t i = s - 1; i >= 0; --i)
+                    {
+                        if (isNaN(pairs[i].value))
+                            ++nans_to_move;
+                        else
+                            break;
+                    }
+                }
+
+                if (reverse)
+                {
+                    if (nans_to_move)
+                    {
+                        for (size_t i = 0; i < s - nans_to_move; ++i)
+                            res[i] = pairs[s - nans_to_move - 1 - i].index;
+                        for (size_t i = s - nans_to_move; i < s; ++i)
+                            res[i] = pairs[s - 1 - (i - (s - nans_to_move))].index;
+                    }
+                    else
+                    {
+                        for (size_t i = 0; i < s; ++i)
+                            res[s - 1 - i] = pairs[i].index;
+                    }
+                }
+                else
+                {
+                    if (nans_to_move)
+                    {
+                        for (size_t i = 0; i < nans_to_move; ++i)
+                            res[i] = pairs[i + s - nans_to_move].index;
+                        for (size_t i = nans_to_move; i < s; ++i)
+                            res[i] = pairs[i - nans_to_move].index;
+                    }
+                    else
+                    {
+                        for (size_t i = 0; i < s; ++i)
+                            res[i] = pairs[i].index;
+                    }
+                }
+
+                return;
+            }
         }
+
+        /// Default sorting algorithm.
+        for (size_t i = 0; i < s; ++i)
+            res[i] = i;
+
+        if (reverse)
+            pdqsort(res.begin(), res.end(), greater(*this, nan_direction_hint));
         else
-        {
-            for (size_t i = 0; i < s; ++i)
-                res[i] = i;
-
-            if (reverse)
-                pdqsort(res.begin(), res.end(), greater(*this, nan_direction_hint));
-            else
-                pdqsort(res.begin(), res.end(), less(*this, nan_direction_hint));
-        }
+            pdqsort(res.begin(), res.end(), less(*this, nan_direction_hint));
     }
 }
 
+
 template <typename T>
 const char * ColumnVector<T>::getFamilyName() const
 {
diff --git a/dbms/src/Common/RadixSort.h b/dbms/src/Common/RadixSort.h
index 2903ba64af5..2fe61c72b6a 100644
--- a/dbms/src/Common/RadixSort.h
+++ b/dbms/src/Common/RadixSort.h
@@ -10,7 +10,6 @@
 #include <cstdlib>
 #include <cstdint>
 #include <type_traits>
-#include <vector>
 
 #include <ext/bit_cast.h>
 #include <Core/Types.h>
@@ -68,15 +67,15 @@ struct RadixSortFloatTransform
 };
 
 
-template <typename _Element, typename _Key = _Element>
+template <typename TElement>
 struct RadixSortFloatTraits
 {
-    using Element = _Element;     /// The type of the element. It can be a structure with a key and some other payload. Or just a key.
-    using Key = _Key;             /// The key to sort.
+    using Element = TElement;     /// The type of the element. It can be a structure with a key and some other payload. Or just a key.
+    using Key = Element;          /// The key to sort by.
     using CountType = uint32_t;   /// Type for calculating histograms. In the case of a known small number of elements, it can be less than size_t.
 
     /// The type to which the key is transformed to do bit operations. This UInt is the same size as the key.
-    using KeyBits = std::conditional_t<sizeof(_Key) == 8, uint64_t, uint32_t>;
+    using KeyBits = std::conditional_t<sizeof(Key) == 8, uint64_t, uint32_t>;
 
     static constexpr size_t PART_SIZE_BITS = 8;    /// With what pieces of the key, in bits, to do one pass - reshuffle of the array.
 
@@ -89,30 +88,7 @@ struct RadixSortFloatTraits
     using Allocator = RadixSortMallocAllocator;
 
     /// The function to get the key from an array element.
-    static Key & extractKey(Element & elem)
-    {
-        if constexpr (std::is_same_v<Element, Key>)
-            return elem;
-        else
-            return *reinterpret_cast<Key *>(&elem);
-    }
-};
-
-template <typename Float>
-struct RadixSortPairFloatKeyTraits
-{
-    using Element = std::pair<Float, size_t>;
-    using Key = Float;
-    using CountType = uint32_t;
-    using KeyBits = std::conditional_t<sizeof(Float) == 8, uint64_t, uint32_t>;
-
-    static constexpr size_t PART_SIZE_BITS = 8;
-
-    using Transform = RadixSortFloatTransform<KeyBits>;
-    using Allocator = RadixSortMallocAllocator;
-
-    /// The function to get the key from an array element.
-    static Key & extractKey(Element & elem) { return elem.first; }
+    static Key & extractKey(Element & elem) { return elem; }
 };
 
 template <typename KeyBits>
@@ -135,92 +111,49 @@ struct RadixSortSignedTransform
 };
 
 
-template <typename _Element, typename _Key = _Element>
+template <typename TElement>
 struct RadixSortUIntTraits
 {
-    using Element = _Element;
-    using Key = _Key;
+    using Element = TElement;
+    using Key = Element;
     using CountType = uint32_t;
-    using KeyBits = _Key;
+    using KeyBits = Key;
 
     static constexpr size_t PART_SIZE_BITS = 8;
 
     using Transform = RadixSortIdentityTransform<KeyBits>;
     using Allocator = RadixSortMallocAllocator;
 
-    /// The function to get the key from an array element.
-    static Key & extractKey(Element & elem)
-    {
-        if constexpr (std::is_same_v<Element, Key>)
-            return elem;
-        else
-            return *reinterpret_cast<Key *>(&elem);
-    }
+    static Key & extractKey(Element & elem) { return elem; }
 };
 
-template <typename UInt>
-struct RadixSortPairUIntKeyTraits
-{
-    using Element = std::pair<UInt, size_t>;
-    using Key = UInt;
-    using CountType = uint32_t;
-    using KeyBits = UInt;
 
-    static constexpr size_t PART_SIZE_BITS = 8;
-
-    using Transform = RadixSortIdentityTransform<KeyBits>;
-    using Allocator = RadixSortMallocAllocator;
-
-    /// The function to get the key from an array element.
-    static Key & extractKey(Element & elem) { return elem.first; }
-};
-
-template <typename _Element, typename _Key = _Element>
+template <typename TElement>
 struct RadixSortIntTraits
 {
-    using Element = _Element;
-    using Key = _Key;
+    using Element = TElement;
+    using Key = Element;
     using CountType = uint32_t;
-    using KeyBits = std::make_unsigned_t<_Key>;
+    using KeyBits = std::make_unsigned_t<Key>;
 
     static constexpr size_t PART_SIZE_BITS = 8;
 
     using Transform = RadixSortSignedTransform<KeyBits>;
     using Allocator = RadixSortMallocAllocator;
 
-    /// The function to get the key from an array element.
-    static Key & extractKey(Element & elem)
-    {
-        if constexpr (std::is_same_v<Element, Key>)
-            return elem;
-        else
-            return *reinterpret_cast<Key *>(&elem);
-    }
-};
-
-template <typename Int>
-struct RadixSortPairIntKeyTraits
-{
-    using Element = std::pair<Int, size_t>;
-    using Key = Int;
-    using CountType = uint32_t;
-    using KeyBits = std::make_unsigned_t<Int>;
-
-    static constexpr size_t PART_SIZE_BITS = 8;
-
-    using Transform = RadixSortSignedTransform<KeyBits>;
-    using Allocator = RadixSortMallocAllocator;
-
-    /// The function to get the key from an array element.
-    static Key & extractKey(Element & elem) { return elem.first; }
+    static Key & extractKey(Element & elem) { return elem; }
 };
 
 
-// Allow std::pair copying
-#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ >= 8)
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wclass-memaccess"
-#endif
+template <typename T>
+using RadixSortNumTraits =
+    std::conditional_t<std::is_integral_v<T>,
+        std::conditional_t<std::is_unsigned_v<T>,
+            RadixSortUIntTraits<T>,
+            RadixSortIntTraits<T>>,
+        RadixSortFloatTraits<T>>;
+
+
 template <typename Traits>
 struct RadixSort
 {
@@ -268,8 +201,8 @@ public:
             if (!Traits::Transform::transform_is_simple)
                 Traits::extractKey(arr[i]) = bitsToKey(Traits::Transform::forward(keyToBits(Traits::extractKey(arr[i]))));
 
-            for (size_t j = 0; j < NUM_PASSES; ++j)
-                ++histograms[j * HISTOGRAM_SIZE + getPart(j, keyToBits(Traits::extractKey(arr[i])))];
+            for (size_t pass = 0; pass < NUM_PASSES; ++pass)
+                ++histograms[pass * HISTOGRAM_SIZE + getPart(pass, keyToBits(Traits::extractKey(arr[i])))];
         }
 
         {
@@ -278,31 +211,31 @@ public:
 
             for (size_t i = 0; i < HISTOGRAM_SIZE; ++i)
             {
-                for (size_t j = 0; j < NUM_PASSES; ++j)
+                for (size_t pass = 0; pass < NUM_PASSES; ++pass)
                 {
-                    size_t tmp = histograms[j * HISTOGRAM_SIZE + i] + sums[j];
-                    histograms[j * HISTOGRAM_SIZE + i] = sums[j] - 1;
-                    sums[j] = tmp;
+                    size_t tmp = histograms[pass * HISTOGRAM_SIZE + i] + sums[pass];
+                    histograms[pass * HISTOGRAM_SIZE + i] = sums[pass] - 1;
+                    sums[pass] = tmp;
                 }
             }
         }
 
         /// Move the elements in the order starting from the least bit piece, and then do a few passes on the number of pieces.
-        for (size_t j = 0; j < NUM_PASSES; ++j)
+        for (size_t pass = 0; pass < NUM_PASSES; ++pass)
         {
-            Element * writer = j % 2 ? arr : swap_buffer;
-            Element * reader = j % 2 ? swap_buffer : arr;
+            Element * writer = pass % 2 ? arr : swap_buffer;
+            Element * reader = pass % 2 ? swap_buffer : arr;
 
             for (size_t i = 0; i < size; ++i)
             {
-                size_t pos = getPart(j, keyToBits(Traits::extractKey(reader[i])));
+                size_t pos = getPart(pass, keyToBits(Traits::extractKey(reader[i])));
 
                 /// Place the element on the next free position.
-                auto & dest = writer[++histograms[j * HISTOGRAM_SIZE + pos]];
+                auto & dest = writer[++histograms[pass * HISTOGRAM_SIZE + pos]];
                 dest = reader[i];
 
                 /// On the last pass, we do the reverse transformation.
-                if (!Traits::Transform::transform_is_simple && j == NUM_PASSES - 1)
+                if (!Traits::Transform::transform_is_simple && pass == NUM_PASSES - 1)
                     Traits::extractKey(dest) = bitsToKey(Traits::Transform::backward(keyToBits(Traits::extractKey(reader[i]))));
             }
         }
@@ -314,80 +247,14 @@ public:
 
         allocator.deallocate(swap_buffer, size * sizeof(Element));
     }
-
-    // For floating point types
-    // Radix sort sometimes incorrectly handles NaNs
-    // Will move them to the right place
-    static void fixNanOrder(Element * arr, size_t size, int nan_direction_hint)
-    {
-        if (nan_direction_hint < 0)
-        {
-            size_t nans_count = std::count_if(arr, arr + size, [](Element d) {return std::isnan(Traits::extractKey(d));});
-            std::vector<Element> nans(nans_count);
-            std::copy(arr + size - nans_count, arr + size, nans.data());
-            std::copy_backward(arr, arr + size - nans_count, arr + size);
-            std::copy(nans.data(), nans.data() + nans_count, arr);
-        }
-    }
 };
-#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ >= 8)
-#pragma GCC diagnostic pop
-#endif
+
+
+/// Helper functions for numeric types.
+/// Use RadixSort with custom traits for complex types instead.
 
 template <typename T>
-std::enable_if_t<std::is_unsigned_v<T> && std::is_integral_v<T>, void>
-radixSort(T * arr, size_t size, int /*nan_direction_hint*/=1)
+void radixSort(T * arr, size_t size)
 {
-    RadixSort<RadixSortUIntTraits<T>>::execute(arr, size);
-}
-
-template <typename T>
-std::enable_if_t<std::is_signed_v<T> && std::is_integral_v<T>, void>
-radixSort(T * arr, size_t size, int /*nan_direction_hint*/=1)
-{
-    RadixSort<RadixSortIntTraits<T>>::execute(arr, size);
-}
-
-template <typename T>
-std::enable_if_t<std::is_floating_point_v<T>, void>
-radixSort(T * arr, size_t size, int nan_direction_hint=1)
-{
-    RadixSort<RadixSortFloatTraits<T>>::execute(arr, size);
-    RadixSort<RadixSortFloatTraits<T>>::fixNanOrder(arr, size, nan_direction_hint);
-}
-
-template <typename _Element, typename _Key>
-std::enable_if_t<std::is_integral_v<_Key>, void>
-radixSort(_Element * arr, size_t size)
-{
-    return RadixSort<RadixSortUIntTraits<_Element, _Key>>::execute(arr, size);
-}
-
-template <typename _Element, typename _Key>
-std::enable_if_t<std::is_floating_point_v<_Key>, void>
-radixSort(_Element * arr, size_t size)
-{
-    return RadixSort<RadixSortFloatTraits<_Element, _Key>>::execute(arr, size);
-}
-
-template <typename T>
-std::enable_if_t<std::is_unsigned_v<T> && !std::is_floating_point_v<T>, void>
-radixSort(std::pair<T, size_t> * arr, size_t size, int /*nan_direction_hint*/=1)
-{
-    RadixSort<RadixSortPairUIntKeyTraits<T>>::execute(arr, size);
-}
-
-template <typename T>
-std::enable_if_t<std::is_signed_v<T> && !std::is_floating_point_v<T>, void>
-radixSort(std::pair<T, size_t> * arr, size_t size, int /*nan_direction_hint*/=1)
-{
-    RadixSort<RadixSortPairIntKeyTraits<T>>::execute(arr, size);
-}
-
-template <typename T>
-std::enable_if_t<std::is_floating_point_v<T>, void>
-radixSort(std::pair<T, size_t> * arr, size_t size, int nan_direction_hint=1)
-{
-    RadixSort<RadixSortPairFloatKeyTraits<T>>::execute(arr, size);
-    RadixSort<RadixSortPairFloatKeyTraits<T>>::fixNanOrder(arr, size, nan_direction_hint);
+    RadixSort<RadixSortNumTraits<T>>::execute(arr, size);
 }
diff --git a/dbms/src/Interpreters/RowRefs.h b/dbms/src/Interpreters/RowRefs.h
index 39bda54cf22..64acba37475 100644
--- a/dbms/src/Interpreters/RowRefs.h
+++ b/dbms/src/Interpreters/RowRefs.h
@@ -40,11 +40,11 @@ struct RowRefList : RowRef
  * references that can be returned by the lookup methods
  */
 
-template <typename _Entry, typename _Key>
+template <typename TEntry, typename TKey>
 class SortedLookupVector
 {
 public:
-    using Base = std::vector<_Entry>;
+    using Base = std::vector<TEntry>;
 
     // First stage, insertions into the vector
     template <typename U, typename ... TAllocatorParams>
@@ -55,7 +55,7 @@ public:
     }
 
     // Transition into second stage, ensures that the vector is sorted
-    typename Base::const_iterator upper_bound(const _Entry & k)
+    typename Base::const_iterator upper_bound(const TEntry & k)
     {
         sort();
         return std::upper_bound(array.cbegin(), array.cend(), k);
@@ -70,6 +70,12 @@ private:
     Base array;
     mutable std::mutex lock;
 
+    struct RadixSortTraits : RadixSortNumTraits<TKey>
+    {
+        using Element = TEntry;
+        static TKey & extractKey(Element & elem) { return elem.asof_value; }
+    };
+
     // Double checked locking with SC atomics works in C++
     // https://preshing.com/20130930/double-checked-locking-is-fixed-in-cpp11/
     // The first thread that calls one of the lookup methods sorts the data
@@ -82,14 +88,14 @@ private:
             std::lock_guard<std::mutex> l(lock);
             if (!sorted.load(std::memory_order_relaxed))
             {
-                /// TODO: It has been tested only for UInt32 yet. It needs to check UInt64, Float32/64.
-                if constexpr (std::is_same_v<_Key, UInt32>)
+                if (!array.empty())
                 {
-                    if (!array.empty())
-                        radixSort<_Entry, _Key>(&array[0], array.size());
+                    /// TODO: It has been tested only for UInt32 yet. It needs to check UInt64, Float32/64.
+                    if constexpr (std::is_same_v<TKey, UInt32>)
+                        RadixSort<RadixSortTraits>::execute(&array[0], array.size());
+                    else
+                        std::sort(array.begin(), array.end());
                 }
-                else
-                    std::sort(array.begin(), array.end());
 
                 sorted.store(true, std::memory_order_release);
             }

From 33922685d5d494283f08198c3b5f7ee5bdb6e1a0 Mon Sep 17 00:00:00 2001
From: proller <proller@users.noreply.github.com>
Date: Thu, 25 Apr 2019 15:29:28 +0300
Subject: [PATCH 053/118] Packager: fixes (#5103)

---
 docker/builder/build.sh             |  2 +-
 docker/packager/binary/Dockerfile   |  4 ++-
 docker/packager/binary/build.sh     | 13 ++++++++++
 docker/packager/deb/Dockerfile      |  6 ++---
 docker/packager/deb/build.sh        | 10 ++++++++
 docker/packager/freebsd/Vagrantfile |  1 -
 docker/packager/packager            | 39 +++++++++++++++++++++--------
 7 files changed, 58 insertions(+), 17 deletions(-)
 mode change 100644 => 100755 docker/builder/build.sh
 create mode 100755 docker/packager/binary/build.sh
 create mode 100755 docker/packager/deb/build.sh

diff --git a/docker/builder/build.sh b/docker/builder/build.sh
old mode 100644
new mode 100755
index 2fc84f227cd..6a5f1359bda
--- a/docker/builder/build.sh
+++ b/docker/builder/build.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 #ccache -s
 mkdir -p /server/build_docker
diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile
index 5b32abfb7ef..67dfda5ec1c 100644
--- a/docker/packager/binary/Dockerfile
+++ b/docker/packager/binary/Dockerfile
@@ -8,6 +8,7 @@ RUN apt-get update -y \
         apt-get install --yes --no-install-recommends \
             bash \
             cmake \
+            ccache \
             curl \
             gcc-7 \
             g++-7 \
@@ -34,4 +35,5 @@ RUN apt-get update -y \
             git \
             tzdata
 
-CMD mkdir -p build/build_result && cd build/build_result && cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DSANITIZE=$SANITIZER $CMAKE_FLAGS && ninja && mv ./dbms/programs/clickhouse* /output && mv ./dbms/unit_tests_dbms /output
+COPY build.sh /
+CMD ["/bin/bash", "/build.sh"]
diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh
new file mode 100755
index 00000000000..bc07140abe4
--- /dev/null
+++ b/docker/packager/binary/build.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -x -e
+
+mkdir -p build/build_docker
+cd build/build_docker
+ccache -s ||:
+cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DSANITIZE=$SANITIZER $CMAKE_FLAGS
+ninja
+ccache -s ||:
+mv ./dbms/programs/clickhouse* /output
+mv ./dbms/unit_tests_dbms /output
+mv ./dbms/programs/odbc-bridge/clickhouse-odbc-bridge /output
diff --git a/docker/packager/deb/Dockerfile b/docker/packager/deb/Dockerfile
index 6a7f7325122..297ce3ad497 100644
--- a/docker/packager/deb/Dockerfile
+++ b/docker/packager/deb/Dockerfile
@@ -9,6 +9,7 @@ RUN apt-get --allow-unauthenticated update -y \
             bash \
             fakeroot \
             cmake \
+            ccache \
             curl \
             gcc-7 \
             g++-7 \
@@ -60,6 +61,5 @@ RUN apt-get --allow-unauthenticated update -y \
             odbcinst \
             tzdata
 
-
-
-CMD /bin/bash build/release --no-pbuilder && mv /*.deb /output && mv *.changes /output && mv *.buildinfo /output
+COPY build.sh /
+CMD ["/bin/bash", "/build.sh"]
diff --git a/docker/packager/deb/build.sh b/docker/packager/deb/build.sh
new file mode 100755
index 00000000000..c9393fc509a
--- /dev/null
+++ b/docker/packager/deb/build.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -x -e
+
+ccache -s ||:
+build/release --no-pbuilder
+mv /*.deb /output
+mv *.changes /output
+mv *.buildinfo /output
+ccache -s ||:
diff --git a/docker/packager/freebsd/Vagrantfile b/docker/packager/freebsd/Vagrantfile
index 3667a9feca7..765f46d5604 100644
--- a/docker/packager/freebsd/Vagrantfile
+++ b/docker/packager/freebsd/Vagrantfile
@@ -2,4 +2,3 @@ Vagrant.configure("2") do |config|
   config.vm.box = "robot-clickhouse/clickhouse-freebsd"
   config.vm.synced_folder ".", "/vagrant", disabled: true
 end
-
diff --git a/docker/packager/packager b/docker/packager/packager
index bceae0d2137..07f4ca579be 100755
--- a/docker/packager/packager
+++ b/docker/packager/packager
@@ -71,7 +71,7 @@ def pull_image(image_name):
 def build_image(image_name, filepath):
     subprocess.check_call("docker build --network=host -t {} -f {} .".format(image_name, filepath), shell=True)
 
-def run_docker_image_with_env(image_name, output, env_variables, ch_root):
+def run_docker_image_with_env(image_name, output, env_variables, ch_root, ccache_dir):
     env_part = " -e ".join(env_variables)
     if env_part:
         env_part = " -e " + env_part
@@ -81,9 +81,10 @@ def run_docker_image_with_env(image_name, output, env_variables, ch_root):
     else:
         interactive = ""
 
-    cmd = "docker run --network=host --rm --volume={output_path}:/output --volume={ch_root}:/build {env} {interactive} {img_name}".format(
+    cmd = "docker run --network=host --rm --volume={output_path}:/output --volume={ch_root}:/build --volume={ccache_dir}:/ccache {env} {interactive} {img_name}".format(
         output_path=output,
         ch_root=ch_root,
+        ccache_dir=ccache_dir,
         env=env_part,
         img_name=image_name,
         interactive=interactive
@@ -104,21 +105,34 @@ def run_vagrant_box_with_env(image_path, output_dir, ch_root):
 
 def parse_env_variables(build_type, compiler, sanitizer, package_type, cache, distcc_hosts, unbundled, split_binary, version, author, official):
     result = []
+    cmake_flags = ['$CMAKE_FLAGS']
+
+    cc = compiler
+    cxx = cc.replace('gcc', 'g++').replace('clang', 'clang++')
     if package_type == "deb":
-        result.append("DEB_CC={}".format(compiler))
-        result.append("DEB_CXX={}".format(compiler.replace('gcc', 'g++').replace('clang', 'clang++')))
+        result.append("DEB_CC={}".format(cc))
+        result.append("DEB_CXX={}".format(cxx))
     elif package_type == "binary":
-        result.append("CC={}".format(compiler))
-        result.append("CXX={}".format(compiler.replace('gcc', 'g++').replace('clang', 'clang++')))
+        result.append("CC={}".format(cc))
+        result.append("CXX={}".format(cxx))
+        cmake_flags.append('-DCMAKE_C_COMPILER=`which {}`'.format(cc))
+        cmake_flags.append('-DCMAKE_CXX_COMPILER=`which {}`'.format(cxx))
 
     if sanitizer:
         result.append("SANITIZER={}".format(sanitizer))
     if build_type:
         result.append("BUILD_TYPE={}".format(build_type))
 
-    if cache:
+    if cache == 'distcc':
         result.append("CCACHE_PREFIX={}".format(cache))
 
+    if cache:
+        result.append("CCACHE_DIR=/ccache")
+        result.append("CCACHE_BASEDIR=/build")
+        result.append("CCACHE_NOHASHDIR=true")
+        result.append("CCACHE_COMPILERCHECK=content")
+        # result.append("CCACHE_UMASK=777")
+
     if distcc_hosts:
         hosts_with_params = ["{}/24,lzo".format(host) for host in distcc_hosts] + ["localhost/`nproc`"]
         result.append('DISTCC_HOSTS="{}"'.format(" ".join(hosts_with_params)))
@@ -126,10 +140,10 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, cache, di
         result.append('DISTCC_HOSTS="{}"'.format("localhost/`nproc`"))
 
     if unbundled:
-        result.append('CMAKE_FLAGS="-DUNBUNDLED=1 -DENABLE_MYSQL=0 -DENABLE_POCO_ODBC=0 -DENABLE_ODBC=0 $CMAKE_FLAGS"')
+        cmake_flags.append('-DUNBUNDLED=1 -DENABLE_MYSQL=0 -DENABLE_POCO_ODBC=0 -DENABLE_ODBC=0')
 
     if split_binary:
-        result.append('CMAKE_FLAGS="-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1 $CMAKE_FLAGS"')
+        cmake_flags.append('-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1')
 
     if version:
         result.append("VERSION_STRING='{}'".format(version))
@@ -138,7 +152,9 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, cache, di
         result.append("AUTHOR='{}'".format(author))
 
     if official:
-        result.append('CMAKE_FLAGS="-DYANDEX_OFFICIAL_BUILD=1 $CMAKE_FLAGS"')
+        cmake_flags.append('-DYANDEX_OFFICIAL_BUILD=1')
+
+    result.append('CMAKE_FLAGS="' + ' '.join(cmake_flags) + '"')
 
     return result
 
@@ -154,6 +170,7 @@ if __name__ == "__main__":
     parser.add_argument("--unbundled", action="store_true")
     parser.add_argument("--split-binary", action="store_true")
     parser.add_argument("--cache", choices=("", "ccache", "distcc"), default="")
+    parser.add_argument("--ccache_dir", default= os.getenv("HOME", "") + '/.ccache')
     parser.add_argument("--distcc-hosts", nargs="+")
     parser.add_argument("--force-build-image", action="store_true")
     parser.add_argument("--version")
@@ -180,7 +197,7 @@ if __name__ == "__main__":
         args.cache, args.distcc_hosts, args.unbundled, args.split_binary,
         args.version, args.author, args.official)
     if args.package_type != "freebsd":
-        run_docker_image_with_env(image_name, args.output_dir, env_prepared, ch_root)
+        run_docker_image_with_env(image_name, args.output_dir, env_prepared, ch_root, args.ccache_dir)
     else:
         logging.info("Running freebsd build, arguments will be ignored")
         run_vagrant_box_with_env(image_name, args.output_dir, ch_root)

From fd630930328495cc3b0ab598cd83b21b98ab4441 Mon Sep 17 00:00:00 2001
From: proller <proller@github.com>
Date: Thu, 25 Apr 2019 17:31:08 +0300
Subject: [PATCH 054/118] Add gperf to build requirements (for upcoming pull
 request #5030 )

---
 debian/control                    | 1 +
 docker/builder/Dockerfile         | 1 +
 docker/packager/binary/Dockerfile | 1 +
 docker/packager/deb/Dockerfile    | 1 +
 utils/build/build_debian.sh       | 2 +-
 5 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/debian/control b/debian/control
index e2b720b2508..5720230e8c1 100644
--- a/debian/control
+++ b/debian/control
@@ -10,6 +10,7 @@ Build-Depends: debhelper (>= 9),
                libc6-dev,
                libicu-dev,
                libreadline-dev,
+               gperf,
                tzdata
 Standards-Version: 3.9.8
 
diff --git a/docker/builder/Dockerfile b/docker/builder/Dockerfile
index 024803e6e6c..14f189f44e7 100644
--- a/docker/builder/Dockerfile
+++ b/docker/builder/Dockerfile
@@ -14,6 +14,7 @@ RUN apt-get update -y \
             libicu-dev \
             liblld-6.0-dev \
             libreadline-dev \
+            gperf \
             ninja-build \
             perl \
             pkg-config \
diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile
index 67dfda5ec1c..2adb9963636 100644
--- a/docker/packager/binary/Dockerfile
+++ b/docker/packager/binary/Dockerfile
@@ -32,6 +32,7 @@ RUN apt-get update -y \
             libicu-dev \
             libreadline-dev \
             ninja-build \
+            gperf \
             git \
             tzdata
 
diff --git a/docker/packager/deb/Dockerfile b/docker/packager/deb/Dockerfile
index 297ce3ad497..b62ed89113f 100644
--- a/docker/packager/deb/Dockerfile
+++ b/docker/packager/deb/Dockerfile
@@ -32,6 +32,7 @@ RUN apt-get --allow-unauthenticated update -y \
             liblld-8-dev \
             libicu-dev \
             libreadline-dev \
+            gperf \
             ninja-build \
             perl \
             pkg-config \
diff --git a/utils/build/build_debian.sh b/utils/build/build_debian.sh
index 5401d1317ad..56dd3f2234d 100755
--- a/utils/build/build_debian.sh
+++ b/utils/build/build_debian.sh
@@ -6,7 +6,7 @@
 # curl https://raw.githubusercontent.com/yandex/ClickHouse/master/utils/build/build_debian.sh | sh
 
 # install compiler and libs
-sudo apt install -y git bash cmake ninja-build gcc-7 g++-7 libicu-dev libreadline-dev
+sudo apt install -y git bash cmake ninja-build gcc-7 g++-7 libicu-dev libreadline-dev gperf
 # for -DUNBUNDLED=1 mode:
 #sudo apt install -y libboost-program-options-dev libboost-system-dev libboost-filesystem-dev libboost-thread-dev zlib1g-dev liblz4-dev libdouble-conversion-dev libzstd-dev libre2-dev libsparsehash-dev librdkafka-dev libcapnp-dev libpoco-dev libsparsehash-dev libgoogle-perftools-dev libunwind-dev googletest libcctz-dev
 

From 7694aa9b1ffc85ba41529f350791dd962e1b1bfb Mon Sep 17 00:00:00 2001
From: Nikolay Semyachkin <letfoolsdie@gmail.com>
Date: Thu, 25 Apr 2019 18:32:29 +0300
Subject: [PATCH 055/118] updated docs for db creation command

added `[ON CLUSTER cluster]` to db creation command
---
 docs/en/query_language/create.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/query_language/create.md b/docs/en/query_language/create.md
index f8e1bab3f78..9ebd50839d3 100644
--- a/docs/en/query_language/create.md
+++ b/docs/en/query_language/create.md
@@ -3,7 +3,7 @@
 Creating db_name databases
 
 ``` sql
-CREATE DATABASE [IF NOT EXISTS] db_name
+CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster]
 ```
 
 `A database` is just a directory for tables.

From 5c68881b72d8e98c25332930f71945f146d5e640 Mon Sep 17 00:00:00 2001
From: Danila Kutenin <danlark@sandbox-286092131>
Date: Thu, 25 Apr 2019 21:34:32 +0300
Subject: [PATCH 056/118] Really faster LZ4 now, cpu dispatch in ClickHouse

---
 dbms/src/Compression/CompressionCodecLZ4.cpp  |   3 +
 .../src/Compression/LZ4_decompress_faster.cpp | 307 +++++++----------
 dbms/src/Compression/LZ4_decompress_faster.h  |   2 +-
 dbms/src/Core/CpuId.h                         | 316 ++++++++++++++++++
 4 files changed, 449 insertions(+), 179 deletions(-)
 create mode 100644 dbms/src/Core/CpuId.h

diff --git a/dbms/src/Compression/CompressionCodecLZ4.cpp b/dbms/src/Compression/CompressionCodecLZ4.cpp
index 08553e0920c..5181ee3df94 100644
--- a/dbms/src/Compression/CompressionCodecLZ4.cpp
+++ b/dbms/src/Compression/CompressionCodecLZ4.cpp
@@ -20,6 +20,7 @@ namespace DB
 namespace ErrorCodes
 {
 extern const int CANNOT_COMPRESS;
+extern const int CANNOT_DECOMPRESS;
 extern const int ILLEGAL_SYNTAX_FOR_CODEC_TYPE;
 extern const int ILLEGAL_CODEC_PARAMETER;
 }
@@ -47,6 +48,8 @@ UInt32 CompressionCodecLZ4::doCompressData(const char * source, UInt32 source_si
 
 void CompressionCodecLZ4::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const
 {
+    // if (LZ4_decompress_safe(source, dest, source_size, uncompressed_size) < 0)
+        // throw Exception("Cannot LZ4_decompress_fast", ErrorCodes::CANNOT_DECOMPRESS);
     LZ4::decompress(source, dest, source_size, uncompressed_size, lz4_stat);
 }
 
diff --git a/dbms/src/Compression/LZ4_decompress_faster.cpp b/dbms/src/Compression/LZ4_decompress_faster.cpp
index 5bfab224538..35e6f403b71 100644
--- a/dbms/src/Compression/LZ4_decompress_faster.cpp
+++ b/dbms/src/Compression/LZ4_decompress_faster.cpp
@@ -1,9 +1,11 @@
 #include "LZ4_decompress_faster.h"
 
+#include <lz4.h>
 #include <string.h>
 #include <iostream>
 #include <random>
 #include <algorithm>
+#include <Core/CpuId.h>
 #include <Core/Defines.h>
 #include <Common/Stopwatch.h>
 #include <common/likely.h>
@@ -14,27 +16,26 @@
 #include <emmintrin.h>
 #endif
 
-#ifdef __AVX__
-#include <immintrin.h>
-#endif
-
 #ifdef __SSSE3__
 #include <tmmintrin.h>
 #endif
 
+#ifdef __AVX__
+#include <immintrin.h>
+#endif
+
 #ifdef __aarch64__
 #include <arm_neon.h>
 #endif
 
-
 namespace LZ4
 {
 
 namespace
 {
 
-template <size_t N> [[maybe_unused]] void copy(UInt8 * dst, const UInt8 * src);
-template <size_t N> [[maybe_unused]] void wildCopy(UInt8 * dst, const UInt8 * src, UInt8 * dst_end);
+template <size_t N, bool use_avx> [[maybe_unused]] void copy(UInt8 * dst, const UInt8 * src);
+template <size_t N, bool use_avx> [[maybe_unused]] void wildCopy(UInt8 * dst, const UInt8 * src, UInt8 * dst_end);
 template <size_t N, bool USE_SHUFFLE> [[maybe_unused]] void copyOverlap(UInt8 * op, const UInt8 *& match, const size_t offset);
 
 
@@ -209,8 +210,10 @@ inline void copyOverlap8Shuffle(UInt8 * op, const UInt8 *& match, const size_t o
 
 
 
-template <> void inline copy<8>(UInt8 * dst, const UInt8 * src) { copy8(dst, src); }
-template <> void inline wildCopy<8>(UInt8 * dst, const UInt8 * src, UInt8 * dst_end) { wildCopy8(dst, src, dst_end); }
+template <> void inline copy<8, false>(UInt8 * dst, const UInt8 * src) { copy8(dst, src); }
+template <> void inline copy<8, true>(UInt8 * dst, const UInt8 * src) { copy8(dst, src); }
+template <> void inline wildCopy<8, false>(UInt8 * dst, const UInt8 * src, UInt8 * dst_end) { wildCopy8(dst, src, dst_end); }
+template <> void inline wildCopy<8, true>(UInt8 * dst, const UInt8 * src, UInt8 * dst_end) { wildCopy8(dst, src, dst_end); }
 template <> void inline copyOverlap<8, false>(UInt8 * op, const UInt8 *& match, const size_t offset) { copyOverlap8(op, match, offset); }
 template <> void inline copyOverlap<8, true>(UInt8 * op, const UInt8 *& match, const size_t offset) { copyOverlap8Shuffle(op, match, offset); }
 
@@ -340,198 +343,126 @@ inline void copyOverlap16Shuffle(UInt8 * op, const UInt8 *& match, const size_t
 #endif
 
 
-template <> void inline copy<16>(UInt8 * dst, const UInt8 * src) { copy16(dst, src); }
-template <> void inline wildCopy<16>(UInt8 * dst, const UInt8 * src, UInt8 * dst_end) { wildCopy16(dst, src, dst_end); }
+template <> void inline copy<16, false>(UInt8 * dst, const UInt8 * src) { copy16(dst, src); }
+template <> void inline copy<16, true>(UInt8 * dst, const UInt8 * src) { copy16(dst, src); }
+template <> void inline wildCopy<16, false>(UInt8 * dst, const UInt8 * src, UInt8 * dst_end) { wildCopy16(dst, src, dst_end); }
+template <> void inline wildCopy<16, true>(UInt8 * dst, const UInt8 * src, UInt8 * dst_end) { wildCopy16(dst, src, dst_end); }
 template <> void inline copyOverlap<16, false>(UInt8 * op, const UInt8 *& match, const size_t offset) { copyOverlap16(op, match, offset); }
 template <> void inline copyOverlap<16, true>(UInt8 * op, const UInt8 *& match, const size_t offset) { copyOverlap16Shuffle(op, match, offset); }
 
+
+template <bool use_avx>
 inline void copy32(UInt8 * dst, const UInt8 * src)
 {
-#if defined(__AVX__)
+    (void)use_avx;
+#ifdef __AVX__
     _mm256_storeu_si256(reinterpret_cast<__m256i *>(dst),
-        _mm256_loadu_si256(reinterpret_cast<const __m256i *>(src)));
+            _mm256_loadu_si256(reinterpret_cast<const __m256i *>(src)));
 #elif defined(__SSE2__)
-    _mm_storeu_si128(reinterpret_cast<__m128i *>(dst),
-        _mm_loadu_si128(reinterpret_cast<const __m128i *>(src)));
-    _mm_storeu_si128(reinterpret_cast<__m128i *>(dst + 16),
-        _mm_loadu_si128(reinterpret_cast<const __m128i *>(src + 16)));
+    if constexpr (use_avx)
+    {
+        __asm__ volatile("vmovups (%0), %%ymm0" : : "r"(src));
+        __asm__ volatile("vmovups %%ymm0, (%0)" : "=r"(dst));
+    }
+    else
+    {
+        _mm_storeu_si128(reinterpret_cast<__m128i *>(dst),
+            _mm_loadu_si128(reinterpret_cast<const __m128i *>(src)));
+        _mm_storeu_si128(reinterpret_cast<__m128i *>(dst + 16),
+            _mm_loadu_si128(reinterpret_cast<const __m128i *>(src + 16)));
+    }
 #else
-    /// This is from reference implementation, actually there can be issues if we write 32 for ARM processor
-    memcpy(dst, src, 16);
-    memcpy(dst + 16, src + 16, 16);
+    memcpy(dst, src, 32);
 #endif
 }
 
+template <bool use_avx>
 inline void wildCopy32(UInt8 * dst, const UInt8 * src, UInt8 * dst_end)
 {
     do
     {
-        copy32(dst, src);
+        copy32<use_avx>(dst, src);
         dst += 32;
         src += 32;
     } while (dst < dst_end);
 }
 
-template <> void inline copy<32>(UInt8 * dst, const UInt8 * src) { copy32(dst, src); }
-template <> void inline wildCopy<32>(UInt8 * dst, const UInt8 * src, UInt8 * dst_end) { wildCopy32(dst, src, dst_end); }
-
-inline void copyUsingOffset(UInt8 * dst, const UInt8 * src, UInt8 * dst_end, const size_t offset)
+inline void copyOverlap32(UInt8 * op, const UInt8 *& match, const size_t offset)
 {
-    UInt8 v[8];
-    switch (offset)
-    {
-        case 1:
-            memset(v, *src, 8);
-            goto copy_loop;
-        case 2:
-            memcpy(v, src, 2);
-            memcpy(&v[2], src, 2);
-            memcpy(&v[4], &v[0], 4);
-            goto copy_loop;
-        case 4:
-            memcpy(v, src, 4);
-            memcpy(&v[4], src, 4);
-            goto copy_loop;
-        default:
-            if (offset < 8)
-            {
-                copyOverlap<8, false>(dst, src, offset);
-                dst += 8;
-            }
-            else
-            {
-                memcpy(dst, src, 8);
-                dst += 8;
-                src += 8;
-            }
-            wildCopy<8>(dst, src, dst_end);
-            return;
-    }
-copy_loop:
-    memcpy(dst, v, 8);
-    dst += 8;
-    while (dst < dst_end)
-    {
-        memcpy(dst, v, 8);
-        dst += 8;
-    }
+    /// 4 % n.
+    static constexpr int shift1[]
+        = { 0,  1,  2,  1,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4 };
+
+    /// 8 % n - 4 % n
+    static constexpr int shift2[]
+        = { 0,  0,  0,  1,  0, -1, -2, -3, -4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4 };
+
+    /// 16 % n - 8 % n
+    static constexpr int shift3[]
+        = { 0,  0,  0, -1,  0, -2,  2,  1,  8, -1, -2, -3, -4, -5, -6, -7,  8,  8,  8,  8,  8,  8,  8,  8,  8,  8,  8,  8,  8,  8,  8,  8 };
+
+    /// 32 % n - 16 % n
+    static constexpr int shift4[]
+        = { 0,  0,  0,  1,  0,  1, -2,  2,  0, -2, -4,  5,  4,  3,  2,  1,  0, -1, -2, -3, -4, -5, -6, -7, -8, -9,-10,-11,-12,-13,-14,-15 };
+
+    op[0] = match[0];
+    op[1] = match[1];
+    op[2] = match[2];
+    op[3] = match[3];
+
+    match += shift1[offset];
+    memcpy(op + 4, match, 4);
+    match += shift2[offset];
+    memcpy(op + 8, match, 8);
+    match += shift3[offset];
+    memcpy(op + 16, match, 16);
+    match += shift4[offset];
 }
 
+template <> void inline copy<32, false>(UInt8 * dst, const UInt8 * src) { copy32<false>(dst, src); }
+template <> void inline copy<32, true>(UInt8 * dst, const UInt8 * src) { copy32<true>(dst, src); }
+template <> void inline wildCopy<32, false>(UInt8 * dst, const UInt8 * src, UInt8 * dst_end) { wildCopy32<false>(dst, src, dst_end); }
+template <> void inline wildCopy<32, true>(UInt8 * dst, const UInt8 * src, UInt8 * dst_end) { wildCopy32<true>(dst, src, dst_end); }
+template <> void inline copyOverlap<32, false>(UInt8 * op, const UInt8 *& match, const size_t offset) { copyOverlap32(op, match, offset); }
+
 /// See also https://stackoverflow.com/a/30669632
 
 
-/// use_optimized_new_lz4_version is for the new optimized LZ4 decompression that was introduced in LZ4 1.9.0
-/// With experiments it turned out to be faster for big columns, so we decided to add it to the statistics choice.
-
-template <size_t copy_amount, size_t guaranteed_minimal_dest_size, bool use_shuffle, bool use_optimized_new_lz4_version>
+template <size_t copy_amount, bool use_shuffle, bool use_avx>
 void NO_INLINE decompressImpl(
      const char * const source,
      char * const dest,
-     [[maybe_unused]] size_t source_size,
      size_t dest_size)
 {
     const UInt8 * ip = reinterpret_cast<const UInt8 *>(source);
-    [[maybe_unused]] const UInt8 * const input_end = reinterpret_cast<const UInt8 *>(source) + source_size;
     UInt8 * op = reinterpret_cast<UInt8 *>(dest);
     UInt8 * const output_end = op + dest_size;
-    UInt8 * copy_end;
-    unsigned token;
-    size_t length;
-    size_t offset;
-    const UInt8 * match;
-
-    auto continue_read_length = [&]
-    {
-        unsigned s;
-        do
-        {
-            s = *ip++;
-            length += s;
-        } while (unlikely(s == 255));
-    };
-
-    /// Don't use `if constexpr` here because compilers start warning about unused labels
-    /// And I don't want to insert pragma push etc. Codegen is the same because of dead code elimination.
-    /// TODO(danlark) Generalize this branch with copy_amount etc.
-    if (use_optimized_new_lz4_version)
-    {
-        if (guaranteed_minimal_dest_size >= 64 || dest_size >= 64)
-        {
-            while (1)
-            {
-                /// Get literal length.
-                token = *ip++;
-                length = token >> 4;
-                if (length == 0x0F)
-                {
-                    continue_read_length();
-                    copy_end = op + length;
-                    if (copy_end > output_end - 32 || ip + length > input_end - 32)
-                        goto literal_copy;
-                    wildCopy<32>(op, ip, copy_end);
-                    ip += length;
-                    op = copy_end;
-                }
-                else
-                {
-                    copy_end = op + length;
-                    if (ip > input_end - 17)
-                        goto literal_copy;
-                    copy<16>(op, ip);
-                    ip += length;
-                    op = copy_end;
-                }
-
-                offset = unalignedLoad<UInt16>(ip);
-                ip += 2;
-                match = op - offset;
-
-                /// Get match length.
-                length = token & 0x0F;
-                if (length == 0x0F)
-                {
-                    continue_read_length();
-                    length += 4;
-                    if (op + length >= output_end - 64)
-                        goto match_copy;
-                }
-                else
-                {
-                    length += 4;
-                    if (op + length >= output_end - 64)
-                        goto match_copy;
-                    if (offset >= 8)
-                    {
-                        memcpy(op, match, 8);
-                        memcpy(op + 8, match + 8, 8);
-                        memcpy(op + 16, match + 16, 2);
-                        op += length;
-                        continue;
-                    }
-                }
-
-                /// Copy match within block, that produce overlapping pattern. Match may replicate itself.
-                copy_end = op + length;
-                if (unlikely(offset < 16))
-                    copyUsingOffset(op, match, copy_end, offset);
-                else
-                    wildCopy<32>(op, match, copy_end);
-
-                op = copy_end;
-            }
-        }
-    }
 
     while (1)
     {
+        size_t length;
+
+        auto continue_read_length = [&]
+        {
+            unsigned s;
+            do
+            {
+                s = *ip++;
+                length += s;
+            } while (unlikely(s == 255));
+        };
+
         /// Get literal length.
-        token = *ip++;
+
+        const unsigned token = *ip++;
         length = token >> 4;
         if (length == 0x0F)
             continue_read_length();
+
         /// Copy literals.
-        copy_end = op + length;
-literal_copy:
+
+        UInt8 * copy_end = op + length;
+
         /// input: Hello, world
         ///        ^-ip
         /// output: xyz
@@ -542,7 +473,8 @@ literal_copy:
         ///              ^-ip
         /// output: xyzHello, w
         ///                  ^-op (we will overwrite excessive bytes on next iteration)
-        wildCopy<copy_amount>(op, ip, copy_end);
+
+        wildCopy<copy_amount, use_avx>(op, ip, copy_end);    /// Here we can write up to copy_amount - 1 bytes after buffer.
 
         ip += length;
         op = copy_end;
@@ -551,18 +483,20 @@ literal_copy:
             return;
 
         /// Get match offset.
-        offset = unalignedLoad<UInt16>(ip);
+
+        size_t offset = unalignedLoad<UInt16>(ip);
         ip += 2;
-        match = op - offset;
+        const UInt8 * match = op - offset;
 
         /// Get match length.
+
         length = token & 0x0F;
         if (length == 0x0F)
             continue_read_length();
         length += 4;
 
-match_copy:
         /// Copy match within block, that produce overlapping pattern. Match may replicate itself.
+
         copy_end = op + length;
 
         /** Here we can write up to copy_amount - 1 - 4 * 2 bytes after buffer.
@@ -581,19 +515,20 @@ match_copy:
             ///
             /// output: HelloHelloHel
             ///            ^-match   ^-op
+
             copyOverlap<copy_amount, use_shuffle>(op, match, offset);
         }
         else
         {
-            copy<copy_amount>(op, match);
+            copy<copy_amount, use_avx>(op, match);
             match += copy_amount;
         }
 
         op += copy_amount;
 
-        copy<copy_amount>(op, match);   /// copy_amount + copy_amount - 1 - 4 * 2 bytes after buffer.
+        copy<copy_amount, use_avx>(op, match);   /// copy_amount + copy_amount - 1 - 4 * 2 bytes after buffer.
         if (length > copy_amount * 2)
-            wildCopy<copy_amount>(op + copy_amount, match + copy_amount, copy_end);
+            wildCopy<copy_amount, use_avx>(op + copy_amount, match + copy_amount, copy_end);
 
         op = copy_end;
     }
@@ -601,6 +536,7 @@ match_copy:
 
 }
 
+
 void decompress(
     const char * const source,
     char * const dest,
@@ -610,24 +546,39 @@ void decompress(
 {
     if (source_size == 0 || dest_size == 0)
         return;
-    static constexpr size_t small_block_threshold = 32768;
+
+
     /// Don't run timer if the block is too small.
-    if (dest_size >= small_block_threshold)
+    if (dest_size >= 32768)
     {
+        bool have_AVX = DB::Cpu::CpuFlagsCache::have_AVX;
         size_t best_variant = statistics.select();
 
         /// Run the selected method and measure time.
 
         Stopwatch watch;
-
-        if (best_variant == 0)
-            decompressImpl<16, small_block_threshold, true, false>(source, dest, source_size, dest_size);
-        if (best_variant == 1)
-            decompressImpl<16, small_block_threshold, false, false>(source, dest, source_size, dest_size);
-        if (best_variant == 2)
-            decompressImpl<8, small_block_threshold, true, false>(source, dest, source_size, dest_size);
-        if (best_variant == 3)
-            decompressImpl<8, small_block_threshold, false, true>(source, dest, source_size, dest_size);
+        if (have_AVX)
+        {
+            if (best_variant == 0)
+                decompressImpl<16, true, true>(source, dest, dest_size);
+            if (best_variant == 1)
+                decompressImpl<16, false, true>(source, dest, dest_size);
+            if (best_variant == 2)
+                decompressImpl<8, true, true>(source, dest, dest_size);
+            if (best_variant == 3)
+                decompressImpl<32, false, true>(source, dest, dest_size);
+        }
+        else
+        {
+            if (best_variant == 0)
+                decompressImpl<16, true, false>(source, dest, dest_size);
+            if (best_variant == 1)
+                decompressImpl<16, false, false>(source, dest, dest_size);
+            if (best_variant == 2)
+                decompressImpl<8, true, false>(source, dest, dest_size);
+            if (best_variant == 3)
+                decompressImpl<32, false, false>(source, dest, dest_size);
+        }
 
         watch.stop();
 
@@ -637,7 +588,7 @@ void decompress(
     }
     else
     {
-        decompressImpl<8, 0, false, false>(source, dest, source_size, dest_size);
+        decompressImpl<8, false, false>(source, dest, dest_size);
     }
 }
 
diff --git a/dbms/src/Compression/LZ4_decompress_faster.h b/dbms/src/Compression/LZ4_decompress_faster.h
index 3b48106c583..68a55c04581 100644
--- a/dbms/src/Compression/LZ4_decompress_faster.h
+++ b/dbms/src/Compression/LZ4_decompress_faster.h
@@ -34,7 +34,7 @@ namespace LZ4
   *  that is allowed to read/write.
   * This value is a little overestimation.
   */
-static constexpr size_t ADDITIONAL_BYTES_AT_END_OF_BUFFER = 32;
+static constexpr size_t ADDITIONAL_BYTES_AT_END_OF_BUFFER = 64;
 
 
 /** When decompressing uniform sequence of blocks (for example, blocks from one file),
diff --git a/dbms/src/Core/CpuId.h b/dbms/src/Core/CpuId.h
new file mode 100644
index 00000000000..dfb3585b038
--- /dev/null
+++ b/dbms/src/Core/CpuId.h
@@ -0,0 +1,316 @@
+#include <Core/Types.h>
+
+#if defined(__x86_64__) || defined(__i386__)
+#include <cpuid.h>
+#endif
+
+#include <cstring>
+
+
+namespace DB
+{
+namespace Cpu
+{
+
+#if defined(__x86_64__) || defined(__i386__)
+inline UInt64 _xgetbv(UInt32 xcr) noexcept
+{
+    UInt32 eax;
+    UInt32 edx;
+    __asm__ volatile(
+        "xgetbv"
+        : "=a"(eax), "=d"(edx)
+        : "c"(xcr));
+    return (static_cast<UInt64>(edx) << 32) | eax;
+}
+#endif
+
+inline bool cpuid(UInt32 op, UInt32 sub_op, UInt32 * res) noexcept
+{
+#if defined(__x86_64__) || defined(__i386__)
+    __cpuid_count(op, sub_op, res[0], res[1], res[2], res[3]);
+    return true;
+#else
+    (void)op;
+    (void)sub_op;
+
+    memset(res, 0, 4 * sizeof(*res));
+
+    return false;
+#endif
+}
+
+inline bool cpuid(UInt32 op, UInt32 * res) noexcept
+{
+#if defined(__x86_64__) || defined(__i386__)
+    __cpuid(op, res[0], res[1], res[2], res[3]);
+    return true;
+#else
+    (void)op;
+
+    memset(res, 0, 4 * sizeof(*res));
+
+    return false;
+#endif
+}
+
+#define CPU_ID_ENUMERATE(OP) \
+    OP(SSE)                  \
+    OP(SSE2)                 \
+    OP(SSE3)                 \
+    OP(SSSE3)                \
+    OP(SSE41)                \
+    OP(SSE42)                \
+    OP(F16C)                 \
+    OP(POPCNT)               \
+    OP(BMI1)                 \
+    OP(BMI2)                 \
+    OP(PCLMUL)               \
+    OP(AES)                  \
+    OP(AVX)                  \
+    OP(FMA)                  \
+    OP(AVX2)                 \
+    OP(AVX512F)              \
+    OP(AVX512DQ)             \
+    OP(AVX512IFMA)           \
+    OP(AVX512PF)             \
+    OP(AVX512ER)             \
+    OP(AVX512CD)             \
+    OP(AVX512BW)             \
+    OP(AVX512VL)             \
+    OP(AVX512VBMI)           \
+    OP(PREFETCHWT1)          \
+    OP(SHA)                  \
+    OP(ADX)                  \
+    OP(RDRAND)               \
+    OP(RDSEED)               \
+    OP(PCOMMIT)              \
+    OP(RDTSCP)               \
+    OP(CLFLUSHOPT)           \
+    OP(CLWB)                 \
+    OP(XSAVE)                \
+    OP(OSXSAVE)
+
+union CpuInfo
+{
+    UInt32 info[4];
+
+    struct
+    {
+        UInt32 eax;
+        UInt32 ebx;
+        UInt32 ecx;
+        UInt32 edx;
+    };
+
+    inline CpuInfo(UInt32 op) noexcept { cpuid(op, info); }
+
+    inline CpuInfo(UInt32 op, UInt32 sub_op) noexcept { cpuid(op, sub_op, info); }
+};
+
+#define DEF_NAME(X) inline bool have##X() noexcept;
+    CPU_ID_ENUMERATE(DEF_NAME)
+#undef DEF_NAME
+
+bool haveRDTSCP() noexcept
+{
+    return (CpuInfo(0x80000001).edx >> 27) & 1u;
+}
+
+bool haveSSE() noexcept
+{
+    return (CpuInfo(0x1).edx >> 25) & 1u;
+}
+
+bool haveSSE2() noexcept
+{
+    return (CpuInfo(0x1).edx >> 26) & 1u;
+}
+
+bool haveSSE3() noexcept
+{
+    return CpuInfo(0x1).ecx & 1u;
+}
+
+bool havePCLMUL() noexcept
+{
+    return (CpuInfo(0x1).ecx >> 1) & 1u;
+}
+
+bool haveSSSE3() noexcept
+{
+    return (CpuInfo(0x1).ecx >> 9) & 1u;
+}
+
+bool haveSSE41() noexcept
+{
+    return (CpuInfo(0x1).ecx >> 19) & 1u;
+}
+
+bool haveSSE42() noexcept
+{
+    return (CpuInfo(0x1).ecx >> 20) & 1u;
+}
+
+bool haveF16C() noexcept
+{
+    return (CpuInfo(0x1).ecx >> 29) & 1u;
+}
+
+bool havePOPCNT() noexcept
+{
+    return (CpuInfo(0x1).ecx >> 23) & 1u;
+}
+
+bool haveAES() noexcept
+{
+    return (CpuInfo(0x1).ecx >> 25) & 1u;
+}
+
+bool haveXSAVE() noexcept
+{
+    return (CpuInfo(0x1).ecx >> 26) & 1u;
+}
+
+bool haveOSXSAVE() noexcept
+{
+    return (CpuInfo(0x1).ecx >> 27) & 1u;
+}
+
+bool haveAVX() noexcept
+{
+#if defined(_x86_)
+    // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-optimization-manual.pdf
+    // https://bugs.chromium.org/p/chromium/issues/detail?id=375968
+    return haveOSXSAVE()                           // implies haveXSAVE()
+           && (_xgetbv(0) & 6u) == 6u              // XMM state and YMM state are enabled by OS
+           && ((CpuInfo(0x1).ecx >> 28) & 1u); // AVX bit
+#else
+    return false;
+#endif
+}
+
+bool haveFMA() noexcept
+{
+    return haveAVX() && ((CpuInfo(0x1).ecx >> 12) & 1u);
+}
+
+bool haveAVX2() noexcept
+{
+    return haveAVX() && ((CpuInfo(0x7, 0).ebx >> 5) & 1u);
+}
+
+bool haveBMI1() noexcept
+{
+    return (CpuInfo(0x7, 0).ebx >> 3) & 1u;
+}
+
+bool haveBMI2() noexcept
+{
+    return (CpuInfo(0x7, 0).ebx >> 8) & 1u;
+}
+
+bool haveAVX512F() noexcept
+{
+#if defined(__x86_64__) || defined(__i386__)
+    // https://software.intel.com/en-us/articles/how-to-detect-knl-instruction-support
+    return haveOSXSAVE()                           // implies haveXSAVE()
+           && (_xgetbv(0) & 6u) == 6u              // XMM state and YMM state are enabled by OS
+           && ((_xgetbv(0) >> 5) & 7u) == 7u       // ZMM state is enabled by OS
+           && CpuInfo(0x0).eax >= 0x7          // leaf 7 is present
+           && ((CpuInfo(0x7).ebx >> 16) & 1u); // AVX512F bit
+#else
+    return false;
+#endif
+}
+
+bool haveAVX512DQ() noexcept
+{
+    return haveAVX512F() && ((CpuInfo(0x7, 0).ebx >> 17) & 1u);
+}
+
+bool haveRDSEED() noexcept
+{
+    return CpuInfo(0x0).eax >= 0x7 && ((CpuInfo(0x7, 0).ebx >> 18) & 1u);
+}
+
+bool haveADX() noexcept
+{
+    return CpuInfo(0x0).eax >= 0x7 && ((CpuInfo(0x7, 0).ebx >> 19) & 1u);
+}
+
+bool haveAVX512IFMA() noexcept
+{
+    return haveAVX512F() && ((CpuInfo(0x7, 0).ebx >> 21) & 1u);
+}
+
+bool havePCOMMIT() noexcept
+{
+    return CpuInfo(0x0).eax >= 0x7 && ((CpuInfo(0x7, 0).ebx >> 22) & 1u);
+}
+
+bool haveCLFLUSHOPT() noexcept
+{
+    return CpuInfo(0x0).eax >= 0x7 && ((CpuInfo(0x7, 0).ebx >> 23) & 1u);
+}
+
+bool haveCLWB() noexcept
+{
+    return CpuInfo(0x0).eax >= 0x7 && ((CpuInfo(0x7, 0).ebx >> 24) & 1u);
+}
+
+bool haveAVX512PF() noexcept
+{
+    return haveAVX512F() && ((CpuInfo(0x7, 0).ebx >> 26) & 1u);
+}
+
+bool haveAVX512ER() noexcept
+{
+    return haveAVX512F() && ((CpuInfo(0x7, 0).ebx >> 27) & 1u);
+}
+
+bool haveAVX512CD() noexcept
+{
+    return haveAVX512F() && ((CpuInfo(0x7, 0).ebx >> 28) & 1u);
+}
+
+bool haveSHA() noexcept
+{
+    return CpuInfo(0x0).eax >= 0x7 && ((CpuInfo(0x7, 0).ebx >> 29) & 1u);
+}
+
+bool haveAVX512BW() noexcept
+{
+    return haveAVX512F() && ((CpuInfo(0x7, 0).ebx >> 30) & 1u);
+}
+
+bool haveAVX512VL() noexcept
+{
+    return haveAVX512F() && ((CpuInfo(0x7, 0).ebx >> 31) & 1u);
+}
+
+bool havePREFETCHWT1() noexcept
+{
+    return CpuInfo(0x0).eax >= 0x7 && ((CpuInfo(0x7, 0).ecx >> 0) & 1u);
+}
+
+bool haveAVX512VBMI() noexcept
+{
+    return haveAVX512F() && ((CpuInfo(0x7, 0).ecx >> 1) & 1u);
+}
+
+bool haveRDRAND() noexcept
+{
+    return CpuInfo(0x0).eax >= 0x7 && ((CpuInfo(0x1).ecx >> 30) & 1u);
+}
+
+struct CpuFlagsCache
+{
+#define DEF_NAME(X) static inline bool have_##X = have##X();
+    CPU_ID_ENUMERATE(DEF_NAME)
+#undef DEF_NAME
+};
+
+}
+}
+

From 3501f633e44d413135f868f6162e853759c55672 Mon Sep 17 00:00:00 2001
From: Danila Kutenin <danlark@sandbox-286092131>
Date: Thu, 25 Apr 2019 21:35:58 +0300
Subject: [PATCH 057/118] Cosmetics

---
 dbms/src/Compression/CompressionCodecLZ4.cpp | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/dbms/src/Compression/CompressionCodecLZ4.cpp b/dbms/src/Compression/CompressionCodecLZ4.cpp
index 5181ee3df94..08553e0920c 100644
--- a/dbms/src/Compression/CompressionCodecLZ4.cpp
+++ b/dbms/src/Compression/CompressionCodecLZ4.cpp
@@ -20,7 +20,6 @@ namespace DB
 namespace ErrorCodes
 {
 extern const int CANNOT_COMPRESS;
-extern const int CANNOT_DECOMPRESS;
 extern const int ILLEGAL_SYNTAX_FOR_CODEC_TYPE;
 extern const int ILLEGAL_CODEC_PARAMETER;
 }
@@ -48,8 +47,6 @@ UInt32 CompressionCodecLZ4::doCompressData(const char * source, UInt32 source_si
 
 void CompressionCodecLZ4::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const
 {
-    // if (LZ4_decompress_safe(source, dest, source_size, uncompressed_size) < 0)
-        // throw Exception("Cannot LZ4_decompress_fast", ErrorCodes::CANNOT_DECOMPRESS);
     LZ4::decompress(source, dest, source_size, uncompressed_size, lz4_stat);
 }
 

From f86cf7bd11d20c03effec2b53e44d660b5c41679 Mon Sep 17 00:00:00 2001
From: Danila Kutenin <danlark@sandbox-286092131>
Date: Fri, 26 Apr 2019 00:53:22 +0300
Subject: [PATCH 058/118] Correct cpu dispatch, nounroll for clang

---
 .../src/Compression/LZ4_decompress_faster.cpp | 64 ++++++++++---------
 dbms/src/Compression/LZ4_decompress_faster.h  | 10 +--
 dbms/src/Core/CpuId.h                         |  2 +-
 3 files changed, 40 insertions(+), 36 deletions(-)

diff --git a/dbms/src/Compression/LZ4_decompress_faster.cpp b/dbms/src/Compression/LZ4_decompress_faster.cpp
index 35e6f403b71..727ba5e05ef 100644
--- a/dbms/src/Compression/LZ4_decompress_faster.cpp
+++ b/dbms/src/Compression/LZ4_decompress_faster.cpp
@@ -34,8 +34,8 @@ namespace LZ4
 namespace
 {
 
-template <size_t N, bool use_avx> [[maybe_unused]] void copy(UInt8 * dst, const UInt8 * src);
-template <size_t N, bool use_avx> [[maybe_unused]] void wildCopy(UInt8 * dst, const UInt8 * src, UInt8 * dst_end);
+template <size_t N, bool USE_AVX> [[maybe_unused]] void copy(UInt8 * dst, const UInt8 * src);
+template <size_t N, bool USE_AVX> [[maybe_unused]] void wildCopy(UInt8 * dst, const UInt8 * src, UInt8 * dst_end);
 template <size_t N, bool USE_SHUFFLE> [[maybe_unused]] void copyOverlap(UInt8 * op, const UInt8 *& match, const size_t offset);
 
 
@@ -46,6 +46,9 @@ inline void copy8(UInt8 * dst, const UInt8 * src)
 
 inline void wildCopy8(UInt8 * dst, const UInt8 * src, UInt8 * dst_end)
 {
+#if defined(__clang__)
+    #pragma nounroll
+#endif
     do
     {
         copy8(dst, src);
@@ -232,6 +235,9 @@ inline void copy16(UInt8 * dst, const UInt8 * src)
 
 inline void wildCopy16(UInt8 * dst, const UInt8 * src, UInt8 * dst_end)
 {
+#if defined(__clang__)
+    #pragma nounroll
+#endif
     do
     {
         copy16(dst, src);
@@ -361,8 +367,12 @@ inline void copy32(UInt8 * dst, const UInt8 * src)
 #elif defined(__SSE2__)
     if constexpr (use_avx)
     {
-        __asm__ volatile("vmovups (%0), %%ymm0" : : "r"(src));
-        __asm__ volatile("vmovups %%ymm0, (%0)" : "=r"(dst));
+        __asm__ volatile(
+            "vmovups (%1), %%ymm0\n\t"
+            "vmovups %%ymm0, (%0)"
+            :  /* output */
+            : "r"(dst), "r"(src) /* input */
+            : "%ymm0"); /* clobber_registers */
     }
     else
     {
@@ -379,6 +389,9 @@ inline void copy32(UInt8 * dst, const UInt8 * src)
 template <bool use_avx>
 inline void wildCopy32(UInt8 * dst, const UInt8 * src, UInt8 * dst_end)
 {
+#if defined(__clang__)
+    #pragma nounroll
+#endif
     do
     {
         copy32<use_avx>(dst, src);
@@ -427,7 +440,7 @@ template <> void inline copyOverlap<32, false>(UInt8 * op, const UInt8 *& match,
 
 /// See also https://stackoverflow.com/a/30669632
 
-
+/// NO_INLINE is vital for hiding avx instructions
 template <size_t copy_amount, bool use_shuffle, bool use_avx>
 void NO_INLINE decompressImpl(
      const char * const source,
@@ -438,6 +451,9 @@ void NO_INLINE decompressImpl(
     UInt8 * op = reinterpret_cast<UInt8 *>(dest);
     UInt8 * const output_end = op + dest_size;
 
+#if defined(__clang__)
+    #pragma nounroll
+#endif
     while (1)
     {
         size_t length;
@@ -551,34 +567,23 @@ void decompress(
     /// Don't run timer if the block is too small.
     if (dest_size >= 32768)
     {
-        bool have_AVX = DB::Cpu::CpuFlagsCache::have_AVX;
-        size_t best_variant = statistics.select();
+        const bool have_AVX = DB::Cpu::CpuFlagsCache::have_AVX;
+        size_t best_variant = statistics.select(PerformanceStatistics::NUM_ELEMENTS - !have_AVX);
 
         /// Run the selected method and measure time.
 
         Stopwatch watch;
-        if (have_AVX)
-        {
-            if (best_variant == 0)
-                decompressImpl<16, true, true>(source, dest, dest_size);
-            if (best_variant == 1)
-                decompressImpl<16, false, true>(source, dest, dest_size);
-            if (best_variant == 2)
-                decompressImpl<8, true, true>(source, dest, dest_size);
-            if (best_variant == 3)
-                decompressImpl<32, false, true>(source, dest, dest_size);
-        }
-        else
-        {
-            if (best_variant == 0)
-                decompressImpl<16, true, false>(source, dest, dest_size);
-            if (best_variant == 1)
-                decompressImpl<16, false, false>(source, dest, dest_size);
-            if (best_variant == 2)
-                decompressImpl<8, true, false>(source, dest, dest_size);
-            if (best_variant == 3)
-                decompressImpl<32, false, false>(source, dest, dest_size);
-        }
+        if (best_variant == 0)
+            decompressImpl<16, true, false>(source, dest, dest_size);
+        if (best_variant == 1)
+            decompressImpl<16, false, false>(source, dest, dest_size);
+        if (best_variant == 2)
+            decompressImpl<8, true, false>(source, dest, dest_size);
+        if (best_variant == 3)
+            decompressImpl<32, false, false>(source, dest, dest_size);
+        /// If we don't have avx, we just never be here becase we select from NUM_EMELENTS minus one, be careful to add new methods
+        if (best_variant == 4)
+            decompressImpl<32, false, true>(source, dest, dest_size);
 
         watch.stop();
 
@@ -631,7 +636,6 @@ void statistics(
     const UInt8 * ip = reinterpret_cast<const UInt8 *>(source);
     UInt8 * op = reinterpret_cast<UInt8 *>(dest);
     UInt8 * const output_end = op + dest_size;
-
     while (1)
     {
         size_t length;
diff --git a/dbms/src/Compression/LZ4_decompress_faster.h b/dbms/src/Compression/LZ4_decompress_faster.h
index 68a55c04581..e83abc945e8 100644
--- a/dbms/src/Compression/LZ4_decompress_faster.h
+++ b/dbms/src/Compression/LZ4_decompress_faster.h
@@ -88,14 +88,14 @@ struct PerformanceStatistics
     };
 
     /// Number of different algorithms to select from.
-    static constexpr size_t NUM_ELEMENTS = 4;
+    static constexpr size_t NUM_ELEMENTS = 5;
 
     /// Cold invocations may be affected by additional memory latencies. Don't take first invocations into account.
     static constexpr double NUM_INVOCATIONS_TO_THROW_OFF = 2;
 
     /// How to select method to run.
     /// -1 - automatically, based on statistics (default);
-    /// 0..3 - always choose specified method (for performance testing);
+    /// 0..4 - always choose specified method (for performance testing);
     /// -2 - choose methods in round robin fashion (for performance testing).
     ssize_t choose_method = -1;
 
@@ -105,17 +105,17 @@ struct PerformanceStatistics
 
     /// To select from different algorithms we use a kind of "bandits" algorithm.
     /// Sample random values from estimated normal distributions and choose the minimal.
-    size_t select()
+    size_t select(size_t max_element)
     {
         if (choose_method < 0)
         {
             double samples[NUM_ELEMENTS];
-            for (size_t i = 0; i < NUM_ELEMENTS; ++i)
+            for (size_t i = 0; i < max_element; ++i)
                 samples[i] = choose_method == -1
                     ? data[i].sample(rng)
                     : data[i].adjustedCount();
 
-            return std::min_element(samples, samples + NUM_ELEMENTS) - samples;
+            return std::min_element(samples, samples + max_element) - samples;
         }
         else
             return choose_method;
diff --git a/dbms/src/Core/CpuId.h b/dbms/src/Core/CpuId.h
index dfb3585b038..b1a2d9fa10a 100644
--- a/dbms/src/Core/CpuId.h
+++ b/dbms/src/Core/CpuId.h
@@ -179,7 +179,7 @@ bool haveOSXSAVE() noexcept
 
 bool haveAVX() noexcept
 {
-#if defined(_x86_)
+#if defined(__x86_64__) || defined(__i386__)
     // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-optimization-manual.pdf
     // https://bugs.chromium.org/p/chromium/issues/detail?id=375968
     return haveOSXSAVE()                           // implies haveXSAVE()

From 97c532c0f0151f91f95a437df90da8dec1089e13 Mon Sep 17 00:00:00 2001
From: proller <proller@users.noreply.github.com>
Date: Fri, 26 Apr 2019 02:21:59 +0300
Subject: [PATCH 059/118] Build fixes (#5114)

Need to push builder docker image
---
 contrib/libxml2-cmake/CMakeLists.txt     |  2 +-
 dbms/programs/CMakeLists.txt             |  5 -----
 dbms/programs/odbc-bridge/CMakeLists.txt | 11 ++++++-----
 docker/packager/binary/build.sh          |  2 +-
 4 files changed, 8 insertions(+), 12 deletions(-)

diff --git a/contrib/libxml2-cmake/CMakeLists.txt b/contrib/libxml2-cmake/CMakeLists.txt
index 326aa5c2d45..cb52e5dad5a 100644
--- a/contrib/libxml2-cmake/CMakeLists.txt
+++ b/contrib/libxml2-cmake/CMakeLists.txt
@@ -48,7 +48,7 @@ set(SRCS
     ${LIBXML2_SOURCE_DIR}/parserInternals.c
     ${LIBXML2_SOURCE_DIR}/xmlwriter.c
     ${LIBXML2_SOURCE_DIR}/xmlunicode.c
-    #${LIBXML2_SOURCE_DIR}/runxmlconf.c
+    ${LIBXML2_SOURCE_DIR}/runxmlconf.c
     ${LIBXML2_SOURCE_DIR}/xmlmemory.c
     ${LIBXML2_SOURCE_DIR}/nanoftp.c
     ${LIBXML2_SOURCE_DIR}/xmlschemastypes.c
diff --git a/dbms/programs/CMakeLists.txt b/dbms/programs/CMakeLists.txt
index 52b58ad0df8..af65df6e8fe 100644
--- a/dbms/programs/CMakeLists.txt
+++ b/dbms/programs/CMakeLists.txt
@@ -209,11 +209,6 @@ else ()
         install (FILES ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-obfuscator DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
         list(APPEND CLICKHOUSE_BUNDLE clickhouse-obfuscator)
     endif ()
-    if (ENABLE_CLICKHOUSE_ODBC_BRIDGE)
-        # just to be able to run integration tests
-        add_custom_target (clickhouse-odbc-bridge-copy ALL COMMAND ${CMAKE_COMMAND} -E create_symlink ${CMAKE_CURRENT_BINARY_DIR}/odbc-bridge/clickhouse-odbc-bridge clickhouse-odbc-bridge DEPENDS clickhouse-odbc-bridge)
-    endif ()
-
 
     # install always because depian package want this files:
     add_custom_target (clickhouse-clang ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-clang DEPENDS clickhouse)
diff --git a/dbms/programs/odbc-bridge/CMakeLists.txt b/dbms/programs/odbc-bridge/CMakeLists.txt
index 38f2bc60b2d..90aad184f3e 100644
--- a/dbms/programs/odbc-bridge/CMakeLists.txt
+++ b/dbms/programs/odbc-bridge/CMakeLists.txt
@@ -39,12 +39,13 @@ clickhouse_program_add_library(odbc-bridge)
 # For this reason, we disabling -rdynamic linker flag. But we do it in strange way:
 SET(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "")
 
-add_executable (clickhouse-odbc-bridge odbc-bridge.cpp)
+add_executable(clickhouse-odbc-bridge odbc-bridge.cpp)
+set_target_properties(clickhouse-odbc-bridge PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..)
 
 clickhouse_program_link_split_binary(odbc-bridge)
 
-install (TARGETS clickhouse-odbc-bridge RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
+install(TARGETS clickhouse-odbc-bridge RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
 
-if (ENABLE_TESTS)
-    add_subdirectory (tests)
-endif ()
+if(ENABLE_TESTS)
+    add_subdirectory(tests)
+endif()
diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh
index bc07140abe4..6fb646e038d 100755
--- a/docker/packager/binary/build.sh
+++ b/docker/packager/binary/build.sh
@@ -5,9 +5,9 @@ set -x -e
 mkdir -p build/build_docker
 cd build/build_docker
 ccache -s ||:
+rm -f CMakeCache.txt
 cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DSANITIZE=$SANITIZER $CMAKE_FLAGS
 ninja
 ccache -s ||:
 mv ./dbms/programs/clickhouse* /output
 mv ./dbms/unit_tests_dbms /output
-mv ./dbms/programs/odbc-bridge/clickhouse-odbc-bridge /output

From 02e9cc8f2d90c0412e4c93f42cadce71dac4e761 Mon Sep 17 00:00:00 2001
From: "821008736@qq.com" <root@server50-ke.mars.ljnode.com>
Date: Fri, 26 Apr 2019 17:36:35 +0800
Subject: [PATCH 060/118] fix retention function

---
 .../AggregateFunctionRetention.h                |  1 -
 .../00938_test_retention_function.reference     | 17 +++++++++++++++++
 .../00938_test_retention_function.sql           | 14 ++++++++++++++
 3 files changed, 31 insertions(+), 1 deletion(-)
 create mode 100644 dbms/tests/queries/0_stateless/00938_test_retention_function.reference
 create mode 100644 dbms/tests/queries/0_stateless/00938_test_retention_function.sql

diff --git a/dbms/src/AggregateFunctions/AggregateFunctionRetention.h b/dbms/src/AggregateFunctions/AggregateFunctionRetention.h
index 525a4d848d2..fb35b7bbb0c 100644
--- a/dbms/src/AggregateFunctions/AggregateFunctionRetention.h
+++ b/dbms/src/AggregateFunctions/AggregateFunctionRetention.h
@@ -104,7 +104,6 @@ public:
             if (event)
             {
                 this->data(place).add(i);
-                break;
             }
         }
     }
diff --git a/dbms/tests/queries/0_stateless/00938_test_retention_function.reference b/dbms/tests/queries/0_stateless/00938_test_retention_function.reference
new file mode 100644
index 00000000000..b6a24fd6890
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00938_test_retention_function.reference
@@ -0,0 +1,17 @@
+0	[1,1]
+1	[1,1]
+2	[1,1]
+3	[1,1]
+4	[1,1]
+----------
+0	[1,1,1]
+1	[1,1,1]
+2	[1,1,1]
+3	[1,1,1]
+4	[1,1,1]
+----------
+0	[1,1,1]
+1	[0,0,0]
+2	[1,1,1]
+3	[0,0,0]
+4	[1,1,1]
diff --git a/dbms/tests/queries/0_stateless/00938_test_retention_function.sql b/dbms/tests/queries/0_stateless/00938_test_retention_function.sql
new file mode 100644
index 00000000000..450fdcc7fc5
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00938_test_retention_function.sql
@@ -0,0 +1,14 @@
+DROP TABLE IF EXISTS retention_test;
+CREATE TABLE retention_test(date Date, uid Int32)ENGINE = Memory;
+
+INSERT INTO retention_test SELECT '2018-08-06', number FROM numbers(8);
+INSERT INTO retention_test SELECT '2018-08-07', number FROM numbers(6);
+INSERT INTO retention_test SELECT '2018-08-08', number FROM numbers(7);
+
+SELECT uid, retention(date = '2018-08-06', date = '2018-08-07') AS r FROM retention_test WHERE date IN ('2018-08-06', '2018-08-07') GROUP BY uid ORDER BY uid LIMIT 5;
+SELECT '----------';
+SELECT uid, retention(1, date = '2018-08-06', date = '2018-08-07') AS r FROM retention_test WHERE date IN ('2018-08-06', '2018-08-07') GROUP BY uid ORDER BY uid LIMIT 5;
+SELECT '----------';
+SELECT uid, retention(uid % 2 = 0, date = '2018-08-06', date = '2018-08-07') AS r FROM retention_test WHERE date IN ('2018-08-06', '2018-08-07') GROUP BY uid ORDER BY uid LIMIT 5;
+
+DROP TABLE IF EXISTS retention_test;

From 8b24fed4860dfd6fdf357f3debb01a7c53ee98ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=BC=A0=E9=A3=8E=E5=95=B8?= <z1079911968@163.com>
Date: Fri, 26 Apr 2019 17:51:15 +0800
Subject: [PATCH 061/118] zhdocs: log_family.md and stripelog.md (#5107)

---
 .../en/operations/table_engines/log_family.md |  2 +-
 .../zh/operations/table_engines/log_family.md | 43 ++++++++-
 docs/zh/operations/table_engines/stripelog.md | 87 ++++++++++++++++++-
 3 files changed, 129 insertions(+), 3 deletions(-)
 mode change 120000 => 100644 docs/zh/operations/table_engines/log_family.md
 mode change 120000 => 100644 docs/zh/operations/table_engines/stripelog.md

diff --git a/docs/en/operations/table_engines/log_family.md b/docs/en/operations/table_engines/log_family.md
index 95b17fb173f..ae9e30d3728 100644
--- a/docs/en/operations/table_engines/log_family.md
+++ b/docs/en/operations/table_engines/log_family.md
@@ -1,4 +1,4 @@
-#Log Engine Family
+# Log Engine Family
 
 These engines were developed for scenarios when you need to write many tables with the small amount of data (less than 1 million rows).
 
diff --git a/docs/zh/operations/table_engines/log_family.md b/docs/zh/operations/table_engines/log_family.md
deleted file mode 120000
index 8c5b5f0365b..00000000000
--- a/docs/zh/operations/table_engines/log_family.md
+++ /dev/null
@@ -1 +0,0 @@
-../../../en/operations/table_engines/log_family.md
\ No newline at end of file
diff --git a/docs/zh/operations/table_engines/log_family.md b/docs/zh/operations/table_engines/log_family.md
new file mode 100644
index 00000000000..a1647cd473c
--- /dev/null
+++ b/docs/zh/operations/table_engines/log_family.md
@@ -0,0 +1,42 @@
+# 日志引擎系列 {#table_engines-log_engine_family}
+
+这些引擎是为了需要写入许多小数据量（少于一百万行）的表的场景而开发的。
+
+这系列的引擎有：
+
+- [StripeLog](stripelog.md)
+- [Log](log.md)
+- [TinyLog](tinylog.md)
+
+## 共同属性 {#table_engines-log_engine_family-common_properties}
+
+引擎：
+
+- 数据存储在磁盘上。
+- 写入时将数据追加在文件末尾。
+- 不支持[突变](../../query_language/alter.md#alter-mutations)操作。
+- 不支持索引。
+
+    这意味着 `SELECT` 在范围查询时效率不高。
+    
+- 非原子地写入数据。
+
+    如果某些事情破坏了写操作，例如服务器的异常关闭，你将会得到一张包含了损坏数据的表。
+
+## 差异 {#table_engines-log_engine_family-differences}
+
+`Log` 和 `StripeLog` 引擎支持：
+
+- 并发访问数据的锁。
+    
+    `INSERT` 请求执行过程中表会被锁定，并且其他的读写数据的请求都会等待直到锁定被解除。如果没有写数据的请求，任意数量的读请求都可以并发执行。
+
+- 并行读取数据。
+
+    在读取数据时，ClickHouse 使用多线程。 每个线程处理不同的数据块。
+
+`Log` 引擎为表中的每一列使用不同的文件。`StripeLog` 将所有的数据存储在一个文件中。因此 `StripeLog` 引擎在操作系统中使用更少的描述符，但是 `Log` 引擎提供更高的读性能。
+
+`TingLog` 引擎是该系列中最简单的引擎并且提供了最少的功能和最低的性能。`TingLog` 引擎不支持并行读取和并发数据访问，并将每一列存储在不同的文件中。它比其余两种支持并行读取的引擎的读取速度更慢，并且使用了和 `Log` 引擎同样多的描述符。你可以在简单的低负载的情景下使用它。
+
+[来源文章](https://clickhouse.yandex/docs/en/operations/table_engines/log_family/) <!--hide-->
diff --git a/docs/zh/operations/table_engines/stripelog.md b/docs/zh/operations/table_engines/stripelog.md
deleted file mode 120000
index f6521a41e3e..00000000000
--- a/docs/zh/operations/table_engines/stripelog.md
+++ /dev/null
@@ -1 +0,0 @@
-../../../en/operations/table_engines/stripelog.md
\ No newline at end of file
diff --git a/docs/zh/operations/table_engines/stripelog.md b/docs/zh/operations/table_engines/stripelog.md
new file mode 100644
index 00000000000..43378293f85
--- /dev/null
+++ b/docs/zh/operations/table_engines/stripelog.md
@@ -0,0 +1,86 @@
+# StripeLog {#table_engines-stripelog}
+
+该引擎属于日志引擎系列。请在[日志引擎系列](log_family.md)文章中查看引擎的共同属性和差异。
+
+在你需要写入许多小数据量（小于一百万行）的表的场景下使用这个引擎。
+
+## 建表 {#table_engines-stripelog-creating-a-table}
+
+```
+CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
+(
+    column1_name [type1] [DEFAULT|MATERIALIZED|ALIAS expr1],
+    column2_name [type2] [DEFAULT|MATERIALIZED|ALIAS expr2],
+    ...
+) ENGINE = StripeLog
+```
+
+查看[建表](../../query_language/create.md#create-table-query)请求的详细说明。
+
+## 写数据 {#table_engines-stripelog-writing-the-data}
+
+`StripeLog` 引擎将所有列存储在一个文件中。对每一次 `Insert` 请求，ClickHouse 将数据块追加在表文件的末尾，逐列写入。
+
+ClickHouse 为每张表写入以下文件：
+
+- `data.bin` — 数据文件。
+- `index.mrk` — 带标记的文件。标记包含了已插入的每个数据块中每列的偏移量。
+
+`StripeLog` 引擎不支持 `ALTER UPDATE` 和 `ALTER DELETE` 操作。
+
+## 读数据 {#table_engines-stripelog-reading-the-data}
+
+带标记的文件使得 ClickHouse 可以并行的读取数据。这意味着 `SELECT` 请求返回行的顺序是不可预测的。使用 `ORDER BY` 子句对行进行排序。
+
+## 使用示例 {#table_engines-stripelog-example-of-use}
+
+建表：
+
+```sql
+CREATE TABLE stripe_log_table
+(
+    timestamp DateTime,
+    message_type String,
+    message String
+)
+ENGINE = StripeLog
+```
+
+插入数据：
+
+```sql
+INSERT INTO stripe_log_table VALUES (now(),'REGULAR','The first regular message')
+INSERT INTO stripe_log_table VALUES (now(),'REGULAR','The second regular message'),(now(),'WARNING','The first warning message')
+```
+
+我们使用两次 `INSERT` 请求从而在 `data.bin` 文件中创建两个数据块。
+ 
+ClickHouse 在查询数据时使用多线程。每个线程读取单独的数据块并在完成后独立的返回结果行。这样的结果是，大多数情况下，输出中块的顺序和输入时相应块的顺序是不同的。例如：
+
+```sql
+SELECT * FROM stripe_log_table
+```
+```
+┌───────────timestamp─┬─message_type─┬─message────────────────────┐
+│ 2019-01-18 14:27:32 │ REGULAR      │ The second regular message │
+│ 2019-01-18 14:34:53 │ WARNING      │ The first warning message  │
+└─────────────────────┴──────────────┴────────────────────────────┘
+┌───────────timestamp─┬─message_type─┬─message───────────────────┐
+│ 2019-01-18 14:23:43 │ REGULAR      │ The first regular message │
+└─────────────────────┴──────────────┴───────────────────────────┘
+```
+
+对结果排序（默认增序）：
+
+```sql
+SELECT * FROM stripe_log_table ORDER BY timestamp
+```
+```
+┌───────────timestamp─┬─message_type─┬─message────────────────────┐
+│ 2019-01-18 14:23:43 │ REGULAR      │ The first regular message  │
+│ 2019-01-18 14:27:32 │ REGULAR      │ The second regular message │
+│ 2019-01-18 14:34:53 │ WARNING      │ The first warning message  │
+└─────────────────────┴──────────────┴────────────────────────────┘
+```
+
+[来源文章](https://clickhouse.yandex/docs/en/operations/table_engines/stripelog/) <!--hide-->

From ae76021237704a19dfbd531d4fe564a9960be5c7 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Fri, 26 Apr 2019 12:55:36 +0300
Subject: [PATCH 062/118] Add missing changelogs

---
 CHANGELOG.md    |  44 ++++++++++++++
 CHANGELOG_RU.md | 158 ++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 202 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7fa13dacdf9..6f03184162f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,8 @@
+## ClickHouse release 19.5.3.8, 2019-04-18
+
+### Bug fixes
+* Fixed type of setting `max_partitions_per_insert_block` from boolean to UInt64. [#5028](https://github.com/yandex/ClickHouse/pull/5028) ([Mohammad Hossein Sekhavat](https://github.com/mhsekhavat))
+
 ## ClickHouse release 19.5.2.6, 2019-04-15
 
 ### New Features
@@ -93,6 +98,45 @@
 * Disable usage of `mremap` when compiled with Thread Sanitizer. Surprisingly enough, TSan does not intercept `mremap` (though it does intercept `mmap`, `munmap`) that leads to false positives. Fixed TSan report in stateful tests. [#4859](https://github.com/yandex/ClickHouse/pull/4859) ([alexey-milovidov](https://github.com/alexey-milovidov))
 * Add test checking using format schema via HTTP interface. [#4864](https://github.com/yandex/ClickHouse/pull/4864) ([Vitaly Baranov](https://github.com/vitlibar))
 
+## ClickHouse release 19.4.4.33, 2019-04-17
+
+### Bug Fixes
+
+* Avoid `std::terminate` in case of memory allocation failure. Now `std::bad_alloc` exception is thrown as expected. [#4665](https://github.com/yandex/ClickHouse/pull/4665) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Fixes capnproto reading from buffer. Sometimes files wasn't loaded successfully by HTTP. [#4674](https://github.com/yandex/ClickHouse/pull/4674) ([Vladislav](https://github.com/smirnov-vs))
+* Fix error `Unknown log entry type: 0` after `OPTIMIZE TABLE FINAL` query. [#4683](https://github.com/yandex/ClickHouse/pull/4683) ([Amos Bird](https://github.com/amosbird))
+* Wrong arguments to `hasAny` or `hasAll` functions may lead to segfault. [#4698](https://github.com/yandex/ClickHouse/pull/4698) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Deadlock may happen while executing `DROP DATABASE dictionary` query. [#4701](https://github.com/yandex/ClickHouse/pull/4701) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Fix undefinied behavior in `median` and `quantile` functions. [#4702](https://github.com/yandex/ClickHouse/pull/4702) ([hcz](https://github.com/hczhcz))
+* Fix compression level detection when `network_compression_method` in lowercase. Broken in v19.1. [#4706](https://github.com/yandex/ClickHouse/pull/4706) ([proller](https://github.com/proller))
+* Keep ordinary, `DEFAULT`, `MATERIALIZED` and `ALIAS` columns in a single list (fixes issue [#2867](https://github.com/yandex/ClickHouse/issues/2867)). [#4707](https://github.com/yandex/ClickHouse/pull/4707) ([Alex Zatelepin](https://github.com/ztlpn))
+* Fixed ignorance of `<timezone>UTC</timezone>` setting (fixes issue [#4658](https://github.com/yandex/ClickHouse/issues/4658)). [#4718](https://github.com/yandex/ClickHouse/pull/4718) ([proller](https://github.com/proller))
+* Fix `histogram` function behaviour with `Distributed` tables. [#4741](https://github.com/yandex/ClickHouse/pull/4741) ([olegkv](https://github.com/olegkv))
+* Fixed tsan report `destroy of a locked mutex`. [#4742](https://github.com/yandex/ClickHouse/pull/4742) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Fixed TSan report on shutdown due to race condition in system logs usage. Fixed potential use-after-free on shutdown when part_log is enabled. [#4758](https://github.com/yandex/ClickHouse/pull/4758) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Fix recheck parts in `ReplicatedMergeTreeAlterThread` in case of error. [#4772](https://github.com/yandex/ClickHouse/pull/4772) ([Nikolai Kochetov](https://github.com/KochetovNicolai))
+* Arithmetic operations on intermediate aggregate function states were not working for constant arguments (such as subquery results). [#4776](https://github.com/yandex/ClickHouse/pull/4776) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Always backquote column names in metadata. Otherwise it's impossible to create a table with column named `index` (server won't restart due to malformed `ATTACH` query in metadata). [#4782](https://github.com/yandex/ClickHouse/pull/4782) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Fix crash in `ALTER ... MODIFY ORDER BY` on `Distributed` table. [#4790](https://github.com/yandex/ClickHouse/pull/4790) ([TCeason](https://github.com/TCeason))
+* Fix segfault in `JOIN ON` with enabled `enable_optimize_predicate_expression`. [#4794](https://github.com/yandex/ClickHouse/pull/4794) ([Winter Zhang](https://github.com/zhang2014))
+* Fix bug with adding an extraneous row after consuming a protobuf message from Kafka. [#4808](https://github.com/yandex/ClickHouse/pull/4808) ([Vitaly Baranov](https://github.com/vitlibar))
+* Fix segmentation fault in `clickhouse-copier`. [#4835](https://github.com/yandex/ClickHouse/pull/4835) ([proller](https://github.com/proller))
+* Fixed race condition in `SELECT` from `system.tables` if the table is renamed or altered concurrently. [#4836](https://github.com/yandex/ClickHouse/pull/4836) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Fixed data race when fetching data part that is already obsolete. [#4839](https://github.com/yandex/ClickHouse/pull/4839) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Fixed rare data race that can happen during `RENAME` table of MergeTree family. [#4844](https://github.com/yandex/ClickHouse/pull/4844) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Fixed segmentation fault in function `arrayIntersect`. Segmentation fault could happen if function was called with mixed constant and ordinary arguments. [#4847](https://github.com/yandex/ClickHouse/pull/4847) ([Lixiang Qian](https://github.com/fancyqlx))
+* Fixed reading from `Array(LowCardinality)` column in rare case when column contained a long sequence of empty arrays. [#4850](https://github.com/yandex/ClickHouse/pull/4850) ([Nikolai Kochetov](https://github.com/KochetovNicolai))
+* Fix `No message received` exception while fetching parts between replicas. [#4856](https://github.com/yandex/ClickHouse/pull/4856) ([alesapin](https://github.com/alesapin))
+* Fixed `arrayIntersect` function wrong result in case of several repeated values in single array. [#4871](https://github.com/yandex/ClickHouse/pull/4871) ([Nikolai Kochetov](https://github.com/KochetovNicolai))
+* Fix a race condition during concurrent `ALTER COLUMN` queries that could lead to a server crash (fixes issue [#3421](https://github.com/yandex/ClickHouse/issues/3421)). [#4592](https://github.com/yandex/ClickHouse/pull/4592) ([Alex Zatelepin](https://github.com/ztlpn))
+* Fix parameter deduction in `ALTER MODIFY` of column `CODEC` when column type is not specified. [#4883](https://github.com/yandex/ClickHouse/pull/4883) ([alesapin](https://github.com/alesapin))
+* Functions `cutQueryStringAndFragment()` and `queryStringAndFragment()` now works correctly when `URL` contains a fragment and no query. [#4894](https://github.com/yandex/ClickHouse/pull/4894) ([Vitaly Baranov](https://github.com/vitlibar))
+* Fix rare bug when setting `min_bytes_to_use_direct_io` is greater than zero, which occures when thread have to seek backward in column file. [#4897](https://github.com/yandex/ClickHouse/pull/4897) ([alesapin](https://github.com/alesapin))
+* Fix wrong argument types for aggregate functions with `LowCardinality` arguments (fixes issue [#4919](https://github.com/yandex/ClickHouse/issues/4919)). [#4922](https://github.com/yandex/ClickHouse/pull/4922) ([Nikolai Kochetov](https://github.com/KochetovNicolai))
+* Function `toISOWeek` result for year 1970. [#4988](https://github.com/yandex/ClickHouse/pull/4988) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Fix `DROP`, `TRUNCATE` and `OPTIMIZE` queries duplication, when executed on `ON CLUSTER` for `ReplicatedMergeTree*` tables family. [#4991](https://github.com/yandex/ClickHouse/pull/4991) ([alesapin](https://github.com/alesapin))
+
+
 ## ClickHouse release 19.4.3.11, 2019-04-02
 
 ### Bug Fixes
diff --git a/CHANGELOG_RU.md b/CHANGELOG_RU.md
index 743edeb7eeb..89946899f22 100644
--- a/CHANGELOG_RU.md
+++ b/CHANGELOG_RU.md
@@ -1,3 +1,161 @@
+## ClickHouse release 19.5.3.8, 2019-04-18
+
+### Исправления ошибок
+* Исправлен тип настройки `max_partitions_per_insert_block` с булевого на UInt64. [#5028](https://github.com/yandex/ClickHouse/pull/5028) ([Mohammad Hossein Sekhavat](https://github.com/mhsekhavat))
+
+## ClickHouse release 19.5.2.6, 2019-04-15
+
+### Новые возможности
+
+* [Hyperscan](https://github.com/intel/hyperscan) Добавлены функции для работы с несколькими регулярными выражениями. (`multiMatchAny`, `multiMatchAnyIndex`, `multiFuzzyMatchAny`, `multiFuzzyMatchAnyIndex`). [#4780](https://github.com/yandex/ClickHouse/pull/4780), [#4841](https://github.com/yandex/ClickHouse/pull/4841) ([Danila Kutenin](https://github.com/danlark1))
+* Добавлена функция `multiSearchFirstPosition`. [#4780](https://github.com/yandex/ClickHouse/pull/4780) ([Danila Kutenin](https://github.com/danlark1))
+* Реализована возможность указания построчного ограничения доступа к таблицам. [#4792](https://github.com/yandex/ClickHouse/pull/4792) ([Ivan](https://github.com/abyss7))
+* Добавлен новый тип вторичного индекса на базе фильтра Блума (используется в функциях `equal`, `in` и `like`). [#4499](https://github.com/yandex/ClickHouse/pull/4499) ([Nikita Vasilev](https://github.com/nikvas0))
+* Добавлен `ASOF JOIN` которые позволяет джойнить строки по наиболее близкому известному значению. [#4774](https://github.com/yandex/ClickHouse/pull/4774) [#4867](https://github.com/yandex/ClickHouse/pull/4867) [#4863](https://github.com/yandex/ClickHouse/pull/4863) [#4875](https://github.com/yandex/ClickHouse/pull/4875) ([Martijn Bakker](https://github.com/Gladdy), [Artem Zuikov](https://github.com/4ertus2))
+* Теперь запрос `COMMA JOIN` переписывается `CROSS JOIN`. И затем оба переписываются в `INNER JOIN`, если это возможно. [#4661](https://github.com/yandex/ClickHouse/pull/4661) ([Artem Zuikov](https://github.com/4ertus2))
+
+### Улучшения
+
+* Функции `topK` и `topKWeighted` теперь поддерживают произвольный `loadFactor` (исправляет issue [#4252](https://github.com/yandex/ClickHouse/issues/4252)). [#4634](https://github.com/yandex/ClickHouse/pull/4634) ([Kirill Danshin](https://github.com/kirillDanshin))
+* Добавлена возможность использования настройки `parallel_replicas_count > 1` для таблиц без семплирования (ранее настройка просто игнорировалась). [#4637](https://github.com/yandex/ClickHouse/pull/4637) ([Alexey Elymanov](https://github.com/digitalist))
+* Поддержан запрос `CREATE OR REPLACE VIEW`. Позволяет создать `VIEW` или изменить запрос в одном выражении. [#4654](https://github.com/yandex/ClickHouse/pull/4654) ([Boris Granveaud](https://github.com/bgranvea))
+* Движок таблиц `Buffer` теперь поддерживает `PREWHERE`. [#4671](https://github.com/yandex/ClickHouse/pull/4671) ([Yangkuan Liu](https://github.com/LiuYangkuan))
+* Теперь реплицируемые таблицы могу стартовать в `readonly` режиме даже при отсутствии zookeeper. [#4691](https://github.com/yandex/ClickHouse/pull/4691) ([alesapin](https://github.com/alesapin))
+* Исправлено мигание прогресс-бара в clickhouse-client. Проблема была наиболее заметна при использовании `FORMAT Null` в потоковых запросах. [#4811](https://github.com/yandex/ClickHouse/pull/4811) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Добавлена возможность отключения функций, использующих библиотеку `hyperscan`, для пользователей, чтобы ограничить возможное неконтролируемое потребление ресурсов. [#4816](https://github.com/yandex/ClickHouse/pull/4816) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Добавлено логирование номера версии во все исключения. [#4824](https://github.com/yandex/ClickHouse/pull/4824) ([proller](https://github.com/proller))
+* Добавлено ограничение на размер строк и количество параметров в функции `multiMatch`. Теперь они принимают строки умещающиеся в `unsigned int`. [#4834](https://github.com/yandex/ClickHouse/pull/4834) ([Danila Kutenin](https://github.com/danlark1))
+* Улучшено использование памяти и обработка ошибок в Hyperscan. [#4866](https://github.com/yandex/ClickHouse/pull/4866) ([Danila Kutenin](https://github.com/danlark1))
+* Теперь системная таблица `system.graphite_detentions` заполняется из конфигурационного файла для таблиц семейства `*GraphiteMergeTree`. [#4584](https://github.com/yandex/ClickHouse/pull/4584) ([Mikhail f. Shiryaev](https://github.com/Felixoid))
+* Функция `trigramDistance` переименована в функцию `ngramDistance`. Добавлено несколько функций с `CaseInsensitive` и `UTF`. [#4602](https://github.com/yandex/ClickHouse/pull/4602) ([Danila Kutenin](https://github.com/danlark1))
+* Улучшено вычисление вторичных индексов. [#4640](https://github.com/yandex/ClickHouse/pull/4640) ([Nikita Vasilev](https://github.com/nikvas0))
+
+### Исправления ошибок
+
+* В случае невозможности выделить память вместо вызова `std::terminate` бросается исключение `std::bad_alloc`. [#4665](https://github.com/yandex/ClickHouse/pull/4665) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Исправлены ошибки чтения capnproto из буфера. Иногда файлы не загружались по HTTP. [#4674](https://github.com/yandex/ClickHouse/pull/4674) ([Vladislav](https://github.com/smirnov-vs))
+* Исправлена ошибка `Unknown log entry type: 0` после запроса `OPTIMIZE TABLE FINAL`. [#4683](https://github.com/yandex/ClickHouse/pull/4683) ([Amos Bird](https://github.com/amosbird))
+* При передаче неправильных аргументов в `hasAny` и `hasAll` могла происходить ошибка сегментирования. [#4698](https://github.com/yandex/ClickHouse/pull/4698) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Исправлен дедлок, который мог происходить при запросе `DROP DATABASE dictionary`. [#4701](https://github.com/yandex/ClickHouse/pull/4701) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Исправлено неопределенное поведение в функциях `median` и `quantile`. [#4702](https://github.com/yandex/ClickHouse/pull/4702) ([hcz](https://github.com/hczhcz))
+* Исправлено определение уровня сжатия при указании настройки `network_compression_method` в нижнем регистре. Было сломано в v19.1. [#4706](https://github.com/yandex/ClickHouse/pull/4706) ([proller](https://github.com/proller))
+* Теперь обычные колонки, а также колонки `DEFAULT`, `MATERIALIZED` и `ALIAS` хранятся в одном списке (исправляет проблему [#2867](https://github.com/yandex/ClickHouse/issues/2867)). [#4707](https://github.com/yandex/ClickHouse/pull/4707) ([Alex Zatelepin](https://github.com/ztlpn))
+* Настройка `<timezone>UTC</timezone>` больше не игнорируется (исправляет проблему [#4658](https://github.com/yandex/ClickHouse/issues/4658)). [#4718](https://github.com/yandex/ClickHouse/pull/4718) ([proller](https://github.com/proller))
+* Исправлено поведение функции `histogram` с `Distributed` таблицами. [#4741](https://github.com/yandex/ClickHouse/pull/4741) ([olegkv](https://github.com/olegkv))
+* Исправлено срабатывание thread-санитайзера с ошибкой `destroy of a locked mutex`. [#4742](https://github.com/yandex/ClickHouse/pull/4742) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Исправлено срабатывание thread-санитайзера при завершении сервера, вызванное гонкой при использовании системных логов. Также исправлена потенциальная ошибка use-after-free при завершении сервера в котором был включен `part_log`. [#4758](https://github.com/yandex/ClickHouse/pull/4758) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Исправлена перепроверка кусков в `ReplicatedMergeTreeAlterThread` при появлении ошибок. [#4772](https://github.com/yandex/ClickHouse/pull/4772) ([Nikolai Kochetov](https://github.com/KochetovNicolai))
+* Исправлена работа арифметических операций с промежуточными состояниями агрегатных функций для константных аргументов (таких как результаты подзапросов). [#4776](https://github.com/yandex/ClickHouse/pull/4776) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Теперь имена колонок всегда экранируются в файлах с метаинформацией. В противном случае было невозможно создать таблицу с колонкой с именем `index`. [#4782](https://github.com/yandex/ClickHouse/pull/4782) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Исправлено падение в запросе `ALTER ... MODIFY ORDER BY` к `Distributed` таблице. [#4790](https://github.com/yandex/ClickHouse/pull/4790) ([TCeason](https://github.com/TCeason))
+* Исправлена ошибка сегментирования при запросах с `JOIN ON` и включенной настройкой `enable_optimize_predicate_expression`. [#4794](https://github.com/yandex/ClickHouse/pull/4794) ([Winter Zhang](https://github.com/zhang2014))
+* Исправлено добавление лишней строки после чтения protobuf-сообщения из таблицы с движком `Kafka`. [#4808](https://github.com/yandex/ClickHouse/pull/4808) ([Vitaly Baranov](https://github.com/vitlibar))
+* Исправлено падение при запросе с `JOIN ON` с не `nullable` и nullable колонкой. Также исправлено поведение при появлении `NULLs` среди ключей справа в`ANY JOIN` + `join_use_nulls`. [#4815](https://github.com/yandex/ClickHouse/pull/4815) ([Artem Zuikov](https://github.com/4ertus2))
+* Исправлена ошибка сегментирования в `clickhouse-copier`. [#4835](https://github.com/yandex/ClickHouse/pull/4835) ([proller](https://github.com/proller))
+* Исправлена гонка при `SELECT` запросе из `system.tables` если таблица была конкурентно переименована или к ней был применен `ALTER` запрос. [#4836](https://github.com/yandex/ClickHouse/pull/4836) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Исправлена гонка при скачивании куска, который уже является устаревшим. [#4839](https://github.com/yandex/ClickHouse/pull/4839) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Исправлена редкая гонка при `RENAME` запросах к таблицам семейства MergeTree. [#4844](https://github.com/yandex/ClickHouse/pull/4844) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Исправлена ошибка сегментирования в функции `arrayIntersect`. Ошибка возникала при вызове функции с константными и не константными аргументами. [#4847](https://github.com/yandex/ClickHouse/pull/4847) ([Lixiang Qian](https://github.com/fancyqlx))
+* Исправлена редкая ошибка при чтении из колонки типа `Array(LowCardinality)`, которая возникала, если в колонке содержалось большее количество подряд идущих пустых массивов. [#4850](https://github.com/yandex/ClickHouse/pull/4850) ([Nikolai Kochetov](https://github.com/KochetovNicolai))
+* Исправлено паление в запроса с `FULL/RIGHT JOIN` когда объединение происходило по nullable и не nullable колонке. [#4855](https://github.com/yandex/ClickHouse/pull/4855) ([Artem Zuikov](https://github.com/4ertus2))
+* Исправлена ошибка `No message received`, возникавшая при скачивании кусков между репликами. [#4856](https://github.com/yandex/ClickHouse/pull/4856) ([alesapin](https://github.com/alesapin))
+* Исправлена ошибка в функции `arrayIntersect` приводившая к неправильным результатам в случае нескольких повторяющихся значений в массиве. [#4871](https://github.com/yandex/ClickHouse/pull/4871) ([Nikolai Kochetov](https://github.com/KochetovNicolai))
+* Исправлена гонка при конкурентных `ALTER COLUMN` запросах, которая могла приводить к падению сервера (исправляет проблему [#3421](https://github.com/yandex/ClickHouse/issues/3421)). [#4592](https://github.com/yandex/ClickHouse/pull/4592) ([Alex Zatelepin](https://github.com/ztlpn))
+* Исправлен некорректный результат в `FULL/RIGHT JOIN` запросах с константной колонкой. [#4723](https://github.com/yandex/ClickHouse/pull/4723) ([Artem Zuikov](https://github.com/4ertus2))
+* Исправлено появление дубликатов в `GLOBAL JOIN` со звездочкой. [#4705](https://github.com/yandex/ClickHouse/pull/4705) ([Artem Zuikov](https://github.com/4ertus2))
+* Исправлено определение параметров кодеков в запросах `ALTER MODIFY`, если тип колонки не был указан. [#4883](https://github.com/yandex/ClickHouse/pull/4883) ([alesapin](https://github.com/alesapin))
+* Функции `cutQueryStringAndFragment()` и `queryStringAndFragment()` теперь работают корректно, когда `URL` содержит фрагмент, но не содержит запроса. [#4894](https://github.com/yandex/ClickHouse/pull/4894) ([Vitaly Baranov](https://github.com/vitlibar))
+* Исправлена редкая ошибка, возникавшая при установке настройки `min_bytes_to_use_direct_io` больше нуля. Она возникла при необходимости сдвинутся в файле, который уже прочитан до конца. [#4897](https://github.com/yandex/ClickHouse/pull/4897) ([alesapin](https://github.com/alesapin))
+* Исправлено неправильное определение типов аргументов для агрегатных функций с `LowCardinality` аргументами (исправляет [#4919](https://github.com/yandex/ClickHouse/issues/4919)). [#4922](https://github.com/yandex/ClickHouse/pull/4922) ([Nikolai Kochetov](https://github.com/KochetovNicolai))
+* Исправлена неверная квалификация имён в `GLOBAL JOIN`. [#4969](https://github.com/yandex/ClickHouse/pull/4969) ([Artem Zuikov](https://github.com/4ertus2))
+* Исправлен результат функции `toISOWeek` для 1970 года. [#4988](https://github.com/yandex/ClickHouse/pull/4988) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Исправлено дублирование `DROP`, `TRUNCATE` и `OPTIMIZE` запросов, когда они выполнялись `ON CLUSTER` для семейства таблиц `ReplicatedMergeTree*`. [#4991](https://github.com/yandex/ClickHouse/pull/4991) ([alesapin](https://github.com/alesapin))
+
+### Обратно несовместимые изменения
+
+* Настройка `insert_sample_with_metadata` переименована в `input_format_defaults_for_omitted_fields`. [#4771](https://github.com/yandex/ClickHouse/pull/4771) ([Artem Zuikov](https://github.com/4ertus2))
+* Добавлена настройка `max_partitions_per_insert_block` (со значением по умолчанию 100). Если вставляемый блок содержит большое количество партиций, то бросается исключение. Лимит можно убрать выставив настройку в 0 (не рекомендуется). [#4845](https://github.com/yandex/ClickHouse/pull/4845) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Функции мультипоиска были переименованы (`multiPosition` в `multiSearchAllPositions`, `multiSearch` в `multiSearchAny`, `firstMatch` в `multiSearchFirstIndex`). [#4780](https://github.com/yandex/ClickHouse/pull/4780) ([Danila Kutenin](https://github.com/danlark1))
+
+### Улучшение производительности
+
+* Оптимизирован поиска с помощью алгоритма Volnitsky с помощью инлайнинга. Это дает около 5-10% улучшения производительности поиска для запросов ищущих множество слов или много одинаковых биграмм. [#4862](https://github.com/yandex/ClickHouse/pull/4862) ([Danila Kutenin](https://github.com/danlark1))
+* Исправлено снижение производительности при выставлении настройки `use_uncompressed_cache` больше нуля для запросов, данные которых целиком лежат в кеше. [#4913](https://github.com/yandex/ClickHouse/pull/4913) ([alesapin](https://github.com/alesapin))
+
+
+### Улучшения сборки/тестирования/пакетирования
+
+* Более строгие настройки для debug-сборок: более гранулярные маппинги памяти и использование ASLR; добавлена защита памяти для кеша засечек и индекса. Это позволяет найти больше ошибок порчи памяти, которые не обнаруживают address-санитайзер и thread-санитайзер. [#4632](https://github.com/yandex/ClickHouse/pull/4632) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Добавлены настройки `ENABLE_PROTOBUF`, `ENABLE_PARQUET` и `ENABLE_BROTLI` которые позволяют отключить соответствующие компоненты. [#4669](https://github.com/yandex/ClickHouse/pull/4669) ([Silviu Caragea](https://github.com/silviucpp))
+* Теперь при зависании запросов во время работы тестов будет показан список запросов и стек-трейсы всех потоков. [#4675](https://github.com/yandex/ClickHouse/pull/4675) ([alesapin](https://github.com/alesapin))
+* Добавлены ретраи при ошибке `Connection loss` в `clickhouse-test`. [#4682](https://github.com/yandex/ClickHouse/pull/4682) ([alesapin](https://github.com/alesapin))
+* Добавлена возможность сборки под FreeBSD в `packager`-скрипт. [#4712](https://github.com/yandex/ClickHouse/pull/4712) [#4748](https://github.com/yandex/ClickHouse/pull/4748) ([alesapin](https://github.com/alesapin))
+* Теперь при установке предлагается установить пароль для пользователя `'default'`. [#4725](https://github.com/yandex/ClickHouse/pull/4725) ([proller](https://github.com/proller))
+* Убраны предупреждения из библиотеки `rdkafka` при сборке. [#4740](https://github.com/yandex/ClickHouse/pull/4740) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Добавлена возможность сборки без поддержки ssl. [#4750](https://github.com/yandex/ClickHouse/pull/4750) ([proller](https://github.com/proller))
+* Добавлена возможность запускать докер-образ с clickhouse-server из под любого пользователя. [#4753](https://github.com/yandex/ClickHouse/pull/4753) ([Mikhail f. Shiryaev](https://github.com/Felixoid))
+* Boost обновлен до 1.69. [#4793](https://github.com/yandex/ClickHouse/pull/4793) ([proller](https://github.com/proller))
+* Отключено использование `mremap` при сборке с thread-санитайзером, что приводило к ложным срабатываниям. Исправлены ошибки thread-санитайзера в stateful-тестах. [#4859](https://github.com/yandex/ClickHouse/pull/4859) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Добавлен тест проверяющий использование схемы форматов для HTTP-интерфейса. [#4864](https://github.com/yandex/ClickHouse/pull/4864) ([Vitaly Baranov](https://github.com/vitlibar))
+
+## ClickHouse release 19.4.4.33, 2019-04-17
+
+### Исправление ошибок
+* В случае невозможности выделить память вместо вызова `std::terminate` бросается исключение `std::bad_alloc`. [#4665](https://github.com/yandex/ClickHouse/pull/4665) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Исправлены ошибки чтения capnproto из буфера. Иногда файлы не загружались по HTTP. [#4674](https://github.com/yandex/ClickHouse/pull/4674) ([Vladislav](https://github.com/smirnov-vs))
+* Исправлена ошибка `Unknown log entry type: 0` после запроса `OPTIMIZE TABLE FINAL`. [#4683](https://github.com/yandex/ClickHouse/pull/4683) ([Amos Bird](https://github.com/amosbird))
+* При передаче неправильных аргументов в `hasAny` и `hasAll` могла происходить ошибка сегментирования. [#4698](https://github.com/yandex/ClickHouse/pull/4698) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Исправлен дедлок, который мог происходить при запросе `DROP DATABASE dictionary`. [#4701](https://github.com/yandex/ClickHouse/pull/4701) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Исправлено неопределенное поведение в функциях `median` и `quantile`. [#4702](https://github.com/yandex/ClickHouse/pull/4702) ([hcz](https://github.com/hczhcz))
+* Исправлено определение уровня сжатия при указании настройки `network_compression_method` в нижнем регистре. Было сломано в v19.1. [#4706](https://github.com/yandex/ClickHouse/pull/4706) ([proller](https://github.com/proller))
+* Теперь обычные колонки, а также колонки `DEFAULT`, `MATERIALIZED` и `ALIAS` хранятся в одном списке (исправляет проблему [#2867](https://github.com/yandex/ClickHouse/issues/2867)). [#4707](https://github.com/yandex/ClickHouse/pull/4707) ([Alex Zatelepin](https://github.com/ztlpn))
+* Настройка `<timezone>UTC</timezone>` больше не игнорируется (исправляет проблему [#4658](https://github.com/yandex/ClickHouse/issues/4658)). [#4718](https://github.com/yandex/ClickHouse/pull/4718) ([proller](https://github.com/proller))
+* Исправлено поведение функции `histogram` с `Distributed` таблицами. [#4741](https://github.com/yandex/ClickHouse/pull/4741) ([olegkv](https://github.com/olegkv))
+* Исправлено срабатывание thread-санитайзера с ошибкой `destroy of a locked mutex`. [#4742](https://github.com/yandex/ClickHouse/pull/4742) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Исправлено срабатывание thread-санитайзера при завершении сервера, вызванное гонкой при использовании системных логов. Также исправлена потенциальная ошибка use-after-free при завершении сервера в котором был включен `part_log`. [#4758](https://github.com/yandex/ClickHouse/pull/4758) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Исправлена перепроверка кусков в `ReplicatedMergeTreeAlterThread` при появлении ошибок. [#4772](https://github.com/yandex/ClickHouse/pull/4772) ([Nikolai Kochetov](https://github.com/KochetovNicolai))
+* Исправлена работа арифметических операций с промежуточными состояниями агрегатных функций для константных аргументов (таких как результаты подзапросов). [#4776](https://github.com/yandex/ClickHouse/pull/4776) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Теперь имена колонок всегда экранируются в файлах с метаинформацией. В противном случае было невозможно создать таблицу с колонкой с именем `index`. [#4782](https://github.com/yandex/ClickHouse/pull/4782) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Исправлено падение в запросе `ALTER ... MODIFY ORDER BY` к `Distributed` таблице. [#4790](https://github.com/yandex/ClickHouse/pull/4790) ([TCeason](https://github.com/TCeason))
+* Исправлена ошибка сегментирования при запросах с `JOIN ON` и включенной настройкой `enable_optimize_predicate_expression`. [#4794](https://github.com/yandex/ClickHouse/pull/4794) ([Winter Zhang](https://github.com/zhang2014))
+* Исправлено добавление лишней строки после чтения protobuf-сообщения из таблицы с движком `Kafka`. [#4808](https://github.com/yandex/ClickHouse/pull/4808) ([Vitaly Baranov](https://github.com/vitlibar))
+* Исправлена ошибка сегментирования в `clickhouse-copier`. [#4835](https://github.com/yandex/ClickHouse/pull/4835) ([proller](https://github.com/proller))
+* Исправлена гонка при `SELECT` запросе из `system.tables` если таблица была конкурентно переименована или к ней был применен `ALTER` запрос. [#4836](https://github.com/yandex/ClickHouse/pull/4836) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Исправлена гонка при скачивании куска, который уже является устаревшим. [#4839](https://github.com/yandex/ClickHouse/pull/4839) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Исправлена редкая гонка при `RENAME` запросах к таблицам семейства MergeTree. [#4844](https://github.com/yandex/ClickHouse/pull/4844) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Исправлена ошибка сегментирования в функции `arrayIntersect`. Ошибка возникала при вызове функции с константными и не константными аргументами. [#4847](https://github.com/yandex/ClickHouse/pull/4847) ([Lixiang Qian](https://github.com/fancyqlx))
+* Исправлена редкая ошибка при чтении из колонки типа `Array(LowCardinality)`, которая возникала, если в колонке содержалось большее количество подряд идущих пустых массивов. [#4850](https://github.com/yandex/ClickHouse/pull/4850) ([Nikolai Kochetov](https://github.com/KochetovNicolai))
+* Исправлена ошибка `No message received`, возникавшая при скачивании кусков между репликами. [#4856](https://github.com/yandex/ClickHouse/pull/4856) ([alesapin](https://github.com/alesapin))
+* Исправлена ошибка в функции `arrayIntersect` приводившая к неправильным результатам в случае нескольких повторяющихся значений в массиве. [#4871](https://github.com/yandex/ClickHouse/pull/4871) ([Nikolai Kochetov](https://github.com/KochetovNicolai))
+* Исправлена гонка при конкурентных `ALTER COLUMN` запросах, которая могла приводить к падению сервера (исправляет проблему [#3421](https://github.com/yandex/ClickHouse/issues/3421)). [#4592](https://github.com/yandex/ClickHouse/pull/4592) ([Alex Zatelepin](https://github.com/ztlpn))
+* Исправлено определение параметров кодеков в запросах `ALTER MODIFY`, если тип колонки не был указан. [#4883](https://github.com/yandex/ClickHouse/pull/4883) ([alesapin](https://github.com/alesapin))
+* Функции `cutQueryStringAndFragment()` и `queryStringAndFragment()` теперь работают корректно, когда `URL` содержит фрагмент, но не содержит запроса. [#4894](https://github.com/yandex/ClickHouse/pull/4894) ([Vitaly Baranov](https://github.com/vitlibar))
+* Исправлена редкая ошибка, возникавшая при установке настройки `min_bytes_to_use_direct_io` больше нуля. Она возникла при необходимости сдвинутся в файле, который уже прочитан до конца. [#4897](https://github.com/yandex/ClickHouse/pull/4897) ([alesapin](https://github.com/alesapin))
+* Исправлено неправильное определение типов аргументов для агрегатных функций с `LowCardinality` аргументами (исправляет [#4919](https://github.com/yandex/ClickHouse/issues/4919)). [#4922](https://github.com/yandex/ClickHouse/pull/4922) ([Nikolai Kochetov](https://github.com/KochetovNicolai))
+* Исправлен результат функции `toISOWeek` для 1970 года. [#4988](https://github.com/yandex/ClickHouse/pull/4988) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Исправлено дублирование `DROP`, `TRUNCATE` и `OPTIMIZE` запросов, когда они выполнялись `ON CLUSTER` для семейства таблиц `ReplicatedMergeTree*`. [#4991](https://github.com/yandex/ClickHouse/pull/4991) ([alesapin](https://github.com/alesapin))
+
+## ClickHouse release 19.4.3.11, 2019-04-02
+
+### Исправление ошибок
+
+* Исправлено паление в запроса с `FULL/RIGHT JOIN` когда объединение происходило по nullable и не nullable колонке. [#4855](https://github.com/yandex/ClickHouse/pull/4855) ([Artem Zuikov](https://github.com/4ertus2))
+* Исправлена ошибка сегментирования в `clickhouse-copier`. [#4835](https://github.com/yandex/ClickHouse/pull/4835) ([proller](https://github.com/proller))
+
+### Улучшения сборки/тестирования/пакетирования
+
+* Добавлена возможность запускать докер-образ с clickhouse-server из под любого пользователя. [#4753](https://github.com/yandex/ClickHouse/pull/4753) ([Mikhail f. Shiryaev](https://github.com/Felixoid))
+
+## ClickHouse release 19.4.2.7, 2019-03-30
+
+### Исправление ошибок
+* Исправлена редкая ошибка при чтении из колонки типа `Array(LowCardinality)`, которая возникала, если в колонке содержалось большее количество подряд идущих пустых массивов. [#4850](https://github.com/yandex/ClickHouse/pull/4850) ([Nikolai Kochetov](https://github.com/KochetovNicolai))
+
+## ClickHouse release 19.4.1.3, 2019-03-19
+
+### Исправление ошибок
+* Исправлено поведение удаленных запросов, которые одновременно содержали `LIMIT BY` и `LIMIT`. Раньше для таких запросов `LIMIT` мог быть выполнен до `LIMIT BY`, что приводило к перефильтрации. [#4708](https://github.com/yandex/ClickHouse/pull/4708) ([Constantin S. Pan](https://github.com/kvap))
+
 ## ClickHouse release 19.4.0.49, 2019-03-09
 
 ### Новые возможности

From 76d40f6dd85b190147eaa12429ad8dd6577abc72 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Fri, 26 Apr 2019 13:05:43 +0300
Subject: [PATCH 063/118] Better naming

---
 CHANGELOG_RU.md | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG_RU.md b/CHANGELOG_RU.md
index 89946899f22..c0547d535e8 100644
--- a/CHANGELOG_RU.md
+++ b/CHANGELOG_RU.md
@@ -39,8 +39,8 @@
 * Исправлен дедлок, который мог происходить при запросе `DROP DATABASE dictionary`. [#4701](https://github.com/yandex/ClickHouse/pull/4701) ([alexey-milovidov](https://github.com/alexey-milovidov))
 * Исправлено неопределенное поведение в функциях `median` и `quantile`. [#4702](https://github.com/yandex/ClickHouse/pull/4702) ([hcz](https://github.com/hczhcz))
 * Исправлено определение уровня сжатия при указании настройки `network_compression_method` в нижнем регистре. Было сломано в v19.1. [#4706](https://github.com/yandex/ClickHouse/pull/4706) ([proller](https://github.com/proller))
-* Теперь обычные колонки, а также колонки `DEFAULT`, `MATERIALIZED` и `ALIAS` хранятся в одном списке (исправляет проблему [#2867](https://github.com/yandex/ClickHouse/issues/2867)). [#4707](https://github.com/yandex/ClickHouse/pull/4707) ([Alex Zatelepin](https://github.com/ztlpn))
-* Настройка `<timezone>UTC</timezone>` больше не игнорируется (исправляет проблему [#4658](https://github.com/yandex/ClickHouse/issues/4658)). [#4718](https://github.com/yandex/ClickHouse/pull/4718) ([proller](https://github.com/proller))
+* Теперь обычные колонки, а также колонки `DEFAULT`, `MATERIALIZED` и `ALIAS` хранятся в одном списке (исправляет issue [#2867](https://github.com/yandex/ClickHouse/issues/2867)). [#4707](https://github.com/yandex/ClickHouse/pull/4707) ([Alex Zatelepin](https://github.com/ztlpn))
+* Настройка `<timezone>UTC</timezone>` больше не игнорируется (исправляет issue [#4658](https://github.com/yandex/ClickHouse/issues/4658)). [#4718](https://github.com/yandex/ClickHouse/pull/4718) ([proller](https://github.com/proller))
 * Исправлено поведение функции `histogram` с `Distributed` таблицами. [#4741](https://github.com/yandex/ClickHouse/pull/4741) ([olegkv](https://github.com/olegkv))
 * Исправлено срабатывание thread-санитайзера с ошибкой `destroy of a locked mutex`. [#4742](https://github.com/yandex/ClickHouse/pull/4742) ([alexey-milovidov](https://github.com/alexey-milovidov))
 * Исправлено срабатывание thread-санитайзера при завершении сервера, вызванное гонкой при использовании системных логов. Также исправлена потенциальная ошибка use-after-free при завершении сервера в котором был включен `part_log`. [#4758](https://github.com/yandex/ClickHouse/pull/4758) ([alexey-milovidov](https://github.com/alexey-milovidov))
@@ -60,7 +60,7 @@
 * Исправлено паление в запроса с `FULL/RIGHT JOIN` когда объединение происходило по nullable и не nullable колонке. [#4855](https://github.com/yandex/ClickHouse/pull/4855) ([Artem Zuikov](https://github.com/4ertus2))
 * Исправлена ошибка `No message received`, возникавшая при скачивании кусков между репликами. [#4856](https://github.com/yandex/ClickHouse/pull/4856) ([alesapin](https://github.com/alesapin))
 * Исправлена ошибка в функции `arrayIntersect` приводившая к неправильным результатам в случае нескольких повторяющихся значений в массиве. [#4871](https://github.com/yandex/ClickHouse/pull/4871) ([Nikolai Kochetov](https://github.com/KochetovNicolai))
-* Исправлена гонка при конкурентных `ALTER COLUMN` запросах, которая могла приводить к падению сервера (исправляет проблему [#3421](https://github.com/yandex/ClickHouse/issues/3421)). [#4592](https://github.com/yandex/ClickHouse/pull/4592) ([Alex Zatelepin](https://github.com/ztlpn))
+* Исправлена гонка при конкурентных `ALTER COLUMN` запросах, которая могла приводить к падению сервера (исправляет issue [#3421](https://github.com/yandex/ClickHouse/issues/3421)). [#4592](https://github.com/yandex/ClickHouse/pull/4592) ([Alex Zatelepin](https://github.com/ztlpn))
 * Исправлен некорректный результат в `FULL/RIGHT JOIN` запросах с константной колонкой. [#4723](https://github.com/yandex/ClickHouse/pull/4723) ([Artem Zuikov](https://github.com/4ertus2))
 * Исправлено появление дубликатов в `GLOBAL JOIN` со звездочкой. [#4705](https://github.com/yandex/ClickHouse/pull/4705) ([Artem Zuikov](https://github.com/4ertus2))
 * Исправлено определение параметров кодеков в запросах `ALTER MODIFY`, если тип колонки не был указан. [#4883](https://github.com/yandex/ClickHouse/pull/4883) ([alesapin](https://github.com/alesapin))
@@ -108,8 +108,8 @@
 * Исправлен дедлок, который мог происходить при запросе `DROP DATABASE dictionary`. [#4701](https://github.com/yandex/ClickHouse/pull/4701) ([alexey-milovidov](https://github.com/alexey-milovidov))
 * Исправлено неопределенное поведение в функциях `median` и `quantile`. [#4702](https://github.com/yandex/ClickHouse/pull/4702) ([hcz](https://github.com/hczhcz))
 * Исправлено определение уровня сжатия при указании настройки `network_compression_method` в нижнем регистре. Было сломано в v19.1. [#4706](https://github.com/yandex/ClickHouse/pull/4706) ([proller](https://github.com/proller))
-* Теперь обычные колонки, а также колонки `DEFAULT`, `MATERIALIZED` и `ALIAS` хранятся в одном списке (исправляет проблему [#2867](https://github.com/yandex/ClickHouse/issues/2867)). [#4707](https://github.com/yandex/ClickHouse/pull/4707) ([Alex Zatelepin](https://github.com/ztlpn))
-* Настройка `<timezone>UTC</timezone>` больше не игнорируется (исправляет проблему [#4658](https://github.com/yandex/ClickHouse/issues/4658)). [#4718](https://github.com/yandex/ClickHouse/pull/4718) ([proller](https://github.com/proller))
+* Теперь обычные колонки, а также колонки `DEFAULT`, `MATERIALIZED` и `ALIAS` хранятся в одном списке (исправляет issue [#2867](https://github.com/yandex/ClickHouse/issues/2867)). [#4707](https://github.com/yandex/ClickHouse/pull/4707) ([Alex Zatelepin](https://github.com/ztlpn))
+* Настройка `<timezone>UTC</timezone>` больше не игнорируется (исправляет issue [#4658](https://github.com/yandex/ClickHouse/issues/4658)). [#4718](https://github.com/yandex/ClickHouse/pull/4718) ([proller](https://github.com/proller))
 * Исправлено поведение функции `histogram` с `Distributed` таблицами. [#4741](https://github.com/yandex/ClickHouse/pull/4741) ([olegkv](https://github.com/olegkv))
 * Исправлено срабатывание thread-санитайзера с ошибкой `destroy of a locked mutex`. [#4742](https://github.com/yandex/ClickHouse/pull/4742) ([alexey-milovidov](https://github.com/alexey-milovidov))
 * Исправлено срабатывание thread-санитайзера при завершении сервера, вызванное гонкой при использовании системных логов. Также исправлена потенциальная ошибка use-after-free при завершении сервера в котором был включен `part_log`. [#4758](https://github.com/yandex/ClickHouse/pull/4758) ([alexey-milovidov](https://github.com/alexey-milovidov))
@@ -127,7 +127,7 @@
 * Исправлена редкая ошибка при чтении из колонки типа `Array(LowCardinality)`, которая возникала, если в колонке содержалось большее количество подряд идущих пустых массивов. [#4850](https://github.com/yandex/ClickHouse/pull/4850) ([Nikolai Kochetov](https://github.com/KochetovNicolai))
 * Исправлена ошибка `No message received`, возникавшая при скачивании кусков между репликами. [#4856](https://github.com/yandex/ClickHouse/pull/4856) ([alesapin](https://github.com/alesapin))
 * Исправлена ошибка в функции `arrayIntersect` приводившая к неправильным результатам в случае нескольких повторяющихся значений в массиве. [#4871](https://github.com/yandex/ClickHouse/pull/4871) ([Nikolai Kochetov](https://github.com/KochetovNicolai))
-* Исправлена гонка при конкурентных `ALTER COLUMN` запросах, которая могла приводить к падению сервера (исправляет проблему [#3421](https://github.com/yandex/ClickHouse/issues/3421)). [#4592](https://github.com/yandex/ClickHouse/pull/4592) ([Alex Zatelepin](https://github.com/ztlpn))
+* Исправлена гонка при конкурентных `ALTER COLUMN` запросах, которая могла приводить к падению сервера (исправляет issue [#3421](https://github.com/yandex/ClickHouse/issues/3421)). [#4592](https://github.com/yandex/ClickHouse/pull/4592) ([Alex Zatelepin](https://github.com/ztlpn))
 * Исправлено определение параметров кодеков в запросах `ALTER MODIFY`, если тип колонки не был указан. [#4883](https://github.com/yandex/ClickHouse/pull/4883) ([alesapin](https://github.com/alesapin))
 * Функции `cutQueryStringAndFragment()` и `queryStringAndFragment()` теперь работают корректно, когда `URL` содержит фрагмент, но не содержит запроса. [#4894](https://github.com/yandex/ClickHouse/pull/4894) ([Vitaly Baranov](https://github.com/vitlibar))
 * Исправлена редкая ошибка, возникавшая при установке настройки `min_bytes_to_use_direct_io` больше нуля. Она возникла при необходимости сдвинутся в файле, который уже прочитан до конца. [#4897](https://github.com/yandex/ClickHouse/pull/4897) ([alesapin](https://github.com/alesapin))

From 5b32756e7e095f95a84df337981185c797b28b68 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Fri, 26 Apr 2019 13:07:32 +0300
Subject: [PATCH 064/118] Order of words

---
 CHANGELOG_RU.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG_RU.md b/CHANGELOG_RU.md
index c0547d535e8..e4827a35a2b 100644
--- a/CHANGELOG_RU.md
+++ b/CHANGELOG_RU.md
@@ -7,7 +7,7 @@
 
 ### Новые возможности
 
-* [Hyperscan](https://github.com/intel/hyperscan) Добавлены функции для работы с несколькими регулярными выражениями. (`multiMatchAny`, `multiMatchAnyIndex`, `multiFuzzyMatchAny`, `multiFuzzyMatchAnyIndex`). [#4780](https://github.com/yandex/ClickHouse/pull/4780), [#4841](https://github.com/yandex/ClickHouse/pull/4841) ([Danila Kutenin](https://github.com/danlark1))
+* Добавлены функции для работы с несколькими регулярными выражениями с помощью библиотеки [Hyperscan](https://github.com/intel/hyperscan). (`multiMatchAny`, `multiMatchAnyIndex`, `multiFuzzyMatchAny`, `multiFuzzyMatchAnyIndex`). [#4780](https://github.com/yandex/ClickHouse/pull/4780), [#4841](https://github.com/yandex/ClickHouse/pull/4841) ([Danila Kutenin](https://github.com/danlark1))
 * Добавлена функция `multiSearchFirstPosition`. [#4780](https://github.com/yandex/ClickHouse/pull/4780) ([Danila Kutenin](https://github.com/danlark1))
 * Реализована возможность указания построчного ограничения доступа к таблицам. [#4792](https://github.com/yandex/ClickHouse/pull/4792) ([Ivan](https://github.com/abyss7))
 * Добавлен новый тип вторичного индекса на базе фильтра Блума (используется в функциях `equal`, `in` и `like`). [#4499](https://github.com/yandex/ClickHouse/pull/4499) ([Nikita Vasilev](https://github.com/nikvas0))

From cc3f372a7b79fa1c0ff2747d5c95b2364ee1df5c Mon Sep 17 00:00:00 2001
From: BayoNet <da-daos@yandex.ru>
Date: Fri, 26 Apr 2019 13:58:33 +0300
Subject: [PATCH 065/118] DOCAPI-6427: Settings for format processing.

---
 docs/en/interfaces/formats.md           |  2 ++
 docs/en/operations/settings/settings.md | 38 ++++++++++++++++++++-----
 2 files changed, 33 insertions(+), 7 deletions(-)

diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md
index 4a7a7b9e8c5..4d38007a050 100644
--- a/docs/en/interfaces/formats.md
+++ b/docs/en/interfaces/formats.md
@@ -30,6 +30,8 @@ The table below lists supported formats and how they can be used in `INSERT` and
 | [XML](#xml) | ✗ | ✔ |
 | [CapnProto](#capnproto) | ✔ | ✗ |
 
+You can control some format processing parameters by the ClickHouse settings. For more information read the [Settings](../operations/settings/settings.md) section.
+
 ## TabSeparated {#tabseparated}
 
 In TabSeparated format, data is written by row. Each row contains values separated by tabs. Each value is follow by a tab, except the last value in the row, which is followed by a line feed. Strictly Unix line feeds are assumed everywhere. The last row also must contain a line feed at the end. Values are written in text format, without enclosing quotation marks, and with special characters escaped.
diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index 2c00f1858a8..7936e287d72 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -184,7 +184,6 @@ INSERT INTO datetime_t SELECT now()
 Ok.
 ```
 
-
 ## input_format_defaults_for_omitted_fields {#session_settings-input_format_defaults_for_omitted_fields}
 
 Turns on/off the extended data exchange between a ClickHouse client and a ClickHouse server. This setting applies for `INSERT` queries.
@@ -202,13 +201,42 @@ For all other operations, ClickHouse doesn't apply the setting.
 !!! note "Note"
     The extended data exchange functionality consumes additional computing resources on the server and can reduce performance.
 
-**Possible values**
+Possible values:
 
 - 0 — Disabled.
 - 1 — Enabled.
 
-**Default value:** 0.
+Default value: 0.
 
+## input_format_skip_unknown_fields {#settings-input_format_skip_unknown_fields}
+
+Enables or disables skipping of insertion of extra data.
+
+When writing data, ClickHouse throws an exception if input data contain columns that do not exist in the target table. If skipping is enabled, ClickHouse doesn't insert extra data and doesn't throw an exception.
+
+Supported formats: [JSONEachRow](../../interfaces/formats.md#jsoneachrow), [CSVWithNames](../../interfaces/formats.md#csvwithnames), [TabSeparatedWithNames](../../interfaces/formats.md#tabseparatedwithnames), [TSKV](../../interfaces/formats.md#tskv).
+
+Possible values:
+
+- 0 — Disabled.
+- 1 — Enabled.
+
+Default value: 0.
+
+## input_format_with_names_use_header {#settings-input_format_with_names_use_header}
+
+Enables or disables checking the column order when inserting data.
+
+We recommend disabling check, if you are sure that the column order of the input data is the same as in the target table. It increases ClickHouse performance.
+
+Supported formats: [CSVWithNames](../../interfaces/formats.md#csvwithnames), [TabSeparatedWithNames](../../interfaces/formats.md#tabseparatedwithnames).
+
+Possible values:
+
+- 0 — Disabled.
+- 1 — Enabled.
+
+Default value: 1.
 
 ## join_default_strictness {#settings-join_default_strictness}
 
@@ -525,10 +553,6 @@ If the value is 1 or more, compilation occurs asynchronously in a separate threa
 Compiled code is required for each different combination of aggregate functions used in the query and the type of keys in the GROUP BY clause.
 The results of compilation are saved in the build directory in the form of .so files. There is no restriction on the number of compilation results, since they don't use very much space. Old results will be used after server restarts, except in the case of a server upgrade – in this case, the old results are deleted.
 
-## input_format_skip_unknown_fields
-
-If the value is true, running INSERT skips input data from columns with unknown names. Otherwise, this situation will generate an exception.
-It works for JSONEachRow and TSKV formats.
 
 ## output_format_json_quote_64bit_integers {#session_settings-output_format_json_quote_64bit_integers}
 

From 99e9c2d534df28d2f5a56883d726447139408005 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Fri, 26 Apr 2019 15:57:56 +0300
Subject: [PATCH 066/118] Temporary disable flapping test

---
 ...adaptive_index_granularity_replicated_merge_tree.reference | 2 --
 ...eeper_adaptive_index_granularity_replicated_merge_tree.sql | 4 ++--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/dbms/tests/queries/0_stateless/00926_zookeeper_adaptive_index_granularity_replicated_merge_tree.reference b/dbms/tests/queries/0_stateless/00926_zookeeper_adaptive_index_granularity_replicated_merge_tree.reference
index 642f836c646..2af6867d8d0 100644
--- a/dbms/tests/queries/0_stateless/00926_zookeeper_adaptive_index_granularity_replicated_merge_tree.reference
+++ b/dbms/tests/queries/0_stateless/00926_zookeeper_adaptive_index_granularity_replicated_merge_tree.reference
@@ -17,9 +17,7 @@
 1
 1
 8
-2
 8
-2
 -----
 4
 1
diff --git a/dbms/tests/queries/0_stateless/00926_zookeeper_adaptive_index_granularity_replicated_merge_tree.sql b/dbms/tests/queries/0_stateless/00926_zookeeper_adaptive_index_granularity_replicated_merge_tree.sql
index df9d791d824..bd70b2723fa 100644
--- a/dbms/tests/queries/0_stateless/00926_zookeeper_adaptive_index_granularity_replicated_merge_tree.sql
+++ b/dbms/tests/queries/0_stateless/00926_zookeeper_adaptive_index_granularity_replicated_merge_tree.sql
@@ -102,13 +102,13 @@ ATTACH TABLE test.four_rows_per_granule2;
 
 SELECT COUNT(*) FROM test.four_rows_per_granule2;
 
-SELECT distinct(marks) from system.parts WHERE table = 'four_rows_per_granule2' and database='test' and active=1;
+--SELECT distinct(marks) from system.parts WHERE table = 'four_rows_per_granule2' and database='test' and active=1;
 
 SYSTEM SYNC REPLICA test.four_rows_per_granule1;
 
 SELECT COUNT(*) FROM test.four_rows_per_granule1;
 
-SELECT distinct(marks) from system.parts WHERE table = 'four_rows_per_granule1' and database='test' and active=1;
+--SELECT distinct(marks) from system.parts WHERE table = 'four_rows_per_granule1' and database='test' and active=1;
 
 DROP TABLE IF EXISTS test.four_rows_per_granule1;
 DROP TABLE IF EXISTS test.four_rows_per_granule2;

From 8b73ff5b3d587d0b3df59a518657c20dc7298572 Mon Sep 17 00:00:00 2001
From: filimonov <1549571+filimonov@users.noreply.github.com>
Date: Fri, 26 Apr 2019 15:45:45 +0200
Subject: [PATCH 067/118] insert_sample_with_metadata small fixes in doc
 (#5121)

---
 docs/en/interfaces/formats.md | 7 +++----
 docs/ru/interfaces/formats.md | 7 +++----
 2 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md
index 4a7a7b9e8c5..e6f89591854 100644
--- a/docs/en/interfaces/formats.md
+++ b/docs/en/interfaces/formats.md
@@ -346,11 +346,10 @@ CREATE TABLE IF NOT EXISTS example_table
 ) ENGINE = Memory;
 ```
 
-- If `insert_sample_with_metadata = 0`, then the default value for `x` and `a` equals `0` (as the default value for the `UInt32` data type).
-- If `insert_sample_with_metadata = 1`, then the default value for `x` equals `0`, but the default value of `a` equals `x * 2`.
+- If `input_format_defaults_for_omitted_fields = 0`, then the default value for `x` and `a` equals `0` (as the default value for the `UInt32` data type).
+- If `input_format_defaults_for_omitted_fields = 1`, then the default value for `x` equals `0`, but the default value of `a` equals `x * 2`.
 
-!!! note "Warning"
-    Use this option carefully. Enabling it negatively affects the performance of the ClickHouse server.
+Enabling the option can affect the performance of inserts.
 
 ### Selecting Data
 
diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md
index 5988bea4b98..ce6cc59e93d 100644
--- a/docs/ru/interfaces/formats.md
+++ b/docs/ru/interfaces/formats.md
@@ -346,11 +346,10 @@ CREATE TABLE IF NOT EXISTS example_table
 ) ENGINE = Memory;
 ```
 
-- Если `insert_sample_with_metadata = 0`, то значение по умолчанию для `x` и `a` равняется `0` (поскольку это значение по умолчанию для типа данных `UInt32`.)
-- Если `insert_sample_with_metadata = 1`, то значение по умолчанию для `x` равно `0`, а значение по умолчанию `a` равно `x * 2`.
+- Если `input_format_defaults_for_omitted_fields = 0`, то значение по умолчанию для `x` и `a` равняется `0` (поскольку это значение по умолчанию для типа данных `UInt32`.)
+- Если `input_format_defaults_for_omitted_fields = 1`, то значение по умолчанию для `x` равно `0`, а значение по умолчанию `a` равно `x * 2`.
 
-!!! note "Предупреждение"
-    Используйте эту опцию осторожно. Её включение негативно влияет на производительность сервера ClickHouse.
+Включение этой опции может негативно влиять на производительность вставок.
 
 ### Выборка данных
 

From f69f03c96f0319abd5f8e800b29b970a112c2d79 Mon Sep 17 00:00:00 2001
From: proller <proller@users.noreply.github.com>
Date: Fri, 26 Apr 2019 18:37:22 +0300
Subject: [PATCH 068/118] Try fix shared libxml2 build (#5123)

---
 contrib/libxml2-cmake/CMakeLists.txt | 88 +++++++++++++---------------
 1 file changed, 41 insertions(+), 47 deletions(-)

diff --git a/contrib/libxml2-cmake/CMakeLists.txt b/contrib/libxml2-cmake/CMakeLists.txt
index cb52e5dad5a..ff14698335e 100644
--- a/contrib/libxml2-cmake/CMakeLists.txt
+++ b/contrib/libxml2-cmake/CMakeLists.txt
@@ -1,60 +1,54 @@
 set(LIBXML2_SOURCE_DIR ${CMAKE_SOURCE_DIR}/contrib/libxml2)
 set(LIBXML2_BINARY_DIR ${CMAKE_BINARY_DIR}/contrib/libxml2)
 
-
 set(SRCS
-    ${LIBXML2_SOURCE_DIR}/parser.c
-    ${LIBXML2_SOURCE_DIR}/HTMLparser.c
-    ${LIBXML2_SOURCE_DIR}/buf.c
-    ${LIBXML2_SOURCE_DIR}/xzlib.c
-    ${LIBXML2_SOURCE_DIR}/xmlregexp.c
-    ${LIBXML2_SOURCE_DIR}/entities.c
-    ${LIBXML2_SOURCE_DIR}/rngparser.c
-    ${LIBXML2_SOURCE_DIR}/encoding.c
-    ${LIBXML2_SOURCE_DIR}/legacy.c
-    ${LIBXML2_SOURCE_DIR}/error.c
-    ${LIBXML2_SOURCE_DIR}/debugXML.c
-    ${LIBXML2_SOURCE_DIR}/xpointer.c
-    ${LIBXML2_SOURCE_DIR}/DOCBparser.c
-    #${LIBXML2_SOURCE_DIR}/xmlcatalog.c
-    ${LIBXML2_SOURCE_DIR}/c14n.c
-    ${LIBXML2_SOURCE_DIR}/xmlreader.c
-    ${LIBXML2_SOURCE_DIR}/xmlstring.c
-    ${LIBXML2_SOURCE_DIR}/dict.c
-    ${LIBXML2_SOURCE_DIR}/xpath.c
-    ${LIBXML2_SOURCE_DIR}/tree.c
-    ${LIBXML2_SOURCE_DIR}/trionan.c
-    ${LIBXML2_SOURCE_DIR}/pattern.c
-    ${LIBXML2_SOURCE_DIR}/globals.c
-    #${LIBXML2_SOURCE_DIR}/xmllint.c
-    ${LIBXML2_SOURCE_DIR}/chvalid.c
-    ${LIBXML2_SOURCE_DIR}/relaxng.c
-    ${LIBXML2_SOURCE_DIR}/list.c
-    ${LIBXML2_SOURCE_DIR}/xinclude.c
-    ${LIBXML2_SOURCE_DIR}/xmlIO.c
-    ${LIBXML2_SOURCE_DIR}/triostr.c
-    ${LIBXML2_SOURCE_DIR}/hash.c
-    ${LIBXML2_SOURCE_DIR}/xmlsave.c
-    ${LIBXML2_SOURCE_DIR}/HTMLtree.c
     ${LIBXML2_SOURCE_DIR}/SAX.c
-    ${LIBXML2_SOURCE_DIR}/xmlschemas.c
-    ${LIBXML2_SOURCE_DIR}/SAX2.c
-    ${LIBXML2_SOURCE_DIR}/threads.c
-    #${LIBXML2_SOURCE_DIR}/runsuite.c
-    ${LIBXML2_SOURCE_DIR}/catalog.c
-    ${LIBXML2_SOURCE_DIR}/uri.c
-    ${LIBXML2_SOURCE_DIR}/xmlmodule.c
-    ${LIBXML2_SOURCE_DIR}/xlink.c
+    ${LIBXML2_SOURCE_DIR}/entities.c
+    ${LIBXML2_SOURCE_DIR}/encoding.c
+    ${LIBXML2_SOURCE_DIR}/error.c
     ${LIBXML2_SOURCE_DIR}/parserInternals.c
-    ${LIBXML2_SOURCE_DIR}/xmlwriter.c
-    ${LIBXML2_SOURCE_DIR}/xmlunicode.c
-    ${LIBXML2_SOURCE_DIR}/runxmlconf.c
+    ${LIBXML2_SOURCE_DIR}/parser.c
+    ${LIBXML2_SOURCE_DIR}/tree.c
+    ${LIBXML2_SOURCE_DIR}/hash.c
+    ${LIBXML2_SOURCE_DIR}/list.c
+    ${LIBXML2_SOURCE_DIR}/xmlIO.c
     ${LIBXML2_SOURCE_DIR}/xmlmemory.c
-    ${LIBXML2_SOURCE_DIR}/nanoftp.c
-    ${LIBXML2_SOURCE_DIR}/xmlschemastypes.c
+    ${LIBXML2_SOURCE_DIR}/uri.c
     ${LIBXML2_SOURCE_DIR}/valid.c
+    ${LIBXML2_SOURCE_DIR}/xlink.c
+    ${LIBXML2_SOURCE_DIR}/HTMLparser.c
+    ${LIBXML2_SOURCE_DIR}/HTMLtree.c
+    ${LIBXML2_SOURCE_DIR}/debugXML.c
+    ${LIBXML2_SOURCE_DIR}/xpath.c
+    ${LIBXML2_SOURCE_DIR}/xpointer.c
+    ${LIBXML2_SOURCE_DIR}/xinclude.c
     ${LIBXML2_SOURCE_DIR}/nanohttp.c
+    ${LIBXML2_SOURCE_DIR}/nanoftp.c
+    ${LIBXML2_SOURCE_DIR}/DOCBparser.c
+    ${LIBXML2_SOURCE_DIR}/catalog.c
+    ${LIBXML2_SOURCE_DIR}/globals.c
+    ${LIBXML2_SOURCE_DIR}/threads.c
+    ${LIBXML2_SOURCE_DIR}/c14n.c
+    ${LIBXML2_SOURCE_DIR}/xmlstring.c
+    ${LIBXML2_SOURCE_DIR}/buf.c
+    ${LIBXML2_SOURCE_DIR}/xmlregexp.c
+    ${LIBXML2_SOURCE_DIR}/xmlschemas.c
+    ${LIBXML2_SOURCE_DIR}/xmlschemastypes.c
+    ${LIBXML2_SOURCE_DIR}/xmlunicode.c
+    ${LIBXML2_SOURCE_DIR}/triostr.c
+    #${LIBXML2_SOURCE_DIR}/trio.c
+    ${LIBXML2_SOURCE_DIR}/xmlreader.c
+    ${LIBXML2_SOURCE_DIR}/relaxng.c
+    ${LIBXML2_SOURCE_DIR}/dict.c
+    ${LIBXML2_SOURCE_DIR}/SAX2.c
+    ${LIBXML2_SOURCE_DIR}/xmlwriter.c
+    ${LIBXML2_SOURCE_DIR}/legacy.c
+    ${LIBXML2_SOURCE_DIR}/chvalid.c
+    ${LIBXML2_SOURCE_DIR}/pattern.c
+    ${LIBXML2_SOURCE_DIR}/xmlsave.c
+    ${LIBXML2_SOURCE_DIR}/xmlmodule.c
     ${LIBXML2_SOURCE_DIR}/schematron.c
+    ${LIBXML2_SOURCE_DIR}/xzlib.c
 )
 add_library(libxml2 ${SRCS})
 

From b24c69743e1fadd64572aed04b543f595ea24c71 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitbar@yandex-team.ru>
Date: Fri, 19 Apr 2019 01:18:50 +0300
Subject: [PATCH 069/118] Use the same parameters in all APPLY_FOR_SETTINGS
 macros.

---
 .../Storages/MergeTree/MergeTreeSettings.cpp  |   4 +-
 .../Storages/MergeTree/MergeTreeSettings.h    | 298 +++++++++---------
 .../System/StorageSystemMergeTreeSettings.cpp |   2 +-
 3 files changed, 152 insertions(+), 152 deletions(-)

diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp b/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp
index 3ba11ba6c3f..899f26e9bca 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp
@@ -26,7 +26,7 @@ void MergeTreeSettings::loadFromConfig(const String & config_elem, const Poco::U
     {
         String value = config.getString(config_elem + "." + key);
 
-#define SET(TYPE, NAME, DEFAULT) \
+#define SET(TYPE, NAME, DEFAULT, DESCRIPTION) \
         else if (key == #NAME) NAME.set(value);
 
         if (false) {}
@@ -43,7 +43,7 @@ void MergeTreeSettings::loadFromQuery(ASTStorage & storage_def)
     {
         for (const ASTSetQuery::Change & setting : storage_def.settings->changes)
         {
-#define SET(TYPE, NAME, DEFAULT) \
+#define SET(TYPE, NAME, DEFAULT, DESCRIPTION) \
             else if (setting.name == #NAME) NAME.set(setting.value);
 
             if (false) {}
diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.h b/dbms/src/Storages/MergeTree/MergeTreeSettings.h
index 612a4030c9b..2d0be1ada68 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeSettings.h
+++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.h
@@ -18,161 +18,161 @@ struct MergeTreeSettings
 {
 
 #define APPLY_FOR_MERGE_TREE_SETTINGS(M) \
-    /** How many rows correspond to one primary key value. */                                                 \
-    M(SettingUInt64, index_granularity, 8192)                                                                 \
-                                                                                                              \
-    /** Merge settings. */                                                                                    \
-                                                                                                              \
-    /** Maximum in total size of parts to merge, when there are maximum (minimum) free threads                \
-     * in background pool (or entries in replication queue). */                                               \
-    M(SettingUInt64, max_bytes_to_merge_at_max_space_in_pool, 150ULL * 1024 * 1024 * 1024)                    \
-    M(SettingUInt64, max_bytes_to_merge_at_min_space_in_pool, 1024 * 1024)                                    \
-                                                                                                              \
-    /** How many tasks of merging parts are allowed simultaneously in ReplicatedMergeTree queue. */           \
-    M(SettingUInt64, max_replicated_merges_in_queue, 16)                                                      \
-                                                                                                              \
-    /** When there is less than specified number of free entries in pool (or replicated queue),               \
-     *  start to lower maximum size of merge to process (or to put in queue).                                 \
-     *  This is to allow small merges to process - not filling the pool with long running merges. */          \
-    M(SettingUInt64, number_of_free_entries_in_pool_to_lower_max_size_of_merge, 8)                            \
-                                                                                                              \
-    /** How many seconds to keep obsolete parts. */                                                           \
-    M(SettingSeconds, old_parts_lifetime, 8 * 60)                                                             \
-                                                                                                              \
-    /** How many seconds to keep tmp_-directories. */                                                         \
-    M(SettingSeconds, temporary_directories_lifetime, 86400)                                                  \
-                                                                                                              \
-    /** Inserts settings. */                                                                                  \
-                                                                                                              \
-    /** If table contains at least that many active parts, artificially slow down insert into table. */       \
-    M(SettingUInt64, parts_to_delay_insert, 150)                                                              \
-                                                                                                              \
-    /** If more than this number active parts, throw 'Too many parts ...' exception */                        \
-    M(SettingUInt64, parts_to_throw_insert, 300)                                                              \
-                                                                                                              \
-    /** Max delay of inserting data into MergeTree table in seconds, if there are a lot of unmerged parts. */ \
-    M(SettingUInt64, max_delay_to_insert, 1)                                                                  \
-                                                                                                              \
-    /** Replication settings. */                                                                              \
-                                                                                                              \
-    /** How many last blocks of hashes should be kept in ZooKeeper (old blocks will be deleted). */           \
-    M(SettingUInt64, replicated_deduplication_window, 100)                                                    \
-    /** Similar to previous, but determines old blocks by their lifetime.                                     \
-     *  Hash of an inserted block will be deleted (and the block will not be deduplicated after)              \
-     *  if it outside of one "window". You can set very big replicated_deduplication_window to avoid          \
-     *  duplicating INSERTs during that period of time. */                                                    \
-    M(SettingUInt64, replicated_deduplication_window_seconds, 7 * 24 * 60 * 60) /** one week */               \
-                                                                                                              \
-    /** How many records may be in log, if there is inactive replica  */                                      \
-    M(SettingUInt64, max_replicated_logs_to_keep, 10000)                                                      \
-                                                                                                              \
-    /** Keep about this number of last records in ZooKeeper log, even if they are obsolete.                   \
-     *  It doesn't affect work of tables: used only to diagnose ZooKeeper log before cleaning. */             \
-    M(SettingUInt64, min_replicated_logs_to_keep, 100)                                                        \
-                                                                                                              \
-    /** After specified amount of time passed after replication log entry creation                            \
-     *  and sum size of parts is greater than threshold,                                                      \
-     *  prefer fetching merged part from replica instead of doing merge locally.                              \
-     *  To speed up very long merges. */                                                                      \
-    M(SettingSeconds, prefer_fetch_merged_part_time_threshold, 3600)                                          \
-    M(SettingUInt64, prefer_fetch_merged_part_size_threshold, 10ULL * 1024 * 1024 * 1024)                     \
-                                                                                                              \
-    /** Max broken parts, if more - deny automatic deletion. */                                               \
-    M(SettingUInt64, max_suspicious_broken_parts, 10)                                                         \
-                                                                                                              \
-    /** Not apply ALTER if number of files for modification(deletion, addition) more than this. */            \
-    M(SettingUInt64, max_files_to_modify_in_alter_columns, 75)                                                \
-    /** Not apply ALTER, if number of files for deletion more than this. */                                   \
-    M(SettingUInt64, max_files_to_remove_in_alter_columns, 50)                                                \
-                                                                                                              \
-    /** If ratio of wrong parts to total number of parts is less than this - allow to start. */               \
-    M(SettingFloat, replicated_max_ratio_of_wrong_parts, 0.5)                                                 \
-                                                                                                              \
-    /** Limit parallel fetches */                                                                             \
-    M(SettingUInt64, replicated_max_parallel_fetches, 0)                                                      \
-    M(SettingUInt64, replicated_max_parallel_fetches_for_table, 0)                                            \
-                                                                                                              \
-    /** Limit parallel fetches from endpoint (actually pool size) */                                          \
-    M(SettingUInt64, replicated_max_parallel_fetches_for_host, DEFAULT_COUNT_OF_HTTP_CONNECTIONS_PER_ENDPOINT)\
-                                                                                                              \
-    /** Limit parallel sends */                                                                               \
-    M(SettingUInt64, replicated_max_parallel_sends, 0)                                                        \
-    M(SettingUInt64, replicated_max_parallel_sends_for_table, 0)                                              \
-                                                                                                              \
-    /** If true, Replicated tables replicas on this node will try to acquire leadership. */                   \
-    M(SettingBool, replicated_can_become_leader, true)                                                        \
-                                                                                                              \
-    M(SettingSeconds, zookeeper_session_expiration_check_period, 60)                                          \
-                                                                                                              \
-    /** Check delay of replicas settings. */                                                                  \
-                                                                                                              \
-    /** Period to check replication delay and compare with other replicas. */                                 \
-    M(SettingUInt64, check_delay_period, 60)                                                                  \
-                                                                                                              \
-    /** Period to clean old queue logs, blocks hashes and parts */                                            \
-    M(SettingUInt64, cleanup_delay_period, 30)                                                                \
-    /** Add uniformly distributed value from 0 to x seconds to cleanup_delay_period                           \
+    /** How many rows correspond to one primary key value. */                                                     \
+    M(SettingUInt64, index_granularity, 8192, "")                                                                 \
+                                                                                                                  \
+    /** Merge settings. */                                                                                        \
+                                                                                                                  \
+    /** Maximum in total size of parts to merge, when there are maximum (minimum) free threads                    \
+     * in background pool (or entries in replication queue). */                                                   \
+    M(SettingUInt64, max_bytes_to_merge_at_max_space_in_pool, 150ULL * 1024 * 1024 * 1024, "")                    \
+    M(SettingUInt64, max_bytes_to_merge_at_min_space_in_pool, 1024 * 1024, "")                                    \
+                                                                                                                  \
+    /** How many tasks of merging parts are allowed simultaneously in ReplicatedMergeTree queue. */               \
+    M(SettingUInt64, max_replicated_merges_in_queue, 16, "")                                                      \
+                                                                                                                  \
+    /** When there is less than specified number of free entries in pool (or replicated queue),                   \
+     *  start to lower maximum size of merge to process (or to put in queue).                                     \
+     *  This is to allow small merges to process - not filling the pool with long running merges. */              \
+    M(SettingUInt64, number_of_free_entries_in_pool_to_lower_max_size_of_merge, 8, "")                            \
+                                                                                                                  \
+    /** How many seconds to keep obsolete parts. */                                                               \
+    M(SettingSeconds, old_parts_lifetime, 8 * 60, "")                                                             \
+                                                                                                                  \
+    /** How many seconds to keep tmp_-directories. */                                                             \
+    M(SettingSeconds, temporary_directories_lifetime, 86400, "")                                                  \
+                                                                                                                  \
+    /** Inserts settings. */                                                                                      \
+                                                                                                                  \
+    /** If table contains at least that many active parts, artificially slow down insert into table. */           \
+    M(SettingUInt64, parts_to_delay_insert, 150, "")                                                              \
+                                                                                                                  \
+    /** If more than this number active parts, throw 'Too many parts ...' exception */                            \
+    M(SettingUInt64, parts_to_throw_insert, 300, "")                                                              \
+                                                                                                                  \
+    /** Max delay of inserting data into MergeTree table in seconds, if there are a lot of unmerged parts. */     \
+    M(SettingUInt64, max_delay_to_insert, 1, "")                                                                  \
+                                                                                                                  \
+    /** Replication settings. */                                                                                  \
+                                                                                                                  \
+    /** How many last blocks of hashes should be kept in ZooKeeper (old blocks will be deleted). */               \
+    M(SettingUInt64, replicated_deduplication_window, 100, "")                                                    \
+    /** Similar to previous, but determines old blocks by their lifetime.                                         \
+     *  Hash of an inserted block will be deleted (and the block will not be deduplicated after)                  \
+     *  if it outside of one "window". You can set very big replicated_deduplication_window to avoid              \
+     *  duplicating INSERTs during that period of time. */                                                        \
+    M(SettingUInt64, replicated_deduplication_window_seconds, 7 * 24 * 60 * 60, "") /** one week */               \
+                                                                                                                  \
+    /** How many records may be in log, if there is inactive replica  */                                          \
+    M(SettingUInt64, max_replicated_logs_to_keep, 10000, "")                                                      \
+                                                                                                                  \
+    /** Keep about this number of last records in ZooKeeper log, even if they are obsolete.                       \
+     *  It doesn't affect work of tables: used only to diagnose ZooKeeper log before cleaning. */                 \
+    M(SettingUInt64, min_replicated_logs_to_keep, 100, "")                                                        \
+                                                                                                                  \
+    /** After specified amount of time passed after replication log entry creation                                \
+     *  and sum size of parts is greater than threshold,                                                          \
+     *  prefer fetching merged part from replica instead of doing merge locally.                                  \
+     *  To speed up very long merges. */                                                                          \
+    M(SettingSeconds, prefer_fetch_merged_part_time_threshold, 3600, "")                                          \
+    M(SettingUInt64, prefer_fetch_merged_part_size_threshold, 10ULL * 1024 * 1024 * 1024, "")                     \
+                                                                                                                  \
+    /** Max broken parts, if more - deny automatic deletion. */                                                   \
+    M(SettingUInt64, max_suspicious_broken_parts, 10, "")                                                         \
+                                                                                                                  \
+    /** Not apply ALTER if number of files for modification(deletion, addition) more than this. */                \
+    M(SettingUInt64, max_files_to_modify_in_alter_columns, 75, "")                                                \
+    /** Not apply ALTER, if number of files for deletion more than this. */                                       \
+    M(SettingUInt64, max_files_to_remove_in_alter_columns, 50, "")                                                \
+                                                                                                                  \
+    /** If ratio of wrong parts to total number of parts is less than this - allow to start. */                   \
+    M(SettingFloat, replicated_max_ratio_of_wrong_parts, 0.5, "")                                                 \
+                                                                                                                  \
+    /** Limit parallel fetches */                                                                                 \
+    M(SettingUInt64, replicated_max_parallel_fetches, 0, "")                                                      \
+    M(SettingUInt64, replicated_max_parallel_fetches_for_table, 0, "")                                            \
+                                                                                                                  \
+    /** Limit parallel fetches from endpoint (actually pool size) */                                              \
+    M(SettingUInt64, replicated_max_parallel_fetches_for_host, DEFAULT_COUNT_OF_HTTP_CONNECTIONS_PER_ENDPOINT, "")\
+                                                                                                                  \
+    /** Limit parallel sends */                                                                                   \
+    M(SettingUInt64, replicated_max_parallel_sends, 0, "")                                                        \
+    M(SettingUInt64, replicated_max_parallel_sends_for_table, 0, "")                                              \
+                                                                                                                  \
+    /** If true, Replicated tables replicas on this node will try to acquire leadership. */                       \
+    M(SettingBool, replicated_can_become_leader, true, "")                                                        \
+                                                                                                                  \
+    M(SettingSeconds, zookeeper_session_expiration_check_period, 60, "")                                          \
+                                                                                                                  \
+    /** Check delay of replicas settings. */                                                                      \
+                                                                                                                  \
+    /** Period to check replication delay and compare with other replicas. */                                     \
+    M(SettingUInt64, check_delay_period, 60, "")                                                                  \
+                                                                                                                  \
+    /** Period to clean old queue logs, blocks hashes and parts */                                                \
+    M(SettingUInt64, cleanup_delay_period, 30, "")                                                                \
+    /** Add uniformly distributed value from 0 to x seconds to cleanup_delay_period                               \
         to avoid thundering herd effect and subsequent DoS of ZooKeeper in case of very large number of tables */ \
-    M(SettingUInt64, cleanup_delay_period_random_add, 10)                                                     \
-                                                                                                              \
-    /** Minimal delay from other replicas to yield leadership. Here and further 0 means unlimited. */         \
-    M(SettingUInt64, min_relative_delay_to_yield_leadership, 120)                                             \
-                                                                                                              \
-    /** Minimal delay from other replicas to close, stop serving requests and not return Ok                   \
-     *  during status check. */                                                                               \
-    M(SettingUInt64, min_relative_delay_to_close, 300)                                                        \
-                                                                                                              \
-    /** Minimal absolute delay to close, stop serving requests and not return Ok during status check. */      \
-    M(SettingUInt64, min_absolute_delay_to_close, 0)                                                          \
-                                                                                                              \
-    /** Enable usage of Vertical merge algorithm. */                                                          \
-    M(SettingUInt64, enable_vertical_merge_algorithm, 1)                                                      \
-                                                                                                              \
-    /** Minimal (approximate) sum of rows in merging parts to activate Vertical merge algorithm */            \
-    M(SettingUInt64, vertical_merge_algorithm_min_rows_to_activate, 16 * DEFAULT_MERGE_BLOCK_SIZE)            \
-                                                                                                              \
-    /** Minimal amount of non-PK columns to activate Vertical merge algorithm */                              \
-    M(SettingUInt64, vertical_merge_algorithm_min_columns_to_activate, 11)                                    \
-                                                                                                              \
-    /** Compatibility settings */                                                                             \
-                                                                                                              \
-    /** Allow to create a table with sampling expression not in primary key.                                  \
-      * This is needed only to temporarily allow to run the server with wrong tables                          \
-      *  for backward compatibility.                                                                          \
-      */                                                                                                      \
-    M(SettingBool, compatibility_allow_sampling_expression_not_in_primary_key, false)                         \
-                                                                                                              \
-    /** Use small format (dozens bytes) for part checksums in ZooKeeper                                       \
-      *  instead of ordinary ones (dozens KB).                                                                \
-      * Before enabling check that all replicas support new format.                                           \
-      */                                                                                                      \
-    M(SettingBool, use_minimalistic_checksums_in_zookeeper, true)                                             \
-                                                                                                              \
-    /** Store part header (checksums and columns) in a compact format and a single part znode                 \
-      *  instead of separate znodes (<part>/columns and <part>/checksums).                                    \
-      * This can dramatically reduce snapshot size in ZooKeeper.                                              \
-      * Before enabling check that all replicas support new format.                                           \
-      */                                                                                                      \
-    M(SettingBool, use_minimalistic_part_header_in_zookeeper, false)                                          \
-                                                                                                              \
-    /** How many records about mutations that are done to keep.                                               \
-     *  If zero, then keep all of them */                                                                     \
-    M(SettingUInt64, finished_mutations_to_keep, 100)                                                         \
-                                                                                                              \
-    /** Minimal amount of bytes to enable O_DIRECT in merge (0 - disabled) */                                 \
-    M(SettingUInt64, min_merge_bytes_to_use_direct_io, 10ULL * 1024 * 1024 * 1024)                            \
-                                                                                                              \
-    /** Approximate amount of bytes in single granule (0 - disabled) */                                       \
-    M(SettingUInt64, index_granularity_bytes, 0)                                                              \
-                                                                                                              \
-    /** Minimal time in seconds, when merge with TTL can be repeated */                                       \
-    M(SettingInt64, merge_with_ttl_timeout, 3600 * 24)
+    M(SettingUInt64, cleanup_delay_period_random_add, 10, "")                                                     \
+                                                                                                                  \
+    /** Minimal delay from other replicas to yield leadership. Here and further 0 means unlimited. */             \
+    M(SettingUInt64, min_relative_delay_to_yield_leadership, 120, "")                                             \
+                                                                                                                  \
+    /** Minimal delay from other replicas to close, stop serving requests and not return Ok                       \
+     *  during status check. */                                                                                   \
+    M(SettingUInt64, min_relative_delay_to_close, 300, "")                                                        \
+                                                                                                                  \
+    /** Minimal absolute delay to close, stop serving requests and not return Ok during status check. */          \
+    M(SettingUInt64, min_absolute_delay_to_close, 0, "")                                                          \
+                                                                                                                  \
+    /** Enable usage of Vertical merge algorithm. */                                                              \
+    M(SettingUInt64, enable_vertical_merge_algorithm, 1, "")                                                      \
+                                                                                                                  \
+    /** Minimal (approximate) sum of rows in merging parts to activate Vertical merge algorithm */                \
+    M(SettingUInt64, vertical_merge_algorithm_min_rows_to_activate, 16 * DEFAULT_MERGE_BLOCK_SIZE, "")            \
+                                                                                                                  \
+    /** Minimal amount of non-PK columns to activate Vertical merge algorithm */                                  \
+    M(SettingUInt64, vertical_merge_algorithm_min_columns_to_activate, 11, "")                                    \
+                                                                                                                  \
+    /** Compatibility settings */                                                                                 \
+                                                                                                                  \
+    /** Allow to create a table with sampling expression not in primary key.                                      \
+      * This is needed only to temporarily allow to run the server with wrong tables                              \
+      *  for backward compatibility.                                                                              \
+      */                                                                                                          \
+    M(SettingBool, compatibility_allow_sampling_expression_not_in_primary_key, false, "")                         \
+                                                                                                                  \
+    /** Use small format (dozens bytes) for part checksums in ZooKeeper                                           \
+      *  instead of ordinary ones (dozens KB).                                                                    \
+      * Before enabling check that all replicas support new format.                                               \
+      */                                                                                                          \
+    M(SettingBool, use_minimalistic_checksums_in_zookeeper, true, "")                                             \
+                                                                                                                  \
+    /** Store part header (checksums and columns) in a compact format and a single part znode                     \
+      *  instead of separate znodes (<part>/columns and <part>/checksums).                                        \
+      * This can dramatically reduce snapshot size in ZooKeeper.                                                  \
+      * Before enabling check that all replicas support new format.                                               \
+      */                                                                                                          \
+    M(SettingBool, use_minimalistic_part_header_in_zookeeper, false, "")                                          \
+                                                                                                                  \
+    /** How many records about mutations that are done to keep.                                                   \
+     *  If zero, then keep all of them */                                                                         \
+    M(SettingUInt64, finished_mutations_to_keep, 100, "")                                                         \
+                                                                                                                  \
+    /** Minimal amount of bytes to enable O_DIRECT in merge (0 - disabled) */                                     \
+    M(SettingUInt64, min_merge_bytes_to_use_direct_io, 10ULL * 1024 * 1024 * 1024, "")                            \
+                                                                                                                  \
+    /** Approximate amount of bytes in single granule (0 - disabled) */                                           \
+    M(SettingUInt64, index_granularity_bytes, 0, "")                                                              \
+                                                                                                                  \
+    /** Minimal time in seconds, when merge with TTL can be repeated */                                           \
+    M(SettingInt64, merge_with_ttl_timeout, 3600 * 24, "")
 
     /// Settings that should not change after the creation of a table.
 #define APPLY_FOR_IMMUTABLE_MERGE_TREE_SETTINGS(M) \
     M(index_granularity)
 
-#define DECLARE(TYPE, NAME, DEFAULT) \
+#define DECLARE(TYPE, NAME, DEFAULT, DESCRIPTION) \
     TYPE NAME {DEFAULT};
 
     APPLY_FOR_MERGE_TREE_SETTINGS(DECLARE)
diff --git a/dbms/src/Storages/System/StorageSystemMergeTreeSettings.cpp b/dbms/src/Storages/System/StorageSystemMergeTreeSettings.cpp
index 485f490210b..f60ed67c56b 100644
--- a/dbms/src/Storages/System/StorageSystemMergeTreeSettings.cpp
+++ b/dbms/src/Storages/System/StorageSystemMergeTreeSettings.cpp
@@ -20,7 +20,7 @@ void SystemMergeTreeSettings::fillData(MutableColumns & res_columns, const Conte
 {
     const MergeTreeSettings & settings = context.getMergeTreeSettings();
 
-#define ADD_SETTING(TYPE, NAME, DEFAULT)              \
+#define ADD_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION)              \
     res_columns[0]->insert(#NAME);            \
     res_columns[1]->insert(settings.NAME.toString()); \
     res_columns[2]->insert(settings.NAME.changed);

From 8277e9d8f125c61df962944e6dabee424cc1c1b1 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitbar@yandex-team.ru>
Date: Fri, 19 Apr 2019 02:29:32 +0300
Subject: [PATCH 070/118] Refactoring of Settings based on new template class
 SettingsCollection. Add new class SettingsConstraints.

---
 dbms/programs/client/Client.cpp               |   5 +-
 dbms/programs/server/HTTPHandler.cpp          |  19 +-
 dbms/src/Client/Connection.cpp                |   2 +-
 dbms/src/Common/SettingsChanges.h             |  17 +
 dbms/src/Core/Settings.cpp                    | 127 +-----
 dbms/src/Core/Settings.h                      |  40 +-
 dbms/src/Core/SettingsCommon.cpp              | 318 ++++++++-----
 dbms/src/Core/SettingsCommon.h                | 419 ++++++++++++++++--
 dbms/src/Interpreters/Context.cpp             |  47 +-
 dbms/src/Interpreters/Context.h               |  10 +-
 dbms/src/Interpreters/InterpreterSetQuery.cpp |  56 +--
 dbms/src/Interpreters/InterpreterSetQuery.h   |   2 -
 dbms/src/Interpreters/SettingsConstraints.cpp |  65 +++
 dbms/src/Interpreters/SettingsConstraints.h   |  25 ++
 dbms/src/Parsers/ASTSetQuery.h                |  13 +-
 dbms/src/Parsers/ParserSetQuery.cpp           |   9 +-
 dbms/src/Storages/Kafka/KafkaSettings.cpp     |  21 +-
 dbms/src/Storages/Kafka/KafkaSettings.h       |  13 +-
 .../Storages/MergeTree/MergeTreeSettings.cpp  |  47 +-
 .../Storages/MergeTree/MergeTreeSettings.h    |  23 +-
 dbms/src/Storages/StorageJoin.cpp             |   2 +-
 .../00474_readonly_settings.reference         |   4 +-
 .../0_stateless/00474_readonly_settings.sh    |   8 +-
 23 files changed, 834 insertions(+), 458 deletions(-)
 create mode 100644 dbms/src/Common/SettingsChanges.h
 create mode 100644 dbms/src/Interpreters/SettingsConstraints.cpp
 create mode 100644 dbms/src/Interpreters/SettingsConstraints.h

diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp
index 805e582e637..20c01f4f79c 100644
--- a/dbms/programs/client/Client.cpp
+++ b/dbms/programs/client/Client.cpp
@@ -816,8 +816,7 @@ private:
             {
                 if (!old_settings)
                     old_settings.emplace(context.getSettingsRef());
-                for (const auto & change : settings_ast.as<ASTSetQuery>()->changes)
-                    context.setSetting(change.name, change.value);
+                context.applySettingsChanges(settings_ast.as<ASTSetQuery>()->changes);
             };
             const auto * insert = parsed_query->as<ASTInsertQuery>();
             if (insert && insert->settings_ast)
@@ -847,7 +846,7 @@ private:
                     if (change.name == "profile")
                         current_profile = change.value.safeGet<String>();
                     else
-                        context.setSetting(change.name, change.value);
+                        context.applySettingChange(change);
                 }
             }
 
diff --git a/dbms/programs/server/HTTPHandler.cpp b/dbms/programs/server/HTTPHandler.cpp
index e310c32034f..ee8a50662c9 100644
--- a/dbms/programs/server/HTTPHandler.cpp
+++ b/dbms/programs/server/HTTPHandler.cpp
@@ -497,8 +497,7 @@ void HTTPHandler::processQuery(
             settings.readonly = 2;
     }
 
-    auto readonly_before_query = settings.readonly;
-
+    SettingsChanges settings_changes;
     for (auto it = params.begin(); it != params.end(); ++it)
     {
         if (it->first == "database")
@@ -515,21 +514,13 @@ void HTTPHandler::processQuery(
         else
         {
             /// All other query parameters are treated as settings.
-            String value;
-            /// Setting is skipped if value wasn't changed.
-            if (!settings.tryGet(it->first, value) || it->second != value)
-            {
-                if (readonly_before_query == 1)
-                    throw Exception("Cannot override setting (" + it->first + ") in readonly mode", ErrorCodes::READONLY);
-
-                if (readonly_before_query && it->first == "readonly")
-                    throw Exception("Setting 'readonly' cannot be overrided in readonly mode", ErrorCodes::READONLY);
-
-                context.setSetting(it->first, it->second);
-            }
+            settings_changes.push_back({it->first, it->second});
         }
     }
 
+    context.checkSettingsConstraints(settings_changes);
+    context.applySettingsChanges(settings_changes);
+
     /// HTTP response compression is turned on only if the client signalled that they support it
     /// (using Accept-Encoding header) and 'enable_http_compression' setting is turned on.
     used_output.out->setCompression(client_supports_http_compression && settings.enable_http_compression);
diff --git a/dbms/src/Client/Connection.cpp b/dbms/src/Client/Connection.cpp
index 5de8e760dc2..a76803baef2 100644
--- a/dbms/src/Client/Connection.cpp
+++ b/dbms/src/Client/Connection.cpp
@@ -401,7 +401,7 @@ void Connection::sendQuery(
     if (settings)
         settings->serialize(*out);
     else
-        writeStringBinary("", *out);
+        writeStringBinary("" /* empty string is a marker of the end of settings */, *out);
 
     writeVarUInt(stage, *out);
     writeVarUInt(static_cast<bool>(compression), *out);
diff --git a/dbms/src/Common/SettingsChanges.h b/dbms/src/Common/SettingsChanges.h
new file mode 100644
index 00000000000..1d4406cf1ea
--- /dev/null
+++ b/dbms/src/Common/SettingsChanges.h
@@ -0,0 +1,17 @@
+#pragma once
+
+#include <Core/Field.h>
+
+
+namespace DB
+{
+
+struct SettingChange
+{
+    String name;
+    Field value;
+};
+
+using SettingsChanges = std::vector<SettingChange>;
+
+}
diff --git a/dbms/src/Core/Settings.cpp b/dbms/src/Core/Settings.cpp
index d5a8bab378b..7653612d11c 100644
--- a/dbms/src/Core/Settings.cpp
+++ b/dbms/src/Core/Settings.cpp
@@ -1,9 +1,6 @@
 #include "Settings.h"
 
 #include <Poco/Util/AbstractConfiguration.h>
-#include <Core/Field.h>
-#include <IO/ReadHelpers.h>
-#include <IO/WriteHelpers.h>
 #include <Columns/ColumnArray.h>
 #include <Common/typeid_cast.h>
 #include <string.h>
@@ -14,94 +11,13 @@ namespace DB
 
 namespace ErrorCodes
 {
-    extern const int UNKNOWN_SETTING;
     extern const int THERE_IS_NO_PROFILE;
     extern const int NO_ELEMENTS_IN_CONFIG;
 }
 
 
-/// Set the configuration by name.
-void Settings::set(const String & name, const Field & value)
-{
-#define TRY_SET(TYPE, NAME, DEFAULT, DESCRIPTION) \
-    else if (name == #NAME) NAME.set(value);
+IMPLEMENT_SETTINGS_COLLECTION(Settings, APPLY_FOR_SETTINGS)
 
-    if (false) {}
-    APPLY_FOR_SETTINGS(TRY_SET)
-    else
-        throw Exception("Unknown setting " + name, ErrorCodes::UNKNOWN_SETTING);
-
-#undef TRY_SET
-}
-
-/// Set the configuration by name. Read the binary serialized value from the buffer (for interserver interaction).
-void Settings::set(const String & name, ReadBuffer & buf)
-{
-#define TRY_SET(TYPE, NAME, DEFAULT, DESCRIPTION) \
-    else if (name == #NAME) NAME.set(buf);
-
-    if (false) {}
-    APPLY_FOR_SETTINGS(TRY_SET)
-    else
-        throw Exception("Unknown setting " + name, ErrorCodes::UNKNOWN_SETTING);
-
-#undef TRY_SET
-}
-
-/// Skip the binary-serialized value from the buffer.
-void Settings::ignore(const String & name, ReadBuffer & buf)
-{
-#define TRY_IGNORE(TYPE, NAME, DEFAULT, DESCRIPTION) \
-    else if (name == #NAME) decltype(NAME)(DEFAULT).set(buf);
-
-    if (false) {}
-    APPLY_FOR_SETTINGS(TRY_IGNORE)
-    else
-        throw Exception("Unknown setting " + name, ErrorCodes::UNKNOWN_SETTING);
-
-    #undef TRY_IGNORE
-}
-
-/** Set the setting by name. Read the value in text form from a string (for example, from a config, or from a URL parameter).
-    */
-void Settings::set(const String & name, const String & value)
-{
-#define TRY_SET(TYPE, NAME, DEFAULT, DESCRIPTION) \
-    else if (name == #NAME) NAME.set(value);
-
-    if (false) {}
-    APPLY_FOR_SETTINGS(TRY_SET)
-    else
-        throw Exception("Unknown setting " + name, ErrorCodes::UNKNOWN_SETTING);
-
-#undef TRY_SET
-}
-
-String Settings::get(const String & name) const
-{
-#define GET(TYPE, NAME, DEFAULT, DESCRIPTION) \
-    else if (name == #NAME) return NAME.toString();
-
-    if (false) {}
-    APPLY_FOR_SETTINGS(GET)
-    else
-        throw Exception("Unknown setting " + name, ErrorCodes::UNKNOWN_SETTING);
-
-#undef GET
-}
-
-bool Settings::tryGet(const String & name, String & value) const
-{
-#define TRY_GET(TYPE, NAME, DEFAULT, DESCRIPTION) \
-    else if (name == #NAME) { value = NAME.toString(); return true; }
-
-    if (false) {}
-    APPLY_FOR_SETTINGS(TRY_GET)
-    else
-        return false;
-
-#undef TRY_GET
-}
 
 /** Set the settings from the profile (in the server configuration, many settings can be listed in one profile).
     * The profile can also be set using the `set` functions, like the `profile` setting.
@@ -139,47 +55,6 @@ void Settings::loadSettingsFromConfig(const String & path, const Poco::Util::Abs
     }
 }
 
-/// Read the settings from the buffer. They are written as a set of name-value pairs that go successively, ending with an empty `name`.
-/// If the `check_readonly` flag is set, `readonly` is set in the preferences, but some changes have occurred - throw an exception.
-void Settings::deserialize(ReadBuffer & buf)
-{
-    auto before_readonly = readonly;
-
-    while (true)
-    {
-        String name;
-        readBinary(name, buf);
-
-        /// An empty string is the marker for the end of the settings.
-        if (name.empty())
-            break;
-
-        /// If readonly = 2, then you can change the settings, except for the readonly setting.
-        if (before_readonly == 0 || (before_readonly == 2 && name != "readonly"))
-            set(name, buf);
-        else
-            ignore(name, buf);
-    }
-}
-
-/// Record the changed settings to the buffer. (For example, to send to a remote server.)
-void Settings::serialize(WriteBuffer & buf) const
-{
-#define WRITE(TYPE, NAME, DEFAULT, DESCRIPTION) \
-    if (NAME.changed) \
-    { \
-        writeStringBinary(#NAME, buf); \
-        NAME.write(buf); \
-    }
-
-    APPLY_FOR_SETTINGS(WRITE)
-
-    /// An empty string is a marker for the end of the settings.
-    writeStringBinary("", buf);
-
-#undef WRITE
-}
-
 void Settings::dumpToArrayColumns(IColumn * column_names_, IColumn * column_values_, bool changed_only)
 {
     /// Convert ptr and make simple check
diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h
index 19278e76515..c7c6fbfb700 100644
--- a/dbms/src/Core/Settings.h
+++ b/dbms/src/Core/Settings.h
@@ -16,11 +16,11 @@ namespace DB
 {
 
 class IColumn;
-class Field;
+
 
 /** Settings of query execution.
   */
-struct Settings
+struct Settings : public SettingsCollection<Settings>
 {
     /// For initialization from empty initializer-list to be "value initialization", not "aggregate initialization" in C++14.
     /// http://en.cppreference.com/w/cpp/language/aggregate_initialization
@@ -315,48 +315,18 @@ struct Settings
     \
     M(SettingUInt64, max_partitions_per_insert_block, 100, "Limit maximum number of partitions in single INSERTed block. Zero means unlimited. Throw exception if the block contains too many partitions. This setting is a safety threshold, because using large number of partitions is a common misconception.") \
 
-#define DECLARE(TYPE, NAME, DEFAULT, DESCRIPTION) \
-    TYPE NAME {DEFAULT};
-
-    APPLY_FOR_SETTINGS(DECLARE)
-
-#undef DECLARE
-
-    /// Set setting by name.
-    void set(const String & name, const Field & value);
-
-    /// Set setting by name. Read value, serialized in binary form from buffer (for inter-server communication).
-    void set(const String & name, ReadBuffer & buf);
-
-    /// Skip value, serialized in binary form in buffer.
-    void ignore(const String & name, ReadBuffer & buf);
-
-    /// Set setting by name. Read value in text form from string (for example, from configuration file or from URL parameter).
-    void set(const String & name, const String & value);
-
-    /// Get setting by name. Converts value to String.
-    String get(const String & name) const;
-
-    bool tryGet(const String & name, String & value) const;
+    DECLARE_SETTINGS_COLLECTION(APPLY_FOR_SETTINGS)
 
     /** Set multiple settings from "profile" (in server configuration file (users.xml), profiles contain groups of multiple settings).
-      * The profile can also be set using the `set` functions, like the profile setting.
-      */
+     * The profile can also be set using the `set` functions, like the profile setting.
+     */
     void setProfile(const String & profile_name, const Poco::Util::AbstractConfiguration & config);
 
     /// Load settings from configuration file, at "path" prefix in configuration.
     void loadSettingsFromConfig(const String & path, const Poco::Util::AbstractConfiguration & config);
 
-    /// Read settings from buffer. They are serialized as list of contiguous name-value pairs, finished with empty name.
-    /// If readonly=1 is set, ignore read settings.
-    void deserialize(ReadBuffer & buf);
-
-    /// Write changed settings to buffer. (For example, to be sent to remote server.)
-    void serialize(WriteBuffer & buf) const;
-
     /// Dumps profile events to two columns of type Array(String)
     void dumpToArrayColumns(IColumn * column_names, IColumn * column_values, bool changed_only = true);
 };
 
-
 }
diff --git a/dbms/src/Core/SettingsCommon.cpp b/dbms/src/Core/SettingsCommon.cpp
index 02a9348dfd6..634d428777d 100644
--- a/dbms/src/Core/SettingsCommon.cpp
+++ b/dbms/src/Core/SettingsCommon.cpp
@@ -25,6 +25,7 @@ namespace ErrorCodes
     extern const int UNKNOWN_LOG_LEVEL;
     extern const int SIZE_OF_FIXED_STRING_DOESNT_MATCH;
     extern const int BAD_ARGUMENTS;
+    extern const int UNKNOWN_SETTING;
 }
 
 template <typename IntType>
@@ -33,6 +34,12 @@ String SettingInt<IntType>::toString() const
     return DB::toString(value);
 }
 
+template <typename IntType>
+Field SettingInt<IntType>::toField() const
+{
+    return value;
+}
+
 template <typename IntType>
 void SettingInt<IntType>::set(IntType x)
 {
@@ -43,7 +50,10 @@ void SettingInt<IntType>::set(IntType x)
 template <typename IntType>
 void SettingInt<IntType>::set(const Field & x)
 {
-    set(applyVisitor(FieldVisitorConvertToNumber<IntType>(), x));
+    if (x.getType() == Field::Types::String)
+        set(get<const String &>(x));
+    else
+        set(applyVisitor(FieldVisitorConvertToNumber<IntType>(), x));
 }
 
 template <typename IntType>
@@ -53,19 +63,19 @@ void SettingInt<IntType>::set(const String & x)
 }
 
 template <typename IntType>
-void SettingInt<IntType>::set(ReadBuffer & buf)
+void SettingInt<IntType>::serialize(WriteBuffer & buf) const
+{
+    writeVarT(value, buf);
+}
+
+template <typename IntType>
+void SettingInt<IntType>::deserialize(ReadBuffer & buf)
 {
     IntType x = 0;
     readVarT(x, buf);
     set(x);
 }
 
-template <typename IntType>
-void SettingInt<IntType>::write(WriteBuffer & buf) const
-{
-    writeVarT(value, buf);
-}
-
 template struct SettingInt<UInt64>;
 template struct SettingInt<Int64>;
 
@@ -76,6 +86,11 @@ String SettingMaxThreads::toString() const
     return DB::toString(value);
 }
 
+Field SettingMaxThreads::toField() const
+{
+    return is_auto ? 0 : value;
+}
+
 void SettingMaxThreads::set(UInt64 x)
 {
     value = x ? x : getAutoValue();
@@ -86,7 +101,7 @@ void SettingMaxThreads::set(UInt64 x)
 void SettingMaxThreads::set(const Field & x)
 {
     if (x.getType() == Field::Types::String)
-        set(safeGet<const String &>(x));
+        set(get<const String &>(x));
     else
         set(safeGet<UInt64>(x));
 }
@@ -99,18 +114,18 @@ void SettingMaxThreads::set(const String & x)
         set(parse<UInt64>(x));
 }
 
-void SettingMaxThreads::set(ReadBuffer & buf)
+void SettingMaxThreads::serialize(WriteBuffer & buf) const
+{
+    writeVarUInt(is_auto ? 0 : value, buf);
+}
+
+void SettingMaxThreads::deserialize(ReadBuffer & buf)
 {
     UInt64 x = 0;
     readVarUInt(x, buf);
     set(x);
 }
 
-void SettingMaxThreads::write(WriteBuffer & buf) const
-{
-    writeVarUInt(is_auto ? 0 : value, buf);
-}
-
 void SettingMaxThreads::setAuto()
 {
     value = getAutoValue();
@@ -135,6 +150,11 @@ String SettingSeconds::toString() const
     return DB::toString(totalSeconds());
 }
 
+Field SettingSeconds::toField() const
+{
+    return totalSeconds();
+}
+
 void SettingSeconds::set(const Poco::Timespan & x)
 {
     value = x;
@@ -148,7 +168,10 @@ void SettingSeconds::set(UInt64 x)
 
 void SettingSeconds::set(const Field & x)
 {
-    set(safeGet<UInt64>(x));
+    if (x.getType() == Field::Types::String)
+        set(get<const String &>(x));
+    else
+        set(safeGet<UInt64>(x));
 }
 
 void SettingSeconds::set(const String & x)
@@ -156,24 +179,29 @@ void SettingSeconds::set(const String & x)
     set(parse<UInt64>(x));
 }
 
-void SettingSeconds::set(ReadBuffer & buf)
+void SettingSeconds::serialize(WriteBuffer & buf) const
+{
+    writeVarUInt(value.totalSeconds(), buf);
+}
+
+void SettingSeconds::deserialize(ReadBuffer & buf)
 {
     UInt64 x = 0;
     readVarUInt(x, buf);
     set(x);
 }
 
-void SettingSeconds::write(WriteBuffer & buf) const
-{
-    writeVarUInt(value.totalSeconds(), buf);
-}
-
 
 String SettingMilliseconds::toString() const
 {
     return DB::toString(totalMilliseconds());
 }
 
+Field SettingMilliseconds::toField() const
+{
+    return totalMilliseconds();
+}
+
 void SettingMilliseconds::set(const Poco::Timespan & x)
 {
     value = x;
@@ -187,7 +215,10 @@ void SettingMilliseconds::set(UInt64 x)
 
 void SettingMilliseconds::set(const Field & x)
 {
-    set(safeGet<UInt64>(x));
+    if (x.getType() == Field::Types::String)
+        set(get<const String &>(x));
+    else
+        set(safeGet<UInt64>(x));
 }
 
 void SettingMilliseconds::set(const String & x)
@@ -195,24 +226,29 @@ void SettingMilliseconds::set(const String & x)
     set(parse<UInt64>(x));
 }
 
-void SettingMilliseconds::set(ReadBuffer & buf)
+void SettingMilliseconds::serialize(WriteBuffer & buf) const
+{
+    writeVarUInt(value.totalMilliseconds(), buf);
+}
+
+void SettingMilliseconds::deserialize(ReadBuffer & buf)
 {
     UInt64 x = 0;
     readVarUInt(x, buf);
     set(x);
 }
 
-void SettingMilliseconds::write(WriteBuffer & buf) const
-{
-    writeVarUInt(value.totalMilliseconds(), buf);
-}
-
 
 String SettingFloat::toString() const
 {
     return DB::toString(value);
 }
 
+Field SettingFloat::toField() const
+{
+    return value;
+}
+
 void SettingFloat::set(float x)
 {
     value = x;
@@ -221,7 +257,10 @@ void SettingFloat::set(float x)
 
 void SettingFloat::set(const Field & x)
 {
-    set(applyVisitor(FieldVisitorConvertToNumber<float>(), x));
+    if (x.getType() == Field::Types::String)
+        set(get<const String &>(x));
+    else
+        set(applyVisitor(FieldVisitorConvertToNumber<float>(), x));
 }
 
 void SettingFloat::set(const String & x)
@@ -229,18 +268,18 @@ void SettingFloat::set(const String & x)
     set(parse<float>(x));
 }
 
-void SettingFloat::set(ReadBuffer & buf)
+void SettingFloat::serialize(WriteBuffer & buf) const
+{
+    writeBinary(toString(), buf);
+}
+
+void SettingFloat::deserialize(ReadBuffer & buf)
 {
     String x;
     readBinary(x, buf);
     set(x);
 }
 
-void SettingFloat::write(WriteBuffer & buf) const
-{
-    writeBinary(toString(), buf);
-}
-
 
 LoadBalancing SettingLoadBalancing::getLoadBalancing(const String & s)
 {
@@ -261,6 +300,11 @@ String SettingLoadBalancing::toString() const
     return strings[static_cast<size_t>(value)];
 }
 
+Field SettingLoadBalancing::toField() const
+{
+    return toString();
+}
+
 void SettingLoadBalancing::set(LoadBalancing x)
 {
     value = x;
@@ -277,18 +321,18 @@ void SettingLoadBalancing::set(const String & x)
     set(getLoadBalancing(x));
 }
 
-void SettingLoadBalancing::set(ReadBuffer & buf)
-{
-    String x;
-    readBinary(x, buf);
-    set(x);
-}
-
-void SettingLoadBalancing::write(WriteBuffer & buf) const
+void SettingLoadBalancing::serialize(WriteBuffer & buf) const
 {
     writeBinary(toString(), buf);
 }
 
+void SettingLoadBalancing::deserialize(ReadBuffer & buf)
+{
+    String s;
+    readBinary(s, buf);
+    set(s);
+}
+
 
 JoinStrictness SettingJoinStrictness::getJoinStrictness(const String & s)
 {
@@ -308,6 +352,11 @@ String SettingJoinStrictness::toString() const
     return strings[static_cast<size_t>(value)];
 }
 
+Field SettingJoinStrictness::toField() const
+{
+    return toString();
+}
+
 void SettingJoinStrictness::set(JoinStrictness x)
 {
     value = x;
@@ -324,18 +373,18 @@ void SettingJoinStrictness::set(const String & x)
     set(getJoinStrictness(x));
 }
 
-void SettingJoinStrictness::set(ReadBuffer & buf)
-{
-    String x;
-    readBinary(x, buf);
-    set(x);
-}
-
-void SettingJoinStrictness::write(WriteBuffer & buf) const
+void SettingJoinStrictness::serialize(WriteBuffer & buf) const
 {
     writeBinary(toString(), buf);
 }
 
+void SettingJoinStrictness::deserialize(ReadBuffer & buf)
+{
+    String s;
+    readBinary(s, buf);
+    set(s);
+}
+
 
 TotalsMode SettingTotalsMode::getTotalsMode(const String & s)
 {
@@ -360,6 +409,11 @@ String SettingTotalsMode::toString() const
     __builtin_unreachable();
 }
 
+Field SettingTotalsMode::toField() const
+{
+    return toString();
+}
+
 void SettingTotalsMode::set(TotalsMode x)
 {
     value = x;
@@ -376,18 +430,19 @@ void SettingTotalsMode::set(const String & x)
     set(getTotalsMode(x));
 }
 
-void SettingTotalsMode::set(ReadBuffer & buf)
-{
-    String x;
-    readBinary(x, buf);
-    set(x);
-}
-
-void SettingTotalsMode::write(WriteBuffer & buf) const
+void SettingTotalsMode::serialize(WriteBuffer & buf) const
 {
     writeBinary(toString(), buf);
 }
 
+void SettingTotalsMode::deserialize(ReadBuffer & buf)
+{
+    String s;
+    readBinary(s, buf);
+    set(s);
+}
+
+
 
 template <bool enable_mode_any>
 OverflowMode SettingOverflowMode<enable_mode_any>::getOverflowModeForGroupBy(const String & s)
@@ -421,6 +476,12 @@ String SettingOverflowMode<enable_mode_any>::toString() const
     return strings[static_cast<size_t>(value)];
 }
 
+template <bool enable_mode_any>
+Field SettingOverflowMode<enable_mode_any>::toField() const
+{
+    return toString();
+}
+
 template <bool enable_mode_any>
 void SettingOverflowMode<enable_mode_any>::set(OverflowMode x)
 {
@@ -441,19 +502,20 @@ void SettingOverflowMode<enable_mode_any>::set(const String & x)
 }
 
 template <bool enable_mode_any>
-void SettingOverflowMode<enable_mode_any>::set(ReadBuffer & buf)
-{
-    String x;
-    readBinary(x, buf);
-    set(x);
-}
-
-template <bool enable_mode_any>
-void SettingOverflowMode<enable_mode_any>::write(WriteBuffer & buf) const
+void SettingOverflowMode<enable_mode_any>::serialize(WriteBuffer & buf) const
 {
     writeBinary(toString(), buf);
 }
 
+template <bool enable_mode_any>
+void SettingOverflowMode<enable_mode_any>::deserialize(ReadBuffer & buf)
+{
+    String s;
+    readBinary(s, buf);
+    set(s);
+}
+
+
 template struct SettingOverflowMode<false>;
 template struct SettingOverflowMode<true>;
 
@@ -476,6 +538,11 @@ String SettingDistributedProductMode::toString() const
     return strings[static_cast<size_t>(value)];
 }
 
+Field SettingDistributedProductMode::toField() const
+{
+    return toString();
+}
+
 void SettingDistributedProductMode::set(DistributedProductMode x)
 {
     value = x;
@@ -492,24 +559,29 @@ void SettingDistributedProductMode::set(const String & x)
     set(getDistributedProductMode(x));
 }
 
-void SettingDistributedProductMode::set(ReadBuffer & buf)
-{
-    String x;
-    readBinary(x, buf);
-    set(x);
-}
-
-void SettingDistributedProductMode::write(WriteBuffer & buf) const
+void SettingDistributedProductMode::serialize(WriteBuffer & buf) const
 {
     writeBinary(toString(), buf);
 }
 
+void SettingDistributedProductMode::deserialize(ReadBuffer & buf)
+{
+    String s;
+    readBinary(s, buf);
+    set(s);
+}
+
 
 String SettingString::toString() const
 {
     return value;
 }
 
+Field SettingString::toField() const
+{
+    return value;
+}
+
 void SettingString::set(const String & x)
 {
     value = x;
@@ -521,24 +593,29 @@ void SettingString::set(const Field & x)
     set(safeGet<const String &>(x));
 }
 
-void SettingString::set(ReadBuffer & buf)
-{
-    String x;
-    readBinary(x, buf);
-    set(x);
-}
-
-void SettingString::write(WriteBuffer & buf) const
+void SettingString::serialize(WriteBuffer & buf) const
 {
     writeBinary(value, buf);
 }
 
+void SettingString::deserialize(ReadBuffer & buf)
+{
+    String s;
+    readBinary(s, buf);
+    set(s);
+}
+
 
 String SettingChar::toString() const
 {
     return String(1, value);
 }
 
+Field SettingChar::toField() const
+{
+    return toString();
+}
+
 void SettingChar::set(char x)
 {
     value = x;
@@ -559,18 +636,18 @@ void SettingChar::set(const Field & x)
     set(s);
 }
 
-void SettingChar::set(ReadBuffer & buf)
+void SettingChar::serialize(WriteBuffer & buf) const
+{
+    writeBinary(toString(), buf);
+}
+
+void SettingChar::deserialize(ReadBuffer & buf)
 {
     String s;
     readBinary(s, buf);
     set(s);
 }
 
-void SettingChar::write(WriteBuffer & buf) const
-{
-    writeBinary(toString(), buf);
-}
-
 
 SettingDateTimeInputFormat::Value SettingDateTimeInputFormat::getValue(const String & s)
 {
@@ -588,6 +665,11 @@ String SettingDateTimeInputFormat::toString() const
     return strings[static_cast<size_t>(value)];
 }
 
+Field SettingDateTimeInputFormat::toField() const
+{
+    return toString();
+}
+
 void SettingDateTimeInputFormat::set(Value x)
 {
     value = x;
@@ -604,18 +686,18 @@ void SettingDateTimeInputFormat::set(const String & x)
     set(getValue(x));
 }
 
-void SettingDateTimeInputFormat::set(ReadBuffer & buf)
-{
-    String x;
-    readBinary(x, buf);
-    set(x);
-}
-
-void SettingDateTimeInputFormat::write(WriteBuffer & buf) const
+void SettingDateTimeInputFormat::serialize(WriteBuffer & buf) const
 {
     writeBinary(toString(), buf);
 }
 
+void SettingDateTimeInputFormat::deserialize(ReadBuffer & buf)
+{
+    String s;
+    readBinary(s, buf);
+    set(s);
+}
+
 
 SettingLogsLevel::Value SettingLogsLevel::getValue(const String & s)
 {
@@ -635,6 +717,11 @@ String SettingLogsLevel::toString() const
     return strings[static_cast<size_t>(value)];
 }
 
+Field SettingLogsLevel::toField() const
+{
+    return toString();
+}
+
 void SettingLogsLevel::set(Value x)
 {
     value = x;
@@ -651,16 +738,33 @@ void SettingLogsLevel::set(const String & x)
     set(getValue(x));
 }
 
-void SettingLogsLevel::set(ReadBuffer & buf)
-{
-    String x;
-    readBinary(x, buf);
-    set(x);
-}
-
-void SettingLogsLevel::write(WriteBuffer & buf) const
+void SettingLogsLevel::serialize(WriteBuffer & buf) const
 {
     writeBinary(toString(), buf);
 }
 
+void SettingLogsLevel::deserialize(ReadBuffer & buf)
+{
+    String s;
+    readBinary(s, buf);
+    set(s);
+}
+
+
+namespace details
+{
+    String SettingsCollectionUtils::deserializeName(ReadBuffer & buf)
+    {
+        String name;
+        readBinary(name, buf);
+        return name;
+    }
+
+    void SettingsCollectionUtils::serializeName(const StringRef & name, WriteBuffer & buf) { writeBinary(name, buf); }
+
+    void SettingsCollectionUtils::throwNameNotFound(const StringRef & name)
+    {
+        throw Exception("Unknown setting " + name.toString(), ErrorCodes::UNKNOWN_SETTING);
+    }
+}
 }
diff --git a/dbms/src/Core/SettingsCommon.h b/dbms/src/Core/SettingsCommon.h
index c661cef1570..b7f7f23474f 100644
--- a/dbms/src/Core/SettingsCommon.h
+++ b/dbms/src/Core/SettingsCommon.h
@@ -3,8 +3,12 @@
 #include <Poco/Timespan.h>
 #include <DataStreams/SizeLimits.h>
 #include <Formats/FormatSettings.h>
+#include <common/StringRef.h>
+#include <Common/SettingsChanges.h>
 #include <Compression/CompressionInfo.h>
 #include <Core/Types.h>
+#include <ext/singleton.h>
+#include <unordered_map>
 
 
 namespace DB
@@ -36,8 +40,8 @@ struct SettingInt
     /// Serialize to a test string.
     String toString() const;
 
-    /// Serialize to binary stream suitable for transfer over network.
-    void write(WriteBuffer & buf) const;
+    /// Converts to a field.
+    Field toField() const;
 
     void set(IntType x);
 
@@ -47,8 +51,11 @@ struct SettingInt
     /// Read from text string.
     void set(const String & x);
 
+    /// Serialize to binary stream suitable for transfer over network.
+    void serialize(WriteBuffer & buf) const;
+
     /// Read from binary stream.
-    void set(ReadBuffer & buf);
+    void deserialize(ReadBuffer & buf);
 };
 
 using SettingUInt64 = SettingInt<UInt64>;
@@ -72,13 +79,14 @@ struct SettingMaxThreads
     SettingMaxThreads & operator= (UInt64 x) { set(x); return *this; }
 
     String toString() const;
+    Field toField() const;
 
     void set(UInt64 x);
     void set(const Field & x);
     void set(const String & x);
-    void set(ReadBuffer & buf);
 
-    void write(WriteBuffer & buf) const;
+    void serialize(WriteBuffer & buf) const;
+    void deserialize(ReadBuffer & buf);
 
     void setAuto();
     UInt64 getAutoValue() const;
@@ -101,15 +109,16 @@ struct SettingSeconds
     Poco::Timespan::TimeDiff totalSeconds() const { return value.totalSeconds(); }
 
     String toString() const;
+    Field toField() const;
 
     void set(const Poco::Timespan & x);
 
     void set(UInt64 x);
     void set(const Field & x);
     void set(const String & x);
-    void set(ReadBuffer & buf);
 
-    void write(WriteBuffer & buf) const;
+    void serialize(WriteBuffer & buf) const;
+    void deserialize(ReadBuffer & buf);
 };
 
 
@@ -126,13 +135,15 @@ struct SettingMilliseconds
     Poco::Timespan::TimeDiff totalMilliseconds() const { return value.totalMilliseconds(); }
 
     String toString() const;
+    Field toField() const;
 
     void set(const Poco::Timespan & x);
     void set(UInt64 x);
     void set(const Field & x);
     void set(const String & x);
-    void set(ReadBuffer & buf);
-    void write(WriteBuffer & buf) const;
+
+    void serialize(WriteBuffer & buf) const;
+    void deserialize(ReadBuffer & buf);
 };
 
 
@@ -147,13 +158,14 @@ struct SettingFloat
     SettingFloat & operator= (float x) { set(x); return *this; }
 
     String toString() const;
+    Field toField() const;
 
     void set(float x);
     void set(const Field & x);
     void set(const String & x);
-    void set(ReadBuffer & buf);
 
-    void write(WriteBuffer & buf) const;
+    void serialize(WriteBuffer & buf) const;
+    void deserialize(ReadBuffer & buf);
 };
 
 
@@ -185,13 +197,14 @@ struct SettingLoadBalancing
     static LoadBalancing getLoadBalancing(const String & s);
 
     String toString() const;
+    Field toField() const;
 
     void set(LoadBalancing x);
     void set(const Field & x);
     void set(const String & x);
-    void set(ReadBuffer & buf);
 
-    void write(WriteBuffer & buf) const;
+    void serialize(WriteBuffer & buf) const;
+    void deserialize(ReadBuffer & buf);
 };
 
 
@@ -216,13 +229,14 @@ struct SettingJoinStrictness
     static JoinStrictness getJoinStrictness(const String & s);
 
     String toString() const;
+    Field toField() const;
 
     void set(JoinStrictness x);
     void set(const Field & x);
     void set(const String & x);
-    void set(ReadBuffer & buf);
 
-    void write(WriteBuffer & buf) const;
+    void serialize(WriteBuffer & buf) const;
+    void deserialize(ReadBuffer & buf);
 };
 
 
@@ -251,13 +265,14 @@ struct SettingTotalsMode
     static TotalsMode getTotalsMode(const String & s);
 
     String toString() const;
+    Field toField() const;
 
     void set(TotalsMode x);
     void set(const Field & x);
     void set(const String & x);
-    void set(ReadBuffer & buf);
 
-    void write(WriteBuffer & buf) const;
+    void serialize(WriteBuffer & buf) const;
+    void deserialize(ReadBuffer & buf);
 };
 
 
@@ -276,13 +291,14 @@ struct SettingOverflowMode
     static OverflowMode getOverflowMode(const String & s);
 
     String toString() const;
+    Field toField() const;
 
     void set(OverflowMode x);
     void set(const Field & x);
     void set(const String & x);
-    void set(ReadBuffer & buf);
 
-    void write(WriteBuffer & buf) const;
+    void serialize(WriteBuffer & buf) const;
+    void deserialize(ReadBuffer & buf);
 };
 
 /// The setting for executing distributed subqueries inside IN or JOIN sections.
@@ -307,12 +323,14 @@ struct SettingDistributedProductMode
     static DistributedProductMode getDistributedProductMode(const String & s);
 
     String toString() const;
+    Field toField() const;
 
     void set(DistributedProductMode x);
     void set(const Field & x);
     void set(const String & x);
-    void set(ReadBuffer & buf);
-    void write(WriteBuffer & buf) const;
+
+    void serialize(WriteBuffer & buf) const;
+    void deserialize(ReadBuffer & buf);
 };
 
 
@@ -327,12 +345,13 @@ struct SettingString
     SettingString & operator= (const String & x) { set(x); return *this; }
 
     String toString() const;
+    Field toField() const;
 
     void set(const String & x);
     void set(const Field & x);
-    void set(ReadBuffer & buf);
 
-    void write(WriteBuffer & buf) const;
+    void serialize(WriteBuffer & buf) const;
+    void deserialize(ReadBuffer & buf);
 };
 
 
@@ -348,13 +367,14 @@ public:
     SettingChar & operator= (char x) { set(x); return *this; }
 
     String toString() const;
+    Field toField() const;
 
     void set(char x);
     void set(const String & x);
     void set(const Field & x);
-    void set(ReadBuffer & buf);
 
-    void write(WriteBuffer & buf) const;
+    void serialize(WriteBuffer & buf) const;
+    void deserialize(ReadBuffer & buf);
 };
 
 
@@ -373,12 +393,14 @@ struct SettingDateTimeInputFormat
     static Value getValue(const String & s);
 
     String toString() const;
+    Field toField() const;
 
     void set(Value x);
     void set(const Field & x);
     void set(const String & x);
-    void set(ReadBuffer & buf);
-    void write(WriteBuffer & buf) const;
+
+    void serialize(WriteBuffer & buf) const;
+    void deserialize(ReadBuffer & buf);
 };
 
 
@@ -408,12 +430,351 @@ public:
     static Value getValue(const String & s);
 
     String toString() const;
+    Field toField() const;
 
     void set(Value x);
     void set(const Field & x);
     void set(const String & x);
-    void set(ReadBuffer & buf);
-    void write(WriteBuffer & buf) const;
+
+    void serialize(WriteBuffer & buf) const;
+    void deserialize(ReadBuffer & buf);
 };
 
+
+namespace details
+{
+    struct SettingsCollectionUtils
+    {
+        static void serializeName(const StringRef & name, WriteBuffer & buf);
+        static String deserializeName(ReadBuffer & buf);
+        static void throwNameNotFound(const StringRef & name);
+    };
+}
+
+
+/** Template class to define collections of settings.
+  * Example of usage:
+  *
+  * mysettings.h:
+  * struct MySettings : public SettingsCollection<MySettings>
+  * {
+  * #   define APPLY_FOR_MYSETTINGS(M) \
+  *         M(SettingUInt64, a, 100, "Description of a") \
+  *         M(SettingFloat, f, 3.11, "Description of f") \
+  *         M(SettingString, s, "default", "Description of s")
+  *
+  *     DECLARE_SETTINGS_COLLECTION(MySettings, APPLY_FOR_MYSETTINGS)
+  * };
+  *
+  * mysettings.cpp:
+  * IMPLEMENT_SETTINGS_COLLECTION(MySettings, APPLY_FOR_MYSETTINGS)
+  */
+template <typename T>
+class SettingsCollection
+{
+private:
+    using Type = T;
+    using GetStringFunction = String (*)(const Type &);
+    using GetFieldFunction = Field (*)(const Type &);
+    using SetStringFunction = void (*)(Type &, const String &);
+    using SetFieldFunction = void (*)(Type &, const Field &);
+    using SerializeFunction = void (*)(const Type &, WriteBuffer & buf);
+    using DeserializeFunction = void (*)(Type &, ReadBuffer & buf);
+
+    struct MemberInfo
+    {
+        size_t offset_of_changed;
+        StringRef name;
+        StringRef description;
+        GetStringFunction get_string;
+        GetFieldFunction get_field;
+        SetStringFunction set_string;
+        SetFieldFunction set_field;
+        SerializeFunction serialize;
+        DeserializeFunction deserialize;
+        bool isChanged(const Type & collection) const { return *reinterpret_cast<const bool*>(reinterpret_cast<const UInt8*>(&collection) + offset_of_changed); }
+    };
+
+    using MemberInfos = std::vector<MemberInfo>;
+
+    class Layout
+    {
+    public:
+        size_t size() const { return infos.size(); }
+        const MemberInfo & operator[](size_t index) const { return infos[index]; }
+
+        static constexpr size_t npos = static_cast<size_t>(-1);
+
+        size_t find(const StringRef & name) const
+        {
+            auto map_it = by_name_map.find(name);
+            if (map_it == by_name_map.end())
+                return npos;
+            else
+                return map_it->second;
+        }
+
+        size_t findStrict(const StringRef & name) const
+        {
+            auto map_it = by_name_map.find(name);
+            if (map_it == by_name_map.end())
+                details::SettingsCollectionUtils::throwNameNotFound(name);
+            return map_it->second;
+        }
+
+        void addMemberInfo(MemberInfo && info)
+        {
+            size_t index = infos.size();
+            infos.emplace_back(info);
+            by_name_map.emplace(infos.back().name, index);
+        }
+
+    private:
+        MemberInfos infos;
+        std::unordered_map<StringRef, size_t> by_name_map;
+    };
+
+    static const Layout & getLayout();
+    Type & castThis() { return static_cast<Type &>(*this); }
+    const Type & castThis() const { return static_cast<const Type &>(*this); }
+
+public:
+    /// Provides access to a setting.
+    class reference
+    {
+    public:
+        const StringRef & getName() const { return member.name; }
+        const StringRef & getDescription() const { return member.description; }
+        bool isChanged() const { return member.isChanged(collection); }
+        Field getValue() const { return member.get_field(collection); }
+        String getValueAsString() const { return member.get_string(collection); }
+        void setValue(const Field & value) { member.set_field(collection, value); }
+        void setValue(const String & value) { member.set_string(collection, value); }
+
+        reference(Type & collection_, const MemberInfo & member_) : collection(collection_), member(member_) {}
+    private:
+        Type & collection;
+        const MemberInfo & member;
+    };
+
+    /// Provides read-only access to a setting.
+    class const_reference
+    {
+    public:
+        const StringRef & getName() const { return member.name; }
+        const StringRef & getDescription() const { return member.description; }
+        bool isChanged() const { return member.isChanged(collection); }
+        Field getValue() const { return member.get_field(collection); }
+        String getValueAsString() const { return member.get_string(collection); }
+
+        const_reference(const Type & collection_, const MemberInfo & member_) : collection(collection_), member(member_) {}
+    private:
+        const Type & collection;
+        const MemberInfo & member;
+    };
+
+    /// Returns number of settings.
+    size_t size() const { return getLayout().size(); }
+
+    /// Returns a setting by its zero-based index.
+    reference operator [](size_t index) { return reference(castThis(), getLayout()[index]); }
+    const_reference operator [](size_t index) const { return const_reference(castThis(), getLayout()[index]); }
+
+    /// Returns a setting by its name. Throws an exception if there is not setting with such name.
+    reference operator [](const String & name)
+    {
+        const Layout & layout = getLayout();
+        return reference(castThis(), layout[layout.findStrict(name)]);
+    }
+
+    const_reference operator [](const String & name) const
+    {
+        const Layout & layout = getLayout();
+        return const_reference(castThis(), layout[layout.findStrict(name)]);
+    }
+
+    /// Finds a setting by name and returns its index. Returns npos if not found.
+    size_t find(const String & name) const { return getLayout().find(name); }
+    static constexpr size_t npos = static_cast<size_t>(-1);
+
+    /// Finds a setting by name and returns its index. Throws an exception if not found.
+    size_t findStrict(const String & name) const { return getLayout().findStrict(name); }
+
+    /// Sets setting by name.
+    void set(const String & name, const Field & value) { (*this)[name].setValue(value); }
+
+    /// Sets setting by name. Read value in text form from string (for example, from configuration file or from URL parameter).
+    void set(const String & name, const String & value) { (*this)[name].setValue(value); }
+
+    /// Returns the value of a setting.
+    Field get(const String & name) const { return (*this)[name].getValue(); }
+
+    /// Returns the value of a setting converted to string.
+    String getAsString(const String & name) const { return (*this)[name].getValueAsString(); }
+
+    /// Returns the value of a setting.
+    bool tryGet(const String & name, Field & value) const
+    {
+        const Layout & layout = getLayout();
+        size_t index = layout.find(name);
+        if (index == npos)
+            return false;
+        value = layout[index].get_field(castThis());
+        return true;
+    }
+
+    /// Returns the value of a setting converted to string.
+    bool tryGet(const String & name, String & value) const
+    {
+        const Layout & layout = getLayout();
+        size_t index = layout.find(name);
+        if (index == npos)
+            return false;
+        value = layout[index].get_string(castThis());
+        return true;
+    }
+
+    /// Compares two collections of settings.
+    bool operator ==(const Type & rhs) const
+    {
+        const Layout & layout = getLayout();
+        for (size_t i = 0; i != layout.size(); ++i)
+        {
+            const auto & member = layout[i];
+            bool left_changed = member.isChanged(castThis());
+            bool right_changed = member.isChanged(rhs);
+            if (left_changed || right_changed)
+            {
+                if (left_changed != right_changed)
+                    return false;
+                if (member.get_field(castThis()) != member.get_field(rhs))
+                    return false;
+            }
+        }
+        return true;
+    }
+
+    bool operator !=(const Type & rhs) const
+    {
+        return !(*this == rhs);
+    }
+
+    /// Gathers all changed values (e.g. for applying them later to another collection of settings).
+    SettingsChanges changes() const
+    {
+        SettingsChanges found_changes;
+        const Layout & layout = getLayout();
+        for (size_t i = 0; i != layout.size(); ++i)
+        {
+            const auto & member = layout[i];
+            if (member.isChanged(castThis()))
+                found_changes.emplace_back(member.name.toString(), member.get_field(castThis()));
+        }
+        return found_changes;
+    }
+
+    /// Applies changes to the settings.
+    void applyChange(const SettingChange & change)
+    {
+        set(change.name, change.value);
+    }
+
+    void applyChanges(const SettingsChanges & changes)
+    {
+        for (const SettingChange & change : changes)
+            applyChange(change);
+    }
+
+    void copyChangesFrom(const Type & src)
+    {
+        const Layout & layout = getLayout();
+        for (size_t i = 0; i != layout.size(); ++i)
+        {
+            const auto & member = layout[i];
+            if (member.isChanged(src))
+                member.set_field(castThis(), member.get_field(src));
+        }
+    }
+
+    void copyChangesTo(Type & dest) const
+    {
+        dest.copyChangesFrom(castThis());
+    }
+
+    /// Writes the settings to buffer (e.g. to be sent to remote server).
+    /// Only changed settings are written. They are written as list of contiguous name-value pairs,
+    /// finished with empty name.
+    void serialize(WriteBuffer & buf) const
+    {
+        const Layout & layout = getLayout();
+        for (size_t i = 0; i != layout.size(); ++i)
+        {
+            const auto & member = layout[i];
+            if (member.isChanged(castThis()))
+            {
+                details::SettingsCollectionUtils::serializeName(member.name, buf);
+                member.serialize(castThis(), buf);
+            }
+        }
+        details::SettingsCollectionUtils::serializeName(StringRef{} /* empty string is a marker of the end of settings */, buf);
+    }
+
+    /// Reads the settings from buffer.
+    void deserialize(ReadBuffer & buf)
+    {
+        const Layout & layout = getLayout();
+        while (true)
+        {
+            String name = details::SettingsCollectionUtils::deserializeName(buf);
+            if (name.empty() /* empty string is a marker of the end of settings */)
+                break; \
+            const auto & member = layout[layout.findStrict(name)];
+            member.deserialize(castThis(), buf); \
+        }
+    }
+};
+
+#define DECLARE_SETTINGS_COLLECTION(APPLY_MACRO) \
+    APPLY_MACRO(DECLARE_SETTINGS_COLLECTION_DECLARE_VARIABLES_HELPER_)
+
+
+#define IMPLEMENT_SETTINGS_COLLECTION(CLASS_NAME, APPLY_MACRO) \
+    template<> \
+    const SettingsCollection<CLASS_NAME>::Layout & SettingsCollection<CLASS_NAME>::getLayout() \
+    { \
+        struct Functions \
+        { \
+            APPLY_MACRO(IMPLEMENT_SETTINGS_COLLECTION_DEFINE_FUNCTIONS_HELPER_) \
+        }; \
+        static const SettingsCollection<Type>::Layout single_instance = [] \
+        { \
+            SettingsCollection<Type>::Layout layout; \
+            APPLY_MACRO(IMPLEMENT_SETTINGS_COLLECTION_ADD_MEMBER_INFO_HELPER_) \
+            return layout; \
+        }(); \
+        return single_instance; \
+    }
+
+
+#define DECLARE_SETTINGS_COLLECTION_DECLARE_VARIABLES_HELPER_(TYPE, NAME, DEFAULT, DESCRIPTION) \
+    TYPE NAME {DEFAULT};
+
+
+#define IMPLEMENT_SETTINGS_COLLECTION_DEFINE_FUNCTIONS_HELPER_(TYPE, NAME, DEFAULT, DESCRIPTION) \
+    static String NAME##_getString(const Type & collection) { return collection.NAME.toString(); } \
+    static Field NAME##_getField(const Type & collection) { return collection.NAME.toField(); } \
+    static void NAME##_setString(Type & collection, const String & value) { collection.NAME.set(value); } \
+    static void NAME##_setField(Type & collection, const Field & value) { collection.NAME.set(value); } \
+    static void NAME##_serialize(const Type & collection, WriteBuffer & buf) { collection.NAME.serialize(buf); } \
+    static void NAME##_deserialize(Type & collection, ReadBuffer & buf) { collection.NAME.deserialize(buf); }
+
+
+#define IMPLEMENT_SETTINGS_COLLECTION_ADD_MEMBER_INFO_HELPER_(TYPE, NAME, DEFAULT, DESCRIPTION) \
+    static_assert(std::is_same_v<decltype(std::declval<TYPE>().changed), bool>); \
+    layout.addMemberInfo({offsetof(Type, NAME.changed), \
+                          StringRef(#NAME, strlen(#NAME)), StringRef(#DESCRIPTION, strlen(#DESCRIPTION)), \
+                          &Functions::NAME##_getString, &Functions::NAME##_getField, \
+                          &Functions::NAME##_setString, &Functions::NAME##_setField, \
+                          &Functions::NAME##_serialize, &Functions::NAME##_deserialize });
+
 }
diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp
index ada120450e8..01fb726273b 100644
--- a/dbms/src/Interpreters/Context.cpp
+++ b/dbms/src/Interpreters/Context.cpp
@@ -36,6 +36,7 @@
 #include <Interpreters/Cluster.h>
 #include <Interpreters/InterserverIOHandler.h>
 #include <Interpreters/Compiler.h>
+#include <Interpreters/SettingsConstraints.h>
 #include <Interpreters/SystemLog.h>
 #include <Interpreters/QueryLog.h>
 #include <Interpreters/QueryThreadLog.h>
@@ -1027,27 +1028,53 @@ void Context::setSettings(const Settings & settings_)
 }
 
 
-void Context::setSetting(const String & name, const Field & value)
+void Context::setSetting(const String & name, const String & value)
 {
+    auto lock = getLock();
     if (name == "profile")
     {
-        auto lock = getLock();
-        settings.setProfile(value.safeGet<String>(), *shared->users_config);
+        settings.setProfile(value, *shared->users_config);
+        return;
     }
-    else
-        settings.set(name, value);
+    settings.set(name, value);
 }
 
 
-void Context::setSetting(const String & name, const std::string & value)
+void Context::setSetting(const String & name, const Field & value)
 {
+    auto lock = getLock();
     if (name == "profile")
     {
-        auto lock = getLock();
-        settings.setProfile(value, *shared->users_config);
+        settings.setProfile(value.safeGet<String>(), *shared->users_config);
+        return;
     }
-    else
-        settings.set(name, value);
+    settings.set(name, value);
+}
+
+
+void Context::applySettingChange(const SettingChange & change)
+{
+    setSetting(change.name, change.value);
+}
+
+
+void Context::applySettingsChanges(const SettingsChanges & changes)
+{
+    auto lock = getLock();
+    for (const SettingChange & change : changes)
+        applySettingChange(change);
+}
+
+
+void Context::checkSettingsConstraints(const SettingChange & change)
+{
+    SettingsConstraints::check(settings, change);
+}
+
+
+void Context::checkSettingsConstraints(const SettingsChanges & changes)
+{
+    SettingsConstraints::check(settings, changes);
 }
 
 
diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h
index 4161c1c5144..b6b42e3f994 100644
--- a/dbms/src/Interpreters/Context.h
+++ b/dbms/src/Interpreters/Context.h
@@ -263,11 +263,15 @@ public:
     Settings getSettings() const;
     void setSettings(const Settings & settings_);
 
-    /// Set a setting by name.
+    /// Set settings by name.
+    void setSetting(const String & name, const String & value);
     void setSetting(const String & name, const Field & value);
+    void applySettingChange(const SettingChange & change);
+    void applySettingsChanges(const SettingsChanges & changes);
 
-    /// Set a setting by name. Read the value in text form from a string (for example, from a config, or from a URL parameter).
-    void setSetting(const String & name, const std::string & value);
+    /// Checks the constraints.
+    void checkSettingsConstraints(const SettingChange & change);
+    void checkSettingsConstraints(const SettingsChanges & changes);
 
     const EmbeddedDictionaries & getEmbeddedDictionaries() const;
     const ExternalDictionaries & getExternalDictionaries() const;
diff --git a/dbms/src/Interpreters/InterpreterSetQuery.cpp b/dbms/src/Interpreters/InterpreterSetQuery.cpp
index 08b5bc5a620..f92e9638822 100644
--- a/dbms/src/Interpreters/InterpreterSetQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterSetQuery.cpp
@@ -1,73 +1,25 @@
 #include <Parsers/ASTSetQuery.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/InterpreterSetQuery.h>
-#include <Common/typeid_cast.h>
-#include <Common/FieldVisitors.h>
 
 namespace DB
 {
 
-namespace ErrorCodes
-{
-    extern const int READONLY;
-    extern const int QUERY_IS_PROHIBITED;
-}
-
 
 BlockIO InterpreterSetQuery::execute()
 {
     const auto & ast = query_ptr->as<ASTSetQuery &>();
-
-    checkAccess(ast);
-
-    Context & target = context.getSessionContext();
-    for (const auto & change : ast.changes)
-        target.setSetting(change.name, change.value);
-
+    context.checkSettingsConstraints(ast.changes);
+    context.getSessionContext().applySettingsChanges(ast.changes);
     return {};
 }
 
-void InterpreterSetQuery::checkAccess(const ASTSetQuery & ast)
-{
-    /** The `readonly` value is understood as follows:
-      * 0 - everything allowed.
-      * 1 - only read queries can be made; you can not change the settings.
-      * 2 - You can only do read queries and you can change the settings, except for the `readonly` setting.
-      */
-
-    const Settings & settings = context.getSettingsRef();
-    auto readonly = settings.readonly;
-    auto allow_ddl = settings.allow_ddl;
-
-    for (const auto & change : ast.changes)
-    {
-        String value;
-        /// Setting isn't checked if value wasn't changed.
-        if (!settings.tryGet(change.name, value) || applyVisitor(FieldVisitorToString(), change.value) != value)
-        {
-
-            if (!allow_ddl && change.name == "allow_ddl")
-                throw Exception("Cannot modify 'allow_ddl' setting when DDL queries are prohibited for the user", ErrorCodes::QUERY_IS_PROHIBITED);
-
-            if (readonly == 1)
-                throw Exception("Cannot execute SET query in readonly mode", ErrorCodes::READONLY);
-
-            if (readonly > 1 && change.name == "readonly")
-                throw Exception("Cannot modify 'readonly' setting in readonly mode", ErrorCodes::READONLY);
-        }
-    }
-}
-
 
 void InterpreterSetQuery::executeForCurrentContext()
 {
     const auto & ast = query_ptr->as<ASTSetQuery &>();
-
-    checkAccess(ast);
-
-    for (const auto & change : ast.changes)
-        context.setSetting(change.name, change.value);
+    context.checkSettingsConstraints(ast.changes);
+    context.applySettingsChanges(ast.changes);
 }
 
-
 }
diff --git a/dbms/src/Interpreters/InterpreterSetQuery.h b/dbms/src/Interpreters/InterpreterSetQuery.h
index 434765cf7d0..6e8e8b8b733 100644
--- a/dbms/src/Interpreters/InterpreterSetQuery.h
+++ b/dbms/src/Interpreters/InterpreterSetQuery.h
@@ -23,8 +23,6 @@ public:
       */
     BlockIO execute() override;
 
-    void checkAccess(const ASTSetQuery & ast);
-
     /** Set setting for current context (query context).
       * It is used for interpretation of SETTINGS clause in SELECT query.
       */
diff --git a/dbms/src/Interpreters/SettingsConstraints.cpp b/dbms/src/Interpreters/SettingsConstraints.cpp
new file mode 100644
index 00000000000..2120356bb7c
--- /dev/null
+++ b/dbms/src/Interpreters/SettingsConstraints.cpp
@@ -0,0 +1,65 @@
+#include <Interpreters/SettingsConstraints.h>
+#include <Core/Settings.h>
+
+
+namespace DB
+{
+namespace ErrorCodes
+{
+    extern const int READONLY;
+    extern const int QUERY_IS_PROHIBITED;
+    extern const int NO_ELEMENTS_IN_CONFIG;
+}
+
+namespace
+{
+    thread_local Settings temp_settings;
+
+    void checkImpl(const Settings & current_settings, size_t index)
+    {
+        const auto & new_setting = temp_settings[index];
+        Field new_value = new_setting.getValue();
+
+        const auto & current_setting = current_settings[index];
+        Field current_value = current_setting.getValue();
+
+        /// Setting isn't checked if value wasn't changed.
+        if (current_value == new_value)
+            return;
+
+        const StringRef & name = new_setting.getName();
+        if (!current_settings.allow_ddl && name == "allow_ddl")
+            throw Exception("Cannot modify 'allow_ddl' setting when DDL queries are prohibited for the user", ErrorCodes::QUERY_IS_PROHIBITED);
+
+        /** The `readonly` value is understood as follows:
+          * 0 - everything allowed.
+          * 1 - only read queries can be made; you can not change the settings.
+          * 2 - You can only do read queries and you can change the settings, except for the `readonly` setting.
+          */
+        if (current_settings.readonly == 1)
+            throw Exception("Cannot modify '" + name.toString() + "' setting in readonly mode", ErrorCodes::READONLY);
+
+        if (current_settings.readonly > 1 && name == "readonly")
+            throw Exception("Cannot modify 'readonly' setting in readonly mode", ErrorCodes::READONLY);
+    }
+}
+
+
+void SettingsConstraints::check(const Settings & current_settings, const SettingChange & change)
+{
+    size_t index = current_settings.find(change.name);
+    if (index == Settings::npos)
+        return;
+    // We store `change.value` to `temp_settings` to ensure it's converted to the correct type.
+    temp_settings[index].setValue(change.value);
+    checkImpl(current_settings, index);
+}
+
+
+void SettingsConstraints::check(const Settings & current_settings, const SettingsChanges & changes)
+{
+    for (const auto & change : changes)
+        check(current_settings, change);
+}
+
+}
diff --git a/dbms/src/Interpreters/SettingsConstraints.h b/dbms/src/Interpreters/SettingsConstraints.h
new file mode 100644
index 00000000000..8272bcfa3ae
--- /dev/null
+++ b/dbms/src/Interpreters/SettingsConstraints.h
@@ -0,0 +1,25 @@
+#pragma once
+
+#include <Common/SettingsChanges.h>
+
+namespace DB
+{
+struct Settings;
+
+/** Checks if specified changes of settings are allowed or not.
+  * If the changes are not allowed (i.e. violates some constraints) this class throws an exception.
+  * This class checks that we are not in the read-only mode.
+  * If a setting cannot be change due to the read-only mode this class throws an exception.
+  * The value of `readonly` value is understood as follows:
+  * 0 - everything allowed.
+  * 1 - only read queries can be made; you can not change the settings.
+  * 2 - you can only do read queries and you can change the settings, except for the `readonly` setting.
+  */
+class SettingsConstraints
+{
+public:
+    static void check(const Settings & current_settings, const SettingChange & change);
+    static void check(const Settings & current_settings, const SettingsChanges & changes);
+};
+
+}
diff --git a/dbms/src/Parsers/ASTSetQuery.h b/dbms/src/Parsers/ASTSetQuery.h
index 756c5bdc058..19277d7f53b 100644
--- a/dbms/src/Parsers/ASTSetQuery.h
+++ b/dbms/src/Parsers/ASTSetQuery.h
@@ -1,7 +1,7 @@
 #pragma once
 
-#include <Core/Field.h>
 #include <Common/FieldVisitors.h>
+#include <Common/SettingsChanges.h>
 #include <Parsers/IAST.h>
 
 
@@ -16,14 +16,7 @@ class ASTSetQuery : public IAST
 public:
     bool is_standalone = true; /// If false, this AST is a part of another query, such as SELECT.
 
-    struct Change
-    {
-        String name;
-        Field value;
-    };
-
-    using Changes = std::vector<Change>;
-    Changes changes;
+    SettingsChanges changes;
 
     /** Get the text that identifies this element. */
     String getID(char) const override { return "Set"; }
@@ -35,7 +28,7 @@ public:
         if (is_standalone)
             settings.ostr << (settings.hilite ? hilite_keyword : "") << "SET " << (settings.hilite ? hilite_none : "");
 
-        for (ASTSetQuery::Changes::const_iterator it = changes.begin(); it != changes.end(); ++it)
+        for (auto it = changes.begin(); it != changes.end(); ++it)
         {
             if (it != changes.begin())
                 settings.ostr << ", ";
diff --git a/dbms/src/Parsers/ParserSetQuery.cpp b/dbms/src/Parsers/ParserSetQuery.cpp
index 9eb94a76364..729d2be786c 100644
--- a/dbms/src/Parsers/ParserSetQuery.cpp
+++ b/dbms/src/Parsers/ParserSetQuery.cpp
@@ -13,7 +13,7 @@ namespace DB
 
 
 /// Parse `name = value`.
-static bool parseNameValuePair(ASTSetQuery::Change & change, IParser::Pos & pos, Expected & expected)
+static bool parseNameValuePair(SettingChange & change, IParser::Pos & pos, Expected & expected)
 {
     ParserIdentifier name_p;
     ParserLiteral value_p;
@@ -31,6 +31,7 @@ static bool parseNameValuePair(ASTSetQuery::Change & change, IParser::Pos & pos,
     if (!value_p.parse(pos, value, expected))
         return false;
 
+    String change_name;
     getIdentifierName(name, change.name);
     change.value = value->as<ASTLiteral &>().value;
 
@@ -50,14 +51,14 @@ bool ParserSetQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
             return false;
     }
 
-    ASTSetQuery::Changes changes;
+    SettingsChanges changes;
 
     while (true)
     {
         if (!changes.empty() && !s_comma.ignore(pos))
             break;
 
-        changes.push_back(ASTSetQuery::Change());
+        changes.push_back(SettingChange{});
 
         if (!parseNameValuePair(changes.back(), pos, expected))
             return false;
@@ -67,7 +68,7 @@ bool ParserSetQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
     node = query;
 
     query->is_standalone = !parse_only_internals;
-    query->changes = changes;
+    query->changes = std::move(changes);
 
     return true;
 }
diff --git a/dbms/src/Storages/Kafka/KafkaSettings.cpp b/dbms/src/Storages/Kafka/KafkaSettings.cpp
index 1d77bdc39af..79223104085 100644
--- a/dbms/src/Storages/Kafka/KafkaSettings.cpp
+++ b/dbms/src/Storages/Kafka/KafkaSettings.cpp
@@ -11,24 +11,25 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int BAD_ARGUMENTS;
+    extern const int UNKNOWN_SETTING;
 }
 
+IMPLEMENT_SETTINGS_COLLECTION(KafkaSettings, APPLY_FOR_KAFKA_SETTINGS)
+
 void KafkaSettings::loadFromQuery(ASTStorage & storage_def)
 {
     if (storage_def.settings)
     {
-        for (const ASTSetQuery::Change & setting : storage_def.settings->changes)
+        try
         {
-#define SET(TYPE, NAME, DEFAULT, DESCRIPTION) \
-            else if (setting.name == #NAME) NAME.set(setting.value);
-
-            if (false) {}
-            APPLY_FOR_KAFKA_SETTINGS(SET)
+            applyChanges(storage_def.settings->changes);
+        }
+        catch (Exception & e)
+        {
+            if (e.code() == ErrorCodes::UNKNOWN_SETTING)
+                throw Exception(e.message() + " for storage " + storage_def.engine->name, ErrorCodes::BAD_ARGUMENTS);
             else
-                throw Exception(
-                    "Unknown setting " + setting.name + " for storage " + storage_def.engine->name,
-                    ErrorCodes::BAD_ARGUMENTS);
-#undef SET
+                e.rethrow();
         }
     }
     else
diff --git a/dbms/src/Storages/Kafka/KafkaSettings.h b/dbms/src/Storages/Kafka/KafkaSettings.h
index 0ddc7ef04f8..0a21adf334b 100644
--- a/dbms/src/Storages/Kafka/KafkaSettings.h
+++ b/dbms/src/Storages/Kafka/KafkaSettings.h
@@ -1,8 +1,5 @@
 #pragma once
 
-#include <Poco/Util/AbstractConfiguration.h>
-#include <Core/Defines.h>
-#include <Core/Types.h>
 #include <Core/SettingsCommon.h>
 
 
@@ -14,7 +11,7 @@ class ASTStorage;
 /** Settings for the Kafka engine.
   * Could be loaded from a CREATE TABLE query (SETTINGS clause).
   */
-struct KafkaSettings
+struct KafkaSettings : public SettingsCollection<KafkaSettings>
 {
 
 #define APPLY_FOR_KAFKA_SETTINGS(M) \
@@ -28,14 +25,8 @@ struct KafkaSettings
     M(SettingUInt64, kafka_max_block_size, 0, "The maximum block size per table for Kafka engine.") \
     M(SettingUInt64, kafka_skip_broken_messages, 0, "Skip at least this number of broken messages from Kafka topic per block")
 
-#define DECLARE(TYPE, NAME, DEFAULT, DESCRIPTION) \
-    TYPE NAME {DEFAULT};
+    DECLARE_SETTINGS_COLLECTION(APPLY_FOR_KAFKA_SETTINGS)
 
-    APPLY_FOR_KAFKA_SETTINGS(DECLARE)
-
-#undef DECLARE
-
-public:
     void loadFromQuery(ASTStorage & storage_def);
 };
 
diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp b/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp
index 899f26e9bca..701e1e2adf2 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp
@@ -1,4 +1,5 @@
 #include <Storages/MergeTree/MergeTreeSettings.h>
+#include <Poco/Util/AbstractConfiguration.h>
 #include <Parsers/ASTCreateQuery.h>
 #include <Parsers/ASTSetQuery.h>
 #include <Parsers/ASTFunction.h>
@@ -12,8 +13,11 @@ namespace ErrorCodes
 {
     extern const int INVALID_CONFIG_PARAMETER;
     extern const int BAD_ARGUMENTS;
+    extern const int UNKNOWN_SETTING;
 }
 
+IMPLEMENT_SETTINGS_COLLECTION(MergeTreeSettings, APPLY_FOR_MERGE_TREE_SETTINGS)
+
 void MergeTreeSettings::loadFromConfig(const String & config_elem, const Poco::Util::AbstractConfiguration & config)
 {
     if (!config.has(config_elem))
@@ -22,18 +26,17 @@ void MergeTreeSettings::loadFromConfig(const String & config_elem, const Poco::U
     Poco::Util::AbstractConfiguration::Keys config_keys;
     config.keys(config_elem, config_keys);
 
-    for (const String & key : config_keys)
+    try
     {
-        String value = config.getString(config_elem + "." + key);
-
-#define SET(TYPE, NAME, DEFAULT, DESCRIPTION) \
-        else if (key == #NAME) NAME.set(value);
-
-        if (false) {}
-        APPLY_FOR_MERGE_TREE_SETTINGS(SET)
+        for (const String & key : config_keys)
+            set(key, config.getString(config_elem + "." + key));
+    }
+    catch (Exception & e)
+    {
+        if (e.code() == ErrorCodes::UNKNOWN_SETTING)
+            throw Exception(e.message() + " in MergeTree config", ErrorCodes::INVALID_CONFIG_PARAMETER);
         else
-            throw Exception("Unknown MergeTree setting " + key + " in config", ErrorCodes::INVALID_CONFIG_PARAMETER);
-#undef SET
+            e.rethrow();
     }
 }
 
@@ -41,18 +44,16 @@ void MergeTreeSettings::loadFromQuery(ASTStorage & storage_def)
 {
     if (storage_def.settings)
     {
-        for (const ASTSetQuery::Change & setting : storage_def.settings->changes)
+        try
         {
-#define SET(TYPE, NAME, DEFAULT, DESCRIPTION) \
-            else if (setting.name == #NAME) NAME.set(setting.value);
-
-            if (false) {}
-            APPLY_FOR_MERGE_TREE_SETTINGS(SET)
+            applyChanges(storage_def.settings->changes);
+        }
+        catch (Exception & e)
+        {
+            if (e.code() == ErrorCodes::UNKNOWN_SETTING)
+                throw Exception(e.message() + " for storage " + storage_def.engine->name, ErrorCodes::BAD_ARGUMENTS);
             else
-                throw Exception(
-                    "Unknown setting " + setting.name + " for storage " + storage_def.engine->name,
-                    ErrorCodes::BAD_ARGUMENTS);
-#undef SET
+                e.rethrow();
         }
     }
     else
@@ -62,13 +63,13 @@ void MergeTreeSettings::loadFromQuery(ASTStorage & storage_def)
         storage_def.set(storage_def.settings, settings_ast);
     }
 
-    ASTSetQuery::Changes & changes = storage_def.settings->changes;
+    SettingsChanges & changes = storage_def.settings->changes;
 
 #define ADD_IF_ABSENT(NAME)                                                                                   \
     if (std::find_if(changes.begin(), changes.end(),                                                          \
-                  [](const ASTSetQuery::Change & c) { return c.name == #NAME; })                              \
+                  [](const SettingChange & c) { return c.name == #NAME; })                              \
             == changes.end())                                                                                 \
-        changes.push_back(ASTSetQuery::Change{#NAME, NAME.value});
+        changes.push_back(SettingChange{#NAME, NAME.value});
 
     APPLY_FOR_IMMUTABLE_MERGE_TREE_SETTINGS(ADD_IF_ABSENT)
 #undef ADD_IF_ABSENT
diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.h b/dbms/src/Storages/MergeTree/MergeTreeSettings.h
index 2d0be1ada68..5cad78ed5ab 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeSettings.h
+++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.h
@@ -1,11 +1,18 @@
 #pragma once
 
-#include <Poco/Util/AbstractConfiguration.h>
 #include <Core/Defines.h>
-#include <Core/Types.h>
 #include <Core/SettingsCommon.h>
 
 
+namespace Poco
+{
+    namespace Util
+    {
+        class AbstractConfiguration;
+    }
+}
+
+
 namespace DB
 {
 
@@ -14,7 +21,7 @@ class ASTStorage;
 /** Settings for the MergeTree family of engines.
   * Could be loaded from config or from a CREATE TABLE query (SETTINGS clause).
   */
-struct MergeTreeSettings
+struct MergeTreeSettings : public SettingsCollection<MergeTreeSettings>
 {
 
 #define APPLY_FOR_MERGE_TREE_SETTINGS(M) \
@@ -168,18 +175,12 @@ struct MergeTreeSettings
     /** Minimal time in seconds, when merge with TTL can be repeated */                                           \
     M(SettingInt64, merge_with_ttl_timeout, 3600 * 24, "")
 
+    DECLARE_SETTINGS_COLLECTION(APPLY_FOR_MERGE_TREE_SETTINGS)
+
     /// Settings that should not change after the creation of a table.
 #define APPLY_FOR_IMMUTABLE_MERGE_TREE_SETTINGS(M) \
     M(index_granularity)
 
-#define DECLARE(TYPE, NAME, DEFAULT, DESCRIPTION) \
-    TYPE NAME {DEFAULT};
-
-    APPLY_FOR_MERGE_TREE_SETTINGS(DECLARE)
-
-#undef DECLARE
-
-public:
     void loadFromConfig(const String & config_elem, const Poco::Util::AbstractConfiguration & config);
 
     /// NOTE: will rewrite the AST to add immutable settings.
diff --git a/dbms/src/Storages/StorageJoin.cpp b/dbms/src/Storages/StorageJoin.cpp
index 60e5f9ee775..14184d8971f 100644
--- a/dbms/src/Storages/StorageJoin.cpp
+++ b/dbms/src/Storages/StorageJoin.cpp
@@ -139,7 +139,7 @@ void registerStorageJoin(StorageFactory & factory)
 
         if (args.storage_def && args.storage_def->settings)
         {
-            for (const ASTSetQuery::Change & setting : args.storage_def->settings->changes)
+            for (const auto & setting : args.storage_def->settings->changes)
             {
                 if (setting.name == "join_use_nulls")
                     join_use_nulls.set(setting.value);
diff --git a/dbms/tests/queries/0_stateless/00474_readonly_settings.reference b/dbms/tests/queries/0_stateless/00474_readonly_settings.reference
index 672b858d944..b1da40ce414 100644
--- a/dbms/tests/queries/0_stateless/00474_readonly_settings.reference
+++ b/dbms/tests/queries/0_stateless/00474_readonly_settings.reference
@@ -4,11 +4,11 @@
 			"value": "4611686018427387904"
 value
 value
-Cannot execute SET query in readonly mode.
+Cannot modify 'output_format_json_quote_64bit_integers' setting in readonly mode
 			"name": "value",
 			"value": "9223372036854775808"
 			"name": "value",
 			"value": 9223372036854775808
 value
 value
-Cannot override setting
+Cannot modify 'output_format_json_quote_64bit_integers' setting in readonly mode
diff --git a/dbms/tests/queries/0_stateless/00474_readonly_settings.sh b/dbms/tests/queries/0_stateless/00474_readonly_settings.sh
index 1907d51e273..0b1f0c415bf 100755
--- a/dbms/tests/queries/0_stateless/00474_readonly_settings.sh
+++ b/dbms/tests/queries/0_stateless/00474_readonly_settings.sh
@@ -6,13 +6,13 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 $CLICKHOUSE_CLIENT --query="select toUInt64(pow(2, 62)) as value format JSON" --output_format_json_quote_64bit_integers=0 | grep value
 $CLICKHOUSE_CLIENT --query="select toUInt64(pow(2, 62)) as value format JSON" --output_format_json_quote_64bit_integers=1 | grep value
 
-$CLICKHOUSE_CLIENT --readonly=1 --multiquery --query="set output_format_json_quote_64bit_integers=1 ; select toUInt64(pow(2, 63)) as value format JSON" --server_logs_file=/dev/null 2>&1 | grep -o 'value\|Cannot execute SET query in readonly mode.'
-$CLICKHOUSE_CLIENT --readonly=1 --multiquery --query="set output_format_json_quote_64bit_integers=0 ; select toUInt64(pow(2, 63)) as value format JSON" --server_logs_file=/dev/null 2>&1 | grep -o 'value\|Cannot execute SET query in readonly mode.'
+$CLICKHOUSE_CLIENT --readonly=1 --multiquery --query="set output_format_json_quote_64bit_integers=1 ; select toUInt64(pow(2, 63)) as value format JSON" --server_logs_file=/dev/null 2>&1 | grep -o 'value\|Cannot modify .* setting in readonly mode'
+$CLICKHOUSE_CLIENT --readonly=1 --multiquery --query="set output_format_json_quote_64bit_integers=0 ; select toUInt64(pow(2, 63)) as value format JSON" --server_logs_file=/dev/null 2>&1 | grep -o 'value\|Cannot modify .* setting in readonly mode'
 
 ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?query=SELECT+toUInt64(pow(2,+63))+as+value+format+JSON&output_format_json_quote_64bit_integers=1" | grep value
 ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?query=SELECT+toUInt64(pow(2,+63))+as+value+format+JSON&output_format_json_quote_64bit_integers=0" | grep value
 
 ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?session_id=readonly&session_timeout=3600" -d 'SET readonly = 1'
 
-${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?session_id=readonly&query=SELECT+toUInt64(pow(2,+63))+as+value+format+JSON&output_format_json_quote_64bit_integers=1" 2>&1 | grep -o 'value\|Cannot override setting'
-${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?session_id=readonly&query=SELECT+toUInt64(pow(2,+63))+as+value+format+JSON&output_format_json_quote_64bit_integers=0" 2>&1 | grep -o 'value\|Cannot override setting'
+${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?session_id=readonly&query=SELECT+toUInt64(pow(2,+63))+as+value+format+JSON&output_format_json_quote_64bit_integers=1" 2>&1 | grep -o 'value\|Cannot modify .* setting in readonly mode.'
+${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?session_id=readonly&query=SELECT+toUInt64(pow(2,+63))+as+value+format+JSON&output_format_json_quote_64bit_integers=0" 2>&1 | grep -o 'value\|Cannot modify .* setting in readonly mode'

From d43e7a89603c0e343eda8be9fc00c7583849bd79 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitbar@yandex-team.ru>
Date: Fri, 19 Apr 2019 03:45:15 +0300
Subject: [PATCH 071/118] Implement `max` constraints for number settings.

---
 dbms/src/Common/ErrorCodes.cpp                |   1 +
 dbms/src/Core/Settings.cpp                    |   2 +
 dbms/src/Interpreters/Context.cpp             |  18 ++-
 dbms/src/Interpreters/Context.h               |   4 +
 dbms/src/Interpreters/SettingsConstraints.cpp | 131 ++++++++++++++----
 dbms/src/Interpreters/SettingsConstraints.h   |  52 ++++++-
 6 files changed, 170 insertions(+), 38 deletions(-)

diff --git a/dbms/src/Common/ErrorCodes.cpp b/dbms/src/Common/ErrorCodes.cpp
index bfe9049fcb3..093f98d22a1 100644
--- a/dbms/src/Common/ErrorCodes.cpp
+++ b/dbms/src/Common/ErrorCodes.cpp
@@ -426,6 +426,7 @@ namespace ErrorCodes
     extern const int BROTLI_WRITE_FAILED = 449;
     extern const int BAD_TTL_EXPRESSION = 450;
     extern const int BAD_TTL_FILE = 451;
+    extern const int SETTING_CONSTRAINT_VIOLATION = 452;
 
     extern const int KEEPER_EXCEPTION = 999;
     extern const int POCO_EXCEPTION = 1000;
diff --git a/dbms/src/Core/Settings.cpp b/dbms/src/Core/Settings.cpp
index 7653612d11c..0540f4b5ec9 100644
--- a/dbms/src/Core/Settings.cpp
+++ b/dbms/src/Core/Settings.cpp
@@ -34,6 +34,8 @@ void Settings::setProfile(const String & profile_name, const Poco::Util::Abstrac
 
     for (const std::string & key : config_keys)
     {
+        if (key == "constraints")
+            continue;
         if (key == "profile")   /// Inheritance of one profile from another.
             setProfile(config.getString(elem + "." + key), config);
         else
diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp
index 01fb726273b..5562a30ec82 100644
--- a/dbms/src/Interpreters/Context.cpp
+++ b/dbms/src/Interpreters/Context.cpp
@@ -620,14 +620,22 @@ void Context::calculateUserSettings()
     /// NOTE: we ignore global_context settings (from which it is usually copied)
     /// NOTE: global_context settings are immutable and not auto updated
     settings = Settings();
+    settings_constraints = SettingsConstraints();
 
     /// 2) Apply settings from default profile
     auto default_profile_name = getDefaultProfileName();
     if (profile != default_profile_name)
-        settings.setProfile(default_profile_name, *shared->users_config);
+        setProfile(default_profile_name);
 
     /// 3) Apply settings from current user
+    setProfile(profile);
+}
+
+
+void Context::setProfile(const String & profile)
+{
     settings.setProfile(profile, *shared->users_config);
+    settings_constraints.setProfile(profile, *shared->users_config);
 }
 
 
@@ -1033,7 +1041,7 @@ void Context::setSetting(const String & name, const String & value)
     auto lock = getLock();
     if (name == "profile")
     {
-        settings.setProfile(value, *shared->users_config);
+        setProfile(value);
         return;
     }
     settings.set(name, value);
@@ -1045,7 +1053,7 @@ void Context::setSetting(const String & name, const Field & value)
     auto lock = getLock();
     if (name == "profile")
     {
-        settings.setProfile(value.safeGet<String>(), *shared->users_config);
+        setProfile(value.safeGet<String>());
         return;
     }
     settings.set(name, value);
@@ -1068,13 +1076,13 @@ void Context::applySettingsChanges(const SettingsChanges & changes)
 
 void Context::checkSettingsConstraints(const SettingChange & change)
 {
-    SettingsConstraints::check(settings, change);
+    settings_constraints.check(settings, change);
 }
 
 
 void Context::checkSettingsConstraints(const SettingsChanges & changes)
 {
-    SettingsConstraints::check(settings, changes);
+    settings_constraints.check(settings, changes);
 }
 
 
diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h
index b6b42e3f994..6962c69d2cf 100644
--- a/dbms/src/Interpreters/Context.h
+++ b/dbms/src/Interpreters/Context.h
@@ -4,6 +4,7 @@
 #include <Core/NamesAndTypes.h>
 #include <Core/Types.h>
 #include <Interpreters/ClientInfo.h>
+#include <Interpreters/SettingsConstraints.h>
 #include <Core/Settings.h>
 #include <Parsers/IAST_fwd.h>
 #include <Common/LRUCache.h>
@@ -125,6 +126,7 @@ private:
     std::shared_ptr<QuotaForIntervals> quota;           /// Current quota. By default - empty quota, that have no limits.
     String current_database;
     Settings settings;                                  /// Setting for query execution.
+    SettingsConstraints settings_constraints;
     using ProgressCallback = std::function<void(const Progress & progress)>;
     ProgressCallback progress_callback;                 /// Callback for tracking progress of query execution.
     QueryStatus * process_list_elem = nullptr;   /// For tracking total resource usage for query.
@@ -484,6 +486,8 @@ private:
       */
     void checkDatabaseAccessRightsImpl(const std::string & database_name) const;
 
+    void setProfile(const String & profile);
+
     EmbeddedDictionaries & getEmbeddedDictionariesImpl(bool throw_on_error) const;
     ExternalDictionaries & getExternalDictionariesImpl(bool throw_on_error) const;
     ExternalModels & getExternalModelsImpl(bool throw_on_error) const;
diff --git a/dbms/src/Interpreters/SettingsConstraints.cpp b/dbms/src/Interpreters/SettingsConstraints.cpp
index 2120356bb7c..e06fb7d84c3 100644
--- a/dbms/src/Interpreters/SettingsConstraints.cpp
+++ b/dbms/src/Interpreters/SettingsConstraints.cpp
@@ -1,5 +1,8 @@
 #include <Interpreters/SettingsConstraints.h>
 #include <Core/Settings.h>
+#include <Common/FieldVisitors.h>
+#include <IO/WriteHelpers.h>
+#include <Poco/Util/AbstractConfiguration.h>
 
 
 namespace DB
@@ -9,43 +12,35 @@ namespace ErrorCodes
     extern const int READONLY;
     extern const int QUERY_IS_PROHIBITED;
     extern const int NO_ELEMENTS_IN_CONFIG;
+    extern const int SETTING_CONSTRAINT_VIOLATION;
 }
 
 namespace
 {
     thread_local Settings temp_settings;
-
-    void checkImpl(const Settings & current_settings, size_t index)
-    {
-        const auto & new_setting = temp_settings[index];
-        Field new_value = new_setting.getValue();
-
-        const auto & current_setting = current_settings[index];
-        Field current_value = current_setting.getValue();
-
-        /// Setting isn't checked if value wasn't changed.
-        if (current_value == new_value)
-            return;
-
-        const StringRef & name = new_setting.getName();
-        if (!current_settings.allow_ddl && name == "allow_ddl")
-            throw Exception("Cannot modify 'allow_ddl' setting when DDL queries are prohibited for the user", ErrorCodes::QUERY_IS_PROHIBITED);
-
-        /** The `readonly` value is understood as follows:
-          * 0 - everything allowed.
-          * 1 - only read queries can be made; you can not change the settings.
-          * 2 - You can only do read queries and you can change the settings, except for the `readonly` setting.
-          */
-        if (current_settings.readonly == 1)
-            throw Exception("Cannot modify '" + name.toString() + "' setting in readonly mode", ErrorCodes::READONLY);
-
-        if (current_settings.readonly > 1 && name == "readonly")
-            throw Exception("Cannot modify 'readonly' setting in readonly mode", ErrorCodes::READONLY);
-    }
 }
 
 
-void SettingsConstraints::check(const Settings & current_settings, const SettingChange & change)
+SettingsConstraints::SettingsConstraints() = default;
+SettingsConstraints::SettingsConstraints(const SettingsConstraints & src) = default;
+SettingsConstraints & SettingsConstraints::operator=(const SettingsConstraints & src) = default;
+SettingsConstraints::SettingsConstraints(SettingsConstraints && src) = default;
+SettingsConstraints & SettingsConstraints::operator=(SettingsConstraints && src) = default;
+SettingsConstraints::~SettingsConstraints() = default;
+
+
+void SettingsConstraints::setMaxValue(const String & name, const String & max_value)
+{
+    max_settings.set(name, max_value);
+}
+
+void SettingsConstraints::setMaxValue(const String & name, const Field & max_value)
+{
+    max_settings.set(name, max_value);
+}
+
+
+void SettingsConstraints::check(const Settings & current_settings, const SettingChange & change) const
 {
     size_t index = current_settings.find(change.name);
     if (index == Settings::npos)
@@ -56,10 +51,86 @@ void SettingsConstraints::check(const Settings & current_settings, const Setting
 }
 
 
-void SettingsConstraints::check(const Settings & current_settings, const SettingsChanges & changes)
+void SettingsConstraints::check(const Settings & current_settings, const SettingsChanges & changes) const
 {
     for (const auto & change : changes)
         check(current_settings, change);
 }
 
+
+void SettingsConstraints::checkImpl(const Settings & current_settings, size_t index) const
+{
+    const auto & new_setting = temp_settings[index];
+    Field new_value = new_setting.getValue();
+
+    const auto & current_setting = current_settings[index];
+    Field current_value = current_setting.getValue();
+
+    /// Setting isn't checked if value wasn't changed.
+    if (current_value == new_value)
+        return;
+
+    const StringRef & name = new_setting.getName();
+    if (!current_settings.allow_ddl && name == "allow_ddl")
+        throw Exception("Cannot modify 'allow_ddl' setting when DDL queries are prohibited for the user", ErrorCodes::QUERY_IS_PROHIBITED);
+
+    /** The `readonly` value is understood as follows:
+      * 0 - everything allowed.
+      * 1 - only read queries can be made; you can not change the settings.
+      * 2 - You can only do read queries and you can change the settings, except for the `readonly` setting.
+      */
+    if (current_settings.readonly == 1)
+        throw Exception("Cannot modify '" + name.toString() + "' setting in readonly mode", ErrorCodes::READONLY);
+
+    if (current_settings.readonly > 1 && name == "readonly")
+        throw Exception("Cannot modify 'readonly' setting in readonly mode", ErrorCodes::READONLY);
+
+    const auto & max_setting = max_settings[index];
+    if (max_setting.isChanged())
+    {
+        Field max_value = max_setting.getValue();
+        if (new_value > max_value)
+            throw Exception(
+                "Setting " + name.toString() + " shouldn't be greater than " + applyVisitor(FieldVisitorToString(), max_value),
+                ErrorCodes::SETTING_CONSTRAINT_VIOLATION);
+    }
+}
+
+
+void SettingsConstraints::setProfile(const String & profile_name, const Poco::Util::AbstractConfiguration & config)
+{
+    String parent_profile = "profiles." + profile_name + ".profile";
+    if (config.has(parent_profile))
+        setProfile(parent_profile, config); // Inheritance of one profile from another.
+
+    String path_to_constraints = "profiles." + profile_name + ".constraints";
+    if (config.has(path_to_constraints))
+        loadFromConfig(path_to_constraints, config);
+}
+
+
+void SettingsConstraints::loadFromConfig(const String & path_to_constraints, const Poco::Util::AbstractConfiguration & config)
+{
+    if (!config.has(path_to_constraints))
+        throw Exception("There is no path '" + path_to_constraints + "' in configuration file.", ErrorCodes::NO_ELEMENTS_IN_CONFIG);
+
+    Poco::Util::AbstractConfiguration::Keys names;
+    config.keys(path_to_constraints, names);
+
+    for (const String & name : names)
+    {
+        String path_to_name = path_to_constraints + "." + name;
+        Poco::Util::AbstractConfiguration::Keys constraint_types;
+        config.keys(path_to_name, constraint_types);
+        for (const String & constraint_type : constraint_types)
+        {
+            String path_to_type = path_to_name + "." + constraint_type;
+            if (constraint_type == "max")
+                setMaxValue(name, config.getString(path_to_type));
+            else
+                throw Exception("Setting " + constraint_type + " value for " + name + " isn't supported", ErrorCodes::NOT_IMPLEMENTED);
+        }
+    }
+}
+
 }
diff --git a/dbms/src/Interpreters/SettingsConstraints.h b/dbms/src/Interpreters/SettingsConstraints.h
index 8272bcfa3ae..5fb3c2adcae 100644
--- a/dbms/src/Interpreters/SettingsConstraints.h
+++ b/dbms/src/Interpreters/SettingsConstraints.h
@@ -1,6 +1,16 @@
 #pragma once
 
 #include <Common/SettingsChanges.h>
+#include <Core/Settings.h>
+
+namespace Poco
+{
+namespace Util
+{
+    class AbstractConfiguration;
+}
+}
+
 
 namespace DB
 {
@@ -8,7 +18,20 @@ struct Settings;
 
 /** Checks if specified changes of settings are allowed or not.
   * If the changes are not allowed (i.e. violates some constraints) this class throws an exception.
-  * This class checks that we are not in the read-only mode.
+  * The constraints are set by editing the `users.xml` file.
+  * For examples, the following lines in `users.xml` will set that `max_memory_usage` cannot be greater than 20000000000:
+  * <profiles>
+  *   <user_profile>
+  *       <max_memory_usage>10000000000</max_memory_usage>
+  *       ...
+  *       <constraints>
+  *           <max_memory_usage>
+  *               <max>20000000000</max>
+  *           </max_memory_usage>
+  *       </constraints>
+  *   </user_profile>
+  * </profiles>
+  * This class also checks that we are not in the read-only mode.
   * If a setting cannot be change due to the read-only mode this class throws an exception.
   * The value of `readonly` value is understood as follows:
   * 0 - everything allowed.
@@ -18,8 +41,31 @@ struct Settings;
 class SettingsConstraints
 {
 public:
-    static void check(const Settings & current_settings, const SettingChange & change);
-    static void check(const Settings & current_settings, const SettingsChanges & changes);
+    SettingsConstraints();
+    SettingsConstraints(const SettingsConstraints & src);
+    SettingsConstraints & operator =(const SettingsConstraints & src);
+    SettingsConstraints(SettingsConstraints && src);
+    SettingsConstraints & operator =(SettingsConstraints && src);
+    ~SettingsConstraints();
+
+    void setMaxValue(const String & name, const String & max_value);
+    void setMaxValue(const String & name, const Field & max_value);
+
+    void check(const Settings & current_settings, const SettingChange & change) const;
+    void check(const Settings & current_settings, const SettingsChanges & changes) const;
+
+    /** Set multiple settings from "profile" (in server configuration file (users.xml), profiles contain groups of multiple settings).
+      * The profile can also be set using the `set` functions, like the profile setting.
+      */
+    void setProfile(const String & profile_name, const Poco::Util::AbstractConfiguration & config);
+
+    /// Loads the constraints from configuration file, at "path" prefix in configuration.
+    void loadFromConfig(const String & path, const Poco::Util::AbstractConfiguration & config);
+
+private:
+    void checkImpl(const Settings & current_settings, size_t index) const;
+
+    Settings max_settings;
 };
 
 }

From d2bfac794905dd626c0c5dc34d3c7e78e8398041 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitbar@yandex-team.ru>
Date: Sun, 7 Apr 2019 03:31:20 +0300
Subject: [PATCH 072/118] Add a test for the 'max' constraint

---
 dbms/tests/integration/helpers/client.py      | 21 ++++++++
 dbms/tests/integration/helpers/cluster.py     |  4 ++
 .../test_settings_constraints/__init__.py     |  0
 .../configs/users.xml                         | 29 +++++++++++
 .../test_settings_constraints/test.py         | 49 +++++++++++++++++++
 5 files changed, 103 insertions(+)
 create mode 100644 dbms/tests/integration/test_settings_constraints/__init__.py
 create mode 100644 dbms/tests/integration/test_settings_constraints/configs/users.xml
 create mode 100644 dbms/tests/integration/test_settings_constraints/test.py

diff --git a/dbms/tests/integration/helpers/client.py b/dbms/tests/integration/helpers/client.py
index f5853f8ee8c..58061114e4f 100644
--- a/dbms/tests/integration/helpers/client.py
+++ b/dbms/tests/integration/helpers/client.py
@@ -40,6 +40,10 @@ class Client:
         return CommandRequest(command, stdin, timeout, ignore_error)
 
 
+    def query_and_get_error(self, sql, stdin=None, timeout=None, settings=None, user=None):
+        return self.get_query_request(sql, stdin=stdin, timeout=timeout, settings=settings, user=user).get_error()
+
+
 class QueryTimeoutExceedException(Exception):
     pass
 
@@ -89,3 +93,20 @@ class CommandRequest:
             raise QueryRuntimeException('Client failed! Return code: {}, stderr: {}'.format(self.process.returncode, stderr))
 
         return stdout
+
+
+    def get_error(self):
+        self.process.wait()
+        self.stdout_file.seek(0)
+        self.stderr_file.seek(0)
+
+        stdout = self.stdout_file.read()
+        stderr = self.stderr_file.read()
+
+        if self.timer is not None and not self.process_finished_before_timeout and not self.ignore_error:
+            raise QueryTimeoutExceedException('Client timed out!')
+
+        if (self.process.returncode == 0):
+            raise QueryRuntimeException('Client expected to be failed but was succeeded! stdout: {}'.format(stdout))
+
+        return stderr
diff --git a/dbms/tests/integration/helpers/cluster.py b/dbms/tests/integration/helpers/cluster.py
index d33c3cfa083..0b314b850bc 100644
--- a/dbms/tests/integration/helpers/cluster.py
+++ b/dbms/tests/integration/helpers/cluster.py
@@ -483,6 +483,10 @@ class ClickHouseInstance:
     def get_query_request(self, *args, **kwargs):
         return self.client.get_query_request(*args, **kwargs)
 
+    # Connects to the instance via clickhouse-client, sends a query (1st argument), expects an error and return its code
+    def query_and_get_error(self, sql, stdin=None, timeout=None, settings=None, user=None):
+        return self.client.query_and_get_error(sql, stdin, timeout, settings, user)
+
     # Connects to the instance via HTTP interface, sends a query and returns the answer
     def http_query(self, sql, data=None):
         return urllib.urlopen("http://"+self.ip_address+":8123/?query="+urllib.quote(sql,safe=''), data).read()
diff --git a/dbms/tests/integration/test_settings_constraints/__init__.py b/dbms/tests/integration/test_settings_constraints/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/dbms/tests/integration/test_settings_constraints/configs/users.xml b/dbms/tests/integration/test_settings_constraints/configs/users.xml
new file mode 100644
index 00000000000..c33f3743ef4
--- /dev/null
+++ b/dbms/tests/integration/test_settings_constraints/configs/users.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0"?>
+<yandex>
+    <profiles>
+        <default>
+            <max_memory_usage>10000000000</max_memory_usage>
+            <constraints>
+                <max_memory_usage>
+                    <max>20000000000</max>
+                </max_memory_usage>
+            </constraints>
+        </default>
+    </profiles>
+
+    <users>
+        <default>
+            <password></password>
+            <networks incl="networks" replace="replace">
+                <ip>::/0</ip>
+            </networks>
+            <profile>default</profile>
+            <quota>default</quota>
+        </default>
+    </users>
+
+    <quotas>
+        <default>
+        </default>
+    </quotas>
+</yandex>
diff --git a/dbms/tests/integration/test_settings_constraints/test.py b/dbms/tests/integration/test_settings_constraints/test.py
new file mode 100644
index 00000000000..1b3604c63dd
--- /dev/null
+++ b/dbms/tests/integration/test_settings_constraints/test.py
@@ -0,0 +1,49 @@
+import pytest
+from helpers.cluster import ClickHouseCluster
+
+cluster = ClickHouseCluster(__file__)
+instance = cluster.add_instance('instance',
+                                config_dir="configs")
+
+
+
+
+@pytest.fixture(scope="module")
+def started_cluster():
+    try:
+        cluster.start()
+        yield cluster
+
+    finally:
+        cluster.shutdown()
+
+
+def test_max_constraint(started_cluster):
+    # Change a setting for session with SET.
+    assert instance.query("SELECT value FROM system.settings WHERE name='max_memory_usage'") ==\
+           "10000000000\n"
+
+    assert instance.query("SET max_memory_usage=20000000000;\n"
+                          "SELECT value FROM system.settings WHERE name='max_memory_usage'") ==\
+           "20000000000\n"
+
+    expected_error = "Setting max_memory_usage shouldn't be greater than 20000000000"
+    assert expected_error in instance.query_and_get_error("SET max_memory_usage=20000000001")
+
+    assert instance.query("SET max_memory_usage=11000000000;\n"
+                          "SELECT value FROM system.settings WHERE name='max_memory_usage'") ==\
+           "11000000000\n"
+
+    # Change a setting for query with SETTINGS.
+    assert instance.query("SELECT value FROM system.settings WHERE name='max_memory_usage'") ==\
+           "10000000000\n"
+
+    assert instance.query("SELECT value FROM system.settings WHERE name='max_memory_usage' "
+                          "SETTINGS max_memory_usage=20000000000") == "20000000000\n"
+
+    assert expected_error in instance.query_and_get_error(
+           "SELECT value FROM system.settings WHERE name='max_memory_usage' "
+           "SETTINGS max_memory_usage=20000000001")
+
+    assert instance.query("SELECT value FROM system.settings WHERE name='max_memory_usage' "
+                          "SETTINGS max_memory_usage=12000000000") == "12000000000\n"

From 77e7838c3022a20f98c173ef41559870bac9a3a8 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitbar@yandex-team.ru>
Date: Wed, 24 Apr 2019 16:22:32 +0300
Subject: [PATCH 073/118] Implemented iterators in the SettingsCollection
 template class

---
 dbms/src/Core/SettingsCommon.h                | 324 ++++++++++--------
 dbms/src/Interpreters/SettingsConstraints.cpp |   2 +-
 2 files changed, 182 insertions(+), 144 deletions(-)

diff --git a/dbms/src/Core/SettingsCommon.h b/dbms/src/Core/SettingsCommon.h
index b7f7f23474f..c3473d46ba6 100644
--- a/dbms/src/Core/SettingsCommon.h
+++ b/dbms/src/Core/SettingsCommon.h
@@ -469,17 +469,19 @@ namespace details
   * mysettings.cpp:
   * IMPLEMENT_SETTINGS_COLLECTION(MySettings, APPLY_FOR_MYSETTINGS)
   */
-template <typename T>
+template <class Derived>
 class SettingsCollection
 {
 private:
-    using Type = T;
-    using GetStringFunction = String (*)(const Type &);
-    using GetFieldFunction = Field (*)(const Type &);
-    using SetStringFunction = void (*)(Type &, const String &);
-    using SetFieldFunction = void (*)(Type &, const Field &);
-    using SerializeFunction = void (*)(const Type &, WriteBuffer & buf);
-    using DeserializeFunction = void (*)(Type &, ReadBuffer & buf);
+    Derived & castToDerived() { return *static_cast<Derived *>(this); }
+    const Derived & castToDerived() const { return *static_cast<const Derived *>(this); }
+
+    using GetStringFunction = String (*)(const Derived &);
+    using GetFieldFunction = Field (*)(const Derived &);
+    using SetStringFunction = void (*)(Derived &, const String &);
+    using SetFieldFunction = void (*)(Derived &, const Field &);
+    using SerializeFunction = void (*)(const Derived &, WriteBuffer & buf);
+    using DeserializeFunction = void (*)(Derived &, ReadBuffer & buf);
 
     struct MemberInfo
     {
@@ -492,169 +494,219 @@ private:
         SetFieldFunction set_field;
         SerializeFunction serialize;
         DeserializeFunction deserialize;
-        bool isChanged(const Type & collection) const { return *reinterpret_cast<const bool*>(reinterpret_cast<const UInt8*>(&collection) + offset_of_changed); }
+        bool isChanged(const Derived & collection) const { return *reinterpret_cast<const bool*>(reinterpret_cast<const UInt8*>(&collection) + offset_of_changed); }
     };
 
-    using MemberInfos = std::vector<MemberInfo>;
-
-    class Layout
+    class MemberInfos
     {
     public:
+        static const MemberInfos & instance()
+        {
+            static const MemberInfos single_instance;
+            return single_instance;
+        }
+
         size_t size() const { return infos.size(); }
         const MemberInfo & operator[](size_t index) const { return infos[index]; }
+        const MemberInfo * begin() const { return infos.data(); }
+        const MemberInfo * end() const { return infos.data() + infos.size(); }
 
-        static constexpr size_t npos = static_cast<size_t>(-1);
-
-        size_t find(const StringRef & name) const
+        size_t findIndex(const StringRef & name) const
         {
-            auto map_it = by_name_map.find(name);
-            if (map_it == by_name_map.end())
-                return npos;
-            else
-                return map_it->second;
+            auto it = by_name_map.find(name);
+            if (it == by_name_map.end())
+                return static_cast<size_t>(-1);
+            return it->second;
         }
 
-        size_t findStrict(const StringRef & name) const
+        size_t findIndexStrict(const StringRef & name) const
         {
-            auto map_it = by_name_map.find(name);
-            if (map_it == by_name_map.end())
+            auto it = by_name_map.find(name);
+            if (it == by_name_map.end())
                 details::SettingsCollectionUtils::throwNameNotFound(name);
-            return map_it->second;
+            return it->second;
         }
 
-        void addMemberInfo(MemberInfo && info)
+        const MemberInfo * find(const StringRef & name) const
+        {
+            auto it = by_name_map.find(name);
+            if (it == by_name_map.end())
+                return end();
+            else
+                return &infos[it->second];
+        }
+
+        const MemberInfo * findStrict(const StringRef & name) const { return &infos[findIndexStrict(name)]; }
+
+    private:
+        MemberInfos();
+
+        void add(MemberInfo && member)
         {
             size_t index = infos.size();
-            infos.emplace_back(info);
+            infos.emplace_back(member);
             by_name_map.emplace(infos.back().name, index);
         }
 
-    private:
-        MemberInfos infos;
+        std::vector<MemberInfo> infos;
         std::unordered_map<StringRef, size_t> by_name_map;
     };
 
-    static const Layout & getLayout();
-    Type & castThis() { return static_cast<Type &>(*this); }
-    const Type & castThis() const { return static_cast<const Type &>(*this); }
+    static const MemberInfos & members() { return MemberInfos::instance(); }
 
 public:
-    /// Provides access to a setting.
-    class reference
-    {
-    public:
-        const StringRef & getName() const { return member.name; }
-        const StringRef & getDescription() const { return member.description; }
-        bool isChanged() const { return member.isChanged(collection); }
-        Field getValue() const { return member.get_field(collection); }
-        String getValueAsString() const { return member.get_string(collection); }
-        void setValue(const Field & value) { member.set_field(collection, value); }
-        void setValue(const String & value) { member.set_string(collection, value); }
-
-        reference(Type & collection_, const MemberInfo & member_) : collection(collection_), member(member_) {}
-    private:
-        Type & collection;
-        const MemberInfo & member;
-    };
+    class const_iterator;
 
     /// Provides read-only access to a setting.
     class const_reference
     {
     public:
-        const StringRef & getName() const { return member.name; }
-        const StringRef & getDescription() const { return member.description; }
-        bool isChanged() const { return member.isChanged(collection); }
-        Field getValue() const { return member.get_field(collection); }
-        String getValueAsString() const { return member.get_string(collection); }
-
-        const_reference(const Type & collection_, const MemberInfo & member_) : collection(collection_), member(member_) {}
-    private:
-        const Type & collection;
-        const MemberInfo & member;
+        const_reference(const Derived & collection_, const MemberInfo & member_) : collection(&collection_), member(&member_) {}
+        const_reference(const const_reference & src) = default;
+        const StringRef & getName() const { return member->name; }
+        const StringRef & getDescription() const { return member->description; }
+        bool isChanged() const { return member->isChanged(*collection); }
+        Field getValue() const { return member->get_field(*collection); }
+        String getValueAsString() const { return member->get_string(*collection); }
+    protected:
+        friend class SettingsCollection<Derived>::const_iterator;
+        const_reference() : collection(nullptr), member(nullptr) {}
+        const_reference & operator=(const const_reference &) = default;
+        const Derived * collection;
+        const MemberInfo * member;
     };
 
-    /// Returns number of settings.
-    size_t size() const { return getLayout().size(); }
-
-    /// Returns a setting by its zero-based index.
-    reference operator [](size_t index) { return reference(castThis(), getLayout()[index]); }
-    const_reference operator [](size_t index) const { return const_reference(castThis(), getLayout()[index]); }
-
-    /// Returns a setting by its name. Throws an exception if there is not setting with such name.
-    reference operator [](const String & name)
+    /// Provides access to a setting.
+    class reference : public const_reference
     {
-        const Layout & layout = getLayout();
-        return reference(castThis(), layout[layout.findStrict(name)]);
-    }
+    public:
+        reference(Derived & collection_, const MemberInfo & member_) : const_reference(collection_, member_) {}
+        reference(const const_reference & src) : const_reference(src) {}
+        void setValue(const Field & value) { this->member->set_field(*const_cast<Derived *>(this->collection), value); }
+        void setValue(const String & value) { this->member->set_string(*const_cast<Derived *>(this->collection), value); }
+    };
 
-    const_reference operator [](const String & name) const
+    /// Iterator to iterating through all the settings.
+    class const_iterator
     {
-        const Layout & layout = getLayout();
-        return const_reference(castThis(), layout[layout.findStrict(name)]);
-    }
+    public:
+        const_iterator(const Derived & collection_, const MemberInfo * member_) : ref(const_cast<Derived &>(collection_), *member_) {}
+        const_iterator() = default;
+        const_iterator(const const_iterator & src) = default;
+        const_iterator & operator =(const const_iterator & src) = default;
+        const const_reference & operator *() const { return ref; }
+        const const_reference * operator ->() const { return &ref; }
+        const_iterator & operator ++() { ++ref.member; return *this; }
+        const_iterator & operator ++(int) { const_iterator tmp = *this; ++*this; return tmp; }
+        bool operator ==(const const_iterator & rhs) const { return ref.member == rhs.ref.member && ref.collection == rhs.ref.collection; }
+        bool operator !=(const const_iterator & rhs) const { return !(*this == rhs); }
+    protected:
+        mutable reference ref;
+    };
 
-    /// Finds a setting by name and returns its index. Returns npos if not found.
-    size_t find(const String & name) const { return getLayout().find(name); }
+    class iterator : public const_iterator
+    {
+    public:
+        iterator(Derived & collection_, const MemberInfo * member_) : const_iterator(collection_, member_) {}
+        iterator() = default;
+        iterator(const const_iterator & src) : const_iterator(src) {}
+        iterator & operator =(const const_iterator & src) { const_iterator::operator =(src); return *this; }
+        reference & operator *() const { return this->ref; }
+        reference * operator ->() const { return &this->ref; }
+        iterator & operator ++() { const_iterator::operator ++(); return *this; }
+        iterator & operator ++(int) { iterator tmp = *this; ++*this; return tmp; }
+    };
+
+    /// Returns the number of settings.
+    static size_t size() { return members().size(); }
+
+    /// Returns name of a setting by its index (0..size()-1).
+    static StringRef getNameByIndex(size_t index) { return members()[index].name; }
+
+    /// Returns description of a setting by its index.
+    static StringRef getDescriptionByIndex(size_t index) { return members()[index].description; }
+
+    /// Searches a setting by its name; returns `npos` if not found.
+    static size_t findIndex(const String & name) { return members().findIndex(name); }
     static constexpr size_t npos = static_cast<size_t>(-1);
 
-    /// Finds a setting by name and returns its index. Throws an exception if not found.
-    size_t findStrict(const String & name) const { return getLayout().findStrict(name); }
+    /// Searches a setting by its name; throws an exception if not found.
+    static size_t findIndexStrict(const String & name) { return members().findIndexStrict(name); }
 
-    /// Sets setting by name.
+    iterator begin() { return iterator(castToDerived(), members().begin()); }
+    const_iterator begin() const { return const_iterator(castToDerived(), members().begin()); }
+    iterator end() { return iterator(castToDerived(), members().end()); }
+    const_iterator end() const { return const_iterator(castToDerived(), members().end()); }
+
+    /// Returns a proxy object for accessing to a setting. Throws an exception if there is not setting with such name.
+    reference operator[](size_t index) { return reference(castToDerived(), members()[index]); }
+    reference operator[](const String & name) { return reference(castToDerived(), *(members().findStrict(name))); }
+    const_reference operator[](size_t index) const { return const_reference(castToDerived(), members()[index]); }
+    const_reference operator[](const String & name) const { return const_reference(castToDerived(), *(members().findStrict(name))); }
+
+    /// Searches a setting by its name; returns end() if not found.
+    iterator find(const String & name) { return iterator(castToDerived(), members().find(name)); }
+    const_iterator find(const String & name) const { return const_iterator(castToDerived(), members().find(name)); }
+
+    /// Searches a setting by its name; throws an exception if not found.
+    iterator findStrict(const String & name) { return iterator(castToDerived(), members().findStrict(name)); }
+    const_iterator findStrict(const String & name) const { return const_iterator(castToDerived(), members().findStrict(name)); }
+
+    /// Sets setting's value.
+    void set(size_t index, const Field & value) { (*this)[index].setValue(value); }
     void set(const String & name, const Field & value) { (*this)[name].setValue(value); }
 
-    /// Sets setting by name. Read value in text form from string (for example, from configuration file or from URL parameter).
+    /// Sets setting's value. Read value in text form from string (for example, from configuration file or from URL parameter).
+    void set(size_t index, const String & value) { (*this)[index].setValue(value); }
     void set(const String & name, const String & value) { (*this)[name].setValue(value); }
 
-    /// Returns the value of a setting.
+    /// Returns value of a setting.
+    Field get(size_t index) const { return (*this)[index].getValue(); }
     Field get(const String & name) const { return (*this)[name].getValue(); }
 
-    /// Returns the value of a setting converted to string.
+    /// Returns value of a setting converted to string.
+    String getAsString(size_t index) const { return (*this)[index].getValueAsString(); }
     String getAsString(const String & name) const { return (*this)[name].getValueAsString(); }
 
-    /// Returns the value of a setting.
+    /// Returns value of a setting; returns false if there is no setting with the specified name.
     bool tryGet(const String & name, Field & value) const
     {
-        const Layout & layout = getLayout();
-        size_t index = layout.find(name);
-        if (index == npos)
+        auto it = find(name);
+        if (it == end())
             return false;
-        value = layout[index].get_field(castThis());
+        value = it->getValue();
         return true;
     }
 
-    /// Returns the value of a setting converted to string.
+    /// Returns value of a setting converted to string; returns false if there is no setting with the specified name.
     bool tryGet(const String & name, String & value) const
     {
-        const Layout & layout = getLayout();
-        size_t index = layout.find(name);
-        if (index == npos)
+        auto it = find(name);
+        if (it == end())
             return false;
-        value = layout[index].get_string(castThis());
+        value = it->getValueAsString();
         return true;
     }
 
     /// Compares two collections of settings.
-    bool operator ==(const Type & rhs) const
+    bool operator ==(const Derived & rhs) const
     {
-        const Layout & layout = getLayout();
-        for (size_t i = 0; i != layout.size(); ++i)
+        for (const auto & member : members())
         {
-            const auto & member = layout[i];
-            bool left_changed = member.isChanged(castThis());
+            bool left_changed = member.isChanged(castToDerived());
             bool right_changed = member.isChanged(rhs);
             if (left_changed || right_changed)
             {
                 if (left_changed != right_changed)
                     return false;
-                if (member.get_field(castThis()) != member.get_field(rhs))
+                if (member.get_field(castToDerived()) != member.get_field(rhs))
                     return false;
             }
         }
         return true;
     }
 
-    bool operator !=(const Type & rhs) const
+    bool operator !=(const Derived & rhs) const
     {
         return !(*this == rhs);
     }
@@ -663,12 +715,10 @@ public:
     SettingsChanges changes() const
     {
         SettingsChanges found_changes;
-        const Layout & layout = getLayout();
-        for (size_t i = 0; i != layout.size(); ++i)
+        for (const auto & member : members())
         {
-            const auto & member = layout[i];
-            if (member.isChanged(castThis()))
-                found_changes.emplace_back(member.name.toString(), member.get_field(castThis()));
+            if (member.isChanged(castToDerived()))
+                found_changes.emplace_back(member.name.toString(), member.get_field(castToDerived()));
         }
         return found_changes;
     }
@@ -685,20 +735,16 @@ public:
             applyChange(change);
     }
 
-    void copyChangesFrom(const Type & src)
+    void copyChangesFrom(const Derived & src)
     {
-        const Layout & layout = getLayout();
-        for (size_t i = 0; i != layout.size(); ++i)
-        {
-            const auto & member = layout[i];
+        for (const auto & member : members())
             if (member.isChanged(src))
-                member.set_field(castThis(), member.get_field(src));
-        }
+                member.set_field(castToDerived(), member.get_field(src));
     }
 
-    void copyChangesTo(Type & dest) const
+    void copyChangesTo(Derived & dest) const
     {
-        dest.copyChangesFrom(castThis());
+        dest.copyChangesFrom(castToDerived());
     }
 
     /// Writes the settings to buffer (e.g. to be sent to remote server).
@@ -706,14 +752,12 @@ public:
     /// finished with empty name.
     void serialize(WriteBuffer & buf) const
     {
-        const Layout & layout = getLayout();
-        for (size_t i = 0; i != layout.size(); ++i)
+        for (const auto & member : members())
         {
-            const auto & member = layout[i];
-            if (member.isChanged(castThis()))
+            if (member.isChanged(castToDerived()))
             {
                 details::SettingsCollectionUtils::serializeName(member.name, buf);
-                member.serialize(castThis(), buf);
+                member.serialize(castToDerived(), buf);
             }
         }
         details::SettingsCollectionUtils::serializeName(StringRef{} /* empty string is a marker of the end of settings */, buf);
@@ -722,14 +766,13 @@ public:
     /// Reads the settings from buffer.
     void deserialize(ReadBuffer & buf)
     {
-        const Layout & layout = getLayout();
+        const auto & the_members = members();
         while (true)
         {
             String name = details::SettingsCollectionUtils::deserializeName(buf);
             if (name.empty() /* empty string is a marker of the end of settings */)
-                break; \
-            const auto & member = layout[layout.findStrict(name)];
-            member.deserialize(castThis(), buf); \
+                break;
+            the_members.findStrict(name)->deserialize(castToDerived(), buf);
         }
     }
 };
@@ -738,21 +781,16 @@ public:
     APPLY_MACRO(DECLARE_SETTINGS_COLLECTION_DECLARE_VARIABLES_HELPER_)
 
 
-#define IMPLEMENT_SETTINGS_COLLECTION(CLASS_NAME, APPLY_MACRO) \
+#define IMPLEMENT_SETTINGS_COLLECTION(DERIVED_CLASS_NAME, APPLY_MACRO) \
     template<> \
-    const SettingsCollection<CLASS_NAME>::Layout & SettingsCollection<CLASS_NAME>::getLayout() \
+    SettingsCollection<DERIVED_CLASS_NAME>::MemberInfos::MemberInfos() \
     { \
+        using Derived = DERIVED_CLASS_NAME; \
         struct Functions \
         { \
             APPLY_MACRO(IMPLEMENT_SETTINGS_COLLECTION_DEFINE_FUNCTIONS_HELPER_) \
         }; \
-        static const SettingsCollection<Type>::Layout single_instance = [] \
-        { \
-            SettingsCollection<Type>::Layout layout; \
-            APPLY_MACRO(IMPLEMENT_SETTINGS_COLLECTION_ADD_MEMBER_INFO_HELPER_) \
-            return layout; \
-        }(); \
-        return single_instance; \
+        APPLY_MACRO(IMPLEMENT_SETTINGS_COLLECTION_ADD_MEMBER_INFO_HELPER_) \
     }
 
 
@@ -761,20 +799,20 @@ public:
 
 
 #define IMPLEMENT_SETTINGS_COLLECTION_DEFINE_FUNCTIONS_HELPER_(TYPE, NAME, DEFAULT, DESCRIPTION) \
-    static String NAME##_getString(const Type & collection) { return collection.NAME.toString(); } \
-    static Field NAME##_getField(const Type & collection) { return collection.NAME.toField(); } \
-    static void NAME##_setString(Type & collection, const String & value) { collection.NAME.set(value); } \
-    static void NAME##_setField(Type & collection, const Field & value) { collection.NAME.set(value); } \
-    static void NAME##_serialize(const Type & collection, WriteBuffer & buf) { collection.NAME.serialize(buf); } \
-    static void NAME##_deserialize(Type & collection, ReadBuffer & buf) { collection.NAME.deserialize(buf); }
+    static String NAME##_getString(const Derived & collection) { return collection.NAME.toString(); } \
+    static Field NAME##_getField(const Derived & collection) { return collection.NAME.toField(); } \
+    static void NAME##_setString(Derived & collection, const String & value) { collection.NAME.set(value); } \
+    static void NAME##_setField(Derived & collection, const Field & value) { collection.NAME.set(value); } \
+    static void NAME##_serialize(const Derived & collection, WriteBuffer & buf) { collection.NAME.serialize(buf); } \
+    static void NAME##_deserialize(Derived & collection, ReadBuffer & buf) { collection.NAME.deserialize(buf); }
 
 
 #define IMPLEMENT_SETTINGS_COLLECTION_ADD_MEMBER_INFO_HELPER_(TYPE, NAME, DEFAULT, DESCRIPTION) \
-    static_assert(std::is_same_v<decltype(std::declval<TYPE>().changed), bool>); \
-    layout.addMemberInfo({offsetof(Type, NAME.changed), \
-                          StringRef(#NAME, strlen(#NAME)), StringRef(#DESCRIPTION, strlen(#DESCRIPTION)), \
-                          &Functions::NAME##_getString, &Functions::NAME##_getField, \
-                          &Functions::NAME##_setString, &Functions::NAME##_setField, \
-                          &Functions::NAME##_serialize, &Functions::NAME##_deserialize });
+    static_assert(std::is_same_v<decltype(std::declval<Derived>().NAME.changed), bool>); \
+    add({offsetof(Derived, NAME.changed), \
+         StringRef(#NAME, strlen(#NAME)), StringRef(#DESCRIPTION, strlen(#DESCRIPTION)), \
+         &Functions::NAME##_getString, &Functions::NAME##_getField, \
+        &Functions::NAME##_setString, &Functions::NAME##_setField, \
+        &Functions::NAME##_serialize, &Functions::NAME##_deserialize });
 
 }
diff --git a/dbms/src/Interpreters/SettingsConstraints.cpp b/dbms/src/Interpreters/SettingsConstraints.cpp
index e06fb7d84c3..6243afc9828 100644
--- a/dbms/src/Interpreters/SettingsConstraints.cpp
+++ b/dbms/src/Interpreters/SettingsConstraints.cpp
@@ -42,7 +42,7 @@ void SettingsConstraints::setMaxValue(const String & name, const Field & max_val
 
 void SettingsConstraints::check(const Settings & current_settings, const SettingChange & change) const
 {
-    size_t index = current_settings.find(change.name);
+    size_t index = current_settings.findIndex(change.name);
     if (index == Settings::npos)
         return;
     // We store `change.value` to `temp_settings` to ensure it's converted to the correct type.

From 5ebb192f4b448857648077a6690e3e47f45e5b2a Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Sat, 27 Apr 2019 00:04:18 +0300
Subject: [PATCH 074/118] Changelog fixes

---
 CHANGELOG.md    | 10 ++++++----
 CHANGELOG_RU.md |  7 +++++--
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6f03184162f..0a8b3b3407e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -29,6 +29,7 @@
 * Fill `system.graphite_detentions` from a table config of `*GraphiteMergeTree` engine tables. [#4584](https://github.com/yandex/ClickHouse/pull/4584) ([Mikhail f. Shiryaev](https://github.com/Felixoid))
 * Rename `trigramDistance` function to `ngramDistance` and add more functions with `CaseInsensitive` and `UTF`. [#4602](https://github.com/yandex/ClickHouse/pull/4602) ([Danila Kutenin](https://github.com/danlark1))
 * Improved data skipping indices calculation. [#4640](https://github.com/yandex/ClickHouse/pull/4640) ([Nikita Vasilev](https://github.com/nikvas0))
+* Keep ordinary, `DEFAULT`, `MATERIALIZED` and `ALIAS` columns in a single list (fixes issue [#2867](https://github.com/yandex/ClickHouse/issues/2867)). [#4707](https://github.com/yandex/ClickHouse/pull/4707) ([Alex Zatelepin](https://github.com/ztlpn))
 
 ### Bug Fix
 
@@ -39,7 +40,6 @@
 * Deadlock may happen while executing `DROP DATABASE dictionary` query. [#4701](https://github.com/yandex/ClickHouse/pull/4701) ([alexey-milovidov](https://github.com/alexey-milovidov))
 * Fix undefinied behavior in `median` and `quantile` functions. [#4702](https://github.com/yandex/ClickHouse/pull/4702) ([hcz](https://github.com/hczhcz))
 * Fix compression level detection when `network_compression_method` in lowercase. Broken in v19.1. [#4706](https://github.com/yandex/ClickHouse/pull/4706) ([proller](https://github.com/proller))
-* Keep ordinary, `DEFAULT`, `MATERIALIZED` and `ALIAS` columns in a single list (fixes issue [#2867](https://github.com/yandex/ClickHouse/issues/2867)). [#4707](https://github.com/yandex/ClickHouse/pull/4707) ([Alex Zatelepin](https://github.com/ztlpn))
 * Fixed ignorance of `<timezone>UTC</timezone>` setting (fixes issue [#4658](https://github.com/yandex/ClickHouse/issues/4658)). [#4718](https://github.com/yandex/ClickHouse/pull/4718) ([proller](https://github.com/proller))
 * Fix `histogram` function behaviour with `Distributed` tables. [#4741](https://github.com/yandex/ClickHouse/pull/4741) ([olegkv](https://github.com/olegkv))
 * Fixed tsan report `destroy of a locked mutex`. [#4742](https://github.com/yandex/ClickHouse/pull/4742) ([alexey-milovidov](https://github.com/alexey-milovidov))
@@ -68,7 +68,7 @@
 * Fix rare bug when setting `min_bytes_to_use_direct_io` is greater than zero, which occures when thread have to seek backward in column file. [#4897](https://github.com/yandex/ClickHouse/pull/4897) ([alesapin](https://github.com/alesapin))
 * Fix wrong argument types for aggregate functions with `LowCardinality` arguments (fixes issue [#4919](https://github.com/yandex/ClickHouse/issues/4919)). [#4922](https://github.com/yandex/ClickHouse/pull/4922) ([Nikolai Kochetov](https://github.com/KochetovNicolai))
 * Fix wrong name qualification in `GLOBAL JOIN`. [#4969](https://github.com/yandex/ClickHouse/pull/4969) ([Artem Zuikov](https://github.com/4ertus2))
-* Function `toISOWeek` result for year 1970. [#4988](https://github.com/yandex/ClickHouse/pull/4988) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Fix function `toISOWeek` result for year 1970. [#4988](https://github.com/yandex/ClickHouse/pull/4988) ([alexey-milovidov](https://github.com/alexey-milovidov))
 * Fix `DROP`, `TRUNCATE` and `OPTIMIZE` queries duplication, when executed on `ON CLUSTER` for `ReplicatedMergeTree*` tables family. [#4991](https://github.com/yandex/ClickHouse/pull/4991) ([alesapin](https://github.com/alesapin))
 
 ### Backward Incompatible Change
@@ -109,7 +109,6 @@
 * Deadlock may happen while executing `DROP DATABASE dictionary` query. [#4701](https://github.com/yandex/ClickHouse/pull/4701) ([alexey-milovidov](https://github.com/alexey-milovidov))
 * Fix undefinied behavior in `median` and `quantile` functions. [#4702](https://github.com/yandex/ClickHouse/pull/4702) ([hcz](https://github.com/hczhcz))
 * Fix compression level detection when `network_compression_method` in lowercase. Broken in v19.1. [#4706](https://github.com/yandex/ClickHouse/pull/4706) ([proller](https://github.com/proller))
-* Keep ordinary, `DEFAULT`, `MATERIALIZED` and `ALIAS` columns in a single list (fixes issue [#2867](https://github.com/yandex/ClickHouse/issues/2867)). [#4707](https://github.com/yandex/ClickHouse/pull/4707) ([Alex Zatelepin](https://github.com/ztlpn))
 * Fixed ignorance of `<timezone>UTC</timezone>` setting (fixes issue [#4658](https://github.com/yandex/ClickHouse/issues/4658)). [#4718](https://github.com/yandex/ClickHouse/pull/4718) ([proller](https://github.com/proller))
 * Fix `histogram` function behaviour with `Distributed` tables. [#4741](https://github.com/yandex/ClickHouse/pull/4741) ([olegkv](https://github.com/olegkv))
 * Fixed tsan report `destroy of a locked mutex`. [#4742](https://github.com/yandex/ClickHouse/pull/4742) ([alexey-milovidov](https://github.com/alexey-milovidov))
@@ -133,9 +132,12 @@
 * Functions `cutQueryStringAndFragment()` and `queryStringAndFragment()` now works correctly when `URL` contains a fragment and no query. [#4894](https://github.com/yandex/ClickHouse/pull/4894) ([Vitaly Baranov](https://github.com/vitlibar))
 * Fix rare bug when setting `min_bytes_to_use_direct_io` is greater than zero, which occures when thread have to seek backward in column file. [#4897](https://github.com/yandex/ClickHouse/pull/4897) ([alesapin](https://github.com/alesapin))
 * Fix wrong argument types for aggregate functions with `LowCardinality` arguments (fixes issue [#4919](https://github.com/yandex/ClickHouse/issues/4919)). [#4922](https://github.com/yandex/ClickHouse/pull/4922) ([Nikolai Kochetov](https://github.com/KochetovNicolai))
-* Function `toISOWeek` result for year 1970. [#4988](https://github.com/yandex/ClickHouse/pull/4988) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Fix function `toISOWeek` result for year 1970. [#4988](https://github.com/yandex/ClickHouse/pull/4988) ([alexey-milovidov](https://github.com/alexey-milovidov))
 * Fix `DROP`, `TRUNCATE` and `OPTIMIZE` queries duplication, when executed on `ON CLUSTER` for `ReplicatedMergeTree*` tables family. [#4991](https://github.com/yandex/ClickHouse/pull/4991) ([alesapin](https://github.com/alesapin))
 
+### Improvements
+
+* Keep ordinary, `DEFAULT`, `MATERIALIZED` and `ALIAS` columns in a single list (fixes issue [#2867](https://github.com/yandex/ClickHouse/issues/2867)). [#4707](https://github.com/yandex/ClickHouse/pull/4707) ([Alex Zatelepin](https://github.com/ztlpn))
 
 ## ClickHouse release 19.4.3.11, 2019-04-02
 
diff --git a/CHANGELOG_RU.md b/CHANGELOG_RU.md
index e4827a35a2b..f862563c3ee 100644
--- a/CHANGELOG_RU.md
+++ b/CHANGELOG_RU.md
@@ -29,6 +29,7 @@
 * Теперь системная таблица `system.graphite_detentions` заполняется из конфигурационного файла для таблиц семейства `*GraphiteMergeTree`. [#4584](https://github.com/yandex/ClickHouse/pull/4584) ([Mikhail f. Shiryaev](https://github.com/Felixoid))
 * Функция `trigramDistance` переименована в функцию `ngramDistance`. Добавлено несколько функций с `CaseInsensitive` и `UTF`. [#4602](https://github.com/yandex/ClickHouse/pull/4602) ([Danila Kutenin](https://github.com/danlark1))
 * Улучшено вычисление вторичных индексов. [#4640](https://github.com/yandex/ClickHouse/pull/4640) ([Nikita Vasilev](https://github.com/nikvas0))
+* Теперь обычные колонки, а также колонки `DEFAULT`, `MATERIALIZED` и `ALIAS` хранятся в одном списке (исправляет issue [#2867](https://github.com/yandex/ClickHouse/issues/2867)). [#4707](https://github.com/yandex/ClickHouse/pull/4707) ([Alex Zatelepin](https://github.com/ztlpn))
 
 ### Исправления ошибок
 
@@ -39,7 +40,6 @@
 * Исправлен дедлок, который мог происходить при запросе `DROP DATABASE dictionary`. [#4701](https://github.com/yandex/ClickHouse/pull/4701) ([alexey-milovidov](https://github.com/alexey-milovidov))
 * Исправлено неопределенное поведение в функциях `median` и `quantile`. [#4702](https://github.com/yandex/ClickHouse/pull/4702) ([hcz](https://github.com/hczhcz))
 * Исправлено определение уровня сжатия при указании настройки `network_compression_method` в нижнем регистре. Было сломано в v19.1. [#4706](https://github.com/yandex/ClickHouse/pull/4706) ([proller](https://github.com/proller))
-* Теперь обычные колонки, а также колонки `DEFAULT`, `MATERIALIZED` и `ALIAS` хранятся в одном списке (исправляет issue [#2867](https://github.com/yandex/ClickHouse/issues/2867)). [#4707](https://github.com/yandex/ClickHouse/pull/4707) ([Alex Zatelepin](https://github.com/ztlpn))
 * Настройка `<timezone>UTC</timezone>` больше не игнорируется (исправляет issue [#4658](https://github.com/yandex/ClickHouse/issues/4658)). [#4718](https://github.com/yandex/ClickHouse/pull/4718) ([proller](https://github.com/proller))
 * Исправлено поведение функции `histogram` с `Distributed` таблицами. [#4741](https://github.com/yandex/ClickHouse/pull/4741) ([olegkv](https://github.com/olegkv))
 * Исправлено срабатывание thread-санитайзера с ошибкой `destroy of a locked mutex`. [#4742](https://github.com/yandex/ClickHouse/pull/4742) ([alexey-milovidov](https://github.com/alexey-milovidov))
@@ -108,7 +108,6 @@
 * Исправлен дедлок, который мог происходить при запросе `DROP DATABASE dictionary`. [#4701](https://github.com/yandex/ClickHouse/pull/4701) ([alexey-milovidov](https://github.com/alexey-milovidov))
 * Исправлено неопределенное поведение в функциях `median` и `quantile`. [#4702](https://github.com/yandex/ClickHouse/pull/4702) ([hcz](https://github.com/hczhcz))
 * Исправлено определение уровня сжатия при указании настройки `network_compression_method` в нижнем регистре. Было сломано в v19.1. [#4706](https://github.com/yandex/ClickHouse/pull/4706) ([proller](https://github.com/proller))
-* Теперь обычные колонки, а также колонки `DEFAULT`, `MATERIALIZED` и `ALIAS` хранятся в одном списке (исправляет issue [#2867](https://github.com/yandex/ClickHouse/issues/2867)). [#4707](https://github.com/yandex/ClickHouse/pull/4707) ([Alex Zatelepin](https://github.com/ztlpn))
 * Настройка `<timezone>UTC</timezone>` больше не игнорируется (исправляет issue [#4658](https://github.com/yandex/ClickHouse/issues/4658)). [#4718](https://github.com/yandex/ClickHouse/pull/4718) ([proller](https://github.com/proller))
 * Исправлено поведение функции `histogram` с `Distributed` таблицами. [#4741](https://github.com/yandex/ClickHouse/pull/4741) ([olegkv](https://github.com/olegkv))
 * Исправлено срабатывание thread-санитайзера с ошибкой `destroy of a locked mutex`. [#4742](https://github.com/yandex/ClickHouse/pull/4742) ([alexey-milovidov](https://github.com/alexey-milovidov))
@@ -135,6 +134,10 @@
 * Исправлен результат функции `toISOWeek` для 1970 года. [#4988](https://github.com/yandex/ClickHouse/pull/4988) ([alexey-milovidov](https://github.com/alexey-milovidov))
 * Исправлено дублирование `DROP`, `TRUNCATE` и `OPTIMIZE` запросов, когда они выполнялись `ON CLUSTER` для семейства таблиц `ReplicatedMergeTree*`. [#4991](https://github.com/yandex/ClickHouse/pull/4991) ([alesapin](https://github.com/alesapin))
 
+### Улучшения
+
+* Теперь обычные колонки, а также колонки `DEFAULT`, `MATERIALIZED` и `ALIAS` хранятся в одном списке (исправляет issue [#2867](https://github.com/yandex/ClickHouse/issues/2867)). [#4707](https://github.com/yandex/ClickHouse/pull/4707) ([Alex Zatelepin](https://github.com/ztlpn))
+
 ## ClickHouse release 19.4.3.11, 2019-04-02
 
 ### Исправление ошибок

From 568ef7880359d63ed800ff0b0ac7995df614df8a Mon Sep 17 00:00:00 2001
From: Evgenii Pravda <evgeniipravda@gmail.com>
Date: Sat, 27 Apr 2019 01:07:32 +0300
Subject: [PATCH 075/118] Add MSD radix sort

---
 dbms/src/AggregateFunctions/QuantileTDigest.h |   2 +-
 dbms/src/Columns/ColumnVector.cpp             |   2 +-
 dbms/src/Common/RadixSort.h                   | 238 ++++++++++++++----
 dbms/src/Common/tests/radix_sort.cpp          |   2 +-
 dbms/src/Interpreters/RowRefs.h               |   2 +-
 5 files changed, 187 insertions(+), 59 deletions(-)

diff --git a/dbms/src/AggregateFunctions/QuantileTDigest.h b/dbms/src/AggregateFunctions/QuantileTDigest.h
index c4ee76b6eed..b32519519f8 100644
--- a/dbms/src/AggregateFunctions/QuantileTDigest.h
+++ b/dbms/src/AggregateFunctions/QuantileTDigest.h
@@ -136,7 +136,7 @@ class QuantileTDigest
     {
         if (unmerged > 0)
         {
-            RadixSort<RadixSortTraits>::execute(summary.data(), summary.size());
+            RadixSort<RadixSortTraits>::executeLsd(summary.data(), summary.size());
 
             if (summary.size() > 3)
             {
diff --git a/dbms/src/Columns/ColumnVector.cpp b/dbms/src/Columns/ColumnVector.cpp
index ea2ce1d7af1..4e806122b03 100644
--- a/dbms/src/Columns/ColumnVector.cpp
+++ b/dbms/src/Columns/ColumnVector.cpp
@@ -120,7 +120,7 @@ void ColumnVector<T>::getPermutation(bool reverse, size_t limit, int nan_directi
                 for (UInt32 i = 0; i < s; ++i)
                     pairs[i] = {data[i], i};
 
-                RadixSort<RadixSortTraits<T>>::execute(pairs.data(), s);
+                RadixSort<RadixSortTraits<T>>::executeLsd(pairs.data(), s);
 
                 /// Radix sort treats all NaNs to be greater than all numbers.
                 /// If the user needs the opposite, we must move them accordingly.
diff --git a/dbms/src/Common/RadixSort.h b/dbms/src/Common/RadixSort.h
index 2fe61c72b6a..2e8b5c81b85 100644
--- a/dbms/src/Common/RadixSort.h
+++ b/dbms/src/Common/RadixSort.h
@@ -43,6 +43,70 @@ struct RadixSortMallocAllocator
 };
 
 
+template <typename KeyBits>
+struct RadixSortIdentityTransform
+{
+    static constexpr bool transform_is_simple = true;
+
+    static KeyBits forward(KeyBits x)     { return x; }
+    static KeyBits backward(KeyBits x)    { return x; }
+};
+
+
+
+template <typename TElement>
+struct RadixSortUIntTraits
+{
+    using Element = TElement;
+    using Key = Element;
+    using CountType = uint32_t;
+    using KeyBits = Key;
+
+    static constexpr size_t PART_SIZE_BITS = 8;
+
+    using Transform = RadixSortIdentityTransform<KeyBits>;
+    using Allocator = RadixSortMallocAllocator;
+
+    static Key & extractKey(Element & elem) { return elem; }
+
+    static bool compare(TElement x, TElement y)
+    {
+        return x < y;
+    }
+};
+
+
+template <typename KeyBits>
+struct RadixSortSignedTransform
+{
+    static constexpr bool transform_is_simple = true;
+
+    static KeyBits forward(KeyBits x)     { return x ^ (KeyBits(1) << (sizeof(KeyBits) * 8 - 1)); }
+    static KeyBits backward(KeyBits x)    { return x ^ (KeyBits(1) << (sizeof(KeyBits) * 8 - 1)); }
+};
+
+template <typename TElement>
+struct RadixSortIntTraits
+{
+    using Element = TElement;
+    using Key = Element;
+    using CountType = uint32_t;
+    using KeyBits = std::make_unsigned_t<Key>;
+
+    static constexpr size_t PART_SIZE_BITS = 8;
+
+    using Transform = RadixSortSignedTransform<KeyBits>;
+    using Allocator = RadixSortMallocAllocator;
+
+    static Key & extractKey(Element & elem) { return elem; }
+
+    static bool compare(TElement x, TElement y)
+    {
+        return x < y;
+    }
+};
+
+
 /** A transformation that transforms the bit representation of a key into an unsigned integer number,
   *  that the order relation over the keys will match the order relation over the obtained unsigned numbers.
   * For floats this conversion does the following:
@@ -89,59 +153,12 @@ struct RadixSortFloatTraits
 
     /// The function to get the key from an array element.
     static Key & extractKey(Element & elem) { return elem; }
-};
 
-template <typename KeyBits>
-struct RadixSortIdentityTransform
-{
-    static constexpr bool transform_is_simple = true;
-
-    static KeyBits forward(KeyBits x)     { return x; }
-    static KeyBits backward(KeyBits x)    { return x; }
-};
-
-
-template <typename KeyBits>
-struct RadixSortSignedTransform
-{
-    static constexpr bool transform_is_simple = true;
-
-    static KeyBits forward(KeyBits x)     { return x ^ (KeyBits(1) << (sizeof(KeyBits) * 8 - 1)); }
-    static KeyBits backward(KeyBits x)    { return x ^ (KeyBits(1) << (sizeof(KeyBits) * 8 - 1)); }
-};
-
-
-template <typename TElement>
-struct RadixSortUIntTraits
-{
-    using Element = TElement;
-    using Key = Element;
-    using CountType = uint32_t;
-    using KeyBits = Key;
-
-    static constexpr size_t PART_SIZE_BITS = 8;
-
-    using Transform = RadixSortIdentityTransform<KeyBits>;
-    using Allocator = RadixSortMallocAllocator;
-
-    static Key & extractKey(Element & elem) { return elem; }
-};
-
-
-template <typename TElement>
-struct RadixSortIntTraits
-{
-    using Element = TElement;
-    using Key = Element;
-    using CountType = uint32_t;
-    using KeyBits = std::make_unsigned_t<Key>;
-
-    static constexpr size_t PART_SIZE_BITS = 8;
-
-    using Transform = RadixSortSignedTransform<KeyBits>;
-    using Allocator = RadixSortMallocAllocator;
-
-    static Key & extractKey(Element & elem) { return elem; }
+    // TODO: Correct handling of NaNs, NULLs, etc
+    static bool compare(TElement x, TElement y)
+    {
+        return x < y;
+    }
 };
 
 
@@ -163,6 +180,8 @@ private:
     using CountType   = typename Traits::CountType;
     using KeyBits     = typename Traits::KeyBits;
 
+    static constexpr size_t INSERT_SORT_THRESHOLD = 64;
+
     static constexpr size_t HISTOGRAM_SIZE = 1 << Traits::PART_SIZE_BITS;
     static constexpr size_t PART_BITMASK = HISTOGRAM_SIZE - 1;
     static constexpr size_t KEY_BITS = sizeof(Key) * 8;
@@ -179,8 +198,101 @@ private:
     static KeyBits keyToBits(Key x) { return ext::bit_cast<KeyBits>(x); }
     static Key bitsToKey(KeyBits x) { return ext::bit_cast<Key>(x); }
 
+    static inline void insertSortInternal(Element * arr, size_t size)
+    {
+        for (Element * i = arr + 1; i < arr + size; ++i)
+        {
+            if (Traits::compare(*i, *(i - 1)))
+            {
+                Element * j;
+                Element tmp = *i;
+                *i = *(i - 1);
+                for (j = i - 1; j > arr && Traits::compare(tmp, *(j - 1)); --j)
+                    *j = *(j - 1);
+                *j = tmp;
+            }
+        }
+    }
+
+    template <int PASS>
+    static inline void msdRadixSortInternal(Element * arr, size_t size, size_t limit)
+    {
+        Element *last_[HISTOGRAM_SIZE + 1];
+        Element ** last = last_ + 1;
+        size_t count[HISTOGRAM_SIZE] = {0};
+
+        for (Element * i = arr; i < arr + size; ++i)
+            ++count[getPart(PASS, *i)];
+
+        last_[0] = last_[1] = arr;
+
+        size_t bucketsForRecursion = HISTOGRAM_SIZE;
+        Element * finish = arr + size;
+        for (size_t i = 1; i < HISTOGRAM_SIZE; ++i)
+        {
+            last[i] = last[i - 1] + count[i - 1];
+            if (last[i] >= arr + limit)
+            {
+                bucketsForRecursion = i;
+                finish = last[i];
+            }
+        }
+
+        for (size_t i = 0; i < bucketsForRecursion; ++i)
+        {
+            Element * end = last[i - 1] + count[i];
+            if (end == finish)
+            {
+                last[i] = end;
+                break;
+            }
+            while (last[i] != end)
+            {
+                Element swapper = *last[i];
+                KeyBits tag = getPart(PASS, swapper);
+                if (tag != i)
+                {
+                    do
+                    {
+                        std::swap(swapper, *last[tag]++);
+                    } while ((tag = getPart(PASS, swapper)) != i);
+                    *last[i] = swapper;
+                }
+                ++last[i];
+            }
+        }
+
+        if constexpr (PASS > 0)
+        {
+            for (size_t i = 0; i < bucketsForRecursion - 1; ++i)
+            {
+                Element * start = last[i - 1];
+                size_t size = last[i] - last[i - 1];
+                msdRadixSortInternalHelper<PASS - 1>(start, size, size);
+            }
+
+            // Sort last necessary bucket with sublimit
+            Element * start = last[bucketsForRecursion - 2];
+            size_t size = last[bucketsForRecursion - 1] - last[bucketsForRecursion - 2];
+            size_t sublimit = limit - (last[bucketsForRecursion - 1] - arr);
+            msdRadixSortInternalHelper<PASS - 1>(start, size, sublimit);
+        }
+    }
+
+    template <int PASS>
+    static inline void msdRadixSortInternalHelper(Element * arr, size_t size, size_t limit)
+    {
+        if (size <= INSERT_SORT_THRESHOLD)
+            insertSortInternal(arr, size);
+        else
+            msdRadixSortInternal<PASS>(arr, size, limit);
+    }
+
 public:
-    static void execute(Element * arr, size_t size)
+    /* Least significant digit radix sort
+     * The most efficient stable general-purpose sorting algorithm
+     */
+    static void executeLsd(Element * arr, size_t size)
     {
         /// If the array is smaller than 256, then it is better to use another algorithm.
 
@@ -247,6 +359,16 @@ public:
 
         allocator.deallocate(swap_buffer, size * sizeof(Element));
     }
+
+    /* Most significant digit radix sort
+     * Usually slower than LSD and is not stable, but allows partial sorting
+     * Based on https://github.com/voutcn/kxsort
+     */
+    static void executeMsd(Element * arr, size_t size, size_t limit)
+    {
+        limit = std::min(limit, size);
+        msdRadixSortInternalHelper<NUM_PASSES - 1>(arr, size, limit);
+    }
 };
 
 
@@ -254,7 +376,13 @@ public:
 /// Use RadixSort with custom traits for complex types instead.
 
 template <typename T>
-void radixSort(T * arr, size_t size)
+void lsdRadixSort(T *arr, size_t size)
 {
-    RadixSort<RadixSortNumTraits<T>>::execute(arr, size);
+    RadixSort<RadixSortNumTraits<T>>::executeLsd(arr, size);
+}
+
+template <typename T>
+void msdRadixSort(T * arr, size_t size, size_t limit)
+{
+    RadixSort<RadixSortNumTraits<T>>::executeMsd(arr, size, limit);
 }
diff --git a/dbms/src/Common/tests/radix_sort.cpp b/dbms/src/Common/tests/radix_sort.cpp
index 7de1f9f170a..d60e453c6bd 100644
--- a/dbms/src/Common/tests/radix_sort.cpp
+++ b/dbms/src/Common/tests/radix_sort.cpp
@@ -16,7 +16,7 @@ void NO_INLINE sort1(Key * data, size_t size)
 
 void NO_INLINE sort2(Key * data, size_t size)
 {
-    radixSort(data, size);
+    lsdRadixSort(data, size);
 }
 
 void NO_INLINE sort3(Key * data, size_t size)
diff --git a/dbms/src/Interpreters/RowRefs.h b/dbms/src/Interpreters/RowRefs.h
index 64acba37475..78e7515729a 100644
--- a/dbms/src/Interpreters/RowRefs.h
+++ b/dbms/src/Interpreters/RowRefs.h
@@ -92,7 +92,7 @@ private:
                 {
                     /// TODO: It has been tested only for UInt32 yet. It needs to check UInt64, Float32/64.
                     if constexpr (std::is_same_v<TKey, UInt32>)
-                        RadixSort<RadixSortTraits>::execute(&array[0], array.size());
+                        RadixSort<RadixSortTraits>::executeLsd(&array[0], array.size());
                     else
                         std::sort(array.begin(), array.end());
                 }

From 7e3c8d6022655aa58d1e006c4de012182057e4d6 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitbar@yandex-team.ru>
Date: Wed, 24 Apr 2019 18:03:15 +0300
Subject: [PATCH 076/118] Implement new constraints: "min" and "readonly".

---
 dbms/src/Interpreters/SettingsConstraints.cpp | 146 ++++++++++++------
 dbms/src/Interpreters/SettingsConstraints.h   |  28 +++-
 .../configs/users.xml                         |   5 +
 .../test_settings_constraints/test.py         |  54 +++++--
 4 files changed, 177 insertions(+), 56 deletions(-)

diff --git a/dbms/src/Interpreters/SettingsConstraints.cpp b/dbms/src/Interpreters/SettingsConstraints.cpp
index 6243afc9828..43b62335a4d 100644
--- a/dbms/src/Interpreters/SettingsConstraints.cpp
+++ b/dbms/src/Interpreters/SettingsConstraints.cpp
@@ -18,6 +18,20 @@ namespace ErrorCodes
 namespace
 {
     thread_local Settings temp_settings;
+
+    Field convertSettingValueToComparableType(const Field & value, size_t setting_index)
+    {
+        auto && temp_setting = temp_settings[setting_index];
+        temp_setting.setValue(value);
+        return temp_setting.getValue();
+    }
+
+    Field convertSettingValueToComparableType(const String & value, size_t setting_index)
+    {
+        auto && temp_setting = temp_settings[setting_index];
+        temp_setting.setValue(value);
+        return temp_setting.getValue();
+    }
 }
 
 
@@ -29,25 +43,87 @@ SettingsConstraints & SettingsConstraints::operator=(SettingsConstraints && src)
 SettingsConstraints::~SettingsConstraints() = default;
 
 
+void SettingsConstraints::clear()
+{
+    constraints_by_index.clear();
+}
+
+
+void SettingsConstraints::setReadOnly(const String & name, bool read_only)
+{
+    size_t setting_index = Settings::findIndexStrict(name);
+    getConstraintRef(setting_index).read_only = read_only;
+}
+
+void SettingsConstraints::setMinValue(const String & name, const String & min_value)
+{
+    size_t setting_index = Settings::findIndexStrict(name);
+    getConstraintRef(setting_index).min_value = convertSettingValueToComparableType(min_value, setting_index);
+}
+
+void SettingsConstraints::setMinValue(const String & name, const Field & min_value)
+{
+    size_t setting_index = Settings::findIndexStrict(name);
+    getConstraintRef(setting_index).min_value = convertSettingValueToComparableType(min_value, setting_index);
+}
+
 void SettingsConstraints::setMaxValue(const String & name, const String & max_value)
 {
-    max_settings.set(name, max_value);
+    size_t setting_index = Settings::findIndexStrict(name);
+    getConstraintRef(setting_index).max_value = convertSettingValueToComparableType(max_value, setting_index);
 }
 
 void SettingsConstraints::setMaxValue(const String & name, const Field & max_value)
 {
-    max_settings.set(name, max_value);
+    size_t setting_index = Settings::findIndexStrict(name);
+    getConstraintRef(setting_index).max_value = convertSettingValueToComparableType(max_value, setting_index);
 }
 
 
 void SettingsConstraints::check(const Settings & current_settings, const SettingChange & change) const
 {
-    size_t index = current_settings.findIndex(change.name);
-    if (index == Settings::npos)
+    const String & name = change.name;
+    size_t setting_index = Settings::findIndex(name);
+    if (setting_index == Settings::npos)
         return;
-    // We store `change.value` to `temp_settings` to ensure it's converted to the correct type.
-    temp_settings[index].setValue(change.value);
-    checkImpl(current_settings, index);
+
+    Field new_value = convertSettingValueToComparableType(change.value, setting_index);
+    Field current_value = current_settings.get(setting_index);
+
+    /// Setting isn't checked if value wasn't changed.
+    if (current_value == new_value)
+        return;
+
+    if (!current_settings.allow_ddl && name == "allow_ddl")
+        throw Exception("Cannot modify 'allow_ddl' setting when DDL queries are prohibited for the user", ErrorCodes::QUERY_IS_PROHIBITED);
+
+    /** The `readonly` value is understood as follows:
+      * 0 - everything allowed.
+      * 1 - only read queries can be made; you can not change the settings.
+      * 2 - You can only do read queries and you can change the settings, except for the `readonly` setting.
+      */
+    if (current_settings.readonly == 1)
+        throw Exception("Cannot modify '" + name + "' setting in readonly mode", ErrorCodes::READONLY);
+
+    if (current_settings.readonly > 1 && name == "readonly")
+        throw Exception("Cannot modify 'readonly' setting in readonly mode", ErrorCodes::READONLY);
+
+    const Constraint * constraint = tryGetConstraint(setting_index);
+    if (constraint)
+    {
+        if (constraint->read_only)
+            throw Exception("Setting " + name + " should not be changed", ErrorCodes::SETTING_CONSTRAINT_VIOLATION);
+
+        if (!constraint->min_value.isNull() && (new_value < constraint->min_value))
+            throw Exception(
+                "Setting " + name + " shouldn't be less than " + applyVisitor(FieldVisitorToString(), constraint->min_value),
+                ErrorCodes::SETTING_CONSTRAINT_VIOLATION);
+
+        if (!constraint->max_value.isNull() && (new_value > constraint->max_value))
+            throw Exception(
+                "Setting " + name + " shouldn't be greater than " + applyVisitor(FieldVisitorToString(), constraint->max_value),
+                ErrorCodes::SETTING_CONSTRAINT_VIOLATION);
+    }
 }
 
 
@@ -58,42 +134,20 @@ void SettingsConstraints::check(const Settings & current_settings, const Setting
 }
 
 
-void SettingsConstraints::checkImpl(const Settings & current_settings, size_t index) const
+SettingsConstraints::Constraint & SettingsConstraints::getConstraintRef(size_t index)
 {
-    const auto & new_setting = temp_settings[index];
-    Field new_value = new_setting.getValue();
+    auto it = constraints_by_index.find(index);
+    if (it == constraints_by_index.end())
+        it = constraints_by_index.emplace(index, Constraint{}).first;
+    return it->second;
+}
 
-    const auto & current_setting = current_settings[index];
-    Field current_value = current_setting.getValue();
-
-    /// Setting isn't checked if value wasn't changed.
-    if (current_value == new_value)
-        return;
-
-    const StringRef & name = new_setting.getName();
-    if (!current_settings.allow_ddl && name == "allow_ddl")
-        throw Exception("Cannot modify 'allow_ddl' setting when DDL queries are prohibited for the user", ErrorCodes::QUERY_IS_PROHIBITED);
-
-    /** The `readonly` value is understood as follows:
-      * 0 - everything allowed.
-      * 1 - only read queries can be made; you can not change the settings.
-      * 2 - You can only do read queries and you can change the settings, except for the `readonly` setting.
-      */
-    if (current_settings.readonly == 1)
-        throw Exception("Cannot modify '" + name.toString() + "' setting in readonly mode", ErrorCodes::READONLY);
-
-    if (current_settings.readonly > 1 && name == "readonly")
-        throw Exception("Cannot modify 'readonly' setting in readonly mode", ErrorCodes::READONLY);
-
-    const auto & max_setting = max_settings[index];
-    if (max_setting.isChanged())
-    {
-        Field max_value = max_setting.getValue();
-        if (new_value > max_value)
-            throw Exception(
-                "Setting " + name.toString() + " shouldn't be greater than " + applyVisitor(FieldVisitorToString(), max_value),
-                ErrorCodes::SETTING_CONSTRAINT_VIOLATION);
-    }
+const SettingsConstraints::Constraint * SettingsConstraints::tryGetConstraint(size_t index) const
+{
+    auto it = constraints_by_index.find(index);
+    if (it == constraints_by_index.end())
+        return nullptr;
+    return &it->second;
 }
 
 
@@ -124,9 +178,13 @@ void SettingsConstraints::loadFromConfig(const String & path_to_constraints, con
         config.keys(path_to_name, constraint_types);
         for (const String & constraint_type : constraint_types)
         {
-            String path_to_type = path_to_name + "." + constraint_type;
-            if (constraint_type == "max")
-                setMaxValue(name, config.getString(path_to_type));
+            auto get_constraint_value = [&]{ return config.getString(path_to_name + "." + constraint_type); };
+            if (constraint_type == "min")
+                setMinValue(name, get_constraint_value());
+            else if (constraint_type == "max")
+                setMaxValue(name, get_constraint_value());
+            else if (constraint_type == "readonly")
+                setReadOnly(name, true);
             else
                 throw Exception("Setting " + constraint_type + " value for " + name + " isn't supported", ErrorCodes::NOT_IMPLEMENTED);
         }
diff --git a/dbms/src/Interpreters/SettingsConstraints.h b/dbms/src/Interpreters/SettingsConstraints.h
index 5fb3c2adcae..cc0e3def441 100644
--- a/dbms/src/Interpreters/SettingsConstraints.h
+++ b/dbms/src/Interpreters/SettingsConstraints.h
@@ -19,18 +19,27 @@ struct Settings;
 /** Checks if specified changes of settings are allowed or not.
   * If the changes are not allowed (i.e. violates some constraints) this class throws an exception.
   * The constraints are set by editing the `users.xml` file.
-  * For examples, the following lines in `users.xml` will set that `max_memory_usage` cannot be greater than 20000000000:
+  *
+  * For examples, the following lines in `users.xml` will set that `max_memory_usage` cannot be greater than 20000000000,
+  * and `force_index_by_date` should be always equal to 0:
+  *
   * <profiles>
   *   <user_profile>
   *       <max_memory_usage>10000000000</max_memory_usage>
+  *       <force_index_by_date>0</force_index_by_date>
   *       ...
   *       <constraints>
   *           <max_memory_usage>
+  *               <min>200000</min>
   *               <max>20000000000</max>
   *           </max_memory_usage>
+  *           <force_index_by_date>
+  *               <readonly/>
+  *           </force_index_by_date>
   *       </constraints>
   *   </user_profile>
   * </profiles>
+  *
   * This class also checks that we are not in the read-only mode.
   * If a setting cannot be change due to the read-only mode this class throws an exception.
   * The value of `readonly` value is understood as follows:
@@ -48,8 +57,13 @@ public:
     SettingsConstraints & operator =(SettingsConstraints && src);
     ~SettingsConstraints();
 
+    void clear();
+
+    void setMinValue(const String & name, const String & min_value);
+    void setMinValue(const String & name, const Field & min_value);
     void setMaxValue(const String & name, const String & max_value);
     void setMaxValue(const String & name, const Field & max_value);
+    void setReadOnly(const String & name, bool read_only);
 
     void check(const Settings & current_settings, const SettingChange & change) const;
     void check(const Settings & current_settings, const SettingsChanges & changes) const;
@@ -63,9 +77,17 @@ public:
     void loadFromConfig(const String & path, const Poco::Util::AbstractConfiguration & config);
 
 private:
-    void checkImpl(const Settings & current_settings, size_t index) const;
+    struct Constraint
+    {
+        bool read_only = false;
+        Field min_value;
+        Field max_value;
+    };
 
-    Settings max_settings;
+    Constraint & getConstraintRef(size_t index);
+    const Constraint * tryGetConstraint(size_t) const;
+
+    std::unordered_map<size_t, Constraint> constraints_by_index;
 };
 
 }
diff --git a/dbms/tests/integration/test_settings_constraints/configs/users.xml b/dbms/tests/integration/test_settings_constraints/configs/users.xml
index c33f3743ef4..8bbe18d7c6c 100644
--- a/dbms/tests/integration/test_settings_constraints/configs/users.xml
+++ b/dbms/tests/integration/test_settings_constraints/configs/users.xml
@@ -3,10 +3,15 @@
     <profiles>
         <default>
             <max_memory_usage>10000000000</max_memory_usage>
+            <force_index_by_date>0</force_index_by_date>
             <constraints>
                 <max_memory_usage>
+                    <min>5000000000</min>
                     <max>20000000000</max>
                 </max_memory_usage>
+               <force_index_by_date>
+                   <readonly/>
+               </force_index_by_date>
             </constraints>
         </default>
     </profiles>
diff --git a/dbms/tests/integration/test_settings_constraints/test.py b/dbms/tests/integration/test_settings_constraints/test.py
index 1b3604c63dd..be296b51827 100644
--- a/dbms/tests/integration/test_settings_constraints/test.py
+++ b/dbms/tests/integration/test_settings_constraints/test.py
@@ -18,6 +18,48 @@ def started_cluster():
         cluster.shutdown()
 
 
+def test_read_only_constraint(started_cluster):
+    # Change a setting for session with SET.
+    assert instance.query("SELECT value FROM system.settings WHERE name='force_index_by_date'") ==\
+           "0\n"
+
+    expected_error = "Setting force_index_by_date should not be changed"
+    assert expected_error in instance.query_and_get_error("SET force_index_by_date=1")
+
+    # Change a setting for query with SETTINGS.
+    assert instance.query("SELECT value FROM system.settings WHERE name='force_index_by_date'") ==\
+           "0\n"
+
+    assert expected_error in instance.query_and_get_error(
+           "SELECT value FROM system.settings WHERE name='force_index_by_date' "
+           "SETTINGS force_index_by_date=1")
+
+
+def test_min_constraint(started_cluster):
+    # Change a setting for session with SET.
+    assert instance.query("SELECT value FROM system.settings WHERE name='max_memory_usage'") ==\
+           "10000000000\n"
+
+    assert instance.query("SET max_memory_usage=5000000000;\n"
+                          "SELECT value FROM system.settings WHERE name='max_memory_usage'") ==\
+           "5000000000\n"
+
+    expected_error = "Setting max_memory_usage shouldn't be less than 5000000000"
+    assert expected_error in instance.query_and_get_error("SET max_memory_usage=4999999999")
+
+    # Change a setting for query with SETTINGS.
+    assert instance.query("SELECT value FROM system.settings WHERE name='max_memory_usage'") ==\
+           "10000000000\n"
+
+    assert instance.query("SET max_memory_usage=5000000001;\n"
+                          "SELECT value FROM system.settings WHERE name='max_memory_usage'") ==\
+           "5000000001\n"
+
+    assert expected_error in instance.query_and_get_error(
+           "SELECT value FROM system.settings WHERE name='max_memory_usage' "
+           "SETTINGS max_memory_usage=4999999999")
+
+
 def test_max_constraint(started_cluster):
     # Change a setting for session with SET.
     assert instance.query("SELECT value FROM system.settings WHERE name='max_memory_usage'") ==\
@@ -30,20 +72,14 @@ def test_max_constraint(started_cluster):
     expected_error = "Setting max_memory_usage shouldn't be greater than 20000000000"
     assert expected_error in instance.query_and_get_error("SET max_memory_usage=20000000001")
 
-    assert instance.query("SET max_memory_usage=11000000000;\n"
-                          "SELECT value FROM system.settings WHERE name='max_memory_usage'") ==\
-           "11000000000\n"
-
-    # Change a setting for query with SETTINGS.
+     # Change a setting for query with SETTINGS.
     assert instance.query("SELECT value FROM system.settings WHERE name='max_memory_usage'") ==\
            "10000000000\n"
 
     assert instance.query("SELECT value FROM system.settings WHERE name='max_memory_usage' "
-                          "SETTINGS max_memory_usage=20000000000") == "20000000000\n"
+                          "SETTINGS max_memory_usage=19999999999") == "19999999999\n"
 
     assert expected_error in instance.query_and_get_error(
            "SELECT value FROM system.settings WHERE name='max_memory_usage' "
            "SETTINGS max_memory_usage=20000000001")
-
-    assert instance.query("SELECT value FROM system.settings WHERE name='max_memory_usage' "
-                          "SETTINGS max_memory_usage=12000000000") == "12000000000\n"
+ 
\ No newline at end of file

From 78016712256a6a77093f87018271ae57aa1045e7 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitbar@yandex-team.ru>
Date: Thu, 25 Apr 2019 01:52:08 +0300
Subject: [PATCH 077/118] Use shared_ptr to avoid copying SettingsConstraints

---
 dbms/src/Interpreters/Context.cpp | 16 ++++++++++++----
 dbms/src/Interpreters/Context.h   |  9 +++++----
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp
index 5562a30ec82..f2b9dfd9508 100644
--- a/dbms/src/Interpreters/Context.cpp
+++ b/dbms/src/Interpreters/Context.cpp
@@ -298,6 +298,8 @@ private:
 
 
 Context::Context() = default;
+Context::Context(const Context &) = default;
+Context & Context::operator=(const Context &) = default;
 
 
 Context Context::createGlobal(std::unique_ptr<IRuntimeComponentsFactory> runtime_components_factory)
@@ -620,7 +622,7 @@ void Context::calculateUserSettings()
     /// NOTE: we ignore global_context settings (from which it is usually copied)
     /// NOTE: global_context settings are immutable and not auto updated
     settings = Settings();
-    settings_constraints = SettingsConstraints();
+    settings_constraints = nullptr;
 
     /// 2) Apply settings from default profile
     auto default_profile_name = getDefaultProfileName();
@@ -635,7 +637,11 @@ void Context::calculateUserSettings()
 void Context::setProfile(const String & profile)
 {
     settings.setProfile(profile, *shared->users_config);
-    settings_constraints.setProfile(profile, *shared->users_config);
+
+    auto new_constraints
+        = settings_constraints ? std::make_shared<SettingsConstraints>(*settings_constraints) : std::make_shared<SettingsConstraints>();
+    new_constraints->setProfile(profile, *shared->users_config);
+    settings_constraints = std::move(new_constraints);
 }
 
 
@@ -1076,13 +1082,15 @@ void Context::applySettingsChanges(const SettingsChanges & changes)
 
 void Context::checkSettingsConstraints(const SettingChange & change)
 {
-    settings_constraints.check(settings, change);
+    if (settings_constraints)
+        settings_constraints->check(settings, change);
 }
 
 
 void Context::checkSettingsConstraints(const SettingsChanges & changes)
 {
-    settings_constraints.check(settings, changes);
+    if (settings_constraints)
+        settings_constraints->check(settings, changes);
 }
 
 
diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h
index 6962c69d2cf..3bfd7d06d3b 100644
--- a/dbms/src/Interpreters/Context.h
+++ b/dbms/src/Interpreters/Context.h
@@ -4,7 +4,6 @@
 #include <Core/NamesAndTypes.h>
 #include <Core/Types.h>
 #include <Interpreters/ClientInfo.h>
-#include <Interpreters/SettingsConstraints.h>
 #include <Core/Settings.h>
 #include <Parsers/IAST_fwd.h>
 #include <Common/LRUCache.h>
@@ -79,6 +78,8 @@ class ActionLocksManager;
 using ActionLocksManagerPtr = std::shared_ptr<ActionLocksManager>;
 class ShellCommand;
 class ICompressionCodec;
+class SettingsConstraints;
+
 
 #if USE_EMBEDDED_COMPILER
 
@@ -126,7 +127,7 @@ private:
     std::shared_ptr<QuotaForIntervals> quota;           /// Current quota. By default - empty quota, that have no limits.
     String current_database;
     Settings settings;                                  /// Setting for query execution.
-    SettingsConstraints settings_constraints;
+    std::shared_ptr<const SettingsConstraints> settings_constraints;
     using ProgressCallback = std::function<void(const Progress & progress)>;
     ProgressCallback progress_callback;                 /// Callback for tracking progress of query execution.
     QueryStatus * process_list_elem = nullptr;   /// For tracking total resource usage for query.
@@ -163,8 +164,8 @@ public:
     static Context createGlobal(std::unique_ptr<IRuntimeComponentsFactory> runtime_components_factory);
     static Context createGlobal();
 
-    Context(const Context &) = default;
-    Context & operator=(const Context &) = default;
+    Context(const Context &);
+    Context & operator=(const Context &);
     ~Context();
 
     String getPath() const;

From 738a49f534e9a75ac9ef2825ffd5bdf9e355ca2c Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitbar@yandex-team.ru>
Date: Thu, 25 Apr 2019 17:08:20 +0300
Subject: [PATCH 078/118] Use iterators instead of macros APPLY_FOR_SETTINGS
 and others.

---
 dbms/programs/benchmark/Benchmark.cpp         | 17 ++-------
 dbms/programs/client/Client.cpp               | 34 ++++++++---------
 dbms/programs/local/LocalServer.cpp           | 20 +++-------
 .../performance-test/PerformanceTestInfo.cpp  | 20 ++--------
 .../performance-test/PerformanceTestSuite.cpp | 22 +++--------
 dbms/src/Core/Settings.cpp                    | 37 ++++++++++++++-----
 dbms/src/Core/Settings.h                      | 13 +++++++
 .../System/StorageSystemMergeTreeSettings.cpp | 14 +++----
 .../Storages/System/StorageSystemSettings.cpp | 16 ++++----
 9 files changed, 89 insertions(+), 104 deletions(-)

diff --git a/dbms/programs/benchmark/Benchmark.cpp b/dbms/programs/benchmark/Benchmark.cpp
index 89b363a2709..534ed415ca0 100644
--- a/dbms/programs/benchmark/Benchmark.cpp
+++ b/dbms/programs/benchmark/Benchmark.cpp
@@ -451,14 +451,14 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv)
             ("password",      value<std::string>()->default_value(""),          "")
             ("database",      value<std::string>()->default_value("default"),   "")
             ("stacktrace",                                                      "print stack traces of exceptions")
-
-        #define DECLARE_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION) (#NAME, boost::program_options::value<std::string> (), DESCRIPTION)
-            APPLY_FOR_SETTINGS(DECLARE_SETTING)
-        #undef DECLARE_SETTING
         ;
 
+        Settings settings;
+        settings.addProgramOptions(desc);
+
         boost::program_options::variables_map options;
         boost::program_options::store(boost::program_options::parse_command_line(argc, argv, desc), options);
+        boost::program_options::notify(options);
 
         if (options.count("help"))
         {
@@ -469,15 +469,6 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv)
 
         print_stacktrace = options.count("stacktrace");
 
-        /// Extract `settings` and `limits` from received `options`
-        Settings settings;
-
-        #define EXTRACT_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION) \
-        if (options.count(#NAME)) \
-            settings.set(#NAME, options[#NAME].as<std::string>());
-        APPLY_FOR_SETTINGS(EXTRACT_SETTING)
-        #undef EXTRACT_SETTING
-
         UseSSL use_ssl;
 
         Benchmark benchmark(
diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp
index 20c01f4f79c..349f6690cbe 100644
--- a/dbms/programs/client/Client.cpp
+++ b/dbms/programs/client/Client.cpp
@@ -217,11 +217,12 @@ private:
         context.setApplicationType(Context::ApplicationType::CLIENT);
 
         /// settings and limits could be specified in config file, but passed settings has higher priority
-#define EXTRACT_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION) \
-        if (config().has(#NAME) && !context.getSettingsRef().NAME.changed) \
-            context.setSetting(#NAME, config().getString(#NAME));
-        APPLY_FOR_SETTINGS(EXTRACT_SETTING)
-#undef EXTRACT_SETTING
+        for (auto && setting : context.getSettingsRef())
+        {
+            const String & name = setting.getName().toString();
+            if (config().has(name) && !setting.isChanged())
+                setting.setValue(config().getString(name));
+        }
 
         /// Set path for format schema files
         if (config().has("format_schema_path"))
@@ -1614,8 +1615,6 @@ public:
             min_description_length = std::min(min_description_length, line_length - 2);
         }
 
-#define DECLARE_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION) (#NAME, po::value<std::string>(), DESCRIPTION)
-
         /// Main commandline options related to client functionality and all parameters from Settings.
         po::options_description main_description("Main options", line_length, min_description_length);
         main_description.add_options()
@@ -1658,9 +1657,9 @@ public:
             ("compression", po::value<bool>(), "enable or disable compression")
             ("log-level", po::value<std::string>(), "client log level")
             ("server_logs_file", po::value<std::string>(), "put server logs into specified file")
-            APPLY_FOR_SETTINGS(DECLARE_SETTING)
         ;
-#undef DECLARE_SETTING
+
+        context.getSettingsRef().addProgramOptions(main_description);
 
         /// Commandline options related to external tables.
         po::options_description external_description("External tables options");
@@ -1676,6 +1675,8 @@ public:
             common_arguments.size(), common_arguments.data()).options(main_description).run();
         po::variables_map options;
         po::store(parsed, options);
+        po::notify(options);
+
         if (options.count("version") || options.count("V"))
         {
             showClientVersion();
@@ -1726,15 +1727,14 @@ public:
             }
         }
 
-        /// Extract settings from the options.
-#define EXTRACT_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION)                \
-        if (options.count(#NAME))                                        \
-        {                                                                \
-            context.setSetting(#NAME, options[#NAME].as<std::string>()); \
-            config().setString(#NAME, options[#NAME].as<std::string>()); \
+        /// Copy settings-related program options to config.
+        /// TODO: Is this code necessary?
+        for (const auto & setting : context.getSettingsRef())
+        {
+            const String name = setting.getName().toString();
+            if (options.count(name))
+                config().setString(name, options[name].as<std::string>());
         }
-        APPLY_FOR_SETTINGS(EXTRACT_SETTING)
-#undef EXTRACT_SETTING
 
         if (options.count("config-file") && options.count("config"))
             throw Exception("Two or more configuration files referenced in arguments", ErrorCodes::BAD_ARGUMENTS);
diff --git a/dbms/programs/local/LocalServer.cpp b/dbms/programs/local/LocalServer.cpp
index 9808cadd303..73e43dffec3 100644
--- a/dbms/programs/local/LocalServer.cpp
+++ b/dbms/programs/local/LocalServer.cpp
@@ -69,11 +69,7 @@ void LocalServer::initialize(Poco::Util::Application & self)
 
 void LocalServer::applyCmdSettings()
 {
-#define EXTRACT_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION) \
-        if (cmd_settings.NAME.changed) \
-            context->getSettingsRef().NAME = cmd_settings.NAME;
-        APPLY_FOR_SETTINGS(EXTRACT_SETTING)
-#undef EXTRACT_SETTING
+    context->getSettingsRef().copyChangesFrom(cmd_settings);
 }
 
 /// If path is specified and not empty, will try to setup server environment and load existing metadata
@@ -414,7 +410,6 @@ void LocalServer::init(int argc, char ** argv)
         min_description_length = std::min(min_description_length, line_length - 2);
     }
 
-#define DECLARE_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION) (#NAME, po::value<std::string> (), DESCRIPTION)
     po::options_description description("Main options", line_length, min_description_length);
     description.add_options()
         ("help", "produce help message")
@@ -435,13 +430,15 @@ void LocalServer::init(int argc, char ** argv)
         ("verbose", "print query and other debugging info")
         ("ignore-error", "do not stop processing if a query failed")
         ("version,V", "print version information and exit")
-        APPLY_FOR_SETTINGS(DECLARE_SETTING);
-#undef DECLARE_SETTING
+        ;
+
+    cmd_settings.addProgramOptions(description);
 
     /// Parse main commandline options.
     po::parsed_options parsed = po::command_line_parser(argc, argv).options(description).run();
     po::variables_map options;
     po::store(parsed, options);
+    po::notify(options);
 
     if (options.count("version") || options.count("V"))
     {
@@ -457,13 +454,6 @@ void LocalServer::init(int argc, char ** argv)
         exit(0);
     }
 
-    /// Extract settings and limits from the options.
-#define EXTRACT_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION) \
-    if (options.count(#NAME)) \
-        cmd_settings.set(#NAME, options[#NAME].as<std::string>());
-    APPLY_FOR_SETTINGS(EXTRACT_SETTING)
-#undef EXTRACT_SETTING
-
     /// Save received data into the internal config.
     if (options.count("config-file"))
         config().setString("config-file", options["config-file"].as<std::string>());
diff --git a/dbms/programs/performance-test/PerformanceTestInfo.cpp b/dbms/programs/performance-test/PerformanceTestInfo.cpp
index 4150ad9461c..4b114823c6d 100644
--- a/dbms/programs/performance-test/PerformanceTestInfo.cpp
+++ b/dbms/programs/performance-test/PerformanceTestInfo.cpp
@@ -21,7 +21,7 @@ void extractSettings(
     const XMLConfigurationPtr & config,
     const std::string & key,
     const Strings & settings_list,
-    std::map<std::string, std::string> & settings_to_apply)
+    SettingsChanges & settings_to_apply)
 {
     for (const std::string & setup : settings_list)
     {
@@ -32,7 +32,7 @@ void extractSettings(
         if (value.empty())
             value = "true";
 
-        settings_to_apply[setup] = value;
+        settings_to_apply.emplace_back(SettingChange{setup, value});
     }
 }
 
@@ -70,7 +70,7 @@ void PerformanceTestInfo::applySettings(XMLConfigurationPtr config)
 {
     if (config->has("settings"))
     {
-        std::map<std::string, std::string> settings_to_apply;
+        SettingsChanges settings_to_apply;
         Strings config_settings;
         config->keys("settings", config_settings);
 
@@ -96,19 +96,7 @@ void PerformanceTestInfo::applySettings(XMLConfigurationPtr config)
         }
 
         extractSettings(config, "settings", config_settings, settings_to_apply);
-
-        /// This macro goes through all settings in the Settings.h
-        /// and, if found any settings in test's xml configuration
-        /// with the same name, sets its value to settings
-        std::map<std::string, std::string>::iterator it;
-#define EXTRACT_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION)   \
-        it = settings_to_apply.find(#NAME);                 \
-        if (it != settings_to_apply.end())                  \
-            settings.set(#NAME, settings_to_apply[#NAME]);
-
-        APPLY_FOR_SETTINGS(EXTRACT_SETTING)
-
-#undef EXTRACT_SETTING
+        settings.applyChanges(settings_to_apply);
 
         if (settings_contain("average_rows_speed_precision"))
             TestStats::avg_rows_speed_precision =
diff --git a/dbms/programs/performance-test/PerformanceTestSuite.cpp b/dbms/programs/performance-test/PerformanceTestSuite.cpp
index 9daf1b86ce7..9c72ba953dc 100644
--- a/dbms/programs/performance-test/PerformanceTestSuite.cpp
+++ b/dbms/programs/performance-test/PerformanceTestSuite.cpp
@@ -61,6 +61,7 @@ public:
         const std::string & default_database_,
         const std::string & user_,
         const std::string & password_,
+        const Settings & cmd_settings,
         const bool lite_output_,
         const std::string & profiles_file_,
         Strings && input_files_,
@@ -87,6 +88,7 @@ public:
         , input_files(input_files_)
         , log(&Poco::Logger::get("PerformanceTestSuite"))
     {
+        global_context.getSettingsRef().copyChangesFrom(cmd_settings);
         if (input_files.size() < 1)
             throw Exception("No tests were specified", ErrorCodes::BAD_ARGUMENTS);
     }
@@ -110,10 +112,6 @@ public:
 
         return 0;
     }
-    void setContextSetting(const String & name, const std::string & value)
-    {
-        global_context.setSetting(name, value);
-    }
 
 private:
     Connection connection;
@@ -326,7 +324,6 @@ try
     using Strings = DB::Strings;
 
 
-#define DECLARE_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION) (#NAME, po::value<std::string>(), DESCRIPTION)
     po::options_description desc("Allowed options");
     desc.add_options()
         ("help", "produce help message")
@@ -348,9 +345,10 @@ try
         ("input-files", value<Strings>()->multitoken(), "Input .xml files")
         ("query-indexes", value<std::vector<size_t>>()->multitoken(), "Input query indexes")
         ("recursive,r", "Recurse in directories to find all xml's")
-    APPLY_FOR_SETTINGS(DECLARE_SETTING);
-#undef DECLARE_SETTING
+    ;
 
+    DB::Settings cmd_settings;
+    cmd_settings.addProgramOptions(desc);
 
     po::options_description cmdline_options;
     cmdline_options.add(desc);
@@ -397,6 +395,7 @@ try
         options["database"].as<std::string>(),
         options["user"].as<std::string>(),
         options["password"].as<std::string>(),
+        cmd_settings,
         options.count("lite") > 0,
         options["profiles-file"].as<std::string>(),
         std::move(input_files),
@@ -408,15 +407,6 @@ try
         std::move(skip_names_regexp),
         queries_with_indexes,
         timeouts);
-    /// Extract settings from the options.
-#define EXTRACT_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION) \
-    if (options.count(#NAME)) \
-    { \
-        performance_test_suite.setContextSetting(#NAME, options[#NAME].as<std::string>()); \
-    }
-    APPLY_FOR_SETTINGS(EXTRACT_SETTING)
-#undef EXTRACT_SETTING
-
     return performance_test_suite.run();
 }
 catch (...)
diff --git a/dbms/src/Core/Settings.cpp b/dbms/src/Core/Settings.cpp
index 0540f4b5ec9..c09ff38a2d8 100644
--- a/dbms/src/Core/Settings.cpp
+++ b/dbms/src/Core/Settings.cpp
@@ -4,6 +4,7 @@
 #include <Columns/ColumnArray.h>
 #include <Common/typeid_cast.h>
 #include <string.h>
+#include <boost/program_options/options_description.hpp>
 
 
 namespace DB
@@ -65,17 +66,20 @@ void Settings::dumpToArrayColumns(IColumn * column_names_, IColumn * column_valu
 
     size_t size = 0;
 
-#define ADD_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION)               \
-    if (!changed_only || NAME.changed)                              \
-    {                                                               \
-        if (column_names)                                           \
-            column_names->getData().insertData(#NAME, strlen(#NAME)); \
-        if (column_values)                                          \
-            column_values->getData().insert(NAME.toString());       \
-        ++size;                                                     \
+    for (const auto & setting : *this)
+    {
+        if (!changed_only || setting.isChanged())
+        {
+            if (column_names)
+            {
+                StringRef name = setting.getName();
+                column_names->getData().insertData(name.data, name.size);
+            }
+            if (column_values)
+                column_values->getData().insert(setting.getValueAsString());
+            ++size;
+        }
     }
-    APPLY_FOR_SETTINGS(ADD_SETTING)
-#undef ADD_SETTING
 
     if (column_names)
     {
@@ -93,4 +97,17 @@ void Settings::dumpToArrayColumns(IColumn * column_names_, IColumn * column_valu
     }
 }
 
+void Settings::addProgramOptions(boost::program_options::options_description & options)
+{
+    for (size_t index = 0; index != Settings::size(); ++index)
+    {
+        auto on_program_option
+            = boost::function1<void, const std::string &>([this, index](const std::string & value) { set(index, value); });
+        options.add(boost::shared_ptr<boost::program_options::option_description>(new boost::program_options::option_description(
+            Settings::getNameByIndex(index).data,
+            boost::program_options::value<std::string>()->composing()->notifier(on_program_option),
+            Settings::getDescriptionByIndex(index).data)));
+    }
+}
+
 }
diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h
index c7c6fbfb700..5533f59e237 100644
--- a/dbms/src/Core/Settings.h
+++ b/dbms/src/Core/Settings.h
@@ -12,6 +12,15 @@ namespace Poco
     }
 }
 
+namespace boost
+{
+    namespace program_options
+    {
+        class options_description;
+    }
+}
+
+
 namespace DB
 {
 
@@ -327,6 +336,10 @@ struct Settings : public SettingsCollection<Settings>
 
     /// Dumps profile events to two columns of type Array(String)
     void dumpToArrayColumns(IColumn * column_names, IColumn * column_values, bool changed_only = true);
+
+    /// Adds program options to set the settings from a command line.
+    /// (Don't forget to call notify() on the `variables_map` after parsing it!)
+    void addProgramOptions(boost::program_options::options_description & options);
 };
 
 }
diff --git a/dbms/src/Storages/System/StorageSystemMergeTreeSettings.cpp b/dbms/src/Storages/System/StorageSystemMergeTreeSettings.cpp
index f60ed67c56b..04dc63537e0 100644
--- a/dbms/src/Storages/System/StorageSystemMergeTreeSettings.cpp
+++ b/dbms/src/Storages/System/StorageSystemMergeTreeSettings.cpp
@@ -18,14 +18,12 @@ NamesAndTypesList SystemMergeTreeSettings::getNamesAndTypes()
 
 void SystemMergeTreeSettings::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const
 {
-    const MergeTreeSettings & settings = context.getMergeTreeSettings();
-
-#define ADD_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION)              \
-    res_columns[0]->insert(#NAME);            \
-    res_columns[1]->insert(settings.NAME.toString()); \
-    res_columns[2]->insert(settings.NAME.changed);
-    APPLY_FOR_MERGE_TREE_SETTINGS(ADD_SETTING)
-#undef ADD_SETTING
+    for (const auto & setting : context.getMergeTreeSettings())
+    {
+        res_columns[0]->insert(setting.getName().toString());
+        res_columns[1]->insert(setting.getValueAsString());
+        res_columns[2]->insert(setting.isChanged());
+    }
 }
 
 }
diff --git a/dbms/src/Storages/System/StorageSystemSettings.cpp b/dbms/src/Storages/System/StorageSystemSettings.cpp
index 5f958e3553f..3963fdb655e 100644
--- a/dbms/src/Storages/System/StorageSystemSettings.cpp
+++ b/dbms/src/Storages/System/StorageSystemSettings.cpp
@@ -23,15 +23,13 @@ NamesAndTypesList StorageSystemSettings::getNamesAndTypes()
 
 void StorageSystemSettings::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const
 {
-    const Settings & settings = context.getSettingsRef();
-
-#define ADD_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION)      \
-    res_columns[0]->insert(#NAME);                 \
-    res_columns[1]->insert(settings.NAME.toString());      \
-    res_columns[2]->insert(settings.NAME.changed); \
-    res_columns[3]->insert(DESCRIPTION);
-    APPLY_FOR_SETTINGS(ADD_SETTING)
-#undef ADD_SETTING
+    for (const auto & setting : context.getSettingsRef())
+    {
+        res_columns[0]->insert(setting.getName().toString());
+        res_columns[1]->insert(setting.getValueAsString());
+        res_columns[2]->insert(setting.isChanged());
+        res_columns[3]->insert(setting.getDescription().toString());
+    }
 }
 
 }

From 1b8c01e9408f1e530b9aebc3b4135d021a3c4a09 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitbar@yandex-team.ru>
Date: Fri, 26 Apr 2019 19:10:37 +0300
Subject: [PATCH 079/118] Transform comments for MergeTreeSettings into
 descriptions.

---
 dbms/src/Core/SettingsCommon.h                |   2 +-
 dbms/src/Parsers/ParserSetQuery.cpp           |   1 -
 .../Storages/MergeTree/MergeTreeSettings.h    | 202 +++++-------------
 .../System/StorageSystemMergeTreeSettings.cpp |   8 +-
 4 files changed, 59 insertions(+), 154 deletions(-)

diff --git a/dbms/src/Core/SettingsCommon.h b/dbms/src/Core/SettingsCommon.h
index c3473d46ba6..0b3ff95daf1 100644
--- a/dbms/src/Core/SettingsCommon.h
+++ b/dbms/src/Core/SettingsCommon.h
@@ -515,7 +515,7 @@ private:
         {
             auto it = by_name_map.find(name);
             if (it == by_name_map.end())
-                return static_cast<size_t>(-1);
+                return static_cast<size_t>(-1); // npos
             return it->second;
         }
 
diff --git a/dbms/src/Parsers/ParserSetQuery.cpp b/dbms/src/Parsers/ParserSetQuery.cpp
index 729d2be786c..1bdb4be6014 100644
--- a/dbms/src/Parsers/ParserSetQuery.cpp
+++ b/dbms/src/Parsers/ParserSetQuery.cpp
@@ -31,7 +31,6 @@ static bool parseNameValuePair(SettingChange & change, IParser::Pos & pos, Expec
     if (!value_p.parse(pos, value, expected))
         return false;
 
-    String change_name;
     getIdentifierName(name, change.name);
     change.value = value->as<ASTLiteral &>().value;
 
diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.h b/dbms/src/Storages/MergeTree/MergeTreeSettings.h
index 5cad78ed5ab..4239c7543ed 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeSettings.h
+++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.h
@@ -25,155 +25,59 @@ struct MergeTreeSettings : public SettingsCollection<MergeTreeSettings>
 {
 
 #define APPLY_FOR_MERGE_TREE_SETTINGS(M) \
-    /** How many rows correspond to one primary key value. */                                                     \
-    M(SettingUInt64, index_granularity, 8192, "")                                                                 \
-                                                                                                                  \
-    /** Merge settings. */                                                                                        \
-                                                                                                                  \
-    /** Maximum in total size of parts to merge, when there are maximum (minimum) free threads                    \
-     * in background pool (or entries in replication queue). */                                                   \
-    M(SettingUInt64, max_bytes_to_merge_at_max_space_in_pool, 150ULL * 1024 * 1024 * 1024, "")                    \
-    M(SettingUInt64, max_bytes_to_merge_at_min_space_in_pool, 1024 * 1024, "")                                    \
-                                                                                                                  \
-    /** How many tasks of merging parts are allowed simultaneously in ReplicatedMergeTree queue. */               \
-    M(SettingUInt64, max_replicated_merges_in_queue, 16, "")                                                      \
-                                                                                                                  \
-    /** When there is less than specified number of free entries in pool (or replicated queue),                   \
-     *  start to lower maximum size of merge to process (or to put in queue).                                     \
-     *  This is to allow small merges to process - not filling the pool with long running merges. */              \
-    M(SettingUInt64, number_of_free_entries_in_pool_to_lower_max_size_of_merge, 8, "")                            \
-                                                                                                                  \
-    /** How many seconds to keep obsolete parts. */                                                               \
-    M(SettingSeconds, old_parts_lifetime, 8 * 60, "")                                                             \
-                                                                                                                  \
-    /** How many seconds to keep tmp_-directories. */                                                             \
-    M(SettingSeconds, temporary_directories_lifetime, 86400, "")                                                  \
-                                                                                                                  \
-    /** Inserts settings. */                                                                                      \
-                                                                                                                  \
-    /** If table contains at least that many active parts, artificially slow down insert into table. */           \
-    M(SettingUInt64, parts_to_delay_insert, 150, "")                                                              \
-                                                                                                                  \
-    /** If more than this number active parts, throw 'Too many parts ...' exception */                            \
-    M(SettingUInt64, parts_to_throw_insert, 300, "")                                                              \
-                                                                                                                  \
-    /** Max delay of inserting data into MergeTree table in seconds, if there are a lot of unmerged parts. */     \
-    M(SettingUInt64, max_delay_to_insert, 1, "")                                                                  \
-                                                                                                                  \
-    /** Replication settings. */                                                                                  \
-                                                                                                                  \
-    /** How many last blocks of hashes should be kept in ZooKeeper (old blocks will be deleted). */               \
-    M(SettingUInt64, replicated_deduplication_window, 100, "")                                                    \
-    /** Similar to previous, but determines old blocks by their lifetime.                                         \
-     *  Hash of an inserted block will be deleted (and the block will not be deduplicated after)                  \
-     *  if it outside of one "window". You can set very big replicated_deduplication_window to avoid              \
-     *  duplicating INSERTs during that period of time. */                                                        \
-    M(SettingUInt64, replicated_deduplication_window_seconds, 7 * 24 * 60 * 60, "") /** one week */               \
-                                                                                                                  \
-    /** How many records may be in log, if there is inactive replica  */                                          \
-    M(SettingUInt64, max_replicated_logs_to_keep, 10000, "")                                                      \
-                                                                                                                  \
-    /** Keep about this number of last records in ZooKeeper log, even if they are obsolete.                       \
-     *  It doesn't affect work of tables: used only to diagnose ZooKeeper log before cleaning. */                 \
-    M(SettingUInt64, min_replicated_logs_to_keep, 100, "")                                                        \
-                                                                                                                  \
-    /** After specified amount of time passed after replication log entry creation                                \
-     *  and sum size of parts is greater than threshold,                                                          \
-     *  prefer fetching merged part from replica instead of doing merge locally.                                  \
-     *  To speed up very long merges. */                                                                          \
-    M(SettingSeconds, prefer_fetch_merged_part_time_threshold, 3600, "")                                          \
-    M(SettingUInt64, prefer_fetch_merged_part_size_threshold, 10ULL * 1024 * 1024 * 1024, "")                     \
-                                                                                                                  \
-    /** Max broken parts, if more - deny automatic deletion. */                                                   \
-    M(SettingUInt64, max_suspicious_broken_parts, 10, "")                                                         \
-                                                                                                                  \
-    /** Not apply ALTER if number of files for modification(deletion, addition) more than this. */                \
-    M(SettingUInt64, max_files_to_modify_in_alter_columns, 75, "")                                                \
-    /** Not apply ALTER, if number of files for deletion more than this. */                                       \
-    M(SettingUInt64, max_files_to_remove_in_alter_columns, 50, "")                                                \
-                                                                                                                  \
-    /** If ratio of wrong parts to total number of parts is less than this - allow to start. */                   \
-    M(SettingFloat, replicated_max_ratio_of_wrong_parts, 0.5, "")                                                 \
-                                                                                                                  \
-    /** Limit parallel fetches */                                                                                 \
-    M(SettingUInt64, replicated_max_parallel_fetches, 0, "")                                                      \
-    M(SettingUInt64, replicated_max_parallel_fetches_for_table, 0, "")                                            \
-                                                                                                                  \
-    /** Limit parallel fetches from endpoint (actually pool size) */                                              \
-    M(SettingUInt64, replicated_max_parallel_fetches_for_host, DEFAULT_COUNT_OF_HTTP_CONNECTIONS_PER_ENDPOINT, "")\
-                                                                                                                  \
-    /** Limit parallel sends */                                                                                   \
-    M(SettingUInt64, replicated_max_parallel_sends, 0, "")                                                        \
-    M(SettingUInt64, replicated_max_parallel_sends_for_table, 0, "")                                              \
-                                                                                                                  \
-    /** If true, Replicated tables replicas on this node will try to acquire leadership. */                       \
-    M(SettingBool, replicated_can_become_leader, true, "")                                                        \
-                                                                                                                  \
-    M(SettingSeconds, zookeeper_session_expiration_check_period, 60, "")                                          \
-                                                                                                                  \
-    /** Check delay of replicas settings. */                                                                      \
-                                                                                                                  \
-    /** Period to check replication delay and compare with other replicas. */                                     \
-    M(SettingUInt64, check_delay_period, 60, "")                                                                  \
-                                                                                                                  \
-    /** Period to clean old queue logs, blocks hashes and parts */                                                \
-    M(SettingUInt64, cleanup_delay_period, 30, "")                                                                \
-    /** Add uniformly distributed value from 0 to x seconds to cleanup_delay_period                               \
-        to avoid thundering herd effect and subsequent DoS of ZooKeeper in case of very large number of tables */ \
-    M(SettingUInt64, cleanup_delay_period_random_add, 10, "")                                                     \
-                                                                                                                  \
-    /** Minimal delay from other replicas to yield leadership. Here and further 0 means unlimited. */             \
-    M(SettingUInt64, min_relative_delay_to_yield_leadership, 120, "")                                             \
-                                                                                                                  \
-    /** Minimal delay from other replicas to close, stop serving requests and not return Ok                       \
-     *  during status check. */                                                                                   \
-    M(SettingUInt64, min_relative_delay_to_close, 300, "")                                                        \
-                                                                                                                  \
-    /** Minimal absolute delay to close, stop serving requests and not return Ok during status check. */          \
-    M(SettingUInt64, min_absolute_delay_to_close, 0, "")                                                          \
-                                                                                                                  \
-    /** Enable usage of Vertical merge algorithm. */                                                              \
-    M(SettingUInt64, enable_vertical_merge_algorithm, 1, "")                                                      \
-                                                                                                                  \
-    /** Minimal (approximate) sum of rows in merging parts to activate Vertical merge algorithm */                \
-    M(SettingUInt64, vertical_merge_algorithm_min_rows_to_activate, 16 * DEFAULT_MERGE_BLOCK_SIZE, "")            \
-                                                                                                                  \
-    /** Minimal amount of non-PK columns to activate Vertical merge algorithm */                                  \
-    M(SettingUInt64, vertical_merge_algorithm_min_columns_to_activate, 11, "")                                    \
-                                                                                                                  \
-    /** Compatibility settings */                                                                                 \
-                                                                                                                  \
-    /** Allow to create a table with sampling expression not in primary key.                                      \
-      * This is needed only to temporarily allow to run the server with wrong tables                              \
-      *  for backward compatibility.                                                                              \
-      */                                                                                                          \
-    M(SettingBool, compatibility_allow_sampling_expression_not_in_primary_key, false, "")                         \
-                                                                                                                  \
-    /** Use small format (dozens bytes) for part checksums in ZooKeeper                                           \
-      *  instead of ordinary ones (dozens KB).                                                                    \
-      * Before enabling check that all replicas support new format.                                               \
-      */                                                                                                          \
-    M(SettingBool, use_minimalistic_checksums_in_zookeeper, true, "")                                             \
-                                                                                                                  \
-    /** Store part header (checksums and columns) in a compact format and a single part znode                     \
-      *  instead of separate znodes (<part>/columns and <part>/checksums).                                        \
-      * This can dramatically reduce snapshot size in ZooKeeper.                                                  \
-      * Before enabling check that all replicas support new format.                                               \
-      */                                                                                                          \
-    M(SettingBool, use_minimalistic_part_header_in_zookeeper, false, "")                                          \
-                                                                                                                  \
-    /** How many records about mutations that are done to keep.                                                   \
-     *  If zero, then keep all of them */                                                                         \
-    M(SettingUInt64, finished_mutations_to_keep, 100, "")                                                         \
-                                                                                                                  \
-    /** Minimal amount of bytes to enable O_DIRECT in merge (0 - disabled) */                                     \
-    M(SettingUInt64, min_merge_bytes_to_use_direct_io, 10ULL * 1024 * 1024 * 1024, "")                            \
-                                                                                                                  \
-    /** Approximate amount of bytes in single granule (0 - disabled) */                                           \
-    M(SettingUInt64, index_granularity_bytes, 0, "")                                                              \
-                                                                                                                  \
-    /** Minimal time in seconds, when merge with TTL can be repeated */                                           \
-    M(SettingInt64, merge_with_ttl_timeout, 3600 * 24, "")
+    M(SettingUInt64, index_granularity, 8192, "How many rows correspond to one primary key value.") \
+    \
+    /** Merge settings. */ \
+    M(SettingUInt64, max_bytes_to_merge_at_max_space_in_pool, 150ULL * 1024 * 1024 * 1024, "Maximum in total size of parts to merge, when there are maximum free threads in background pool (or entries in replication queue).") \
+    M(SettingUInt64, max_bytes_to_merge_at_min_space_in_pool, 1024 * 1024, "Maximum in total size of parts to merge, when there are minimum free threads in background pool (or entries in replication queue).") \
+    M(SettingUInt64, max_replicated_merges_in_queue, 16, "How many tasks of merging parts are allowed simultaneously in ReplicatedMergeTree queue.") \
+    M(SettingUInt64, number_of_free_entries_in_pool_to_lower_max_size_of_merge, 8, "When there is less than specified number of free entries in pool (or replicated queue), start to lower maximum size of merge to process (or to put in queue). This is to allow small merges to process - not filling the pool with long running merges.") \
+    M(SettingSeconds, old_parts_lifetime, 8 * 60, "How many seconds to keep obsolete parts.") \
+    M(SettingSeconds, temporary_directories_lifetime, 86400, "How many seconds to keep tmp_-directories.") \
+    \
+    /** Inserts settings. */ \
+    M(SettingUInt64, parts_to_delay_insert, 150, "If table contains at least that many active parts, artificially slow down insert into table.") \
+    M(SettingUInt64, parts_to_throw_insert, 300, "If more than this number active parts, throw 'Too many parts ...' exception.") \
+    M(SettingUInt64, max_delay_to_insert, 1, "Max delay of inserting data into MergeTree table in seconds, if there are a lot of unmerged parts.") \
+    \
+    /** Replication settings. */ \
+    M(SettingUInt64, replicated_deduplication_window, 100, "How many last blocks of hashes should be kept in ZooKeeper (old blocks will be deleted).") \
+    M(SettingUInt64, replicated_deduplication_window_seconds, 7 * 24 * 60 * 60 /* one week */, "Similar to \"replicated_deduplication_window\", but determines old blocks by their lifetime. Hash of an inserted block will be deleted (and the block will not be deduplicated after) if it outside of one \"window\". You can set very big replicated_deduplication_window to avoid duplicating INSERTs during that period of time.") \
+    M(SettingUInt64, max_replicated_logs_to_keep, 10000, "How many records may be in log, if there is inactive replica.") \
+    M(SettingUInt64, min_replicated_logs_to_keep, 100, "Keep about this number of last records in ZooKeeper log, even if they are obsolete. It doesn't affect work of tables: used only to diagnose ZooKeeper log before cleaning.") \
+    M(SettingSeconds, prefer_fetch_merged_part_time_threshold, 3600, "If time passed after replication log entry creation exceeds this threshold and sum size of parts is greater than \"prefer_fetch_merged_part_size_threshold\", prefer fetching merged part from replica instead of doing merge locally. To speed up very long merges.") \
+    M(SettingUInt64, prefer_fetch_merged_part_size_threshold, 10ULL * 1024 * 1024 * 1024, "If sum size of parts exceeds this threshold and time passed after replication log entry creation is greater than \"prefer_fetch_merged_part_time_threshold\", prefer fetching merged part from replica instead of doing merge locally. To speed up very long merges.") \
+    M(SettingUInt64, max_suspicious_broken_parts, 10, "Max broken parts, if more - deny automatic deletion.") \
+    M(SettingUInt64, max_files_to_modify_in_alter_columns, 75, "Not apply ALTER if number of files for modification(deletion, addition) more than this.") \
+    M(SettingUInt64, max_files_to_remove_in_alter_columns, 50, "Not apply ALTER, if number of files for deletion more than this.") \
+    M(SettingFloat, replicated_max_ratio_of_wrong_parts, 0.5, "If ratio of wrong parts to total number of parts is less than this - allow to start.") \
+    M(SettingUInt64, replicated_max_parallel_fetches, 0, "Limit parallel fetches.") \
+    M(SettingUInt64, replicated_max_parallel_fetches_for_table, 0, "Limit parallel fetches for one table.") \
+    M(SettingUInt64, replicated_max_parallel_fetches_for_host, DEFAULT_COUNT_OF_HTTP_CONNECTIONS_PER_ENDPOINT, "Limit parallel fetches from endpoint (actually pool size).") \
+    M(SettingUInt64, replicated_max_parallel_sends, 0, "Limit parallel sends.") \
+    M(SettingUInt64, replicated_max_parallel_sends_for_table, 0, "Limit parallel sends for one table.") \
+    M(SettingBool, replicated_can_become_leader, true, "If true, Replicated tables replicas on this node will try to acquire leadership.") \
+    M(SettingSeconds, zookeeper_session_expiration_check_period, 60, "ZooKeeper session expiration check period, in seconds.") \
+    \
+    /** Check delay of replicas settings. */ \
+    M(SettingUInt64, check_delay_period, 60, "Period to check replication delay and compare with other replicas.") \
+    M(SettingUInt64, cleanup_delay_period, 30, "Period to clean old queue logs, blocks hashes and parts.") \
+    M(SettingUInt64, cleanup_delay_period_random_add, 10, "Add uniformly distributed value from 0 to x seconds to cleanup_delay_period to avoid thundering herd effect and subsequent DoS of ZooKeeper in case of very large number of tables.") \
+    M(SettingUInt64, min_relative_delay_to_yield_leadership, 120, "Minimal delay from other replicas to yield leadership. Here and further 0 means unlimited.") \
+    M(SettingUInt64, min_relative_delay_to_close, 300, "Minimal delay from other replicas to close, stop serving requests and not return Ok during status check.") \
+    M(SettingUInt64, min_absolute_delay_to_close, 0, "Minimal absolute delay to close, stop serving requests and not return Ok during status check.") \
+    M(SettingUInt64, enable_vertical_merge_algorithm, 1, "Enable usage of Vertical merge algorithm.") \
+    M(SettingUInt64, vertical_merge_algorithm_min_rows_to_activate, 16 * DEFAULT_MERGE_BLOCK_SIZE, "Minimal (approximate) sum of rows in merging parts to activate Vertical merge algorithm.") \
+    M(SettingUInt64, vertical_merge_algorithm_min_columns_to_activate, 11, "Minimal amount of non-PK columns to activate Vertical merge algorithm.") \
+    \
+    /** Compatibility settings */ \
+    M(SettingBool, compatibility_allow_sampling_expression_not_in_primary_key, false, "Allow to create a table with sampling expression not in primary key. This is needed only to temporarily allow to run the server with wrong tables for backward compatibility.") \
+    M(SettingBool, use_minimalistic_checksums_in_zookeeper, true, "Use small format (dozens bytes) for part checksums in ZooKeeper instead of ordinary ones (dozens KB). Before enabling check that all replicas support new format.") \
+    M(SettingBool, use_minimalistic_part_header_in_zookeeper, false, "Store part header (checksums and columns) in a compact format and a single part znode instead of separate znodes (<part>/columns and <part>/checksums). This can dramatically reduce snapshot size in ZooKeeper. Before enabling check that all replicas support new format.") \
+    M(SettingUInt64, finished_mutations_to_keep, 100, "How many records about mutations that are done to keep. If zero, then keep all of them.") \
+    M(SettingUInt64, min_merge_bytes_to_use_direct_io, 10ULL * 1024 * 1024 * 1024, "Minimal amount of bytes to enable O_DIRECT in merge (0 - disabled).") \
+    M(SettingUInt64, index_granularity_bytes, 0, "Approximate amount of bytes in single granule (0 - disabled).") \
+    M(SettingInt64, merge_with_ttl_timeout, 3600 * 24, "Minimal time in seconds, when merge with TTL can be repeated.")
 
     DECLARE_SETTINGS_COLLECTION(APPLY_FOR_MERGE_TREE_SETTINGS)
 
diff --git a/dbms/src/Storages/System/StorageSystemMergeTreeSettings.cpp b/dbms/src/Storages/System/StorageSystemMergeTreeSettings.cpp
index 04dc63537e0..bb0918454ef 100644
--- a/dbms/src/Storages/System/StorageSystemMergeTreeSettings.cpp
+++ b/dbms/src/Storages/System/StorageSystemMergeTreeSettings.cpp
@@ -10,9 +10,10 @@ namespace DB
 NamesAndTypesList SystemMergeTreeSettings::getNamesAndTypes()
 {
     return {
-        {"name",    std::make_shared<DataTypeString>()},
-        {"value",   std::make_shared<DataTypeString>()},
-        {"changed", std::make_shared<DataTypeUInt8>()},
+        {"name",        std::make_shared<DataTypeString>()},
+        {"value",       std::make_shared<DataTypeString>()},
+        {"changed",     std::make_shared<DataTypeUInt8>()},
+        {"description", std::make_shared<DataTypeString>()},
     };
 }
 
@@ -23,6 +24,7 @@ void SystemMergeTreeSettings::fillData(MutableColumns & res_columns, const Conte
         res_columns[0]->insert(setting.getName().toString());
         res_columns[1]->insert(setting.getValueAsString());
         res_columns[2]->insert(setting.isChanged());
+        res_columns[3]->insert(setting.getDescription().toString());
     }
 }
 

From 91ccf2ab2085115e7dbe38b9adc76e30b4280b92 Mon Sep 17 00:00:00 2001
From: Evgenii Pravda <evgeniipravda@gmail.com>
Date: Sat, 27 Apr 2019 20:52:43 +0300
Subject: [PATCH 080/118] Fixes

---
 dbms/src/AggregateFunctions/QuantileTDigest.h |   2 +-
 dbms/src/Columns/ColumnVector.cpp             |   2 +-
 dbms/src/Common/RadixSort.h                   | 118 +++++++++++-------
 dbms/src/Common/tests/radix_sort.cpp          |   2 +-
 dbms/src/Interpreters/RowRefs.h               |   2 +-
 5 files changed, 80 insertions(+), 46 deletions(-)

diff --git a/dbms/src/AggregateFunctions/QuantileTDigest.h b/dbms/src/AggregateFunctions/QuantileTDigest.h
index b32519519f8..e9f261d4c21 100644
--- a/dbms/src/AggregateFunctions/QuantileTDigest.h
+++ b/dbms/src/AggregateFunctions/QuantileTDigest.h
@@ -136,7 +136,7 @@ class QuantileTDigest
     {
         if (unmerged > 0)
         {
-            RadixSort<RadixSortTraits>::executeLsd(summary.data(), summary.size());
+            RadixSort<RadixSortTraits>::executeLSD(summary.data(), summary.size());
 
             if (summary.size() > 3)
             {
diff --git a/dbms/src/Columns/ColumnVector.cpp b/dbms/src/Columns/ColumnVector.cpp
index 4e806122b03..6d1925a2492 100644
--- a/dbms/src/Columns/ColumnVector.cpp
+++ b/dbms/src/Columns/ColumnVector.cpp
@@ -120,7 +120,7 @@ void ColumnVector<T>::getPermutation(bool reverse, size_t limit, int nan_directi
                 for (UInt32 i = 0; i < s; ++i)
                     pairs[i] = {data[i], i};
 
-                RadixSort<RadixSortTraits<T>>::executeLsd(pairs.data(), s);
+                RadixSort<RadixSortTraits<T>>::executeLSD(pairs.data(), s);
 
                 /// Radix sort treats all NaNs to be greater than all numbers.
                 /// If the user needs the opposite, we must move them accordingly.
diff --git a/dbms/src/Common/RadixSort.h b/dbms/src/Common/RadixSort.h
index 2e8b5c81b85..c661a622881 100644
--- a/dbms/src/Common/RadixSort.h
+++ b/dbms/src/Common/RadixSort.h
@@ -69,7 +69,7 @@ struct RadixSortUIntTraits
 
     static Key & extractKey(Element & elem) { return elem; }
 
-    static bool compare(TElement x, TElement y)
+    static bool less(Key x, Key y)
     {
         return x < y;
     }
@@ -100,7 +100,7 @@ struct RadixSortIntTraits
 
     static Key & extractKey(Element & elem) { return elem; }
 
-    static bool compare(TElement x, TElement y)
+    static bool less(Key x, Key y)
     {
         return x < y;
     }
@@ -155,7 +155,7 @@ struct RadixSortFloatTraits
     static Key & extractKey(Element & elem) { return elem; }
 
     // TODO: Correct handling of NaNs, NULLs, etc
-    static bool compare(TElement x, TElement y)
+    static bool less(Key x, Key y)
     {
         return x < y;
     }
@@ -180,7 +180,8 @@ private:
     using CountType   = typename Traits::CountType;
     using KeyBits     = typename Traits::KeyBits;
 
-    static constexpr size_t INSERT_SORT_THRESHOLD = 64;
+    // Use insertion sort if the size of the array is less than equal to this threshold
+    static constexpr size_t INSERTION_SORT_THRESHOLD = 64;
 
     static constexpr size_t HISTOGRAM_SIZE = 1 << Traits::PART_SIZE_BITS;
     static constexpr size_t PART_BITMASK = HISTOGRAM_SIZE - 1;
@@ -198,47 +199,57 @@ private:
     static KeyBits keyToBits(Key x) { return ext::bit_cast<KeyBits>(x); }
     static Key bitsToKey(KeyBits x) { return ext::bit_cast<Key>(x); }
 
-    static inline void insertSortInternal(Element * arr, size_t size)
+    static void insertionSortInternal(Element *arr, size_t size)
     {
-        for (Element * i = arr + 1; i < arr + size; ++i)
-        {
-            if (Traits::compare(*i, *(i - 1)))
+        Element * end = arr + size;
+        for (Element * i = arr + 1; i < end; ++i)
+            if (Traits::less(Traits::extractKey(*i), Traits::extractKey(*(i - 1))))
             {
                 Element * j;
                 Element tmp = *i;
                 *i = *(i - 1);
-                for (j = i - 1; j > arr && Traits::compare(tmp, *(j - 1)); --j)
+                for (j = i - 1; j > arr && Traits::less(Traits::extractKey(tmp), Traits::extractKey(*(j - 1))); --j)
                     *j = *(j - 1);
                 *j = tmp;
             }
-        }
     }
 
-    template <int PASS>
-    static inline void msdRadixSortInternal(Element * arr, size_t size, size_t limit)
+    /* Main MSD radix sort subroutine
+     * Puts elements to buckets based on PASS-th digit, then recursively calls insertion sort or itself on the buckets
+     */
+    template <size_t PASS>
+    static inline void radixSortMSDInternal(Element *arr, size_t size, size_t limit)
     {
-        Element *last_[HISTOGRAM_SIZE + 1];
-        Element ** last = last_ + 1;
+        Element *last_list[HISTOGRAM_SIZE + 1];
+        Element ** last = last_list + 1;
         size_t count[HISTOGRAM_SIZE] = {0};
 
         for (Element * i = arr; i < arr + size; ++i)
             ++count[getPart(PASS, *i)];
 
-        last_[0] = last_[1] = arr;
+        last_list[0] = last_list[1] = arr;
 
-        size_t bucketsForRecursion = HISTOGRAM_SIZE;
+        size_t buckets_for_recursion = HISTOGRAM_SIZE;
         Element * finish = arr + size;
         for (size_t i = 1; i < HISTOGRAM_SIZE; ++i)
         {
             last[i] = last[i - 1] + count[i - 1];
             if (last[i] >= arr + limit)
             {
-                bucketsForRecursion = i;
+                buckets_for_recursion = i;
                 finish = last[i];
             }
         }
 
-        for (size_t i = 0; i < bucketsForRecursion; ++i)
+        /* At this point, we have the following variables:
+         * count[i] is the size of i-th bucket
+         * last[i] is a pointer to the beginning of i-th bucket, last[-1] == last[0]
+         * buckets_for_recursion is the number of buckets that should be sorted, the last of them only partially
+         * finish is a pointer to the end of the first buckets_for_recursion buckets
+         */
+
+        // Scatter array elements to buckets until the first buckets_for_recursion buckets are full
+        for (size_t i = 0; i < buckets_for_recursion; ++i)
         {
             Element * end = last[i - 1] + count[i];
             if (end == finish)
@@ -264,35 +275,35 @@ private:
 
         if constexpr (PASS > 0)
         {
-            for (size_t i = 0; i < bucketsForRecursion - 1; ++i)
+            // Recursively sort buckets, except the last one
+            for (size_t i = 0; i < buckets_for_recursion - 1; ++i)
             {
                 Element * start = last[i - 1];
-                size_t size = last[i] - last[i - 1];
-                msdRadixSortInternalHelper<PASS - 1>(start, size, size);
+                size_t subsize = last[i] - last[i - 1];
+                radixSortMSDInternalHelper<PASS - 1>(start, subsize, subsize);
             }
 
-            // Sort last necessary bucket with sublimit
-            Element * start = last[bucketsForRecursion - 2];
-            size_t size = last[bucketsForRecursion - 1] - last[bucketsForRecursion - 2];
-            size_t sublimit = limit - (last[bucketsForRecursion - 1] - arr);
-            msdRadixSortInternalHelper<PASS - 1>(start, size, sublimit);
+            // Sort last necessary bucket with limit
+            Element * start = last[buckets_for_recursion - 2];
+            size_t subsize = last[buckets_for_recursion - 1] - last[buckets_for_recursion - 2];
+            size_t sublimit = limit - (last[buckets_for_recursion - 1] - arr);
+            radixSortMSDInternalHelper<PASS - 1>(start, subsize, sublimit);
         }
     }
 
-    template <int PASS>
-    static inline void msdRadixSortInternalHelper(Element * arr, size_t size, size_t limit)
+    // A helper to choose sorting algorithm based on array length
+    template <size_t PASS>
+    static inline void radixSortMSDInternalHelper(Element *arr, size_t size, size_t limit)
     {
-        if (size <= INSERT_SORT_THRESHOLD)
-            insertSortInternal(arr, size);
+        if (size <= INSERTION_SORT_THRESHOLD)
+            insertionSortInternal(arr, size);
         else
-            msdRadixSortInternal<PASS>(arr, size, limit);
+            radixSortMSDInternal<PASS>(arr, size, limit);
     }
 
 public:
-    /* Least significant digit radix sort
-     * The most efficient stable general-purpose sorting algorithm
-     */
-    static void executeLsd(Element * arr, size_t size)
+    // Least significant digit radix sort (stable)
+    static void executeLSD(Element *arr, size_t size)
     {
         /// If the array is smaller than 256, then it is better to use another algorithm.
 
@@ -362,12 +373,35 @@ public:
 
     /* Most significant digit radix sort
      * Usually slower than LSD and is not stable, but allows partial sorting
-     * Based on https://github.com/voutcn/kxsort
+     *
+     * Based on https://github.com/voutcn/kxsort, license:
+     * The MIT License
+     * Copyright (c) 2016 Dinghua Li <voutcn@gmail.com>
+     *
+     * Permission is hereby granted, free of charge, to any person obtaining
+     * a copy of this software and associated documentation files (the
+     * "Software"), to deal in the Software without restriction, including
+     * without limitation the rights to use, copy, modify, merge, publish,
+     * distribute, sublicense, and/or sell copies of the Software, and to
+     * permit persons to whom the Software is furnished to do so, subject to
+     * the following conditions:
+     *
+     * The above copyright notice and this permission notice shall be
+     * included in all copies or substantial portions of the Software.
+     *
+     * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+     * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+     * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+     * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+     * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+     * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+     * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+     * SOFTWARE.
      */
-    static void executeMsd(Element * arr, size_t size, size_t limit)
+    static void executeMSD(Element *arr, size_t size, size_t limit)
     {
         limit = std::min(limit, size);
-        msdRadixSortInternalHelper<NUM_PASSES - 1>(arr, size, limit);
+        radixSortMSDInternalHelper<NUM_PASSES - 1>(arr, size, limit);
     }
 };
 
@@ -376,13 +410,13 @@ public:
 /// Use RadixSort with custom traits for complex types instead.
 
 template <typename T>
-void lsdRadixSort(T *arr, size_t size)
+void radixSortLSD(T *arr, size_t size)
 {
-    RadixSort<RadixSortNumTraits<T>>::executeLsd(arr, size);
+    RadixSort<RadixSortNumTraits<T>>::executeLSD(arr, size);
 }
 
 template <typename T>
-void msdRadixSort(T * arr, size_t size, size_t limit)
+void radixSortMSD(T *arr, size_t size, size_t limit)
 {
-    RadixSort<RadixSortNumTraits<T>>::executeMsd(arr, size, limit);
+    RadixSort<RadixSortNumTraits<T>>::executeMSD(arr, size, limit);
 }
diff --git a/dbms/src/Common/tests/radix_sort.cpp b/dbms/src/Common/tests/radix_sort.cpp
index d60e453c6bd..4cd9da2a926 100644
--- a/dbms/src/Common/tests/radix_sort.cpp
+++ b/dbms/src/Common/tests/radix_sort.cpp
@@ -16,7 +16,7 @@ void NO_INLINE sort1(Key * data, size_t size)
 
 void NO_INLINE sort2(Key * data, size_t size)
 {
-    lsdRadixSort(data, size);
+    radixSortLSD(data, size);
 }
 
 void NO_INLINE sort3(Key * data, size_t size)
diff --git a/dbms/src/Interpreters/RowRefs.h b/dbms/src/Interpreters/RowRefs.h
index 78e7515729a..76efb33543d 100644
--- a/dbms/src/Interpreters/RowRefs.h
+++ b/dbms/src/Interpreters/RowRefs.h
@@ -92,7 +92,7 @@ private:
                 {
                     /// TODO: It has been tested only for UInt32 yet. It needs to check UInt64, Float32/64.
                     if constexpr (std::is_same_v<TKey, UInt32>)
-                        RadixSort<RadixSortTraits>::executeLsd(&array[0], array.size());
+                        RadixSort<RadixSortTraits>::executeLSD(&array[0], array.size());
                     else
                         std::sort(array.begin(), array.end());
                 }

From 9edbbc637abc1c02c3b6f5449f787a6c763b412f Mon Sep 17 00:00:00 2001
From: Ivan <5627721+abyss7@users.noreply.github.com>
Date: Sat, 27 Apr 2019 22:12:35 +0300
Subject: [PATCH 081/118] Fix segmentation fault on iterator double-free inside
 RWLock (#5127)

* Store the key instead of iterator, which may get invalidated by erasure from another instance.
* Add test
---
 dbms/src/Common/RWLock.cpp                     | 18 ++++++++----------
 .../00938_fix_rwlock_segfault.reference        |  0
 .../0_stateless/00938_fix_rwlock_segfault.sh   | 12 ++++++++++++
 3 files changed, 20 insertions(+), 10 deletions(-)
 create mode 100644 dbms/tests/queries/0_stateless/00938_fix_rwlock_segfault.reference
 create mode 100755 dbms/tests/queries/0_stateless/00938_fix_rwlock_segfault.sh

diff --git a/dbms/src/Common/RWLock.cpp b/dbms/src/Common/RWLock.cpp
index aa658e964d7..a6df0f741e0 100644
--- a/dbms/src/Common/RWLock.cpp
+++ b/dbms/src/Common/RWLock.cpp
@@ -37,8 +37,8 @@ class RWLockImpl::LockHolderImpl
     RWLock parent;
     GroupsContainer::iterator it_group;
     ClientsContainer::iterator it_client;
-    ThreadToHolder::iterator it_thread;
-    QueryIdToHolder::iterator it_query;
+    ThreadToHolder::key_type thread_id;
+    QueryIdToHolder::key_type query_id;
     CurrentMetrics::Increment active_client_increment;
 
     LockHolderImpl(RWLock && parent, GroupsContainer::iterator it_group, ClientsContainer::iterator it_client);
@@ -123,12 +123,12 @@ RWLockImpl::LockHolder RWLockImpl::getLock(RWLockImpl::Type type, const String &
     LockHolder res(new LockHolderImpl(shared_from_this(), it_group, it_client));
 
     /// Insert myself (weak_ptr to the holder) to threads set to implement recursive lock
-    it_thread = thread_to_holder.emplace(this_thread_id, res).first;
-    res->it_thread = it_thread;
+    thread_to_holder.emplace(this_thread_id, res);
+    res->thread_id = this_thread_id;
 
     if (query_id != RWLockImpl::NO_QUERY)
-        it_query = query_id_to_holder.emplace(query_id, res).first;
-    res->it_query = it_query;
+        query_id_to_holder.emplace(query_id, res);
+    res->query_id = query_id;
 
     /// We are first, we should not wait anything
     /// If we are not the first client in the group, a notification could be already sent
@@ -151,10 +151,8 @@ RWLockImpl::LockHolderImpl::~LockHolderImpl()
     std::unique_lock lock(parent->mutex);
 
     /// Remove weak_ptrs to the holder, since there are no owners of the current lock
-    parent->thread_to_holder.erase(it_thread);
-
-    if (it_query != parent->query_id_to_holder.end())
-        parent->query_id_to_holder.erase(it_query);
+    parent->thread_to_holder.erase(thread_id);
+    parent->query_id_to_holder.erase(query_id);
 
     /// Removes myself from client list of our group
     it_group->clients.erase(it_client);
diff --git a/dbms/tests/queries/0_stateless/00938_fix_rwlock_segfault.reference b/dbms/tests/queries/0_stateless/00938_fix_rwlock_segfault.reference
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/dbms/tests/queries/0_stateless/00938_fix_rwlock_segfault.sh b/dbms/tests/queries/0_stateless/00938_fix_rwlock_segfault.sh
new file mode 100755
index 00000000000..cc0577ff15d
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00938_fix_rwlock_segfault.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+# Test fix for issue #5066
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+. $CURDIR/../shell_config.sh
+
+set -e
+
+for i in {1..100}; do \
+$CLICKHOUSE_CLIENT -q "SELECT name FROM system.tables UNION ALL SELECT name FROM system.columns format Null";
+done

From 3c1e5348d15943d988283e49f84cf6245d45d783 Mon Sep 17 00:00:00 2001
From: Danila Kutenin <kutdanila@yandex.ru>
Date: Sun, 28 Apr 2019 13:04:47 +0300
Subject: [PATCH 082/118] LfAlloc more reliable choice of hint

---
 contrib/lfalloc/src/lf_allocX64.h | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/contrib/lfalloc/src/lf_allocX64.h b/contrib/lfalloc/src/lf_allocX64.h
index 2c4cf3f1021..63500426163 100644
--- a/contrib/lfalloc/src/lf_allocX64.h
+++ b/contrib/lfalloc/src/lf_allocX64.h
@@ -358,8 +358,15 @@ static char* AllocWithMMap(uintptr_t sz, EMMapMode mode) {
     }
 #if defined(USE_LFALLOC_RANDOM_HINT)
     static thread_local std::mt19937_64 generator(std::random_device{}());
-    std::uniform_int_distribution<intptr_t> distr(areaStart, areaFinish / 2);
-    char* largeBlock = (char*)mmap(reinterpret_cast<void*>(distr(generator)), sz, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
+    std::uniform_int_distribution<intptr_t> distr(areaStart, areaFinish - sz);
+    char* largeBlock;
+    static constexpr size_t maxAttempts = 10;
+    size_t attempt = 0;
+    do
+    {
+        largeBlock = (char*)mmap(reinterpret_cast<void*>(distr(generator)), sz, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
+        ++attempt;
+    } while (uintptr_t(((char*)largeBlock - ALLOC_START) + sz) >= areaFinish && attempt < maxAttempts && munmap(largeBlock, sz) == 0);
 #else
     char* largeBlock = (char*)mmap(0, sz, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
 #endif

From 852ea727fb23b80e708b3cbf61c0fba6e757e733 Mon Sep 17 00:00:00 2001
From: Danila Kutenin <kutdanila@yandex.ru>
Date: Sun, 28 Apr 2019 13:09:01 +0300
Subject: [PATCH 083/118] Exponentially small

---
 contrib/lfalloc/src/lf_allocX64.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/contrib/lfalloc/src/lf_allocX64.h b/contrib/lfalloc/src/lf_allocX64.h
index 63500426163..2047d64e4b1 100644
--- a/contrib/lfalloc/src/lf_allocX64.h
+++ b/contrib/lfalloc/src/lf_allocX64.h
@@ -360,13 +360,13 @@ static char* AllocWithMMap(uintptr_t sz, EMMapMode mode) {
     static thread_local std::mt19937_64 generator(std::random_device{}());
     std::uniform_int_distribution<intptr_t> distr(areaStart, areaFinish - sz);
     char* largeBlock;
-    static constexpr size_t maxAttempts = 10;
+    static constexpr size_t MaxAttempts = 100;
     size_t attempt = 0;
     do
     {
         largeBlock = (char*)mmap(reinterpret_cast<void*>(distr(generator)), sz, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
         ++attempt;
-    } while (uintptr_t(((char*)largeBlock - ALLOC_START) + sz) >= areaFinish && attempt < maxAttempts && munmap(largeBlock, sz) == 0);
+    } while (uintptr_t(((char*)largeBlock - ALLOC_START) + sz) >= areaFinish && attempt < MaxAttempts && munmap(largeBlock, sz) == 0);
 #else
     char* largeBlock = (char*)mmap(0, sz, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
 #endif

From 89b0b3c3421220c4edd70e488897f9e69f6ecea6 Mon Sep 17 00:00:00 2001
From: Danila Kutenin <kutdanila@yandex.ru>
Date: Sun, 28 Apr 2019 13:12:36 +0300
Subject: [PATCH 084/118] Fix

---
 contrib/lfalloc/src/lf_allocX64.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/lfalloc/src/lf_allocX64.h b/contrib/lfalloc/src/lf_allocX64.h
index 2047d64e4b1..12190f0712f 100644
--- a/contrib/lfalloc/src/lf_allocX64.h
+++ b/contrib/lfalloc/src/lf_allocX64.h
@@ -358,7 +358,7 @@ static char* AllocWithMMap(uintptr_t sz, EMMapMode mode) {
     }
 #if defined(USE_LFALLOC_RANDOM_HINT)
     static thread_local std::mt19937_64 generator(std::random_device{}());
-    std::uniform_int_distribution<intptr_t> distr(areaStart, areaFinish - sz);
+    std::uniform_int_distribution<intptr_t> distr(areaStart, areaFinish - sz - 1);
     char* largeBlock;
     static constexpr size_t MaxAttempts = 100;
     size_t attempt = 0;

From 5beaf33a9f54fb57654aac24d290187171d75b47 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitbar@yandex-team.ru>
Date: Sat, 27 Apr 2019 16:58:20 +0300
Subject: [PATCH 085/118] Get rid of code duplication, use templates.

---
 dbms/src/Core/Settings.h         |  18 +-
 dbms/src/Core/SettingsCommon.cpp | 679 ++++++++-----------------------
 dbms/src/Core/SettingsCommon.h   | 360 +++++-----------
 3 files changed, 278 insertions(+), 779 deletions(-)

diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h
index 5533f59e237..272f1ce2373 100644
--- a/dbms/src/Core/Settings.h
+++ b/dbms/src/Core/Settings.h
@@ -226,25 +226,25 @@ struct Settings : public SettingsCollection<Settings>
     \
     M(SettingUInt64, max_rows_to_read, 0, "Limit on read rows from the most 'deep' sources. That is, only in the deepest subquery. When reading from a remote server, it is only checked on a remote server.") \
     M(SettingUInt64, max_bytes_to_read, 0, "Limit on read bytes (after decompression) from the most 'deep' sources. That is, only in the deepest subquery. When reading from a remote server, it is only checked on a remote server.") \
-    M(SettingOverflowMode<false>, read_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.") \
+    M(SettingOverflowMode, read_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.") \
     \
     M(SettingUInt64, max_rows_to_group_by, 0, "") \
-    M(SettingOverflowMode<true>, group_by_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.") \
+    M(SettingOverflowModeGroupBy, group_by_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.") \
     M(SettingUInt64, max_bytes_before_external_group_by, 0, "") \
     \
     M(SettingUInt64, max_rows_to_sort, 0, "") \
     M(SettingUInt64, max_bytes_to_sort, 0, "") \
-    M(SettingOverflowMode<false>, sort_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.") \
+    M(SettingOverflowMode, sort_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.") \
     M(SettingUInt64, max_bytes_before_external_sort, 0, "") \
     M(SettingUInt64, max_bytes_before_remerge_sort, 1000000000, "In case of ORDER BY with LIMIT, when memory usage is higher than specified threshold, perform additional steps of merging blocks before final merge to keep just top LIMIT rows.") \
     \
     M(SettingUInt64, max_result_rows, 0, "Limit on result size in rows. Also checked for intermediate data sent from remote servers.") \
     M(SettingUInt64, max_result_bytes, 0, "Limit on result size in bytes (uncompressed). Also checked for intermediate data sent from remote servers.") \
-    M(SettingOverflowMode<false>, result_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.") \
+    M(SettingOverflowMode, result_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.") \
     \
     /* TODO: Check also when merging and finalizing aggregate functions. */ \
     M(SettingSeconds, max_execution_time, 0, "") \
-    M(SettingOverflowMode<false>, timeout_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.") \
+    M(SettingOverflowMode, timeout_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.") \
     \
     M(SettingUInt64, min_execution_speed, 0, "Minimum number of execution rows per second.") \
     M(SettingUInt64, max_execution_speed, 0, "Maximum number of execution rows per second.") \
@@ -266,20 +266,20 @@ struct Settings : public SettingsCollection<Settings>
     \
     M(SettingUInt64, max_rows_in_set, 0, "Maximum size of the set (in number of elements) resulting from the execution of the IN section.") \
     M(SettingUInt64, max_bytes_in_set, 0, "Maximum size of the set (in bytes in memory) resulting from the execution of the IN section.") \
-    M(SettingOverflowMode<false>, set_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.") \
+    M(SettingOverflowMode, set_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.") \
     \
     M(SettingUInt64, max_rows_in_join, 0, "Maximum size of the hash table for JOIN (in number of rows).") \
     M(SettingUInt64, max_bytes_in_join, 0, "Maximum size of the hash table for JOIN (in number of bytes in memory).") \
-    M(SettingOverflowMode<false>, join_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.") \
+    M(SettingOverflowMode, join_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.") \
     M(SettingBool, join_any_take_last_row, false, "When disabled (default) ANY JOIN will take the first found row for a key. When enabled, it will take the last row seen if there are multiple rows for the same key.") \
     \
     M(SettingUInt64, max_rows_to_transfer, 0, "Maximum size (in rows) of the transmitted external table obtained when the GLOBAL IN/JOIN section is executed.") \
     M(SettingUInt64, max_bytes_to_transfer, 0, "Maximum size (in uncompressed bytes) of the transmitted external table obtained when the GLOBAL IN/JOIN section is executed.") \
-    M(SettingOverflowMode<false>, transfer_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.") \
+    M(SettingOverflowMode, transfer_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.") \
     \
     M(SettingUInt64, max_rows_in_distinct, 0, "Maximum number of elements during execution of DISTINCT.") \
     M(SettingUInt64, max_bytes_in_distinct, 0, "Maximum total size of state (in uncompressed bytes) in memory for the execution of DISTINCT.") \
-    M(SettingOverflowMode<false>, distinct_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.") \
+    M(SettingOverflowMode, distinct_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.") \
     \
     M(SettingUInt64, max_memory_usage, 0, "Maximum memory usage for processing of single query. Zero means unlimited.") \
     M(SettingUInt64, max_memory_usage_for_user, 0, "Maximum memory usage for processing all concurrently running queries for the user. Zero means unlimited.") \
diff --git a/dbms/src/Core/SettingsCommon.cpp b/dbms/src/Core/SettingsCommon.cpp
index 634d428777d..8f56a6d1454 100644
--- a/dbms/src/Core/SettingsCommon.cpp
+++ b/dbms/src/Core/SettingsCommon.cpp
@@ -28,62 +28,89 @@ namespace ErrorCodes
     extern const int UNKNOWN_SETTING;
 }
 
-template <typename IntType>
-String SettingInt<IntType>::toString() const
+
+template <typename Type>
+String SettingNumber<Type>::toString() const
 {
     return DB::toString(value);
 }
 
-template <typename IntType>
-Field SettingInt<IntType>::toField() const
+template <typename Type>
+Field SettingNumber<Type>::toField() const
 {
     return value;
 }
 
-template <typename IntType>
-void SettingInt<IntType>::set(IntType x)
+template <typename Type>
+void SettingNumber<Type>::set(Type x)
 {
     value = x;
     changed = true;
 }
 
-template <typename IntType>
-void SettingInt<IntType>::set(const Field & x)
+template <typename Type>
+void SettingNumber<Type>::set(const Field & x)
 {
     if (x.getType() == Field::Types::String)
         set(get<const String &>(x));
     else
-        set(applyVisitor(FieldVisitorConvertToNumber<IntType>(), x));
+        set(applyVisitor(FieldVisitorConvertToNumber<Type>(), x));
 }
 
-template <typename IntType>
-void SettingInt<IntType>::set(const String & x)
+template <typename Type>
+void SettingNumber<Type>::set(const String & x)
 {
-    set(parse<IntType>(x));
+    set(parse<Type>(x));
 }
 
-template <typename IntType>
-void SettingInt<IntType>::serialize(WriteBuffer & buf) const
+template <typename Type>
+void SettingNumber<Type>::serialize(WriteBuffer & buf) const
 {
-    writeVarT(value, buf);
+    if constexpr (std::is_integral_v<Type> && std::is_unsigned_v<Type>)
+        writeVarUInt(static_cast<UInt64>(value), buf);
+    else if constexpr (std::is_integral_v<Type> && std::is_signed_v<Type>)
+        writeVarInt(static_cast<Int64>(value), buf);
+    else
+    {
+        static_assert(std::is_floating_point_v<Type>);
+        writeBinary(toString(), buf);
+    }
 }
 
-template <typename IntType>
-void SettingInt<IntType>::deserialize(ReadBuffer & buf)
+template <typename Type>
+void SettingNumber<Type>::deserialize(ReadBuffer & buf)
 {
-    IntType x = 0;
-    readVarT(x, buf);
-    set(x);
+    if constexpr (std::is_integral_v<Type> && std::is_unsigned_v<Type>)
+    {
+        UInt64 x;
+        readVarUInt(x, buf);
+        set(static_cast<Type>(x));
+    }
+    else if constexpr (std::is_integral_v<Type> && std::is_signed_v<Type>)
+    {
+        Int64 x;
+        readVarInt(x, buf);
+        set(static_cast<Type>(x));
+    }
+    else
+    {
+        static_assert(std::is_floating_point_v<Type>);
+        String x;
+        readBinary(x, buf);
+        set(x);
+    }
 }
 
-template struct SettingInt<UInt64>;
-template struct SettingInt<Int64>;
+template struct SettingNumber<UInt64>;
+template struct SettingNumber<Int64>;
+template struct SettingNumber<float>;
+template struct SettingNumber<bool>;
 
 
 String SettingMaxThreads::toString() const
 {
     /// Instead of the `auto` value, we output the actual value to make it easier to see.
-    return DB::toString(value);
+    return is_auto ? ("auto(" + DB::toString(value) + ")") : DB::toString(value);
 }
 
 Field SettingMaxThreads::toField() const
@@ -108,7 +135,7 @@ void SettingMaxThreads::set(const Field & x)
 
 void SettingMaxThreads::set(const String & x)
 {
-    if (x == "auto")
+    if (startsWith(x, "auto"))
         setAuto();
     else
         set(parse<UInt64>(x));
@@ -134,39 +161,38 @@ void SettingMaxThreads::setAuto()
 
 UInt64 SettingMaxThreads::getAutoValue() const
 {
-    static auto res = getAutoValueImpl();
+    static auto res = getNumberOfPhysicalCPUCores();
     return res;
 }
 
-/// Executed once for all time. Executed from one thread.
-UInt64 SettingMaxThreads::getAutoValueImpl() const
+
+template <SettingTimespanIO io_unit>
+String SettingTimespan<io_unit>::toString() const
 {
-    return getNumberOfPhysicalCPUCores();
+    return DB::toString(value.totalMicroseconds() / microseconds_per_io_unit);
 }
 
-
-String SettingSeconds::toString() const
+template <SettingTimespanIO io_unit>
+Field SettingTimespan<io_unit>::toField() const
 {
-    return DB::toString(totalSeconds());
+    return value.totalMicroseconds() / microseconds_per_io_unit;
 }
 
-Field SettingSeconds::toField() const
-{
-    return totalSeconds();
-}
-
-void SettingSeconds::set(const Poco::Timespan & x)
+template <SettingTimespanIO io_unit>
+void SettingTimespan<io_unit>::set(const Poco::Timespan & x)
 {
     value = x;
     changed = true;
 }
 
-void SettingSeconds::set(UInt64 x)
+template <SettingTimespanIO io_unit>
+void SettingTimespan<io_unit>::set(UInt64 x)
 {
-    set(Poco::Timespan(x, 0));
+    set(Poco::Timespan(x * microseconds_per_io_unit));
 }
 
-void SettingSeconds::set(const Field & x)
+template <SettingTimespanIO io_unit>
+void SettingTimespan<io_unit>::set(const Field & x)
 {
     if (x.getType() == Field::Types::String)
         set(get<const String &>(x));
@@ -174,402 +200,28 @@ void SettingSeconds::set(const Field & x)
         set(safeGet<UInt64>(x));
 }
 
-void SettingSeconds::set(const String & x)
+template <SettingTimespanIO io_unit>
+void SettingTimespan<io_unit>::set(const String & x)
 {
     set(parse<UInt64>(x));
 }
 
-void SettingSeconds::serialize(WriteBuffer & buf) const
+template <SettingTimespanIO io_unit>
+void SettingTimespan<io_unit>::serialize(WriteBuffer & buf) const
 {
-    writeVarUInt(value.totalSeconds(), buf);
+    writeVarUInt(value.totalMicroseconds() / microseconds_per_io_unit, buf);
 }
 
-void SettingSeconds::deserialize(ReadBuffer & buf)
+template <SettingTimespanIO io_unit>
+void SettingTimespan<io_unit>::deserialize(ReadBuffer & buf)
 {
     UInt64 x = 0;
     readVarUInt(x, buf);
     set(x);
 }
 
-
-String SettingMilliseconds::toString() const
-{
-    return DB::toString(totalMilliseconds());
-}
-
-Field SettingMilliseconds::toField() const
-{
-    return totalMilliseconds();
-}
-
-void SettingMilliseconds::set(const Poco::Timespan & x)
-{
-    value = x;
-    changed = true;
-}
-
-void SettingMilliseconds::set(UInt64 x)
-{
-    set(Poco::Timespan(x * 1000));
-}
-
-void SettingMilliseconds::set(const Field & x)
-{
-    if (x.getType() == Field::Types::String)
-        set(get<const String &>(x));
-    else
-        set(safeGet<UInt64>(x));
-}
-
-void SettingMilliseconds::set(const String & x)
-{
-    set(parse<UInt64>(x));
-}
-
-void SettingMilliseconds::serialize(WriteBuffer & buf) const
-{
-    writeVarUInt(value.totalMilliseconds(), buf);
-}
-
-void SettingMilliseconds::deserialize(ReadBuffer & buf)
-{
-    UInt64 x = 0;
-    readVarUInt(x, buf);
-    set(x);
-}
-
-
-String SettingFloat::toString() const
-{
-    return DB::toString(value);
-}
-
-Field SettingFloat::toField() const
-{
-    return value;
-}
-
-void SettingFloat::set(float x)
-{
-    value = x;
-    changed = true;
-}
-
-void SettingFloat::set(const Field & x)
-{
-    if (x.getType() == Field::Types::String)
-        set(get<const String &>(x));
-    else
-        set(applyVisitor(FieldVisitorConvertToNumber<float>(), x));
-}
-
-void SettingFloat::set(const String & x)
-{
-    set(parse<float>(x));
-}
-
-void SettingFloat::serialize(WriteBuffer & buf) const
-{
-    writeBinary(toString(), buf);
-}
-
-void SettingFloat::deserialize(ReadBuffer & buf)
-{
-    String x;
-    readBinary(x, buf);
-    set(x);
-}
-
-
-LoadBalancing SettingLoadBalancing::getLoadBalancing(const String & s)
-{
-    if (s == "random")           return LoadBalancing::RANDOM;
-    if (s == "nearest_hostname") return LoadBalancing::NEAREST_HOSTNAME;
-    if (s == "in_order")         return LoadBalancing::IN_ORDER;
-    if (s == "first_or_random")  return LoadBalancing::FIRST_OR_RANDOM;
-
-    throw Exception("Unknown load balancing mode: '" + s + "', must be one of 'random', 'nearest_hostname', 'in_order', 'first_or_random'",
-        ErrorCodes::UNKNOWN_LOAD_BALANCING);
-}
-
-String SettingLoadBalancing::toString() const
-{
-    const char * strings[] = {"random", "nearest_hostname", "in_order", "first_or_random"};
-    if (value < LoadBalancing::RANDOM || value > LoadBalancing::FIRST_OR_RANDOM)
-        throw Exception("Unknown load balancing mode", ErrorCodes::UNKNOWN_LOAD_BALANCING);
-    return strings[static_cast<size_t>(value)];
-}
-
-Field SettingLoadBalancing::toField() const
-{
-    return toString();
-}
-
-void SettingLoadBalancing::set(LoadBalancing x)
-{
-    value = x;
-    changed = true;
-}
-
-void SettingLoadBalancing::set(const Field & x)
-{
-    set(safeGet<const String &>(x));
-}
-
-void SettingLoadBalancing::set(const String & x)
-{
-    set(getLoadBalancing(x));
-}
-
-void SettingLoadBalancing::serialize(WriteBuffer & buf) const
-{
-    writeBinary(toString(), buf);
-}
-
-void SettingLoadBalancing::deserialize(ReadBuffer & buf)
-{
-    String s;
-    readBinary(s, buf);
-    set(s);
-}
-
-
-JoinStrictness SettingJoinStrictness::getJoinStrictness(const String & s)
-{
-    if (s == "")       return JoinStrictness::Unspecified;
-    if (s == "ALL")    return JoinStrictness::ALL;
-    if (s == "ANY")    return JoinStrictness::ANY;
-
-    throw Exception("Unknown join strictness mode: '" + s + "', must be one of '', 'ALL', 'ANY'",
-        ErrorCodes::UNKNOWN_JOIN_STRICTNESS);
-}
-
-String SettingJoinStrictness::toString() const
-{
-    const char * strings[] = {"", "ALL", "ANY"};
-    if (value < JoinStrictness::Unspecified || value > JoinStrictness::ANY)
-        throw Exception("Unknown join strictness mode", ErrorCodes::UNKNOWN_JOIN_STRICTNESS);
-    return strings[static_cast<size_t>(value)];
-}
-
-Field SettingJoinStrictness::toField() const
-{
-    return toString();
-}
-
-void SettingJoinStrictness::set(JoinStrictness x)
-{
-    value = x;
-    changed = true;
-}
-
-void SettingJoinStrictness::set(const Field & x)
-{
-    set(safeGet<const String &>(x));
-}
-
-void SettingJoinStrictness::set(const String & x)
-{
-    set(getJoinStrictness(x));
-}
-
-void SettingJoinStrictness::serialize(WriteBuffer & buf) const
-{
-    writeBinary(toString(), buf);
-}
-
-void SettingJoinStrictness::deserialize(ReadBuffer & buf)
-{
-    String s;
-    readBinary(s, buf);
-    set(s);
-}
-
-
-TotalsMode SettingTotalsMode::getTotalsMode(const String & s)
-{
-    if (s == "before_having")          return TotalsMode::BEFORE_HAVING;
-    if (s == "after_having_exclusive") return TotalsMode::AFTER_HAVING_EXCLUSIVE;
-    if (s == "after_having_inclusive") return TotalsMode::AFTER_HAVING_INCLUSIVE;
-    if (s == "after_having_auto")      return TotalsMode::AFTER_HAVING_AUTO;
-
-    throw Exception("Unknown totals mode: '" + s + "', must be one of 'before_having', 'after_having_exclusive', 'after_having_inclusive', 'after_having_auto'", ErrorCodes::UNKNOWN_TOTALS_MODE);
-}
-
-String SettingTotalsMode::toString() const
-{
-    switch (value)
-    {
-        case TotalsMode::BEFORE_HAVING:          return "before_having";
-        case TotalsMode::AFTER_HAVING_EXCLUSIVE: return "after_having_exclusive";
-        case TotalsMode::AFTER_HAVING_INCLUSIVE: return "after_having_inclusive";
-        case TotalsMode::AFTER_HAVING_AUTO:      return "after_having_auto";
-    }
-
-    __builtin_unreachable();
-}
-
-Field SettingTotalsMode::toField() const
-{
-    return toString();
-}
-
-void SettingTotalsMode::set(TotalsMode x)
-{
-    value = x;
-    changed = true;
-}
-
-void SettingTotalsMode::set(const Field & x)
-{
-    set(safeGet<const String &>(x));
-}
-
-void SettingTotalsMode::set(const String & x)
-{
-    set(getTotalsMode(x));
-}
-
-void SettingTotalsMode::serialize(WriteBuffer & buf) const
-{
-    writeBinary(toString(), buf);
-}
-
-void SettingTotalsMode::deserialize(ReadBuffer & buf)
-{
-    String s;
-    readBinary(s, buf);
-    set(s);
-}
-
-
-
-template <bool enable_mode_any>
-OverflowMode SettingOverflowMode<enable_mode_any>::getOverflowModeForGroupBy(const String & s)
-{
-    if (s == "throw") return OverflowMode::THROW;
-    if (s == "break") return OverflowMode::BREAK;
-    if (s == "any")   return OverflowMode::ANY;
-
-    throw Exception("Unknown overflow mode: '" + s + "', must be one of 'throw', 'break', 'any'", ErrorCodes::UNKNOWN_OVERFLOW_MODE);
-}
-
-template <bool enable_mode_any>
-OverflowMode SettingOverflowMode<enable_mode_any>::getOverflowMode(const String & s)
-{
-    OverflowMode mode = getOverflowModeForGroupBy(s);
-
-    if (mode == OverflowMode::ANY && !enable_mode_any)
-        throw Exception("Illegal overflow mode: 'any' is only for 'group_by_overflow_mode'", ErrorCodes::ILLEGAL_OVERFLOW_MODE);
-
-    return mode;
-}
-
-template <bool enable_mode_any>
-String SettingOverflowMode<enable_mode_any>::toString() const
-{
-    const char * strings[] = { "throw", "break", "any" };
-
-    if (value < OverflowMode::THROW || value > OverflowMode::ANY)
-        throw Exception("Unknown overflow mode", ErrorCodes::UNKNOWN_OVERFLOW_MODE);
-
-    return strings[static_cast<size_t>(value)];
-}
-
-template <bool enable_mode_any>
-Field SettingOverflowMode<enable_mode_any>::toField() const
-{
-    return toString();
-}
-
-template <bool enable_mode_any>
-void SettingOverflowMode<enable_mode_any>::set(OverflowMode x)
-{
-    value = x;
-    changed = true;
-}
-
-template <bool enable_mode_any>
-void SettingOverflowMode<enable_mode_any>::set(const Field & x)
-{
-    set(safeGet<const String &>(x));
-}
-
-template <bool enable_mode_any>
-void SettingOverflowMode<enable_mode_any>::set(const String & x)
-{
-    set(getOverflowMode(x));
-}
-
-template <bool enable_mode_any>
-void SettingOverflowMode<enable_mode_any>::serialize(WriteBuffer & buf) const
-{
-    writeBinary(toString(), buf);
-}
-
-template <bool enable_mode_any>
-void SettingOverflowMode<enable_mode_any>::deserialize(ReadBuffer & buf)
-{
-    String s;
-    readBinary(s, buf);
-    set(s);
-}
-
-
-template struct SettingOverflowMode<false>;
-template struct SettingOverflowMode<true>;
-
-DistributedProductMode SettingDistributedProductMode::getDistributedProductMode(const String & s)
-{
-    if (s == "deny") return DistributedProductMode::DENY;
-    if (s == "local") return DistributedProductMode::LOCAL;
-    if (s == "global") return DistributedProductMode::GLOBAL;
-    if (s == "allow") return DistributedProductMode::ALLOW;
-
-    throw Exception("Unknown distributed product mode: '" + s + "', must be one of 'deny', 'local', 'global', 'allow'",
-        ErrorCodes::UNKNOWN_DISTRIBUTED_PRODUCT_MODE);
-}
-
-String SettingDistributedProductMode::toString() const
-{
-    const char * strings[] = {"deny", "local", "global", "allow"};
-    if (value < DistributedProductMode::DENY || value > DistributedProductMode::ALLOW)
-        throw Exception("Unknown distributed product mode", ErrorCodes::UNKNOWN_DISTRIBUTED_PRODUCT_MODE);
-    return strings[static_cast<size_t>(value)];
-}
-
-Field SettingDistributedProductMode::toField() const
-{
-    return toString();
-}
-
-void SettingDistributedProductMode::set(DistributedProductMode x)
-{
-    value = x;
-    changed = true;
-}
-
-void SettingDistributedProductMode::set(const Field & x)
-{
-    set(safeGet<const String &>(x));
-}
-
-void SettingDistributedProductMode::set(const String & x)
-{
-    set(getDistributedProductMode(x));
-}
-
-void SettingDistributedProductMode::serialize(WriteBuffer & buf) const
-{
-    writeBinary(toString(), buf);
-}
-
-void SettingDistributedProductMode::deserialize(ReadBuffer & buf)
-{
-    String s;
-    readBinary(s, buf);
-    set(s);
-}
+template struct SettingTimespan<SettingTimespanIO::SECOND>;
+template struct SettingTimespan<SettingTimespanIO::MILLISECOND>;
 
 
 String SettingString::toString() const
@@ -649,49 +301,14 @@ void SettingChar::deserialize(ReadBuffer & buf)
 }
 
 
-SettingDateTimeInputFormat::Value SettingDateTimeInputFormat::getValue(const String & s)
-{
-    if (s == "basic") return Value::Basic;
-    if (s == "best_effort") return Value::BestEffort;
-
-    throw Exception("Unknown DateTime input format: '" + s + "', must be one of 'basic', 'best_effort'", ErrorCodes::BAD_ARGUMENTS);
-}
-
-String SettingDateTimeInputFormat::toString() const
-{
-    const char * strings[] = {"basic", "best_effort"};
-    if (value < Value::Basic || value > Value::BestEffort)
-        throw Exception("Unknown DateTime input format", ErrorCodes::BAD_ARGUMENTS);
-    return strings[static_cast<size_t>(value)];
-}
-
-Field SettingDateTimeInputFormat::toField() const
-{
-    return toString();
-}
-
-void SettingDateTimeInputFormat::set(Value x)
-{
-    value = x;
-    changed = true;
-}
-
-void SettingDateTimeInputFormat::set(const Field & x)
-{
-    set(safeGet<const String &>(x));
-}
-
-void SettingDateTimeInputFormat::set(const String & x)
-{
-    set(getValue(x));
-}
-
-void SettingDateTimeInputFormat::serialize(WriteBuffer & buf) const
+template <typename EnumType, typename Tag>
+void SettingEnum<EnumType, Tag>::serialize(WriteBuffer & buf) const
 {
     writeBinary(toString(), buf);
 }
 
-void SettingDateTimeInputFormat::deserialize(ReadBuffer & buf)
+template <typename EnumType, typename Tag>
+void SettingEnum<EnumType, Tag>::deserialize(ReadBuffer & buf)
 {
     String s;
     readBinary(s, buf);
@@ -699,56 +316,110 @@ void SettingDateTimeInputFormat::deserialize(ReadBuffer & buf)
 }
 
 
-SettingLogsLevel::Value SettingLogsLevel::getValue(const String & s)
-{
-    if (s == "none") return Value::none;
-    if (s == "error") return Value::error;
-    if (s == "warning") return Value::warning;
-    if (s == "information") return Value::information;
-    if (s == "debug") return Value::debug;
-    if (s == "trace") return Value::trace;
+#define IMPLEMENT_SETTING_ENUM(ENUM_NAME, LIST_OF_NAMES_MACRO, ERROR_CODE_FOR_UNEXPECTED_NAME) \
+    IMPLEMENT_SETTING_ENUM_WITH_TAG(ENUM_NAME, void, LIST_OF_NAMES_MACRO, ERROR_CODE_FOR_UNEXPECTED_NAME)
 
-    throw Exception("Unknown logs level: '" + s + "', must be one of: none, error, warning, information, debug, trace", ErrorCodes::BAD_ARGUMENTS);
-}
+#define IMPLEMENT_SETTING_ENUM_WITH_TAG(ENUM_NAME, TAG, LIST_OF_NAMES_MACRO, ERROR_CODE_FOR_UNEXPECTED_NAME) \
+    template <> \
+    String SettingEnum<ENUM_NAME, TAG>::toString() const \
+    { \
+        using EnumType = ENUM_NAME; \
+        using UnderlyingType = std::underlying_type<EnumType>::type; \
+        switch (static_cast<UnderlyingType>(value)) \
+        { \
+            LIST_OF_NAMES_MACRO(IMPLEMENT_SETTING_ENUM_TO_STRING_HELPER_) \
+        } \
+        throw Exception("Unknown " #ENUM_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME); \
+    } \
+    \
+    template <> \
+    void SettingEnum<ENUM_NAME, TAG>::set(const String & s) \
+    { \
+        using EnumType = ENUM_NAME; \
+        LIST_OF_NAMES_MACRO(IMPLEMENT_SETTING_ENUM_FROM_STRING_HELPER_) \
+        \
+        String all_io_names; \
+        LIST_OF_NAMES_MACRO(IMPLEMENT_SETTING_ENUM_CONCAT_NAMES_HELPER_) \
+        throw Exception("Unknown " #ENUM_NAME " : '" + s + "', must be one of " + all_io_names, \
+            ERROR_CODE_FOR_UNEXPECTED_NAME); \
+    } \
+    \
+    template struct SettingEnum<ENUM_NAME, TAG>;
 
-String SettingLogsLevel::toString() const
-{
-    const char * strings[] = {"none", "error", "warning", "information", "debug", "trace"};
-    return strings[static_cast<size_t>(value)];
-}
+#define IMPLEMENT_SETTING_ENUM_TO_STRING_HELPER_(NAME, IO_NAME) \
+    case static_cast<UnderlyingType>(EnumType::NAME): return IO_NAME;
 
-Field SettingLogsLevel::toField() const
-{
-    return toString();
-}
+#define IMPLEMENT_SETTING_ENUM_FROM_STRING_HELPER_(NAME, IO_NAME) \
+    if (s == IO_NAME) \
+    { \
+        set(EnumType::NAME); \
+        return; \
+    }
 
-void SettingLogsLevel::set(Value x)
-{
-    value = x;
-    changed = true;
-}
+#define IMPLEMENT_SETTING_ENUM_CONCAT_NAMES_HELPER_(NAME, IO_NAME) \
+    if (!all_io_names.empty()) \
+        all_io_names += ", "; \
+    all_io_names += String("'") + IO_NAME + "'";
 
-void SettingLogsLevel::set(const Field & x)
-{
-    set(safeGet<const String &>(x));
-}
 
-void SettingLogsLevel::set(const String & x)
-{
-    set(getValue(x));
-}
+#define LOAD_BALANCING_LIST_OF_NAMES(M) \
+    M(RANDOM, "random") \
+    M(NEAREST_HOSTNAME, "nearest_hostname") \
+    M(IN_ORDER, "in_order") \
+    M(FIRST_OR_RANDOM, "first_or_random")
+IMPLEMENT_SETTING_ENUM(LoadBalancing, LOAD_BALANCING_LIST_OF_NAMES, ErrorCodes::UNKNOWN_LOAD_BALANCING)
 
-void SettingLogsLevel::serialize(WriteBuffer & buf) const
-{
-    writeBinary(toString(), buf);
-}
 
-void SettingLogsLevel::deserialize(ReadBuffer & buf)
-{
-    String s;
-    readBinary(s, buf);
-    set(s);
-}
+#define JOIN_STRICTNESS_LIST_OF_NAMES(M) \
+    M(Unspecified, "") \
+    M(ALL, "ALL") \
+    M(ANY, "ANY")
+IMPLEMENT_SETTING_ENUM(JoinStrictness, JOIN_STRICTNESS_LIST_OF_NAMES, ErrorCodes::UNKNOWN_JOIN_STRICTNESS)
+
+
+#define TOTALS_MODE_LIST_OF_NAMES(M) \
+    M(BEFORE_HAVING, "before_having") \
+    M(AFTER_HAVING_EXCLUSIVE, "after_having_exclusive") \
+    M(AFTER_HAVING_INCLUSIVE, "after_having_inclusive") \
+    M(AFTER_HAVING_AUTO, "after_having_auto")
+IMPLEMENT_SETTING_ENUM(TotalsMode, TOTALS_MODE_LIST_OF_NAMES, ErrorCodes::UNKNOWN_TOTALS_MODE)
+
+
+#define OVERFLOW_MODE_LIST_OF_NAMES(M) \
+    M(THROW, "throw") \
+    M(BREAK, "break")
+IMPLEMENT_SETTING_ENUM(OverflowMode, OVERFLOW_MODE_LIST_OF_NAMES, ErrorCodes::UNKNOWN_OVERFLOW_MODE)
+
+
+#define OVERFLOW_MODE_LIST_OF_NAMES_WITH_ANY(M) \
+    M(THROW, "throw") \
+    M(BREAK, "break") \
+    M(ANY, "any")
+IMPLEMENT_SETTING_ENUM_WITH_TAG(OverflowMode, SettingOverflowModeGroupByTag, OVERFLOW_MODE_LIST_OF_NAMES_WITH_ANY, ErrorCodes::UNKNOWN_OVERFLOW_MODE)
+
+
+#define DISTRIBUTED_PRODUCT_MODE_LIST_OF_NAMES(M) \
+    M(DENY, "deny") \
+    M(LOCAL, "local") \
+    M(GLOBAL, "global") \
+    M(ALLOW, "allow")
+IMPLEMENT_SETTING_ENUM(DistributedProductMode, DISTRIBUTED_PRODUCT_MODE_LIST_OF_NAMES, ErrorCodes::UNKNOWN_DISTRIBUTED_PRODUCT_MODE)
+
+
+#define DATE_TIME_INPUT_FORMAT_LIST_OF_NAMES(M) \
+    M(Basic, "basic") \
+    M(BestEffort, "best_effort")
+IMPLEMENT_SETTING_ENUM(FormatSettings::DateTimeInputFormat, DATE_TIME_INPUT_FORMAT_LIST_OF_NAMES, ErrorCodes::BAD_ARGUMENTS)
+
+
+#define LOGS_LEVEL_LIST_OF_NAMES(M) \
+    M(none, "none") \
+    M(error, "error") \
+    M(warning, "warning") \
+    M(information, "information") \
+    M(debug, "debug") \
+    M(trace, "trace")
+IMPLEMENT_SETTING_ENUM(LogsLevel, LOGS_LEVEL_LIST_OF_NAMES, ErrorCodes::BAD_ARGUMENTS)
 
 
 namespace details
diff --git a/dbms/src/Core/SettingsCommon.h b/dbms/src/Core/SettingsCommon.h
index 0b3ff95daf1..6bc24839145 100644
--- a/dbms/src/Core/SettingsCommon.h
+++ b/dbms/src/Core/SettingsCommon.h
@@ -5,7 +5,6 @@
 #include <Formats/FormatSettings.h>
 #include <common/StringRef.h>
 #include <Common/SettingsChanges.h>
-#include <Compression/CompressionInfo.h>
 #include <Core/Types.h>
 #include <ext/singleton.h>
 #include <unordered_map>
@@ -26,16 +25,16 @@ class WriteBuffer;
   *  and the remote server will use its default value.
   */
 
-template <typename IntType>
-struct SettingInt
+template <typename Type>
+struct SettingNumber
 {
-    IntType value;
+    Type value;
     bool changed = false;
 
-    SettingInt(IntType x = 0) : value(x) {}
+    SettingNumber(Type x = 0) : value(x) {}
 
-    operator IntType() const { return value; }
-    SettingInt & operator= (IntType x) { set(x); return *this; }
+    operator Type() const { return value; }
+    SettingNumber & operator= (Type x) { set(x); return *this; }
 
     /// Serialize to a test string.
     String toString() const;
@@ -43,7 +42,7 @@ struct SettingInt
     /// Converts to a field.
     Field toField() const;
 
-    void set(IntType x);
+    void set(Type x);
 
     /// Read from SQL literal.
     void set(const Field & x);
@@ -58,9 +57,10 @@ struct SettingInt
     void deserialize(ReadBuffer & buf);
 };
 
-using SettingUInt64 = SettingInt<UInt64>;
-using SettingInt64 = SettingInt<Int64>;
-using SettingBool = SettingUInt64;
+using SettingUInt64 = SettingNumber<UInt64>;
+using SettingInt64 = SettingNumber<Int64>;
+using SettingFloat = SettingNumber<float>;
+using SettingBool = SettingNumber<bool>;
 
 
 /** Unlike SettingUInt64, supports the value of 'auto' - the number of processor cores without taking into account SMT.
@@ -90,248 +90,42 @@ struct SettingMaxThreads
 
     void setAuto();
     UInt64 getAutoValue() const;
-
-    /// Executed once for all time. Executed from one thread.
-    UInt64 getAutoValueImpl() const;
 };
 
 
-struct SettingSeconds
+enum class SettingTimespanIO { MILLISECOND, SECOND };
+
+template <SettingTimespanIO io_unit>
+struct SettingTimespan
 {
     Poco::Timespan value;
     bool changed = false;
 
-    SettingSeconds(UInt64 seconds = 0) : value(seconds, 0) {}
+    SettingTimespan(UInt64 x = 0) : value(x * microseconds_per_io_unit) {}
 
     operator Poco::Timespan() const { return value; }
-    SettingSeconds & operator= (const Poco::Timespan & x) { set(x); return *this; }
+    SettingTimespan & operator= (const Poco::Timespan & x) { set(x); return *this; }
 
     Poco::Timespan::TimeDiff totalSeconds() const { return value.totalSeconds(); }
-
-    String toString() const;
-    Field toField() const;
-
-    void set(const Poco::Timespan & x);
-
-    void set(UInt64 x);
-    void set(const Field & x);
-    void set(const String & x);
-
-    void serialize(WriteBuffer & buf) const;
-    void deserialize(ReadBuffer & buf);
-};
-
-
-struct SettingMilliseconds
-{
-    Poco::Timespan value;
-    bool changed = false;
-
-    SettingMilliseconds(UInt64 milliseconds = 0) : value(milliseconds * 1000) {}
-
-    operator Poco::Timespan() const { return value; }
-    SettingMilliseconds & operator= (const Poco::Timespan & x) { set(x); return *this; }
-
     Poco::Timespan::TimeDiff totalMilliseconds() const { return value.totalMilliseconds(); }
 
     String toString() const;
     Field toField() const;
 
     void set(const Poco::Timespan & x);
+
     void set(UInt64 x);
     void set(const Field & x);
     void set(const String & x);
 
     void serialize(WriteBuffer & buf) const;
     void deserialize(ReadBuffer & buf);
+
+    static constexpr UInt64 microseconds_per_io_unit = (io_unit == SettingTimespanIO::MILLISECOND) ? 1000 : 1000000;
 };
 
-
-struct SettingFloat
-{
-    float value;
-    bool changed = false;
-
-    SettingFloat(float x = 0) : value(x) {}
-
-    operator float() const { return value; }
-    SettingFloat & operator= (float x) { set(x); return *this; }
-
-    String toString() const;
-    Field toField() const;
-
-    void set(float x);
-    void set(const Field & x);
-    void set(const String & x);
-
-    void serialize(WriteBuffer & buf) const;
-    void deserialize(ReadBuffer & buf);
-};
-
-
-/// TODO: X macro
-enum class LoadBalancing
-{
-    /// among replicas with a minimum number of errors selected randomly
-    RANDOM = 0,
-    /// a replica is selected among the replicas with the minimum number of errors
-    /// with the minimum number of distinguished characters in the replica name and local hostname
-    NEAREST_HOSTNAME,
-    /// replicas are walked through strictly in order; the number of errors does not matter
-    IN_ORDER,
-    /// if first replica one has higher number of errors,
-    ///   pick a random one from replicas with minimum number of errors
-    FIRST_OR_RANDOM,
-};
-
-struct SettingLoadBalancing
-{
-    LoadBalancing value;
-    bool changed = false;
-
-    SettingLoadBalancing(LoadBalancing x) : value(x) {}
-
-    operator LoadBalancing() const { return value; }
-    SettingLoadBalancing & operator= (LoadBalancing x) { set(x); return *this; }
-
-    static LoadBalancing getLoadBalancing(const String & s);
-
-    String toString() const;
-    Field toField() const;
-
-    void set(LoadBalancing x);
-    void set(const Field & x);
-    void set(const String & x);
-
-    void serialize(WriteBuffer & buf) const;
-    void deserialize(ReadBuffer & buf);
-};
-
-
-enum class JoinStrictness
-{
-    Unspecified = 0, /// Query JOIN without strictness will throw Exception.
-    ALL, /// Query JOIN without strictness -> ALL JOIN ...
-    ANY, /// Query JOIN without strictness -> ANY JOIN ...
-};
-
-
-struct SettingJoinStrictness
-{
-    JoinStrictness value;
-    bool changed = false;
-
-    SettingJoinStrictness(JoinStrictness x) : value(x) {}
-
-    operator JoinStrictness() const { return value; }
-    SettingJoinStrictness & operator= (JoinStrictness x) { set(x); return *this; }
-
-    static JoinStrictness getJoinStrictness(const String & s);
-
-    String toString() const;
-    Field toField() const;
-
-    void set(JoinStrictness x);
-    void set(const Field & x);
-    void set(const String & x);
-
-    void serialize(WriteBuffer & buf) const;
-    void deserialize(ReadBuffer & buf);
-};
-
-
-/// Which rows should be included in TOTALS.
-enum class TotalsMode
-{
-    BEFORE_HAVING            = 0, /// Count HAVING for all read rows;
-                                  ///  including those not in max_rows_to_group_by
-                                  ///  and have not passed HAVING after grouping.
-    AFTER_HAVING_INCLUSIVE    = 1, /// Count on all rows except those that have not passed HAVING;
-                                   ///  that is, to include in TOTALS all the rows that did not pass max_rows_to_group_by.
-    AFTER_HAVING_EXCLUSIVE    = 2, /// Include only the rows that passed and max_rows_to_group_by, and HAVING.
-    AFTER_HAVING_AUTO         = 3, /// Automatically select between INCLUSIVE and EXCLUSIVE,
-};
-
-struct SettingTotalsMode
-{
-    TotalsMode value;
-    bool changed = false;
-
-    SettingTotalsMode(TotalsMode x) : value(x) {}
-
-    operator TotalsMode() const { return value; }
-    SettingTotalsMode & operator= (TotalsMode x) { set(x); return *this; }
-
-    static TotalsMode getTotalsMode(const String & s);
-
-    String toString() const;
-    Field toField() const;
-
-    void set(TotalsMode x);
-    void set(const Field & x);
-    void set(const String & x);
-
-    void serialize(WriteBuffer & buf) const;
-    void deserialize(ReadBuffer & buf);
-};
-
-
-template <bool enable_mode_any>
-struct SettingOverflowMode
-{
-    OverflowMode value;
-    bool changed = false;
-
-    SettingOverflowMode(OverflowMode x = OverflowMode::THROW) : value(x) {}
-
-    operator OverflowMode() const { return value; }
-    SettingOverflowMode & operator= (OverflowMode x) { set(x); return *this; }
-
-    static OverflowMode getOverflowModeForGroupBy(const String & s);
-    static OverflowMode getOverflowMode(const String & s);
-
-    String toString() const;
-    Field toField() const;
-
-    void set(OverflowMode x);
-    void set(const Field & x);
-    void set(const String & x);
-
-    void serialize(WriteBuffer & buf) const;
-    void deserialize(ReadBuffer & buf);
-};
-
-/// The setting for executing distributed subqueries inside IN or JOIN sections.
-enum class DistributedProductMode
-{
-    DENY = 0,    /// Disable
-    LOCAL,       /// Convert to local query
-    GLOBAL,      /// Convert to global query
-    ALLOW        /// Enable
-};
-
-struct SettingDistributedProductMode
-{
-    DistributedProductMode value;
-    bool changed = false;
-
-    SettingDistributedProductMode(DistributedProductMode x) : value(x) {}
-
-    operator DistributedProductMode() const { return value; }
-    SettingDistributedProductMode & operator= (DistributedProductMode x) { set(x); return *this; }
-
-    static DistributedProductMode getDistributedProductMode(const String & s);
-
-    String toString() const;
-    Field toField() const;
-
-    void set(DistributedProductMode x);
-    void set(const Field & x);
-    void set(const String & x);
-
-    void serialize(WriteBuffer & buf) const;
-    void deserialize(ReadBuffer & buf);
-};
+using SettingSeconds = SettingTimespan<SettingTimespanIO::SECOND>;
+using SettingMilliseconds = SettingTimespan<SettingTimespanIO::MILLISECOND>;
 
 
 struct SettingString
@@ -378,25 +172,23 @@ public:
 };
 
 
-struct SettingDateTimeInputFormat
+/// Template class to define enum-based settings.
+template <typename EnumType, typename Tag = void>
+struct SettingEnum
 {
-    using Value = FormatSettings::DateTimeInputFormat;
-
-    Value value;
+    EnumType value;
     bool changed = false;
 
-    SettingDateTimeInputFormat(Value x) : value(x) {}
+    SettingEnum(EnumType x) : value(x) {}
 
-    operator Value() const { return value; }
-    SettingDateTimeInputFormat & operator= (Value x) { set(x); return *this; }
-
-    static Value getValue(const String & s);
+    operator EnumType() const { return value; }
+    SettingEnum & operator= (EnumType x) { set(x); return *this; }
 
     String toString() const;
-    Field toField() const;
+    Field toField() const { return toString(); }
 
-    void set(Value x);
-    void set(const Field & x);
+    void set(EnumType x) { value = x; changed = true; }
+    void set(const Field & x) { set(safeGet<const String &>(x)); }
     void set(const String & x);
 
     void serialize(WriteBuffer & buf) const;
@@ -404,6 +196,67 @@ struct SettingDateTimeInputFormat
 };
 
 
+enum class LoadBalancing
+{
+    /// among replicas with a minimum number of errors selected randomly
+    RANDOM = 0,
+    /// a replica is selected among the replicas with the minimum number of errors
+    /// with the minimum number of distinguished characters in the replica name and local hostname
+    NEAREST_HOSTNAME,
+    /// replicas are walked through strictly in order; the number of errors does not matter
+    IN_ORDER,
+    /// if first replica one has higher number of errors,
+    ///   pick a random one from replicas with minimum number of errors
+    FIRST_OR_RANDOM,
+};
+using SettingLoadBalancing = SettingEnum<LoadBalancing>;
+
+
+enum class JoinStrictness
+{
+    Unspecified = 0, /// Query JOIN without strictness will throw Exception.
+    ALL, /// Query JOIN without strictness -> ALL JOIN ...
+    ANY, /// Query JOIN without strictness -> ANY JOIN ...
+};
+using SettingJoinStrictness = SettingEnum<JoinStrictness>;
+
+
+/// Which rows should be included in TOTALS.
+enum class TotalsMode
+{
+    BEFORE_HAVING            = 0, /// Count HAVING for all read rows;
+                                  ///  including those not in max_rows_to_group_by
+                                  ///  and have not passed HAVING after grouping.
+    AFTER_HAVING_INCLUSIVE    = 1, /// Count on all rows except those that have not passed HAVING;
+                                   ///  that is, to include in TOTALS all the rows that did not pass max_rows_to_group_by.
+    AFTER_HAVING_EXCLUSIVE    = 2, /// Include only the rows that passed and max_rows_to_group_by, and HAVING.
+    AFTER_HAVING_AUTO         = 3, /// Automatically select between INCLUSIVE and EXCLUSIVE,
+};
+using SettingTotalsMode = SettingEnum<TotalsMode>;
+
+
+/// The settings keeps OverflowMode which cannot be OverflowMode::ANY.
+using SettingOverflowMode = SettingEnum<OverflowMode>;
+struct SettingOverflowModeGroupByTag;
+
+/// The settings keeps OverflowMode which can be OverflowMode::ANY.
+using SettingOverflowModeGroupBy = SettingEnum<OverflowMode, SettingOverflowModeGroupByTag>;
+
+
+/// The setting for executing distributed subqueries inside IN or JOIN sections.
+enum class DistributedProductMode
+{
+    DENY = 0,    /// Disable
+    LOCAL,       /// Convert to local query
+    GLOBAL,      /// Convert to global query
+    ALLOW        /// Enable
+};
+using SettingDistributedProductMode = SettingEnum<DistributedProductMode>;
+
+
+using SettingDateTimeInputFormat = SettingEnum<FormatSettings::DateTimeInputFormat>;
+
+
 enum class LogsLevel
 {
     none = 0,    /// Disable
@@ -413,32 +266,7 @@ enum class LogsLevel
     debug,
     trace,
 };
-
-class SettingLogsLevel
-{
-public:
-    using Value = LogsLevel;
-
-    Value value;
-    bool changed = false;
-
-    SettingLogsLevel(Value x) : value(x) {}
-
-    operator Value() const { return value; }
-    SettingLogsLevel & operator= (Value x) { set(x); return *this; }
-
-    static Value getValue(const String & s);
-
-    String toString() const;
-    Field toField() const;
-
-    void set(Value x);
-    void set(const Field & x);
-    void set(const String & x);
-
-    void serialize(WriteBuffer & buf) const;
-    void deserialize(ReadBuffer & buf);
-};
+using SettingLogsLevel = SettingEnum<LogsLevel>;
 
 
 namespace details

From 28d9a80072dd38cc7bc395f4c0b8482b98d81e7f Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitbar@yandex-team.ru>
Date: Mon, 29 Apr 2019 00:03:19 +0300
Subject: [PATCH 086/118] Rename macro APPLY_FOR_SETTINGS => LIST_OF_SETTINGS.

---
 dbms/src/Core/Settings.cpp                        |  2 +-
 dbms/src/Core/Settings.h                          |  4 ++--
 dbms/src/Core/SettingsCommon.h                    | 10 +++++-----
 dbms/src/Storages/Kafka/KafkaSettings.cpp         |  2 +-
 dbms/src/Storages/Kafka/KafkaSettings.h           |  4 ++--
 dbms/src/Storages/MergeTree/MergeTreeSettings.cpp |  2 +-
 dbms/src/Storages/MergeTree/MergeTreeSettings.h   |  4 ++--
 7 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/dbms/src/Core/Settings.cpp b/dbms/src/Core/Settings.cpp
index c09ff38a2d8..3b7c83dbf97 100644
--- a/dbms/src/Core/Settings.cpp
+++ b/dbms/src/Core/Settings.cpp
@@ -17,7 +17,7 @@ namespace ErrorCodes
 }
 
 
-IMPLEMENT_SETTINGS_COLLECTION(Settings, APPLY_FOR_SETTINGS)
+IMPLEMENT_SETTINGS_COLLECTION(Settings, LIST_OF_SETTINGS)
 
 
 /** Set the settings from the profile (in the server configuration, many settings can be listed in one profile).
diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h
index 272f1ce2373..c8a65ea59d2 100644
--- a/dbms/src/Core/Settings.h
+++ b/dbms/src/Core/Settings.h
@@ -42,7 +42,7 @@ struct Settings : public SettingsCollection<Settings>
       *  but we are not going to do it, because settings is used everywhere as static struct fields.
       */
 
-#define APPLY_FOR_SETTINGS(M) \
+#define LIST_OF_SETTINGS(M) \
     M(SettingUInt64, min_compress_block_size, 65536, "The actual size of the block to compress, if the uncompressed data less than max_compress_block_size is no less than this value and no less than the volume of data for one mark.") \
     M(SettingUInt64, max_compress_block_size, 1048576, "The maximum size of blocks of uncompressed data before compressing for writing to a table.") \
     M(SettingUInt64, max_block_size, DEFAULT_BLOCK_SIZE, "Maximum block size for reading") \
@@ -324,7 +324,7 @@ struct Settings : public SettingsCollection<Settings>
     \
     M(SettingUInt64, max_partitions_per_insert_block, 100, "Limit maximum number of partitions in single INSERTed block. Zero means unlimited. Throw exception if the block contains too many partitions. This setting is a safety threshold, because using large number of partitions is a common misconception.") \
 
-    DECLARE_SETTINGS_COLLECTION(APPLY_FOR_SETTINGS)
+    DECLARE_SETTINGS_COLLECTION(LIST_OF_SETTINGS)
 
     /** Set multiple settings from "profile" (in server configuration file (users.xml), profiles contain groups of multiple settings).
      * The profile can also be set using the `set` functions, like the profile setting.
diff --git a/dbms/src/Core/SettingsCommon.h b/dbms/src/Core/SettingsCommon.h
index 6bc24839145..287f30c5879 100644
--- a/dbms/src/Core/SettingsCommon.h
+++ b/dbms/src/Core/SettingsCommon.h
@@ -605,20 +605,20 @@ public:
     }
 };
 
-#define DECLARE_SETTINGS_COLLECTION(APPLY_MACRO) \
-    APPLY_MACRO(DECLARE_SETTINGS_COLLECTION_DECLARE_VARIABLES_HELPER_)
+#define DECLARE_SETTINGS_COLLECTION(LIST_OF_SETTINGS_MACRO) \
+    LIST_OF_SETTINGS_MACRO(DECLARE_SETTINGS_COLLECTION_DECLARE_VARIABLES_HELPER_)
 
 
-#define IMPLEMENT_SETTINGS_COLLECTION(DERIVED_CLASS_NAME, APPLY_MACRO) \
+#define IMPLEMENT_SETTINGS_COLLECTION(DERIVED_CLASS_NAME, LIST_OF_SETTINGS_MACRO) \
     template<> \
     SettingsCollection<DERIVED_CLASS_NAME>::MemberInfos::MemberInfos() \
     { \
         using Derived = DERIVED_CLASS_NAME; \
         struct Functions \
         { \
-            APPLY_MACRO(IMPLEMENT_SETTINGS_COLLECTION_DEFINE_FUNCTIONS_HELPER_) \
+            LIST_OF_SETTINGS_MACRO(IMPLEMENT_SETTINGS_COLLECTION_DEFINE_FUNCTIONS_HELPER_) \
         }; \
-        APPLY_MACRO(IMPLEMENT_SETTINGS_COLLECTION_ADD_MEMBER_INFO_HELPER_) \
+        LIST_OF_SETTINGS_MACRO(IMPLEMENT_SETTINGS_COLLECTION_ADD_MEMBER_INFO_HELPER_) \
     }
 
 
diff --git a/dbms/src/Storages/Kafka/KafkaSettings.cpp b/dbms/src/Storages/Kafka/KafkaSettings.cpp
index 79223104085..d08282a9794 100644
--- a/dbms/src/Storages/Kafka/KafkaSettings.cpp
+++ b/dbms/src/Storages/Kafka/KafkaSettings.cpp
@@ -14,7 +14,7 @@ namespace ErrorCodes
     extern const int UNKNOWN_SETTING;
 }
 
-IMPLEMENT_SETTINGS_COLLECTION(KafkaSettings, APPLY_FOR_KAFKA_SETTINGS)
+IMPLEMENT_SETTINGS_COLLECTION(KafkaSettings, LIST_OF_KAFKA_SETTINGS)
 
 void KafkaSettings::loadFromQuery(ASTStorage & storage_def)
 {
diff --git a/dbms/src/Storages/Kafka/KafkaSettings.h b/dbms/src/Storages/Kafka/KafkaSettings.h
index 0a21adf334b..595a620c697 100644
--- a/dbms/src/Storages/Kafka/KafkaSettings.h
+++ b/dbms/src/Storages/Kafka/KafkaSettings.h
@@ -14,7 +14,7 @@ class ASTStorage;
 struct KafkaSettings : public SettingsCollection<KafkaSettings>
 {
 
-#define APPLY_FOR_KAFKA_SETTINGS(M) \
+#define LIST_OF_KAFKA_SETTINGS(M) \
     M(SettingString, kafka_broker_list, "", "A comma-separated list of brokers for Kafka engine.") \
     M(SettingString, kafka_topic_list, "", "A list of Kafka topics.") \
     M(SettingString, kafka_group_name, "", "A group of Kafka consumers.") \
@@ -25,7 +25,7 @@ struct KafkaSettings : public SettingsCollection<KafkaSettings>
     M(SettingUInt64, kafka_max_block_size, 0, "The maximum block size per table for Kafka engine.") \
     M(SettingUInt64, kafka_skip_broken_messages, 0, "Skip at least this number of broken messages from Kafka topic per block")
 
-    DECLARE_SETTINGS_COLLECTION(APPLY_FOR_KAFKA_SETTINGS)
+    DECLARE_SETTINGS_COLLECTION(LIST_OF_KAFKA_SETTINGS)
 
     void loadFromQuery(ASTStorage & storage_def);
 };
diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp b/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp
index 701e1e2adf2..e3600b6ac4a 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.cpp
@@ -16,7 +16,7 @@ namespace ErrorCodes
     extern const int UNKNOWN_SETTING;
 }
 
-IMPLEMENT_SETTINGS_COLLECTION(MergeTreeSettings, APPLY_FOR_MERGE_TREE_SETTINGS)
+IMPLEMENT_SETTINGS_COLLECTION(MergeTreeSettings, LIST_OF_MERGE_TREE_SETTINGS)
 
 void MergeTreeSettings::loadFromConfig(const String & config_elem, const Poco::Util::AbstractConfiguration & config)
 {
diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.h b/dbms/src/Storages/MergeTree/MergeTreeSettings.h
index 4239c7543ed..5deaa0816b4 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeSettings.h
+++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.h
@@ -24,7 +24,7 @@ class ASTStorage;
 struct MergeTreeSettings : public SettingsCollection<MergeTreeSettings>
 {
 
-#define APPLY_FOR_MERGE_TREE_SETTINGS(M) \
+#define LIST_OF_MERGE_TREE_SETTINGS(M) \
     M(SettingUInt64, index_granularity, 8192, "How many rows correspond to one primary key value.") \
     \
     /** Merge settings. */ \
@@ -79,7 +79,7 @@ struct MergeTreeSettings : public SettingsCollection<MergeTreeSettings>
     M(SettingUInt64, index_granularity_bytes, 0, "Approximate amount of bytes in single granule (0 - disabled).") \
     M(SettingInt64, merge_with_ttl_timeout, 3600 * 24, "Minimal time in seconds, when merge with TTL can be repeated.")
 
-    DECLARE_SETTINGS_COLLECTION(APPLY_FOR_MERGE_TREE_SETTINGS)
+    DECLARE_SETTINGS_COLLECTION(LIST_OF_MERGE_TREE_SETTINGS)
 
     /// Settings that should not change after the creation of a table.
 #define APPLY_FOR_IMMUTABLE_MERGE_TREE_SETTINGS(M) \

From 8d2941fa607e3fbcf8a4633ff5a895dbf54e66f4 Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Mon, 29 Apr 2019 04:50:18 +0300
Subject: [PATCH 087/118] Update RadixSort.h

---
 dbms/src/Common/RadixSort.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/dbms/src/Common/RadixSort.h b/dbms/src/Common/RadixSort.h
index c661a622881..c4be1bf4a4b 100644
--- a/dbms/src/Common/RadixSort.h
+++ b/dbms/src/Common/RadixSort.h
@@ -203,6 +203,7 @@ private:
     {
         Element * end = arr + size;
         for (Element * i = arr + 1; i < end; ++i)
+        {
             if (Traits::less(Traits::extractKey(*i), Traits::extractKey(*(i - 1))))
             {
                 Element * j;
@@ -212,6 +213,7 @@ private:
                     *j = *(j - 1);
                 *j = tmp;
             }
+        }
     }
 
     /* Main MSD radix sort subroutine

From 95141804fbefb675649124dcb875a03c672cfe08 Mon Sep 17 00:00:00 2001
From: Guillaume Tassery <tassery.guillaume@gmail.com>
Date: Mon, 29 Apr 2019 16:40:44 +0700
Subject: [PATCH 088/118] Add a basename function

---
 dbms/src/Functions/basename.cpp               | 38 +++++++++++++++++++
 .../registerFunctionsMiscellaneous.cpp        |  2 +
 2 files changed, 40 insertions(+)
 create mode 100644 dbms/src/Functions/basename.cpp

diff --git a/dbms/src/Functions/basename.cpp b/dbms/src/Functions/basename.cpp
new file mode 100644
index 00000000000..dacf08ae1e3
--- /dev/null
+++ b/dbms/src/Functions/basename.cpp
@@ -0,0 +1,38 @@
+#include <Functions/FunctionFactory.h>
+#include <Functions/FunctionStringToString.h>
+#include <Functions/FunctionsURL.h>
+#include <common/find_symbols.h>
+
+namespace DB
+{
+
+struct ExtractBasename
+{
+    static size_t getReserveLengthForElement() { return 25; }
+
+    static void execute(Pos data, size_t size, Pos & res_data, size_t & res_size)
+    {
+        res_data = data;
+        res_size = size;
+
+        Pos pos = data;
+        Pos end = pos + size;
+
+        if ((pos = find_last_symbols_or_null<'/', '\\'>(pos, end)))
+        {
+            ++pos;
+            res_data = pos;
+            res_size = end - pos;
+        }
+    }
+};
+
+struct NameBasename { static constexpr auto name = "basename"; };
+using FunctionBasename = FunctionStringToString<ExtractSubstringImpl<ExtractBasename>, NameBasename>;
+
+void registerFunctionBasename(FunctionFactory & factory)
+{
+    factory.registerFunction<FunctionBasename>();
+}
+
+}
diff --git a/dbms/src/Functions/registerFunctionsMiscellaneous.cpp b/dbms/src/Functions/registerFunctionsMiscellaneous.cpp
index d985fb6bf97..05e888dbbe9 100644
--- a/dbms/src/Functions/registerFunctionsMiscellaneous.cpp
+++ b/dbms/src/Functions/registerFunctionsMiscellaneous.cpp
@@ -42,6 +42,7 @@ void registerFunctionLowCardinalityKeys(FunctionFactory &);
 void registerFunctionsIn(FunctionFactory &);
 void registerFunctionJoinGet(FunctionFactory &);
 void registerFunctionFilesystem(FunctionFactory &);
+void registerFunctionBasename(FunctionFactory &);
 
 void registerFunctionsMiscellaneous(FunctionFactory & factory)
 {
@@ -84,6 +85,7 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory)
     registerFunctionsIn(factory);
     registerFunctionJoinGet(factory);
     registerFunctionFilesystem(factory);
+    registerFunctionBasename(factory);
 }
 
 }

From 294b694a8adbb60a5f42436a56f45e88944853fb Mon Sep 17 00:00:00 2001
From: Serge Rider <serge@jkiss.org>
Date: Mon, 29 Apr 2019 13:13:18 +0300
Subject: [PATCH 089/118] DBeaver information update

DBeaver for Clickhouse is NOT a commercial software, it is free and open-source (Apache license). Source code is at https://github.com/dbeaver/dbeaver/issues
---
 docs/en/interfaces/third-party/gui.md | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/docs/en/interfaces/third-party/gui.md b/docs/en/interfaces/third-party/gui.md
index 727d1ba7270..5f7ef803bbb 100644
--- a/docs/en/interfaces/third-party/gui.md
+++ b/docs/en/interfaces/third-party/gui.md
@@ -48,6 +48,17 @@ Features:
 - Table preview with filtering and sorting.
 - Read-only queries execution.
 
+### DBeaver
+
+[DBeaver](https://dbeaver.io/) - universal desktop database client with ClickHouse support.
+
+Features:
+
+- Query development with syntax highlight and autocompletion.
+- Table list with filters and metadata search.
+- Table data preview.
+- Full text search.
+
 ### clickhouse-cli
 
 [clickhouse-cli](https://github.com/hatarist/clickhouse-cli) is an alternative command line client for ClickHouse, written in Python 3.
@@ -60,16 +71,6 @@ Features:
 
 ## Commercial
 
-### DBeaver
-
-[DBeaver](https://dbeaver.io/) - universal desktop database client with ClickHouse support.
-
-Features:
-
-- Query development with syntax highlight.
-- Table preview.
-- Autocompletion.
-
 ### DataGrip
 
 [DataGrip](https://www.jetbrains.com/datagrip/) is a database IDE from JetBrains with dedicated support for ClickHouse. It is also embedded into other IntelliJ-based tools: PyCharm, IntelliJ IDEA, GoLand, PhpStorm and others.

From 87b60f0a3b41499a8e04dda5138b80ecc4e6500d Mon Sep 17 00:00:00 2001
From: Guillaume Tassery <tassery.guillaume@gmail.com>
Date: Mon, 29 Apr 2019 17:18:02 +0700
Subject: [PATCH 090/118] Add test for basename function

---
 dbms/tests/queries/0_stateless/00938_basename.reference | 5 +++++
 dbms/tests/queries/0_stateless/00938_basename.sql       | 5 +++++
 2 files changed, 10 insertions(+)
 create mode 100644 dbms/tests/queries/0_stateless/00938_basename.reference
 create mode 100644 dbms/tests/queries/0_stateless/00938_basename.sql

diff --git a/dbms/tests/queries/0_stateless/00938_basename.reference b/dbms/tests/queries/0_stateless/00938_basename.reference
new file mode 100644
index 00000000000..37da7973757
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00938_basename.reference
@@ -0,0 +1,5 @@
+bash
+
+bash
+test_file
+script.php
diff --git a/dbms/tests/queries/0_stateless/00938_basename.sql b/dbms/tests/queries/0_stateless/00938_basename.sql
new file mode 100644
index 00000000000..cea11e9a479
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00938_basename.sql
@@ -0,0 +1,5 @@
+SELECT basename('/usr/bin/bash');
+SELECT basename('/usr/bin/bash/');
+SELECT basename('bash');
+SELECT basename('C:\\\\Users\\Documents\\test_file');
+SELECT basename(path('http://example.com/folder_1/folder_2/script.php'))

From 7766f5c0c31c09b4ec9d65d99c04db2c17238edc Mon Sep 17 00:00:00 2001
From: CurtizJ <pad11rus@gmail.com>
Date: Mon, 29 Apr 2019 16:12:39 +0300
Subject: [PATCH 091/118] added offset for limit by

---
 .../DataStreams/LimitByBlockInputStream.cpp   |  9 +++--
 .../src/DataStreams/LimitByBlockInputStream.h |  5 +--
 .../Interpreters/InterpreterSelectQuery.cpp   |  8 ++---
 dbms/src/Parsers/ASTSelectQuery.cpp           | 12 +++++--
 dbms/src/Parsers/ASTSelectQuery.h             |  6 ++--
 dbms/src/Parsers/ParserSelectQuery.cpp        | 33 ++++++++++++-------
 .../00939_limit_by_offset.reference           | 13 ++++++++
 .../0_stateless/00939_limit_by_offset.sql     | 10 ++++++
 8 files changed, 70 insertions(+), 26 deletions(-)
 create mode 100644 dbms/tests/queries/0_stateless/00939_limit_by_offset.reference
 create mode 100644 dbms/tests/queries/0_stateless/00939_limit_by_offset.sql

diff --git a/dbms/src/DataStreams/LimitByBlockInputStream.cpp b/dbms/src/DataStreams/LimitByBlockInputStream.cpp
index b77c2b6e659..93f1ce83ae8 100644
--- a/dbms/src/DataStreams/LimitByBlockInputStream.cpp
+++ b/dbms/src/DataStreams/LimitByBlockInputStream.cpp
@@ -5,9 +5,11 @@
 namespace DB
 {
 
-LimitByBlockInputStream::LimitByBlockInputStream(const BlockInputStreamPtr & input, size_t group_size_, const Names & columns)
+LimitByBlockInputStream::LimitByBlockInputStream(const BlockInputStreamPtr & input,
+    size_t group_length_, size_t group_offset_, const Names & columns)
     : columns_names(columns)
-    , group_size(group_size_)
+    , group_length(group_length_)
+    , group_offset(group_offset_)
 {
     children.push_back(input);
 }
@@ -37,7 +39,8 @@ Block LimitByBlockInputStream::readImpl()
 
             hash.get128(key.low, key.high);
 
-            if (keys_counts[key]++ < group_size)
+            auto count = keys_counts[key]++;
+            if (count >= group_offset && count < group_length + group_offset)
             {
                 inserted_count++;
                 filter[i] = 1;
diff --git a/dbms/src/DataStreams/LimitByBlockInputStream.h b/dbms/src/DataStreams/LimitByBlockInputStream.h
index 4244908ba1c..8fca64be606 100644
--- a/dbms/src/DataStreams/LimitByBlockInputStream.h
+++ b/dbms/src/DataStreams/LimitByBlockInputStream.h
@@ -18,7 +18,7 @@ namespace DB
 class LimitByBlockInputStream : public IBlockInputStream
 {
 public:
-    LimitByBlockInputStream(const BlockInputStreamPtr & input, size_t group_size_, const Names & columns);
+    LimitByBlockInputStream(const BlockInputStreamPtr & input, size_t group_length_, size_t group_offset_, const Names & columns);
 
     String getName() const override { return "LimitBy"; }
 
@@ -34,7 +34,8 @@ private:
     using MapHashed = HashMap<UInt128, UInt64, UInt128TrivialHash>;
 
     const Names columns_names;
-    const size_t group_size;
+    const size_t group_length;
+    const size_t group_offset;
     MapHashed keys_counts;
 };
 
diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp
index 04896e85d62..658333202f2 100644
--- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp
@@ -1562,18 +1562,18 @@ void InterpreterSelectQuery::executePreLimit(Pipeline & pipeline)
 void InterpreterSelectQuery::executeLimitBy(Pipeline & pipeline)
 {
     auto & query = getSelectQuery();
-    if (!query.limitByValue() || !query.limitBy())
+    if (!query.limitByLength() || !query.limitBy())
         return;
 
     Names columns;
     for (const auto & elem : query.limitBy()->children)
         columns.emplace_back(elem->getColumnName());
-
-    UInt64 value = getLimitUIntValue(query.limitByValue(), context);
+    UInt64 length = getLimitUIntValue(query.limitByLength(), context);
+    UInt64 offset = (query.limitByOffset() ? getLimitUIntValue(query.limitByOffset(), context) : 0);
 
     pipeline.transform([&](auto & stream)
     {
-        stream = std::make_shared<LimitByBlockInputStream>(stream, value, columns);
+        stream = std::make_shared<LimitByBlockInputStream>(stream, length, offset, columns);
     });
 }
 
diff --git a/dbms/src/Parsers/ASTSelectQuery.cpp b/dbms/src/Parsers/ASTSelectQuery.cpp
index ed40f0adbcf..44012cd071c 100644
--- a/dbms/src/Parsers/ASTSelectQuery.cpp
+++ b/dbms/src/Parsers/ASTSelectQuery.cpp
@@ -42,7 +42,8 @@ ASTPtr ASTSelectQuery::clone() const
     CLONE(Expression::GROUP_BY);
     CLONE(Expression::HAVING);
     CLONE(Expression::ORDER_BY);
-    CLONE(Expression::LIMIT_BY_VALUE);
+    CLONE(Expression::LIMIT_BY_OFFSET);
+    CLONE(Expression::LIMIT_BY_LENGTH);
     CLONE(Expression::LIMIT_BY);
     CLONE(Expression::LIMIT_OFFSET);
     CLONE(Expression::LIMIT_LENGTH);
@@ -124,10 +125,15 @@ void ASTSelectQuery::formatImpl(const FormatSettings & s, FormatState & state, F
             : orderBy()->as<ASTExpressionList &>().formatImplMultiline(s, state, frame);
     }
 
-    if (limitByValue())
+    if (limitByLength())
     {
         s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "LIMIT " << (s.hilite ? hilite_none : "");
-        limitByValue()->formatImpl(s, state, frame);
+        if (limitByOffset())
+        {
+            limitByOffset()->formatImpl(s, state, frame);
+            s.ostr << ", ";
+        }
+        limitByLength()->formatImpl(s, state, frame);
         s.ostr << (s.hilite ? hilite_keyword : "") << " BY " << (s.hilite ? hilite_none : "");
         s.one_line
             ? limitBy()->formatImpl(s, state, frame)
diff --git a/dbms/src/Parsers/ASTSelectQuery.h b/dbms/src/Parsers/ASTSelectQuery.h
index 374933a1d57..b94800ee0be 100644
--- a/dbms/src/Parsers/ASTSelectQuery.h
+++ b/dbms/src/Parsers/ASTSelectQuery.h
@@ -25,7 +25,8 @@ public:
         GROUP_BY,
         HAVING,
         ORDER_BY,
-        LIMIT_BY_VALUE,
+        LIMIT_BY_OFFSET,
+        LIMIT_BY_LENGTH,
         LIMIT_BY,
         LIMIT_OFFSET,
         LIMIT_LENGTH,
@@ -56,7 +57,8 @@ public:
     const ASTPtr groupBy()        const { return getExpression(Expression::GROUP_BY); }
     const ASTPtr having()         const { return getExpression(Expression::HAVING); }
     const ASTPtr orderBy()        const { return getExpression(Expression::ORDER_BY); }
-    const ASTPtr limitByValue()   const { return getExpression(Expression::LIMIT_BY_VALUE); }
+    const ASTPtr limitByOffset()  const { return getExpression(Expression::LIMIT_BY_OFFSET); }
+    const ASTPtr limitByLength()  const { return getExpression(Expression::LIMIT_BY_LENGTH); }
     const ASTPtr limitBy()        const { return getExpression(Expression::LIMIT_BY); }
     const ASTPtr limitOffset()    const { return getExpression(Expression::LIMIT_OFFSET); }
     const ASTPtr limitLength()    const { return getExpression(Expression::LIMIT_LENGTH); }
diff --git a/dbms/src/Parsers/ParserSelectQuery.cpp b/dbms/src/Parsers/ParserSelectQuery.cpp
index c521ba4ec40..644d63788cb 100644
--- a/dbms/src/Parsers/ParserSelectQuery.cpp
+++ b/dbms/src/Parsers/ParserSelectQuery.cpp
@@ -60,7 +60,8 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
     ASTPtr group_expression_list;
     ASTPtr having_expression;
     ASTPtr order_expression_list;
-    ASTPtr limit_by_value;
+    ASTPtr limit_by_length;
+    ASTPtr limit_by_offset;
     ASTPtr limit_by_expression_list;
     ASTPtr limit_offset;
     ASTPtr limit_length;
@@ -180,7 +181,7 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
             return false;
     }
 
-    /// LIMIT length | LIMIT offset, length | LIMIT count BY expr-list
+    /// LIMIT length | LIMIT offset, length | LIMIT count BY expr-list | LIMIT offset, length BY expr-list
     if (s_limit.ignore(pos, expected))
     {
         if (limit_length)
@@ -197,25 +198,27 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
             if (!exp_elem.parse(pos, limit_length, expected))
                 return false;
         }
-        else if (s_by.ignore(pos, expected))
-        {
-            limit_by_value = limit_length;
-            limit_length = nullptr;
-
-            if (!exp_list.parse(pos, limit_by_expression_list, expected))
-                return false;
-        }
         else if (s_offset.ignore(pos, expected))
         {
             if (!exp_elem.parse(pos, limit_offset, expected))
                 return false;
         }
+        if (s_by.ignore(pos, expected))
+        {
+            limit_by_length = limit_length;
+            limit_by_offset = limit_offset;
+            limit_length = nullptr;
+            limit_offset = nullptr;
+
+            if (!exp_list.parse(pos, limit_by_expression_list, expected))
+                return false;
+        }
     }
 
     /// LIMIT length | LIMIT offset, length
     if (s_limit.ignore(pos, expected))
     {
-        if (!limit_by_value || limit_length)
+        if (!limit_by_length|| limit_length)
             return false;
 
         ParserToken s_comma(TokenType::Comma);
@@ -229,6 +232,11 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
             if (!exp_elem.parse(pos, limit_length, expected))
                 return false;
         }
+        else if (s_offset.ignore(pos, expected))
+        {
+            if (!exp_elem.parse(pos, limit_offset, expected))
+                return false;
+        }
     }
 
     /// SETTINGS key1 = value1, key2 = value2, ...
@@ -248,7 +256,8 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
     select_query->setExpression(ASTSelectQuery::Expression::GROUP_BY, std::move(group_expression_list));
     select_query->setExpression(ASTSelectQuery::Expression::HAVING, std::move(having_expression));
     select_query->setExpression(ASTSelectQuery::Expression::ORDER_BY, std::move(order_expression_list));
-    select_query->setExpression(ASTSelectQuery::Expression::LIMIT_BY_VALUE, std::move(limit_by_value));
+    select_query->setExpression(ASTSelectQuery::Expression::LIMIT_BY_OFFSET, std::move(limit_by_offset));
+    select_query->setExpression(ASTSelectQuery::Expression::LIMIT_BY_LENGTH, std::move(limit_by_length));
     select_query->setExpression(ASTSelectQuery::Expression::LIMIT_BY, std::move(limit_by_expression_list));
     select_query->setExpression(ASTSelectQuery::Expression::LIMIT_OFFSET, std::move(limit_offset));
     select_query->setExpression(ASTSelectQuery::Expression::LIMIT_LENGTH, std::move(limit_length));
diff --git a/dbms/tests/queries/0_stateless/00939_limit_by_offset.reference b/dbms/tests/queries/0_stateless/00939_limit_by_offset.reference
new file mode 100644
index 00000000000..982ff0b863a
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00939_limit_by_offset.reference
@@ -0,0 +1,13 @@
+1	120
+1	130
+2	220
+1	110
+1	120
+2	210
+2	220
+1	110
+1	120
+2	210
+1	120
+2	210
+2	220
diff --git a/dbms/tests/queries/0_stateless/00939_limit_by_offset.sql b/dbms/tests/queries/0_stateless/00939_limit_by_offset.sql
new file mode 100644
index 00000000000..30cb1276486
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00939_limit_by_offset.sql
@@ -0,0 +1,10 @@
+drop table if exists test.limit_by;
+
+create table test.limit_by(id Int, val Int) engine = Memory;
+
+insert into test.limit_by values(1, 100), (1, 110), (1, 120), (1, 130), (2, 200), (2, 210), (2, 220), (3, 300);
+
+select * from test.limit_by order by id, val limit 2, 2 by id;
+select * from test.limit_by order by id, val limit 2 offset 1 by id;
+select * from test.limit_by order by id, val limit 1, 2 by id limit 3;
+select * from test.limit_by order by id, val limit 1, 2 by id limit 3 offset 1;

From 9a1aee4c2ebb802f83b86a4db2dcd953afe14d46 Mon Sep 17 00:00:00 2001
From: Ivan Lezhankin <ilezhankin@yandex-team.ru>
Date: Mon, 29 Apr 2019 16:37:38 +0300
Subject: [PATCH 092/118] Remove move-assignment to prevent possible self-move

---
 dbms/src/Storages/ColumnsDescription.h  | 1 -
 dbms/src/Storages/ITableDeclaration.cpp | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/dbms/src/Storages/ColumnsDescription.h b/dbms/src/Storages/ColumnsDescription.h
index 0ff98473c99..3712178156c 100644
--- a/dbms/src/Storages/ColumnsDescription.h
+++ b/dbms/src/Storages/ColumnsDescription.h
@@ -38,7 +38,6 @@ public:
     ColumnsDescription(const ColumnsDescription & other);
     ColumnsDescription & operator=(const ColumnsDescription & other);
     ColumnsDescription(ColumnsDescription &&) noexcept = default;
-    ColumnsDescription & operator=(ColumnsDescription &&) noexcept = default;
 
     /// `after_column` can be a Nested column name;
     void add(ColumnDescription column, const String & after_column = String());
diff --git a/dbms/src/Storages/ITableDeclaration.cpp b/dbms/src/Storages/ITableDeclaration.cpp
index c9385c24cbe..ec62b0a568a 100644
--- a/dbms/src/Storages/ITableDeclaration.cpp
+++ b/dbms/src/Storages/ITableDeclaration.cpp
@@ -28,7 +28,7 @@ void ITableDeclaration::setColumns(ColumnsDescription columns_)
 {
     if (columns_.getOrdinary().empty())
         throw Exception("Empty list of columns passed", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED);
-    columns = std::move(columns_);
+    columns = columns_;
 }
 
 void ITableDeclaration::setIndicesDescription(IndicesDescription indices_)

From 1eb835be354f43a3ea91dd536cef7722101ea2c7 Mon Sep 17 00:00:00 2001
From: Ivan <5627721+abyss7@users.noreply.github.com>
Date: Mon, 29 Apr 2019 17:22:04 +0300
Subject: [PATCH 093/118] Reduce maximum page size (#5140)

---
 utils/github/__main__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utils/github/__main__.py b/utils/github/__main__.py
index e5a5a89e8d4..836ad734fce 100644
--- a/utils/github/__main__.py
+++ b/utils/github/__main__.py
@@ -47,7 +47,7 @@ parser.add_argument('--login', type=str,
 
 args = parser.parse_args()
 
-github = query.Query(args.token)
+github = query.Query(args.token, 50)
 repo = local.Local(args.repo, args.remote, github.get_default_branch())
 
 stables = repo.get_stables()[-args.number:] # [(branch name, base)]

From 81d276a0a4d73b5b87665d51bbe93cd96f75f5fb Mon Sep 17 00:00:00 2001
From: never lee <neverlea@foxmail.com>
Date: Mon, 29 Apr 2019 23:08:11 +0800
Subject: [PATCH 094/118] translate
 docs/zh/operations/table_engines/custom_partitioning_key.md (#5134)

---
 .../table_engines/custom_partitioning_key.md  |  22 ++--
 .../table_engines/custom_partitioning_key.md  | 116 ++++++++++++++----
 2 files changed, 105 insertions(+), 33 deletions(-)

diff --git a/docs/en/operations/table_engines/custom_partitioning_key.md b/docs/en/operations/table_engines/custom_partitioning_key.md
index 7e57f9252fb..e53cf903aed 100644
--- a/docs/en/operations/table_engines/custom_partitioning_key.md
+++ b/docs/en/operations/table_engines/custom_partitioning_key.md
@@ -9,8 +9,8 @@ The partition is specified in the `PARTITION BY expr` clause when [creating a ta
 ``` sql
 CREATE TABLE visits
 (
-    VisitDate Date, 
-    Hour UInt8, 
+    VisitDate Date,
+    Hour UInt8,
     ClientID UUID
 )
 ENGINE = MergeTree()
@@ -36,11 +36,11 @@ When inserting new data to a table, this data is stored as a separate part (chun
 Use the [system.parts](../system_tables.md#system_tables-parts) table to view the table parts and partitions. For example, let's assume that we have a `visits` table with partitioning by month. Let's perform the `SELECT` query for the `system.parts` table:
 
 ``` sql
-SELECT 
+SELECT
     partition,
-    name, 
+    name,
     active
-FROM system.parts 
+FROM system.parts
 WHERE table = 'visits'
 ```
 
@@ -59,7 +59,7 @@ WHERE table = 'visits'
 The `partition` column contains the names of the partitions. There are two partitions in this example: `201901` and `201902`. You can use this column value to specify the partition name in [ALTER ... PARTITION](#alter_manipulations-with-partitions) queries.
 
 The `name` column contains the names of the partition data parts. You can use this column to specify the name of the part in the [ALTER ATTACH PART](#alter_attach-partition) query.
- 
+
 Let's break down the name of the first part: `201901_1_3_1`:
 
 - `201901` is the partition name.
@@ -91,9 +91,9 @@ OPTIMIZE TABLE visits PARTITION 201902;
 └───────────┴────────────────┴────────┘
 ```
 
-Inactive parts will be deleted approximately 10 minutes after merging. 
- 
-Another way to view a set of parts and partitions is to go into the directory of the table: `/var/lib/clickhouse/data/<database>/<table>/`. For example: 
+Inactive parts will be deleted approximately 10 minutes after merging.
+
+Another way to view a set of parts and partitions is to go into the directory of the table: `/var/lib/clickhouse/data/<database>/<table>/`. For example:
 
 ```bash
 dev:/var/lib/clickhouse/data/default/visits$ ls -l
@@ -114,7 +114,7 @@ The folders '201901_1_1_0', '201901_1_7_1' and so on are the directories of the
 The `detached` directory contains parts that were detached from the table using the [DETACH](#alter_detach-partition) query. The corrupted parts are also moved to this directory, instead of being deleted. The server does not use the parts from the `detached` directory. You can add, delete, or modify the data in this directory at any time – the server will not know about this until you run the [ATTACH](../../query_language/alter.md#alter_attach-partition) query.
 
 Note that on the operating server, you cannot manually change the set of parts or their data on the file system, since the server will not know about it. For non-replicated tables, you can do this when the server is stopped, but it isn't recommended. For replicated tables, the set of parts cannot be changed in any case.
-    
-ClickHouse allows you to perform operations with the partitions: delete them, copy from one table to another, or create a backup. See the list of all operations in the section [Manipulations With Partitions and Parts](../../query_language/alter.md#alter_manipulations-with-partitions). 
+
+ClickHouse allows you to perform operations with the partitions: delete them, copy from one table to another, or create a backup. See the list of all operations in the section [Manipulations With Partitions and Parts](../../query_language/alter.md#alter_manipulations-with-partitions).
 
 [Original article](https://clickhouse.yandex/docs/en/operations/table_engines/custom_partitioning_key/) <!--hide-->
diff --git a/docs/zh/operations/table_engines/custom_partitioning_key.md b/docs/zh/operations/table_engines/custom_partitioning_key.md
index 9e723efd42e..8ee25c19944 100644
--- a/docs/zh/operations/table_engines/custom_partitioning_key.md
+++ b/docs/zh/operations/table_engines/custom_partitioning_key.md
@@ -1,48 +1,120 @@
+# 自定义分区键
 
-# Custom Partitioning Key
+[MergeTree](mergetree.md) 系列的表（包括 [可复制表](replication.md) ）可以使用分区。基于 MergeTree 表的 [物化视图](materializedview.md) 也支持分区。
 
-Starting with version 1.1.54310, you can create tables in the MergeTree family with any partitioning expression (not only partitioning by month).
+一个分区是指按指定规则逻辑组合一起的表的记录集。可以按任意标准进行分区，如按月，按日或按事件类型。为了减少需要操作的数据，每个分区都是分开存储的。访问数据时，ClickHouse 尽量使用这些分区的最小子集。
 
-The partition key can be an expression from the table columns, or a tuple of such expressions (similar to the primary key). The partition key can be omitted. When creating a table, specify the partition key in the ENGINE description with the new syntax:
+分区是在 [建表](mergetree.md#table_engine-mergetree-creating-a-table) 的 `PARTITION BY expr` 子句中指定。分区键可以是关于列的任何表达式。例如，指定按月分区，表达式为 `toYYYYMM(date_column)`：
 
-```
-ENGINE [=] Name(...) [PARTITION BY expr] [ORDER BY expr] [SAMPLE BY expr] [SETTINGS name=value, ...]
+``` sql
+CREATE TABLE visits
+(
+    VisitDate Date, 
+    Hour UInt8, 
+    ClientID UUID
+)
+ENGINE = MergeTree()
+PARTITION BY toYYYYMM(VisitDate)
+ORDER BY Hour;
 ```
 
-For MergeTree tables, the partition expression is specified after `PARTITION BY`, the primary key after `ORDER BY`, the sampling key after `SAMPLE BY`, and `SETTINGS` can specify `index_granularity` (optional; the default value is 8192), as well as other settings from [MergeTreeSettings.h](https://github.com/yandex/ClickHouse/blob/master/dbms/src/Storages/MergeTree/MergeTreeSettings.h). The other engine parameters are specified in parentheses after the engine name, as previously. Example:
+分区键也可以是表达式元组（类似 [主键](mergetree.md#primary-keys-and-indexes-in-queries) ）。例如：
 
 ``` sql
 ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/tables/name', 'replica1', Sign)
-    PARTITION BY (toMonday(StartDate), EventType)
-    ORDER BY (CounterID, StartDate, intHash32(UserID))
-    SAMPLE BY intHash32(UserID)
+PARTITION BY (toMonday(StartDate), EventType)
+ORDER BY (CounterID, StartDate, intHash32(UserID));
 ```
 
-The traditional partitioning by month is expressed as `toYYYYMM(date_column)`.
+上例中，我们设置按一周内的事件类型分区。
 
-You can't convert an old-style table to a table with custom partitions (only via INSERT SELECT).
+新数据插入到表中时，这些数据会存储为按主键排序的新片段（块）。插入后 10-15 分钟，同一分区的各个片段会合并为一整个片段。
 
-After this table is created, merge will only work for data parts that have the same value for the partitioning expression. Note: This means that you shouldn't make overly granular partitions (more than about a thousand partitions), or SELECT will perform poorly.
+!!! 注意
+    那些有相同分区表达式值的数据片段才会合并。这意味着 **你不应该用太精细的分区方案**（超过一千个分区）。否则，会因为文件系统中的文件数量和需要找开的文件描述符过多，导致 `SELECT` 查询效率不佳。
 
-To specify a partition in ALTER PARTITION commands, specify the value of the partition expression (or a tuple). Constants and constant expressions are supported. Example:
+可以通过 [system.parts](../system_tables.md#system_tables-parts) 表查看表片段和分区信息。例如，假设我们有一个 `visits` 表，按月分区。对 `system.parts` 表执行 `SELECT`：
 
 ``` sql
-ALTER TABLE table DROP PARTITION (toMonday(today()), 1)
+SELECT 
+    partition,
+    name, 
+    active
+FROM system.parts 
+WHERE table = 'visits'
 ```
 
-Deletes the partition for the current week with event type 1. The same is true for the OPTIMIZE query. To specify the only partition in a non-partitioned table, specify `PARTITION tuple()`.
+```
+┌─partition─┬─name───────────┬─active─┐
+│ 201901    │ 201901_1_3_1   │      0 │
+│ 201901    │ 201901_1_9_2   │      1 │
+│ 201901    │ 201901_8_8_0   │      0 │
+│ 201901    │ 201901_9_9_0   │      0 │
+│ 201902    │ 201902_4_6_1   │      1 │
+│ 201902    │ 201902_10_10_0 │      1 │
+│ 201902    │ 201902_11_11_0 │      1 │
+└───────────┴────────────────┴────────┘
+```
 
-Note: For old-style tables, the partition can be specified either as a number `201710` or a string `'201710'`. The syntax for the new style of tables is stricter with types (similar to the parser for the VALUES input format). In addition, ALTER TABLE FREEZE PARTITION uses exact match for new-style tables (not prefix match).
+`partition` 列存储分区的名称。此示例中有两个分区：`201901` 和 `201902`。在 [ALTER ... PARTITION](#alter_manipulations-with-partitions) 语句中你可以使用该列值来指定分区名称。
 
-In the `system.parts` table, the `partition` column specifies the value of the partition expression to use in ALTER queries (if quotas are removed). The `name` column should specify the name of the data part that has a new format.
+`name` 列为分区中数据片段的名称。在 [ALTER ATTACH PART](#alter_attach-partition) 语句中你可以使用此列值中来指定片段名称。
 
-Old: `20140317_20140323_2_2_0` (minimum date - maximum date - minimum block number - maximum block number - level).
+这里我们拆解下第一部分的名称：`201901_1_3_1`：
 
-Now: `201403_2_2_0` (partition ID - minimum block number - maximum block number - level).
+- `201901` 是分区名称。
+- `1` 是数据块的最小编号。
+- `3` 是数据块的最大编号。
+- `1` 是块级别（即在由块组成的合并树中，该块在树中的深度）。
 
-The partition ID is its string identifier (human-readable, if possible) that is used for the names of data parts in the file system and in ZooKeeper. You can specify it in ALTER queries in place of the partition key. Example: Partition key `toYYYYMM(EventDate)`; ALTER can specify either `PARTITION 201710` or `PARTITION ID '201710'`.
+!!! 注意
+    旧类型表的片段名称为：`20190117_20190123_2_2_0`（最小日期 - 最大日期 - 最小块编号 - 最大块编号 - 块级别）。
 
-For more examples, see the tests [`00502_custom_partitioning_local`](https://github.com/yandex/ClickHouse/blob/master/dbms/tests/queries/0_stateless/00502_custom_partitioning_local.sql) and [`00502_custom_partitioning_replicated_zookeeper`](https://github.com/yandex/ClickHouse/blob/master/dbms/tests/queries/0_stateless/00502_custom_partitioning_replicated_zookeeper.sql).
+`active` 列为片段状态。`1` 激活状态；`0` 非激活状态。非激活片段是那些在合并到较大片段之后剩余的源数据片段。损坏的数据片段也表示为非活动状态。
 
+正如在示例中所看到的，同一分区中有几个独立的片段（例如，`201901_1_3_1`和`201901_1_9_2`）。这意味着这些片段尚未合并。ClickHouse 大约在插入后15分钟定期报告合并操作，合并插入的数据片段。此外，你也可以使用 [OPTIMIZE](../../query_language/misc.md#misc_operations-optimize) 语句直接执行合并。例：
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/custom_partitioning_key/) <!--hide-->
+``` sql
+OPTIMIZE TABLE visits PARTITION 201902;
+```
+
+```
+┌─partition─┬─name───────────┬─active─┐
+│ 201901    │ 201901_1_3_1   │      0 │
+│ 201901    │ 201901_1_9_2   │      1 │
+│ 201901    │ 201901_8_8_0   │      0 │
+│ 201901    │ 201901_9_9_0   │      0 │
+│ 201902    │ 201902_4_6_1   │      0 │
+│ 201902    │ 201902_4_11_2  │      1 │
+│ 201902    │ 201902_10_10_0 │      0 │
+│ 201902    │ 201902_11_11_0 │      0 │
+└───────────┴────────────────┴────────┘
+```
+
+非激活片段会在合并后的10分钟左右删除。
+
+查看片段和分区信息的另一种方法是进入表的目录：`/var/lib/clickhouse/data/<database>/<table>/`。例如：
+
+```bash
+dev:/var/lib/clickhouse/data/default/visits$ ls -l
+total 40
+drwxr-xr-x 2 clickhouse clickhouse 4096 Feb  1 16:48 201901_1_3_1
+drwxr-xr-x 2 clickhouse clickhouse 4096 Feb  5 16:17 201901_1_9_2
+drwxr-xr-x 2 clickhouse clickhouse 4096 Feb  5 15:52 201901_8_8_0
+drwxr-xr-x 2 clickhouse clickhouse 4096 Feb  5 15:52 201901_9_9_0
+drwxr-xr-x 2 clickhouse clickhouse 4096 Feb  5 16:17 201902_10_10_0
+drwxr-xr-x 2 clickhouse clickhouse 4096 Feb  5 16:17 201902_11_11_0
+drwxr-xr-x 2 clickhouse clickhouse 4096 Feb  5 16:19 201902_4_11_2
+drwxr-xr-x 2 clickhouse clickhouse 4096 Feb  5 12:09 201902_4_6_1
+drwxr-xr-x 2 clickhouse clickhouse 4096 Feb  1 16:48 detached
+```
+
+文件夹 '201901_1_1_0'，'201901_1_7_1' 等是片段的目录。每个片段都与一个对应的分区相关，并且只包含这个月的数据（本例中的表按月分区）。
+
+`detached` 目录存放着使用 [DETACH](#alter_detach-partition) 语句从表中分离的片段。损坏的片段也会移到该目录，而不是删除。服务器不使用`detached`目录中的片段。可以随时添加，删除或修改此目录中的数据 – 在运行 [ATTACH](../../query_language/alter.md#alter_attach-partition) 语句前，服务器不会感知到。
+
+注意，在操作服务器时，你不能手动更改文件系统上的片段集或其数据，因为服务器不会感知到这些修改。对于非复制表，可以在服务器停止时执行这些操作，但不建议这样做。对于复制表，在任何情况下都不要更改片段文件。
+
+ClickHouse 支持对分区执行这些操作：删除分区，从一个表复制到另一个表，或创建备份。了解分区的所有操作，请参阅 [分区和片段的操作](../../query_language/alter.md#alter_manipulations-with-partitions) 一节。
+
+[来源文章](https://clickhouse.yandex/docs/en/operations/table_engines/custom_partitioning_key/) <!--hide-->

From 2cf0ab341ecec546022a61f9706210221d135cb9 Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Mon, 29 Apr 2019 19:30:39 +0300
Subject: [PATCH 095/118] Update gui.md

---
 docs/ru/interfaces/third-party/gui.md | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/docs/ru/interfaces/third-party/gui.md b/docs/ru/interfaces/third-party/gui.md
index 9eddc4461f0..d561f54548c 100644
--- a/docs/ru/interfaces/third-party/gui.md
+++ b/docs/ru/interfaces/third-party/gui.md
@@ -51,6 +51,17 @@
 - Предварительный просмотр таблицы с фильтрацией и сортировкой;
 - Выполнение запросов только для чтения.
 
+### DBeaver
+
+[DBeaver](https://dbeaver.io/) - универсальный desktop клиент баз данных с поддержкой ClickHouse.
+
+Основные возможности:
+
+- Построение запросов с подсветкой синтаксиса;
+- Просмотр таблиц;
+- Автодополнение команд;
+- Полнотекстовый поиск.
+
 ### clickhouse-cli
 
 [clickhouse-cli](https://github.com/hatarist/clickhouse-cli) - это альтернативный клиент командной строки для ClickHouse, написанный на Python 3.
@@ -63,16 +74,6 @@
 
 ## Коммерческие
 
-### DBeaver
-
-[DBeaver](https://dbeaver.io/) - универсальный desktop клиент баз данных с поддержкой ClickHouse.
-
-Основные возможности:
-
-- Построение запросов с подсветкой синтаксиса;
-- Просмотр таблиц;
-- Автодополнение команд.
-
 ### DataGrip
 
 [DataGrip](https://www.jetbrains.com/datagrip/) — это IDE для баз данных о JetBrains с выделенной поддержкой ClickHouse. Он также встроен в другие инструменты на основе IntelliJ: PyCharm, IntelliJ IDEA, GoLand, PhpStorm и другие.

From 3c8dce62a4d6553e181ae3bd9b00fe441ddac062 Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Mon, 29 Apr 2019 19:31:44 +0300
Subject: [PATCH 096/118] Update gui.md

---
 docs/zh/interfaces/third-party/gui.md | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/docs/zh/interfaces/third-party/gui.md b/docs/zh/interfaces/third-party/gui.md
index 35e4717a497..457ae3af902 100644
--- a/docs/zh/interfaces/third-party/gui.md
+++ b/docs/zh/interfaces/third-party/gui.md
@@ -47,6 +47,16 @@ ClickHouse Web 界面 [Tabix](https://github.com/tabixio/tabix).
  - 包含过滤和元数据的表列表。
  - 带有过滤和排序的表格预览。
  - 只读查询执行。
+ 
+### DBeaver
+
+[DBeaver](https://dbeaver.io/) 具有ClickHouse支持的通用桌面数据库客户端。
+
+特征：
+
+ - 使用语法高亮显示查询开发。
+ - 表格预览。
+ - 自动完成。
 
 ### clickhouse-cli
 
@@ -60,16 +70,6 @@ ClickHouse Web 界面 [Tabix](https://github.com/tabixio/tabix).
 
 ## 商业
 
-### DBeaver
-
-[DBeaver](https://dbeaver.io/) 具有ClickHouse支持的通用桌面数据库客户端。
-
-特征：
-
- - 使用语法高亮显示查询开发。
- - 表格预览。
- - 自动完成。
-
 ### DataGrip
 
 [DataGrip](https://www.jetbrains.com/datagrip/) 是JetBrains的数据库IDE，专门支持ClickHouse。 它还嵌入到其他基于IntelliJ的工具中：PyCharm，IntelliJ IDEA，GoLand，PhpStorm等。

From f999100e562672e7be0d890e3b1897514fedd7ed Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Mon, 29 Apr 2019 19:31:56 +0300
Subject: [PATCH 097/118] Update gui.md

---
 docs/fa/interfaces/third-party/gui.md | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/docs/fa/interfaces/third-party/gui.md b/docs/fa/interfaces/third-party/gui.md
index bb9dcbe9c07..a02d8a3cfc5 100644
--- a/docs/fa/interfaces/third-party/gui.md
+++ b/docs/fa/interfaces/third-party/gui.md
@@ -49,6 +49,16 @@ interface تحت وب برای ClickHouse در پروژه [Tabix](https://github
 - پیش نمایش جدول با فیلتر کردن و مرتب سازی.
 - اعداد نمایش داده شده فقط خواندنی
 
+### DBeaver
+
+[DBeaver](https://dbeaver.io/) - مشتری دسکتاپ دسکتاپ دسکتاپ با پشتیبانی ClickHouse.
+
+امکانات:
+
+- توسعه پرس و جو با برجسته نحو
+- پیش نمایش جدول
+- تکمیل خودکار
+
 ### clickhouse-cli
 
 [clickhouse-cli](https://github.com/hatarist/clickhouse-cli) یک مشتری خط فرمان برای ClickHouse است که در پایتون 3 نوشته شده است.
@@ -61,16 +71,6 @@ interface تحت وب برای ClickHouse در پروژه [Tabix](https://github
 
 ## تجاری
 
-### DBeaver
-
-[DBeaver](https://dbeaver.io/) - مشتری دسکتاپ دسکتاپ دسکتاپ با پشتیبانی ClickHouse.
-
-امکانات:
-
-- توسعه پرس و جو با برجسته نحو
-- پیش نمایش جدول
-- تکمیل خودکار
-
 ### DataGrip
 
 [DataGrip](https://www.jetbrains.com/datagrip/) IDE پایگاه داده از JetBrains با پشتیبانی اختصاصی برای ClickHouse است. این ابزار همچنین به سایر ابزارهای مبتنی بر IntelliJ تعبیه شده است: PyCharm، IntelliJ IDEA، GoLand، PhpStorm و دیگران.

From 17b2def426140063356d968cb8aef869693cced1 Mon Sep 17 00:00:00 2001
From: Ivan Blinkov <github@blinkov.ru>
Date: Mon, 29 Apr 2019 19:48:23 +0300
Subject: [PATCH 098/118] [docs] a bit less confusing

---
 docs/tools/mkdocs-material-theme/partials/language/en.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/tools/mkdocs-material-theme/partials/language/en.html b/docs/tools/mkdocs-material-theme/partials/language/en.html
index bcd2debaac5..47e40c800bd 100644
--- a/docs/tools/mkdocs-material-theme/partials/language/en.html
+++ b/docs/tools/mkdocs-material-theme/partials/language/en.html
@@ -7,7 +7,7 @@
   "footer.next": "Next",
   "meta.comments": "Comments",
   "meta.source": "Source",
-  "nav.latest": "latest",
+  "nav.latest": "master",
   "nav.multi_page": "Multi page version",
   "nav.pdf": "PDF version",
   "nav.release": "Release",

From b1831a9ab52b9e386acf295643b307e584eeb31d Mon Sep 17 00:00:00 2001
From: Ivan Blinkov <github@blinkov.ru>
Date: Mon, 29 Apr 2019 19:50:02 +0300
Subject: [PATCH 099/118] Add link to horgh-replicator

---
 docs/en/interfaces/third-party/integrations.md | 1 +
 docs/fa/interfaces/third-party/integrations.md | 1 +
 docs/ru/interfaces/third-party/integrations.md | 1 +
 docs/zh/interfaces/third-party/integrations.md | 1 +
 4 files changed, 4 insertions(+)

diff --git a/docs/en/interfaces/third-party/integrations.md b/docs/en/interfaces/third-party/integrations.md
index 6533145c866..7c302884631 100644
--- a/docs/en/interfaces/third-party/integrations.md
+++ b/docs/en/interfaces/third-party/integrations.md
@@ -9,6 +9,7 @@
     - [MySQL](https://www.mysql.com)
         - [ProxySQL](https://github.com/sysown/proxysql/wiki/ClickHouse-Support)
         - [clickhouse-mysql-data-reader](https://github.com/Altinity/clickhouse-mysql-data-reader)
+        - [horgh-replicator](https://github.com/larsnovikov/horgh-replicator)
     - [PostgreSQL](https://www.postgresql.org)
         - [clickhousedb_fdw](https://github.com/Percona-Lab/clickhousedb_fdw)
         - [infi.clickhouse_fdw](https://github.com/Infinidat/infi.clickhouse_fdw) (uses [infi.clickhouse_orm](https://github.com/Infinidat/infi.clickhouse_orm))
diff --git a/docs/fa/interfaces/third-party/integrations.md b/docs/fa/interfaces/third-party/integrations.md
index b1d806f776e..e7def6bca58 100644
--- a/docs/fa/interfaces/third-party/integrations.md
+++ b/docs/fa/interfaces/third-party/integrations.md
@@ -11,6 +11,7 @@
     - [MySQL](https://www.mysql.com)
         - [ProxySQL](https://github.com/sysown/proxysql/wiki/ClickHouse-Support)
         - [clickhouse-mysql-data-reader](https://github.com/Altinity/clickhouse-mysql-data-reader)
+        - [horgh-replicator](https://github.com/larsnovikov/horgh-replicator)
     - [PostgreSQL](https://www.postgresql.org)
         - [clickhousedb_fdw](https://github.com/Percona-Lab/clickhousedb_fdw)
         - [infi.clickhouse_fdw](https://github.com/Infinidat/infi.clickhouse_fdw) (استفاده می کند [infi.clickhouse_orm](https://github.com/Infinidat/infi.clickhouse_orm))
diff --git a/docs/ru/interfaces/third-party/integrations.md b/docs/ru/interfaces/third-party/integrations.md
index 43d736c8e7c..a00239b1579 100644
--- a/docs/ru/interfaces/third-party/integrations.md
+++ b/docs/ru/interfaces/third-party/integrations.md
@@ -8,6 +8,7 @@
     - [MySQL](https://www.mysql.com)
         - [ProxySQL](https://github.com/sysown/proxysql/wiki/ClickHouse-Support)
         - [clickhouse-mysql-data-reader](https://github.com/Altinity/clickhouse-mysql-data-reader)
+        - [horgh-replicator](https://github.com/larsnovikov/horgh-replicator)
     - [PostgreSQL](https://www.postgresql.org)
         - [clickhousedb_fdw](https://github.com/Percona-Lab/clickhousedb_fdw)
         - [infi.clickhouse_fdw](https://github.com/Infinidat/infi.clickhouse_fdw) (использует [infi.clickhouse_orm](https://github.com/Infinidat/infi.clickhouse_orm))
diff --git a/docs/zh/interfaces/third-party/integrations.md b/docs/zh/interfaces/third-party/integrations.md
index fab2bdd07b0..3e9fcfcd410 100644
--- a/docs/zh/interfaces/third-party/integrations.md
+++ b/docs/zh/interfaces/third-party/integrations.md
@@ -8,6 +8,7 @@
     - [MySQL](https://www.mysql.com)
         - [ProxySQL](https://github.com/sysown/proxysql/wiki/ClickHouse-Support)
         - [clickhouse-mysql-data-reader](https://github.com/Altinity/clickhouse-mysql-data-reader)
+        - [horgh-replicator](https://github.com/larsnovikov/horgh-replicator)
     - [PostgreSQL](https://www.postgresql.org)
         - [clickhousedb_fdw](https://github.com/Percona-Lab/clickhousedb_fdw)
         - [infi.clickhouse_fdw](https://github.com/Infinidat/infi.clickhouse_fdw) (使用 [infi.clickhouse_orm](https://github.com/Infinidat/infi.clickhouse_orm))

From 436029bf64319350047e2e44391a40eb5f66e541 Mon Sep 17 00:00:00 2001
From: Danila Kutenin <kutdanila@yandex.ru>
Date: Mon, 29 Apr 2019 22:30:14 +0300
Subject: [PATCH 100/118] No AVX but remain wildcopy 32

---
 .../src/Compression/LZ4_decompress_faster.cpp | 93 +++++++------------
 dbms/src/Compression/LZ4_decompress_faster.h  | 10 +-
 2 files changed, 37 insertions(+), 66 deletions(-)

diff --git a/dbms/src/Compression/LZ4_decompress_faster.cpp b/dbms/src/Compression/LZ4_decompress_faster.cpp
index 727ba5e05ef..51ce1bfa7b7 100644
--- a/dbms/src/Compression/LZ4_decompress_faster.cpp
+++ b/dbms/src/Compression/LZ4_decompress_faster.cpp
@@ -20,10 +20,6 @@
 #include <tmmintrin.h>
 #endif
 
-#ifdef __AVX__
-#include <immintrin.h>
-#endif
-
 #ifdef __aarch64__
 #include <arm_neon.h>
 #endif
@@ -34,8 +30,8 @@ namespace LZ4
 namespace
 {
 
-template <size_t N, bool USE_AVX> [[maybe_unused]] void copy(UInt8 * dst, const UInt8 * src);
-template <size_t N, bool USE_AVX> [[maybe_unused]] void wildCopy(UInt8 * dst, const UInt8 * src, UInt8 * dst_end);
+template <size_t N> [[maybe_unused]] void copy(UInt8 * dst, const UInt8 * src);
+template <size_t N> [[maybe_unused]] void wildCopy(UInt8 * dst, const UInt8 * src, UInt8 * dst_end);
 template <size_t N, bool USE_SHUFFLE> [[maybe_unused]] void copyOverlap(UInt8 * op, const UInt8 *& match, const size_t offset);
 
 
@@ -46,6 +42,7 @@ inline void copy8(UInt8 * dst, const UInt8 * src)
 
 inline void wildCopy8(UInt8 * dst, const UInt8 * src, UInt8 * dst_end)
 {
+    /// Unrolling with clang doing >10% performance degrade.
 #if defined(__clang__)
     #pragma nounroll
 #endif
@@ -213,10 +210,8 @@ inline void copyOverlap8Shuffle(UInt8 * op, const UInt8 *& match, const size_t o
 
 
 
-template <> void inline copy<8, false>(UInt8 * dst, const UInt8 * src) { copy8(dst, src); }
-template <> void inline copy<8, true>(UInt8 * dst, const UInt8 * src) { copy8(dst, src); }
-template <> void inline wildCopy<8, false>(UInt8 * dst, const UInt8 * src, UInt8 * dst_end) { wildCopy8(dst, src, dst_end); }
-template <> void inline wildCopy<8, true>(UInt8 * dst, const UInt8 * src, UInt8 * dst_end) { wildCopy8(dst, src, dst_end); }
+template <> void inline copy<8>(UInt8 * dst, const UInt8 * src) { copy8(dst, src); }
+template <> void inline wildCopy<8>(UInt8 * dst, const UInt8 * src, UInt8 * dst_end) { wildCopy8(dst, src, dst_end); }
 template <> void inline copyOverlap<8, false>(UInt8 * op, const UInt8 *& match, const size_t offset) { copyOverlap8(op, match, offset); }
 template <> void inline copyOverlap<8, true>(UInt8 * op, const UInt8 *& match, const size_t offset) { copyOverlap8Shuffle(op, match, offset); }
 
@@ -231,10 +226,9 @@ inline void copy16(UInt8 * dst, const UInt8 * src)
 #endif
 }
 
-
-
 inline void wildCopy16(UInt8 * dst, const UInt8 * src, UInt8 * dst_end)
 {
+    /// Unrolling with clang doing >10% performance degrade.
 #if defined(__clang__)
     #pragma nounroll
 #endif
@@ -349,52 +343,35 @@ inline void copyOverlap16Shuffle(UInt8 * op, const UInt8 *& match, const size_t
 #endif
 
 
-template <> void inline copy<16, false>(UInt8 * dst, const UInt8 * src) { copy16(dst, src); }
-template <> void inline copy<16, true>(UInt8 * dst, const UInt8 * src) { copy16(dst, src); }
-template <> void inline wildCopy<16, false>(UInt8 * dst, const UInt8 * src, UInt8 * dst_end) { wildCopy16(dst, src, dst_end); }
-template <> void inline wildCopy<16, true>(UInt8 * dst, const UInt8 * src, UInt8 * dst_end) { wildCopy16(dst, src, dst_end); }
+template <> void inline copy<16>(UInt8 * dst, const UInt8 * src) { copy16(dst, src); }
+template <> void inline wildCopy<16>(UInt8 * dst, const UInt8 * src, UInt8 * dst_end) { wildCopy16(dst, src, dst_end); }
 template <> void inline copyOverlap<16, false>(UInt8 * op, const UInt8 *& match, const size_t offset) { copyOverlap16(op, match, offset); }
 template <> void inline copyOverlap<16, true>(UInt8 * op, const UInt8 *& match, const size_t offset) { copyOverlap16Shuffle(op, match, offset); }
 
 
-template <bool use_avx>
 inline void copy32(UInt8 * dst, const UInt8 * src)
 {
-    (void)use_avx;
-#ifdef __AVX__
-    _mm256_storeu_si256(reinterpret_cast<__m256i *>(dst),
-            _mm256_loadu_si256(reinterpret_cast<const __m256i *>(src)));
-#elif defined(__SSE2__)
-    if constexpr (use_avx)
-    {
-        __asm__ volatile(
-            "vmovups (%1), %%ymm0\n\t"
-            "vmovups %%ymm0, (%0)"
-            :  /* output */
-            : "r"(dst), "r"(src) /* input */
-            : "%ymm0"); /* clobber_registers */
-    }
-    else
-    {
-        _mm_storeu_si128(reinterpret_cast<__m128i *>(dst),
-            _mm_loadu_si128(reinterpret_cast<const __m128i *>(src)));
-        _mm_storeu_si128(reinterpret_cast<__m128i *>(dst + 16),
-            _mm_loadu_si128(reinterpret_cast<const __m128i *>(src + 16)));
-    }
+    /// There was an AVX here but with mash with SSE instructions, we got a big slowdown.
+#if defined(__SSE2__)
+    _mm_storeu_si128(reinterpret_cast<__m128i *>(dst),
+        _mm_loadu_si128(reinterpret_cast<const __m128i *>(src)));
+    _mm_storeu_si128(reinterpret_cast<__m128i *>(dst + 16),
+        _mm_loadu_si128(reinterpret_cast<const __m128i *>(src + 16)));
 #else
-    memcpy(dst, src, 32);
+    memcpy(dst, src, 16);
+    memcpy(dst + 16, src + 16, 16);
 #endif
 }
 
-template <bool use_avx>
 inline void wildCopy32(UInt8 * dst, const UInt8 * src, UInt8 * dst_end)
 {
+    /// Unrolling with clang doing >10% performance degrade.
 #if defined(__clang__)
     #pragma nounroll
 #endif
     do
     {
-        copy32<use_avx>(dst, src);
+        copy32(dst, src);
         dst += 32;
         src += 32;
     } while (dst < dst_end);
@@ -432,16 +409,13 @@ inline void copyOverlap32(UInt8 * op, const UInt8 *& match, const size_t offset)
     match += shift4[offset];
 }
 
-template <> void inline copy<32, false>(UInt8 * dst, const UInt8 * src) { copy32<false>(dst, src); }
-template <> void inline copy<32, true>(UInt8 * dst, const UInt8 * src) { copy32<true>(dst, src); }
-template <> void inline wildCopy<32, false>(UInt8 * dst, const UInt8 * src, UInt8 * dst_end) { wildCopy32<false>(dst, src, dst_end); }
-template <> void inline wildCopy<32, true>(UInt8 * dst, const UInt8 * src, UInt8 * dst_end) { wildCopy32<true>(dst, src, dst_end); }
+template <> void inline copy<32>(UInt8 * dst, const UInt8 * src) { copy32(dst, src); }
+template <> void inline wildCopy<32>(UInt8 * dst, const UInt8 * src, UInt8 * dst_end) { wildCopy32(dst, src, dst_end); }
 template <> void inline copyOverlap<32, false>(UInt8 * op, const UInt8 *& match, const size_t offset) { copyOverlap32(op, match, offset); }
 
 /// See also https://stackoverflow.com/a/30669632
 
-/// NO_INLINE is vital for hiding avx instructions
-template <size_t copy_amount, bool use_shuffle, bool use_avx>
+template <size_t copy_amount, bool use_shuffle>
 void NO_INLINE decompressImpl(
      const char * const source,
      char * const dest,
@@ -451,6 +425,7 @@ void NO_INLINE decompressImpl(
     UInt8 * op = reinterpret_cast<UInt8 *>(dest);
     UInt8 * const output_end = op + dest_size;
 
+    /// Unrolling with clang doing >10% performance degrade.
 #if defined(__clang__)
     #pragma nounroll
 #endif
@@ -490,7 +465,7 @@ void NO_INLINE decompressImpl(
         /// output: xyzHello, w
         ///                  ^-op (we will overwrite excessive bytes on next iteration)
 
-        wildCopy<copy_amount, use_avx>(op, ip, copy_end);    /// Here we can write up to copy_amount - 1 bytes after buffer.
+        wildCopy<copy_amount>(op, ip, copy_end);    /// Here we can write up to copy_amount - 1 bytes after buffer.
 
         ip += length;
         op = copy_end;
@@ -536,15 +511,15 @@ void NO_INLINE decompressImpl(
         }
         else
         {
-            copy<copy_amount, use_avx>(op, match);
+            copy<copy_amount>(op, match);
             match += copy_amount;
         }
 
         op += copy_amount;
 
-        copy<copy_amount, use_avx>(op, match);   /// copy_amount + copy_amount - 1 - 4 * 2 bytes after buffer.
+        copy<copy_amount>(op, match);   /// copy_amount + copy_amount - 1 - 4 * 2 bytes after buffer.
         if (length > copy_amount * 2)
-            wildCopy<copy_amount, use_avx>(op + copy_amount, match + copy_amount, copy_end);
+            wildCopy<copy_amount>(op + copy_amount, match + copy_amount, copy_end);
 
         op = copy_end;
     }
@@ -567,23 +542,19 @@ void decompress(
     /// Don't run timer if the block is too small.
     if (dest_size >= 32768)
     {
-        const bool have_AVX = DB::Cpu::CpuFlagsCache::have_AVX;
-        size_t best_variant = statistics.select(PerformanceStatistics::NUM_ELEMENTS - !have_AVX);
+        size_t best_variant = statistics.select();
 
         /// Run the selected method and measure time.
 
         Stopwatch watch;
         if (best_variant == 0)
-            decompressImpl<16, true, false>(source, dest, dest_size);
+            decompressImpl<16, true>(source, dest, dest_size);
         if (best_variant == 1)
-            decompressImpl<16, false, false>(source, dest, dest_size);
+            decompressImpl<16, false>(source, dest, dest_size);
         if (best_variant == 2)
-            decompressImpl<8, true, false>(source, dest, dest_size);
+            decompressImpl<8, true>(source, dest, dest_size);
         if (best_variant == 3)
-            decompressImpl<32, false, false>(source, dest, dest_size);
-        /// If we don't have avx, we just never be here becase we select from NUM_EMELENTS minus one, be careful to add new methods
-        if (best_variant == 4)
-            decompressImpl<32, false, true>(source, dest, dest_size);
+            decompressImpl<32, false>(source, dest, dest_size);
 
         watch.stop();
 
@@ -593,7 +564,7 @@ void decompress(
     }
     else
     {
-        decompressImpl<8, false, false>(source, dest, dest_size);
+        decompressImpl<8, false>(source, dest, dest_size);
     }
 }
 
diff --git a/dbms/src/Compression/LZ4_decompress_faster.h b/dbms/src/Compression/LZ4_decompress_faster.h
index e83abc945e8..68a55c04581 100644
--- a/dbms/src/Compression/LZ4_decompress_faster.h
+++ b/dbms/src/Compression/LZ4_decompress_faster.h
@@ -88,14 +88,14 @@ struct PerformanceStatistics
     };
 
     /// Number of different algorithms to select from.
-    static constexpr size_t NUM_ELEMENTS = 5;
+    static constexpr size_t NUM_ELEMENTS = 4;
 
     /// Cold invocations may be affected by additional memory latencies. Don't take first invocations into account.
     static constexpr double NUM_INVOCATIONS_TO_THROW_OFF = 2;
 
     /// How to select method to run.
     /// -1 - automatically, based on statistics (default);
-    /// 0..4 - always choose specified method (for performance testing);
+    /// 0..3 - always choose specified method (for performance testing);
     /// -2 - choose methods in round robin fashion (for performance testing).
     ssize_t choose_method = -1;
 
@@ -105,17 +105,17 @@ struct PerformanceStatistics
 
     /// To select from different algorithms we use a kind of "bandits" algorithm.
     /// Sample random values from estimated normal distributions and choose the minimal.
-    size_t select(size_t max_element)
+    size_t select()
     {
         if (choose_method < 0)
         {
             double samples[NUM_ELEMENTS];
-            for (size_t i = 0; i < max_element; ++i)
+            for (size_t i = 0; i < NUM_ELEMENTS; ++i)
                 samples[i] = choose_method == -1
                     ? data[i].sample(rng)
                     : data[i].adjustedCount();
 
-            return std::min_element(samples, samples + max_element) - samples;
+            return std::min_element(samples, samples + NUM_ELEMENTS) - samples;
         }
         else
             return choose_method;

From 2e7f3f6e173daa8a001a3c46c19514a2c46925a9 Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@yandex-team.ru>
Date: Mon, 29 Apr 2019 22:33:13 +0300
Subject: [PATCH 101/118] Auto version update to [19.8.1.1] [54420]

---
 dbms/cmake/version.cmake | 10 +++++-----
 debian/changelog         |  4 ++--
 docker/client/Dockerfile |  2 +-
 docker/server/Dockerfile |  2 +-
 docker/test/Dockerfile   |  2 +-
 5 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake
index 3557d3d97df..78a4230c9d9 100644
--- a/dbms/cmake/version.cmake
+++ b/dbms/cmake/version.cmake
@@ -1,11 +1,11 @@
 # This strings autochanged from release_lib.sh:
-set(VERSION_REVISION 54419)
+set(VERSION_REVISION 54420)
 set(VERSION_MAJOR 19)
-set(VERSION_MINOR 7)
+set(VERSION_MINOR 8)
 set(VERSION_PATCH 1)
-set(VERSION_GITHASH b0b369b30f04a5026d1da5c7d3fd5998d6de1fe4)
-set(VERSION_DESCRIBE v19.7.1.1-testing)
-set(VERSION_STRING 19.7.1.1)
+set(VERSION_GITHASH a76e504f45ff4a74e8c492bd269f022352d5f6d9)
+set(VERSION_DESCRIBE v19.8.1.1-testing)
+set(VERSION_STRING 19.8.1.1)
 # end of autochange
 
 set(VERSION_EXTRA "" CACHE STRING "")
diff --git a/debian/changelog b/debian/changelog
index 9cc1178d4c0..87201a8f297 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,5 @@
-clickhouse (19.7.1.1) unstable; urgency=low
+clickhouse (19.8.1.1) unstable; urgency=low
 
   * Modified source code
 
- -- clickhouse-release <clickhouse-release@yandex-team.ru>  Fri, 19 Apr 2019 00:01:38 +0300
+ -- clickhouse-release <clickhouse-release@yandex-team.ru>  Mon, 29 Apr 2019 22:33:09 +0300
diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile
index ffd196bc27c..29a325428d1 100644
--- a/docker/client/Dockerfile
+++ b/docker/client/Dockerfile
@@ -1,7 +1,7 @@
 FROM ubuntu:18.04
 
 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/"
-ARG version=19.7.1.*
+ARG version=19.8.1.*
 
 RUN apt-get update \
     && apt-get install --yes --no-install-recommends \
diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile
index aaf395ee216..631d0a3a96a 100644
--- a/docker/server/Dockerfile
+++ b/docker/server/Dockerfile
@@ -1,7 +1,7 @@
 FROM ubuntu:18.04
 
 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/"
-ARG version=19.7.1.*
+ARG version=19.8.1.*
 ARG gosu_ver=1.10
 
 RUN apt-get update \
diff --git a/docker/test/Dockerfile b/docker/test/Dockerfile
index 0b7740bf098..04f638a7728 100644
--- a/docker/test/Dockerfile
+++ b/docker/test/Dockerfile
@@ -1,7 +1,7 @@
 FROM ubuntu:18.04
 
 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/"
-ARG version=19.7.1.*
+ARG version=19.8.1.*
 
 RUN apt-get update && \
     apt-get install -y apt-transport-https dirmngr && \

From 6291c4098fc8de4e89514d2842a7272aea265627 Mon Sep 17 00:00:00 2001
From: Ivan <5627721+abyss7@users.noreply.github.com>
Date: Mon, 29 Apr 2019 23:33:26 +0300
Subject: [PATCH 102/118] Revert "Merge pull request #5139 from
 abyss7/fix-self-move" (#5150)

This reverts commit bd1f8fe97a354151b9c6e50bee6af15d2d67ac45, reversing
changes made to 5d1091bfeea2937a1354d09651c6decb94e74866.
---
 dbms/src/Storages/ColumnsDescription.h  | 1 +
 dbms/src/Storages/ITableDeclaration.cpp | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/dbms/src/Storages/ColumnsDescription.h b/dbms/src/Storages/ColumnsDescription.h
index 3712178156c..0ff98473c99 100644
--- a/dbms/src/Storages/ColumnsDescription.h
+++ b/dbms/src/Storages/ColumnsDescription.h
@@ -38,6 +38,7 @@ public:
     ColumnsDescription(const ColumnsDescription & other);
     ColumnsDescription & operator=(const ColumnsDescription & other);
     ColumnsDescription(ColumnsDescription &&) noexcept = default;
+    ColumnsDescription & operator=(ColumnsDescription &&) noexcept = default;
 
     /// `after_column` can be a Nested column name;
     void add(ColumnDescription column, const String & after_column = String());
diff --git a/dbms/src/Storages/ITableDeclaration.cpp b/dbms/src/Storages/ITableDeclaration.cpp
index ec62b0a568a..c9385c24cbe 100644
--- a/dbms/src/Storages/ITableDeclaration.cpp
+++ b/dbms/src/Storages/ITableDeclaration.cpp
@@ -28,7 +28,7 @@ void ITableDeclaration::setColumns(ColumnsDescription columns_)
 {
     if (columns_.getOrdinary().empty())
         throw Exception("Empty list of columns passed", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED);
-    columns = columns_;
+    columns = std::move(columns_);
 }
 
 void ITableDeclaration::setIndicesDescription(IndicesDescription indices_)

From d38f4cd3a7c008b707f21a93ad57f26a6e989eed Mon Sep 17 00:00:00 2001
From: Danila Kutenin <kutdanila@yandex.ru>
Date: Tue, 30 Apr 2019 00:36:53 +0300
Subject: [PATCH 103/118] Almost empty

---
 dbms/src/Compression/LZ4_decompress_faster.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/dbms/src/Compression/LZ4_decompress_faster.cpp b/dbms/src/Compression/LZ4_decompress_faster.cpp
index 51ce1bfa7b7..aed18cbe6f8 100644
--- a/dbms/src/Compression/LZ4_decompress_faster.cpp
+++ b/dbms/src/Compression/LZ4_decompress_faster.cpp
@@ -209,7 +209,6 @@ inline void copyOverlap8Shuffle(UInt8 * op, const UInt8 *& match, const size_t o
 #endif
 
 
-
 template <> void inline copy<8>(UInt8 * dst, const UInt8 * src) { copy8(dst, src); }
 template <> void inline wildCopy<8>(UInt8 * dst, const UInt8 * src, UInt8 * dst_end) { wildCopy8(dst, src, dst_end); }
 template <> void inline copyOverlap<8, false>(UInt8 * op, const UInt8 *& match, const size_t offset) { copyOverlap8(op, match, offset); }
@@ -409,10 +408,12 @@ inline void copyOverlap32(UInt8 * op, const UInt8 *& match, const size_t offset)
     match += shift4[offset];
 }
 
+
 template <> void inline copy<32>(UInt8 * dst, const UInt8 * src) { copy32(dst, src); }
 template <> void inline wildCopy<32>(UInt8 * dst, const UInt8 * src, UInt8 * dst_end) { wildCopy32(dst, src, dst_end); }
 template <> void inline copyOverlap<32, false>(UInt8 * op, const UInt8 *& match, const size_t offset) { copyOverlap32(op, match, offset); }
 
+
 /// See also https://stackoverflow.com/a/30669632
 
 template <size_t copy_amount, bool use_shuffle>

From 109726d5339dc2c32b8a794564f74a1c32b05e7d Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Tue, 30 Apr 2019 00:40:54 +0300
Subject: [PATCH 104/118] Update basename.cpp

---
 dbms/src/Functions/basename.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dbms/src/Functions/basename.cpp b/dbms/src/Functions/basename.cpp
index dacf08ae1e3..f7e1778da5b 100644
--- a/dbms/src/Functions/basename.cpp
+++ b/dbms/src/Functions/basename.cpp
@@ -8,7 +8,7 @@ namespace DB
 
 struct ExtractBasename
 {
-    static size_t getReserveLengthForElement() { return 25; }
+    static size_t getReserveLengthForElement() { return 16; } /// Just a guess.
 
     static void execute(Pos data, size_t size, Pos & res_data, size_t & res_size)
     {

From 80897592052d4bc97301232e10e1f32933dde908 Mon Sep 17 00:00:00 2001
From: Danila Kutenin <kutdanila@yandex.ru>
Date: Tue, 30 Apr 2019 00:43:05 +0300
Subject: [PATCH 105/118] Remove bad headers

---
 dbms/src/Compression/LZ4_decompress_faster.cpp | 2 --
 1 file changed, 2 deletions(-)

diff --git a/dbms/src/Compression/LZ4_decompress_faster.cpp b/dbms/src/Compression/LZ4_decompress_faster.cpp
index aed18cbe6f8..7d90e88cdb7 100644
--- a/dbms/src/Compression/LZ4_decompress_faster.cpp
+++ b/dbms/src/Compression/LZ4_decompress_faster.cpp
@@ -1,11 +1,9 @@
 #include "LZ4_decompress_faster.h"
 
-#include <lz4.h>
 #include <string.h>
 #include <iostream>
 #include <random>
 #include <algorithm>
-#include <Core/CpuId.h>
 #include <Core/Defines.h>
 #include <Common/Stopwatch.h>
 #include <common/likely.h>

From 0d3ff520c6cd8d643bbf0958833ce18adbf1ae38 Mon Sep 17 00:00:00 2001
From: Danila Kutenin <kutdanila@yandex.ru>
Date: Tue, 30 Apr 2019 00:44:17 +0300
Subject: [PATCH 106/118] Typos

---
 dbms/src/Compression/LZ4_decompress_faster.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/dbms/src/Compression/LZ4_decompress_faster.cpp b/dbms/src/Compression/LZ4_decompress_faster.cpp
index 7d90e88cdb7..387650d3dcc 100644
--- a/dbms/src/Compression/LZ4_decompress_faster.cpp
+++ b/dbms/src/Compression/LZ4_decompress_faster.cpp
@@ -40,7 +40,7 @@ inline void copy8(UInt8 * dst, const UInt8 * src)
 
 inline void wildCopy8(UInt8 * dst, const UInt8 * src, UInt8 * dst_end)
 {
-    /// Unrolling with clang doing >10% performance degrade.
+    /// Unrolling with clang is doing >10% performance degrade.
 #if defined(__clang__)
     #pragma nounroll
 #endif
@@ -225,7 +225,7 @@ inline void copy16(UInt8 * dst, const UInt8 * src)
 
 inline void wildCopy16(UInt8 * dst, const UInt8 * src, UInt8 * dst_end)
 {
-    /// Unrolling with clang doing >10% performance degrade.
+    /// Unrolling with clang is doing >10% performance degrade.
 #if defined(__clang__)
     #pragma nounroll
 #endif
@@ -362,7 +362,7 @@ inline void copy32(UInt8 * dst, const UInt8 * src)
 
 inline void wildCopy32(UInt8 * dst, const UInt8 * src, UInt8 * dst_end)
 {
-    /// Unrolling with clang doing >10% performance degrade.
+    /// Unrolling with clang is doing >10% performance degrade.
 #if defined(__clang__)
     #pragma nounroll
 #endif
@@ -424,7 +424,7 @@ void NO_INLINE decompressImpl(
     UInt8 * op = reinterpret_cast<UInt8 *>(dest);
     UInt8 * const output_end = op + dest_size;
 
-    /// Unrolling with clang doing >10% performance degrade.
+    /// Unrolling with clang is doing >10% performance degrade.
 #if defined(__clang__)
     #pragma nounroll
 #endif

From 6b2656b02ce88504961b26251eb6614dfdb725dd Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Tue, 30 Apr 2019 13:34:12 +0300
Subject: [PATCH 107/118] Remove LowCardinality from types in PreparedSetKey.
 #5031

---
 dbms/src/Interpreters/PreparedSets.h | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/dbms/src/Interpreters/PreparedSets.h b/dbms/src/Interpreters/PreparedSets.h
index a2c6f6de40a..7dacdd24ce3 100644
--- a/dbms/src/Interpreters/PreparedSets.h
+++ b/dbms/src/Interpreters/PreparedSets.h
@@ -4,6 +4,7 @@
 #include <DataTypes/IDataType.h>
 #include <memory>
 #include <unordered_map>
+#include <DataTypes/DataTypeLowCardinality.h>
 
 namespace DB
 {
@@ -15,6 +16,9 @@ struct PreparedSetKey
     /// if left hand sides of the IN operators have different types).
     static PreparedSetKey forLiteral(const IAST & ast, DataTypes types_)
     {
+        for (auto & type : types_)
+            type = recursiveRemoveLowCardinality(type);
+
         PreparedSetKey key;
         key.ast_hash = ast.getTreeHash();
         key.types = std::move(types_);

From 0b7bf0c794ca8b27c77ddb66233064f2085cd6d4 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Tue, 30 Apr 2019 13:46:04 +0300
Subject: [PATCH 108/118] Added test. #5031

---
 ...1_low_cardinality_set_index_in_key_condition.reference | 1 +
 .../00931_low_cardinality_set_index_in_key_condition.sql  | 8 ++++++++
 2 files changed, 9 insertions(+)
 create mode 100644 dbms/tests/queries/0_stateless/00931_low_cardinality_set_index_in_key_condition.reference
 create mode 100644 dbms/tests/queries/0_stateless/00931_low_cardinality_set_index_in_key_condition.sql

diff --git a/dbms/tests/queries/0_stateless/00931_low_cardinality_set_index_in_key_condition.reference b/dbms/tests/queries/0_stateless/00931_low_cardinality_set_index_in_key_condition.reference
new file mode 100644
index 00000000000..78981922613
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00931_low_cardinality_set_index_in_key_condition.reference
@@ -0,0 +1 @@
+a
diff --git a/dbms/tests/queries/0_stateless/00931_low_cardinality_set_index_in_key_condition.sql b/dbms/tests/queries/0_stateless/00931_low_cardinality_set_index_in_key_condition.sql
new file mode 100644
index 00000000000..649c19b15ca
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00931_low_cardinality_set_index_in_key_condition.sql
@@ -0,0 +1,8 @@
+drop table if exists test_in;
+create table test_in (a LowCardinality(String)) Engine = MergeTree order by a;
+
+insert into test_in values ('a');
+select * from test_in where a in ('a');
+
+drop table if exists test_in;
+

From 19259ab5e33a7260684a51ab1453bea49980c5a3 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitbar@yandex-team.ru>
Date: Tue, 30 Apr 2019 17:09:10 +0300
Subject: [PATCH 109/118] Get rid of thread_local in SettingsConstraints.cpp

---
 dbms/src/Core/Settings.cpp                    |  4 +-
 dbms/src/Core/SettingsCommon.h                | 23 ++++++++---
 dbms/src/Interpreters/SettingsConstraints.cpp | 38 ++-----------------
 dbms/src/Interpreters/SettingsConstraints.h   |  2 -
 4 files changed, 22 insertions(+), 45 deletions(-)

diff --git a/dbms/src/Core/Settings.cpp b/dbms/src/Core/Settings.cpp
index 3b7c83dbf97..8ebb7d9b28b 100644
--- a/dbms/src/Core/Settings.cpp
+++ b/dbms/src/Core/Settings.cpp
@@ -104,9 +104,9 @@ void Settings::addProgramOptions(boost::program_options::options_description & o
         auto on_program_option
             = boost::function1<void, const std::string &>([this, index](const std::string & value) { set(index, value); });
         options.add(boost::shared_ptr<boost::program_options::option_description>(new boost::program_options::option_description(
-            Settings::getNameByIndex(index).data,
+            Settings::getName(index).data,
             boost::program_options::value<std::string>()->composing()->notifier(on_program_option),
-            Settings::getDescriptionByIndex(index).data)));
+            Settings::getDescription(index).data)));
     }
 }
 
diff --git a/dbms/src/Core/SettingsCommon.h b/dbms/src/Core/SettingsCommon.h
index 287f30c5879..db4500bb72a 100644
--- a/dbms/src/Core/SettingsCommon.h
+++ b/dbms/src/Core/SettingsCommon.h
@@ -310,6 +310,7 @@ private:
     using SetFieldFunction = void (*)(Derived &, const Field &);
     using SerializeFunction = void (*)(const Derived &, WriteBuffer & buf);
     using DeserializeFunction = void (*)(Derived &, ReadBuffer & buf);
+    using CastValueWithoutApplyingFunction = Field (*)(const Field &);
 
     struct MemberInfo
     {
@@ -322,6 +323,8 @@ private:
         SetFieldFunction set_field;
         SerializeFunction serialize;
         DeserializeFunction deserialize;
+        CastValueWithoutApplyingFunction cast_value_without_applying;
+
         bool isChanged(const Derived & collection) const { return *reinterpret_cast<const bool*>(reinterpret_cast<const UInt8*>(&collection) + offset_of_changed); }
     };
 
@@ -449,10 +452,11 @@ public:
     static size_t size() { return members().size(); }
 
     /// Returns name of a setting by its index (0..size()-1).
-    static StringRef getNameByIndex(size_t index) { return members()[index].name; }
+    static StringRef getName(size_t index) { return members()[index].name; }
 
-    /// Returns description of a setting by its index.
-    static StringRef getDescriptionByIndex(size_t index) { return members()[index].description; }
+    /// Returns description of a setting.
+    static StringRef getDescription(size_t index) { return members()[index].description; }
+    static StringRef getDescription(const String & name) { return members().findStrict(name)->description; }
 
     /// Searches a setting by its name; returns `npos` if not found.
     static size_t findIndex(const String & name) { return members().findIndex(name); }
@@ -461,6 +465,11 @@ public:
     /// Searches a setting by its name; throws an exception if not found.
     static size_t findIndexStrict(const String & name) { return members().findIndexStrict(name); }
 
+    /// Casts a value to a type according to a specified setting without actual changing this settings.
+    /// E.g. for SettingInt64 it casts Field to Field::Types::Int64.
+    static Field castValueWithoutApplying(size_t index, const Field & value) { return members()[index].cast_value_without_applying(value); }
+    static Field castValueWithoutApplying(const String & name, const Field & value) { return members().findStrict(name)->cast_value_without_applying(value); }
+
     iterator begin() { return iterator(castToDerived(), members().begin()); }
     const_iterator begin() const { return const_iterator(castToDerived(), members().begin()); }
     iterator end() { return iterator(castToDerived(), members().end()); }
@@ -632,7 +641,8 @@ public:
     static void NAME##_setString(Derived & collection, const String & value) { collection.NAME.set(value); } \
     static void NAME##_setField(Derived & collection, const Field & value) { collection.NAME.set(value); } \
     static void NAME##_serialize(const Derived & collection, WriteBuffer & buf) { collection.NAME.serialize(buf); } \
-    static void NAME##_deserialize(Derived & collection, ReadBuffer & buf) { collection.NAME.deserialize(buf); }
+    static void NAME##_deserialize(Derived & collection, ReadBuffer & buf) { collection.NAME.deserialize(buf); } \
+    static Field NAME##_castValueWithoutApplying(const Field & value) { TYPE temp{DEFAULT}; temp.set(value); return temp.toField(); }
 
 
 #define IMPLEMENT_SETTINGS_COLLECTION_ADD_MEMBER_INFO_HELPER_(TYPE, NAME, DEFAULT, DESCRIPTION) \
@@ -640,7 +650,8 @@ public:
     add({offsetof(Derived, NAME.changed), \
          StringRef(#NAME, strlen(#NAME)), StringRef(#DESCRIPTION, strlen(#DESCRIPTION)), \
          &Functions::NAME##_getString, &Functions::NAME##_getField, \
-        &Functions::NAME##_setString, &Functions::NAME##_setField, \
-        &Functions::NAME##_serialize, &Functions::NAME##_deserialize });
+         &Functions::NAME##_setString, &Functions::NAME##_setField, \
+         &Functions::NAME##_serialize, &Functions::NAME##_deserialize, \
+         &Functions::NAME##_castValueWithoutApplying });
 
 }
diff --git a/dbms/src/Interpreters/SettingsConstraints.cpp b/dbms/src/Interpreters/SettingsConstraints.cpp
index 43b62335a4d..35c431d7ed7 100644
--- a/dbms/src/Interpreters/SettingsConstraints.cpp
+++ b/dbms/src/Interpreters/SettingsConstraints.cpp
@@ -15,26 +15,6 @@ namespace ErrorCodes
     extern const int SETTING_CONSTRAINT_VIOLATION;
 }
 
-namespace
-{
-    thread_local Settings temp_settings;
-
-    Field convertSettingValueToComparableType(const Field & value, size_t setting_index)
-    {
-        auto && temp_setting = temp_settings[setting_index];
-        temp_setting.setValue(value);
-        return temp_setting.getValue();
-    }
-
-    Field convertSettingValueToComparableType(const String & value, size_t setting_index)
-    {
-        auto && temp_setting = temp_settings[setting_index];
-        temp_setting.setValue(value);
-        return temp_setting.getValue();
-    }
-}
-
-
 SettingsConstraints::SettingsConstraints() = default;
 SettingsConstraints::SettingsConstraints(const SettingsConstraints & src) = default;
 SettingsConstraints & SettingsConstraints::operator=(const SettingsConstraints & src) = default;
@@ -55,28 +35,16 @@ void SettingsConstraints::setReadOnly(const String & name, bool read_only)
     getConstraintRef(setting_index).read_only = read_only;
 }
 
-void SettingsConstraints::setMinValue(const String & name, const String & min_value)
-{
-    size_t setting_index = Settings::findIndexStrict(name);
-    getConstraintRef(setting_index).min_value = convertSettingValueToComparableType(min_value, setting_index);
-}
-
 void SettingsConstraints::setMinValue(const String & name, const Field & min_value)
 {
     size_t setting_index = Settings::findIndexStrict(name);
-    getConstraintRef(setting_index).min_value = convertSettingValueToComparableType(min_value, setting_index);
-}
-
-void SettingsConstraints::setMaxValue(const String & name, const String & max_value)
-{
-    size_t setting_index = Settings::findIndexStrict(name);
-    getConstraintRef(setting_index).max_value = convertSettingValueToComparableType(max_value, setting_index);
+    getConstraintRef(setting_index).min_value = Settings::castValueWithoutApplying(setting_index, min_value);
 }
 
 void SettingsConstraints::setMaxValue(const String & name, const Field & max_value)
 {
     size_t setting_index = Settings::findIndexStrict(name);
-    getConstraintRef(setting_index).max_value = convertSettingValueToComparableType(max_value, setting_index);
+    getConstraintRef(setting_index).max_value = Settings::castValueWithoutApplying(setting_index, max_value);
 }
 
 
@@ -87,7 +55,7 @@ void SettingsConstraints::check(const Settings & current_settings, const Setting
     if (setting_index == Settings::npos)
         return;
 
-    Field new_value = convertSettingValueToComparableType(change.value, setting_index);
+    Field new_value = Settings::castValueWithoutApplying(setting_index, change.value);
     Field current_value = current_settings.get(setting_index);
 
     /// Setting isn't checked if value wasn't changed.
diff --git a/dbms/src/Interpreters/SettingsConstraints.h b/dbms/src/Interpreters/SettingsConstraints.h
index cc0e3def441..f0ba3b7eb16 100644
--- a/dbms/src/Interpreters/SettingsConstraints.h
+++ b/dbms/src/Interpreters/SettingsConstraints.h
@@ -59,9 +59,7 @@ public:
 
     void clear();
 
-    void setMinValue(const String & name, const String & min_value);
     void setMinValue(const String & name, const Field & min_value);
-    void setMaxValue(const String & name, const String & max_value);
     void setMaxValue(const String & name, const Field & max_value);
     void setReadOnly(const String & name, bool read_only);
 

From 6a3966cd08018a8da6d538fa2388dbd06f0c507a Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <KochetovNicolai@users.noreply.github.com>
Date: Tue, 30 Apr 2019 17:46:17 +0300
Subject: [PATCH 110/118] Update PreparedSets.h

---
 dbms/src/Interpreters/PreparedSets.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/dbms/src/Interpreters/PreparedSets.h b/dbms/src/Interpreters/PreparedSets.h
index 7dacdd24ce3..0faa748303d 100644
--- a/dbms/src/Interpreters/PreparedSets.h
+++ b/dbms/src/Interpreters/PreparedSets.h
@@ -16,6 +16,8 @@ struct PreparedSetKey
     /// if left hand sides of the IN operators have different types).
     static PreparedSetKey forLiteral(const IAST & ast, DataTypes types_)
     {
+        /// Remove LowCardinality types from type list because Set doesn't support LowCardinality keys now, 
+        ///   just converts LowCardinality to ordinary types.
         for (auto & type : types_)
             type = recursiveRemoveLowCardinality(type);
 

From dede124449b93edfb82bd5d6eec65075f706f017 Mon Sep 17 00:00:00 2001
From: Ivan <5627721+abyss7@users.noreply.github.com>
Date: Tue, 30 Apr 2019 19:05:39 +0300
Subject: [PATCH 111/118] Fixes after premature merge of #5139 (#5143)

* Implement move-constructor via move-assignment operator.
* Reimplement copy-ctor via copy-assignment operator too.
---
 dbms/src/Storages/ColumnsDescription.cpp | 23 +++++++++++++++++++----
 dbms/src/Storages/ColumnsDescription.h   |  4 ++--
 2 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/dbms/src/Storages/ColumnsDescription.cpp b/dbms/src/Storages/ColumnsDescription.cpp
index 89cf986f165..a91b1f6c67c 100644
--- a/dbms/src/Storages/ColumnsDescription.cpp
+++ b/dbms/src/Storages/ColumnsDescription.cpp
@@ -124,16 +124,31 @@ ColumnsDescription::ColumnsDescription(NamesAndTypesList ordinary)
 }
 
 ColumnsDescription::ColumnsDescription(const ColumnsDescription & other)
-    : columns(other.columns)
 {
-    for (auto it = columns.begin(); it != columns.end(); ++it)
-        name_to_column.emplace(it->name, it);
+    *this = other;
 }
 
 ColumnsDescription & ColumnsDescription::operator=(const ColumnsDescription & other)
 {
     if (&other != this)
-        *this = ColumnsDescription(other);
+    {
+        columns = other.columns;
+        name_to_column.clear();
+        for (auto it = columns.begin(); it != columns.end(); ++it)
+            name_to_column.emplace(it->name, it);
+    }
+    return *this;
+}
+
+ColumnsDescription::ColumnsDescription(ColumnsDescription && other) noexcept {
+    *this = std::move(other);
+}
+
+ColumnsDescription & ColumnsDescription::operator=(ColumnsDescription && other) noexcept
+{
+    assert(&other != this);
+    columns = std::move(other.columns);
+    name_to_column = std::move(other.name_to_column);
     return *this;
 }
 
diff --git a/dbms/src/Storages/ColumnsDescription.h b/dbms/src/Storages/ColumnsDescription.h
index 0ff98473c99..b203a639537 100644
--- a/dbms/src/Storages/ColumnsDescription.h
+++ b/dbms/src/Storages/ColumnsDescription.h
@@ -37,8 +37,8 @@ public:
     explicit ColumnsDescription(NamesAndTypesList ordinary_);
     ColumnsDescription(const ColumnsDescription & other);
     ColumnsDescription & operator=(const ColumnsDescription & other);
-    ColumnsDescription(ColumnsDescription &&) noexcept = default;
-    ColumnsDescription & operator=(ColumnsDescription &&) noexcept = default;
+    ColumnsDescription(ColumnsDescription &&) noexcept;
+    ColumnsDescription & operator=(ColumnsDescription && other) noexcept;
 
     /// `after_column` can be a Nested column name;
     void add(ColumnDescription column, const String & after_column = String());

From 39b696f9dfc257323c8acf51845fb9ec55bb260a Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Tue, 30 Apr 2019 19:14:53 +0300
Subject: [PATCH 112/118] Update basename.cpp

---
 dbms/src/Functions/basename.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/dbms/src/Functions/basename.cpp b/dbms/src/Functions/basename.cpp
index f7e1778da5b..615a8c01de2 100644
--- a/dbms/src/Functions/basename.cpp
+++ b/dbms/src/Functions/basename.cpp
@@ -6,6 +6,10 @@
 namespace DB
 {
 
+/** Extract substring after the last slash or backslash.
+  * If there is no slashes, return the string unchanged.
+  * It is used to extract filename from path.
+  */
 struct ExtractBasename
 {
     static size_t getReserveLengthForElement() { return 16; } /// Just a guess.

From b88c981902f7833055fc6bc86796d9f4e8382ef7 Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Tue, 30 Apr 2019 19:15:08 +0300
Subject: [PATCH 113/118] Update basename.cpp

---
 dbms/src/Functions/basename.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dbms/src/Functions/basename.cpp b/dbms/src/Functions/basename.cpp
index 615a8c01de2..53320014766 100644
--- a/dbms/src/Functions/basename.cpp
+++ b/dbms/src/Functions/basename.cpp
@@ -7,7 +7,7 @@ namespace DB
 {
 
 /** Extract substring after the last slash or backslash.
-  * If there is no slashes, return the string unchanged.
+  * If there are no slashes, return the string unchanged.
   * It is used to extract filename from path.
   */
 struct ExtractBasename

From cd16c3a3c90810a6ce9277e0272ddf36c7200b12 Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Tue, 30 Apr 2019 19:21:33 +0300
Subject: [PATCH 114/118] Update
 00931_low_cardinality_set_index_in_key_condition.sql

---
 .../00931_low_cardinality_set_index_in_key_condition.sql         | 1 -
 1 file changed, 1 deletion(-)

diff --git a/dbms/tests/queries/0_stateless/00931_low_cardinality_set_index_in_key_condition.sql b/dbms/tests/queries/0_stateless/00931_low_cardinality_set_index_in_key_condition.sql
index 649c19b15ca..4a43b4dced8 100644
--- a/dbms/tests/queries/0_stateless/00931_low_cardinality_set_index_in_key_condition.sql
+++ b/dbms/tests/queries/0_stateless/00931_low_cardinality_set_index_in_key_condition.sql
@@ -5,4 +5,3 @@ insert into test_in values ('a');
 select * from test_in where a in ('a');
 
 drop table if exists test_in;
-

From 7a2e40da289b2e67f107b10e602cc09c25ef1f67 Mon Sep 17 00:00:00 2001
From: Danila Kutenin <kutdanila@yandex.ru>
Date: Tue, 30 Apr 2019 19:48:33 +0300
Subject: [PATCH 115/118] pragma once in header file

---
 dbms/src/Core/CpuId.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/dbms/src/Core/CpuId.h b/dbms/src/Core/CpuId.h
index b1a2d9fa10a..1bae7407818 100644
--- a/dbms/src/Core/CpuId.h
+++ b/dbms/src/Core/CpuId.h
@@ -1,3 +1,5 @@
+#pragma once
+
 #include <Core/Types.h>
 
 #if defined(__x86_64__) || defined(__i386__)

From b02588eb623dcc1b71525fd68811f3c8b0a1ff2a Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Tue, 30 Apr 2019 20:11:43 +0300
Subject: [PATCH 116/118] Move performance tests for convenience

---
 dbms/tests/performance/{ipv4_ipv6 => }/IPv4.xml                   | 0
 dbms/tests/performance/{ipv4_ipv6 => }/IPv6.xml                   | 0
 dbms/tests/performance/{array => }/array_element.xml              | 0
 dbms/tests/performance/{array => }/array_join.xml                 | 0
 dbms/tests/performance/{base64 => }/base64.xml                    | 0
 dbms/tests/performance/{base64 => }/base64_hits.xml               | 0
 dbms/tests/performance/{no_data => }/bounding_ratio.xml           | 0
 .../{string_comparisions => }/column_column_comparison.xml        | 0
 dbms/tests/performance/{columns_hashing => }/columns_hashing.xml  | 0
 dbms/tests/performance/{array => }/complex_array_creation.xml     | 0
 dbms/tests/performance/{gather => }/concat_hits.xml               | 0
 dbms/tests/performance/{date_time => }/conditional.xml            | 0
 .../performance/{functions_hashing => }/consistent_hashes.xml     | 0
 .../{string_comparisions => }/constant_column_comparison.xml      | 0
 .../constant_column_comparison.xml => constant_column_search.xml} | 0
 .../performance/{functions_hashing => }/cryptographic_hashes.xml  | 0
 dbms/tests/performance/{date_parsing => }/date_parsing.xml        | 0
 dbms/tests/performance/{date_time => }/date_time.xml              | 0
 dbms/tests/performance/{entropy => }/entropy.xml                  | 0
 .../performance/{test_hits => }/first_significant_subdomain.xml   | 0
 .../performance/{string_comparisions => }/fixed_string16.xml      | 0
 dbms/tests/performance/{float_parsing => }/float_parsing.xml      | 0
 dbms/tests/performance/{date_time => }/format_date_time.xml       | 0
 .../tests/performance/{functions_coding => }/functions_coding.xml | 0
 .../{functions_hashing => }/general_purpose_hashes.xml            | 0
 dbms/tests/performance/{gather => }/if_array_num.xml              | 0
 dbms/tests/performance/{gather => }/if_array_string.xml           | 0
 dbms/tests/performance/{gather => }/if_string_const.xml           | 0
 dbms/tests/performance/{gather => }/if_string_hits.xml            | 0
 dbms/tests/performance/{int_parsing => }/int_parsing.xml          | 0
 dbms/tests/performance/{leftpad => }/leftpad.xml                  | 0
 dbms/tests/performance/{string_search => }/ngram_distance.xml     | 0
 dbms/tests/performance/{nyc_taxi => }/nyc_taxi.xml                | 0
 .../{string_comparisions => }/order_by_single_column.xml          | 0
 .../performance/{merge_tree_with_aio => }/read_hits_with_aio.xml  | 0
 dbms/tests/performance/{right => }/right.xml                      | 0
 dbms/tests/performance/{roundDown => }/roundDown.xml              | 0
 dbms/tests/performance/{set => }/set.xml                          | 0
 dbms/tests/performance/{set => }/set_hits.xml                     | 0
 dbms/tests/performance/{gather => }/slices_hits.xml               | 0
 .../performance/{jit_query_compilation => }/small_requests.xml    | 0
 dbms/tests/performance/{sort => }/sort.xml                        | 0
 dbms/tests/performance/{string_sort => }/string_sort.xml          | 0
 dbms/tests/performance/{no_data => }/system_numbers.xml           | 0
 dbms/tests/performance/{test_hits => }/test_hits.xml              | 0
 .../{test_hits => }/test_hits_agg_functions_min_max_any.xml       | 0
 dbms/tests/performance/{trim => }/trim_numbers.xml                | 0
 dbms/tests/performance/{trim => }/trim_urls.xml                   | 0
 dbms/tests/performance/{trim => }/trim_whitespace.xml             | 0
 dbms/tests/performance/{uniq => }/uniq.xml                        | 0
 dbms/tests/performance/{url => }/url_hits.xml                     | 0
 .../performance/{functions_json => }/visit_param_extract_raw.xml  | 0
 dbms/tests/performance/{website/url_hits.xml => website.xml}      | 0
 53 files changed, 0 insertions(+), 0 deletions(-)
 rename dbms/tests/performance/{ipv4_ipv6 => }/IPv4.xml (100%)
 rename dbms/tests/performance/{ipv4_ipv6 => }/IPv6.xml (100%)
 rename dbms/tests/performance/{array => }/array_element.xml (100%)
 rename dbms/tests/performance/{array => }/array_join.xml (100%)
 rename dbms/tests/performance/{base64 => }/base64.xml (100%)
 rename dbms/tests/performance/{base64 => }/base64_hits.xml (100%)
 rename dbms/tests/performance/{no_data => }/bounding_ratio.xml (100%)
 rename dbms/tests/performance/{string_comparisions => }/column_column_comparison.xml (100%)
 rename dbms/tests/performance/{columns_hashing => }/columns_hashing.xml (100%)
 rename dbms/tests/performance/{array => }/complex_array_creation.xml (100%)
 rename dbms/tests/performance/{gather => }/concat_hits.xml (100%)
 rename dbms/tests/performance/{date_time => }/conditional.xml (100%)
 rename dbms/tests/performance/{functions_hashing => }/consistent_hashes.xml (100%)
 rename dbms/tests/performance/{string_comparisions => }/constant_column_comparison.xml (100%)
 rename dbms/tests/performance/{string_search/constant_column_comparison.xml => constant_column_search.xml} (100%)
 rename dbms/tests/performance/{functions_hashing => }/cryptographic_hashes.xml (100%)
 rename dbms/tests/performance/{date_parsing => }/date_parsing.xml (100%)
 rename dbms/tests/performance/{date_time => }/date_time.xml (100%)
 rename dbms/tests/performance/{entropy => }/entropy.xml (100%)
 rename dbms/tests/performance/{test_hits => }/first_significant_subdomain.xml (100%)
 rename dbms/tests/performance/{string_comparisions => }/fixed_string16.xml (100%)
 rename dbms/tests/performance/{float_parsing => }/float_parsing.xml (100%)
 rename dbms/tests/performance/{date_time => }/format_date_time.xml (100%)
 rename dbms/tests/performance/{functions_coding => }/functions_coding.xml (100%)
 rename dbms/tests/performance/{functions_hashing => }/general_purpose_hashes.xml (100%)
 rename dbms/tests/performance/{gather => }/if_array_num.xml (100%)
 rename dbms/tests/performance/{gather => }/if_array_string.xml (100%)
 rename dbms/tests/performance/{gather => }/if_string_const.xml (100%)
 rename dbms/tests/performance/{gather => }/if_string_hits.xml (100%)
 rename dbms/tests/performance/{int_parsing => }/int_parsing.xml (100%)
 rename dbms/tests/performance/{leftpad => }/leftpad.xml (100%)
 rename dbms/tests/performance/{string_search => }/ngram_distance.xml (100%)
 rename dbms/tests/performance/{nyc_taxi => }/nyc_taxi.xml (100%)
 rename dbms/tests/performance/{string_comparisions => }/order_by_single_column.xml (100%)
 rename dbms/tests/performance/{merge_tree_with_aio => }/read_hits_with_aio.xml (100%)
 rename dbms/tests/performance/{right => }/right.xml (100%)
 rename dbms/tests/performance/{roundDown => }/roundDown.xml (100%)
 rename dbms/tests/performance/{set => }/set.xml (100%)
 rename dbms/tests/performance/{set => }/set_hits.xml (100%)
 rename dbms/tests/performance/{gather => }/slices_hits.xml (100%)
 rename dbms/tests/performance/{jit_query_compilation => }/small_requests.xml (100%)
 rename dbms/tests/performance/{sort => }/sort.xml (100%)
 rename dbms/tests/performance/{string_sort => }/string_sort.xml (100%)
 rename dbms/tests/performance/{no_data => }/system_numbers.xml (100%)
 rename dbms/tests/performance/{test_hits => }/test_hits.xml (100%)
 rename dbms/tests/performance/{test_hits => }/test_hits_agg_functions_min_max_any.xml (100%)
 rename dbms/tests/performance/{trim => }/trim_numbers.xml (100%)
 rename dbms/tests/performance/{trim => }/trim_urls.xml (100%)
 rename dbms/tests/performance/{trim => }/trim_whitespace.xml (100%)
 rename dbms/tests/performance/{uniq => }/uniq.xml (100%)
 rename dbms/tests/performance/{url => }/url_hits.xml (100%)
 rename dbms/tests/performance/{functions_json => }/visit_param_extract_raw.xml (100%)
 rename dbms/tests/performance/{website/url_hits.xml => website.xml} (100%)

diff --git a/dbms/tests/performance/ipv4_ipv6/IPv4.xml b/dbms/tests/performance/IPv4.xml
similarity index 100%
rename from dbms/tests/performance/ipv4_ipv6/IPv4.xml
rename to dbms/tests/performance/IPv4.xml
diff --git a/dbms/tests/performance/ipv4_ipv6/IPv6.xml b/dbms/tests/performance/IPv6.xml
similarity index 100%
rename from dbms/tests/performance/ipv4_ipv6/IPv6.xml
rename to dbms/tests/performance/IPv6.xml
diff --git a/dbms/tests/performance/array/array_element.xml b/dbms/tests/performance/array_element.xml
similarity index 100%
rename from dbms/tests/performance/array/array_element.xml
rename to dbms/tests/performance/array_element.xml
diff --git a/dbms/tests/performance/array/array_join.xml b/dbms/tests/performance/array_join.xml
similarity index 100%
rename from dbms/tests/performance/array/array_join.xml
rename to dbms/tests/performance/array_join.xml
diff --git a/dbms/tests/performance/base64/base64.xml b/dbms/tests/performance/base64.xml
similarity index 100%
rename from dbms/tests/performance/base64/base64.xml
rename to dbms/tests/performance/base64.xml
diff --git a/dbms/tests/performance/base64/base64_hits.xml b/dbms/tests/performance/base64_hits.xml
similarity index 100%
rename from dbms/tests/performance/base64/base64_hits.xml
rename to dbms/tests/performance/base64_hits.xml
diff --git a/dbms/tests/performance/no_data/bounding_ratio.xml b/dbms/tests/performance/bounding_ratio.xml
similarity index 100%
rename from dbms/tests/performance/no_data/bounding_ratio.xml
rename to dbms/tests/performance/bounding_ratio.xml
diff --git a/dbms/tests/performance/string_comparisions/column_column_comparison.xml b/dbms/tests/performance/column_column_comparison.xml
similarity index 100%
rename from dbms/tests/performance/string_comparisions/column_column_comparison.xml
rename to dbms/tests/performance/column_column_comparison.xml
diff --git a/dbms/tests/performance/columns_hashing/columns_hashing.xml b/dbms/tests/performance/columns_hashing.xml
similarity index 100%
rename from dbms/tests/performance/columns_hashing/columns_hashing.xml
rename to dbms/tests/performance/columns_hashing.xml
diff --git a/dbms/tests/performance/array/complex_array_creation.xml b/dbms/tests/performance/complex_array_creation.xml
similarity index 100%
rename from dbms/tests/performance/array/complex_array_creation.xml
rename to dbms/tests/performance/complex_array_creation.xml
diff --git a/dbms/tests/performance/gather/concat_hits.xml b/dbms/tests/performance/concat_hits.xml
similarity index 100%
rename from dbms/tests/performance/gather/concat_hits.xml
rename to dbms/tests/performance/concat_hits.xml
diff --git a/dbms/tests/performance/date_time/conditional.xml b/dbms/tests/performance/conditional.xml
similarity index 100%
rename from dbms/tests/performance/date_time/conditional.xml
rename to dbms/tests/performance/conditional.xml
diff --git a/dbms/tests/performance/functions_hashing/consistent_hashes.xml b/dbms/tests/performance/consistent_hashes.xml
similarity index 100%
rename from dbms/tests/performance/functions_hashing/consistent_hashes.xml
rename to dbms/tests/performance/consistent_hashes.xml
diff --git a/dbms/tests/performance/string_comparisions/constant_column_comparison.xml b/dbms/tests/performance/constant_column_comparison.xml
similarity index 100%
rename from dbms/tests/performance/string_comparisions/constant_column_comparison.xml
rename to dbms/tests/performance/constant_column_comparison.xml
diff --git a/dbms/tests/performance/string_search/constant_column_comparison.xml b/dbms/tests/performance/constant_column_search.xml
similarity index 100%
rename from dbms/tests/performance/string_search/constant_column_comparison.xml
rename to dbms/tests/performance/constant_column_search.xml
diff --git a/dbms/tests/performance/functions_hashing/cryptographic_hashes.xml b/dbms/tests/performance/cryptographic_hashes.xml
similarity index 100%
rename from dbms/tests/performance/functions_hashing/cryptographic_hashes.xml
rename to dbms/tests/performance/cryptographic_hashes.xml
diff --git a/dbms/tests/performance/date_parsing/date_parsing.xml b/dbms/tests/performance/date_parsing.xml
similarity index 100%
rename from dbms/tests/performance/date_parsing/date_parsing.xml
rename to dbms/tests/performance/date_parsing.xml
diff --git a/dbms/tests/performance/date_time/date_time.xml b/dbms/tests/performance/date_time.xml
similarity index 100%
rename from dbms/tests/performance/date_time/date_time.xml
rename to dbms/tests/performance/date_time.xml
diff --git a/dbms/tests/performance/entropy/entropy.xml b/dbms/tests/performance/entropy.xml
similarity index 100%
rename from dbms/tests/performance/entropy/entropy.xml
rename to dbms/tests/performance/entropy.xml
diff --git a/dbms/tests/performance/test_hits/first_significant_subdomain.xml b/dbms/tests/performance/first_significant_subdomain.xml
similarity index 100%
rename from dbms/tests/performance/test_hits/first_significant_subdomain.xml
rename to dbms/tests/performance/first_significant_subdomain.xml
diff --git a/dbms/tests/performance/string_comparisions/fixed_string16.xml b/dbms/tests/performance/fixed_string16.xml
similarity index 100%
rename from dbms/tests/performance/string_comparisions/fixed_string16.xml
rename to dbms/tests/performance/fixed_string16.xml
diff --git a/dbms/tests/performance/float_parsing/float_parsing.xml b/dbms/tests/performance/float_parsing.xml
similarity index 100%
rename from dbms/tests/performance/float_parsing/float_parsing.xml
rename to dbms/tests/performance/float_parsing.xml
diff --git a/dbms/tests/performance/date_time/format_date_time.xml b/dbms/tests/performance/format_date_time.xml
similarity index 100%
rename from dbms/tests/performance/date_time/format_date_time.xml
rename to dbms/tests/performance/format_date_time.xml
diff --git a/dbms/tests/performance/functions_coding/functions_coding.xml b/dbms/tests/performance/functions_coding.xml
similarity index 100%
rename from dbms/tests/performance/functions_coding/functions_coding.xml
rename to dbms/tests/performance/functions_coding.xml
diff --git a/dbms/tests/performance/functions_hashing/general_purpose_hashes.xml b/dbms/tests/performance/general_purpose_hashes.xml
similarity index 100%
rename from dbms/tests/performance/functions_hashing/general_purpose_hashes.xml
rename to dbms/tests/performance/general_purpose_hashes.xml
diff --git a/dbms/tests/performance/gather/if_array_num.xml b/dbms/tests/performance/if_array_num.xml
similarity index 100%
rename from dbms/tests/performance/gather/if_array_num.xml
rename to dbms/tests/performance/if_array_num.xml
diff --git a/dbms/tests/performance/gather/if_array_string.xml b/dbms/tests/performance/if_array_string.xml
similarity index 100%
rename from dbms/tests/performance/gather/if_array_string.xml
rename to dbms/tests/performance/if_array_string.xml
diff --git a/dbms/tests/performance/gather/if_string_const.xml b/dbms/tests/performance/if_string_const.xml
similarity index 100%
rename from dbms/tests/performance/gather/if_string_const.xml
rename to dbms/tests/performance/if_string_const.xml
diff --git a/dbms/tests/performance/gather/if_string_hits.xml b/dbms/tests/performance/if_string_hits.xml
similarity index 100%
rename from dbms/tests/performance/gather/if_string_hits.xml
rename to dbms/tests/performance/if_string_hits.xml
diff --git a/dbms/tests/performance/int_parsing/int_parsing.xml b/dbms/tests/performance/int_parsing.xml
similarity index 100%
rename from dbms/tests/performance/int_parsing/int_parsing.xml
rename to dbms/tests/performance/int_parsing.xml
diff --git a/dbms/tests/performance/leftpad/leftpad.xml b/dbms/tests/performance/leftpad.xml
similarity index 100%
rename from dbms/tests/performance/leftpad/leftpad.xml
rename to dbms/tests/performance/leftpad.xml
diff --git a/dbms/tests/performance/string_search/ngram_distance.xml b/dbms/tests/performance/ngram_distance.xml
similarity index 100%
rename from dbms/tests/performance/string_search/ngram_distance.xml
rename to dbms/tests/performance/ngram_distance.xml
diff --git a/dbms/tests/performance/nyc_taxi/nyc_taxi.xml b/dbms/tests/performance/nyc_taxi.xml
similarity index 100%
rename from dbms/tests/performance/nyc_taxi/nyc_taxi.xml
rename to dbms/tests/performance/nyc_taxi.xml
diff --git a/dbms/tests/performance/string_comparisions/order_by_single_column.xml b/dbms/tests/performance/order_by_single_column.xml
similarity index 100%
rename from dbms/tests/performance/string_comparisions/order_by_single_column.xml
rename to dbms/tests/performance/order_by_single_column.xml
diff --git a/dbms/tests/performance/merge_tree_with_aio/read_hits_with_aio.xml b/dbms/tests/performance/read_hits_with_aio.xml
similarity index 100%
rename from dbms/tests/performance/merge_tree_with_aio/read_hits_with_aio.xml
rename to dbms/tests/performance/read_hits_with_aio.xml
diff --git a/dbms/tests/performance/right/right.xml b/dbms/tests/performance/right.xml
similarity index 100%
rename from dbms/tests/performance/right/right.xml
rename to dbms/tests/performance/right.xml
diff --git a/dbms/tests/performance/roundDown/roundDown.xml b/dbms/tests/performance/roundDown.xml
similarity index 100%
rename from dbms/tests/performance/roundDown/roundDown.xml
rename to dbms/tests/performance/roundDown.xml
diff --git a/dbms/tests/performance/set/set.xml b/dbms/tests/performance/set.xml
similarity index 100%
rename from dbms/tests/performance/set/set.xml
rename to dbms/tests/performance/set.xml
diff --git a/dbms/tests/performance/set/set_hits.xml b/dbms/tests/performance/set_hits.xml
similarity index 100%
rename from dbms/tests/performance/set/set_hits.xml
rename to dbms/tests/performance/set_hits.xml
diff --git a/dbms/tests/performance/gather/slices_hits.xml b/dbms/tests/performance/slices_hits.xml
similarity index 100%
rename from dbms/tests/performance/gather/slices_hits.xml
rename to dbms/tests/performance/slices_hits.xml
diff --git a/dbms/tests/performance/jit_query_compilation/small_requests.xml b/dbms/tests/performance/small_requests.xml
similarity index 100%
rename from dbms/tests/performance/jit_query_compilation/small_requests.xml
rename to dbms/tests/performance/small_requests.xml
diff --git a/dbms/tests/performance/sort/sort.xml b/dbms/tests/performance/sort.xml
similarity index 100%
rename from dbms/tests/performance/sort/sort.xml
rename to dbms/tests/performance/sort.xml
diff --git a/dbms/tests/performance/string_sort/string_sort.xml b/dbms/tests/performance/string_sort.xml
similarity index 100%
rename from dbms/tests/performance/string_sort/string_sort.xml
rename to dbms/tests/performance/string_sort.xml
diff --git a/dbms/tests/performance/no_data/system_numbers.xml b/dbms/tests/performance/system_numbers.xml
similarity index 100%
rename from dbms/tests/performance/no_data/system_numbers.xml
rename to dbms/tests/performance/system_numbers.xml
diff --git a/dbms/tests/performance/test_hits/test_hits.xml b/dbms/tests/performance/test_hits.xml
similarity index 100%
rename from dbms/tests/performance/test_hits/test_hits.xml
rename to dbms/tests/performance/test_hits.xml
diff --git a/dbms/tests/performance/test_hits/test_hits_agg_functions_min_max_any.xml b/dbms/tests/performance/test_hits_agg_functions_min_max_any.xml
similarity index 100%
rename from dbms/tests/performance/test_hits/test_hits_agg_functions_min_max_any.xml
rename to dbms/tests/performance/test_hits_agg_functions_min_max_any.xml
diff --git a/dbms/tests/performance/trim/trim_numbers.xml b/dbms/tests/performance/trim_numbers.xml
similarity index 100%
rename from dbms/tests/performance/trim/trim_numbers.xml
rename to dbms/tests/performance/trim_numbers.xml
diff --git a/dbms/tests/performance/trim/trim_urls.xml b/dbms/tests/performance/trim_urls.xml
similarity index 100%
rename from dbms/tests/performance/trim/trim_urls.xml
rename to dbms/tests/performance/trim_urls.xml
diff --git a/dbms/tests/performance/trim/trim_whitespace.xml b/dbms/tests/performance/trim_whitespace.xml
similarity index 100%
rename from dbms/tests/performance/trim/trim_whitespace.xml
rename to dbms/tests/performance/trim_whitespace.xml
diff --git a/dbms/tests/performance/uniq/uniq.xml b/dbms/tests/performance/uniq.xml
similarity index 100%
rename from dbms/tests/performance/uniq/uniq.xml
rename to dbms/tests/performance/uniq.xml
diff --git a/dbms/tests/performance/url/url_hits.xml b/dbms/tests/performance/url_hits.xml
similarity index 100%
rename from dbms/tests/performance/url/url_hits.xml
rename to dbms/tests/performance/url_hits.xml
diff --git a/dbms/tests/performance/functions_json/visit_param_extract_raw.xml b/dbms/tests/performance/visit_param_extract_raw.xml
similarity index 100%
rename from dbms/tests/performance/functions_json/visit_param_extract_raw.xml
rename to dbms/tests/performance/visit_param_extract_raw.xml
diff --git a/dbms/tests/performance/website/url_hits.xml b/dbms/tests/performance/website.xml
similarity index 100%
rename from dbms/tests/performance/website/url_hits.xml
rename to dbms/tests/performance/website.xml

From 546b4815d13e454ebc9e77b95ca194f5803ddd46 Mon Sep 17 00:00:00 2001
From: Hasnat <hasnat.ullah@gmail.com>
Date: Tue, 30 Apr 2019 18:18:21 +0100
Subject: [PATCH 117/118] Update access_rights.md

---
 docs/en/operations/access_rights.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/en/operations/access_rights.md b/docs/en/operations/access_rights.md
index 13458a39d47..0984eea43ae 100644
--- a/docs/en/operations/access_rights.md
+++ b/docs/en/operations/access_rights.md
@@ -57,6 +57,7 @@ Users are recorded in the `users` section. Here is a fragment of the `users.xml`
            <database>test</database>
         </allow_databases>
     </web>
+</users>
 ```
 
 You can see a declaration from two users: `default`and`web`. We added the `web` user separately.

From 83a35d7644cad823533c3a6459c8c777f9d10481 Mon Sep 17 00:00:00 2001
From: proller <proller@github.com>
Date: Tue, 30 Apr 2019 22:46:07 +0300
Subject: [PATCH 118/118] Update contributors

---
 .../StorageSystemContributors.generated.cpp   | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/dbms/src/Storages/System/StorageSystemContributors.generated.cpp b/dbms/src/Storages/System/StorageSystemContributors.generated.cpp
index c8489a949c8..7076a9af5a8 100644
--- a/dbms/src/Storages/System/StorageSystemContributors.generated.cpp
+++ b/dbms/src/Storages/System/StorageSystemContributors.generated.cpp
@@ -1,6 +1,7 @@
 // autogenerated by ./StorageSystemContributors.sh
 const char * auto_contributors[] {
     "0xflotus",
+    "821008736@qq.com",
     "Alberto",
     "Aleksandra (Ася)",
     "Alex Bocharov",
@@ -9,12 +10,14 @@ const char * auto_contributors[] {
     "Alexander Avdonkin",
     "Alexander Ermolaev",
     "Alexander GQ Gerasiov",
+    "Alexander Kazakov",
     "Alexander Krasheninnikov",
     "Alexander Kuranoff",
     "Alexander Lukin",
     "Alexander Makarov",
     "Alexander Marshalov",
     "Alexander Millin",
+    "Alexander Mochalin",
     "Alexander Prudaev",
     "Alexander Sapin",
     "Alexander Tokmakov",
@@ -54,6 +57,7 @@ const char * auto_contributors[] {
     "Artemkin Pavel",
     "Arthur Tokarchuk",
     "Atri Sharma",
+    "Azat Khuzhin",
     "BSD_Conqueror",
     "Babacar Diassé",
     "BanyRule",
@@ -63,6 +67,7 @@ const char * auto_contributors[] {
     "Bogdan Voronin",
     "Bolinov",
     "Boris Granveaud",
+    "Bowen Masco",
     "Brett Hoerner",
     "Bulat Gaifullin",
     "Chen Yufei",
@@ -94,12 +99,14 @@ const char * auto_contributors[] {
     "Fadi Hadzh",
     "Flowyi",
     "Fruit of Eden",
+    "Gary Dotzler",
     "George",
     "George G",
     "George3d6",
     "Gleb Kanterov",
     "Guillaume Tassery",
     "Hamoon",
+    "Hasnat",
     "Hiroaki Nakamura",
     "Igor",
     "Igor Hatarist",
@@ -146,6 +153,7 @@ const char * auto_contributors[] {
     "Marek Vavruša",
     "Marsel Arduanov",
     "Marti Raudsepp",
+    "Martijn Bakker",
     "Max",
     "Max Akhmedov",
     "Max Vetrov",
@@ -170,14 +178,17 @@ const char * auto_contributors[] {
     "Mikhail Surin",
     "Mikhail f. Shiryaev",
     "Milad Arabi",
+    "Mohammad Hossein Sekhavat",
     "Murat Kabilov",
     "Narek Galstyan",
     "NeZeD [Mac Pro]",
+    "Neeke Gao",
     "Nicolae Vartolomei",
     "Nikhil Raman",
     "Nikita Vasilev",
     "Nikolai Kochetov",
     "Nikolay Kirsh",
+    "Nikolay Semyachkin",
     "Nikolay Vasiliev",
     "Nikolay Volosatov",
     "Odin Hultgren Van Der Horst",
@@ -198,12 +209,14 @@ const char * auto_contributors[] {
     "Ravengg",
     "Reto Kromer",
     "Roman Lipovsky",
+    "Roman Nikolaev",
     "Roman Nozdrin",
     "Roman Peshkurov",
     "Roman Tsisyk",
     "Sabyanin Maxim",
     "SaltTan",
     "Samuel Chou",
+    "Serge Rider",
     "Sergei Semin",
     "Sergei Tsetlin (rekub)",
     "Sergey Elantsev",
@@ -212,14 +225,17 @@ const char * auto_contributors[] {
     "Sergey Magidovich",
     "Sergey V. Galtsev",
     "Sergey Zaikin",
+    "SevaCode",
     "Silviu Caragea",
     "Simon Podlipsky",
     "Sjoerd Mulder",
     "Snow",
     "Stanislav Pavlovichev",
     "Stas Pavlovichev",
+    "Stefan Thies",
     "Stupnikov Andrey",
     "SuperBot",
+    "Sébastien Launay",
     "Tangaev",
     "Tema Novikov",
     "The-Alchemist",
@@ -247,6 +263,7 @@ const char * auto_contributors[] {
     "Vladimir Kozbin",
     "Vladimir Smirnov",
     "Vladislav Rassokhin",
+    "Vladislav Smirnov",
     "Vojtech Splichal",
     "Vsevolod Orlov",
     "Vyacheslav Alipov",
@@ -301,7 +318,9 @@ const char * auto_contributors[] {
     "ks1322",
     "kshvakov",
     "leozhang",
+    "levysh",
     "liangqian",
+    "liuyangkuan",
     "liuyimin",
     "lomberts",
     "luc1ph3r",
@@ -311,10 +330,12 @@ const char * auto_contributors[] {
     "miha-g",
     "morty",
     "moscas",
+    "never lee",
     "nicelulu",
     "ns-vasilev",
     "objatie_groba",
     "ogorbacheva",
+    "olegkv",
     "orantius",
     "peshkurov",
     "proller",
@@ -332,6 +353,7 @@ const char * auto_contributors[] {
     "stavrolia",
     "sundy-li",
     "sundyli",
+    "tai",
     "topvisor",
     "urgordeadbeef",
     "velom",
@@ -346,5 +368,6 @@ const char * auto_contributors[] {
     "Сундуков Алексей",
     "小路",
     "张健",
+    "张风啸",
     "谢磊",
     nullptr};