From 0f20952f2ce98329ed64c567484272905bcfc6d9 Mon Sep 17 00:00:00 2001
From: zhang2014 <coswde@gmail.com>
Date: Sat, 10 Feb 2018 17:21:54 +0800
Subject: [PATCH 001/244] ISSUES-1885 UTF8 countCodePoints use simd

---
 dbms/src/Common/UTF8Helpers.h | 35 +++++++++++++++++++++++++++++++++--
 1 file changed, 33 insertions(+), 2 deletions(-)

diff --git a/dbms/src/Common/UTF8Helpers.h b/dbms/src/Common/UTF8Helpers.h
index 1ce31426e85..ba1685c89f3 100644
--- a/dbms/src/Common/UTF8Helpers.h
+++ b/dbms/src/Common/UTF8Helpers.h
@@ -3,6 +3,9 @@
 #include <Core/Types.h>
 #include <Common/BitHelpers.h>
 
+#if __SSE2__
+#include <emmintrin.h>
+#endif
 
 namespace DB
 {
@@ -49,9 +52,37 @@ inline size_t seqLength(const UInt8 first_octet)
 inline size_t countCodePoints(const UInt8 * data, size_t size)
 {
     size_t res = 0;
+    const auto end = data + size;
 
-    /// TODO SIMD implementation looks quite simple.
-    for (auto end = data + size; data < end; ++data) /// Skip UTF-8 continuation bytes.
+#if __SSE2__
+    const auto bytes_sse = sizeof(__m128i);
+    const auto src_end_sse = (data + size) - (size % bytes_sse);
+
+    const auto upper_bound = _mm_set1_epi8(0x7F + 1);
+    const auto lower_bound = _mm_set1_epi8(0xC0 - 1);
+
+    for (; data < src_end_sse;)
+    {
+        UInt8 mem_res[16] = {0};
+        auto sse_res = _mm_set1_epi8(0);
+
+        for (int i = 0; i < 0XFF && data < src_end_sse; ++i, data += bytes_sse)
+        {
+            const auto chars = _mm_loadu_si128(reinterpret_cast<const __m128i *>(data));
+            sse_res = _mm_add_epi8(sse_res,
+                                   _mm_or_si128(_mm_cmplt_epi8(chars, upper_bound),
+                                                _mm_cmpgt_epi8(chars, lower_bound)));
+        }
+
+        _mm_store_si128(reinterpret_cast<__m128i *>(mem_res), sse_res);
+
+        for (auto count : mem_res)
+            res += count;
+    }
+
+#endif
+
+    for (; data < end; ++data) /// Skip UTF-8 continuation bytes.
         res += (*data <= 0x7F || *data >= 0xC0);
 
     return res;

From 73077e02459fb1a6da866aa785c21c4fba2d5d01 Mon Sep 17 00:00:00 2001
From: zhang2014 <coswde@gmail.com>
Date: Sun, 11 Feb 2018 22:38:51 +0800
Subject: [PATCH 002/244] ISSUES-1885 fix failed test

---
 dbms/src/Common/UTF8Helpers.h | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/dbms/src/Common/UTF8Helpers.h b/dbms/src/Common/UTF8Helpers.h
index ba1685c89f3..c2f57b8107a 100644
--- a/dbms/src/Common/UTF8Helpers.h
+++ b/dbms/src/Common/UTF8Helpers.h
@@ -2,6 +2,7 @@
 
 #include <Core/Types.h>
 #include <Common/BitHelpers.h>
+#include <iostream>
 
 #if __SSE2__
 #include <emmintrin.h>
@@ -58,23 +59,27 @@ inline size_t countCodePoints(const UInt8 * data, size_t size)
     const auto bytes_sse = sizeof(__m128i);
     const auto src_end_sse = (data + size) - (size % bytes_sse);
 
+    const auto fill_sse = _mm_set1_epi8(0xFF);
     const auto upper_bound = _mm_set1_epi8(0x7F + 1);
     const auto lower_bound = _mm_set1_epi8(0xC0 - 1);
 
     for (; data < src_end_sse;)
     {
-        UInt8 mem_res[16] = {0};
+        UInt16 mem_res[8] = {0};
         auto sse_res = _mm_set1_epi8(0);
 
-        for (int i = 0; i < 0XFF && data < src_end_sse; ++i, data += bytes_sse)
+        for (int i = 0, limit = 0xFF; i < limit && data < src_end_sse; ++i, data += bytes_sse)
         {
             const auto chars = _mm_loadu_si128(reinterpret_cast<const __m128i *>(data));
-            sse_res = _mm_add_epi8(sse_res,
-                                   _mm_or_si128(_mm_cmplt_epi8(chars, upper_bound),
-                                                _mm_cmpgt_epi8(chars, lower_bound)));
+
+            const auto chars_cmpgt_res = _mm_or_si128(_mm_cmplt_epi8(chars, upper_bound),
+                                                      _mm_cmpgt_epi8(chars, lower_bound));
+
+            sse_res = _mm_add_epi8(sse_res, _mm_add_epi8(_mm_xor_si128(chars_cmpgt_res, fill_sse), _mm_set1_epi8(1)));
         }
 
-        _mm_store_si128(reinterpret_cast<__m128i *>(mem_res), sse_res);
+        auto horizontal_add = _mm_sad_epu8(sse_res, _mm_set1_epi8(0));
+        _mm_store_si128(reinterpret_cast<__m128i *>(mem_res), horizontal_add);
 
         for (auto count : mem_res)
             res += count;

From 5f9292ac30056d23425399b451df9f24a14f09f1 Mon Sep 17 00:00:00 2001
From: zhang2014 <coswde@gmail.com>
Date: Mon, 12 Feb 2018 15:27:51 +0800
Subject: [PATCH 003/244] ISSUES-1885 resolve some opinions

---
 dbms/src/Common/UTF8Helpers.h | 32 ++++++++++++++++++++------------
 1 file changed, 20 insertions(+), 12 deletions(-)

diff --git a/dbms/src/Common/UTF8Helpers.h b/dbms/src/Common/UTF8Helpers.h
index c2f57b8107a..71ce6e4af51 100644
--- a/dbms/src/Common/UTF8Helpers.h
+++ b/dbms/src/Common/UTF8Helpers.h
@@ -2,12 +2,15 @@
 
 #include <Core/Types.h>
 #include <Common/BitHelpers.h>
-#include <iostream>
 
 #if __SSE2__
 #include <emmintrin.h>
 #endif
 
+#if __SSE4_1__
+#include <smmintrin.h>
+#endif
+
 namespace DB
 {
 
@@ -55,34 +58,39 @@ inline size_t countCodePoints(const UInt8 * data, size_t size)
     size_t res = 0;
     const auto end = data + size;
 
-#if __SSE2__
+#if __SSE2__ && __SSE4_1__
     const auto bytes_sse = sizeof(__m128i);
     const auto src_end_sse = (data + size) - (size % bytes_sse);
 
-    const auto fill_sse = _mm_set1_epi8(0xFF);
-    const auto upper_bound = _mm_set1_epi8(0x7F + 1);
-    const auto lower_bound = _mm_set1_epi8(0xC0 - 1);
+    const auto one_sse = _mm_set1_epi8(1);
+    const auto zero_sse = _mm_set1_epi8(0);
+
+    const auto align_sse = _mm_set1_epi8(0xFF - 0xC0);
+    const auto upper_bound = _mm_set1_epi8(0x7F + (0xFF - 0xC0) + 1);
 
     for (; data < src_end_sse;)
     {
-        UInt16 mem_res[8] = {0};
         auto sse_res = _mm_set1_epi8(0);
 
         for (int i = 0, limit = 0xFF; i < limit && data < src_end_sse; ++i, data += bytes_sse)
         {
             const auto chars = _mm_loadu_si128(reinterpret_cast<const __m128i *>(data));
 
-            const auto chars_cmpgt_res = _mm_or_si128(_mm_cmplt_epi8(chars, upper_bound),
-                                                      _mm_cmpgt_epi8(chars, lower_bound));
+            ///Align to zero for the solve two case
+            const auto align_res = _mm_add_epi8(chars, align_sse);
+            ///Because _mm_cmmpgt_epid8 return 0xFF if true
+            const auto choose_res = _mm_blendv_epi8(zero_sse, one_sse, _mm_cmpgt_epi8(align_res, upper_bound));
 
-            sse_res = _mm_add_epi8(sse_res, _mm_add_epi8(_mm_xor_si128(chars_cmpgt_res, fill_sse), _mm_set1_epi8(1)));
+            sse_res = _mm_add_epi8(sse_res, choose_res);
         }
 
-        auto horizontal_add = _mm_sad_epu8(sse_res, _mm_set1_epi8(0));
+        UInt16 mem_res[8] = {0};
+        auto horizontal_add = _mm_sad_epu8(sse_res, zero_sse);
         _mm_store_si128(reinterpret_cast<__m128i *>(mem_res), horizontal_add);
 
-        for (auto count : mem_res)
-            res += count;
+        /// Hack,Because only bytes MSB to LSB
+        res += mem_res[0];
+        res += mem_res[4];
     }
 
 #endif

From acb5b99195171b104c5a04c918e3b12f23517dd1 Mon Sep 17 00:00:00 2001
From: zhang2014 <coswde@gmail.com>
Date: Mon, 12 Feb 2018 16:05:10 +0800
Subject: [PATCH 004/244] ISSUES-1885 fix align bound

---
 dbms/src/Common/UTF8Helpers.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dbms/src/Common/UTF8Helpers.h b/dbms/src/Common/UTF8Helpers.h
index 71ce6e4af51..c1aa9930eaa 100644
--- a/dbms/src/Common/UTF8Helpers.h
+++ b/dbms/src/Common/UTF8Helpers.h
@@ -65,8 +65,8 @@ inline size_t countCodePoints(const UInt8 * data, size_t size)
     const auto one_sse = _mm_set1_epi8(1);
     const auto zero_sse = _mm_set1_epi8(0);
 
-    const auto align_sse = _mm_set1_epi8(0xFF - 0xC0);
-    const auto upper_bound = _mm_set1_epi8(0x7F + (0xFF - 0xC0) + 1);
+    const auto align_sse = _mm_set1_epi8(0xFF - 0xC0 + 1);
+    const auto upper_bound = _mm_set1_epi8(0x7F + (0xFF - 0xC0 + 1) + 1);
 
     for (; data < src_end_sse;)
     {

From 0d8730c687f2ebaa67a3a2b3bd0ca0faaf2e82ec Mon Sep 17 00:00:00 2001
From: zhang2014 <coswde@gmail.com>
Date: Tue, 13 Feb 2018 13:46:16 +0800
Subject: [PATCH 005/244] ISSUES-1885 fix signed problem

---
 dbms/src/Common/UTF8Helpers.h | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/dbms/src/Common/UTF8Helpers.h b/dbms/src/Common/UTF8Helpers.h
index c1aa9930eaa..698ff369684 100644
--- a/dbms/src/Common/UTF8Helpers.h
+++ b/dbms/src/Common/UTF8Helpers.h
@@ -63,10 +63,10 @@ inline size_t countCodePoints(const UInt8 * data, size_t size)
     const auto src_end_sse = (data + size) - (size % bytes_sse);
 
     const auto one_sse = _mm_set1_epi8(1);
-    const auto zero_sse = _mm_set1_epi8(0);
+    const auto zero_sse = _mm_setzero_si128();
 
-    const auto align_sse = _mm_set1_epi8(0xFF - 0xC0 + 1);
-    const auto upper_bound = _mm_set1_epi8(0x7F + (0xFF - 0xC0 + 1) + 1);
+    const auto align_sse = _mm_set1_epi8(0x40);
+    const auto upper_bound = _mm_set1_epi8(0xBF);
 
     for (; data < src_end_sse;)
     {
@@ -77,11 +77,12 @@ inline size_t countCodePoints(const UInt8 * data, size_t size)
             const auto chars = _mm_loadu_si128(reinterpret_cast<const __m128i *>(data));
 
             ///Align to zero for the solve two case
-            const auto align_res = _mm_add_epi8(chars, align_sse);
-            ///Because _mm_cmmpgt_epid8 return 0xFF if true
-            const auto choose_res = _mm_blendv_epi8(zero_sse, one_sse, _mm_cmpgt_epi8(align_res, upper_bound));
+            const auto align_res = _mm_adds_epu8(chars, align_sse);
+            const auto less_than_and_equals = _mm_cmpeq_epi8(_mm_min_epu8(align_res, upper_bound), align_res);
+            ///Because _mm_cmpeq_epi8 return 0xFF if true
+            const auto choose_res = _mm_blendv_epi8(zero_sse, one_sse, less_than_and_equals);
 
-            sse_res = _mm_add_epi8(sse_res, choose_res);
+            sse_res = _mm_adds_epu8(sse_res, choose_res);
         }
 
         UInt16 mem_res[8] = {0};

From 6033096fd1d6aabd4fe6dafd1c8c52c1a16ae2e0 Mon Sep 17 00:00:00 2001
From: zhang2014 <coswde@gmail.com>
Date: Tue, 13 Feb 2018 16:12:20 +0800
Subject: [PATCH 006/244] ISSUES-1885 add performance test

---
 .../functions_length/functions_length.xml     | 38 +++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100644 dbms/tests/performance/functions_length/functions_length.xml

diff --git a/dbms/tests/performance/functions_length/functions_length.xml b/dbms/tests/performance/functions_length/functions_length.xml
new file mode 100644
index 00000000000..d285cd2422f
--- /dev/null
+++ b/dbms/tests/performance/functions_length/functions_length.xml
@@ -0,0 +1,38 @@
+<test>
+    <name>functions_length</name>
+    <type>once</type>
+
+    <stop_conditions>
+        <all_of>
+            <total_time_ms>10000</total_time_ms>
+        </all_of>
+        <any_of>
+            <average_speed_not_changing_for_ms>5000</average_speed_not_changing_for_ms>
+            <total_time_ms>20000</total_time_ms>
+        </any_of>
+    </stop_conditions>
+
+    <main_metric>
+        <avg_rows_per_second/>
+    </main_metric>
+
+    <substitutions>
+        <substitution>
+            <name>string</name>
+            <values>
+                <value>materialize('')</value>
+                <value>materialize('Hello, world')</value>
+                <value>toString(number)</value>
+                <value>reinterpretAsString(number)</value>
+                <value>materialize('中文测试字符串')</value>
+                <value>materialize('https://github.com/yandex/ClickHouse/pull/1882')</value>
+                <value>materialize('https://zh.wikipedia.org/wiki/%E4%B8%AD%E6%97%A5%E9%9F%93%E7%B5%B1%E4%B8%80%E8%A1%A8%E6%84%8F%E6%96%87%E5%AD%97%E6%93%B4%E5%B1%95%E5%8D%80F')</value>
+                <value>concat('中文测试字符串 ', toString(number), ' Привет, мир!')</value>
+                <value>concat(concat('中文测试字符串 ', toString(number), ' Привет, мир!') AS x, x, x, x, x, x, x, x, x, x)</value>
+                <value>convertCharset(concat(reinterpretAsString(rand64(1)), reinterpretAsString(rand64(2)), reinterpretAsString(rand64(3)), reinterpretAsString(rand64(4)), reinterpretAsString(rand64(5)), reinterpretAsString(rand64(6)), reinterpretAsString(rand64(7)), reinterpretAsString(rand64(8)), reinterpretAsString(rand64(9)), reinterpretAsString(rand64(10))), 'UTF-16', 'UTF-8')</value>
+            </values>
+        </substitution>
+    </substitutions>
+
+    <query>SELECT count() FROM system.numbers WHERE NOT ignore(lengthUTF8({string}))</query>
+</test>

From eeae014a42affed560781499625c6ed6f705cb99 Mon Sep 17 00:00:00 2001
From: zhang2014 <coswde@gmail.com>
Date: Wed, 14 Feb 2018 08:11:12 +0800
Subject: [PATCH 007/244] ISSUES-1885 use movemask & popcnt

---
 dbms/src/Common/UTF8Helpers.h | 36 +++++++----------------------------
 1 file changed, 7 insertions(+), 29 deletions(-)

diff --git a/dbms/src/Common/UTF8Helpers.h b/dbms/src/Common/UTF8Helpers.h
index 698ff369684..0237b6f036c 100644
--- a/dbms/src/Common/UTF8Helpers.h
+++ b/dbms/src/Common/UTF8Helpers.h
@@ -7,10 +7,6 @@
 #include <emmintrin.h>
 #endif
 
-#if __SSE4_1__
-#include <smmintrin.h>
-#endif
-
 namespace DB
 {
 
@@ -58,40 +54,22 @@ inline size_t countCodePoints(const UInt8 * data, size_t size)
     size_t res = 0;
     const auto end = data + size;
 
-#if __SSE2__ && __SSE4_1__
+#if __SSE2__
     const auto bytes_sse = sizeof(__m128i);
     const auto src_end_sse = (data + size) - (size % bytes_sse);
 
-    const auto one_sse = _mm_set1_epi8(1);
-    const auto zero_sse = _mm_setzero_si128();
-
     const auto align_sse = _mm_set1_epi8(0x40);
     const auto upper_bound = _mm_set1_epi8(0xBF);
 
-    for (; data < src_end_sse;)
+    for (; data < src_end_sse; data += bytes_sse)
     {
-        auto sse_res = _mm_set1_epi8(0);
+        const auto chars = _mm_loadu_si128(reinterpret_cast<const __m128i *>(data));
 
-        for (int i = 0, limit = 0xFF; i < limit && data < src_end_sse; ++i, data += bytes_sse)
-        {
-            const auto chars = _mm_loadu_si128(reinterpret_cast<const __m128i *>(data));
+        ///Align to zero for the solve two case
+        const auto align_res = _mm_adds_epu8(chars, align_sse);
+        const auto less_than_and_equals = _mm_cmpeq_epi8(_mm_min_epu8(align_res, upper_bound), align_res);
 
-            ///Align to zero for the solve two case
-            const auto align_res = _mm_adds_epu8(chars, align_sse);
-            const auto less_than_and_equals = _mm_cmpeq_epi8(_mm_min_epu8(align_res, upper_bound), align_res);
-            ///Because _mm_cmpeq_epi8 return 0xFF if true
-            const auto choose_res = _mm_blendv_epi8(zero_sse, one_sse, less_than_and_equals);
-
-            sse_res = _mm_adds_epu8(sse_res, choose_res);
-        }
-
-        UInt16 mem_res[8] = {0};
-        auto horizontal_add = _mm_sad_epu8(sse_res, zero_sse);
-        _mm_store_si128(reinterpret_cast<__m128i *>(mem_res), horizontal_add);
-
-        /// Hack,Because only bytes MSB to LSB
-        res += mem_res[0];
-        res += mem_res[4];
+        res += __builtin_popcount(_mm_movemask_epi8(less_than_and_equals));
     }
 
 #endif

From 0b26189fabfa5e3fce522c53fb0895031360a3be Mon Sep 17 00:00:00 2001
From: Kirill Shvakov <shvakov@gmail.com>
Date: Sat, 24 Feb 2018 14:06:53 +0200
Subject: [PATCH 008/244] prototype: show server group name in client

---
 dbms/src/Client/Connection.cpp | 12 ++++++++++++
 dbms/src/Client/Connection.h   |  2 ++
 dbms/src/Core/Defines.h        |  1 +
 dbms/src/Server/Client.cpp     |  5 ++++-
 dbms/src/Server/TCPHandler.cpp |  4 ++++
 dbms/src/Server/config.xml     |  2 +-
 6 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/dbms/src/Client/Connection.cpp b/dbms/src/Client/Connection.cpp
index 3c6c8d75d0e..9b57c1b39b2 100644
--- a/dbms/src/Client/Connection.cpp
+++ b/dbms/src/Client/Connection.cpp
@@ -148,6 +148,10 @@ void Connection::receiveHello()
         {
             readStringBinary(server_timezone, *in);
         }
+        if (server_revision >= DBMS_MIN_REVISION_WITH_SERVER_GROUP_NAME)
+        {
+            readStringBinary(server_group_name, *in);
+        }
     }
     else if (packet_type == Protocol::Server::Exception)
         receiveException()->rethrow();
@@ -203,6 +207,14 @@ const String & Connection::getServerTimezone()
     return server_timezone;
 }
 
+const String & Connection::getServerGroupname()
+{
+    if (!connected)
+        connect();
+
+    return server_group_name;
+}
+
 void Connection::forceConnected()
 {
     if (!connected)
diff --git a/dbms/src/Client/Connection.h b/dbms/src/Client/Connection.h
index c9b401c91ee..06f8e94e29c 100644
--- a/dbms/src/Client/Connection.h
+++ b/dbms/src/Client/Connection.h
@@ -134,6 +134,7 @@ public:
     void getServerVersion(String & name, UInt64 & version_major, UInt64 & version_minor, UInt64 & revision);
 
     const String & getServerTimezone();
+    const String & getServerGroupname();
 
     /// For log and exception messages.
     const String & getDescription() const;
@@ -213,6 +214,7 @@ private:
     UInt64 server_version_minor = 0;
     UInt64 server_revision = 0;
     String server_timezone;
+    String server_group_name;
 
     std::unique_ptr<Poco::Net::StreamSocket> socket;
     std::shared_ptr<ReadBuffer> in;
diff --git a/dbms/src/Core/Defines.h b/dbms/src/Core/Defines.h
index 8b69d83d69e..180d2a0dc2b 100644
--- a/dbms/src/Core/Defines.h
+++ b/dbms/src/Core/Defines.h
@@ -60,6 +60,7 @@
 #define DBMS_MIN_REVISION_WITH_QUOTA_KEY_IN_CLIENT_INFO 54060
 #define DBMS_MIN_REVISION_WITH_TABLES_STATUS 54226
 #define DBMS_MIN_REVISION_WITH_TIME_ZONE_PARAMETER_IN_DATETIME_DATA_TYPE 54337
+#define DBMS_MIN_REVISION_WITH_SERVER_GROUP_NAME 54355
 
 /// Version of ClickHouse TCP protocol. Set to git tag with latest protocol change.
 #define DBMS_TCP_PROTOCOL_VERSION 54226
diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp
index 238be504aac..1189dd4d6bc 100644
--- a/dbms/src/Server/Client.cpp
+++ b/dbms/src/Server/Client.cpp
@@ -153,6 +153,7 @@ private:
     /// If the last query resulted in exception.
     bool got_exception = false;
     String server_version;
+    String server_group_name;
 
     Stopwatch watch;
 
@@ -425,6 +426,7 @@ private:
         connection->getServerVersion(server_name, server_version_major, server_version_minor, server_revision);
 
         server_version = toString(server_version_major) + "." + toString(server_version_minor) + "." + toString(server_revision);
+        server_group_name = connection->getServerGroupname();
         if (is_interactive)
         {
             std::cout << "Connected to " << server_name
@@ -450,7 +452,8 @@ private:
     {
         String query;
         String prev_query;
-        while (char * line_ = readline(query.empty() ? ":) " : ":-] "))
+        String prompt = server_group_name.length() ? "[" + server_group_name + "] :) " : ":) ";
+        while (char * line_ = readline(query.empty() ? prompt.c_str() : ":-] "))
         {
             String line = line_;
             free(line_);
diff --git a/dbms/src/Server/TCPHandler.cpp b/dbms/src/Server/TCPHandler.cpp
index 7dadeef966b..70d16d6a16b 100644
--- a/dbms/src/Server/TCPHandler.cpp
+++ b/dbms/src/Server/TCPHandler.cpp
@@ -508,6 +508,10 @@ void TCPHandler::sendHello()
     {
         writeStringBinary(DateLUT::instance().getTimeZone(), *out);
     }
+    if (client_revision >= DBMS_MIN_REVISION_WITH_SERVER_GROUP_NAME)
+    {
+        writeStringBinary(server.config().getString("group_name"), *out);
+    }
     out->next();
 }
 
diff --git a/dbms/src/Server/config.xml b/dbms/src/Server/config.xml
index e0b4ea563ac..3fe4f580f40 100644
--- a/dbms/src/Server/config.xml
+++ b/dbms/src/Server/config.xml
@@ -8,7 +8,7 @@
         <count>10</count>
         <!-- <console>1</console> --> <!-- Default behavior is autodetection (log to console if not daemon mode and is tty) -->
     </logger>
-
+    <group_name>staging</group_name>
     <http_port>8123</http_port>
     <tcp_port>9000</tcp_port>
 

From 2216f9e0cccbed113744d475c13087ade37ff2c9 Mon Sep 17 00:00:00 2001
From: Kirill Shvakov <shvakov@gmail.com>
Date: Sat, 24 Feb 2018 14:36:19 +0200
Subject: [PATCH 009/244] fix function name

---
 dbms/src/Client/Connection.cpp | 2 +-
 dbms/src/Client/Connection.h   | 2 +-
 dbms/src/Server/Client.cpp     | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/dbms/src/Client/Connection.cpp b/dbms/src/Client/Connection.cpp
index 9b57c1b39b2..4e5ee88c782 100644
--- a/dbms/src/Client/Connection.cpp
+++ b/dbms/src/Client/Connection.cpp
@@ -207,7 +207,7 @@ const String & Connection::getServerTimezone()
     return server_timezone;
 }
 
-const String & Connection::getServerGroupname()
+const String & Connection::getServerGroupName()
 {
     if (!connected)
         connect();
diff --git a/dbms/src/Client/Connection.h b/dbms/src/Client/Connection.h
index 06f8e94e29c..0caf6ead9f2 100644
--- a/dbms/src/Client/Connection.h
+++ b/dbms/src/Client/Connection.h
@@ -134,7 +134,7 @@ public:
     void getServerVersion(String & name, UInt64 & version_major, UInt64 & version_minor, UInt64 & revision);
 
     const String & getServerTimezone();
-    const String & getServerGroupname();
+    const String & getServerGroupName();
 
     /// For log and exception messages.
     const String & getDescription() const;
diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp
index 1189dd4d6bc..f644da2f3ea 100644
--- a/dbms/src/Server/Client.cpp
+++ b/dbms/src/Server/Client.cpp
@@ -426,7 +426,7 @@ private:
         connection->getServerVersion(server_name, server_version_major, server_version_minor, server_revision);
 
         server_version = toString(server_version_major) + "." + toString(server_version_minor) + "." + toString(server_revision);
-        server_group_name = connection->getServerGroupname();
+        server_group_name = connection->getServerGroupName();
         if (is_interactive)
         {
             std::cout << "Connected to " << server_name

From d3b6aafa097c3c772531ea30b07bd9348bf43856 Mon Sep 17 00:00:00 2001
From: Kirill Shvakov <shvakov@gmail.com>
Date: Mon, 26 Feb 2018 08:49:17 +0200
Subject: [PATCH 010/244] Rename group_name to display_name

---
 dbms/src/Client/Connection.cpp | 8 ++++----
 dbms/src/Client/Connection.h   | 4 ++--
 dbms/src/Core/Defines.h        | 2 +-
 dbms/src/Server/Client.cpp     | 6 +++---
 dbms/src/Server/TCPHandler.cpp | 5 +++--
 dbms/src/Server/config.xml     | 2 +-
 6 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/dbms/src/Client/Connection.cpp b/dbms/src/Client/Connection.cpp
index 4e5ee88c782..0960ac8e77d 100644
--- a/dbms/src/Client/Connection.cpp
+++ b/dbms/src/Client/Connection.cpp
@@ -148,9 +148,9 @@ void Connection::receiveHello()
         {
             readStringBinary(server_timezone, *in);
         }
-        if (server_revision >= DBMS_MIN_REVISION_WITH_SERVER_GROUP_NAME)
+        if (server_revision >= DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME)
         {
-            readStringBinary(server_group_name, *in);
+            readStringBinary(server_display_name, *in);
         }
     }
     else if (packet_type == Protocol::Server::Exception)
@@ -207,12 +207,12 @@ const String & Connection::getServerTimezone()
     return server_timezone;
 }
 
-const String & Connection::getServerGroupName()
+const String & Connection::getServerDisplayName()
 {
     if (!connected)
         connect();
 
-    return server_group_name;
+    return server_display_name;
 }
 
 void Connection::forceConnected()
diff --git a/dbms/src/Client/Connection.h b/dbms/src/Client/Connection.h
index 0caf6ead9f2..cac188f12c8 100644
--- a/dbms/src/Client/Connection.h
+++ b/dbms/src/Client/Connection.h
@@ -134,7 +134,7 @@ public:
     void getServerVersion(String & name, UInt64 & version_major, UInt64 & version_minor, UInt64 & revision);
 
     const String & getServerTimezone();
-    const String & getServerGroupName();
+    const String & getServerDisplayName();
 
     /// For log and exception messages.
     const String & getDescription() const;
@@ -214,7 +214,7 @@ private:
     UInt64 server_version_minor = 0;
     UInt64 server_revision = 0;
     String server_timezone;
-    String server_group_name;
+    String server_display_name;
 
     std::unique_ptr<Poco::Net::StreamSocket> socket;
     std::shared_ptr<ReadBuffer> in;
diff --git a/dbms/src/Core/Defines.h b/dbms/src/Core/Defines.h
index 180d2a0dc2b..1cc14a62155 100644
--- a/dbms/src/Core/Defines.h
+++ b/dbms/src/Core/Defines.h
@@ -60,7 +60,7 @@
 #define DBMS_MIN_REVISION_WITH_QUOTA_KEY_IN_CLIENT_INFO 54060
 #define DBMS_MIN_REVISION_WITH_TABLES_STATUS 54226
 #define DBMS_MIN_REVISION_WITH_TIME_ZONE_PARAMETER_IN_DATETIME_DATA_TYPE 54337
-#define DBMS_MIN_REVISION_WITH_SERVER_GROUP_NAME 54355
+#define DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME 54355
 
 /// Version of ClickHouse TCP protocol. Set to git tag with latest protocol change.
 #define DBMS_TCP_PROTOCOL_VERSION 54226
diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp
index f644da2f3ea..4760c8d4874 100644
--- a/dbms/src/Server/Client.cpp
+++ b/dbms/src/Server/Client.cpp
@@ -153,7 +153,7 @@ private:
     /// If the last query resulted in exception.
     bool got_exception = false;
     String server_version;
-    String server_group_name;
+    String server_display_name;
 
     Stopwatch watch;
 
@@ -426,7 +426,7 @@ private:
         connection->getServerVersion(server_name, server_version_major, server_version_minor, server_revision);
 
         server_version = toString(server_version_major) + "." + toString(server_version_minor) + "." + toString(server_revision);
-        server_group_name = connection->getServerGroupName();
+        server_display_name = connection->getServerDisplayName();
         if (is_interactive)
         {
             std::cout << "Connected to " << server_name
@@ -452,7 +452,7 @@ private:
     {
         String query;
         String prev_query;
-        String prompt = server_group_name.length() ? "[" + server_group_name + "] :) " : ":) ";
+        String prompt = server_display_name.length() ? "[" + server_display_name + "] :) " : ":) ";
         while (char * line_ = readline(query.empty() ? prompt.c_str() : ":-] "))
         {
             String line = line_;
diff --git a/dbms/src/Server/TCPHandler.cpp b/dbms/src/Server/TCPHandler.cpp
index 70d16d6a16b..e6ac7b5d5d6 100644
--- a/dbms/src/Server/TCPHandler.cpp
+++ b/dbms/src/Server/TCPHandler.cpp
@@ -31,6 +31,7 @@
 #include "TCPHandler.h"
 
 #include <Common/NetException.h>
+#include <Common/getFQDNOrHostName.h>
 
 
 namespace DB
@@ -508,9 +509,9 @@ void TCPHandler::sendHello()
     {
         writeStringBinary(DateLUT::instance().getTimeZone(), *out);
     }
-    if (client_revision >= DBMS_MIN_REVISION_WITH_SERVER_GROUP_NAME)
+    if (client_revision >= DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME)
     {
-        writeStringBinary(server.config().getString("group_name"), *out);
+        writeStringBinary(server.config().getString("display_name", getFQDNOrHostName()), *out);
     }
     out->next();
 }
diff --git a/dbms/src/Server/config.xml b/dbms/src/Server/config.xml
index 3fe4f580f40..4c1afb53639 100644
--- a/dbms/src/Server/config.xml
+++ b/dbms/src/Server/config.xml
@@ -8,7 +8,7 @@
         <count>10</count>
         <!-- <console>1</console> --> <!-- Default behavior is autodetection (log to console if not daemon mode and is tty) -->
     </logger>
-    <group_name>staging</group_name>
+    <!--display_name>production</display_name--> <!-- It is the name that will be shown in the client -->
     <http_port>8123</http_port>
     <tcp_port>9000</tcp_port>
 

From f15b55f9295b6422ece51bad2c0e0fccf4e5fd6e Mon Sep 17 00:00:00 2001
From: Kirill Shvakov <shvakov@gmail.com>
Date: Mon, 26 Feb 2018 15:24:06 +0200
Subject: [PATCH 011/244] prototype: custom prompt

---
 dbms/src/Server/Client.cpp            | 59 ++++++++++++++++++++++++++-
 dbms/src/Server/clickhouse-client.xml |  4 ++
 2 files changed, 61 insertions(+), 2 deletions(-)

diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp
index 4760c8d4874..989a83a0030 100644
--- a/dbms/src/Server/Client.cpp
+++ b/dbms/src/Server/Client.cpp
@@ -3,6 +3,7 @@
 #include <fcntl.h>
 #include <signal.h>
 
+#include <map>
 #include <iostream>
 #include <fstream>
 #include <iomanip>
@@ -10,6 +11,7 @@
 #include <algorithm>
 #include <optional>
 #include <boost/program_options.hpp>
+#include <boost/algorithm/string/replace.hpp>
 
 #include <Poco/File.h>
 #include <Poco/Util/Application.h>
@@ -138,6 +140,8 @@ private:
 
     String current_profile;
 
+    String prompt_by_server_display_name = "%display_name% :)";
+
     /// Path to a file containing command history.
     String history_file;
 
@@ -335,6 +339,18 @@ private:
             }
         }
 
+        Strings keys;
+        config().keys("prompt_by_server_display_name", keys);
+
+        for (const String & key : keys)
+        {
+            if (server_display_name.find(key) != std::string::npos) 
+            {
+                prompt_by_server_display_name = config().getString("prompt_by_server_display_name." + key);
+                break;
+            }
+        }
+
         if (is_interactive)
         {
             if (print_time_to_stderr)
@@ -447,13 +463,52 @@ private:
         return select(1, &fds, 0, 0, &timeout) == 1;
     }
 
+    const String prompt() const
+    {
+        std::map<String, String> colors = {
+            { "red",     "31" },
+            { "green",   "32" },
+            { "yellow",  "33" },
+            { "blue",    "34" },
+            { "magenta", "35" },
+            { "cyan",    "36" },
+        };
+
+        String pattern = prompt_by_server_display_name;
+
+        for (const auto & pair: colors)
+        {
+            String name = static_cast<String>(pair.first);
+            String code = static_cast<String>(pair.second);
+
+            boost::replace_all(pattern, "[" + name + "]",  "\33[1;" + code + "m");
+            boost::replace_all(pattern, "[/" + name + "]", "\033[0m");
+        }
+
+        std::map<String, String> environment = {
+            {"host",         config().getString("host", "localhost")},
+            {"display_name", server_display_name},
+            {"user",         config().getString("user",     "default")},
+            {"database",     config().getString("database", "default")},
+        };
+
+        for (const auto & pair: environment)
+        {
+            String key   = static_cast<String>(pair.first);
+            String value = static_cast<String>(pair.second);
+
+            boost::replace_all(pattern, "%" + key + "%",  value);
+        }
+
+        return pattern;
+    }
 
     void loop()
     {
         String query;
         String prev_query;
-        String prompt = server_display_name.length() ? "[" + server_display_name + "] :) " : ":) ";
-        while (char * line_ = readline(query.empty() ? prompt.c_str() : ":-] "))
+
+        while (char * line_ = readline(query.empty() ? prompt().c_str() : ":-] "))
         {
             String line = line_;
             free(line_);
diff --git a/dbms/src/Server/clickhouse-client.xml b/dbms/src/Server/clickhouse-client.xml
index c19606c518b..e5ec8318e94 100644
--- a/dbms/src/Server/clickhouse-client.xml
+++ b/dbms/src/Server/clickhouse-client.xml
@@ -13,4 +13,8 @@
             </invalidCertificateHandler>
         </client>
     </openSSL>
+    <prompt_by_server_display_name>
+        <test>%display_name% [green]:)[/green]</test>
+        <production>%display_name% [red]:)[/red]</production> 
+    </prompt_by_server_display_name>
 </config>

From 213caffddeda54f92392a058a36df10816a1d42d Mon Sep 17 00:00:00 2001
From: Kirill Shvakov <shvakov@gmail.com>
Date: Mon, 26 Feb 2018 15:48:07 +0200
Subject: [PATCH 012/244] add default section to client config

---
 dbms/src/Server/Client.cpp            | 4 +++-
 dbms/src/Server/clickhouse-client.xml | 1 +
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp
index 989a83a0030..229167763e9 100644
--- a/dbms/src/Server/Client.cpp
+++ b/dbms/src/Server/Client.cpp
@@ -140,7 +140,7 @@ private:
 
     String current_profile;
 
-    String prompt_by_server_display_name = "%display_name% :)";
+    String prompt_by_server_display_name;
 
     /// Path to a file containing command history.
     String history_file;
@@ -342,6 +342,8 @@ private:
         Strings keys;
         config().keys("prompt_by_server_display_name", keys);
 
+        prompt_by_server_display_name = config().getString("prompt_by_server_display_name.default", "%display_name% :)");
+
         for (const String & key : keys)
         {
             if (server_display_name.find(key) != std::string::npos) 
diff --git a/dbms/src/Server/clickhouse-client.xml b/dbms/src/Server/clickhouse-client.xml
index e5ec8318e94..6b569f02bc6 100644
--- a/dbms/src/Server/clickhouse-client.xml
+++ b/dbms/src/Server/clickhouse-client.xml
@@ -14,6 +14,7 @@
         </client>
     </openSSL>
     <prompt_by_server_display_name>
+        <default>%display_name% :)</default>
         <test>%display_name% [green]:)[/green]</test>
         <production>%display_name% [red]:)[/red]</production> 
     </prompt_by_server_display_name>

From 390ce9428b2754a956e2e288f87d7755aa3c856a Mon Sep 17 00:00:00 2001
From: Kirill Shvakov <shvakov@gmail.com>
Date: Tue, 27 Feb 2018 09:47:15 +0200
Subject: [PATCH 013/244] add background colors

---
 dbms/src/Server/Client.cpp | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp
index 229167763e9..a13dd195eff 100644
--- a/dbms/src/Server/Client.cpp
+++ b/dbms/src/Server/Client.cpp
@@ -468,12 +468,43 @@ private:
     const String prompt() const
     {
         std::map<String, String> colors = {
+            /* Foreground */
+            { "black",   "30" },
             { "red",     "31" },
             { "green",   "32" },
             { "yellow",  "33" },
             { "blue",    "34" },
             { "magenta", "35" },
             { "cyan",    "36" },
+            { "white",   "97" },
+
+            { "light gray",    "37" },
+            { "dark gray",     "90" },
+            { "light red",     "91" },
+            { "light green",   "92" },
+            { "light yellow",  "93" },
+            { "light blue",    "94" },
+            { "light magenta", "95" },
+            { "light cyan",    "96" },
+    
+            /* Background*/
+            { "background black",   "40" },
+            { "background red",     "41" },
+            { "background green",   "42" },
+            { "background yellow",  "43" },
+            { "background blue",    "44" },
+            { "background magenta", "45" },
+            { "background cyan",    "46" },
+            { "background white",   "107" },
+
+            { "background light gray",    "47" },
+            { "background dark gray",     "100" },
+            { "background light red",     "101" },
+            { "background light green",   "102" },
+            { "background light yellow",  "103" },
+            { "background light blue",    "104" },
+            { "background light magenta", "105" },
+            { "background light cyan",    "106" },
         };
 
         String pattern = prompt_by_server_display_name;
@@ -489,6 +520,7 @@ private:
 
         std::map<String, String> environment = {
             {"host",         config().getString("host", "localhost")},
+            {"port",         config().getString("port", "9000")},
             {"display_name", server_display_name},
             {"user",         config().getString("user",     "default")},
             {"database",     config().getString("database", "default")},

From 053b3f21a02a9ca35e2ec05b56961bbcb6d605fc Mon Sep 17 00:00:00 2001
From: Kirill Shvakov <shvakov@gmail.com>
Date: Tue, 27 Feb 2018 09:57:18 +0200
Subject: [PATCH 014/244] move 'colors' from function body

---
 dbms/src/Server/Client.cpp | 78 +++++++++++++++++++-------------------
 1 file changed, 38 insertions(+), 40 deletions(-)

diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp
index a13dd195eff..741d2effd98 100644
--- a/dbms/src/Server/Client.cpp
+++ b/dbms/src/Server/Client.cpp
@@ -103,7 +103,45 @@ private:
         "учшеж", "йгшеж", "дщпщгеж",
         "q", "й", "\\q", "\\Q", "\\й", "\\Й", ":q", "Жй"
     };
+    std::map<String, String> colors = {
+        /* Foreground */
+        { "black",   "30" },
+        { "red",     "31" },
+        { "green",   "32" },
+        { "yellow",  "33" },
+        { "blue",    "34" },
+        { "magenta", "35" },
+        { "cyan",    "36" },
+        { "white",   "97" },
 
+        { "light gray",    "37" },
+        { "dark gray",     "90" },
+        { "light red",     "91" },
+        { "light green",   "92" },
+        { "light yellow",  "93" },
+        { "light blue",    "94" },
+        { "light magenta", "95" },
+        { "light cyan",    "96" },
+
+        /* Background*/
+        { "background black",   "40" },
+        { "background red",     "41" },
+        { "background green",   "42" },
+        { "background yellow",  "43" },
+        { "background blue",    "44" },
+        { "background magenta", "45" },
+        { "background cyan",    "46" },
+        { "background white",   "107" },
+
+        { "background light gray",    "47" },
+        { "background dark gray",     "100" },
+        { "background light red",     "101" },
+        { "background light green",   "102" },
+        { "background light yellow",  "103" },
+        { "background light blue",    "104" },
+        { "background light magenta", "105" },
+        { "background light cyan",    "106" },
+    };
     bool is_interactive = true;          /// Use either readline interface or batch mode.
     bool need_render_progress = true;    /// Render query execution progress.
     bool echo_queries = false;           /// Print queries before execution in batch mode.
@@ -467,46 +505,6 @@ private:
 
     const String prompt() const
     {
-        std::map<String, String> colors = {
-            /* Foreground */
-            { "black",   "30" },
-            { "red",     "31" },
-            { "green",   "32" },
-            { "yellow",  "33" },
-            { "blue",    "34" },
-            { "magenta", "35" },
-            { "cyan",    "36" },
-            { "white",   "97" },
-
-            { "light gray",    "37" },
-            { "dark gray",     "90" },
-            { "light red",     "91" },
-            { "light green",   "92" },
-            { "light yellow",  "93" },
-            { "light blue",    "94" },
-            { "light magenta", "95" },
-            { "light cyan",    "96" },
-    
-            /* Background*/
-            { "background black",   "40" },
-            { "background red",     "41" },
-            { "background green",   "42" },
-            { "background yellow",  "43" },
-            { "background blue",    "44" },
-            { "background magenta", "45" },
-            { "background cyan",    "46" },
-            { "background white",   "107" },
-
-            { "background light gray",    "47" },
-            { "background dark gray",     "100" },
-            { "background light red",     "101" },
-            { "background light green",   "102" },
-            { "background light yellow",  "103" },
-            { "background light blue",    "104" },
-            { "background light magenta", "105" },
-            { "background light cyan",    "106" },
-        };
-
         String pattern = prompt_by_server_display_name;
 
         for (const auto & pair: colors)

From bba059145299d7015ad1087282f79d86d44fdc9d Mon Sep 17 00:00:00 2001
From: Kirill Shvakov <shvakov@gmail.com>
Date: Mon, 5 Mar 2018 12:20:23 +0200
Subject: [PATCH 015/244] add X-ClickHouse-Display-Name header to HTTPHandler

---
 dbms/src/Server/Client.cpp      | 13 +++++--------
 dbms/src/Server/HTTPHandler.cpp |  5 +++--
 dbms/src/Server/HTTPHandler.h   |  1 +
 3 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp
index 741d2effd98..bf08072a20d 100644
--- a/dbms/src/Server/Client.cpp
+++ b/dbms/src/Server/Client.cpp
@@ -507,10 +507,10 @@ private:
     {
         String pattern = prompt_by_server_display_name;
 
-        for (const auto & pair: colors)
+        for (const auto & [key, value]: colors)
         {
-            String name = static_cast<String>(pair.first);
-            String code = static_cast<String>(pair.second);
+            String name = static_cast<String>(key);
+            String code = static_cast<String>(value);
 
             boost::replace_all(pattern, "[" + name + "]",  "\33[1;" + code + "m");
             boost::replace_all(pattern, "[/" + name + "]", "\033[0m");
@@ -524,12 +524,9 @@ private:
             {"database",     config().getString("database", "default")},
         };
 
-        for (const auto & pair: environment)
+        for (const auto & [key, value]: environment)
         {
-            String key   = static_cast<String>(pair.first);
-            String value = static_cast<String>(pair.second);
-
-            boost::replace_all(pattern, "%" + key + "%",  value);
+            boost::replace_all(pattern, "%" + static_cast<String>(key) + "%",  static_cast<String>(value));
         }
 
         return pattern;
diff --git a/dbms/src/Server/HTTPHandler.cpp b/dbms/src/Server/HTTPHandler.cpp
index 555d4b5f3ba..3d99257f70a 100644
--- a/dbms/src/Server/HTTPHandler.cpp
+++ b/dbms/src/Server/HTTPHandler.cpp
@@ -12,7 +12,7 @@
 #include <Common/ExternalTable.h>
 #include <Common/StringUtils/StringUtils.h>
 #include <Common/escapeForFileName.h>
-
+#include <Common/getFQDNOrHostName.h>
 #include <IO/ReadBufferFromIStream.h>
 #include <IO/ZlibInflatingReadBuffer.h>
 #include <IO/ReadBufferFromString.h>
@@ -198,6 +198,7 @@ HTTPHandler::HTTPHandler(IServer & server_)
     : server(server_)
     , log(&Logger::get("HTTPHandler"))
 {
+    display_name = server.config().getString("display_name", getFQDNOrHostName());
 }
 
 
@@ -631,7 +632,7 @@ void HTTPHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne
     try
     {
         response.setContentType("text/plain; charset=UTF-8");
-
+        response.set("X-ClickHouse-Display-Name", display_name);
         /// For keep-alive to work.
         if (request.getVersion() == Poco::Net::HTTPServerRequest::HTTP_1_1)
             response.setChunkedTransferEncoding(true);
diff --git a/dbms/src/Server/HTTPHandler.h b/dbms/src/Server/HTTPHandler.h
index b0b7164852e..255da56c8d9 100644
--- a/dbms/src/Server/HTTPHandler.h
+++ b/dbms/src/Server/HTTPHandler.h
@@ -54,6 +54,7 @@ private:
 
     IServer & server;
     Poco::Logger * log;
+    String display_name;
 
     CurrentMetrics::Increment metric_increment{CurrentMetrics::HTTPConnection};
 

From 3c09dacb5f1e049ffdaaddd624d18252a72e198a Mon Sep 17 00:00:00 2001
From: Kirill Shvakov <shvakov@gmail.com>
Date: Mon, 5 Mar 2018 14:05:48 +0200
Subject: [PATCH 016/244] fix test

---
 dbms/tests/queries/0_stateless/00501_http_head.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dbms/tests/queries/0_stateless/00501_http_head.sh b/dbms/tests/queries/0_stateless/00501_http_head.sh
index c15b8679fab..7fb675e5b8a 100755
--- a/dbms/tests/queries/0_stateless/00501_http_head.sh
+++ b/dbms/tests/queries/0_stateless/00501_http_head.sh
@@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . $CURDIR/../shell_config.sh
 
 ( ${CLICKHOUSE_CURL} -s --head "${CLICKHOUSE_URL}?query=SELECT%201";
-  ${CLICKHOUSE_CURL} -s --head "${CLICKHOUSE_URL}?query=select+*+from+system.numbers+limit+1000000" ) | grep -v "Date:"
+  ${CLICKHOUSE_CURL} -s --head "${CLICKHOUSE_URL}?query=select+*+from+system.numbers+limit+1000000" ) | grep -v "Date:" | grep -v "X-ClickHouse-Display-Name:"
 
 if [[ `${CLICKHOUSE_CURL} -sS -X POST -I "${CLICKHOUSE_URL}?query=SELECT+1" | grep -c '411 Length Required'` -ne 1 ]]; then
     echo FAIL

From 5e55a605f09d593b59d32a5fd354c0a21eba17fe Mon Sep 17 00:00:00 2001
From: Kirill Shvakov <shvakov@gmail.com>
Date: Mon, 5 Mar 2018 14:18:53 +0200
Subject: [PATCH 017/244] add bold font

---
 dbms/src/Server/Client.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp
index bf08072a20d..e47c4b5b5ce 100644
--- a/dbms/src/Server/Client.cpp
+++ b/dbms/src/Server/Client.cpp
@@ -104,6 +104,7 @@ private:
         "q", "й", "\\q", "\\Q", "\\й", "\\Й", ":q", "Жй"
     };
     std::map<String, String> colors = {
+        { "bold",    "1"  },
         /* Foreground */
         { "black",   "30" },
         { "red",     "31" },

From 27a9ce8dd74a64dcfcd4a90805cd7d752f985cbf Mon Sep 17 00:00:00 2001
From: Kirill Shvakov <shvakov@gmail.com>
Date: Mon, 5 Mar 2018 16:47:10 +0200
Subject: [PATCH 018/244] Display host if display name is empty

---
 dbms/src/Server/Client.cpp | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp
index e47c4b5b5ce..a7394fa7c7a 100644
--- a/dbms/src/Server/Client.cpp
+++ b/dbms/src/Server/Client.cpp
@@ -484,6 +484,12 @@ private:
 
         server_version = toString(server_version_major) + "." + toString(server_version_minor) + "." + toString(server_revision);
         server_display_name = connection->getServerDisplayName();
+        
+        if (server_display_name.length() == 0)
+        {
+            server_display_name = config().getString("host", "localhost");
+        }
+
         if (is_interactive)
         {
             std::cout << "Connected to " << server_name

From 950a7f83cfdfd675558f6607e413278f214a1896 Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Wed, 7 Mar 2018 23:54:36 +0300
Subject: [PATCH 019/244] Update Defines.h

---
 dbms/src/Core/Defines.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dbms/src/Core/Defines.h b/dbms/src/Core/Defines.h
index 1cc14a62155..f860bd8c71c 100644
--- a/dbms/src/Core/Defines.h
+++ b/dbms/src/Core/Defines.h
@@ -60,7 +60,7 @@
 #define DBMS_MIN_REVISION_WITH_QUOTA_KEY_IN_CLIENT_INFO 54060
 #define DBMS_MIN_REVISION_WITH_TABLES_STATUS 54226
 #define DBMS_MIN_REVISION_WITH_TIME_ZONE_PARAMETER_IN_DATETIME_DATA_TYPE 54337
-#define DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME 54355
+#define DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME 54358
 
 /// Version of ClickHouse TCP protocol. Set to git tag with latest protocol change.
 #define DBMS_TCP_PROTOCOL_VERSION 54226

From fdcc88b5a28464b46333b84ffbbd70e9c18dc3a3 Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Wed, 7 Mar 2018 23:56:15 +0300
Subject: [PATCH 020/244] Update Client.cpp

---
 dbms/src/Server/Client.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp
index a7394fa7c7a..fff59cd6e4e 100644
--- a/dbms/src/Server/Client.cpp
+++ b/dbms/src/Server/Client.cpp
@@ -124,7 +124,7 @@ private:
         { "light magenta", "95" },
         { "light cyan",    "96" },
 
-        /* Background*/
+        /* Background */
         { "background black",   "40" },
         { "background red",     "41" },
         { "background green",   "42" },

From 052079b73a49852765e5a03d482f14d9b18775a2 Mon Sep 17 00:00:00 2001
From: Kirill Shvakov <shvakov@gmail.com>
Date: Thu, 8 Mar 2018 09:36:58 +0200
Subject: [PATCH 021/244] Use standard markup '{}' like macros. Add coments.

---
 dbms/src/Server/Client.cpp            |  9 +++------
 dbms/src/Server/HTTPHandler.cpp       |  4 ++--
 dbms/src/Server/HTTPHandler.h         |  4 +++-
 dbms/src/Server/TCPHandler.cpp        |  3 +--
 dbms/src/Server/TCPHandler.h          |  6 +++++-
 dbms/src/Server/clickhouse-client.xml | 15 ++++++++++++---
 6 files changed, 26 insertions(+), 15 deletions(-)

diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp
index a7394fa7c7a..82e15d9d302 100644
--- a/dbms/src/Server/Client.cpp
+++ b/dbms/src/Server/Client.cpp
@@ -381,7 +381,7 @@ private:
         Strings keys;
         config().keys("prompt_by_server_display_name", keys);
 
-        prompt_by_server_display_name = config().getString("prompt_by_server_display_name.default", "%display_name% :)");
+        prompt_by_server_display_name = config().getString("prompt_by_server_display_name.default", "{display_name} :) ");
 
         for (const String & key : keys)
         {
@@ -514,11 +514,8 @@ private:
     {
         String pattern = prompt_by_server_display_name;
 
-        for (const auto & [key, value]: colors)
+        for (const auto & [name, code]: colors)
         {
-            String name = static_cast<String>(key);
-            String code = static_cast<String>(value);
-
             boost::replace_all(pattern, "[" + name + "]",  "\33[1;" + code + "m");
             boost::replace_all(pattern, "[/" + name + "]", "\033[0m");
         }
@@ -533,7 +530,7 @@ private:
 
         for (const auto & [key, value]: environment)
         {
-            boost::replace_all(pattern, "%" + static_cast<String>(key) + "%",  static_cast<String>(value));
+            boost::replace_all(pattern, "{" + key + "}", value);
         }
 
         return pattern;
diff --git a/dbms/src/Server/HTTPHandler.cpp b/dbms/src/Server/HTTPHandler.cpp
index 3d99257f70a..539573fc8fb 100644
--- a/dbms/src/Server/HTTPHandler.cpp
+++ b/dbms/src/Server/HTTPHandler.cpp
@@ -198,7 +198,7 @@ HTTPHandler::HTTPHandler(IServer & server_)
     : server(server_)
     , log(&Logger::get("HTTPHandler"))
 {
-    display_name = server.config().getString("display_name", getFQDNOrHostName());
+    server_display_name = server.config().getString("display_name", getFQDNOrHostName());
 }
 
 
@@ -632,7 +632,7 @@ void HTTPHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne
     try
     {
         response.setContentType("text/plain; charset=UTF-8");
-        response.set("X-ClickHouse-Display-Name", display_name);
+        response.set("X-ClickHouse-Server-Display-Name", server_display_name);
         /// For keep-alive to work.
         if (request.getVersion() == Poco::Net::HTTPServerRequest::HTTP_1_1)
             response.setChunkedTransferEncoding(true);
diff --git a/dbms/src/Server/HTTPHandler.h b/dbms/src/Server/HTTPHandler.h
index 255da56c8d9..72606afa827 100644
--- a/dbms/src/Server/HTTPHandler.h
+++ b/dbms/src/Server/HTTPHandler.h
@@ -54,7 +54,9 @@ private:
 
     IServer & server;
     Poco::Logger * log;
-    String display_name;
+
+    /// It is the name of the server that will be sent in an http-header X-ClickHouse-Server-Display-Name. 
+    String server_display_name;
 
     CurrentMetrics::Increment metric_increment{CurrentMetrics::HTTPConnection};
 
diff --git a/dbms/src/Server/TCPHandler.cpp b/dbms/src/Server/TCPHandler.cpp
index e6ac7b5d5d6..729916738c3 100644
--- a/dbms/src/Server/TCPHandler.cpp
+++ b/dbms/src/Server/TCPHandler.cpp
@@ -31,7 +31,6 @@
 #include "TCPHandler.h"
 
 #include <Common/NetException.h>
-#include <Common/getFQDNOrHostName.h>
 
 
 namespace DB
@@ -511,7 +510,7 @@ void TCPHandler::sendHello()
     }
     if (client_revision >= DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME)
     {
-        writeStringBinary(server.config().getString("display_name", getFQDNOrHostName()), *out);
+        writeStringBinary(server_display_name, *out);
     }
     out->next();
 }
diff --git a/dbms/src/Server/TCPHandler.h b/dbms/src/Server/TCPHandler.h
index 444d8eb4990..f53850ee487 100644
--- a/dbms/src/Server/TCPHandler.h
+++ b/dbms/src/Server/TCPHandler.h
@@ -2,6 +2,7 @@
 
 #include <Poco/Net/TCPServerConnection.h>
 
+#include <Common/getFQDNOrHostName.h>
 #include <Common/CurrentMetrics.h>
 #include <Common/Stopwatch.h>
 #include <IO/Progress.h>
@@ -81,6 +82,7 @@ public:
         , connection_context(server.context())
         , query_context(server.context())
     {
+        server_display_name =  server.config().getString("display_name", getFQDNOrHostName());
     }
 
     void run();
@@ -111,7 +113,9 @@ private:
     QueryState state;
 
     CurrentMetrics::Increment metric_increment{CurrentMetrics::TCPConnection};
-
+    
+    /// It is the name of the server that will be sent to the client. 
+    String server_display_name;
 
     void runImpl();
 
diff --git a/dbms/src/Server/clickhouse-client.xml b/dbms/src/Server/clickhouse-client.xml
index 6b569f02bc6..ecd19d6c623 100644
--- a/dbms/src/Server/clickhouse-client.xml
+++ b/dbms/src/Server/clickhouse-client.xml
@@ -13,9 +13,18 @@
             </invalidCertificateHandler>
         </client>
     </openSSL>
+    <!--
+        It's a custom prompt settings for the clickhouse-client
+        Possible macros: 
+            {host}
+            {port}
+            {user}
+            {database} 
+            {display_name}
+    -->
     <prompt_by_server_display_name>
-        <default>%display_name% :)</default>
-        <test>%display_name% [green]:)[/green]</test>
-        <production>%display_name% [red]:)[/red]</production> 
+        <default>{display_name} :) </default>
+        <test>{display_name} [green]:)[/green </test> <!-- if it matched to the substring "test" in the server display name - -->
+        <production>{display_name} [red]:)[/red] </production> <!-- if it matched to the substring "production" in the server display name -->
     </prompt_by_server_display_name>
 </config>

From 8c4a7b774efc7e352347c0415c20e4774b5508b0 Mon Sep 17 00:00:00 2001
From: Kirill Shvakov <shvakov@gmail.com>
Date: Thu, 8 Mar 2018 10:55:27 +0200
Subject: [PATCH 022/244] Removed BB-Style macros for colors. Preparing prompt
 pattern once and replace current database in the cycle only.

---
 dbms/src/Server/Client.cpp                    | 93 ++++++-------------
 dbms/src/Server/clickhouse-client.xml         |  5 +-
 .../queries/0_stateless/00501_http_head.sh    |  2 +-
 3 files changed, 33 insertions(+), 67 deletions(-)

diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp
index f490b61b12a..959dfe9a199 100644
--- a/dbms/src/Server/Client.cpp
+++ b/dbms/src/Server/Client.cpp
@@ -103,46 +103,6 @@ private:
         "учшеж", "йгшеж", "дщпщгеж",
         "q", "й", "\\q", "\\Q", "\\й", "\\Й", ":q", "Жй"
     };
-    std::map<String, String> colors = {
-        { "bold",    "1"  },
-        /* Foreground */
-        { "black",   "30" },
-        { "red",     "31" },
-        { "green",   "32" },
-        { "yellow",  "33" },
-        { "blue",    "34" },
-        { "magenta", "35" },
-        { "cyan",    "36" },
-        { "white",   "97" },
-
-        { "light gray",    "37" },
-        { "dark gray",     "90" },
-        { "light red",     "91" },
-        { "light green",   "92" },
-        { "light yellow",  "93" },
-        { "light blue",    "94" },
-        { "light magenta", "95" },
-        { "light cyan",    "96" },
-
-        /* Background */
-        { "background black",   "40" },
-        { "background red",     "41" },
-        { "background green",   "42" },
-        { "background yellow",  "43" },
-        { "background blue",    "44" },
-        { "background magenta", "45" },
-        { "background cyan",    "46" },
-        { "background white",   "107" },
-
-        { "background light gray",    "47" },
-        { "background dark gray",     "100" },
-        { "background light red",     "101" },
-        { "background light green",   "102" },
-        { "background light yellow",  "103" },
-        { "background light blue",    "104" },
-        { "background light magenta", "105" },
-        { "background light cyan",    "106" },
-    };
     bool is_interactive = true;          /// Use either readline interface or batch mode.
     bool need_render_progress = true;    /// Render query execution progress.
     bool echo_queries = false;           /// Print queries before execution in batch mode.
@@ -379,10 +339,11 @@ private:
         }
 
         Strings keys;
-        config().keys("prompt_by_server_display_name", keys);
 
         prompt_by_server_display_name = config().getString("prompt_by_server_display_name.default", "{display_name} :) ");
 
+        config().keys("prompt_by_server_display_name", keys);
+
         for (const String & key : keys)
         {
             if (server_display_name.find(key) != std::string::npos) 
@@ -392,6 +353,30 @@ private:
             }
         }
 
+        std::map<String, String> terminalCharacters = {
+            { "\\e[",   "\e["  },
+            { "\\33[",  "\33[" },
+            { "\\033[", "\033["},
+            { "\\x1B[", "\x1B["},
+        };
+
+        for (const auto & [key, value]: terminalCharacters)
+        {
+            boost::replace_all(prompt_by_server_display_name, key, value);
+        }
+
+        std::map<String, String> environment = {
+            {"host",         config().getString("host", "localhost")},
+            {"port",         config().getString("port", "9000")},
+            {"user",         config().getString("user", "default")},
+            {"display_name", server_display_name},
+        };
+
+        for (const auto & [key, value]: environment)
+        {
+            boost::replace_all(prompt_by_server_display_name, "{" + key + "}", value);
+        }
+
         if (is_interactive)
         {
             if (print_time_to_stderr)
@@ -483,9 +468,8 @@ private:
         connection->getServerVersion(server_name, server_version_major, server_version_minor, server_revision);
 
         server_version = toString(server_version_major) + "." + toString(server_version_minor) + "." + toString(server_revision);
-        server_display_name = connection->getServerDisplayName();
-        
-        if (server_display_name.length() == 0)
+
+        if (server_display_name = connection->getServerDisplayName(); server_display_name.length() == 0)
         {
             server_display_name = config().getString("host", "localhost");
         }
@@ -513,26 +497,7 @@ private:
     const String prompt() const
     {
         String pattern = prompt_by_server_display_name;
-
-        for (const auto & [name, code]: colors)
-        {
-            boost::replace_all(pattern, "[" + name + "]",  "\33[1;" + code + "m");
-            boost::replace_all(pattern, "[/" + name + "]", "\033[0m");
-        }
-
-        std::map<String, String> environment = {
-            {"host",         config().getString("host", "localhost")},
-            {"port",         config().getString("port", "9000")},
-            {"display_name", server_display_name},
-            {"user",         config().getString("user",     "default")},
-            {"database",     config().getString("database", "default")},
-        };
-
-        for (const auto & [key, value]: environment)
-        {
-            boost::replace_all(pattern, "{" + key + "}", value);
-        }
-
+        boost::replace_all(pattern, "{database}", config().getString("database", "default"));
         return pattern;
     }
 
diff --git a/dbms/src/Server/clickhouse-client.xml b/dbms/src/Server/clickhouse-client.xml
index ecd19d6c623..3cd05d7fec9 100644
--- a/dbms/src/Server/clickhouse-client.xml
+++ b/dbms/src/Server/clickhouse-client.xml
@@ -21,10 +21,11 @@
             {user}
             {database} 
             {display_name}
+        Terminal colors: https://misc.flogisoft.com/bash/tip_colors_and_formatting
     -->
     <prompt_by_server_display_name>
         <default>{display_name} :) </default>
-        <test>{display_name} [green]:)[/green </test> <!-- if it matched to the substring "test" in the server display name - -->
-        <production>{display_name} [red]:)[/red] </production> <!-- if it matched to the substring "production" in the server display name -->
+        <test>{display_name} \e[1;32m:)\e[0m  </test> <!-- if it matched to the substring "test" in the server display name - -->
+        <production>{display_name} \e[1;31m:)\e[0m </production> <!-- if it matched to the substring "production" in the server display name -->
     </prompt_by_server_display_name>
 </config>
diff --git a/dbms/tests/queries/0_stateless/00501_http_head.sh b/dbms/tests/queries/0_stateless/00501_http_head.sh
index 7fb675e5b8a..578e27da751 100755
--- a/dbms/tests/queries/0_stateless/00501_http_head.sh
+++ b/dbms/tests/queries/0_stateless/00501_http_head.sh
@@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . $CURDIR/../shell_config.sh
 
 ( ${CLICKHOUSE_CURL} -s --head "${CLICKHOUSE_URL}?query=SELECT%201";
-  ${CLICKHOUSE_CURL} -s --head "${CLICKHOUSE_URL}?query=select+*+from+system.numbers+limit+1000000" ) | grep -v "Date:" | grep -v "X-ClickHouse-Display-Name:"
+  ${CLICKHOUSE_CURL} -s --head "${CLICKHOUSE_URL}?query=select+*+from+system.numbers+limit+1000000" ) | grep -v "Date:" | grep -v "X-ClickHouse-Server-Display-Name:"
 
 if [[ `${CLICKHOUSE_CURL} -sS -X POST -I "${CLICKHOUSE_URL}?query=SELECT+1" | grep -c '411 Length Required'` -ne 1 ]]; then
     echo FAIL

From 8cb5733b4460f3101e5d9fcf0a4f25c7a376a06a Mon Sep 17 00:00:00 2001
From: Michael Furmur <m.furmur@gmail.com>
Date: Thu, 8 Mar 2018 15:23:28 +0200
Subject: [PATCH 023/244] add aggregate functions IntersectionsMax and
 IntersectionsMaxPos

---
 .../AggregateFunctionIntersectionsMax.cpp     | 145 ++++++++++++++++++
 .../AggregateFunctionIntersectionsMax.h       | 132 ++++++++++++++++
 .../registerAggregateFunctions.cpp            |   2 +
 3 files changed, 279 insertions(+)
 create mode 100644 dbms/src/AggregateFunctions/AggregateFunctionIntersectionsMax.cpp
 create mode 100644 dbms/src/AggregateFunctions/AggregateFunctionIntersectionsMax.h

diff --git a/dbms/src/AggregateFunctions/AggregateFunctionIntersectionsMax.cpp b/dbms/src/AggregateFunctions/AggregateFunctionIntersectionsMax.cpp
new file mode 100644
index 00000000000..854737ceba4
--- /dev/null
+++ b/dbms/src/AggregateFunctions/AggregateFunctionIntersectionsMax.cpp
@@ -0,0 +1,145 @@
+#include <AggregateFunctions/AggregateFunctionFactory.h>
+#include <AggregateFunctions/AggregateFunctionIntersectionsMax.h>
+#include <AggregateFunctions/FactoryHelpers.h>
+
+#include <IO/ReadHelpers.h>
+#include <IO/WriteHelpers.h>
+
+namespace DB
+{
+
+template <typename T>
+typename Intersections<T>::PointsMap::iterator
+Intersections<T>::insert_point(const T &v)
+{
+    auto res = points.emplace(v,0);
+    auto &i = res.first;
+    if(!res.second) return i;
+    if(i==points.begin()) return i;
+    auto prev = i;
+    prev--;
+    i->second=prev->second;
+    return i;
+}
+
+template <typename T>
+void Intersections<T>::add(const T &start, const T &end, T weight)
+{
+    auto sp = insert_point(start);
+    auto ep = end ? insert_point(end) : points.end();
+    do {
+        sp->second+=weight;
+        if(sp->second > max_weight) {
+            max_weight = sp->second;
+            max_weight_pos = sp->first;
+        }
+    } while(++sp != ep);
+}
+
+template <typename T>
+void Intersections<T>::merge(const Intersections &other)
+{
+    if(other.points.empty())
+        return;
+
+    typename PointsMap::const_iterator prev, i = other.points.begin();
+    prev = i;
+    i++;
+
+    while(i != other.points.end()) {
+        add(prev->first,i->first,prev->second);
+        prev = i;
+        i++;
+    }
+
+    if(prev != other.points.end())
+        add(prev->first,0,prev->second);
+}
+
+template <typename T>
+void Intersections<T>::serialize(WriteBuffer & buf) const
+{
+    writeBinary(points.size(),buf);
+    for(const auto &p: points) {
+        writeBinary(p.first,buf);
+        writeBinary(p.second,buf);
+    }
+}
+
+template <typename T>
+void Intersections<T>::deserialize(ReadBuffer & buf)
+{
+    std::size_t size;
+    T point;
+    T weight;
+
+    readBinary(size, buf);
+    for (std::size_t i = 0; i < size; ++i) {
+        readBinary(point, buf);
+        readBinary(weight,buf);
+        points.emplace(point,weight);
+    }
+}
+
+void AggregateFunctionIntersectionsMax::_add(
+        AggregateDataPtr place,
+        const IColumn & column_start,
+        const IColumn & column_end,
+        size_t row_num) const
+{
+    PointType start_time, end_time;
+    Field tmp_start_time_field, tmp_end_time_field;
+
+    column_start.get(row_num,tmp_start_time_field);
+    if(tmp_start_time_field.isNull())
+        return;
+    start_time = tmp_start_time_field.template get<PointType>();
+    if(0==start_time)
+        return;
+
+    column_end.get(row_num,tmp_end_time_field);
+    if(tmp_end_time_field.isNull()) {
+        end_time = 0;
+    } else {
+        end_time = tmp_end_time_field.template get<PointType>();
+        if(0!=end_time) {
+            if(end_time==start_time) {
+                end_time = 0;
+            } else if(end_time < start_time) {
+                return;
+            }
+        }
+    }
+
+    data(place).add(start_time,end_time);
+}
+
+namespace
+{
+
+AggregateFunctionPtr createAggregateFunctionIntersectionsMax(const std::string & name, const DataTypes & argument_types, const Array & parameters)
+{
+    assertBinary(name, argument_types);
+    return std::make_shared<AggregateFunctionIntersectionsMax>(argument_types,parameters,false);
+}
+
+AggregateFunctionPtr createAggregateFunctionIntersectionsMaxPos(const std::string & name, const DataTypes & argument_types, const Array & parameters)
+{
+    assertBinary(name, argument_types);
+    return std::make_shared<AggregateFunctionIntersectionsMax>(argument_types,parameters,true);
+}
+
+}
+
+void registerAggregateFunctionIntersectionsMax(AggregateFunctionFactory & factory)
+{
+    factory.registerFunction("IntersectionsMax",
+        createAggregateFunctionIntersectionsMax,
+        AggregateFunctionFactory::CaseInsensitive);
+    factory.registerFunction("IntersectionsMaxPos",
+        createAggregateFunctionIntersectionsMaxPos,
+        AggregateFunctionFactory::CaseInsensitive);
+}
+
+}
+
diff --git a/dbms/src/AggregateFunctions/AggregateFunctionIntersectionsMax.h b/dbms/src/AggregateFunctions/AggregateFunctionIntersectionsMax.h
new file mode 100644
index 00000000000..ca97986e678
--- /dev/null
+++ b/dbms/src/AggregateFunctions/AggregateFunctionIntersectionsMax.h
@@ -0,0 +1,132 @@
+#pragma once
+
+#include <common/logger_useful.h>
+
+#include <DataTypes/DataTypesNumber.h>
+#include <DataTypes/DataTypeNullable.h>
+#include <DataTypes/DataTypeDate.h>
+#include <DataTypes/DataTypeDateTime.h>
+
+#include <Columns/ColumnNullable.h>
+
+#include <Common/Allocator.h>
+
+#include <AggregateFunctions/IAggregateFunction.h>
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int AGGREGATE_FUNCTION_DOESNT_ALLOW_PARAMETERS;
+    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+}
+
+template <typename T>
+class Intersections final
+{
+
+    using PointsMap = std::map<T, T>;
+
+    PointsMap points;
+    T max_weight;
+    T max_weight_pos;
+
+    typename PointsMap::iterator insert_point(const T &v);
+  public:
+
+    Intersections()
+      : max_weight(0)
+    { }
+
+    void add(const T &start, const T &end, T weight = 1);
+    void merge(const Intersections &other);
+
+    void serialize(WriteBuffer & buf) const;
+    void deserialize(ReadBuffer & buf);
+
+    T max() const { return max_weight; }
+    T max_pos() const { return max_weight_pos; }
+};
+
+class AggregateFunctionIntersectionsMax final
+  : public IAggregateFunctionDataHelper<Intersections<UInt64>, AggregateFunctionIntersectionsMax>
+{
+    using PointType = UInt64;
+
+    bool return_position;
+    void _add(AggregateDataPtr place, const IColumn & column_start, const IColumn & column_end, size_t row_num) const;
+
+  public:
+    AggregateFunctionIntersectionsMax(const DataTypes & arguments, const Array & params, bool return_position)
+      : return_position(return_position)
+    {
+        if (!params.empty()) {
+            throw Exception(
+                "Aggregate function " + getName() + " does not allow paremeters.",
+                ErrorCodes::AGGREGATE_FUNCTION_DOESNT_ALLOW_PARAMETERS);
+        }
+
+        if (arguments.size() != 2)
+            throw Exception("Aggregate function " + getName() + " requires two arguments.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
+
+        if(!arguments[0]->isValueRepresentedByInteger())
+            throw Exception {
+                getName() + ": first argument must be represented by integer",
+                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT
+            };
+
+        if(!arguments[1]->isValueRepresentedByInteger())
+            throw Exception {
+                getName() + ": second argument must be represented by integer",
+                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT
+            };
+
+        if(!arguments[0]->equals(*arguments[1]))
+            throw Exception {
+                getName() + ": arguments must have the same type",
+                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT
+            };
+    }
+
+    String getName() const override { return "IntersectionsMax"; }
+
+    DataTypePtr getReturnType() const override
+    {
+        return std::make_shared<DataTypeUInt64>();
+    }
+
+    void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override
+    {
+        _add(place,*columns[0],*columns[1],row_num);
+    }
+
+    void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override
+    {
+        this->data(place).merge(data(rhs));
+    }
+
+    void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override
+    {
+        this->data(place).serialize(buf);
+    }
+
+    void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena *) const override
+    {
+        this->data(place).deserialize(buf);
+    }
+
+    void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override
+    {
+        auto &ret = static_cast<ColumnUInt64 &>(to).getData();
+        ret.push_back(data(place).max());
+        if(return_position)
+            ret.push_back(data(place).max_pos());
+    }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
+};
+
+}
+
diff --git a/dbms/src/AggregateFunctions/registerAggregateFunctions.cpp b/dbms/src/AggregateFunctions/registerAggregateFunctions.cpp
index 59bda8bb8b5..d054107c705 100644
--- a/dbms/src/AggregateFunctions/registerAggregateFunctions.cpp
+++ b/dbms/src/AggregateFunctions/registerAggregateFunctions.cpp
@@ -23,6 +23,7 @@ void registerAggregateFunctionsUniq(AggregateFunctionFactory &);
 void registerAggregateFunctionUniqUpTo(AggregateFunctionFactory &);
 void registerAggregateFunctionTopK(AggregateFunctionFactory &);
 void registerAggregateFunctionsBitwise(AggregateFunctionFactory &);
+void registerAggregateFunctionIntersectionsMax(AggregateFunctionFactory &);
 
 void registerAggregateFunctionCombinatorIf(AggregateFunctionCombinatorFactory &);
 void registerAggregateFunctionCombinatorArray(AggregateFunctionCombinatorFactory &);
@@ -53,6 +54,7 @@ void registerAggregateFunctions()
         registerAggregateFunctionUniqUpTo(factory);
         registerAggregateFunctionTopK(factory);
         registerAggregateFunctionsBitwise(factory);
+        registerAggregateFunctionIntersectionsMax(factory);
     }
 
     {

From ab9553711a1839a2466be099620894e2fdcbcbb4 Mon Sep 17 00:00:00 2001
From: BayoNet <bayonet@yandex-team.ru>
Date: Thu, 8 Mar 2018 20:35:42 +0300
Subject: [PATCH 024/244] Contents bag is fixed.

---
 docs/mkdocs_en.yml | 2 +-
 docs/mkdocs_ru.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/mkdocs_en.yml b/docs/mkdocs_en.yml
index 5d0723f0042..551ce240f3b 100644
--- a/docs/mkdocs_en.yml
+++ b/docs/mkdocs_en.yml
@@ -218,7 +218,7 @@ pages:
   - 'Internal dictionaries': 'dicts/internal_dicts.md'
 
 - 'Operation':
-  - 'Operation': 'operations/index.md'
+#  - 'Operation': 'operations/index.md'
   - 'Access rights': 'operations/access_rights.md'
   - 'Configuration files': 'operations/configuration_files.md'
   - 'Quotas': 'operations/quotas.md'
diff --git a/docs/mkdocs_ru.yml b/docs/mkdocs_ru.yml
index df06d235c99..bc8b74a3d17 100644
--- a/docs/mkdocs_ru.yml
+++ b/docs/mkdocs_ru.yml
@@ -219,7 +219,7 @@ pages:
   - 'Встроенные словари': 'dicts/internal_dicts.md'
 
 - 'Эксплуатация':
-  - 'Эксплуатация': 'operations/index.md'
+#  - 'Эксплуатация': 'operations/index.md'
   - 'Права доступа': 'operations/access_rights.md'
   - 'Конфигурационные файлы': 'operations/configuration_files.md'
   - 'Квоты': 'operations/quotas.md'

From 808bfef3453fe016a4f560867e4b4d31b38409ae Mon Sep 17 00:00:00 2001
From: Kirill Shvakov <shvakov@gmail.com>
Date: Mon, 12 Mar 2018 15:23:15 +0200
Subject: [PATCH 025/244] inline prompt function

---
 dbms/src/Server/Client.cpp | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp
index 959dfe9a199..53752f1d173 100644
--- a/dbms/src/Server/Client.cpp
+++ b/dbms/src/Server/Client.cpp
@@ -346,7 +346,7 @@ private:
 
         for (const String & key : keys)
         {
-            if (server_display_name.find(key) != std::string::npos) 
+            if (key != "default" && server_display_name.find(key) != std::string::npos) 
             {
                 prompt_by_server_display_name = config().getString("prompt_by_server_display_name." + key);
                 break;
@@ -494,11 +494,9 @@ private:
         return select(1, &fds, 0, 0, &timeout) == 1;
     }
 
-    const String prompt() const
+    inline const String prompt() const
     {
-        String pattern = prompt_by_server_display_name;
-        boost::replace_all(pattern, "{database}", config().getString("database", "default"));
-        return pattern;
+        return boost::replace_all_copy(prompt_by_server_display_name, "{database}", config().getString("database", "default"));
     }
 
     void loop()

From 2b3e897aa1b75b292c9f325322002038af4e34a5 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Mon, 12 Mar 2018 17:14:56 +0300
Subject: [PATCH 026/244] added SHOW CREATE DATABASE query [#CLICKHOUSE-3590]

---
 dbms/src/Databases/DatabaseOrdinary.cpp       | 17 ++++++--
 dbms/src/Databases/IDatabase.h                |  3 +-
 dbms/src/Interpreters/InterpreterFactory.cpp  |  6 ++-
 .../InterpreterShowCreateQuery.cpp            |  2 +-
 .../Parsers/ParserTablePropertiesQuery.cpp    | 39 +++++++++++++------
 dbms/src/Parsers/TablePropertiesQueriesASTs.h | 22 +++++++++--
 6 files changed, 69 insertions(+), 20 deletions(-)

diff --git a/dbms/src/Databases/DatabaseOrdinary.cpp b/dbms/src/Databases/DatabaseOrdinary.cpp
index 0021f47c4c7..b0719649094 100644
--- a/dbms/src/Databases/DatabaseOrdinary.cpp
+++ b/dbms/src/Databases/DatabaseOrdinary.cpp
@@ -45,6 +45,12 @@ namespace detail
     {
         return base_path + (endsWith(base_path, "/") ? "" : "/") + escapeForFileName(table_name) + ".sql";
     }
+
+    String getDatabaseMetadataPath(const String & base_path)
+    {
+        return (endsWith(base_path, "/") ? base_path.substr(0, base_path.size() - 1) : base_path) + ".sql";
+    }
+
 }
 
 static void loadTable(
@@ -332,16 +338,21 @@ void DatabaseOrdinary::removeTable(
 
 static ASTPtr getCreateQueryImpl(const String & path, const String & table_name)
 {
-    String table_metadata_path = detail::getTableMetadataPath(path, table_name);
+    String metadata_path;
+
+    if (table_name.empty())
+        metadata_path = detail::getDatabaseMetadataPath(path);
+    else
+        metadata_path = detail::getTableMetadataPath(path, table_name);
 
     String query;
     {
-        ReadBufferFromFile in(table_metadata_path, 4096);
+        ReadBufferFromFile in(metadata_path, 4096);
         readStringUntilEOF(query, in);
     }
 
     ParserCreateQuery parser;
-    return parseQuery(parser, query.data(), query.data() + query.size(), "in file " + table_metadata_path);
+    return parseQuery(parser, query.data(), query.data() + query.size(), "in file " + metadata_path);
 }
 
 
diff --git a/dbms/src/Databases/IDatabase.h b/dbms/src/Databases/IDatabase.h
index 74cbe0b2202..8d62c0e3d45 100644
--- a/dbms/src/Databases/IDatabase.h
+++ b/dbms/src/Databases/IDatabase.h
@@ -124,7 +124,8 @@ public:
         const Context & context,
         const String & name) = 0;
 
-    /// Get the CREATE TABLE query for the table. It can also provide information for detached tables for which there is metadata.
+    /// Get the CREATE TABLE query for the table or CREATE DATABASE query for database if name is empty.
+    /// It can also provide information for detached tables for which there is metadata.
     virtual ASTPtr getCreateQuery(
         const Context & context,
         const String & name) const = 0;
diff --git a/dbms/src/Interpreters/InterpreterFactory.cpp b/dbms/src/Interpreters/InterpreterFactory.cpp
index 9d30c586129..2e43efbb12e 100644
--- a/dbms/src/Interpreters/InterpreterFactory.cpp
+++ b/dbms/src/Interpreters/InterpreterFactory.cpp
@@ -107,7 +107,11 @@ std::unique_ptr<IInterpreter> InterpreterFactory::get(ASTPtr & query, Context &
     {
         return std::make_unique<InterpreterExistsQuery>(query, context);
     }
-    else if (typeid_cast<ASTShowCreateQuery *>(query.get()))
+    else if (typeid_cast<ASTShowCreateTableQuery *>(query.get()))
+    {
+        return std::make_unique<InterpreterShowCreateQuery>(query, context);
+    }
+    else if (typeid_cast<ASTShowCreateDatabaseQuery *>(query.get()))
     {
         return std::make_unique<InterpreterShowCreateQuery>(query, context);
     }
diff --git a/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp b/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp
index d4bd116bcf7..705b9bb4120 100644
--- a/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp
@@ -39,7 +39,7 @@ Block InterpreterShowCreateQuery::getSampleBlock()
 
 BlockInputStreamPtr InterpreterShowCreateQuery::executeImpl()
 {
-    const ASTShowCreateQuery & ast = typeid_cast<const ASTShowCreateQuery &>(*query_ptr);
+    const auto & ast = dynamic_cast<const ASTQueryWithTableAndOutput &>(*query_ptr);
 
     if (ast.temporary && !ast.database.empty())
         throw Exception("Temporary databases are not possible.", ErrorCodes::SYNTAX_ERROR);
diff --git a/dbms/src/Parsers/ParserTablePropertiesQuery.cpp b/dbms/src/Parsers/ParserTablePropertiesQuery.cpp
index 570597b0fa6..45e92f9e181 100644
--- a/dbms/src/Parsers/ParserTablePropertiesQuery.cpp
+++ b/dbms/src/Parsers/ParserTablePropertiesQuery.cpp
@@ -19,6 +19,7 @@ bool ParserTablePropertiesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected &
     ParserKeyword s_desc("DESC");
     ParserKeyword s_show("SHOW");
     ParserKeyword s_create("CREATE");
+    ParserKeyword s_database("DATABASE");
     ParserKeyword s_table("TABLE");
     ParserToken s_dot(TokenType::Dot);
     ParserIdentifier name_p;
@@ -27,6 +28,8 @@ bool ParserTablePropertiesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected &
     ASTPtr table;
     std::shared_ptr<ASTQueryWithTableAndOutput> query;
 
+    bool parse_only_database_name = false;
+
     if (s_exists.ignore(pos, expected))
     {
         query = std::make_shared<ASTExistsQuery>();
@@ -36,26 +39,40 @@ bool ParserTablePropertiesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected &
         if (!s_create.ignore(pos, expected))
             return false;
 
-        query = std::make_shared<ASTShowCreateQuery>();
+        if (s_database.ignore(pos, expected))
+        {
+            parse_only_database_name = true;
+            query = std::make_shared<ASTShowCreateDatabaseQuery>();
+        }
+        else
+            query = std::make_shared<ASTShowCreateTableQuery>();
     }
     else
     {
         return false;
     }
 
-    if (s_temporary.ignore(pos, expected))
-        query->temporary = true;
-
-    s_table.ignore(pos, expected);
-
-    if (!name_p.parse(pos, table, expected))
-        return false;
-
-    if (s_dot.ignore(pos, expected))
+    if (parse_only_database_name)
     {
-        database = table;
+        if (!name_p.parse(pos, database, expected))
+            return false;
+    }
+    else
+    {
+        if (s_temporary.ignore(pos, expected))
+            query->temporary = true;
+
+        s_table.ignore(pos, expected);
+
         if (!name_p.parse(pos, table, expected))
             return false;
+
+        if (s_dot.ignore(pos, expected))
+        {
+            database = table;
+            if (!name_p.parse(pos, table, expected))
+                return false;
+        }
     }
 
     if (database)
diff --git a/dbms/src/Parsers/TablePropertiesQueriesASTs.h b/dbms/src/Parsers/TablePropertiesQueriesASTs.h
index 312b112f494..15a1d397fa4 100644
--- a/dbms/src/Parsers/TablePropertiesQueriesASTs.h
+++ b/dbms/src/Parsers/TablePropertiesQueriesASTs.h
@@ -12,12 +12,18 @@ struct ASTExistsQueryIDAndQueryNames
     static constexpr auto Query = "EXISTS TABLE";
 };
 
-struct ASTShowCreateQueryIDAndQueryNames
+struct ASTShowCreateTableQueryIDAndQueryNames
 {
-    static constexpr auto ID = "ShowCreateQuery";
+    static constexpr auto ID = "ShowCreateTableQuery";
     static constexpr auto Query = "SHOW CREATE TABLE";
 };
 
+struct ASTShowCreateDatabaseQueryIDAndQueryNames
+{
+    static constexpr auto ID = "ShowCreateDatabaseQuery";
+    static constexpr auto Query = "SHOW CREATE DATABASE";
+};
+
 struct ASTDescribeQueryExistsQueryIDAndQueryNames
 {
     static constexpr auto ID = "DescribeQuery";
@@ -25,7 +31,17 @@ struct ASTDescribeQueryExistsQueryIDAndQueryNames
 };
 
 using ASTExistsQuery = ASTQueryWithTableAndOutputImpl<ASTExistsQueryIDAndQueryNames>;
-using ASTShowCreateQuery = ASTQueryWithTableAndOutputImpl<ASTShowCreateQueryIDAndQueryNames>;
+using ASTShowCreateTableQuery = ASTQueryWithTableAndOutputImpl<ASTShowCreateTableQueryIDAndQueryNames>;
+
+class ASTShowCreateDatabaseQuery : public ASTQueryWithTableAndOutputImpl<ASTShowCreateDatabaseQueryIDAndQueryNames>
+{
+protected:
+    void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override
+    {
+        settings.ostr << (settings.hilite ? hilite_keyword : "") << ASTShowCreateDatabaseQueryIDAndQueryNames::Query
+                      << " " << (settings.hilite ? hilite_none : "") << backQuoteIfNeed(database);
+    }
+};
 
 class ASTDescribeQuery : public ASTQueryWithOutput
 {

From 5a07569c2a57f95197dd8af42232bf08553cdae6 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Mon, 12 Mar 2018 18:21:17 +0300
Subject: [PATCH 027/244] added test [#CLICKHOUSE-3590]

---
 .../queries/0_stateless/00604_show_create_database.reference    | 1 +
 dbms/tests/queries/0_stateless/00604_show_create_database.sql   | 2 ++
 2 files changed, 3 insertions(+)
 create mode 100644 dbms/tests/queries/0_stateless/00604_show_create_database.reference
 create mode 100644 dbms/tests/queries/0_stateless/00604_show_create_database.sql

diff --git a/dbms/tests/queries/0_stateless/00604_show_create_database.reference b/dbms/tests/queries/0_stateless/00604_show_create_database.reference
new file mode 100644
index 00000000000..ab242ee47fd
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00604_show_create_database.reference
@@ -0,0 +1 @@
+CREATE DATABASE test ENGINE = Ordinary
diff --git a/dbms/tests/queries/0_stateless/00604_show_create_database.sql b/dbms/tests/queries/0_stateless/00604_show_create_database.sql
new file mode 100644
index 00000000000..b5671b3a984
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00604_show_create_database.sql
@@ -0,0 +1,2 @@
+create database if not exists test;
+show create database test;

From 4dd7f47ad52151494551498aadc9c13fd6dbf45b Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Mon, 12 Mar 2018 21:16:51 +0300
Subject: [PATCH 028/244] reload macros with config [#CLICKHOUSE-3591]

---
 dbms/src/Common/Macros.h                         |  2 ++
 dbms/src/Interpreters/Context.cpp                | 10 ++++++----
 dbms/src/Interpreters/Context.h                  |  3 ++-
 dbms/src/Interpreters/DDLWorker.cpp              |  2 +-
 dbms/src/Server/Server.cpp                       |  1 +
 dbms/src/Storages/StorageDistributed.cpp         |  2 +-
 dbms/src/Storages/StorageReplicatedMergeTree.cpp |  4 ++--
 7 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/dbms/src/Common/Macros.h b/dbms/src/Common/Macros.h
index 0ebf52afd02..5b5435ca9e6 100644
--- a/dbms/src/Common/Macros.h
+++ b/dbms/src/Common/Macros.h
@@ -27,4 +27,6 @@ private:
     MacroMap macros;
 };
 
+using MacrosPtr = std::shared_ptr<Macros>;
+
 }
diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp
index 6fddcbc2f05..7a2358f36c0 100644
--- a/dbms/src/Interpreters/Context.cpp
+++ b/dbms/src/Interpreters/Context.cpp
@@ -130,7 +130,8 @@ struct ContextShared
     ConfigurationPtr users_config;                          /// Config with the users, profiles and quotas sections.
     InterserverIOHandler interserver_io_handler;            /// Handler for interserver communication.
     BackgroundProcessingPoolPtr background_pool;            /// The thread pool for the background work performed by the tables.
-    Macros macros;                                          /// Substitutions extracted from config.
+    mutable std::mutex macros_mutex;
+    MacrosPtr macros;                                       /// Substitutions extracted from config.
     std::unique_ptr<Compiler> compiler;                     /// Used for dynamic compilation of queries' parts if it necessary.
     std::shared_ptr<DDLWorker> ddl_worker;                  /// Process ddl commands from zk.
     /// Rules for selecting the compression settings, depending on the size of the part.
@@ -1042,15 +1043,16 @@ void Context::setDefaultFormat(const String & name)
     default_format = name;
 }
 
-const Macros & Context::getMacros() const
+const MacrosPtr & Context::getMacros() const
 {
+    std::unique_lock<std::mutex> lock(shared->macros_mutex);
     return shared->macros;
 }
 
 void Context::setMacros(Macros && macros)
 {
-    /// We assume that this assignment occurs once when the server starts. If this is not the case, you need to use a mutex.
-    shared->macros = macros;
+    std::unique_lock<std::mutex> lock(shared->macros_mutex);
+    shared->macros = std::make_shared<Macros>(std::move(macros));
 }
 
 const Context & Context::getQueryContext() const
diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h
index bab3be07add..4148b79923b 100644
--- a/dbms/src/Interpreters/Context.h
+++ b/dbms/src/Interpreters/Context.h
@@ -80,6 +80,7 @@ using Dependencies = std::vector<DatabaseAndTableName>;
 using TableAndCreateAST = std::pair<StoragePtr, ASTPtr>;
 using TableAndCreateASTs = std::map<String, TableAndCreateAST>;
 
+using MacrosPtr = std::shared_ptr<Macros>;
 
 /** A set of known objects that can be used in the query.
   * Consists of a shared part (always common to all sessions and queries)
@@ -206,7 +207,7 @@ public:
     String getDefaultFormat() const;    /// If default_format is not specified, some global default format is returned.
     void setDefaultFormat(const String & name);
 
-    const Macros & getMacros() const;
+    const MacrosPtr & getMacros() const;
     void setMacros(Macros && macros);
 
     Settings getSettings() const;
diff --git a/dbms/src/Interpreters/DDLWorker.cpp b/dbms/src/Interpreters/DDLWorker.cpp
index 1c60b6d766c..c0262358fe1 100644
--- a/dbms/src/Interpreters/DDLWorker.cpp
+++ b/dbms/src/Interpreters/DDLWorker.cpp
@@ -1131,7 +1131,7 @@ BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr_, const Context & cont
         }
     }
 
-    query->cluster = context.getMacros().expand(query->cluster);
+    query->cluster = context.getMacros()->expand(query->cluster);
     ClusterPtr cluster = context.getCluster(query->cluster);
     DDLWorker & ddl_worker = context.getDDLWorker();
 
diff --git a/dbms/src/Server/Server.cpp b/dbms/src/Server/Server.cpp
index db20c194573..08153b00b1d 100644
--- a/dbms/src/Server/Server.cpp
+++ b/dbms/src/Server/Server.cpp
@@ -231,6 +231,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
         [&](ConfigurationPtr config)
         {
             global_context->setClustersConfig(config);
+            global_context->setMacros(Macros(*config, "macros"));
         },
         /* already_loaded = */ true);
 
diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp
index c1681e6fbea..f25b0ef6deb 100644
--- a/dbms/src/Storages/StorageDistributed.cpp
+++ b/dbms/src/Storages/StorageDistributed.cpp
@@ -144,7 +144,7 @@ StorageDistributed::StorageDistributed(
     : IStorage{columns_, materialized_columns_, alias_columns_, column_defaults_},
     name(name_),
     remote_database(remote_database_), remote_table(remote_table_),
-    context(context_), cluster_name(context.getMacros().expand(cluster_name_)), has_sharding_key(sharding_key_),
+    context(context_), cluster_name(context.getMacros()->expand(cluster_name_)), has_sharding_key(sharding_key_),
     sharding_key_expr(sharding_key_ ? ExpressionAnalyzer(sharding_key_, context, nullptr, columns).getActions(false) : nullptr),
     sharding_key_column_name(sharding_key_ ? sharding_key_->getColumnName() : String{}),
     path(data_path_.empty() ? "" : (data_path_ + escapeForFileName(name) + '/'))
diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp
index 9917b7e7a94..56b9b6443cd 100644
--- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp
@@ -190,8 +190,8 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree(
     : IStorage{columns_, materialized_columns_, alias_columns_, column_defaults_}, context(context_),
     current_zookeeper(context.getZooKeeper()), database_name(database_name_),
     table_name(name_), full_path(path_ + escapeForFileName(table_name) + '/'),
-    zookeeper_path(context.getMacros().expand(zookeeper_path_)),
-    replica_name(context.getMacros().expand(replica_name_)),
+    zookeeper_path(context.getMacros()->expand(zookeeper_path_)),
+    replica_name(context.getMacros()->expand(replica_name_)),
     data(database_name, table_name,
         full_path, columns_,
         materialized_columns_, alias_columns_, column_defaults_,

From 8b09e5510267ef5a08a825abcb382dcfc2e2ce3f Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Mon, 12 Mar 2018 21:38:00 +0300
Subject: [PATCH 029/244] added system.macros table [#CLICKHOUSE-3591]

---
 dbms/src/Common/Macros.h                      |  3 +-
 .../Storages/System/StorageSystemMacros.cpp   | 48 +++++++++++++++++++
 .../src/Storages/System/StorageSystemMacros.h | 36 ++++++++++++++
 .../Storages/System/attachSystemTables.cpp    |  2 +
 4 files changed, 88 insertions(+), 1 deletion(-)
 create mode 100644 dbms/src/Storages/System/StorageSystemMacros.cpp
 create mode 100644 dbms/src/Storages/System/StorageSystemMacros.h

diff --git a/dbms/src/Common/Macros.h b/dbms/src/Common/Macros.h
index 5b5435ca9e6..316d526b5a7 100644
--- a/dbms/src/Common/Macros.h
+++ b/dbms/src/Common/Macros.h
@@ -21,9 +21,10 @@ public:
       */
     String expand(const String & s, size_t level = 0) const;
 
-private:
     using MacroMap = std::map<String, String>;
+    const MacroMap getMacroMap() const { return macros; }
 
+private:
     MacroMap macros;
 };
 
diff --git a/dbms/src/Storages/System/StorageSystemMacros.cpp b/dbms/src/Storages/System/StorageSystemMacros.cpp
new file mode 100644
index 00000000000..37c3197fd60
--- /dev/null
+++ b/dbms/src/Storages/System/StorageSystemMacros.cpp
@@ -0,0 +1,48 @@
+#include <Common/Macros.h>
+#include <Columns/ColumnString.h>
+#include <DataTypes/DataTypeString.h>
+#include <DataStreams/OneBlockInputStream.h>
+#include <Storages/System/StorageSystemMacros.h>
+#include <Interpreters/Context.h>
+
+
+namespace DB
+{
+
+
+StorageSystemMacros::StorageSystemMacros(const std::string & name_)
+        : name(name_)
+{
+    columns = NamesAndTypesList{
+            {"macro", std::make_shared<DataTypeString>()},
+            {"substitution", std::make_shared<DataTypeString>()},
+    };
+}
+
+
+BlockInputStreams StorageSystemMacros::read(
+        const Names & column_names,
+        const SelectQueryInfo &,
+        const Context & context,
+        QueryProcessingStage::Enum & processed_stage,
+        const size_t /*max_block_size*/,
+        const unsigned /*num_streams*/)
+{
+    check(column_names);
+    processed_stage = QueryProcessingStage::FetchColumns;
+
+    MutableColumns res_columns = getSampleBlock().cloneEmptyColumns();
+
+    auto macros = context.getMacros();
+
+    for (const auto & macro : macros->getMacroMap())
+    {
+        res_columns[0]->insert(macro.first);
+        res_columns[1]->insert(macro.second);
+    }
+
+    return BlockInputStreams(1, std::make_shared<OneBlockInputStream>(getSampleBlock().cloneWithColumns(std::move(res_columns))));
+}
+
+
+}
diff --git a/dbms/src/Storages/System/StorageSystemMacros.h b/dbms/src/Storages/System/StorageSystemMacros.h
new file mode 100644
index 00000000000..ec8af55a0d3
--- /dev/null
+++ b/dbms/src/Storages/System/StorageSystemMacros.h
@@ -0,0 +1,36 @@
+#pragma once
+
+#include <ext/shared_ptr_helper.h>
+#include <Storages/IStorage.h>
+
+
+namespace DB
+{
+
+class Context;
+
+
+/** Implements `databases` system table, which allows you to get information about all databases.
+  */
+class StorageSystemMacros : public ext::shared_ptr_helper<StorageSystemMacros>, public IStorage
+{
+public:
+    std::string getName() const override { return "SystemMacros"; }
+    std::string getTableName() const override { return name; }
+
+    BlockInputStreams read(
+            const Names & column_names,
+            const SelectQueryInfo & query_info,
+            const Context & context,
+            QueryProcessingStage::Enum & processed_stage,
+            size_t max_block_size,
+            unsigned num_streams) override;
+
+private:
+    const std::string name;
+
+protected:
+    StorageSystemMacros(const std::string & name_);
+};
+
+}
diff --git a/dbms/src/Storages/System/attachSystemTables.cpp b/dbms/src/Storages/System/attachSystemTables.cpp
index cc49f047d9d..93d4809b3c7 100644
--- a/dbms/src/Storages/System/attachSystemTables.cpp
+++ b/dbms/src/Storages/System/attachSystemTables.cpp
@@ -10,6 +10,7 @@
 #include <Storages/System/StorageSystemEvents.h>
 #include <Storages/System/StorageSystemFunctions.h>
 #include <Storages/System/StorageSystemGraphite.h>
+#include <Storages/System/StorageSystemMacros.h>
 #include <Storages/System/StorageSystemMerges.h>
 #include <Storages/System/StorageSystemMetrics.h>
 #include <Storages/System/StorageSystemModels.h>
@@ -56,6 +57,7 @@ void attachSystemTablesServer(IDatabase & system_database, bool has_zookeeper)
     system_database.attachTable("models", StorageSystemModels::create("models"));
     system_database.attachTable("clusters", StorageSystemClusters::create("clusters"));
     system_database.attachTable("graphite_retentions", StorageSystemGraphite::create("graphite_retentions"));
+    system_database.attachTable("macros", StorageSystemMacros::create("macros"));
 
     if (has_zookeeper)
         system_database.attachTable("zookeeper", StorageSystemZooKeeper::create("zookeeper"));

From 71642b08809d8e6e29fc1e46d22a7e5829886116 Mon Sep 17 00:00:00 2001
From: Alexey Zatelepin <ztlpn@yandex-team.ru>
Date: Tue, 6 Mar 2018 23:18:34 +0300
Subject: [PATCH 030/244] use ColumnsDescription everywhere [#CLICKHOUSE-3241]

---
 dbms/src/Analyzers/AnalyzeColumns.cpp         |   4 +-
 dbms/src/Analyzers/ExecuteTableFunctions.cpp  |   2 +-
 dbms/src/Common/ExternalTable.h               |   4 +-
 dbms/src/Core/iostream_debug_helpers.cpp      |   2 +-
 .../DataStreams/RemoteBlockInputStream.cpp    |   2 +-
 dbms/src/Databases/DatabaseDictionary.cpp     |   9 +-
 dbms/src/Databases/DatabaseDictionary.h       |   5 +-
 dbms/src/Databases/DatabaseMemory.cpp         |   5 +-
 dbms/src/Databases/DatabaseMemory.h           |   5 +-
 dbms/src/Databases/DatabaseOrdinary.cpp       |   7 +-
 dbms/src/Databases/DatabaseOrdinary.h         |   5 +-
 dbms/src/Databases/DatabasesCommon.cpp        |   4 +-
 dbms/src/Databases/IDatabase.h                |   7 +-
 dbms/src/Interpreters/ExpressionAnalyzer.cpp  |  12 +-
 .../Interpreters/InterpreterAlterQuery.cpp    |   2 +-
 .../Interpreters/InterpreterCreateQuery.cpp   |  54 ++++----
 .../src/Interpreters/InterpreterCreateQuery.h |  20 +--
 .../Interpreters/InterpreterDescribeQuery.cpp |   6 +-
 .../Interpreters/InterpreterInsertQuery.cpp   |   6 +-
 .../Interpreters/InterpreterSelectQuery.cpp   |  10 +-
 dbms/src/Server/TCPHandler.cpp                |   3 +-
 dbms/src/Storages/AlterCommands.cpp           |  99 +++++++--------
 dbms/src/Storages/AlterCommands.h             |  12 +-
 dbms/src/Storages/ColumnsDescription.cpp      |  63 +++++++---
 dbms/src/Storages/ColumnsDescription.h        |  45 +++++--
 .../Storages/Distributed/DirectoryMonitor.cpp |   2 +-
 dbms/src/Storages/IStorage.h                  |   4 +
 dbms/src/Storages/ITableDeclaration.cpp       | 115 ++++--------------
 dbms/src/Storages/ITableDeclaration.h         |  65 +---------
 .../Storages/MergeTree/DataPartsExchange.cpp  |   1 +
 .../MergeTree/MergeTreeBlockInputStream.cpp   |   4 +-
 .../MergeTree/MergeTreeBlockReadUtils.cpp     |   4 +-
 dbms/src/Storages/MergeTree/MergeTreeData.cpp |  36 ++----
 dbms/src/Storages/MergeTree/MergeTreeData.h   |  21 ++--
 .../MergeTree/MergeTreeDataMerger.cpp         |   4 +-
 .../Storages/MergeTree/MergeTreeDataPart.cpp  |   6 +-
 .../MergeTree/MergeTreeDataSelectExecutor.cpp |   4 +-
 .../MergeTree/MergeTreeDataSelectExecutor.h   |   2 +
 .../MergeTree/MergeTreeDataWriter.cpp         |   2 +-
 .../Storages/MergeTree/MergeTreeReadPool.cpp  |   4 +-
 .../Storages/MergeTree/MergeTreeReader.cpp    |   4 +-
 .../MergeTree/MergeTreeWhereOptimizer.cpp     |   4 +-
 .../ReplicatedMergeTreeAlterThread.cpp        |  44 +------
 .../MergeTree/registerStorageMergeTree.cpp    |   5 +-
 dbms/src/Storages/StorageBuffer.cpp           |  15 +--
 dbms/src/Storages/StorageBuffer.h             |   5 +-
 dbms/src/Storages/StorageCatBoostPool.cpp     |  11 +-
 dbms/src/Storages/StorageDictionary.cpp       |  12 +-
 dbms/src/Storages/StorageDictionary.h         |   6 +-
 dbms/src/Storages/StorageDistributed.cpp      |  30 ++---
 dbms/src/Storages/StorageDistributed.h        |  16 +--
 dbms/src/Storages/StorageFactory.cpp          |  14 +--
 dbms/src/Storages/StorageFactory.h            |  12 +-
 dbms/src/Storages/StorageFile.cpp             |   8 +-
 dbms/src/Storages/StorageFile.h               |  12 +-
 dbms/src/Storages/StorageJoin.cpp             |  22 +---
 dbms/src/Storages/StorageJoin.h               |   5 +-
 dbms/src/Storages/StorageKafka.cpp            |   8 +-
 dbms/src/Storages/StorageKafka.h              |   5 +-
 dbms/src/Storages/StorageLog.cpp              |  18 ++-
 dbms/src/Storages/StorageLog.h                |   5 +-
 dbms/src/Storages/StorageMaterializedView.cpp |  10 +-
 dbms/src/Storages/StorageMaterializedView.h   |   6 +-
 dbms/src/Storages/StorageMemory.cpp           |  12 +-
 dbms/src/Storages/StorageMemory.h             |  13 +-
 dbms/src/Storages/StorageMerge.cpp            |  16 +--
 dbms/src/Storages/StorageMerge.h              |   6 +-
 dbms/src/Storages/StorageMergeTree.cpp        |  48 ++------
 dbms/src/Storages/StorageMergeTree.h          |   8 +-
 dbms/src/Storages/StorageMySQL.cpp            |   6 +-
 dbms/src/Storages/StorageMySQL.h              |   2 +-
 dbms/src/Storages/StorageNull.cpp             |   9 +-
 dbms/src/Storages/StorageNull.h               |  17 ++-
 dbms/src/Storages/StorageODBC.cpp             |   6 +-
 dbms/src/Storages/StorageODBC.h               |   2 +-
 .../Storages/StorageReplicatedMergeTree.cpp   |  70 +++--------
 .../src/Storages/StorageReplicatedMergeTree.h |   7 +-
 dbms/src/Storages/StorageSet.cpp              |  25 ++--
 dbms/src/Storages/StorageSet.h                |  16 +--
 dbms/src/Storages/StorageStripeLog.cpp        |   8 +-
 dbms/src/Storages/StorageStripeLog.h          |   5 +-
 dbms/src/Storages/StorageTinyLog.cpp          |  12 +-
 dbms/src/Storages/StorageTinyLog.h            |   5 +-
 dbms/src/Storages/StorageView.cpp             |   4 +-
 dbms/src/Storages/StorageView.h               |   2 +-
 .../StorageSystemAsynchronousMetrics.cpp      |   2 +-
 .../System/StorageSystemBuildOptions.cpp      |   2 +-
 .../Storages/System/StorageSystemClusters.cpp |   2 +-
 .../Storages/System/StorageSystemColumns.cpp  |   8 +-
 .../System/StorageSystemDatabases.cpp         |   2 +-
 .../System/StorageSystemDictionaries.cpp      |   2 +-
 .../Storages/System/StorageSystemEvents.cpp   |   2 +-
 .../System/StorageSystemFunctions.cpp         |   2 +-
 .../Storages/System/StorageSystemGraphite.cpp |   2 +-
 .../Storages/System/StorageSystemMerges.cpp   |   2 +-
 .../Storages/System/StorageSystemMetrics.cpp  |   2 +-
 .../Storages/System/StorageSystemModels.cpp   |   2 +-
 .../Storages/System/StorageSystemNumbers.cpp  |   2 +-
 dbms/src/Storages/System/StorageSystemOne.cpp |   2 +-
 .../System/StorageSystemPartsBase.cpp         |  10 +-
 .../System/StorageSystemPartsColumns.cpp      |   6 +-
 .../System/StorageSystemProcesses.cpp         |   2 +-
 .../Storages/System/StorageSystemReplicas.cpp |   2 +-
 .../System/StorageSystemReplicationQueue.cpp  |   2 +-
 .../Storages/System/StorageSystemSettings.cpp |   2 +-
 .../Storages/System/StorageSystemTables.cpp   |   2 +-
 .../System/StorageSystemZooKeeper.cpp         |   2 +-
 dbms/src/Storages/tests/storage_log.cpp       |   4 +-
 .../src/TableFunctions/TableFunctionMerge.cpp |  12 +-
 .../src/TableFunctions/TableFunctionMySQL.cpp |   2 +-
 dbms/src/TableFunctions/TableFunctionODBC.cpp |   3 +-
 .../getStructureOfRemoteTable.cpp             |   2 +-
 112 files changed, 478 insertions(+), 850 deletions(-)

diff --git a/dbms/src/Analyzers/AnalyzeColumns.cpp b/dbms/src/Analyzers/AnalyzeColumns.cpp
index 6367699ea8e..78568b080c6 100644
--- a/dbms/src/Analyzers/AnalyzeColumns.cpp
+++ b/dbms/src/Analyzers/AnalyzeColumns.cpp
@@ -173,7 +173,7 @@ ASTPtr createASTIdentifierForColumnInTable(const String & column, const CollectT
 void createASTsForAllColumnsInTable(const CollectTables::TableInfo & table, ASTs & res)
 {
     if (table.storage)
-        for (const auto & name : table.storage->getColumnNamesList())
+        for (const auto & name : table.storage->columns.getNames())
             res.emplace_back(createASTIdentifierForColumnInTable(name, table));
     else
         for (size_t i = 0, size = table.structure_of_subquery.columns(); i < size; ++i)
@@ -315,7 +315,7 @@ void processIdentifier(
     }
     else if (table->storage)
     {
-        info.data_type = table->storage->getDataTypeByName(column_name);
+        info.data_type = table->storage->getColumn(column_name).type;
     }
     else
         throw Exception("Logical error: no storage and no structure of subquery is specified for table", ErrorCodes::LOGICAL_ERROR);
diff --git a/dbms/src/Analyzers/ExecuteTableFunctions.cpp b/dbms/src/Analyzers/ExecuteTableFunctions.cpp
index fab9c6c3a3f..326c5df62ed 100644
--- a/dbms/src/Analyzers/ExecuteTableFunctions.cpp
+++ b/dbms/src/Analyzers/ExecuteTableFunctions.cpp
@@ -72,7 +72,7 @@ void ExecuteTableFunctions::dump(WriteBuffer & out) const
     {
         writeString(table.second->getName(), out);
         writeCString("\n\n", out);
-        writeString(table.second->getColumnsList().toString(), out);
+        writeString(table.second->columns.getList().toString(), out);
         writeCString("\n", out);
     }
 }
diff --git a/dbms/src/Common/ExternalTable.h b/dbms/src/Common/ExternalTable.h
index 624604d30a1..b5b0c522339 100644
--- a/dbms/src/Common/ExternalTable.h
+++ b/dbms/src/Common/ExternalTable.h
@@ -198,7 +198,9 @@ public:
 
         /// Create table
         NamesAndTypesList columns = sample_block.getNamesAndTypesList();
-        StoragePtr storage = StorageMemory::create(data.second, columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{});
+        StoragePtr storage = StorageMemory::create(
+            data.second,
+            ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}});
         storage->startup();
         context.addExternalTable(data.second, storage);
         BlockOutputStreamPtr output = storage->write(ASTPtr(), context.getSettingsRef());
diff --git a/dbms/src/Core/iostream_debug_helpers.cpp b/dbms/src/Core/iostream_debug_helpers.cpp
index 3fbbb02a98f..822bbf58376 100644
--- a/dbms/src/Core/iostream_debug_helpers.cpp
+++ b/dbms/src/Core/iostream_debug_helpers.cpp
@@ -44,7 +44,7 @@ std::ostream & operator<<(std::ostream & stream, const IDataType & what)
 std::ostream & operator<<(std::ostream & stream, const IStorage & what)
 {
     stream << "IStorage(name = " << what.getName() << ", tableName = " << what.getTableName() << ") {"
-           << what.getColumnsList().toString()
+           << what.columns.getList().toString()
            << "}";
     // isRemote supportsSampling supportsFinal supportsPrewhere
     return stream;
diff --git a/dbms/src/DataStreams/RemoteBlockInputStream.cpp b/dbms/src/DataStreams/RemoteBlockInputStream.cpp
index 7b63b36cadc..079cabe8dee 100644
--- a/dbms/src/DataStreams/RemoteBlockInputStream.cpp
+++ b/dbms/src/DataStreams/RemoteBlockInputStream.cpp
@@ -138,7 +138,7 @@ void RemoteBlockInputStream::sendExternalTables()
             {
                 StoragePtr cur = table.second;
                 QueryProcessingStage::Enum stage = QueryProcessingStage::Complete;
-                BlockInputStreams input = cur->read(cur->getColumnNamesList(), {}, context,
+                BlockInputStreams input = cur->read(cur->columns.getNames(), {}, context,
                     stage, DEFAULT_BLOCK_SIZE, 1);
                 if (input.size() == 0)
                     res.push_back(std::make_pair(std::make_shared<OneBlockInputStream>(cur->getSampleBlock()), table.first));
diff --git a/dbms/src/Databases/DatabaseDictionary.cpp b/dbms/src/Databases/DatabaseDictionary.cpp
index 8a10535fd09..439fe8f4d79 100644
--- a/dbms/src/Databases/DatabaseDictionary.cpp
+++ b/dbms/src/Databases/DatabaseDictionary.cpp
@@ -42,7 +42,7 @@ Tables DatabaseDictionary::loadTables()
             const DictionaryStructure & dictionary_structure = dict_ptr->getStructure();
             auto columns = StorageDictionary::getNamesAndTypes(dictionary_structure);
             tables[name] = StorageDictionary::create(name,
-                columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}, dictionary_structure, name);
+                ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}}, dictionary_structure, name);
         }
     }
 
@@ -77,7 +77,7 @@ StoragePtr DatabaseDictionary::tryGetTable(
                 const DictionaryStructure & dictionary_structure = dict_ptr->getStructure();
                 auto columns = StorageDictionary::getNamesAndTypes(dictionary_structure);
                 return StorageDictionary::create(table_name,
-                    columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}, dictionary_structure, table_name);
+                    ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}}, dictionary_structure, table_name);
             }
         }
     }
@@ -142,10 +142,7 @@ void DatabaseDictionary::renameTable(
 void DatabaseDictionary::alterTable(
     const Context &,
     const String &,
-    const NamesAndTypesList &,
-    const NamesAndTypesList &,
-    const NamesAndTypesList &,
-    const ColumnDefaults &,
+    const ColumnsDescription &,
     const ASTModifier &)
 {
     throw Exception("DatabaseDictionary: alterTable() is not supported", ErrorCodes::NOT_IMPLEMENTED);
diff --git a/dbms/src/Databases/DatabaseDictionary.h b/dbms/src/Databases/DatabaseDictionary.h
index 1308ea20a40..aba93faa5b1 100644
--- a/dbms/src/Databases/DatabaseDictionary.h
+++ b/dbms/src/Databases/DatabaseDictionary.h
@@ -79,10 +79,7 @@ public:
     void alterTable(
         const Context & context,
         const String & name,
-        const NamesAndTypesList & columns,
-        const NamesAndTypesList & materialized_columns,
-        const NamesAndTypesList & alias_columns,
-        const ColumnDefaults & column_defaults,
+        const ColumnsDescription & columns,
         const ASTModifier & engine_modifier) override;
 
     time_t getTableMetadataModificationTime(
diff --git a/dbms/src/Databases/DatabaseMemory.cpp b/dbms/src/Databases/DatabaseMemory.cpp
index 66b23f07ca6..57089862863 100644
--- a/dbms/src/Databases/DatabaseMemory.cpp
+++ b/dbms/src/Databases/DatabaseMemory.cpp
@@ -105,10 +105,7 @@ void DatabaseMemory::renameTable(
 void DatabaseMemory::alterTable(
     const Context &,
     const String &,
-    const NamesAndTypesList &,
-    const NamesAndTypesList &,
-    const NamesAndTypesList &,
-    const ColumnDefaults &,
+    const ColumnsDescription &,
     const ASTModifier &)
 {
     throw Exception("DatabaseMemory: alterTable() is not supported", ErrorCodes::NOT_IMPLEMENTED);
diff --git a/dbms/src/Databases/DatabaseMemory.h b/dbms/src/Databases/DatabaseMemory.h
index be095ad3755..586dcfbe4da 100644
--- a/dbms/src/Databases/DatabaseMemory.h
+++ b/dbms/src/Databases/DatabaseMemory.h
@@ -70,10 +70,7 @@ public:
     void alterTable(
         const Context & context,
         const String & name,
-        const NamesAndTypesList & columns,
-        const NamesAndTypesList & materialized_columns,
-        const NamesAndTypesList & alias_columns,
-        const ColumnDefaults & column_defaults,
+        const ColumnsDescription & columns,
         const ASTModifier & engine_modifier) override;
 
     time_t getTableMetadataModificationTime(
diff --git a/dbms/src/Databases/DatabaseOrdinary.cpp b/dbms/src/Databases/DatabaseOrdinary.cpp
index 0021f47c4c7..9dd705fbf9a 100644
--- a/dbms/src/Databases/DatabaseOrdinary.cpp
+++ b/dbms/src/Databases/DatabaseOrdinary.cpp
@@ -462,10 +462,7 @@ void DatabaseOrdinary::drop()
 void DatabaseOrdinary::alterTable(
     const Context & context,
     const String & name,
-    const NamesAndTypesList & columns,
-    const NamesAndTypesList & materialized_columns,
-    const NamesAndTypesList & alias_columns,
-    const ColumnDefaults & column_defaults,
+    const ColumnsDescription & columns,
     const ASTModifier & storage_modifier)
 {
     /// Read the definition of the table and replace the necessary parts with new ones.
@@ -486,7 +483,7 @@ void DatabaseOrdinary::alterTable(
 
     ASTCreateQuery & ast_create_query = typeid_cast<ASTCreateQuery &>(*ast);
 
-    ASTPtr new_columns = InterpreterCreateQuery::formatColumns(columns, materialized_columns, alias_columns, column_defaults);
+    ASTPtr new_columns = InterpreterCreateQuery::formatColumns(columns);
     ast_create_query.replace(ast_create_query.columns, new_columns);
 
     if (storage_modifier)
diff --git a/dbms/src/Databases/DatabaseOrdinary.h b/dbms/src/Databases/DatabaseOrdinary.h
index dbd44586c74..25e99114430 100644
--- a/dbms/src/Databases/DatabaseOrdinary.h
+++ b/dbms/src/Databases/DatabaseOrdinary.h
@@ -45,10 +45,7 @@ public:
     void alterTable(
         const Context & context,
         const String & name,
-        const NamesAndTypesList & columns,
-        const NamesAndTypesList & materialized_columns,
-        const NamesAndTypesList & alias_columns,
-        const ColumnDefaults & column_defaults,
+        const ColumnsDescription & columns,
         const ASTModifier & engine_modifier) override;
 
     time_t getTableMetadataModificationTime(
diff --git a/dbms/src/Databases/DatabasesCommon.cpp b/dbms/src/Databases/DatabasesCommon.cpp
index 83a14145d58..66d05d11184 100644
--- a/dbms/src/Databases/DatabasesCommon.cpp
+++ b/dbms/src/Databases/DatabasesCommon.cpp
@@ -67,7 +67,7 @@ std::pair<String, StoragePtr> createTableFromDefinition(
     if (!ast_create_query.columns)
         throw Exception("Missing definition of columns.", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED);
 
-    InterpreterCreateQuery::ColumnsInfo columns_info = InterpreterCreateQuery::getColumnsInfo(*ast_create_query.columns, context);
+    ColumnsDescription columns = InterpreterCreateQuery::getColumnsDescription(*ast_create_query.columns, context);
 
     return
     {
@@ -75,7 +75,7 @@ std::pair<String, StoragePtr> createTableFromDefinition(
         StorageFactory::instance().get(
             ast_create_query,
             database_data_path, ast_create_query.table, database_name, context, context.getGlobalContext(),
-            columns_info.columns, columns_info.materialized_columns, columns_info.alias_columns, columns_info.column_defaults,
+            columns,
             true, has_force_restore_data_flag)
     };
 }
diff --git a/dbms/src/Databases/IDatabase.h b/dbms/src/Databases/IDatabase.h
index 74cbe0b2202..f7b8af632ca 100644
--- a/dbms/src/Databases/IDatabase.h
+++ b/dbms/src/Databases/IDatabase.h
@@ -2,7 +2,7 @@
 
 #include <Core/Types.h>
 #include <Core/NamesAndTypes.h>
-#include <Storages/ColumnDefault.h>
+#include <Storages/ColumnsDescription.h>
 #include <ctime>
 #include <memory>
 #include <functional>
@@ -113,10 +113,7 @@ public:
     virtual void alterTable(
         const Context & context,
         const String & name,
-        const NamesAndTypesList & columns,
-        const NamesAndTypesList & materialized_columns,
-        const NamesAndTypesList & alias_columns,
-        const ColumnDefaults & column_defaults,
+        const ColumnsDescription & columns,
         const ASTModifier & engine_modifier) = 0;
 
     /// Returns time of table's metadata change, 0 if there is no corresponding metadata file.
diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp
index c3c663d59ad..fe85e154e92 100644
--- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp
+++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp
@@ -190,7 +190,7 @@ ExpressionAnalyzer::ExpressionAnalyzer(
     }
 
     if (storage && source_columns.empty())
-        source_columns = storage->getSampleBlock().getNamesAndTypesList();
+        source_columns = storage->columns.getList();
     else
         removeDuplicateColumns(source_columns);
 
@@ -712,7 +712,7 @@ static std::shared_ptr<InterpreterSelectWithUnionQuery> interpretSubquery(
         /// get columns list for target table
         auto database_table = getDatabaseAndTableNameFromIdentifier(*table);
         const auto & storage = context.getTable(database_table.first, database_table.second);
-        const auto & columns = storage->getColumnsListNonMaterialized();
+        const auto & columns = storage->columns.getListNonMaterialized();
         select_expression_list->children.reserve(columns.size());
 
         /// manually substitute column names in place of asterisk
@@ -826,7 +826,9 @@ void ExpressionAnalyzer::addExternalStorage(ASTPtr & subquery_or_table_name_or_t
     Block sample = interpreter->getSampleBlock();
     NamesAndTypesList columns = sample.getNamesAndTypesList();
 
-    StoragePtr external_storage = StorageMemory::create(external_table_name, columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{});
+    StoragePtr external_storage = StorageMemory::create(
+        external_table_name,
+        ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}});
     external_storage->startup();
 
     /** We replace the subquery with the name of the temporary table.
@@ -1050,7 +1052,7 @@ void ExpressionAnalyzer::normalizeTreeImpl(
                 if (storage)
                 {
                     /// If we select from a table, get only not MATERIALIZED, not ALIAS columns.
-                    for (const auto & name_type : storage->getColumnsListNonMaterialized())
+                    for (const auto & name_type : storage->columns.getListNonMaterialized())
                         all_columns.emplace_back(std::make_shared<ASTIdentifier>(name_type.name));
                 }
                 else
@@ -1147,7 +1149,7 @@ void ExpressionAnalyzer::addAliasColumns()
     if (!storage)
         return;
 
-    source_columns.insert(std::end(source_columns), std::begin(storage->alias_columns), std::end(storage->alias_columns));
+    source_columns.insert(std::end(source_columns), std::begin(storage->columns.aliases), std::end(storage->columns.aliases));
 }
 
 
diff --git a/dbms/src/Interpreters/InterpreterAlterQuery.cpp b/dbms/src/Interpreters/InterpreterAlterQuery.cpp
index f990fd8570f..1fb3e1e80db 100644
--- a/dbms/src/Interpreters/InterpreterAlterQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterAlterQuery.cpp
@@ -200,7 +200,7 @@ void InterpreterAlterQuery::PartitionCommands::validate(const IStorage * table)
         {
             String column_name = command.column_name.safeGet<String>();
 
-            if (!table->hasRealColumn(column_name))
+            if (!table->columns.has(column_name))
             {
                 throw Exception("Wrong column name. Cannot find column " + column_name + " to clear it from partition",
                     DB::ErrorCodes::ILLEGAL_COLUMN);
diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp
index 2b6d333d1f6..6033947b474 100644
--- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp
@@ -301,15 +301,11 @@ ASTPtr InterpreterCreateQuery::formatColumns(const NamesAndTypesList & columns)
     return columns_list;
 }
 
-ASTPtr InterpreterCreateQuery::formatColumns(
-    const NamesAndTypesList & columns,
-    const NamesAndTypesList & materialized_columns,
-    const NamesAndTypesList & alias_columns,
-    const ColumnDefaults & column_defaults)
+ASTPtr InterpreterCreateQuery::formatColumns(const ColumnsDescription & columns)
 {
     auto columns_list = std::make_shared<ASTExpressionList>();
 
-    for (const auto & column : boost::join(columns, boost::join(materialized_columns, alias_columns)))
+    for (const auto & column : boost::join(columns.ordinary, boost::join(columns.materialized, columns.aliases)))
     {
         const auto column_declaration = std::make_shared<ASTColumnDeclaration>();
         ASTPtr column_declaration_ptr{column_declaration};
@@ -324,8 +320,8 @@ ASTPtr InterpreterCreateQuery::formatColumns(
         column_declaration->type = parseQuery(storage_p, pos, end, "data type");
         column_declaration->type->owned_string = type_name;
 
-        const auto it = column_defaults.find(column.name);
-        if (it != std::end(column_defaults))
+        const auto it = columns.defaults.find(column.name);
+        if (it != std::end(columns.defaults))
         {
             column_declaration->default_specifier = toString(it->second.type);
             column_declaration->default_expression = it->second.expression->clone();
@@ -338,49 +334,46 @@ ASTPtr InterpreterCreateQuery::formatColumns(
 }
 
 
-InterpreterCreateQuery::ColumnsInfo InterpreterCreateQuery::getColumnsInfo(const ASTExpressionList & columns, const Context & context)
+ColumnsDescription InterpreterCreateQuery::getColumnsDescription(const ASTExpressionList & columns, const Context & context)
 {
-    ColumnsInfo res;
+    ColumnsDescription res;
 
     auto && columns_and_defaults = parseColumns(columns, context);
-    res.materialized_columns = removeAndReturnColumns(columns_and_defaults, ColumnDefaultType::Materialized);
-    res.alias_columns = removeAndReturnColumns(columns_and_defaults, ColumnDefaultType::Alias);
-    res.columns = std::move(columns_and_defaults.first);
-    res.column_defaults = std::move(columns_and_defaults.second);
+    res.materialized = removeAndReturnColumns(columns_and_defaults, ColumnDefaultType::Materialized);
+    res.aliases = removeAndReturnColumns(columns_and_defaults, ColumnDefaultType::Alias);
+    res.ordinary = std::move(columns_and_defaults.first);
+    res.defaults = std::move(columns_and_defaults.second);
 
-    if (res.columns.size() + res.materialized_columns.size() == 0)
+    if (res.ordinary.size() + res.materialized.size() == 0)
         throw Exception{"Cannot CREATE table without physical columns", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED};
 
     return res;
 }
 
 
-InterpreterCreateQuery::ColumnsInfo InterpreterCreateQuery::setColumns(
+ColumnsDescription InterpreterCreateQuery::setColumns(
     ASTCreateQuery & create, const Block & as_select_sample, const StoragePtr & as_storage) const
 {
-    ColumnsInfo res;
+    ColumnsDescription res;
 
     if (create.columns)
     {
-        res = getColumnsInfo(*create.columns, context);
+        res = getColumnsDescription(*create.columns, context);
     }
     else if (!create.as_table.empty())
     {
-        res.columns = as_storage->getColumnsListNonMaterialized();
-        res.materialized_columns = as_storage->materialized_columns;
-        res.alias_columns = as_storage->alias_columns;
-        res.column_defaults = as_storage->column_defaults;
+        res = as_storage->columns;
     }
     else if (create.select)
     {
         for (size_t i = 0; i < as_select_sample.columns(); ++i)
-            res.columns.emplace_back(as_select_sample.safeGetByPosition(i).name, as_select_sample.safeGetByPosition(i).type);
+            res.ordinary.emplace_back(as_select_sample.safeGetByPosition(i).name, as_select_sample.safeGetByPosition(i).type);
     }
     else
         throw Exception("Incorrect CREATE query: required list of column descriptions or AS section or SELECT.", ErrorCodes::INCORRECT_QUERY);
 
     /// Even if query has list of columns, canonicalize it (unfold Nested columns).
-    ASTPtr new_columns = formatColumns(res.columns, res.materialized_columns, res.alias_columns, res.column_defaults);
+    ASTPtr new_columns = formatColumns(res);
     if (create.columns)
         create.replace(create.columns, new_columns);
     else
@@ -394,11 +387,11 @@ InterpreterCreateQuery::ColumnsInfo InterpreterCreateQuery::setColumns(
             throw Exception("Column " + backQuoteIfNeed(column_name_and_type.name) + " already exists", ErrorCodes::DUPLICATE_COLUMN);
     };
 
-    for (const auto & elem : res.columns)
+    for (const auto & elem : res.ordinary)
         check_column_already_exists(elem);
-    for (const auto & elem : res.materialized_columns)
+    for (const auto & elem : res.materialized)
         check_column_already_exists(elem);
-    for (const auto & elem : res.alias_columns)
+    for (const auto & elem : res.aliases)
         check_column_already_exists(elem);
 
     return res;
@@ -489,7 +482,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create)
     }
 
     /// Set and retrieve list of columns.
-    ColumnsInfo columns = setColumns(create, as_select_sample, as_storage);
+    ColumnsDescription columns = setColumns(create, as_select_sample, as_storage);
 
     /// Set the table engine if it was not specified explicitly.
     setEngine(create);
@@ -530,10 +523,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create)
             database_name,
             context,
             context.getGlobalContext(),
-            columns.columns,
-            columns.materialized_columns,
-            columns.alias_columns,
-            columns.column_defaults,
+            columns,
             create.attach,
             false);
 
diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.h b/dbms/src/Interpreters/InterpreterCreateQuery.h
index b72b7eb7d09..8bec93a0fdc 100644
--- a/dbms/src/Interpreters/InterpreterCreateQuery.h
+++ b/dbms/src/Interpreters/InterpreterCreateQuery.h
@@ -1,7 +1,7 @@
 #pragma once
 
 #include <Interpreters/IInterpreter.h>
-#include <Storages/ColumnDefault.h>
+#include <Storages/ColumnsDescription.h>
 
 
 class ThreadPool;
@@ -27,11 +27,7 @@ public:
 
     /// List of columns and their types in AST.
     static ASTPtr formatColumns(const NamesAndTypesList & columns);
-    static ASTPtr formatColumns(
-        const NamesAndTypesList & columns,
-        const NamesAndTypesList & materialized_columns,
-        const NamesAndTypesList & alias_columns,
-        const ColumnDefaults & column_defaults);
+    static ASTPtr formatColumns(const ColumnsDescription & columns);
 
     void setDatabaseLoadingThreadpool(ThreadPool & thread_pool_)
     {
@@ -48,23 +44,15 @@ public:
         internal = internal_;
     }
 
-    struct ColumnsInfo
-    {
-        NamesAndTypesList columns;
-        NamesAndTypesList materialized_columns;
-        NamesAndTypesList alias_columns;
-        ColumnDefaults column_defaults;
-    };
-
     /// Obtain information about columns, their types and default values, for case when columns in CREATE query is specified explicitly.
-    static ColumnsInfo getColumnsInfo(const ASTExpressionList & columns, const Context & context);
+    static ColumnsDescription getColumnsDescription(const ASTExpressionList & columns, const Context & context);
 
 private:
     BlockIO createDatabase(ASTCreateQuery & create);
     BlockIO createTable(ASTCreateQuery & create);
 
     /// Calculate list of columns of table and return it.
-    ColumnsInfo setColumns(ASTCreateQuery & create, const Block & as_select_sample, const StoragePtr & as_storage) const;
+    ColumnsDescription setColumns(ASTCreateQuery & create, const Block & as_select_sample, const StoragePtr & as_storage) const;
     void setEngine(ASTCreateQuery & create) const;
     void checkAccess(const ASTCreateQuery & create);
 
diff --git a/dbms/src/Interpreters/InterpreterDescribeQuery.cpp b/dbms/src/Interpreters/InterpreterDescribeQuery.cpp
index 65b0fea25f1..b0d7e4c0363 100644
--- a/dbms/src/Interpreters/InterpreterDescribeQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterDescribeQuery.cpp
@@ -99,9 +99,9 @@ BlockInputStreamPtr InterpreterDescribeQuery::executeImpl()
         }
 
         auto table_lock = table->lockStructure(false, __PRETTY_FUNCTION__);
-        columns = table->getColumnsList();
-        columns.insert(std::end(columns), std::begin(table->alias_columns), std::end(table->alias_columns));
-        column_defaults = table->column_defaults;
+        columns = table->columns.getList();
+        columns.insert(std::end(columns), std::begin(table->columns.aliases), std::end(table->columns.aliases));
+        column_defaults = table->columns.defaults;
     }
 
     Block sample_block = getSampleBlock();
diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp
index 014c8d287ad..5f0b8201ac3 100644
--- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp
@@ -95,7 +95,7 @@ BlockIO InterpreterInsertQuery::execute()
 
     auto table_lock = table->lockStructure(true, __PRETTY_FUNCTION__);
 
-    NamesAndTypesList required_columns = table->getColumnsList();
+    NamesAndTypesList required_columns = table->columns.getList();
 
     /// We create a pipeline of several streams, into which we will write data.
     BlockOutputStreamPtr out;
@@ -103,7 +103,7 @@ BlockIO InterpreterInsertQuery::execute()
     out = std::make_shared<PushingToViewsBlockOutputStream>(query.database, query.table, table, context, query_ptr, query.no_destination);
 
     out = std::make_shared<AddingDefaultBlockOutputStream>(
-        out, getSampleBlock(query, table), required_columns, table->column_defaults, context);
+        out, getSampleBlock(query, table), required_columns, table->columns.defaults, context);
 
     /// Do not squash blocks if it is a sync INSERT into Distributed
     if (!(context.getSettingsRef().insert_distributed_sync && table->getName() == "Distributed"))
@@ -135,7 +135,7 @@ BlockIO InterpreterInsertQuery::execute()
         if (!allow_materialized)
         {
             Block in_header = res.in->getHeader();
-            for (const auto & name_type : table->materialized_columns)
+            for (const auto & name_type : table->columns.materialized)
                 if (in_header.has(name_type.name))
                     throw Exception("Cannot insert column " + name_type.name + ", because it is MATERIALIZED column.", ErrorCodes::ILLEGAL_COLUMN);
         }
diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp
index 1d5a442bb62..04e05e975ac 100644
--- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp
@@ -516,12 +516,12 @@ QueryProcessingStage::Enum InterpreterSelectQuery::executeFetchColumns(Pipeline
     /// Are ALIAS columns required for query execution?
     auto alias_columns_required = false;
 
-    if (storage && !storage->alias_columns.empty())
+    if (storage && !storage->columns.aliases.empty())
     {
         for (const auto & column : required_columns)
         {
-            const auto default_it = storage->column_defaults.find(column);
-            if (default_it != std::end(storage->column_defaults) && default_it->second.type == ColumnDefaultType::Alias)
+            const auto default_it = storage->columns.defaults.find(column);
+            if (default_it != std::end(storage->columns.defaults) && default_it->second.type == ColumnDefaultType::Alias)
             {
                 alias_columns_required = true;
                 break;
@@ -535,8 +535,8 @@ QueryProcessingStage::Enum InterpreterSelectQuery::executeFetchColumns(Pipeline
 
             for (const auto & column : required_columns)
             {
-                const auto default_it = storage->column_defaults.find(column);
-                if (default_it != std::end(storage->column_defaults) && default_it->second.type == ColumnDefaultType::Alias)
+                const auto default_it = storage->columns.defaults.find(column);
+                if (default_it != std::end(storage->columns.defaults) && default_it->second.type == ColumnDefaultType::Alias)
                     required_columns_expr_list->children.emplace_back(setAlias(default_it->second.expression->clone(), column));
                 else
                     required_columns_expr_list->children.emplace_back(std::make_shared<ASTIdentifier>(column));
diff --git a/dbms/src/Server/TCPHandler.cpp b/dbms/src/Server/TCPHandler.cpp
index 55f85c662dc..76a79d09722 100644
--- a/dbms/src/Server/TCPHandler.cpp
+++ b/dbms/src/Server/TCPHandler.cpp
@@ -630,7 +630,8 @@ bool TCPHandler::receiveData()
             if (!(storage = query_context.tryGetExternalTable(external_table_name)))
             {
                 NamesAndTypesList columns = block.getNamesAndTypesList();
-                storage = StorageMemory::create(external_table_name, columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{});
+                storage = StorageMemory::create(external_table_name,
+                    ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}});
                 storage->startup();
                 query_context.addExternalTable(external_table_name, storage);
             }
diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp
index 8411dfe1bf6..defad6f5cba 100644
--- a/dbms/src/Storages/AlterCommands.cpp
+++ b/dbms/src/Storages/AlterCommands.cpp
@@ -21,9 +21,7 @@ namespace ErrorCodes
 }
 
 
-void AlterCommand::apply(
-    NamesAndTypesList & columns, NamesAndTypesList & materialized_columns, NamesAndTypesList & alias_columns,
-    ColumnDefaults & column_defaults) const
+void AlterCommand::apply(ColumnsDescription & columns_description) const
 {
     if (type == ADD_COLUMN)
     {
@@ -33,9 +31,9 @@ void AlterCommand::apply(
                 std::bind(namesEqual, std::cref(column_name), std::placeholders::_1));
         };
 
-        if (exists_in(columns) ||
-            exists_in(materialized_columns) ||
-            exists_in(alias_columns))
+        if (exists_in(columns_description.ordinary) ||
+            exists_in(columns_description.materialized) ||
+            exists_in(columns_description.aliases))
         {
             throw Exception{
                 "Cannot add column " + column_name + ": column with this name already exists",
@@ -68,24 +66,24 @@ void AlterCommand::apply(
         };
 
         if (default_type == ColumnDefaultType::Default)
-            add_column(columns);
+            add_column(columns_description.ordinary);
         else if (default_type == ColumnDefaultType::Materialized)
-            add_column(materialized_columns);
+            add_column(columns_description.materialized);
         else if (default_type == ColumnDefaultType::Alias)
-            add_column(alias_columns);
+            add_column(columns_description.aliases);
         else
             throw Exception{"Unknown ColumnDefaultType value", ErrorCodes::LOGICAL_ERROR};
 
         if (default_expression)
-            column_defaults.emplace(column_name, ColumnDefault{default_type, default_expression});
+            columns_description.defaults.emplace(column_name, ColumnDefault{default_type, default_expression});
 
         /// Slow, because each time a list is copied
-        columns = Nested::flatten(columns);
+        columns_description.ordinary = Nested::flatten(columns_description.ordinary);
     }
     else if (type == DROP_COLUMN)
     {
         /// look for a column in list and remove it if present, also removing corresponding entry from column_defaults
-        const auto remove_column = [&column_defaults, this] (NamesAndTypesList & columns)
+        const auto remove_column = [&columns_description, this] (NamesAndTypesList & columns)
         {
             auto removed = false;
             NamesAndTypesList::iterator column_it;
@@ -95,15 +93,15 @@ void AlterCommand::apply(
             {
                 removed = true;
                 column_it = columns.erase(column_it);
-                column_defaults.erase(column_name);
+                columns_description.defaults.erase(column_name);
             }
 
             return removed;
         };
 
-        if (!remove_column(columns) &&
-            !remove_column(materialized_columns) &&
-            !remove_column(alias_columns))
+        if (!remove_column(columns_description.ordinary) &&
+            !remove_column(columns_description.materialized) &&
+            !remove_column(columns_description.aliases))
         {
             throw Exception("Wrong column name. Cannot find column " + column_name + " to drop",
                             ErrorCodes::ILLEGAL_COLUMN);
@@ -111,14 +109,15 @@ void AlterCommand::apply(
     }
     else if (type == MODIFY_COLUMN)
     {
-        const auto default_it = column_defaults.find(column_name);
-        const auto had_default_expr = default_it != std::end(column_defaults);
+        const auto default_it = columns_description.defaults.find(column_name);
+        const auto had_default_expr = default_it != std::end(columns_description.defaults);
         const auto old_default_type = had_default_expr ? default_it->second.type : ColumnDefaultType{};
 
         /// target column list
-        auto & new_columns = default_type == ColumnDefaultType::Default ?
-            columns : default_type == ColumnDefaultType::Materialized ?
-            materialized_columns : alias_columns;
+        auto & new_columns =
+            default_type == ColumnDefaultType::Default ? columns_description.ordinary
+            : default_type == ColumnDefaultType::Materialized ? columns_description.materialized
+            : columns_description.aliases;
 
         /// find column or throw exception
         const auto find_column = [this] (NamesAndTypesList & columns)
@@ -136,9 +135,10 @@ void AlterCommand::apply(
         if (default_type != old_default_type)
         {
             /// source column list
-            auto & old_columns = old_default_type == ColumnDefaultType::Default ?
-                columns : old_default_type == ColumnDefaultType::Materialized ?
-                materialized_columns : alias_columns;
+            auto & old_columns =
+                old_default_type == ColumnDefaultType::Default ? columns_description.ordinary
+                : old_default_type == ColumnDefaultType::Materialized ? columns_description.materialized
+                : columns_description.aliases;
 
             const auto old_column_it = find_column(old_columns);
             new_columns.emplace_back(*old_column_it);
@@ -146,7 +146,7 @@ void AlterCommand::apply(
 
             /// do not forget to change the default type of old column
             if (had_default_expr)
-                column_defaults[column_name].type = default_type;
+                columns_description.defaults[column_name].type = default_type;
         }
 
         /// find column in one of three column lists
@@ -155,13 +155,13 @@ void AlterCommand::apply(
 
         if (!default_expression && had_default_expr)
             /// new column has no default expression, remove it from column_defaults along with it's type
-            column_defaults.erase(column_name);
+            columns_description.defaults.erase(column_name);
         else if (default_expression && !had_default_expr)
             /// new column has a default expression while the old one had not, add it it column_defaults
-            column_defaults.emplace(column_name, ColumnDefault{default_type, default_expression});
+            columns_description.defaults.emplace(column_name, ColumnDefault{default_type, default_expression});
         else if (had_default_expr)
             /// both old and new columns have default expression, update it
-            column_defaults[column_name].expression = default_expression;
+            columns_description.defaults[column_name].expression = default_expression;
     }
     else if (type == MODIFY_PRIMARY_KEY)
     {
@@ -173,30 +173,21 @@ void AlterCommand::apply(
 }
 
 
-void AlterCommands::apply(NamesAndTypesList & columns,
-            NamesAndTypesList & materialized_columns,
-            NamesAndTypesList & alias_columns,
-            ColumnDefaults & column_defaults) const
+void AlterCommands::apply(ColumnsDescription & columns_description) const
 {
-    auto new_columns = columns;
-    auto new_materialized_columns = materialized_columns;
-    auto new_alias_columns = alias_columns;
-    auto new_column_defaults = column_defaults;
+    auto new_columns_description = columns_description;
 
     for (const AlterCommand & command : *this)
-        command.apply(new_columns, new_materialized_columns, new_alias_columns, new_column_defaults);
+        command.apply(new_columns_description);
 
-    columns = std::move(new_columns);
-    materialized_columns = std::move(new_materialized_columns);
-    alias_columns = std::move(new_alias_columns);
-    column_defaults = std::move(new_column_defaults);
+    columns_description = std::move(new_columns_description);
 }
 
 void AlterCommands::validate(IStorage * table, const Context & context)
 {
-    auto columns = table->getColumnsList();
-    columns.insert(std::end(columns), std::begin(table->alias_columns), std::end(table->alias_columns));
-    auto defaults = table->column_defaults;
+    auto all_columns = table->columns.getList();
+    all_columns.insert(std::end(all_columns), std::begin(table->columns.aliases), std::end(table->columns.aliases));
+    auto defaults = table->columns.defaults;
 
     std::vector<std::pair<NameAndTypePair, AlterCommand *>> defaulted_columns{};
 
@@ -208,12 +199,12 @@ void AlterCommands::validate(IStorage * table, const Context & context)
         if (command.type == AlterCommand::ADD_COLUMN || command.type == AlterCommand::MODIFY_COLUMN)
         {
             const auto & column_name = command.column_name;
-            const auto column_it = std::find_if(std::begin(columns), std::end(columns),
+            const auto column_it = std::find_if(std::begin(all_columns), std::end(all_columns),
                 std::bind(AlterCommand::namesEqual, std::cref(command.column_name), std::placeholders::_1));
 
             if (command.type == AlterCommand::ADD_COLUMN)
             {
-                if (std::end(columns) != column_it)
+                if (std::end(all_columns) != column_it)
                     throw Exception{
                         "Cannot add column " + column_name + ": column with this name already exists",
                         ErrorCodes::ILLEGAL_COLUMN};
@@ -221,17 +212,17 @@ void AlterCommands::validate(IStorage * table, const Context & context)
             else if (command.type == AlterCommand::MODIFY_COLUMN)
             {
 
-                if (std::end(columns) == column_it)
+                if (std::end(all_columns) == column_it)
                     throw Exception{
                         "Wrong column name. Cannot find column " + column_name + " to modify",
                         ErrorCodes::ILLEGAL_COLUMN};
 
-                columns.erase(column_it);
+                all_columns.erase(column_it);
                 defaults.erase(column_name);
             }
 
             /// we're creating dummy DataTypeUInt8 in order to prevent the NullPointerException in ExpressionActions
-            columns.emplace_back(column_name, command.data_type ? command.data_type : std::make_shared<DataTypeUInt8>());
+            all_columns.emplace_back(column_name, command.data_type ? command.data_type : std::make_shared<DataTypeUInt8>());
 
             if (command.default_expression)
             {
@@ -265,7 +256,7 @@ void AlterCommands::validate(IStorage * table, const Context & context)
             for (const auto & default_column : defaults)
             {
                 const auto & default_expression = default_column.second.expression;
-                const auto actions = ExpressionAnalyzer{default_expression, context, {}, columns}.getActions(true);
+                const auto actions = ExpressionAnalyzer{default_expression, context, {}, all_columns}.getActions(true);
                 const auto required_columns = actions->getRequiredColumns();
 
                 if (required_columns.end() != std::find(required_columns.begin(), required_columns.end(), command.column_name))
@@ -275,12 +266,12 @@ void AlterCommands::validate(IStorage * table, const Context & context)
             }
 
             auto found = false;
-            for (auto it = std::begin(columns); it != std::end(columns);)
+            for (auto it = std::begin(all_columns); it != std::end(all_columns);)
             {
                 if (AlterCommand::namesEqual(command.column_name, *it))
                 {
                     found = true;
-                    it = columns.erase(it);
+                    it = all_columns.erase(it);
                 }
                 else
                     ++it;
@@ -305,7 +296,7 @@ void AlterCommands::validate(IStorage * table, const Context & context)
     for (const auto & col_def : defaults)
     {
         const auto & column_name = col_def.first;
-        const auto column_it = std::find_if(columns.begin(), columns.end(), [&] (const NameAndTypePair & name_type)
+        const auto column_it = std::find_if(all_columns.begin(), all_columns.end(), [&] (const NameAndTypePair & name_type)
             { return AlterCommand::namesEqual(column_name, name_type); });
 
         const auto tmp_column_name = column_name + "_tmp";
@@ -321,7 +312,7 @@ void AlterCommands::validate(IStorage * table, const Context & context)
         defaulted_columns.emplace_back(NameAndTypePair{column_name, column_type_ptr}, nullptr);
     }
 
-    const auto actions = ExpressionAnalyzer{default_expr_list, context, {}, columns}.getActions(true);
+    const auto actions = ExpressionAnalyzer{default_expr_list, context, {}, all_columns}.getActions(true);
     const auto block = actions->getSampleBlock();
 
     /// set deduced types, modify default expression if necessary
diff --git a/dbms/src/Storages/AlterCommands.h b/dbms/src/Storages/AlterCommands.h
index 543106254e6..44c44e8bfd0 100644
--- a/dbms/src/Storages/AlterCommands.h
+++ b/dbms/src/Storages/AlterCommands.h
@@ -1,7 +1,7 @@
 #pragma once
 
 #include <Core/NamesAndTypes.h>
-#include <Storages/ColumnDefault.h>
+#include <Storages/ColumnsDescription.h>
 
 namespace DB
 {
@@ -43,10 +43,7 @@ struct AlterCommand
         return (name_with_dot == name_type.name.substr(0, name_without_dot.length() + 1) || name_without_dot == name_type.name);
     }
 
-    void apply(NamesAndTypesList & columns,
-               NamesAndTypesList & materialized_columns,
-               NamesAndTypesList & alias_columns,
-               ColumnDefaults & column_defaults) const;
+    void apply(ColumnsDescription & columns_description) const;
 
     AlterCommand() = default;
     AlterCommand(const Type type, const String & column_name, const DataTypePtr & data_type,
@@ -63,10 +60,7 @@ class Context;
 class AlterCommands : public std::vector<AlterCommand>
 {
 public:
-    void apply(NamesAndTypesList & columns,
-               NamesAndTypesList & materialized_columns,
-               NamesAndTypesList & alias_columns,
-               ColumnDefaults & column_defaults) const;
+    void apply(ColumnsDescription & columns_description) const;
 
     void validate(IStorage * table, const Context & context);
 };
diff --git a/dbms/src/Storages/ColumnsDescription.cpp b/dbms/src/Storages/ColumnsDescription.cpp
index c0bfabea727..b8fae53ed4e 100644
--- a/dbms/src/Storages/ColumnsDescription.cpp
+++ b/dbms/src/Storages/ColumnsDescription.cpp
@@ -1,3 +1,4 @@
+#include <Storages/ColumnsDescription.h>
 #include <Parsers/ExpressionListParsers.h>
 #include <Parsers/parseQuery.h>
 #include <Parsers/queryToString.h>
@@ -7,8 +8,13 @@
 #include <IO/ReadHelpers.h>
 #include <IO/WriteBufferFromString.h>
 #include <IO/ReadBufferFromString.h>
-#include <Storages/ColumnsDescription.h>
 #include <DataTypes/DataTypeFactory.h>
+#include <Common/Exception.h>
+
+#include <ext/collection_cast.h>
+#include <ext/map.h>
+
+#include <boost/range/join.hpp>
 
 
 namespace DB
@@ -16,17 +22,47 @@ namespace DB
 
 namespace ErrorCodes
 {
+    extern const int NO_SUCH_COLUMN_IN_TABLE;
     extern const int CANNOT_PARSE_TEXT;
 }
 
 
-template <bool store>
-String ColumnsDescription<store>::toString() const
+NamesAndTypesList ColumnsDescription::getList() const
+{
+    return ext::collection_cast<NamesAndTypesList>(boost::join(ordinary, materialized));
+}
+
+
+Names ColumnsDescription::getNames() const
+{
+    return ext::map<Names>(boost::join(ordinary, materialized), [] (const auto & it) { return it.name; });
+}
+
+
+NameAndTypePair ColumnsDescription::get(const String & column_name) const
+{
+    for (auto & it : boost::join(ordinary, materialized))
+        if (it.name == column_name)
+            return it;
+    throw Exception("There is no column " + column_name + " in table.", ErrorCodes::NO_SUCH_COLUMN_IN_TABLE);
+}
+
+
+bool ColumnsDescription::has(const String & column_name) const
+{
+    for (auto & it : boost::join(ordinary, materialized))
+        if (it.name == column_name)
+            return true;
+    return false;
+}
+
+
+String ColumnsDescription::toString() const
 {
     WriteBufferFromOwnString buf;
 
     writeString("columns format version: 1\n", buf);
-    writeText(columns.size() + materialized.size() + alias.size(), buf);
+    writeText(ordinary.size() + materialized.size() + aliases.size(), buf);
     writeString(" columns:\n", buf);
 
     const auto write_columns = [this, &buf] (const NamesAndTypesList & columns)
@@ -53,16 +89,15 @@ String ColumnsDescription<store>::toString() const
         }
     };
 
-    write_columns(columns);
+    write_columns(ordinary);
     write_columns(materialized);
-    write_columns(alias);
+    write_columns(aliases);
 
     return buf.str();
 }
 
 
-template <>
-ColumnsDescription<true> ColumnsDescription<true>::parse(const String & str)
+ColumnsDescription ColumnsDescription::parse(const String & str)
 {
     ReadBufferFromString buf{str};
 
@@ -74,7 +109,7 @@ ColumnsDescription<true> ColumnsDescription<true>::parse(const String & str)
     ParserExpression expr_parser;
     const DataTypeFactory & data_type_factory = DataTypeFactory::instance();
 
-    ColumnsDescription<true> result{};
+    ColumnsDescription result;
     for (size_t i = 0; i < count; ++i)
     {
         String column_name;
@@ -88,7 +123,7 @@ ColumnsDescription<true> ColumnsDescription<true>::parse(const String & str)
         {
             assertChar('\n', buf);
 
-            result.columns.emplace_back(column_name, std::move(type));
+            result.ordinary.emplace_back(column_name, std::move(type));
             continue;
         }
         assertChar('\t', buf);
@@ -107,11 +142,11 @@ ColumnsDescription<true> ColumnsDescription<true>::parse(const String & str)
         ASTPtr default_expr = parseQuery(expr_parser, begin, end, "default expression");
 
         if (ColumnDefaultType::Default == default_type)
-            result.columns.emplace_back(column_name, std::move(type));
+            result.ordinary.emplace_back(column_name, std::move(type));
         else if (ColumnDefaultType::Materialized == default_type)
             result.materialized.emplace_back(column_name, std::move(type));
         else if (ColumnDefaultType::Alias == default_type)
-            result.alias.emplace_back(column_name, std::move(type));
+            result.aliases.emplace_back(column_name, std::move(type));
 
         result.defaults.emplace(column_name, ColumnDefault{default_type, default_expr});
     }
@@ -121,8 +156,4 @@ ColumnsDescription<true> ColumnsDescription<true>::parse(const String & str)
     return result;
 }
 
-
-template struct ColumnsDescription<false>;
-template struct ColumnsDescription<true>;
-
 }
diff --git a/dbms/src/Storages/ColumnsDescription.h b/dbms/src/Storages/ColumnsDescription.h
index cce6232d33b..888f0864701 100644
--- a/dbms/src/Storages/ColumnsDescription.h
+++ b/dbms/src/Storages/ColumnsDescription.h
@@ -1,28 +1,53 @@
 #pragma once
 
-#include <Storages/ColumnDefault.h>
 #include <Core/NamesAndTypes.h>
+#include <Core/Names.h>
+#include <Storages/ColumnDefault.h>
+#include <Core/Block.h>
 
 
 namespace DB
 {
 
-
-template <bool store>
 struct ColumnsDescription
 {
-    template <typename T>
-    using by_value_or_cref = std::conditional_t<store, T, const T &>;
+    NamesAndTypesList ordinary;
+    NamesAndTypesList materialized;
+    NamesAndTypesList aliases;
+    ColumnDefaults defaults;
+
+
+    bool operator==(const ColumnsDescription & other) const
+    {
+        return ordinary == other.ordinary
+            && materialized == other.materialized
+            && aliases == other.aliases
+            && defaults == other.defaults;
+    }
+
+    bool operator!=(const ColumnsDescription & other) const { return !(*this == other); }
+
+    /** Get a list of names and table column types, only non-virtual.
+     */
+    NamesAndTypesList getList() const;
+    const NamesAndTypesList & getListNonMaterialized() const { return ordinary; }
+
+    /** Get a list of column names.
+     */
+    Names getNames() const;
+
+    /** Get a description of any column by its name.
+     */
+    NameAndTypePair get(const String & column_name) const;
+
+    /** Is there a column with that name.
+      */
+    bool has(const String & column_name) const;
 
-    by_value_or_cref<NamesAndTypesList> columns;
-    by_value_or_cref<NamesAndTypesList> materialized;
-    by_value_or_cref<NamesAndTypesList> alias;
-    by_value_or_cref<ColumnDefaults> defaults;
 
     String toString() const;
 
     static ColumnsDescription parse(const String & str);
 };
 
-
 }
diff --git a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp
index 66a4fc84434..6347510f7b1 100644
--- a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp
+++ b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp
@@ -468,7 +468,7 @@ bool StorageDistributedDirectoryMonitor::maybeMarkAsBroken(const std::string & f
 
 std::string StorageDistributedDirectoryMonitor::getLoggerName() const
 {
-    return storage.name + '.' + storage.getName() + ".DirectoryMonitor";
+    return storage.table_name + '.' + storage.getName() + ".DirectoryMonitor";
 }
 
 }
diff --git a/dbms/src/Storages/IStorage.h b/dbms/src/Storages/IStorage.h
index a2970b0de96..ef9fd8004ce 100644
--- a/dbms/src/Storages/IStorage.h
+++ b/dbms/src/Storages/IStorage.h
@@ -83,6 +83,10 @@ public:
     /// The main name of the table type (for example, StorageMergeTree).
     virtual std::string getName() const = 0;
 
+    /** The name of the table.
+      */
+    virtual std::string getTableName() const = 0;
+
     /** Returns true if the storage receives data from a remote server or servers. */
     virtual bool isRemote() const { return false; }
 
diff --git a/dbms/src/Storages/ITableDeclaration.cpp b/dbms/src/Storages/ITableDeclaration.cpp
index da4cc920608..d4bbb4b501e 100644
--- a/dbms/src/Storages/ITableDeclaration.cpp
+++ b/dbms/src/Storages/ITableDeclaration.cpp
@@ -1,13 +1,12 @@
-#include <unordered_set>
+#include <Storages/ITableDeclaration.h>
+#include <Common/Exception.h>
+
+#include <boost/range/join.hpp>
 #include <sparsehash/dense_hash_map>
 #include <sparsehash/dense_hash_set>
-#include <Storages/ITableDeclaration.h>
-#include <Parsers/ASTIdentifier.h>
-#include <Parsers/ASTNameTypePair.h>
-#include <Interpreters/Context.h>
-#include <ext/map.h>
-#include <ext/identity.h>
-#include <ext/collection_cast.h>
+
+#include <unordered_set>
+#include <sstream>
 
 
 namespace DB
@@ -25,77 +24,14 @@ namespace ErrorCodes
 }
 
 
-NamesAndTypesList ITableDeclaration::getColumnsList() const
-{
-    return ext::collection_cast<NamesAndTypesList>(getColumnsListRange());
-}
-
-
-ITableDeclaration::ColumnsListRange ITableDeclaration::getColumnsListRange() const
-{
-    return boost::join(getColumnsListImpl(), materialized_columns);
-}
-
-
-bool ITableDeclaration::hasRealColumn(const String & column_name) const
-{
-    for (auto & it : getColumnsListRange())
-        if (it.name == column_name)
-            return true;
-    return false;
-}
-
-
-Names ITableDeclaration::getColumnNamesList() const
-{
-    return ext::map<Names>(getColumnsListRange(), [] (const auto & it) { return it.name; });
-}
-
-
-NameAndTypePair ITableDeclaration::getRealColumn(const String & column_name) const
-{
-    for (auto & it : getColumnsListRange())
-        if (it.name == column_name)
-            return it;
-    throw Exception("There is no column " + column_name + " in table.", ErrorCodes::NO_SUCH_COLUMN_IN_TABLE);
-}
-
-NameAndTypePair ITableDeclaration::getMaterializedColumn(const String & column_name) const
-{
-    for (auto & column : materialized_columns)
-        if (column.name == column_name)
-            return column;
-
-    throw Exception("There is no column " + column_name + " in table.", ErrorCodes::NO_SUCH_COLUMN_IN_TABLE);
-}
-
-bool ITableDeclaration::hasMaterializedColumn(const String & column_name) const
-{
-    for (auto & column : materialized_columns)
-        if (column.name == column_name)
-            return true;
-
-    return false;
-}
-
 bool ITableDeclaration::hasColumn(const String & column_name) const
 {
-    return hasRealColumn(column_name); /// By default, we assume that there are no virtual columns in the storage.
+    return columns.has(column_name); /// By default, we assume that there are no virtual columns in the storage.
 }
 
 NameAndTypePair ITableDeclaration::getColumn(const String & column_name) const
 {
-    return getRealColumn(column_name); /// By default, we assume that there are no virtual columns in the storage.
-}
-
-
-const DataTypePtr ITableDeclaration::getDataTypeByName(const String & column_name) const
-{
-    for (const auto & column : getColumnsListRange())
-        if (column.name == column_name)
-            return column.type;
-
-    throw Exception("There is no column " + column_name + " in table.", ErrorCodes::NO_SUCH_COLUMN_IN_TABLE);
+    return columns.get(column_name); /// By default, we assume that there are no virtual columns in the storage.
 }
 
 
@@ -103,7 +39,7 @@ Block ITableDeclaration::getSampleBlock() const
 {
     Block res;
 
-    for (const auto & col : getColumnsListRange())
+    for (const auto & col : boost::join(columns.ordinary, columns.materialized))
         res.insert({ col.type->createColumn(), col.type, col.name });
 
     return res;
@@ -114,7 +50,7 @@ Block ITableDeclaration::getSampleBlockNonMaterialized() const
 {
     Block res;
 
-    for (const auto & col : getColumnsListNonMaterialized())
+    for (const auto & col : columns.ordinary)
         res.insert({ col.type->createColumn(), col.type, col.name });
 
     return res;
@@ -138,7 +74,7 @@ Block ITableDeclaration::getSampleBlockForColumns(const Names & column_names) co
 static std::string listOfColumns(const NamesAndTypesList & available_columns)
 {
     std::stringstream s;
-    for (NamesAndTypesList::const_iterator it = available_columns.begin(); it != available_columns.end(); ++it)
+    for (auto it = available_columns.begin(); it != available_columns.end(); ++it)
     {
         if (it != available_columns.begin())
             s << ", ";
@@ -175,7 +111,7 @@ static NamesAndTypesMap getColumnsMap(const Args &... args)
 
 void ITableDeclaration::check(const Names & column_names) const
 {
-    const NamesAndTypesList & available_columns = getColumnsList();
+    const NamesAndTypesList & available_columns = columns.getList();
 
     if (column_names.empty())
         throw Exception("Empty list of columns queried. There are columns: " + listOfColumns(available_columns),
@@ -201,16 +137,16 @@ void ITableDeclaration::check(const Names & column_names) const
 }
 
 
-void ITableDeclaration::check(const NamesAndTypesList & columns) const
+void ITableDeclaration::check(const NamesAndTypesList & provided_columns) const
 {
-    const NamesAndTypesList & available_columns = getColumnsList();
+    const NamesAndTypesList & available_columns = columns.getList();
     const auto columns_map = getColumnsMap(available_columns);
 
     using UniqueStrings = google::dense_hash_set<StringRef, StringRefHash>;
     UniqueStrings unique_names;
     unique_names.set_empty_key(StringRef());
 
-    for (const NameAndTypePair & column : columns)
+    for (const NameAndTypePair & column : provided_columns)
     {
         NamesAndTypesMap::const_iterator it = columns_map.find(column.name);
         if (columns_map.end() == it)
@@ -229,11 +165,11 @@ void ITableDeclaration::check(const NamesAndTypesList & columns) const
 }
 
 
-void ITableDeclaration::check(const NamesAndTypesList & columns, const Names & column_names) const
+void ITableDeclaration::check(const NamesAndTypesList & provided_columns, const Names & column_names) const
 {
-    const NamesAndTypesList & available_columns = getColumnsList();
+    const NamesAndTypesList & available_columns = columns.getList();
     const auto available_columns_map = getColumnsMap(available_columns);
-    const NamesAndTypesMap & provided_columns_map = getColumnsMap(columns);
+    const NamesAndTypesMap & provided_columns_map = getColumnsMap(provided_columns);
 
     if (column_names.empty())
         throw Exception("Empty list of columns queried. There are columns: " + listOfColumns(available_columns),
@@ -268,7 +204,7 @@ void ITableDeclaration::check(const NamesAndTypesList & columns, const Names & c
 
 void ITableDeclaration::check(const Block & block, bool need_all) const
 {
-    const NamesAndTypesList & available_columns = getColumnsList();
+    const NamesAndTypesList & available_columns = columns.getList();
     const auto columns_map = getColumnsMap(available_columns);
 
     using NameSet = std::unordered_set<String>;
@@ -304,14 +240,11 @@ void ITableDeclaration::check(const Block & block, bool need_all) const
     }
 }
 
-ITableDeclaration::ITableDeclaration(const NamesAndTypesList & columns, const NamesAndTypesList & materialized_columns,
-                                     const NamesAndTypesList & alias_columns, const ColumnDefaults & column_defaults)
-    : columns{columns},
-      materialized_columns{materialized_columns},
-      alias_columns{alias_columns},
-      column_defaults{column_defaults}
+
+ITableDeclaration::ITableDeclaration(ColumnsDescription columns_)
+    : columns{std::move(columns_)}
 {
-    if (columns.empty())
+    if (columns.ordinary.empty())
         throw Exception("Empty list of columns passed to storage constructor", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED);
 }
 
diff --git a/dbms/src/Storages/ITableDeclaration.h b/dbms/src/Storages/ITableDeclaration.h
index 841ebc6c753..4c4ea59e4b5 100644
--- a/dbms/src/Storages/ITableDeclaration.h
+++ b/dbms/src/Storages/ITableDeclaration.h
@@ -1,50 +1,17 @@
 #pragma once
 
-#include <Core/Names.h>
-#include <Core/NamesAndTypes.h>
-#include <Common/Exception.h>
-#include <Core/Block.h>
-#include <Storages/ColumnDefault.h>
-
-#include <boost/range/iterator_range.hpp>
-#include <boost/range/join.hpp>
+#include <Storages/ColumnsDescription.h>
 
 
 namespace DB
 {
 
-class Context;
-
 /** Description of the table.
-  * Do not thread safe. See IStorage::lockStructure ().
+  * Is not thread safe. See IStorage::lockStructure ().
   */
 class ITableDeclaration
 {
 public:
-    /** The name of the table.
-      */
-    virtual std::string getTableName() const = 0;
-
-    /** Get a list of names and table column types, only non-virtual.
-      */
-    NamesAndTypesList getColumnsList() const;
-    const NamesAndTypesList & getColumnsListNonMaterialized() const { return getColumnsListImpl(); }
-
-    /** Get a list of column table names, only non-virtual.
-      */
-    virtual Names getColumnNamesList() const;
-
-    /** Get a description of the real (non-virtual) column by its name.
-      */
-    virtual NameAndTypePair getRealColumn(const String & column_name) const;
-
-    /** Is there a real (non-virtual) column with that name.
-      */
-    virtual bool hasRealColumn(const String & column_name) const;
-
-    NameAndTypePair getMaterializedColumn(const String & column_name) const;
-    bool hasMaterializedColumn(const String & column_name) const;
-
     /** Get a description of any column by its name.
       */
     virtual NameAndTypePair getColumn(const String & column_name) const;
@@ -53,10 +20,6 @@ public:
       */
     virtual bool hasColumn(const String & column_name) const;
 
-    const DataTypePtr getDataTypeByName(const String & column_name) const;
-
-    /** The same, but in the form of a block-sample.
-      */
     Block getSampleBlock() const;
     Block getSampleBlockNonMaterialized() const;
     Block getSampleBlockForColumns(const Names & column_names) const;
@@ -81,29 +44,11 @@ public:
     void check(const Block & block, bool need_all = false) const;
 
 
+    ITableDeclaration() = default;
+    explicit ITableDeclaration(ColumnsDescription columns_);
     virtual ~ITableDeclaration() = default;
 
-    ITableDeclaration() = default;
-    ITableDeclaration(
-        const NamesAndTypesList & columns,
-        const NamesAndTypesList & materialized_columns,
-        const NamesAndTypesList & alias_columns,
-        const ColumnDefaults & column_defaults);
-
-    NamesAndTypesList columns;
-    NamesAndTypesList materialized_columns;
-    NamesAndTypesList alias_columns;
-    ColumnDefaults column_defaults;
-
-private:
-    virtual const NamesAndTypesList & getColumnsListImpl() const
-    {
-        return columns;
-    }
-
-    using ColumnsListRange = boost::range::joined_range<const NamesAndTypesList, const NamesAndTypesList>;
-    /// Returns a lazily joined range of table's ordinary and materialized columns, without unnecessary copying
-    ColumnsListRange getColumnsListRange() const;
+    ColumnsDescription columns;
 };
 
 }
diff --git a/dbms/src/Storages/MergeTree/DataPartsExchange.cpp b/dbms/src/Storages/MergeTree/DataPartsExchange.cpp
index e2c884681d3..b2ff96e0fe3 100644
--- a/dbms/src/Storages/MergeTree/DataPartsExchange.cpp
+++ b/dbms/src/Storages/MergeTree/DataPartsExchange.cpp
@@ -1,4 +1,5 @@
 #include <Storages/MergeTree/DataPartsExchange.h>
+#include <Storages/IStorage.h>
 #include <Common/CurrentMetrics.h>
 #include <Common/NetException.h>
 #include <Common/typeid_cast.h>
diff --git a/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp
index e0a8dc94400..5a67714db64 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp
@@ -141,8 +141,8 @@ try
         if (!column_names.empty())
             storage.check(data_part->columns, column_names);
 
-        pre_columns = storage.getColumnsList().addTypes(pre_column_names);
-        columns = storage.getColumnsList().addTypes(column_names);
+        pre_columns = storage.columns.getList().addTypes(pre_column_names);
+        columns = storage.columns.getList().addTypes(column_names);
     }
     else
     {
diff --git a/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp
index b0b09ae06eb..b6d8ff3cd5a 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp
@@ -26,9 +26,9 @@ NameSet injectRequiredColumns(const MergeTreeData & storage, const MergeTreeData
             continue;
         }
 
-        const auto default_it = storage.column_defaults.find(column_name);
+        const auto default_it = storage.columns.defaults.find(column_name);
         /// columns has no explicit default expression
-        if (default_it == std::end(storage.column_defaults))
+        if (default_it == std::end(storage.columns.defaults))
             continue;
 
         /// collect identifiers required for evaluation
diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp
index 392c45a7cc7..7933a9454c7 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp
@@ -80,10 +80,7 @@ namespace ErrorCodes
 
 MergeTreeData::MergeTreeData(
     const String & database_, const String & table_,
-    const String & full_path_, const NamesAndTypesList & columns_,
-    const NamesAndTypesList & materialized_columns_,
-    const NamesAndTypesList & alias_columns_,
-    const ColumnDefaults & column_defaults_,
+    const String & full_path_, const ColumnsDescription & columns_,
     Context & context_,
     const ASTPtr & primary_expr_ast_,
     const ASTPtr & secondary_sort_expr_ast_,
@@ -95,7 +92,7 @@ MergeTreeData::MergeTreeData(
     bool require_part_metadata_,
     bool attach,
     BrokenPartCallback broken_part_callback_)
-    : ITableDeclaration{columns_, materialized_columns_, alias_columns_, column_defaults_},
+    : ITableDeclaration{columns_},
     context(context_),
     sampling_expression(sampling_expression_),
     index_granularity(settings_.index_granularity),
@@ -113,7 +110,7 @@ MergeTreeData::MergeTreeData(
     data_parts_by_state_and_info(data_parts_indexes.get<TagByStateAndInfo>())
 {
     /// NOTE: using the same columns list as is read when performing actual merges.
-    merging_params.check(getColumnsList());
+    merging_params.check(columns.getList());
 
     if (!primary_expr_ast)
         throw Exception("Primary key cannot be empty", ErrorCodes::BAD_ARGUMENTS);
@@ -225,11 +222,11 @@ void MergeTreeData::initPrimaryKey()
     primary_sort_descr.clear();
     addSortDescription(primary_sort_descr, primary_expr_ast);
 
-    primary_expr = ExpressionAnalyzer(primary_expr_ast, context, nullptr, getColumnsList()).getActions(false);
+    primary_expr = ExpressionAnalyzer(primary_expr_ast, context, nullptr, columns.getList()).getActions(false);
 
     {
         ExpressionActionsPtr projected_expr =
-                ExpressionAnalyzer(primary_expr_ast, context, nullptr, getColumnsList()).getActions(true);
+                ExpressionAnalyzer(primary_expr_ast, context, nullptr, columns.getList()).getActions(true);
         primary_key_sample = projected_expr->getSampleBlock();
     }
 
@@ -244,10 +241,10 @@ void MergeTreeData::initPrimaryKey()
     if (secondary_sort_expr_ast)
     {
         addSortDescription(sort_descr, secondary_sort_expr_ast);
-        secondary_sort_expr = ExpressionAnalyzer(secondary_sort_expr_ast, context, nullptr, getColumnsList()).getActions(false);
+        secondary_sort_expr = ExpressionAnalyzer(secondary_sort_expr_ast, context, nullptr, columns.getList()).getActions(false);
 
         ExpressionActionsPtr projected_expr =
-                ExpressionAnalyzer(secondary_sort_expr_ast, context, nullptr, getColumnsList()).getActions(true);
+                ExpressionAnalyzer(secondary_sort_expr_ast, context, nullptr, columns.getList()).getActions(true);
         auto secondary_key_sample = projected_expr->getSampleBlock();
 
         checkKeyExpression(*secondary_sort_expr, secondary_key_sample, "Secondary");
@@ -260,7 +257,7 @@ void MergeTreeData::initPartitionKey()
     if (!partition_expr_ast || partition_expr_ast->children.empty())
         return;
 
-    partition_expr = ExpressionAnalyzer(partition_expr_ast, context, nullptr, getColumnsList()).getActions(false);
+    partition_expr = ExpressionAnalyzer(partition_expr_ast, context, nullptr, columns.getList()).getActions(false);
     for (const ASTPtr & ast : partition_expr_ast->children)
     {
         String col_name = ast->getColumnName();
@@ -865,10 +862,7 @@ void MergeTreeData::checkAlter(const AlterCommands & commands)
 {
     /// Check that needed transformations can be applied to the list of columns without considering type conversions.
     auto new_columns = columns;
-    auto new_materialized_columns = materialized_columns;
-    auto new_alias_columns = alias_columns;
-    auto new_column_defaults = column_defaults;
-    commands.apply(new_columns, new_materialized_columns, new_alias_columns, new_column_defaults);
+    commands.apply(new_columns);
 
     /// Set of columns that shouldn't be altered.
     NameSet columns_alter_forbidden;
@@ -909,7 +903,7 @@ void MergeTreeData::checkAlter(const AlterCommands & commands)
         columns_alter_forbidden.insert(merging_params.sign_column);
 
     std::map<String, const IDataType *> old_types;
-    for (const auto & column : columns)
+    for (const auto & column : columns.ordinary)
         old_types.emplace(column.name, column.type.get());
 
     for (const AlterCommand & command : commands)
@@ -937,11 +931,7 @@ void MergeTreeData::checkAlter(const AlterCommands & commands)
     NameToNameMap unused_map;
     bool unused_bool;
 
-    /// augment plain columns with materialized columns for convert expression creation
-    new_columns.insert(std::end(new_columns),
-        std::begin(new_materialized_columns), std::end(new_materialized_columns));
-
-    createConvertExpression(nullptr, getColumnsList(), new_columns, unused_expression, unused_map, unused_bool);
+    createConvertExpression(nullptr, columns.getList(), new_columns.getList(), unused_expression, unused_map, unused_bool);
 }
 
 void MergeTreeData::createConvertExpression(const DataPartPtr & part, const NamesAndTypesList & old_columns, const NamesAndTypesList & new_columns,
@@ -1845,7 +1835,7 @@ void MergeTreeData::addPartContributionToColumnSizes(const DataPartPtr & part)
     const auto & files = part->checksums.files;
 
     /// TODO This method doesn't take into account columns with multiple files.
-    for (const auto & column : getColumnsList())
+    for (const auto & column : columns.getList())
     {
         const auto escaped_name = escapeForFileName(column.name);
         const auto bin_file_name = escaped_name + ".bin";
@@ -1878,7 +1868,7 @@ void MergeTreeData::removePartContributionToColumnSizes(const DataPartPtr & part
     const auto & files = part->checksums.files;
 
     /// TODO This method doesn't take into account columns with multiple files.
-    for (const auto & column : columns)
+    for (const auto & column : columns.ordinary)
     {
         const auto escaped_name = escapeForFileName(column.name);
         const auto bin_file_name = escaped_name + ".bin";
diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h
index 519e7223d13..a32baf797ef 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeData.h
+++ b/dbms/src/Storages/MergeTree/MergeTreeData.h
@@ -4,7 +4,8 @@
 #include <Common/SimpleIncrement.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/ExpressionActions.h>
-#include <Storages/IStorage.h>
+#include <Storages/ITableDeclaration.h>
+#include <Storages/AlterCommands.h>
 #include <Storages/MergeTree/MergeTreePartInfo.h>
 #include <Storages/MergeTree/MergeTreeSettings.h>
 #include <IO/ReadBufferFromString.h>
@@ -270,10 +271,8 @@ public:
     /// require_part_metadata - should checksums.txt and columns.txt exist in the part directory.
     /// attach - whether the existing table is attached or the new table is created.
     MergeTreeData(const String & database_, const String & table_,
-                  const String & full_path_, const NamesAndTypesList & columns_,
-                  const NamesAndTypesList & materialized_columns_,
-                  const NamesAndTypesList & alias_columns_,
-                  const ColumnDefaults & column_defaults_,
+                  const String & full_path_,
+                  const ColumnsDescription & columns_,
                   Context & context_,
                   const ASTPtr & primary_expr_ast_,
                   const ASTPtr & secondary_sort_expr_ast_,
@@ -305,8 +304,6 @@ public:
 
     Int64 getMaxDataPartIndex();
 
-    const NamesAndTypesList & getColumnsListImpl() const override { return columns; }
-
     NameAndTypePair getColumn(const String & column_name) const override
     {
         if (column_name == "_part")
@@ -316,12 +313,12 @@ public:
         if (column_name == "_sample_factor")
             return NameAndTypePair("_sample_factor", std::make_shared<DataTypeFloat64>());
 
-        return ITableDeclaration::getColumn(column_name);
+        return columns.get(column_name);
     }
 
     bool hasColumn(const String & column_name) const override
     {
-        return ITableDeclaration::hasColumn(column_name)
+        return columns.has(column_name)
             || column_name == "_part"
             || column_name == "_part_index"
             || column_name == "_sample_factor";
@@ -329,7 +326,7 @@ public:
 
     String getDatabaseName() const { return database_name; }
 
-    String getTableName() const override { return table_name; }
+    String getTableName() const { return table_name; }
 
     String getFullPath() const { return full_path; }
 
@@ -431,9 +428,6 @@ public:
         const ASTPtr & new_primary_key,
         bool skip_sanity_checks);
 
-    /// Must be called with locked lockStructureForAlter().
-    void setColumnsList(const NamesAndTypesList & new_columns) { columns = new_columns; }
-
     /// Should be called if part data is suspected to be corrupted.
     void reportBrokenPart(const String & name)
     {
@@ -507,6 +501,7 @@ public:
     /// For ATTACH/DETACH/DROP PARTITION.
     String getPartitionIDFromQuery(const ASTPtr & partition, const Context & context);
 
+
     MergeTreeDataFormatVersion format_version;
 
     Context & context;
diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp
index cf9e6eef15c..4e3e9788aaa 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp
@@ -536,8 +536,8 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMerger::mergePartsToTemporaryPart
     for (const MergeTreeData::DataPartPtr & part : parts)
         part->accumulateColumnSizes(merged_column_to_size);
 
-    Names all_column_names = data.getColumnNamesList();
-    NamesAndTypesList all_columns = data.getColumnsList();
+    Names all_column_names = data.columns.getNames();
+    NamesAndTypesList all_columns = data.columns.getList();
     const SortDescription sort_desc = data.getSortDescription();
 
     NamesAndTypesList gathering_columns, merging_columns;
diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp
index df273f2d6b5..9c0269c7437 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp
@@ -418,7 +418,7 @@ UInt64 MergeTreeDataPart::getColumnMrkSize(const String & name) const
   */
 String MergeTreeDataPart::getColumnNameWithMinumumCompressedSize() const
 {
-    const auto & columns = storage.getColumnsList();
+    const auto & columns = storage.columns.getList();
     const std::string * minimum_size_column = nullptr;
     UInt64 minimum_size = std::numeric_limits<UInt64>::max();
 
@@ -774,7 +774,7 @@ void MergeTreeDataPart::accumulateColumnSizes(ColumnToSize & column_to_size) con
 {
     std::shared_lock<std::shared_mutex> part_lock(columns_lock);
 
-    for (const NameAndTypePair & name_type : storage.columns)
+    for (const NameAndTypePair & name_type : storage.columns.getList())
     {
         name_type.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path)
         {
@@ -794,7 +794,7 @@ void MergeTreeDataPart::loadColumns(bool require)
             throw Exception("No columns.txt in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART);
 
         /// If there is no file with a list of columns, write it down.
-        for (const NameAndTypePair & column : storage.getColumnsList())
+        for (const NameAndTypePair & column : storage.columns.getList())
             if (Poco::File(getFullPath() + escapeForFileName(column.name) + ".bin").exists())
                 columns.push_back(column);
 
diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
index fb7c1735ced..605091e6386 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
@@ -175,7 +175,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::read(
         }
     }
 
-    NamesAndTypesList available_real_columns = data.getColumnsList();
+    NamesAndTypesList available_real_columns = data.columns.getList();
 
     NamesAndTypesList available_real_and_virtual_columns = available_real_columns;
     for (const auto & name : virt_column_names)
@@ -857,7 +857,7 @@ void MergeTreeDataSelectExecutor::createPositiveSignCondition(
     arguments->children.push_back(sign);
     arguments->children.push_back(one);
 
-    out_expression = ExpressionAnalyzer(function, context, {}, data.getColumnsList()).getActions(false);
+    out_expression = ExpressionAnalyzer(function, context, {}, data.columns.getList()).getActions(false);
     out_column = function->getColumnName();
 }
 
diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h
index fd0b0265ced..1b3caeeb259 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h
@@ -1,5 +1,7 @@
 #pragma once
 
+#include <Core/QueryProcessingStage.h>
+#include <Storages/SelectQueryInfo.h>
 #include <Storages/MergeTree/MergeTreeData.h>
 #include <Storages/MergeTree/RangesInDataPart.h>
 
diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp
index fabbca3a473..f4db6b934ba 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp
@@ -205,7 +205,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa
     ///  either default lz4 or compression method with zero thresholds on absolute and relative part size.
     auto compression_settings = data.context.chooseCompressionSettings(0, 0);
 
-    NamesAndTypesList columns = data.getColumnsList().filter(block.getNames());
+    NamesAndTypesList columns = data.columns.getList().filter(block.getNames());
     MergedBlockOutputStream out(data, new_data_part->getFullPath(), columns, compression_settings);
 
     out.writePrefix();
diff --git a/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp b/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp
index e95c576274c..c32aa71a731 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp
@@ -238,8 +238,8 @@ std::vector<size_t> MergeTreeReadPool::fillPerPartInfo(
             if (!required_column_names.empty())
                 data.check(part.data_part->columns, required_column_names);
 
-            per_part_pre_columns.push_back(data.getColumnsList().addTypes(required_pre_column_names));
-            per_part_columns.push_back(data.getColumnsList().addTypes(required_column_names));
+            per_part_pre_columns.push_back(data.columns.getList().addTypes(required_pre_column_names));
+            per_part_columns.push_back(data.columns.getList().addTypes(required_column_names));
         }
         else
         {
diff --git a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp
index 63d763d5939..7bce149776a 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp
@@ -463,7 +463,7 @@ void MergeTreeReader::fillMissingColumns(Block & res, const Names & ordered_name
             if (!has_column)
             {
                 should_sort = true;
-                if (storage.column_defaults.count(requested_column.name) != 0)
+                if (storage.columns.defaults.count(requested_column.name) != 0)
                 {
                     should_evaluate_defaults = true;
                     continue;
@@ -498,7 +498,7 @@ void MergeTreeReader::fillMissingColumns(Block & res, const Names & ordered_name
 
         /// evaluate defaulted columns if necessary
         if (should_evaluate_defaults)
-            evaluateMissingDefaults(res, columns, storage.column_defaults, storage.context);
+            evaluateMissingDefaults(res, columns, storage.columns.defaults, storage.context);
 
         /// sort columns to ensure consistent order among all blocks
         if (should_sort)
diff --git a/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp
index cf7bc0d33f7..005ca750226 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp
@@ -41,9 +41,9 @@ MergeTreeWhereOptimizer::MergeTreeWhereOptimizer(
     Logger * log)
         : primary_key_columns{ext::map<std::unordered_set>(data.getPrimarySortDescription(),
             [] (const SortColumnDescription & col) { return col.column_name; })},
-        table_columns{ext::map<std::unordered_set>(data.getColumnsList(),
+        table_columns{ext::map<std::unordered_set>(data.columns.getList(),
             [] (const NameAndTypePair & col) { return col.name; })},
-        block_with_constants{PKCondition::getBlockWithConstants(query_info.query, context, data.getColumnsList())},
+        block_with_constants{PKCondition::getBlockWithConstants(query_info.query, context, data.columns.getList())},
         prepared_sets(query_info.sets),
         log{log}
 {
diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp
index ef7582c497c..c359a889bfb 100644
--- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp
+++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp
@@ -56,12 +56,7 @@ void ReplicatedMergeTreeAlterThread::run()
 
             zkutil::Stat stat;
             const String columns_str = zookeeper->get(storage.zookeeper_path + "/columns", &stat, wakeup_event);
-            auto columns_desc = ColumnsDescription<true>::parse(columns_str);
-
-            auto & columns = columns_desc.columns;
-            auto & materialized_columns = columns_desc.materialized;
-            auto & alias_columns = columns_desc.alias;
-            auto & column_defaults = columns_desc.defaults;
+            auto columns_in_zk = ColumnsDescription::parse(columns_str);
 
             bool changed_version = (stat.version != storage.columns_version);
 
@@ -92,42 +87,15 @@ void ReplicatedMergeTreeAlterThread::run()
 
                     auto table_lock = storage.lockStructureForAlter(__PRETTY_FUNCTION__);
 
-                    const auto columns_changed = columns != storage.data.getColumnsListNonMaterialized();
-                    const auto materialized_columns_changed = materialized_columns != storage.data.materialized_columns;
-                    const auto alias_columns_changed = alias_columns != storage.data.alias_columns;
-                    const auto column_defaults_changed = column_defaults != storage.data.column_defaults;
-
-                    if (columns_changed || materialized_columns_changed || alias_columns_changed ||
-                        column_defaults_changed)
+                    if (columns_in_zk != storage.data.columns)
                     {
                         LOG_INFO(log, "Columns list changed in ZooKeeper. Applying changes locally.");
 
                         storage.context.getDatabase(storage.database_name)->alterTable(
-                            storage.context, storage.table_name,
-                            columns, materialized_columns, alias_columns, column_defaults, {});
+                            storage.context, storage.table_name, columns_in_zk, {});
 
-                        if (columns_changed)
-                        {
-                            storage.data.setColumnsList(columns);
-                        }
-
-                        if (materialized_columns_changed)
-                        {
-                            storage.materialized_columns = materialized_columns;
-                            storage.data.materialized_columns = std::move(materialized_columns);
-                        }
-
-                        if (alias_columns_changed)
-                        {
-                            storage.alias_columns = alias_columns;
-                            storage.data.alias_columns = std::move(alias_columns);
-                        }
-
-                        if (column_defaults_changed)
-                        {
-                            storage.column_defaults = column_defaults;
-                            storage.data.column_defaults = std::move(column_defaults);
-                        }
+                        storage.columns = columns_in_zk;
+                        storage.data.columns = std::move(columns_in_zk);
 
                         /// Reinitialize primary key because primary key column types might have changed.
                         storage.data.initPrimaryKey();
@@ -158,7 +126,7 @@ void ReplicatedMergeTreeAlterThread::run()
                     if (!changed_version)
                         parts = storage.data.getDataParts();
 
-                    const auto columns_plus_materialized = storage.data.getColumnsList();
+                    const auto columns_plus_materialized = storage.data.columns.getList();
 
                     for (const MergeTreeData::DataPartPtr & part : parts)
                     {
diff --git a/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp b/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp
index 89787fca4f0..4cdff05c2e3 100644
--- a/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp
+++ b/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp
@@ -628,14 +628,13 @@ static StoragePtr create(const StorageFactory::Arguments & args)
     if (replicated)
         return StorageReplicatedMergeTree::create(
             zookeeper_path, replica_name, args.attach, args.data_path, args.database_name, args.table_name,
-            args.columns, args.materialized_columns, args.alias_columns, args.column_defaults,
+            args.columns,
             args.context, primary_expr_list, secondary_sorting_expr_list, date_column_name, partition_expr_list,
             sampling_expression, merging_params, storage_settings,
             args.has_force_restore_data_flag);
     else
         return StorageMergeTree::create(
-            args.data_path, args.database_name, args.table_name,
-            args.columns, args.materialized_columns, args.alias_columns, args.column_defaults, args.attach,
+            args.data_path, args.database_name, args.table_name, args.columns, args.attach,
             args.context, primary_expr_list, secondary_sorting_expr_list, date_column_name, partition_expr_list,
             sampling_expression, merging_params, storage_settings,
             args.has_force_restore_data_flag);
diff --git a/dbms/src/Storages/StorageBuffer.cpp b/dbms/src/Storages/StorageBuffer.cpp
index 6b9ec92ce75..c76a1edc81b 100644
--- a/dbms/src/Storages/StorageBuffer.cpp
+++ b/dbms/src/Storages/StorageBuffer.cpp
@@ -48,14 +48,11 @@ namespace ErrorCodes
 }
 
 
-StorageBuffer::StorageBuffer(const std::string & name_, const NamesAndTypesList & columns_,
-    const NamesAndTypesList & materialized_columns_,
-    const NamesAndTypesList & alias_columns_,
-    const ColumnDefaults & column_defaults_,
+StorageBuffer::StorageBuffer(const std::string & name_, const ColumnsDescription & columns_,
     Context & context_,
     size_t num_shards_, const Thresholds & min_thresholds_, const Thresholds & max_thresholds_,
     const String & destination_database_, const String & destination_table_, bool allow_materialized_)
-    : IStorage{columns_, materialized_columns_, alias_columns_, column_defaults_},
+    : IStorage{columns_},
     name(name_), context(context_),
     num_shards(num_shards_), buffers(num_shards_),
     min_thresholds(min_thresholds_), max_thresholds(max_thresholds_),
@@ -598,11 +595,8 @@ void StorageBuffer::alter(const AlterCommands & params, const String & database_
     /// So that no blocks of the old structure remain.
     optimize({} /*query*/, {} /*partition_id*/, false /*final*/, false /*deduplicate*/, context);
 
-    params.apply(columns, materialized_columns, alias_columns, column_defaults);
-
-    context.getDatabase(database_name)->alterTable(
-        context, table_name,
-        columns, materialized_columns, alias_columns, column_defaults, {});
+    params.apply(columns);
+    context.getDatabase(database_name)->alterTable(context, table_name, columns, {});
 }
 
 
@@ -641,7 +635,6 @@ void registerStorageBuffer(StorageFactory & factory)
 
         return StorageBuffer::create(
             args.table_name, args.columns,
-            args.materialized_columns, args.alias_columns, args.column_defaults,
             args.context,
             num_buckets,
             StorageBuffer::Thresholds{min_time, min_rows, min_bytes},
diff --git a/dbms/src/Storages/StorageBuffer.h b/dbms/src/Storages/StorageBuffer.h
index 9d9f72340c5..02c2ef9b09b 100644
--- a/dbms/src/Storages/StorageBuffer.h
+++ b/dbms/src/Storages/StorageBuffer.h
@@ -123,10 +123,7 @@ protected:
     /** num_shards - the level of internal parallelism (the number of independent buffers)
       * The buffer is flushed if all minimum thresholds or at least one of the maximum thresholds are exceeded.
       */
-    StorageBuffer(const std::string & name_, const NamesAndTypesList & columns_,
-        const NamesAndTypesList & materialized_columns_,
-        const NamesAndTypesList & alias_columns_,
-        const ColumnDefaults & column_defaults_,
+    StorageBuffer(const std::string & name_, const ColumnsDescription & columns_,
         Context & context_,
         size_t num_shards_, const Thresholds & min_thresholds_, const Thresholds & max_thresholds_,
         const String & destination_database_, const String & destination_table_, bool allow_materialized_);
diff --git a/dbms/src/Storages/StorageCatBoostPool.cpp b/dbms/src/Storages/StorageCatBoostPool.cpp
index cbaf523e3bf..1a973f7dd75 100644
--- a/dbms/src/Storages/StorageCatBoostPool.cpp
+++ b/dbms/src/Storages/StorageCatBoostPool.cpp
@@ -220,7 +220,6 @@ void StorageCatBoostPool::parseColumnDescription()
 
 void StorageCatBoostPool::createSampleBlockAndColumns()
 {
-    columns.clear();
     NamesAndTypesList cat_columns;
     NamesAndTypesList num_columns;
     sample_block.clear();
@@ -239,19 +238,19 @@ void StorageCatBoostPool::createSampleBlockAndColumns()
         else if (desc.column_type == DatasetColumnType::Num)
             num_columns.emplace_back(desc.column_name, type);
         else
-            materialized_columns.emplace_back(desc.column_name, type);
+            columns.materialized.emplace_back(desc.column_name, type);
 
         if (!desc.alias.empty())
         {
             auto alias = std::make_shared<ASTIdentifier>(desc.column_name);
-            column_defaults[desc.alias] = {ColumnDefaultType::Alias, alias};
-            alias_columns.emplace_back(desc.alias, type);
+            columns.defaults[desc.alias] = {ColumnDefaultType::Alias, alias};
+            columns.aliases.emplace_back(desc.alias, type);
         }
 
         sample_block.insert(ColumnWithTypeAndName(type, desc.column_name));
     }
-    columns.insert(columns.end(), num_columns.begin(), num_columns.end());
-    columns.insert(columns.end(), cat_columns.begin(), cat_columns.end());
+    columns.ordinary.insert(columns.ordinary.end(), num_columns.begin(), num_columns.end());
+    columns.ordinary.insert(columns.ordinary.end(), cat_columns.begin(), cat_columns.end());
 }
 
 BlockInputStreams StorageCatBoostPool::read(const Names & column_names,
diff --git a/dbms/src/Storages/StorageDictionary.cpp b/dbms/src/Storages/StorageDictionary.cpp
index 2bb69748c6d..954b8ae898b 100644
--- a/dbms/src/Storages/StorageDictionary.cpp
+++ b/dbms/src/Storages/StorageDictionary.cpp
@@ -24,13 +24,10 @@ namespace ErrorCodes
 
 StorageDictionary::StorageDictionary(
     const String & table_name_,
-    const NamesAndTypesList & columns_,
-    const NamesAndTypesList & materialized_columns_,
-    const NamesAndTypesList & alias_columns_,
-    const ColumnDefaults & column_defaults_,
+    const ColumnsDescription & columns_,
     const DictionaryStructure & dictionary_structure_,
     const String & dictionary_name_)
-    : IStorage{columns_, materialized_columns_, alias_columns_, column_defaults_}, table_name(table_name_),
+    : IStorage{columns_}, table_name(table_name_),
     dictionary_name(dictionary_name_),
     logger(&Poco::Logger::get("StorageDictionary"))
 {
@@ -75,7 +72,7 @@ void StorageDictionary::checkNamesAndTypesCompatibleWithDictionary(const Diction
     auto dictionary_names_and_types = getNamesAndTypes(dictionary_structure);
     std::set<NameAndTypePair> namesAndTypesSet(dictionary_names_and_types.begin(), dictionary_names_and_types.end());
 
-    for (auto & column : columns)
+    for (auto & column : columns.ordinary)
     {
         if (namesAndTypesSet.find(column) == namesAndTypesSet.end())
         {
@@ -105,8 +102,7 @@ void registerStorageDictionary(StorageFactory & factory)
         const DictionaryStructure & dictionary_structure = dictionary->getStructure();
 
         return StorageDictionary::create(
-            args.table_name, args.columns, args.materialized_columns, args.alias_columns,
-            args.column_defaults, dictionary_structure, dictionary_name);
+            args.table_name, args.columns, dictionary_structure, dictionary_name);
     });
 }
 
diff --git a/dbms/src/Storages/StorageDictionary.h b/dbms/src/Storages/StorageDictionary.h
index 0f21373f546..0320a514839 100644
--- a/dbms/src/Storages/StorageDictionary.h
+++ b/dbms/src/Storages/StorageDictionary.h
@@ -22,7 +22,6 @@ class StorageDictionary : public ext::shared_ptr_helper<StorageDictionary>, publ
 public:
     std::string getName() const override { return "Dictionary"; }
     std::string getTableName() const override { return table_name; }
-    const NamesAndTypesList & getColumnsListImpl() const override { return columns; }
     BlockInputStreams read(const Names & column_names,
         const SelectQueryInfo & query_info,
         const Context & context,
@@ -62,10 +61,7 @@ private:
 
 protected:
     StorageDictionary(const String & table_name_,
-        const NamesAndTypesList & columns_,
-        const NamesAndTypesList & materialized_columns_,
-        const NamesAndTypesList & alias_columns_,
-        const ColumnDefaults & column_defaults_,
+        const ColumnsDescription & columns_,
         const DictionaryStructure & dictionary_structure_,
         const String & dictionary_name_);
 };
diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp
index c1681e6fbea..1b7c5fdf69a 100644
--- a/dbms/src/Storages/StorageDistributed.cpp
+++ b/dbms/src/Storages/StorageDistributed.cpp
@@ -130,24 +130,21 @@ StorageDistributed::~StorageDistributed() = default;
 
 
 StorageDistributed::StorageDistributed(
-    const std::string & name_,
-    const NamesAndTypesList & columns_,
-    const NamesAndTypesList & materialized_columns_,
-    const NamesAndTypesList & alias_columns_,
-    const ColumnDefaults & column_defaults_,
+    const std::string & table_name_,
+    const ColumnsDescription & columns_,
     const String & remote_database_,
     const String & remote_table_,
     const String & cluster_name_,
     const Context & context_,
     const ASTPtr & sharding_key_,
     const String & data_path_)
-    : IStorage{columns_, materialized_columns_, alias_columns_, column_defaults_},
-    name(name_),
+    : IStorage{columns_},
+    table_name(table_name_),
     remote_database(remote_database_), remote_table(remote_table_),
     context(context_), cluster_name(context.getMacros().expand(cluster_name_)), has_sharding_key(sharding_key_),
-    sharding_key_expr(sharding_key_ ? ExpressionAnalyzer(sharding_key_, context, nullptr, columns).getActions(false) : nullptr),
+    sharding_key_expr(sharding_key_ ? ExpressionAnalyzer(sharding_key_, context, nullptr, columns.ordinary).getActions(false) : nullptr),
     sharding_key_column_name(sharding_key_ ? sharding_key_->getColumnName() : String{}),
-    path(data_path_.empty() ? "" : (data_path_ + escapeForFileName(name) + '/'))
+    path(data_path_.empty() ? "" : (data_path_ + escapeForFileName(table_name) + '/'))
 {
 }
 
@@ -161,7 +158,7 @@ StoragePtr StorageDistributed::createWithOwnCluster(
     const Context & context_)
 {
     auto res = ext::shared_ptr_helper<StorageDistributed>::create(
-        name_, columns_, NamesAndTypesList(), NamesAndTypesList(), ColumnDefaults(),
+        name_, ColumnsDescription{columns_, NamesAndTypesList(), NamesAndTypesList(), ColumnDefaults()},
         remote_database_, remote_table_, String{}, context_, ASTPtr(), String());
 
     res->owned_cluster = owned_cluster_;
@@ -236,11 +233,9 @@ void StorageDistributed::alter(const AlterCommands & params, const String & data
             throw Exception("Storage engine " + getName() + " doesn't support primary key.", ErrorCodes::NOT_IMPLEMENTED);
 
     auto lock = lockStructureForAlter(__PRETTY_FUNCTION__);
-    params.apply(columns, materialized_columns, alias_columns, column_defaults);
+    params.apply(columns);
 
-    context.getDatabase(database_name)->alterTable(
-        context, table_name,
-        columns, materialized_columns, alias_columns, column_defaults, {});
+    context.getDatabase(database_name)->alterTable(context, table_name, columns, {});
 }
 
 
@@ -291,13 +286,13 @@ NameAndTypePair StorageDistributed::getColumn(const String & column_name) const
     if (const auto & type = VirtualColumnFactory::tryGetType(column_name))
         return { column_name, type };
 
-    return getRealColumn(column_name);
+    return columns.get(column_name);
 }
 
 
 bool StorageDistributed::hasColumn(const String & column_name) const
 {
-    return VirtualColumnFactory::hasColumn(column_name) || IStorage::hasColumn(column_name);
+    return VirtualColumnFactory::hasColumn(column_name) || columns.has(column_name);
 }
 
 void StorageDistributed::createDirectoryMonitors()
@@ -390,7 +385,7 @@ void registerStorageDistributed(StorageFactory & factory)
         /// Check that sharding_key exists in the table and has numeric type.
         if (sharding_key)
         {
-            auto sharding_expr = ExpressionAnalyzer(sharding_key, args.context, nullptr, args.columns).getActions(true);
+            auto sharding_expr = ExpressionAnalyzer(sharding_key, args.context, nullptr, args.columns.ordinary).getActions(true);
             const Block & block = sharding_expr->getSampleBlock();
 
             if (block.columns() != 1)
@@ -405,7 +400,6 @@ void registerStorageDistributed(StorageFactory & factory)
 
         return StorageDistributed::create(
             args.table_name, args.columns,
-            args.materialized_columns, args.alias_columns, args.column_defaults,
             remote_database, remote_table, cluster_name,
             args.context, sharding_key, args.data_path);
     });
diff --git a/dbms/src/Storages/StorageDistributed.h b/dbms/src/Storages/StorageDistributed.h
index 6afb54ba007..22e24c3f9fb 100644
--- a/dbms/src/Storages/StorageDistributed.h
+++ b/dbms/src/Storages/StorageDistributed.h
@@ -34,20 +34,19 @@ public:
     ~StorageDistributed() override;
 
     static StoragePtr createWithOwnCluster(
-        const std::string & name_,            /// The name of the table.
-        const NamesAndTypesList & columns_,        /// List of columns.
+        const std::string & table_name_,
+        const NamesAndTypesList & columns_,
         const String & remote_database_,      /// database on remote servers.
         const String & remote_table_,         /// The name of the table on the remote servers.
         ClusterPtr & owned_cluster_,
         const Context & context_);
 
     std::string getName() const override { return "Distributed"; }
-    std::string getTableName() const override { return name; }
+    std::string getTableName() const override { return table_name; }
     bool supportsSampling() const override { return true; }
     bool supportsFinal() const override { return true; }
     bool supportsPrewhere() const override { return true; }
 
-    const NamesAndTypesList & getColumnsListImpl() const override { return columns; }
     NameAndTypePair getColumn(const String & column_name) const override;
     bool hasColumn(const String & column_name) const override;
 
@@ -64,7 +63,7 @@ public:
     BlockOutputStreamPtr write(const ASTPtr & query, const Settings & settings) override;
 
     void drop() override {}
-    void rename(const String & /*new_path_to_db*/, const String & /*new_database_name*/, const String & new_table_name) override { name = new_table_name; }
+    void rename(const String & /*new_path_to_db*/, const String & /*new_database_name*/, const String & new_table_name) override { table_name = new_table_name; }
     /// in the sub-tables, you need to manually add and delete columns
     /// the structure of the sub-table is not checked
     void alter(const AlterCommands & params, const String & database_name, const String & table_name, const Context & context) override;
@@ -95,7 +94,7 @@ public:
     ClusterPtr getCluster() const;
 
 
-    String name;
+    String table_name;
     String remote_database;
     String remote_table;
 
@@ -132,10 +131,7 @@ public:
 protected:
     StorageDistributed(
         const std::string & name_,
-        const NamesAndTypesList & columns_,
-        const NamesAndTypesList & materialized_columns_,
-        const NamesAndTypesList & alias_columns_,
-        const ColumnDefaults & column_defaults_,
+        const ColumnsDescription & columns_,
         const String & remote_database_,
         const String & remote_table_,
         const String & cluster_name_,
diff --git a/dbms/src/Storages/StorageFactory.cpp b/dbms/src/Storages/StorageFactory.cpp
index 0bcc7166c1d..1b8970aba24 100644
--- a/dbms/src/Storages/StorageFactory.cpp
+++ b/dbms/src/Storages/StorageFactory.cpp
@@ -45,10 +45,7 @@ StoragePtr StorageFactory::get(
     const String & database_name,
     Context & local_context,
     Context & context,
-    const NamesAndTypesList & columns,
-    const NamesAndTypesList & materialized_columns,
-    const NamesAndTypesList & alias_columns,
-    const ColumnDefaults & column_defaults,
+    const ColumnsDescription & columns,
     bool attach,
     bool has_force_restore_data_flag) const
 {
@@ -67,9 +64,9 @@ StoragePtr StorageFactory::get(
     {
         /// Check for some special types, that are not allowed to be stored in tables. Example: NULL data type.
         /// Exception: any type is allowed in View, because plain (non-materialized) View does not store anything itself.
-        checkAllTypesAreAllowedInTable(columns);
-        checkAllTypesAreAllowedInTable(materialized_columns);
-        checkAllTypesAreAllowedInTable(alias_columns);
+        checkAllTypesAreAllowedInTable(columns.ordinary);
+        checkAllTypesAreAllowedInTable(columns.materialized);
+        checkAllTypesAreAllowedInTable(columns.aliases);
 
         if (query.is_materialized_view)
         {
@@ -130,9 +127,6 @@ StoragePtr StorageFactory::get(
         .local_context = local_context,
         .context = context,
         .columns = columns,
-        .materialized_columns = materialized_columns,
-        .alias_columns = alias_columns,
-        .column_defaults = column_defaults,
         .attach = attach,
         .has_force_restore_data_flag = has_force_restore_data_flag
     };
diff --git a/dbms/src/Storages/StorageFactory.h b/dbms/src/Storages/StorageFactory.h
index 1af131943ba..2acb9fb7c00 100644
--- a/dbms/src/Storages/StorageFactory.h
+++ b/dbms/src/Storages/StorageFactory.h
@@ -14,7 +14,7 @@ class ASTStorage;
 
 
 /** Allows to create a table by the name and parameters of the engine.
-  * In 'columns', 'materialized_columns', etc., Nested data structures must be flattened.
+  * In 'columns' Nested data structures must be flattened.
   * You should subsequently call IStorage::startup method to work with table.
   */
 class StorageFactory : public ext::singleton<StorageFactory>
@@ -31,10 +31,7 @@ public:
         const String & database_name;
         Context & local_context;
         Context & context;
-        const NamesAndTypesList & columns;
-        const NamesAndTypesList & materialized_columns;
-        const NamesAndTypesList & alias_columns;
-        const ColumnDefaults & column_defaults;
+        const ColumnsDescription & columns;
         bool attach;
         bool has_force_restore_data_flag;
     };
@@ -48,10 +45,7 @@ public:
         const String & database_name,
         Context & local_context,
         Context & context,
-        const NamesAndTypesList & columns,
-        const NamesAndTypesList & materialized_columns,
-        const NamesAndTypesList & alias_columns,
-        const ColumnDefaults & column_defaults,
+        const ColumnsDescription & columns,
         bool attach,
         bool has_force_restore_data_flag) const;
 
diff --git a/dbms/src/Storages/StorageFile.cpp b/dbms/src/Storages/StorageFile.cpp
index 78fb3990978..2f606d5cbcf 100644
--- a/dbms/src/Storages/StorageFile.cpp
+++ b/dbms/src/Storages/StorageFile.cpp
@@ -54,12 +54,9 @@ StorageFile::StorageFile(
         const std::string & db_dir_path,
         const std::string & table_name_,
         const std::string & format_name_,
-        const NamesAndTypesList & columns_,
-        const NamesAndTypesList & materialized_columns_,
-        const NamesAndTypesList & alias_columns_,
-        const ColumnDefaults & column_defaults_,
+        const ColumnsDescription & columns_,
         Context & context_)
-    : IStorage(columns_, materialized_columns_, alias_columns_, column_defaults_),
+    : IStorage(columns_),
     table_name(table_name_), format_name(format_name_), context_global(context_), table_fd(table_fd_)
 {
     if (table_fd < 0) /// Will use file
@@ -311,7 +308,6 @@ void registerStorageFile(StorageFactory & factory)
         return StorageFile::create(
             source_path, source_fd,
             args.data_path, args.table_name, format_name, args.columns,
-            args.materialized_columns, args.alias_columns, args.column_defaults,
             args.context);
     });
 }
diff --git a/dbms/src/Storages/StorageFile.h b/dbms/src/Storages/StorageFile.h
index 8501415001c..4292ea6d7f9 100644
--- a/dbms/src/Storages/StorageFile.h
+++ b/dbms/src/Storages/StorageFile.h
@@ -31,11 +31,6 @@ public:
         return table_name;
     }
 
-    const NamesAndTypesList & getColumnsListImpl() const override
-    {
-        return columns;
-    }
-
     BlockInputStreams read(
         const Names & column_names,
         const SelectQueryInfo & query_info,
@@ -61,7 +56,7 @@ protected:
     /** there are three options (ordered by priority):
     - use specified file descriptor if (fd >= 0)
     - use specified table_path if it isn't empty
-    - create own tabale inside data/db/table/
+    - create own table inside data/db/table/
     */
     StorageFile(
         const std::string & table_path_,
@@ -69,10 +64,7 @@ protected:
         const std::string & db_dir_path,
         const std::string & table_name_,
         const std::string & format_name_,
-        const NamesAndTypesList & columns_,
-        const NamesAndTypesList & materialized_columns_,
-        const NamesAndTypesList & alias_columns_,
-        const ColumnDefaults & column_defaults_,
+        const ColumnsDescription & columns_,
         Context & context_);
 
 private:
diff --git a/dbms/src/Storages/StorageJoin.cpp b/dbms/src/Storages/StorageJoin.cpp
index 846c18d7055..c3dfc16c5d7 100644
--- a/dbms/src/Storages/StorageJoin.cpp
+++ b/dbms/src/Storages/StorageJoin.cpp
@@ -24,24 +24,12 @@ StorageJoin::StorageJoin(
     const String & name_,
     const Names & key_names_,
     ASTTableJoin::Kind kind_, ASTTableJoin::Strictness strictness_,
-    const NamesAndTypesList & columns_,
-    const NamesAndTypesList & materialized_columns_,
-    const NamesAndTypesList & alias_columns_,
-    const ColumnDefaults & column_defaults_)
-    : StorageSetOrJoinBase{path_, name_, columns_, materialized_columns_, alias_columns_, column_defaults_},
+    const ColumnsDescription & columns_)
+    : StorageSetOrJoinBase{path_, name_, columns_},
     key_names(key_names_), kind(kind_), strictness(strictness_)
 {
-    /// Check that key exists in table definition.
-    const auto check_key_exists = [] (const NamesAndTypesList & columns, const String & key)
-    {
-        for (const auto & column : columns)
-            if (column.name == key)
-                return true;
-        return false;
-    };
-
     for (const auto & key : key_names)
-        if (!check_key_exists(columns, key) && !check_key_exists(materialized_columns, key))
+        if (!columns.has(key))
             throw Exception{
                 "Key column (" + key + ") does not exist in table declaration.",
                 ErrorCodes::NO_SUCH_COLUMN_IN_TABLE};
@@ -58,7 +46,7 @@ void StorageJoin::assertCompatible(ASTTableJoin::Kind kind_, ASTTableJoin::Stric
 {
     /// NOTE Could be more loose.
     if (!(kind == kind_ && strictness == strictness_))
-        throw Exception("Table " + name + " has incompatible type of JOIN.", ErrorCodes::INCOMPATIBLE_TYPE_OF_JOIN);
+        throw Exception("Table " + table_name + " has incompatible type of JOIN.", ErrorCodes::INCOMPATIBLE_TYPE_OF_JOIN);
 }
 
 
@@ -123,7 +111,7 @@ void registerStorageJoin(StorageFactory & factory)
         return StorageJoin::create(
             args.data_path, args.table_name,
             key_names, kind, strictness,
-            args.columns, args.materialized_columns, args.alias_columns, args.column_defaults);
+            args.columns);
     });
 }
 
diff --git a/dbms/src/Storages/StorageJoin.h b/dbms/src/Storages/StorageJoin.h
index 135236beddb..c005450e772 100644
--- a/dbms/src/Storages/StorageJoin.h
+++ b/dbms/src/Storages/StorageJoin.h
@@ -47,10 +47,7 @@ protected:
         const String & name_,
         const Names & key_names_,
         ASTTableJoin::Kind kind_, ASTTableJoin::Strictness strictness_,
-        const NamesAndTypesList & columns_,
-        const NamesAndTypesList & materialized_columns_,
-        const NamesAndTypesList & alias_columns_,
-        const ColumnDefaults & column_defaults_);
+        const ColumnsDescription & columns_);
 };
 
 }
diff --git a/dbms/src/Storages/StorageKafka.cpp b/dbms/src/Storages/StorageKafka.cpp
index c3b14a468ca..a822fe47a92 100644
--- a/dbms/src/Storages/StorageKafka.cpp
+++ b/dbms/src/Storages/StorageKafka.cpp
@@ -218,13 +218,10 @@ StorageKafka::StorageKafka(
     const std::string & table_name_,
     const std::string & database_name_,
     Context & context_,
-    const NamesAndTypesList & columns_,
-    const NamesAndTypesList & materialized_columns_,
-    const NamesAndTypesList & alias_columns_,
-    const ColumnDefaults & column_defaults_,
+    const ColumnsDescription & columns_,
     const String & brokers_, const String & group_, const Names & topics_,
     const String & format_name_, const String & schema_name_, size_t num_consumers_)
-    : IStorage{columns_, materialized_columns_, alias_columns_, column_defaults_},
+    : IStorage{columns_},
     table_name(table_name_), database_name(database_name_), context(context_),
     topics(topics_), brokers(brokers_), group(group_), format_name(format_name_), schema_name(schema_name_),
     num_consumers(num_consumers_), log(&Logger::get("StorageKafka (" + table_name_ + ")")),
@@ -591,7 +588,6 @@ void registerStorageKafka(StorageFactory & factory)
 
         return StorageKafka::create(
             args.table_name, args.database_name, args.context, args.columns,
-            args.materialized_columns, args.alias_columns, args.column_defaults,
             brokers, group, topics, format, schema, num_consumers);
     });
 }
diff --git a/dbms/src/Storages/StorageKafka.h b/dbms/src/Storages/StorageKafka.h
index 39df9c136a1..655b0bfe5e8 100644
--- a/dbms/src/Storages/StorageKafka.h
+++ b/dbms/src/Storages/StorageKafka.h
@@ -102,10 +102,7 @@ protected:
         const std::string & table_name_,
         const std::string & database_name_,
         Context & context_,
-        const NamesAndTypesList & columns_,
-        const NamesAndTypesList & materialized_columns_,
-        const NamesAndTypesList & alias_columns_,
-        const ColumnDefaults & column_defaults_,
+        const ColumnsDescription & columns_,
         const String & brokers_, const String & group_, const Names & topics_,
         const String & format_name_, const String & schema_name_, size_t num_consumers_);
 };
diff --git a/dbms/src/Storages/StorageLog.cpp b/dbms/src/Storages/StorageLog.cpp
index 711d48621f0..27881d8b8fe 100644
--- a/dbms/src/Storages/StorageLog.cpp
+++ b/dbms/src/Storages/StorageLog.cpp
@@ -358,12 +358,9 @@ void LogBlockOutputStream::writeMarks(MarksForColumns && marks)
 StorageLog::StorageLog(
     const std::string & path_,
     const std::string & name_,
-    const NamesAndTypesList & columns_,
-    const NamesAndTypesList & materialized_columns_,
-    const NamesAndTypesList & alias_columns_,
-    const ColumnDefaults & column_defaults_,
+    const ColumnsDescription & columns_,
     size_t max_compress_block_size_)
-    : IStorage{columns_, materialized_columns_, alias_columns_, column_defaults_},
+    : IStorage{columns_},
     path(path_), name(name_),
     max_compress_block_size(max_compress_block_size_),
     file_checker(path + escapeForFileName(name) + '/' + "sizes.json")
@@ -374,7 +371,7 @@ StorageLog::StorageLog(
      /// create files if they do not exist
     Poco::File(path + escapeForFileName(name) + '/').createDirectories();
 
-    for (const auto & column : getColumnsList())
+    for (const auto & column : columns.getList())
         addFiles(column.name, *column.type);
 
     marks_file = Poco::File(path + escapeForFileName(name) + '/' + DBMS_STORAGE_LOG_MARKS_FILE_NAME);
@@ -468,8 +465,8 @@ void StorageLog::rename(const String & new_path_to_db, const String & /*new_data
 
 const StorageLog::Marks & StorageLog::getMarksWithRealRowCount() const
 {
-    const String & column_name = columns.front().name;
-    const IDataType & column_type = *columns.front().type;
+    const String & column_name = columns.ordinary.front().name;
+    const IDataType & column_type = *columns.ordinary.front().type;
     String filename;
 
     /** We take marks from first column.
@@ -502,7 +499,7 @@ BlockInputStreams StorageLog::read(
     processed_stage = QueryProcessingStage::FetchColumns;
     loadMarks();
 
-    NamesAndTypesList columns = Nested::collect(getColumnsList().addTypes(column_names));
+    NamesAndTypesList all_columns = Nested::collect(columns.getList().addTypes(column_names));
 
     std::shared_lock<std::shared_mutex> lock(rwlock);
 
@@ -526,7 +523,7 @@ BlockInputStreams StorageLog::read(
 
         res.emplace_back(std::make_shared<LogBlockInputStream>(
             max_block_size,
-            columns,
+            all_columns,
             *this,
             mark_begin,
             rows_end - rows_begin,
@@ -562,7 +559,6 @@ void registerStorageLog(StorageFactory & factory)
 
         return StorageLog::create(
             args.data_path, args.table_name, args.columns,
-            args.materialized_columns, args.alias_columns, args.column_defaults,
             args.context.getSettings().max_compress_block_size);
     });
 }
diff --git a/dbms/src/Storages/StorageLog.h b/dbms/src/Storages/StorageLog.h
index 4f38f403138..b52a90ec563 100644
--- a/dbms/src/Storages/StorageLog.h
+++ b/dbms/src/Storages/StorageLog.h
@@ -52,10 +52,7 @@ protected:
     StorageLog(
         const std::string & path_,
         const std::string & name_,
-        const NamesAndTypesList & columns_,
-        const NamesAndTypesList & materialized_columns_,
-        const NamesAndTypesList & alias_columns_,
-        const ColumnDefaults & column_defaults_,
+        const ColumnsDescription & columns_,
         size_t max_compress_block_size_);
 
 private:
diff --git a/dbms/src/Storages/StorageMaterializedView.cpp b/dbms/src/Storages/StorageMaterializedView.cpp
index 33eec0ad8f7..1146028830b 100644
--- a/dbms/src/Storages/StorageMaterializedView.cpp
+++ b/dbms/src/Storages/StorageMaterializedView.cpp
@@ -64,12 +64,9 @@ StorageMaterializedView::StorageMaterializedView(
     const String & database_name_,
     Context & local_context,
     const ASTCreateQuery & query,
-    const NamesAndTypesList & columns_,
-    const NamesAndTypesList & materialized_columns_,
-    const NamesAndTypesList & alias_columns_,
-    const ColumnDefaults & column_defaults_,
+    const ColumnsDescription & columns_,
     bool attach_)
-    : IStorage{columns_, materialized_columns_, alias_columns_, column_defaults_}, table_name(table_name_),
+    : IStorage{columns_}, table_name(table_name_),
     database_name(database_name_), global_context(local_context.getGlobalContext())
 {
     if (!query.select)
@@ -213,8 +210,7 @@ void registerStorageMaterializedView(StorageFactory & factory)
         /// Pass local_context here to convey setting for inner table
         return StorageMaterializedView::create(
             args.table_name, args.database_name, args.local_context, args.query,
-            args.columns, args.materialized_columns, args.alias_columns, args.column_defaults,
-            args.attach);
+            args.columns, args.attach);
     });
 }
 
diff --git a/dbms/src/Storages/StorageMaterializedView.h b/dbms/src/Storages/StorageMaterializedView.h
index 4dd0980e874..82a5b0822b4 100644
--- a/dbms/src/Storages/StorageMaterializedView.h
+++ b/dbms/src/Storages/StorageMaterializedView.h
@@ -17,7 +17,6 @@ class StorageMaterializedView : public ext::shared_ptr_helper<StorageMaterialize
 public:
     std::string getName() const override { return "MaterializedView"; }
     std::string getTableName() const override { return table_name; }
-    const NamesAndTypesList & getColumnsListImpl() const override { return columns; }
     ASTPtr getInnerQuery() const { return inner_query->clone(); };
     StoragePtr getTargetTable() const;
 
@@ -62,10 +61,7 @@ protected:
         const String & database_name_,
         Context & local_context,
         const ASTCreateQuery & query,
-        const NamesAndTypesList & columns_,
-        const NamesAndTypesList & materialized_columns_,
-        const NamesAndTypesList & alias_columns_,
-        const ColumnDefaults & column_defaults_,
+        const ColumnsDescription & columns_,
         bool attach_);
 };
 
diff --git a/dbms/src/Storages/StorageMemory.cpp b/dbms/src/Storages/StorageMemory.cpp
index 96af6a9a138..7b36796c324 100644
--- a/dbms/src/Storages/StorageMemory.cpp
+++ b/dbms/src/Storages/StorageMemory.cpp
@@ -74,14 +74,8 @@ private:
 };
 
 
-StorageMemory::StorageMemory(
-    const std::string & name_,
-    const NamesAndTypesList & columns_,
-    const NamesAndTypesList & materialized_columns_,
-    const NamesAndTypesList & alias_columns_,
-    const ColumnDefaults & column_defaults_)
-    : IStorage{columns_, materialized_columns_, alias_columns_, column_defaults_},
-    name(name_)
+StorageMemory::StorageMemory(String table_name_, ColumnsDescription columns_description_)
+    : IStorage{std::move(columns_description_)}, table_name(std::move(table_name_))
 {
 }
 
@@ -144,7 +138,7 @@ void registerStorageMemory(StorageFactory & factory)
                 "Engine " + args.engine_name + " doesn't support any arguments (" + toString(args.engine_args.size()) + " given)",
                 ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
 
-        return StorageMemory::create(args.table_name, args.columns, args.materialized_columns, args.alias_columns, args.column_defaults);
+        return StorageMemory::create(args.table_name, args.columns);
     });
 }
 
diff --git a/dbms/src/Storages/StorageMemory.h b/dbms/src/Storages/StorageMemory.h
index 956e4354727..454fdcc2823 100644
--- a/dbms/src/Storages/StorageMemory.h
+++ b/dbms/src/Storages/StorageMemory.h
@@ -24,7 +24,7 @@ friend class MemoryBlockOutputStream;
 
 public:
     std::string getName() const override { return "Memory"; }
-    std::string getTableName() const override { return name; }
+    std::string getTableName() const override { return table_name; }
 
     size_t getSize() const { return data.size(); }
 
@@ -39,10 +39,10 @@ public:
     BlockOutputStreamPtr write(const ASTPtr & query, const Settings & settings) override;
 
     void drop() override;
-    void rename(const String & /*new_path_to_db*/, const String & /*new_database_name*/, const String & new_table_name) override { name = new_table_name; }
+    void rename(const String & /*new_path_to_db*/, const String & /*new_database_name*/, const String & new_table_name) override { table_name = new_table_name; }
 
 private:
-    String name;
+    String table_name;
 
     /// The data itself. `list` - so that when inserted to the end, the existing iterators are not invalidated.
     BlocksList data;
@@ -50,12 +50,7 @@ private:
     std::mutex mutex;
 
 protected:
-    StorageMemory(
-        const std::string & name_,
-        const NamesAndTypesList & columns_,
-        const NamesAndTypesList & materialized_columns_,
-        const NamesAndTypesList & alias_columns_,
-        const ColumnDefaults & column_defaults_);
+    StorageMemory(String table_name_, ColumnsDescription columns_description_);
 };
 
 }
diff --git a/dbms/src/Storages/StorageMerge.cpp b/dbms/src/Storages/StorageMerge.cpp
index 7ceecc62a43..39fed81e538 100644
--- a/dbms/src/Storages/StorageMerge.cpp
+++ b/dbms/src/Storages/StorageMerge.cpp
@@ -37,14 +37,11 @@ namespace ErrorCodes
 
 StorageMerge::StorageMerge(
     const std::string & name_,
-    const NamesAndTypesList & columns_,
-    const NamesAndTypesList & materialized_columns_,
-    const NamesAndTypesList & alias_columns_,
-    const ColumnDefaults & column_defaults_,
+    const ColumnsDescription & columns_,
     const String & source_database_,
     const String & table_name_regexp_,
     const Context & context_)
-    : IStorage{columns_, materialized_columns_, alias_columns_, column_defaults_},
+    : IStorage{columns_},
     name(name_), source_database(source_database_),
     table_name_regexp(table_name_regexp_), context(context_)
 {
@@ -184,7 +181,7 @@ BlockInputStreams StorageMerge::read(
 
         /// If there are only virtual columns in query, you must request at least one other column.
         if (real_column_names.size() == 0)
-            real_column_names.push_back(ExpressionActions::getSmallestColumn(table->getColumnsList()));
+            real_column_names.push_back(ExpressionActions::getSmallestColumn(table->columns.getList()));
 
         /// Substitute virtual column for its value when querying tables.
         ASTPtr modified_query_ast = query->clone();
@@ -336,11 +333,9 @@ void StorageMerge::alter(const AlterCommands & params, const String & database_n
             throw Exception("Storage engine " + getName() + " doesn't support primary key.", ErrorCodes::NOT_IMPLEMENTED);
 
     auto lock = lockStructureForAlter(__PRETTY_FUNCTION__);
-    params.apply(columns, materialized_columns, alias_columns, column_defaults);
 
-    context.getDatabase(database_name)->alterTable(
-        context, table_name,
-        columns, materialized_columns, alias_columns, column_defaults, {});
+    params.apply(columns);
+    context.getDatabase(database_name)->alterTable(context, table_name, columns, {});
 }
 
 
@@ -367,7 +362,6 @@ void registerStorageMerge(StorageFactory & factory)
 
         return StorageMerge::create(
             args.table_name, args.columns,
-            args.materialized_columns, args.alias_columns, args.column_defaults,
             source_database, table_name_regexp, args.context);
     });
 }
diff --git a/dbms/src/Storages/StorageMerge.h b/dbms/src/Storages/StorageMerge.h
index 9bef645c0e0..0c37e4e7fb4 100644
--- a/dbms/src/Storages/StorageMerge.h
+++ b/dbms/src/Storages/StorageMerge.h
@@ -26,7 +26,6 @@ public:
     bool supportsFinal() const override { return true; }
     bool supportsIndexForIn() const override { return true; }
 
-    const NamesAndTypesList & getColumnsListImpl() const override { return columns; }
     NameAndTypePair getColumn(const String & column_name) const override;
     bool hasColumn(const String & column_name) const override;
 
@@ -60,10 +59,7 @@ private:
 protected:
     StorageMerge(
         const std::string & name_,
-        const NamesAndTypesList & columns_,
-        const NamesAndTypesList & materialized_columns_,
-        const NamesAndTypesList & alias_columns_,
-        const ColumnDefaults & column_defaults_,
+        const ColumnsDescription & columns_,
         const String & source_database_,
         const String & table_name_regexp_,
         const Context & context_);
diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp
index 8ee9f3c7c5b..71d8013e4db 100644
--- a/dbms/src/Storages/StorageMergeTree.cpp
+++ b/dbms/src/Storages/StorageMergeTree.cpp
@@ -35,10 +35,7 @@ StorageMergeTree::StorageMergeTree(
     const String & path_,
     const String & database_name_,
     const String & table_name_,
-    const NamesAndTypesList & columns_,
-    const NamesAndTypesList & materialized_columns_,
-    const NamesAndTypesList & alias_columns_,
-    const ColumnDefaults & column_defaults_,
+    const ColumnsDescription & columns_,
     bool attach,
     Context & context_,
     const ASTPtr & primary_expr_ast_,
@@ -49,12 +46,11 @@ StorageMergeTree::StorageMergeTree(
     const MergeTreeData::MergingParams & merging_params_,
     const MergeTreeSettings & settings_,
     bool has_force_restore_data_flag)
-    : IStorage{columns_, materialized_columns_, alias_columns_, column_defaults_},
+    : IStorage{columns_},
     path(path_), database_name(database_name_), table_name(table_name_), full_path(path + escapeForFileName(table_name) + '/'),
     context(context_), background_pool(context_.getBackgroundPool()),
     data(database_name, table_name,
          full_path, columns_,
-         materialized_columns_, alias_columns_, column_defaults_,
          context_, primary_expr_ast_, secondary_sorting_expr_list_, date_column_name, partition_expr_ast_,
          sampling_expression_, merging_params_,
          settings_, false, attach),
@@ -157,16 +153,9 @@ void StorageMergeTree::alter(
 
     data.checkAlter(params);
 
-    auto new_columns = data.getColumnsListNonMaterialized();
-    auto new_materialized_columns = data.materialized_columns;
-    auto new_alias_columns = data.alias_columns;
-    auto new_column_defaults = data.column_defaults;
+    auto new_columns = data.columns;
 
-    params.apply(new_columns, new_materialized_columns, new_alias_columns, new_column_defaults);
-
-    auto columns_for_parts = new_columns;
-    columns_for_parts.insert(std::end(columns_for_parts),
-        std::begin(new_materialized_columns), std::end(new_materialized_columns));
+    params.apply(new_columns);
 
     std::vector<MergeTreeData::AlterDataPartTransactionPtr> transactions;
 
@@ -187,6 +176,7 @@ void StorageMergeTree::alter(
         throw Exception("MODIFY PRIMARY KEY only supported for tables without sampling key", ErrorCodes::BAD_ARGUMENTS);
 
     auto parts = data.getDataParts({MergeTreeDataPartState::PreCommitted, MergeTreeDataPartState::Committed, MergeTreeDataPartState::Outdated});
+    auto columns_for_parts = new_columns.getList();
     for (const MergeTreeData::DataPartPtr & part : parts)
     {
         if (auto transaction = data.alterDataPart(part, columns_for_parts, new_primary_key_ast, false))
@@ -212,19 +202,10 @@ void StorageMergeTree::alter(
         };
     }
 
-    context.getDatabase(database_name)->alterTable(
-        context, table_name,
-        new_columns, new_materialized_columns, new_alias_columns, new_column_defaults,
-        storage_modifier);
+    context.getDatabase(database_name)->alterTable(context, table_name, new_columns, storage_modifier);
 
-    materialized_columns = new_materialized_columns;
-    alias_columns = new_alias_columns;
-    column_defaults = new_column_defaults;
-
-    data.setColumnsList(new_columns);
-    data.materialized_columns = std::move(new_materialized_columns);
-    data.alias_columns = std::move(new_alias_columns);
-    data.column_defaults = std::move(new_column_defaults);
+    columns = new_columns;
+    data.columns = std::move(new_columns);
 
     if (primary_key_is_modified)
     {
@@ -438,17 +419,10 @@ void StorageMergeTree::clearColumnInPartition(const ASTPtr & partition, const Fi
     alter_command.type = AlterCommand::DROP_COLUMN;
     alter_command.column_name = get<String>(column_name);
 
-    auto new_columns = data.getColumnsListNonMaterialized();
-    auto new_materialized_columns = data.materialized_columns;
-    auto new_alias_columns = data.alias_columns;
-    auto new_column_defaults = data.column_defaults;
-
-    alter_command.apply(new_columns, new_materialized_columns, new_alias_columns, new_column_defaults);
-
-    auto columns_for_parts = new_columns;
-    columns_for_parts.insert(std::end(columns_for_parts),
-        std::begin(new_materialized_columns), std::end(new_materialized_columns));
+    auto new_columns = data.columns;
+    alter_command.apply(new_columns);
 
+    auto columns_for_parts = new_columns.getList();
     for (const auto & part : parts)
     {
         if (part->info.partition_id != partition_id)
diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h
index e1121ec84d0..e2ad6177a82 100644
--- a/dbms/src/Storages/StorageMergeTree.h
+++ b/dbms/src/Storages/StorageMergeTree.h
@@ -2,6 +2,7 @@
 
 #include <ext/shared_ptr_helper.h>
 
+#include <Storages/IStorage.h>
 #include <Storages/MergeTree/MergeTreeData.h>
 #include <Storages/MergeTree/MergeTreeDataSelectExecutor.h>
 #include <Storages/MergeTree/MergeTreeDataWriter.h>
@@ -35,8 +36,6 @@ public:
     bool supportsFinal() const override { return data.supportsFinal(); }
     bool supportsPrewhere() const override { return data.supportsPrewhere(); }
 
-    const NamesAndTypesList & getColumnsListImpl() const override { return data.getColumnsListNonMaterialized(); }
-
     NameAndTypePair getColumn(const String & column_name) const override
     {
         return data.getColumn(column_name);
@@ -135,10 +134,7 @@ protected:
         const String & path_,
         const String & database_name_,
         const String & table_name_,
-        const NamesAndTypesList & columns_,
-        const NamesAndTypesList & materialized_columns_,
-        const NamesAndTypesList & alias_columns_,
-        const ColumnDefaults & column_defaults_,
+        const ColumnsDescription & columns_,
         bool attach,
         Context & context_,
         const ASTPtr & primary_expr_ast_,
diff --git a/dbms/src/Storages/StorageMySQL.cpp b/dbms/src/Storages/StorageMySQL.cpp
index f3f18518d83..28a367f17e0 100644
--- a/dbms/src/Storages/StorageMySQL.cpp
+++ b/dbms/src/Storages/StorageMySQL.cpp
@@ -23,8 +23,8 @@ StorageMySQL::StorageMySQL(
     mysqlxx::Pool && pool,
     const std::string & remote_database_name,
     const std::string & remote_table_name,
-    const NamesAndTypesList & columns_)
-    : IStorage{columns_, {}, {}, {}}
+    const ColumnsDescription & columns_)
+    : IStorage{columns_}
     , name(name)
     , remote_database_name(remote_database_name)
     , remote_table_name(remote_table_name)
@@ -43,7 +43,7 @@ BlockInputStreams StorageMySQL::read(
 {
     check(column_names);
     processed_stage = QueryProcessingStage::FetchColumns;
-    String query = transformQueryForExternalDatabase(*query_info.query, columns, remote_database_name, remote_table_name, context);
+    String query = transformQueryForExternalDatabase(*query_info.query, columns.ordinary, remote_database_name, remote_table_name, context);
 
     Block sample_block;
     for (const String & name : column_names)
diff --git a/dbms/src/Storages/StorageMySQL.h b/dbms/src/Storages/StorageMySQL.h
index 9558605ff0e..9e2b233283e 100644
--- a/dbms/src/Storages/StorageMySQL.h
+++ b/dbms/src/Storages/StorageMySQL.h
@@ -24,7 +24,7 @@ public:
         mysqlxx::Pool && pool,
         const std::string & remote_database_name,
         const std::string & remote_table_name,
-        const NamesAndTypesList & columns_);
+        const ColumnsDescription & columns);
 
     std::string getName() const override { return "MySQL"; }
     std::string getTableName() const override { return name; }
diff --git a/dbms/src/Storages/StorageNull.cpp b/dbms/src/Storages/StorageNull.cpp
index 0cee00aa1b8..13232068ca9 100644
--- a/dbms/src/Storages/StorageNull.cpp
+++ b/dbms/src/Storages/StorageNull.cpp
@@ -25,8 +25,7 @@ void registerStorageNull(StorageFactory & factory)
                 "Engine " + args.engine_name + " doesn't support any arguments (" + toString(args.engine_args.size()) + " given)",
                 ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
 
-        return StorageNull::create(args.table_name,
-            args.columns, args.materialized_columns, args.alias_columns, args.column_defaults);
+        return StorageNull::create(args.table_name, args.columns);
     });
 }
 
@@ -34,11 +33,9 @@ void StorageNull::alter(const AlterCommands & params, const String & database_na
 {
     auto lock = lockStructureForAlter(__PRETTY_FUNCTION__);
 
-    params.apply(columns, materialized_columns, alias_columns, column_defaults);
+    params.apply(columns);
 
-    context.getDatabase(database_name)->alterTable(
-        context, table_name,
-        columns, materialized_columns, alias_columns, column_defaults, {});
+    context.getDatabase(database_name)->alterTable(context, table_name, columns, {});
 }
 
 }
diff --git a/dbms/src/Storages/StorageNull.h b/dbms/src/Storages/StorageNull.h
index ef3f0f6fcd1..cdde480a951 100644
--- a/dbms/src/Storages/StorageNull.h
+++ b/dbms/src/Storages/StorageNull.h
@@ -18,7 +18,7 @@ class StorageNull : public ext::shared_ptr_helper<StorageNull>, public IStorage
 {
 public:
     std::string getName() const override { return "Null"; }
-    std::string getTableName() const override { return name; }
+    std::string getTableName() const override { return table_name; }
 
     BlockInputStreams read(
         const Names & column_names,
@@ -38,22 +38,19 @@ public:
 
     void rename(const String & /*new_path_to_db*/, const String & /*new_database_name*/, const String & new_table_name) override
     {
-        name = new_table_name;
+        table_name = new_table_name;
     }
 
     void alter(const AlterCommands & params, const String & database_name, const String & table_name, const Context & context) override;
 
 private:
-    String name;
+    String table_name;
 
 protected:
-    StorageNull(
-        const std::string & name_,
-        const NamesAndTypesList & columns_,
-        const NamesAndTypesList & materialized_columns_,
-        const NamesAndTypesList & alias_columns_,
-        const ColumnDefaults & column_defaults_)
-        : IStorage{columns_, materialized_columns_, alias_columns_, column_defaults_}, name(name_)  {}
+    StorageNull(String table_name_, ColumnsDescription columns_description_)
+        : IStorage{std::move(columns_description_)}, table_name(std::move(table_name_))
+    {
+    }
 };
 
 }
diff --git a/dbms/src/Storages/StorageODBC.cpp b/dbms/src/Storages/StorageODBC.cpp
index 67362bf2f90..7856ee9a231 100644
--- a/dbms/src/Storages/StorageODBC.cpp
+++ b/dbms/src/Storages/StorageODBC.cpp
@@ -21,8 +21,8 @@ StorageODBC::StorageODBC(
     const std::string & connection_string,
     const std::string & remote_database_name,
     const std::string & remote_table_name,
-    const NamesAndTypesList & columns_)
-    : IStorage{columns_, {}, {}, {}}
+    const ColumnsDescription & columns_)
+    : IStorage{columns_}
     , name(name)
     , remote_database_name(remote_database_name)
     , remote_table_name(remote_table_name)
@@ -43,7 +43,7 @@ BlockInputStreams StorageODBC::read(
 {
     check(column_names);
     processed_stage = QueryProcessingStage::FetchColumns;
-    String query = transformQueryForExternalDatabase(*query_info.query, columns, remote_database_name, remote_table_name, context);
+    String query = transformQueryForExternalDatabase(*query_info.query, columns.ordinary, remote_database_name, remote_table_name, context);
 
     Block sample_block;
     for (const String & name : column_names)
diff --git a/dbms/src/Storages/StorageODBC.h b/dbms/src/Storages/StorageODBC.h
index f582c74a263..5632fe38282 100644
--- a/dbms/src/Storages/StorageODBC.h
+++ b/dbms/src/Storages/StorageODBC.h
@@ -30,7 +30,7 @@ public:
         const std::string & connection_string,
         const std::string & remote_database_name,
         const std::string & remote_table_name,
-        const NamesAndTypesList & columns_);
+        const ColumnsDescription & columns_);
 
     std::string getName() const override { return "ODBC"; }
     std::string getTableName() const override { return name; }
diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp
index 9917b7e7a94..9f6553f6512 100644
--- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp
@@ -174,10 +174,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree(
     const String & replica_name_,
     bool attach,
     const String & path_, const String & database_name_, const String & name_,
-    const NamesAndTypesList & columns_,
-    const NamesAndTypesList & materialized_columns_,
-    const NamesAndTypesList & alias_columns_,
-    const ColumnDefaults & column_defaults_,
+    const ColumnsDescription & columns_,
     Context & context_,
     const ASTPtr & primary_expr_ast_,
     const ASTPtr & secondary_sorting_expr_list_,
@@ -187,14 +184,13 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree(
     const MergeTreeData::MergingParams & merging_params_,
     const MergeTreeSettings & settings_,
     bool has_force_restore_data_flag)
-    : IStorage{columns_, materialized_columns_, alias_columns_, column_defaults_}, context(context_),
+    : IStorage{columns_}, context(context_),
     current_zookeeper(context.getZooKeeper()), database_name(database_name_),
     table_name(name_), full_path(path_ + escapeForFileName(table_name) + '/'),
     zookeeper_path(context.getMacros().expand(zookeeper_path_)),
     replica_name(context.getMacros().expand(replica_name_)),
     data(database_name, table_name,
         full_path, columns_,
-        materialized_columns_, alias_columns_, column_defaults_,
         context_, primary_expr_ast_, secondary_sorting_expr_list_, date_column_name, partition_expr_ast_,
         sampling_expression_, merging_params_,
         settings_, true, attach,
@@ -488,9 +484,7 @@ void StorageReplicatedMergeTree::createTableIfNotExists()
         acl, zkutil::CreateMode::Persistent));
     ops.emplace_back(std::make_unique<zkutil::Op::Create>(zookeeper_path + "/metadata", metadata,
         acl, zkutil::CreateMode::Persistent));
-    ops.emplace_back(std::make_unique<zkutil::Op::Create>(zookeeper_path + "/columns", ColumnsDescription<false>{
-        data.getColumnsListNonMaterialized(), data.materialized_columns,
-        data.alias_columns, data.column_defaults}.toString(),
+    ops.emplace_back(std::make_unique<zkutil::Op::Create>(zookeeper_path + "/columns", data.columns.toString(),
         acl, zkutil::CreateMode::Persistent));
     ops.emplace_back(std::make_unique<zkutil::Op::Create>(zookeeper_path + "/log", "",
         acl, zkutil::CreateMode::Persistent));
@@ -524,39 +518,28 @@ void StorageReplicatedMergeTree::checkTableStructure(bool skip_sanity_checks, bo
     TableMetadata(data).check(metadata_str);
 
     zkutil::Stat stat;
-    auto columns_desc = ColumnsDescription<true>::parse(zookeeper->get(zookeeper_path + "/columns", &stat));
+    auto columns_from_zk = ColumnsDescription::parse(zookeeper->get(zookeeper_path + "/columns", &stat));
 
-    auto & columns = columns_desc.columns;
-    auto & materialized_columns = columns_desc.materialized;
-    auto & alias_columns = columns_desc.alias;
-    auto & column_defaults = columns_desc.defaults;
     columns_version = stat.version;
 
-    if (columns != data.getColumnsListNonMaterialized() ||
-        materialized_columns != data.materialized_columns ||
-        alias_columns != data.alias_columns ||
-        column_defaults != data.column_defaults)
+    if (columns_from_zk != data.columns)
     {
         if (allow_alter &&
             (skip_sanity_checks ||
-             data.getColumnsListNonMaterialized().sizeOfDifference(columns) +
-             data.materialized_columns.sizeOfDifference(materialized_columns) <= 2))
+             data.columns.ordinary.sizeOfDifference(columns_from_zk.ordinary) +
+             data.columns.materialized.sizeOfDifference(columns_from_zk.materialized) <= 2))
         {
             LOG_WARNING(log, "Table structure in ZooKeeper is a little different from local table structure. Assuming ALTER.");
 
             /// Without any locks, because table has not been created yet.
-            context.getDatabase(database_name)->alterTable(
-                context, table_name,
-                columns, materialized_columns, alias_columns, column_defaults, {});
+            context.getDatabase(database_name)->alterTable(context, table_name, columns_from_zk, {});
 
-            data.setColumnsList(columns);
-            data.materialized_columns = std::move(materialized_columns);
-            data.alias_columns = std::move(alias_columns);
-            data.column_defaults = std::move(column_defaults);
+            columns = columns_from_zk;
+            data.columns = std::move(columns_from_zk);
         }
         else
         {
-            throw Exception("Table structure in ZooKeeper is too much different from local table structure.",
+            throw Exception("Table structure in ZooKeeper is too different from local table structure",
                             ErrorCodes::INCOMPATIBLE_COLUMNS);
         }
     }
@@ -714,12 +697,7 @@ void StorageReplicatedMergeTree::createReplica()
         LOG_DEBUG(log, "Copied " << source_queue.size() << " queue entries");
     }
 
-    zookeeper->create(replica_path + "/columns", ColumnsDescription<false>{
-            data.getColumnsListNonMaterialized(),
-            data.materialized_columns,
-            data.alias_columns,
-            data.column_defaults
-        }.toString(), zkutil::CreateMode::Persistent);
+    zookeeper->create(replica_path + "/columns", data.columns.toString(), zkutil::CreateMode::Persistent);
 }
 
 
@@ -1529,19 +1507,12 @@ void StorageReplicatedMergeTree::executeClearColumnInPartition(const LogEntry &
     alter_command.type = AlterCommand::DROP_COLUMN;
     alter_command.column_name = entry.column_name;
 
-    auto new_columns = data.getColumnsListNonMaterialized();
-    auto new_materialized_columns = data.materialized_columns;
-    auto new_alias_columns = data.alias_columns;
-    auto new_column_defaults = data.column_defaults;
-
-    alter_command.apply(new_columns, new_materialized_columns, new_alias_columns, new_column_defaults);
-
-    auto columns_for_parts = new_columns;
-    columns_for_parts.insert(std::end(columns_for_parts),
-        std::begin(new_materialized_columns), std::end(new_materialized_columns));
+    auto new_columns = data.columns;
+    alter_command.apply(new_columns);
 
     size_t modified_parts = 0;
     auto parts = data.getDataParts();
+    auto columns_for_parts = new_columns.getList();
     for (const auto & part : parts)
     {
         if (!entry_part_info.contains(part->info))
@@ -2573,15 +2544,10 @@ void StorageReplicatedMergeTree::alter(const AlterCommands & params,
             if (param.type == AlterCommand::MODIFY_PRIMARY_KEY)
                 throw Exception("Modification of primary key is not supported for replicated tables", ErrorCodes::NOT_IMPLEMENTED);
 
-        NamesAndTypesList new_columns = data.getColumnsListNonMaterialized();
-        NamesAndTypesList new_materialized_columns = data.materialized_columns;
-        NamesAndTypesList new_alias_columns = data.alias_columns;
-        ColumnDefaults new_column_defaults = data.column_defaults;
-        params.apply(new_columns, new_materialized_columns, new_alias_columns, new_column_defaults);
+        ColumnsDescription new_columns = data.columns;
+        params.apply(new_columns);
 
-        new_columns_str = ColumnsDescription<false>{
-            new_columns, new_materialized_columns,
-            new_alias_columns, new_column_defaults}.toString();
+        new_columns_str = new_columns.toString();
 
         /// Do ALTER.
         getZooKeeper()->set(zookeeper_path + "/columns", new_columns_str, -1, &stat);
diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h
index b1cbf630b1b..468f097534d 100644
--- a/dbms/src/Storages/StorageReplicatedMergeTree.h
+++ b/dbms/src/Storages/StorageReplicatedMergeTree.h
@@ -84,8 +84,6 @@ public:
     bool supportsPrewhere() const override { return data.supportsPrewhere(); }
     bool supportsReplication() const override { return true; }
 
-    const NamesAndTypesList & getColumnsListImpl() const override { return data.getColumnsListNonMaterialized(); }
-
     NameAndTypePair getColumn(const String & column_name) const override
     {
         return data.getColumn(column_name);
@@ -443,10 +441,7 @@ protected:
         const String & replica_name_,
         bool attach,
         const String & path_, const String & database_name_, const String & name_,
-        const NamesAndTypesList & columns_,
-        const NamesAndTypesList & materialized_columns_,
-        const NamesAndTypesList & alias_columns_,
-        const ColumnDefaults & column_defaults_,
+        const ColumnsDescription & columns_,
         Context & context_,
         const ASTPtr & primary_expr_ast_,
         const ASTPtr & secondary_sorting_expr_list_,
diff --git a/dbms/src/Storages/StorageSet.cpp b/dbms/src/Storages/StorageSet.cpp
index 1303aa6271f..5ba5d737435 100644
--- a/dbms/src/Storages/StorageSet.cpp
+++ b/dbms/src/Storages/StorageSet.cpp
@@ -88,18 +88,14 @@ BlockOutputStreamPtr StorageSetOrJoinBase::write(const ASTPtr & /*query*/, const
 
 StorageSetOrJoinBase::StorageSetOrJoinBase(
     const String & path_,
-    const String & name_,
-    const NamesAndTypesList & columns_,
-    const NamesAndTypesList & materialized_columns_,
-    const NamesAndTypesList & alias_columns_,
-    const ColumnDefaults & column_defaults_)
-    : IStorage{columns_, materialized_columns_, alias_columns_, column_defaults_},
-    name(name_)
+    const String & table_name_,
+    const ColumnsDescription & columns_)
+    : IStorage{columns_}, table_name(table_name_)
 {
     if (path_.empty())
         throw Exception("Join and Set storages require data path", ErrorCodes::INCORRECT_FILE_NAME);
 
-    path = path_ + escapeForFileName(name_) + '/';
+    path = path_ + escapeForFileName(table_name_) + '/';
 }
 
 
@@ -107,11 +103,8 @@ StorageSetOrJoinBase::StorageSetOrJoinBase(
 StorageSet::StorageSet(
     const String & path_,
     const String & name_,
-    const NamesAndTypesList & columns_,
-    const NamesAndTypesList & materialized_columns_,
-    const NamesAndTypesList & alias_columns_,
-    const ColumnDefaults & column_defaults_)
-    : StorageSetOrJoinBase{path_, name_, columns_, materialized_columns_, alias_columns_, column_defaults_},
+    const ColumnsDescription & columns_)
+    : StorageSetOrJoinBase{path_, name_, columns_},
     set(std::make_shared<Set>(SizeLimits()))
 {
     restore();
@@ -181,7 +174,7 @@ void StorageSetOrJoinBase::rename(const String & new_path_to_db, const String &
     Poco::File(path).renameTo(new_path);
 
     path = new_path + "/";
-    name = new_table_name;
+    table_name = new_table_name;
 }
 
 
@@ -194,9 +187,7 @@ void registerStorageSet(StorageFactory & factory)
                 "Engine " + args.engine_name + " doesn't support any arguments (" + toString(args.engine_args.size()) + " given)",
                 ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
 
-        return StorageSet::create(
-            args.data_path, args.table_name, args.columns,
-            args.materialized_columns, args.alias_columns, args.column_defaults);
+        return StorageSet::create(args.data_path, args.table_name, args.columns);
     });
 }
 
diff --git a/dbms/src/Storages/StorageSet.h b/dbms/src/Storages/StorageSet.h
index 2546967b24a..d1e4ae89226 100644
--- a/dbms/src/Storages/StorageSet.h
+++ b/dbms/src/Storages/StorageSet.h
@@ -19,7 +19,7 @@ class StorageSetOrJoinBase : public IStorage
     friend class SetOrJoinBlockOutputStream;
 
 public:
-    String getTableName() const override { return name; }
+    String getTableName() const override { return table_name; }
 
     void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override;
 
@@ -30,14 +30,11 @@ public:
 protected:
     StorageSetOrJoinBase(
         const String & path_,
-        const String & name_,
-        const NamesAndTypesList & columns_,
-        const NamesAndTypesList & materialized_columns_,
-        const NamesAndTypesList & alias_columns_,
-        const ColumnDefaults & column_defaults_);
+        const String & table_name_,
+        const ColumnsDescription & columns_);
 
     String path;
-    String name;
+    String table_name;
 
     UInt64 increment = 0;    /// For the backup file names.
 
@@ -78,10 +75,7 @@ protected:
     StorageSet(
         const String & path_,
         const String & name_,
-        const NamesAndTypesList & columns_,
-        const NamesAndTypesList & materialized_columns_,
-        const NamesAndTypesList & alias_columns_,
-        const ColumnDefaults & column_defaults_);
+        const ColumnsDescription & columns_);
 };
 
 }
diff --git a/dbms/src/Storages/StorageStripeLog.cpp b/dbms/src/Storages/StorageStripeLog.cpp
index 5f746dc246c..1bebca62322 100644
--- a/dbms/src/Storages/StorageStripeLog.cpp
+++ b/dbms/src/Storages/StorageStripeLog.cpp
@@ -193,13 +193,10 @@ private:
 StorageStripeLog::StorageStripeLog(
     const std::string & path_,
     const std::string & name_,
-    const NamesAndTypesList & columns_,
-    const NamesAndTypesList & materialized_columns_,
-    const NamesAndTypesList & alias_columns_,
-    const ColumnDefaults & column_defaults_,
+    const ColumnsDescription & columns_,
     bool attach,
     size_t max_compress_block_size_)
-    : IStorage{columns_, materialized_columns_, alias_columns_, column_defaults_},
+    : IStorage{columns_},
     path(path_), name(name_),
     max_compress_block_size(max_compress_block_size_),
     file_checker(path + escapeForFileName(name) + '/' + "sizes.json"),
@@ -301,7 +298,6 @@ void registerStorageStripeLog(StorageFactory & factory)
 
         return StorageStripeLog::create(
             args.data_path, args.table_name, args.columns,
-            args.materialized_columns, args.alias_columns, args.column_defaults,
             args.attach, args.context.getSettings().max_compress_block_size);
     });
 }
diff --git a/dbms/src/Storages/StorageStripeLog.h b/dbms/src/Storages/StorageStripeLog.h
index e1514f8329f..115f070f8d0 100644
--- a/dbms/src/Storages/StorageStripeLog.h
+++ b/dbms/src/Storages/StorageStripeLog.h
@@ -68,10 +68,7 @@ protected:
     StorageStripeLog(
         const std::string & path_,
         const std::string & name_,
-        const NamesAndTypesList & columns_,
-        const NamesAndTypesList & materialized_columns_,
-        const NamesAndTypesList & alias_columns_,
-        const ColumnDefaults & column_defaults_,
+        const ColumnsDescription & columns_,
         bool attach,
         size_t max_compress_block_size_);
 };
diff --git a/dbms/src/Storages/StorageTinyLog.cpp b/dbms/src/Storages/StorageTinyLog.cpp
index 4ed7d6dfff2..039f470c366 100644
--- a/dbms/src/Storages/StorageTinyLog.cpp
+++ b/dbms/src/Storages/StorageTinyLog.cpp
@@ -276,13 +276,10 @@ void TinyLogBlockOutputStream::write(const Block & block)
 StorageTinyLog::StorageTinyLog(
     const std::string & path_,
     const std::string & name_,
-    const NamesAndTypesList & columns_,
-    const NamesAndTypesList & materialized_columns_,
-    const NamesAndTypesList & alias_columns_,
-    const ColumnDefaults & column_defaults_,
+    const ColumnsDescription & columns_,
     bool attach,
     size_t max_compress_block_size_)
-    : IStorage{columns_, materialized_columns_, alias_columns_, column_defaults_},
+    : IStorage{columns_},
     path(path_), name(name_),
     max_compress_block_size(max_compress_block_size_),
     file_checker(path + escapeForFileName(name) + '/' + "sizes.json"),
@@ -299,7 +296,7 @@ StorageTinyLog::StorageTinyLog(
             throwFromErrno("Cannot create directory " + full_path, ErrorCodes::CANNOT_CREATE_DIRECTORY);
     }
 
-    for (const auto & col : getColumnsList())
+    for (const auto & col : columns.getList())
         addFiles(col.name, *col.type);
 }
 
@@ -351,7 +348,7 @@ BlockInputStreams StorageTinyLog::read(
     check(column_names);
     processed_stage = QueryProcessingStage::FetchColumns;
     return BlockInputStreams(1, std::make_shared<TinyLogBlockInputStream>(
-        max_block_size, Nested::collect(getColumnsList().addTypes(column_names)), *this, context.getSettingsRef().max_read_buffer_size));
+        max_block_size, Nested::collect(columns.getList().addTypes(column_names)), *this, context.getSettingsRef().max_read_buffer_size));
 }
 
 
@@ -379,7 +376,6 @@ void registerStorageTinyLog(StorageFactory & factory)
 
         return StorageTinyLog::create(
             args.data_path, args.table_name, args.columns,
-            args.materialized_columns, args.alias_columns, args.column_defaults,
             args.attach, args.context.getSettings().max_compress_block_size);
     });
 }
diff --git a/dbms/src/Storages/StorageTinyLog.h b/dbms/src/Storages/StorageTinyLog.h
index e64301227ad..acaf7944f30 100644
--- a/dbms/src/Storages/StorageTinyLog.h
+++ b/dbms/src/Storages/StorageTinyLog.h
@@ -71,10 +71,7 @@ protected:
     StorageTinyLog(
         const std::string & path_,
         const std::string & name_,
-        const NamesAndTypesList & columns_,
-        const NamesAndTypesList & materialized_columns_,
-        const NamesAndTypesList & alias_columns_,
-        const ColumnDefaults & column_defaults_,
+        const ColumnsDescription & columns_,
         bool attach,
         size_t max_compress_block_size_);
 };
diff --git a/dbms/src/Storages/StorageView.cpp b/dbms/src/Storages/StorageView.cpp
index 76cd481a855..06809ffb4e3 100644
--- a/dbms/src/Storages/StorageView.cpp
+++ b/dbms/src/Storages/StorageView.cpp
@@ -19,8 +19,8 @@ namespace ErrorCodes
 StorageView::StorageView(
     const String & table_name_,
     const ASTCreateQuery & query,
-    const NamesAndTypesList & columns_)
-    : IStorage{columns_, {}, {}, {}}, table_name(table_name_)
+    const ColumnsDescription & columns_)
+    : IStorage{columns_}, table_name(table_name_)
 {
     if (!query.select)
         throw Exception("SELECT query is not specified for " + getName(), ErrorCodes::INCORRECT_QUERY);
diff --git a/dbms/src/Storages/StorageView.h b/dbms/src/Storages/StorageView.h
index 6a16e0d16e6..20a8f76189c 100644
--- a/dbms/src/Storages/StorageView.h
+++ b/dbms/src/Storages/StorageView.h
@@ -42,7 +42,7 @@ protected:
     StorageView(
         const String & table_name_,
         const ASTCreateQuery & query,
-        const NamesAndTypesList & columns_);
+        const ColumnsDescription & columns_);
 };
 
 }
diff --git a/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.cpp b/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.cpp
index 44f8d933c99..f08808f799a 100644
--- a/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.cpp
+++ b/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.cpp
@@ -16,7 +16,7 @@ StorageSystemAsynchronousMetrics::StorageSystemAsynchronousMetrics(const std::st
     : name(name_),
     async_metrics(async_metrics_)
 {
-    columns = NamesAndTypesList{
+    columns.ordinary = NamesAndTypesList{
         {"metric", std::make_shared<DataTypeString>()},
         {"value", std::make_shared<DataTypeFloat64>()},
     };
diff --git a/dbms/src/Storages/System/StorageSystemBuildOptions.cpp b/dbms/src/Storages/System/StorageSystemBuildOptions.cpp
index c8da9f12c40..7a864090e59 100644
--- a/dbms/src/Storages/System/StorageSystemBuildOptions.cpp
+++ b/dbms/src/Storages/System/StorageSystemBuildOptions.cpp
@@ -13,7 +13,7 @@ namespace DB
 StorageSystemBuildOptions::StorageSystemBuildOptions(const std::string & name_)
     : name(name_)
 {
-    columns = NamesAndTypesList{
+    columns.ordinary = NamesAndTypesList{
         { "name", std::make_shared<DataTypeString>() },
         { "value", std::make_shared<DataTypeString>() },
     };
diff --git a/dbms/src/Storages/System/StorageSystemClusters.cpp b/dbms/src/Storages/System/StorageSystemClusters.cpp
index 084f295de74..8d772e5ebb8 100644
--- a/dbms/src/Storages/System/StorageSystemClusters.cpp
+++ b/dbms/src/Storages/System/StorageSystemClusters.cpp
@@ -14,7 +14,7 @@ namespace DB
 StorageSystemClusters::StorageSystemClusters(const std::string & name_)
     : name(name_)
 {
-    columns = NamesAndTypesList{
+    columns.ordinary = NamesAndTypesList{
         { "cluster",      std::make_shared<DataTypeString>() },
         { "shard_num",    std::make_shared<DataTypeUInt32>() },
         { "shard_weight", std::make_shared<DataTypeUInt32>() },
diff --git a/dbms/src/Storages/System/StorageSystemColumns.cpp b/dbms/src/Storages/System/StorageSystemColumns.cpp
index 5f9116b08a5..adec6a70c58 100644
--- a/dbms/src/Storages/System/StorageSystemColumns.cpp
+++ b/dbms/src/Storages/System/StorageSystemColumns.cpp
@@ -18,7 +18,7 @@ namespace DB
 StorageSystemColumns::StorageSystemColumns(const std::string & name_)
     : name(name_)
 {
-    columns = NamesAndTypesList{
+    columns.ordinary = NamesAndTypesList{
         { "database",           std::make_shared<DataTypeString>() },
         { "table",              std::make_shared<DataTypeString>() },
         { "name",               std::make_shared<DataTypeString>() },
@@ -137,9 +137,9 @@ BlockInputStreams StorageSystemColumns::read(
                     throw;
             }
 
-            columns = storage->getColumnsList();
-            columns.insert(std::end(columns), std::begin(storage->alias_columns), std::end(storage->alias_columns));
-            column_defaults = storage->column_defaults;
+            columns = storage->columns.getList();
+            columns.insert(std::end(columns), std::begin(storage->columns.aliases), std::end(storage->columns.aliases));
+            column_defaults = storage->columns.defaults;
 
             /** Info about sizes of columns for tables of MergeTree family.
               * NOTE: It is possible to add getter for this info to IStorage interface.
diff --git a/dbms/src/Storages/System/StorageSystemDatabases.cpp b/dbms/src/Storages/System/StorageSystemDatabases.cpp
index 93917b81000..12c35c7ec9e 100644
--- a/dbms/src/Storages/System/StorageSystemDatabases.cpp
+++ b/dbms/src/Storages/System/StorageSystemDatabases.cpp
@@ -13,7 +13,7 @@ namespace DB
 StorageSystemDatabases::StorageSystemDatabases(const std::string & name_)
     : name(name_)
 {
-    columns = NamesAndTypesList{
+    columns.ordinary = NamesAndTypesList{
         {"name", std::make_shared<DataTypeString>()},
         {"engine", std::make_shared<DataTypeString>()},
         {"data_path", std::make_shared<DataTypeString>()},
diff --git a/dbms/src/Storages/System/StorageSystemDictionaries.cpp b/dbms/src/Storages/System/StorageSystemDictionaries.cpp
index 93c742ef1a7..3cb4bd7962b 100644
--- a/dbms/src/Storages/System/StorageSystemDictionaries.cpp
+++ b/dbms/src/Storages/System/StorageSystemDictionaries.cpp
@@ -21,7 +21,7 @@ namespace DB
 StorageSystemDictionaries::StorageSystemDictionaries(const std::string & name)
     : name{name}
 {
-    columns = NamesAndTypesList{
+    columns.ordinary = NamesAndTypesList{
         { "name", std::make_shared<DataTypeString>() },
         { "origin", std::make_shared<DataTypeString>() },
         { "type", std::make_shared<DataTypeString>() },
diff --git a/dbms/src/Storages/System/StorageSystemEvents.cpp b/dbms/src/Storages/System/StorageSystemEvents.cpp
index 96a02d72332..0838eb739a5 100644
--- a/dbms/src/Storages/System/StorageSystemEvents.cpp
+++ b/dbms/src/Storages/System/StorageSystemEvents.cpp
@@ -13,7 +13,7 @@ namespace DB
 StorageSystemEvents::StorageSystemEvents(const std::string & name_)
     : name(name_)
 {
-    columns = NamesAndTypesList
+    columns.ordinary = NamesAndTypesList
     {
         {"event", std::make_shared<DataTypeString>()},
         {"value", std::make_shared<DataTypeUInt64>()}
diff --git a/dbms/src/Storages/System/StorageSystemFunctions.cpp b/dbms/src/Storages/System/StorageSystemFunctions.cpp
index 74e58338d40..4d013eb2268 100644
--- a/dbms/src/Storages/System/StorageSystemFunctions.cpp
+++ b/dbms/src/Storages/System/StorageSystemFunctions.cpp
@@ -16,7 +16,7 @@ namespace DB
 StorageSystemFunctions::StorageSystemFunctions(const std::string & name_)
     : name(name_)
 {
-    columns = NamesAndTypesList{
+    columns.ordinary = NamesAndTypesList{
         { "name",         std::make_shared<DataTypeString>() },
         { "is_aggregate", std::make_shared<DataTypeUInt8>()  }
     };
diff --git a/dbms/src/Storages/System/StorageSystemGraphite.cpp b/dbms/src/Storages/System/StorageSystemGraphite.cpp
index 76f3defe885..e99c037ea78 100644
--- a/dbms/src/Storages/System/StorageSystemGraphite.cpp
+++ b/dbms/src/Storages/System/StorageSystemGraphite.cpp
@@ -125,7 +125,7 @@ static Strings getAllGraphiteSections(const AbstractConfiguration & config)
 StorageSystemGraphite::StorageSystemGraphite(const std::string & name_)
     : name(name_)
 {
-    columns = NamesAndTypesList{
+    columns.ordinary = NamesAndTypesList{
         {"config_name", std::make_shared<DataTypeString>()},
         {"regexp",      std::make_shared<DataTypeString>()},
         {"function",    std::make_shared<DataTypeString>()},
diff --git a/dbms/src/Storages/System/StorageSystemMerges.cpp b/dbms/src/Storages/System/StorageSystemMerges.cpp
index f6667c19908..fea37df35a1 100644
--- a/dbms/src/Storages/System/StorageSystemMerges.cpp
+++ b/dbms/src/Storages/System/StorageSystemMerges.cpp
@@ -14,7 +14,7 @@ namespace DB
 StorageSystemMerges::StorageSystemMerges(const std::string & name)
     : name{name}
 {
-    columns = NamesAndTypesList{
+    columns.ordinary = NamesAndTypesList{
         { "database",                    std::make_shared<DataTypeString>() },
         { "table",                       std::make_shared<DataTypeString>() },
         { "elapsed",                     std::make_shared<DataTypeFloat64>() },
diff --git a/dbms/src/Storages/System/StorageSystemMetrics.cpp b/dbms/src/Storages/System/StorageSystemMetrics.cpp
index c60d7f4b4f5..54002318ef6 100644
--- a/dbms/src/Storages/System/StorageSystemMetrics.cpp
+++ b/dbms/src/Storages/System/StorageSystemMetrics.cpp
@@ -14,7 +14,7 @@ namespace DB
 StorageSystemMetrics::StorageSystemMetrics(const std::string & name_)
     : name(name_)
 {
-    columns = NamesAndTypesList{
+    columns.ordinary = NamesAndTypesList{
         {"metric", std::make_shared<DataTypeString>()},
         {"value",  std::make_shared<DataTypeInt64>()},
     };
diff --git a/dbms/src/Storages/System/StorageSystemModels.cpp b/dbms/src/Storages/System/StorageSystemModels.cpp
index da0249b1705..9057c6de626 100644
--- a/dbms/src/Storages/System/StorageSystemModels.cpp
+++ b/dbms/src/Storages/System/StorageSystemModels.cpp
@@ -14,7 +14,7 @@ namespace DB
 StorageSystemModels::StorageSystemModels(const std::string & name)
     : name{name}
 {
-    columns = NamesAndTypesList{
+    columns.ordinary = NamesAndTypesList{
         { "name", std::make_shared<DataTypeString>() },
         { "origin", std::make_shared<DataTypeString>() },
         { "type", std::make_shared<DataTypeString>() },
diff --git a/dbms/src/Storages/System/StorageSystemNumbers.cpp b/dbms/src/Storages/System/StorageSystemNumbers.cpp
index 30586bbbba3..803f651639d 100644
--- a/dbms/src/Storages/System/StorageSystemNumbers.cpp
+++ b/dbms/src/Storages/System/StorageSystemNumbers.cpp
@@ -47,7 +47,7 @@ private:
 StorageSystemNumbers::StorageSystemNumbers(const std::string & name_, bool multithreaded_, size_t limit_)
     : name(name_), multithreaded(multithreaded_), limit(limit_)
 {
-    columns = NamesAndTypesList{{"number", std::make_shared<DataTypeUInt64>()}};
+    columns.ordinary = NamesAndTypesList{{"number", std::make_shared<DataTypeUInt64>()}};
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemOne.cpp b/dbms/src/Storages/System/StorageSystemOne.cpp
index 4ddc64031e3..f35580c3882 100644
--- a/dbms/src/Storages/System/StorageSystemOne.cpp
+++ b/dbms/src/Storages/System/StorageSystemOne.cpp
@@ -13,7 +13,7 @@ namespace DB
 StorageSystemOne::StorageSystemOne(const std::string & name_)
     : name(name_)
 {
-    columns = NamesAndTypesList{{"dummy", std::make_shared<DataTypeUInt8>()}};
+    columns.ordinary = NamesAndTypesList{{"dummy", std::make_shared<DataTypeUInt8>()}};
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemPartsBase.cpp b/dbms/src/Storages/System/StorageSystemPartsBase.cpp
index f1f99c312f4..c6dc2a2c551 100644
--- a/dbms/src/Storages/System/StorageSystemPartsBase.cpp
+++ b/dbms/src/Storages/System/StorageSystemPartsBase.cpp
@@ -245,20 +245,20 @@ BlockInputStreams StorageSystemPartsBase::read(
 
     /// Create the result.
 
-    MutableColumns columns = getSampleBlock().cloneEmptyColumns();
+    MutableColumns res_columns = getSampleBlock().cloneEmptyColumns();
     if (has_state_column)
-        columns.push_back(ColumnString::create());
+        res_columns.push_back(ColumnString::create());
 
     while (StoragesInfo info = stream.next())
     {
-        processNextStorage(columns, info, has_state_column);
+        processNextStorage(res_columns, info, has_state_column);
     }
 
     Block block = getSampleBlock();
     if (has_state_column)
         block.insert(ColumnWithTypeAndName(std::make_shared<DataTypeString>(), "_state"));
 
-    return BlockInputStreams(1, std::make_shared<OneBlockInputStream>(block.cloneWithColumns(std::move(columns))));
+    return BlockInputStreams(1, std::make_shared<OneBlockInputStream>(block.cloneWithColumns(std::move(res_columns))));
 }
 
 NameAndTypePair StorageSystemPartsBase::getColumn(const String & column_name) const
@@ -280,7 +280,7 @@ bool StorageSystemPartsBase::hasColumn(const String & column_name) const
 StorageSystemPartsBase::StorageSystemPartsBase(std::string name_, NamesAndTypesList && columns_)
     : name(std::move(name_))
 {
-    columns = columns_;
+    columns.ordinary = columns_;
 }
 
 }
diff --git a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp
index 9667399c534..5f0a584c1d6 100644
--- a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp
+++ b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp
@@ -61,9 +61,9 @@ void StorageSystemPartsColumns::processNextStorage(MutableColumns & columns, con
         String default_expression;
     };
 
-    NamesAndTypesList columns_list = info.storage->getColumnsList();
-    columns_list.insert(std::end(columns_list), std::begin(info.storage->alias_columns), std::end(info.storage->alias_columns));
-    const auto & column_defaults = info.storage->column_defaults;
+    NamesAndTypesList columns_list = info.storage->columns.getList();
+    columns_list.insert(std::end(columns_list), std::begin(info.storage->columns.aliases), std::end(info.storage->columns.aliases));
+    const auto & column_defaults = info.storage->columns.defaults;
     std::unordered_map<String, ColumnInfo> columns_info;
 
     for (const auto & column : columns_list)
diff --git a/dbms/src/Storages/System/StorageSystemProcesses.cpp b/dbms/src/Storages/System/StorageSystemProcesses.cpp
index 55f3bb120db..e8262723d89 100644
--- a/dbms/src/Storages/System/StorageSystemProcesses.cpp
+++ b/dbms/src/Storages/System/StorageSystemProcesses.cpp
@@ -14,7 +14,7 @@ namespace DB
 StorageSystemProcesses::StorageSystemProcesses(const std::string & name_)
     : name(name_)
 {
-    columns = NamesAndTypesList{
+    columns.ordinary = NamesAndTypesList{
         { "is_initial_query",     std::make_shared<DataTypeUInt8>() },
 
         { "user",                 std::make_shared<DataTypeString>() },
diff --git a/dbms/src/Storages/System/StorageSystemReplicas.cpp b/dbms/src/Storages/System/StorageSystemReplicas.cpp
index 6dc11214b7c..ca240a80568 100644
--- a/dbms/src/Storages/System/StorageSystemReplicas.cpp
+++ b/dbms/src/Storages/System/StorageSystemReplicas.cpp
@@ -17,7 +17,7 @@ namespace DB
 StorageSystemReplicas::StorageSystemReplicas(const std::string & name_)
     : name(name_)
 {
-    columns = NamesAndTypesList{
+    columns.ordinary = NamesAndTypesList{
         { "database",                             std::make_shared<DataTypeString>()   },
         { "table",                                std::make_shared<DataTypeString>()   },
         { "engine",                               std::make_shared<DataTypeString>()   },
diff --git a/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp b/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp
index a2432525044..363306437bb 100644
--- a/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp
+++ b/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp
@@ -20,7 +20,7 @@ namespace DB
 StorageSystemReplicationQueue::StorageSystemReplicationQueue(const std::string & name_)
     : name(name_)
 {
-    columns = NamesAndTypesList{
+    columns.ordinary = NamesAndTypesList{
         /// Table properties.
         { "database",                std::make_shared<DataTypeString>() },
         { "table",                   std::make_shared<DataTypeString>() },
diff --git a/dbms/src/Storages/System/StorageSystemSettings.cpp b/dbms/src/Storages/System/StorageSystemSettings.cpp
index 6099883eeb6..3848583688b 100644
--- a/dbms/src/Storages/System/StorageSystemSettings.cpp
+++ b/dbms/src/Storages/System/StorageSystemSettings.cpp
@@ -14,7 +14,7 @@ namespace DB
 StorageSystemSettings::StorageSystemSettings(const std::string & name_)
     : name(name_)
 {
-    columns = NamesAndTypesList{
+    columns.ordinary = NamesAndTypesList{
         { "name",        std::make_shared<DataTypeString>() },
         { "value",       std::make_shared<DataTypeString>() },
         { "changed",     std::make_shared<DataTypeUInt8>() },
diff --git a/dbms/src/Storages/System/StorageSystemTables.cpp b/dbms/src/Storages/System/StorageSystemTables.cpp
index 852392a63d1..91ab3581bc8 100644
--- a/dbms/src/Storages/System/StorageSystemTables.cpp
+++ b/dbms/src/Storages/System/StorageSystemTables.cpp
@@ -114,7 +114,7 @@ protected:
 StorageSystemTables::StorageSystemTables(const std::string & name_)
     : name(name_)
 {
-    columns = NamesAndTypesList
+    columns.ordinary = NamesAndTypesList
     {
         {"database", std::make_shared<DataTypeString>()},
         {"name", std::make_shared<DataTypeString>()},
diff --git a/dbms/src/Storages/System/StorageSystemZooKeeper.cpp b/dbms/src/Storages/System/StorageSystemZooKeeper.cpp
index 80ca3bc2f0e..9aae0ce5b3b 100644
--- a/dbms/src/Storages/System/StorageSystemZooKeeper.cpp
+++ b/dbms/src/Storages/System/StorageSystemZooKeeper.cpp
@@ -22,7 +22,7 @@ namespace DB
 StorageSystemZooKeeper::StorageSystemZooKeeper(const std::string & name_)
     : name(name_)
 {
-    columns = NamesAndTypesList{
+    columns.ordinary = NamesAndTypesList{
         { "name",           std::make_shared<DataTypeString>() },
         { "value",          std::make_shared<DataTypeString>() },
         { "czxid",          std::make_shared<DataTypeInt64>() },
diff --git a/dbms/src/Storages/tests/storage_log.cpp b/dbms/src/Storages/tests/storage_log.cpp
index 6d9cb5d0def..da53fcc356d 100644
--- a/dbms/src/Storages/tests/storage_log.cpp
+++ b/dbms/src/Storages/tests/storage_log.cpp
@@ -36,7 +36,7 @@ try
         {
             ColumnWithTypeAndName column;
             column.name = "a";
-            column.type = table->getDataTypeByName("a");
+            column.type = table->columns.getColumn("a").type;
             auto col = column.type->createColumn();
             ColumnUInt64::Container & vec = typeid_cast<ColumnUInt64 &>(*col).getData();
 
@@ -51,7 +51,7 @@ try
         {
             ColumnWithTypeAndName column;
             column.name = "b";
-            column.type = table->getDataTypeByName("b");
+            column.type = table->columns.getColumn("b").type;
             auto col = column.type->createColumn();
             ColumnUInt8::Container & vec = typeid_cast<ColumnUInt8 &>(*col).getData();
 
diff --git a/dbms/src/TableFunctions/TableFunctionMerge.cpp b/dbms/src/TableFunctions/TableFunctionMerge.cpp
index d37d2002d5a..b471969b847 100644
--- a/dbms/src/TableFunctions/TableFunctionMerge.cpp
+++ b/dbms/src/TableFunctions/TableFunctionMerge.cpp
@@ -51,7 +51,7 @@ static NamesAndTypesList chooseColumns(const String & source_database, const Str
         throw Exception("Error while executing table function merge. In database " + source_database + " no one matches regular expression: "
             + table_name_regexp_, ErrorCodes::UNKNOWN_TABLE);
 
-    return any_table->getColumnsList();
+    return any_table->columns.getList();
 }
 
 
@@ -79,10 +79,12 @@ StoragePtr TableFunctionMerge::executeImpl(const ASTPtr & ast_function, const Co
 
     auto res = StorageMerge::create(
         getName(),
-        chooseColumns(source_database, table_name_regexp, context),
-        NamesAndTypesList{},
-        NamesAndTypesList{},
-        ColumnDefaults{},
+        ColumnsDescription{
+            chooseColumns(source_database, table_name_regexp, context),
+            NamesAndTypesList{},
+            NamesAndTypesList{},
+            ColumnDefaults{},
+        },
         source_database,
         table_name_regexp,
         context);
diff --git a/dbms/src/TableFunctions/TableFunctionMySQL.cpp b/dbms/src/TableFunctions/TableFunctionMySQL.cpp
index f1b4e911385..1ced8800255 100644
--- a/dbms/src/TableFunctions/TableFunctionMySQL.cpp
+++ b/dbms/src/TableFunctions/TableFunctionMySQL.cpp
@@ -152,7 +152,7 @@ StoragePtr TableFunctionMySQL::executeImpl(const ASTPtr & ast_function, const Co
         std::move(pool),
         database_name,
         table_name,
-        columns);
+        ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}});
 
     res->startup();
     return res;
diff --git a/dbms/src/TableFunctions/TableFunctionODBC.cpp b/dbms/src/TableFunctions/TableFunctionODBC.cpp
index cc4c430ae8f..cacdbe29670 100644
--- a/dbms/src/TableFunctions/TableFunctionODBC.cpp
+++ b/dbms/src/TableFunctions/TableFunctionODBC.cpp
@@ -113,7 +113,8 @@ StoragePtr TableFunctionODBC::executeImpl(const ASTPtr & ast_function, const Con
         columns.emplace_back(reinterpret_cast<char *>(column_name), getDataType(type));
     }
 
-    auto result = StorageODBC::create(table_name, connection_string, "", table_name, columns);
+    auto result = StorageODBC::create(table_name, connection_string, "", table_name,
+        ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}});
     result->startup();
     return result;
 }
diff --git a/dbms/src/TableFunctions/getStructureOfRemoteTable.cpp b/dbms/src/TableFunctions/getStructureOfRemoteTable.cpp
index 3cb68d25416..181448714ba 100644
--- a/dbms/src/TableFunctions/getStructureOfRemoteTable.cpp
+++ b/dbms/src/TableFunctions/getStructureOfRemoteTable.cpp
@@ -32,7 +32,7 @@ NamesAndTypesList getStructureOfRemoteTable(
     const auto & shard_info = cluster.getAnyShardInfo();
 
     if (shard_info.isLocal())
-        return context.getTable(database, table)->getColumnsList();
+        return context.getTable(database, table)->columns.getList();
 
     auto input = std::make_shared<RemoteBlockInputStream>(shard_info.pool, query, InterpreterDescribeQuery::getSampleBlock(), context);
     input->setPoolMode(PoolMode::GET_ONE);

From 972db7b0723db78f45554acd2cc5f7d195ae7c70 Mon Sep 17 00:00:00 2001
From: Alexey Zatelepin <ztlpn@yandex-team.ru>
Date: Sat, 10 Mar 2018 23:37:26 +0300
Subject: [PATCH 031/244] iterate over materialized columns where it makes
 sense

---
 dbms/src/Storages/MergeTree/MergeTreeData.cpp | 4 ++--
 dbms/src/Storages/StorageDistributed.cpp      | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp
index 7933a9454c7..ea36f75a96c 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp
@@ -903,7 +903,7 @@ void MergeTreeData::checkAlter(const AlterCommands & commands)
         columns_alter_forbidden.insert(merging_params.sign_column);
 
     std::map<String, const IDataType *> old_types;
-    for (const auto & column : columns.ordinary)
+    for (const auto & column : columns.getList())
         old_types.emplace(column.name, column.type.get());
 
     for (const AlterCommand & command : commands)
@@ -1868,7 +1868,7 @@ void MergeTreeData::removePartContributionToColumnSizes(const DataPartPtr & part
     const auto & files = part->checksums.files;
 
     /// TODO This method doesn't take into account columns with multiple files.
-    for (const auto & column : columns.ordinary)
+    for (const auto & column : columns.getList())
     {
         const auto escaped_name = escapeForFileName(column.name);
         const auto bin_file_name = escaped_name + ".bin";
diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp
index 1b7c5fdf69a..b543e802ef9 100644
--- a/dbms/src/Storages/StorageDistributed.cpp
+++ b/dbms/src/Storages/StorageDistributed.cpp
@@ -142,7 +142,7 @@ StorageDistributed::StorageDistributed(
     table_name(table_name_),
     remote_database(remote_database_), remote_table(remote_table_),
     context(context_), cluster_name(context.getMacros().expand(cluster_name_)), has_sharding_key(sharding_key_),
-    sharding_key_expr(sharding_key_ ? ExpressionAnalyzer(sharding_key_, context, nullptr, columns.ordinary).getActions(false) : nullptr),
+    sharding_key_expr(sharding_key_ ? ExpressionAnalyzer(sharding_key_, context, nullptr, columns.getList()).getActions(false) : nullptr),
     sharding_key_column_name(sharding_key_ ? sharding_key_->getColumnName() : String{}),
     path(data_path_.empty() ? "" : (data_path_ + escapeForFileName(table_name) + '/'))
 {
@@ -385,7 +385,7 @@ void registerStorageDistributed(StorageFactory & factory)
         /// Check that sharding_key exists in the table and has numeric type.
         if (sharding_key)
         {
-            auto sharding_expr = ExpressionAnalyzer(sharding_key, args.context, nullptr, args.columns.ordinary).getActions(true);
+            auto sharding_expr = ExpressionAnalyzer(sharding_key, args.context, nullptr, args.columns.getList()).getActions(true);
             const Block & block = sharding_expr->getSampleBlock();
 
             if (block.columns() != 1)

From 78d92f67945db374033f13214ed3c62b27ac07d7 Mon Sep 17 00:00:00 2001
From: Alexey Zatelepin <ztlpn@yandex-team.ru>
Date: Sat, 10 Mar 2018 20:03:57 +0300
Subject: [PATCH 032/244] add some useful methods, rename for clarity
 [#CLICKHOUSE-3241]

---
 dbms/src/Analyzers/ExecuteTableFunctions.cpp  |  2 +-
 dbms/src/Common/ExternalTable.h               |  4 +--
 dbms/src/Core/NamesAndTypes.cpp               | 10 ++++++
 dbms/src/Core/NamesAndTypes.h                 |  2 ++
 dbms/src/Core/iostream_debug_helpers.cpp      |  2 +-
 .../tests/filter_stream_hitlog.cpp            |  4 +--
 dbms/src/DataStreams/tests/native_streams.cpp |  4 +--
 dbms/src/DataStreams/tests/sorting_stream.cpp |  4 +--
 dbms/src/Databases/DatabaseDictionary.cpp     |  6 ++--
 dbms/src/Interpreters/ExpressionAnalyzer.cpp  | 10 +++---
 .../Interpreters/InterpreterCreateQuery.cpp   |  2 +-
 .../Interpreters/InterpreterDescribeQuery.cpp |  3 +-
 .../Interpreters/InterpreterInsertQuery.cpp   |  2 +-
 .../tests/in_join_subqueries_preprocessor.cpp |  2 --
 dbms/src/Storages/AlterCommands.cpp           | 13 ++------
 dbms/src/Storages/ColumnsDescription.cpp      |  8 ++++-
 dbms/src/Storages/ColumnsDescription.h        | 32 +++++++++++++------
 dbms/src/Storages/ITableDeclaration.cpp       |  8 ++---
 .../MergeTree/MergeTreeBlockInputStream.cpp   |  4 +--
 dbms/src/Storages/MergeTree/MergeTreeData.cpp | 20 ++++++------
 .../MergeTree/MergeTreeDataMerger.cpp         |  2 +-
 .../Storages/MergeTree/MergeTreeDataPart.cpp  |  6 ++--
 .../MergeTree/MergeTreeDataSelectExecutor.cpp |  4 +--
 .../MergeTree/MergeTreeDataWriter.cpp         |  2 +-
 .../Storages/MergeTree/MergeTreeReadPool.cpp  |  4 +--
 .../MergeTree/MergeTreeWhereOptimizer.cpp     |  4 +--
 .../ReplicatedMergeTreeAlterThread.cpp        |  4 +--
 dbms/src/Storages/StorageDistributed.cpp      |  6 ++--
 dbms/src/Storages/StorageFactory.cpp          |  4 +--
 dbms/src/Storages/StorageLog.cpp              |  4 +--
 dbms/src/Storages/StorageMerge.cpp            |  2 +-
 dbms/src/Storages/StorageMergeTree.cpp        |  4 +--
 .../Storages/StorageReplicatedMergeTree.cpp   |  2 +-
 dbms/src/Storages/StorageTinyLog.cpp          |  4 +--
 .../StorageSystemAsynchronousMetrics.cpp      |  4 +--
 .../System/StorageSystemBuildOptions.cpp      |  4 +--
 .../Storages/System/StorageSystemClusters.cpp |  6 ++--
 .../Storages/System/StorageSystemColumns.cpp  |  7 ++--
 .../System/StorageSystemDatabases.cpp         |  4 +--
 .../System/StorageSystemDictionaries.cpp      |  6 ++--
 .../Storages/System/StorageSystemEvents.cpp   |  6 ++--
 .../System/StorageSystemFunctions.cpp         |  6 ++--
 .../Storages/System/StorageSystemGraphite.cpp |  6 ++--
 .../Storages/System/StorageSystemMerges.cpp   |  4 +--
 .../Storages/System/StorageSystemMetrics.cpp  |  4 +--
 .../Storages/System/StorageSystemModels.cpp   |  6 ++--
 .../Storages/System/StorageSystemNumbers.cpp  |  2 +-
 dbms/src/Storages/System/StorageSystemOne.cpp |  2 +-
 .../System/StorageSystemPartsBase.cpp         |  2 +-
 .../System/StorageSystemPartsColumns.cpp      |  3 +-
 .../System/StorageSystemProcesses.cpp         |  6 ++--
 .../Storages/System/StorageSystemReplicas.cpp |  4 +--
 .../System/StorageSystemReplicationQueue.cpp  |  4 +--
 .../Storages/System/StorageSystemSettings.cpp |  6 ++--
 .../Storages/System/StorageSystemTables.cpp   |  4 +--
 .../System/StorageSystemZooKeeper.cpp         |  4 +--
 dbms/src/Storages/tests/hit_log.cpp           |  4 +--
 dbms/src/Storages/tests/storage_log.cpp       |  8 ++---
 .../src/TableFunctions/TableFunctionMerge.cpp |  9 ++----
 .../src/TableFunctions/TableFunctionMySQL.cpp |  2 +-
 dbms/src/TableFunctions/TableFunctionODBC.cpp |  3 +-
 .../getStructureOfRemoteTable.cpp             |  2 +-
 62 files changed, 162 insertions(+), 160 deletions(-)

diff --git a/dbms/src/Analyzers/ExecuteTableFunctions.cpp b/dbms/src/Analyzers/ExecuteTableFunctions.cpp
index 326c5df62ed..3feabc8f490 100644
--- a/dbms/src/Analyzers/ExecuteTableFunctions.cpp
+++ b/dbms/src/Analyzers/ExecuteTableFunctions.cpp
@@ -72,7 +72,7 @@ void ExecuteTableFunctions::dump(WriteBuffer & out) const
     {
         writeString(table.second->getName(), out);
         writeCString("\n\n", out);
-        writeString(table.second->columns.getList().toString(), out);
+        writeString(table.second->columns.getPhysical().toString(), out);
         writeCString("\n", out);
     }
 }
diff --git a/dbms/src/Common/ExternalTable.h b/dbms/src/Common/ExternalTable.h
index b5b0c522339..7cc8069108b 100644
--- a/dbms/src/Common/ExternalTable.h
+++ b/dbms/src/Common/ExternalTable.h
@@ -198,9 +198,7 @@ public:
 
         /// Create table
         NamesAndTypesList columns = sample_block.getNamesAndTypesList();
-        StoragePtr storage = StorageMemory::create(
-            data.second,
-            ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}});
+        StoragePtr storage = StorageMemory::create(data.second, ColumnsDescription{columns});
         storage->startup();
         context.addExternalTable(data.second, storage);
         BlockOutputStreamPtr output = storage->write(ASTPtr(), context.getSettingsRef());
diff --git a/dbms/src/Core/NamesAndTypes.cpp b/dbms/src/Core/NamesAndTypes.cpp
index 14faaa133ea..a5240ba4a91 100644
--- a/dbms/src/Core/NamesAndTypes.cpp
+++ b/dbms/src/Core/NamesAndTypes.cpp
@@ -138,4 +138,14 @@ NamesAndTypesList NamesAndTypesList::addTypes(const Names & names) const
     return res;
 }
 
+bool NamesAndTypesList::contains(const String & name) const
+{
+    for (const NameAndTypePair & column : *this)
+    {
+        if (column.name == name)
+            return true;
+    }
+    return false;
+}
+
 }
diff --git a/dbms/src/Core/NamesAndTypes.h b/dbms/src/Core/NamesAndTypes.h
index 7aa9f3b3ff4..849141e43f7 100644
--- a/dbms/src/Core/NamesAndTypes.h
+++ b/dbms/src/Core/NamesAndTypes.h
@@ -69,6 +69,8 @@ public:
 
     /// Unlike `filter`, returns columns in the order in which they go in `names`.
     NamesAndTypesList addTypes(const Names & names) const;
+
+    bool contains(const String & name) const;
 };
 
 }
diff --git a/dbms/src/Core/iostream_debug_helpers.cpp b/dbms/src/Core/iostream_debug_helpers.cpp
index 822bbf58376..723b3dc2e90 100644
--- a/dbms/src/Core/iostream_debug_helpers.cpp
+++ b/dbms/src/Core/iostream_debug_helpers.cpp
@@ -44,7 +44,7 @@ std::ostream & operator<<(std::ostream & stream, const IDataType & what)
 std::ostream & operator<<(std::ostream & stream, const IStorage & what)
 {
     stream << "IStorage(name = " << what.getName() << ", tableName = " << what.getTableName() << ") {"
-           << what.columns.getList().toString()
+           << what.columns.getPhysical().toString()
            << "}";
     // isRemote supportsSampling supportsFinal supportsPrewhere
     return stream;
diff --git a/dbms/src/DataStreams/tests/filter_stream_hitlog.cpp b/dbms/src/DataStreams/tests/filter_stream_hitlog.cpp
index fe9ce592a43..2eb65c30df3 100644
--- a/dbms/src/DataStreams/tests/filter_stream_hitlog.cpp
+++ b/dbms/src/DataStreams/tests/filter_stream_hitlog.cpp
@@ -104,8 +104,8 @@ int main(int, char **)
 
         /// create an object of an existing hit log table
 
-        StoragePtr table = StorageLog::create("./", "HitLog", names_and_types_list,
-            NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}, DEFAULT_MAX_COMPRESS_BLOCK_SIZE);
+        StoragePtr table = StorageLog::create(
+            "./", "HitLog", ColumnsDescription{names_and_types_list}, DEFAULT_MAX_COMPRESS_BLOCK_SIZE);
         table->startup();
 
         /// read from it, apply the expression, filter, and write in tsv form to the console
diff --git a/dbms/src/DataStreams/tests/native_streams.cpp b/dbms/src/DataStreams/tests/native_streams.cpp
index bd0a998f88e..5d02d491ec4 100644
--- a/dbms/src/DataStreams/tests/native_streams.cpp
+++ b/dbms/src/DataStreams/tests/native_streams.cpp
@@ -95,8 +95,8 @@ try
 
     /// create an object of an existing hit log table
 
-    StoragePtr table = StorageLog::create("./", "HitLog", names_and_types_list,
-        NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}, DEFAULT_MAX_COMPRESS_BLOCK_SIZE);
+    StoragePtr table = StorageLog::create(
+        "./", "HitLog", ColumnsDescription{names_and_types_list}, DEFAULT_MAX_COMPRESS_BLOCK_SIZE);
     table->startup();
 
     /// read from it
diff --git a/dbms/src/DataStreams/tests/sorting_stream.cpp b/dbms/src/DataStreams/tests/sorting_stream.cpp
index cd6fe515a53..0cfffec295f 100644
--- a/dbms/src/DataStreams/tests/sorting_stream.cpp
+++ b/dbms/src/DataStreams/tests/sorting_stream.cpp
@@ -107,8 +107,8 @@ try
 
     /// create an object of an existing hit log table
 
-    StoragePtr table = StorageLog::create("./", "HitLog", names_and_types_list,
-        NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}, DEFAULT_MAX_COMPRESS_BLOCK_SIZE);
+    StoragePtr table = StorageLog::create(
+        "./", "HitLog", ColumnsDescription{names_and_types_list}, DEFAULT_MAX_COMPRESS_BLOCK_SIZE);
     table->startup();
 
     /// read from it, sort it, and write it in tsv form to the console
diff --git a/dbms/src/Databases/DatabaseDictionary.cpp b/dbms/src/Databases/DatabaseDictionary.cpp
index 439fe8f4d79..503d0195168 100644
--- a/dbms/src/Databases/DatabaseDictionary.cpp
+++ b/dbms/src/Databases/DatabaseDictionary.cpp
@@ -41,8 +41,7 @@ Tables DatabaseDictionary::loadTables()
         {
             const DictionaryStructure & dictionary_structure = dict_ptr->getStructure();
             auto columns = StorageDictionary::getNamesAndTypes(dictionary_structure);
-            tables[name] = StorageDictionary::create(name,
-                ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}}, dictionary_structure, name);
+            tables[name] = StorageDictionary::create(name, ColumnsDescription{columns}, dictionary_structure, name);
         }
     }
 
@@ -76,8 +75,7 @@ StoragePtr DatabaseDictionary::tryGetTable(
             {
                 const DictionaryStructure & dictionary_structure = dict_ptr->getStructure();
                 auto columns = StorageDictionary::getNamesAndTypes(dictionary_structure);
-                return StorageDictionary::create(table_name,
-                    ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}}, dictionary_structure, table_name);
+                return StorageDictionary::create(table_name, ColumnsDescription{columns}, dictionary_structure, table_name);
             }
         }
     }
diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp
index fe85e154e92..e6190b8f1df 100644
--- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp
+++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp
@@ -190,7 +190,7 @@ ExpressionAnalyzer::ExpressionAnalyzer(
     }
 
     if (storage && source_columns.empty())
-        source_columns = storage->columns.getList();
+        source_columns = storage->columns.getPhysical();
     else
         removeDuplicateColumns(source_columns);
 
@@ -712,7 +712,7 @@ static std::shared_ptr<InterpreterSelectWithUnionQuery> interpretSubquery(
         /// get columns list for target table
         auto database_table = getDatabaseAndTableNameFromIdentifier(*table);
         const auto & storage = context.getTable(database_table.first, database_table.second);
-        const auto & columns = storage->columns.getListNonMaterialized();
+        const auto & columns = storage->columns.ordinary;
         select_expression_list->children.reserve(columns.size());
 
         /// manually substitute column names in place of asterisk
@@ -826,9 +826,7 @@ void ExpressionAnalyzer::addExternalStorage(ASTPtr & subquery_or_table_name_or_t
     Block sample = interpreter->getSampleBlock();
     NamesAndTypesList columns = sample.getNamesAndTypesList();
 
-    StoragePtr external_storage = StorageMemory::create(
-        external_table_name,
-        ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}});
+    StoragePtr external_storage = StorageMemory::create(external_table_name, ColumnsDescription{columns});
     external_storage->startup();
 
     /** We replace the subquery with the name of the temporary table.
@@ -1052,7 +1050,7 @@ void ExpressionAnalyzer::normalizeTreeImpl(
                 if (storage)
                 {
                     /// If we select from a table, get only not MATERIALIZED, not ALIAS columns.
-                    for (const auto & name_type : storage->columns.getListNonMaterialized())
+                    for (const auto & name_type : storage->columns.ordinary)
                         all_columns.emplace_back(std::make_shared<ASTIdentifier>(name_type.name));
                 }
                 else
diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp
index 6033947b474..e5d20648fbd 100644
--- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp
@@ -305,7 +305,7 @@ ASTPtr InterpreterCreateQuery::formatColumns(const ColumnsDescription & columns)
 {
     auto columns_list = std::make_shared<ASTExpressionList>();
 
-    for (const auto & column : boost::join(columns.ordinary, boost::join(columns.materialized, columns.aliases)))
+    for (const auto & column : columns.getAll())
     {
         const auto column_declaration = std::make_shared<ASTColumnDeclaration>();
         ASTPtr column_declaration_ptr{column_declaration};
diff --git a/dbms/src/Interpreters/InterpreterDescribeQuery.cpp b/dbms/src/Interpreters/InterpreterDescribeQuery.cpp
index b0d7e4c0363..c03034af49a 100644
--- a/dbms/src/Interpreters/InterpreterDescribeQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterDescribeQuery.cpp
@@ -99,8 +99,7 @@ BlockInputStreamPtr InterpreterDescribeQuery::executeImpl()
         }
 
         auto table_lock = table->lockStructure(false, __PRETTY_FUNCTION__);
-        columns = table->columns.getList();
-        columns.insert(std::end(columns), std::begin(table->columns.aliases), std::end(table->columns.aliases));
+        columns = table->columns.getAll();
         column_defaults = table->columns.defaults;
     }
 
diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp
index 5f0b8201ac3..973fdc31a27 100644
--- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp
@@ -95,7 +95,7 @@ BlockIO InterpreterInsertQuery::execute()
 
     auto table_lock = table->lockStructure(true, __PRETTY_FUNCTION__);
 
-    NamesAndTypesList required_columns = table->columns.getList();
+    NamesAndTypesList required_columns = table->columns.getPhysical();
 
     /// We create a pipeline of several streams, into which we will write data.
     BlockOutputStreamPtr out;
diff --git a/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp b/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp
index 0a67da3a5d8..e5c04025c59 100644
--- a/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp
+++ b/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp
@@ -38,7 +38,6 @@ public:
     std::string getRemoteTableName() const { return remote_table; }
 
     std::string getTableName() const override { return ""; }
-    const DB::NamesAndTypesList & getColumnsListImpl() const override { return names_and_types; }
 
 protected:
     StorageDistributedFake(const std::string & remote_database_, const std::string & remote_table_, size_t shard_count_)
@@ -50,7 +49,6 @@ private:
     const std::string remote_database;
     const std::string remote_table;
     size_t shard_count;
-    DB::NamesAndTypesList names_and_types;
 };
 
 
diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp
index defad6f5cba..184f63c070f 100644
--- a/dbms/src/Storages/AlterCommands.cpp
+++ b/dbms/src/Storages/AlterCommands.cpp
@@ -25,15 +25,7 @@ void AlterCommand::apply(ColumnsDescription & columns_description) const
 {
     if (type == ADD_COLUMN)
     {
-        const auto exists_in = [this] (const NamesAndTypesList & columns)
-        {
-            return columns.end() != std::find_if(columns.begin(), columns.end(),
-                std::bind(namesEqual, std::cref(column_name), std::placeholders::_1));
-        };
-
-        if (exists_in(columns_description.ordinary) ||
-            exists_in(columns_description.materialized) ||
-            exists_in(columns_description.aliases))
+        if (columns_description.getAll().contains(column_name))
         {
             throw Exception{
                 "Cannot add column " + column_name + ": column with this name already exists",
@@ -185,8 +177,7 @@ void AlterCommands::apply(ColumnsDescription & columns_description) const
 
 void AlterCommands::validate(IStorage * table, const Context & context)
 {
-    auto all_columns = table->columns.getList();
-    all_columns.insert(std::end(all_columns), std::begin(table->columns.aliases), std::end(table->columns.aliases));
+    auto all_columns = table->columns.getAll();
     auto defaults = table->columns.defaults;
 
     std::vector<std::pair<NameAndTypePair, AlterCommand *>> defaulted_columns{};
diff --git a/dbms/src/Storages/ColumnsDescription.cpp b/dbms/src/Storages/ColumnsDescription.cpp
index b8fae53ed4e..95b5001a1f9 100644
--- a/dbms/src/Storages/ColumnsDescription.cpp
+++ b/dbms/src/Storages/ColumnsDescription.cpp
@@ -27,12 +27,18 @@ namespace ErrorCodes
 }
 
 
-NamesAndTypesList ColumnsDescription::getList() const
+NamesAndTypesList ColumnsDescription::getPhysical() const
 {
     return ext::collection_cast<NamesAndTypesList>(boost::join(ordinary, materialized));
 }
 
 
+NamesAndTypesList ColumnsDescription::getAll() const
+{
+    return ext::collection_cast<NamesAndTypesList>(boost::join(ordinary, boost::join(materialized, aliases)));
+}
+
+
 Names ColumnsDescription::getNames() const
 {
     return ext::map<Names>(boost::join(ordinary, materialized), [] (const auto & it) { return it.name; });
diff --git a/dbms/src/Storages/ColumnsDescription.h b/dbms/src/Storages/ColumnsDescription.h
index 888f0864701..e4e45ff6fe2 100644
--- a/dbms/src/Storages/ColumnsDescription.h
+++ b/dbms/src/Storages/ColumnsDescription.h
@@ -16,6 +16,20 @@ struct ColumnsDescription
     NamesAndTypesList aliases;
     ColumnDefaults defaults;
 
+    ColumnsDescription() = default;
+
+    ColumnsDescription(
+        NamesAndTypesList ordinary_,
+        NamesAndTypesList materialized_,
+        NamesAndTypesList aliases_,
+        ColumnDefaults defaults_)
+        : ordinary(std::move(ordinary_))
+        , materialized(std::move(materialized_))
+        , aliases(std::move(aliases_))
+        , defaults(std::move(defaults_))
+    {}
+
+    explicit ColumnsDescription(NamesAndTypesList ordinary_) : ordinary(std::move(ordinary_)) {}
 
     bool operator==(const ColumnsDescription & other) const
     {
@@ -27,21 +41,19 @@ struct ColumnsDescription
 
     bool operator!=(const ColumnsDescription & other) const { return !(*this == other); }
 
-    /** Get a list of names and table column types, only non-virtual.
-     */
-    NamesAndTypesList getList() const;
-    const NamesAndTypesList & getListNonMaterialized() const { return ordinary; }
+    /// ordinary + materialized.
+    NamesAndTypesList getPhysical() const;
 
-    /** Get a list of column names.
-     */
+    /// ordinary + materialized + aliases.
+    NamesAndTypesList getAll() const;
+
+    /// Get names of physical columns.
     Names getNames() const;
 
-    /** Get a description of any column by its name.
-     */
+    /// Get a physical column by name.
     NameAndTypePair get(const String & column_name) const;
 
-    /** Is there a column with that name.
-      */
+    /// Is there a physical column with the given name.
     bool has(const String & column_name) const;
 
 
diff --git a/dbms/src/Storages/ITableDeclaration.cpp b/dbms/src/Storages/ITableDeclaration.cpp
index d4bbb4b501e..b5a6c035b96 100644
--- a/dbms/src/Storages/ITableDeclaration.cpp
+++ b/dbms/src/Storages/ITableDeclaration.cpp
@@ -111,7 +111,7 @@ static NamesAndTypesMap getColumnsMap(const Args &... args)
 
 void ITableDeclaration::check(const Names & column_names) const
 {
-    const NamesAndTypesList & available_columns = columns.getList();
+    const NamesAndTypesList & available_columns = columns.getPhysical();
 
     if (column_names.empty())
         throw Exception("Empty list of columns queried. There are columns: " + listOfColumns(available_columns),
@@ -139,7 +139,7 @@ void ITableDeclaration::check(const Names & column_names) const
 
 void ITableDeclaration::check(const NamesAndTypesList & provided_columns) const
 {
-    const NamesAndTypesList & available_columns = columns.getList();
+    const NamesAndTypesList & available_columns = columns.getPhysical();
     const auto columns_map = getColumnsMap(available_columns);
 
     using UniqueStrings = google::dense_hash_set<StringRef, StringRefHash>;
@@ -167,7 +167,7 @@ void ITableDeclaration::check(const NamesAndTypesList & provided_columns) const
 
 void ITableDeclaration::check(const NamesAndTypesList & provided_columns, const Names & column_names) const
 {
-    const NamesAndTypesList & available_columns = columns.getList();
+    const NamesAndTypesList & available_columns = columns.getPhysical();
     const auto available_columns_map = getColumnsMap(available_columns);
     const NamesAndTypesMap & provided_columns_map = getColumnsMap(provided_columns);
 
@@ -204,7 +204,7 @@ void ITableDeclaration::check(const NamesAndTypesList & provided_columns, const
 
 void ITableDeclaration::check(const Block & block, bool need_all) const
 {
-    const NamesAndTypesList & available_columns = columns.getList();
+    const NamesAndTypesList & available_columns = columns.getPhysical();
     const auto columns_map = getColumnsMap(available_columns);
 
     using NameSet = std::unordered_set<String>;
diff --git a/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp
index 5a67714db64..6fb7c8fb611 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp
@@ -141,8 +141,8 @@ try
         if (!column_names.empty())
             storage.check(data_part->columns, column_names);
 
-        pre_columns = storage.columns.getList().addTypes(pre_column_names);
-        columns = storage.columns.getList().addTypes(column_names);
+        pre_columns = storage.columns.getPhysical().addTypes(pre_column_names);
+        columns = storage.columns.getPhysical().addTypes(column_names);
     }
     else
     {
diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp
index ea36f75a96c..de16c22593d 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp
@@ -110,7 +110,7 @@ MergeTreeData::MergeTreeData(
     data_parts_by_state_and_info(data_parts_indexes.get<TagByStateAndInfo>())
 {
     /// NOTE: using the same columns list as is read when performing actual merges.
-    merging_params.check(columns.getList());
+    merging_params.check(columns.getPhysical());
 
     if (!primary_expr_ast)
         throw Exception("Primary key cannot be empty", ErrorCodes::BAD_ARGUMENTS);
@@ -222,11 +222,11 @@ void MergeTreeData::initPrimaryKey()
     primary_sort_descr.clear();
     addSortDescription(primary_sort_descr, primary_expr_ast);
 
-    primary_expr = ExpressionAnalyzer(primary_expr_ast, context, nullptr, columns.getList()).getActions(false);
+    primary_expr = ExpressionAnalyzer(primary_expr_ast, context, nullptr, columns.getPhysical()).getActions(false);
 
     {
         ExpressionActionsPtr projected_expr =
-                ExpressionAnalyzer(primary_expr_ast, context, nullptr, columns.getList()).getActions(true);
+                ExpressionAnalyzer(primary_expr_ast, context, nullptr, columns.getPhysical()).getActions(true);
         primary_key_sample = projected_expr->getSampleBlock();
     }
 
@@ -241,10 +241,10 @@ void MergeTreeData::initPrimaryKey()
     if (secondary_sort_expr_ast)
     {
         addSortDescription(sort_descr, secondary_sort_expr_ast);
-        secondary_sort_expr = ExpressionAnalyzer(secondary_sort_expr_ast, context, nullptr, columns.getList()).getActions(false);
+        secondary_sort_expr = ExpressionAnalyzer(secondary_sort_expr_ast, context, nullptr, columns.getPhysical()).getActions(false);
 
         ExpressionActionsPtr projected_expr =
-                ExpressionAnalyzer(secondary_sort_expr_ast, context, nullptr, columns.getList()).getActions(true);
+                ExpressionAnalyzer(secondary_sort_expr_ast, context, nullptr, columns.getPhysical()).getActions(true);
         auto secondary_key_sample = projected_expr->getSampleBlock();
 
         checkKeyExpression(*secondary_sort_expr, secondary_key_sample, "Secondary");
@@ -257,7 +257,7 @@ void MergeTreeData::initPartitionKey()
     if (!partition_expr_ast || partition_expr_ast->children.empty())
         return;
 
-    partition_expr = ExpressionAnalyzer(partition_expr_ast, context, nullptr, columns.getList()).getActions(false);
+    partition_expr = ExpressionAnalyzer(partition_expr_ast, context, nullptr, columns.getPhysical()).getActions(false);
     for (const ASTPtr & ast : partition_expr_ast->children)
     {
         String col_name = ast->getColumnName();
@@ -903,7 +903,7 @@ void MergeTreeData::checkAlter(const AlterCommands & commands)
         columns_alter_forbidden.insert(merging_params.sign_column);
 
     std::map<String, const IDataType *> old_types;
-    for (const auto & column : columns.getList())
+    for (const auto & column : columns.getPhysical())
         old_types.emplace(column.name, column.type.get());
 
     for (const AlterCommand & command : commands)
@@ -931,7 +931,7 @@ void MergeTreeData::checkAlter(const AlterCommands & commands)
     NameToNameMap unused_map;
     bool unused_bool;
 
-    createConvertExpression(nullptr, columns.getList(), new_columns.getList(), unused_expression, unused_map, unused_bool);
+    createConvertExpression(nullptr, columns.getPhysical(), new_columns.getPhysical(), unused_expression, unused_map, unused_bool);
 }
 
 void MergeTreeData::createConvertExpression(const DataPartPtr & part, const NamesAndTypesList & old_columns, const NamesAndTypesList & new_columns,
@@ -1835,7 +1835,7 @@ void MergeTreeData::addPartContributionToColumnSizes(const DataPartPtr & part)
     const auto & files = part->checksums.files;
 
     /// TODO This method doesn't take into account columns with multiple files.
-    for (const auto & column : columns.getList())
+    for (const auto & column : columns.getPhysical())
     {
         const auto escaped_name = escapeForFileName(column.name);
         const auto bin_file_name = escaped_name + ".bin";
@@ -1868,7 +1868,7 @@ void MergeTreeData::removePartContributionToColumnSizes(const DataPartPtr & part
     const auto & files = part->checksums.files;
 
     /// TODO This method doesn't take into account columns with multiple files.
-    for (const auto & column : columns.getList())
+    for (const auto & column : columns.getPhysical())
     {
         const auto escaped_name = escapeForFileName(column.name);
         const auto bin_file_name = escaped_name + ".bin";
diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp
index 4e3e9788aaa..8e30320aefb 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp
@@ -537,7 +537,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMerger::mergePartsToTemporaryPart
         part->accumulateColumnSizes(merged_column_to_size);
 
     Names all_column_names = data.columns.getNames();
-    NamesAndTypesList all_columns = data.columns.getList();
+    NamesAndTypesList all_columns = data.columns.getPhysical();
     const SortDescription sort_desc = data.getSortDescription();
 
     NamesAndTypesList gathering_columns, merging_columns;
diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp
index 9c0269c7437..f37cd8e579c 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp
@@ -418,7 +418,7 @@ UInt64 MergeTreeDataPart::getColumnMrkSize(const String & name) const
   */
 String MergeTreeDataPart::getColumnNameWithMinumumCompressedSize() const
 {
-    const auto & columns = storage.columns.getList();
+    const auto & columns = storage.columns.getPhysical();
     const std::string * minimum_size_column = nullptr;
     UInt64 minimum_size = std::numeric_limits<UInt64>::max();
 
@@ -774,7 +774,7 @@ void MergeTreeDataPart::accumulateColumnSizes(ColumnToSize & column_to_size) con
 {
     std::shared_lock<std::shared_mutex> part_lock(columns_lock);
 
-    for (const NameAndTypePair & name_type : storage.columns.getList())
+    for (const NameAndTypePair & name_type : storage.columns.getPhysical())
     {
         name_type.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path)
         {
@@ -794,7 +794,7 @@ void MergeTreeDataPart::loadColumns(bool require)
             throw Exception("No columns.txt in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART);
 
         /// If there is no file with a list of columns, write it down.
-        for (const NameAndTypePair & column : storage.columns.getList())
+        for (const NameAndTypePair & column : storage.columns.getPhysical())
             if (Poco::File(getFullPath() + escapeForFileName(column.name) + ".bin").exists())
                 columns.push_back(column);
 
diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
index 605091e6386..0890163078d 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
@@ -175,7 +175,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::read(
         }
     }
 
-    NamesAndTypesList available_real_columns = data.columns.getList();
+    NamesAndTypesList available_real_columns = data.columns.getPhysical();
 
     NamesAndTypesList available_real_and_virtual_columns = available_real_columns;
     for (const auto & name : virt_column_names)
@@ -857,7 +857,7 @@ void MergeTreeDataSelectExecutor::createPositiveSignCondition(
     arguments->children.push_back(sign);
     arguments->children.push_back(one);
 
-    out_expression = ExpressionAnalyzer(function, context, {}, data.columns.getList()).getActions(false);
+    out_expression = ExpressionAnalyzer(function, context, {}, data.columns.getPhysical()).getActions(false);
     out_column = function->getColumnName();
 }
 
diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp
index f4db6b934ba..19eb2e8eecd 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp
@@ -205,7 +205,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa
     ///  either default lz4 or compression method with zero thresholds on absolute and relative part size.
     auto compression_settings = data.context.chooseCompressionSettings(0, 0);
 
-    NamesAndTypesList columns = data.columns.getList().filter(block.getNames());
+    NamesAndTypesList columns = data.columns.getPhysical().filter(block.getNames());
     MergedBlockOutputStream out(data, new_data_part->getFullPath(), columns, compression_settings);
 
     out.writePrefix();
diff --git a/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp b/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp
index c32aa71a731..2c600dbd32d 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp
@@ -238,8 +238,8 @@ std::vector<size_t> MergeTreeReadPool::fillPerPartInfo(
             if (!required_column_names.empty())
                 data.check(part.data_part->columns, required_column_names);
 
-            per_part_pre_columns.push_back(data.columns.getList().addTypes(required_pre_column_names));
-            per_part_columns.push_back(data.columns.getList().addTypes(required_column_names));
+            per_part_pre_columns.push_back(data.columns.getPhysical().addTypes(required_pre_column_names));
+            per_part_columns.push_back(data.columns.getPhysical().addTypes(required_column_names));
         }
         else
         {
diff --git a/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp
index 005ca750226..ce4047fc85f 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp
@@ -41,9 +41,9 @@ MergeTreeWhereOptimizer::MergeTreeWhereOptimizer(
     Logger * log)
         : primary_key_columns{ext::map<std::unordered_set>(data.getPrimarySortDescription(),
             [] (const SortColumnDescription & col) { return col.column_name; })},
-        table_columns{ext::map<std::unordered_set>(data.columns.getList(),
+        table_columns{ext::map<std::unordered_set>(data.columns.getPhysical(),
             [] (const NameAndTypePair & col) { return col.name; })},
-        block_with_constants{PKCondition::getBlockWithConstants(query_info.query, context, data.columns.getList())},
+        block_with_constants{PKCondition::getBlockWithConstants(query_info.query, context, data.columns.getPhysical())},
         prepared_sets(query_info.sets),
         log{log}
 {
diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp
index c359a889bfb..948902d8302 100644
--- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp
+++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp
@@ -126,7 +126,7 @@ void ReplicatedMergeTreeAlterThread::run()
                     if (!changed_version)
                         parts = storage.data.getDataParts();
 
-                    const auto columns_plus_materialized = storage.data.columns.getList();
+                    const auto columns_for_parts = storage.data.columns.getPhysical();
 
                     for (const MergeTreeData::DataPartPtr & part : parts)
                     {
@@ -134,7 +134,7 @@ void ReplicatedMergeTreeAlterThread::run()
                         /// TODO: You can skip checking for too large changes if ZooKeeper has, for example,
                         /// node /flags/force_alter.
                         auto transaction = storage.data.alterDataPart(
-                            part, columns_plus_materialized, storage.data.primary_expr_ast, false);
+                            part, columns_for_parts, storage.data.primary_expr_ast, false);
 
                         if (!transaction)
                             continue;
diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp
index b543e802ef9..c010ca768d1 100644
--- a/dbms/src/Storages/StorageDistributed.cpp
+++ b/dbms/src/Storages/StorageDistributed.cpp
@@ -142,7 +142,7 @@ StorageDistributed::StorageDistributed(
     table_name(table_name_),
     remote_database(remote_database_), remote_table(remote_table_),
     context(context_), cluster_name(context.getMacros().expand(cluster_name_)), has_sharding_key(sharding_key_),
-    sharding_key_expr(sharding_key_ ? ExpressionAnalyzer(sharding_key_, context, nullptr, columns.getList()).getActions(false) : nullptr),
+    sharding_key_expr(sharding_key_ ? ExpressionAnalyzer(sharding_key_, context, nullptr, columns.getPhysical()).getActions(false) : nullptr),
     sharding_key_column_name(sharding_key_ ? sharding_key_->getColumnName() : String{}),
     path(data_path_.empty() ? "" : (data_path_ + escapeForFileName(table_name) + '/'))
 {
@@ -158,7 +158,7 @@ StoragePtr StorageDistributed::createWithOwnCluster(
     const Context & context_)
 {
     auto res = ext::shared_ptr_helper<StorageDistributed>::create(
-        name_, ColumnsDescription{columns_, NamesAndTypesList(), NamesAndTypesList(), ColumnDefaults()},
+        name_, ColumnsDescription{columns_},
         remote_database_, remote_table_, String{}, context_, ASTPtr(), String());
 
     res->owned_cluster = owned_cluster_;
@@ -385,7 +385,7 @@ void registerStorageDistributed(StorageFactory & factory)
         /// Check that sharding_key exists in the table and has numeric type.
         if (sharding_key)
         {
-            auto sharding_expr = ExpressionAnalyzer(sharding_key, args.context, nullptr, args.columns.getList()).getActions(true);
+            auto sharding_expr = ExpressionAnalyzer(sharding_key, args.context, nullptr, args.columns.getPhysical()).getActions(true);
             const Block & block = sharding_expr->getSampleBlock();
 
             if (block.columns() != 1)
diff --git a/dbms/src/Storages/StorageFactory.cpp b/dbms/src/Storages/StorageFactory.cpp
index 1b8970aba24..e6e661571aa 100644
--- a/dbms/src/Storages/StorageFactory.cpp
+++ b/dbms/src/Storages/StorageFactory.cpp
@@ -64,9 +64,7 @@ StoragePtr StorageFactory::get(
     {
         /// Check for some special types, that are not allowed to be stored in tables. Example: NULL data type.
         /// Exception: any type is allowed in View, because plain (non-materialized) View does not store anything itself.
-        checkAllTypesAreAllowedInTable(columns.ordinary);
-        checkAllTypesAreAllowedInTable(columns.materialized);
-        checkAllTypesAreAllowedInTable(columns.aliases);
+        checkAllTypesAreAllowedInTable(columns.getAll());
 
         if (query.is_materialized_view)
         {
diff --git a/dbms/src/Storages/StorageLog.cpp b/dbms/src/Storages/StorageLog.cpp
index 27881d8b8fe..137c1d55616 100644
--- a/dbms/src/Storages/StorageLog.cpp
+++ b/dbms/src/Storages/StorageLog.cpp
@@ -371,7 +371,7 @@ StorageLog::StorageLog(
      /// create files if they do not exist
     Poco::File(path + escapeForFileName(name) + '/').createDirectories();
 
-    for (const auto & column : columns.getList())
+    for (const auto & column : columns.getPhysical())
         addFiles(column.name, *column.type);
 
     marks_file = Poco::File(path + escapeForFileName(name) + '/' + DBMS_STORAGE_LOG_MARKS_FILE_NAME);
@@ -499,7 +499,7 @@ BlockInputStreams StorageLog::read(
     processed_stage = QueryProcessingStage::FetchColumns;
     loadMarks();
 
-    NamesAndTypesList all_columns = Nested::collect(columns.getList().addTypes(column_names));
+    NamesAndTypesList all_columns = Nested::collect(columns.getPhysical().addTypes(column_names));
 
     std::shared_lock<std::shared_mutex> lock(rwlock);
 
diff --git a/dbms/src/Storages/StorageMerge.cpp b/dbms/src/Storages/StorageMerge.cpp
index 39fed81e538..f26544975cd 100644
--- a/dbms/src/Storages/StorageMerge.cpp
+++ b/dbms/src/Storages/StorageMerge.cpp
@@ -181,7 +181,7 @@ BlockInputStreams StorageMerge::read(
 
         /// If there are only virtual columns in query, you must request at least one other column.
         if (real_column_names.size() == 0)
-            real_column_names.push_back(ExpressionActions::getSmallestColumn(table->columns.getList()));
+            real_column_names.push_back(ExpressionActions::getSmallestColumn(table->columns.getPhysical()));
 
         /// Substitute virtual column for its value when querying tables.
         ASTPtr modified_query_ast = query->clone();
diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp
index 71d8013e4db..5e799c91909 100644
--- a/dbms/src/Storages/StorageMergeTree.cpp
+++ b/dbms/src/Storages/StorageMergeTree.cpp
@@ -176,7 +176,7 @@ void StorageMergeTree::alter(
         throw Exception("MODIFY PRIMARY KEY only supported for tables without sampling key", ErrorCodes::BAD_ARGUMENTS);
 
     auto parts = data.getDataParts({MergeTreeDataPartState::PreCommitted, MergeTreeDataPartState::Committed, MergeTreeDataPartState::Outdated});
-    auto columns_for_parts = new_columns.getList();
+    auto columns_for_parts = new_columns.getPhysical();
     for (const MergeTreeData::DataPartPtr & part : parts)
     {
         if (auto transaction = data.alterDataPart(part, columns_for_parts, new_primary_key_ast, false))
@@ -422,7 +422,7 @@ void StorageMergeTree::clearColumnInPartition(const ASTPtr & partition, const Fi
     auto new_columns = data.columns;
     alter_command.apply(new_columns);
 
-    auto columns_for_parts = new_columns.getList();
+    auto columns_for_parts = new_columns.getPhysical();
     for (const auto & part : parts)
     {
         if (part->info.partition_id != partition_id)
diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp
index 9f6553f6512..d69db2d388e 100644
--- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp
@@ -1512,7 +1512,7 @@ void StorageReplicatedMergeTree::executeClearColumnInPartition(const LogEntry &
 
     size_t modified_parts = 0;
     auto parts = data.getDataParts();
-    auto columns_for_parts = new_columns.getList();
+    auto columns_for_parts = new_columns.getPhysical();
     for (const auto & part : parts)
     {
         if (!entry_part_info.contains(part->info))
diff --git a/dbms/src/Storages/StorageTinyLog.cpp b/dbms/src/Storages/StorageTinyLog.cpp
index 039f470c366..8c7fe1963a8 100644
--- a/dbms/src/Storages/StorageTinyLog.cpp
+++ b/dbms/src/Storages/StorageTinyLog.cpp
@@ -296,7 +296,7 @@ StorageTinyLog::StorageTinyLog(
             throwFromErrno("Cannot create directory " + full_path, ErrorCodes::CANNOT_CREATE_DIRECTORY);
     }
 
-    for (const auto & col : columns.getList())
+    for (const auto & col : columns.getPhysical())
         addFiles(col.name, *col.type);
 }
 
@@ -348,7 +348,7 @@ BlockInputStreams StorageTinyLog::read(
     check(column_names);
     processed_stage = QueryProcessingStage::FetchColumns;
     return BlockInputStreams(1, std::make_shared<TinyLogBlockInputStream>(
-        max_block_size, Nested::collect(columns.getList().addTypes(column_names)), *this, context.getSettingsRef().max_read_buffer_size));
+        max_block_size, Nested::collect(columns.getPhysical().addTypes(column_names)), *this, context.getSettingsRef().max_read_buffer_size));
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.cpp b/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.cpp
index f08808f799a..3c77244c557 100644
--- a/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.cpp
+++ b/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.cpp
@@ -16,10 +16,10 @@ StorageSystemAsynchronousMetrics::StorageSystemAsynchronousMetrics(const std::st
     : name(name_),
     async_metrics(async_metrics_)
 {
-    columns.ordinary = NamesAndTypesList{
+    columns = ColumnsDescription({
         {"metric", std::make_shared<DataTypeString>()},
         {"value", std::make_shared<DataTypeFloat64>()},
-    };
+    });
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemBuildOptions.cpp b/dbms/src/Storages/System/StorageSystemBuildOptions.cpp
index 7a864090e59..f354da97944 100644
--- a/dbms/src/Storages/System/StorageSystemBuildOptions.cpp
+++ b/dbms/src/Storages/System/StorageSystemBuildOptions.cpp
@@ -13,10 +13,10 @@ namespace DB
 StorageSystemBuildOptions::StorageSystemBuildOptions(const std::string & name_)
     : name(name_)
 {
-    columns.ordinary = NamesAndTypesList{
+    columns = ColumnsDescription({
         { "name", std::make_shared<DataTypeString>() },
         { "value", std::make_shared<DataTypeString>() },
-    };
+    });
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemClusters.cpp b/dbms/src/Storages/System/StorageSystemClusters.cpp
index 8d772e5ebb8..1e67e433d48 100644
--- a/dbms/src/Storages/System/StorageSystemClusters.cpp
+++ b/dbms/src/Storages/System/StorageSystemClusters.cpp
@@ -14,7 +14,7 @@ namespace DB
 StorageSystemClusters::StorageSystemClusters(const std::string & name_)
     : name(name_)
 {
-    columns.ordinary = NamesAndTypesList{
+    columns = ColumnsDescription({
         { "cluster",      std::make_shared<DataTypeString>() },
         { "shard_num",    std::make_shared<DataTypeUInt32>() },
         { "shard_weight", std::make_shared<DataTypeUInt32>() },
@@ -24,8 +24,8 @@ StorageSystemClusters::StorageSystemClusters(const std::string & name_)
         { "port",         std::make_shared<DataTypeUInt16>() },
         { "is_local",     std::make_shared<DataTypeUInt8>() },
         { "user",         std::make_shared<DataTypeString>() },
-        { "default_database", std::make_shared<DataTypeString>() }
-    };
+        { "default_database", std::make_shared<DataTypeString>() },
+    });
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemColumns.cpp b/dbms/src/Storages/System/StorageSystemColumns.cpp
index adec6a70c58..75c8afa1515 100644
--- a/dbms/src/Storages/System/StorageSystemColumns.cpp
+++ b/dbms/src/Storages/System/StorageSystemColumns.cpp
@@ -18,7 +18,7 @@ namespace DB
 StorageSystemColumns::StorageSystemColumns(const std::string & name_)
     : name(name_)
 {
-    columns.ordinary = NamesAndTypesList{
+    columns = ColumnsDescription({
         { "database",           std::make_shared<DataTypeString>() },
         { "table",              std::make_shared<DataTypeString>() },
         { "name",               std::make_shared<DataTypeString>() },
@@ -28,7 +28,7 @@ StorageSystemColumns::StorageSystemColumns(const std::string & name_)
         { "data_compressed_bytes",      std::make_shared<DataTypeUInt64>() },
         { "data_uncompressed_bytes",    std::make_shared<DataTypeUInt64>() },
         { "marks_bytes",                std::make_shared<DataTypeUInt64>() },
-    };
+    });
 }
 
 
@@ -137,8 +137,7 @@ BlockInputStreams StorageSystemColumns::read(
                     throw;
             }
 
-            columns = storage->columns.getList();
-            columns.insert(std::end(columns), std::begin(storage->columns.aliases), std::end(storage->columns.aliases));
+            columns = storage->columns.getAll();
             column_defaults = storage->columns.defaults;
 
             /** Info about sizes of columns for tables of MergeTree family.
diff --git a/dbms/src/Storages/System/StorageSystemDatabases.cpp b/dbms/src/Storages/System/StorageSystemDatabases.cpp
index 12c35c7ec9e..79a3b3483a6 100644
--- a/dbms/src/Storages/System/StorageSystemDatabases.cpp
+++ b/dbms/src/Storages/System/StorageSystemDatabases.cpp
@@ -13,12 +13,12 @@ namespace DB
 StorageSystemDatabases::StorageSystemDatabases(const std::string & name_)
     : name(name_)
 {
-    columns.ordinary = NamesAndTypesList{
+    columns = ColumnsDescription({
         {"name", std::make_shared<DataTypeString>()},
         {"engine", std::make_shared<DataTypeString>()},
         {"data_path", std::make_shared<DataTypeString>()},
         {"metadata_path", std::make_shared<DataTypeString>()},
-    };
+    });
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemDictionaries.cpp b/dbms/src/Storages/System/StorageSystemDictionaries.cpp
index 3cb4bd7962b..84bf4a2c98f 100644
--- a/dbms/src/Storages/System/StorageSystemDictionaries.cpp
+++ b/dbms/src/Storages/System/StorageSystemDictionaries.cpp
@@ -21,7 +21,7 @@ namespace DB
 StorageSystemDictionaries::StorageSystemDictionaries(const std::string & name)
     : name{name}
 {
-    columns.ordinary = NamesAndTypesList{
+    columns = ColumnsDescription({
         { "name", std::make_shared<DataTypeString>() },
         { "origin", std::make_shared<DataTypeString>() },
         { "type", std::make_shared<DataTypeString>() },
@@ -35,8 +35,8 @@ StorageSystemDictionaries::StorageSystemDictionaries(const std::string & name)
         { "load_factor", std::make_shared<DataTypeFloat64>() },
         { "creation_time", std::make_shared<DataTypeDateTime>() },
         { "source", std::make_shared<DataTypeString>() },
-        { "last_exception", std::make_shared<DataTypeString>() }
-    };
+        { "last_exception", std::make_shared<DataTypeString>() },
+    });
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemEvents.cpp b/dbms/src/Storages/System/StorageSystemEvents.cpp
index 0838eb739a5..ac613a7e84e 100644
--- a/dbms/src/Storages/System/StorageSystemEvents.cpp
+++ b/dbms/src/Storages/System/StorageSystemEvents.cpp
@@ -13,11 +13,11 @@ namespace DB
 StorageSystemEvents::StorageSystemEvents(const std::string & name_)
     : name(name_)
 {
-    columns.ordinary = NamesAndTypesList
+    columns = ColumnsDescription(
     {
         {"event", std::make_shared<DataTypeString>()},
-        {"value", std::make_shared<DataTypeUInt64>()}
-    };
+        {"value", std::make_shared<DataTypeUInt64>()},
+    });
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemFunctions.cpp b/dbms/src/Storages/System/StorageSystemFunctions.cpp
index 4d013eb2268..2d42bee9a46 100644
--- a/dbms/src/Storages/System/StorageSystemFunctions.cpp
+++ b/dbms/src/Storages/System/StorageSystemFunctions.cpp
@@ -16,10 +16,10 @@ namespace DB
 StorageSystemFunctions::StorageSystemFunctions(const std::string & name_)
     : name(name_)
 {
-    columns.ordinary = NamesAndTypesList{
+    columns = ColumnsDescription({
         { "name",         std::make_shared<DataTypeString>() },
-        { "is_aggregate", std::make_shared<DataTypeUInt8>()  }
-    };
+        { "is_aggregate", std::make_shared<DataTypeUInt8>()  },
+    });
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemGraphite.cpp b/dbms/src/Storages/System/StorageSystemGraphite.cpp
index e99c037ea78..30cc56c55e3 100644
--- a/dbms/src/Storages/System/StorageSystemGraphite.cpp
+++ b/dbms/src/Storages/System/StorageSystemGraphite.cpp
@@ -125,15 +125,15 @@ static Strings getAllGraphiteSections(const AbstractConfiguration & config)
 StorageSystemGraphite::StorageSystemGraphite(const std::string & name_)
     : name(name_)
 {
-    columns.ordinary = NamesAndTypesList{
+    columns = ColumnsDescription({
         {"config_name", std::make_shared<DataTypeString>()},
         {"regexp",      std::make_shared<DataTypeString>()},
         {"function",    std::make_shared<DataTypeString>()},
         {"age",         std::make_shared<DataTypeUInt64>()},
         {"precision",   std::make_shared<DataTypeUInt64>()},
         {"priority",    std::make_shared<DataTypeUInt16>()},
-        {"is_default",  std::make_shared<DataTypeUInt8>()}
-    };
+        {"is_default",  std::make_shared<DataTypeUInt8>()},
+    });
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemMerges.cpp b/dbms/src/Storages/System/StorageSystemMerges.cpp
index fea37df35a1..c32cad1265b 100644
--- a/dbms/src/Storages/System/StorageSystemMerges.cpp
+++ b/dbms/src/Storages/System/StorageSystemMerges.cpp
@@ -14,7 +14,7 @@ namespace DB
 StorageSystemMerges::StorageSystemMerges(const std::string & name)
     : name{name}
 {
-    columns.ordinary = NamesAndTypesList{
+    columns = ColumnsDescription({
         { "database",                    std::make_shared<DataTypeString>() },
         { "table",                       std::make_shared<DataTypeString>() },
         { "elapsed",                     std::make_shared<DataTypeFloat64>() },
@@ -31,7 +31,7 @@ StorageSystemMerges::StorageSystemMerges(const std::string & name)
         { "columns_written",             std::make_shared<DataTypeUInt64>() },
         { "memory_usage",                std::make_shared<DataTypeUInt64>() },
         { "thread_number",               std::make_shared<DataTypeUInt64>() },
-    };
+    });
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemMetrics.cpp b/dbms/src/Storages/System/StorageSystemMetrics.cpp
index 54002318ef6..57935b13fea 100644
--- a/dbms/src/Storages/System/StorageSystemMetrics.cpp
+++ b/dbms/src/Storages/System/StorageSystemMetrics.cpp
@@ -14,10 +14,10 @@ namespace DB
 StorageSystemMetrics::StorageSystemMetrics(const std::string & name_)
     : name(name_)
 {
-    columns.ordinary = NamesAndTypesList{
+    columns = ColumnsDescription({
         {"metric", std::make_shared<DataTypeString>()},
         {"value",  std::make_shared<DataTypeInt64>()},
-    };
+    });
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemModels.cpp b/dbms/src/Storages/System/StorageSystemModels.cpp
index 9057c6de626..6ef9d157303 100644
--- a/dbms/src/Storages/System/StorageSystemModels.cpp
+++ b/dbms/src/Storages/System/StorageSystemModels.cpp
@@ -14,13 +14,13 @@ namespace DB
 StorageSystemModels::StorageSystemModels(const std::string & name)
     : name{name}
 {
-    columns.ordinary = NamesAndTypesList{
+    columns = ColumnsDescription({
         { "name", std::make_shared<DataTypeString>() },
         { "origin", std::make_shared<DataTypeString>() },
         { "type", std::make_shared<DataTypeString>() },
         { "creation_time", std::make_shared<DataTypeDateTime>() },
-        { "last_exception", std::make_shared<DataTypeString>() }
-    };
+        { "last_exception", std::make_shared<DataTypeString>() },
+    });
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemNumbers.cpp b/dbms/src/Storages/System/StorageSystemNumbers.cpp
index 803f651639d..d5e72bcbdc3 100644
--- a/dbms/src/Storages/System/StorageSystemNumbers.cpp
+++ b/dbms/src/Storages/System/StorageSystemNumbers.cpp
@@ -47,7 +47,7 @@ private:
 StorageSystemNumbers::StorageSystemNumbers(const std::string & name_, bool multithreaded_, size_t limit_)
     : name(name_), multithreaded(multithreaded_), limit(limit_)
 {
-    columns.ordinary = NamesAndTypesList{{"number", std::make_shared<DataTypeUInt64>()}};
+    columns = ColumnsDescription({{"number", std::make_shared<DataTypeUInt64>()}});
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemOne.cpp b/dbms/src/Storages/System/StorageSystemOne.cpp
index f35580c3882..9cd74a7525c 100644
--- a/dbms/src/Storages/System/StorageSystemOne.cpp
+++ b/dbms/src/Storages/System/StorageSystemOne.cpp
@@ -13,7 +13,7 @@ namespace DB
 StorageSystemOne::StorageSystemOne(const std::string & name_)
     : name(name_)
 {
-    columns.ordinary = NamesAndTypesList{{"dummy", std::make_shared<DataTypeUInt8>()}};
+    columns = ColumnsDescription({{"dummy", std::make_shared<DataTypeUInt8>()}});
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemPartsBase.cpp b/dbms/src/Storages/System/StorageSystemPartsBase.cpp
index c6dc2a2c551..5143b73955f 100644
--- a/dbms/src/Storages/System/StorageSystemPartsBase.cpp
+++ b/dbms/src/Storages/System/StorageSystemPartsBase.cpp
@@ -280,7 +280,7 @@ bool StorageSystemPartsBase::hasColumn(const String & column_name) const
 StorageSystemPartsBase::StorageSystemPartsBase(std::string name_, NamesAndTypesList && columns_)
     : name(std::move(name_))
 {
-    columns.ordinary = columns_;
+    columns = ColumnsDescription(std::move(columns_));
 }
 
 }
diff --git a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp
index 5f0a584c1d6..326dfbf30ad 100644
--- a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp
+++ b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp
@@ -61,8 +61,7 @@ void StorageSystemPartsColumns::processNextStorage(MutableColumns & columns, con
         String default_expression;
     };
 
-    NamesAndTypesList columns_list = info.storage->columns.getList();
-    columns_list.insert(std::end(columns_list), std::begin(info.storage->columns.aliases), std::end(info.storage->columns.aliases));
+    NamesAndTypesList columns_list = info.storage->columns.getAll();
     const auto & column_defaults = info.storage->columns.defaults;
     std::unordered_map<String, ColumnInfo> columns_info;
 
diff --git a/dbms/src/Storages/System/StorageSystemProcesses.cpp b/dbms/src/Storages/System/StorageSystemProcesses.cpp
index e8262723d89..15ecc1a3d95 100644
--- a/dbms/src/Storages/System/StorageSystemProcesses.cpp
+++ b/dbms/src/Storages/System/StorageSystemProcesses.cpp
@@ -14,7 +14,7 @@ namespace DB
 StorageSystemProcesses::StorageSystemProcesses(const std::string & name_)
     : name(name_)
 {
-    columns.ordinary = NamesAndTypesList{
+    columns = ColumnsDescription({
         { "is_initial_query",     std::make_shared<DataTypeUInt8>() },
 
         { "user",                 std::make_shared<DataTypeString>() },
@@ -50,8 +50,8 @@ StorageSystemProcesses::StorageSystemProcesses(const std::string & name_)
         { "written_bytes",        std::make_shared<DataTypeUInt64>() },
         { "memory_usage",         std::make_shared<DataTypeInt64>() },
         { "peak_memory_usage",    std::make_shared<DataTypeInt64>() },
-        { "query",                std::make_shared<DataTypeString>() }
-    };
+        { "query",                std::make_shared<DataTypeString>() },
+    });
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemReplicas.cpp b/dbms/src/Storages/System/StorageSystemReplicas.cpp
index ca240a80568..fdcc6984861 100644
--- a/dbms/src/Storages/System/StorageSystemReplicas.cpp
+++ b/dbms/src/Storages/System/StorageSystemReplicas.cpp
@@ -17,7 +17,7 @@ namespace DB
 StorageSystemReplicas::StorageSystemReplicas(const std::string & name_)
     : name(name_)
 {
-    columns.ordinary = NamesAndTypesList{
+    columns = ColumnsDescription({
         { "database",                             std::make_shared<DataTypeString>()   },
         { "table",                                std::make_shared<DataTypeString>()   },
         { "engine",                               std::make_shared<DataTypeString>()   },
@@ -44,7 +44,7 @@ StorageSystemReplicas::StorageSystemReplicas(const std::string & name_)
         { "absolute_delay",                       std::make_shared<DataTypeUInt64>()   },
         { "total_replicas",                       std::make_shared<DataTypeUInt8>()    },
         { "active_replicas",                      std::make_shared<DataTypeUInt8>()    },
-    };
+    });
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp b/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp
index 363306437bb..b0ba714fef4 100644
--- a/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp
+++ b/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp
@@ -20,7 +20,7 @@ namespace DB
 StorageSystemReplicationQueue::StorageSystemReplicationQueue(const std::string & name_)
     : name(name_)
 {
-    columns.ordinary = NamesAndTypesList{
+    columns = ColumnsDescription({
         /// Table properties.
         { "database",                std::make_shared<DataTypeString>() },
         { "table",                   std::make_shared<DataTypeString>() },
@@ -43,7 +43,7 @@ StorageSystemReplicationQueue::StorageSystemReplicationQueue(const std::string &
         { "num_postponed",           std::make_shared<DataTypeUInt32>() },
         { "postpone_reason",         std::make_shared<DataTypeString>() },
         { "last_postpone_time",      std::make_shared<DataTypeDateTime>() },
-    };
+    });
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemSettings.cpp b/dbms/src/Storages/System/StorageSystemSettings.cpp
index 3848583688b..d8a758becc7 100644
--- a/dbms/src/Storages/System/StorageSystemSettings.cpp
+++ b/dbms/src/Storages/System/StorageSystemSettings.cpp
@@ -14,12 +14,12 @@ namespace DB
 StorageSystemSettings::StorageSystemSettings(const std::string & name_)
     : name(name_)
 {
-    columns.ordinary = NamesAndTypesList{
+    columns = ColumnsDescription({
         { "name",        std::make_shared<DataTypeString>() },
         { "value",       std::make_shared<DataTypeString>() },
         { "changed",     std::make_shared<DataTypeUInt8>() },
-        { "description", std::make_shared<DataTypeString>() }
-    };
+        { "description", std::make_shared<DataTypeString>() },
+    });
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemTables.cpp b/dbms/src/Storages/System/StorageSystemTables.cpp
index 91ab3581bc8..2db2a8052cc 100644
--- a/dbms/src/Storages/System/StorageSystemTables.cpp
+++ b/dbms/src/Storages/System/StorageSystemTables.cpp
@@ -114,7 +114,7 @@ protected:
 StorageSystemTables::StorageSystemTables(const std::string & name_)
     : name(name_)
 {
-    columns.ordinary = NamesAndTypesList
+    columns = ColumnsDescription(
     {
         {"database", std::make_shared<DataTypeString>()},
         {"name", std::make_shared<DataTypeString>()},
@@ -122,7 +122,7 @@ StorageSystemTables::StorageSystemTables(const std::string & name_)
         {"is_temporary", std::make_shared<DataTypeUInt8>()},
         {"data_path", std::make_shared<DataTypeString>()},
         {"metadata_path", std::make_shared<DataTypeString>()},
-    };
+    });
 
     virtual_columns =
     {
diff --git a/dbms/src/Storages/System/StorageSystemZooKeeper.cpp b/dbms/src/Storages/System/StorageSystemZooKeeper.cpp
index 9aae0ce5b3b..a5d5e2dab96 100644
--- a/dbms/src/Storages/System/StorageSystemZooKeeper.cpp
+++ b/dbms/src/Storages/System/StorageSystemZooKeeper.cpp
@@ -22,7 +22,7 @@ namespace DB
 StorageSystemZooKeeper::StorageSystemZooKeeper(const std::string & name_)
     : name(name_)
 {
-    columns.ordinary = NamesAndTypesList{
+    columns = ColumnsDescription({
         { "name",           std::make_shared<DataTypeString>() },
         { "value",          std::make_shared<DataTypeString>() },
         { "czxid",          std::make_shared<DataTypeInt64>() },
@@ -37,7 +37,7 @@ StorageSystemZooKeeper::StorageSystemZooKeeper(const std::string & name_)
         { "numChildren",    std::make_shared<DataTypeInt32>() },
         { "pzxid",          std::make_shared<DataTypeInt64>() },
         { "path",           std::make_shared<DataTypeString>() },
-    };
+    });
 }
 
 
diff --git a/dbms/src/Storages/tests/hit_log.cpp b/dbms/src/Storages/tests/hit_log.cpp
index 3dd75206a2b..305974b22d9 100644
--- a/dbms/src/Storages/tests/hit_log.cpp
+++ b/dbms/src/Storages/tests/hit_log.cpp
@@ -98,8 +98,8 @@ try
 
     /// create a hit log table
 
-    StoragePtr table = StorageLog::create("./", "HitLog", names_and_types_list,
-        NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}, DEFAULT_MAX_COMPRESS_BLOCK_SIZE);
+    StoragePtr table = StorageLog::create(
+        "./", "HitLog", ColumnsDescription{names_and_types_list}, DEFAULT_MAX_COMPRESS_BLOCK_SIZE);
     table->startup();
 
     /// create a description of how to read data from the tab separated dump
diff --git a/dbms/src/Storages/tests/storage_log.cpp b/dbms/src/Storages/tests/storage_log.cpp
index da53fcc356d..edad84b5b8c 100644
--- a/dbms/src/Storages/tests/storage_log.cpp
+++ b/dbms/src/Storages/tests/storage_log.cpp
@@ -25,8 +25,8 @@ try
     names_and_types.emplace_back("a", std::make_shared<DataTypeUInt64>());
     names_and_types.emplace_back("b", std::make_shared<DataTypeUInt8>());
 
-    StoragePtr table = StorageLog::create("./", "test", names_and_types,
-        NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}, DEFAULT_MAX_COMPRESS_BLOCK_SIZE);
+    StoragePtr table = StorageLog::create(
+        "./", "test", ColumnsDescription{names_and_types}, DEFAULT_MAX_COMPRESS_BLOCK_SIZE);
     table->startup();
 
     /// write into it
@@ -36,7 +36,7 @@ try
         {
             ColumnWithTypeAndName column;
             column.name = "a";
-            column.type = table->columns.getColumn("a").type;
+            column.type = table->getColumn("a").type;
             auto col = column.type->createColumn();
             ColumnUInt64::Container & vec = typeid_cast<ColumnUInt64 &>(*col).getData();
 
@@ -51,7 +51,7 @@ try
         {
             ColumnWithTypeAndName column;
             column.name = "b";
-            column.type = table->columns.getColumn("b").type;
+            column.type = table->getColumn("b").type;
             auto col = column.type->createColumn();
             ColumnUInt8::Container & vec = typeid_cast<ColumnUInt8 &>(*col).getData();
 
diff --git a/dbms/src/TableFunctions/TableFunctionMerge.cpp b/dbms/src/TableFunctions/TableFunctionMerge.cpp
index b471969b847..2a12dbca275 100644
--- a/dbms/src/TableFunctions/TableFunctionMerge.cpp
+++ b/dbms/src/TableFunctions/TableFunctionMerge.cpp
@@ -51,7 +51,7 @@ static NamesAndTypesList chooseColumns(const String & source_database, const Str
         throw Exception("Error while executing table function merge. In database " + source_database + " no one matches regular expression: "
             + table_name_regexp_, ErrorCodes::UNKNOWN_TABLE);
 
-    return any_table->columns.getList();
+    return any_table->columns.getPhysical();
 }
 
 
@@ -79,12 +79,7 @@ StoragePtr TableFunctionMerge::executeImpl(const ASTPtr & ast_function, const Co
 
     auto res = StorageMerge::create(
         getName(),
-        ColumnsDescription{
-            chooseColumns(source_database, table_name_regexp, context),
-            NamesAndTypesList{},
-            NamesAndTypesList{},
-            ColumnDefaults{},
-        },
+        ColumnsDescription{chooseColumns(source_database, table_name_regexp, context)},
         source_database,
         table_name_regexp,
         context);
diff --git a/dbms/src/TableFunctions/TableFunctionMySQL.cpp b/dbms/src/TableFunctions/TableFunctionMySQL.cpp
index 1ced8800255..1f7839ada86 100644
--- a/dbms/src/TableFunctions/TableFunctionMySQL.cpp
+++ b/dbms/src/TableFunctions/TableFunctionMySQL.cpp
@@ -152,7 +152,7 @@ StoragePtr TableFunctionMySQL::executeImpl(const ASTPtr & ast_function, const Co
         std::move(pool),
         database_name,
         table_name,
-        ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}});
+        ColumnsDescription{columns});
 
     res->startup();
     return res;
diff --git a/dbms/src/TableFunctions/TableFunctionODBC.cpp b/dbms/src/TableFunctions/TableFunctionODBC.cpp
index cacdbe29670..b75e48de054 100644
--- a/dbms/src/TableFunctions/TableFunctionODBC.cpp
+++ b/dbms/src/TableFunctions/TableFunctionODBC.cpp
@@ -113,8 +113,7 @@ StoragePtr TableFunctionODBC::executeImpl(const ASTPtr & ast_function, const Con
         columns.emplace_back(reinterpret_cast<char *>(column_name), getDataType(type));
     }
 
-    auto result = StorageODBC::create(table_name, connection_string, "", table_name,
-        ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}});
+    auto result = StorageODBC::create(table_name, connection_string, "", table_name, ColumnsDescription{columns});
     result->startup();
     return result;
 }
diff --git a/dbms/src/TableFunctions/getStructureOfRemoteTable.cpp b/dbms/src/TableFunctions/getStructureOfRemoteTable.cpp
index 181448714ba..2c56688c522 100644
--- a/dbms/src/TableFunctions/getStructureOfRemoteTable.cpp
+++ b/dbms/src/TableFunctions/getStructureOfRemoteTable.cpp
@@ -32,7 +32,7 @@ NamesAndTypesList getStructureOfRemoteTable(
     const auto & shard_info = cluster.getAnyShardInfo();
 
     if (shard_info.isLocal())
-        return context.getTable(database, table)->columns.getList();
+        return context.getTable(database, table)->columns.getPhysical();
 
     auto input = std::make_shared<RemoteBlockInputStream>(shard_info.pool, query, InterpreterDescribeQuery::getSampleBlock(), context);
     input->setPoolMode(PoolMode::GET_ONE);

From de1db216a50ea83323d8c2cb9dae21e80d0fbee2 Mon Sep 17 00:00:00 2001
From: Alexey Zatelepin <ztlpn@yandex-team.ru>
Date: Mon, 12 Mar 2018 16:47:01 +0300
Subject: [PATCH 033/244] getStructureOfRemoteTable returns info about column
 default kinds [#CLICKHOUSE-3241]

---
 .../Interpreters/InterpreterAlterQuery.cpp    |  4 +-
 .../Interpreters/InterpreterCreateQuery.cpp   | 12 +++---
 .../Interpreters/InterpreterDescribeQuery.cpp |  2 +-
 .../Interpreters/InterpreterSelectQuery.cpp   |  5 ++-
 dbms/src/Server/ClusterCopier.cpp             |  4 +-
 dbms/src/Storages/AlterCommands.cpp           | 28 ++++++------
 dbms/src/Storages/AlterCommands.h             |  6 +--
 dbms/src/Storages/ColumnDefault.cpp           | 26 +++++------
 dbms/src/Storages/ColumnDefault.h             |  8 ++--
 dbms/src/Storages/ColumnsDescription.cpp      | 16 +++----
 dbms/src/Storages/StorageCatBoostPool.cpp     |  2 +-
 dbms/src/Storages/StorageDistributed.cpp      |  5 +--
 dbms/src/Storages/StorageDistributed.h        |  2 +-
 .../Storages/System/StorageSystemColumns.cpp  |  2 +-
 .../System/StorageSystemPartsColumns.cpp      |  2 +-
 .../getStructureOfRemoteTable.cpp             | 43 ++++++++++++++-----
 .../getStructureOfRemoteTable.h               |  4 +-
 17 files changed, 97 insertions(+), 74 deletions(-)

diff --git a/dbms/src/Interpreters/InterpreterAlterQuery.cpp b/dbms/src/Interpreters/InterpreterAlterQuery.cpp
index 1fb3e1e80db..08254c8e8f8 100644
--- a/dbms/src/Interpreters/InterpreterAlterQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterAlterQuery.cpp
@@ -110,7 +110,7 @@ void InterpreterAlterQuery::parseAlter(
             }
             if (ast_col_decl.default_expression)
             {
-                command.default_type = columnDefaultTypeFromString(ast_col_decl.default_specifier);
+                command.default_kind = columnDefaultKindFromString(ast_col_decl.default_specifier);
                 command.default_expression = ast_col_decl.default_expression;
             }
 
@@ -157,7 +157,7 @@ void InterpreterAlterQuery::parseAlter(
 
             if (ast_col_decl.default_expression)
             {
-                command.default_type = columnDefaultTypeFromString(ast_col_decl.default_specifier);
+                command.default_kind = columnDefaultKindFromString(ast_col_decl.default_specifier);
                 command.default_expression = ast_col_decl.default_expression;
             }
 
diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp
index e5d20648fbd..43f4336f896 100644
--- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp
@@ -246,7 +246,7 @@ static ColumnsAndDefaults parseColumns(const ASTExpressionList & column_list_ast
                 explicit_type = block.getByName(column_name).type;
 
             defaults.emplace(column_name, ColumnDefault{
-                columnDefaultTypeFromString(col_decl_ptr->default_specifier),
+                columnDefaultKindFromString(col_decl_ptr->default_specifier),
                 col_decl_ptr->default_expression
             });
         }
@@ -256,7 +256,7 @@ static ColumnsAndDefaults parseColumns(const ASTExpressionList & column_list_ast
 }
 
 
-static NamesAndTypesList removeAndReturnColumns(ColumnsAndDefaults & columns_and_defaults, const ColumnDefaultType type)
+static NamesAndTypesList removeAndReturnColumns(ColumnsAndDefaults & columns_and_defaults, const ColumnDefaultKind kind)
 {
     auto & columns = columns_and_defaults.first;
     auto & defaults = columns_and_defaults.second;
@@ -266,7 +266,7 @@ static NamesAndTypesList removeAndReturnColumns(ColumnsAndDefaults & columns_and
     for (auto it = std::begin(columns); it != std::end(columns);)
     {
         const auto jt = defaults.find(it->name);
-        if (jt != std::end(defaults) && jt->second.type == type)
+        if (jt != std::end(defaults) && jt->second.kind == kind)
         {
             removed.push_back(*it);
             it = columns.erase(it);
@@ -323,7 +323,7 @@ ASTPtr InterpreterCreateQuery::formatColumns(const ColumnsDescription & columns)
         const auto it = columns.defaults.find(column.name);
         if (it != std::end(columns.defaults))
         {
-            column_declaration->default_specifier = toString(it->second.type);
+            column_declaration->default_specifier = toString(it->second.kind);
             column_declaration->default_expression = it->second.expression->clone();
         }
 
@@ -339,8 +339,8 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription(const ASTExpres
     ColumnsDescription res;
 
     auto && columns_and_defaults = parseColumns(columns, context);
-    res.materialized = removeAndReturnColumns(columns_and_defaults, ColumnDefaultType::Materialized);
-    res.aliases = removeAndReturnColumns(columns_and_defaults, ColumnDefaultType::Alias);
+    res.materialized = removeAndReturnColumns(columns_and_defaults, ColumnDefaultKind::Materialized);
+    res.aliases = removeAndReturnColumns(columns_and_defaults, ColumnDefaultKind::Alias);
     res.ordinary = std::move(columns_and_defaults.first);
     res.defaults = std::move(columns_and_defaults.second);
 
diff --git a/dbms/src/Interpreters/InterpreterDescribeQuery.cpp b/dbms/src/Interpreters/InterpreterDescribeQuery.cpp
index c03034af49a..07eefbd913b 100644
--- a/dbms/src/Interpreters/InterpreterDescribeQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterDescribeQuery.cpp
@@ -119,7 +119,7 @@ BlockInputStreamPtr InterpreterDescribeQuery::executeImpl()
         }
         else
         {
-            res_columns[2]->insert(toString(it->second.type));
+            res_columns[2]->insert(toString(it->second.kind));
             res_columns[3]->insert(queryToString(it->second.expression));
         }
     }
diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp
index 04e05e975ac..ba93c737f08 100644
--- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp
@@ -521,7 +521,7 @@ QueryProcessingStage::Enum InterpreterSelectQuery::executeFetchColumns(Pipeline
         for (const auto & column : required_columns)
         {
             const auto default_it = storage->columns.defaults.find(column);
-            if (default_it != std::end(storage->columns.defaults) && default_it->second.type == ColumnDefaultType::Alias)
+            if (default_it != std::end(storage->columns.defaults) && default_it->second.kind == ColumnDefaultKind::Alias)
             {
                 alias_columns_required = true;
                 break;
@@ -536,7 +536,7 @@ QueryProcessingStage::Enum InterpreterSelectQuery::executeFetchColumns(Pipeline
             for (const auto & column : required_columns)
             {
                 const auto default_it = storage->columns.defaults.find(column);
-                if (default_it != std::end(storage->columns.defaults) && default_it->second.type == ColumnDefaultType::Alias)
+                if (default_it != std::end(storage->columns.defaults) && default_it->second.kind == ColumnDefaultKind::Alias)
                     required_columns_expr_list->children.emplace_back(setAlias(default_it->second.expression->clone(), column));
                 else
                     required_columns_expr_list->children.emplace_back(std::make_shared<ASTIdentifier>(column));
@@ -711,6 +711,7 @@ QueryProcessingStage::Enum InterpreterSelectQuery::executeFetchColumns(Pipeline
     else
         throw Exception("Logical error in InterpreterSelectQuery: nowhere to read", ErrorCodes::LOGICAL_ERROR);
 
+    StackTrace st;
     /// Aliases in table declaration.
     if (from_stage == QueryProcessingStage::FetchColumns && alias_actions)
     {
diff --git a/dbms/src/Server/ClusterCopier.cpp b/dbms/src/Server/ClusterCopier.cpp
index 77172a22b23..3ae6a9ed205 100644
--- a/dbms/src/Server/ClusterCopier.cpp
+++ b/dbms/src/Server/ClusterCopier.cpp
@@ -1120,8 +1120,8 @@ protected:
 
             if (!column.default_specifier.empty())
             {
-                ColumnDefaultType type = columnDefaultTypeFromString(column.default_specifier);
-                if (type == ColumnDefaultType::Materialized || type == ColumnDefaultType::Alias)
+                ColumnDefaultKind kind = columnDefaultKindFromString(column.default_specifier);
+                if (kind == ColumnDefaultKind::Materialized || kind == ColumnDefaultKind::Alias)
                     continue;
             }
 
diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp
index 184f63c070f..6c16acf5300 100644
--- a/dbms/src/Storages/AlterCommands.cpp
+++ b/dbms/src/Storages/AlterCommands.cpp
@@ -57,17 +57,17 @@ void AlterCommand::apply(ColumnsDescription & columns_description) const
             columns.emplace(insert_it, column_name, data_type);
         };
 
-        if (default_type == ColumnDefaultType::Default)
+        if (default_kind == ColumnDefaultKind::Default)
             add_column(columns_description.ordinary);
-        else if (default_type == ColumnDefaultType::Materialized)
+        else if (default_kind == ColumnDefaultKind::Materialized)
             add_column(columns_description.materialized);
-        else if (default_type == ColumnDefaultType::Alias)
+        else if (default_kind == ColumnDefaultKind::Alias)
             add_column(columns_description.aliases);
         else
-            throw Exception{"Unknown ColumnDefaultType value", ErrorCodes::LOGICAL_ERROR};
+            throw Exception{"Unknown ColumnDefaultKind value", ErrorCodes::LOGICAL_ERROR};
 
         if (default_expression)
-            columns_description.defaults.emplace(column_name, ColumnDefault{default_type, default_expression});
+            columns_description.defaults.emplace(column_name, ColumnDefault{default_kind, default_expression});
 
         /// Slow, because each time a list is copied
         columns_description.ordinary = Nested::flatten(columns_description.ordinary);
@@ -103,12 +103,12 @@ void AlterCommand::apply(ColumnsDescription & columns_description) const
     {
         const auto default_it = columns_description.defaults.find(column_name);
         const auto had_default_expr = default_it != std::end(columns_description.defaults);
-        const auto old_default_type = had_default_expr ? default_it->second.type : ColumnDefaultType{};
+        const auto old_default_kind = had_default_expr ? default_it->second.kind : ColumnDefaultKind{};
 
         /// target column list
         auto & new_columns =
-            default_type == ColumnDefaultType::Default ? columns_description.ordinary
-            : default_type == ColumnDefaultType::Materialized ? columns_description.materialized
+            default_kind == ColumnDefaultKind::Default ? columns_description.ordinary
+            : default_kind == ColumnDefaultKind::Materialized ? columns_description.materialized
             : columns_description.aliases;
 
         /// find column or throw exception
@@ -124,12 +124,12 @@ void AlterCommand::apply(ColumnsDescription & columns_description) const
         };
 
         /// if default types differ, remove column from the old list, then add to the new list
-        if (default_type != old_default_type)
+        if (default_kind != old_default_kind)
         {
             /// source column list
             auto & old_columns =
-                old_default_type == ColumnDefaultType::Default ? columns_description.ordinary
-                : old_default_type == ColumnDefaultType::Materialized ? columns_description.materialized
+                old_default_kind == ColumnDefaultKind::Default ? columns_description.ordinary
+                : old_default_kind == ColumnDefaultKind::Materialized ? columns_description.materialized
                 : columns_description.aliases;
 
             const auto old_column_it = find_column(old_columns);
@@ -138,7 +138,7 @@ void AlterCommand::apply(ColumnsDescription & columns_description) const
 
             /// do not forget to change the default type of old column
             if (had_default_expr)
-                columns_description.defaults[column_name].type = default_type;
+                columns_description.defaults[column_name].kind = default_kind;
         }
 
         /// find column in one of three column lists
@@ -150,7 +150,7 @@ void AlterCommand::apply(ColumnsDescription & columns_description) const
             columns_description.defaults.erase(column_name);
         else if (default_expression && !had_default_expr)
             /// new column has a default expression while the old one had not, add it it column_defaults
-            columns_description.defaults.emplace(column_name, ColumnDefault{default_type, default_expression});
+            columns_description.defaults.emplace(column_name, ColumnDefault{default_kind, default_expression});
         else if (had_default_expr)
             /// both old and new columns have default expression, update it
             columns_description.defaults[column_name].expression = default_expression;
@@ -333,7 +333,7 @@ void AlterCommands::validate(IStorage * table, const Context & context)
                     /// add a new alter command to modify existing column
                     this->emplace_back(AlterCommand{
                         AlterCommand::MODIFY_COLUMN, column_name, explicit_type,
-                        default_it->second.type, default_it->second.expression
+                        default_it->second.kind, default_it->second.expression
                     });
 
                     command_ptr = &this->back();
diff --git a/dbms/src/Storages/AlterCommands.h b/dbms/src/Storages/AlterCommands.h
index 44c44e8bfd0..306d5cec54c 100644
--- a/dbms/src/Storages/AlterCommands.h
+++ b/dbms/src/Storages/AlterCommands.h
@@ -27,7 +27,7 @@ struct AlterCommand
     /// For ADD and MODIFY, a new column type.
     DataTypePtr data_type;
 
-    ColumnDefaultType default_type{};
+    ColumnDefaultKind default_kind{};
     ASTPtr default_expression{};
 
     /// For ADD - after which column to add a new one. If an empty string, add to the end. To add to the beginning now it is impossible.
@@ -47,9 +47,9 @@ struct AlterCommand
 
     AlterCommand() = default;
     AlterCommand(const Type type, const String & column_name, const DataTypePtr & data_type,
-                 const ColumnDefaultType default_type, const ASTPtr & default_expression,
+                 const ColumnDefaultKind default_kind, const ASTPtr & default_expression,
                  const String & after_column = String{})
-        : type{type}, column_name{column_name}, data_type{data_type}, default_type{default_type},
+        : type{type}, column_name{column_name}, data_type{data_type}, default_kind{default_kind},
         default_expression{default_expression}, after_column{after_column}
     {}
 };
diff --git a/dbms/src/Storages/ColumnDefault.cpp b/dbms/src/Storages/ColumnDefault.cpp
index dcdfe2a6e51..0ff885ab1e5 100644
--- a/dbms/src/Storages/ColumnDefault.cpp
+++ b/dbms/src/Storages/ColumnDefault.cpp
@@ -6,12 +6,12 @@ namespace DB
 {
 
 
-ColumnDefaultType columnDefaultTypeFromString(const std::string & str)
+ColumnDefaultKind columnDefaultKindFromString(const std::string & str)
 {
-    static const std::unordered_map<std::string, ColumnDefaultType> map{
-        { "DEFAULT", ColumnDefaultType::Default },
-        { "MATERIALIZED", ColumnDefaultType::Materialized },
-        { "ALIAS", ColumnDefaultType::Alias }
+    static const std::unordered_map<std::string, ColumnDefaultKind> map{
+        { "DEFAULT", ColumnDefaultKind::Default },
+        { "MATERIALIZED", ColumnDefaultKind::Materialized },
+        { "ALIAS", ColumnDefaultKind::Alias }
     };
 
     const auto it = map.find(str);
@@ -19,22 +19,22 @@ ColumnDefaultType columnDefaultTypeFromString(const std::string & str)
 }
 
 
-std::string toString(const ColumnDefaultType type)
+std::string toString(const ColumnDefaultKind kind)
 {
-    static const std::unordered_map<ColumnDefaultType, std::string> map{
-        { ColumnDefaultType::Default, "DEFAULT" },
-        { ColumnDefaultType::Materialized, "MATERIALIZED" },
-        { ColumnDefaultType::Alias, "ALIAS" }
+    static const std::unordered_map<ColumnDefaultKind, std::string> map{
+        { ColumnDefaultKind::Default, "DEFAULT" },
+        { ColumnDefaultKind::Materialized, "MATERIALIZED" },
+        { ColumnDefaultKind::Alias, "ALIAS" }
     };
 
-    const auto it = map.find(type);
-    return it != std::end(map) ? it->second : throw Exception{"Invalid ColumnDefaultType"};
+    const auto it = map.find(kind);
+    return it != std::end(map) ? it->second : throw Exception{"Invalid ColumnDefaultKind"};
 }
 
 
 bool operator==(const ColumnDefault & lhs, const ColumnDefault & rhs)
 {
-    return lhs.type == rhs.type && queryToString(lhs.expression) == queryToString(rhs.expression);
+    return lhs.kind == rhs.kind && queryToString(lhs.expression) == queryToString(rhs.expression);
 }
 
 }
diff --git a/dbms/src/Storages/ColumnDefault.h b/dbms/src/Storages/ColumnDefault.h
index 33f17952d08..95eb4d5b597 100644
--- a/dbms/src/Storages/ColumnDefault.h
+++ b/dbms/src/Storages/ColumnDefault.h
@@ -9,7 +9,7 @@
 namespace DB
 {
 
-enum class ColumnDefaultType
+enum class ColumnDefaultKind
 {
     Default,
     Materialized,
@@ -17,13 +17,13 @@ enum class ColumnDefaultType
 };
 
 
-ColumnDefaultType columnDefaultTypeFromString(const std::string & str);
-std::string toString(const ColumnDefaultType type);
+ColumnDefaultKind columnDefaultKindFromString(const std::string & str);
+std::string toString(const ColumnDefaultKind type);
 
 
 struct ColumnDefault
 {
-    ColumnDefaultType type;
+    ColumnDefaultKind kind;
     ASTPtr expression;
 };
 
diff --git a/dbms/src/Storages/ColumnsDescription.cpp b/dbms/src/Storages/ColumnsDescription.cpp
index 95b5001a1f9..683f10f324d 100644
--- a/dbms/src/Storages/ColumnsDescription.cpp
+++ b/dbms/src/Storages/ColumnsDescription.cpp
@@ -88,7 +88,7 @@ String ColumnsDescription::toString() const
             else
                 writeChar('\t', buf);
 
-            writeString(DB::toString(it->second.type), buf);
+            writeString(DB::toString(it->second.kind), buf);
             writeChar('\t', buf);
             writeString(queryToString(it->second.expression), buf);
             writeChar('\n', buf);
@@ -134,9 +134,9 @@ ColumnsDescription ColumnsDescription::parse(const String & str)
         }
         assertChar('\t', buf);
 
-        String default_type_str;
-        readString(default_type_str, buf);
-        const auto default_type = columnDefaultTypeFromString(default_type_str);
+        String default_kind_str;
+        readString(default_kind_str, buf);
+        const auto default_kind = columnDefaultKindFromString(default_kind_str);
         assertChar('\t', buf);
 
         String default_expr_str;
@@ -147,14 +147,14 @@ ColumnsDescription ColumnsDescription::parse(const String & str)
         const auto end = begin + default_expr_str.size();
         ASTPtr default_expr = parseQuery(expr_parser, begin, end, "default expression");
 
-        if (ColumnDefaultType::Default == default_type)
+        if (ColumnDefaultKind::Default == default_kind)
             result.ordinary.emplace_back(column_name, std::move(type));
-        else if (ColumnDefaultType::Materialized == default_type)
+        else if (ColumnDefaultKind::Materialized == default_kind)
             result.materialized.emplace_back(column_name, std::move(type));
-        else if (ColumnDefaultType::Alias == default_type)
+        else if (ColumnDefaultKind::Alias == default_kind)
             result.aliases.emplace_back(column_name, std::move(type));
 
-        result.defaults.emplace(column_name, ColumnDefault{default_type, default_expr});
+        result.defaults.emplace(column_name, ColumnDefault{default_kind, default_expr});
     }
 
     assertEOF(buf);
diff --git a/dbms/src/Storages/StorageCatBoostPool.cpp b/dbms/src/Storages/StorageCatBoostPool.cpp
index 1a973f7dd75..d20da89b170 100644
--- a/dbms/src/Storages/StorageCatBoostPool.cpp
+++ b/dbms/src/Storages/StorageCatBoostPool.cpp
@@ -243,7 +243,7 @@ void StorageCatBoostPool::createSampleBlockAndColumns()
         if (!desc.alias.empty())
         {
             auto alias = std::make_shared<ASTIdentifier>(desc.column_name);
-            columns.defaults[desc.alias] = {ColumnDefaultType::Alias, alias};
+            columns.defaults[desc.alias] = {ColumnDefaultKind::Alias, alias};
             columns.aliases.emplace_back(desc.alias, type);
         }
 
diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp
index c010ca768d1..e3ff7f30369 100644
--- a/dbms/src/Storages/StorageDistributed.cpp
+++ b/dbms/src/Storages/StorageDistributed.cpp
@@ -151,15 +151,14 @@ StorageDistributed::StorageDistributed(
 
 StoragePtr StorageDistributed::createWithOwnCluster(
     const std::string & name_,
-    const NamesAndTypesList & columns_,
+    const ColumnsDescription & columns_,
     const String & remote_database_,
     const String & remote_table_,
     ClusterPtr & owned_cluster_,
     const Context & context_)
 {
     auto res = ext::shared_ptr_helper<StorageDistributed>::create(
-        name_, ColumnsDescription{columns_},
-        remote_database_, remote_table_, String{}, context_, ASTPtr(), String());
+        name_, columns_, remote_database_, remote_table_, String{}, context_, ASTPtr(), String());
 
     res->owned_cluster = owned_cluster_;
 
diff --git a/dbms/src/Storages/StorageDistributed.h b/dbms/src/Storages/StorageDistributed.h
index 22e24c3f9fb..451c9f12d8a 100644
--- a/dbms/src/Storages/StorageDistributed.h
+++ b/dbms/src/Storages/StorageDistributed.h
@@ -35,7 +35,7 @@ public:
 
     static StoragePtr createWithOwnCluster(
         const std::string & table_name_,
-        const NamesAndTypesList & columns_,
+        const ColumnsDescription & columns_,
         const String & remote_database_,      /// database on remote servers.
         const String & remote_table_,         /// The name of the table on the remote servers.
         ClusterPtr & owned_cluster_,
diff --git a/dbms/src/Storages/System/StorageSystemColumns.cpp b/dbms/src/Storages/System/StorageSystemColumns.cpp
index 75c8afa1515..83c5dca9117 100644
--- a/dbms/src/Storages/System/StorageSystemColumns.cpp
+++ b/dbms/src/Storages/System/StorageSystemColumns.cpp
@@ -171,7 +171,7 @@ BlockInputStreams StorageSystemColumns::read(
                 }
                 else
                 {
-                    res_columns[i++]->insert(toString(it->second.type));
+                    res_columns[i++]->insert(toString(it->second.kind));
                     res_columns[i++]->insert(queryToString(it->second.expression));
                 }
             }
diff --git a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp
index 326dfbf30ad..d804504f13a 100644
--- a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp
+++ b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp
@@ -72,7 +72,7 @@ void StorageSystemPartsColumns::processNextStorage(MutableColumns & columns, con
         const auto it = column_defaults.find(column.name);
         if (it != std::end(column_defaults))
         {
-            column_info.default_kind = toString(it->second.type);
+            column_info.default_kind = toString(it->second.kind);
             column_info.default_expression = queryToString(it->second.expression);
         }
 
diff --git a/dbms/src/TableFunctions/getStructureOfRemoteTable.cpp b/dbms/src/TableFunctions/getStructureOfRemoteTable.cpp
index 2c56688c522..6eede709352 100644
--- a/dbms/src/TableFunctions/getStructureOfRemoteTable.cpp
+++ b/dbms/src/TableFunctions/getStructureOfRemoteTable.cpp
@@ -1,12 +1,12 @@
+#include <TableFunctions/getStructureOfRemoteTable.h>
 #include <Interpreters/Cluster.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/InterpreterDescribeQuery.h>
 #include <DataStreams/RemoteBlockInputStream.h>
 #include <DataTypes/DataTypeFactory.h>
 #include <Storages/IStorage.h>
-#include <Parsers/IAST.h>
-
-#include <TableFunctions/getStructureOfRemoteTable.h>
+#include <Parsers/ExpressionListParsers.h>
+#include <Parsers/parseQuery.h>
 
 
 namespace DB
@@ -18,21 +18,21 @@ namespace ErrorCodes
 }
 
 
-NamesAndTypesList getStructureOfRemoteTable(
+ColumnsDescription getStructureOfRemoteTable(
     const Cluster & cluster,
     const std::string & database,
     const std::string & table,
     const Context & context)
 {
-    /// Request for a table description
-    String query = "DESC TABLE " + backQuoteIfNeed(database) + "." + backQuoteIfNeed(table);
-    NamesAndTypesList res;
-
     /// Send to the first any remote shard.
     const auto & shard_info = cluster.getAnyShardInfo();
 
     if (shard_info.isLocal())
-        return context.getTable(database, table)->columns.getPhysical();
+        return context.getTable(database, table)->columns;
+
+    /// Request for a table description
+    String query = "DESC TABLE " + backQuoteIfNeed(database) + "." + backQuoteIfNeed(table);
+    ColumnsDescription res;
 
     auto input = std::make_shared<RemoteBlockInputStream>(shard_info.pool, query, InterpreterDescribeQuery::getSampleBlock(), context);
     input->setPoolMode(PoolMode::GET_ONE);
@@ -41,18 +41,41 @@ NamesAndTypesList getStructureOfRemoteTable(
 
     const DataTypeFactory & data_type_factory = DataTypeFactory::instance();
 
+    ParserExpression expr_parser;
+
     while (Block current = input->read())
     {
         ColumnPtr name = current.getByName("name").column;
         ColumnPtr type = current.getByName("type").column;
+        ColumnPtr default_kind = current.getByName("default_type").column;
+        ColumnPtr default_expr = current.getByName("default_expression").column;
         size_t size = name->size();
 
         for (size_t i = 0; i < size; ++i)
         {
             String column_name = (*name)[i].get<const String &>();
             String data_type_name = (*type)[i].get<const String &>();
+            String kind_name = (*default_kind)[i].get<const String &>();
 
-            res.emplace_back(column_name, data_type_factory.get(data_type_name));
+            auto data_type = data_type_factory.get(data_type_name);
+
+            if (kind_name.empty())
+                res.ordinary.emplace_back(column_name, std::move(data_type));
+            else
+            {
+                auto kind = columnDefaultKindFromString(kind_name);
+
+                String expr_str = (*default_expr)[i].get<const String &>();
+                ASTPtr expr = parseQuery(expr_parser, expr_str.data(), expr_str.data() + expr_str.size(), "default expression");
+                res.defaults.emplace(column_name, ColumnDefault{kind, expr});
+
+                if (ColumnDefaultKind::Default == kind)
+                    res.ordinary.emplace_back(column_name, std::move(data_type));
+                else if (ColumnDefaultKind::Materialized == kind)
+                    res.materialized.emplace_back(column_name, std::move(data_type));
+                else if (ColumnDefaultKind::Alias == kind)
+                    res.aliases.emplace_back(column_name, std::move(data_type));
+            }
         }
     }
 
diff --git a/dbms/src/TableFunctions/getStructureOfRemoteTable.h b/dbms/src/TableFunctions/getStructureOfRemoteTable.h
index 70b3d83b8f1..20417ef50e1 100644
--- a/dbms/src/TableFunctions/getStructureOfRemoteTable.h
+++ b/dbms/src/TableFunctions/getStructureOfRemoteTable.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#include <Core/NamesAndTypes.h>
+#include <Storages/ColumnsDescription.h>
 
 
 namespace DB
@@ -11,7 +11,7 @@ class Context;
 
 /// Find the names and types of the table columns on any server in the cluster.
 /// Used to implement the `remote` table function and others.
-NamesAndTypesList getStructureOfRemoteTable(
+ColumnsDescription getStructureOfRemoteTable(
     const Cluster & cluster,
     const std::string & database,
     const std::string & table,

From d51e0b72563a2492cbf6d2d20020f5fa36c7a1c5 Mon Sep 17 00:00:00 2001
From: Ivan Blinkov <ivan@blinkov.ru>
Date: Tue, 13 Mar 2018 12:58:21 +0300
Subject: [PATCH 034/244] Update reference.md

---
 docs/en/agg_functions/reference.md | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/docs/en/agg_functions/reference.md b/docs/en/agg_functions/reference.md
index d907f79647c..90ff9da58e7 100644
--- a/docs/en/agg_functions/reference.md
+++ b/docs/en/agg_functions/reference.md
@@ -19,7 +19,7 @@ In some cases, you can rely on the order of execution. This applies to cases whe
 
 When a `SELECT` query has the `GROUP BY` clause or at least one aggregate function, ClickHouse (in contrast to MySQL) requires that all expressions in the `SELECT`, `HAVING`, and `ORDER BY` clauses be calculated from keys or from aggregate functions. In other words, each column selected from the table must be used either in keys or inside aggregate functions. To get behavior like in MySQL, you can put the other columns in the `any` aggregate function.
 
-## anyHeavy
+## anyHeavy(x)
 
 Selects a frequently occurring value using the [heavy hitters](http://www.cs.umd.edu/~samir/498/karp.pdf) algorithm. If there is a value that occurs more than in half the cases in each of the query's execution threads, this value is returned. Normally, the result is nondeterministic.
 
@@ -33,7 +33,7 @@ anyHeavy(column)
 
 **Example**
 
-Take the [OnTime](../getting_started/example_datasets/ontime.md#example_datasets-ontime)data set and select any frequently occurring value in the `AirlineID` column.
+Take the [OnTime](../getting_started/example_datasets/ontime.md#example_datasets-ontime) data set and select any frequently occurring value in the `AirlineID` column.
 
 ```sql
 SELECT anyHeavy(AirlineID) AS res
@@ -170,7 +170,7 @@ In some cases, you can still rely on the order of execution. This applies to cas
 
 <a name="agg_functions_groupArrayInsertAt"></a>
 
-## groupArrayInsertAt
+## groupArrayInsertAt(x)
 
 Inserts a value into the array in the specified position.
 
@@ -257,7 +257,7 @@ The performance of the function is lower than for ` quantile`, ` quantileTiming`
 
 The result depends on the order of running the query, and is nondeterministic.
 
-## median
+## median(x)
 
 All the quantile functions have corresponding median functions: `median`, `medianDeterministic`, `medianTiming`, `medianTimingWeighted`, `medianExact`, `medianExactWeighted`, `medianTDigest`. They are synonyms and their behavior is identical.
 
@@ -287,16 +287,12 @@ The result is equal to the square root of `varSamp(x)`.
 
 The result is equal to the square root of `varPop(x)`.
 
-## topK
+## topK(N)(column)
 
 Returns an array of the most frequent values in the specified column. The resulting array is sorted in descending order of frequency of values (not by the values themselves).
 
 Implements the [ Filtered Space-Saving](http://www.l2f.inesc-id.pt/~fmmb/wiki/uploads/Work/misnis.ref0a.pdf)  algorithm for analyzing TopK, based on the reduce-and-combine algorithm from [Parallel Space Saving](https://arxiv.org/pdf/1401.0702.pdf).
 
-```
-topK(N)(column)
-```
-
 This function doesn't provide a guaranteed result. In certain situations, errors might occur and it might return frequent values that aren't the most frequent values.
 
 We recommend using the `N < 10 ` value; performance is reduced with large `N` values. Maximum value of ` N = 65536`.

From 632a2ecabc59f78de86c52f84b122149b6dd8841 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Tue, 13 Mar 2018 13:41:47 +0300
Subject: [PATCH 035/244] added SYSTEM RELOAD CONFIG query [#CLICKHOUSE-3591]

---
 dbms/src/Common/Config/ConfigReloader.cpp      |  2 ++
 dbms/src/Common/Config/ConfigReloader.h        |  6 ++++++
 dbms/src/Interpreters/Context.cpp              | 18 ++++++++++++++++++
 dbms/src/Interpreters/Context.h                |  4 ++++
 .../Interpreters/InterpreterSystemQuery.cpp    |  3 +++
 dbms/src/Parsers/ASTSystemQuery.cpp            |  2 ++
 dbms/src/Parsers/ASTSystemQuery.h              |  1 +
 dbms/src/Server/Server.cpp                     |  6 ++++++
 8 files changed, 42 insertions(+)

diff --git a/dbms/src/Common/Config/ConfigReloader.cpp b/dbms/src/Common/Config/ConfigReloader.cpp
index 54cbf507055..e06fca6316a 100644
--- a/dbms/src/Common/Config/ConfigReloader.cpp
+++ b/dbms/src/Common/Config/ConfigReloader.cpp
@@ -73,6 +73,8 @@ void ConfigReloader::run()
 
 void ConfigReloader::reloadIfNewer(bool force, bool throw_on_error, bool fallback_to_preprocessed)
 {
+    std::lock_guard<std::mutex> lock(reload_mutex);
+
     FilesChangesTracker new_files = getNewFileList();
     if (force || new_files.isDifferOrNewerThan(files))
     {
diff --git a/dbms/src/Common/Config/ConfigReloader.h b/dbms/src/Common/Config/ConfigReloader.h
index 94b2d3c629c..74a5294832b 100644
--- a/dbms/src/Common/Config/ConfigReloader.h
+++ b/dbms/src/Common/Config/ConfigReloader.h
@@ -42,6 +42,9 @@ public:
     /// Call this method to run the backround thread.
     void start();
 
+    /// Reload immediately. For SYSTEM RELOAD CONFIG query.
+    void reload() { reloadIfNewer(/* force */ true, /* throw_on_error */ true, /* fallback_to_preprocessed */ false); }
+
 private:
     void run();
 
@@ -74,6 +77,9 @@ private:
 
     std::atomic<bool> quit{false};
     std::thread thread;
+
+    /// Is taken after entering into reloadIfNewer.
+    std::mutex reload_mutex;
 };
 
 }
diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp
index 7a2358f36c0..6bc4cde2d06 100644
--- a/dbms/src/Interpreters/Context.cpp
+++ b/dbms/src/Interpreters/Context.cpp
@@ -186,6 +186,8 @@ struct ContextShared
 
     pcg64 rng{randomSeed()};
 
+    Context::ConfigReloadCallback config_reload_callback;
+
     ContextShared(std::shared_ptr<IRuntimeComponentsFactory> runtime_components_factory_)
         : runtime_components_factory(std::move(runtime_components_factory_))
     {
@@ -1624,6 +1626,22 @@ time_t Context::getUptimeSeconds() const
 }
 
 
+void Context::setConfigReloadCallback(ConfigReloadCallback && callback)
+{
+    /// Is initialized at server startup, so lock isn't required. Otherwise use mutex.
+    shared->config_reload_callback = std::move(callback);
+}
+
+void Context::reloadConfig() const
+{
+    /// Use mutex if callback may be changed after startup.
+    if (!shared->config_reload_callback)
+        throw Exception("Can't reload config beacuse config_reload_callback is not set.", ErrorCodes::LOGICAL_ERROR);
+
+    shared->config_reload_callback();
+}
+
+
 void Context::shutdown()
 {
     system_logs.reset();
diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h
index 4148b79923b..32ba0b8cd7f 100644
--- a/dbms/src/Interpreters/Context.h
+++ b/dbms/src/Interpreters/Context.h
@@ -361,6 +361,10 @@ public:
     /// Get the server uptime in seconds.
     time_t getUptimeSeconds() const;
 
+    using ConfigReloadCallback = std::function<void()>;
+    void setConfigReloadCallback(ConfigReloadCallback && callback);
+    void reloadConfig() const;
+
     void shutdown();
 
     enum class ApplicationType
diff --git a/dbms/src/Interpreters/InterpreterSystemQuery.cpp b/dbms/src/Interpreters/InterpreterSystemQuery.cpp
index 419a4937a72..ef8197f5ffb 100644
--- a/dbms/src/Interpreters/InterpreterSystemQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterSystemQuery.cpp
@@ -97,6 +97,9 @@ BlockIO InterpreterSystemQuery::execute()
                 throw Exception(status.message, status.code);
             break;
         }
+        case Type::RELOAD_CONFIG:
+            context.reloadConfig();
+            break;
         case Type::STOP_LISTEN_QUERIES:
         case Type::START_LISTEN_QUERIES:
         case Type::RESTART_REPLICAS:
diff --git a/dbms/src/Parsers/ASTSystemQuery.cpp b/dbms/src/Parsers/ASTSystemQuery.cpp
index 9f6914e7f4c..a50a98f03d6 100644
--- a/dbms/src/Parsers/ASTSystemQuery.cpp
+++ b/dbms/src/Parsers/ASTSystemQuery.cpp
@@ -39,6 +39,8 @@ const char * ASTSystemQuery::typeToString(Type type)
             return "RELOAD DICTIONARY";
         case Type::RELOAD_DICTIONARIES:
             return "RELOAD DICTIONARIES";
+        case Type::RELOAD_CONFIG:
+            return "RELOAD CONFIG";
         case Type::STOP_MERGES:
             return "STOP MERGES";
         case Type::START_MERGES:
diff --git a/dbms/src/Parsers/ASTSystemQuery.h b/dbms/src/Parsers/ASTSystemQuery.h
index 1821b73bc12..87eaded44ef 100644
--- a/dbms/src/Parsers/ASTSystemQuery.h
+++ b/dbms/src/Parsers/ASTSystemQuery.h
@@ -24,6 +24,7 @@ public:
         SYNC_REPLICA,
         RELOAD_DICTIONARY,
         RELOAD_DICTIONARIES,
+        RELOAD_CONFIG,
         STOP_MERGES,
         START_MERGES,
         STOP_REPLICATION_QUEUES,
diff --git a/dbms/src/Server/Server.cpp b/dbms/src/Server/Server.cpp
index 08153b00b1d..390eec74f43 100644
--- a/dbms/src/Server/Server.cpp
+++ b/dbms/src/Server/Server.cpp
@@ -251,6 +251,12 @@ int Server::main(const std::vector<std::string> & /*args*/)
         [&](ConfigurationPtr config) { global_context->setUsersConfig(config); },
         /* already_loaded = */ false);
 
+    /// Reload config in SYSTEM RELOAD CONFIG query.
+    global_context->setConfigReloadCallback([&]() {
+        main_config_reloader->reload();
+        users_config_reloader->reload();
+    });
+
     /// Limit on total number of concurrently executed queries.
     global_context->getProcessList().setMaxSize(config().getInt("max_concurrent_queries", 0));
 

From 8b802ad7743803df206a194a8c577102994361a7 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Tue, 13 Mar 2018 14:47:32 +0300
Subject: [PATCH 036/244] added test for SYSTEM RELOAD CONFIG and system.macros
 [#CLICKHOUSE-3591]

---
 dbms/tests/integration/test_system_queries/test.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/dbms/tests/integration/test_system_queries/test.py b/dbms/tests/integration/test_system_queries/test.py
index 063d9c005bd..46ab2561a83 100644
--- a/dbms/tests/integration/test_system_queries/test.py
+++ b/dbms/tests/integration/test_system_queries/test.py
@@ -74,6 +74,17 @@ def test_DROP_DNS_CACHE(started_cluster):
     assert TSV(instance.query("SELECT DISTINCT host_name, host_address FROM system.clusters WHERE cluster='lost_host_cluster'")) == TSV("lost_host\t127.0.0.1\n")
 
 
+def test_RELOAD_CONFIG_AND_MACROS(started_cluster):
+
+    macros = "<yandex><macros><mac>ro</mac></macros></yandex>"
+    create_macros = 'echo "{}" > /etc/clickhouse-server/config.d/macros.xml'.format(macros)
+
+    instance = cluster.instances['ch1']
+
+    instance.exec_in_container(['bash', '-c', create_macros], privileged=True, user='root')
+    instance.query("SYSTEM RELOAD CONFIG")
+    assert TSV(instance.query("select * from system.macros")) == TSV("mac\tro\n")
+
 if __name__ == '__main__':
     with contextmanager(started_cluster)() as cluster:
        for name, instance in cluster.instances.items():

From 324cda232ceb5be9212f3513fab68a3e1ca28231 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Tue, 13 Mar 2018 14:53:27 +0300
Subject: [PATCH 037/244] fix clang build [#CLICKHOUSE-3591]

---
 dbms/src/Common/Macros.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/dbms/src/Common/Macros.h b/dbms/src/Common/Macros.h
index 316d526b5a7..dae3bd10aa9 100644
--- a/dbms/src/Common/Macros.h
+++ b/dbms/src/Common/Macros.h
@@ -3,6 +3,7 @@
 #include <Core/Types.h>
 #include <Poco/Util/AbstractConfiguration.h>
 #include <map>
+#include <memory>
 
 
 namespace DB

From cda51eb920ce3adbb96a15bd01de6bb8b44627bb Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Tue, 13 Mar 2018 15:31:58 +0300
Subject: [PATCH 038/244] added fake create query for default database
 [#CLICKHOUSE-3590]

---
 dbms/src/Databases/DatabaseOrdinary.cpp | 9 +++++++--
 dbms/src/Databases/DatabaseOrdinary.h   | 1 +
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/dbms/src/Databases/DatabaseOrdinary.cpp b/dbms/src/Databases/DatabaseOrdinary.cpp
index b0719649094..73152387cf6 100644
--- a/dbms/src/Databases/DatabaseOrdinary.cpp
+++ b/dbms/src/Databases/DatabaseOrdinary.cpp
@@ -336,16 +336,21 @@ void DatabaseOrdinary::removeTable(
 }
 
 
-static ASTPtr getCreateQueryImpl(const String & path, const String & table_name)
+ASTPtr DatabaseOrdinary::getCreateQueryImpl(const String & path, const String & table_name) const
 {
     String metadata_path;
+    String query;
 
     if (table_name.empty())
+    {
         metadata_path = detail::getDatabaseMetadataPath(path);
+        if (!Poco::File(metadata_path).exists())
+            query = "CREATE DATABASE " + backQuoteIfNeed(name) + " ENGINE = Ordinary";
+    }
     else
         metadata_path = detail::getTableMetadataPath(path, table_name);
 
-    String query;
+    if (query.empty())
     {
         ReadBufferFromFile in(metadata_path, 4096);
         readStringUntilEOF(query, in);
diff --git a/dbms/src/Databases/DatabaseOrdinary.h b/dbms/src/Databases/DatabaseOrdinary.h
index dbd44586c74..4cd77c85895 100644
--- a/dbms/src/Databases/DatabaseOrdinary.h
+++ b/dbms/src/Databases/DatabaseOrdinary.h
@@ -68,6 +68,7 @@ public:
 
 private:
     void startupTables(ThreadPool * thread_pool);
+    ASTPtr getCreateQueryImpl(const String & path, const String & table_name) const;
 };
 
 }

From 2f573e4f7714e595d993a4c9eafbebe5245fdd18 Mon Sep 17 00:00:00 2001
From: Alexey Zatelepin <ztlpn@yandex-team.ru>
Date: Mon, 12 Mar 2018 21:38:24 +0300
Subject: [PATCH 039/244] add tests [#CLICKHOUSE-3241]

---
 ...c_table_functions_and_subqueries.reference | 12 +++---
 ...remote_and_columns_with_defaults.reference | 13 +++++++
 ...shard_remote_and_columns_with_defaults.sql | 38 +++++++++++++++++++
 3 files changed, 57 insertions(+), 6 deletions(-)
 create mode 100644 dbms/tests/queries/0_stateless/00604_shard_remote_and_columns_with_defaults.reference
 create mode 100644 dbms/tests/queries/0_stateless/00604_shard_remote_and_columns_with_defaults.sql

diff --git a/dbms/tests/queries/0_stateless/00515_shard_desc_table_functions_and_subqueries.reference b/dbms/tests/queries/0_stateless/00515_shard_desc_table_functions_and_subqueries.reference
index 20d07b74593..b0615b7993f 100644
--- a/dbms/tests/queries/0_stateless/00515_shard_desc_table_functions_and_subqueries.reference
+++ b/dbms/tests/queries/0_stateless/00515_shard_desc_table_functions_and_subqueries.reference
@@ -12,15 +12,15 @@ val4	UInt64	ALIAS	val
 -
 date	Date		
 val	UInt64		
-val2	UInt8		
-val3	UInt8		
-val4	UInt64		
+val2	UInt8	DEFAULT	42
+val3	UInt8	DEFAULT	CAST((val2 + 1) AS UInt8)
+val4	UInt64	ALIAS	val
 -
 date	Date		
 val	UInt64		
-val2	UInt8		
-val3	UInt8		
-val4	UInt64		
+val2	UInt8	DEFAULT	42
+val3	UInt8	DEFAULT	CAST((val2 + 1) AS UInt8)
+val4	UInt64	ALIAS	val
 -
 1	UInt8		
 -
diff --git a/dbms/tests/queries/0_stateless/00604_shard_remote_and_columns_with_defaults.reference b/dbms/tests/queries/0_stateless/00604_shard_remote_and_columns_with_defaults.reference
new file mode 100644
index 00000000000..8a5c72b9000
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00604_shard_remote_and_columns_with_defaults.reference
@@ -0,0 +1,13 @@
+1	1
+2	2
+*** With a DEFAULT column ***
+1	1
+2	2
+*** With a MATERIALIZED column ***
+1
+1	2
+*** With an ALIAS column ***
+1
+2
+1	2
+2	3
diff --git a/dbms/tests/queries/0_stateless/00604_shard_remote_and_columns_with_defaults.sql b/dbms/tests/queries/0_stateless/00604_shard_remote_and_columns_with_defaults.sql
new file mode 100644
index 00000000000..ba955f7848b
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00604_shard_remote_and_columns_with_defaults.sql
@@ -0,0 +1,38 @@
+DROP TABLE IF EXISTS test.t1;
+DROP TABLE IF EXISTS test.t2;
+DROP TABLE IF EXISTS test.t3;
+DROP TABLE IF EXISTS test.t4;
+
+CREATE TABLE test.t1(x UInt32, y UInt32) ENGINE TinyLog;
+CREATE TABLE test.t2(x UInt32, y UInt32 DEFAULT x + 1) ENGINE TinyLog;
+CREATE TABLE test.t3(x UInt32, y UInt32 MATERIALIZED x + 1) ENGINE TinyLog;
+CREATE TABLE test.t4(x UInt32, y UInt32 ALIAS x + 1) ENGINE TinyLog;
+
+INSERT INTO test.t1 VALUES (1, 1);
+INSERT INTO test.t2 VALUES (1, 1);
+INSERT INTO test.t3 VALUES (1);
+INSERT INTO test.t4 VALUES (1);
+
+INSERT INTO FUNCTION remote('127.0.0.2', test.t1) VALUES (2, 2);
+INSERT INTO FUNCTION remote('127.0.0.2', test.t2) VALUES (2, 2);
+--TODO: INSERT into remote tables with MATERIALIZED columns.
+--INSERT INTO FUNCTION remote('127.0.0.2', test.t3) VALUES (2);
+INSERT INTO FUNCTION remote('127.0.0.2', test.t4) VALUES (2);
+
+SELECT * FROM remote('127.0.0.2', test.t1);
+
+SELECT '*** With a DEFAULT column ***';
+SELECT * FROM remote('127.0.0.2', test.t2);
+
+SELECT '*** With a MATERIALIZED column ***';
+SELECT * FROM remote('127.0.0.2', test.t3);
+SELECT x, y FROM remote('127.0.0.2', test.t3);
+
+SELECT '*** With an ALIAS column ***';
+SELECT * FROM remote('127.0.0.2', test.t4);
+SELECT x, y FROM remote('127.0.0.2', test.t4);
+
+DROP TABLE IF EXISTS test.t1;
+DROP TABLE IF EXISTS test.t2;
+DROP TABLE IF EXISTS test.t3;
+DROP TABLE IF EXISTS test.t4;

From fdffbe53ac010d2a54822d16c67944b1451b4218 Mon Sep 17 00:00:00 2001
From: Alexey Zatelepin <ztlpn@yandex-team.ru>
Date: Tue, 13 Mar 2018 17:18:11 +0300
Subject: [PATCH 040/244] encapsulate ColumnsDeclaration [#CLICKHOUSE-3241]

---
 dbms/src/Analyzers/AnalyzeColumns.cpp         |  2 +-
 dbms/src/Analyzers/ExecuteTableFunctions.cpp  |  2 +-
 dbms/src/Core/iostream_debug_helpers.cpp      |  2 +-
 .../DataStreams/RemoteBlockInputStream.cpp    |  2 +-
 dbms/src/Interpreters/ExpressionAnalyzer.cpp  |  9 +++---
 .../Interpreters/InterpreterAlterQuery.cpp    |  2 +-
 .../Interpreters/InterpreterCreateQuery.cpp   |  2 +-
 .../Interpreters/InterpreterDescribeQuery.cpp |  4 +--
 .../Interpreters/InterpreterInsertQuery.cpp   |  6 ++--
 .../Interpreters/InterpreterSelectQuery.cpp   | 11 ++++----
 dbms/src/Storages/AlterCommands.cpp           |  4 +--
 dbms/src/Storages/ITableDeclaration.cpp       | 28 +++++++++++--------
 dbms/src/Storages/ITableDeclaration.h         | 11 ++++----
 .../MergeTree/MergeTreeBlockInputStream.cpp   |  5 ++--
 .../MergeTree/MergeTreeBlockReadUtils.cpp     |  4 +--
 dbms/src/Storages/MergeTree/MergeTreeData.cpp | 22 +++++++--------
 dbms/src/Storages/MergeTree/MergeTreeData.h   |  4 +--
 .../MergeTree/MergeTreeDataMerger.cpp         |  4 +--
 .../Storages/MergeTree/MergeTreeDataPart.cpp  |  6 ++--
 .../MergeTree/MergeTreeDataSelectExecutor.cpp |  4 +--
 .../MergeTree/MergeTreeDataWriter.cpp         |  2 +-
 .../Storages/MergeTree/MergeTreeReadPool.cpp  |  5 ++--
 .../Storages/MergeTree/MergeTreeReader.cpp    |  4 +--
 .../MergeTree/MergeTreeWhereOptimizer.cpp     |  4 +--
 .../ReplicatedMergeTreeAlterThread.cpp        |  8 ++----
 dbms/src/Storages/StorageBuffer.cpp           |  6 ++--
 dbms/src/Storages/StorageCatBoostPool.cpp     |  3 ++
 dbms/src/Storages/StorageDictionary.cpp       |  2 +-
 dbms/src/Storages/StorageDistributed.cpp      | 12 ++++----
 dbms/src/Storages/StorageJoin.cpp             |  2 +-
 dbms/src/Storages/StorageLog.cpp              |  8 +++---
 dbms/src/Storages/StorageMerge.cpp            |  8 ++++--
 dbms/src/Storages/StorageMergeTree.cpp        | 12 +++-----
 dbms/src/Storages/StorageMergeTree.h          |  3 ++
 dbms/src/Storages/StorageMySQL.cpp            |  2 +-
 dbms/src/Storages/StorageNull.cpp             |  7 +++--
 dbms/src/Storages/StorageODBC.cpp             |  3 +-
 .../Storages/StorageReplicatedMergeTree.cpp   | 21 +++++++-------
 .../src/Storages/StorageReplicatedMergeTree.h |  3 ++
 dbms/src/Storages/StorageTinyLog.cpp          |  4 +--
 .../StorageSystemAsynchronousMetrics.cpp      |  4 +--
 .../System/StorageSystemBuildOptions.cpp      |  4 +--
 .../Storages/System/StorageSystemClusters.cpp |  4 +--
 .../Storages/System/StorageSystemColumns.cpp  |  8 +++---
 .../System/StorageSystemDatabases.cpp         |  4 +--
 .../System/StorageSystemDictionaries.cpp      |  4 +--
 .../Storages/System/StorageSystemEvents.cpp   |  4 +--
 .../System/StorageSystemFunctions.cpp         |  4 +--
 .../Storages/System/StorageSystemGraphite.cpp |  4 +--
 .../Storages/System/StorageSystemMerges.cpp   |  4 +--
 .../Storages/System/StorageSystemMetrics.cpp  |  4 +--
 .../Storages/System/StorageSystemModels.cpp   |  4 +--
 .../Storages/System/StorageSystemNumbers.cpp  |  2 +-
 dbms/src/Storages/System/StorageSystemOne.cpp |  2 +-
 .../System/StorageSystemPartsBase.cpp         |  2 +-
 .../System/StorageSystemPartsColumns.cpp      |  4 +--
 .../System/StorageSystemProcesses.cpp         |  4 +--
 .../Storages/System/StorageSystemReplicas.cpp |  4 +--
 .../System/StorageSystemReplicationQueue.cpp  |  4 +--
 .../Storages/System/StorageSystemSettings.cpp |  4 +--
 .../Storages/System/StorageSystemTables.cpp   |  4 +--
 .../System/StorageSystemZooKeeper.cpp         |  4 +--
 .../src/TableFunctions/TableFunctionMerge.cpp |  2 +-
 .../getStructureOfRemoteTable.cpp             |  2 +-
 64 files changed, 182 insertions(+), 161 deletions(-)

diff --git a/dbms/src/Analyzers/AnalyzeColumns.cpp b/dbms/src/Analyzers/AnalyzeColumns.cpp
index 78568b080c6..efcbaf66888 100644
--- a/dbms/src/Analyzers/AnalyzeColumns.cpp
+++ b/dbms/src/Analyzers/AnalyzeColumns.cpp
@@ -173,7 +173,7 @@ ASTPtr createASTIdentifierForColumnInTable(const String & column, const CollectT
 void createASTsForAllColumnsInTable(const CollectTables::TableInfo & table, ASTs & res)
 {
     if (table.storage)
-        for (const auto & name : table.storage->columns.getNames())
+        for (const auto & name : table.storage->getColumns().getNames())
             res.emplace_back(createASTIdentifierForColumnInTable(name, table));
     else
         for (size_t i = 0, size = table.structure_of_subquery.columns(); i < size; ++i)
diff --git a/dbms/src/Analyzers/ExecuteTableFunctions.cpp b/dbms/src/Analyzers/ExecuteTableFunctions.cpp
index 3feabc8f490..37cc3d66eec 100644
--- a/dbms/src/Analyzers/ExecuteTableFunctions.cpp
+++ b/dbms/src/Analyzers/ExecuteTableFunctions.cpp
@@ -72,7 +72,7 @@ void ExecuteTableFunctions::dump(WriteBuffer & out) const
     {
         writeString(table.second->getName(), out);
         writeCString("\n\n", out);
-        writeString(table.second->columns.getPhysical().toString(), out);
+        writeString(table.second->getColumns().getPhysical().toString(), out);
         writeCString("\n", out);
     }
 }
diff --git a/dbms/src/Core/iostream_debug_helpers.cpp b/dbms/src/Core/iostream_debug_helpers.cpp
index 723b3dc2e90..bd19c453d8d 100644
--- a/dbms/src/Core/iostream_debug_helpers.cpp
+++ b/dbms/src/Core/iostream_debug_helpers.cpp
@@ -44,7 +44,7 @@ std::ostream & operator<<(std::ostream & stream, const IDataType & what)
 std::ostream & operator<<(std::ostream & stream, const IStorage & what)
 {
     stream << "IStorage(name = " << what.getName() << ", tableName = " << what.getTableName() << ") {"
-           << what.columns.getPhysical().toString()
+           << what.getColumns().getPhysical().toString()
            << "}";
     // isRemote supportsSampling supportsFinal supportsPrewhere
     return stream;
diff --git a/dbms/src/DataStreams/RemoteBlockInputStream.cpp b/dbms/src/DataStreams/RemoteBlockInputStream.cpp
index 079cabe8dee..b5e20898b89 100644
--- a/dbms/src/DataStreams/RemoteBlockInputStream.cpp
+++ b/dbms/src/DataStreams/RemoteBlockInputStream.cpp
@@ -138,7 +138,7 @@ void RemoteBlockInputStream::sendExternalTables()
             {
                 StoragePtr cur = table.second;
                 QueryProcessingStage::Enum stage = QueryProcessingStage::Complete;
-                BlockInputStreams input = cur->read(cur->columns.getNames(), {}, context,
+                BlockInputStreams input = cur->read(cur->getColumns().getNames(), {}, context,
                     stage, DEFAULT_BLOCK_SIZE, 1);
                 if (input.size() == 0)
                     res.push_back(std::make_pair(std::make_shared<OneBlockInputStream>(cur->getSampleBlock()), table.first));
diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp
index e6190b8f1df..4f0fe52bcff 100644
--- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp
+++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp
@@ -190,7 +190,7 @@ ExpressionAnalyzer::ExpressionAnalyzer(
     }
 
     if (storage && source_columns.empty())
-        source_columns = storage->columns.getPhysical();
+        source_columns = storage->getColumns().getPhysical();
     else
         removeDuplicateColumns(source_columns);
 
@@ -712,7 +712,7 @@ static std::shared_ptr<InterpreterSelectWithUnionQuery> interpretSubquery(
         /// get columns list for target table
         auto database_table = getDatabaseAndTableNameFromIdentifier(*table);
         const auto & storage = context.getTable(database_table.first, database_table.second);
-        const auto & columns = storage->columns.ordinary;
+        const auto & columns = storage->getColumns().ordinary;
         select_expression_list->children.reserve(columns.size());
 
         /// manually substitute column names in place of asterisk
@@ -1050,7 +1050,7 @@ void ExpressionAnalyzer::normalizeTreeImpl(
                 if (storage)
                 {
                     /// If we select from a table, get only not MATERIALIZED, not ALIAS columns.
-                    for (const auto & name_type : storage->columns.ordinary)
+                    for (const auto & name_type : storage->getColumns().ordinary)
                         all_columns.emplace_back(std::make_shared<ASTIdentifier>(name_type.name));
                 }
                 else
@@ -1147,7 +1147,8 @@ void ExpressionAnalyzer::addAliasColumns()
     if (!storage)
         return;
 
-    source_columns.insert(std::end(source_columns), std::begin(storage->columns.aliases), std::end(storage->columns.aliases));
+    const auto & aliases = storage->getColumns().aliases;
+    source_columns.insert(std::end(source_columns), std::begin(aliases), std::end(aliases));
 }
 
 
diff --git a/dbms/src/Interpreters/InterpreterAlterQuery.cpp b/dbms/src/Interpreters/InterpreterAlterQuery.cpp
index 08254c8e8f8..377f92d6c8d 100644
--- a/dbms/src/Interpreters/InterpreterAlterQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterAlterQuery.cpp
@@ -200,7 +200,7 @@ void InterpreterAlterQuery::PartitionCommands::validate(const IStorage * table)
         {
             String column_name = command.column_name.safeGet<String>();
 
-            if (!table->columns.has(column_name))
+            if (!table->getColumns().has(column_name))
             {
                 throw Exception("Wrong column name. Cannot find column " + column_name + " to clear it from partition",
                     DB::ErrorCodes::ILLEGAL_COLUMN);
diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp
index 43f4336f896..dd594d2993a 100644
--- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp
@@ -362,7 +362,7 @@ ColumnsDescription InterpreterCreateQuery::setColumns(
     }
     else if (!create.as_table.empty())
     {
-        res = as_storage->columns;
+        res = as_storage->getColumns();
     }
     else if (create.select)
     {
diff --git a/dbms/src/Interpreters/InterpreterDescribeQuery.cpp b/dbms/src/Interpreters/InterpreterDescribeQuery.cpp
index 07eefbd913b..536c554e996 100644
--- a/dbms/src/Interpreters/InterpreterDescribeQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterDescribeQuery.cpp
@@ -99,8 +99,8 @@ BlockInputStreamPtr InterpreterDescribeQuery::executeImpl()
         }
 
         auto table_lock = table->lockStructure(false, __PRETTY_FUNCTION__);
-        columns = table->columns.getAll();
-        column_defaults = table->columns.defaults;
+        columns = table->getColumns().getAll();
+        column_defaults = table->getColumns().defaults;
     }
 
     Block sample_block = getSampleBlock();
diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp
index 973fdc31a27..3e054f576a4 100644
--- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp
@@ -95,7 +95,7 @@ BlockIO InterpreterInsertQuery::execute()
 
     auto table_lock = table->lockStructure(true, __PRETTY_FUNCTION__);
 
-    NamesAndTypesList required_columns = table->columns.getPhysical();
+    NamesAndTypesList required_columns = table->getColumns().getPhysical();
 
     /// We create a pipeline of several streams, into which we will write data.
     BlockOutputStreamPtr out;
@@ -103,7 +103,7 @@ BlockIO InterpreterInsertQuery::execute()
     out = std::make_shared<PushingToViewsBlockOutputStream>(query.database, query.table, table, context, query_ptr, query.no_destination);
 
     out = std::make_shared<AddingDefaultBlockOutputStream>(
-        out, getSampleBlock(query, table), required_columns, table->columns.defaults, context);
+        out, getSampleBlock(query, table), required_columns, table->getColumns().defaults, context);
 
     /// Do not squash blocks if it is a sync INSERT into Distributed
     if (!(context.getSettingsRef().insert_distributed_sync && table->getName() == "Distributed"))
@@ -135,7 +135,7 @@ BlockIO InterpreterInsertQuery::execute()
         if (!allow_materialized)
         {
             Block in_header = res.in->getHeader();
-            for (const auto & name_type : table->columns.materialized)
+            for (const auto & name_type : table->getColumns().materialized)
                 if (in_header.has(name_type.name))
                     throw Exception("Cannot insert column " + name_type.name + ", because it is MATERIALIZED column.", ErrorCodes::ILLEGAL_COLUMN);
         }
diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp
index ba93c737f08..49a46abeb13 100644
--- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp
@@ -516,12 +516,13 @@ QueryProcessingStage::Enum InterpreterSelectQuery::executeFetchColumns(Pipeline
     /// Are ALIAS columns required for query execution?
     auto alias_columns_required = false;
 
-    if (storage && !storage->columns.aliases.empty())
+    if (storage && !storage->getColumns().aliases.empty())
     {
+        const auto & column_defaults = storage->getColumns().defaults;
         for (const auto & column : required_columns)
         {
-            const auto default_it = storage->columns.defaults.find(column);
-            if (default_it != std::end(storage->columns.defaults) && default_it->second.kind == ColumnDefaultKind::Alias)
+            const auto default_it = column_defaults.find(column);
+            if (default_it != std::end(column_defaults) && default_it->second.kind == ColumnDefaultKind::Alias)
             {
                 alias_columns_required = true;
                 break;
@@ -535,8 +536,8 @@ QueryProcessingStage::Enum InterpreterSelectQuery::executeFetchColumns(Pipeline
 
             for (const auto & column : required_columns)
             {
-                const auto default_it = storage->columns.defaults.find(column);
-                if (default_it != std::end(storage->columns.defaults) && default_it->second.kind == ColumnDefaultKind::Alias)
+                const auto default_it = column_defaults.find(column);
+                if (default_it != std::end(column_defaults) && default_it->second.kind == ColumnDefaultKind::Alias)
                     required_columns_expr_list->children.emplace_back(setAlias(default_it->second.expression->clone(), column));
                 else
                     required_columns_expr_list->children.emplace_back(std::make_shared<ASTIdentifier>(column));
diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp
index 6c16acf5300..274f9315f5f 100644
--- a/dbms/src/Storages/AlterCommands.cpp
+++ b/dbms/src/Storages/AlterCommands.cpp
@@ -177,8 +177,8 @@ void AlterCommands::apply(ColumnsDescription & columns_description) const
 
 void AlterCommands::validate(IStorage * table, const Context & context)
 {
-    auto all_columns = table->columns.getAll();
-    auto defaults = table->columns.defaults;
+    auto all_columns = table->getColumns().getAll();
+    auto defaults = table->getColumns().defaults;
 
     std::vector<std::pair<NameAndTypePair, AlterCommand *>> defaulted_columns{};
 
diff --git a/dbms/src/Storages/ITableDeclaration.cpp b/dbms/src/Storages/ITableDeclaration.cpp
index b5a6c035b96..783b0b2a7ed 100644
--- a/dbms/src/Storages/ITableDeclaration.cpp
+++ b/dbms/src/Storages/ITableDeclaration.cpp
@@ -24,14 +24,22 @@ namespace ErrorCodes
 }
 
 
+void ITableDeclaration::setColumns(ColumnsDescription columns_)
+{
+    if (columns_.ordinary.empty())
+        throw Exception("Empty list of columns passed", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED);
+    columns = std::move(columns_);
+}
+
+
 bool ITableDeclaration::hasColumn(const String & column_name) const
 {
-    return columns.has(column_name); /// By default, we assume that there are no virtual columns in the storage.
+    return getColumns().has(column_name); /// By default, we assume that there are no virtual columns in the storage.
 }
 
 NameAndTypePair ITableDeclaration::getColumn(const String & column_name) const
 {
-    return columns.get(column_name); /// By default, we assume that there are no virtual columns in the storage.
+    return getColumns().get(column_name); /// By default, we assume that there are no virtual columns in the storage.
 }
 
 
@@ -39,7 +47,7 @@ Block ITableDeclaration::getSampleBlock() const
 {
     Block res;
 
-    for (const auto & col : boost::join(columns.ordinary, columns.materialized))
+    for (const auto & col : boost::join(getColumns().ordinary, getColumns().materialized))
         res.insert({ col.type->createColumn(), col.type, col.name });
 
     return res;
@@ -50,7 +58,7 @@ Block ITableDeclaration::getSampleBlockNonMaterialized() const
 {
     Block res;
 
-    for (const auto & col : columns.ordinary)
+    for (const auto & col : getColumns().ordinary)
         res.insert({ col.type->createColumn(), col.type, col.name });
 
     return res;
@@ -111,7 +119,7 @@ static NamesAndTypesMap getColumnsMap(const Args &... args)
 
 void ITableDeclaration::check(const Names & column_names) const
 {
-    const NamesAndTypesList & available_columns = columns.getPhysical();
+    const NamesAndTypesList & available_columns = getColumns().getPhysical();
 
     if (column_names.empty())
         throw Exception("Empty list of columns queried. There are columns: " + listOfColumns(available_columns),
@@ -139,7 +147,7 @@ void ITableDeclaration::check(const Names & column_names) const
 
 void ITableDeclaration::check(const NamesAndTypesList & provided_columns) const
 {
-    const NamesAndTypesList & available_columns = columns.getPhysical();
+    const NamesAndTypesList & available_columns = getColumns().getPhysical();
     const auto columns_map = getColumnsMap(available_columns);
 
     using UniqueStrings = google::dense_hash_set<StringRef, StringRefHash>;
@@ -167,7 +175,7 @@ void ITableDeclaration::check(const NamesAndTypesList & provided_columns) const
 
 void ITableDeclaration::check(const NamesAndTypesList & provided_columns, const Names & column_names) const
 {
-    const NamesAndTypesList & available_columns = columns.getPhysical();
+    const NamesAndTypesList & available_columns = getColumns().getPhysical();
     const auto available_columns_map = getColumnsMap(available_columns);
     const NamesAndTypesMap & provided_columns_map = getColumnsMap(provided_columns);
 
@@ -204,7 +212,7 @@ void ITableDeclaration::check(const NamesAndTypesList & provided_columns, const
 
 void ITableDeclaration::check(const Block & block, bool need_all) const
 {
-    const NamesAndTypesList & available_columns = columns.getPhysical();
+    const NamesAndTypesList & available_columns = getColumns().getPhysical();
     const auto columns_map = getColumnsMap(available_columns);
 
     using NameSet = std::unordered_set<String>;
@@ -242,10 +250,8 @@ void ITableDeclaration::check(const Block & block, bool need_all) const
 
 
 ITableDeclaration::ITableDeclaration(ColumnsDescription columns_)
-    : columns{std::move(columns_)}
 {
-    if (columns.ordinary.empty())
-        throw Exception("Empty list of columns passed to storage constructor", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED);
+    setColumns(std::move(columns_));
 }
 
 }
diff --git a/dbms/src/Storages/ITableDeclaration.h b/dbms/src/Storages/ITableDeclaration.h
index 4c4ea59e4b5..270ab175576 100644
--- a/dbms/src/Storages/ITableDeclaration.h
+++ b/dbms/src/Storages/ITableDeclaration.h
@@ -12,12 +12,12 @@ namespace DB
 class ITableDeclaration
 {
 public:
-    /** Get a description of any column by its name.
-      */
-    virtual NameAndTypePair getColumn(const String & column_name) const;
+    virtual const ColumnsDescription & getColumns() const { return columns; }
+    virtual void setColumns(ColumnsDescription columns_);
 
-    /** Is there a column with that name.
-      */
+    /// NOTE: These methods should include virtual columns, but should NOT include ALIAS columns
+    /// (they are treated separately).
+    virtual NameAndTypePair getColumn(const String & column_name) const;
     virtual bool hasColumn(const String & column_name) const;
 
     Block getSampleBlock() const;
@@ -48,6 +48,7 @@ public:
     explicit ITableDeclaration(ColumnsDescription columns_);
     virtual ~ITableDeclaration() = default;
 
+private:
     ColumnsDescription columns;
 };
 
diff --git a/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp
index 6fb7c8fb611..a6d2958a786 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp
@@ -141,8 +141,9 @@ try
         if (!column_names.empty())
             storage.check(data_part->columns, column_names);
 
-        pre_columns = storage.columns.getPhysical().addTypes(pre_column_names);
-        columns = storage.columns.getPhysical().addTypes(column_names);
+        const NamesAndTypesList & physical_columns = storage.getColumns().getPhysical();
+        pre_columns = physical_columns.addTypes(pre_column_names);
+        columns = physical_columns.addTypes(column_names);
     }
     else
     {
diff --git a/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp
index b6d8ff3cd5a..5d52eedfda1 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp
@@ -26,9 +26,9 @@ NameSet injectRequiredColumns(const MergeTreeData & storage, const MergeTreeData
             continue;
         }
 
-        const auto default_it = storage.columns.defaults.find(column_name);
+        const auto default_it = storage.getColumns().defaults.find(column_name);
         /// columns has no explicit default expression
-        if (default_it == std::end(storage.columns.defaults))
+        if (default_it == std::end(storage.getColumns().defaults))
             continue;
 
         /// collect identifiers required for evaluation
diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp
index de16c22593d..ed16d1536de 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp
@@ -110,7 +110,7 @@ MergeTreeData::MergeTreeData(
     data_parts_by_state_and_info(data_parts_indexes.get<TagByStateAndInfo>())
 {
     /// NOTE: using the same columns list as is read when performing actual merges.
-    merging_params.check(columns.getPhysical());
+    merging_params.check(getColumns().getPhysical());
 
     if (!primary_expr_ast)
         throw Exception("Primary key cannot be empty", ErrorCodes::BAD_ARGUMENTS);
@@ -222,11 +222,11 @@ void MergeTreeData::initPrimaryKey()
     primary_sort_descr.clear();
     addSortDescription(primary_sort_descr, primary_expr_ast);
 
-    primary_expr = ExpressionAnalyzer(primary_expr_ast, context, nullptr, columns.getPhysical()).getActions(false);
+    primary_expr = ExpressionAnalyzer(primary_expr_ast, context, nullptr, getColumns().getPhysical()).getActions(false);
 
     {
         ExpressionActionsPtr projected_expr =
-                ExpressionAnalyzer(primary_expr_ast, context, nullptr, columns.getPhysical()).getActions(true);
+            ExpressionAnalyzer(primary_expr_ast, context, nullptr, getColumns().getPhysical()).getActions(true);
         primary_key_sample = projected_expr->getSampleBlock();
     }
 
@@ -241,10 +241,10 @@ void MergeTreeData::initPrimaryKey()
     if (secondary_sort_expr_ast)
     {
         addSortDescription(sort_descr, secondary_sort_expr_ast);
-        secondary_sort_expr = ExpressionAnalyzer(secondary_sort_expr_ast, context, nullptr, columns.getPhysical()).getActions(false);
+        secondary_sort_expr = ExpressionAnalyzer(secondary_sort_expr_ast, context, nullptr, getColumns().getPhysical()).getActions(false);
 
         ExpressionActionsPtr projected_expr =
-                ExpressionAnalyzer(secondary_sort_expr_ast, context, nullptr, columns.getPhysical()).getActions(true);
+            ExpressionAnalyzer(secondary_sort_expr_ast, context, nullptr, getColumns().getPhysical()).getActions(true);
         auto secondary_key_sample = projected_expr->getSampleBlock();
 
         checkKeyExpression(*secondary_sort_expr, secondary_key_sample, "Secondary");
@@ -257,7 +257,7 @@ void MergeTreeData::initPartitionKey()
     if (!partition_expr_ast || partition_expr_ast->children.empty())
         return;
 
-    partition_expr = ExpressionAnalyzer(partition_expr_ast, context, nullptr, columns.getPhysical()).getActions(false);
+    partition_expr = ExpressionAnalyzer(partition_expr_ast, context, nullptr, getColumns().getPhysical()).getActions(false);
     for (const ASTPtr & ast : partition_expr_ast->children)
     {
         String col_name = ast->getColumnName();
@@ -861,7 +861,7 @@ bool isMetadataOnlyConversion(const IDataType * from, const IDataType * to)
 void MergeTreeData::checkAlter(const AlterCommands & commands)
 {
     /// Check that needed transformations can be applied to the list of columns without considering type conversions.
-    auto new_columns = columns;
+    auto new_columns = getColumns();
     commands.apply(new_columns);
 
     /// Set of columns that shouldn't be altered.
@@ -903,7 +903,7 @@ void MergeTreeData::checkAlter(const AlterCommands & commands)
         columns_alter_forbidden.insert(merging_params.sign_column);
 
     std::map<String, const IDataType *> old_types;
-    for (const auto & column : columns.getPhysical())
+    for (const auto & column : getColumns().getPhysical())
         old_types.emplace(column.name, column.type.get());
 
     for (const AlterCommand & command : commands)
@@ -931,7 +931,7 @@ void MergeTreeData::checkAlter(const AlterCommands & commands)
     NameToNameMap unused_map;
     bool unused_bool;
 
-    createConvertExpression(nullptr, columns.getPhysical(), new_columns.getPhysical(), unused_expression, unused_map, unused_bool);
+    createConvertExpression(nullptr, getColumns().getPhysical(), new_columns.getPhysical(), unused_expression, unused_map, unused_bool);
 }
 
 void MergeTreeData::createConvertExpression(const DataPartPtr & part, const NamesAndTypesList & old_columns, const NamesAndTypesList & new_columns,
@@ -1835,7 +1835,7 @@ void MergeTreeData::addPartContributionToColumnSizes(const DataPartPtr & part)
     const auto & files = part->checksums.files;
 
     /// TODO This method doesn't take into account columns with multiple files.
-    for (const auto & column : columns.getPhysical())
+    for (const auto & column : getColumns().getPhysical())
     {
         const auto escaped_name = escapeForFileName(column.name);
         const auto bin_file_name = escaped_name + ".bin";
@@ -1868,7 +1868,7 @@ void MergeTreeData::removePartContributionToColumnSizes(const DataPartPtr & part
     const auto & files = part->checksums.files;
 
     /// TODO This method doesn't take into account columns with multiple files.
-    for (const auto & column : columns.getPhysical())
+    for (const auto & column : getColumns().getPhysical())
     {
         const auto escaped_name = escapeForFileName(column.name);
         const auto bin_file_name = escaped_name + ".bin";
diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h
index a32baf797ef..da6999ac965 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeData.h
+++ b/dbms/src/Storages/MergeTree/MergeTreeData.h
@@ -313,12 +313,12 @@ public:
         if (column_name == "_sample_factor")
             return NameAndTypePair("_sample_factor", std::make_shared<DataTypeFloat64>());
 
-        return columns.get(column_name);
+        return getColumns().get(column_name);
     }
 
     bool hasColumn(const String & column_name) const override
     {
-        return columns.has(column_name)
+        return getColumns().has(column_name)
             || column_name == "_part"
             || column_name == "_part_index"
             || column_name == "_sample_factor";
diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp
index 8e30320aefb..dafa27e2346 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp
@@ -536,8 +536,8 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMerger::mergePartsToTemporaryPart
     for (const MergeTreeData::DataPartPtr & part : parts)
         part->accumulateColumnSizes(merged_column_to_size);
 
-    Names all_column_names = data.columns.getNames();
-    NamesAndTypesList all_columns = data.columns.getPhysical();
+    Names all_column_names = data.getColumns().getNames();
+    NamesAndTypesList all_columns = data.getColumns().getPhysical();
     const SortDescription sort_desc = data.getSortDescription();
 
     NamesAndTypesList gathering_columns, merging_columns;
diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp
index f37cd8e579c..92936e2adc7 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp
@@ -418,7 +418,7 @@ UInt64 MergeTreeDataPart::getColumnMrkSize(const String & name) const
   */
 String MergeTreeDataPart::getColumnNameWithMinumumCompressedSize() const
 {
-    const auto & columns = storage.columns.getPhysical();
+    const auto & columns = storage.getColumns().getPhysical();
     const std::string * minimum_size_column = nullptr;
     UInt64 minimum_size = std::numeric_limits<UInt64>::max();
 
@@ -774,7 +774,7 @@ void MergeTreeDataPart::accumulateColumnSizes(ColumnToSize & column_to_size) con
 {
     std::shared_lock<std::shared_mutex> part_lock(columns_lock);
 
-    for (const NameAndTypePair & name_type : storage.columns.getPhysical())
+    for (const NameAndTypePair & name_type : storage.getColumns().getPhysical())
     {
         name_type.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path)
         {
@@ -794,7 +794,7 @@ void MergeTreeDataPart::loadColumns(bool require)
             throw Exception("No columns.txt in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART);
 
         /// If there is no file with a list of columns, write it down.
-        for (const NameAndTypePair & column : storage.columns.getPhysical())
+        for (const NameAndTypePair & column : storage.getColumns().getPhysical())
             if (Poco::File(getFullPath() + escapeForFileName(column.name) + ".bin").exists())
                 columns.push_back(column);
 
diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
index 0890163078d..77ba844ee0b 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
@@ -175,7 +175,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::read(
         }
     }
 
-    NamesAndTypesList available_real_columns = data.columns.getPhysical();
+    NamesAndTypesList available_real_columns = data.getColumns().getPhysical();
 
     NamesAndTypesList available_real_and_virtual_columns = available_real_columns;
     for (const auto & name : virt_column_names)
@@ -857,7 +857,7 @@ void MergeTreeDataSelectExecutor::createPositiveSignCondition(
     arguments->children.push_back(sign);
     arguments->children.push_back(one);
 
-    out_expression = ExpressionAnalyzer(function, context, {}, data.columns.getPhysical()).getActions(false);
+    out_expression = ExpressionAnalyzer(function, context, {}, data.getColumns().getPhysical()).getActions(false);
     out_column = function->getColumnName();
 }
 
diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp
index 19eb2e8eecd..c57d1d5bfdd 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp
@@ -205,7 +205,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa
     ///  either default lz4 or compression method with zero thresholds on absolute and relative part size.
     auto compression_settings = data.context.chooseCompressionSettings(0, 0);
 
-    NamesAndTypesList columns = data.columns.getPhysical().filter(block.getNames());
+    NamesAndTypesList columns = data.getColumns().getPhysical().filter(block.getNames());
     MergedBlockOutputStream out(data, new_data_part->getFullPath(), columns, compression_settings);
 
     out.writePrefix();
diff --git a/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp b/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp
index 2c600dbd32d..452aee79bf3 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp
@@ -238,8 +238,9 @@ std::vector<size_t> MergeTreeReadPool::fillPerPartInfo(
             if (!required_column_names.empty())
                 data.check(part.data_part->columns, required_column_names);
 
-            per_part_pre_columns.push_back(data.columns.getPhysical().addTypes(required_pre_column_names));
-            per_part_columns.push_back(data.columns.getPhysical().addTypes(required_column_names));
+            const NamesAndTypesList & physical_columns = data.getColumns().getPhysical();
+            per_part_pre_columns.push_back(physical_columns.addTypes(required_pre_column_names));
+            per_part_columns.push_back(physical_columns.addTypes(required_column_names));
         }
         else
         {
diff --git a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp
index 7bce149776a..e172760b17f 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp
@@ -463,7 +463,7 @@ void MergeTreeReader::fillMissingColumns(Block & res, const Names & ordered_name
             if (!has_column)
             {
                 should_sort = true;
-                if (storage.columns.defaults.count(requested_column.name) != 0)
+                if (storage.getColumns().defaults.count(requested_column.name) != 0)
                 {
                     should_evaluate_defaults = true;
                     continue;
@@ -498,7 +498,7 @@ void MergeTreeReader::fillMissingColumns(Block & res, const Names & ordered_name
 
         /// evaluate defaulted columns if necessary
         if (should_evaluate_defaults)
-            evaluateMissingDefaults(res, columns, storage.columns.defaults, storage.context);
+            evaluateMissingDefaults(res, columns, storage.getColumns().defaults, storage.context);
 
         /// sort columns to ensure consistent order among all blocks
         if (should_sort)
diff --git a/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp
index ce4047fc85f..ac28d3e782d 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp
@@ -41,9 +41,9 @@ MergeTreeWhereOptimizer::MergeTreeWhereOptimizer(
     Logger * log)
         : primary_key_columns{ext::map<std::unordered_set>(data.getPrimarySortDescription(),
             [] (const SortColumnDescription & col) { return col.column_name; })},
-        table_columns{ext::map<std::unordered_set>(data.columns.getPhysical(),
+        table_columns{ext::map<std::unordered_set>(data.getColumns().getPhysical(),
             [] (const NameAndTypePair & col) { return col.name; })},
-        block_with_constants{PKCondition::getBlockWithConstants(query_info.query, context, data.columns.getPhysical())},
+        block_with_constants{PKCondition::getBlockWithConstants(query_info.query, context, data.getColumns().getPhysical())},
         prepared_sets(query_info.sets),
         log{log}
 {
diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp
index 948902d8302..1fcba112fed 100644
--- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp
+++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp
@@ -87,15 +87,13 @@ void ReplicatedMergeTreeAlterThread::run()
 
                     auto table_lock = storage.lockStructureForAlter(__PRETTY_FUNCTION__);
 
-                    if (columns_in_zk != storage.data.columns)
+                    if (columns_in_zk != storage.getColumns())
                     {
                         LOG_INFO(log, "Columns list changed in ZooKeeper. Applying changes locally.");
 
                         storage.context.getDatabase(storage.database_name)->alterTable(
                             storage.context, storage.table_name, columns_in_zk, {});
-
-                        storage.columns = columns_in_zk;
-                        storage.data.columns = std::move(columns_in_zk);
+                        storage.setColumns(std::move(columns_in_zk));
 
                         /// Reinitialize primary key because primary key column types might have changed.
                         storage.data.initPrimaryKey();
@@ -126,7 +124,7 @@ void ReplicatedMergeTreeAlterThread::run()
                     if (!changed_version)
                         parts = storage.data.getDataParts();
 
-                    const auto columns_for_parts = storage.data.columns.getPhysical();
+                    const auto columns_for_parts = storage.getColumns().getPhysical();
 
                     for (const MergeTreeData::DataPartPtr & part : parts)
                     {
diff --git a/dbms/src/Storages/StorageBuffer.cpp b/dbms/src/Storages/StorageBuffer.cpp
index c76a1edc81b..2072ac72c5c 100644
--- a/dbms/src/Storages/StorageBuffer.cpp
+++ b/dbms/src/Storages/StorageBuffer.cpp
@@ -595,8 +595,10 @@ void StorageBuffer::alter(const AlterCommands & params, const String & database_
     /// So that no blocks of the old structure remain.
     optimize({} /*query*/, {} /*partition_id*/, false /*final*/, false /*deduplicate*/, context);
 
-    params.apply(columns);
-    context.getDatabase(database_name)->alterTable(context, table_name, columns, {});
+    ColumnsDescription new_columns = getColumns();
+    params.apply(new_columns);
+    context.getDatabase(database_name)->alterTable(context, table_name, new_columns, {});
+    setColumns(std::move(new_columns));
 }
 
 
diff --git a/dbms/src/Storages/StorageCatBoostPool.cpp b/dbms/src/Storages/StorageCatBoostPool.cpp
index d20da89b170..9fa8133617c 100644
--- a/dbms/src/Storages/StorageCatBoostPool.cpp
+++ b/dbms/src/Storages/StorageCatBoostPool.cpp
@@ -220,6 +220,7 @@ void StorageCatBoostPool::parseColumnDescription()
 
 void StorageCatBoostPool::createSampleBlockAndColumns()
 {
+    ColumnsDescription columns;
     NamesAndTypesList cat_columns;
     NamesAndTypesList num_columns;
     sample_block.clear();
@@ -251,6 +252,8 @@ void StorageCatBoostPool::createSampleBlockAndColumns()
     }
     columns.ordinary.insert(columns.ordinary.end(), num_columns.begin(), num_columns.end());
     columns.ordinary.insert(columns.ordinary.end(), cat_columns.begin(), cat_columns.end());
+
+    setColumns(columns);
 }
 
 BlockInputStreams StorageCatBoostPool::read(const Names & column_names,
diff --git a/dbms/src/Storages/StorageDictionary.cpp b/dbms/src/Storages/StorageDictionary.cpp
index 954b8ae898b..b3dbf39e9c2 100644
--- a/dbms/src/Storages/StorageDictionary.cpp
+++ b/dbms/src/Storages/StorageDictionary.cpp
@@ -72,7 +72,7 @@ void StorageDictionary::checkNamesAndTypesCompatibleWithDictionary(const Diction
     auto dictionary_names_and_types = getNamesAndTypes(dictionary_structure);
     std::set<NameAndTypePair> namesAndTypesSet(dictionary_names_and_types.begin(), dictionary_names_and_types.end());
 
-    for (auto & column : columns.ordinary)
+    for (auto & column : getColumns().ordinary)
     {
         if (namesAndTypesSet.find(column) == namesAndTypesSet.end())
         {
diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp
index e3ff7f30369..972a4817009 100644
--- a/dbms/src/Storages/StorageDistributed.cpp
+++ b/dbms/src/Storages/StorageDistributed.cpp
@@ -142,7 +142,7 @@ StorageDistributed::StorageDistributed(
     table_name(table_name_),
     remote_database(remote_database_), remote_table(remote_table_),
     context(context_), cluster_name(context.getMacros().expand(cluster_name_)), has_sharding_key(sharding_key_),
-    sharding_key_expr(sharding_key_ ? ExpressionAnalyzer(sharding_key_, context, nullptr, columns.getPhysical()).getActions(false) : nullptr),
+      sharding_key_expr(sharding_key_ ? ExpressionAnalyzer(sharding_key_, context, nullptr, getColumns().getPhysical()).getActions(false) : nullptr),
     sharding_key_column_name(sharding_key_ ? sharding_key_->getColumnName() : String{}),
     path(data_path_.empty() ? "" : (data_path_ + escapeForFileName(table_name) + '/'))
 {
@@ -232,9 +232,11 @@ void StorageDistributed::alter(const AlterCommands & params, const String & data
             throw Exception("Storage engine " + getName() + " doesn't support primary key.", ErrorCodes::NOT_IMPLEMENTED);
 
     auto lock = lockStructureForAlter(__PRETTY_FUNCTION__);
-    params.apply(columns);
 
-    context.getDatabase(database_name)->alterTable(context, table_name, columns, {});
+    ColumnsDescription new_columns = getColumns();
+    params.apply(new_columns);
+    context.getDatabase(database_name)->alterTable(context, table_name, new_columns, {});
+    setColumns(std::move(new_columns));
 }
 
 
@@ -285,13 +287,13 @@ NameAndTypePair StorageDistributed::getColumn(const String & column_name) const
     if (const auto & type = VirtualColumnFactory::tryGetType(column_name))
         return { column_name, type };
 
-    return columns.get(column_name);
+    return getColumns().get(column_name);
 }
 
 
 bool StorageDistributed::hasColumn(const String & column_name) const
 {
-    return VirtualColumnFactory::hasColumn(column_name) || columns.has(column_name);
+    return VirtualColumnFactory::hasColumn(column_name) || getColumns().has(column_name);
 }
 
 void StorageDistributed::createDirectoryMonitors()
diff --git a/dbms/src/Storages/StorageJoin.cpp b/dbms/src/Storages/StorageJoin.cpp
index c3dfc16c5d7..746f15bab50 100644
--- a/dbms/src/Storages/StorageJoin.cpp
+++ b/dbms/src/Storages/StorageJoin.cpp
@@ -29,7 +29,7 @@ StorageJoin::StorageJoin(
     key_names(key_names_), kind(kind_), strictness(strictness_)
 {
     for (const auto & key : key_names)
-        if (!columns.has(key))
+        if (!getColumns().has(key))
             throw Exception{
                 "Key column (" + key + ") does not exist in table declaration.",
                 ErrorCodes::NO_SUCH_COLUMN_IN_TABLE};
diff --git a/dbms/src/Storages/StorageLog.cpp b/dbms/src/Storages/StorageLog.cpp
index 137c1d55616..c23d92de8fa 100644
--- a/dbms/src/Storages/StorageLog.cpp
+++ b/dbms/src/Storages/StorageLog.cpp
@@ -371,7 +371,7 @@ StorageLog::StorageLog(
      /// create files if they do not exist
     Poco::File(path + escapeForFileName(name) + '/').createDirectories();
 
-    for (const auto & column : columns.getPhysical())
+    for (const auto & column : getColumns().getPhysical())
         addFiles(column.name, *column.type);
 
     marks_file = Poco::File(path + escapeForFileName(name) + '/' + DBMS_STORAGE_LOG_MARKS_FILE_NAME);
@@ -465,8 +465,8 @@ void StorageLog::rename(const String & new_path_to_db, const String & /*new_data
 
 const StorageLog::Marks & StorageLog::getMarksWithRealRowCount() const
 {
-    const String & column_name = columns.ordinary.front().name;
-    const IDataType & column_type = *columns.ordinary.front().type;
+    const String & column_name = getColumns().ordinary.front().name;
+    const IDataType & column_type = *getColumns().ordinary.front().type;
     String filename;
 
     /** We take marks from first column.
@@ -499,7 +499,7 @@ BlockInputStreams StorageLog::read(
     processed_stage = QueryProcessingStage::FetchColumns;
     loadMarks();
 
-    NamesAndTypesList all_columns = Nested::collect(columns.getPhysical().addTypes(column_names));
+    NamesAndTypesList all_columns = Nested::collect(getColumns().getPhysical().addTypes(column_names));
 
     std::shared_lock<std::shared_mutex> lock(rwlock);
 
diff --git a/dbms/src/Storages/StorageMerge.cpp b/dbms/src/Storages/StorageMerge.cpp
index f26544975cd..6f29850c8d1 100644
--- a/dbms/src/Storages/StorageMerge.cpp
+++ b/dbms/src/Storages/StorageMerge.cpp
@@ -181,7 +181,7 @@ BlockInputStreams StorageMerge::read(
 
         /// If there are only virtual columns in query, you must request at least one other column.
         if (real_column_names.size() == 0)
-            real_column_names.push_back(ExpressionActions::getSmallestColumn(table->columns.getPhysical()));
+            real_column_names.push_back(ExpressionActions::getSmallestColumn(table->getColumns().getPhysical()));
 
         /// Substitute virtual column for its value when querying tables.
         ASTPtr modified_query_ast = query->clone();
@@ -334,8 +334,10 @@ void StorageMerge::alter(const AlterCommands & params, const String & database_n
 
     auto lock = lockStructureForAlter(__PRETTY_FUNCTION__);
 
-    params.apply(columns);
-    context.getDatabase(database_name)->alterTable(context, table_name, columns, {});
+    ColumnsDescription new_columns = getColumns();
+    params.apply(new_columns);
+    context.getDatabase(database_name)->alterTable(context, table_name, new_columns, {});
+    setColumns(new_columns);
 }
 
 
diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp
index 5e799c91909..f22f24082fb 100644
--- a/dbms/src/Storages/StorageMergeTree.cpp
+++ b/dbms/src/Storages/StorageMergeTree.cpp
@@ -46,8 +46,7 @@ StorageMergeTree::StorageMergeTree(
     const MergeTreeData::MergingParams & merging_params_,
     const MergeTreeSettings & settings_,
     bool has_force_restore_data_flag)
-    : IStorage{columns_},
-    path(path_), database_name(database_name_), table_name(table_name_), full_path(path + escapeForFileName(table_name) + '/'),
+    : path(path_), database_name(database_name_), table_name(table_name_), full_path(path + escapeForFileName(table_name) + '/'),
     context(context_), background_pool(context_.getBackgroundPool()),
     data(database_name, table_name,
          full_path, columns_,
@@ -153,8 +152,7 @@ void StorageMergeTree::alter(
 
     data.checkAlter(params);
 
-    auto new_columns = data.columns;
-
+    auto new_columns = data.getColumns();
     params.apply(new_columns);
 
     std::vector<MergeTreeData::AlterDataPartTransactionPtr> transactions;
@@ -203,9 +201,7 @@ void StorageMergeTree::alter(
     }
 
     context.getDatabase(database_name)->alterTable(context, table_name, new_columns, storage_modifier);
-
-    columns = new_columns;
-    data.columns = std::move(new_columns);
+    setColumns(std::move(new_columns));
 
     if (primary_key_is_modified)
     {
@@ -419,7 +415,7 @@ void StorageMergeTree::clearColumnInPartition(const ASTPtr & partition, const Fi
     alter_command.type = AlterCommand::DROP_COLUMN;
     alter_command.column_name = get<String>(column_name);
 
-    auto new_columns = data.columns;
+    auto new_columns = getColumns();
     alter_command.apply(new_columns);
 
     auto columns_for_parts = new_columns.getPhysical();
diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h
index e2ad6177a82..acb0e03a60a 100644
--- a/dbms/src/Storages/StorageMergeTree.h
+++ b/dbms/src/Storages/StorageMergeTree.h
@@ -36,6 +36,9 @@ public:
     bool supportsFinal() const override { return data.supportsFinal(); }
     bool supportsPrewhere() const override { return data.supportsPrewhere(); }
 
+    const ColumnsDescription & getColumns() const override { return data.getColumns(); }
+    void setColumns(ColumnsDescription columns_) override { return data.setColumns(std::move(columns_)); }
+
     NameAndTypePair getColumn(const String & column_name) const override
     {
         return data.getColumn(column_name);
diff --git a/dbms/src/Storages/StorageMySQL.cpp b/dbms/src/Storages/StorageMySQL.cpp
index 28a367f17e0..3c63ebe4ca5 100644
--- a/dbms/src/Storages/StorageMySQL.cpp
+++ b/dbms/src/Storages/StorageMySQL.cpp
@@ -43,7 +43,7 @@ BlockInputStreams StorageMySQL::read(
 {
     check(column_names);
     processed_stage = QueryProcessingStage::FetchColumns;
-    String query = transformQueryForExternalDatabase(*query_info.query, columns.ordinary, remote_database_name, remote_table_name, context);
+    String query = transformQueryForExternalDatabase(*query_info.query, getColumns().ordinary, remote_database_name, remote_table_name, context);
 
     Block sample_block;
     for (const String & name : column_names)
diff --git a/dbms/src/Storages/StorageNull.cpp b/dbms/src/Storages/StorageNull.cpp
index 13232068ca9..25ecfecf2c9 100644
--- a/dbms/src/Storages/StorageNull.cpp
+++ b/dbms/src/Storages/StorageNull.cpp
@@ -33,9 +33,10 @@ void StorageNull::alter(const AlterCommands & params, const String & database_na
 {
     auto lock = lockStructureForAlter(__PRETTY_FUNCTION__);
 
-    params.apply(columns);
-
-    context.getDatabase(database_name)->alterTable(context, table_name, columns, {});
+    ColumnsDescription new_columns = getColumns();
+    params.apply(new_columns);
+    context.getDatabase(database_name)->alterTable(context, table_name, new_columns, {});
+    setColumns(std::move(new_columns));
 }
 
 }
diff --git a/dbms/src/Storages/StorageODBC.cpp b/dbms/src/Storages/StorageODBC.cpp
index 7856ee9a231..09791c0e314 100644
--- a/dbms/src/Storages/StorageODBC.cpp
+++ b/dbms/src/Storages/StorageODBC.cpp
@@ -43,7 +43,8 @@ BlockInputStreams StorageODBC::read(
 {
     check(column_names);
     processed_stage = QueryProcessingStage::FetchColumns;
-    String query = transformQueryForExternalDatabase(*query_info.query, columns.ordinary, remote_database_name, remote_table_name, context);
+    String query = transformQueryForExternalDatabase(
+        *query_info.query, getColumns().ordinary, remote_database_name, remote_table_name, context);
 
     Block sample_block;
     for (const String & name : column_names)
diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp
index d69db2d388e..c2cf65e1c85 100644
--- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp
@@ -184,7 +184,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree(
     const MergeTreeData::MergingParams & merging_params_,
     const MergeTreeSettings & settings_,
     bool has_force_restore_data_flag)
-    : IStorage{columns_}, context(context_),
+    : context(context_),
     current_zookeeper(context.getZooKeeper()), database_name(database_name_),
     table_name(name_), full_path(path_ + escapeForFileName(table_name) + '/'),
     zookeeper_path(context.getMacros().expand(zookeeper_path_)),
@@ -484,7 +484,7 @@ void StorageReplicatedMergeTree::createTableIfNotExists()
         acl, zkutil::CreateMode::Persistent));
     ops.emplace_back(std::make_unique<zkutil::Op::Create>(zookeeper_path + "/metadata", metadata,
         acl, zkutil::CreateMode::Persistent));
-    ops.emplace_back(std::make_unique<zkutil::Op::Create>(zookeeper_path + "/columns", data.columns.toString(),
+    ops.emplace_back(std::make_unique<zkutil::Op::Create>(zookeeper_path + "/columns", getColumns().toString(),
         acl, zkutil::CreateMode::Persistent));
     ops.emplace_back(std::make_unique<zkutil::Op::Create>(zookeeper_path + "/log", "",
         acl, zkutil::CreateMode::Persistent));
@@ -519,23 +519,22 @@ void StorageReplicatedMergeTree::checkTableStructure(bool skip_sanity_checks, bo
 
     zkutil::Stat stat;
     auto columns_from_zk = ColumnsDescription::parse(zookeeper->get(zookeeper_path + "/columns", &stat));
-
     columns_version = stat.version;
 
-    if (columns_from_zk != data.columns)
+    const ColumnsDescription & old_columns = getColumns();
+    if (columns_from_zk != old_columns)
     {
         if (allow_alter &&
             (skip_sanity_checks ||
-             data.columns.ordinary.sizeOfDifference(columns_from_zk.ordinary) +
-             data.columns.materialized.sizeOfDifference(columns_from_zk.materialized) <= 2))
+             old_columns.ordinary.sizeOfDifference(columns_from_zk.ordinary) +
+             old_columns.materialized.sizeOfDifference(columns_from_zk.materialized) <= 2))
         {
             LOG_WARNING(log, "Table structure in ZooKeeper is a little different from local table structure. Assuming ALTER.");
 
             /// Without any locks, because table has not been created yet.
             context.getDatabase(database_name)->alterTable(context, table_name, columns_from_zk, {});
 
-            columns = columns_from_zk;
-            data.columns = std::move(columns_from_zk);
+            setColumns(std::move(columns_from_zk));
         }
         else
         {
@@ -697,7 +696,7 @@ void StorageReplicatedMergeTree::createReplica()
         LOG_DEBUG(log, "Copied " << source_queue.size() << " queue entries");
     }
 
-    zookeeper->create(replica_path + "/columns", data.columns.toString(), zkutil::CreateMode::Persistent);
+    zookeeper->create(replica_path + "/columns", getColumns().toString(), zkutil::CreateMode::Persistent);
 }
 
 
@@ -1507,7 +1506,7 @@ void StorageReplicatedMergeTree::executeClearColumnInPartition(const LogEntry &
     alter_command.type = AlterCommand::DROP_COLUMN;
     alter_command.column_name = entry.column_name;
 
-    auto new_columns = data.columns;
+    auto new_columns = getColumns();
     alter_command.apply(new_columns);
 
     size_t modified_parts = 0;
@@ -2544,7 +2543,7 @@ void StorageReplicatedMergeTree::alter(const AlterCommands & params,
             if (param.type == AlterCommand::MODIFY_PRIMARY_KEY)
                 throw Exception("Modification of primary key is not supported for replicated tables", ErrorCodes::NOT_IMPLEMENTED);
 
-        ColumnsDescription new_columns = data.columns;
+        ColumnsDescription new_columns = data.getColumns();
         params.apply(new_columns);
 
         new_columns_str = new_columns.toString();
diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h
index 468f097534d..4a2193b05db 100644
--- a/dbms/src/Storages/StorageReplicatedMergeTree.h
+++ b/dbms/src/Storages/StorageReplicatedMergeTree.h
@@ -84,6 +84,9 @@ public:
     bool supportsPrewhere() const override { return data.supportsPrewhere(); }
     bool supportsReplication() const override { return true; }
 
+    const ColumnsDescription & getColumns() const override { return data.getColumns(); }
+    void setColumns(ColumnsDescription columns_) override { return data.setColumns(std::move(columns_)); }
+
     NameAndTypePair getColumn(const String & column_name) const override
     {
         return data.getColumn(column_name);
diff --git a/dbms/src/Storages/StorageTinyLog.cpp b/dbms/src/Storages/StorageTinyLog.cpp
index 8c7fe1963a8..bd4a8ca304f 100644
--- a/dbms/src/Storages/StorageTinyLog.cpp
+++ b/dbms/src/Storages/StorageTinyLog.cpp
@@ -296,7 +296,7 @@ StorageTinyLog::StorageTinyLog(
             throwFromErrno("Cannot create directory " + full_path, ErrorCodes::CANNOT_CREATE_DIRECTORY);
     }
 
-    for (const auto & col : columns.getPhysical())
+    for (const auto & col : getColumns().getPhysical())
         addFiles(col.name, *col.type);
 }
 
@@ -348,7 +348,7 @@ BlockInputStreams StorageTinyLog::read(
     check(column_names);
     processed_stage = QueryProcessingStage::FetchColumns;
     return BlockInputStreams(1, std::make_shared<TinyLogBlockInputStream>(
-        max_block_size, Nested::collect(columns.getPhysical().addTypes(column_names)), *this, context.getSettingsRef().max_read_buffer_size));
+        max_block_size, Nested::collect(getColumns().getPhysical().addTypes(column_names)), *this, context.getSettingsRef().max_read_buffer_size));
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.cpp b/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.cpp
index 3c77244c557..bc2f76379e9 100644
--- a/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.cpp
+++ b/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.cpp
@@ -16,10 +16,10 @@ StorageSystemAsynchronousMetrics::StorageSystemAsynchronousMetrics(const std::st
     : name(name_),
     async_metrics(async_metrics_)
 {
-    columns = ColumnsDescription({
+    setColumns(ColumnsDescription({
         {"metric", std::make_shared<DataTypeString>()},
         {"value", std::make_shared<DataTypeFloat64>()},
-    });
+    }));
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemBuildOptions.cpp b/dbms/src/Storages/System/StorageSystemBuildOptions.cpp
index f354da97944..e62e6e9bbfd 100644
--- a/dbms/src/Storages/System/StorageSystemBuildOptions.cpp
+++ b/dbms/src/Storages/System/StorageSystemBuildOptions.cpp
@@ -13,10 +13,10 @@ namespace DB
 StorageSystemBuildOptions::StorageSystemBuildOptions(const std::string & name_)
     : name(name_)
 {
-    columns = ColumnsDescription({
+    setColumns(ColumnsDescription({
         { "name", std::make_shared<DataTypeString>() },
         { "value", std::make_shared<DataTypeString>() },
-    });
+    }));
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemClusters.cpp b/dbms/src/Storages/System/StorageSystemClusters.cpp
index 1e67e433d48..904d22d180e 100644
--- a/dbms/src/Storages/System/StorageSystemClusters.cpp
+++ b/dbms/src/Storages/System/StorageSystemClusters.cpp
@@ -14,7 +14,7 @@ namespace DB
 StorageSystemClusters::StorageSystemClusters(const std::string & name_)
     : name(name_)
 {
-    columns = ColumnsDescription({
+    setColumns(ColumnsDescription({
         { "cluster",      std::make_shared<DataTypeString>() },
         { "shard_num",    std::make_shared<DataTypeUInt32>() },
         { "shard_weight", std::make_shared<DataTypeUInt32>() },
@@ -25,7 +25,7 @@ StorageSystemClusters::StorageSystemClusters(const std::string & name_)
         { "is_local",     std::make_shared<DataTypeUInt8>() },
         { "user",         std::make_shared<DataTypeString>() },
         { "default_database", std::make_shared<DataTypeString>() },
-    });
+    }));
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemColumns.cpp b/dbms/src/Storages/System/StorageSystemColumns.cpp
index 83c5dca9117..18598b1c1dc 100644
--- a/dbms/src/Storages/System/StorageSystemColumns.cpp
+++ b/dbms/src/Storages/System/StorageSystemColumns.cpp
@@ -18,7 +18,7 @@ namespace DB
 StorageSystemColumns::StorageSystemColumns(const std::string & name_)
     : name(name_)
 {
-    columns = ColumnsDescription({
+    setColumns(ColumnsDescription({
         { "database",           std::make_shared<DataTypeString>() },
         { "table",              std::make_shared<DataTypeString>() },
         { "name",               std::make_shared<DataTypeString>() },
@@ -28,7 +28,7 @@ StorageSystemColumns::StorageSystemColumns(const std::string & name_)
         { "data_compressed_bytes",      std::make_shared<DataTypeUInt64>() },
         { "data_uncompressed_bytes",    std::make_shared<DataTypeUInt64>() },
         { "marks_bytes",                std::make_shared<DataTypeUInt64>() },
-    });
+    }));
 }
 
 
@@ -137,8 +137,8 @@ BlockInputStreams StorageSystemColumns::read(
                     throw;
             }
 
-            columns = storage->columns.getAll();
-            column_defaults = storage->columns.defaults;
+            columns = storage->getColumns().getAll();
+            column_defaults = storage->getColumns().defaults;
 
             /** Info about sizes of columns for tables of MergeTree family.
               * NOTE: It is possible to add getter for this info to IStorage interface.
diff --git a/dbms/src/Storages/System/StorageSystemDatabases.cpp b/dbms/src/Storages/System/StorageSystemDatabases.cpp
index 79a3b3483a6..49c78688616 100644
--- a/dbms/src/Storages/System/StorageSystemDatabases.cpp
+++ b/dbms/src/Storages/System/StorageSystemDatabases.cpp
@@ -13,12 +13,12 @@ namespace DB
 StorageSystemDatabases::StorageSystemDatabases(const std::string & name_)
     : name(name_)
 {
-    columns = ColumnsDescription({
+    setColumns(ColumnsDescription({
         {"name", std::make_shared<DataTypeString>()},
         {"engine", std::make_shared<DataTypeString>()},
         {"data_path", std::make_shared<DataTypeString>()},
         {"metadata_path", std::make_shared<DataTypeString>()},
-    });
+    }));
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemDictionaries.cpp b/dbms/src/Storages/System/StorageSystemDictionaries.cpp
index 84bf4a2c98f..c57c1c7f459 100644
--- a/dbms/src/Storages/System/StorageSystemDictionaries.cpp
+++ b/dbms/src/Storages/System/StorageSystemDictionaries.cpp
@@ -21,7 +21,7 @@ namespace DB
 StorageSystemDictionaries::StorageSystemDictionaries(const std::string & name)
     : name{name}
 {
-    columns = ColumnsDescription({
+    setColumns(ColumnsDescription({
         { "name", std::make_shared<DataTypeString>() },
         { "origin", std::make_shared<DataTypeString>() },
         { "type", std::make_shared<DataTypeString>() },
@@ -36,7 +36,7 @@ StorageSystemDictionaries::StorageSystemDictionaries(const std::string & name)
         { "creation_time", std::make_shared<DataTypeDateTime>() },
         { "source", std::make_shared<DataTypeString>() },
         { "last_exception", std::make_shared<DataTypeString>() },
-    });
+    }));
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemEvents.cpp b/dbms/src/Storages/System/StorageSystemEvents.cpp
index ac613a7e84e..1dc49ad37b2 100644
--- a/dbms/src/Storages/System/StorageSystemEvents.cpp
+++ b/dbms/src/Storages/System/StorageSystemEvents.cpp
@@ -13,11 +13,11 @@ namespace DB
 StorageSystemEvents::StorageSystemEvents(const std::string & name_)
     : name(name_)
 {
-    columns = ColumnsDescription(
+    setColumns(ColumnsDescription(
     {
         {"event", std::make_shared<DataTypeString>()},
         {"value", std::make_shared<DataTypeUInt64>()},
-    });
+    }));
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemFunctions.cpp b/dbms/src/Storages/System/StorageSystemFunctions.cpp
index 2d42bee9a46..909bf1d8089 100644
--- a/dbms/src/Storages/System/StorageSystemFunctions.cpp
+++ b/dbms/src/Storages/System/StorageSystemFunctions.cpp
@@ -16,10 +16,10 @@ namespace DB
 StorageSystemFunctions::StorageSystemFunctions(const std::string & name_)
     : name(name_)
 {
-    columns = ColumnsDescription({
+    setColumns(ColumnsDescription({
         { "name",         std::make_shared<DataTypeString>() },
         { "is_aggregate", std::make_shared<DataTypeUInt8>()  },
-    });
+    }));
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemGraphite.cpp b/dbms/src/Storages/System/StorageSystemGraphite.cpp
index 30cc56c55e3..f558adc1d95 100644
--- a/dbms/src/Storages/System/StorageSystemGraphite.cpp
+++ b/dbms/src/Storages/System/StorageSystemGraphite.cpp
@@ -125,7 +125,7 @@ static Strings getAllGraphiteSections(const AbstractConfiguration & config)
 StorageSystemGraphite::StorageSystemGraphite(const std::string & name_)
     : name(name_)
 {
-    columns = ColumnsDescription({
+    setColumns(ColumnsDescription({
         {"config_name", std::make_shared<DataTypeString>()},
         {"regexp",      std::make_shared<DataTypeString>()},
         {"function",    std::make_shared<DataTypeString>()},
@@ -133,7 +133,7 @@ StorageSystemGraphite::StorageSystemGraphite(const std::string & name_)
         {"precision",   std::make_shared<DataTypeUInt64>()},
         {"priority",    std::make_shared<DataTypeUInt16>()},
         {"is_default",  std::make_shared<DataTypeUInt8>()},
-    });
+    }));
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemMerges.cpp b/dbms/src/Storages/System/StorageSystemMerges.cpp
index c32cad1265b..d3af993e29e 100644
--- a/dbms/src/Storages/System/StorageSystemMerges.cpp
+++ b/dbms/src/Storages/System/StorageSystemMerges.cpp
@@ -14,7 +14,7 @@ namespace DB
 StorageSystemMerges::StorageSystemMerges(const std::string & name)
     : name{name}
 {
-    columns = ColumnsDescription({
+    setColumns(ColumnsDescription({
         { "database",                    std::make_shared<DataTypeString>() },
         { "table",                       std::make_shared<DataTypeString>() },
         { "elapsed",                     std::make_shared<DataTypeFloat64>() },
@@ -31,7 +31,7 @@ StorageSystemMerges::StorageSystemMerges(const std::string & name)
         { "columns_written",             std::make_shared<DataTypeUInt64>() },
         { "memory_usage",                std::make_shared<DataTypeUInt64>() },
         { "thread_number",               std::make_shared<DataTypeUInt64>() },
-    });
+    }));
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemMetrics.cpp b/dbms/src/Storages/System/StorageSystemMetrics.cpp
index 57935b13fea..9d3b1cc9fbc 100644
--- a/dbms/src/Storages/System/StorageSystemMetrics.cpp
+++ b/dbms/src/Storages/System/StorageSystemMetrics.cpp
@@ -14,10 +14,10 @@ namespace DB
 StorageSystemMetrics::StorageSystemMetrics(const std::string & name_)
     : name(name_)
 {
-    columns = ColumnsDescription({
+    setColumns(ColumnsDescription({
         {"metric", std::make_shared<DataTypeString>()},
         {"value",  std::make_shared<DataTypeInt64>()},
-    });
+    }));
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemModels.cpp b/dbms/src/Storages/System/StorageSystemModels.cpp
index 6ef9d157303..5175989b861 100644
--- a/dbms/src/Storages/System/StorageSystemModels.cpp
+++ b/dbms/src/Storages/System/StorageSystemModels.cpp
@@ -14,13 +14,13 @@ namespace DB
 StorageSystemModels::StorageSystemModels(const std::string & name)
     : name{name}
 {
-    columns = ColumnsDescription({
+    setColumns(ColumnsDescription({
         { "name", std::make_shared<DataTypeString>() },
         { "origin", std::make_shared<DataTypeString>() },
         { "type", std::make_shared<DataTypeString>() },
         { "creation_time", std::make_shared<DataTypeDateTime>() },
         { "last_exception", std::make_shared<DataTypeString>() },
-    });
+    }));
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemNumbers.cpp b/dbms/src/Storages/System/StorageSystemNumbers.cpp
index d5e72bcbdc3..49a3c7ca513 100644
--- a/dbms/src/Storages/System/StorageSystemNumbers.cpp
+++ b/dbms/src/Storages/System/StorageSystemNumbers.cpp
@@ -47,7 +47,7 @@ private:
 StorageSystemNumbers::StorageSystemNumbers(const std::string & name_, bool multithreaded_, size_t limit_)
     : name(name_), multithreaded(multithreaded_), limit(limit_)
 {
-    columns = ColumnsDescription({{"number", std::make_shared<DataTypeUInt64>()}});
+    setColumns(ColumnsDescription({{"number", std::make_shared<DataTypeUInt64>()}}));
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemOne.cpp b/dbms/src/Storages/System/StorageSystemOne.cpp
index 9cd74a7525c..7714a0ea53c 100644
--- a/dbms/src/Storages/System/StorageSystemOne.cpp
+++ b/dbms/src/Storages/System/StorageSystemOne.cpp
@@ -13,7 +13,7 @@ namespace DB
 StorageSystemOne::StorageSystemOne(const std::string & name_)
     : name(name_)
 {
-    columns = ColumnsDescription({{"dummy", std::make_shared<DataTypeUInt8>()}});
+    setColumns(ColumnsDescription({{"dummy", std::make_shared<DataTypeUInt8>()}}));
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemPartsBase.cpp b/dbms/src/Storages/System/StorageSystemPartsBase.cpp
index 5143b73955f..9d94655fc18 100644
--- a/dbms/src/Storages/System/StorageSystemPartsBase.cpp
+++ b/dbms/src/Storages/System/StorageSystemPartsBase.cpp
@@ -280,7 +280,7 @@ bool StorageSystemPartsBase::hasColumn(const String & column_name) const
 StorageSystemPartsBase::StorageSystemPartsBase(std::string name_, NamesAndTypesList && columns_)
     : name(std::move(name_))
 {
-    columns = ColumnsDescription(std::move(columns_));
+    setColumns(ColumnsDescription(std::move(columns_)));
 }
 
 }
diff --git a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp
index d804504f13a..8e55625c5cd 100644
--- a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp
+++ b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp
@@ -61,8 +61,8 @@ void StorageSystemPartsColumns::processNextStorage(MutableColumns & columns, con
         String default_expression;
     };
 
-    NamesAndTypesList columns_list = info.storage->columns.getAll();
-    const auto & column_defaults = info.storage->columns.defaults;
+    NamesAndTypesList columns_list = info.storage->getColumns().getAll();
+    const auto & column_defaults = info.storage->getColumns().defaults;
     std::unordered_map<String, ColumnInfo> columns_info;
 
     for (const auto & column : columns_list)
diff --git a/dbms/src/Storages/System/StorageSystemProcesses.cpp b/dbms/src/Storages/System/StorageSystemProcesses.cpp
index 15ecc1a3d95..793e3124a2a 100644
--- a/dbms/src/Storages/System/StorageSystemProcesses.cpp
+++ b/dbms/src/Storages/System/StorageSystemProcesses.cpp
@@ -14,7 +14,7 @@ namespace DB
 StorageSystemProcesses::StorageSystemProcesses(const std::string & name_)
     : name(name_)
 {
-    columns = ColumnsDescription({
+    setColumns(ColumnsDescription({
         { "is_initial_query",     std::make_shared<DataTypeUInt8>() },
 
         { "user",                 std::make_shared<DataTypeString>() },
@@ -51,7 +51,7 @@ StorageSystemProcesses::StorageSystemProcesses(const std::string & name_)
         { "memory_usage",         std::make_shared<DataTypeInt64>() },
         { "peak_memory_usage",    std::make_shared<DataTypeInt64>() },
         { "query",                std::make_shared<DataTypeString>() },
-    });
+    }));
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemReplicas.cpp b/dbms/src/Storages/System/StorageSystemReplicas.cpp
index fdcc6984861..ea2c8b1cde2 100644
--- a/dbms/src/Storages/System/StorageSystemReplicas.cpp
+++ b/dbms/src/Storages/System/StorageSystemReplicas.cpp
@@ -17,7 +17,7 @@ namespace DB
 StorageSystemReplicas::StorageSystemReplicas(const std::string & name_)
     : name(name_)
 {
-    columns = ColumnsDescription({
+    setColumns(ColumnsDescription({
         { "database",                             std::make_shared<DataTypeString>()   },
         { "table",                                std::make_shared<DataTypeString>()   },
         { "engine",                               std::make_shared<DataTypeString>()   },
@@ -44,7 +44,7 @@ StorageSystemReplicas::StorageSystemReplicas(const std::string & name_)
         { "absolute_delay",                       std::make_shared<DataTypeUInt64>()   },
         { "total_replicas",                       std::make_shared<DataTypeUInt8>()    },
         { "active_replicas",                      std::make_shared<DataTypeUInt8>()    },
-    });
+    }));
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp b/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp
index b0ba714fef4..77b265ad065 100644
--- a/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp
+++ b/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp
@@ -20,7 +20,7 @@ namespace DB
 StorageSystemReplicationQueue::StorageSystemReplicationQueue(const std::string & name_)
     : name(name_)
 {
-    columns = ColumnsDescription({
+    setColumns(ColumnsDescription({
         /// Table properties.
         { "database",                std::make_shared<DataTypeString>() },
         { "table",                   std::make_shared<DataTypeString>() },
@@ -43,7 +43,7 @@ StorageSystemReplicationQueue::StorageSystemReplicationQueue(const std::string &
         { "num_postponed",           std::make_shared<DataTypeUInt32>() },
         { "postpone_reason",         std::make_shared<DataTypeString>() },
         { "last_postpone_time",      std::make_shared<DataTypeDateTime>() },
-    });
+    }));
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemSettings.cpp b/dbms/src/Storages/System/StorageSystemSettings.cpp
index d8a758becc7..efb50c559cc 100644
--- a/dbms/src/Storages/System/StorageSystemSettings.cpp
+++ b/dbms/src/Storages/System/StorageSystemSettings.cpp
@@ -14,12 +14,12 @@ namespace DB
 StorageSystemSettings::StorageSystemSettings(const std::string & name_)
     : name(name_)
 {
-    columns = ColumnsDescription({
+    setColumns(ColumnsDescription({
         { "name",        std::make_shared<DataTypeString>() },
         { "value",       std::make_shared<DataTypeString>() },
         { "changed",     std::make_shared<DataTypeUInt8>() },
         { "description", std::make_shared<DataTypeString>() },
-    });
+    }));
 }
 
 
diff --git a/dbms/src/Storages/System/StorageSystemTables.cpp b/dbms/src/Storages/System/StorageSystemTables.cpp
index 2db2a8052cc..99ed89739f9 100644
--- a/dbms/src/Storages/System/StorageSystemTables.cpp
+++ b/dbms/src/Storages/System/StorageSystemTables.cpp
@@ -114,7 +114,7 @@ protected:
 StorageSystemTables::StorageSystemTables(const std::string & name_)
     : name(name_)
 {
-    columns = ColumnsDescription(
+    setColumns(ColumnsDescription(
     {
         {"database", std::make_shared<DataTypeString>()},
         {"name", std::make_shared<DataTypeString>()},
@@ -122,7 +122,7 @@ StorageSystemTables::StorageSystemTables(const std::string & name_)
         {"is_temporary", std::make_shared<DataTypeUInt8>()},
         {"data_path", std::make_shared<DataTypeString>()},
         {"metadata_path", std::make_shared<DataTypeString>()},
-    });
+    }));
 
     virtual_columns =
     {
diff --git a/dbms/src/Storages/System/StorageSystemZooKeeper.cpp b/dbms/src/Storages/System/StorageSystemZooKeeper.cpp
index a5d5e2dab96..67660ea3446 100644
--- a/dbms/src/Storages/System/StorageSystemZooKeeper.cpp
+++ b/dbms/src/Storages/System/StorageSystemZooKeeper.cpp
@@ -22,7 +22,7 @@ namespace DB
 StorageSystemZooKeeper::StorageSystemZooKeeper(const std::string & name_)
     : name(name_)
 {
-    columns = ColumnsDescription({
+    setColumns(ColumnsDescription({
         { "name",           std::make_shared<DataTypeString>() },
         { "value",          std::make_shared<DataTypeString>() },
         { "czxid",          std::make_shared<DataTypeInt64>() },
@@ -37,7 +37,7 @@ StorageSystemZooKeeper::StorageSystemZooKeeper(const std::string & name_)
         { "numChildren",    std::make_shared<DataTypeInt32>() },
         { "pzxid",          std::make_shared<DataTypeInt64>() },
         { "path",           std::make_shared<DataTypeString>() },
-    });
+    }));
 }
 
 
diff --git a/dbms/src/TableFunctions/TableFunctionMerge.cpp b/dbms/src/TableFunctions/TableFunctionMerge.cpp
index 2a12dbca275..6b0710d4ba0 100644
--- a/dbms/src/TableFunctions/TableFunctionMerge.cpp
+++ b/dbms/src/TableFunctions/TableFunctionMerge.cpp
@@ -51,7 +51,7 @@ static NamesAndTypesList chooseColumns(const String & source_database, const Str
         throw Exception("Error while executing table function merge. In database " + source_database + " no one matches regular expression: "
             + table_name_regexp_, ErrorCodes::UNKNOWN_TABLE);
 
-    return any_table->columns.getPhysical();
+    return any_table->getColumns().getPhysical();
 }
 
 
diff --git a/dbms/src/TableFunctions/getStructureOfRemoteTable.cpp b/dbms/src/TableFunctions/getStructureOfRemoteTable.cpp
index 6eede709352..56bb257f252 100644
--- a/dbms/src/TableFunctions/getStructureOfRemoteTable.cpp
+++ b/dbms/src/TableFunctions/getStructureOfRemoteTable.cpp
@@ -28,7 +28,7 @@ ColumnsDescription getStructureOfRemoteTable(
     const auto & shard_info = cluster.getAnyShardInfo();
 
     if (shard_info.isLocal())
-        return context.getTable(database, table)->columns;
+        return context.getTable(database, table)->getColumns();
 
     /// Request for a table description
     String query = "DESC TABLE " + backQuoteIfNeed(database) + "." + backQuoteIfNeed(table);

From 5906c9a9886ad74e4d4a95ebb454fadc4bb8987a Mon Sep 17 00:00:00 2001
From: Alexey Zatelepin <ztlpn@yandex-team.ru>
Date: Tue, 13 Mar 2018 18:00:28 +0300
Subject: [PATCH 041/244] rename for clarity

---
 dbms/src/Analyzers/AnalyzeColumns.cpp         |  2 +-
 dbms/src/Analyzers/ExecuteTableFunctions.cpp  |  2 +-
 dbms/src/Core/iostream_debug_helpers.cpp      |  2 +-
 .../DataStreams/RemoteBlockInputStream.cpp    |  2 +-
 dbms/src/Functions/FunctionsMiscellaneous.cpp |  5 ++---
 dbms/src/Interpreters/ExpressionAnalyzer.cpp  |  2 +-
 .../Interpreters/InterpreterAlterQuery.cpp    |  2 +-
 .../Interpreters/InterpreterInsertQuery.cpp   |  2 +-
 dbms/src/Storages/ColumnsDescription.cpp      |  8 ++++----
 dbms/src/Storages/ColumnsDescription.h        | 11 ++++------
 dbms/src/Storages/ITableDeclaration.cpp       | 12 +++++------
 .../MergeTree/MergeTreeBlockInputStream.cpp   |  2 +-
 dbms/src/Storages/MergeTree/MergeTreeData.cpp | 20 +++++++++----------
 dbms/src/Storages/MergeTree/MergeTreeData.h   |  4 ++--
 .../MergeTree/MergeTreeDataMerger.cpp         |  4 ++--
 .../Storages/MergeTree/MergeTreeDataPart.cpp  |  6 +++---
 .../MergeTree/MergeTreeDataSelectExecutor.cpp |  4 ++--
 .../MergeTree/MergeTreeDataWriter.cpp         |  2 +-
 .../Storages/MergeTree/MergeTreeReadPool.cpp  |  2 +-
 .../MergeTree/MergeTreeWhereOptimizer.cpp     |  4 ++--
 .../ReplicatedMergeTreeAlterThread.cpp        |  2 +-
 dbms/src/Storages/StorageDistributed.cpp      |  8 ++++----
 dbms/src/Storages/StorageJoin.cpp             |  2 +-
 dbms/src/Storages/StorageLog.cpp              |  4 ++--
 dbms/src/Storages/StorageMerge.cpp            |  2 +-
 dbms/src/Storages/StorageMergeTree.cpp        |  4 ++--
 .../Storages/StorageReplicatedMergeTree.cpp   |  2 +-
 dbms/src/Storages/StorageTinyLog.cpp          |  4 ++--
 .../src/TableFunctions/TableFunctionMerge.cpp |  2 +-
 29 files changed, 62 insertions(+), 66 deletions(-)

diff --git a/dbms/src/Analyzers/AnalyzeColumns.cpp b/dbms/src/Analyzers/AnalyzeColumns.cpp
index efcbaf66888..73760c4ff01 100644
--- a/dbms/src/Analyzers/AnalyzeColumns.cpp
+++ b/dbms/src/Analyzers/AnalyzeColumns.cpp
@@ -173,7 +173,7 @@ ASTPtr createASTIdentifierForColumnInTable(const String & column, const CollectT
 void createASTsForAllColumnsInTable(const CollectTables::TableInfo & table, ASTs & res)
 {
     if (table.storage)
-        for (const auto & name : table.storage->getColumns().getNames())
+        for (const auto & name : table.storage->getColumns().getNamesOfPhysical())
             res.emplace_back(createASTIdentifierForColumnInTable(name, table));
     else
         for (size_t i = 0, size = table.structure_of_subquery.columns(); i < size; ++i)
diff --git a/dbms/src/Analyzers/ExecuteTableFunctions.cpp b/dbms/src/Analyzers/ExecuteTableFunctions.cpp
index 37cc3d66eec..d79fa1481f0 100644
--- a/dbms/src/Analyzers/ExecuteTableFunctions.cpp
+++ b/dbms/src/Analyzers/ExecuteTableFunctions.cpp
@@ -72,7 +72,7 @@ void ExecuteTableFunctions::dump(WriteBuffer & out) const
     {
         writeString(table.second->getName(), out);
         writeCString("\n\n", out);
-        writeString(table.second->getColumns().getPhysical().toString(), out);
+        writeString(table.second->getColumns().getAllPhysical().toString(), out);
         writeCString("\n", out);
     }
 }
diff --git a/dbms/src/Core/iostream_debug_helpers.cpp b/dbms/src/Core/iostream_debug_helpers.cpp
index bd19c453d8d..2ad56c402b7 100644
--- a/dbms/src/Core/iostream_debug_helpers.cpp
+++ b/dbms/src/Core/iostream_debug_helpers.cpp
@@ -44,7 +44,7 @@ std::ostream & operator<<(std::ostream & stream, const IDataType & what)
 std::ostream & operator<<(std::ostream & stream, const IStorage & what)
 {
     stream << "IStorage(name = " << what.getName() << ", tableName = " << what.getTableName() << ") {"
-           << what.getColumns().getPhysical().toString()
+           << what.getColumns().getAllPhysical().toString()
            << "}";
     // isRemote supportsSampling supportsFinal supportsPrewhere
     return stream;
diff --git a/dbms/src/DataStreams/RemoteBlockInputStream.cpp b/dbms/src/DataStreams/RemoteBlockInputStream.cpp
index b5e20898b89..7b851ec8469 100644
--- a/dbms/src/DataStreams/RemoteBlockInputStream.cpp
+++ b/dbms/src/DataStreams/RemoteBlockInputStream.cpp
@@ -138,7 +138,7 @@ void RemoteBlockInputStream::sendExternalTables()
             {
                 StoragePtr cur = table.second;
                 QueryProcessingStage::Enum stage = QueryProcessingStage::Complete;
-                BlockInputStreams input = cur->read(cur->getColumns().getNames(), {}, context,
+                BlockInputStreams input = cur->read(cur->getColumns().getNamesOfPhysical(), {}, context,
                     stage, DEFAULT_BLOCK_SIZE, 1);
                 if (input.size() == 0)
                     res.push_back(std::make_pair(std::make_shared<OneBlockInputStream>(cur->getSampleBlock()), table.first));
diff --git a/dbms/src/Functions/FunctionsMiscellaneous.cpp b/dbms/src/Functions/FunctionsMiscellaneous.cpp
index 99ed8b2dda6..475fd47ce6f 100644
--- a/dbms/src/Functions/FunctionsMiscellaneous.cpp
+++ b/dbms/src/Functions/FunctionsMiscellaneous.cpp
@@ -1750,9 +1750,8 @@ void FunctionHasColumnInTable::executeImpl(Block & block, const ColumnNumbers &
     {
         std::vector<std::vector<String>> host_names = {{ host_name }};
         auto cluster = std::make_shared<Cluster>(global_context.getSettings(), host_names, !user_name.empty() ? user_name : "default", password, global_context.getTCPPort(), false);
-        auto names_and_types_list = getStructureOfRemoteTable(*cluster, database_name, table_name, global_context);
-        const auto & names = names_and_types_list.getNames();
-        has_column = std::find(names.begin(), names.end(), column_name) != names.end();
+        auto remote_columns = getStructureOfRemoteTable(*cluster, database_name, table_name, global_context);
+        has_column = remote_columns.hasPhysical(column_name);
     }
 
     block.getByPosition(result).column = DataTypeUInt8().createColumnConst(block.rows(), UInt64(has_column));
diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp
index 4f0fe52bcff..f0bc8137029 100644
--- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp
+++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp
@@ -190,7 +190,7 @@ ExpressionAnalyzer::ExpressionAnalyzer(
     }
 
     if (storage && source_columns.empty())
-        source_columns = storage->getColumns().getPhysical();
+        source_columns = storage->getColumns().getAllPhysical();
     else
         removeDuplicateColumns(source_columns);
 
diff --git a/dbms/src/Interpreters/InterpreterAlterQuery.cpp b/dbms/src/Interpreters/InterpreterAlterQuery.cpp
index 377f92d6c8d..f4708a67c3d 100644
--- a/dbms/src/Interpreters/InterpreterAlterQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterAlterQuery.cpp
@@ -200,7 +200,7 @@ void InterpreterAlterQuery::PartitionCommands::validate(const IStorage * table)
         {
             String column_name = command.column_name.safeGet<String>();
 
-            if (!table->getColumns().has(column_name))
+            if (!table->getColumns().hasPhysical(column_name))
             {
                 throw Exception("Wrong column name. Cannot find column " + column_name + " to clear it from partition",
                     DB::ErrorCodes::ILLEGAL_COLUMN);
diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp
index 3e054f576a4..c21e3c26f93 100644
--- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp
@@ -95,7 +95,7 @@ BlockIO InterpreterInsertQuery::execute()
 
     auto table_lock = table->lockStructure(true, __PRETTY_FUNCTION__);
 
-    NamesAndTypesList required_columns = table->getColumns().getPhysical();
+    NamesAndTypesList required_columns = table->getColumns().getAllPhysical();
 
     /// We create a pipeline of several streams, into which we will write data.
     BlockOutputStreamPtr out;
diff --git a/dbms/src/Storages/ColumnsDescription.cpp b/dbms/src/Storages/ColumnsDescription.cpp
index 683f10f324d..d17f532b0ef 100644
--- a/dbms/src/Storages/ColumnsDescription.cpp
+++ b/dbms/src/Storages/ColumnsDescription.cpp
@@ -27,7 +27,7 @@ namespace ErrorCodes
 }
 
 
-NamesAndTypesList ColumnsDescription::getPhysical() const
+NamesAndTypesList ColumnsDescription::getAllPhysical() const
 {
     return ext::collection_cast<NamesAndTypesList>(boost::join(ordinary, materialized));
 }
@@ -39,13 +39,13 @@ NamesAndTypesList ColumnsDescription::getAll() const
 }
 
 
-Names ColumnsDescription::getNames() const
+Names ColumnsDescription::getNamesOfPhysical() const
 {
     return ext::map<Names>(boost::join(ordinary, materialized), [] (const auto & it) { return it.name; });
 }
 
 
-NameAndTypePair ColumnsDescription::get(const String & column_name) const
+NameAndTypePair ColumnsDescription::getPhysical(const String & column_name) const
 {
     for (auto & it : boost::join(ordinary, materialized))
         if (it.name == column_name)
@@ -54,7 +54,7 @@ NameAndTypePair ColumnsDescription::get(const String & column_name) const
 }
 
 
-bool ColumnsDescription::has(const String & column_name) const
+bool ColumnsDescription::hasPhysical(const String & column_name) const
 {
     for (auto & it : boost::join(ordinary, materialized))
         if (it.name == column_name)
diff --git a/dbms/src/Storages/ColumnsDescription.h b/dbms/src/Storages/ColumnsDescription.h
index e4e45ff6fe2..288d2712b3b 100644
--- a/dbms/src/Storages/ColumnsDescription.h
+++ b/dbms/src/Storages/ColumnsDescription.h
@@ -42,19 +42,16 @@ struct ColumnsDescription
     bool operator!=(const ColumnsDescription & other) const { return !(*this == other); }
 
     /// ordinary + materialized.
-    NamesAndTypesList getPhysical() const;
+    NamesAndTypesList getAllPhysical() const;
 
     /// ordinary + materialized + aliases.
     NamesAndTypesList getAll() const;
 
-    /// Get names of physical columns.
-    Names getNames() const;
+    Names getNamesOfPhysical() const;
 
-    /// Get a physical column by name.
-    NameAndTypePair get(const String & column_name) const;
+    NameAndTypePair getPhysical(const String & column_name) const;
 
-    /// Is there a physical column with the given name.
-    bool has(const String & column_name) const;
+    bool hasPhysical(const String & column_name) const;
 
 
     String toString() const;
diff --git a/dbms/src/Storages/ITableDeclaration.cpp b/dbms/src/Storages/ITableDeclaration.cpp
index 783b0b2a7ed..9a6ac0e2538 100644
--- a/dbms/src/Storages/ITableDeclaration.cpp
+++ b/dbms/src/Storages/ITableDeclaration.cpp
@@ -34,12 +34,12 @@ void ITableDeclaration::setColumns(ColumnsDescription columns_)
 
 bool ITableDeclaration::hasColumn(const String & column_name) const
 {
-    return getColumns().has(column_name); /// By default, we assume that there are no virtual columns in the storage.
+    return getColumns().hasPhysical(column_name); /// By default, we assume that there are no virtual columns in the storage.
 }
 
 NameAndTypePair ITableDeclaration::getColumn(const String & column_name) const
 {
-    return getColumns().get(column_name); /// By default, we assume that there are no virtual columns in the storage.
+    return getColumns().getPhysical(column_name); /// By default, we assume that there are no virtual columns in the storage.
 }
 
 
@@ -119,7 +119,7 @@ static NamesAndTypesMap getColumnsMap(const Args &... args)
 
 void ITableDeclaration::check(const Names & column_names) const
 {
-    const NamesAndTypesList & available_columns = getColumns().getPhysical();
+    const NamesAndTypesList & available_columns = getColumns().getAllPhysical();
 
     if (column_names.empty())
         throw Exception("Empty list of columns queried. There are columns: " + listOfColumns(available_columns),
@@ -147,7 +147,7 @@ void ITableDeclaration::check(const Names & column_names) const
 
 void ITableDeclaration::check(const NamesAndTypesList & provided_columns) const
 {
-    const NamesAndTypesList & available_columns = getColumns().getPhysical();
+    const NamesAndTypesList & available_columns = getColumns().getAllPhysical();
     const auto columns_map = getColumnsMap(available_columns);
 
     using UniqueStrings = google::dense_hash_set<StringRef, StringRefHash>;
@@ -175,7 +175,7 @@ void ITableDeclaration::check(const NamesAndTypesList & provided_columns) const
 
 void ITableDeclaration::check(const NamesAndTypesList & provided_columns, const Names & column_names) const
 {
-    const NamesAndTypesList & available_columns = getColumns().getPhysical();
+    const NamesAndTypesList & available_columns = getColumns().getAllPhysical();
     const auto available_columns_map = getColumnsMap(available_columns);
     const NamesAndTypesMap & provided_columns_map = getColumnsMap(provided_columns);
 
@@ -212,7 +212,7 @@ void ITableDeclaration::check(const NamesAndTypesList & provided_columns, const
 
 void ITableDeclaration::check(const Block & block, bool need_all) const
 {
-    const NamesAndTypesList & available_columns = getColumns().getPhysical();
+    const NamesAndTypesList & available_columns = getColumns().getAllPhysical();
     const auto columns_map = getColumnsMap(available_columns);
 
     using NameSet = std::unordered_set<String>;
diff --git a/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp
index a6d2958a786..1e9cea1887f 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp
@@ -141,7 +141,7 @@ try
         if (!column_names.empty())
             storage.check(data_part->columns, column_names);
 
-        const NamesAndTypesList & physical_columns = storage.getColumns().getPhysical();
+        const NamesAndTypesList & physical_columns = storage.getColumns().getAllPhysical();
         pre_columns = physical_columns.addTypes(pre_column_names);
         columns = physical_columns.addTypes(column_names);
     }
diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp
index ed16d1536de..b5200441383 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp
@@ -110,7 +110,7 @@ MergeTreeData::MergeTreeData(
     data_parts_by_state_and_info(data_parts_indexes.get<TagByStateAndInfo>())
 {
     /// NOTE: using the same columns list as is read when performing actual merges.
-    merging_params.check(getColumns().getPhysical());
+    merging_params.check(getColumns().getAllPhysical());
 
     if (!primary_expr_ast)
         throw Exception("Primary key cannot be empty", ErrorCodes::BAD_ARGUMENTS);
@@ -222,11 +222,11 @@ void MergeTreeData::initPrimaryKey()
     primary_sort_descr.clear();
     addSortDescription(primary_sort_descr, primary_expr_ast);
 
-    primary_expr = ExpressionAnalyzer(primary_expr_ast, context, nullptr, getColumns().getPhysical()).getActions(false);
+    primary_expr = ExpressionAnalyzer(primary_expr_ast, context, nullptr, getColumns().getAllPhysical()).getActions(false);
 
     {
         ExpressionActionsPtr projected_expr =
-            ExpressionAnalyzer(primary_expr_ast, context, nullptr, getColumns().getPhysical()).getActions(true);
+            ExpressionAnalyzer(primary_expr_ast, context, nullptr, getColumns().getAllPhysical()).getActions(true);
         primary_key_sample = projected_expr->getSampleBlock();
     }
 
@@ -241,10 +241,10 @@ void MergeTreeData::initPrimaryKey()
     if (secondary_sort_expr_ast)
     {
         addSortDescription(sort_descr, secondary_sort_expr_ast);
-        secondary_sort_expr = ExpressionAnalyzer(secondary_sort_expr_ast, context, nullptr, getColumns().getPhysical()).getActions(false);
+        secondary_sort_expr = ExpressionAnalyzer(secondary_sort_expr_ast, context, nullptr, getColumns().getAllPhysical()).getActions(false);
 
         ExpressionActionsPtr projected_expr =
-            ExpressionAnalyzer(secondary_sort_expr_ast, context, nullptr, getColumns().getPhysical()).getActions(true);
+            ExpressionAnalyzer(secondary_sort_expr_ast, context, nullptr, getColumns().getAllPhysical()).getActions(true);
         auto secondary_key_sample = projected_expr->getSampleBlock();
 
         checkKeyExpression(*secondary_sort_expr, secondary_key_sample, "Secondary");
@@ -257,7 +257,7 @@ void MergeTreeData::initPartitionKey()
     if (!partition_expr_ast || partition_expr_ast->children.empty())
         return;
 
-    partition_expr = ExpressionAnalyzer(partition_expr_ast, context, nullptr, getColumns().getPhysical()).getActions(false);
+    partition_expr = ExpressionAnalyzer(partition_expr_ast, context, nullptr, getColumns().getAllPhysical()).getActions(false);
     for (const ASTPtr & ast : partition_expr_ast->children)
     {
         String col_name = ast->getColumnName();
@@ -903,7 +903,7 @@ void MergeTreeData::checkAlter(const AlterCommands & commands)
         columns_alter_forbidden.insert(merging_params.sign_column);
 
     std::map<String, const IDataType *> old_types;
-    for (const auto & column : getColumns().getPhysical())
+    for (const auto & column : getColumns().getAllPhysical())
         old_types.emplace(column.name, column.type.get());
 
     for (const AlterCommand & command : commands)
@@ -931,7 +931,7 @@ void MergeTreeData::checkAlter(const AlterCommands & commands)
     NameToNameMap unused_map;
     bool unused_bool;
 
-    createConvertExpression(nullptr, getColumns().getPhysical(), new_columns.getPhysical(), unused_expression, unused_map, unused_bool);
+    createConvertExpression(nullptr, getColumns().getAllPhysical(), new_columns.getAllPhysical(), unused_expression, unused_map, unused_bool);
 }
 
 void MergeTreeData::createConvertExpression(const DataPartPtr & part, const NamesAndTypesList & old_columns, const NamesAndTypesList & new_columns,
@@ -1835,7 +1835,7 @@ void MergeTreeData::addPartContributionToColumnSizes(const DataPartPtr & part)
     const auto & files = part->checksums.files;
 
     /// TODO This method doesn't take into account columns with multiple files.
-    for (const auto & column : getColumns().getPhysical())
+    for (const auto & column : getColumns().getAllPhysical())
     {
         const auto escaped_name = escapeForFileName(column.name);
         const auto bin_file_name = escaped_name + ".bin";
@@ -1868,7 +1868,7 @@ void MergeTreeData::removePartContributionToColumnSizes(const DataPartPtr & part
     const auto & files = part->checksums.files;
 
     /// TODO This method doesn't take into account columns with multiple files.
-    for (const auto & column : getColumns().getPhysical())
+    for (const auto & column : getColumns().getAllPhysical())
     {
         const auto escaped_name = escapeForFileName(column.name);
         const auto bin_file_name = escaped_name + ".bin";
diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h
index da6999ac965..d440d86d62b 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeData.h
+++ b/dbms/src/Storages/MergeTree/MergeTreeData.h
@@ -313,12 +313,12 @@ public:
         if (column_name == "_sample_factor")
             return NameAndTypePair("_sample_factor", std::make_shared<DataTypeFloat64>());
 
-        return getColumns().get(column_name);
+        return getColumns().getPhysical(column_name);
     }
 
     bool hasColumn(const String & column_name) const override
     {
-        return getColumns().has(column_name)
+        return getColumns().hasPhysical(column_name)
             || column_name == "_part"
             || column_name == "_part_index"
             || column_name == "_sample_factor";
diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp
index dafa27e2346..229c40404bb 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp
@@ -536,8 +536,8 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMerger::mergePartsToTemporaryPart
     for (const MergeTreeData::DataPartPtr & part : parts)
         part->accumulateColumnSizes(merged_column_to_size);
 
-    Names all_column_names = data.getColumns().getNames();
-    NamesAndTypesList all_columns = data.getColumns().getPhysical();
+    Names all_column_names = data.getColumns().getNamesOfPhysical();
+    NamesAndTypesList all_columns = data.getColumns().getAllPhysical();
     const SortDescription sort_desc = data.getSortDescription();
 
     NamesAndTypesList gathering_columns, merging_columns;
diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp
index 92936e2adc7..38608f6ea32 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp
@@ -418,7 +418,7 @@ UInt64 MergeTreeDataPart::getColumnMrkSize(const String & name) const
   */
 String MergeTreeDataPart::getColumnNameWithMinumumCompressedSize() const
 {
-    const auto & columns = storage.getColumns().getPhysical();
+    const auto & columns = storage.getColumns().getAllPhysical();
     const std::string * minimum_size_column = nullptr;
     UInt64 minimum_size = std::numeric_limits<UInt64>::max();
 
@@ -774,7 +774,7 @@ void MergeTreeDataPart::accumulateColumnSizes(ColumnToSize & column_to_size) con
 {
     std::shared_lock<std::shared_mutex> part_lock(columns_lock);
 
-    for (const NameAndTypePair & name_type : storage.getColumns().getPhysical())
+    for (const NameAndTypePair & name_type : storage.getColumns().getAllPhysical())
     {
         name_type.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path)
         {
@@ -794,7 +794,7 @@ void MergeTreeDataPart::loadColumns(bool require)
             throw Exception("No columns.txt in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART);
 
         /// If there is no file with a list of columns, write it down.
-        for (const NameAndTypePair & column : storage.getColumns().getPhysical())
+        for (const NameAndTypePair & column : storage.getColumns().getAllPhysical())
             if (Poco::File(getFullPath() + escapeForFileName(column.name) + ".bin").exists())
                 columns.push_back(column);
 
diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
index 77ba844ee0b..afc9700cf0b 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
@@ -175,7 +175,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::read(
         }
     }
 
-    NamesAndTypesList available_real_columns = data.getColumns().getPhysical();
+    NamesAndTypesList available_real_columns = data.getColumns().getAllPhysical();
 
     NamesAndTypesList available_real_and_virtual_columns = available_real_columns;
     for (const auto & name : virt_column_names)
@@ -857,7 +857,7 @@ void MergeTreeDataSelectExecutor::createPositiveSignCondition(
     arguments->children.push_back(sign);
     arguments->children.push_back(one);
 
-    out_expression = ExpressionAnalyzer(function, context, {}, data.getColumns().getPhysical()).getActions(false);
+    out_expression = ExpressionAnalyzer(function, context, {}, data.getColumns().getAllPhysical()).getActions(false);
     out_column = function->getColumnName();
 }
 
diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp
index c57d1d5bfdd..e85efa435fd 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp
@@ -205,7 +205,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa
     ///  either default lz4 or compression method with zero thresholds on absolute and relative part size.
     auto compression_settings = data.context.chooseCompressionSettings(0, 0);
 
-    NamesAndTypesList columns = data.getColumns().getPhysical().filter(block.getNames());
+    NamesAndTypesList columns = data.getColumns().getAllPhysical().filter(block.getNames());
     MergedBlockOutputStream out(data, new_data_part->getFullPath(), columns, compression_settings);
 
     out.writePrefix();
diff --git a/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp b/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp
index 452aee79bf3..370797968b3 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp
@@ -238,7 +238,7 @@ std::vector<size_t> MergeTreeReadPool::fillPerPartInfo(
             if (!required_column_names.empty())
                 data.check(part.data_part->columns, required_column_names);
 
-            const NamesAndTypesList & physical_columns = data.getColumns().getPhysical();
+            const NamesAndTypesList & physical_columns = data.getColumns().getAllPhysical();
             per_part_pre_columns.push_back(physical_columns.addTypes(required_pre_column_names));
             per_part_columns.push_back(physical_columns.addTypes(required_column_names));
         }
diff --git a/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp
index ac28d3e782d..077ea9fed5d 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp
@@ -41,9 +41,9 @@ MergeTreeWhereOptimizer::MergeTreeWhereOptimizer(
     Logger * log)
         : primary_key_columns{ext::map<std::unordered_set>(data.getPrimarySortDescription(),
             [] (const SortColumnDescription & col) { return col.column_name; })},
-        table_columns{ext::map<std::unordered_set>(data.getColumns().getPhysical(),
+        table_columns{ext::map<std::unordered_set>(data.getColumns().getAllPhysical(),
             [] (const NameAndTypePair & col) { return col.name; })},
-        block_with_constants{PKCondition::getBlockWithConstants(query_info.query, context, data.getColumns().getPhysical())},
+        block_with_constants{PKCondition::getBlockWithConstants(query_info.query, context, data.getColumns().getAllPhysical())},
         prepared_sets(query_info.sets),
         log{log}
 {
diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp
index 1fcba112fed..35a2dfcc6b7 100644
--- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp
+++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp
@@ -124,7 +124,7 @@ void ReplicatedMergeTreeAlterThread::run()
                     if (!changed_version)
                         parts = storage.data.getDataParts();
 
-                    const auto columns_for_parts = storage.getColumns().getPhysical();
+                    const auto columns_for_parts = storage.getColumns().getAllPhysical();
 
                     for (const MergeTreeData::DataPartPtr & part : parts)
                     {
diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp
index 972a4817009..690d513f50e 100644
--- a/dbms/src/Storages/StorageDistributed.cpp
+++ b/dbms/src/Storages/StorageDistributed.cpp
@@ -142,7 +142,7 @@ StorageDistributed::StorageDistributed(
     table_name(table_name_),
     remote_database(remote_database_), remote_table(remote_table_),
     context(context_), cluster_name(context.getMacros().expand(cluster_name_)), has_sharding_key(sharding_key_),
-      sharding_key_expr(sharding_key_ ? ExpressionAnalyzer(sharding_key_, context, nullptr, getColumns().getPhysical()).getActions(false) : nullptr),
+      sharding_key_expr(sharding_key_ ? ExpressionAnalyzer(sharding_key_, context, nullptr, getColumns().getAllPhysical()).getActions(false) : nullptr),
     sharding_key_column_name(sharding_key_ ? sharding_key_->getColumnName() : String{}),
     path(data_path_.empty() ? "" : (data_path_ + escapeForFileName(table_name) + '/'))
 {
@@ -287,13 +287,13 @@ NameAndTypePair StorageDistributed::getColumn(const String & column_name) const
     if (const auto & type = VirtualColumnFactory::tryGetType(column_name))
         return { column_name, type };
 
-    return getColumns().get(column_name);
+    return getColumns().getPhysical(column_name);
 }
 
 
 bool StorageDistributed::hasColumn(const String & column_name) const
 {
-    return VirtualColumnFactory::hasColumn(column_name) || getColumns().has(column_name);
+    return VirtualColumnFactory::hasColumn(column_name) || getColumns().hasPhysical(column_name);
 }
 
 void StorageDistributed::createDirectoryMonitors()
@@ -386,7 +386,7 @@ void registerStorageDistributed(StorageFactory & factory)
         /// Check that sharding_key exists in the table and has numeric type.
         if (sharding_key)
         {
-            auto sharding_expr = ExpressionAnalyzer(sharding_key, args.context, nullptr, args.columns.getPhysical()).getActions(true);
+            auto sharding_expr = ExpressionAnalyzer(sharding_key, args.context, nullptr, args.columns.getAllPhysical()).getActions(true);
             const Block & block = sharding_expr->getSampleBlock();
 
             if (block.columns() != 1)
diff --git a/dbms/src/Storages/StorageJoin.cpp b/dbms/src/Storages/StorageJoin.cpp
index 746f15bab50..aab48c3ed2a 100644
--- a/dbms/src/Storages/StorageJoin.cpp
+++ b/dbms/src/Storages/StorageJoin.cpp
@@ -29,7 +29,7 @@ StorageJoin::StorageJoin(
     key_names(key_names_), kind(kind_), strictness(strictness_)
 {
     for (const auto & key : key_names)
-        if (!getColumns().has(key))
+        if (!getColumns().hasPhysical(key))
             throw Exception{
                 "Key column (" + key + ") does not exist in table declaration.",
                 ErrorCodes::NO_SUCH_COLUMN_IN_TABLE};
diff --git a/dbms/src/Storages/StorageLog.cpp b/dbms/src/Storages/StorageLog.cpp
index c23d92de8fa..04d9f18ffd1 100644
--- a/dbms/src/Storages/StorageLog.cpp
+++ b/dbms/src/Storages/StorageLog.cpp
@@ -371,7 +371,7 @@ StorageLog::StorageLog(
      /// create files if they do not exist
     Poco::File(path + escapeForFileName(name) + '/').createDirectories();
 
-    for (const auto & column : getColumns().getPhysical())
+    for (const auto & column : getColumns().getAllPhysical())
         addFiles(column.name, *column.type);
 
     marks_file = Poco::File(path + escapeForFileName(name) + '/' + DBMS_STORAGE_LOG_MARKS_FILE_NAME);
@@ -499,7 +499,7 @@ BlockInputStreams StorageLog::read(
     processed_stage = QueryProcessingStage::FetchColumns;
     loadMarks();
 
-    NamesAndTypesList all_columns = Nested::collect(getColumns().getPhysical().addTypes(column_names));
+    NamesAndTypesList all_columns = Nested::collect(getColumns().getAllPhysical().addTypes(column_names));
 
     std::shared_lock<std::shared_mutex> lock(rwlock);
 
diff --git a/dbms/src/Storages/StorageMerge.cpp b/dbms/src/Storages/StorageMerge.cpp
index 6f29850c8d1..e7090ad66ed 100644
--- a/dbms/src/Storages/StorageMerge.cpp
+++ b/dbms/src/Storages/StorageMerge.cpp
@@ -181,7 +181,7 @@ BlockInputStreams StorageMerge::read(
 
         /// If there are only virtual columns in query, you must request at least one other column.
         if (real_column_names.size() == 0)
-            real_column_names.push_back(ExpressionActions::getSmallestColumn(table->getColumns().getPhysical()));
+            real_column_names.push_back(ExpressionActions::getSmallestColumn(table->getColumns().getAllPhysical()));
 
         /// Substitute virtual column for its value when querying tables.
         ASTPtr modified_query_ast = query->clone();
diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp
index f22f24082fb..293c3254a8f 100644
--- a/dbms/src/Storages/StorageMergeTree.cpp
+++ b/dbms/src/Storages/StorageMergeTree.cpp
@@ -174,7 +174,7 @@ void StorageMergeTree::alter(
         throw Exception("MODIFY PRIMARY KEY only supported for tables without sampling key", ErrorCodes::BAD_ARGUMENTS);
 
     auto parts = data.getDataParts({MergeTreeDataPartState::PreCommitted, MergeTreeDataPartState::Committed, MergeTreeDataPartState::Outdated});
-    auto columns_for_parts = new_columns.getPhysical();
+    auto columns_for_parts = new_columns.getAllPhysical();
     for (const MergeTreeData::DataPartPtr & part : parts)
     {
         if (auto transaction = data.alterDataPart(part, columns_for_parts, new_primary_key_ast, false))
@@ -418,7 +418,7 @@ void StorageMergeTree::clearColumnInPartition(const ASTPtr & partition, const Fi
     auto new_columns = getColumns();
     alter_command.apply(new_columns);
 
-    auto columns_for_parts = new_columns.getPhysical();
+    auto columns_for_parts = new_columns.getAllPhysical();
     for (const auto & part : parts)
     {
         if (part->info.partition_id != partition_id)
diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp
index c2cf65e1c85..7ff7b1373e0 100644
--- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp
@@ -1511,7 +1511,7 @@ void StorageReplicatedMergeTree::executeClearColumnInPartition(const LogEntry &
 
     size_t modified_parts = 0;
     auto parts = data.getDataParts();
-    auto columns_for_parts = new_columns.getPhysical();
+    auto columns_for_parts = new_columns.getAllPhysical();
     for (const auto & part : parts)
     {
         if (!entry_part_info.contains(part->info))
diff --git a/dbms/src/Storages/StorageTinyLog.cpp b/dbms/src/Storages/StorageTinyLog.cpp
index bd4a8ca304f..cb62c911e63 100644
--- a/dbms/src/Storages/StorageTinyLog.cpp
+++ b/dbms/src/Storages/StorageTinyLog.cpp
@@ -296,7 +296,7 @@ StorageTinyLog::StorageTinyLog(
             throwFromErrno("Cannot create directory " + full_path, ErrorCodes::CANNOT_CREATE_DIRECTORY);
     }
 
-    for (const auto & col : getColumns().getPhysical())
+    for (const auto & col : getColumns().getAllPhysical())
         addFiles(col.name, *col.type);
 }
 
@@ -348,7 +348,7 @@ BlockInputStreams StorageTinyLog::read(
     check(column_names);
     processed_stage = QueryProcessingStage::FetchColumns;
     return BlockInputStreams(1, std::make_shared<TinyLogBlockInputStream>(
-        max_block_size, Nested::collect(getColumns().getPhysical().addTypes(column_names)), *this, context.getSettingsRef().max_read_buffer_size));
+        max_block_size, Nested::collect(getColumns().getAllPhysical().addTypes(column_names)), *this, context.getSettingsRef().max_read_buffer_size));
 }
 
 
diff --git a/dbms/src/TableFunctions/TableFunctionMerge.cpp b/dbms/src/TableFunctions/TableFunctionMerge.cpp
index 6b0710d4ba0..45aa42fea32 100644
--- a/dbms/src/TableFunctions/TableFunctionMerge.cpp
+++ b/dbms/src/TableFunctions/TableFunctionMerge.cpp
@@ -51,7 +51,7 @@ static NamesAndTypesList chooseColumns(const String & source_database, const Str
         throw Exception("Error while executing table function merge. In database " + source_database + " no one matches regular expression: "
             + table_name_regexp_, ErrorCodes::UNKNOWN_TABLE);
 
-    return any_table->getColumns().getPhysical();
+    return any_table->getColumns().getAllPhysical();
 }
 
 

From 13879f749107bd946fc4b038f1128de40f471104 Mon Sep 17 00:00:00 2001
From: proller <proller@github.com>
Date: Tue, 13 Mar 2018 22:49:21 +0300
Subject: [PATCH 042/244] Test client-server ssl

---
 dbms/src/Server/Server.cpp          |  4 ++--
 dbms/src/Server/TCPHandlerFactory.h |  4 ++--
 dbms/tests/clickhouse-test-server   |  8 ++++++--
 dbms/tests/client-test.xml          |  1 +
 dbms/tests/server-test.xml          | 30 ++++++++++++++++++++++++++++-
 5 files changed, 40 insertions(+), 7 deletions(-)

diff --git a/dbms/src/Server/Server.cpp b/dbms/src/Server/Server.cpp
index db20c194573..9f09ffd549b 100644
--- a/dbms/src/Server/Server.cpp
+++ b/dbms/src/Server/Server.cpp
@@ -387,7 +387,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
                     http_socket.setSendTimeout(settings.http_send_timeout);
 
                     servers.emplace_back(new Poco::Net::HTTPServer(
-                        new HTTPHandlerFactory(*this, "HTTPHandler-factory"),
+                        new HTTPHandlerFactory(*this, "HTTPSHandler-factory"),
                         server_pool,
                         http_socket,
                         http_params));
@@ -425,7 +425,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
                     tcp_socket.setReceiveTimeout(settings.receive_timeout);
                     tcp_socket.setSendTimeout(settings.send_timeout);
                     servers.emplace_back(new Poco::Net::TCPServer(
-                        new TCPHandlerFactory(*this),
+                        new TCPHandlerFactory(*this, /* secure= */ true ),
                                                                   server_pool,
                                                                   tcp_socket,
                                                                   new Poco::Net::TCPServerParams));
diff --git a/dbms/src/Server/TCPHandlerFactory.h b/dbms/src/Server/TCPHandlerFactory.h
index 1f454b35a8f..99cc23a45da 100644
--- a/dbms/src/Server/TCPHandlerFactory.h
+++ b/dbms/src/Server/TCPHandlerFactory.h
@@ -17,9 +17,9 @@ private:
     Poco::Logger * log;
 
 public:
-    explicit TCPHandlerFactory(IServer & server_)
+    explicit TCPHandlerFactory(IServer & server_, bool secure_ = false)
         : server(server_)
-        , log(&Logger::get("TCPHandlerFactory"))
+        , log(&Logger::get("TCP" + (secure_ ? "S" : "") + "HandlerFactory"))
     {
     }
 
diff --git a/dbms/tests/clickhouse-test-server b/dbms/tests/clickhouse-test-server
index 82d254017bf..7fb6918bfe5 100755
--- a/dbms/tests/clickhouse-test-server
+++ b/dbms/tests/clickhouse-test-server
@@ -22,8 +22,10 @@ export CLICKHOUSE_CONFIG=${CLICKHOUSE_CONFIG:=${CONFIG_SERVER_DIR}server-test.xm
 [ ! -d "$QUERIES_DIR" ] && QUERIES_DIR=${QUERIES_DIR=/usr/share/clickhouse-test/queries}
 
 rm -rf $DATA_DIR
-mkdir -p $LOG_DIR
+mkdir -p $LOG_DIR $DATA_DIR
 
+openssl dhparam -out `clickhouse-extract-from-config --config=$CLICKHOUSE_CONFIG  --key=openSSL.server.dhParamsFile` 256
+openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout `clickhouse-extract-from-config --config=$CLICKHOUSE_CONFIG  --key=openSSL.server.privateKeyFile` -out `clickhouse-extract-from-config --config=$CLICKHOUSE_CONFIG  --key=openSSL.server.certificateFile`
 
 # Start a local clickhouse server which will be used to run tests
 #PATH=$PATH:$BIN_DIR \
@@ -31,6 +33,8 @@ ${BIN_DIR}clickhouse-server --config-file=${CLICKHOUSE_CONFIG} > $LOG_DIR/stdout
 CH_PID=$!
 sleep 3
 
+tail -n50 $LOG_DIR/*
+
 # Define needed stuff to kill test clickhouse server after tests completion
 function finish {
   kill $CH_PID || true
@@ -38,7 +42,7 @@ function finish {
   tail -n 50 $LOG_DIR/stdout
   rm -rf $DATA_DIR
 }
-trap finish EXIT
+trap finish EXIT SIGINT SIGQUIT SIGTERM
 
 # Do tests
 if [ -n "$*" ]; then
diff --git a/dbms/tests/client-test.xml b/dbms/tests/client-test.xml
index 0cd3bdd5dc0..c5478dcb80d 100644
--- a/dbms/tests/client-test.xml
+++ b/dbms/tests/client-test.xml
@@ -1,4 +1,5 @@
 <!-- Config for connecting to test server -->
 <yandex>
     <tcp_port>59000</tcp_port>
+    <tcp_ssl_port>59440</tcp_ssl_port>
 </yandex>
diff --git a/dbms/tests/server-test.xml b/dbms/tests/server-test.xml
index cdb8e9caf5d..57f3f8ad9ea 100644
--- a/dbms/tests/server-test.xml
+++ b/dbms/tests/server-test.xml
@@ -13,8 +13,36 @@
     <listen_try>1</listen_try>
     <http_port>58123</http_port>
     <tcp_port>59000</tcp_port>
-    <listen_host>0.0.0.0</listen_host>
+    <https_port>58443</https_port>
+    <tcp_ssl_port>59440</tcp_ssl_port>
     <interserver_http_port>59009</interserver_http_port>
+    <openSSL>
+        <server> <!-- Used for https server AND secure tcp port -->
+            <!-- openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout /etc/clickhouse-server/server.key -out /etc/clickhouse-server/server.crt -->
+            <certificateFile>/tmp/clickhouse/server.crt</certificateFile>
+            <privateKeyFile>/tmp/clickhouse/server.key</privateKeyFile>
+            <!-- openssl dhparam -out /etc/clickhouse-server/dhparam.pem 4096 -->
+            <dhParamsFile>/tmp/clickhouse/dhparam.pem</dhParamsFile>
+            <verificationMode>none</verificationMode>
+            <loadDefaultCAFile>true</loadDefaultCAFile>
+            <cacheSessions>true</cacheSessions>
+            <disableProtocols>sslv2,sslv3</disableProtocols>
+            <preferServerCiphers>true</preferServerCiphers>
+        </server>
+
+        <client> <!-- Used for connecting to https dictionary source -->
+            <loadDefaultCAFile>true</loadDefaultCAFile>
+            <cacheSessions>true</cacheSessions>
+            <disableProtocols>sslv2,sslv3</disableProtocols>
+            <preferServerCiphers>true</preferServerCiphers>
+            <!-- Use for self-signed: <verificationMode>none</verificationMode> -->
+            <invalidCertificateHandler>
+                <!-- Use for self-signed: <name>AcceptCertificateHandler</name> -->
+                <name>RejectCertificateHandler</name>
+            </invalidCertificateHandler>
+        </client>
+    </openSSL>
+
     <keep_alive_timeout>3</keep_alive_timeout>
     <path>/tmp/clickhouse/data/</path>
     <tmp_path>/tmp/clickhouse/tmp/</tmp_path>

From 9f22e4677472bf4a6304866fe4c4168027260a49 Mon Sep 17 00:00:00 2001
From: proller <proller@github.com>
Date: Tue, 13 Mar 2018 22:51:12 +0300
Subject: [PATCH 043/244] Add openssl to debian test

---
 debian/control | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/control b/debian/control
index df43f4538ee..79666ace2e5 100644
--- a/debian/control
+++ b/debian/control
@@ -51,5 +51,5 @@ Description: debugging symbols for clickhouse-server-base
 Package: clickhouse-test
 Priority: extra
 Architecture: any
-Depends: ${shlibs:Depends}, ${misc:Depends}, clickhouse-client, bash, expect, python, python-lxml, python-termcolor, curl, perl, sudo
+Depends: ${shlibs:Depends}, ${misc:Depends}, clickhouse-client, bash, expect, python, python-lxml, python-termcolor, curl, perl, sudo, openssl
 Description: Clickhouse tests

From 53095aba45ad6ecbedd06c79420dced1f78255f7 Mon Sep 17 00:00:00 2001
From: proller <proller@github.com>
Date: Tue, 13 Mar 2018 22:58:26 +0300
Subject: [PATCH 044/244] fix

---
 dbms/src/Server/TCPHandlerFactory.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dbms/src/Server/TCPHandlerFactory.h b/dbms/src/Server/TCPHandlerFactory.h
index 99cc23a45da..0eb8be13a2d 100644
--- a/dbms/src/Server/TCPHandlerFactory.h
+++ b/dbms/src/Server/TCPHandlerFactory.h
@@ -19,7 +19,7 @@ private:
 public:
     explicit TCPHandlerFactory(IServer & server_, bool secure_ = false)
         : server(server_)
-        , log(&Logger::get("TCP" + (secure_ ? "S" : "") + "HandlerFactory"))
+        , log(&Logger::get(std::string("TCP") + (secure_ ? "S" : "") + "HandlerFactory"))
     {
     }
 

From 24651f7dd688a29d53036ed1305654b066d31332 Mon Sep 17 00:00:00 2001
From: proller <proller@github.com>
Date: Tue, 13 Mar 2018 23:38:35 +0300
Subject: [PATCH 045/244] Fixes

---
 dbms/tests/clickhouse-test-server                      | 5 +++--
 dbms/tests/queries/0_stateless/00505_tcp_ssl.reference | 1 +
 dbms/tests/queries/0_stateless/00505_tcp_ssl.sh        | 2 ++
 dbms/tests/queries/shell_config.sh                     | 1 +
 4 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/dbms/tests/clickhouse-test-server b/dbms/tests/clickhouse-test-server
index 7fb6918bfe5..b22058d5d7e 100755
--- a/dbms/tests/clickhouse-test-server
+++ b/dbms/tests/clickhouse-test-server
@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 
+set -x
 set -o errexit
 set -o pipefail
 
@@ -24,8 +25,8 @@ export CLICKHOUSE_CONFIG=${CLICKHOUSE_CONFIG:=${CONFIG_SERVER_DIR}server-test.xm
 rm -rf $DATA_DIR
 mkdir -p $LOG_DIR $DATA_DIR
 
-openssl dhparam -out `clickhouse-extract-from-config --config=$CLICKHOUSE_CONFIG  --key=openSSL.server.dhParamsFile` 256
-openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout `clickhouse-extract-from-config --config=$CLICKHOUSE_CONFIG  --key=openSSL.server.privateKeyFile` -out `clickhouse-extract-from-config --config=$CLICKHOUSE_CONFIG  --key=openSSL.server.certificateFile`
+openssl dhparam -out `${BIN_DIR}clickhouse-extract-from-config --config=$CLICKHOUSE_CONFIG  --key=openSSL.server.dhParamsFile` 256
+openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout `${BIN_DIR}clickhouse-extract-from-config --config=$CLICKHOUSE_CONFIG  --key=openSSL.server.privateKeyFile` -out `${BIN_DIR}clickhouse-extract-from-config --config=$CLICKHOUSE_CONFIG  --key=openSSL.server.certificateFile`
 
 # Start a local clickhouse server which will be used to run tests
 #PATH=$PATH:$BIN_DIR \
diff --git a/dbms/tests/queries/0_stateless/00505_tcp_ssl.reference b/dbms/tests/queries/0_stateless/00505_tcp_ssl.reference
index 1191247b6d9..01e79c32a8c 100644
--- a/dbms/tests/queries/0_stateless/00505_tcp_ssl.reference
+++ b/dbms/tests/queries/0_stateless/00505_tcp_ssl.reference
@@ -1,2 +1,3 @@
 1
 2
+3
diff --git a/dbms/tests/queries/0_stateless/00505_tcp_ssl.sh b/dbms/tests/queries/0_stateless/00505_tcp_ssl.sh
index 40b90d572b0..47f3c3b35da 100755
--- a/dbms/tests/queries/0_stateless/00505_tcp_ssl.sh
+++ b/dbms/tests/queries/0_stateless/00505_tcp_ssl.sh
@@ -13,4 +13,6 @@ else
     # Auto port detect
     ${CLICKHOUSE_CLIENT} --ssl -q "SELECT 1";
     ${CLICKHOUSE_CLIENT} --ssl --port=${CLICKHOUSE_PORT_TCP_SSL} -q "SELECT 2";
+
+    ${CLICKHOUSE_CURL} -sS --insecure ${CLICKHOUSE_URL_HTTPS}?query=SELECT%203
 fi
diff --git a/dbms/tests/queries/shell_config.sh b/dbms/tests/queries/shell_config.sh
index b060c56877d..0aec2aab89e 100644
--- a/dbms/tests/queries/shell_config.sh
+++ b/dbms/tests/queries/shell_config.sh
@@ -18,6 +18,7 @@ export CLICKHOUSE_PORT_HTTPS=${CLICKHOUSE_PORT_HTTPS:=`${CLICKHOUSE_EXTRACT_CONF
 export CLICKHOUSE_PORT_HTTPS=${CLICKHOUSE_PORT_HTTPS:="8443"}
 export CLICKHOUSE_PORT_HTTP_PROTO=${CLICKHOUSE_PORT_HTTP_PROTO:="http"}
 export CLICKHOUSE_URL=${CLICKHOUSE_URL:="${CLICKHOUSE_PORT_HTTP_PROTO}://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/"}
+export CLICKHOUSE_URL_HTTPS=${CLICKHOUSE_URL_HTTPS:="https://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTPS}/"}
 export CLICKHOUSE_CURL=${CLICKHOUSE_CURL:="curl --max-time 10"}
 export CLICKHOUSE_TMP=${CLICKHOUSE_TMP:="."}
 mkdir -p ${CLICKHOUSE_TMP}

From 16c8aefba24e83c85b38ed6b80cc34fdd09b3310 Mon Sep 17 00:00:00 2001
From: Vitaliy Lyudvichenko <vludv@yandex-team.ru>
Date: Tue, 13 Mar 2018 23:49:13 +0300
Subject: [PATCH 046/244] Fixed test after refactoring. [#CLICKHOUSE-2]

---
 .../queries/0_stateless/00429_long_http_bufferization.sh    | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/dbms/tests/queries/0_stateless/00429_long_http_bufferization.sh b/dbms/tests/queries/0_stateless/00429_long_http_bufferization.sh
index 27a79f37cd1..e94db0d951f 100755
--- a/dbms/tests/queries/0_stateless/00429_long_http_bufferization.sh
+++ b/dbms/tests/queries/0_stateless/00429_long_http_bufferization.sh
@@ -20,15 +20,15 @@ function ch_url() {
 
 # Check correct exceptions handling
 
-exception_pattern="Code: 307, e\.displayText() = DB::Exception:[[:print:]]* e\.what() = DB::Exception$"
+exception_pattern="displayText() = DB::Exception:[[:print:]]* e\.what() = DB::Exception"
 
 function check_only_exception() {
     local res=`ch_url "$1" "$2"`
     #(echo "$res")
     #(echo "$res" | wc -l)
-    #(echo "$res" | grep -c "^$exception_pattern")
+    #(echo "$res" | grep -c "$exception_pattern")
     [[ `echo "$res" | wc -l` -eq 1 ]] || echo FAIL 1 $@
-    [[ $(echo "$res" | grep -c "^$exception_pattern") -eq 1 ]] || echo FAIL 2 $@
+    [[ $(echo "$res" | grep -c "$exception_pattern") -eq 1 ]] || echo FAIL 2 $@
 }
 
 function check_last_line_exception() {

From f11f8e0365c7f1f4a22b68de3de57b7235c102d5 Mon Sep 17 00:00:00 2001
From: Vitaliy Lyudvichenko <vludv@yandex-team.ru>
Date: Wed, 14 Mar 2018 00:20:08 +0300
Subject: [PATCH 047/244] Fixed DDLWorker after recent changes. [#CLICKHOUSE-2]

---
 dbms/src/Interpreters/DDLWorker.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/dbms/src/Interpreters/DDLWorker.cpp b/dbms/src/Interpreters/DDLWorker.cpp
index 1c60b6d766c..30cccb5d848 100644
--- a/dbms/src/Interpreters/DDLWorker.cpp
+++ b/dbms/src/Interpreters/DDLWorker.cpp
@@ -523,6 +523,7 @@ bool DDLWorker::tryExecuteQuery(const String & query, const DDLTask & task, Exec
     try
     {
         Context local_context(context);
+        local_context.setCurrentQueryId(""); // generate random query_id
         executeQuery(istr, ostr, false, local_context, nullptr);
     }
     catch (...)

From 40845794350c911c4ad8b4988e645e8af6f64cac Mon Sep 17 00:00:00 2001
From: proller <proller@github.com>
Date: Wed, 14 Mar 2018 00:19:42 +0300
Subject: [PATCH 048/244] Use openssl

---
 .travis.yml                          | 4 ++--
 debian/changelog                     | 2 +-
 debian/pbuilder-hooks/B90test-server | 5 +++++
 debian/pbuilder-hooks/ssl.xml        | 4 ++++
 release                              | 2 +-
 5 files changed, 13 insertions(+), 4 deletions(-)
 create mode 100644 debian/pbuilder-hooks/ssl.xml

diff --git a/.travis.yml b/.travis.yml
index 6dab6f478ab..7af12080396 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -13,7 +13,7 @@ matrix:
 #        apt:
 #          sources:
 #            - ubuntu-toolchain-r-test
-#          packages: [ g++-7,            libicu-dev, libreadline-dev, libmysqlclient-dev, unixodbc-dev, libltdl-dev, libssl-dev, libboost-dev, zlib1g-dev, libdouble-conversion-dev, libzookeeper-mt-dev, libsparsehash-dev, librdkafka-dev, libcapnp-dev, libsparsehash-dev, libgoogle-perftools-dev, bash, expect, python, python-lxml, python-termcolor, curl, perl, sudo ]
+#          packages: [ g++-7,            libicu-dev, libreadline-dev, libmysqlclient-dev, unixodbc-dev, libltdl-dev, libssl-dev, libboost-dev, zlib1g-dev, libdouble-conversion-dev, libzookeeper-mt-dev, libsparsehash-dev, librdkafka-dev, libcapnp-dev, libsparsehash-dev, libgoogle-perftools-dev, bash, expect, python, python-lxml, python-termcolor, curl, perl, sudo, openssl ]
 #
 #      env:
 #        - MATRIX_EVAL="export CC=gcc-7 && export CXX=g++-7"
@@ -36,7 +36,7 @@ matrix:
           sources:
             - ubuntu-toolchain-r-test
             - llvm-toolchain-trusty-5.0
-          packages: [ g++-7, clang-5.0, lld-5.0, libicu-dev, libreadline-dev, libmysqlclient-dev, unixodbc-dev, libltdl-dev, libssl-dev, libboost-dev, zlib1g-dev, libdouble-conversion-dev, libzookeeper-mt-dev, libsparsehash-dev, librdkafka-dev, libcapnp-dev, libsparsehash-dev, libgoogle-perftools-dev, bash, expect, python, python-lxml, python-termcolor, curl, perl, sudo ]
+          packages: [ g++-7, clang-5.0, lld-5.0, libicu-dev, libreadline-dev, libmysqlclient-dev, unixodbc-dev, libltdl-dev, libssl-dev, libboost-dev, zlib1g-dev, libdouble-conversion-dev, libzookeeper-mt-dev, libsparsehash-dev, librdkafka-dev, libcapnp-dev, libsparsehash-dev, libgoogle-perftools-dev, bash, expect, python, python-lxml, python-termcolor, curl, perl, sudo, openssl]
 
       env:
         - MATRIX_EVAL="export CC=clang-5.0 && export CXX=clang++-5.0"
diff --git a/debian/changelog b/debian/changelog
index b5eaa0afae3..64bd8096d89 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,4 +2,4 @@ clickhouse (1.1.54363) unstable; urgency=low
 
   * Modified source code
 
- --  <robot-metrika-test@yandex-team.ru>  Tue, 13 Mar 2018 21:18:53 +0300
+ -- proller <proller@yandex-team.ru>  Wed, 14 Mar 2018 00:06:37 +0300
diff --git a/debian/pbuilder-hooks/B90test-server b/debian/pbuilder-hooks/B90test-server
index 5f163771582..6efbd387e31 100755
--- a/debian/pbuilder-hooks/B90test-server
+++ b/debian/pbuilder-hooks/B90test-server
@@ -16,6 +16,11 @@ ln -fs /usr/share/zoneinfo/Europe/Moscow /etc/localtime
 echo 'Europe/Moscow' > /etc/timezone
 dpkg-reconfigure -f noninteractive tzdata
 
+pwd
+mkdir -p /etc/clickhouse-server/config.d
+cp ssl.xml /etc/clickhouse-server/config.d
+openssl dhparam -out /etc/clickhouse-server/dhparam.pem 256
+openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout /etc/clickhouse-server/server.key -out /etc/clickhouse-server/server.crt
 
 service clickhouse-server start
 sleep 3
diff --git a/debian/pbuilder-hooks/ssl.xml b/debian/pbuilder-hooks/ssl.xml
new file mode 100644
index 00000000000..8b4580a14d4
--- /dev/null
+++ b/debian/pbuilder-hooks/ssl.xml
@@ -0,0 +1,4 @@
+<yandex>
+    <https_port>8443</https_port>
+    <tcp_ssl_port>9440</tcp_ssl_port>
+</yandex>
diff --git a/release b/release
index 0fba786373b..6c36a594cfe 100755
--- a/release
+++ b/release
@@ -39,7 +39,7 @@ do
         shift
     elif [[ $1 == '--fast' ]]; then
         # Wrong but fast pbuilder mode: create base package with all depends
-        EXTRAPACKAGES="$EXTRAPACKAGES debhelper cmake gcc-7 g++-7 libc6-dev libmariadbclient-dev libicu-dev libltdl-dev libreadline-dev libssl-dev unixodbc-dev psmisc bash expect python python-lxml python-termcolor curl perl sudo"
+        EXTRAPACKAGES="$EXTRAPACKAGES debhelper cmake gcc-7 g++-7 libc6-dev libmariadbclient-dev libicu-dev libltdl-dev libreadline-dev libssl-dev unixodbc-dev psmisc bash expect python python-lxml python-termcolor curl perl sudo openssl"
         shift
     else
         echo "Unknown option $1"

From b5b1f7c8bd516f9a0a1e76359b24446f9b308147 Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Wed, 14 Mar 2018 02:01:03 +0300
Subject: [PATCH 049/244] Update ConfigReloader.h

---
 dbms/src/Common/Config/ConfigReloader.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dbms/src/Common/Config/ConfigReloader.h b/dbms/src/Common/Config/ConfigReloader.h
index 74a5294832b..e50b9e34837 100644
--- a/dbms/src/Common/Config/ConfigReloader.h
+++ b/dbms/src/Common/Config/ConfigReloader.h
@@ -78,7 +78,7 @@ private:
     std::atomic<bool> quit{false};
     std::thread thread;
 
-    /// Is taken after entering into reloadIfNewer.
+    /// Locked inside reloadIfNewer.
     std::mutex reload_mutex;
 };
 

From 165de1bc3686b34b6640e31928ef217772a4da42 Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Wed, 14 Mar 2018 02:03:00 +0300
Subject: [PATCH 050/244] Update Context.cpp

---
 dbms/src/Interpreters/Context.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp
index 6bc4cde2d06..acb9816699a 100644
--- a/dbms/src/Interpreters/Context.cpp
+++ b/dbms/src/Interpreters/Context.cpp
@@ -130,8 +130,8 @@ struct ContextShared
     ConfigurationPtr users_config;                          /// Config with the users, profiles and quotas sections.
     InterserverIOHandler interserver_io_handler;            /// Handler for interserver communication.
     BackgroundProcessingPoolPtr background_pool;            /// The thread pool for the background work performed by the tables.
-    mutable std::mutex macros_mutex;
     MacrosPtr macros;                                       /// Substitutions extracted from config.
+    mutable std::mutex macros_mutex;
     std::unique_ptr<Compiler> compiler;                     /// Used for dynamic compilation of queries' parts if it necessary.
     std::shared_ptr<DDLWorker> ddl_worker;                  /// Process ddl commands from zk.
     /// Rules for selecting the compression settings, depending on the size of the part.

From 8522a1b4d187b72ca1c473bd1de8258c5a993244 Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Wed, 14 Mar 2018 02:09:26 +0300
Subject: [PATCH 051/244] Update StorageSystemMacros.h

---
 dbms/src/Storages/System/StorageSystemMacros.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dbms/src/Storages/System/StorageSystemMacros.h b/dbms/src/Storages/System/StorageSystemMacros.h
index ec8af55a0d3..d4bb5ab3732 100644
--- a/dbms/src/Storages/System/StorageSystemMacros.h
+++ b/dbms/src/Storages/System/StorageSystemMacros.h
@@ -10,7 +10,7 @@ namespace DB
 class Context;
 
 
-/** Implements `databases` system table, which allows you to get information about all databases.
+/** Information about macros for introspection.
   */
 class StorageSystemMacros : public ext::shared_ptr_helper<StorageSystemMacros>, public IStorage
 {

From 153fcd6c9ff797c3c85e90b9b7ad23d495b56520 Mon Sep 17 00:00:00 2001
From: proller <proller@users.noreply.github.com>
Date: Wed, 14 Mar 2018 02:13:39 +0300
Subject: [PATCH 052/244] Fix double free in LibraryDictionarySource (#2028)

* Fix double free in LibraryDictionarySource

* Fix {}

* wip

* wip

* Better Exception

* Fix possible leak
---
 .../ComplexKeyCacheDictionary.cpp             |  1 -
 .../Dictionaries/LibraryDictionarySource.cpp  | 62 ++++++++++---------
 .../Dictionaries/LibraryDictionarySource.h    | 19 +++---
 .../dictionary_library/dictionary_library.cpp |  2 +-
 .../dictionary_library/dictionary_library_c.c |  2 +-
 5 files changed, 44 insertions(+), 42 deletions(-)

diff --git a/dbms/src/Dictionaries/ComplexKeyCacheDictionary.cpp b/dbms/src/Dictionaries/ComplexKeyCacheDictionary.cpp
index 540b71a80dd..fcd73cdb889 100644
--- a/dbms/src/Dictionaries/ComplexKeyCacheDictionary.cpp
+++ b/dbms/src/Dictionaries/ComplexKeyCacheDictionary.cpp
@@ -8,7 +8,6 @@
 #include <Common/ProfileEvents.h>
 #include <Common/CurrentMetrics.h>
 #include <ext/range.h>
-#include <ext/scope_guard.h>
 #include <ext/map.h>
 
 
diff --git a/dbms/src/Dictionaries/LibraryDictionarySource.cpp b/dbms/src/Dictionaries/LibraryDictionarySource.cpp
index 04ee831bb1b..8c911771a20 100644
--- a/dbms/src/Dictionaries/LibraryDictionarySource.cpp
+++ b/dbms/src/Dictionaries/LibraryDictionarySource.cpp
@@ -6,6 +6,7 @@
 #include <common/logger_useful.h>
 #include <ext/bit_cast.h>
 #include <ext/range.h>
+#include <ext/scope_guard.h>
 
 namespace DB
 {
@@ -70,11 +71,11 @@ namespace
     Block dataToBlock(const Block & sample_block, const void * data)
     {
         if (!data)
-            return sample_block.cloneEmpty();
+            throw Exception("LibraryDictionarySource: No data returned", ErrorCodes::EXTERNAL_LIBRARY_ERROR);
 
         auto columns_received = static_cast<const ClickHouseLibrary::Table *>(data);
         if (columns_received->error_code)
-            throw Exception("Received error: " + std::to_string(columns_received->error_code) + " "
+            throw Exception("LibraryDictionarySource: Returned error: " + std::to_string(columns_received->error_code) + " "
                     + (columns_received->error_string ? columns_received->error_string : ""),
                 ErrorCodes::EXTERNAL_LIBRARY_ERROR);
 
@@ -85,7 +86,7 @@ namespace
         for (size_t col_n = 0; col_n < columns_received->size; ++col_n)
         {
             if (columns.size() != columns_received->data[col_n].size)
-                throw Exception("Received unexpected number of columns: " + std::to_string(columns_received->data[col_n].size)
+                throw Exception("LibraryDictionarySource: Returned unexpected number of columns: " + std::to_string(columns_received->data[col_n].size)
                         + ", must be " + std::to_string(columns.size()),
                     ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
 
@@ -123,9 +124,8 @@ LibraryDictionarySource::LibraryDictionarySource(const DictionaryStructure & dic
     description.init(sample_block);
     library = std::make_shared<SharedLibrary>(path);
     settings = std::make_shared<CStringsHolder>(getLibSettings(config, config_prefix + lib_config_settings));
-    auto fptr = library->tryGet<void * (*)()>("ClickHouseDictionary_v2_libNew");
-    if (fptr)
-        lib_data = fptr();
+    if (auto libNew = library->tryGet<decltype(lib_data) (*)(decltype(&settings->strings))>("ClickHouseDictionary_v2_libNew"))
+        lib_data = libNew(&settings->strings);
 }
 
 LibraryDictionarySource::LibraryDictionarySource(const LibraryDictionarySource & other)
@@ -138,15 +138,17 @@ LibraryDictionarySource::LibraryDictionarySource(const LibraryDictionarySource &
     , library{other.library}
     , description{other.description}
     , settings{other.settings}
-    , lib_data{other.lib_data}
 {
+    if (auto libClone = library->tryGet<decltype(lib_data) (*)(decltype(other.lib_data))>("ClickHouseDictionary_v2_libClone"))
+        lib_data = libClone(other.lib_data);
+    else if (auto libNew = library->tryGet<decltype(lib_data) (*)(decltype(&settings->strings))>("ClickHouseDictionary_v2_libNew"))
+        lib_data = libNew(&settings->strings);
 }
 
 LibraryDictionarySource::~LibraryDictionarySource()
 {
-    auto fptr = library->tryGet<void * (*)(void *)>("ClickHouseDictionary_v2_libDelete");
-    if (fptr)
-        fptr(lib_data);
+    if (auto libDelete = library->tryGet<void (*)(decltype(lib_data))>("ClickHouseDictionary_v2_libDelete"))
+        libDelete(lib_data);
 }
 
 BlockInputStreamPtr LibraryDictionarySource::loadAll()
@@ -165,12 +167,12 @@ BlockInputStreamPtr LibraryDictionarySource::loadAll()
     void * data_ptr = nullptr;
 
     /// Get function pointer before dataNew call because library->get may throw.
-    auto fptr
+    auto func_loadAll
         = library->get<void * (*)(decltype(data_ptr), decltype(&settings->strings), decltype(&columns))>("ClickHouseDictionary_v2_loadAll");
-    data_ptr = library->get<void * (*)(void *)>("ClickHouseDictionary_v2_dataNew")(lib_data);
-    auto data = fptr(data_ptr, &settings->strings, &columns);
+    data_ptr = library->get<decltype(data_ptr) (*)(decltype(lib_data))>("ClickHouseDictionary_v2_dataNew")(lib_data);
+    auto data = func_loadAll(data_ptr, &settings->strings, &columns);
     auto block = dataToBlock(description.sample_block, data);
-    library->get<void (*)(void *, void *)>("ClickHouseDictionary_v2_dataDelete")(lib_data, data_ptr);
+    SCOPE_EXIT(library->get<void (*)(decltype(lib_data), decltype(data_ptr))>("ClickHouseDictionary_v2_dataDelete")(lib_data, data_ptr));
     return std::make_shared<OneBlockInputStream>(block);
 }
 
@@ -191,12 +193,13 @@ BlockInputStreamPtr LibraryDictionarySource::loadIds(const std::vector<UInt64> &
     void * data_ptr = nullptr;
 
     /// Get function pointer before dataNew call because library->get may throw.
-    auto fptr = library->get<void * (*)(decltype(data_ptr), decltype(&settings->strings), decltype(&columns_pass), decltype(&ids_data))>(
-        "ClickHouseDictionary_v2_loadIds");
-    data_ptr = library->get<void * (*)(void *)>("ClickHouseDictionary_v2_dataNew")(lib_data);
-    auto data = fptr(data_ptr, &settings->strings, &columns_pass, &ids_data);
+    auto func_loadIds
+        = library->get<void * (*)(decltype(data_ptr), decltype(&settings->strings), decltype(&columns_pass), decltype(&ids_data))>(
+            "ClickHouseDictionary_v2_loadIds");
+    data_ptr = library->get<decltype(data_ptr) (*)(decltype(lib_data))>("ClickHouseDictionary_v2_dataNew")(lib_data);
+    auto data = func_loadIds(data_ptr, &settings->strings, &columns_pass, &ids_data);
     auto block = dataToBlock(description.sample_block, data);
-    library->get<void (*)(void *, void *)>("ClickHouseDictionary_v2_dataDelete")(lib_data, data_ptr);
+    SCOPE_EXIT(library->get<void (*)(decltype(lib_data), decltype(data_ptr))>("ClickHouseDictionary_v2_dataDelete")(lib_data, data_ptr));
     return std::make_shared<OneBlockInputStream>(block);
 }
 
@@ -218,30 +221,29 @@ BlockInputStreamPtr LibraryDictionarySource::loadKeys(const Columns & key_column
     void * data_ptr = nullptr;
 
     /// Get function pointer before dataNew call because library->get may throw.
-    auto fptr
+    auto func_loadKeys
         = library->get<void * (*)(decltype(data_ptr), decltype(&settings->strings), decltype(&columns_pass), decltype(&requested_rows_c))>(
             "ClickHouseDictionary_v2_loadKeys");
-    data_ptr = library->get<void * (*)(void *)>("ClickHouseDictionary_v2_dataNew")(lib_data);
-    auto data = fptr(data_ptr, &settings->strings, &columns_pass, &requested_rows_c);
+    data_ptr = library->get<decltype(data_ptr) (*)(decltype(lib_data))>("ClickHouseDictionary_v2_dataNew")(lib_data);
+    auto data = func_loadKeys(data_ptr, &settings->strings, &columns_pass, &requested_rows_c);
     auto block = dataToBlock(description.sample_block, data);
-    library->get<void (*)(void *, void *)>("ClickHouseDictionary_v2_dataDelete")(lib_data, data_ptr);
+    SCOPE_EXIT(library->get<void (*)(decltype(lib_data), decltype(data_ptr))>("ClickHouseDictionary_v2_dataDelete")(lib_data, data_ptr));
     return std::make_shared<OneBlockInputStream>(block);
 }
 
 bool LibraryDictionarySource::isModified() const
 {
-    auto fptr = library->tryGet<void * (*)(decltype(lib_data), decltype(&settings->strings))>("ClickHouseDictionary_v2_isModified");
-    if (fptr)
-        return fptr(lib_data, &settings->strings);
+    if (auto func_isModified
+        = library->tryGet<bool (*)(decltype(lib_data), decltype(&settings->strings))>("ClickHouseDictionary_v2_isModified"))
+        return func_isModified(lib_data, &settings->strings);
     return true;
 }
 
 bool LibraryDictionarySource::supportsSelectiveLoad() const
 {
-    auto fptr
-        = library->tryGet<void * (*)(decltype(lib_data), decltype(&settings->strings))>("ClickHouseDictionary_v2_supportsSelectiveLoad");
-    if (fptr)
-        return fptr(lib_data, &settings->strings);
+    if (auto func_supportsSelectiveLoad
+        = library->tryGet<bool (*)(decltype(lib_data), decltype(&settings->strings))>("ClickHouseDictionary_v2_supportsSelectiveLoad"))
+        return func_supportsSelectiveLoad(lib_data, &settings->strings);
     return true;
 }
 
diff --git a/dbms/src/Dictionaries/LibraryDictionarySource.h b/dbms/src/Dictionaries/LibraryDictionarySource.h
index ce136b90318..1765968adca 100644
--- a/dbms/src/Dictionaries/LibraryDictionarySource.h
+++ b/dbms/src/Dictionaries/LibraryDictionarySource.h
@@ -1,26 +1,25 @@
 #pragma once
 
-#include <Common/SharedLibrary.h>
 #include <Dictionaries/DictionaryStructure.h>
 #include <Dictionaries/ExternalResultDescription.h>
 #include <Dictionaries/IDictionarySource.h>
+#include <Common/SharedLibrary.h>
 #include <common/LocalDateTime.h>
 
 
 namespace Poco
 {
-    class Logger;
+class Logger;
 
-    namespace Util
-    {
-        class AbstractConfiguration;
-    }
+namespace Util
+{
+    class AbstractConfiguration;
+}
 }
 
 
 namespace DB
 {
-
 class CStringsHolder;
 
 /// Allows loading dictionaries from dynamic libraries (.so)
@@ -55,7 +54,10 @@ public:
     bool supportsSelectiveLoad() const override;
 
     ///Not yet supported
-    bool hasUpdateField() const override { return false; }
+    bool hasUpdateField() const override
+    {
+        return false;
+    }
 
     DictionarySourcePtr clone() const override;
 
@@ -76,5 +78,4 @@ private:
     std::shared_ptr<CStringsHolder> settings;
     void * lib_data = nullptr;
 };
-
 }
diff --git a/dbms/tests/external_dictionaries/dictionary_library/dictionary_library.cpp b/dbms/tests/external_dictionaries/dictionary_library/dictionary_library.cpp
index 7bb6e2eb726..9ddaf69e1fd 100644
--- a/dbms/tests/external_dictionaries/dictionary_library/dictionary_library.cpp
+++ b/dbms/tests/external_dictionaries/dictionary_library/dictionary_library.cpp
@@ -158,7 +158,7 @@ void * ClickHouseDictionary_v2_loadKeys(void * data_ptr,
     return nullptr;
 }
 
-void * ClickHouseDictionary_v2_libNew()
+void * ClickHouseDictionary_v2_libNew(ClickHouseLibrary::CStrings * /*settings*/)
 {
     auto lib_ptr = new LibHolder;
     return lib_ptr;
diff --git a/dbms/tests/external_dictionaries/dictionary_library/dictionary_library_c.c b/dbms/tests/external_dictionaries/dictionary_library/dictionary_library_c.c
index 9d95894302a..6caea3ac2cf 100644
--- a/dbms/tests/external_dictionaries/dictionary_library/dictionary_library_c.c
+++ b/dbms/tests/external_dictionaries/dictionary_library/dictionary_library_c.c
@@ -39,7 +39,7 @@ void * ClickHouseDictionary_v2_loadKeys(
     return 0;
 }
 
-void * ClickHouseDictionary_v2_libNew()
+void * ClickHouseDictionary_v2_libNew(ClickHouseLibCStrings * settings)
 {
     int size = 101;
     void * lib_ptr = malloc(size);

From 8b59db58c3ce92a82037c643302027d18e0bf586 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Wed, 14 Mar 2018 02:44:23 +0300
Subject: [PATCH 053/244] Fixed uneven code #2032

---
 dbms/src/Common/Macros.cpp                    |  2 ++
 dbms/src/Common/Macros.h                      | 12 +++++--
 dbms/src/Interpreters/Context.cpp             | 13 +++----
 dbms/src/Interpreters/Context.h               |  7 ++--
 .../src/Interpreters/EmbeddedDictionaries.cpp |  2 +-
 dbms/src/Server/LocalServer.cpp               |  2 +-
 dbms/src/Server/Server.cpp                    |  4 +--
 libs/libcommon/include/common/MultiVersion.h  | 34 ++-----------------
 libs/libdaemon/src/BaseDaemon.cpp             |  1 +
 9 files changed, 27 insertions(+), 50 deletions(-)

diff --git a/dbms/src/Common/Macros.cpp b/dbms/src/Common/Macros.cpp
index cb3d43cd21c..56b766eda68 100644
--- a/dbms/src/Common/Macros.cpp
+++ b/dbms/src/Common/Macros.cpp
@@ -1,6 +1,8 @@
+#include <Poco/Util/AbstractConfiguration.h>
 #include <Common/Macros.h>
 #include <Common/Exception.h>
 
+
 namespace DB
 {
 
diff --git a/dbms/src/Common/Macros.h b/dbms/src/Common/Macros.h
index dae3bd10aa9..d2602cf62e7 100644
--- a/dbms/src/Common/Macros.h
+++ b/dbms/src/Common/Macros.h
@@ -1,9 +1,16 @@
 #pragma once
 
 #include <Core/Types.h>
-#include <Poco/Util/AbstractConfiguration.h>
 #include <map>
-#include <memory>
+
+
+namespace Poco
+{
+    namespace Util
+    {
+        class AbstractConfiguration;
+    }
+}
 
 
 namespace DB
@@ -29,6 +36,5 @@ private:
     MacroMap macros;
 };
 
-using MacrosPtr = std::shared_ptr<Macros>;
 
 }
diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp
index acb9816699a..8fe51bf5c08 100644
--- a/dbms/src/Interpreters/Context.cpp
+++ b/dbms/src/Interpreters/Context.cpp
@@ -130,8 +130,7 @@ struct ContextShared
     ConfigurationPtr users_config;                          /// Config with the users, profiles and quotas sections.
     InterserverIOHandler interserver_io_handler;            /// Handler for interserver communication.
     BackgroundProcessingPoolPtr background_pool;            /// The thread pool for the background work performed by the tables.
-    MacrosPtr macros;                                       /// Substitutions extracted from config.
-    mutable std::mutex macros_mutex;
+    MultiVersion<Macros> macros;                            /// Substitutions extracted from config.
     std::unique_ptr<Compiler> compiler;                     /// Used for dynamic compilation of queries' parts if it necessary.
     std::shared_ptr<DDLWorker> ddl_worker;                  /// Process ddl commands from zk.
     /// Rules for selecting the compression settings, depending on the size of the part.
@@ -1045,16 +1044,14 @@ void Context::setDefaultFormat(const String & name)
     default_format = name;
 }
 
-const MacrosPtr & Context::getMacros() const
+MultiVersion<Macros>::Version Context::getMacros() const
 {
-    std::unique_lock<std::mutex> lock(shared->macros_mutex);
-    return shared->macros;
+    return shared->macros.get();
 }
 
-void Context::setMacros(Macros && macros)
+void Context::setMacros(std::unique_ptr<Macros> && macros)
 {
-    std::unique_lock<std::mutex> lock(shared->macros_mutex);
-    shared->macros = std::make_shared<Macros>(std::move(macros));
+    shared->macros.set(std::move(macros));
 }
 
 const Context & Context::getQueryContext() const
diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h
index 32ba0b8cd7f..4f714842b62 100644
--- a/dbms/src/Interpreters/Context.h
+++ b/dbms/src/Interpreters/Context.h
@@ -7,6 +7,7 @@
 #include <mutex>
 #include <thread>
 
+#include <common/MultiVersion.h>
 #include <Core/Types.h>
 #include <Core/NamesAndTypes.h>
 #include <Interpreters/Settings.h>
@@ -80,8 +81,6 @@ using Dependencies = std::vector<DatabaseAndTableName>;
 using TableAndCreateAST = std::pair<StoragePtr, ASTPtr>;
 using TableAndCreateASTs = std::map<String, TableAndCreateAST>;
 
-using MacrosPtr = std::shared_ptr<Macros>;
-
 /** A set of known objects that can be used in the query.
   * Consists of a shared part (always common to all sessions and queries)
   *  and copied part (which can be its own for each session or query).
@@ -207,8 +206,8 @@ public:
     String getDefaultFormat() const;    /// If default_format is not specified, some global default format is returned.
     void setDefaultFormat(const String & name);
 
-    const MacrosPtr & getMacros() const;
-    void setMacros(Macros && macros);
+    MultiVersion<Macros>::Version getMacros() const;
+    void setMacros(std::unique_ptr<Macros> && macros);
 
     Settings getSettings() const;
     void setSettings(const Settings & settings_);
diff --git a/dbms/src/Interpreters/EmbeddedDictionaries.cpp b/dbms/src/Interpreters/EmbeddedDictionaries.cpp
index f5567e0a1bc..441bb7c92d8 100644
--- a/dbms/src/Interpreters/EmbeddedDictionaries.cpp
+++ b/dbms/src/Interpreters/EmbeddedDictionaries.cpp
@@ -49,7 +49,7 @@ bool EmbeddedDictionaries::reloadDictionary(
         {
             auto new_dictionary = reload_dictionary(config);
             if (new_dictionary)
-                dictionary.set(new_dictionary.release());
+                dictionary.set(std::move(new_dictionary));
         }
         catch (...)
         {
diff --git a/dbms/src/Server/LocalServer.cpp b/dbms/src/Server/LocalServer.cpp
index aec3f4152f1..1196c62f06f 100644
--- a/dbms/src/Server/LocalServer.cpp
+++ b/dbms/src/Server/LocalServer.cpp
@@ -275,7 +275,7 @@ try
 
     /// Maybe useless
     if (config().has("macros"))
-        context->setMacros(Macros(config(), "macros"));
+        context->setMacros(std::make_unique<Macros>(config(), "macros"));
 
     /// Skip networking
 
diff --git a/dbms/src/Server/Server.cpp b/dbms/src/Server/Server.cpp
index 25a55f1ff8d..12598af18e6 100644
--- a/dbms/src/Server/Server.cpp
+++ b/dbms/src/Server/Server.cpp
@@ -221,7 +221,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
     }
 
     if (config().has("macros"))
-        global_context->setMacros(Macros(config(), "macros"));
+        global_context->setMacros(std::make_unique<Macros>(config(), "macros"));
 
     /// Initialize main config reloader.
     std::string include_from_path = config().getString("include_from", "/etc/metrika.xml");
@@ -231,7 +231,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
         [&](ConfigurationPtr config)
         {
             global_context->setClustersConfig(config);
-            global_context->setMacros(Macros(*config, "macros"));
+            global_context->setMacros(std::make_unique<Macros>(*config, "macros"));
         },
         /* already_loaded = */ true);
 
diff --git a/libs/libcommon/include/common/MultiVersion.h b/libs/libcommon/include/common/MultiVersion.h
index 6b55bfa9e05..3014689f861 100644
--- a/libs/libcommon/include/common/MultiVersion.h
+++ b/libs/libcommon/include/common/MultiVersion.h
@@ -30,30 +30,13 @@ public:
     /// Default initialization - by nullptr.
     MultiVersion() = default;
 
-    /// Initialization with first version.
-    MultiVersion(const Version & value)
-    {
-        set(value);
-    }
-
-    /// Take an ownership of first version.
-    MultiVersion(const T * value)
-    {
-        set(value);
-    }
-
-    MultiVersion(Version && value)
-    {
-        set(std::move(value));
-    }
-
     MultiVersion(std::unique_ptr<const T> && value)
     {
         set(std::move(value));
     }
 
     /// Obtain current version for read-only usage. Returns shared_ptr, that manages lifetime of version.
-    const Version get() const
+    Version get() const
     {
         /// NOTE: is it possible to lock-free replace of shared_ptr?
         std::lock_guard lock(mutex);
@@ -61,21 +44,10 @@ public:
     }
 
     /// Update an object with new version.
-    void set(const Version & value)
-    {
-        std::lock_guard lock(mutex);
-        current_version = value;
-    }
-
-    /// Update an object with new version and take an ownership of it.
-    void set(const T * value)
-    {
-        set(Version(value));
-    }
-
     void set(std::unique_ptr<const T> && value)
     {
-        set(Version(value.release()));
+        std::lock_guard lock(mutex);
+        current_version = std::move(value);
     }
 
 private:
diff --git a/libs/libdaemon/src/BaseDaemon.cpp b/libs/libdaemon/src/BaseDaemon.cpp
index 1d4a4dffc2a..29405901206 100644
--- a/libs/libdaemon/src/BaseDaemon.cpp
+++ b/libs/libdaemon/src/BaseDaemon.cpp
@@ -63,6 +63,7 @@
 #include <Common/ClickHouseRevision.h>
 #include <daemon/OwnPatternFormatter.h>
 
+
 using Poco::Logger;
 using Poco::AutoPtr;
 using Poco::Observer;

From 0a33b774a3f34ddc80e10262e100e4ab94f0e63c Mon Sep 17 00:00:00 2001
From: Kirill Shvakov <shvakov@gmail.com>
Date: Sat, 10 Mar 2018 14:41:11 +0200
Subject: [PATCH 054/244] Fix quotes in partition display name based on Date,
 DateTime or UUID type

---
 dbms/src/Storages/MergeTree/MergeTreeData.cpp          | 2 +-
 dbms/src/Storages/MergeTree/MergeTreePartition.cpp     | 4 ++--
 dbms/src/Storages/MergeTree/MergeTreePartition.h       | 2 +-
 dbms/src/Storages/System/StorageSystemParts.cpp        | 2 +-
 dbms/src/Storages/System/StorageSystemPartsColumns.cpp | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp
index 392c45a7cc7..1270c645a2b 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp
@@ -2050,7 +2050,7 @@ String MergeTreeData::getPartitionIDFromQuery(const ASTPtr & ast, const Context
         {
             WriteBufferFromOwnString buf;
             writeCString("Parsed partition value: ", buf);
-            partition.serializeTextQuoted(*this, buf);
+            partition.serializeText(*this, buf);
             writeCString(" doesn't match partition value for an existing part with the same partition ID: ", buf);
             writeString(existing_part_in_partition->name, buf);
             throw Exception(buf.str(), ErrorCodes::INVALID_PARTITION_VALUE);
diff --git a/dbms/src/Storages/MergeTree/MergeTreePartition.cpp b/dbms/src/Storages/MergeTree/MergeTreePartition.cpp
index b95916b2164..0857f285d9b 100644
--- a/dbms/src/Storages/MergeTree/MergeTreePartition.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreePartition.cpp
@@ -77,7 +77,7 @@ String MergeTreePartition::getID(const MergeTreeData & storage) const
     return result;
 }
 
-void MergeTreePartition::serializeTextQuoted(const MergeTreeData & storage, WriteBuffer & out) const
+void MergeTreePartition::serializeText(const MergeTreeData & storage, WriteBuffer & out) const
 {
     size_t key_size = storage.partition_key_sample.columns();
 
@@ -98,7 +98,7 @@ void MergeTreePartition::serializeTextQuoted(const MergeTreeData & storage, Writ
         const DataTypePtr & type = storage.partition_key_sample.getByPosition(i).type;
         auto column = type->createColumn();
         column->insert(value[i]);
-        type->serializeTextQuoted(*column, 0, out);
+        type->serializeText(*column, 0, out);
     }
 
     if (key_size > 1)
diff --git a/dbms/src/Storages/MergeTree/MergeTreePartition.h b/dbms/src/Storages/MergeTree/MergeTreePartition.h
index d32b5f4401a..f3ae17b468c 100644
--- a/dbms/src/Storages/MergeTree/MergeTreePartition.h
+++ b/dbms/src/Storages/MergeTree/MergeTreePartition.h
@@ -25,7 +25,7 @@ public:
 
     String getID(const MergeTreeData & storage) const;
 
-    void serializeTextQuoted(const MergeTreeData & storage, WriteBuffer & out) const;
+    void serializeText(const MergeTreeData & storage, WriteBuffer & out) const;
 
     void load(const MergeTreeData & storage, const String & part_path);
     void store(const MergeTreeData & storage, const String & part_path, MergeTreeDataPartChecksums & checksums) const;
diff --git a/dbms/src/Storages/System/StorageSystemParts.cpp b/dbms/src/Storages/System/StorageSystemParts.cpp
index 0914cae72da..9972c4832f3 100644
--- a/dbms/src/Storages/System/StorageSystemParts.cpp
+++ b/dbms/src/Storages/System/StorageSystemParts.cpp
@@ -56,7 +56,7 @@ void StorageSystemParts::processNextStorage(MutableColumns & columns, const Stor
         size_t i = 0;
         {
             WriteBufferFromOwnString out;
-            part->partition.serializeTextQuoted(*info.data, out);
+            part->partition.serializeText(*info.data, out);
             columns[i++]->insert(out.str());
         }
         columns[i++]->insert(part->name);
diff --git a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp
index 9667399c534..5e73a14c970 100644
--- a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp
+++ b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp
@@ -101,7 +101,7 @@ void StorageSystemPartsColumns::processNextStorage(MutableColumns & columns, con
             size_t j = 0;
             {
                 WriteBufferFromOwnString out;
-                part->partition.serializeTextQuoted(*info.data, out);
+                part->partition.serializeText(*info.data, out);
                 columns[j++]->insert(out.str());
             }
             columns[j++]->insert(part->name);

From 3ae2e443fdf81d7480d9179ba42ff74afeff91ec Mon Sep 17 00:00:00 2001
From: kshvakov <shvakov@gmail.com>
Date: Sat, 10 Mar 2018 16:00:32 +0200
Subject: [PATCH 055/244] Revert "Fix quotes in partition display name based on
 Date, DateTime or UUID type"

This reverts commit 10aa9cfc8a90f272200b343b45e9e8447aa92558.
---
 dbms/src/Storages/MergeTree/MergeTreeData.cpp          | 2 +-
 dbms/src/Storages/MergeTree/MergeTreePartition.cpp     | 4 ++--
 dbms/src/Storages/MergeTree/MergeTreePartition.h       | 2 +-
 dbms/src/Storages/System/StorageSystemParts.cpp        | 2 +-
 dbms/src/Storages/System/StorageSystemPartsColumns.cpp | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp
index 1270c645a2b..392c45a7cc7 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp
@@ -2050,7 +2050,7 @@ String MergeTreeData::getPartitionIDFromQuery(const ASTPtr & ast, const Context
         {
             WriteBufferFromOwnString buf;
             writeCString("Parsed partition value: ", buf);
-            partition.serializeText(*this, buf);
+            partition.serializeTextQuoted(*this, buf);
             writeCString(" doesn't match partition value for an existing part with the same partition ID: ", buf);
             writeString(existing_part_in_partition->name, buf);
             throw Exception(buf.str(), ErrorCodes::INVALID_PARTITION_VALUE);
diff --git a/dbms/src/Storages/MergeTree/MergeTreePartition.cpp b/dbms/src/Storages/MergeTree/MergeTreePartition.cpp
index 0857f285d9b..b95916b2164 100644
--- a/dbms/src/Storages/MergeTree/MergeTreePartition.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreePartition.cpp
@@ -77,7 +77,7 @@ String MergeTreePartition::getID(const MergeTreeData & storage) const
     return result;
 }
 
-void MergeTreePartition::serializeText(const MergeTreeData & storage, WriteBuffer & out) const
+void MergeTreePartition::serializeTextQuoted(const MergeTreeData & storage, WriteBuffer & out) const
 {
     size_t key_size = storage.partition_key_sample.columns();
 
@@ -98,7 +98,7 @@ void MergeTreePartition::serializeText(const MergeTreeData & storage, WriteBuffe
         const DataTypePtr & type = storage.partition_key_sample.getByPosition(i).type;
         auto column = type->createColumn();
         column->insert(value[i]);
-        type->serializeText(*column, 0, out);
+        type->serializeTextQuoted(*column, 0, out);
     }
 
     if (key_size > 1)
diff --git a/dbms/src/Storages/MergeTree/MergeTreePartition.h b/dbms/src/Storages/MergeTree/MergeTreePartition.h
index f3ae17b468c..d32b5f4401a 100644
--- a/dbms/src/Storages/MergeTree/MergeTreePartition.h
+++ b/dbms/src/Storages/MergeTree/MergeTreePartition.h
@@ -25,7 +25,7 @@ public:
 
     String getID(const MergeTreeData & storage) const;
 
-    void serializeText(const MergeTreeData & storage, WriteBuffer & out) const;
+    void serializeTextQuoted(const MergeTreeData & storage, WriteBuffer & out) const;
 
     void load(const MergeTreeData & storage, const String & part_path);
     void store(const MergeTreeData & storage, const String & part_path, MergeTreeDataPartChecksums & checksums) const;
diff --git a/dbms/src/Storages/System/StorageSystemParts.cpp b/dbms/src/Storages/System/StorageSystemParts.cpp
index 9972c4832f3..0914cae72da 100644
--- a/dbms/src/Storages/System/StorageSystemParts.cpp
+++ b/dbms/src/Storages/System/StorageSystemParts.cpp
@@ -56,7 +56,7 @@ void StorageSystemParts::processNextStorage(MutableColumns & columns, const Stor
         size_t i = 0;
         {
             WriteBufferFromOwnString out;
-            part->partition.serializeText(*info.data, out);
+            part->partition.serializeTextQuoted(*info.data, out);
             columns[i++]->insert(out.str());
         }
         columns[i++]->insert(part->name);
diff --git a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp
index 5e73a14c970..9667399c534 100644
--- a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp
+++ b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp
@@ -101,7 +101,7 @@ void StorageSystemPartsColumns::processNextStorage(MutableColumns & columns, con
             size_t j = 0;
             {
                 WriteBufferFromOwnString out;
-                part->partition.serializeText(*info.data, out);
+                part->partition.serializeTextQuoted(*info.data, out);
                 columns[j++]->insert(out.str());
             }
             columns[j++]->insert(part->name);

From b28cef987132b8b90c50ac2b4c2c2c031d555504 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Wed, 14 Mar 2018 04:42:51 +0300
Subject: [PATCH 056/244] Disabled ill-formed test #2035

---
 .../0_stateless/{00505_tcp_ssl.sh => 00505_tcp_ssl.sh.disabled}   | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename dbms/tests/queries/0_stateless/{00505_tcp_ssl.sh => 00505_tcp_ssl.sh.disabled} (100%)

diff --git a/dbms/tests/queries/0_stateless/00505_tcp_ssl.sh b/dbms/tests/queries/0_stateless/00505_tcp_ssl.sh.disabled
similarity index 100%
rename from dbms/tests/queries/0_stateless/00505_tcp_ssl.sh
rename to dbms/tests/queries/0_stateless/00505_tcp_ssl.sh.disabled

From 1c0999932f5de655b1db265658371123e1ed7d76 Mon Sep 17 00:00:00 2001
From: Mikhail Filimonov <1549571+filimonov@users.noreply.github.com>
Date: Wed, 14 Mar 2018 00:33:56 +0100
Subject: [PATCH 057/244] Documenting numbers table function, VerticalRaw
 format, HTTP sessions, HTTP compression.

---
 docs/en/formats/verticalraw.md       | 24 ++++++++++++++++++++++++
 docs/en/interfaces/http_interface.md | 13 ++++++++++---
 docs/en/table_functions/numbers.md   | 17 +++++++++++++++++
 docs/mkdocs_en.yml                   |  2 ++
 docs/mkdocs_ru.yml                   |  2 ++
 docs/ru/formats/verticalraw.md       | 26 ++++++++++++++++++++++++++
 docs/ru/interfaces/http_interface.md | 11 +++++++++--
 docs/ru/table_functions/numbers.md   | 16 ++++++++++++++++
 8 files changed, 106 insertions(+), 5 deletions(-)
 create mode 100644 docs/en/formats/verticalraw.md
 create mode 100644 docs/en/table_functions/numbers.md
 create mode 100644 docs/ru/formats/verticalraw.md
 create mode 100644 docs/ru/table_functions/numbers.md

diff --git a/docs/en/formats/verticalraw.md b/docs/en/formats/verticalraw.md
new file mode 100644
index 00000000000..440967eb598
--- /dev/null
+++ b/docs/en/formats/verticalraw.md
@@ -0,0 +1,24 @@
+# VerticalRaw
+
+Differs from `TabSeparated` format in that the rows are written without escaping.
+This format is only appropriate for outputting a query result, but not for parsing (retrieving data to insert in a table).
+
+Samples:
+```
+:) SHOW CREATE TABLE geonames FORMAT VerticalRaw;
+Row 1:
+──────
+statement: CREATE TABLE default.geonames ( geonameid UInt32,  name String,  asciiname String, alternatenames String,  latitude Float32,  longitude Float32,  feature_class String,  feature_code String,  country_code String,  cc2 String,  admin1_code String,  admin2_code String,  admin3_code String,  admin4_code String,  population Int64,  elevation String,  dem String,  timezone String,  modification_date Date, date Date DEFAULT CAST('2017-12-08' AS Date)) ENGINE = MergeTree(date, geonameid, 8192)
+
+:) SELECT 'string with \'quotes\' and \t with some special \n characters' AS test FORMAT VerticalRaw;
+Row 1:
+──────
+test: string with 'quotes' and   with some special
+ characters
+
+-- the same in Vertical format:
+:) SELECT 'string with \'quotes\' and \t with some special \n characters' AS test FORMAT Vertical;
+Row 1:
+──────
+test: string with \'quotes\' and \t with some special \n characters
+```
diff --git a/docs/en/interfaces/http_interface.md b/docs/en/interfaces/http_interface.md
index 91c6790f975..8c223cf69cf 100644
--- a/docs/en/interfaces/http_interface.md
+++ b/docs/en/interfaces/http_interface.md
@@ -130,11 +130,15 @@ POST 'http://localhost:8123/?query=DROP TABLE t'
 
 For successful requests that don't return a data table, an empty response body is returned.
 
-You can use compression when transmitting data. The compressed data has a non-standard format, and you will need to use the special compressor program to work with it (sudo apt-get install compressor-metrika-yandex).
+You can use compression when transmitting data.
 
+For using ClickHouse internal compression format, and you will need to use the special compressor program to work with it (sudo apt-get install compressor-metrika-yandex).
 If you specified 'compress=1' in the URL, the server will compress the data it sends you.
 If you specified 'decompress=1' in the URL, the server will decompress the same data that you pass in the POST method.
 
+Also standard gzip-based HTTP compression can be used. To send gzip compressed POST data just add `Content-Encoding: gzip` to request headers, and gzip POST body.
+To get response compressed, you need to add `Accept-Encoding: gzip` to request headers, and turn on ClickHouse setting called `enable_http_compression`.
+
 You can use this to reduce network traffic when transmitting a large amount of data, or for creating dumps that are immediately compressed.
 
 You can use the 'database' URL parameter to specify the default database.
@@ -190,7 +194,11 @@ $ echo 'SELECT number FROM system.numbers LIMIT 10' | curl 'http://localhost:812
 
 For information about other parameters, see the section "SET".
 
-In contrast to the native interface, the HTTP interface does not support the concept of sessions or session settings, does not allow aborting a query (to be exact, it allows this in only a few cases),  and does not show the progress of query processing. Parsing and data formatting are performed on the server side, and using the network might be ineffective.
+You can use ClickHouse sessions in the HTTP protocol. To do this, you need to specify the `session_id` GET parameter in HTTP request. You can use any alphanumeric string as a session_id. By default session will be timed out after 60 seconds of inactivity. You can change that by setting `default_session_timeout` in server config file, or by adding GET parameter `session_timeout`. You can also check the status of the session by using GET parameter `session_check=1`. When using sessions you can't run 2 queries with the same session_id simultaneously.
+
+You can get the progress of query execution in X-ClickHouse-Progress headers, by enabling setting send_progress_in_http_headers.
+
+Running query are not aborted automatically after closing HTTP connection. Parsing and data formatting are performed on the server side, and using the network might be ineffective.
 The optional 'query_id' parameter can be passed as the query ID (any string). For more information, see the section "Settings, replace_running_query".
 
 The optional 'quota_key' parameter can be passed as the quota key (any string). For more information, see the section "Quotas".
@@ -212,4 +220,3 @@ curl -sS 'http://localhost:8123/?max_result_bytes=4000000&buffer_size=3000000&wa
 ```
 
 Use buffering to avoid situations where a query processing error occurred after the response code and HTTP headers were sent to the client. In this situation, an error message is written at the end of the response body, and on the client side, the error can only be detected at the parsing stage.
-
diff --git a/docs/en/table_functions/numbers.md b/docs/en/table_functions/numbers.md
new file mode 100644
index 00000000000..b055f1cd56e
--- /dev/null
+++ b/docs/en/table_functions/numbers.md
@@ -0,0 +1,17 @@
+# numbers
+
+`numbers(N)` - returns the table with one column named `number` (UInt64 type), containing integer numbers from 0 to N-1.
+
+`numbers(N)` (like a table `system.numbers`) can be used in tests or for sequences generation.
+
+Two following queries are equal:
+```sql
+SELECT * FROM numbers(10);
+SELECT * FROM system.numbers LIMIT 10;
+```
+
+Samples:
+```sql
+-- generation of sequence of dates from 2010-01-01 to 2010-12-31
+select toDate('2010-01-01') + number as d FROM numbers(365);
+```
diff --git a/docs/mkdocs_en.yml b/docs/mkdocs_en.yml
index 1daf36ecfd8..c94e33c7818 100644
--- a/docs/mkdocs_en.yml
+++ b/docs/mkdocs_en.yml
@@ -122,6 +122,7 @@ pages:
   - 'Introduction': 'table_functions/index.md'
   - 'remote': 'table_functions/remote.md'
   - 'merge': 'table_functions/merge.md'
+  - 'numbers': 'table_functions/numbers.md'
 
 - 'Formats':
   - 'Introduction': 'formats/index.md'
@@ -133,6 +134,7 @@ pages:
   - 'CSVWithNames': 'formats/csvwithnames.md'
   - 'Values': 'formats/values.md'
   - 'Vertical': 'formats/vertical.md'
+  - 'VerticalRaw': 'formats/verticalraw.md'
   - 'JSON': 'formats/json.md'
   - 'JSONCompact': 'formats/jsoncompact.md'
   - 'JSONEachRow': 'formats/jsoneachrow.md'
diff --git a/docs/mkdocs_ru.yml b/docs/mkdocs_ru.yml
index 23734934bb5..eb8c002847d 100644
--- a/docs/mkdocs_ru.yml
+++ b/docs/mkdocs_ru.yml
@@ -124,6 +124,7 @@ pages:
   - 'Введение': 'table_functions/index.md'
   - 'remote': 'table_functions/remote.md'
   - 'merge': 'table_functions/merge.md'
+  - 'numbers': 'table_functions/numbers.md'
 
 - 'Форматы':
   - 'Введение': 'formats/index.md'
@@ -135,6 +136,7 @@ pages:
   - 'CSVWithNames': 'formats/csvwithnames.md'
   - 'Values': 'formats/values.md'
   - 'Vertical': 'formats/vertical.md'
+  - 'VerticalRaw': 'formats/verticalraw.md'
   - 'JSON': 'formats/json.md'
   - 'JSONCompact': 'formats/jsoncompact.md'
   - 'JSONEachRow': 'formats/jsoneachrow.md'
diff --git a/docs/ru/formats/verticalraw.md b/docs/ru/formats/verticalraw.md
new file mode 100644
index 00000000000..9693cabe3c3
--- /dev/null
+++ b/docs/ru/formats/verticalraw.md
@@ -0,0 +1,26 @@
+# VerticalRaw
+
+Отличается от формата `Vertical` тем, что строки выводятся без экранирования.
+Этот формат подходит только для вывода результата выполнения запроса, но не для парсинга (приёма данных для вставки в таблицу).
+
+Примеры:
+```
+:) SHOW CREATE TABLE geonames FORMAT VerticalRaw;
+Row 1:
+──────
+statement: CREATE TABLE default.geonames ( geonameid UInt32,  name String,  asciiname String, alternatenames String,  latitude Float32,  longitude Float32,  feature_class String,  feature_code String,  country_code String,  cc2 String,  admin1_code String,  admin2_code String,  admin3_code String,  admin4_code String,  population Int64,  elevation String,  dem String,  timezone String,  modification_date Date, date Date DEFAULT CAST('2017-12-08' AS Date)) ENGINE = MergeTree(date, geonameid, 8192)
+
+:) SELECT 'string with \'quotes\' and \t with some special \n characters' AS test FORMAT VerticalRaw;
+Row 1:
+──────
+test: string with 'quotes' and   with some special
+ characters
+```
+
+Для сравнения - формат Vertical:
+```
+:) SELECT 'string with \'quotes\' and \t with some special \n characters' AS test FORMAT Vertical;
+Row 1:
+──────
+test: string with \'quotes\' and \t with some special \n characters
+```
diff --git a/docs/ru/interfaces/http_interface.md b/docs/ru/interfaces/http_interface.md
index 28cb8ce6c66..a20ca4d844e 100644
--- a/docs/ru/interfaces/http_interface.md
+++ b/docs/ru/interfaces/http_interface.md
@@ -132,11 +132,14 @@ POST 'http://localhost:8123/?query=DROP TABLE t'
 
 Для запросов, которые не возвращают таблицу с данными, в случае успеха, выдаётся пустое тело ответа.
 
-Вы можете использовать сжатие при передаче данных. Формат сжатых данных нестандартный, и вам придётся использовать для работы с ним специальную программу compressor (sudo apt-get install compressor-metrika-yandex).
+Вы можете использовать внутренний формат сжатия Clickhouse при передаче данных. Формат сжатых данных нестандартный, и вам придётся использовать для работы с ним специальную программу compressor (sudo apt-get install compressor-metrika-yandex).
 
 Если вы указали в URL compress=1, то сервер будет сжимать отправляемые вам данные.
 Если вы указали в URL decompress=1, то сервер будет разжимать те данные, которые вы передаёте ему POST-ом.
 
+Также имеется возможность использования стандартного сжатия HTTP, на основе gzip. Чтобы отправить POST-запрос, сжатый с помощью gzip, добавьте к запросу заголовок `Content-Encoding: gzip`.
+Чтобы ClickHouse сжимал ответ на запрос с помощью gzip, необходимо добавить `Accept-Encoding: gzip` к заголовкам запроса, и включить настройку ClickHouse  `enable_http_compression`.
+
 Это может быть использовано для уменьшения трафика по сети при передаче большого количества данных, а также для создания сразу сжатых дампов.
 
 В параметре URL database может быть указана БД по умолчанию.
@@ -193,7 +196,11 @@ $ echo 'SELECT number FROM system.numbers LIMIT 10' | curl 'http://localhost:812
 
 Об остальных параметрах смотри раздел "SET".
 
-В отличие от родного интерфейса, HTTP интерфейс не поддерживает понятие сессии и настройки в пределах сессии, не позволяет (вернее, позволяет лишь в некоторых случаях) прервать выполнение запроса, не показывает прогресс выполнения запроса. Парсинг и форматирование данных производится на стороне сервера и использование сети может быть неэффективным.
+В HTTP-протоколе можно использовать ClickHouse-сессии, для этого необходимо добавить к запросу GET-пaраметр `session_id`. В качестве идентификатора сессии можно использовать произвольную строку. По умолчанию через 60 секунд бездействия сессия будет прервана. Можно изменить этот таймаут, изменяя настройку `default_session_timeout` в конфигурации сервера, или добавив к запросу GET параметр `session_timeout`. Статус сессии можно проверить с помощью параметра `session_check=1`. В рамках одной сессии одновременно может испольняться только один запрос.
+
+Имеется возможность получать информацию о прогрессе выполнения запроса в залоголвках X-ClickHouse-Progress, для этого нужно включить настройку send_progress_in_http_headers.
+
+Запущенные запросы не останавливаются автоматически при разрыве HTTP соединения. Парсинг и форматирование данных производится на стороне сервера и использование сети может быть неэффективным.
 Может быть передан необязательный параметр query_id - идентификатор запроса, произвольная строка. Подробнее смотрите раздел "Настройки, replace_running_query".
 
 Может быть передан необязательный параметр quota_key - ключ квоты, произвольная строка. Подробнее смотрите раздел "Квоты".
diff --git a/docs/ru/table_functions/numbers.md b/docs/ru/table_functions/numbers.md
new file mode 100644
index 00000000000..21bf3492ca1
--- /dev/null
+++ b/docs/ru/table_functions/numbers.md
@@ -0,0 +1,16 @@
+# numbers
+
+`numbers(N)` - возвращает таблицу с единственным столбцом number (тип UInt64), содержащую натуральные числа от 0 до N-1.
+
+Так же как и таблица `system.numbers` может использоваться для тестов и генерации последовательных значений.
+
+Следующие 2 запроса эквивалентны:
+```sql
+SELECT * FROM numbers(10);
+SELECT * FROM system.numbers LIMIT 10;
+```
+Примеры:
+```sql
+-- генарация последовательности всех дат от 2010-01-01 до 2010-12-31
+select toDate('2010-01-01') + number as d FROM numbers(365);
+```

From 04235f2e510b66ff7879fdb01811bc683292c8d2 Mon Sep 17 00:00:00 2001
From: Mikhail Filimonov <1549571+filimonov@users.noreply.github.com>
Date: Wed, 14 Mar 2018 00:42:06 +0100
Subject: [PATCH 058/244] Fixing bad copy-paste, shorten sample

---
 docs/en/formats/verticalraw.md | 4 ++--
 docs/ru/formats/verticalraw.md | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/en/formats/verticalraw.md b/docs/en/formats/verticalraw.md
index 440967eb598..9bb53ee1260 100644
--- a/docs/en/formats/verticalraw.md
+++ b/docs/en/formats/verticalraw.md
@@ -1,6 +1,6 @@
 # VerticalRaw
 
-Differs from `TabSeparated` format in that the rows are written without escaping.
+Differs from `Vertical` format in that the rows are written without escaping.
 This format is only appropriate for outputting a query result, but not for parsing (retrieving data to insert in a table).
 
 Samples:
@@ -8,7 +8,7 @@ Samples:
 :) SHOW CREATE TABLE geonames FORMAT VerticalRaw;
 Row 1:
 ──────
-statement: CREATE TABLE default.geonames ( geonameid UInt32,  name String,  asciiname String, alternatenames String,  latitude Float32,  longitude Float32,  feature_class String,  feature_code String,  country_code String,  cc2 String,  admin1_code String,  admin2_code String,  admin3_code String,  admin4_code String,  population Int64,  elevation String,  dem String,  timezone String,  modification_date Date, date Date DEFAULT CAST('2017-12-08' AS Date)) ENGINE = MergeTree(date, geonameid, 8192)
+statement: CREATE TABLE default.geonames ( geonameid UInt32, date Date DEFAULT CAST('2017-12-08' AS Date)) ENGINE = MergeTree(date, geonameid, 8192)
 
 :) SELECT 'string with \'quotes\' and \t with some special \n characters' AS test FORMAT VerticalRaw;
 Row 1:
diff --git a/docs/ru/formats/verticalraw.md b/docs/ru/formats/verticalraw.md
index 9693cabe3c3..fb497430fcd 100644
--- a/docs/ru/formats/verticalraw.md
+++ b/docs/ru/formats/verticalraw.md
@@ -8,7 +8,7 @@
 :) SHOW CREATE TABLE geonames FORMAT VerticalRaw;
 Row 1:
 ──────
-statement: CREATE TABLE default.geonames ( geonameid UInt32,  name String,  asciiname String, alternatenames String,  latitude Float32,  longitude Float32,  feature_class String,  feature_code String,  country_code String,  cc2 String,  admin1_code String,  admin2_code String,  admin3_code String,  admin4_code String,  population Int64,  elevation String,  dem String,  timezone String,  modification_date Date, date Date DEFAULT CAST('2017-12-08' AS Date)) ENGINE = MergeTree(date, geonameid, 8192)
+statement: CREATE TABLE default.geonames ( geonameid UInt32, date Date DEFAULT CAST('2017-12-08' AS Date)) ENGINE = MergeTree(date, geonameid, 8192)
 
 :) SELECT 'string with \'quotes\' and \t with some special \n characters' AS test FORMAT VerticalRaw;
 Row 1:

From 2c083b9b5916c0a868ebe27a5727a45e3703f3fa Mon Sep 17 00:00:00 2001
From: Mikhail Filimonov <1549571+filimonov@users.noreply.github.com>
Date: Wed, 14 Mar 2018 00:49:12 +0100
Subject: [PATCH 059/244] Fixing obsolete name of clickhouse-compressor

---
 docs/en/interfaces/http_interface.md | 2 +-
 docs/ru/interfaces/http_interface.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/en/interfaces/http_interface.md b/docs/en/interfaces/http_interface.md
index 8c223cf69cf..5c989a59d65 100644
--- a/docs/en/interfaces/http_interface.md
+++ b/docs/en/interfaces/http_interface.md
@@ -132,7 +132,7 @@ For successful requests that don't return a data table, an empty response body i
 
 You can use compression when transmitting data.
 
-For using ClickHouse internal compression format, and you will need to use the special compressor program to work with it (sudo apt-get install compressor-metrika-yandex).
+For using ClickHouse internal compression format, and you will need to use the special clickhouse-compressor program to work with it (installed as a part of clickhouse-client package).
 If you specified 'compress=1' in the URL, the server will compress the data it sends you.
 If you specified 'decompress=1' in the URL, the server will decompress the same data that you pass in the POST method.
 
diff --git a/docs/ru/interfaces/http_interface.md b/docs/ru/interfaces/http_interface.md
index a20ca4d844e..8f2ae4377f4 100644
--- a/docs/ru/interfaces/http_interface.md
+++ b/docs/ru/interfaces/http_interface.md
@@ -132,7 +132,7 @@ POST 'http://localhost:8123/?query=DROP TABLE t'
 
 Для запросов, которые не возвращают таблицу с данными, в случае успеха, выдаётся пустое тело ответа.
 
-Вы можете использовать внутренний формат сжатия Clickhouse при передаче данных. Формат сжатых данных нестандартный, и вам придётся использовать для работы с ним специальную программу compressor (sudo apt-get install compressor-metrika-yandex).
+Вы можете использовать внутренний формат сжатия Clickhouse при передаче данных. Формат сжатых данных нестандартный, и вам придётся использовать для работы с ним специальную программу clickhouse-compressor (устанавливается вместе с пакетом clickhouse-client).
 
 Если вы указали в URL compress=1, то сервер будет сжимать отправляемые вам данные.
 Если вы указали в URL decompress=1, то сервер будет разжимать те данные, которые вы передаёте ему POST-ом.

From 614cc887101a35a0db24729da41cdaa66b3d8f5d Mon Sep 17 00:00:00 2001
From: filimonov <1549571+filimonov@users.noreply.github.com>
Date: Wed, 14 Mar 2018 00:52:38 +0100
Subject: [PATCH 060/244] fix word end

---
 docs/ru/table_functions/numbers.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/ru/table_functions/numbers.md b/docs/ru/table_functions/numbers.md
index 21bf3492ca1..bd5d566f78e 100644
--- a/docs/ru/table_functions/numbers.md
+++ b/docs/ru/table_functions/numbers.md
@@ -1,6 +1,6 @@
 # numbers
 
-`numbers(N)` - возвращает таблицу с единственным столбцом number (тип UInt64), содержащую натуральные числа от 0 до N-1.
+`numbers(N)` - возвращает таблицу с единственным столбцом number (тип UInt64), содержащим натуральные числа от 0 до N-1.
 
 Так же как и таблица `system.numbers` может использоваться для тестов и генерации последовательных значений.
 

From 8bb3672beac6b2c659f13ff09b1e95da38a2263f Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Wed, 14 Mar 2018 05:42:48 +0300
Subject: [PATCH 061/244] Update InterpreterSelectQuery.cpp

---
 dbms/src/Interpreters/InterpreterSelectQuery.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp
index 49a46abeb13..49b3ba27004 100644
--- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp
@@ -712,7 +712,6 @@ QueryProcessingStage::Enum InterpreterSelectQuery::executeFetchColumns(Pipeline
     else
         throw Exception("Logical error in InterpreterSelectQuery: nowhere to read", ErrorCodes::LOGICAL_ERROR);
 
-    StackTrace st;
     /// Aliases in table declaration.
     if (from_stage == QueryProcessingStage::FetchColumns && alias_actions)
     {

From cbab87f18eb6c295de5f23eb91b003a8059a34ee Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Wed, 14 Mar 2018 05:59:27 +0300
Subject: [PATCH 062/244] Update AggregateFunctionIntersectionsMax.cpp

---
 .../AggregateFunctions/AggregateFunctionIntersectionsMax.cpp  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dbms/src/AggregateFunctions/AggregateFunctionIntersectionsMax.cpp b/dbms/src/AggregateFunctions/AggregateFunctionIntersectionsMax.cpp
index 854737ceba4..1292bcb8561 100644
--- a/dbms/src/AggregateFunctions/AggregateFunctionIntersectionsMax.cpp
+++ b/dbms/src/AggregateFunctions/AggregateFunctionIntersectionsMax.cpp
@@ -133,10 +133,10 @@ AggregateFunctionPtr createAggregateFunctionIntersectionsMaxPos(const std::strin
 
 void registerAggregateFunctionIntersectionsMax(AggregateFunctionFactory & factory)
 {
-    factory.registerFunction("IntersectionsMax",
+    factory.registerFunction("intersectionsMax",
         createAggregateFunctionIntersectionsMax,
         AggregateFunctionFactory::CaseInsensitive);
-    factory.registerFunction("IntersectionsMaxPos",
+    factory.registerFunction("intersectionsMaxPos",
         createAggregateFunctionIntersectionsMaxPos,
         AggregateFunctionFactory::CaseInsensitive);
 }

From b4f1ecb7f8f8f9198fb071af5f509a71a8599945 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Wed, 14 Mar 2018 06:02:18 +0300
Subject: [PATCH 063/244] clang-format #2012

---
 .../AggregateFunctionIntersectionsMax.cpp     | 123 +++++++++---------
 .../AggregateFunctionIntersectionsMax.h       |  79 ++++++-----
 2 files changed, 101 insertions(+), 101 deletions(-)

diff --git a/dbms/src/AggregateFunctions/AggregateFunctionIntersectionsMax.cpp b/dbms/src/AggregateFunctions/AggregateFunctionIntersectionsMax.cpp
index 1292bcb8561..e9e6c739422 100644
--- a/dbms/src/AggregateFunctions/AggregateFunctionIntersectionsMax.cpp
+++ b/dbms/src/AggregateFunctions/AggregateFunctionIntersectionsMax.cpp
@@ -7,62 +7,66 @@
 
 namespace DB
 {
-
 template <typename T>
-typename Intersections<T>::PointsMap::iterator
-Intersections<T>::insert_point(const T &v)
+typename Intersections<T>::PointsMap::iterator Intersections<T>::insert_point(const T & v)
 {
-    auto res = points.emplace(v,0);
-    auto &i = res.first;
-    if(!res.second) return i;
-    if(i==points.begin()) return i;
+    auto res = points.emplace(v, 0);
+    auto & i = res.first;
+    if (!res.second)
+        return i;
+    if (i == points.begin())
+        return i;
     auto prev = i;
     prev--;
-    i->second=prev->second;
+    i->second = prev->second;
     return i;
 }
 
 template <typename T>
-void Intersections<T>::add(const T &start, const T &end, T weight)
+void Intersections<T>::add(const T & start, const T & end, T weight)
 {
     auto sp = insert_point(start);
     auto ep = end ? insert_point(end) : points.end();
-    do {
-        sp->second+=weight;
-        if(sp->second > max_weight) {
+    do
+    {
+        sp->second += weight;
+        if (sp->second > max_weight)
+        {
             max_weight = sp->second;
             max_weight_pos = sp->first;
         }
-    } while(++sp != ep);
+    } while (++sp != ep);
 }
 
 template <typename T>
-void Intersections<T>::merge(const Intersections &other)
+void Intersections<T>::merge(const Intersections & other)
 {
-    if(other.points.empty())
+    if (other.points.empty())
         return;
 
     typename PointsMap::const_iterator prev, i = other.points.begin();
     prev = i;
     i++;
 
-    while(i != other.points.end()) {
-        add(prev->first,i->first,prev->second);
+    while (i != other.points.end())
+    {
+        add(prev->first, i->first, prev->second);
         prev = i;
         i++;
     }
 
-    if(prev != other.points.end())
-        add(prev->first,0,prev->second);
+    if (prev != other.points.end())
+        add(prev->first, 0, prev->second);
 }
 
 template <typename T>
 void Intersections<T>::serialize(WriteBuffer & buf) const
 {
-    writeBinary(points.size(),buf);
-    for(const auto &p: points) {
-        writeBinary(p.first,buf);
-        writeBinary(p.second,buf);
+    writeBinary(points.size(), buf);
+    for (const auto & p : points)
+    {
+        writeBinary(p.first, buf);
+        writeBinary(p.second, buf);
     }
 }
 
@@ -74,72 +78,71 @@ void Intersections<T>::deserialize(ReadBuffer & buf)
     T weight;
 
     readBinary(size, buf);
-    for (std::size_t i = 0; i < size; ++i) {
+    for (std::size_t i = 0; i < size; ++i)
+    {
         readBinary(point, buf);
-        readBinary(weight,buf);
-        points.emplace(point,weight);
+        readBinary(weight, buf);
+        points.emplace(point, weight);
     }
 }
 
 void AggregateFunctionIntersectionsMax::_add(
-        AggregateDataPtr place,
-        const IColumn & column_start,
-        const IColumn & column_end,
-        size_t row_num) const
+    AggregateDataPtr place, const IColumn & column_start, const IColumn & column_end, size_t row_num) const
 {
     PointType start_time, end_time;
     Field tmp_start_time_field, tmp_end_time_field;
 
-    column_start.get(row_num,tmp_start_time_field);
-    if(tmp_start_time_field.isNull())
+    column_start.get(row_num, tmp_start_time_field);
+    if (tmp_start_time_field.isNull())
         return;
     start_time = tmp_start_time_field.template get<PointType>();
-    if(0==start_time)
+    if (0 == start_time)
         return;
 
-    column_end.get(row_num,tmp_end_time_field);
-    if(tmp_end_time_field.isNull()) {
+    column_end.get(row_num, tmp_end_time_field);
+    if (tmp_end_time_field.isNull())
+    {
         end_time = 0;
-    } else {
+    }
+    else
+    {
         end_time = tmp_end_time_field.template get<PointType>();
-        if(0!=end_time) {
-            if(end_time==start_time) {
+        if (0 != end_time)
+        {
+            if (end_time == start_time)
+            {
                 end_time = 0;
-            } else if(end_time < start_time) {
+            }
+            else if (end_time < start_time)
+            {
                 return;
             }
         }
     }
 
-    data(place).add(start_time,end_time);
+    data(place).add(start_time, end_time);
 }
 
 namespace
 {
+    AggregateFunctionPtr createAggregateFunctionIntersectionsMax(
+        const std::string & name, const DataTypes & argument_types, const Array & parameters)
+    {
+        assertBinary(name, argument_types);
+        return std::make_shared<AggregateFunctionIntersectionsMax>(argument_types, parameters, false);
+    }
 
-AggregateFunctionPtr createAggregateFunctionIntersectionsMax(const std::string & name, const DataTypes & argument_types, const Array & parameters)
-{
-    assertBinary(name, argument_types);
-    return std::make_shared<AggregateFunctionIntersectionsMax>(argument_types,parameters,false);
-}
-
-AggregateFunctionPtr createAggregateFunctionIntersectionsMaxPos(const std::string & name, const DataTypes & argument_types, const Array & parameters)
-{
-    assertBinary(name, argument_types);
-    return std::make_shared<AggregateFunctionIntersectionsMax>(argument_types,parameters,true);
-}
-
+    AggregateFunctionPtr createAggregateFunctionIntersectionsMaxPos(
+        const std::string & name, const DataTypes & argument_types, const Array & parameters)
+    {
+        assertBinary(name, argument_types);
+        return std::make_shared<AggregateFunctionIntersectionsMax>(argument_types, parameters, true);
+    }
 }
 
 void registerAggregateFunctionIntersectionsMax(AggregateFunctionFactory & factory)
 {
-    factory.registerFunction("intersectionsMax",
-        createAggregateFunctionIntersectionsMax,
-        AggregateFunctionFactory::CaseInsensitive);
-    factory.registerFunction("intersectionsMaxPos",
-        createAggregateFunctionIntersectionsMaxPos,
-        AggregateFunctionFactory::CaseInsensitive);
+    factory.registerFunction("intersectionsMax", createAggregateFunctionIntersectionsMax, AggregateFunctionFactory::CaseInsensitive);
+    factory.registerFunction("intersectionsMaxPos", createAggregateFunctionIntersectionsMaxPos, AggregateFunctionFactory::CaseInsensitive);
 }
-
 }
-
diff --git a/dbms/src/AggregateFunctions/AggregateFunctionIntersectionsMax.h b/dbms/src/AggregateFunctions/AggregateFunctionIntersectionsMax.h
index ca97986e678..c15ae1ae33d 100644
--- a/dbms/src/AggregateFunctions/AggregateFunctionIntersectionsMax.h
+++ b/dbms/src/AggregateFunctions/AggregateFunctionIntersectionsMax.h
@@ -2,10 +2,10 @@
 
 #include <common/logger_useful.h>
 
-#include <DataTypes/DataTypesNumber.h>
-#include <DataTypes/DataTypeNullable.h>
 #include <DataTypes/DataTypeDate.h>
 #include <DataTypes/DataTypeDateTime.h>
+#include <DataTypes/DataTypeNullable.h>
+#include <DataTypes/DataTypesNumber.h>
 
 #include <Columns/ColumnNullable.h>
 
@@ -15,7 +15,6 @@
 
 namespace DB
 {
-
 namespace ErrorCodes
 {
     extern const int AGGREGATE_FUNCTION_DOESNT_ALLOW_PARAMETERS;
@@ -26,71 +25,68 @@ namespace ErrorCodes
 template <typename T>
 class Intersections final
 {
-
     using PointsMap = std::map<T, T>;
 
     PointsMap points;
     T max_weight;
     T max_weight_pos;
 
-    typename PointsMap::iterator insert_point(const T &v);
-  public:
+    typename PointsMap::iterator insert_point(const T & v);
 
-    Intersections()
-      : max_weight(0)
-    { }
+public:
+    Intersections() : max_weight(0) {}
 
-    void add(const T &start, const T &end, T weight = 1);
-    void merge(const Intersections &other);
+    void add(const T & start, const T & end, T weight = 1);
+    void merge(const Intersections & other);
 
     void serialize(WriteBuffer & buf) const;
     void deserialize(ReadBuffer & buf);
 
-    T max() const { return max_weight; }
-    T max_pos() const { return max_weight_pos; }
+    T max() const
+    {
+        return max_weight;
+    }
+    T max_pos() const
+    {
+        return max_weight_pos;
+    }
 };
 
 class AggregateFunctionIntersectionsMax final
-  : public IAggregateFunctionDataHelper<Intersections<UInt64>, AggregateFunctionIntersectionsMax>
+    : public IAggregateFunctionDataHelper<Intersections<UInt64>, AggregateFunctionIntersectionsMax>
 {
     using PointType = UInt64;
 
     bool return_position;
     void _add(AggregateDataPtr place, const IColumn & column_start, const IColumn & column_end, size_t row_num) const;
 
-  public:
+public:
     AggregateFunctionIntersectionsMax(const DataTypes & arguments, const Array & params, bool return_position)
-      : return_position(return_position)
+        : return_position(return_position)
     {
-        if (!params.empty()) {
+        if (!params.empty())
+        {
             throw Exception(
-                "Aggregate function " + getName() + " does not allow paremeters.",
-                ErrorCodes::AGGREGATE_FUNCTION_DOESNT_ALLOW_PARAMETERS);
+                "Aggregate function " + getName() + " does not allow paremeters.", ErrorCodes::AGGREGATE_FUNCTION_DOESNT_ALLOW_PARAMETERS);
         }
 
         if (arguments.size() != 2)
             throw Exception("Aggregate function " + getName() + " requires two arguments.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
 
-        if(!arguments[0]->isValueRepresentedByInteger())
-            throw Exception {
-                getName() + ": first argument must be represented by integer",
-                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT
-            };
+        if (!arguments[0]->isValueRepresentedByInteger())
+            throw Exception{getName() + ": first argument must be represented by integer", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
 
-        if(!arguments[1]->isValueRepresentedByInteger())
-            throw Exception {
-                getName() + ": second argument must be represented by integer",
-                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT
-            };
+        if (!arguments[1]->isValueRepresentedByInteger())
+            throw Exception{getName() + ": second argument must be represented by integer", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
 
-        if(!arguments[0]->equals(*arguments[1]))
-            throw Exception {
-                getName() + ": arguments must have the same type",
-                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT
-            };
+        if (!arguments[0]->equals(*arguments[1]))
+            throw Exception{getName() + ": arguments must have the same type", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
     }
 
-    String getName() const override { return "IntersectionsMax"; }
+    String getName() const override
+    {
+        return "IntersectionsMax";
+    }
 
     DataTypePtr getReturnType() const override
     {
@@ -99,7 +95,7 @@ class AggregateFunctionIntersectionsMax final
 
     void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override
     {
-        _add(place,*columns[0],*columns[1],row_num);
+        _add(place, *columns[0], *columns[1], row_num);
     }
 
     void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override
@@ -119,14 +115,15 @@ class AggregateFunctionIntersectionsMax final
 
     void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override
     {
-        auto &ret = static_cast<ColumnUInt64 &>(to).getData();
+        auto & ret = static_cast<ColumnUInt64 &>(to).getData();
         ret.push_back(data(place).max());
-        if(return_position)
+        if (return_position)
             ret.push_back(data(place).max_pos());
     }
 
-    const char * getHeaderFilePath() const override { return __FILE__; }
+    const char * getHeaderFilePath() const override
+    {
+        return __FILE__;
+    }
 };
-
 }
-

From c9d1b61ef7a8a3c67c89354e7f3d502a3b62f561 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Wed, 14 Mar 2018 06:19:23 +0300
Subject: [PATCH 064/244] Fixed build after merge #2029

---
 dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.h | 3 +++
 dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp     | 2 ++
 dbms/src/Storages/System/StorageSystemMacros.cpp      | 4 ++--
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.h b/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.h
index b694b4535da..055581eb518 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.h
+++ b/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.h
@@ -1,8 +1,11 @@
 #pragma once
+
+#include <optional>
 #include <Core/NamesAndTypes.h>
 #include <Storages/MergeTree/RangesInDataPart.h>
 #include <Storages/MergeTree/MergeTreeRangeReader.h>
 
+
 namespace DB
 {
 
diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp
index 38608f6ea32..8ab7699a03b 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp
@@ -1,3 +1,5 @@
+#include <optional>
+
 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
 #include <IO/CompressedStream.h>
diff --git a/dbms/src/Storages/System/StorageSystemMacros.cpp b/dbms/src/Storages/System/StorageSystemMacros.cpp
index 37c3197fd60..456730bde4b 100644
--- a/dbms/src/Storages/System/StorageSystemMacros.cpp
+++ b/dbms/src/Storages/System/StorageSystemMacros.cpp
@@ -13,10 +13,10 @@ namespace DB
 StorageSystemMacros::StorageSystemMacros(const std::string & name_)
         : name(name_)
 {
-    columns = NamesAndTypesList{
+    setColumns(ColumnsDescription({
             {"macro", std::make_shared<DataTypeString>()},
             {"substitution", std::make_shared<DataTypeString>()},
-    };
+    }));
 }
 
 

From 688d277ad4bcd3826f5ceb40ca32c1ca9c894132 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Wed, 14 Mar 2018 07:36:41 +0300
Subject: [PATCH 065/244] Adaptation #2012

---
 .../AggregateFunctionIntersectionsMax.cpp     | 148 ----------------
 .../AggregateFunctionIntersectionsMax.h       | 129 --------------
 .../AggregateFunctionMaxIntersections.cpp     |  32 ++++
 .../AggregateFunctionMaxIntersections.h       | 167 ++++++++++++++++++
 .../registerAggregateFunctions.cpp            |   4 +-
 ...ntersections_aggregate_functions.reference |   2 +
 ...0605_intersections_aggregate_functions.sql |  20 +++
 7 files changed, 223 insertions(+), 279 deletions(-)
 delete mode 100644 dbms/src/AggregateFunctions/AggregateFunctionIntersectionsMax.cpp
 delete mode 100644 dbms/src/AggregateFunctions/AggregateFunctionIntersectionsMax.h
 create mode 100644 dbms/src/AggregateFunctions/AggregateFunctionMaxIntersections.cpp
 create mode 100644 dbms/src/AggregateFunctions/AggregateFunctionMaxIntersections.h
 create mode 100644 dbms/tests/queries/0_stateless/00605_intersections_aggregate_functions.reference
 create mode 100644 dbms/tests/queries/0_stateless/00605_intersections_aggregate_functions.sql

diff --git a/dbms/src/AggregateFunctions/AggregateFunctionIntersectionsMax.cpp b/dbms/src/AggregateFunctions/AggregateFunctionIntersectionsMax.cpp
deleted file mode 100644
index e9e6c739422..00000000000
--- a/dbms/src/AggregateFunctions/AggregateFunctionIntersectionsMax.cpp
+++ /dev/null
@@ -1,148 +0,0 @@
-#include <AggregateFunctions/AggregateFunctionFactory.h>
-#include <AggregateFunctions/AggregateFunctionIntersectionsMax.h>
-#include <AggregateFunctions/FactoryHelpers.h>
-
-#include <IO/ReadHelpers.h>
-#include <IO/WriteHelpers.h>
-
-namespace DB
-{
-template <typename T>
-typename Intersections<T>::PointsMap::iterator Intersections<T>::insert_point(const T & v)
-{
-    auto res = points.emplace(v, 0);
-    auto & i = res.first;
-    if (!res.second)
-        return i;
-    if (i == points.begin())
-        return i;
-    auto prev = i;
-    prev--;
-    i->second = prev->second;
-    return i;
-}
-
-template <typename T>
-void Intersections<T>::add(const T & start, const T & end, T weight)
-{
-    auto sp = insert_point(start);
-    auto ep = end ? insert_point(end) : points.end();
-    do
-    {
-        sp->second += weight;
-        if (sp->second > max_weight)
-        {
-            max_weight = sp->second;
-            max_weight_pos = sp->first;
-        }
-    } while (++sp != ep);
-}
-
-template <typename T>
-void Intersections<T>::merge(const Intersections & other)
-{
-    if (other.points.empty())
-        return;
-
-    typename PointsMap::const_iterator prev, i = other.points.begin();
-    prev = i;
-    i++;
-
-    while (i != other.points.end())
-    {
-        add(prev->first, i->first, prev->second);
-        prev = i;
-        i++;
-    }
-
-    if (prev != other.points.end())
-        add(prev->first, 0, prev->second);
-}
-
-template <typename T>
-void Intersections<T>::serialize(WriteBuffer & buf) const
-{
-    writeBinary(points.size(), buf);
-    for (const auto & p : points)
-    {
-        writeBinary(p.first, buf);
-        writeBinary(p.second, buf);
-    }
-}
-
-template <typename T>
-void Intersections<T>::deserialize(ReadBuffer & buf)
-{
-    std::size_t size;
-    T point;
-    T weight;
-
-    readBinary(size, buf);
-    for (std::size_t i = 0; i < size; ++i)
-    {
-        readBinary(point, buf);
-        readBinary(weight, buf);
-        points.emplace(point, weight);
-    }
-}
-
-void AggregateFunctionIntersectionsMax::_add(
-    AggregateDataPtr place, const IColumn & column_start, const IColumn & column_end, size_t row_num) const
-{
-    PointType start_time, end_time;
-    Field tmp_start_time_field, tmp_end_time_field;
-
-    column_start.get(row_num, tmp_start_time_field);
-    if (tmp_start_time_field.isNull())
-        return;
-    start_time = tmp_start_time_field.template get<PointType>();
-    if (0 == start_time)
-        return;
-
-    column_end.get(row_num, tmp_end_time_field);
-    if (tmp_end_time_field.isNull())
-    {
-        end_time = 0;
-    }
-    else
-    {
-        end_time = tmp_end_time_field.template get<PointType>();
-        if (0 != end_time)
-        {
-            if (end_time == start_time)
-            {
-                end_time = 0;
-            }
-            else if (end_time < start_time)
-            {
-                return;
-            }
-        }
-    }
-
-    data(place).add(start_time, end_time);
-}
-
-namespace
-{
-    AggregateFunctionPtr createAggregateFunctionIntersectionsMax(
-        const std::string & name, const DataTypes & argument_types, const Array & parameters)
-    {
-        assertBinary(name, argument_types);
-        return std::make_shared<AggregateFunctionIntersectionsMax>(argument_types, parameters, false);
-    }
-
-    AggregateFunctionPtr createAggregateFunctionIntersectionsMaxPos(
-        const std::string & name, const DataTypes & argument_types, const Array & parameters)
-    {
-        assertBinary(name, argument_types);
-        return std::make_shared<AggregateFunctionIntersectionsMax>(argument_types, parameters, true);
-    }
-}
-
-void registerAggregateFunctionIntersectionsMax(AggregateFunctionFactory & factory)
-{
-    factory.registerFunction("intersectionsMax", createAggregateFunctionIntersectionsMax, AggregateFunctionFactory::CaseInsensitive);
-    factory.registerFunction("intersectionsMaxPos", createAggregateFunctionIntersectionsMaxPos, AggregateFunctionFactory::CaseInsensitive);
-}
-}
diff --git a/dbms/src/AggregateFunctions/AggregateFunctionIntersectionsMax.h b/dbms/src/AggregateFunctions/AggregateFunctionIntersectionsMax.h
deleted file mode 100644
index c15ae1ae33d..00000000000
--- a/dbms/src/AggregateFunctions/AggregateFunctionIntersectionsMax.h
+++ /dev/null
@@ -1,129 +0,0 @@
-#pragma once
-
-#include <common/logger_useful.h>
-
-#include <DataTypes/DataTypeDate.h>
-#include <DataTypes/DataTypeDateTime.h>
-#include <DataTypes/DataTypeNullable.h>
-#include <DataTypes/DataTypesNumber.h>
-
-#include <Columns/ColumnNullable.h>
-
-#include <Common/Allocator.h>
-
-#include <AggregateFunctions/IAggregateFunction.h>
-
-namespace DB
-{
-namespace ErrorCodes
-{
-    extern const int AGGREGATE_FUNCTION_DOESNT_ALLOW_PARAMETERS;
-    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
-    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
-}
-
-template <typename T>
-class Intersections final
-{
-    using PointsMap = std::map<T, T>;
-
-    PointsMap points;
-    T max_weight;
-    T max_weight_pos;
-
-    typename PointsMap::iterator insert_point(const T & v);
-
-public:
-    Intersections() : max_weight(0) {}
-
-    void add(const T & start, const T & end, T weight = 1);
-    void merge(const Intersections & other);
-
-    void serialize(WriteBuffer & buf) const;
-    void deserialize(ReadBuffer & buf);
-
-    T max() const
-    {
-        return max_weight;
-    }
-    T max_pos() const
-    {
-        return max_weight_pos;
-    }
-};
-
-class AggregateFunctionIntersectionsMax final
-    : public IAggregateFunctionDataHelper<Intersections<UInt64>, AggregateFunctionIntersectionsMax>
-{
-    using PointType = UInt64;
-
-    bool return_position;
-    void _add(AggregateDataPtr place, const IColumn & column_start, const IColumn & column_end, size_t row_num) const;
-
-public:
-    AggregateFunctionIntersectionsMax(const DataTypes & arguments, const Array & params, bool return_position)
-        : return_position(return_position)
-    {
-        if (!params.empty())
-        {
-            throw Exception(
-                "Aggregate function " + getName() + " does not allow paremeters.", ErrorCodes::AGGREGATE_FUNCTION_DOESNT_ALLOW_PARAMETERS);
-        }
-
-        if (arguments.size() != 2)
-            throw Exception("Aggregate function " + getName() + " requires two arguments.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
-
-        if (!arguments[0]->isValueRepresentedByInteger())
-            throw Exception{getName() + ": first argument must be represented by integer", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
-
-        if (!arguments[1]->isValueRepresentedByInteger())
-            throw Exception{getName() + ": second argument must be represented by integer", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
-
-        if (!arguments[0]->equals(*arguments[1]))
-            throw Exception{getName() + ": arguments must have the same type", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
-    }
-
-    String getName() const override
-    {
-        return "IntersectionsMax";
-    }
-
-    DataTypePtr getReturnType() const override
-    {
-        return std::make_shared<DataTypeUInt64>();
-    }
-
-    void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override
-    {
-        _add(place, *columns[0], *columns[1], row_num);
-    }
-
-    void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override
-    {
-        this->data(place).merge(data(rhs));
-    }
-
-    void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override
-    {
-        this->data(place).serialize(buf);
-    }
-
-    void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena *) const override
-    {
-        this->data(place).deserialize(buf);
-    }
-
-    void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override
-    {
-        auto & ret = static_cast<ColumnUInt64 &>(to).getData();
-        ret.push_back(data(place).max());
-        if (return_position)
-            ret.push_back(data(place).max_pos());
-    }
-
-    const char * getHeaderFilePath() const override
-    {
-        return __FILE__;
-    }
-};
-}
diff --git a/dbms/src/AggregateFunctions/AggregateFunctionMaxIntersections.cpp b/dbms/src/AggregateFunctions/AggregateFunctionMaxIntersections.cpp
new file mode 100644
index 00000000000..e81dfe98259
--- /dev/null
+++ b/dbms/src/AggregateFunctions/AggregateFunctionMaxIntersections.cpp
@@ -0,0 +1,32 @@
+#include <AggregateFunctions/AggregateFunctionFactory.h>
+#include <AggregateFunctions/AggregateFunctionMaxIntersections.h>
+#include <AggregateFunctions/FactoryHelpers.h>
+#include <AggregateFunctions/Helpers.h>
+
+
+namespace DB
+{
+
+namespace
+{
+    AggregateFunctionPtr createAggregateFunctionMaxIntersections(
+        AggregateFunctionIntersectionsKind kind,
+        const std::string & name, const DataTypes & argument_types, const Array & parameters)
+    {
+        assertBinary(name, argument_types);
+        assertNoParameters(name, parameters);
+
+        return AggregateFunctionPtr{createWithNumericType<AggregateFunctionIntersectionsMax>(*argument_types[0], kind, argument_types)};
+    }
+}
+
+void registerAggregateFunctionsMaxIntersections(AggregateFunctionFactory & factory)
+{
+    factory.registerFunction("maxIntersections", [](const std::string & name, const DataTypes & argument_types, const Array & parameters)
+        { return createAggregateFunctionMaxIntersections(AggregateFunctionIntersectionsKind::Count, name, argument_types, parameters); });
+
+    factory.registerFunction("maxIntersectionsPosition", [](const std::string & name, const DataTypes & argument_types, const Array & parameters)
+        { return createAggregateFunctionMaxIntersections(AggregateFunctionIntersectionsKind::Position, name, argument_types, parameters); });
+}
+
+}
diff --git a/dbms/src/AggregateFunctions/AggregateFunctionMaxIntersections.h b/dbms/src/AggregateFunctions/AggregateFunctionMaxIntersections.h
new file mode 100644
index 00000000000..ebc283c03fc
--- /dev/null
+++ b/dbms/src/AggregateFunctions/AggregateFunctionMaxIntersections.h
@@ -0,0 +1,167 @@
+#pragma once
+
+#include <common/logger_useful.h>
+
+#include <DataTypes/DataTypesNumber.h>
+#include <Columns/ColumnsNumber.h>
+
+#include <IO/ReadHelpers.h>
+#include <IO/WriteHelpers.h>
+
+#include <Common/ArenaAllocator.h>
+
+#include <AggregateFunctions/IAggregateFunction.h>
+
+#define AGGREGATE_FUNCTION_MAX_INTERSECTIONS_MAX_ARRAY_SIZE 0xFFFFFF
+
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+    extern const int TOO_LARGE_ARRAY_SIZE;
+}
+
+
+/** maxIntersections: returns maximum count of the intersected intervals defined by start_column and end_column values,
+  * maxIntersectionsPosition: returns leftmost position of maximum intersection of intervals.
+  */
+
+/// Similar to GroupArrayNumericData.
+template <typename T>
+struct MaxIntersectionsData
+{
+    /// Left or right end of the interval and signed weight; with positive sign for begin of interval and negative sign for end of interval.
+    using Value = std::pair<T, Int64>;
+
+    // Switch to ordinary Allocator after 4096 bytes to avoid fragmentation and trash in Arena
+    using Allocator = MixedArenaAllocator<4096>;
+    using Array = PODArray<Value, 32, Allocator>;
+
+    Array value;
+};
+
+enum class AggregateFunctionIntersectionsKind
+{
+    Count,
+    Position
+};
+
+template <typename PointType>
+class AggregateFunctionIntersectionsMax final
+    : public IAggregateFunctionDataHelper<MaxIntersectionsData<PointType>, AggregateFunctionIntersectionsMax<PointType>>
+{
+private:
+    AggregateFunctionIntersectionsKind kind;
+
+public:
+    AggregateFunctionIntersectionsMax(AggregateFunctionIntersectionsKind kind_, const DataTypes & arguments)
+        : kind(kind_)
+    {
+        if (!arguments[0]->isNumber())
+            throw Exception{getName() + ": first argument must be represented by integer", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
+
+        if (!arguments[1]->isNumber())
+            throw Exception{getName() + ": second argument must be represented by integer", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
+
+        if (!arguments[0]->equals(*arguments[1]))
+            throw Exception{getName() + ": arguments must have the same type", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
+    }
+
+    String getName() const override
+    {
+        return kind == AggregateFunctionIntersectionsKind::Count
+            ? "maxIntersections"
+            : "maxIntersectionsPosition";
+    }
+
+    DataTypePtr getReturnType() const override
+    {
+        if (kind == AggregateFunctionIntersectionsKind::Count)
+            return std::make_shared<DataTypeUInt64>();
+        else
+            return std::make_shared<DataTypeNumber<PointType>>();
+    }
+
+    void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena * arena) const override
+    {
+        PointType left = static_cast<const ColumnVector<PointType> &>(*columns[0]).getData()[row_num];
+        PointType right = static_cast<const ColumnVector<PointType> &>(*columns[1]).getData()[row_num];
+
+        this->data(place).value.push_back(std::make_pair(left, Int64(1)), arena);
+        this->data(place).value.push_back(std::make_pair(right, Int64(-1)), arena);
+    }
+
+    void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena * arena) const override
+    {
+        auto & cur_elems = this->data(place);
+        auto & rhs_elems = this->data(rhs);
+
+        cur_elems.value.insert(rhs_elems.value.begin(), rhs_elems.value.end(), arena);
+    }
+
+    void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override
+    {
+        const auto & value = this->data(place).value;
+        size_t size = value.size();
+        writeVarUInt(size, buf);
+        buf.write(reinterpret_cast<const char *>(&value[0]), size * sizeof(value[0]));
+    }
+
+    void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena * arena) const override
+    {
+        size_t size = 0;
+        readVarUInt(size, buf);
+
+        if (unlikely(size > AGGREGATE_FUNCTION_MAX_INTERSECTIONS_MAX_ARRAY_SIZE))
+            throw Exception("Too large array size", ErrorCodes::TOO_LARGE_ARRAY_SIZE);
+
+        auto & value = this->data(place).value;
+
+        value.resize(size, arena);
+        buf.read(reinterpret_cast<char *>(&value[0]), size * sizeof(value[0]));
+    }
+
+    void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override
+    {
+        Int64 current_intersections = 0;
+        Int64 max_intersections = 0;
+        PointType position_of_max_intersections = 0;
+
+        /// const_cast because we will sort the array
+        auto & array = const_cast<typename MaxIntersectionsData<PointType>::Array &>(this->data(place).value);
+
+        /// TODO NaNs?
+        std::sort(array.begin(), array.end(), [](const auto & a, const auto & b) { return a.first < b.first; });
+
+        for (const auto & point_weight : array)
+        {
+            current_intersections += point_weight.second;
+            if (current_intersections > max_intersections)
+            {
+                max_intersections = current_intersections;
+                position_of_max_intersections = point_weight.first;
+            }
+        }
+
+        if (kind == AggregateFunctionIntersectionsKind::Count)
+        {
+            auto & result_column = static_cast<ColumnUInt64 &>(to).getData();
+            result_column.push_back(max_intersections);
+        }
+        else
+        {
+            auto & result_column = static_cast<ColumnVector<PointType> &>(to).getData();
+            result_column.push_back(position_of_max_intersections);
+        }
+    }
+
+    const char * getHeaderFilePath() const override
+    {
+        return __FILE__;
+    }
+};
+
+}
diff --git a/dbms/src/AggregateFunctions/registerAggregateFunctions.cpp b/dbms/src/AggregateFunctions/registerAggregateFunctions.cpp
index d054107c705..644543e4f2e 100644
--- a/dbms/src/AggregateFunctions/registerAggregateFunctions.cpp
+++ b/dbms/src/AggregateFunctions/registerAggregateFunctions.cpp
@@ -23,7 +23,7 @@ void registerAggregateFunctionsUniq(AggregateFunctionFactory &);
 void registerAggregateFunctionUniqUpTo(AggregateFunctionFactory &);
 void registerAggregateFunctionTopK(AggregateFunctionFactory &);
 void registerAggregateFunctionsBitwise(AggregateFunctionFactory &);
-void registerAggregateFunctionIntersectionsMax(AggregateFunctionFactory &);
+void registerAggregateFunctionsMaxIntersections(AggregateFunctionFactory &);
 
 void registerAggregateFunctionCombinatorIf(AggregateFunctionCombinatorFactory &);
 void registerAggregateFunctionCombinatorArray(AggregateFunctionCombinatorFactory &);
@@ -54,7 +54,7 @@ void registerAggregateFunctions()
         registerAggregateFunctionUniqUpTo(factory);
         registerAggregateFunctionTopK(factory);
         registerAggregateFunctionsBitwise(factory);
-        registerAggregateFunctionIntersectionsMax(factory);
+        registerAggregateFunctionsMaxIntersections(factory);
     }
 
     {
diff --git a/dbms/tests/queries/0_stateless/00605_intersections_aggregate_functions.reference b/dbms/tests/queries/0_stateless/00605_intersections_aggregate_functions.reference
new file mode 100644
index 00000000000..61c83cba41c
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00605_intersections_aggregate_functions.reference
@@ -0,0 +1,2 @@
+4
+5
diff --git a/dbms/tests/queries/0_stateless/00605_intersections_aggregate_functions.sql b/dbms/tests/queries/0_stateless/00605_intersections_aggregate_functions.sql
new file mode 100644
index 00000000000..c23583dd8c8
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00605_intersections_aggregate_functions.sql
@@ -0,0 +1,20 @@
+DROP TABLE IF EXISTS test.test;
+CREATE TABLE test.test(start Integer, end Integer) engine = Memory;
+INSERT INTO test.test(start,end) VALUES (1,3),(2,7),(3,999),(4,7),(5,8);
+
+/*
+1 2 3 4 5 6 7 8 9
+------------------>
+1---3
+  2---------7
+    3-------------
+      4-----7
+        5-----8
+------------------>
+1 2 3 3 4 4 4 2 1  //intersections count for each point
+*/
+
+SELECT maxIntersections(start,end) FROM test.test;
+SELECT maxIntersectionsPosition(start,end) FROM test.test;
+
+DROP TABLE test.test;

From f08940c04c020e2ecdd5a1998571ebab4042a01e Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Wed, 14 Mar 2018 08:03:51 +0300
Subject: [PATCH 066/244] Fixed handling of NaNs in aggregate functions that
 use comparison based sorting #2012

---
 .../AggregateFunctionMaxIntersections.h                  | 9 ++++++---
 dbms/src/AggregateFunctions/QuantileExact.h              | 5 ++++-
 dbms/src/AggregateFunctions/QuantileExactWeighted.h      | 8 ++++++--
 dbms/src/AggregateFunctions/QuantilesCommon.h            | 6 ++++++
 dbms/src/AggregateFunctions/ReservoirSampler.h           | 4 ++++
 .../AggregateFunctions/ReservoirSamplerDeterministic.h   | 4 ++++
 .../0_stateless/00606_quantiles_and_nans.reference       | 1 +
 .../queries/0_stateless/00606_quantiles_and_nans.sql     | 1 +
 8 files changed, 32 insertions(+), 6 deletions(-)
 create mode 100644 dbms/tests/queries/0_stateless/00606_quantiles_and_nans.reference
 create mode 100644 dbms/tests/queries/0_stateless/00606_quantiles_and_nans.sql

diff --git a/dbms/src/AggregateFunctions/AggregateFunctionMaxIntersections.h b/dbms/src/AggregateFunctions/AggregateFunctionMaxIntersections.h
index ebc283c03fc..11ec01e52e4 100644
--- a/dbms/src/AggregateFunctions/AggregateFunctionMaxIntersections.h
+++ b/dbms/src/AggregateFunctions/AggregateFunctionMaxIntersections.h
@@ -9,6 +9,7 @@
 #include <IO/WriteHelpers.h>
 
 #include <Common/ArenaAllocator.h>
+#include <Common/NaNUtils.h>
 
 #include <AggregateFunctions/IAggregateFunction.h>
 
@@ -90,8 +91,11 @@ public:
         PointType left = static_cast<const ColumnVector<PointType> &>(*columns[0]).getData()[row_num];
         PointType right = static_cast<const ColumnVector<PointType> &>(*columns[1]).getData()[row_num];
 
-        this->data(place).value.push_back(std::make_pair(left, Int64(1)), arena);
-        this->data(place).value.push_back(std::make_pair(right, Int64(-1)), arena);
+        if (!isNaN(left))
+            this->data(place).value.push_back(std::make_pair(left, Int64(1)), arena);
+
+        if (!isNaN(right))
+            this->data(place).value.push_back(std::make_pair(right, Int64(-1)), arena);
     }
 
     void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena * arena) const override
@@ -133,7 +137,6 @@ public:
         /// const_cast because we will sort the array
         auto & array = const_cast<typename MaxIntersectionsData<PointType>::Array &>(this->data(place).value);
 
-        /// TODO NaNs?
         std::sort(array.begin(), array.end(), [](const auto & a, const auto & b) { return a.first < b.first; });
 
         for (const auto & point_weight : array)
diff --git a/dbms/src/AggregateFunctions/QuantileExact.h b/dbms/src/AggregateFunctions/QuantileExact.h
index c134ad1653d..568ad8d0950 100644
--- a/dbms/src/AggregateFunctions/QuantileExact.h
+++ b/dbms/src/AggregateFunctions/QuantileExact.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <Common/PODArray.h>
+#include <Common/NaNUtils.h>
 #include <IO/WriteBuffer.h>
 #include <IO/ReadBuffer.h>
 #include <Core/Types.h>
@@ -32,7 +33,9 @@ struct QuantileExact
 
     void add(const Value & x)
     {
-        array.push_back(x);
+        /// We must skip NaNs as they are not compatible with comparison sorting.
+        if (!isNaN(x))
+            array.push_back(x);
     }
 
     template <typename Weight>
diff --git a/dbms/src/AggregateFunctions/QuantileExactWeighted.h b/dbms/src/AggregateFunctions/QuantileExactWeighted.h
index f524426392a..76e65f07dac 100644
--- a/dbms/src/AggregateFunctions/QuantileExactWeighted.h
+++ b/dbms/src/AggregateFunctions/QuantileExactWeighted.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <Common/HashTable/HashMap.h>
+#include <Common/NaNUtils.h>
 
 
 namespace DB
@@ -33,12 +34,15 @@ struct QuantileExactWeighted
 
     void add(const Value & x)
     {
-        ++map[x];
+        /// We must skip NaNs as they are not compatible with comparison sorting.
+        if (!isNaN(x))
+            ++map[x];
     }
 
     void add(const Value & x, const Weight & weight)
     {
-        map[x] += weight;
+        if (!isNaN(x))
+            map[x] += weight;
     }
 
     void merge(const QuantileExactWeighted & rhs)
diff --git a/dbms/src/AggregateFunctions/QuantilesCommon.h b/dbms/src/AggregateFunctions/QuantilesCommon.h
index 53802834379..1f70fde74dd 100644
--- a/dbms/src/AggregateFunctions/QuantilesCommon.h
+++ b/dbms/src/AggregateFunctions/QuantilesCommon.h
@@ -4,6 +4,7 @@
 
 #include <Core/Field.h>
 #include <Common/FieldVisitors.h>
+#include <Common/NaNUtils.h>
 
 
 namespace DB
@@ -12,6 +13,7 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+    extern const int PARAMETER_OUT_OF_BOUND;
 }
 
 
@@ -55,6 +57,10 @@ struct QuantileLevels
         for (size_t i = 0; i < size; ++i)
         {
             levels[i] = applyVisitor(FieldVisitorConvertToNumber<Float64>(), params[i]);
+
+            if (isNaN(levels[i]) || levels[i] < 0 || levels[i] > 1)
+                throw Exception("Quantile level is out of range [0..1]", ErrorCodes::PARAMETER_OUT_OF_BOUND);
+
             permutation[i] = i;
         }
 
diff --git a/dbms/src/AggregateFunctions/ReservoirSampler.h b/dbms/src/AggregateFunctions/ReservoirSampler.h
index 77d9f7285b7..ad5bf10f48f 100644
--- a/dbms/src/AggregateFunctions/ReservoirSampler.h
+++ b/dbms/src/AggregateFunctions/ReservoirSampler.h
@@ -9,6 +9,7 @@
 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
 #include <Common/PODArray.h>
+#include <Common/NaNUtils.h>
 #include <Poco/Exception.h>
 #include <pcg_random.hpp>
 
@@ -67,6 +68,9 @@ public:
 
     void insert(const T & v)
     {
+        if (isNaN(v))
+            return;
+
         sorted = false;
         ++total_values;
         if (samples.size() < sample_count)
diff --git a/dbms/src/AggregateFunctions/ReservoirSamplerDeterministic.h b/dbms/src/AggregateFunctions/ReservoirSamplerDeterministic.h
index bff3fcb78ea..44760d8e5a5 100644
--- a/dbms/src/AggregateFunctions/ReservoirSamplerDeterministic.h
+++ b/dbms/src/AggregateFunctions/ReservoirSamplerDeterministic.h
@@ -11,6 +11,7 @@
 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
 #include <Common/PODArray.h>
+#include <Common/NaNUtils.h>
 #include <Poco/Exception.h>
 
 
@@ -66,6 +67,9 @@ public:
 
     void insert(const T & v, const UInt64 determinator)
     {
+        if (isNaN(v))
+            return;
+
         const UInt32 hash = intHash64(determinator);
         if (!good(hash))
             return;
diff --git a/dbms/tests/queries/0_stateless/00606_quantiles_and_nans.reference b/dbms/tests/queries/0_stateless/00606_quantiles_and_nans.reference
new file mode 100644
index 00000000000..d00491fd7e5
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00606_quantiles_and_nans.reference
@@ -0,0 +1 @@
+1
diff --git a/dbms/tests/queries/0_stateless/00606_quantiles_and_nans.sql b/dbms/tests/queries/0_stateless/00606_quantiles_and_nans.sql
new file mode 100644
index 00000000000..c89883e356b
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00606_quantiles_and_nans.sql
@@ -0,0 +1 @@
+SELECT DISTINCT eq FROM (WITH range(number % 10) AS arr, arrayMap(x -> x = intDiv(number, 10) ? nan : x, arr) AS arr_with_nan, arrayFilter(x -> x != intDiv(number, 10), arr) AS arr_filtered SELECT number, arrayReduce('quantileExact', arr_with_nan) AS q1, arrayReduce('quantileExact', arr_filtered) AS q2, q1 = q2 AS eq FROM numbers(100));
\ No newline at end of file

From 6d6eca5e23253e8677825dbd2039d897b9186f4d Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Wed, 14 Mar 2018 08:29:05 +0300
Subject: [PATCH 067/244] Reverted bad modification #2035

---
 debian/pbuilder-hooks/B90test-server | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/debian/pbuilder-hooks/B90test-server b/debian/pbuilder-hooks/B90test-server
index 6efbd387e31..bf0bbfbba63 100755
--- a/debian/pbuilder-hooks/B90test-server
+++ b/debian/pbuilder-hooks/B90test-server
@@ -3,8 +3,8 @@ set -e
 set -x
 
 for PKG in $(ls /tmp/buildd/*.deb | sed -e's,.*/,,;s,_.*,,' ); do
-    apt-get install -y --force-yes "$PKG" || true 
-    apt-get remove -y "$PKG" || true 
+    apt-get install -y --force-yes "$PKG" || true
+    apt-get remove -y "$PKG" || true
 done
 
 dpkg -i /tmp/buildd/*.deb || true
@@ -18,9 +18,6 @@ dpkg-reconfigure -f noninteractive tzdata
 
 pwd
 mkdir -p /etc/clickhouse-server/config.d
-cp ssl.xml /etc/clickhouse-server/config.d
-openssl dhparam -out /etc/clickhouse-server/dhparam.pem 256
-openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout /etc/clickhouse-server/server.key -out /etc/clickhouse-server/server.crt
 
 service clickhouse-server start
 sleep 3

From 905652d0baa313cd467c8fa42f47731349d6c74d Mon Sep 17 00:00:00 2001
From: robot-metrika-test <robot-metrika-test@yandex-team.ru>
Date: Wed, 14 Mar 2018 10:03:38 +0300
Subject: [PATCH 068/244] Auto version update to [54364]

---
 dbms/cmake/version.cmake | 4 ++--
 debian/changelog         | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake
index b82d8afccbb..4f6d925f969 100644
--- a/dbms/cmake/version.cmake
+++ b/dbms/cmake/version.cmake
@@ -1,6 +1,6 @@
 # This strings autochanged from release_lib.sh:
-set(VERSION_DESCRIBE v1.1.54363-testing)
-set(VERSION_REVISION 54363)
+set(VERSION_DESCRIBE v1.1.54364-testing)
+set(VERSION_REVISION 54364)
 # end of autochange
 
 set (VERSION_MAJOR 1)
diff --git a/debian/changelog b/debian/changelog
index 64bd8096d89..7623f7307d3 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,5 @@
-clickhouse (1.1.54363) unstable; urgency=low
+clickhouse (1.1.54364) unstable; urgency=low
 
   * Modified source code
 
- -- proller <proller@yandex-team.ru>  Wed, 14 Mar 2018 00:06:37 +0300
+ --  <robot-metrika-test@yandex-team.ru>  Wed, 14 Mar 2018 10:03:38 +0300

From 974e2cf5e7b728f774a4de58acd2f9c83607570b Mon Sep 17 00:00:00 2001
From: Alexander Millin <millin7@gmail.com>
Date: Wed, 14 Mar 2018 12:08:37 +0300
Subject: [PATCH 069/244] Add docs for "ip_trie" dict layout

---
 docs/en/dicts/external_dicts_dict_layout.md | 87 +++++++++++++++++++--
 docs/ru/dicts/external_dicts_dict_layout.md | 76 ++++++++++++++++--
 2 files changed, 148 insertions(+), 15 deletions(-)

diff --git a/docs/en/dicts/external_dicts_dict_layout.md b/docs/en/dicts/external_dicts_dict_layout.md
index 38be4187de4..ad635db94f5 100644
--- a/docs/en/dicts/external_dicts_dict_layout.md
+++ b/docs/en/dicts/external_dicts_dict_layout.md
@@ -2,11 +2,11 @@
 
 # Storing dictionaries in memory
 
-There are [many different ways](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-manner) to store dictionaries in memory.
+There are [many different ways](external_dicts_dict_layout#dicts-external_dicts_dict_layout-manner) to store dictionaries in memory.
 
-We recommend [flat](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-flat), [hashed](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-hashed), and [complex_key_hashed](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-complex_key_hashed). which provide optimal processing speed.
+We recommend [flat](external_dicts_dict_layout#dicts-external_dicts_dict_layout-flat), [hashed](external_dicts_dict_layout#dicts-external_dicts_dict_layout-hashed), and [complex_key_hashed](external_dicts_dict_layout#dicts-external_dicts_dict_layout-complex_key_hashed). which provide optimal processing speed.
 
-Caching is not recommended because of potentially poor performance and difficulties in selecting optimal parameters. Read more about this in the "[cache](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-cache)" section.
+Caching is not recommended because of potentially poor performance and difficulties in selecting optimal parameters. Read more about this in the "[cache](external_dicts_dict_layout#dicts-external_dicts_dict_layout-cache)" section.
 
 There are several ways to improve dictionary performance:
 
@@ -46,6 +46,7 @@ The configuration looks like this:
 - [range_hashed](#dicts-external_dicts_dict_layout-range_hashed)
 - [complex_key_hashed](#dicts-external_dicts_dict_layout-complex_key_hashed)
 - [complex_key_cache](#dicts-external_dicts_dict_layout-complex_key_cache)
+- [ip_trie](#dicts-external_dicts_dict_layout-ip_trie)
 
 <a name="dicts-external_dicts_dict_layout-flat"></a>
 
@@ -87,7 +88,7 @@ Configuration example:
 
 ### complex_key_hashed
 
-This type of storage is designed for use with compound [keys](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure). It is similar to hashed.
+This type of storage is designed for use with compound [keys](external_dicts_dict_structure#dicts-external_dicts_dict_structure). It is similar to hashed.
 
 Configuration example:
 
@@ -108,10 +109,18 @@ This storage method works the same way as hashed and allows using date/time rang
 Example: The table contains discounts for each advertiser in the format:
 
 ```
-+------------------+-----------------------------+------------+----------+  | advertiser id | discount start date | discount end date | amount |  +==================+=============================+============+==========+  | 123              | 2015-01-01                  | 2015-01-15 | 0.15     |  +------------------+-----------------------------+------------+----------+  | 123              | 2015-01-16                  | 2015-01-31 | 0.25     |  +------------------+-----------------------------+------------+----------+  | 456              | 2015-01-01                  | 2015-01-15 | 0.05     |  +------------------+-----------------------------+------------+----------+
+  +---------------+---------------------+-------------------+--------+
+  | advertiser id | discount start date | discount end date | amount |
+  +===============+=====================+===================+========+
+  | 123           | 2015-01-01          | 2015-01-15        | 0.15   |
+  +---------------+---------------------+-------------------+--------+
+  | 123           | 2015-01-16          | 2015-01-31        | 0.25   |
+  +---------------+---------------------+-------------------+--------+
+  | 456           | 2015-01-01          | 2015-01-15        | 0.05   |
+  +---------------+---------------------+-------------------+--------+
 ```
 
-To use a sample for date ranges, define `range_min` and `range_max` in [structure](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure).
+To use a sample for date ranges, define `range_min` and `range_max` in [structure](external_dicts_dict_structure#dicts-external_dicts_dict_structure).
 
 Example:
 
@@ -188,7 +197,7 @@ This is the least effective of all the ways to store dictionaries. The speed of
 
 To improve cache performance, use a subquery with ` LIMIT`, and call the function with the dictionary externally.
 
-Supported [sources](external_dicts_dict_sources.md#dicts-external_dicts_dict_sources): MySQL, ClickHouse, executable, HTTP.
+Supported [sources](external_dicts_dict_sources#dicts-external_dicts_dict_sources): MySQL, ClickHouse, executable, HTTP.
 
 Example of settings:
 
@@ -218,4 +227,66 @@ Do not use ClickHouse as a source, because it is slow to process queries with ra
 
 ### complex_key_cache
 
-This type of storage is designed for use with compound [keys](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure). Similar to `cache`.
+This type of storage is designed for use with compound [keys](external_dicts_dict_structure#dicts-external_dicts_dict_structure). Similar to `cache`.
+
+<a name="dicts-external_dicts_dict_layout-ip_trie"></a>
+
+### ip_trie
+
+
+The table stores IP prefixes for each key (IP address), which makes it possible to map IP addresses to metadata such as ASN or threat score.
+
+Example: in the table there are prefixes matches to AS number and country:
+
+```
+  +-----------------+-------+--------+
+  | prefix          | asn   | cca2   |
+  +=================+=======+========+
+  | 202.79.32.0/20  | 17501 | NP     |
+  +-----------------+-------+--------+
+  | 2620:0:870::/48 | 3856  | US     |
+  +-----------------+-------+--------+
+  | 2a02:6b8:1::/48 | 13238 | RU     |
+  +-----------------+-------+--------+
+  | 2001:db8::/32   | 65536 | ZZ     |
+  +-----------------+-------+--------+
+```
+
+When using such a layout, the structure should have the "key" element.
+
+Example:
+
+```xml
+<structure>
+    <key>
+        <attribute>
+            <name>prefix</name>
+            <type>String</type>
+        </attribute>
+    </key>
+    <attribute>
+            <name>asn</name>
+            <type>UInt32</type>
+            <null_value />
+    </attribute>
+    <attribute>
+            <name>cca2</name>
+            <type>String</type>
+            <null_value>??</null_value>
+    </attribute>
+    ...
+```
+
+These key must have only one attribute of type String, containing a valid IP prefix. Other types are not yet supported.
+
+For querying, same functions (dictGetT with tuple) as for complex key dictionaries have to be used:
+
+    dictGetT('dict_name', 'attr_name', tuple(ip))
+
+The function accepts either UInt32 for IPv4 address or FixedString(16) for IPv6 address in wire format:
+
+    dictGetString('prefix', 'asn', tuple(IPv6StringToNum('2001:db8::1')))
+
+No other type is supported. The function returns attribute for a prefix matching the given IP address. If there are overlapping prefixes, the most specific one is returned.
+
+The data is stored currently in a bitwise trie, it has to fit in memory.
diff --git a/docs/ru/dicts/external_dicts_dict_layout.md b/docs/ru/dicts/external_dicts_dict_layout.md
index e88ddf7c735..ff1b9c0cdd5 100644
--- a/docs/ru/dicts/external_dicts_dict_layout.md
+++ b/docs/ru/dicts/external_dicts_dict_layout.md
@@ -2,11 +2,11 @@
 
 # Хранение словарей в памяти
 
-Словари можно размещать в памяти [множеством способов](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-manner).
+Словари можно размещать в памяти [множеством способов](external_dicts_dict_layout#dicts-external_dicts_dict_layout-manner).
 
-Рекомендуем [flat](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-flat), [hashed](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-hashed) и [complex_key_hashed](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-complex_key_hashed). Скорость обработки словарей при этом максимальна.
+Рекомендуем [flat](external_dicts_dict_layout#dicts-external_dicts_dict_layout-flat), [hashed](external_dicts_dict_layout#dicts-external_dicts_dict_layout-hashed) и [complex_key_hashed](external_dicts_dict_layout#dicts-external_dicts_dict_layout-complex_key_hashed). Скорость обработки словарей при этом максимальна.
 
-Размещение с кэшированием не рекомендуется использовать из-за потенциально низкой производительности и сложностей в подборе оптимальных параметров. Читайте об этом подробнее в разделе " [cache](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-cache)".
+Размещение с кэшированием не рекомендуется использовать из-за потенциально низкой производительности и сложностей в подборе оптимальных параметров. Читайте об этом подробнее в разделе " [cache](external_dicts_dict_layout#dicts-external_dicts_dict_layout-cache)".
 
 Повысить производительнось словарей можно следующими способами:
 
@@ -46,6 +46,7 @@
 -   [range_hashed](#dicts-external_dicts_dict_layout-range_hashed)
 -   [complex_key_hashed](#dicts-external_dicts_dict_layout-complex_key_hashed)
 -   [complex_key_cache](#dicts-external_dicts_dict_layout-complex_key_cache)
+-   [ip_trie](#dicts-external_dicts_dict_layout-ip_trie)
 
 <a name="dicts-external_dicts_dict_layout-flat"></a>
 
@@ -87,7 +88,7 @@
 
 ### complex_key_hashed
 
-Тип размещения предназначен для использования с составными [ключами](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure). Аналогичен `hashed`.
+Тип размещения предназначен для использования с составными [ключами](external_dicts_dict_structure#dicts-external_dicts_dict_structure). Аналогичен `hashed`.
 
 Пример конфигурации:
 
@@ -119,7 +120,7 @@
   +------------------+-----------------------------+------------+----------+
 ```
 
-Чтобы использовать выборку по диапазонам дат, необходимо в [structure](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure) определить элементы `range_min`, `range_max`.
+Чтобы использовать выборку по диапазонам дат, необходимо в [structure](external_dicts_dict_structure#dicts-external_dicts_dict_structure) определить элементы `range_min`, `range_max`.
 
 Пример:
 
@@ -196,7 +197,7 @@
 
 Чтобы увеличить производительность кэша, используйте подзапрос с `LIMIT`, а снаружи вызывайте функцию со словарём.
 
-Поддерживаются [источники](external_dicts_dict_sources.md#dicts-external_dicts_dict_sources): MySQL, ClickHouse, executable, HTTP.
+Поддерживаются [источники](external_dicts_dict_sources#dicts-external_dicts_dict_sources): MySQL, ClickHouse, executable, HTTP.
 
 Пример настройки:
 
@@ -226,4 +227,65 @@
 
 ### complex_key_cache
 
-Тип размещения предназначен для использования с составными [ключами](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure). Аналогичен `cache`.
+Тип размещения предназначен для использования с составными [ключами](external_dicts_dict_structure#dicts-external_dicts_dict_structure). Аналогичен `cache`.
+
+<a name="dicts-external_dicts_dict_layout-ip_trie"></a>
+
+### ip_trie
+
+Тип размещения предназначен для сопоставления префиксов сети (IP адресов) с метаданными, такими как ASN.
+
+Пример: таблица содержит префиксы сети и соответствующие им номера AS и коды стран:
+
+```
+  +-----------------+-------+--------+
+  | prefix          | asn   | cca2   |
+  +=================+=======+========+
+  | 202.79.32.0/20  | 17501 | NP     |
+  +-----------------+-------+--------+
+  | 2620:0:870::/48 | 3856  | US     |
+  +-----------------+-------+--------+
+  | 2a02:6b8:1::/48 | 13238 | RU     |
+  +-----------------+-------+--------+
+  | 2001:db8::/32   | 65536 | ZZ     |
+  +-----------------+-------+--------+
+```
+
+При использовании такого макета структура должна иметь составной ключ.
+
+Пример:
+
+```xml
+<structure>
+    <key>
+        <attribute>
+            <name>prefix</name>
+            <type>String</type>
+        </attribute>
+    </key>
+    <attribute>
+            <name>asn</name>
+            <type>UInt32</type>
+            <null_value />
+    </attribute>
+    <attribute>
+            <name>cca2</name>
+            <type>String</type>
+            <null_value>??</null_value>
+    </attribute>
+    ...
+```
+
+Этот ключ должен иметь только один атрибут типа String, содержащий допустимый префикс IP. Другие типы еще не поддерживаются.
+
+Для запросов необходимо использовать те же функции (`dictGetT` с кортежем), что и для словарей с составными ключами:
+
+    dictGetT('dict_name', 'attr_name', tuple(ip))
+
+Функция принимает либо UInt32 для адреса IPv4, либо FixedString(16) для адреса IPv6:
+
+    dictGetString('prefix', 'asn', tuple(IPv6StringToNum('2001:db8::1')))
+
+Никакие другие типы не поддерживаются. Функция возвращает атрибут для префикса, соответствующего данному IP-адресу. Если есть перекрывающиеся префиксы, возвращается наиболее специфический.
+
+Данные хранятся в побитовом дереве (trie), он должены полностью помещаться в оперативной памяти.

From c04ad1aa3eb7df39fc3ed7f5b4961576347e9b83 Mon Sep 17 00:00:00 2001
From: Pavel <pavel@tyavin.name>
Date: Wed, 14 Mar 2018 13:35:07 +0300
Subject: [PATCH 070/244] typo

---
 docs/ru/query_language/queries.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/ru/query_language/queries.md b/docs/ru/query_language/queries.md
index 53063705410..c5a1d46273e 100644
--- a/docs/ru/query_language/queries.md
+++ b/docs/ru/query_language/queries.md
@@ -76,7 +76,7 @@ CREATE [TEMPORARY] TABLE [IF NOT EXISTS] [db.]name ENGINE = engine AS SELECT ...
 Его значения не могут быть вставлены в таблицу, он не подставляется при использовании звёздочки в запросе SELECT.
 Он может быть использован в SELECT-ах - в таком случае, во время разбора запроса, алиас раскрывается.
 
-При добавлении новых столбцов с помощью запроса ALTER, старые данные для этих столбцов не записываются. Вместо этого, при чтении старых данных, для которых отсутствуют значения новых столбцов, выполняется вычисление выражений по умолчанию налету. При этом, если выполнение выражения требует использования других столбцов, не указанных в запросе, то эти столбцы будут дополнительно прочитаны, но только для тех блоков данных, для которых это необходимо.
+При добавлении новых столбцов с помощью запроса ALTER, старые данные для этих столбцов не записываются. Вместо этого, при чтении старых данных, для которых отсутствуют значения новых столбцов, выполняется вычисление выражений по умолчанию на лету. При этом, если выполнение выражения требует использования других столбцов, не указанных в запросе, то эти столбцы будут дополнительно прочитаны, но только для тех блоков данных, для которых это необходимо.
 
 Если добавить в таблицу новый столбец, а через некоторое время изменить его выражение по умолчанию, то используемые значения для старых данных (для данных, где значения не хранились на диске) поменяются. Также заметим, что при выполнении фоновых слияний, данные для столбцов, отсутствующих в одном из сливаемых кусков, записываются в объединённый кусок.
 

From b910e729460e7da4072e0c182d0c796c3a96433a Mon Sep 17 00:00:00 2001
From: Alexey Zatelepin <ztlpn@yandex-team.ru>
Date: Wed, 14 Mar 2018 15:30:39 +0300
Subject: [PATCH 071/244] stable order in a test

---
 ...shard_remote_and_columns_with_defaults.sql | 20 +++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/dbms/tests/queries/0_stateless/00604_shard_remote_and_columns_with_defaults.sql b/dbms/tests/queries/0_stateless/00604_shard_remote_and_columns_with_defaults.sql
index ba955f7848b..c1c47d225db 100644
--- a/dbms/tests/queries/0_stateless/00604_shard_remote_and_columns_with_defaults.sql
+++ b/dbms/tests/queries/0_stateless/00604_shard_remote_and_columns_with_defaults.sql
@@ -19,20 +19,20 @@ INSERT INTO FUNCTION remote('127.0.0.2', test.t2) VALUES (2, 2);
 --INSERT INTO FUNCTION remote('127.0.0.2', test.t3) VALUES (2);
 INSERT INTO FUNCTION remote('127.0.0.2', test.t4) VALUES (2);
 
-SELECT * FROM remote('127.0.0.2', test.t1);
+SELECT * FROM remote('127.0.0.2', test.t1) ORDER BY x;
 
 SELECT '*** With a DEFAULT column ***';
-SELECT * FROM remote('127.0.0.2', test.t2);
+SELECT * FROM remote('127.0.0.2', test.t2) ORDER BY x;
 
 SELECT '*** With a MATERIALIZED column ***';
-SELECT * FROM remote('127.0.0.2', test.t3);
-SELECT x, y FROM remote('127.0.0.2', test.t3);
+SELECT * FROM remote('127.0.0.2', test.t3) ORDER BY x;
+SELECT x, y FROM remote('127.0.0.2', test.t3) ORDER BY x;
 
 SELECT '*** With an ALIAS column ***';
-SELECT * FROM remote('127.0.0.2', test.t4);
-SELECT x, y FROM remote('127.0.0.2', test.t4);
+SELECT * FROM remote('127.0.0.2', test.t4) ORDER BY x;
+SELECT x, y FROM remote('127.0.0.2', test.t4) ORDER BY x;
 
-DROP TABLE IF EXISTS test.t1;
-DROP TABLE IF EXISTS test.t2;
-DROP TABLE IF EXISTS test.t3;
-DROP TABLE IF EXISTS test.t4;
+DROP TABLE test.t1;
+DROP TABLE test.t2;
+DROP TABLE test.t3;
+DROP TABLE test.t4;

From d6e9d08d5ef49b82e210c42f401d8c0aad7d43e5 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Tue, 13 Mar 2018 16:28:32 +0300
Subject: [PATCH 072/244] getCreateTableQuery and getCreateDatabaseQuery
 [#CLICKHOUSE-3590]

---
 dbms/src/Databases/DatabaseDictionary.cpp     | 36 ++++++++-
 dbms/src/Databases/DatabaseDictionary.h       |  4 +-
 dbms/src/Databases/DatabaseMemory.cpp         |  7 +-
 dbms/src/Databases/DatabaseMemory.h           |  8 +-
 dbms/src/Databases/DatabaseOrdinary.cpp       | 76 ++++++++++++-------
 dbms/src/Databases/DatabaseOrdinary.h         |  5 +-
 dbms/src/Databases/IDatabase.h                |  8 +-
 dbms/src/Interpreters/Context.cpp             | 15 +++-
 dbms/src/Interpreters/Context.h               |  5 +-
 .../Interpreters/InterpreterCreateQuery.cpp   |  4 +-
 .../InterpreterShowCreateQuery.cpp            |  9 ++-
 dbms/src/Storages/StorageDictionary.cpp       |  1 -
 dbms/src/Storages/StorageDictionary.h         | 40 +++++-----
 .../Storages/System/StorageSystemTables.cpp   |  2 +-
 14 files changed, 149 insertions(+), 71 deletions(-)

diff --git a/dbms/src/Databases/DatabaseDictionary.cpp b/dbms/src/Databases/DatabaseDictionary.cpp
index 8a10535fd09..6c14312c2e3 100644
--- a/dbms/src/Databases/DatabaseDictionary.cpp
+++ b/dbms/src/Databases/DatabaseDictionary.cpp
@@ -3,6 +3,11 @@
 #include <Interpreters/ExternalDictionaries.h>
 #include <Storages/StorageDictionary.h>
 #include <common/logger_useful.h>
+#include <Parsers/IAST.h>
+#include <IO/WriteBufferFromString.h>
+#include <IO/Operators.h>
+#include <Parsers/ParserCreateQuery.h>
+#include <Parsers/parseQuery.h>
 
 namespace DB
 {
@@ -158,11 +163,34 @@ time_t DatabaseDictionary::getTableMetadataModificationTime(
     return static_cast<time_t>(0);
 }
 
-ASTPtr DatabaseDictionary::getCreateQuery(
-    const Context &,
-    const String &) const
+ASTPtr DatabaseDictionary::getCreateTableQuery(
+    const Context & context,
+    const String & table_name) const
 {
-    throw Exception("There is no CREATE TABLE query for DatabaseDictionary tables", ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY);
+    String query;
+    {
+        WriteBufferFromString buffer(query);
+
+        auto dictionary = context.getExternalDictionaries().getDictionary(table_name);
+        auto names_and_types = StorageDictionary::getNamesAndTypes(dictionary->getStructure());
+        buffer << "CREATE TABLE " << backQuoteIfNeed(name) << '.' << backQuoteIfNeed(table_name) << " (";
+        buffer << StorageDictionary::generateNamesAndTypesDescription(names_and_types.begin(), names_and_types.end());
+        buffer << ") Engine = Dictionary(" << backQuoteIfNeed(table_name) << ")";
+    }
+
+    ParserCreateQuery parser;
+    return parseQuery(parser, query.data(), query.data() + query.size(), "");
+}
+
+ASTPtr DatabaseDictionary::getCreateDatabaseQuery(const Context & /*context*/) const
+{
+    String query;
+    {
+        WriteBufferFromString buffer(query);
+        buffer << "CREATE DATABASE " << backQuoteIfNeed(name) << " ENGINE = Dictionary";
+    }
+    ParserCreateQuery parser;
+    return parseQuery(parser, query.data(), query.data() + query.size(), "");
 }
 
 void DatabaseDictionary::shutdown()
diff --git a/dbms/src/Databases/DatabaseDictionary.h b/dbms/src/Databases/DatabaseDictionary.h
index 1308ea20a40..f62bfc3e1c3 100644
--- a/dbms/src/Databases/DatabaseDictionary.h
+++ b/dbms/src/Databases/DatabaseDictionary.h
@@ -89,10 +89,12 @@ public:
         const Context & context,
         const String & table_name) override;
 
-    ASTPtr getCreateQuery(
+    ASTPtr getCreateTableQuery(
         const Context & context,
         const String & table_name) const override;
 
+    ASTPtr getCreateDatabaseQuery(const Context & context) const override;
+
     void shutdown() override;
     void drop() override;
 };
diff --git a/dbms/src/Databases/DatabaseMemory.cpp b/dbms/src/Databases/DatabaseMemory.cpp
index 66b23f07ca6..b50fb78468e 100644
--- a/dbms/src/Databases/DatabaseMemory.cpp
+++ b/dbms/src/Databases/DatabaseMemory.cpp
@@ -121,13 +121,18 @@ time_t DatabaseMemory::getTableMetadataModificationTime(
     return static_cast<time_t>(0);
 }
 
-ASTPtr DatabaseMemory::getCreateQuery(
+ASTPtr DatabaseMemory::getCreateTableQuery(
     const Context &,
     const String &) const
 {
     throw Exception("There is no CREATE TABLE query for DatabaseMemory tables", ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY);
 }
 
+ASTPtr DatabaseMemory::getCreateDatabaseQuery(const Context &) const
+{
+    throw Exception("There is no CREATE DATABASE query for DatabaseMemory", ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY);
+}
+
 void DatabaseMemory::shutdown()
 {
     /// You can not hold a lock during shutdown.
diff --git a/dbms/src/Databases/DatabaseMemory.h b/dbms/src/Databases/DatabaseMemory.h
index be095ad3755..dbf37acb5f6 100644
--- a/dbms/src/Databases/DatabaseMemory.h
+++ b/dbms/src/Databases/DatabaseMemory.h
@@ -80,9 +80,11 @@ public:
         const Context & context,
         const String & table_name) override;
 
-    ASTPtr getCreateQuery(
-        const Context & context,
-        const String & table_name) const override;
+    ASTPtr getCreateTableQuery(
+            const Context & context,
+            const String & table_name) const override;
+
+    ASTPtr getCreateDatabaseQuery(const Context & context) const override;
 
     void shutdown() override;
     void drop() override;
diff --git a/dbms/src/Databases/DatabaseOrdinary.cpp b/dbms/src/Databases/DatabaseOrdinary.cpp
index 73152387cf6..1f09c118666 100644
--- a/dbms/src/Databases/DatabaseOrdinary.cpp
+++ b/dbms/src/Databases/DatabaseOrdinary.cpp
@@ -335,22 +335,13 @@ void DatabaseOrdinary::removeTable(
     }
 }
 
-
-ASTPtr DatabaseOrdinary::getCreateQueryImpl(const String & path, const String & table_name) const
+static ASTPtr getQueryFromMetadata(const String & metadata_path)
 {
-    String metadata_path;
+    if (!Poco::File(metadata_path).exists())
+        return nullptr;
+
     String query;
 
-    if (table_name.empty())
-    {
-        metadata_path = detail::getDatabaseMetadataPath(path);
-        if (!Poco::File(metadata_path).exists())
-            query = "CREATE DATABASE " + backQuoteIfNeed(name) + " ENGINE = Ordinary";
-    }
-    else
-        metadata_path = detail::getTableMetadataPath(path, table_name);
-
-    if (query.empty())
     {
         ReadBufferFromFile in(metadata_path, 4096);
         readStringUntilEOF(query, in);
@@ -360,6 +351,20 @@ ASTPtr DatabaseOrdinary::getCreateQueryImpl(const String & path, const String &
     return parseQuery(parser, query.data(), query.data() + query.size(), "in file " + metadata_path);
 }
 
+static ASTPtr getCreateQueryFromMetadata(const String & metadata_path, const String & database)
+{
+    ASTPtr ast = getQueryFromMetadata(metadata_path);
+
+    if (ast)
+    {
+        ASTCreateQuery & ast_create_query = typeid_cast<ASTCreateQuery &>(*ast);
+        ast_create_query.attach = false;
+        ast_create_query.database = database;
+    }
+
+    return ast;
+}
+
 
 void DatabaseOrdinary::renameTable(
     const Context & context,
@@ -394,7 +399,9 @@ void DatabaseOrdinary::renameTable(
         throw Exception{e};
     }
 
-    ASTPtr ast = getCreateQueryImpl(metadata_path, table_name);
+    ASTPtr ast = getQueryFromMetadata(detail::getTableMetadataPath(metadata_path, table_name));
+    if (!ast)
+        throw Exception("There is no metadata file for table " + table_name, ErrorCodes::FILE_DOESNT_EXIST);
     ASTCreateQuery & ast_create_query = typeid_cast<ASTCreateQuery &>(*ast);
     ast_create_query.table = to_table_name;
 
@@ -422,27 +429,40 @@ time_t DatabaseOrdinary::getTableMetadataModificationTime(
 }
 
 
-ASTPtr DatabaseOrdinary::getCreateQuery(
+ASTPtr DatabaseOrdinary::getCreateTableQuery(
     const Context & context,
     const String & table_name) const
 {
     ASTPtr ast;
-    try
-    {
-        ast = getCreateQueryImpl(metadata_path, table_name);
-    }
-    catch (const Exception & e)
-    {
-        /// Handle system.* tables for which there are no table.sql files
-        if (e.code() == ErrorCodes::FILE_DOESNT_EXIST && tryGetTable(context, table_name) != nullptr)
-            throw Exception("There is no CREATE TABLE query for table " + table_name, ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY);
 
-        throw;
+    auto table_metadata_path = detail::getTableMetadataPath(metadata_path, table_name);
+    ast = getCreateQueryFromMetadata(table_metadata_path, name);
+    if (!ast)
+    {
+        /// Handle system.* tables for which there are no table.sql files.
+        auto msg = tryGetTable(context, table_name)
+                   ? "There is no CREATE TABLE query for table "
+                   : "There is no metadata file for table ";
+
+        throw Exception(msg + table_name, ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY);
     }
 
-    ASTCreateQuery & ast_create_query = typeid_cast<ASTCreateQuery &>(*ast);
-    ast_create_query.attach = false;
-    ast_create_query.database = name;
+    return ast;
+}
+
+ASTPtr DatabaseOrdinary::getCreateDatabaseQuery(const Context & /*context*/) const
+{
+    ASTPtr ast;
+
+    auto database_metadata_path = detail::getDatabaseMetadataPath(metadata_path);
+    ast = getCreateQueryFromMetadata(database_metadata_path, name);
+    if (!ast)
+    {
+        /// Handle databases (such as default) for which there are no database.sql files.
+        String query = "CREATE DATABASE " + backQuoteIfNeed(name) + " ENGINE = Ordinary";
+        ParserCreateQuery parser;
+        ast = parseQuery(parser, query.data(), query.data() + query.size(), "");
+    }
 
     return ast;
 }
diff --git a/dbms/src/Databases/DatabaseOrdinary.h b/dbms/src/Databases/DatabaseOrdinary.h
index 4cd77c85895..9d8d2458df6 100644
--- a/dbms/src/Databases/DatabaseOrdinary.h
+++ b/dbms/src/Databases/DatabaseOrdinary.h
@@ -55,10 +55,12 @@ public:
         const Context & context,
         const String & table_name) override;
 
-    ASTPtr getCreateQuery(
+    ASTPtr getCreateTableQuery(
         const Context & context,
         const String & table_name) const override;
 
+    ASTPtr getCreateDatabaseQuery(const Context & context) const override;
+
     String getDataPath() const override;
     String getMetadataPath() const override;
     String getTableMetadataPath(const String & table_name) const override;
@@ -68,7 +70,6 @@ public:
 
 private:
     void startupTables(ThreadPool * thread_pool);
-    ASTPtr getCreateQueryImpl(const String & path, const String & table_name) const;
 };
 
 }
diff --git a/dbms/src/Databases/IDatabase.h b/dbms/src/Databases/IDatabase.h
index 8d62c0e3d45..01c8970901d 100644
--- a/dbms/src/Databases/IDatabase.h
+++ b/dbms/src/Databases/IDatabase.h
@@ -124,12 +124,14 @@ public:
         const Context & context,
         const String & name) = 0;
 
-    /// Get the CREATE TABLE query for the table or CREATE DATABASE query for database if name is empty.
-    /// It can also provide information for detached tables for which there is metadata.
-    virtual ASTPtr getCreateQuery(
+    /// Get the CREATE TABLE query for the table. It can also provide information for detached tables for which there is metadata.
+    virtual ASTPtr getCreateTableQuery(
         const Context & context,
         const String & name) const = 0;
 
+    /// Get the CREATE DATABASE query for current database.
+    virtual ASTPtr getCreateDatabaseQuery(const Context & context) const = 0;
+
     /// Returns path for persistent data storage if the database supports it, empty string otherwise
     virtual String getDataPath() const { return {}; }
     /// Returns metadata path if the database supports it, empty string otherwise
diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp
index 6fddcbc2f05..e3e0e268f98 100644
--- a/dbms/src/Interpreters/Context.cpp
+++ b/dbms/src/Interpreters/Context.cpp
@@ -910,17 +910,17 @@ DatabasePtr Context::detachDatabase(const String & database_name)
 }
 
 
-ASTPtr Context::getCreateQuery(const String & database_name, const String & table_name) const
+ASTPtr Context::getCreateTableQuery(const String & database_name, const String & table_name) const
 {
     auto lock = getLock();
 
     String db = resolveDatabase(database_name, current_database);
     assertDatabaseExists(db);
 
-    return shared->databases[db]->getCreateQuery(*this, table_name);
+    return shared->databases[db]->getCreateTableQuery(*this, table_name);
 }
 
-ASTPtr Context::getCreateExternalQuery(const String & table_name) const
+ASTPtr Context::getCreateExternalTableQuery(const String & table_name) const
 {
     TableAndCreateASTs::const_iterator jt = external_tables.find(table_name);
     if (external_tables.end() == jt)
@@ -929,6 +929,15 @@ ASTPtr Context::getCreateExternalQuery(const String & table_name) const
     return jt->second.second;
 }
 
+ASTPtr Context::getCreateDatabaseQuery(const String & database_name) const
+{
+    auto lock = getLock();
+
+    String db = resolveDatabase(database_name, current_database);
+    assertDatabaseExists(db);
+
+    return shared->databases[db]->getCreateDatabaseQuery(*this);
+}
 
 Settings Context::getSettings() const
 {
diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h
index bab3be07add..f1e44705f3a 100644
--- a/dbms/src/Interpreters/Context.h
+++ b/dbms/src/Interpreters/Context.h
@@ -241,8 +241,9 @@ public:
     UInt16 getTCPPort() const;
 
     /// Get query for the CREATE table.
-    ASTPtr getCreateQuery(const String & database_name, const String & table_name) const;
-    ASTPtr getCreateExternalQuery(const String & table_name) const;
+    ASTPtr getCreateTableQuery(const String & database_name, const String & table_name) const;
+    ASTPtr getCreateExternalTableQuery(const String & table_name) const;
+    ASTPtr getCreateDatabaseQuery(const String & database_name) const;
 
     const DatabasePtr getDatabase(const String & database_name) const;
     DatabasePtr getDatabase(const String & database_name);
diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp
index 2b6d333d1f6..347d6208e70 100644
--- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp
@@ -432,7 +432,7 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const
         String as_database_name = create.as_database.empty() ? context.getCurrentDatabase() : create.as_database;
         String as_table_name = create.as_table;
 
-        ASTPtr as_create_ptr = context.getCreateQuery(as_database_name, as_table_name);
+        ASTPtr as_create_ptr = context.getCreateTableQuery(as_database_name, as_table_name);
         const auto & as_create = typeid_cast<const ASTCreateQuery &>(*as_create_ptr);
 
         if (as_create.is_view)
@@ -461,7 +461,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create)
     if (create.attach && !create.storage && !create.columns)
     {
         // Table SQL definition is available even if the table is detached
-        auto query = context.getCreateQuery(database_name, table_name);
+        auto query = context.getCreateTableQuery(database_name, table_name);
         auto & as_create = typeid_cast<const ASTCreateQuery &>(*query);
         create = as_create; // Copy the saved create query, but use ATTACH instead of CREATE
         create.attach = true;
diff --git a/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp b/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp
index 705b9bb4120..a4c3bd7224c 100644
--- a/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp
@@ -44,8 +44,13 @@ BlockInputStreamPtr InterpreterShowCreateQuery::executeImpl()
     if (ast.temporary && !ast.database.empty())
         throw Exception("Temporary databases are not possible.", ErrorCodes::SYNTAX_ERROR);
 
-    ASTPtr create_query = (ast.temporary ? context.getCreateExternalQuery(ast.table) :
-                          context.getCreateQuery(ast.database, ast.table));
+    ASTPtr create_query;
+    if (ast.temporary)
+        create_query = context.getCreateExternalTableQuery(ast.table);
+    else if (ast.table.empty())
+        create_query = context.getCreateDatabaseQuery(ast.database);
+    else
+        create_query = context.getCreateTableQuery(ast.database, ast.table);
 
     if (!create_query && ast.temporary)
         throw Exception("Unable to show the create query of " + ast.table + ". Maybe it was created by the system.", ErrorCodes::THERE_IS_NO_QUERY);
diff --git a/dbms/src/Storages/StorageDictionary.cpp b/dbms/src/Storages/StorageDictionary.cpp
index 2bb69748c6d..89d59e11b38 100644
--- a/dbms/src/Storages/StorageDictionary.cpp
+++ b/dbms/src/Storages/StorageDictionary.cpp
@@ -89,7 +89,6 @@ void StorageDictionary::checkNamesAndTypesCompatibleWithDictionary(const Diction
     }
 }
 
-
 void registerStorageDictionary(StorageFactory & factory)
 {
     factory.registerStorage("Dictionary", [](const StorageFactory::Arguments & args)
diff --git a/dbms/src/Storages/StorageDictionary.h b/dbms/src/Storages/StorageDictionary.h
index 0f21373f546..079c65ff134 100644
--- a/dbms/src/Storages/StorageDictionary.h
+++ b/dbms/src/Storages/StorageDictionary.h
@@ -4,6 +4,8 @@
 #include <Core/Defines.h>
 #include <common/MultiVersion.h>
 #include <ext/shared_ptr_helper.h>
+#include <IO/WriteBufferFromString.h>
+#include <IO/Operators.h>
 
 
 namespace Poco
@@ -33,6 +35,26 @@ public:
     void drop() override {}
     static NamesAndTypesList getNamesAndTypes(const DictionaryStructure & dictionary_structure);
 
+    template <typename ForwardIterator>
+    static std::string generateNamesAndTypesDescription(ForwardIterator begin, ForwardIterator end)
+    {
+        std::string description;
+        {
+            WriteBufferFromString buffer(description);
+            bool first = true;
+            for (; begin != end; ++begin)
+            {
+                if (!first)
+                    buffer << ", ";
+                first = false;
+
+                buffer << begin->name << ' ' << begin->type->getName();
+            }
+        }
+
+        return description;
+    }
+
 private:
     using Ptr = MultiVersion<IDictionaryBase>::Version;
 
@@ -42,24 +64,6 @@ private:
 
     void checkNamesAndTypesCompatibleWithDictionary(const DictionaryStructure & dictionary_structure) const;
 
-    template <typename ForwardIterator>
-    std::string generateNamesAndTypesDescription(ForwardIterator begin, ForwardIterator end) const
-    {
-        if (begin == end)
-        {
-            return "";
-        }
-        std::string description;
-        for (; begin != end; ++begin)
-        {
-            description += ", ";
-            description += begin->name;
-            description += ' ';
-            description += begin->type->getName();
-        }
-        return description.substr(2, description.size());
-    }
-
 protected:
     StorageDictionary(const String & table_name_,
         const NamesAndTypesList & columns_,
diff --git a/dbms/src/Storages/System/StorageSystemTables.cpp b/dbms/src/Storages/System/StorageSystemTables.cpp
index 852392a63d1..493750d5dfa 100644
--- a/dbms/src/Storages/System/StorageSystemTables.cpp
+++ b/dbms/src/Storages/System/StorageSystemTables.cpp
@@ -204,7 +204,7 @@ BlockInputStreams StorageSystemTables::read(
 
                 try
                 {
-                    ast = database->getCreateQuery(context, table_name);
+                    ast = database->getCreateTableQuery(context, table_name);
                 }
                 catch (const Exception & e)
                 {

From 5e7ba54e0635c9db2d7961eeb9a21513b48fb232 Mon Sep 17 00:00:00 2001
From: BayoNet <bayonet@yandex-team.ru>
Date: Wed, 14 Mar 2018 15:42:02 +0300
Subject: [PATCH 073/244] max_memory_usage, max_memory_usage_for_user,
 max_memory_usage_for_all_queries parameter descriptions are added.

---
 .../operations/settings/query_complexity.md   | 36 +++++++++++++++----
 1 file changed, 29 insertions(+), 7 deletions(-)

diff --git a/docs/ru/operations/settings/query_complexity.md b/docs/ru/operations/settings/query_complexity.md
index 83251f6813c..c964ad6e41c 100644
--- a/docs/ru/operations/settings/query_complexity.md
+++ b/docs/ru/operations/settings/query_complexity.md
@@ -27,21 +27,43 @@
 
 При использовании метода GET HTTP интерфейса, автоматически выставляется readonly = 1. То есть, для запросов, модифицирующие данные, можно использовать только метод POST. Сам запрос при этом можно отправлять как в теле POST-а, так и в параметре URL.
 
+<a name="settings_max_memory_usage"></a>
+
 ## max_memory_usage
 
-Максимальное количество потребляемой памяти при выполнении запроса на одном сервере. По умолчанию - 10 GB.
+Максимальный возможный объем оперативной памяти для выполнения запроса на одном сервере.
+
+Значения по умолчанию определены в файле [Settings.h](https://github.com/yandex/ClickHouse/blob/master/dbms/src/Interpreters/Settings.h#L243). По умолчанию размер не ограничен (`max_memory_usage = 0`).
 
 Настройка не учитывает объём свободной памяти или общий объём памяти на машине.
 Ограничение действует на один запрос, в пределах одного сервера.
-Текущее потребление оперативки для каждого запроса можно посмотреть с помощью SHOW PROCESSLIST.
-Также отслеживается пиковое потребление оперативки для каждого запроса, и выводится в лог.
+Текущее потребление памяти для каждого запроса можно посмотреть с помощью `SHOW PROCESSLIST`.
+Также отслеживается и выводится в лог пиковое потребление памяти для каждого запроса.
 
-Некоторые случаи потребления оперативки не отслеживаются:
+Потребление памяти не отслеживается для:
 
--   большие константы (например, очень длинная константная строка);
--   состояния некоторых агрегатных функций;
+- Констант.
+- Состояний некоторых агрегатных функций.
 
-Потребление оперативки не полностью учитывается для состояний агрегатных функций min, max, any, anyLast, argMin, argMax от аргументов String и Array.
+Потребление памяти не полностью учитывается для состояний агрегатных функций `min`, `max`, `any`, `anyLast`, `argMin`, `argMax` от аргументов `String` и `Array`.
+
+Потребление памяти ограничивается также параметрами `max_memory_usage_for_user` и `max_memory_usage_for_all_queries`.
+
+## max_memory_usage_for_user
+
+Максимальный возможный объем оперативной памяти для запросов пользователя на одном сервере.
+
+Значения по умолчанию определены в файле [Settings.h](https://github.com/yandex/ClickHouse/blob/master/dbms/src/Interpreters/Settings.h#L244). По умолчанию размер не ограничен (`max_memory_usage_for_user = 0`).
+
+Смотрите также описание настройки [max_memory_usage](#settings_max_memory_usage).
+
+## max_memory_usage_for_all_queries
+
+Максимальный возможный объем оперативной памяти для всех запросов на одном сервере.
+
+Значения по умолчанию определены в файле [Settings.h](https://github.com/yandex/ClickHouse/blob/master/dbms/src/Interpreters/Settings.h#L245). По умолчанию размер не ограничен (`max_memory_usage_for_all_queries = 0`).
+
+Смотрите также описание настройки [max_memory_usage](#settings_max_memory_usage).
 
 ## max_rows_to_read
 

From 22794bc93ea1fb0d3c678838391c5b0c3791cecd Mon Sep 17 00:00:00 2001
From: Vitaliy Lyudvichenko <vludv@yandex-team.ru>
Date: Mon, 12 Mar 2018 20:45:24 +0300
Subject: [PATCH 074/244] ZooKeeper multi op is transparent with chroot
 prefixes. [#CLICKHOUSE-3639]

---
 dbms/src/Common/ZooKeeper/Types.h             |  13 ++-
 dbms/src/Common/ZooKeeper/ZooKeeper.cpp       | 100 ++++++++++++------
 dbms/src/Common/ZooKeeper/ZooKeeper.h         |   6 +-
 .../gtest_zkutil_test_multi_exception.cpp     |  12 ++-
 4 files changed, 89 insertions(+), 42 deletions(-)

diff --git a/dbms/src/Common/ZooKeeper/Types.h b/dbms/src/Common/ZooKeeper/Types.h
index 4097627f395..08183f8128d 100644
--- a/dbms/src/Common/ZooKeeper/Types.h
+++ b/dbms/src/Common/ZooKeeper/Types.h
@@ -12,6 +12,8 @@ namespace zkutil
 
 using ACLPtr = const ACL_vector *;
 using Stat = ::Stat;
+class ZooKeeper;
+
 
 struct Op
 {
@@ -136,10 +138,15 @@ private:
     int32_t version;
 };
 
-struct OpResult : public zoo_op_result_t
+/// C++ version of zoo_op_result_t
+struct OpResult
 {
-    /// Pointers in this class point to fields of class Op.
-    /// Op instances have the same (or longer lifetime), therefore destructor is not required.
+    int err;
+    std::string value;
+    std::unique_ptr<Stat> stat;
+
+    /// ZooKeeper is required for correct chroot path prefixes handling
+    explicit OpResult(const zoo_op_result_t & op_result, const ZooKeeper * zookeeper = nullptr);
 };
 
 using OpPtr = std::unique_ptr<Op>;
diff --git a/dbms/src/Common/ZooKeeper/ZooKeeper.cpp b/dbms/src/Common/ZooKeeper/ZooKeeper.cpp
index 85bbe140d39..105c275e711 100644
--- a/dbms/src/Common/ZooKeeper/ZooKeeper.cpp
+++ b/dbms/src/Common/ZooKeeper/ZooKeeper.cpp
@@ -82,19 +82,21 @@ void ZooKeeper::processCallback(zhandle_t *, int type, int state, const char * p
 }
 
 void ZooKeeper::init(const std::string & hosts_, const std::string & identity_,
-                     int32_t session_timeout_ms_, bool check_root_exists)
+                     int32_t session_timeout_ms_, const std::string & chroot_)
 {
     log = &Logger::get("ZooKeeper");
-    zoo_set_debug_level(ZOO_LOG_LEVEL_ERROR);
+    zoo_set_debug_level(ZOO_LOG_LEVEL_DEBUG);
     hosts = hosts_;
     identity = identity_;
     session_timeout_ms = session_timeout_ms_;
+    chroot = chroot_;
 
-    impl = zookeeper_init(hosts.c_str(), nullptr, session_timeout_ms, nullptr, nullptr, 0);
+    std::string hosts_for_lib = hosts + chroot;
+    impl = zookeeper_init(hosts_for_lib.c_str(), nullptr, session_timeout_ms, nullptr, nullptr, 0);
     ProfileEvents::increment(ProfileEvents::ZooKeeperInit);
 
     if (!impl)
-        throw KeeperException("Fail to initialize zookeeper. Hosts are " + hosts);
+        throw KeeperException("Fail to initialize zookeeper. Hosts are " + hosts_for_lib);
 
     if (!identity.empty())
     {
@@ -107,16 +109,16 @@ void ZooKeeper::init(const std::string & hosts_, const std::string & identity_,
     else
         default_acl = &ZOO_OPEN_ACL_UNSAFE;
 
-    LOG_TRACE(log, "initialized, hosts: " << hosts);
+    LOG_TRACE(log, "initialized, hosts: " << hosts << (chroot.empty() ? "" : ", chroot: " + chroot));
 
-    if (check_root_exists && !exists("/"))
-        throw KeeperException("Zookeeper root doesn't exist. You should create root node before start.");
+    if (!chroot.empty() && !exists("/"))
+        throw KeeperException("Zookeeper root doesn't exist. You should create root node " + chroot + " before start.");
 }
 
 ZooKeeper::ZooKeeper(const std::string & hosts, const std::string & identity,
-                     int32_t session_timeout_ms, bool check_root_exists)
+                     int32_t session_timeout_ms, const std::string & chroot)
 {
-    init(hosts, identity, session_timeout_ms, check_root_exists);
+    init(hosts, identity, session_timeout_ms, chroot);
 }
 
 struct ZooKeeperArgs
@@ -127,10 +129,8 @@ struct ZooKeeperArgs
         config.keys(config_name, keys);
 
         std::vector<std::string> hosts_strings;
-        std::string root;
 
         session_timeout_ms = DEFAULT_SESSION_TIMEOUT;
-        has_chroot = false;
         for (const auto & key : keys)
         {
             if (startsWith(key, "node"))
@@ -150,7 +150,7 @@ struct ZooKeeperArgs
             }
             else if (key == "root")
             {
-                root = config.getString(config_name + "." + key);
+                chroot = config.getString(config_name + "." + key);
             }
             else throw KeeperException(std::string("Unknown key ") + key + " in config file");
         }
@@ -166,28 +166,25 @@ struct ZooKeeperArgs
             hosts += host;
         }
 
-        if (!root.empty())
+        if (!chroot.empty())
         {
-            if (root.front() != '/')
-                throw KeeperException(std::string("Root path in config file should start with '/', but got ") + root);
-            if (root.back() == '/')
-                root.pop_back();
-
-            hosts += root;
-            has_chroot = true;
+            if (chroot.front() != '/')
+                throw KeeperException(std::string("Root path in config file should start with '/', but got ") + chroot);
+            if (chroot.back() == '/')
+                chroot.pop_back();
         }
     }
 
     std::string hosts;
     std::string identity;
     int session_timeout_ms;
-    bool has_chroot;
+    std::string chroot;
 };
 
 ZooKeeper::ZooKeeper(const Poco::Util::AbstractConfiguration & config, const std::string & config_name)
 {
     ZooKeeperArgs args(config, config_name);
-    init(args.hosts, args.identity, args.session_timeout_ms, args.has_chroot);
+    init(args.hosts, args.identity, args.session_timeout_ms, args.chroot);
 }
 
 WatchCallback ZooKeeper::callbackForEvent(const EventPtr & event)
@@ -290,19 +287,18 @@ int32_t ZooKeeper::createImpl(const std::string & path, const std::string & data
     int code;
     /// The name of the created node can be longer than path if the sequential node is created.
     size_t name_buffer_size = path.size() + SEQUENTIAL_SUFFIX_SIZE;
-    char * name_buffer = new char[name_buffer_size];
+    std::string name_buffer(name_buffer_size, '\0');
 
-    code = zoo_create(impl, path.c_str(), data.c_str(), data.size(), getDefaultACL(), mode,  name_buffer, name_buffer_size);
+    code = zoo_create(impl, path.c_str(), data.c_str(), data.size(), getDefaultACL(), mode,  name_buffer.data(), name_buffer_size);
     ProfileEvents::increment(ProfileEvents::ZooKeeperCreate);
     ProfileEvents::increment(ProfileEvents::ZooKeeperTransactions);
 
     if (code == ZOK)
     {
-        path_created = std::string(name_buffer);
+        name_buffer.resize(strlen(name_buffer.data()));
+        path_created = std::move(name_buffer);
     }
 
-    delete[] name_buffer;
-
     return code;
 }
 
@@ -571,6 +567,18 @@ int32_t ZooKeeper::trySet(const std::string & path, const std::string & data,
     return code;
 }
 
+
+static void convertOpResults(const std::vector<zoo_op_result_t> & out_results_native, OpResultsPtr & out_results,
+                             const ZooKeeper * zookeeper = nullptr)
+{
+    if (!out_results)
+        out_results = std::make_shared<OpResults>();
+
+    out_results->reserve(out_results_native.size());
+    for (const zoo_op_result_t & res_native : out_results_native)
+            out_results->emplace_back(res_native, zookeeper);
+}
+
 int32_t ZooKeeper::multiImpl(const Ops & ops_, OpResultsPtr * out_results_)
 {
     if (ops_.empty())
@@ -585,7 +593,7 @@ int32_t ZooKeeper::multiImpl(const Ops & ops_, OpResultsPtr * out_results_)
         return ZINVALIDSTATE;
 
     size_t count = ops_.size();
-    OpResultsPtr out_results(new OpResults(count));
+    std::vector<zoo_op_result_t> out_results_native(count);
 
     /// Copy the struct containing pointers with default copy-constructor.
     /// It is safe because it hasn't got a destructor.
@@ -594,12 +602,12 @@ int32_t ZooKeeper::multiImpl(const Ops & ops_, OpResultsPtr * out_results_)
     for (const auto & op : ops_)
         ops.push_back(*(op->data));
 
-    int32_t code = zoo_multi(impl, static_cast<int>(ops.size()), ops.data(), out_results->data());
+    int32_t code = zoo_multi(impl, static_cast<int>(ops.size()), ops.data(), out_results_native.data());
     ProfileEvents::increment(ProfileEvents::ZooKeeperMulti);
     ProfileEvents::increment(ProfileEvents::ZooKeeperTransactions);
 
     if (out_results_)
-        *out_results_ = out_results;
+        convertOpResults(out_results_native, *out_results_, this);
 
     return code;
 }
@@ -754,7 +762,7 @@ ZooKeeper::~ZooKeeper()
 
 ZooKeeperPtr ZooKeeper::startNewSession() const
 {
-    return std::make_shared<ZooKeeper>(hosts, identity, session_timeout_ms);
+    return std::make_shared<ZooKeeper>(hosts, identity, session_timeout_ms, chroot);
 }
 
 Op::Create::Create(const std::string & path_pattern_, const std::string & value_, ACLPtr acl_, int32_t flags_)
@@ -982,7 +990,7 @@ ZooKeeper::TryRemoveFuture ZooKeeper::asyncTryRemove(const std::string & path, i
 ZooKeeper::MultiFuture ZooKeeper::asyncMultiImpl(const zkutil::Ops & ops_, bool throw_exception)
 {
     size_t count = ops_.size();
-    OpResultsPtr results(new OpResults(count));
+    auto results_native = std::make_shared<std::vector<zoo_op_result_t>>(count);
 
     /// We need to hold all references to ops data until the end of multi callback
     struct OpsHolder
@@ -997,10 +1005,10 @@ ZooKeeper::MultiFuture ZooKeeper::asyncMultiImpl(const zkutil::Ops & ops_, bool
         holder.ops_raw_ptr->push_back(*holder.ops_ptr->back()->data);
     }
 
-    MultiFuture future{ [throw_exception, results, holder] (int rc) {
+    MultiFuture future{ [throw_exception, results_native, holder, zookeeper=this] (int rc) {
         OpResultsAndCode res;
         res.code = rc;
-        res.results = results;
+        convertOpResults(*results_native, res.results, zookeeper);
         res.ops_ptr = holder.ops_ptr;
         if (throw_exception && rc != ZOK)
             throw zkutil::KeeperException(rc);
@@ -1020,7 +1028,7 @@ ZooKeeper::MultiFuture ZooKeeper::asyncMultiImpl(const zkutil::Ops & ops_, bool
 
     auto & ops = *holder.ops_raw_ptr;
 
-    int32_t code = zoo_amulti(impl, static_cast<int>(ops.size()), ops.data(), results->data(),
+    int32_t code = zoo_amulti(impl, static_cast<int>(ops.size()), ops.data(), results_native->data(),
                               [] (int rc, const void * data)
                               {
                                   MultiFuture::TaskPtr owned_task =
@@ -1069,4 +1077,26 @@ size_t getFailedOpIndex(const OpResultsPtr & op_results, int32_t transaction_ret
 }
 
 
+OpResult::OpResult(const zoo_op_result_t & op_result, const ZooKeeper * zookeeper)
+    : err(op_result.err)
+{
+    if (op_result.value)
+    {
+        value = std::string(op_result.value, op_result.value + op_result.valuelen);
+
+        /// Current version of libzookeeper does not cut chroot path prefixes
+        /// We do it here manually
+        if (zookeeper && !zookeeper->chroot.empty())
+        {
+            if (startsWith(value, zookeeper->chroot))
+                value = value.substr(zookeeper->chroot.length());
+            else
+                throw DB::Exception("Expected ZooKeeper path with chroot " + zookeeper->chroot + ", got " + value,
+                                    DB::ErrorCodes::LOGICAL_ERROR);
+        }
+    }
+
+    if (op_result.stat)
+        stat = std::make_unique<Stat>(*op_result.stat);
+}
 }
diff --git a/dbms/src/Common/ZooKeeper/ZooKeeper.h b/dbms/src/Common/ZooKeeper/ZooKeeper.h
index 8d35e37d27f..29b1d2fb69b 100644
--- a/dbms/src/Common/ZooKeeper/ZooKeeper.h
+++ b/dbms/src/Common/ZooKeeper/ZooKeeper.h
@@ -56,7 +56,7 @@ public:
     using Ptr = std::shared_ptr<ZooKeeper>;
 
     ZooKeeper(const std::string & hosts, const std::string & identity = "",
-              int32_t session_timeout_ms = DEFAULT_SESSION_TIMEOUT, bool check_root_exists = false);
+              int32_t session_timeout_ms = DEFAULT_SESSION_TIMEOUT, const std::string & chroot = "");
 
     /** Config of the form:
         <zookeeper>
@@ -366,9 +366,10 @@ public:
 private:
     friend struct WatchContext;
     friend class EphemeralNodeHolder;
+    friend class OpResult;
 
     void init(const std::string & hosts, const std::string & identity,
-              int32_t session_timeout_ms, bool check_root_exists);
+              int32_t session_timeout_ms, const std::string & chroot);
     void removeChildrenRecursive(const std::string & path);
     void tryRemoveChildrenRecursive(const std::string & path);
 
@@ -414,6 +415,7 @@ private:
     std::string hosts;
     std::string identity;
     int32_t session_timeout_ms;
+    std::string chroot;
 
     std::mutex mutex;
     ACLPtr default_acl;
diff --git a/dbms/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp b/dbms/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp
index df71feab43e..4d1d05b54e1 100644
--- a/dbms/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp
+++ b/dbms/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp
@@ -134,10 +134,13 @@ TEST(zkutil, multi_create_sequential)
     try
     {
         auto zookeeper = std::make_unique<zkutil::ZooKeeper>("localhost:2181");
+        zookeeper->createAncestors("/clickhouse_test/");
+
+        zookeeper = std::make_unique<zkutil::ZooKeeper>("localhost:2181", "", zkutil::DEFAULT_SESSION_TIMEOUT, "/clickhouse_test");
         auto acl = zookeeper->getDefaultACL();
         zkutil::Ops ops;
 
-        String base_path = "/clickhouse_test/zkutil/multi_create_sequential";
+        String base_path = "/zkutil/multi_create_sequential";
         zookeeper->tryRemoveRecursive(base_path);
         zookeeper->createAncestors(base_path + "/");
 
@@ -147,6 +150,10 @@ TEST(zkutil, multi_create_sequential)
         zkutil::OpResult & result = results->at(0);
 
         EXPECT_TRUE(result.value != nullptr);
+
+        String path = result.value;
+        std::cout << path << "\n";
+
         EXPECT_TRUE(startsWith(result.value, entry_path));
     }
     catch (...)
@@ -154,5 +161,6 @@ TEST(zkutil, multi_create_sequential)
         std::cerr << getCurrentExceptionMessage(false);
         throw;
     }
-
 }
+
+

From 8028ba0de98ad8e187fdc05ee042f2b4426c8163 Mon Sep 17 00:00:00 2001
From: Vitaliy Lyudvichenko <vludv@yandex-team.ru>
Date: Tue, 13 Mar 2018 23:36:22 +0300
Subject: [PATCH 075/244] Refactoring of zkutil::ZooKeeper::multi* functions.
 [#CLICKHOUSE-2]

---
 dbms/src/Common/ZooKeeper/KeeperException.h   | 15 +++
 dbms/src/Common/ZooKeeper/Types.h             | 50 +++++++---
 dbms/src/Common/ZooKeeper/ZooKeeper.cpp       | 93 ++++++++++++-------
 dbms/src/Common/ZooKeeper/ZooKeeper.h         | 53 ++---------
 .../tests/zkutil_expiration_test.cpp          |  4 +-
 dbms/src/Interpreters/DDLWorker.cpp           | 12 +--
 dbms/src/Server/ClusterCopier.cpp             | 28 +++---
 .../MergeTree/AbandonableLockInZooKeeper.h    | 59 ++++++------
 .../ReplicatedMergeTreeAlterThread.cpp        |  4 +-
 .../ReplicatedMergeTreeBlockOutputStream.cpp  | 59 ++++++------
 .../ReplicatedMergeTreeCleanupThread.cpp      |  4 +-
 .../MergeTree/ReplicatedMergeTreeQueue.cpp    | 10 +-
 .../ReplicatedMergeTreeRestartingThread.cpp   |  4 +-
 .../Storages/StorageReplicatedMergeTree.cpp   | 66 ++++++-------
 14 files changed, 247 insertions(+), 214 deletions(-)

diff --git a/dbms/src/Common/ZooKeeper/KeeperException.h b/dbms/src/Common/ZooKeeper/KeeperException.h
index b6a46930b97..37b0a04b564 100644
--- a/dbms/src/Common/ZooKeeper/KeeperException.h
+++ b/dbms/src/Common/ZooKeeper/KeeperException.h
@@ -102,4 +102,19 @@ private:
 
 };
 
+
+class KeeperMultiException : public KeeperException
+{
+public:
+    MultiTransactionInfo info;
+
+    /// If it is user error throws KeeperMultiException else throws ordinary KeeperException
+    /// If it is ZOK does nothing
+    static void check(const MultiTransactionInfo & info);
+    static void check(int code, const Ops & ops, const OpResultsPtr & op_results);
+
+protected:
+    KeeperMultiException(const MultiTransactionInfo & info, size_t failed_op_index);
+};
+
 };
diff --git a/dbms/src/Common/ZooKeeper/Types.h b/dbms/src/Common/ZooKeeper/Types.h
index 08183f8128d..1938081bb2e 100644
--- a/dbms/src/Common/ZooKeeper/Types.h
+++ b/dbms/src/Common/ZooKeeper/Types.h
@@ -21,7 +21,7 @@ public:
     Op() : data(new zoo_op_t) {}
     virtual ~Op() {}
 
-    virtual std::unique_ptr<Op> clone() const = 0;
+    virtual std::shared_ptr<Op> clone() const = 0;
 
     virtual std::string getPath() const = 0;
 
@@ -35,6 +35,9 @@ public:
     struct Check;
 };
 
+using OpPtr = std::shared_ptr<Op>;
+
+
 struct Op::Remove : public Op
 {
     Remove(const std::string & path_, int32_t version_) :
@@ -43,9 +46,9 @@ struct Op::Remove : public Op
         zoo_delete_op_init(data.get(), path.c_str(), version);
     }
 
-    std::unique_ptr<Op> clone() const override
+    OpPtr clone() const override
     {
-        return std::unique_ptr<zkutil::Op>(new Remove(path, version));
+        return std::make_shared<Remove>(path, version);
     }
 
     std::string getPath() const override { return path; }
@@ -61,9 +64,9 @@ struct Op::Create : public Op
 {
     Create(const std::string & path_pattern_, const std::string & value_, ACLPtr acl_, int32_t flags_);
 
-    std::unique_ptr<Op> clone() const override
+    OpPtr clone() const override
     {
-        return std::unique_ptr<zkutil::Op>(new Create(path_pattern, value, acl, flags));
+        return std::make_shared<Create>(path_pattern, value, acl, flags);
     }
 
     std::string getPathCreated() { return created_path.data(); }
@@ -93,9 +96,9 @@ struct Op::SetData : public Op
         zoo_set_op_init(data.get(), path.c_str(), value.c_str(), value.size(), version, &stat);
     }
 
-    std::unique_ptr<Op> clone() const override
+    OpPtr clone() const override
     {
-        return std::unique_ptr<zkutil::Op>(new SetData(path, value, version));
+        return std::make_shared<SetData>(path, value, version);
     }
 
     std::string getPath() const override { return path; }
@@ -124,9 +127,9 @@ struct Op::Check : public Op
         zoo_check_op_init(data.get(), path.c_str(), version);
     }
 
-    std::unique_ptr<Op> clone() const override
+    OpPtr clone() const override
     {
-        return std::unique_ptr<zkutil::Op>(new Check(path, version));
+        return std::make_shared<Check>(path, version);
     }
 
     std::string getPath() const override { return path; }
@@ -138,6 +141,9 @@ private:
     int32_t version;
 };
 
+using Ops = std::vector<OpPtr>;
+
+
 /// C++ version of zoo_op_result_t
 struct OpResult
 {
@@ -148,13 +154,33 @@ struct OpResult
     /// ZooKeeper is required for correct chroot path prefixes handling
     explicit OpResult(const zoo_op_result_t & op_result, const ZooKeeper * zookeeper = nullptr);
 };
-
-using OpPtr = std::unique_ptr<Op>;
-using Ops = std::vector<OpPtr>;
 using OpResults = std::vector<OpResult>;
 using OpResultsPtr = std::shared_ptr<OpResults>;
 using Strings = std::vector<std::string>;
 
+
+/// Simple structure to handle transaction execution results
+struct MultiTransactionInfo
+{
+    Ops ops;
+    int32_t code = ZOK;
+    OpResultsPtr op_results;
+
+    MultiTransactionInfo() = default;
+
+    MultiTransactionInfo(int32_t code_, const Ops & ops_, const OpResultsPtr & op_results_)
+        : ops(ops_), code(code_), op_results(op_results_) {}
+
+    bool empty() const
+    {
+        return ops.empty();
+    }
+
+    /// Returns failed op if zkutil::isUserError(code) is true
+    const Op & getFailedOp() const;
+};
+
+
 namespace CreateMode
 {
     extern const int Persistent;
diff --git a/dbms/src/Common/ZooKeeper/ZooKeeper.cpp b/dbms/src/Common/ZooKeeper/ZooKeeper.cpp
index 105c275e711..8d3866a0008 100644
--- a/dbms/src/Common/ZooKeeper/ZooKeeper.cpp
+++ b/dbms/src/Common/ZooKeeper/ZooKeeper.cpp
@@ -85,7 +85,7 @@ void ZooKeeper::init(const std::string & hosts_, const std::string & identity_,
                      int32_t session_timeout_ms_, const std::string & chroot_)
 {
     log = &Logger::get("ZooKeeper");
-    zoo_set_debug_level(ZOO_LOG_LEVEL_DEBUG);
+    zoo_set_debug_level(ZOO_LOG_LEVEL_ERROR);
     hosts = hosts_;
     identity = identity_;
     session_timeout_ms = session_timeout_ms_;
@@ -567,19 +567,19 @@ int32_t ZooKeeper::trySet(const std::string & path, const std::string & data,
     return code;
 }
 
-
-static void convertOpResults(const std::vector<zoo_op_result_t> & out_results_native, OpResultsPtr & out_results,
+/// Makes deep copy of zoo_op_result_t and removes chroot prefix from paths
+static void convertOpResults(const std::vector<zoo_op_result_t> & op_results_native, OpResultsPtr & out_op_results,
                              const ZooKeeper * zookeeper = nullptr)
 {
-    if (!out_results)
-        out_results = std::make_shared<OpResults>();
+    if (!out_op_results)
+        out_op_results = std::make_shared<OpResults>();
 
-    out_results->reserve(out_results_native.size());
-    for (const zoo_op_result_t & res_native : out_results_native)
-            out_results->emplace_back(res_native, zookeeper);
+    out_op_results->reserve(op_results_native.size());
+    for (const zoo_op_result_t & res_native : op_results_native)
+        out_op_results->emplace_back(res_native, zookeeper);
 }
 
-int32_t ZooKeeper::multiImpl(const Ops & ops_, OpResultsPtr * out_results_)
+int32_t ZooKeeper::multiImpl(const Ops & ops_, OpResultsPtr * out_op_results, MultiTransactionInfo * out_info)
 {
     if (ops_.empty())
         return ZOK;
@@ -606,30 +606,27 @@ int32_t ZooKeeper::multiImpl(const Ops & ops_, OpResultsPtr * out_results_)
     ProfileEvents::increment(ProfileEvents::ZooKeeperMulti);
     ProfileEvents::increment(ProfileEvents::ZooKeeperTransactions);
 
-    if (out_results_)
-        convertOpResults(out_results_native, *out_results_, this);
+    if (out_op_results || out_info)
+    {
+        OpResultsPtr op_results;
+        convertOpResults(out_results_native, op_results, this);
+
+        if (out_op_results)
+            *out_op_results = op_results;
+
+        if (out_info)
+            *out_info = MultiTransactionInfo(code, ops_, op_results);
+    }
 
     return code;
 }
 
 OpResultsPtr ZooKeeper::multi(const Ops & ops)
 {
-    OpResultsPtr results;
-    int code = tryMulti(ops, &results);
-    if (code != ZOK)
-    {
-        if (results && results->size() == ops.size())
-        {
-            for (size_t i = 0; i < ops.size(); ++i)
-            {
-                if (results->at(i).err == code)
-                    throw KeeperException("Transaction failed at op #" + std::to_string(i) + ": " + ops[i]->describe(), code);
-            }
-        }
-
-        throw KeeperException(code);
-    }
-    return results;
+    OpResultsPtr op_results;
+    int code = multiImpl(ops, &op_results);
+    KeeperMultiException::check(code, ops, op_results);
+    return op_results;
 }
 
 int32_t ZooKeeper::tryMulti(const Ops & ops_, OpResultsPtr * out_results_)
@@ -646,17 +643,9 @@ int32_t ZooKeeper::tryMulti(const Ops & ops_, OpResultsPtr * out_results_)
     return code;
 }
 
-int32_t ZooKeeper::tryMultiUnsafe(const Ops & ops, MultiTransactionInfo & info)
-{
-    info.code = multiImpl(ops, &info.op_results);
-    for (const OpPtr & op : ops)
-        info.ops.emplace_back(op->clone());
-    return info.code;
-}
-
 int32_t ZooKeeper::tryMultiWithRetries(const Ops & ops, OpResultsPtr * out_results, size_t * attempt)
 {
-    int32_t code = retry(std::bind(&ZooKeeper::multiImpl, this, std::ref(ops), out_results), attempt);
+    int32_t code = retry(std::bind(&ZooKeeper::multiImpl, this, std::ref(ops), out_results, nullptr), attempt);
     if (!(code == ZOK ||
         code == ZNONODE ||
         code == ZNODEEXISTS ||
@@ -1099,4 +1088,36 @@ OpResult::OpResult(const zoo_op_result_t & op_result, const ZooKeeper * zookeepe
     if (op_result.stat)
         stat = std::make_unique<Stat>(*op_result.stat);
 }
+
+
+KeeperMultiException::KeeperMultiException(const MultiTransactionInfo & info_, size_t failed_op_index_)
+    :KeeperException(
+        "Transaction failed at op #" + std::to_string(failed_op_index_) + ": " + info_.ops.at(failed_op_index_)->describe(),
+        info_.code),
+    info(info_) {}
+
+void KeeperMultiException::check(int code, const Ops & ops, const OpResultsPtr & op_results)
+{
+    if (code == ZOK) {}
+    else if (zkutil::isUserError(code))
+        throw KeeperMultiException(MultiTransactionInfo(code, ops, op_results), getFailedOpIndex(op_results, code));
+    else
+        throw KeeperException(code);
+}
+
+void KeeperMultiException::check(const MultiTransactionInfo & info)
+{
+    if (info.code == ZOK) {}
+    else if (zkutil::isUserError(info.code))
+        throw KeeperMultiException(info, getFailedOpIndex(info.op_results, info.code));
+    else
+        throw KeeperException(info.code);
+}
+
+
+const Op & MultiTransactionInfo::getFailedOp() const
+{
+    return *ops.at(getFailedOpIndex(op_results, code));
+}
+
 }
diff --git a/dbms/src/Common/ZooKeeper/ZooKeeper.h b/dbms/src/Common/ZooKeeper/ZooKeeper.h
index 29b1d2fb69b..2b0d195fd80 100644
--- a/dbms/src/Common/ZooKeeper/ZooKeeper.h
+++ b/dbms/src/Common/ZooKeeper/ZooKeeper.h
@@ -196,14 +196,16 @@ public:
     /// Performs several operations in a transaction.
     /// Throws on every error.
     OpResultsPtr multi(const Ops & ops);
-
     /// Throws only if some operation has returned an "unexpected" error
     /// - an error that would cause the corresponding try- method to throw.
     int32_t tryMulti(const Ops & ops, OpResultsPtr * out_results = nullptr);
-    /// Like previous one, but does not throw any ZooKeeper exceptions
-    int32_t tryMultiUnsafe(const Ops & ops, MultiTransactionInfo & info);
     /// Use only with read-only operations.
     int32_t tryMultiWithRetries(const Ops & ops, OpResultsPtr * out_results = nullptr, size_t * attempt = nullptr);
+    /// Throws nothing, just alias of multiImpl
+    int32_t tryMultiNoThrow(const Ops & ops, OpResultsPtr * out_op_results = nullptr, MultiTransactionInfo * out_info = nullptr)
+    {
+        return multiImpl(ops, out_op_results, out_info);
+    }
 
     Int64 getClientID();
 
@@ -368,8 +370,8 @@ private:
     friend class EphemeralNodeHolder;
     friend class OpResult;
 
-    void init(const std::string & hosts, const std::string & identity,
-              int32_t session_timeout_ms, const std::string & chroot);
+    void init(const std::string & hosts, const std::string & identity, int32_t session_timeout_ms, const std::string & chroot);
+
     void removeChildrenRecursive(const std::string & path);
     void tryRemoveChildrenRecursive(const std::string & path);
 
@@ -392,7 +394,7 @@ private:
             /// If the connection has been lost, wait timeout/3 hoping for connection re-establishment.
             static const int MAX_SLEEP_TIME = 10;
             if (code == ZCONNECTIONLOSS)
-                usleep(std::min(session_timeout_ms * 1000 / 3, MAX_SLEEP_TIME * 1000 * 1000));
+                usleep(std::min(session_timeout_ms * 1000u / 3, MAX_SLEEP_TIME * 1000u * 1000u));
 
             LOG_WARNING(log, "Error on attempt " << i << ": " << error2string(code)  << ". Retry");
             code = operation();
@@ -407,7 +409,7 @@ private:
     int32_t getImpl(const std::string & path, std::string & res, Stat * stat, WatchCallback watch_callback);
     int32_t setImpl(const std::string & path, const std::string & data, int32_t version = -1, Stat * stat = nullptr);
     int32_t getChildrenImpl(const std::string & path, Strings & res, Stat * stat, WatchCallback watch_callback);
-    int32_t multiImpl(const Ops & ops, OpResultsPtr * out_results = nullptr);
+    int32_t multiImpl(const Ops & ops, OpResultsPtr * out_op_results = nullptr, MultiTransactionInfo * out_info = nullptr);
     int32_t existsImpl(const std::string & path, Stat * stat_, WatchCallback watch_callback);
 
     MultiFuture asyncMultiImpl(const zkutil::Ops & ops_, bool throw_exception);
@@ -495,41 +497,4 @@ private:
 
 using EphemeralNodeHolderPtr = EphemeralNodeHolder::Ptr;
 
-
-/// Simple structure to handle transaction execution results
-struct MultiTransactionInfo
-{
-    MultiTransactionInfo() = default;
-
-    Ops ops;
-    int32_t code = ZOK;
-    OpResultsPtr op_results;
-
-    bool empty() const
-    {
-        return ops.empty();
-    }
-
-    bool hasFailedOp() const
-    {
-        return zkutil::isUserError(code);
-    }
-
-    const Op & getFailedOp() const
-    {
-        return *ops.at(getFailedOpIndex(op_results, code));
-    }
-
-    KeeperException getException() const
-    {
-        if (hasFailedOp())
-        {
-            size_t i = getFailedOpIndex(op_results, code);
-            return KeeperException("Transaction failed at op #" + std::to_string(i) + ": " + ops.at(i)->describe(), code);
-        }
-        else
-            return KeeperException(code);
-    }
-};
-
 }
diff --git a/dbms/src/Common/ZooKeeper/tests/zkutil_expiration_test.cpp b/dbms/src/Common/ZooKeeper/tests/zkutil_expiration_test.cpp
index 2079f7a84da..54648bdad8f 100644
--- a/dbms/src/Common/ZooKeeper/tests/zkutil_expiration_test.cpp
+++ b/dbms/src/Common/ZooKeeper/tests/zkutil_expiration_test.cpp
@@ -34,8 +34,8 @@ int main(int argc, char ** argv)
         {
             {
                 zkutil::Ops ops;
-                ops.emplace_back(std::make_unique<zkutil::Op::Create>("/test/zk_expiration_test", "hello", zk.getDefaultACL(), zkutil::CreateMode::Persistent));
-                ops.emplace_back(std::make_unique<zkutil::Op::Remove>("/test/zk_expiration_test", -1));
+                ops.emplace_back(std::make_shared<zkutil::Op::Create>("/test/zk_expiration_test", "hello", zk.getDefaultACL(), zkutil::CreateMode::Persistent));
+                ops.emplace_back(std::make_shared<zkutil::Op::Remove>("/test/zk_expiration_test", -1));
 
                 zkutil::MultiTransactionInfo info;
                 zk.tryMultiUnsafe(ops, info);
diff --git a/dbms/src/Interpreters/DDLWorker.cpp b/dbms/src/Interpreters/DDLWorker.cpp
index bc9cac4e206..7deabb7872e 100644
--- a/dbms/src/Interpreters/DDLWorker.cpp
+++ b/dbms/src/Interpreters/DDLWorker.cpp
@@ -600,8 +600,8 @@ void DDLWorker::processTask(DDLTask & task)
 
     /// Delete active flag and create finish flag
     zkutil::Ops ops;
-    ops.emplace_back(std::make_unique<zkutil::Op::Remove>(active_node_path, -1));
-    ops.emplace_back(std::make_unique<zkutil::Op::Create>(finished_node_path, task.execution_status.serializeText(),
+    ops.emplace_back(std::make_shared<zkutil::Op::Remove>(active_node_path, -1));
+    ops.emplace_back(std::make_shared<zkutil::Op::Create>(finished_node_path, task.execution_status.serializeText(),
                                                           zookeeper->getDefaultACL(), zkutil::CreateMode::Persistent));
     zookeeper->multi(ops);
 }
@@ -780,8 +780,8 @@ void DDLWorker::cleanupQueue()
 
                 /// Remove the lock node and its parent atomically
                 zkutil::Ops ops;
-                ops.emplace_back(std::make_unique<zkutil::Op::Remove>(lock_path, -1));
-                ops.emplace_back(std::make_unique<zkutil::Op::Remove>(node_path, -1));
+                ops.emplace_back(std::make_shared<zkutil::Op::Remove>(lock_path, -1));
+                ops.emplace_back(std::make_shared<zkutil::Op::Remove>(node_path, -1));
                 zookeeper->multi(ops);
 
                 lock->unlockAssumeLockNodeRemovedManually();
@@ -800,8 +800,8 @@ void DDLWorker::createStatusDirs(const std::string & node_path)
 {
     zkutil::Ops ops;
     auto acl = zookeeper->getDefaultACL();
-    ops.emplace_back(std::make_unique<zkutil::Op::Create>(node_path + "/active", "", acl, zkutil::CreateMode::Persistent));
-    ops.emplace_back(std::make_unique<zkutil::Op::Create>(node_path + "/finished", "", acl, zkutil::CreateMode::Persistent));
+    ops.emplace_back(std::make_shared<zkutil::Op::Create>(node_path + "/active", "", acl, zkutil::CreateMode::Persistent));
+    ops.emplace_back(std::make_shared<zkutil::Op::Create>(node_path + "/finished", "", acl, zkutil::CreateMode::Persistent));
 
     int code = zookeeper->tryMulti(ops);
     if (code != ZOK && code != ZNODEEXISTS)
diff --git a/dbms/src/Server/ClusterCopier.cpp b/dbms/src/Server/ClusterCopier.cpp
index 3ae6a9ed205..077dc43c5f6 100644
--- a/dbms/src/Server/ClusterCopier.cpp
+++ b/dbms/src/Server/ClusterCopier.cpp
@@ -355,20 +355,23 @@ struct TaskCluster
 
 /// Atomically checks that is_dirty node is not exists, and made the remaining op
 /// Returns relative number of failed operation in the second field (the passed op has 0 index)
-static void checkNoNodeAndCommit(
+static zkutil::MultiTransactionInfo checkNoNodeAndCommit(
     const zkutil::ZooKeeperPtr & zookeeper,
     const String & checking_node_path,
-    zkutil::OpPtr && op,
-    zkutil::MultiTransactionInfo & info)
+    zkutil::OpPtr && op)
 {
     zkutil::Ops ops;
-    ops.emplace_back(std::make_unique<zkutil::Op::Create>(checking_node_path, "", zookeeper->getDefaultACL(), zkutil::CreateMode::Persistent));
-    ops.emplace_back(std::make_unique<zkutil::Op::Remove>(checking_node_path, -1));
+    ops.emplace_back(std::make_shared<zkutil::Op::Create>(checking_node_path, "", zookeeper->getDefaultACL(), zkutil::CreateMode::Persistent));
+    ops.emplace_back(std::make_shared<zkutil::Op::Remove>(checking_node_path, -1));
     ops.emplace_back(std::move(op));
 
-    zookeeper->tryMultiUnsafe(ops, info);
+    zkutil::MultiTransactionInfo info;
+    zookeeper->tryMultiNoThrow(ops, nullptr, &info);
+
     if (info.code != ZOK && !zkutil::isUserError(info.code))
-        throw info.getException();
+        throw zkutil::KeeperException(info.code);
+
+    return info;
 }
 
 
@@ -1542,10 +1545,8 @@ protected:
         /// Try start processing, create node about it
         {
             String start_state = TaskStateWithOwner::getData(TaskState::Started, host_id);
-            auto op_create = std::make_unique<zkutil::Op::Create>(current_task_status_path, start_state, acl, zkutil::CreateMode::Persistent);
-
-            zkutil::MultiTransactionInfo info;
-            checkNoNodeAndCommit(zookeeper, is_dirty_flag_path, std::move(op_create), info);
+            auto op_create = std::make_shared<zkutil::Op::Create>(current_task_status_path, start_state, acl, zkutil::CreateMode::Persistent);
+            zkutil::MultiTransactionInfo info = checkNoNodeAndCommit(zookeeper, is_dirty_flag_path, std::move(op_create));
 
             if (info.code != ZOK)
             {
@@ -1683,9 +1684,8 @@ protected:
         /// Finalize the processing, change state of current partition task (and also check is_dirty flag)
         {
             String state_finished = TaskStateWithOwner::getData(TaskState::Finished, host_id);
-            auto op_set = std::make_unique<zkutil::Op::SetData>(current_task_status_path, state_finished, 0);
-            zkutil::MultiTransactionInfo info;
-            checkNoNodeAndCommit(zookeeper, is_dirty_flag_path, std::move(op_set), info);
+            auto op_set = std::make_shared<zkutil::Op::SetData>(current_task_status_path, state_finished, 0);
+            zkutil::MultiTransactionInfo info = checkNoNodeAndCommit(zookeeper, is_dirty_flag_path, std::move(op_set));
 
             if (info.code != ZOK)
             {
diff --git a/dbms/src/Storages/MergeTree/AbandonableLockInZooKeeper.h b/dbms/src/Storages/MergeTree/AbandonableLockInZooKeeper.h
index 96ccd5d4dfc..690217302bf 100644
--- a/dbms/src/Storages/MergeTree/AbandonableLockInZooKeeper.h
+++ b/dbms/src/Storages/MergeTree/AbandonableLockInZooKeeper.h
@@ -19,7 +19,7 @@ namespace ErrorCodes
   * When the destructor is called or the session ends in ZooKeeper, it goes into the ABANDONED state.
   * (Including when the program is halted).
   */
-class AbandonableLockInZooKeeper : private boost::noncopyable
+class AbandonableLockInZooKeeper : public boost::noncopyable
 {
 public:
     enum State
@@ -31,44 +31,52 @@ public:
 
     AbandonableLockInZooKeeper(
         const String & path_prefix_, const String & temp_path, zkutil::ZooKeeper & zookeeper_, zkutil::Ops * precheck_ops = nullptr)
-        : zookeeper(zookeeper_), path_prefix(path_prefix_)
+        : zookeeper(&zookeeper_), path_prefix(path_prefix_)
     {
         String abandonable_path = temp_path + "/abandonable_lock-";
 
         /// Let's create an secondary ephemeral node.
         if (!precheck_ops || precheck_ops->empty())
         {
-            holder_path = zookeeper.create(abandonable_path, "", zkutil::CreateMode::EphemeralSequential);
+            holder_path = zookeeper->create(abandonable_path, "", zkutil::CreateMode::EphemeralSequential);
         }
         else
         {
-            precheck_ops->emplace_back(std::make_unique<zkutil::Op::Create>(
-                abandonable_path, "", zookeeper.getDefaultACL(), zkutil::CreateMode::EphemeralSequential));
+            precheck_ops->emplace_back(std::make_shared<zkutil::Op::Create>(
+                abandonable_path, "", zookeeper->getDefaultACL(), zkutil::CreateMode::EphemeralSequential));
 
-            if (zookeeper.tryMultiUnsafe(*precheck_ops, precheck_info) != ZOK)
-                return;
+            zkutil::OpResultsPtr op_results = zookeeper->multi(*precheck_ops);
 
-            holder_path = precheck_info.op_results->back().value;
+            holder_path = op_results->back().value;
         }
 
         /// Write the path to the secondary node in the main node.
-        path = zookeeper.create(path_prefix, holder_path, zkutil::CreateMode::PersistentSequential);
+        path = zookeeper->create(path_prefix, holder_path, zkutil::CreateMode::PersistentSequential);
 
         if (path.size() <= path_prefix.size())
             throw Exception("Logical error: name of sequential node is shorter than prefix.", ErrorCodes::LOGICAL_ERROR);
     }
 
-    AbandonableLockInZooKeeper(AbandonableLockInZooKeeper && rhs)
-        : zookeeper(rhs.zookeeper)
+    AbandonableLockInZooKeeper() = default;
+
+    AbandonableLockInZooKeeper(AbandonableLockInZooKeeper && rhs) noexcept
     {
-        std::swap(path_prefix, rhs.path_prefix);
-        std::swap(path, rhs.path);
-        std::swap(holder_path, rhs.holder_path);
+        *this = std::move(rhs);
+    }
+
+    AbandonableLockInZooKeeper & operator=(AbandonableLockInZooKeeper && rhs) noexcept
+    {
+        zookeeper = rhs.zookeeper;
+        rhs.zookeeper = nullptr;
+        path_prefix = std::move(rhs.path_prefix);
+        path = std::move(rhs.path);
+        holder_path = std::move(rhs.holder_path);
+        return *this;
     }
 
     bool isCreated() const
     {
-        return !holder_path.empty() && !path.empty();
+        return zookeeper && !holder_path.empty() && !path.empty();
     }
 
     String getPath() const
@@ -87,8 +95,8 @@ public:
     void unlock()
     {
         checkCreated();
-        zookeeper.remove(path);
-        zookeeper.remove(holder_path);
+        zookeeper->remove(path);
+        zookeeper->remove(holder_path);
         holder_path = "";
     }
 
@@ -96,8 +104,8 @@ public:
     void getUnlockOps(zkutil::Ops & ops)
     {
         checkCreated();
-        ops.emplace_back(std::make_unique<zkutil::Op::Remove>(path, -1));
-        ops.emplace_back(std::make_unique<zkutil::Op::Remove>(holder_path, -1));
+        ops.emplace_back(std::make_shared<zkutil::Op::Remove>(path, -1));
+        ops.emplace_back(std::make_shared<zkutil::Op::Remove>(holder_path, -1));
     }
 
     void checkCreated() const
@@ -108,13 +116,13 @@ public:
 
     ~AbandonableLockInZooKeeper()
     {
-        if (holder_path.empty())
+        if (!zookeeper || holder_path.empty())
             return;
 
         try
         {
-            zookeeper.tryRemoveEphemeralNodeWithRetries(holder_path);
-            zookeeper.trySet(path, ""); /// It's not necessary.
+            zookeeper->tryRemoveEphemeralNodeWithRetries(holder_path);
+            zookeeper->trySet(path, ""); /// It's not necessary.
         }
         catch (...)
         {
@@ -153,15 +161,10 @@ public:
     }
 
 private:
-    zkutil::ZooKeeper & zookeeper;
+    zkutil::ZooKeeper * zookeeper = nullptr;
     String path_prefix;
     String path;
     String holder_path;
-
-public:
-    /// Contains information about execution of passed precheck ops and block-creation payload
-    /// TODO: quite ugly interface to handle precheck ops
-    zkutil::MultiTransactionInfo precheck_info;
 };
 
 }
diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp
index 35a2dfcc6b7..72d7cd91e3a 100644
--- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp
+++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp
@@ -141,9 +141,9 @@ void ReplicatedMergeTreeAlterThread::run()
 
                         /// Update part metadata in ZooKeeper.
                         zkutil::Ops ops;
-                        ops.emplace_back(std::make_unique<zkutil::Op::SetData>(
+                        ops.emplace_back(std::make_shared<zkutil::Op::SetData>(
                             storage.replica_path + "/parts/" + part->name + "/columns", transaction->getNewColumns().toString(), -1));
-                        ops.emplace_back(std::make_unique<zkutil::Op::SetData>(
+                        ops.emplace_back(std::make_shared<zkutil::Op::SetData>(
                             storage.replica_path + "/parts/" + part->name + "/checksums", transaction->getNewChecksums().toString(), -1));
 
                         try
diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp
index e34f0f68b6d..7e8ae99cad2 100644
--- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp
+++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp
@@ -218,32 +218,35 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo
         block_id_path = storage.zookeeper_path + "/blocks/" + block_id;
 
         /// Lets check for duplicates in advance, to avoid superflous block numbers allocation
-        deduplication_check_ops.emplace_back(std::make_unique<zkutil::Op::Create>(block_id_path, "", acl, zkutil::CreateMode::Persistent));
-        deduplication_check_ops.emplace_back(std::make_unique<zkutil::Op::Remove>(block_id_path, -1));
+        deduplication_check_ops.emplace_back(std::make_shared<zkutil::Op::Create>(block_id_path, "", acl, zkutil::CreateMode::Persistent));
+        deduplication_check_ops.emplace_back(std::make_shared<zkutil::Op::Remove>(block_id_path, -1));
         deduplication_check_ops_ptr = &deduplication_check_ops;
     }
 
-    /// 2 RTT
-    AbandonableLockInZooKeeper block_number_lock = storage.allocateBlockNumber(part->info.partition_id, zookeeper, deduplication_check_ops_ptr);
-    if (!block_number_lock.isCreated())
+    AbandonableLockInZooKeeper block_number_lock;
+    try
     {
-        zkutil::MultiTransactionInfo & info = block_number_lock.precheck_info;
+        /// 2 RTT
+        block_number_lock = storage.allocateBlockNumber(part->info.partition_id, zookeeper, deduplication_check_ops_ptr);
+    }
+    catch (zkutil::KeeperMultiException & e)
+    {
+        zkutil::MultiTransactionInfo & info = e.info;
 
-        if (!info.empty() && info.code != ZOK)
+        if (deduplicate_block && info.code == ZNODEEXISTS && info.getFailedOp().getPath() == block_id_path)
         {
-            if (deduplicate_block && info.code == ZNODEEXISTS && info.getFailedOp().getPath() == block_id_path)
-            {
-                LOG_INFO(log, "Block with ID " << block_id << " already exists; ignoring it (skip the insertion)");
-                part->is_duplicate = true;
-                last_block_is_duplicate = true;
-                ProfileEvents::increment(ProfileEvents::DuplicatedInsertedBlocks);
-                return;
-            }
-
-            throw Exception("Cannot allocate block number in ZooKeeper: " + info.getException().displayText(), ErrorCodes::KEEPER_EXCEPTION);
+            LOG_INFO(log, "Block with ID " << block_id << " already exists; ignoring it (skip the insertion)");
+            part->is_duplicate = true;
+            last_block_is_duplicate = true;
+            ProfileEvents::increment(ProfileEvents::DuplicatedInsertedBlocks);
+            return;
         }
 
-        throw Exception("Cannot allocate block number in ZooKeeper", ErrorCodes::KEEPER_EXCEPTION);
+        throw Exception("Cannot allocate block number in ZooKeeper: " + e.displayText(), ErrorCodes::KEEPER_EXCEPTION);
+    }
+    catch (zkutil::KeeperException & e)
+    {
+        throw Exception("Cannot allocate block number in ZooKeeper: " + e.displayText(), ErrorCodes::KEEPER_EXCEPTION);
     }
 
     Int64 block_number = block_number_lock.getNumber();
@@ -279,7 +282,7 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo
     {
         /// Make final duplicate check and commit block_id
         ops.emplace_back(
-            std::make_unique<zkutil::Op::Create>(
+            std::make_shared<zkutil::Op::Create>(
                 block_id_path,
                 toString(block_number),  /// We will able to know original part number for duplicate blocks, if we want.
                 acl,
@@ -288,27 +291,27 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo
 
     /// Information about the part, in the replica data.
 
-    ops.emplace_back(std::make_unique<zkutil::Op::Check>(
+    ops.emplace_back(std::make_shared<zkutil::Op::Check>(
         storage.zookeeper_path + "/columns",
         storage.columns_version));
-    ops.emplace_back(std::make_unique<zkutil::Op::Create>(
+    ops.emplace_back(std::make_shared<zkutil::Op::Create>(
         storage.replica_path + "/parts/" + part->name,
         "",
         acl,
         zkutil::CreateMode::Persistent));
-    ops.emplace_back(std::make_unique<zkutil::Op::Create>(
+    ops.emplace_back(std::make_shared<zkutil::Op::Create>(
         storage.replica_path + "/parts/" + part->name + "/columns",
         part->columns.toString(),
         acl,
         zkutil::CreateMode::Persistent));
-    ops.emplace_back(std::make_unique<zkutil::Op::Create>(
+    ops.emplace_back(std::make_shared<zkutil::Op::Create>(
         storage.replica_path + "/parts/" + part->name + "/checksums",
         part->checksums.toString(),
         acl,
         zkutil::CreateMode::Persistent));
 
     /// Replication log.
-    ops.emplace_back(std::make_unique<zkutil::Op::Create>(
+    ops.emplace_back(std::make_shared<zkutil::Op::Create>(
         storage.zookeeper_path + "/log/log-",
         log_entry.toString(),
         acl,
@@ -336,7 +339,7 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo
             */
 
         ops.emplace_back(
-            std::make_unique<zkutil::Op::Create>(
+            std::make_shared<zkutil::Op::Create>(
                 quorum_info.status_path,
                 quorum_entry.toString(),
                 acl,
@@ -344,7 +347,7 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo
 
         /// Make sure that during the insertion time, the replica was not reinitialized or disabled (when the server is finished).
         ops.emplace_back(
-            std::make_unique<zkutil::Op::Check>(
+            std::make_shared<zkutil::Op::Check>(
                 storage.replica_path + "/is_active",
                 quorum_info.is_active_node_version));
 
@@ -352,7 +355,7 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo
         /// But then the `host` value will change. We will check this.
         /// It's great that these two nodes change in the same transaction (see MergeTreeRestartingThread).
         ops.emplace_back(
-            std::make_unique<zkutil::Op::Check>(
+            std::make_shared<zkutil::Op::Check>(
                 storage.replica_path + "/host",
                 quorum_info.host_node_version));
     }
@@ -361,7 +364,7 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo
     storage.data.renameTempPartAndAdd(part, nullptr, &transaction);
 
     zkutil::MultiTransactionInfo info;
-    zookeeper->tryMultiUnsafe(ops, info); /// 1 RTT
+    zookeeper->tryMultiNoThrow(ops, nullptr, &info); /// 1 RTT
 
     if (info.code == ZOK)
     {
diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp
index ecd201ef765..0352374fd0f 100644
--- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp
+++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp
@@ -98,12 +98,12 @@ void ReplicatedMergeTreeCleanupThread::clearOldLogs()
     zkutil::Ops ops;
     for (size_t i = 0; i < entries.size(); ++i)
     {
-        ops.emplace_back(std::make_unique<zkutil::Op::Remove>(storage.zookeeper_path + "/log/" + entries[i], -1));
+        ops.emplace_back(std::make_shared<zkutil::Op::Remove>(storage.zookeeper_path + "/log/" + entries[i], -1));
 
         if (ops.size() > 4 * zkutil::MULTI_BATCH_SIZE || i + 1 == entries.size())
         {
             /// Simultaneously with clearing the log, we check to see if replica was added since we received replicas list.
-            ops.emplace_back(std::make_unique<zkutil::Op::Check>(storage.zookeeper_path + "/replicas", stat.version));
+            ops.emplace_back(std::make_shared<zkutil::Op::Check>(storage.zookeeper_path + "/replicas", stat.version));
             zookeeper->multi(ops);
             ops.clear();
         }
diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp
index b2fe028f5d7..c5d4a41d496 100644
--- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp
+++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp
@@ -164,11 +164,11 @@ void ReplicatedMergeTreeQueue::updateTimesInZooKeeper(
     zkutil::Ops ops;
 
     if (min_unprocessed_insert_time_changed)
-        ops.emplace_back(std::make_unique<zkutil::Op::SetData>(
+        ops.emplace_back(std::make_shared<zkutil::Op::SetData>(
             replica_path + "/min_unprocessed_insert_time", toString(*min_unprocessed_insert_time_changed), -1));
 
     if (max_processed_insert_time_changed)
-        ops.emplace_back(std::make_unique<zkutil::Op::SetData>(
+        ops.emplace_back(std::make_shared<zkutil::Op::SetData>(
             replica_path + "/max_processed_insert_time", toString(*max_processed_insert_time_changed), -1));
 
     if (!ops.empty())
@@ -334,7 +334,7 @@ bool ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper, z
                 zkutil::ZooKeeper::ValueAndStat res = future.second.get();
                 copied_entries.emplace_back(LogEntry::parse(res.value, res.stat));
 
-                ops.emplace_back(std::make_unique<zkutil::Op::Create>(
+                ops.emplace_back(std::make_shared<zkutil::Op::Create>(
                     replica_path + "/queue/queue-", res.value, zookeeper->getDefaultACL(), zkutil::CreateMode::PersistentSequential));
 
                 const auto & entry = *copied_entries.back();
@@ -349,11 +349,11 @@ bool ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper, z
                 }
             }
 
-            ops.emplace_back(std::make_unique<zkutil::Op::SetData>(
+            ops.emplace_back(std::make_shared<zkutil::Op::SetData>(
                 replica_path + "/log_pointer", toString(last_entry_index + 1), -1));
 
             if (min_unprocessed_insert_time_changed)
-                ops.emplace_back(std::make_unique<zkutil::Op::SetData>(
+                ops.emplace_back(std::make_shared<zkutil::Op::SetData>(
                     replica_path + "/min_unprocessed_insert_time", toString(*min_unprocessed_insert_time_changed), -1));
 
             try
diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp
index 64e4c31e4a8..f6686c273ce 100644
--- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp
+++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp
@@ -322,9 +322,9 @@ void ReplicatedMergeTreeRestartingThread::activateReplica()
 
     /// Simultaneously declare that this replica is active, and update the host.
     zkutil::Ops ops;
-    ops.emplace_back(std::make_unique<zkutil::Op::Create>(is_active_path,
+    ops.emplace_back(std::make_shared<zkutil::Op::Create>(is_active_path,
         active_node_identifier, zookeeper->getDefaultACL(), zkutil::CreateMode::Ephemeral));
-    ops.emplace_back(std::make_unique<zkutil::Op::SetData>(storage.replica_path + "/host", address.toString(), -1));
+    ops.emplace_back(std::make_shared<zkutil::Op::SetData>(storage.replica_path + "/host", address.toString(), -1));
 
     try
     {
diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp
index d619f37d696..ebdc5320ddf 100644
--- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp
@@ -480,25 +480,25 @@ void StorageReplicatedMergeTree::createTableIfNotExists()
     auto acl = zookeeper->getDefaultACL();
 
     zkutil::Ops ops;
-    ops.emplace_back(std::make_unique<zkutil::Op::Create>(zookeeper_path, "",
+    ops.emplace_back(std::make_shared<zkutil::Op::Create>(zookeeper_path, "",
         acl, zkutil::CreateMode::Persistent));
-    ops.emplace_back(std::make_unique<zkutil::Op::Create>(zookeeper_path + "/metadata", metadata,
+    ops.emplace_back(std::make_shared<zkutil::Op::Create>(zookeeper_path + "/metadata", metadata,
         acl, zkutil::CreateMode::Persistent));
     ops.emplace_back(std::make_unique<zkutil::Op::Create>(zookeeper_path + "/columns", getColumns().toString(),
         acl, zkutil::CreateMode::Persistent));
-    ops.emplace_back(std::make_unique<zkutil::Op::Create>(zookeeper_path + "/log", "",
+    ops.emplace_back(std::make_shared<zkutil::Op::Create>(zookeeper_path + "/log", "",
         acl, zkutil::CreateMode::Persistent));
-    ops.emplace_back(std::make_unique<zkutil::Op::Create>(zookeeper_path + "/blocks", "",
+    ops.emplace_back(std::make_shared<zkutil::Op::Create>(zookeeper_path + "/blocks", "",
         acl, zkutil::CreateMode::Persistent));
-    ops.emplace_back(std::make_unique<zkutil::Op::Create>(zookeeper_path + "/block_numbers", "",
+    ops.emplace_back(std::make_shared<zkutil::Op::Create>(zookeeper_path + "/block_numbers", "",
         acl, zkutil::CreateMode::Persistent));
-    ops.emplace_back(std::make_unique<zkutil::Op::Create>(zookeeper_path + "/nonincrement_block_numbers", "",
+    ops.emplace_back(std::make_shared<zkutil::Op::Create>(zookeeper_path + "/nonincrement_block_numbers", "",
         acl, zkutil::CreateMode::Persistent));
-    ops.emplace_back(std::make_unique<zkutil::Op::Create>(zookeeper_path + "/leader_election", "",
+    ops.emplace_back(std::make_shared<zkutil::Op::Create>(zookeeper_path + "/leader_election", "",
         acl, zkutil::CreateMode::Persistent));
-    ops.emplace_back(std::make_unique<zkutil::Op::Create>(zookeeper_path + "/temp", "",
+    ops.emplace_back(std::make_shared<zkutil::Op::Create>(zookeeper_path + "/temp", "",
         acl, zkutil::CreateMode::Persistent));
-    ops.emplace_back(std::make_unique<zkutil::Op::Create>(zookeeper_path + "/replicas", "",
+    ops.emplace_back(std::make_shared<zkutil::Op::Create>(zookeeper_path + "/replicas", "",
         acl, zkutil::CreateMode::Persistent));
 
     auto code = zookeeper->tryMulti(ops);
@@ -573,12 +573,12 @@ void StorageReplicatedMergeTree::createReplica()
     /// Create an empty replica. We'll create `columns` node at the end - we'll use it as a sign that replica creation is complete.
     auto acl = zookeeper->getDefaultACL();
     zkutil::Ops ops;
-    ops.emplace_back(std::make_unique<zkutil::Op::Create>(replica_path, "", acl, zkutil::CreateMode::Persistent));
-    ops.emplace_back(std::make_unique<zkutil::Op::Create>(replica_path + "/host", "", acl, zkutil::CreateMode::Persistent));
-    ops.emplace_back(std::make_unique<zkutil::Op::Create>(replica_path + "/log_pointer", "", acl, zkutil::CreateMode::Persistent));
-    ops.emplace_back(std::make_unique<zkutil::Op::Create>(replica_path + "/queue", "", acl, zkutil::CreateMode::Persistent));
-    ops.emplace_back(std::make_unique<zkutil::Op::Create>(replica_path + "/parts", "", acl, zkutil::CreateMode::Persistent));
-    ops.emplace_back(std::make_unique<zkutil::Op::Create>(replica_path + "/flags", "", acl, zkutil::CreateMode::Persistent));
+    ops.emplace_back(std::make_shared<zkutil::Op::Create>(replica_path, "", acl, zkutil::CreateMode::Persistent));
+    ops.emplace_back(std::make_shared<zkutil::Op::Create>(replica_path + "/host", "", acl, zkutil::CreateMode::Persistent));
+    ops.emplace_back(std::make_shared<zkutil::Op::Create>(replica_path + "/log_pointer", "", acl, zkutil::CreateMode::Persistent));
+    ops.emplace_back(std::make_shared<zkutil::Op::Create>(replica_path + "/queue", "", acl, zkutil::CreateMode::Persistent));
+    ops.emplace_back(std::make_shared<zkutil::Op::Create>(replica_path + "/parts", "", acl, zkutil::CreateMode::Persistent));
+    ops.emplace_back(std::make_shared<zkutil::Op::Create>(replica_path + "/flags", "", acl, zkutil::CreateMode::Persistent));
 
     try
     {
@@ -866,7 +866,7 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks)
         /// We assume that this occurs before the queue is loaded (queue.initialize).
         zkutil::Ops ops;
         removePartFromZooKeeper(name, ops);
-        ops.emplace_back(std::make_unique<zkutil::Op::Create>(
+        ops.emplace_back(std::make_shared<zkutil::Op::Create>(
             replica_path + "/queue/queue-", log_entry.toString(), zookeeper->getDefaultACL(), zkutil::CreateMode::PersistentSequential));
         zookeeper->multi(ops);
     }
@@ -931,20 +931,20 @@ void StorageReplicatedMergeTree::checkPartAndAddToZooKeeper(
 
     auto acl = zookeeper->getDefaultACL();
 
-    ops.emplace_back(std::make_unique<zkutil::Op::Check>(
+    ops.emplace_back(std::make_shared<zkutil::Op::Check>(
         zookeeper_path + "/columns",
         expected_columns_version));
-    ops.emplace_back(std::make_unique<zkutil::Op::Create>(
+    ops.emplace_back(std::make_shared<zkutil::Op::Create>(
         replica_path + "/parts/" + part_name,
         "",
         acl,
         zkutil::CreateMode::Persistent));
-    ops.emplace_back(std::make_unique<zkutil::Op::Create>(
+    ops.emplace_back(std::make_shared<zkutil::Op::Create>(
         replica_path + "/parts/" + part_name + "/columns",
         part->columns.toString(),
         acl,
         zkutil::CreateMode::Persistent));
-    ops.emplace_back(std::make_unique<zkutil::Op::Create>(
+    ops.emplace_back(std::make_shared<zkutil::Op::Create>(
         replica_path + "/parts/" + part_name + "/checksums",
         part->checksums.toString(),
         acl,
@@ -1307,7 +1307,7 @@ bool StorageReplicatedMergeTree::executeFetch(const StorageReplicatedMergeTree::
                     Stat stat;
                     String path = zookeeper_path + "/replicas/" + replicas[i] + "/host";
                     zookeeper->get(path, &stat);
-                    ops.emplace_back(std::make_unique<zkutil::Op::Check>(path, stat.version));
+                    ops.emplace_back(std::make_shared<zkutil::Op::Check>(path, stat.version));
                 }
 
                 /// We verify that while we were collecting versions, the replica with the necessary part did not come alive.
@@ -1326,7 +1326,7 @@ bool StorageReplicatedMergeTree::executeFetch(const StorageReplicatedMergeTree::
 
                     if (quorum_entry.part_name == entry.new_part_name)
                     {
-                        ops.emplace_back(std::make_unique<zkutil::Op::Remove>(quorum_path, quorum_stat.version));
+                        ops.emplace_back(std::make_shared<zkutil::Op::Remove>(quorum_path, quorum_stat.version));
 
                         auto part_info = MergeTreePartInfo::fromPartName(entry.new_part_name, data.format_version);
 
@@ -1338,13 +1338,13 @@ bool StorageReplicatedMergeTree::executeFetch(const StorageReplicatedMergeTree::
 
                         auto acl = zookeeper->getDefaultACL();
 
-                        ops.emplace_back(std::make_unique<zkutil::Op::Create>(
+                        ops.emplace_back(std::make_shared<zkutil::Op::Create>(
                             zookeeper_path + "/nonincrement_block_numbers/" + part_info.partition_id + "/block-" + padIndex(part_info.min_block),
                             "",
                             acl,
                             zkutil::CreateMode::Persistent));
 
-                        ops.emplace_back(std::make_unique<zkutil::Op::Create>(
+                        ops.emplace_back(std::make_shared<zkutil::Op::Create>(
                             zookeeper_path + "/quorum/failed_parts/" + entry.new_part_name,
                             "",
                             acl,
@@ -1352,7 +1352,7 @@ bool StorageReplicatedMergeTree::executeFetch(const StorageReplicatedMergeTree::
 
                         /// Deleting from `blocks`.
                         if (!entry.block_id.empty() && zookeeper->exists(zookeeper_path + "/blocks/" + entry.block_id))
-                            ops.emplace_back(std::make_unique<zkutil::Op::Remove>(zookeeper_path + "/blocks/" + entry.block_id, -1));
+                            ops.emplace_back(std::make_shared<zkutil::Op::Remove>(zookeeper_path + "/blocks/" + entry.block_id, -1));
 
                         auto code = zookeeper->tryMulti(ops);
 
@@ -1525,9 +1525,9 @@ void StorageReplicatedMergeTree::executeClearColumnInPartition(const LogEntry &
 
         /// Update part metadata in ZooKeeper.
         zkutil::Ops ops;
-        ops.emplace_back(std::make_unique<zkutil::Op::SetData>(
+        ops.emplace_back(std::make_shared<zkutil::Op::SetData>(
             replica_path + "/parts/" + part->name + "/columns", transaction->getNewColumns().toString(), -1));
-        ops.emplace_back(std::make_unique<zkutil::Op::SetData>(
+        ops.emplace_back(std::make_shared<zkutil::Op::SetData>(
             replica_path + "/parts/" + part->name + "/checksums", transaction->getNewChecksums().toString(), -1));
 
         zookeeper->multi(ops);
@@ -1970,9 +1970,9 @@ void StorageReplicatedMergeTree::removePartFromZooKeeper(const String & part_nam
 {
     String part_path = replica_path + "/parts/" + part_name;
 
-    ops.emplace_back(std::make_unique<zkutil::Op::Remove>(part_path + "/checksums", -1));
-    ops.emplace_back(std::make_unique<zkutil::Op::Remove>(part_path + "/columns", -1));
-    ops.emplace_back(std::make_unique<zkutil::Op::Remove>(part_path, -1));
+    ops.emplace_back(std::make_shared<zkutil::Op::Remove>(part_path + "/checksums", -1));
+    ops.emplace_back(std::make_shared<zkutil::Op::Remove>(part_path + "/columns", -1));
+    ops.emplace_back(std::make_shared<zkutil::Op::Remove>(part_path, -1));
 }
 
 
@@ -1989,7 +1989,7 @@ void StorageReplicatedMergeTree::removePartAndEnqueueFetch(const String & part_n
     log_entry->new_part_name = part_name;
 
     zkutil::Ops ops;
-    ops.emplace_back(std::make_unique<zkutil::Op::Create>(
+    ops.emplace_back(std::make_shared<zkutil::Op::Create>(
         replica_path + "/queue/queue-", log_entry->toString(), zookeeper->getDefaultACL(),
         zkutil::CreateMode::PersistentSequential));
 
@@ -2139,8 +2139,8 @@ void StorageReplicatedMergeTree::updateQuorum(const String & part_name)
             /// The quorum is reached. Delete the node, and update information about the last part that was successfully written with quorum.
 
             zkutil::Ops ops;
-            ops.emplace_back(std::make_unique<zkutil::Op::Remove>(quorum_status_path, stat.version));
-            ops.emplace_back(std::make_unique<zkutil::Op::SetData>(quorum_last_part_path, part_name, -1));
+            ops.emplace_back(std::make_shared<zkutil::Op::Remove>(quorum_status_path, stat.version));
+            ops.emplace_back(std::make_shared<zkutil::Op::SetData>(quorum_last_part_path, part_name, -1));
             auto code = zookeeper->tryMulti(ops);
 
             if (code == ZOK)

From f38dc334d68147a1b5981476930e4221996abace Mon Sep 17 00:00:00 2001
From: Vitaliy Lyudvichenko <vludv@yandex-team.ru>
Date: Wed, 14 Mar 2018 16:19:30 +0300
Subject: [PATCH 076/244] Refactor of asyncMulti. [#CLICKHOUSE-2]

---
 dbms/src/Common/ZooKeeper/ZooKeeper.cpp       | 31 ++++----
 .../gtest_zkutil_test_multi_exception.cpp     | 77 ++++++++++++-------
 2 files changed, 64 insertions(+), 44 deletions(-)

diff --git a/dbms/src/Common/ZooKeeper/ZooKeeper.cpp b/dbms/src/Common/ZooKeeper/ZooKeeper.cpp
index 8d3866a0008..746ed4c609a 100644
--- a/dbms/src/Common/ZooKeeper/ZooKeeper.cpp
+++ b/dbms/src/Common/ZooKeeper/ZooKeeper.cpp
@@ -978,26 +978,27 @@ ZooKeeper::TryRemoveFuture ZooKeeper::asyncTryRemove(const std::string & path, i
 
 ZooKeeper::MultiFuture ZooKeeper::asyncMultiImpl(const zkutil::Ops & ops_, bool throw_exception)
 {
-    size_t count = ops_.size();
-    auto results_native = std::make_shared<std::vector<zoo_op_result_t>>(count);
-
     /// We need to hold all references to ops data until the end of multi callback
     struct OpsHolder
     {
-        std::shared_ptr<zkutil::Ops> ops_ptr = std::make_shared<zkutil::Ops>();
-        std::shared_ptr<std::vector<zoo_op_t>> ops_raw_ptr = std::make_shared<std::vector<zoo_op_t>>();
+        std::shared_ptr<zkutil::Ops> ops_ptr;
+        std::shared_ptr<std::vector<zoo_op_t>> ops_native;
+        std::shared_ptr<std::vector<zoo_op_result_t>> op_results_native;
     } holder;
 
-    for (const auto & op : ops_)
-    {
-        holder.ops_ptr->emplace_back(op->clone());
-        holder.ops_raw_ptr->push_back(*holder.ops_ptr->back()->data);
-    }
+    /// Copy ops (swallow copy)
+    holder.ops_ptr = std::make_shared<zkutil::Ops>(ops_);
+    /// Copy native ops to contiguous vector
+    holder.ops_native = std::make_shared<std::vector<zoo_op_t>>();
+    for (const OpPtr & op : *holder.ops_ptr)
+        holder.ops_native->push_back(*op->data);
+    /// Allocate native result holders
+    holder.op_results_native = std::make_shared<std::vector<zoo_op_result_t>>(holder.ops_ptr->size());
 
-    MultiFuture future{ [throw_exception, results_native, holder, zookeeper=this] (int rc) {
+    MultiFuture future{ [throw_exception, holder, zookeeper=this] (int rc) {
         OpResultsAndCode res;
         res.code = rc;
-        convertOpResults(*results_native, res.results, zookeeper);
+        convertOpResults(*holder.op_results_native, res.results, zookeeper);
         res.ops_ptr = holder.ops_ptr;
         if (throw_exception && rc != ZOK)
             throw zkutil::KeeperException(rc);
@@ -1015,9 +1016,9 @@ ZooKeeper::MultiFuture ZooKeeper::asyncMultiImpl(const zkutil::Ops & ops_, bool
     if (expired())
         throw KeeperException(ZINVALIDSTATE);
 
-    auto & ops = *holder.ops_raw_ptr;
-
-    int32_t code = zoo_amulti(impl, static_cast<int>(ops.size()), ops.data(), results_native->data(),
+    int32_t code = zoo_amulti(impl, static_cast<int>(holder.ops_native->size()),
+                              holder.ops_native->data(),
+                              holder.op_results_native->data(),
                               [] (int rc, const void * data)
                               {
                                   MultiFuture::TaskPtr owned_task =
diff --git a/dbms/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp b/dbms/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp
index 4d1d05b54e1..695b9949a66 100644
--- a/dbms/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp
+++ b/dbms/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp
@@ -6,6 +6,9 @@
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wsign-compare"
 #include <gtest/gtest.h>
+#include <Common/ShellCommand.h>
+
+
 #pragma GCC diagnostic pop
 
 using namespace DB;
@@ -32,32 +35,32 @@ TEST(zkutil, multi_nice_exception_msg)
     zkutil::Ops ops;
 
     ASSERT_NO_THROW(
-        zookeeper->tryRemoveRecursive("/clickhouse_test_zkutil_multi");
+        zookeeper->tryRemoveRecursive("/clickhouse_test/zkutil_multi");
 
-        ops.emplace_back(new zkutil::Op::Create("/clickhouse_test_zkutil_multi", "_", acl, zkutil::CreateMode::Persistent));
-        ops.emplace_back(new zkutil::Op::Create("/clickhouse_test_zkutil_multi/a", "_", acl, zkutil::CreateMode::Persistent));
+        ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi", "_", acl, zkutil::CreateMode::Persistent));
+        ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi/a", "_", acl, zkutil::CreateMode::Persistent));
         zookeeper->multi(ops);
     );
 
     try
     {
         ops.clear();
-        ops.emplace_back(new zkutil::Op::Create("/clickhouse_test_zkutil_multi/c", "_", acl, zkutil::CreateMode::Persistent));
-        ops.emplace_back(new zkutil::Op::Remove("/clickhouse_test_zkutil_multi/c", -1));
-        ops.emplace_back(new zkutil::Op::Create("/clickhouse_test_zkutil_multi/a", "BadBoy", acl, zkutil::CreateMode::Persistent));
-        ops.emplace_back(new zkutil::Op::Create("/clickhouse_test_zkutil_multi/b", "_", acl, zkutil::CreateMode::Persistent));
-        ops.emplace_back(new zkutil::Op::Create("/clickhouse_test_zkutil_multi/a", "_", acl, zkutil::CreateMode::Persistent));
+        ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi/c", "_", acl, zkutil::CreateMode::Persistent));
+        ops.emplace_back(new zkutil::Op::Remove("/clickhouse_test/zkutil_multi/c", -1));
+        ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi/a", "BadBoy", acl, zkutil::CreateMode::Persistent));
+        ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi/b", "_", acl, zkutil::CreateMode::Persistent));
+        ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi/a", "_", acl, zkutil::CreateMode::Persistent));
 
         zookeeper->multi(ops);
         FAIL();
     }
     catch (...)
     {
-        zookeeper->tryRemoveRecursive("/clickhouse_test_zkutil_multi");
+        zookeeper->tryRemoveRecursive("/clickhouse_test/zkutil_multi");
 
         String msg = getCurrentExceptionMessage(false);
 
-        bool msg_has_reqired_patterns = msg.find("/clickhouse_test_zkutil_multi/a") != std::string::npos && msg.find("#2") != std::string::npos;
+        bool msg_has_reqired_patterns = msg.find("/clickhouse_test/zkutil_multi/a") != std::string::npos && msg.find("#2") != std::string::npos;
         EXPECT_TRUE(msg_has_reqired_patterns) << msg;
     }
 }
@@ -69,7 +72,7 @@ TEST(zkutil, multi_async)
     auto acl = zookeeper->getDefaultACL();
     zkutil::Ops ops;
 
-    zookeeper->tryRemoveRecursive("/clickhouse_test_zkutil_multi");
+    zookeeper->tryRemoveRecursive("/clickhouse_test/zkutil_multi");
 
     {
         ops.clear();
@@ -78,8 +81,8 @@ TEST(zkutil, multi_async)
 
     {
         ops.clear();
-        ops.emplace_back(new zkutil::Op::Create("/clickhouse_test_zkutil_multi", "", acl, zkutil::CreateMode::Persistent));
-        ops.emplace_back(new zkutil::Op::Create("/clickhouse_test_zkutil_multi/a", "", acl, zkutil::CreateMode::Persistent));
+        ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi", "", acl, zkutil::CreateMode::Persistent));
+        ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi/a", "", acl, zkutil::CreateMode::Persistent));
 
         auto fut = zookeeper->tryAsyncMulti(ops);
         ops.clear();
@@ -97,11 +100,11 @@ TEST(zkutil, multi_async)
         for (size_t i = 0; i < 10000; ++i)
         {
             ops.clear();
-            ops.emplace_back(new zkutil::Op::Remove("/clickhouse_test_zkutil_multi", -1));
-            ops.emplace_back(new zkutil::Op::Create("/clickhouse_test_zkutil_multi", "_", acl, zkutil::CreateMode::Persistent));
-            ops.emplace_back(new zkutil::Op::Check("/clickhouse_test_zkutil_multi", -1));
-            ops.emplace_back(new zkutil::Op::SetData("/clickhouse_test_zkutil_multi", "xxx", 42));
-            ops.emplace_back(new zkutil::Op::Create("/clickhouse_test_zkutil_multi/a", "_", acl, zkutil::CreateMode::Persistent));
+            ops.emplace_back(new zkutil::Op::Remove("/clickhouse_test/zkutil_multi", -1));
+            ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi", "_", acl, zkutil::CreateMode::Persistent));
+            ops.emplace_back(new zkutil::Op::Check("/clickhouse_test/zkutil_multi", -1));
+            ops.emplace_back(new zkutil::Op::SetData("/clickhouse_test/zkutil_multi", "xxx", 42));
+            ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi/a", "_", acl, zkutil::CreateMode::Persistent));
 
             futures.emplace_back(zookeeper->asyncMulti(ops));
         }
@@ -115,8 +118,8 @@ TEST(zkutil, multi_async)
 
     {
         ops.clear();
-        ops.emplace_back(new zkutil::Op::Create("/clickhouse_test_zkutil_multi", "_", acl, zkutil::CreateMode::Persistent));
-        ops.emplace_back(new zkutil::Op::Create("/clickhouse_test_zkutil_multi/a", "_", acl, zkutil::CreateMode::Persistent));
+        ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi", "_", acl, zkutil::CreateMode::Persistent));
+        ops.emplace_back(new zkutil::Op::Create("/clickhouse_test/zkutil_multi/a", "_", acl, zkutil::CreateMode::Persistent));
 
         auto fut = zookeeper->tryAsyncMulti(ops);
         ops.clear();
@@ -128,11 +131,30 @@ TEST(zkutil, multi_async)
     }
 }
 
+/// Run this test under sudo
+TEST(zkutil, multi_async_libzookeeper_segfault)
+{
+    auto zookeeper = std::make_unique<zkutil::ZooKeeper>("localhost:2181", "", 1000);
+    zkutil::Ops ops;
+
+    ops.emplace_back(new zkutil::Op::Check("/clickhouse_test/zkutil_multi", 0));
+
+    /// Uncomment to test
+    //auto cmd = ShellCommand::execute("sudo service zookeeper restart");
+    //cmd->wait();
+
+    auto future = zookeeper->asyncMulti(ops);
+    auto res = future.get();
+
+    EXPECT_TRUE(zkutil::isUnrecoverableErrorCode(res.code));
+}
+
 
 TEST(zkutil, multi_create_sequential)
 {
     try
     {
+        /// Create chroot node firstly
         auto zookeeper = std::make_unique<zkutil::ZooKeeper>("localhost:2181");
         zookeeper->createAncestors("/clickhouse_test/");
 
@@ -144,17 +166,14 @@ TEST(zkutil, multi_create_sequential)
         zookeeper->tryRemoveRecursive(base_path);
         zookeeper->createAncestors(base_path + "/");
 
-        String entry_path = base_path + "/queue-";
-        ops.emplace_back(new zkutil::Op::Create(entry_path, "", acl, zkutil::CreateMode::EphemeralSequential));
+        String sequential_node_prefix = base_path + "/queue-";
+        ops.emplace_back(new zkutil::Op::Create(sequential_node_prefix, "", acl, zkutil::CreateMode::EphemeralSequential));
         zkutil::OpResultsPtr results = zookeeper->multi(ops);
-        zkutil::OpResult & result = results->at(0);
+        zkutil::OpResult & sequential_node_result_op = results->at(0);
 
-        EXPECT_TRUE(result.value != nullptr);
-
-        String path = result.value;
-        std::cout << path << "\n";
-
-        EXPECT_TRUE(startsWith(result.value, entry_path));
+        EXPECT_FALSE(sequential_node_result_op.value.empty());
+        EXPECT_GT(sequential_node_result_op.value.length(), sequential_node_prefix.length());
+        EXPECT_EQ(sequential_node_result_op.value.substr(0, sequential_node_prefix.length()), sequential_node_prefix);
     }
     catch (...)
     {

From a2560045b39b64e580d5141581b20ed13790468b Mon Sep 17 00:00:00 2001
From: robot-metrika-test <robot-metrika-test@yandex-team.ru>
Date: Wed, 14 Mar 2018 16:50:31 +0300
Subject: [PATCH 077/244] Auto version update to [54365]

---
 dbms/cmake/version.cmake | 4 ++--
 debian/changelog         | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake
index 4f6d925f969..49bda24e075 100644
--- a/dbms/cmake/version.cmake
+++ b/dbms/cmake/version.cmake
@@ -1,6 +1,6 @@
 # This strings autochanged from release_lib.sh:
-set(VERSION_DESCRIBE v1.1.54364-testing)
-set(VERSION_REVISION 54364)
+set(VERSION_DESCRIBE v1.1.54365-testing)
+set(VERSION_REVISION 54365)
 # end of autochange
 
 set (VERSION_MAJOR 1)
diff --git a/debian/changelog b/debian/changelog
index 7623f7307d3..8ee6e628a17 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,5 @@
-clickhouse (1.1.54364) unstable; urgency=low
+clickhouse (1.1.54365) unstable; urgency=low
 
   * Modified source code
 
- --  <robot-metrika-test@yandex-team.ru>  Wed, 14 Mar 2018 10:03:38 +0300
+ --  <robot-metrika-test@yandex-team.ru>  Wed, 14 Mar 2018 16:50:31 +0300

From 93a693f84fed485c09c4fde8e61283431f6cbbd3 Mon Sep 17 00:00:00 2001
From: BayoNet <bayonet@yandex-team.ru>
Date: Wed, 14 Mar 2018 16:52:35 +0300
Subject: [PATCH 078/244] Data replication topic is actualized.

---
 docs/mkdocs_ru.yml                   |  2 +-
 docs/ru/table_engines/replication.md | 76 +++++++++++++++-------------
 2 files changed, 41 insertions(+), 37 deletions(-)

diff --git a/docs/mkdocs_ru.yml b/docs/mkdocs_ru.yml
index bc8b74a3d17..2d2bed6baca 100644
--- a/docs/mkdocs_ru.yml
+++ b/docs/mkdocs_ru.yml
@@ -84,7 +84,7 @@ pages:
   - 'AggregatingMergeTree': 'table_engines/aggregatingmergetree.md'
   - 'CollapsingMergeTree': 'table_engines/collapsingmergetree.md'
   - 'GraphiteMergeTree': 'table_engines/graphitemergetree.md'
-  - 'Data replication': 'table_engines/replication.md'
+  - 'Репликация данных': 'table_engines/replication.md'
   - 'Distributed': 'table_engines/distributed.md'
   - 'Dictionary': 'table_engines/dictionary.md'
   - 'Merge': 'table_engines/merge.md'
diff --git a/docs/ru/table_engines/replication.md b/docs/ru/table_engines/replication.md
index f76f7644a77..970e431370e 100644
--- a/docs/ru/table_engines/replication.md
+++ b/docs/ru/table_engines/replication.md
@@ -2,26 +2,28 @@
 
 # Репликация данных
 
-## ReplicatedAggregatingMergeTree
+Репликация поддерживается только для таблиц семейства MergeTree:
 
-## ReplicatedCollapsingMergeTree
+- ReplicatedAggregatingMergeTree
+- ReplicatedCollapsingMergeTree
+- ReplicatedGraphiteMergeTree
+- ReplicatedMergeTree
+- ReplicatedReplacingMergeTree
+- ReplicatedSummingMergeTree
 
-## ReplicatedGraphiteMergeTree
+Репликация работает на уровне отдельных таблиц, а не всего сервера. То есть, на сервере могут быть расположены одновременно реплицируемые и не реплицируемые таблицы.
 
-## ReplicatedMergeTree
+Репликация не зависит от шардирования. На каждом шарде репликация работает независимо.
 
-## ReplicatedReplacingMergeTree
+Реплицируются сжатые данные запросов `INSERT`, `ALTER` (см. подробности в описании запроса [ALTER](../query_language/queries.md#query_language_queries_alter)).
 
-## ReplicatedSummingMergeTree
+Запросы `CREATE`, `DROP`, `ATTACH`, `DETACH`, `RENAME` выполняются на одном сервере и не реплицируются:
 
-Репликация поддерживается только для таблиц семейства MergeTree. Репликация работает на уровне отдельных таблиц, а не всего сервера. То есть, на сервере могут быть расположены одновременно реплицируемые и не реплицируемые таблицы.
+- `CREATE TABLE` создаёт новую реплицируемую таблицу на том сервере, где выполняется запрос, а если на других серверах такая таблица уже есть - добавляет новую реплику.
+- `DROP TABLE` удаляет реплику, расположенную на том сервере, где выполняется запрос.
+- `RENAME` переименовывает таблицу на одной из реплик - то есть, реплицируемые таблицы на разных репликах могут называться по разному.
 
-Реплицируются INSERT, ALTER (см. подробности в описании запроса ALTER). Реплицируются сжатые данные, а не тексты запросов.
-Запросы CREATE, DROP, ATTACH, DETACH, RENAME не реплицируются - то есть, относятся к одному серверу. Запрос CREATE TABLE создаёт новую реплицируемую таблицу на том сервере, где выполняется запрос; а если на других серверах такая таблица уже есть - добавляет новую реплику. Запрос DROP TABLE удаляет реплику, расположенную на том сервере, где выполняется запрос. Запрос RENAME переименовывает таблицу на одной из реплик - то есть, реплицируемые таблицы на разных репликах могут называться по разному.
-
-Репликация никак не связана с шардированием. На каждом шарде репликация работает независимо.
-
-Репликация является опциональной возможностью. Для использования репликации, укажите в конфигурационном файле адреса ZooKeeper кластера. Пример:
+Чтобы использовать репликацию, укажите в конфигурационном файле адреса ZooKeeper кластера. Пример:
 
 ```xml
 <zookeeper>
@@ -40,25 +42,25 @@
 </zookeeper>
 ```
 
-**Используйте версию ZooKeeper не старее 3.4.5.** Для примера, в Ubuntu Precise слишком старая версия в пакете.
+Используйте ZooKeeper 3.4.5 и больше.
 
 Можно указать любой имеющийся у вас ZooKeeper-кластер - система будет использовать в нём одну директорию для своих данных (директория указывается при создании реплицируемой таблицы).
 
 Если в конфигурационном файле не настроен ZooKeeper, то вы не сможете создать реплицируемые таблицы, а уже имеющиеся реплицируемые таблицы будут доступны в режиме только на чтение.
 
-При запросах SELECT, ZooKeeper не используется. То есть, репликация никак не влияет на производительность SELECT-ов - запросы работают так же быстро, как и для нереплицируемых таблиц. При запросах к распределенным реплицированным таблицам поведение ClickHouse регулируется настройками [max_replica_delay_for_distributed_queries](../operations/settings/settings.md#settings_settings_max_replica_delay_for_distributed_queries) и [fallback_to_stale_replicas_for_distributed_queries](../operations/settings/settings.md#settings-settings-fallback_to_stale_replicas_for_distributed_queries).
+При запросах `SELECT`, ZooKeeper не используется, т.е. репликация не влияет на производительность `SELECT` и запросы работают так же быстро, как и для нереплицируемых таблиц. При запросах к распределенным реплицированным таблицам поведение ClickHouse регулируется настройками [max_replica_delay_for_distributed_queries](../operations/settings/settings.md#settings_settings_max_replica_delay_for_distributed_queries) и [fallback_to_stale_replicas_for_distributed_queries](../operations/settings/settings.md#settings-settings-fallback_to_stale_replicas_for_distributed_queries).
 
-При каждом запросе INSERT (точнее, на каждый вставляемый блок данных; запрос INSERT содержит один блок, или по блоку на каждые max_insert_block_size = 1048576 строк), делается около десятка записей в ZooKeeper в рамках нескольких транзакций. Это приводит к некоторому увеличению задержек при INSERT-е, по сравнению с нереплицируемыми таблицами. Но если придерживаться обычных рекомендаций - вставлять данные пачками не более одного INSERT-а в секунду, то это не составляет проблем. На всём кластере ClickHouse, использующим для координации один кластер ZooKeeper, может быть в совокупности несколько сотен INSERT-ов в секунду. Пропускная способность при вставке данных (количество строчек в секунду) такая же высокая, как для нереплицируемых таблиц.
+При каждом запросе `INSERT` (точнее, на каждый вставляемый блок данных; запрос INSERT содержит один блок, или по блоку на каждые `max_insert_block_size = 1048576` строк), делается около десятка записей в ZooKeeper в рамках нескольких транзакций. Это приводит к некоторому увеличению задержек при `INSERT`, по сравнению с нереплицируемыми таблицами. Но если придерживаться обычных рекомендаций - вставлять данные пачками не более одного `INSERT` в секунду, то это не составляет проблем. На всём кластере ClickHouse, использующим для координации один кластер ZooKeeper, может быть в совокупности несколько сотен `INSERT` в секунду. Пропускная способность при вставке данных (количество строчек в секунду) такая же высокая, как для нереплицируемых таблиц.
 
 Для очень больших кластеров, можно использовать разные кластеры ZooKeeper для разных шардов. Впрочем, на кластере Яндекс.Метрики (примерно 300 серверов) такой необходимости не возникает.
 
-Репликация асинхронная, мульти-мастер. Запросы INSERT (а также ALTER) можно отправлять на любой доступный сервер. Данные вставятся на этот сервер, а затем приедут на остальные серверы. В связи с асинхронностью, только что вставленные данные, появляются на остальных репликах с небольшой задержкой. Если часть реплик недоступна - данные на них запишутся тогда, когда они станут доступны. Если реплика доступна, то задержка составляет столько времени, сколько требуется для передачи блока сжатых данных по сети.
+Репликация асинхронная, мульти-мастер. Запросы `INSERT` (а также `ALTER`) можно отправлять на любой доступный сервер. Данные вставятся на сервер, где выполнен запрос, а затем скопируются на остальные серверы. В связи с асинхронностью, только что вставленные данные появляются на остальных репликах с небольшой задержкой. Если часть реплик недоступна, данные на них запишутся тогда, когда они станут доступны. Если реплика доступна, то задержка составляет столько времени, сколько требуется для передачи блока сжатых данных по сети.
 
-Кворумная запись отсутствует. То есть, вы не можете записать данные с подтверждением их получения более одной репликой. Если вы записали пачку данных на одну реплику, и данные ещё не успели разъехаться по остальным репликам, после чего сервер с этими данными перестал существовать, то эта пачка данных будет потеряна.
+Кворумная запись отсутствует. Вы можете записать данные с подтвержением только от одной реплики. Если вы записали данные на одну реплику, а сервер с этой репликой перестал существовать, то записанные данные будут потеряны, если репликация еще не завершилась.
 
-Каждый блок данных записывается атомарно. Запрос INSERT разбивается на блоки данных размером до max_insert_block_size = 1048576 строк. То есть, если в запросе INSERT менее 1048576 строк, то он делается атомарно.
+Каждый блок данных записывается атомарно. Запрос INSERT разбивается на блоки данных размером до `max_insert_block_size = 1048576` строк. То есть, если в запросе `INSERT` менее 1048576 строк, то он делается атомарно.
 
-Блоки данных дедуплицируются. При многократной записи одного и того же блока данных (блоков данных одинакового размера, содержащих одни и те же строчки в одном и том же порядке), блок будет записан только один раз. Это сделано для того, чтобы в случае сбоя в сети, когда клиентское приложение не может понять, были ли данные записаны в БД, можно было просто повторить запрос INSERT. При этом не имеет значения, на какую реплику будут отправлены INSERT-ы с одинаковыми данными. То есть, обеспечивается идемпотентность INSERT-ов. Это работает только для последних 100 вставленных в таблицу блоков.
+Блоки данных дедуплицируются. При многократной записи одного и того же блока данных (блоков данных одинакового размера, содержащих одни и те же строчки в одном и том же порядке), блок будет записан только один раз. Это сделано для того, чтобы в случае сбоя в сети, когда клиентское приложение не может понять, были ли данные записаны в БД, можно было просто повторить запрос `INSERT`. При этом не имеет значения, на какую реплику будут отправлены INSERT-ы с одинаковыми данными. То есть, обеспечивается идемпотентность `INSERT`. Параметры дедупликации регулируются настройками сервера [merge_tree](../operations/server_settings/settings.md#server_settings-merge_tree).
 
 При репликации, по сети передаются только исходные вставляемые данные. Дальнейшие преобразования данных (слияния) координируются и делаются на всех репликах одинаковым образом. За счёт этого минимизируется использование сети, и благодаря этому, репликация хорошо работает при расположении реплик в разных датацентрах. (Стоит заметить, что дублирование данных в разных датацентрах, по сути, является основной задачей репликации).
 
@@ -101,19 +103,21 @@ ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/hits', '{replica}', Even
 
 Имя реплики - то, что идентифицирует разные реплики одной и той же таблицы. Можно использовать для него имя сервера, как показано в примере. Впрочем, достаточно, чтобы имя было уникально лишь в пределах каждого шарда.
 
-Можно не использовать подстановки, а прописать всё явно. Это может быть удобным для тестирования и при настройке маленьких кластеров, но менее удобным при работе с большими кластерами.
+Можно не использовать подстановки, а указать соответствующие параметры явно. Это может быть удобным для тестирования и при настройке маленьких кластеров. Однако в этом случае нельзя пользоваться распределенными DDL-запросами (`ON CLUSTER`).
 
-Выполните запрос CREATE TABLE на каждой реплике. Запрос создаёт новую реплицируемую таблицу, или добавляет новую реплику к имеющимся.
+При работе с большими кластерами мы рекомендуем использовать подстановки, они уменьшают вероятность ошибки.
+
+Выполните запрос `CREATE TABLE` на каждой реплике. Запрос создаёт новую реплицируемую таблицу, или добавляет новую реплику к имеющимся.
 
 Если вы добавляете новую реплику после того, как таблица на других репликах уже содержит некоторые данные, то после выполнения запроса, данные на новую реплику будут скачаны с других реплик. То есть, новая реплика синхронизирует себя с остальными.
 
-Для удаления реплики, выполните запрос DROP TABLE. При этом, удаляется только одна реплика - расположенная на том сервере, где вы выполняете запрос.
+Для удаления реплики, выполните запрос `DROP TABLE`. При этом, удаляется только одна реплика - расположенная на том сервере, где вы выполняете запрос.
 
 ## Восстановление после сбоя
 
 Если при старте сервера, недоступен ZooKeeper, реплицируемые таблицы переходят в режим только для чтения. Система будет пытаться периодически установить соединение с ZooKeeper.
 
-Если при INSERT-е недоступен ZooKeeper, или происходит ошибка при взаимодействии с ним, будет выкинуто исключение.
+Если при `INSERT` недоступен ZooKeeper, или происходит ошибка при взаимодействии с ним, будет выкинуто исключение.
 
 При подключении к ZooKeeper, система проверяет соответствие между имеющимся в локальной файловой системе набором данных и ожидаемым набором данных (информация о котором хранится в ZooKeeper). Если имеются небольшие несоответствия, то система устраняет их, синхронизируя данные с реплик.
 
@@ -121,7 +125,7 @@ ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/hits', '{replica}', Even
 
 Стоит заметить, что ClickHouse не делает самостоятельно никаких деструктивных действий типа автоматического удаления большого количества данных.
 
-При старте сервера (или создании новой сессии с ZooKeeper), проверяется только количество и размеры всех файлов. Если у файлов совпадают размеры, но изменены байты где-то посередине, то это обнаруживается не сразу, а только при попытке их прочитать при каком-либо запросе SELECT - запрос кинет исключение о несоответствующей чексумме или размере сжатого блока. В этом случае, куски данных добавляются в очередь на проверку, и при необходимости, скачиваются с реплик.
+При старте сервера (или создании новой сессии с ZooKeeper), проверяется только количество и размеры всех файлов. Если у файлов совпадают размеры, но изменены байты где-то посередине, то это обнаруживается не сразу, а только при попытке их прочитать при каком-либо запросе `SELECT` - запрос кинет исключение о несоответствующей чексумме или размере сжатого блока. В этом случае, куски данных добавляются в очередь на проверку, и при необходимости, скачиваются с реплик.
 
 Если обнаруживается, что локальный набор данных слишком сильно отличается от ожидаемого, то срабатывает защитный механизм - сервер сообщает об этом в лог и отказывается запускаться. Это сделано, так как такой случай может свидетельствовать об ошибке конфигурации - например, если реплика одного шарда была случайно сконфигурирована, как реплика другого шарда. Тем не менее, пороги защитного механизма поставлены довольно низкими, и такая ситуация может возникнуть и при обычном восстановлении после сбоя. В этом случае, восстановление делается полуавтоматически - "по кнопке".
 
@@ -138,13 +142,13 @@ sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data
 Если на одном из серверов исчезли все данные и метаданные, восстановление делается следующим образом:
 
 1.  Установите на сервер ClickHouse. Корректно пропишите подстановки в конфигурационном файле, отвечающие за идентификатор шарда и реплики, если вы их используете.
-2.  Если у вас были нереплицируемые таблицы, которые должны быть вручную продублированы на серверах, скопируйте их данные (в директории /var/lib/clickhouse/data/db_name/table_name/) с реплики.
-3.  Скопируйте с реплики определения таблиц, находящиеся в /var/lib/clickhouse/metadata/. Если в определениях таблиц, идентификатор шарда или реплики, прописаны в явном виде - исправьте их, чтобы они соответствовали данной реплике. (Альтернативный вариант - запустить сервер и сделать самостоятельно все запросы ATTACH TABLE, которые должны были бы быть в соответствующих .sql файлах в /var/lib/clickhouse/metadata/.)
-4.  Создайте в ZooKeeper узел /path_to_table/replica_name/flags/force_restore_data с любым содержимым или выполните команду для восстановления всех реплицируемых таблиц: `sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data`
+2.  Если у вас были нереплицируемые таблицы, которые должны быть вручную продублированы на серверах, скопируйте их данные (в директории `/var/lib/clickhouse/data/db_name/table_name/`) с реплики.
+3.  Скопируйте с реплики определения таблиц, находящиеся в `/var/lib/clickhouse/metadata/`. Если в определениях таблиц, идентификатор шарда или реплики, прописаны в явном виде - исправьте их, чтобы они соответствовали данной реплике. (Альтернативный вариант - запустить сервер и сделать самостоятельно все запросы `ATTACH TABLE`, которые должны были бы быть в соответствующих .sql файлах в `/var/lib/clickhouse/metadata/`.)
+4.  Создайте в ZooKeeper узел `/path_to_table/replica_name/flags/force_restore_data` с любым содержимым или выполните команду для восстановления всех реплицируемых таблиц: `sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data`
 
 Затем запустите сервер (перезапустите, если уже запущен). Данные будут скачаны с реплик.
 
-В качестве альтернативного варианта восстановления, вы можете удалить из ZooKeeper информацию о потерянной реплике - `/path_to_table/replica_name`, и затем создать реплику заново, как написано в разделе "Создание реплицируемых таблиц".
+В качестве альтернативного варианта восстановления, вы можете удалить из ZooKeeper информацию о потерянной реплике - `/path_to_table/replica_name`, и затем создать реплику заново, как написано в разделе "[Создание реплицируемых таблиц](#table_engines-replication-creation_of_rep_tables)".
 
 Отсутствует ограничение на использование сетевой полосы при восстановлении. Имейте это ввиду, если восстанавливаете сразу много реплик.
 
@@ -152,24 +156,24 @@ sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data
 
 Здесь и далее, под `MergeTree` подразумеваются все движки таблиц семейства `MergeTree`, так же для `ReplicatedMergeTree`.
 
-Если у вас была таблица типа MergeTree, репликация которой делалась вручную, вы можете преобразовать её в реплицируемую таблицу. Это может понадобиться лишь в случаях, когда вы уже успели накопить большое количество данных в таблице типа MergeTree, а сейчас хотите включить репликацию.
+Если у вас была таблица типа `MergeTree`, репликация которой делалась вручную, вы можете преобразовать её в реплицируемую таблицу. Это может понадобиться лишь в случаях, когда вы уже успели накопить большое количество данных в таблице типа `MergeTree`, а сейчас хотите включить репликацию.
 
 Если на разных репликах данные отличаются, то сначала синхронизируйте их, либо удалите эти данные на всех репликах кроме одной.
 
-Переименуйте имеющуюся MergeTree таблицу, затем создайте со старым именем таблицу типа ReplicatedMergeTree.
+Переименуйте имеющуюся MergeTree таблицу, затем создайте со старым именем таблицу типа `ReplicatedMergeTree`.
 Перенесите данные из старой таблицы в поддиректорию detached в директории с данными новой таблицы (`/var/lib/clickhouse/data/db_name/table_name/`).
-Затем добавьте эти куски данных в рабочий набор с помощью выполнения запросов ALTER TABLE ATTACH PARTITION на одной из реплик.
+Затем добавьте эти куски данных в рабочий набор с помощью выполнения запросов `ALTER TABLE ATTACH PARTITION` на одной из реплик.
 
 ## Преобразование из ReplicatedMergeTree в MergeTree
 
-Создайте таблицу типа MergeTree с другим именем. Перенесите в её директорию с данными все данные из директории с данными таблицы типа ReplicatedMergeTree. Затем удалите таблицу типа ReplicatedMergeTree и перезапустите сервер.
+Создайте таблицу типа MergeTree с другим именем. Перенесите в её директорию с данными все данные из директории с данными таблицы типа `ReplicatedMergeTree`. Затем удалите таблицу типа `ReplicatedMergeTree` и перезапустите сервер.
 
-Если вы хотите избавиться от таблицы ReplicatedMergeTree, не запуская сервер, то
+Если вы хотите избавиться от таблицы `ReplicatedMergeTree`, не запуская сервер, то
 
--   удалите соответствующий файл .sql в директории с метаданными (`/var/lib/clickhouse/metadata/`);
+-   удалите соответствующий файл `.sql` в директории с метаданными (`/var/lib/clickhouse/metadata/`);
 -   удалите соответствующий путь в ZooKeeper (`/path_to_table/replica_name`);
 
-После этого, вы можете запустить сервер, создать таблицу типа MergeTree, перенести данные в её директорию, и перезапустить сервер.
+После этого, вы можете запустить сервер, создать таблицу типа `MergeTree`, перенести данные в её директорию, и перезапустить сервер.
 
 ## Восстановление в случае потери или повреждения метаданных на ZooKeeper кластере
 

From 56bfb05ebb70fa7fb2c0a54c6876219a49d89984 Mon Sep 17 00:00:00 2001
From: Vitaliy Lyudvichenko <vludv@yandex-team.ru>
Date: Wed, 14 Mar 2018 18:57:13 +0300
Subject: [PATCH 079/244] Fix insane DateTime in system.parts. [#CLICKHOUSE-3]

---
 dbms/src/Storages/MergeTree/MergeTreeDataPart.h | 4 +++-
 libs/libcommon/include/common/DateLUTImpl.h     | 5 ++++-
 libs/libcommon/src/DateLUTImpl.cpp              | 2 +-
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.h b/dbms/src/Storages/MergeTree/MergeTreeDataPart.h
index 43ddf7abfdd..cb9a5cf79f1 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.h
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.h
@@ -139,7 +139,9 @@ struct MergeTreeDataPart
     std::atomic<UInt64> size_in_bytes {0};  /// size in bytes, 0 - if not counted;
                                             ///  is used from several threads without locks (it is changed with ALTER).
     time_t modification_time = 0;
-    mutable std::atomic<time_t> remove_time { std::numeric_limits<time_t>::max() }; /// When the part is removed from the working set. Changes once.
+    /// When the part is removed from the working set. Changes once.
+    /// Use DATE_LUT_MAX which is almost 2^32 to produce correct time_t to String conversions using DateLUT
+    mutable std::atomic<time_t> remove_time { DATE_LUT_MAX };
 
     /// If true, the destructor will delete the directory with the part.
     bool is_temp = false;
diff --git a/libs/libcommon/include/common/DateLUTImpl.h b/libs/libcommon/include/common/DateLUTImpl.h
index 669f228cb34..754a83d4023 100644
--- a/libs/libcommon/include/common/DateLUTImpl.h
+++ b/libs/libcommon/include/common/DateLUTImpl.h
@@ -70,6 +70,7 @@ private:
     std::string time_zone;
 
 
+    /// We can correctly process only timestamps that less DATE_LUT_MAX (i.e. up to 2015 year inclusively)
     inline size_t findIndex(time_t t) const
     {
         /// First guess.
@@ -242,8 +243,10 @@ public:
     {
         size_t index = findIndex(t);
 
+        /// If it is not 1970 year (findIndex found nothing appropriate),
+        ///  than limit number of hours to avoid insane results like 1970-01-01 89:28:15
         if (unlikely(index == 0))
-            return (t + offset_at_start_of_epoch) / 3600;
+            return static_cast<unsigned>((t + offset_at_start_of_epoch) / 3600) % 24;
 
         time_t res = t - lut[index].date;
 
diff --git a/libs/libcommon/src/DateLUTImpl.cpp b/libs/libcommon/src/DateLUTImpl.cpp
index 570b277602c..5b3fc2a295e 100644
--- a/libs/libcommon/src/DateLUTImpl.cpp
+++ b/libs/libcommon/src/DateLUTImpl.cpp
@@ -64,7 +64,7 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_)
     {
         cctz::time_zone::civil_lookup lookup = cctz_time_zone.lookup(date);
 
-        start_of_day = std::chrono::system_clock::to_time_t(lookup.pre);    /// Ambiguouty is possible.
+        start_of_day = std::chrono::system_clock::to_time_t(lookup.pre);    /// Ambiguity is possible.
 
         Values & values = lut[i];
         values.year = date.year();

From 7abbe093e98bc21f60630d7246ef62ea9feb1dd6 Mon Sep 17 00:00:00 2001
From: Vitaliy Lyudvichenko <vludv@yandex-team.ru>
Date: Wed, 14 Mar 2018 19:04:19 +0300
Subject: [PATCH 080/244] Avoid magic constants in system.parts.
 [#CLICKHOUSE-3]

Resolves #2039
---
 dbms/src/Storages/MergeTree/MergeTreeDataPart.h | 3 +--
 dbms/src/Storages/System/StorageSystemParts.cpp | 4 +++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.h b/dbms/src/Storages/MergeTree/MergeTreeDataPart.h
index cb9a5cf79f1..95bed39eaec 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.h
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.h
@@ -140,8 +140,7 @@ struct MergeTreeDataPart
                                             ///  is used from several threads without locks (it is changed with ALTER).
     time_t modification_time = 0;
     /// When the part is removed from the working set. Changes once.
-    /// Use DATE_LUT_MAX which is almost 2^32 to produce correct time_t to String conversions using DateLUT
-    mutable std::atomic<time_t> remove_time { DATE_LUT_MAX };
+    mutable std::atomic<time_t> remove_time { std::numeric_limits<time_t>::max() };
 
     /// If true, the destructor will delete the directory with the part.
     bool is_temp = false;
diff --git a/dbms/src/Storages/System/StorageSystemParts.cpp b/dbms/src/Storages/System/StorageSystemParts.cpp
index 0914cae72da..a07e8bc26dd 100644
--- a/dbms/src/Storages/System/StorageSystemParts.cpp
+++ b/dbms/src/Storages/System/StorageSystemParts.cpp
@@ -66,7 +66,9 @@ void StorageSystemParts::processNextStorage(MutableColumns & columns, const Stor
         columns[i++]->insert(static_cast<UInt64>(part->rows_count));
         columns[i++]->insert(static_cast<UInt64>(part->size_in_bytes));
         columns[i++]->insert(static_cast<UInt64>(part->modification_time));
-        columns[i++]->insert(static_cast<UInt64>(part->remove_time.load(std::memory_order_relaxed)));
+
+        time_t remove_time = part->remove_time.load(std::memory_order_relaxed);
+        columns[i++]->insert(static_cast<UInt64>(remove_time == std::numeric_limits<time_t>::max() ? 0 : remove_time));
 
         /// For convenience, in returned refcount, don't add references that was due to local variables in this method: all_parts, active_parts.
         columns[i++]->insert(static_cast<UInt64>(part.use_count() - 1));

From bfc568741066ac684b8c189d285a963998dc6847 Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Wed, 14 Mar 2018 19:19:22 +0300
Subject: [PATCH 081/244] Update DateLUTImpl.h

---
 libs/libcommon/include/common/DateLUTImpl.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libs/libcommon/include/common/DateLUTImpl.h b/libs/libcommon/include/common/DateLUTImpl.h
index 754a83d4023..535c863eeb3 100644
--- a/libs/libcommon/include/common/DateLUTImpl.h
+++ b/libs/libcommon/include/common/DateLUTImpl.h
@@ -70,7 +70,7 @@ private:
     std::string time_zone;
 
 
-    /// We can correctly process only timestamps that less DATE_LUT_MAX (i.e. up to 2015 year inclusively)
+    /// We can correctly process only timestamps that less DATE_LUT_MAX (i.e. up to 2105 year inclusively)
     inline size_t findIndex(time_t t) const
     {
         /// First guess.

From fd4c0403ba82b8e80ca2589aaef8d3184a8bb430 Mon Sep 17 00:00:00 2001
From: proller <proller@github.com>
Date: Wed, 14 Mar 2018 19:37:56 +0300
Subject: [PATCH 082/244] Tests compile fix

---
 libs/libcommon/src/tests/CMakeLists.txt    | 1 +
 libs/libcommon/src/tests/multi_version.cpp | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/libs/libcommon/src/tests/CMakeLists.txt b/libs/libcommon/src/tests/CMakeLists.txt
index 56b9a927e76..68d8a7e68ce 100644
--- a/libs/libcommon/src/tests/CMakeLists.txt
+++ b/libs/libcommon/src/tests/CMakeLists.txt
@@ -15,6 +15,7 @@ target_link_libraries (date_lut3 common ${PLATFORM_LIBS})
 target_link_libraries (date_lut4 common ${PLATFORM_LIBS})
 target_link_libraries (date_lut_default_timezone common ${PLATFORM_LIBS})
 target_link_libraries (multi_version common)
+add_check(multi_version)
 
 add_executable (unit_tests_libcommon gtest_json_test.cpp gtest_strong_typedef.cpp)
 target_link_libraries (unit_tests_libcommon gtest_main common)
diff --git a/libs/libcommon/src/tests/multi_version.cpp b/libs/libcommon/src/tests/multi_version.cpp
index 94035360008..9cab2e095e4 100644
--- a/libs/libcommon/src/tests/multi_version.cpp
+++ b/libs/libcommon/src/tests/multi_version.cpp
@@ -19,7 +19,7 @@ void thread1(MV & x, T & result)
 
 void thread2(MV & x, const char * result)
 {
-    x.set(std::make_shared<T>(result));
+    x.set(std::make_unique<T>(result));
 }
 
 
@@ -31,7 +31,7 @@ int main(int argc, char ** argv)
         const char * s2 = "Goodbye!";
 
         size_t n = 1000;
-        MV x(std::make_shared<T>(s1));
+        MV x(std::make_unique<T>(s1));
         Results results(n);
 
         ThreadPool tp(8);

From 57279a87c7e654b4ef5d535586909ffea5687ed7 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Wed, 14 Mar 2018 19:40:08 +0300
Subject: [PATCH 083/244] Whitespaces [#CLICKHOUSE-2].

---
 dbms/src/Storages/StorageReplicatedMergeTree.cpp | 2 --
 1 file changed, 2 deletions(-)

diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp
index d619f37d696..1a63b0b52c5 100644
--- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp
@@ -773,7 +773,6 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks)
         unexpected_parts_rows += part->rows_count;
     }
 
-
     /// Additional helpful statistics
     auto get_blocks_count_in_data_part = [&] (const String & part_name) -> UInt64
     {
@@ -793,7 +792,6 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks)
     for (const String & name : expected_parts)
         expected_parts_blocks += get_blocks_count_in_data_part(name);
 
-
     std::stringstream sanity_report;
     sanity_report << "There are "
         << unexpected_parts.size() << " unexpected parts with " << unexpected_parts_rows << " rows ("

From d056d7e516443f97ef1d8eba6f6f037957612e24 Mon Sep 17 00:00:00 2001
From: proller <proller@users.noreply.github.com>
Date: Wed, 14 Mar 2018 19:57:15 +0300
Subject: [PATCH 084/244] Fix ssl tests (#2042)

* Revert "Reverted bad modification #2035"

This reverts commit 6d6eca5e23253e8677825dbd2039d897b9186f4d.

* spaces

* fix

* better

* Revert "Disabled ill-formed test #2035"

This reverts commit b28cef987132b8b90c50ac2b4c2c2c031d555504.

* debug

* fix

* fix

* fix

* fix

* fix

* fix

* clean

* Tests compile fix
---
 dbms/tests/CMakeLists.txt                            |  8 +++++++-
 dbms/tests/clickhouse-test-server                    |  2 +-
 dbms/tests/client-test.xml                           | 12 ++++++++++++
 .../{00505_tcp_ssl.sh.disabled => 00505_tcp_ssl.sh}  |  0
 dbms/tests/server-test.xml                           |  5 ++---
 debian/pbuilder-hooks/B90test-server                 | 12 ++++++++----
 debian/pbuilder-hooks/ssl.xml                        |  4 ----
 7 files changed, 30 insertions(+), 13 deletions(-)
 rename dbms/tests/queries/0_stateless/{00505_tcp_ssl.sh.disabled => 00505_tcp_ssl.sh} (100%)
 delete mode 100644 debian/pbuilder-hooks/ssl.xml

diff --git a/dbms/tests/CMakeLists.txt b/dbms/tests/CMakeLists.txt
index 30978652315..be51bcb8d69 100644
--- a/dbms/tests/CMakeLists.txt
+++ b/dbms/tests/CMakeLists.txt
@@ -25,6 +25,12 @@ add_subdirectory (external_dictionaries)
 if (ENABLE_TESTS)
     configure_file(${CMAKE_CURRENT_SOURCE_DIR}/CTestCustom.cmake ${CMAKE_BINARY_DIR})
 
+    if ($ENV{TEST_OPT})
+        set(TEST_OPT $ENV{TEST_OPT})
+    else ()
+        set(TEST_OPT "--skip compile")
+    endif ()
+
     # maybe add --no-long ?
-    add_test(NAME with_server COMMAND bash -c "env BUILD_DIR=${ClickHouse_BINARY_DIR} TEST_OPT='--skip compile' ${CMAKE_CURRENT_SOURCE_DIR}/clickhouse-test-server")
+    add_test(NAME with_server COMMAND bash -c "env BUILD_DIR=${ClickHouse_BINARY_DIR} TEST_OPT='${TEST_OPT}' ${CMAKE_CURRENT_SOURCE_DIR}/clickhouse-test-server")
 endif ()
diff --git a/dbms/tests/clickhouse-test-server b/dbms/tests/clickhouse-test-server
index b22058d5d7e..b3c9e4eaae5 100755
--- a/dbms/tests/clickhouse-test-server
+++ b/dbms/tests/clickhouse-test-server
@@ -23,7 +23,7 @@ export CLICKHOUSE_CONFIG=${CLICKHOUSE_CONFIG:=${CONFIG_SERVER_DIR}server-test.xm
 [ ! -d "$QUERIES_DIR" ] && QUERIES_DIR=${QUERIES_DIR=/usr/share/clickhouse-test/queries}
 
 rm -rf $DATA_DIR
-mkdir -p $LOG_DIR $DATA_DIR
+mkdir -p $LOG_DIR
 
 openssl dhparam -out `${BIN_DIR}clickhouse-extract-from-config --config=$CLICKHOUSE_CONFIG  --key=openSSL.server.dhParamsFile` 256
 openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout `${BIN_DIR}clickhouse-extract-from-config --config=$CLICKHOUSE_CONFIG  --key=openSSL.server.privateKeyFile` -out `${BIN_DIR}clickhouse-extract-from-config --config=$CLICKHOUSE_CONFIG  --key=openSSL.server.certificateFile`
diff --git a/dbms/tests/client-test.xml b/dbms/tests/client-test.xml
index c5478dcb80d..8747c773a66 100644
--- a/dbms/tests/client-test.xml
+++ b/dbms/tests/client-test.xml
@@ -2,4 +2,16 @@
 <yandex>
     <tcp_port>59000</tcp_port>
     <tcp_ssl_port>59440</tcp_ssl_port>
+    <openSSL>
+        <client>
+            <loadDefaultCAFile>true</loadDefaultCAFile>
+            <cacheSessions>true</cacheSessions>
+            <disableProtocols>sslv2,sslv3</disableProtocols>
+            <preferServerCiphers>true</preferServerCiphers>
+            <verificationMode>none</verificationMode>
+            <invalidCertificateHandler>
+                <name>AcceptCertificateHandler</name>
+            </invalidCertificateHandler>
+        </client>
+    </openSSL>
 </yandex>
diff --git a/dbms/tests/queries/0_stateless/00505_tcp_ssl.sh.disabled b/dbms/tests/queries/0_stateless/00505_tcp_ssl.sh
similarity index 100%
rename from dbms/tests/queries/0_stateless/00505_tcp_ssl.sh.disabled
rename to dbms/tests/queries/0_stateless/00505_tcp_ssl.sh
diff --git a/dbms/tests/server-test.xml b/dbms/tests/server-test.xml
index 57f3f8ad9ea..78e53e576e2 100644
--- a/dbms/tests/server-test.xml
+++ b/dbms/tests/server-test.xml
@@ -35,10 +35,9 @@
             <cacheSessions>true</cacheSessions>
             <disableProtocols>sslv2,sslv3</disableProtocols>
             <preferServerCiphers>true</preferServerCiphers>
-            <!-- Use for self-signed: <verificationMode>none</verificationMode> -->
+            <verificationMode>none</verificationMode>
             <invalidCertificateHandler>
-                <!-- Use for self-signed: <name>AcceptCertificateHandler</name> -->
-                <name>RejectCertificateHandler</name>
+                <name>AcceptCertificateHandler</name>
             </invalidCertificateHandler>
         </client>
     </openSSL>
diff --git a/debian/pbuilder-hooks/B90test-server b/debian/pbuilder-hooks/B90test-server
index bf0bbfbba63..e710cea45c1 100755
--- a/debian/pbuilder-hooks/B90test-server
+++ b/debian/pbuilder-hooks/B90test-server
@@ -16,8 +16,12 @@ ln -fs /usr/share/zoneinfo/Europe/Moscow /etc/localtime
 echo 'Europe/Moscow' > /etc/timezone
 dpkg-reconfigure -f noninteractive tzdata
 
-pwd
-mkdir -p /etc/clickhouse-server/config.d
+mkdir -p /etc/clickhouse-server/config.d /etc/clickhouse-client/config.d
+echo "<yandex><https_port>8443</https_port><tcp_ssl_port>9440</tcp_ssl_port></yandex>" > /etc/clickhouse-server/config.d/ssl.xml
+echo "<yandex><openSSL><client><verificationMode>none</verificationMode><invalidCertificateHandler><name>AcceptCertificateHandler</name></invalidCertificateHandler></client></openSSL></yandex>" > /etc/clickhouse-client/config.d/ssl.xml
+
+openssl dhparam -out /etc/clickhouse-server/dhparam.pem 256
+openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout /etc/clickhouse-server/server.key -out /etc/clickhouse-server/server.crt
 
 service clickhouse-server start
 sleep 3
@@ -25,8 +29,8 @@ sleep 3
 # TODO: remove me or make only on error:
 tail -n100 /var/log/clickhouse-server/*
 
-clickhouse-client -q "SELECT * from system.build_options;"
-clickhouse-client -q "SELECT toDateTime(1);"
+clickhouse-client --ssl -q "SELECT * from system.build_options;"
+clickhouse-client --ssl -q "SELECT toDateTime(1);"
 
 
 ( [ "${TEST_RUN}" ] && clickhouse-test --queries /usr/share/clickhouse-test/queries --tmp /tmp/clickhouse-test/ ${TEST_OPT} ) || ${TEST_TRUE:=true}
diff --git a/debian/pbuilder-hooks/ssl.xml b/debian/pbuilder-hooks/ssl.xml
deleted file mode 100644
index 8b4580a14d4..00000000000
--- a/debian/pbuilder-hooks/ssl.xml
+++ /dev/null
@@ -1,4 +0,0 @@
-<yandex>
-    <https_port>8443</https_port>
-    <tcp_ssl_port>9440</tcp_ssl_port>
-</yandex>

From 0ee9b8ee369928d37eb7e4367d9e315ff7379d69 Mon Sep 17 00:00:00 2001
From: Eugene Klimov <bloodjazman@gmail.com>
Date: Wed, 14 Mar 2018 23:20:33 +0500
Subject: [PATCH 085/244] remove xenial, trusty etc...

currently
---
 docs/ru/getting_started/index.md | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/ru/getting_started/index.md b/docs/ru/getting_started/index.md
index be4ce4f0333..3847663b3d5 100644
--- a/docs/ru/getting_started/index.md
+++ b/docs/ru/getting_started/index.md
@@ -16,7 +16,7 @@ grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not su
 
 В целях тестирования и разработки, система может быть установлена на один сервер или на рабочий компьютер.
 
-### Установка из пакетов
+### Установка из пакетов для Debian/Ubuntu 
 
 Пропишите в `/etc/apt/sources.list` (или в отдельный файл `/etc/apt/sources.list.d/clickhouse.list`) репозитории:
 
@@ -24,7 +24,6 @@ grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not su
 deb http://repo.yandex.ru/clickhouse/deb/stable/ main/
 ```
 
-На других версиях Ubuntu, замените `trusty` на `xenial` или `precise`.
 Если вы хотите использовать наиболее свежую тестовую версию, замените stable на testing.
 
 Затем выполните:

From 595bc175db44ef6d53bb988d7341e65318e76165 Mon Sep 17 00:00:00 2001
From: Eugene Klimov <bloodjazman@gmail.com>
Date: Wed, 14 Mar 2018 23:21:22 +0500
Subject: [PATCH 086/244] remove xenial, trusty etc.

currently one common package for debian\ubuntu
---
 docs/en/getting_started/index.md | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/en/getting_started/index.md b/docs/en/getting_started/index.md
index 3d81e359227..42fc1c75551 100644
--- a/docs/en/getting_started/index.md
+++ b/docs/en/getting_started/index.md
@@ -16,7 +16,7 @@ The terminal must use UTF-8 encoding (the default in Ubuntu).
 
 For testing and development, the system can be installed on a single server or on a desktop computer.
 
-### Installing from packages
+### Installing from packages Debian/Ubuntu
 
 In `/etc/apt/sources.list` (or in a separate `/etc/apt/sources.list.d/clickhouse.list` file), add the repository:
 
@@ -24,7 +24,6 @@ In `/etc/apt/sources.list` (or in a separate `/etc/apt/sources.list.d/clickhouse
 deb http://repo.yandex.ru/clickhouse/deb/stable/ main/
 ```
 
-On other versions of Ubuntu, replace `trusty` with `xenial` or `precise`.
 If you want to use the most recent test version, replace 'stable' with 'testing'.
 
 Then run:

From 597160443a7bd0f992178098daff256a70ecd56d Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Wed, 14 Mar 2018 21:58:40 +0300
Subject: [PATCH 087/244] Fixed build after merge #2036

---
 dbms/src/Common/ZooKeeper/ZooKeeper.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dbms/src/Common/ZooKeeper/ZooKeeper.h b/dbms/src/Common/ZooKeeper/ZooKeeper.h
index 2b0d195fd80..52f1968eba6 100644
--- a/dbms/src/Common/ZooKeeper/ZooKeeper.h
+++ b/dbms/src/Common/ZooKeeper/ZooKeeper.h
@@ -368,7 +368,7 @@ public:
 private:
     friend struct WatchContext;
     friend class EphemeralNodeHolder;
-    friend class OpResult;
+    friend struct OpResult;
 
     void init(const std::string & hosts, const std::string & identity, int32_t session_timeout_ms, const std::string & chroot);
 

From 6a965bb70727891b3948c52063137edcd4a54c09 Mon Sep 17 00:00:00 2001
From: proller <proller@users.noreply.github.com>
Date: Wed, 14 Mar 2018 22:38:48 +0300
Subject: [PATCH 088/244]  Client: use ConfigProcessor (#2049)

* Client: use ConfigProcessor
---
 dbms/src/Server/Client.cpp | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp
index 661d91e68d7..1580f45640b 100644
--- a/dbms/src/Server/Client.cpp
+++ b/dbms/src/Server/Client.cpp
@@ -28,6 +28,7 @@
 #include <Common/Throttler.h>
 #include <Common/StringUtils/StringUtils.h>
 #include <Common/typeid_cast.h>
+#include <Common/Config/ConfigProcessor.h>
 #include <Core/Types.h>
 #include <Core/QueryProcessingStage.h>
 #include <IO/ReadBufferFromFileDescriptor.h>
@@ -175,14 +176,22 @@ private:
         if (home_path_cstr)
             home_path = home_path_cstr;
 
+        std::string config_path;
         if (config().has("config-file"))
-            loadConfiguration(config().getString("config-file"));
+            config_path = config().getString("config-file");
         else if (Poco::File("./clickhouse-client.xml").exists())
-            loadConfiguration("./clickhouse-client.xml");
+            config_path = "./clickhouse-client.xml";
         else if (!home_path.empty() && Poco::File(home_path + "/.clickhouse-client/config.xml").exists())
-            loadConfiguration(home_path + "/.clickhouse-client/config.xml");
+            config_path = home_path + "/.clickhouse-client/config.xml";
         else if (Poco::File("/etc/clickhouse-client/config.xml").exists())
-            loadConfiguration("/etc/clickhouse-client/config.xml");
+            config_path = "/etc/clickhouse-client/config.xml";
+
+        if (!config_path.empty())
+        {
+            ConfigProcessor config_processor(config_path);
+            auto loaded_config = config_processor.loadConfig();
+            config().add(loaded_config.configuration);
+        }
 
         context.setApplicationType(Context::ApplicationType::CLIENT);
 

From 51301d38cd81b0b1757163023af92cf7275ea390 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Wed, 14 Mar 2018 22:56:44 +0300
Subject: [PATCH 089/244] added tryGetCreateTableQuery [#CLICKHOUSE-3590]

---
 dbms/src/Databases/DatabaseDictionary.cpp     |  5 +++++
 dbms/src/Databases/DatabaseDictionary.h       |  4 ++++
 dbms/src/Databases/DatabaseMemory.cpp         |  7 +++++++
 dbms/src/Databases/DatabaseMemory.h           |  4 ++++
 dbms/src/Databases/DatabaseOrdinary.cpp       | 21 +++++++++++++++----
 dbms/src/Databases/DatabaseOrdinary.h         |  5 +++++
 dbms/src/Databases/IDatabase.h                |  5 +++++
 .../Storages/System/StorageSystemTables.cpp   | 12 +----------
 8 files changed, 48 insertions(+), 15 deletions(-)

diff --git a/dbms/src/Databases/DatabaseDictionary.cpp b/dbms/src/Databases/DatabaseDictionary.cpp
index 6c14312c2e3..c68c801e4cb 100644
--- a/dbms/src/Databases/DatabaseDictionary.cpp
+++ b/dbms/src/Databases/DatabaseDictionary.cpp
@@ -182,6 +182,11 @@ ASTPtr DatabaseDictionary::getCreateTableQuery(
     return parseQuery(parser, query.data(), query.data() + query.size(), "");
 }
 
+ASTPtr DatabaseDictionary::tryGetCreateTableQuery(const Context & context, const String & table_name) const
+{
+    return getCreateTableQuery(context, table_name);
+}
+
 ASTPtr DatabaseDictionary::getCreateDatabaseQuery(const Context & /*context*/) const
 {
     String query;
diff --git a/dbms/src/Databases/DatabaseDictionary.h b/dbms/src/Databases/DatabaseDictionary.h
index f62bfc3e1c3..e2a036362b0 100644
--- a/dbms/src/Databases/DatabaseDictionary.h
+++ b/dbms/src/Databases/DatabaseDictionary.h
@@ -93,6 +93,10 @@ public:
         const Context & context,
         const String & table_name) const override;
 
+    ASTPtr tryGetCreateTableQuery(
+        const Context & context,
+        const String & table_name) const override;
+
     ASTPtr getCreateDatabaseQuery(const Context & context) const override;
 
     void shutdown() override;
diff --git a/dbms/src/Databases/DatabaseMemory.cpp b/dbms/src/Databases/DatabaseMemory.cpp
index b50fb78468e..8eacc7f856f 100644
--- a/dbms/src/Databases/DatabaseMemory.cpp
+++ b/dbms/src/Databases/DatabaseMemory.cpp
@@ -128,6 +128,13 @@ ASTPtr DatabaseMemory::getCreateTableQuery(
     throw Exception("There is no CREATE TABLE query for DatabaseMemory tables", ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY);
 }
 
+ASTPtr DatabaseMemory::tryGetCreateTableQuery(
+        const Context &,
+        const String &) const
+{
+    throw Exception("There is no CREATE TABLE query for DatabaseMemory tables", ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY);
+}
+
 ASTPtr DatabaseMemory::getCreateDatabaseQuery(const Context &) const
 {
     throw Exception("There is no CREATE DATABASE query for DatabaseMemory", ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY);
diff --git a/dbms/src/Databases/DatabaseMemory.h b/dbms/src/Databases/DatabaseMemory.h
index dbf37acb5f6..1b5f0a20c62 100644
--- a/dbms/src/Databases/DatabaseMemory.h
+++ b/dbms/src/Databases/DatabaseMemory.h
@@ -84,6 +84,10 @@ public:
             const Context & context,
             const String & table_name) const override;
 
+    ASTPtr tryGetCreateTableQuery(
+            const Context & context,
+            const String & table_name) const override;
+
     ASTPtr getCreateDatabaseQuery(const Context & context) const override;
 
     void shutdown() override;
diff --git a/dbms/src/Databases/DatabaseOrdinary.cpp b/dbms/src/Databases/DatabaseOrdinary.cpp
index 1f09c118666..4f63834d396 100644
--- a/dbms/src/Databases/DatabaseOrdinary.cpp
+++ b/dbms/src/Databases/DatabaseOrdinary.cpp
@@ -429,9 +429,7 @@ time_t DatabaseOrdinary::getTableMetadataModificationTime(
 }
 
 
-ASTPtr DatabaseOrdinary::getCreateTableQuery(
-    const Context & context,
-    const String & table_name) const
+ASTPtr DatabaseOrdinary::getCreateTableQueryImpl(const Context & context, const String & table_name, bool try_get) const
 {
     ASTPtr ast;
 
@@ -440,7 +438,12 @@ ASTPtr DatabaseOrdinary::getCreateTableQuery(
     if (!ast)
     {
         /// Handle system.* tables for which there are no table.sql files.
-        auto msg = tryGetTable(context, table_name)
+        bool has_table = tryGetTable(context, table_name) != nullptr;
+
+        if (has_table && try_get)
+            return nullptr;
+
+        auto msg = has_table
                    ? "There is no CREATE TABLE query for table "
                    : "There is no metadata file for table ";
 
@@ -450,6 +453,16 @@ ASTPtr DatabaseOrdinary::getCreateTableQuery(
     return ast;
 }
 
+ASTPtr DatabaseOrdinary::getCreateTableQuery(const Context & context, const String & table_name) const
+{
+    return getCreateTableQueryImpl(context, table_name, false);
+}
+
+ASTPtr DatabaseOrdinary::tryGetCreateTableQuery(const Context & context, const String & table_name) const
+{
+    return getCreateTableQueryImpl(context, table_name, true);
+}
+
 ASTPtr DatabaseOrdinary::getCreateDatabaseQuery(const Context & /*context*/) const
 {
     ASTPtr ast;
diff --git a/dbms/src/Databases/DatabaseOrdinary.h b/dbms/src/Databases/DatabaseOrdinary.h
index 9d8d2458df6..cae6686d5ba 100644
--- a/dbms/src/Databases/DatabaseOrdinary.h
+++ b/dbms/src/Databases/DatabaseOrdinary.h
@@ -59,6 +59,10 @@ public:
         const Context & context,
         const String & table_name) const override;
 
+    ASTPtr tryGetCreateTableQuery(
+            const Context & context,
+            const String & table_name) const override;
+
     ASTPtr getCreateDatabaseQuery(const Context & context) const override;
 
     String getDataPath() const override;
@@ -70,6 +74,7 @@ public:
 
 private:
     void startupTables(ThreadPool * thread_pool);
+    ASTPtr getCreateTableQueryImpl(const Context & context, const String & table_name, bool try_get) const;
 };
 
 }
diff --git a/dbms/src/Databases/IDatabase.h b/dbms/src/Databases/IDatabase.h
index 01c8970901d..6b84f3fa165 100644
--- a/dbms/src/Databases/IDatabase.h
+++ b/dbms/src/Databases/IDatabase.h
@@ -129,6 +129,11 @@ public:
         const Context & context,
         const String & name) const = 0;
 
+    /// Like getCreateTableQuery, but don't throw for existing tables without metadata.
+    virtual ASTPtr tryGetCreateTableQuery(
+        const Context & context,
+        const String & name) const = 0;
+
     /// Get the CREATE DATABASE query for current database.
     virtual ASTPtr getCreateDatabaseQuery(const Context & context) const = 0;
 
diff --git a/dbms/src/Storages/System/StorageSystemTables.cpp b/dbms/src/Storages/System/StorageSystemTables.cpp
index 493750d5dfa..77700c2a256 100644
--- a/dbms/src/Storages/System/StorageSystemTables.cpp
+++ b/dbms/src/Storages/System/StorageSystemTables.cpp
@@ -200,17 +200,7 @@ BlockInputStreams StorageSystemTables::read(
 
             if (has_create_table_query || has_engine_full)
             {
-                ASTPtr ast;
-
-                try
-                {
-                    ast = database->getCreateTableQuery(context, table_name);
-                }
-                catch (const Exception & e)
-                {
-                    if (e.code() != ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY)
-                        throw;
-                }
+                ASTPtr ast = database->tryGetCreateTableQuery(context, table_name);
 
                 if (has_create_table_query)
                     res_columns[j++]->insert(ast ? queryToString(ast) : "");

From f1b32e8982f1f5b5d2d19f8884183fa5eab085e6 Mon Sep 17 00:00:00 2001
From: Vitaliy Lyudvichenko <vludv@yandex-team.ru>
Date: Wed, 14 Mar 2018 19:43:18 +0300
Subject: [PATCH 090/244] Fixed ordinary MergeTree merges in case of empty
 ORDER BY key. [#CLICKHOUSE-2]

---
 dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp  | 12 ++++++++----
 .../00578_merge_trees_without_primary_key.sql        |  2 +-
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp
index 229c40404bb..ae8677c57be 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp
@@ -330,12 +330,12 @@ static void extractMergingAndGatheringColumns(const NamesAndTypesList & all_colu
     NamesAndTypesList & merging_columns, Names & merging_column_names
 )
 {
-    Names primary_key_columns_dup = primary_key_expressions->getRequiredColumns();
-    std::set<String> key_columns(primary_key_columns_dup.cbegin(), primary_key_columns_dup.cend());
+    Names primary_key_columns_vec = primary_key_expressions->getRequiredColumns();
+    std::set<String> key_columns(primary_key_columns_vec.cbegin(), primary_key_columns_vec.cend());
     if (secondary_key_expressions)
     {
-        Names secondary_key_columns_dup = secondary_key_expressions->getRequiredColumns();
-        key_columns.insert(secondary_key_columns_dup.begin(), secondary_key_columns_dup.end());
+        Names secondary_key_columns_vec = secondary_key_expressions->getRequiredColumns();
+        key_columns.insert(secondary_key_columns_vec.begin(), secondary_key_columns_vec.end());
     }
 
     /// Force sign column for Collapsing mode
@@ -350,6 +350,10 @@ static void extractMergingAndGatheringColumns(const NamesAndTypesList & all_colu
     if (merging_params.mode == MergeTreeData::MergingParams::VersionedCollapsing)
         key_columns.emplace(merging_params.sign_column);
 
+    /// Force to merge at least one column in case of empty key
+    if (key_columns.empty())
+        key_columns.emplace(all_columns.front().name);
+
     /// TODO: also force "summing" and "aggregating" columns to make Horizontal merge only for such columns
 
     for (auto & column : all_columns)
diff --git a/dbms/tests/queries/0_stateless/00578_merge_trees_without_primary_key.sql b/dbms/tests/queries/0_stateless/00578_merge_trees_without_primary_key.sql
index efd881ae006..2e1c0d24024 100644
--- a/dbms/tests/queries/0_stateless/00578_merge_trees_without_primary_key.sql
+++ b/dbms/tests/queries/0_stateless/00578_merge_trees_without_primary_key.sql
@@ -1,7 +1,7 @@
 SELECT '*** MergeTree ***';
 
 DROP TABLE IF EXISTS test.unsorted;
-CREATE TABLE test.unsorted (x UInt32, y String) ENGINE MergeTree ORDER BY tuple();
+CREATE TABLE test.unsorted (x UInt32, y String) ENGINE MergeTree ORDER BY tuple() SETTINGS vertical_merge_algorithm_min_rows_to_activate=0, vertical_merge_algorithm_min_columns_to_activate=0;
 
 INSERT INTO test.unsorted VALUES (1, 'a'), (5, 'b');
 INSERT INTO test.unsorted VALUES (2, 'c'), (4, 'd');

From 4011afa399593c2e207e07b03a441b88b30bf00f Mon Sep 17 00:00:00 2001
From: Vitaliy Lyudvichenko <vludv@yandex-team.ru>
Date: Wed, 14 Mar 2018 22:07:57 +0300
Subject: [PATCH 091/244] Parallel SELECT and INSERT in ClusterCopier.
 [#CLICKHOUSE-2]

---
 dbms/src/Server/ClusterCopier.cpp             | 26 ++++++++++++-------
 .../DistributedBlockOutputStream.cpp          |  5 +++-
 .../DistributedBlockOutputStream.h            |  2 +-
 3 files changed, 22 insertions(+), 11 deletions(-)

diff --git a/dbms/src/Server/ClusterCopier.cpp b/dbms/src/Server/ClusterCopier.cpp
index 077dc43c5f6..c38c81e73ed 100644
--- a/dbms/src/Server/ClusterCopier.cpp
+++ b/dbms/src/Server/ClusterCopier.cpp
@@ -20,6 +20,7 @@
 #include <Common/Exception.h>
 #include <Common/ZooKeeper/ZooKeeper.h>
 #include <Common/getFQDNOrHostName.h>
+#include <Common/isLocalAddress.h>
 #include <Client/Connection.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/Cluster.h>
@@ -37,7 +38,7 @@
 #include <Common/escapeForFileName.h>
 #include <Columns/ColumnString.h>
 #include <Columns/ColumnsNumber.h>
-#include <Storages/StorageDistributed.h>
+#include <DataTypes/DataTypeString.h>
 #include <Parsers/ParserCreateQuery.h>
 #include <Parsers/parseQuery.h>
 #include <Parsers/ParserQuery.h>
@@ -46,20 +47,20 @@
 #include <Parsers/ASTDropQuery.h>
 #include <Parsers/ASTLiteral.h>
 #include <Parsers/ASTExpressionList.h>
-#include <Databases/DatabaseMemory.h>
 #include <DataStreams/RemoteBlockInputStream.h>
 #include <DataStreams/SquashingBlockInputStream.h>
-#include <Common/isLocalAddress.h>
+#include <DataStreams/AsynchronousBlockInputStream.h>
 #include <DataStreams/copyData.h>
-#include <DataTypes/DataTypeString.h>
 #include <DataStreams/NullBlockOutputStream.h>
 #include <IO/Operators.h>
 #include <IO/ReadBufferFromString.h>
 #include <Functions/registerFunctions.h>
 #include <TableFunctions/registerTableFunctions.h>
 #include <AggregateFunctions/registerAggregateFunctions.h>
-#include <Server/StatusFile.h>
 #include <Storages/registerStorages.h>
+#include <Storages/StorageDistributed.h>
+#include <Databases/DatabaseMemory.h>
+#include <Server/StatusFile.h>
 #include <Common/formatReadable.h>
 #include <daemon/OwnPatternFormatter.h>
 
@@ -697,7 +698,7 @@ void DB::TaskCluster::reloadSettings(const Poco::Util::AbstractConfiguration & c
     settings_pull.load_balancing = LoadBalancing::NEAREST_HOSTNAME;
     settings_pull.readonly = 1;
     settings_pull.max_threads = 1;
-    settings_pull.max_block_size = std::min(8192UL, settings_pull.max_block_size.value);
+    settings_pull.max_block_size = settings_pull.max_block_size.changed ? settings_pull.max_block_size.value : 8192UL;
     settings_pull.preferred_block_size_bytes = 0;
 
     settings_push.insert_distributed_timeout = 0;
@@ -1609,8 +1610,15 @@ protected:
                 Context context_insert = context;
                 context_insert.getSettingsRef() = task_cluster->settings_push;
 
-                BlockIO io_select = InterpreterFactory::get(query_select_ast, context_select)->execute();
-                BlockIO io_insert = InterpreterFactory::get(query_insert_ast, context_insert)->execute();
+                BlockInputStreamPtr input;
+                BlockOutputStreamPtr output;
+                {
+                    BlockIO io_select = InterpreterFactory::get(query_select_ast, context_select)->execute();
+                    BlockIO io_insert = InterpreterFactory::get(query_insert_ast, context_insert)->execute();
+
+                    input = std::make_shared<AsynchronousBlockInputStream>(io_select.in);
+                    output = io_insert.out;
+                }
 
                 using ExistsFuture = zkutil::ZooKeeper::ExistsFuture;
                 auto future_is_dirty_checker = std::make_unique<ExistsFuture>(zookeeper->asyncExists(is_dirty_flag_path));
@@ -1665,7 +1673,7 @@ protected:
                 };
 
                 /// Main work is here
-                copyData(*io_select.in, *io_insert.out, cancel_check, update_stats);
+                copyData(*input, *output, cancel_check, update_stats);
 
                 // Just in case
                 if (future_is_dirty_checker != nullptr)
diff --git a/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp
index 8db08dfb09a..607eb9efb67 100644
--- a/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp
+++ b/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp
@@ -337,7 +337,10 @@ void DistributedBlockOutputStream::writeSuffix()
             throw;
         }
 
-        LOG_DEBUG(log, getCurrentStateDescription());
+        double elapsed = watch.elapsedSeconds();
+        LOG_DEBUG(log, "It took " << std::fixed << std::setprecision(1) << elapsed << " sec. to insert " << blocks_inserted << " blocks"
+                       << " (average " << std::fixed << std::setprecision(1) << elapsed / blocks_inserted * 1000 << " ms. per block)"
+                       << ". " << getCurrentStateDescription());
     }
 }
 
diff --git a/dbms/src/Storages/Distributed/DistributedBlockOutputStream.h b/dbms/src/Storages/Distributed/DistributedBlockOutputStream.h
index 199700d742c..eed38adf417 100644
--- a/dbms/src/Storages/Distributed/DistributedBlockOutputStream.h
+++ b/dbms/src/Storages/Distributed/DistributedBlockOutputStream.h
@@ -118,7 +118,7 @@ private:
 
     size_t remote_jobs_count = 0;
     size_t local_jobs_count = 0;
-
+    
     std::atomic<unsigned> finished_jobs_count{0};
 
     Poco::Logger * log;

From 1f1870abd41b5044b30c1a449660feb82cde4dee Mon Sep 17 00:00:00 2001
From: Vitaliy Lyudvichenko <vludv@yandex-team.ru>
Date: Wed, 14 Mar 2018 23:19:25 +0300
Subject: [PATCH 092/244] Add more priority to local ClusterCopier workers.
 [#CLICKHOUSE-2]

---
 dbms/src/Server/ClusterCopier.cpp             | 51 +++++++++++++++----
 .../test_cluster_copier/task0_description.xml |  2 +-
 2 files changed, 43 insertions(+), 10 deletions(-)

diff --git a/dbms/src/Server/ClusterCopier.cpp b/dbms/src/Server/ClusterCopier.cpp
index c38c81e73ed..4233ce70e3f 100644
--- a/dbms/src/Server/ClusterCopier.cpp
+++ b/dbms/src/Server/ClusterCopier.cpp
@@ -760,6 +760,7 @@ public:
 
         /// Do not initialize tables, will make deferred initialization in process()
 
+        getZooKeeper()->createAncestors(getWorkersPathVersion() + "/");
         getZooKeeper()->createAncestors(getWorkersPath() + "/");
     }
 
@@ -1012,32 +1013,65 @@ protected:
         return task_cluster->task_zookeeper_path + "/task_active_workers";
     }
 
+    String getWorkersPathVersion() const
+    {
+        return getWorkersPath() + "_version";
+    }
+
     String getCurrentWorkerNodePath() const
     {
         return getWorkersPath() + "/" + host_id;
     }
 
     zkutil::EphemeralNodeHolder::Ptr createTaskWorkerNodeAndWaitIfNeed(const zkutil::ZooKeeperPtr & zookeeper,
-                                                                       const String & description)
+                                                                       const String & description, bool unprioritized)
     {
+        std::chrono::milliseconds current_sleep_time = default_sleep_time;
+        static constexpr std::chrono::milliseconds max_sleep_time(30000); // 30 sec
+
+        if (unprioritized)
+            std::this_thread::sleep_for(current_sleep_time);
+
+        String workers_version_path = getWorkersPathVersion();
+        String workers_path = getWorkersPath();
+        String current_worker_path = getCurrentWorkerNodePath();
+
         while (true)
         {
             zkutil::Stat stat;
-            zookeeper->get(getWorkersPath(), &stat);
+            zookeeper->get(workers_version_path, &stat);
+            auto version = stat.version;
+            zookeeper->get(workers_path, &stat);
 
             if (static_cast<size_t>(stat.numChildren) >= task_cluster->max_workers)
             {
                 LOG_DEBUG(log, "Too many workers (" << stat.numChildren << ", maximum " << task_cluster->max_workers << ")"
                     << ". Postpone processing " << description);
-
-                std::this_thread::sleep_for(default_sleep_time);
-
-                updateConfigIfNeeded();
             }
             else
             {
-                return std::make_shared<zkutil::EphemeralNodeHolder>(getCurrentWorkerNodePath(), *zookeeper, true, false, description);
+                zkutil::Ops ops;
+                ops.emplace_back(new zkutil::Op::SetData(workers_version_path, description, version));
+                ops.emplace_back(new zkutil::Op::Create(current_worker_path, description, zookeeper->getDefaultACL(), zkutil::CreateMode::Ephemeral));
+                auto code = zookeeper->tryMulti(ops);
+
+                if (code == ZOK || code == ZNODEEXISTS)
+                    return std::make_shared<zkutil::EphemeralNodeHolder>(current_worker_path, *zookeeper, false, false, description);
+
+                if (code == ZBADVERSION)
+                {
+                    LOG_DEBUG(log, "A concurrent worker has just been added, will check free worker slots again");
+                }
+                else
+                    throw zkutil::KeeperException(code);
             }
+
+            if (unprioritized)
+                current_sleep_time = std::min(max_sleep_time, current_sleep_time + default_sleep_time);
+
+            std::this_thread::sleep_for(current_sleep_time);
+
+            updateConfigIfNeeded();
         }
     }
 
@@ -1445,9 +1479,8 @@ protected:
             return parseQuery(p_query, query);
         };
 
-
         /// Load balancing
-        auto worker_node_holder = createTaskWorkerNodeAndWaitIfNeed(zookeeper, current_task_status_path);
+        auto worker_node_holder = createTaskWorkerNodeAndWaitIfNeed(zookeeper, current_task_status_path, task_shard.priority.is_remote);
 
         LOG_DEBUG(log, "Processing " << current_task_status_path);
 
diff --git a/dbms/tests/integration/test_cluster_copier/task0_description.xml b/dbms/tests/integration/test_cluster_copier/task0_description.xml
index 0031a18c552..e8e4df99254 100644
--- a/dbms/tests/integration/test_cluster_copier/task0_description.xml
+++ b/dbms/tests/integration/test_cluster_copier/task0_description.xml
@@ -1,7 +1,7 @@
 <?xml version="1.0"?>
 <yandex>
     <!-- How many simualteneous workers are posssible -->
-    <max_workers>4</max_workers>
+    <max_workers>3</max_workers>
 
     <!-- Common setting for pull and push operations -->
     <settings>

From 9e9739ce2d8559b88171e83d9bb3e476864400cb Mon Sep 17 00:00:00 2001
From: Vitaliy Lyudvichenko <vludv@yandex-team.ru>
Date: Wed, 14 Mar 2018 23:54:16 +0300
Subject: [PATCH 093/244] Fixed a test. [#CLICKHOUSE-2]

---
 dbms/src/Common/ZooKeeper/tests/zkutil_expiration_test.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dbms/src/Common/ZooKeeper/tests/zkutil_expiration_test.cpp b/dbms/src/Common/ZooKeeper/tests/zkutil_expiration_test.cpp
index 54648bdad8f..073e482e109 100644
--- a/dbms/src/Common/ZooKeeper/tests/zkutil_expiration_test.cpp
+++ b/dbms/src/Common/ZooKeeper/tests/zkutil_expiration_test.cpp
@@ -38,7 +38,7 @@ int main(int argc, char ** argv)
                 ops.emplace_back(std::make_shared<zkutil::Op::Remove>("/test/zk_expiration_test", -1));
 
                 zkutil::MultiTransactionInfo info;
-                zk.tryMultiUnsafe(ops, info);
+                zk.tryMultiNoThrow(ops, nullptr, &info);
 
                 std::cout << time(nullptr) - time0 << "s: " << zkutil::ZooKeeper::error2string(info.code) << std::endl;
                 try

From d52f688883fe2a703dd83164dc172b76745d8e0f Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Thu, 15 Mar 2018 00:19:47 +0300
Subject: [PATCH 094/244] Attempt to fix regression: missing support for remote
 tables in IN section when querying Distributed tables [#CLICKHOUSE-2]

---
 dbms/src/Interpreters/ExpressionAnalyzer.cpp  |  5 +++++
 .../Interpreters/InterpreterSelectQuery.cpp   |  3 ++-
 dbms/src/Storages/IStorage.h                  | 14 +++++++++++++
 dbms/src/Storages/StorageDistributed.cpp      | 20 +++++++++++++++++++
 dbms/src/Storages/StorageDistributed.h        |  6 ++++++
 5 files changed, 47 insertions(+), 1 deletion(-)

diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp
index f0bc8137029..5360e750a33 100644
--- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp
+++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp
@@ -61,6 +61,8 @@
 #include <DataTypes/DataTypeFunction.h>
 #include <Functions/FunctionsMiscellaneous.h>
 
+#include <Core/iostream_debug_helpers.h>
+
 
 namespace DB
 {
@@ -2438,6 +2440,9 @@ bool ExpressionAnalyzer::appendWhere(ExpressionActionsChain & chain, bool only_t
     initChain(chain, source_columns);
     ExpressionActionsChain::Step & step = chain.steps.back();
 
+    DUMP(only_types);
+    DUMP(StackTrace().toString());
+
     step.required_output.push_back(select_query->where_expression->getColumnName());
     getRootActions(select_query->where_expression, only_types, false, step.actions);
 
diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp
index 49b3ba27004..1f6aeb9d3c5 100644
--- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp
@@ -676,7 +676,8 @@ QueryProcessingStage::Enum InterpreterSelectQuery::executeFetchColumns(Pipeline
             pipeline.streams = storage->read(required_columns, query_info, context, from_stage, max_block_size, max_streams);
 
         if (pipeline.streams.empty())
-            pipeline.streams.emplace_back(std::make_shared<NullBlockInputStream>(storage->getSampleBlockForColumns(required_columns)));
+            pipeline.streams.emplace_back(std::make_shared<NullBlockInputStream>(
+                storage->analyze(required_columns, query_info, context, from_stage)));
 
         pipeline.transform([&](auto & stream)
         {
diff --git a/dbms/src/Storages/IStorage.h b/dbms/src/Storages/IStorage.h
index ef9fd8004ce..b30aeb2bfb8 100644
--- a/dbms/src/Storages/IStorage.h
+++ b/dbms/src/Storages/IStorage.h
@@ -180,6 +180,20 @@ public:
         throw Exception("Method read is not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED);
     }
 
+    /** Without executing a query determine,
+      *  what structure of result and what query processed stage
+      *  we will get if we will call 'read' method.
+      */
+    virtual Block analyze(
+        const Names & column_names,
+        const SelectQueryInfo & /*query_info*/,
+        const Context & /*context*/,
+        QueryProcessingStage::Enum & processed_stage)
+    {
+        processed_stage = QueryProcessingStage::FetchColumns;
+        return getSampleBlockForColumns(column_names);
+    }
+
     /** Writes the data to a table.
       * Receives a description of the query, which can contain information about the data write method.
       * Returns an object by which you can write data sequentially.
diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp
index 2b1dfe66523..f73f23e5a13 100644
--- a/dbms/src/Storages/StorageDistributed.cpp
+++ b/dbms/src/Storages/StorageDistributed.cpp
@@ -197,6 +197,26 @@ BlockInputStreams StorageDistributed::read(
 }
 
 
+Block StorageDistributed::analyze(
+    const Names & /*column_names*/,
+    const SelectQueryInfo & query_info,
+    const Context & context,
+    QueryProcessingStage::Enum & processed_stage)
+{
+    auto cluster = getCluster();
+
+    const Settings & settings = context.getSettingsRef();
+
+    size_t result_size = (cluster->getRemoteShardCount() * settings.max_parallel_replicas) + cluster->getLocalShardCount();
+
+    processed_stage = result_size == 1 || settings.distributed_group_by_no_merge
+        ? QueryProcessingStage::Complete
+        : QueryProcessingStage::WithMergeableState;
+
+    return materializeBlock(InterpreterSelectQuery(query_info.query, context, {}, processed_stage).getSampleBlock());
+}
+
+
 BlockOutputStreamPtr StorageDistributed::write(const ASTPtr & query, const Settings & settings)
 {
     auto cluster = (owned_cluster) ? owned_cluster : context.getCluster(cluster_name);
diff --git a/dbms/src/Storages/StorageDistributed.h b/dbms/src/Storages/StorageDistributed.h
index 451c9f12d8a..63fefbfe501 100644
--- a/dbms/src/Storages/StorageDistributed.h
+++ b/dbms/src/Storages/StorageDistributed.h
@@ -60,6 +60,12 @@ public:
         size_t max_block_size,
         unsigned num_streams) override;
 
+    Block analyze(
+        const Names & column_names,
+        const SelectQueryInfo & query_info,
+        const Context & context,
+        QueryProcessingStage::Enum & processed_stage) override;
+
     BlockOutputStreamPtr write(const ASTPtr & query, const Settings & settings) override;
 
     void drop() override {}

From 56947de18a120d0522f7c359866b2feb8c3dc062 Mon Sep 17 00:00:00 2001
From: proller <proller@users.noreply.github.com>
Date: Thu, 15 Mar 2018 02:04:35 +0300
Subject: [PATCH 095/244] Try fix ssl (#2051)

* Revert "Reverted bad modification #2035"

This reverts commit 6d6eca5e23253e8677825dbd2039d897b9186f4d.

* spaces

* fix

* better

* Revert "Disabled ill-formed test #2035"

This reverts commit b28cef987132b8b90c50ac2b4c2c2c031d555504.

* debug

* fix

* fix

* fix

* fix

* fix

* fix

* clean

* Tests compile fix

* fix

* Try fix ssl

* version

* Test fixes, ./release now uses pbuilder by default

* fix
---
 debian/pbuilder-hooks/B90test-server | 17 +++++++++++------
 release                              |  5 +++--
 utils/travis/normal.sh               |  8 +++++---
 utils/travis/pbuilder.sh             |  7 +++++--
 4 files changed, 24 insertions(+), 13 deletions(-)

diff --git a/debian/pbuilder-hooks/B90test-server b/debian/pbuilder-hooks/B90test-server
index e710cea45c1..fc7c0a04167 100755
--- a/debian/pbuilder-hooks/B90test-server
+++ b/debian/pbuilder-hooks/B90test-server
@@ -17,11 +17,16 @@ echo 'Europe/Moscow' > /etc/timezone
 dpkg-reconfigure -f noninteractive tzdata
 
 mkdir -p /etc/clickhouse-server/config.d /etc/clickhouse-client/config.d
-echo "<yandex><https_port>8443</https_port><tcp_ssl_port>9440</tcp_ssl_port></yandex>" > /etc/clickhouse-server/config.d/ssl.xml
-echo "<yandex><openSSL><client><verificationMode>none</verificationMode><invalidCertificateHandler><name>AcceptCertificateHandler</name></invalidCertificateHandler></client></openSSL></yandex>" > /etc/clickhouse-client/config.d/ssl.xml
 
-openssl dhparam -out /etc/clickhouse-server/dhparam.pem 256
-openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout /etc/clickhouse-server/server.key -out /etc/clickhouse-server/server.crt
+TEST_SSL=${TEST_SSL=1}
+if  [ "${TEST_SSL}" ]; then
+    echo "<yandex><https_port>8443</https_port><tcp_ssl_port>9440</tcp_ssl_port></yandex>" > /etc/clickhouse-server/config.d/ssl.xml
+    echo "<yandex><openSSL><client><verificationMode>none</verificationMode><invalidCertificateHandler><name>AcceptCertificateHandler</name></invalidCertificateHandler></client></openSSL></yandex>" > /etc/clickhouse-client/config.d/ssl.xml
+
+    openssl dhparam -out /etc/clickhouse-server/dhparam.pem 256
+    openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout /etc/clickhouse-server/server.key -out /etc/clickhouse-server/server.crt
+    CLIENT_ADD="--ssl"
+fi
 
 service clickhouse-server start
 sleep 3
@@ -29,8 +34,8 @@ sleep 3
 # TODO: remove me or make only on error:
 tail -n100 /var/log/clickhouse-server/*
 
-clickhouse-client --ssl -q "SELECT * from system.build_options;"
-clickhouse-client --ssl -q "SELECT toDateTime(1);"
+clickhouse-client -q "SELECT * from system.build_options;"
+clickhouse-client ${CLIENT_ADD} -q "SELECT toDateTime(1);"
 
 
 ( [ "${TEST_RUN}" ] && clickhouse-test --queries /usr/share/clickhouse-test/queries --tmp /tmp/clickhouse-test/ ${TEST_OPT} ) || ${TEST_TRUE:=true}
diff --git a/release b/release
index 6c36a594cfe..fb0ce279a9d 100755
--- a/release
+++ b/release
@@ -13,6 +13,7 @@ DEB_CXX=${DEB_CXX:=g++-7}
 CONTROL=debian/control
 DEBUILD_NOSIGN_OPTIONS="-us -uc"
 DEBUILD_NODEPS_OPTIONS="-d"
+USE_PBUILDER=1
 
 if [ -z "$REVISION" ] ; then
     get_revision_author
@@ -34,8 +35,8 @@ do
     elif [[ $1 == '--head' ]]; then
         REVISION=`git rev-parse HEAD`
         shift
-    elif [[ $1 == '--pbuilder' ]]; then
-        USE_PBUILDER=1
+    elif [[ $1 == '--no-pbuilder' ]]; then
+        USE_PBUILDER=
         shift
     elif [[ $1 == '--fast' ]]; then
         # Wrong but fast pbuilder mode: create base package with all depends
diff --git a/utils/travis/normal.sh b/utils/travis/normal.sh
index 6f94981b735..ba5b294535f 100755
--- a/utils/travis/normal.sh
+++ b/utils/travis/normal.sh
@@ -1,9 +1,11 @@
-#!/bin/sh
+#!/bin/bash
 
 # Manual run:
 # env CXX=g++-7 CC=gcc-7 utils/travis/normal.sh
 # env CXX=clang++-5.0 CC=clang-5.0 utils/travis/normal.sh
 
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+
 set -e
 set -x
 
@@ -20,7 +22,7 @@ date
 
 mkdir -p build
 cd build
-cmake .. -DCMAKE_CXX_COMPILER=`which $DEB_CXX $CXX` -DCMAKE_C_COMPILER=`which $DEB_CC $CC` \
+cmake $CUR_DIR/../.. -DCMAKE_CXX_COMPILER=`which $DEB_CXX $CXX` -DCMAKE_C_COMPILER=`which $DEB_CC $CC` \
     `# Does not optimize to speedup build, skip debug info to use less disk` \
     -DCMAKE_C_FLAGS_ADD="-O0 -g0" -DCMAKE_CXX_FLAGS_ADD="-O0 -g0" \
     `# ignore ccache disabler on trusty` \
@@ -33,6 +35,6 @@ cmake .. -DCMAKE_CXX_COMPILER=`which $DEB_CXX $CXX` -DCMAKE_C_COMPILER=`which $D
     `# Skip tests:` \
     `# 00281 requires internal compiler` \
     `# 00428 requires sudo (not all vms allow this)` \
-    && ( [ ${TEST_RUN=1} ] && ( ( cd .. && env TEST_OPT="--skip long compile 00428 $TEST_OPT" bash -x dbms/tests/clickhouse-test-server ) || ${TEST_TRUE=false} ) || true )
+    && ( [ ${TEST_RUN=1} ] && ( ( cd $CUR_DIR/../.. && env TEST_OPT="--skip long compile 00428 $TEST_OPT" bash -x dbms/tests/clickhouse-test-server ) || ${TEST_TRUE=false} ) || true )
 
 date
diff --git a/utils/travis/pbuilder.sh b/utils/travis/pbuilder.sh
index 635db59fdaf..9d5cce6a366 100755
--- a/utils/travis/pbuilder.sh
+++ b/utils/travis/pbuilder.sh
@@ -1,4 +1,6 @@
-#!/bin/sh
+#!/bin/bash
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 # env CXX=clang++-5.0 CC=clang-5.0 DH_VERBOSE=1 utils/travis/pbuilder.sh
 
@@ -14,6 +16,7 @@ env TEST_RUN=${TEST_RUN=1} \
     `# 00281 requires internal compiler` \
     `# 00416 requires patched poco from contrib/` \
     TEST_OPT="--skip long pocopatch $TEST_OPT" \
+    TEST_SSL="" `# <Error> Application: SSL context exception: Error loading certificate from file /etc/clickhouse-server/server.crt: No error -- when using system poco on artful` \
     TEST_TRUE=${TEST_TRUE=false} \
     `# travisci will not upload ccache cache after timeout (48min), use our less timeout` \
     PBUILDER_OPT="--timeout ${PBUILDER_TIMEOUT:=35m} $PBUILDER_OPT" \
@@ -25,6 +28,6 @@ env TEST_RUN=${TEST_RUN=1} \
     `# Use all possible contrib libs from system` \
     `# psmisc - killall` \
     EXTRAPACKAGES="psmisc clang-5.0 lld-5.0 liblld-5.0-dev libclang-5.0-dev liblld-5.0 libc++abi-dev libc++-dev libboost-program-options-dev libboost-system-dev libboost-filesystem-dev libboost-thread-dev zlib1g-dev liblz4-dev libdouble-conversion-dev libzookeeper-mt-dev libsparsehash-dev librdkafka-dev libpoco-dev libsparsehash-dev libgoogle-perftools-dev libzstd-dev libre2-dev $EXTRAPACKAGES" \
-    ./release --pbuilder $RELEASE_OPT
+    $CUR_DIR/../../release $RELEASE_OPT
 
 date

From b39fa2027d415a123ce87fb292508138f77dfadf Mon Sep 17 00:00:00 2001
From: proller <proller@github.com>
Date: Thu, 15 Mar 2018 12:41:06 +0300
Subject: [PATCH 096/244] release: ignore --pbuilder flag

---
 release | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/release b/release
index fb0ce279a9d..ae19a8ada46 100755
--- a/release
+++ b/release
@@ -35,6 +35,9 @@ do
     elif [[ $1 == '--head' ]]; then
         REVISION=`git rev-parse HEAD`
         shift
+    elif [[ $1 == '--pbuilder' ]]; then
+        # Default
+        shift
     elif [[ $1 == '--no-pbuilder' ]]; then
         USE_PBUILDER=
         shift

From b1967ff0d50977483e9c61f8ae4710e0314616f5 Mon Sep 17 00:00:00 2001
From: Valera Ryaboshapko <valera@creator.su>
Date: Thu, 15 Mar 2018 14:48:19 +0300
Subject: [PATCH 097/244] Update mergetree.md

---
 docs/ru/table_engines/mergetree.md | 36 +++++++++++++++---------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/docs/ru/table_engines/mergetree.md b/docs/ru/table_engines/mergetree.md
index 4da83967e26..714d368909d 100644
--- a/docs/ru/table_engines/mergetree.md
+++ b/docs/ru/table_engines/mergetree.md
@@ -2,45 +2,45 @@
 
 # MergeTree
 
-Движок MergeTree поддерживает индекс по первичному ключу и по дате, и обеспечивает возможность обновления данных в реальном времени.
+Движок MergeTree поддерживает индекс по первичному ключу и по дате и обеспечивает возможность обновления данных в реальном времени.
 Это наиболее продвинутый движок таблиц в ClickHouse. Не путайте с движком Merge.
 
-Движок принимает параметры: имя столбца типа Date, содержащего дату; выражение для семплирования (не обязательно); кортеж, определяющий первичный ключ таблицы; гранулированность индекса. Пример:
+Движок принимает параметры: имя столбца типа Date, содержащего дату; выражение для семплирования (не обязательно); кортеж, определяющий первичный ключ таблицы; гранулированность индекса.
 
-Пример без поддержки сэмплирования:
+Пример без поддержки сэмплирования.
 
 ```text
 MergeTree(EventDate, (CounterID, EventDate), 8192)
 ```
 
-Пример с поддержкой сэмплирования:
+Пример с поддержкой сэмплирования.
 
 ```text
 MergeTree(EventDate, intHash32(UserID), (CounterID, EventDate, intHash32(UserID)), 8192)
 ```
 
-В таблице типа MergeTree обязательно должен быть отдельный столбец, содержащий дату. В этом примере, это - столбец EventDate. Тип столбца с датой - обязательно Date (а не DateTime).
+В таблице типа MergeTree обязательно должен быть отдельный столбец, содержащий дату, здесь это столбец EventDate. Тип столбца с датой — обязательно Date (а не DateTime).
 
 Первичным ключом может быть кортеж из произвольных выражений (обычно это просто кортеж столбцов) или одно выражение.
 
-Выражение для сэмплирования (использовать не обязательно) - произвольное выражение. Оно должно также присутствовать в первичном ключе. В примере используется хэширование по идентификатору посетителя, чтобы псевдослучайно перемешать данные в таблице для каждого CounterID и EventDate. То есть, при использовании секции SAMPLE в запросе, вы получите равномерно-псевдослучайную выборку данных для подмножества посетителей.
+Выражение для сэмплирования (использовать не обязательно) — произвольное выражение. Оно должно также присутствовать в первичном ключе. В примере используется хэширование по идентификатору посетителя, чтобы псевдослучайно перемешать данные в таблице для каждого CounterID и EventDate. То есть, при использовании секции SAMPLE в запросе вы получите равномерно-псевдослучайную выборку данных для подмножества посетителей.
 
-Таблица реализована, как набор кусочков. Каждый кусочек сортирован по первичному ключу. Также, для каждого кусочка прописана минимальная и максимальная дата. При вставке в таблицу, создаётся новый сортированный кусочек. В фоне, периодически инициируется процесс слияния. При слиянии, выбирается несколько кусочков, обычно наименьших, и сливаются в один большой сортированный кусочек.
+Таблица реализована как набор кусочков. Каждый кусочек сортирован по первичному ключу. Также для каждого кусочка прописана минимальная и максимальная дата. При вставке в таблицу создаётся новый сортированный кусочек. В фоне периодически инициируется процесс слияния. При слиянии выбирается несколько кусочков, обычно наименьших, и сливаются в один большой сортированный кусочек.
 
 То есть, при вставке в таблицу производится инкрементальная сортировка. Слияние реализовано таким образом, что таблица постоянно состоит из небольшого количества сортированных кусочков, а также само слияние делает не слишком много работы.
 
-При вставке, данные относящиеся к разным месяцам, разбиваются на разные кусочки. Кусочки, соответствующие разным месяцам, никогда не объединяются. Это сделано, чтобы обеспечить локальность модификаций данных (для упрощения бэкапов).
+При вставке данные, относящиеся к разным месяцам, разбиваются на разные кусочки. Кусочки, соответствующие разным месяцам, никогда не объединяются. Это сделано, чтобы обеспечить локальность модификаций данных (для упрощения бэкапов).
 
-Кусочки объединяются до некоторого предельного размера - чтобы не было слишком длительных слияний.
+Кусочки объединяются до некоторого предельного размера, чтобы не было слишком длительных слияний.
 
-Для каждого кусочка также пишется индексный файл. Индексный файл содержит значение первичного ключа для каждой index_granularity строки таблицы. То есть, это - разреженный индекс сортированных данных.
+Для каждого кусочка также пишется индексный файл. Индексный файл содержит значение первичного ключа для каждой index_granularity строки таблицы. То есть, это разреженный индекс сортированных данных.
 
-Для столбцов также пишутся "засечки" каждую index_granularity строку, чтобы данные можно было читать в определённом диапазоне.
+Для столбцов также пишутся «засечки» каждую index_granularity строку, чтобы данные можно было читать в определённом диапазоне.
 
 При чтении из таблицы, запрос SELECT анализируется на предмет того, можно ли использовать индексы.
 Индекс может использоваться, если в секции WHERE/PREWHERE, в качестве одного из элементов конъюнкции, или целиком, есть выражение, представляющее операции сравнения на равенства, неравенства, а также IN над столбцами, входящими в первичный ключ / дату, а также логические связки над ними.
 
-Таким образом, обеспечивается возможность быстро выполнять запросы по одному или многим диапазонам первичного ключа. Например, в указанном примере, будут быстро работать запросы для конкретного счётчика; для конкретного счётчика и диапазона дат; для конкретного счётчика и даты, для нескольких счётчиков и диапазона дат и т. п.
+Таким образом, обеспечивается возможность быстро выполнять запросы по одному или многим диапазонам первичного ключа. Например, в указанном примере будут быстро работать запросы для конкретного счётчика; для конкретного счётчика и диапазона дат; для конкретного счётчика и даты, для нескольких счётчиков и диапазона дат и т. п.
 
 ```sql
 SELECT count() FROM table WHERE EventDate = toDate(now()) AND CounterID = 34
@@ -50,22 +50,22 @@ SELECT count() FROM table WHERE ((EventDate >= toDate('2014-01-01') AND EventDat
 
 Во всех этих случаях будет использоваться индекс по дате и по первичному ключу. Видно, что индекс используется даже для достаточно сложных выражений. Чтение из таблицы организовано так, что использование индекса не может быть медленнее full scan-а.
 
-В этом примере, индекс не может использоваться:
+В этом примере индекс не может использоваться.
 
 ```sql
 SELECT count() FROM table WHERE CounterID = 34 OR URL LIKE '%upyachka%'
 ```
 
-Чтобы проверить сможет ли ClickHouse использовать индекс при выполнении запроса, используйте настройки [force_index_by_date](../operations/settings/settings.md#settings-settings-force_index_by_date) и [force_primary_key](../operations/settings/settings.md#settings-settings-force_primary_key).
+Чтобы проверить, сможет ли ClickHouse использовать индекс при выполнении запроса, используйте настройки [force_index_by_date](../operations/settings/settings.md#settings-settings-force_index_by_date) и [force_primary_key](../operations/settings/settings.md#settings-settings-force_primary_key).
 
-Индекс по дате обеспечивает чтение только кусков, содержащих даты из нужного диапазона. При этом, кусок данных может содержать данные за многие даты (до целого месяца), а в пределах одного куска, данные лежат упорядоченными по первичному ключу, который может не содержать дату в качестве первого столбца. В связи с этим, при использовании запроса с указанием условия только на дату, но не на префикс первичного ключа, будет читаться данных больше, чем за одну дату.
+Индекс по дате обеспечивает чтение только кусков, содержащих даты из нужного диапазона. При этом кусок данных может содержать данные за многие даты (до целого месяца), а в пределах одного куска данные лежат упорядоченными по первичному ключу, который может не содержать дату в качестве первого столбца. В связи с этим, при использовании запроса с указанием условия только на дату, но не на префикс первичного ключа, будет читаться данных больше, чем за одну дату.
 
-Для конкуррентного доступа к таблице, используется мульти-версионность. То есть, при одновременном чтении и обновлении таблицы, данные будут читаться из набора кусочков, актуального на момент запроса. Длинных блокировок нет. Вставки никак не мешают чтениям.
+Для конкуррентного доступа к таблице используется мультиверсионность. То есть, при одновременном чтении и обновлении таблицы, данные будут читаться из набора кусочков, актуального на момент запроса. Длинных блокировок нет. Вставки никак не мешают чтениям.
 
 Чтения из таблицы автоматически распараллеливаются.
 
 Поддерживается запрос `OPTIMIZE`, который вызывает один внеочередной шаг слияния.
 
-Вы можете использовать одну большую таблицу, постоянно добавляя в неё данные небольшими пачками - именно для этого предназначен движок MergeTree.
+Вы можете использовать одну большую таблицу, постоянно добавляя в неё данные небольшими пачками, именно для этого предназначен движок MergeTree.
 
-Для всех типов таблиц семейства MergeTree возможна репликация данных - смотрите раздел "Репликация данных".
+Для всех типов таблиц семейства MergeTree возможна репликация данных — смотрите раздел «Репликация данных».

From b4bb6f4bc3c36822902daa7e6098858f0a41de90 Mon Sep 17 00:00:00 2001
From: Valera Ryaboshapko <valera@creator.su>
Date: Thu, 15 Mar 2018 15:33:12 +0300
Subject: [PATCH 098/244] Update replacingmergetree.md

---
 docs/ru/table_engines/replacingmergetree.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/ru/table_engines/replacingmergetree.md b/docs/ru/table_engines/replacingmergetree.md
index 4c86b489f99..8e09810ca2b 100644
--- a/docs/ru/table_engines/replacingmergetree.md
+++ b/docs/ru/table_engines/replacingmergetree.md
@@ -2,15 +2,15 @@
 
 Движок таблиц отличается от `MergeTree` тем, что выполняет удаление дублирующихся записей с одинаковым значением первичного ключа.
 
-Последний, необязательный параметр движка таблицы - столбец с "версией". При слиянии, для всех строк с одинаковым значением первичного ключа, оставляет только одну строку: если задан столбец версии - строку с максимальной версией, иначе - последнюю строку.
+Последний, необязательный параметр движка таблицы — столбец с версией. При слиянии для всех строк с одинаковым значением первичного ключа оставляет только одну строку: если задан столбец версии — строку с максимальной версией, иначе — последнюю строку.
 
-Столбец с версией должен иметь тип из семейства `UInt`, либо `Date` или `DateTime`.
+Столбец с версией должен иметь тип из семейства `UInt`, или `Date`, или `DateTime`.
 
 ```sql
 ReplacingMergeTree(EventDate, (OrderID, EventDate, BannerID, ...), 8192, ver)
 ```
 
-Обратите внимание, что дедупликация данных производится лишь во время слияний. Слияние происходят в фоне, в неизвестный момент времени, на который вы не можете ориентироваться. Некоторая часть данных может так и остаться необработанной. Хотя вы можете вызвать внеочередное слияние с помощью запроса OPTIMIZE, на это не стоит рассчитывать, так как запрос OPTIMIZE приводит к чтению и записи большого объёма данных.
+Обратите внимание, что дедупликация данных производится лишь во время слияний. Слияние происходят в фоне в неизвестный момент времени, на который вы не можете ориентироваться. Некоторая часть данных может так и остаться необработанной. Хотя вы можете вызвать внеочередное слияние с помощью запроса OPTIMIZE, на это не стоит рассчитывать, так как запрос OPTIMIZE приводит к чтению и записи большого объёма данных.
 
 Таким образом, `ReplacingMergeTree` подходит для фоновой чистки дублирующихся данных в целях экономии места, но не даёт гарантий отсутствия дубликатов.
 

From 12d9c30d2b28783e61355bcbed13cb560fd3d298 Mon Sep 17 00:00:00 2001
From: Valera Ryaboshapko <valera@creator.su>
Date: Thu, 15 Mar 2018 16:26:19 +0300
Subject: [PATCH 099/244] Update merge.md

---
 docs/ru/table_engines/merge.md | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/docs/ru/table_engines/merge.md b/docs/ru/table_engines/merge.md
index b3d9ced8f89..661d979fdf5 100644
--- a/docs/ru/table_engines/merge.md
+++ b/docs/ru/table_engines/merge.md
@@ -2,33 +2,33 @@
 
 Движок Merge (не путайте с движком `MergeTree`) не хранит данные самостоятельно, а позволяет читать одновременно из произвольного количества других таблиц.
 Чтение автоматически распараллеливается. Запись в таблицу не поддерживается. При чтении будут использованы индексы тех таблиц, из которых реально идёт чтение, если они существуют.
-Движок Merge принимает параметры: имя базы данных и регулярное выражение для таблиц. Пример:
+Движок Merge принимает параметры: имя базы данных и регулярное выражение для таблиц. Пример.
 
 ```text
 Merge(hits, '^WatchLog')
 ```
 
--   данные будут читаться из таблиц в базе hits, имена которых соответствуют регулярному выражению '`^WatchLog`'.
+Данные будут читаться из таблиц в базе hits, имена которых соответствуют регулярному выражению '`^WatchLog`'.
 
 Вместо имени базы данных может использоваться константное выражение, возвращающее строку. Например, `currentDatabase()`.
 
-Регулярные выражения - re2 (как PCRE, но без особых извратов), регистрозависимые.
+Регулярные выражения — [re2](https://github.com/google/re2) (облегчённая версия PCRE), регистрозависимые.
 Смотрите замечание об экранировании в регулярных выражениях в разделе "match".
 
-При выборе таблиц для чтения, сама Merge-таблица не будет выбрана, даже если попадает под регулярное выражение - чтобы не возникло циклов.
-Впрочем, вы можете создать две Merge-таблицы, которые будут пытаться бесконечно читать данные друг-друга. Этого делать не нужно.
+При выборе таблиц для чтения, сама Merge-таблица не будет выбрана, даже если попадает под регулярное выражение, чтобы не возникло циклов.
+Впрочем, вы можете создать две Merge-таблицы, которые будут пытаться бесконечно читать данные друг друга, но делать этого не нужно.
 
-Типичный способ использования движка Merge - возможность работы с большим количеством таблиц типа TinyLog, как с одной.
+Типичный способ использования движка Merge — работа с большим количеством таблиц типа TinyLog, как с одной.
 
 ## Виртуальные столбцы
 
-Виртуальные столбцы - столбцы, предоставляемые движком таблиц, независимо от определения таблицы. То есть, такие столбцы не указываются в CREATE TABLE, но доступны для SELECT-а.
+Виртуальные столбцы — столбцы, предоставляемые движком таблиц независимо от определения таблицы. То есть, такие столбцы не указываются в CREATE TABLE, но доступны для SELECT-а.
 
 Виртуальные столбцы отличаются от обычных следующими особенностями:
 
 -   они не указываются в определении таблицы;
 -   в них нельзя вставить данные при INSERT-е;
--   при INSERT-е без указания списка столбцов, виртуальные столбцы не учитываются;
+-   при INSERT-е без указания списка столбцов виртуальные столбцы не учитываются;
 -   они не выбираются при использовании звёздочки (`SELECT *`);
 -   виртуальные столбцы не показываются в запросах `SHOW CREATE TABLE` и `DESC TABLE`;
 

From d9848d89047519f51626eef0882f4be69c483b19 Mon Sep 17 00:00:00 2001
From: proller <proller@github.com>
Date: Thu, 15 Mar 2018 18:09:44 +0300
Subject: [PATCH 100/244] Allow run test under gdb

---
 dbms/tests/clickhouse-test-server | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/dbms/tests/clickhouse-test-server b/dbms/tests/clickhouse-test-server
index b3c9e4eaae5..778dc29b041 100755
--- a/dbms/tests/clickhouse-test-server
+++ b/dbms/tests/clickhouse-test-server
@@ -28,20 +28,33 @@ mkdir -p $LOG_DIR
 openssl dhparam -out `${BIN_DIR}clickhouse-extract-from-config --config=$CLICKHOUSE_CONFIG  --key=openSSL.server.dhParamsFile` 256
 openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout `${BIN_DIR}clickhouse-extract-from-config --config=$CLICKHOUSE_CONFIG  --key=openSSL.server.privateKeyFile` -out `${BIN_DIR}clickhouse-extract-from-config --config=$CLICKHOUSE_CONFIG  --key=openSSL.server.certificateFile`
 
+if [ "$TEST_GDB" ]; then
+    echo -e "run \nset pagination off \nset logging file $DATA_DIR/gdb.log \nset logging on \nthread apply all backtrace \ndetach \nquit " > $DATA_DIR/gdb.cmd
+    GDB="gdb -x $DATA_DIR/gdb.cmd --args "
+fi
+
 # Start a local clickhouse server which will be used to run tests
 #PATH=$PATH:$BIN_DIR \
-${BIN_DIR}clickhouse-server --config-file=${CLICKHOUSE_CONFIG} > $LOG_DIR/stdout 2>&1 &
+$GDB ${BIN_DIR}clickhouse-server --config-file=${CLICKHOUSE_CONFIG} > $LOG_DIR/stdout 2>&1 &
 CH_PID=$!
 sleep 3
 
+if [ "$GDB" ]; then
+    # Long symbols read
+    sleep 40
+fi
+
 tail -n50 $LOG_DIR/*
 
 # Define needed stuff to kill test clickhouse server after tests completion
 function finish {
-  kill $CH_PID || true
-  wait
-  tail -n 50 $LOG_DIR/stdout
-  rm -rf $DATA_DIR
+    kill $CH_PID || true
+    wait
+    tail -n 50 $LOG_DIR/*
+    if [ "$GDB" ]; then
+        cat $DATA_DIR/gdb.log || true
+    fi
+    rm -rf $DATA_DIR
 }
 trap finish EXIT SIGINT SIGQUIT SIGTERM
 

From 1656eafd8ce561c66829455f59e8044d50b65055 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Thu, 15 Mar 2018 19:22:43 +0300
Subject: [PATCH 101/244] Revert "Attempt to fix regression: missing support
 for remote tables in IN section when querying Distributed tables
 [#CLICKHOUSE-2]"

This reverts commit d52f688883fe2a703dd83164dc172b76745d8e0f.
---
 dbms/src/Interpreters/ExpressionAnalyzer.cpp  |  5 -----
 .../Interpreters/InterpreterSelectQuery.cpp   |  3 +--
 dbms/src/Storages/IStorage.h                  | 14 -------------
 dbms/src/Storages/StorageDistributed.cpp      | 20 -------------------
 dbms/src/Storages/StorageDistributed.h        |  6 ------
 5 files changed, 1 insertion(+), 47 deletions(-)

diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp
index 5360e750a33..f0bc8137029 100644
--- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp
+++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp
@@ -61,8 +61,6 @@
 #include <DataTypes/DataTypeFunction.h>
 #include <Functions/FunctionsMiscellaneous.h>
 
-#include <Core/iostream_debug_helpers.h>
-
 
 namespace DB
 {
@@ -2440,9 +2438,6 @@ bool ExpressionAnalyzer::appendWhere(ExpressionActionsChain & chain, bool only_t
     initChain(chain, source_columns);
     ExpressionActionsChain::Step & step = chain.steps.back();
 
-    DUMP(only_types);
-    DUMP(StackTrace().toString());
-
     step.required_output.push_back(select_query->where_expression->getColumnName());
     getRootActions(select_query->where_expression, only_types, false, step.actions);
 
diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp
index 1f6aeb9d3c5..49b3ba27004 100644
--- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp
@@ -676,8 +676,7 @@ QueryProcessingStage::Enum InterpreterSelectQuery::executeFetchColumns(Pipeline
             pipeline.streams = storage->read(required_columns, query_info, context, from_stage, max_block_size, max_streams);
 
         if (pipeline.streams.empty())
-            pipeline.streams.emplace_back(std::make_shared<NullBlockInputStream>(
-                storage->analyze(required_columns, query_info, context, from_stage)));
+            pipeline.streams.emplace_back(std::make_shared<NullBlockInputStream>(storage->getSampleBlockForColumns(required_columns)));
 
         pipeline.transform([&](auto & stream)
         {
diff --git a/dbms/src/Storages/IStorage.h b/dbms/src/Storages/IStorage.h
index b30aeb2bfb8..ef9fd8004ce 100644
--- a/dbms/src/Storages/IStorage.h
+++ b/dbms/src/Storages/IStorage.h
@@ -180,20 +180,6 @@ public:
         throw Exception("Method read is not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED);
     }
 
-    /** Without executing a query determine,
-      *  what structure of result and what query processed stage
-      *  we will get if we will call 'read' method.
-      */
-    virtual Block analyze(
-        const Names & column_names,
-        const SelectQueryInfo & /*query_info*/,
-        const Context & /*context*/,
-        QueryProcessingStage::Enum & processed_stage)
-    {
-        processed_stage = QueryProcessingStage::FetchColumns;
-        return getSampleBlockForColumns(column_names);
-    }
-
     /** Writes the data to a table.
       * Receives a description of the query, which can contain information about the data write method.
       * Returns an object by which you can write data sequentially.
diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp
index f73f23e5a13..2b1dfe66523 100644
--- a/dbms/src/Storages/StorageDistributed.cpp
+++ b/dbms/src/Storages/StorageDistributed.cpp
@@ -197,26 +197,6 @@ BlockInputStreams StorageDistributed::read(
 }
 
 
-Block StorageDistributed::analyze(
-    const Names & /*column_names*/,
-    const SelectQueryInfo & query_info,
-    const Context & context,
-    QueryProcessingStage::Enum & processed_stage)
-{
-    auto cluster = getCluster();
-
-    const Settings & settings = context.getSettingsRef();
-
-    size_t result_size = (cluster->getRemoteShardCount() * settings.max_parallel_replicas) + cluster->getLocalShardCount();
-
-    processed_stage = result_size == 1 || settings.distributed_group_by_no_merge
-        ? QueryProcessingStage::Complete
-        : QueryProcessingStage::WithMergeableState;
-
-    return materializeBlock(InterpreterSelectQuery(query_info.query, context, {}, processed_stage).getSampleBlock());
-}
-
-
 BlockOutputStreamPtr StorageDistributed::write(const ASTPtr & query, const Settings & settings)
 {
     auto cluster = (owned_cluster) ? owned_cluster : context.getCluster(cluster_name);
diff --git a/dbms/src/Storages/StorageDistributed.h b/dbms/src/Storages/StorageDistributed.h
index 63fefbfe501..451c9f12d8a 100644
--- a/dbms/src/Storages/StorageDistributed.h
+++ b/dbms/src/Storages/StorageDistributed.h
@@ -60,12 +60,6 @@ public:
         size_t max_block_size,
         unsigned num_streams) override;
 
-    Block analyze(
-        const Names & column_names,
-        const SelectQueryInfo & query_info,
-        const Context & context,
-        QueryProcessingStage::Enum & processed_stage) override;
-
     BlockOutputStreamPtr write(const ASTPtr & query, const Settings & settings) override;
 
     void drop() override {}

From 16dad209f7ffd91d0d2b38fffff548044ed7a0e7 Mon Sep 17 00:00:00 2001
From: Vitaliy Lyudvichenko <vludv@yandex-team.ru>
Date: Thu, 15 Mar 2018 20:04:47 +0300
Subject: [PATCH 102/244] Do not make extra ZK RTTs in inserts. [#CLICKHOUSE-3]

Resolves #2054
---
 dbms/src/Storages/MergeTree/AbandonableLockInZooKeeper.h    | 6 ++++++
 .../MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp      | 3 +++
 2 files changed, 9 insertions(+)

diff --git a/dbms/src/Storages/MergeTree/AbandonableLockInZooKeeper.h b/dbms/src/Storages/MergeTree/AbandonableLockInZooKeeper.h
index 690217302bf..75c2c01eaaf 100644
--- a/dbms/src/Storages/MergeTree/AbandonableLockInZooKeeper.h
+++ b/dbms/src/Storages/MergeTree/AbandonableLockInZooKeeper.h
@@ -108,6 +108,12 @@ public:
         ops.emplace_back(std::make_shared<zkutil::Op::Remove>(holder_path, -1));
     }
 
+    /// Do not delete nodes in destructor
+    void setIsUnlocked()
+    {
+        holder_path = "";
+    }
+
     void checkCreated() const
     {
         if (!isCreated())
diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp
index 7e8ae99cad2..22fb1794756 100644
--- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp
+++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp
@@ -370,6 +370,9 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo
     {
         transaction.commit();
         storage.merge_selecting_event.set();
+
+        /// Lock nodes have been already deleted, do not delete them in destructor
+        block_number_lock.setIsUnlocked();
     }
     else if (zkutil::isUserError(info.code))
     {

From e77701bd4bd6b6a205445abc1a21b90cb38ffd99 Mon Sep 17 00:00:00 2001
From: Vitaliy Lyudvichenko <vludv@yandex-team.ru>
Date: Thu, 15 Mar 2018 19:57:10 +0300
Subject: [PATCH 103/244] Enable alisas for sets and tuples for IN operator.
 [#CLICKHOUSE-3]

Resolves #2044
---
 dbms/src/Interpreters/ExpressionAnalyzer.cpp                   | 3 ++-
 .../queries/0_stateless/00172_constexprs_in_set.reference      | 1 +
 dbms/tests/queries/0_stateless/00172_constexprs_in_set.sql     | 1 +
 3 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp
index f0bc8137029..411c868d6f0 100644
--- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp
+++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp
@@ -980,7 +980,8 @@ void ExpressionAnalyzer::normalizeTreeImpl(
         /// `IN t` can be specified, where t is a table, which is equivalent to `IN (SELECT * FROM t)`.
         if (functionIsInOrGlobalInOperator(func_node->name))
             if (ASTIdentifier * right = typeid_cast<ASTIdentifier *>(func_node->arguments->children.at(1).get()))
-                right->kind = ASTIdentifier::Table;
+                if (!aliases.count(right->name))
+                    right->kind = ASTIdentifier::Table;
 
         /// Special cases for count function.
         String func_name_lowercase = Poco::toLower(func_node->name);
diff --git a/dbms/tests/queries/0_stateless/00172_constexprs_in_set.reference b/dbms/tests/queries/0_stateless/00172_constexprs_in_set.reference
index c06d3de5a56..ac284bbc084 100644
--- a/dbms/tests/queries/0_stateless/00172_constexprs_in_set.reference
+++ b/dbms/tests/queries/0_stateless/00172_constexprs_in_set.reference
@@ -4,3 +4,4 @@
 1
 1
 1
+1	0
diff --git a/dbms/tests/queries/0_stateless/00172_constexprs_in_set.sql b/dbms/tests/queries/0_stateless/00172_constexprs_in_set.sql
index 3c438417053..5d79185ee8d 100644
--- a/dbms/tests/queries/0_stateless/00172_constexprs_in_set.sql
+++ b/dbms/tests/queries/0_stateless/00172_constexprs_in_set.sql
@@ -4,3 +4,4 @@ SELECT toDate('2015-06-12') IN (toDate('2015-06-12'));
 SELECT today() IN (toDate('2014-01-01'), toDate(now()));
 SELECT - -1 IN (2 - 1);
 SELECT - -1 IN (2 - 1, 3);
+WITH (1, 2) AS a SELECT 1 IN a, 3 IN a;

From 2097ee9a825ecbde1ac6ddcad93d34749ce9aa63 Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Thu, 15 Mar 2018 21:44:04 +0300
Subject: [PATCH 104/244] Update merge.md

---
 docs/ru/table_engines/merge.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/ru/table_engines/merge.md b/docs/ru/table_engines/merge.md
index 661d979fdf5..1124e54b5bb 100644
--- a/docs/ru/table_engines/merge.md
+++ b/docs/ru/table_engines/merge.md
@@ -12,7 +12,7 @@ Merge(hits, '^WatchLog')
 
 Вместо имени базы данных может использоваться константное выражение, возвращающее строку. Например, `currentDatabase()`.
 
-Регулярные выражения — [re2](https://github.com/google/re2) (облегчённая версия PCRE), регистрозависимые.
+Регулярные выражения — [re2](https://github.com/google/re2) (поддерживает подмножество PCRE), регистрозависимые.
 Смотрите замечание об экранировании в регулярных выражениях в разделе "match".
 
 При выборе таблиц для чтения, сама Merge-таблица не будет выбрана, даже если попадает под регулярное выражение, чтобы не возникло циклов.

From 1554390baba5bfcb7b53054a589c212b377c38dd Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Thu, 15 Mar 2018 22:05:58 +0300
Subject: [PATCH 105/244] Update AbandonableLockInZooKeeper.h

---
 dbms/src/Storages/MergeTree/AbandonableLockInZooKeeper.h | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/dbms/src/Storages/MergeTree/AbandonableLockInZooKeeper.h b/dbms/src/Storages/MergeTree/AbandonableLockInZooKeeper.h
index 75c2c01eaaf..b1c82483e38 100644
--- a/dbms/src/Storages/MergeTree/AbandonableLockInZooKeeper.h
+++ b/dbms/src/Storages/MergeTree/AbandonableLockInZooKeeper.h
@@ -108,8 +108,9 @@ public:
         ops.emplace_back(std::make_shared<zkutil::Op::Remove>(holder_path, -1));
     }
 
-    /// Do not delete nodes in destructor
-    void setIsUnlocked()
+    /// Do not delete nodes in destructor. You may call this method after 'getUnlockOps' and successful execution of these ops,
+    ///  because the nodes will be already deleted.
+    void assumeUnlocked()
     {
         holder_path = "";
     }

From e64b2df87492d0cb1da61c1101e4d5d45f16fec3 Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Thu, 15 Mar 2018 22:06:23 +0300
Subject: [PATCH 106/244] Update ReplicatedMergeTreeBlockOutputStream.cpp

---
 .../Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp
index 22fb1794756..1d5ec76c652 100644
--- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp
+++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp
@@ -372,7 +372,7 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo
         storage.merge_selecting_event.set();
 
         /// Lock nodes have been already deleted, do not delete them in destructor
-        block_number_lock.setIsUnlocked();
+        block_number_lock.assumeUnlocked();
     }
     else if (zkutil::isUserError(info.code))
     {

From 1e88f2fe5c44d8b649eee658b04be8670a140ea7 Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Thu, 15 Mar 2018 22:06:53 +0300
Subject: [PATCH 107/244] Update AbandonableLockInZooKeeper.h

---
 dbms/src/Storages/MergeTree/AbandonableLockInZooKeeper.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dbms/src/Storages/MergeTree/AbandonableLockInZooKeeper.h b/dbms/src/Storages/MergeTree/AbandonableLockInZooKeeper.h
index b1c82483e38..35f56fa429f 100644
--- a/dbms/src/Storages/MergeTree/AbandonableLockInZooKeeper.h
+++ b/dbms/src/Storages/MergeTree/AbandonableLockInZooKeeper.h
@@ -112,7 +112,7 @@ public:
     ///  because the nodes will be already deleted.
     void assumeUnlocked()
     {
-        holder_path = "";
+        holder_path.clear();
     }
 
     void checkCreated() const

From 8a432e40b95105c3cfa65b9d6d004d7410ddafe5 Mon Sep 17 00:00:00 2001
From: Vitaliy Lyudvichenko <vludv@yandex-team.ru>
Date: Thu, 15 Mar 2018 22:24:20 +0300
Subject: [PATCH 108/244] Fixed 569 test. [#CLICKHOUSE-2]

---
 .../0_stateless/00569_parse_date_time_best_effort.reference     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dbms/tests/queries/0_stateless/00569_parse_date_time_best_effort.reference b/dbms/tests/queries/0_stateless/00569_parse_date_time_best_effort.reference
index c0ad225ca1e..38264a337f0 100644
--- a/dbms/tests/queries/0_stateless/00569_parse_date_time_best_effort.reference
+++ b/dbms/tests/queries/0_stateless/00569_parse_date_time_best_effort.reference
@@ -7,7 +7,7 @@ s                                                   a                     b
 02/01/17 010203 MSK               2017-01-01 22:02:03   2017-01-01 22:02:03
 02/01/17 010203 MSK+0100          2017-01-01 21:02:03   2017-01-01 21:02:03
 02/01/17 010203 UTC+0300          2017-01-01 22:02:03   2017-01-01 22:02:03
-020117 010203 UTC+0300            1970-01-01 84:30:19   1970-01-01 84:30:19
+020117 010203 UTC+0300            1970-01-01 04:30:19   1970-01-01 04:30:19
 02/01/17 010203Z                  2017-01-02 01:02:03   2017-01-02 01:02:03
 02/01/1970 010203Z                1970-01-02 01:02:03   1970-01-02 01:02:03
 02/01/70 010203Z                  1970-01-02 01:02:03   1970-01-02 01:02:03

From b2e4fd30981f9bd700f25dc3bd0adbbbc2a0259e Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Thu, 15 Mar 2018 22:29:53 +0300
Subject: [PATCH 109/244] Updated test [#CLICKHOUSE-2]

---
 .../0_stateless/00569_parse_date_time_best_effort.reference     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dbms/tests/queries/0_stateless/00569_parse_date_time_best_effort.reference b/dbms/tests/queries/0_stateless/00569_parse_date_time_best_effort.reference
index c0ad225ca1e..38264a337f0 100644
--- a/dbms/tests/queries/0_stateless/00569_parse_date_time_best_effort.reference
+++ b/dbms/tests/queries/0_stateless/00569_parse_date_time_best_effort.reference
@@ -7,7 +7,7 @@ s                                                   a                     b
 02/01/17 010203 MSK               2017-01-01 22:02:03   2017-01-01 22:02:03
 02/01/17 010203 MSK+0100          2017-01-01 21:02:03   2017-01-01 21:02:03
 02/01/17 010203 UTC+0300          2017-01-01 22:02:03   2017-01-01 22:02:03
-020117 010203 UTC+0300            1970-01-01 84:30:19   1970-01-01 84:30:19
+020117 010203 UTC+0300            1970-01-01 04:30:19   1970-01-01 04:30:19
 02/01/17 010203Z                  2017-01-02 01:02:03   2017-01-02 01:02:03
 02/01/1970 010203Z                1970-01-02 01:02:03   1970-01-02 01:02:03
 02/01/70 010203Z                  1970-01-02 01:02:03   1970-01-02 01:02:03

From cc19e40fa02b8519e0238c6f863a722f8fa0726a Mon Sep 17 00:00:00 2001
From: robot-metrika-test <robot-metrika-test@yandex-team.ru>
Date: Fri, 16 Mar 2018 00:07:09 +0300
Subject: [PATCH 110/244] Auto version update to [54366]

---
 dbms/cmake/version.cmake | 4 ++--
 debian/changelog         | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake
index 49bda24e075..80e9aaf2e57 100644
--- a/dbms/cmake/version.cmake
+++ b/dbms/cmake/version.cmake
@@ -1,6 +1,6 @@
 # This strings autochanged from release_lib.sh:
-set(VERSION_DESCRIBE v1.1.54365-testing)
-set(VERSION_REVISION 54365)
+set(VERSION_DESCRIBE v1.1.54366-testing)
+set(VERSION_REVISION 54366)
 # end of autochange
 
 set (VERSION_MAJOR 1)
diff --git a/debian/changelog b/debian/changelog
index 8ee6e628a17..6f142ef0c16 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,5 @@
-clickhouse (1.1.54365) unstable; urgency=low
+clickhouse (1.1.54366) unstable; urgency=low
 
   * Modified source code
 
- --  <robot-metrika-test@yandex-team.ru>  Wed, 14 Mar 2018 16:50:31 +0300
+ --  <robot-metrika-test@yandex-team.ru>  Fri, 16 Mar 2018 00:07:09 +0300

From 3a8d6c28ef6333c3f7e3e168978ba73d5eee594a Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 16 Mar 2018 00:44:09 +0300
Subject: [PATCH 111/244] Attempt to fix a regression [#CLICKHOUSE-2]

---
 dbms/src/Interpreters/ExpressionAnalyzer.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp
index 411c868d6f0..ca9bc493566 100644
--- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp
+++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp
@@ -1580,7 +1580,7 @@ void ExpressionAnalyzer::makeSet(const ASTFunction * node, const Block & sample_
           *   in the subquery_for_set object, this subquery is set as source and the temporary table _data1 as the table.
           * - this function shows the expression IN_data1.
           */
-        if (!subquery_for_set.source)
+        if (!subquery_for_set.source && (!storage || !storage->isRemote()))
         {
             auto interpreter = interpretSubquery(arg, context, subquery_depth, {});
             subquery_for_set.source = std::make_shared<LazyBlockInputStream>(

From 9017f1d66427e14b17c77c4f4e6b175baec84bb7 Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Fri, 16 Mar 2018 02:22:36 +0300
Subject: [PATCH 112/244] Update query_complexity.md

---
 docs/ru/operations/settings/query_complexity.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/ru/operations/settings/query_complexity.md b/docs/ru/operations/settings/query_complexity.md
index c964ad6e41c..3c4ae65b370 100644
--- a/docs/ru/operations/settings/query_complexity.md
+++ b/docs/ru/operations/settings/query_complexity.md
@@ -33,7 +33,7 @@
 
 Максимальный возможный объем оперативной памяти для выполнения запроса на одном сервере.
 
-Значения по умолчанию определены в файле [Settings.h](https://github.com/yandex/ClickHouse/blob/master/dbms/src/Interpreters/Settings.h#L243). По умолчанию размер не ограничен (`max_memory_usage = 0`).
+В конфигурационном файле по-умолчанию, ограничение равно 10 ГБ.
 
 Настройка не учитывает объём свободной памяти или общий объём памяти на машине.
 Ограничение действует на один запрос, в пределах одного сервера.

From 70ab16e864239c3baff6148716854cf0d51f754b Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Fri, 16 Mar 2018 02:23:13 +0300
Subject: [PATCH 113/244] Update query_complexity.md

---
 docs/ru/operations/settings/query_complexity.md | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/docs/ru/operations/settings/query_complexity.md b/docs/ru/operations/settings/query_complexity.md
index 3c4ae65b370..afbba3bc688 100644
--- a/docs/ru/operations/settings/query_complexity.md
+++ b/docs/ru/operations/settings/query_complexity.md
@@ -40,10 +40,7 @@
 Текущее потребление памяти для каждого запроса можно посмотреть с помощью `SHOW PROCESSLIST`.
 Также отслеживается и выводится в лог пиковое потребление памяти для каждого запроса.
 
-Потребление памяти не отслеживается для:
-
-- Констант.
-- Состояний некоторых агрегатных функций.
+Потребление памяти не отслеживается для состояний некоторых агрегатных функций.
 
 Потребление памяти не полностью учитывается для состояний агрегатных функций `min`, `max`, `any`, `anyLast`, `argMin`, `argMax` от аргументов `String` и `Array`.
 

From bbb101845aedfd3cc1731291df3cdd70faa21038 Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Fri, 16 Mar 2018 02:24:21 +0300
Subject: [PATCH 114/244] Update replication.md

---
 docs/ru/table_engines/replication.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/ru/table_engines/replication.md b/docs/ru/table_engines/replication.md
index 970e431370e..0a2aa2e1e71 100644
--- a/docs/ru/table_engines/replication.md
+++ b/docs/ru/table_engines/replication.md
@@ -4,12 +4,12 @@
 
 Репликация поддерживается только для таблиц семейства MergeTree:
 
+- ReplicatedMergeTree
+- ReplicatedSummingMergeTree
+- ReplicatedReplacingMergeTree
 - ReplicatedAggregatingMergeTree
 - ReplicatedCollapsingMergeTree
 - ReplicatedGraphiteMergeTree
-- ReplicatedMergeTree
-- ReplicatedReplacingMergeTree
-- ReplicatedSummingMergeTree
 
 Репликация работает на уровне отдельных таблиц, а не всего сервера. То есть, на сервере могут быть расположены одновременно реплицируемые и не реплицируемые таблицы.
 

From 93c0dd3f428563a62dbc9d9ddfc98828bd12f899 Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Fri, 16 Mar 2018 02:25:57 +0300
Subject: [PATCH 115/244] Update replication.md

---
 docs/ru/table_engines/replication.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/ru/table_engines/replication.md b/docs/ru/table_engines/replication.md
index 0a2aa2e1e71..8bdd9c13c36 100644
--- a/docs/ru/table_engines/replication.md
+++ b/docs/ru/table_engines/replication.md
@@ -42,7 +42,7 @@
 </zookeeper>
 ```
 
-Используйте ZooKeeper 3.4.5 и больше.
+Используйте ZooKeeper версии 3.4.5 или более новый.
 
 Можно указать любой имеющийся у вас ZooKeeper-кластер - система будет использовать в нём одну директорию для своих данных (директория указывается при создании реплицируемой таблицы).
 

From ee88475c5b87d39ae9c11b76e2113bd0ed58682e Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Fri, 16 Mar 2018 02:30:26 +0300
Subject: [PATCH 116/244] Update replication.md

---
 docs/ru/table_engines/replication.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/ru/table_engines/replication.md b/docs/ru/table_engines/replication.md
index 8bdd9c13c36..30d8882d4eb 100644
--- a/docs/ru/table_engines/replication.md
+++ b/docs/ru/table_engines/replication.md
@@ -56,7 +56,7 @@
 
 Репликация асинхронная, мульти-мастер. Запросы `INSERT` (а также `ALTER`) можно отправлять на любой доступный сервер. Данные вставятся на сервер, где выполнен запрос, а затем скопируются на остальные серверы. В связи с асинхронностью, только что вставленные данные появляются на остальных репликах с небольшой задержкой. Если часть реплик недоступна, данные на них запишутся тогда, когда они станут доступны. Если реплика доступна, то задержка составляет столько времени, сколько требуется для передачи блока сжатых данных по сети.
 
-Кворумная запись отсутствует. Вы можете записать данные с подтвержением только от одной реплики. Если вы записали данные на одну реплику, а сервер с этой репликой перестал существовать, то записанные данные будут потеряны, если репликация еще не завершилась.
+По-умолчанию, запрос INSERT ждёт подтверждения записи только от одной реплики. Если данные были успешно записаны только на одну реплику, и сервер с этой репликой перестал существовать, то записанные данные будут потеряны. Вы можете включить подтверждение записи от нескольких реплик, используя настройку `insert_quorum`.
 
 Каждый блок данных записывается атомарно. Запрос INSERT разбивается на блоки данных размером до `max_insert_block_size = 1048576` строк. То есть, если в запросе `INSERT` менее 1048576 строк, то он делается атомарно.
 

From b949865d34f858c53f89e23409c833ea998940d2 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 16 Mar 2018 04:06:02 +0300
Subject: [PATCH 117/244] Added configuration for ad-hoc testing purposes
 [#CLICKHOUSE-2].

---
 dbms/src/Server/config.d/more_clusters.xml | 44 ++++++++++++++++++++++
 1 file changed, 44 insertions(+)
 create mode 100644 dbms/src/Server/config.d/more_clusters.xml

diff --git a/dbms/src/Server/config.d/more_clusters.xml b/dbms/src/Server/config.d/more_clusters.xml
new file mode 100644
index 00000000000..ca31b883540
--- /dev/null
+++ b/dbms/src/Server/config.d/more_clusters.xml
@@ -0,0 +1,44 @@
+<yandex>
+    <remote_servers>
+
+        <single_remote_shard_at_port_9001>
+            <shard>
+                <replica>
+                    <host>localhost</host>
+                    <port>9001</port>
+                </replica>
+            </shard>
+        </single_remote_shard_at_port_9001>
+
+        <two_remote_shards_at_port_9001_9002>
+            <shard>
+                <replica>
+                    <host>localhost</host>
+                    <port>9001</port>
+                </replica>
+            </shard>
+            <shard>
+                <replica>
+                    <host>localhost</host>
+                    <port>9002</port>
+                </replica>
+            </shard>
+        </two_remote_shards_at_port_9001_9002>
+
+        <two_shards_one_local_one_remote_at_port_9001>
+            <shard>
+                <replica>
+                    <host>localhost</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+            <shard>
+                <replica>
+                    <host>localhost</host>
+                    <port>9001</port>
+                </replica>
+            </shard>
+        </two_shards_one_local_one_remote_at_port_9001>
+
+    </remote_servers>
+</yandex>

From ace7046acb0bc81d11ecb1e9b96ce000da5ec2b9 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 16 Mar 2018 04:09:31 +0300
Subject: [PATCH 118/244] Added comment [#CLICKHOUSE-2].

---
 dbms/src/Server/config.d/more_clusters.xml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/dbms/src/Server/config.d/more_clusters.xml b/dbms/src/Server/config.d/more_clusters.xml
index ca31b883540..aecbf9e0ba7 100644
--- a/dbms/src/Server/config.d/more_clusters.xml
+++ b/dbms/src/Server/config.d/more_clusters.xml
@@ -1,6 +1,11 @@
 <yandex>
     <remote_servers>
 
+        <![CDATA[
+            You can run additional servers simply as
+             ./clickhouse-server -- --path=9001 --tcp_port=9001
+        ]]>
+
         <single_remote_shard_at_port_9001>
             <shard>
                 <replica>

From e845d3456988df9d196733ac189434bb5c184579 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 16 Mar 2018 04:15:50 +0300
Subject: [PATCH 119/244] Miscellaneous [#CLICKHOUSE-2]

---
 dbms/tests/integration/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dbms/tests/integration/README.md b/dbms/tests/integration/README.md
index 0f8f615b616..cc704022e79 100644
--- a/dbms/tests/integration/README.md
+++ b/dbms/tests/integration/README.md
@@ -5,7 +5,7 @@ This directory contains tests that involve several ClickHouse instances, custom
 ### Running
 
 Prerequisites:
-* Ubuntu 14.04 (Trusty).
+* Ubuntu 14.04 (Trusty) or higher.
 * [docker](https://www.docker.com/community-edition#/download). Minimum required API version: 1.25, check with `docker version`.
 
 You must install latest Docker from

From a16dd1cee73312da30cfd8f3a8122391a801b640 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 16 Mar 2018 04:22:02 +0300
Subject: [PATCH 120/244] Little better [#CLICKHOUSE-3650]

---
 .../Functions/FunctionsExternalDictionaries.h  | 18 ++++--------------
 1 file changed, 4 insertions(+), 14 deletions(-)

diff --git a/dbms/src/Functions/FunctionsExternalDictionaries.h b/dbms/src/Functions/FunctionsExternalDictionaries.h
index cb77d84b751..b921ba50b9a 100644
--- a/dbms/src/Functions/FunctionsExternalDictionaries.h
+++ b/dbms/src/Functions/FunctionsExternalDictionaries.h
@@ -76,6 +76,9 @@ public:
 private:
     size_t getNumberOfArguments() const override { return 2; }
 
+    bool useDefaultImplementationForConstants() const final { return true; }
+    ColumnNumbers getArgumentsThatAreAlwaysConstant() const final { return {0}; }
+
     DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
     {
         if (!arguments[0]->isString())
@@ -135,15 +138,6 @@ private:
             dict->has(ids, out->getData());
             block.getByPosition(result).column = std::move(out);
         }
-        else if (const auto id_col = checkAndGetColumnConst<ColumnVector<UInt64>>(id_col_untyped))
-        {
-            const PaddedPODArray<UInt64> ids(1, id_col->getValue<UInt64>());
-            PaddedPODArray<UInt8> out(1);
-
-            dict->has(ids, out);
-
-            block.getByPosition(result).column = DataTypeUInt8().createColumnConst(id_col->size(), toField(out.front()));
-        }
         else
             throw Exception{
                 "Second argument of function " + getName() + " must be UInt64",
@@ -161,11 +155,7 @@ private:
             return false;
 
         const ColumnWithTypeAndName & key_col_with_type = block.getByPosition(arguments[1]);
-        ColumnPtr key_col = key_col_with_type.column;
-
-        /// Functions in external dictionaries only support full-value (not constant) columns with keys.
-        if (ColumnPtr key_col_materialized = key_col_with_type.column->convertToFullColumnIfConst())
-            key_col = key_col_materialized;
+        const ColumnPtr & key_col = key_col_with_type.column;
 
         if (checkColumn<ColumnTuple>(key_col.get()))
         {

From b5e4d0692a6794d9ef07b0476d294c6be2ff526a Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 16 Mar 2018 04:23:37 +0300
Subject: [PATCH 121/244] Disallow PREWHERE, SAMPLE, FINAL in MATERIALIZED
 VIEWs [#CLICKHOUSE-2].

---
 dbms/src/Common/ErrorCodes.cpp                |  1 +
 dbms/src/Storages/StorageMaterializedView.cpp | 32 +++++++++++++++++--
 2 files changed, 30 insertions(+), 3 deletions(-)

diff --git a/dbms/src/Common/ErrorCodes.cpp b/dbms/src/Common/ErrorCodes.cpp
index 6c07a5e3422..46b4c3394e5 100644
--- a/dbms/src/Common/ErrorCodes.cpp
+++ b/dbms/src/Common/ErrorCodes.cpp
@@ -372,6 +372,7 @@ namespace ErrorCodes
     extern const int QUERY_WAS_CANCELLED = 394;
     extern const int FUNCTION_THROW_IF_VALUE_IS_NON_ZERO = 395;
     extern const int TOO_MANY_ROWS_OR_BYTES = 396;
+    extern const int QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW = 397;
 
 
     extern const int KEEPER_EXCEPTION = 999;
diff --git a/dbms/src/Storages/StorageMaterializedView.cpp b/dbms/src/Storages/StorageMaterializedView.cpp
index 1146028830b..ccdbcf204fc 100644
--- a/dbms/src/Storages/StorageMaterializedView.cpp
+++ b/dbms/src/Storages/StorageMaterializedView.cpp
@@ -1,4 +1,5 @@
 #include <Parsers/ASTSelectQuery.h>
+#include <Parsers/ASTSelectWithUnionQuery.h>
 #include <Parsers/ASTCreateQuery.h>
 #include <Parsers/ASTDropQuery.h>
 #include <Parsers/ASTIdentifier.h>
@@ -22,6 +23,7 @@ namespace ErrorCodes
 {
     extern const int LOGICAL_ERROR;
     extern const int INCORRECT_QUERY;
+    extern const int QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW;
 }
 
 
@@ -46,7 +48,7 @@ static void extractDependentTable(ASTSelectQuery & query, String & select_databa
     else if (auto ast_select = typeid_cast<ASTSelectWithUnionQuery *>(query_table.get()))
     {
         if (ast_select->list_of_selects->children.size() != 1)
-            throw Exception("UNION is not supported for MATERIALIZED VIEW", ErrorCodes::INCORRECT_QUERY);
+            throw Exception("UNION is not supported for MATERIALIZED VIEW", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW);
 
         auto & inner_query = ast_select->list_of_selects->children.at(0);
 
@@ -59,6 +61,28 @@ static void extractDependentTable(ASTSelectQuery & query, String & select_databa
 }
 
 
+static void checkAllowedQueries(const ASTSelectQuery & query)
+{
+    if (query.prewhere_expression || query.final() || query.sample_size())
+        throw Exception("MATERIALIZED VIEW cannot have PREWHERE, SAMPLE or FINAL.", DB::ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW);
+
+    auto query_table = query.table();
+
+    if (!query_table)
+        return;
+
+    if (auto ast_select = typeid_cast<const ASTSelectWithUnionQuery *>(query_table.get()))
+    {
+        if (ast_select->list_of_selects->children.size() != 1)
+            throw Exception("UNION is not supported for MATERIALIZED VIEW", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW);
+
+        const auto & inner_query = ast_select->list_of_selects->children.at(0);
+
+        checkAllowedQueries(typeid_cast<const ASTSelectQuery &>(*inner_query));
+    }
+}
+
+
 StorageMaterializedView::StorageMaterializedView(
     const String & table_name_,
     const String & database_name_,
@@ -80,11 +104,13 @@ StorageMaterializedView::StorageMaterializedView(
     /// Default value, if only table name exist in the query
     select_database_name = local_context.getCurrentDatabase();
     if (query.select->list_of_selects->children.size() != 1)
-        throw Exception("UNION is not supported for MATERIALIZED VIEW", ErrorCodes::INCORRECT_QUERY);
+        throw Exception("UNION is not supported for MATERIALIZED VIEW", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW);
 
     inner_query = query.select->list_of_selects->children.at(0);
 
-    extractDependentTable(typeid_cast<ASTSelectQuery &>(*inner_query), select_database_name, select_table_name);
+    ASTSelectQuery & select_query = typeid_cast<ASTSelectQuery &>(*inner_query);
+    extractDependentTable(select_query, select_database_name, select_table_name);
+    checkAllowedQueries(select_query);
 
     if (!select_table_name.empty())
         global_context.addDependency(

From 25a691f03d906d872ef4d99353bf83ed53fa9561 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 16 Mar 2018 05:08:31 +0300
Subject: [PATCH 122/244] Don't allow Distributed table that look at itself
 [#CLICKHOUSE-2919].

---
 .../Interpreters/InterpreterSelectQuery.cpp   |  4 +++
 dbms/src/Storages/StorageDistributed.cpp      | 31 +++++++++++++------
 dbms/src/Storages/StorageDistributed.h        |  6 ++--
 3 files changed, 30 insertions(+), 11 deletions(-)

diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp
index 49b3ba27004..ec6b9395701 100644
--- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp
@@ -63,6 +63,7 @@ namespace ErrorCodes
     extern const int ILLEGAL_PREWHERE;
     extern const int TOO_MANY_COLUMNS;
     extern const int LOGICAL_ERROR;
+    extern const int NOT_IMPLEMENTED;
 }
 
 InterpreterSelectQuery::InterpreterSelectQuery(
@@ -338,6 +339,9 @@ void InterpreterSelectQuery::executeImpl(Pipeline & pipeline, const BlockInputSt
     /** Read the data from Storage. from_stage - to what stage the request was completed in Storage. */
     QueryProcessingStage::Enum from_stage = executeFetchColumns(pipeline, dry_run);
 
+    if (from_stage == QueryProcessingStage::WithMergeableState && to_stage == QueryProcessingStage::WithMergeableState)
+        throw Exception("Distributed on Distributed is not supported", ErrorCodes::NOT_IMPLEMENTED);
+
     if (!dry_run)
         LOG_TRACE(log, QueryProcessingStage::toString(from_stage) << " -> " << QueryProcessingStage::toString(to_stage));
 
diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp
index 2b1dfe66523..e239ae3d65e 100644
--- a/dbms/src/Storages/StorageDistributed.cpp
+++ b/dbms/src/Storages/StorageDistributed.cpp
@@ -52,6 +52,7 @@ namespace ErrorCodes
     extern const int READONLY;
     extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
     extern const int INCORRECT_NUMBER_OF_COLUMNS;
+    extern const int INFINITE_LOOP;
 }
 
 
@@ -130,14 +131,16 @@ StorageDistributed::~StorageDistributed() = default;
 
 
 StorageDistributed::StorageDistributed(
-    const std::string & table_name_,
+    const String & database_name,
+    const String & table_name_,
     const ColumnsDescription & columns_,
     const String & remote_database_,
     const String & remote_table_,
     const String & cluster_name_,
     const Context & context_,
     const ASTPtr & sharding_key_,
-    const String & data_path_)
+    const String & data_path_,
+    bool attach)
     : IStorage{columns_},
     table_name(table_name_),
     remote_database(remote_database_), remote_table(remote_table_),
@@ -146,6 +149,13 @@ StorageDistributed::StorageDistributed(
     sharding_key_column_name(sharding_key_ ? sharding_key_->getColumnName() : String{}),
     path(data_path_.empty() ? "" : (data_path_ + escapeForFileName(table_name) + '/'))
 {
+    /// Sanity check. Skip check if the table is already created to allow the server to start.
+    if (!attach && !cluster_name.empty())
+    {
+        size_t num_local_shards = context.getCluster(cluster_name)->getLocalShardCount();
+        if (num_local_shards && remote_database == database_name && remote_table == table_name)
+            throw Exception("Distributed table " + table_name + " looks at itself", ErrorCodes::INFINITE_LOOP);
+    }
 }
 
 
@@ -158,7 +168,7 @@ StoragePtr StorageDistributed::createWithOwnCluster(
     const Context & context_)
 {
     auto res = ext::shared_ptr_helper<StorageDistributed>::create(
-        name_, columns_, remote_database_, remote_table_, String{}, context_, ASTPtr(), String());
+        String{}, name_, columns_, remote_database_, remote_table_, String{}, context_, ASTPtr(), String(), false);
 
     res->owned_cluster = owned_cluster_;
 
@@ -178,7 +188,9 @@ BlockInputStreams StorageDistributed::read(
 
     const Settings & settings = context.getSettingsRef();
 
-    size_t result_size = (cluster->getRemoteShardCount() * settings.max_parallel_replicas) + cluster->getLocalShardCount();
+    size_t num_local_shards = cluster->getLocalShardCount();
+    size_t num_remote_shards = cluster->getRemoteShardCount();
+    size_t result_size = (num_remote_shards * settings.max_parallel_replicas) + num_local_shards;
 
     processed_stage = result_size == 1 || settings.distributed_group_by_no_merge
         ? QueryProcessingStage::Complete
@@ -199,7 +211,7 @@ BlockInputStreams StorageDistributed::read(
 
 BlockOutputStreamPtr StorageDistributed::write(const ASTPtr & query, const Settings & settings)
 {
-    auto cluster = (owned_cluster) ? owned_cluster : context.getCluster(cluster_name);
+    auto cluster = getCluster();
 
     /// Ban an attempt to make async insert into the table belonging to DatabaseMemory
     if (path.empty() && !owned_cluster && !settings.insert_distributed_sync.value)
@@ -327,13 +339,13 @@ ConnectionPoolPtr StorageDistributed::requireConnectionPool(const std::string &
 
 size_t StorageDistributed::getShardCount() const
 {
-    return getCluster()->getRemoteShardCount();
+    return getCluster()->getShardCount();
 }
 
 
 ClusterPtr StorageDistributed::getCluster() const
 {
-    return (owned_cluster) ? owned_cluster : context.getCluster(cluster_name);
+    return owned_cluster ? owned_cluster : context.getCluster(cluster_name);
 }
 
 void StorageDistributed::ClusterNodeData::requireConnectionPool(const std::string & name, const StorageDistributed & storage)
@@ -400,9 +412,10 @@ void registerStorageDistributed(StorageFactory & factory)
         }
 
         return StorageDistributed::create(
-            args.table_name, args.columns,
+            args.database_name, args.table_name, args.columns,
             remote_database, remote_table, cluster_name,
-            args.context, sharding_key, args.data_path);
+            args.context, sharding_key, args.data_path,
+            args.attach);
     });
 }
 
diff --git a/dbms/src/Storages/StorageDistributed.h b/dbms/src/Storages/StorageDistributed.h
index 451c9f12d8a..41a11fa3199 100644
--- a/dbms/src/Storages/StorageDistributed.h
+++ b/dbms/src/Storages/StorageDistributed.h
@@ -130,14 +130,16 @@ public:
 
 protected:
     StorageDistributed(
-        const std::string & name_,
+        const String & database_name,
+        const String & table_name_,
         const ColumnsDescription & columns_,
         const String & remote_database_,
         const String & remote_table_,
         const String & cluster_name_,
         const Context & context_,
         const ASTPtr & sharding_key_,
-        const String & data_path_);
+        const String & data_path_,
+        bool attach);
 };
 
 }

From e272f6d79e9dbb2652771807d3a124e6ed19fdd4 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 16 Mar 2018 06:03:17 +0300
Subject: [PATCH 123/244] Not a fix [#CLICKHOUSE-3650].

---
 dbms/src/Common/isLocalAddress.h              |   1 -
 .../Functions/FunctionsExternalDictionaries.h | 460 +++++-------------
 2 files changed, 134 insertions(+), 327 deletions(-)

diff --git a/dbms/src/Common/isLocalAddress.h b/dbms/src/Common/isLocalAddress.h
index d90d7aef63a..981d343a97f 100644
--- a/dbms/src/Common/isLocalAddress.h
+++ b/dbms/src/Common/isLocalAddress.h
@@ -12,7 +12,6 @@ namespace Poco
 
 namespace DB
 {
-
     /** Lets you check if the address is similar to `localhost`.
      * The purpose of this check is usually to make an assumption,
      *  that when we go to this address via the Internet, we'll get to ourselves.
diff --git a/dbms/src/Functions/FunctionsExternalDictionaries.h b/dbms/src/Functions/FunctionsExternalDictionaries.h
index b921ba50b9a..abb8c518aa6 100644
--- a/dbms/src/Functions/FunctionsExternalDictionaries.h
+++ b/dbms/src/Functions/FunctionsExternalDictionaries.h
@@ -82,17 +82,13 @@ private:
     DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
     {
         if (!arguments[0]->isString())
-            throw Exception{
-                "Illegal type " + arguments[0]->getName() + " of first argument of function " + getName()
-                    + ", expected a string.",
-                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
+            throw Exception{"Illegal type " + arguments[0]->getName() + " of first argument of function " + getName()
+                + ", expected a string.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
 
         if (!checkDataType<DataTypeUInt64>(arguments[1].get()) &&
             !checkDataType<DataTypeTuple>(arguments[1].get()))
-            throw Exception{
-                "Illegal type " + arguments[1]->getName() + " of second argument of function " + getName()
-                    + ", must be UInt64 or tuple(...).",
-                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
+            throw Exception{"Illegal type " + arguments[1]->getName() + " of second argument of function " + getName()
+                + ", must be UInt64 or tuple(...).", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
 
         return std::make_shared<DataTypeUInt8>();
     }
@@ -103,9 +99,7 @@ private:
     {
         const auto dict_name_col = checkAndGetColumnConst<ColumnString>(block.getByPosition(arguments[0]).column.get());
         if (!dict_name_col)
-            throw Exception{
-                "First argument of function " + getName() + " must be a constant string",
-                ErrorCodes::ILLEGAL_COLUMN};
+            throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN};
 
         auto dict = dictionaries.getDictionary(dict_name_col->getValue<String>());
         const auto dict_ptr = dict.get();
@@ -116,9 +110,7 @@ private:
             !executeDispatchComplex<ComplexKeyHashedDictionary>(block, arguments, result, dict_ptr) &&
             !executeDispatchComplex<ComplexKeyCacheDictionary>(block, arguments, result, dict_ptr) &&
             !executeDispatchComplex<TrieDictionary>(block, arguments, result, dict_ptr))
-            throw Exception{
-                "Unsupported dictionary type " + dict_ptr->getTypeName(),
-                ErrorCodes::UNKNOWN_TYPE};
+            throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE};
     }
 
     template <typename DictionaryType>
@@ -139,9 +131,7 @@ private:
             block.getByPosition(result).column = std::move(out);
         }
         else
-            throw Exception{
-                "Second argument of function " + getName() + " must be UInt64",
-                ErrorCodes::ILLEGAL_COLUMN};
+            throw Exception{"Second argument of function " + getName() + " must be UInt64", ErrorCodes::ILLEGAL_COLUMN};
 
         return true;
     }
@@ -167,9 +157,7 @@ private:
             block.getByPosition(result).column = std::move(out);
         }
         else
-            throw Exception{
-                "Second argument of function " + getName() + " must be " + dict->getKeyDescription(),
-                ErrorCodes::TYPE_MISMATCH};
+            throw Exception{"Second argument of function " + getName() + " must be " + dict->getKeyDescription(), ErrorCodes::TYPE_MISMATCH};
 
         return true;
     }
@@ -185,15 +173,11 @@ static bool isDictGetFunctionInjective(const ExternalDictionaries & dictionaries
 
     const auto dict_name_col = checkAndGetColumnConst<ColumnString>(sample_block.getByPosition(0).column.get());
     if (!dict_name_col)
-        throw Exception{
-            "First argument of function dictGet... must be a constant string",
-            ErrorCodes::ILLEGAL_COLUMN};
+        throw Exception{"First argument of function dictGet... must be a constant string", ErrorCodes::ILLEGAL_COLUMN};
 
     const auto attr_name_col = checkAndGetColumnConst<ColumnString>(sample_block.getByPosition(1).column.get());
     if (!attr_name_col)
-        throw Exception{
-            "Second argument of function dictGet... must be a constant string",
-            ErrorCodes::ILLEGAL_COLUMN};
+        throw Exception{"Second argument of function dictGet... must be a constant string", ErrorCodes::ILLEGAL_COLUMN};
 
     return dictionaries.getDictionary(dict_name_col->getValue<String>())->isInjective(attr_name_col->getValue<String>());
 }
@@ -217,6 +201,9 @@ private:
     bool isVariadic() const override { return true; }
     size_t getNumberOfArguments() const override { return 0; }
 
+    bool useDefaultImplementationForConstants() const final { return true; }
+    ColumnNumbers getArgumentsThatAreAlwaysConstant() const final { return {0, 1}; }
+
     bool isInjective(const Block & sample_block) override
     {
         return isDictGetFunctionInjective(dictionaries, sample_block);
@@ -225,42 +212,33 @@ private:
     DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
     {
         if (arguments.size() != 3 && arguments.size() != 4)
-            throw Exception{
-                "Number of arguments for function " + getName() + " doesn't match: passed "
-                    + toString(arguments.size()) + ", should be 3 or 4.",
-                ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH};
+            throw Exception{"Number of arguments for function " + getName() + " doesn't match: passed "
+                + toString(arguments.size()) + ", should be 3 or 4.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH};
 
         if (!arguments[0]->isString())
         {
-            throw Exception{
-                "Illegal type " + arguments[0]->getName() + " of first argument of function " + getName()
-                    + ", expected a string.",
-                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
+            throw Exception{"Illegal type " + arguments[0]->getName() + " of first argument of function " + getName()
+                + ", expected a string.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
         }
 
         if (!arguments[1]->isString())
         {
-            throw Exception{
-                "Illegal type " + arguments[1]->getName() + " of second argument of function " + getName()
-                    + ", expected a string.",
-                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
+            throw Exception{"Illegal type " + arguments[1]->getName() + " of second argument of function " + getName()
+                + ", expected a string.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
         }
 
         if (!checkDataType<DataTypeUInt64>(arguments[2].get()) &&
             !checkDataType<DataTypeTuple>(arguments[2].get()))
         {
-            throw Exception{
-                "Illegal type " + arguments[2]->getName() + " of third argument of function " + getName()
-                    + ", must be UInt64 or tuple(...).",
-                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
+            throw Exception{"Illegal type " + arguments[2]->getName() + " of third argument of function " + getName()
+                + ", must be UInt64 or tuple(...).", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
         }
 
+        /// This is for the case of range dictionaries.
         if (arguments.size() == 4 && !checkDataType<DataTypeDate>(arguments[3].get()))
         {
-            throw Exception{
-                "Illegal type " + arguments[3]->getName() + " of fourth argument of function " + getName()
-                    + ", must be Date.",
-                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
+            throw Exception{"Illegal type " + arguments[3]->getName() + " of fourth argument of function " + getName()
+                + ", must be Date.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
         }
 
         return std::make_shared<DataTypeString>();
@@ -272,9 +250,7 @@ private:
     {
         const auto dict_name_col = checkAndGetColumnConst<ColumnString>(block.getByPosition(arguments[0]).column.get());
         if (!dict_name_col)
-            throw Exception{
-                "First argument of function " + getName() + " must be a constant string",
-                ErrorCodes::ILLEGAL_COLUMN};
+            throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN};
 
         auto dict = dictionaries.getDictionary(dict_name_col->getValue<String>());
         const auto dict_ptr = dict.get();
@@ -286,9 +262,7 @@ private:
             !executeDispatchComplex<ComplexKeyCacheDictionary>(block, arguments, result, dict_ptr) &&
             !executeDispatchComplex<TrieDictionary>(block, arguments, result, dict_ptr) &&
             !executeDispatchRange<RangeHashedDictionary>(block, arguments, result, dict_ptr))
-            throw Exception{
-                "Unsupported dictionary type " + dict_ptr->getTypeName(),
-                ErrorCodes::UNKNOWN_TYPE};
+            throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE};
     }
 
     template <typename DictionaryType>
@@ -300,16 +274,12 @@ private:
             return false;
 
         if (arguments.size() != 3)
-            throw Exception{
-                "Function " + getName() + " for dictionary of type " + dict->getTypeName() +
-                " requires exactly 3 arguments",
-                ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH};
+            throw Exception{"Function " + getName() + " for dictionary of type " + dict->getTypeName() +
+                " requires exactly 3 arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH};
 
         const auto attr_name_col = checkAndGetColumnConst<ColumnString>(block.getByPosition(arguments[1]).column.get());
         if (!attr_name_col)
-            throw Exception{
-                "Second argument of function " + getName() + " must be a constant string",
-                ErrorCodes::ILLEGAL_COLUMN};
+            throw Exception{"Second argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN};
 
         String attr_name = attr_name_col->getValue<String>();
 
@@ -320,19 +290,8 @@ private:
             dict->getString(attr_name, id_col->getData(), out.get());
             block.getByPosition(result).column = std::move(out);
         }
-        else if (const auto id_col = checkAndGetColumnConst<ColumnVector<UInt64>>(id_col_untyped))
-        {
-            const PaddedPODArray<UInt64> ids(1, id_col->getValue<UInt64>());
-            auto out = ColumnString::create();
-            dict->getString(attr_name, ids, out.get());
-            block.getByPosition(result).column = DataTypeString().createColumnConst(id_col->size(), out->getDataAt(0).toString());
-        }
         else
-        {
-            throw Exception{
-                "Third argument of function " + getName() + " must be UInt64",
-                ErrorCodes::ILLEGAL_COLUMN};
-        }
+            throw Exception{"Third argument of function " + getName() + " must be UInt64", ErrorCodes::ILLEGAL_COLUMN};
 
         return true;
     }
@@ -346,16 +305,12 @@ private:
             return false;
 
         if (arguments.size() != 3)
-            throw Exception{
-                "Function " + getName() + " for dictionary of type " + dict->getTypeName() +
-                    " requires exactly 3 arguments",
-                ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH};
+            throw Exception{"Function " + getName() + " for dictionary of type " + dict->getTypeName() +
+                " requires exactly 3 arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH};
 
         const auto attr_name_col = checkAndGetColumnConst<ColumnString>(block.getByPosition(arguments[1]).column.get());
         if (!attr_name_col)
-            throw Exception{
-                "Second argument of function " + getName() + " must be a constant string",
-                ErrorCodes::ILLEGAL_COLUMN};
+            throw Exception{"Second argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN};
 
         String attr_name = attr_name_col->getValue<String>();
 
@@ -376,9 +331,7 @@ private:
             block.getByPosition(result).column = std::move(out);
         }
         else
-            throw Exception{
-                "Third argument of function " + getName() + " must be " + dict->getKeyDescription(),
-                ErrorCodes::TYPE_MISMATCH};
+            throw Exception{"Third argument of function " + getName() + " must be " + dict->getKeyDescription(), ErrorCodes::TYPE_MISMATCH};
 
         return true;
     }
@@ -392,16 +345,12 @@ private:
             return false;
 
         if (arguments.size() != 4)
-            throw Exception{
-                "Function " + getName() + " for dictionary of type " + dict->getTypeName() +
-                " requires exactly 4 arguments",
-                ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH};
+            throw Exception{"Function " + getName() + " for dictionary of type " + dict->getTypeName() +
+                " requires exactly 4 arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH};
 
         const auto attr_name_col = checkAndGetColumnConst<ColumnString>(block.getByPosition(arguments[1]).column.get());
         if (!attr_name_col)
-            throw Exception{
-                "Second argument of function " + getName() + " must be a constant string",
-                ErrorCodes::ILLEGAL_COLUMN};
+            throw Exception{"Second argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN};
 
         String attr_name = attr_name_col->getValue<String>();
 
@@ -413,9 +362,7 @@ private:
             executeRange(block, result, dict, attr_name, id_col, date_col_untyped);
         else
         {
-            throw Exception{
-                "Third argument of function " + getName() + " must be UInt64",
-                ErrorCodes::ILLEGAL_COLUMN};
+            throw Exception{"Third argument of function " + getName() + " must be UInt64", ErrorCodes::ILLEGAL_COLUMN};
         }
 
         return true;
@@ -441,9 +388,7 @@ private:
         }
         else
         {
-            throw Exception{
-                "Fourth argument of function " + getName() + " must be Date",
-                ErrorCodes::ILLEGAL_COLUMN};
+            throw Exception{"Fourth argument of function " + getName() + " must be Date", ErrorCodes::ILLEGAL_COLUMN};
         }
     }
 
@@ -470,9 +415,7 @@ private:
         }
         else
         {
-            throw Exception{
-                "Fourth argument of function " + getName() + " must be Date",
-                ErrorCodes::ILLEGAL_COLUMN};
+            throw Exception{"Fourth argument of function " + getName() + " must be Date", ErrorCodes::ILLEGAL_COLUMN};
         }
     }
 
@@ -497,34 +440,29 @@ public:
 private:
     size_t getNumberOfArguments() const override { return 4; }
 
+    bool useDefaultImplementationForConstants() const final { return true; }
+    ColumnNumbers getArgumentsThatAreAlwaysConstant() const final { return {0, 1}; }
+
     DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
     {
         if (!arguments[0]->isString())
-            throw Exception{
-                "Illegal type " + arguments[0]->getName() + " of first argument of function " + getName() +
-                    ", expected a string.",
-                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
+            throw Exception{"Illegal type " + arguments[0]->getName() + " of first argument of function " + getName() +
+                ", expected a string.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
 
         if (!arguments[1]->isString())
-            throw Exception{
-                "Illegal type " + arguments[1]->getName() + " of second argument of function " + getName() +
-                    ", expected a string.",
-                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
+            throw Exception{"Illegal type " + arguments[1]->getName() + " of second argument of function " + getName() +
+                ", expected a string.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
 
         if (!checkDataType<DataTypeUInt64>(arguments[2].get()) &&
             !checkDataType<DataTypeTuple>(arguments[2].get()))
         {
-            throw Exception{
-                "Illegal type " + arguments[2]->getName() + " of third argument of function " + getName()
-                    + ", must be UInt64 or tuple(...).",
-                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
+            throw Exception{"Illegal type " + arguments[2]->getName() + " of third argument of function " + getName()
+                + ", must be UInt64 or tuple(...).", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
         }
 
         if (!arguments[3]->isString())
-            throw Exception{
-                "Illegal type " + arguments[3]->getName() + " of fourth argument of function " + getName() +
-                    ", must be String.",
-                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
+            throw Exception{"Illegal type " + arguments[3]->getName() + " of fourth argument of function " + getName() +
+                ", must be String.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
 
         return std::make_shared<DataTypeString>();
     }
@@ -535,9 +473,7 @@ private:
     {
         const auto dict_name_col = checkAndGetColumnConst<ColumnString>(block.getByPosition(arguments[0]).column.get());
         if (!dict_name_col)
-            throw Exception{
-                "First argument of function " + getName() + " must be a constant string",
-                ErrorCodes::ILLEGAL_COLUMN};
+            throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN};
 
         auto dict = dictionaries.getDictionary(dict_name_col->getValue<String>());
         const auto dict_ptr = dict.get();
@@ -548,9 +484,7 @@ private:
             !executeDispatchComplex<ComplexKeyHashedDictionary>(block, arguments, result, dict_ptr) &&
             !executeDispatchComplex<ComplexKeyCacheDictionary>(block, arguments, result, dict_ptr) &&
             !executeDispatchComplex<TrieDictionary>(block, arguments, result, dict_ptr))
-            throw Exception{
-                "Unsupported dictionary type " + dict_ptr->getTypeName(),
-                ErrorCodes::UNKNOWN_TYPE};
+            throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE};
     }
 
     template <typename DictionaryType>
@@ -563,9 +497,7 @@ private:
 
         const auto attr_name_col = checkAndGetColumnConst<ColumnString>(block.getByPosition(arguments[1]).column.get());
         if (!attr_name_col)
-            throw Exception{
-                "Second argument of function " + getName() + " must be a constant string",
-                    ErrorCodes::ILLEGAL_COLUMN};
+            throw Exception{"Second argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN};
 
         String attr_name = attr_name_col->getValue<String>();
 
@@ -575,9 +507,7 @@ private:
         else if (const auto id_col = checkAndGetColumnConst<ColumnVector<UInt64>>(id_col_untyped))
             executeDispatch(block, arguments, result, dict, attr_name, id_col);
         else
-            throw Exception{
-                "Third argument of function " + getName() + " must be UInt64",
-                ErrorCodes::ILLEGAL_COLUMN};
+            throw Exception{"Third argument of function " + getName() + " must be UInt64", ErrorCodes::ILLEGAL_COLUMN};
 
         return true;
     }
@@ -607,9 +537,7 @@ private:
             block.getByPosition(result).column = std::move(out);
         }
         else
-            throw Exception{
-                "Fourth argument of function " + getName() + " must be String",
-                ErrorCodes::ILLEGAL_COLUMN};
+            throw Exception{"Fourth argument of function " + getName() + " must be String", ErrorCodes::ILLEGAL_COLUMN};
     }
 
     template <typename DictionaryType>
@@ -638,9 +566,7 @@ private:
             block.getByPosition(result).column = DataTypeString().createColumnConst(id_col->size(), out->getDataAt(0).toString());
         }
         else
-            throw Exception{
-                "Fourth argument of function " + getName() + " must be String",
-                ErrorCodes::ILLEGAL_COLUMN};
+            throw Exception{"Fourth argument of function " + getName() + " must be String", ErrorCodes::ILLEGAL_COLUMN};
     }
 
     template <typename DictionaryType>
@@ -653,9 +579,7 @@ private:
 
         const auto attr_name_col = checkAndGetColumnConst<ColumnString>(block.getByPosition(arguments[1]).column.get());
         if (!attr_name_col)
-            throw Exception{
-                "Second argument of function " + getName() + " must be a constant string",
-                    ErrorCodes::ILLEGAL_COLUMN};
+            throw Exception{"Second argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN};
 
         String attr_name = attr_name_col->getValue<String>();
 
@@ -682,9 +606,7 @@ private:
             dict->getString(attr_name, key_columns, key_types, def, out.get());
         }
         else
-            throw Exception{
-                "Fourth argument of function " + getName() + " must be String",
-                ErrorCodes::ILLEGAL_COLUMN};
+            throw Exception{"Fourth argument of function " + getName() + " must be String", ErrorCodes::ILLEGAL_COLUMN};
 
         block.getByPosition(result).column = std::move(out);
         return true;
@@ -771,6 +693,9 @@ private:
     bool isVariadic() const override { return true; }
     size_t getNumberOfArguments() const override { return 0; }
 
+    bool useDefaultImplementationForConstants() const final { return true; }
+    ColumnNumbers getArgumentsThatAreAlwaysConstant() const final { return {0, 1}; }
+
     bool isInjective(const Block & sample_block) override
     {
         return isDictGetFunctionInjective(dictionaries, sample_block);
@@ -782,37 +707,21 @@ private:
             throw Exception{"Function " + getName() + " takes 3 or 4 arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH};
 
         if (!arguments[0]->isString())
-        {
-            throw Exception{
-                "Illegal type " + arguments[0]->getName() + " of first argument of function " + getName()
-                    + ", expected a string.",
-                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
-        }
+            throw Exception{"Illegal type " + arguments[0]->getName() + " of first argument of function " + getName()
+                + ", expected a string.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
 
         if (!arguments[1]->isString())
-        {
-            throw Exception{
-                "Illegal type " + arguments[1]->getName() + " of second argument of function " + getName()
-                    + ", expected a string.",
-                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
-        }
+            throw Exception{"Illegal type " + arguments[1]->getName() + " of second argument of function " + getName()
+                + ", expected a string.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
 
         if (!checkDataType<DataTypeUInt64>(arguments[2].get()) &&
             !checkDataType<DataTypeTuple>(arguments[2].get()))
-        {
-            throw Exception{
-                "Illegal type " + arguments[2]->getName() + " of third argument of function " + getName()
-                    + ", must be UInt64 or tuple(...).",
-                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
-        }
+            throw Exception{"Illegal type " + arguments[2]->getName() + " of third argument of function " + getName()
+                + ", must be UInt64 or tuple(...).", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
 
         if (arguments.size() == 4 && !checkDataType<DataTypeDate>(arguments[3].get()))
-        {
-            throw Exception{
-                "Illegal type " + arguments[3]->getName() + " of fourth argument of function " + getName()
-                + ", must be Date.",
-                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
-        }
+            throw Exception{"Illegal type " + arguments[3]->getName() + " of fourth argument of function " + getName()
+                + ", must be Date.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
 
         return std::make_shared<DataType>();
     }
@@ -823,9 +732,7 @@ private:
     {
         const auto dict_name_col = checkAndGetColumnConst<ColumnString>(block.getByPosition(arguments[0]).column.get());
         if (!dict_name_col)
-            throw Exception{
-                "First argument of function " + getName() + " must be a constant string",
-                ErrorCodes::ILLEGAL_COLUMN};
+            throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN};
 
         auto dict = dictionaries.getDictionary(dict_name_col->getValue<String>());
         const auto dict_ptr = dict.get();
@@ -837,9 +744,7 @@ private:
             !executeDispatchComplex<ComplexKeyCacheDictionary>(block, arguments, result, dict_ptr) &&
             !executeDispatchComplex<TrieDictionary>(block, arguments, result, dict_ptr) &&
             !executeDispatchRange<RangeHashedDictionary>(block, arguments, result, dict_ptr))
-            throw Exception{
-                "Unsupported dictionary type " + dict_ptr->getTypeName(),
-                ErrorCodes::UNKNOWN_TYPE};
+            throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE};
     }
 
     template <typename DictionaryType>
@@ -851,16 +756,12 @@ private:
             return false;
 
         if (arguments.size() != 3)
-            throw Exception{
-                "Function " + getName() + " for dictionary of type " + dict->getTypeName() +
-                " requires exactly 3 arguments.",
-                ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH};
+            throw Exception{"Function " + getName() + " for dictionary of type " + dict->getTypeName() +
+                " requires exactly 3 arguments.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH};
 
         const auto attr_name_col = checkAndGetColumnConst<ColumnString>(block.getByPosition(arguments[1]).column.get());
         if (!attr_name_col)
-            throw Exception{
-                "Second argument of function " + getName() + " must be a constant string",
-                ErrorCodes::ILLEGAL_COLUMN};
+            throw Exception{"Second argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN};
 
         String attr_name = attr_name_col->getValue<String>();
 
@@ -881,11 +782,7 @@ private:
             block.getByPosition(result).column = DataTypeNumber<Type>().createColumnConst(id_col->size(), toField(data.front()));
         }
         else
-        {
-            throw Exception{
-                "Third argument of function " + getName() + " must be UInt64",
-                ErrorCodes::ILLEGAL_COLUMN};
-        }
+            throw Exception{"Third argument of function " + getName() + " must be UInt64", ErrorCodes::ILLEGAL_COLUMN};
 
         return true;
     }
@@ -899,16 +796,12 @@ private:
             return false;
 
         if (arguments.size() != 3)
-            throw Exception{
-                "Function " + getName() + " for dictionary of type " + dict->getTypeName() +
-                    " requires exactly 3 arguments",
-                ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH};
+            throw Exception{"Function " + getName() + " for dictionary of type " + dict->getTypeName() +
+                " requires exactly 3 arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH};
 
         const auto attr_name_col = checkAndGetColumnConst<ColumnString>(block.getByPosition(arguments[1]).column.get());
         if (!attr_name_col)
-            throw Exception{
-                "Second argument of function " + getName() + " must be a constant string",
-                ErrorCodes::ILLEGAL_COLUMN};
+            throw Exception{"Second argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN};
 
         String attr_name = attr_name_col->getValue<String>();
 
@@ -930,9 +823,7 @@ private:
             block.getByPosition(result).column = std::move(out);
         }
         else
-            throw Exception{
-                "Third argument of function " + getName() + " must be " + dict->getKeyDescription(),
-                ErrorCodes::TYPE_MISMATCH};
+            throw Exception{"Third argument of function " + getName() + " must be " + dict->getKeyDescription(), ErrorCodes::TYPE_MISMATCH};
 
         return true;
     }
@@ -946,16 +837,12 @@ private:
             return false;
 
         if (arguments.size() != 4)
-            throw Exception{
-                "Function " + getName() + " for dictionary of type " + dict->getTypeName() +
-                " requires exactly 4 arguments",
-                ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH};
+            throw Exception{"Function " + getName() + " for dictionary of type " + dict->getTypeName() +
+                " requires exactly 4 arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH};
 
         const auto attr_name_col = checkAndGetColumnConst<ColumnString>(block.getByPosition(arguments[1]).column.get());
         if (!attr_name_col)
-            throw Exception{
-                "Second argument of function " + getName() + " must be a constant string",
-                ErrorCodes::ILLEGAL_COLUMN};
+            throw Exception{"Second argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN};
 
         String attr_name = attr_name_col->getValue<String>();
 
@@ -966,11 +853,7 @@ private:
         else if (const auto id_col = checkAndGetColumnConst<ColumnVector<UInt64>>(id_col_untyped))
             executeRange(block, result, dict, attr_name, id_col, date_col_untyped);
         else
-        {
-            throw Exception{
-                "Third argument of function " + getName() + " must be UInt64",
-                ErrorCodes::ILLEGAL_COLUMN};
-        }
+            throw Exception{"Third argument of function " + getName() + " must be UInt64", ErrorCodes::ILLEGAL_COLUMN};
 
         return true;
     }
@@ -1003,11 +886,7 @@ private:
             block.getByPosition(result).column = std::move(out);
         }
         else
-        {
-            throw Exception{
-                "Fourth argument of function " + getName() + " must be Date",
-                ErrorCodes::ILLEGAL_COLUMN};
-        }
+            throw Exception{"Fourth argument of function " + getName() + " must be Date", ErrorCodes::ILLEGAL_COLUMN};
     }
 
     template <typename DictionaryType>
@@ -1035,11 +914,7 @@ private:
             block.getByPosition(result).column = DataTypeNumber<Type>().createColumnConst(id_col->size(), toField(data.front()));
         }
         else
-        {
-            throw Exception{
-                "Fourth argument of function " + getName() + " must be Date",
-                ErrorCodes::ILLEGAL_COLUMN};
-        }
+            throw Exception{"Fourth argument of function " + getName() + " must be Date", ErrorCodes::ILLEGAL_COLUMN};
     }
 
     const ExternalDictionaries & dictionaries;
@@ -1094,40 +969,27 @@ public:
 private:
     size_t getNumberOfArguments() const override { return 4; }
 
+    bool useDefaultImplementationForConstants() const final { return true; }
+    ColumnNumbers getArgumentsThatAreAlwaysConstant() const final { return {0, 1}; }
+
     DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
     {
         if (!arguments[0]->isString())
-        {
-            throw Exception{
-                "Illegal type " + arguments[0]->getName() + " of first argument of function " + getName()
-                    + ", expected a string.",
-                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
-        }
+            throw Exception{"Illegal type " + arguments[0]->getName() + " of first argument of function " + getName()
+                + ", expected a string.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
 
         if (!arguments[1]->isString())
-        {
-            throw Exception{
-                "Illegal type " + arguments[1]->getName() + " of second argument of function " + getName()
-                    + ", expected a string.",
-                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
-        }
+            throw Exception{"Illegal type " + arguments[1]->getName() + " of second argument of function " + getName()
+                + ", expected a string.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
 
         if (!checkDataType<DataTypeUInt64>(arguments[2].get()) &&
             !checkDataType<DataTypeTuple>(arguments[2].get()))
-        {
-            throw Exception{
-                "Illegal type " + arguments[2]->getName() + " of third argument of function " + getName()
-                    + ", must be UInt64 or tuple(...).",
-                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
-        }
+            throw Exception{"Illegal type " + arguments[2]->getName() + " of third argument of function " + getName()
+                + ", must be UInt64 or tuple(...).", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
 
         if (!checkDataType<DataType>(arguments[3].get()))
-        {
-            throw Exception{
-                "Illegal type " + arguments[3]->getName() + " of fourth argument of function " + getName()
-                    + ", must be " + String(DataType{}.getFamilyName()) + ".",
-                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
-        }
+            throw Exception{"Illegal type " + arguments[3]->getName() + " of fourth argument of function " + getName()
+                + ", must be " + String(DataType{}.getFamilyName()) + ".", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
 
         return std::make_shared<DataType>();
     }
@@ -1138,9 +1000,7 @@ private:
     {
         const auto dict_name_col = checkAndGetColumnConst<ColumnString>(block.getByPosition(arguments[0]).column.get());
         if (!dict_name_col)
-            throw Exception{
-                "First argument of function " + getName() + " must be a constant string",
-                ErrorCodes::ILLEGAL_COLUMN};
+            throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN};
 
         auto dict = dictionaries.getDictionary(dict_name_col->getValue<String>());
         const auto dict_ptr = dict.get();
@@ -1151,9 +1011,7 @@ private:
             !executeDispatchComplex<ComplexKeyHashedDictionary>(block, arguments, result, dict_ptr) &&
             !executeDispatchComplex<ComplexKeyCacheDictionary>(block, arguments, result, dict_ptr) &&
             !executeDispatchComplex<TrieDictionary>(block, arguments, result, dict_ptr))
-            throw Exception{
-                "Unsupported dictionary type " + dict_ptr->getTypeName(),
-                ErrorCodes::UNKNOWN_TYPE};
+            throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE};
     }
 
     template <typename DictionaryType>
@@ -1166,9 +1024,7 @@ private:
 
         const auto attr_name_col = checkAndGetColumnConst<ColumnString>(block.getByPosition(arguments[1]).column.get());
         if (!attr_name_col)
-            throw Exception{
-                "Second argument of function " + getName() + " must be a constant string",
-                ErrorCodes::ILLEGAL_COLUMN};
+            throw Exception{"Second argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN};
 
         String attr_name = attr_name_col->getValue<String>();
 
@@ -1178,9 +1034,7 @@ private:
         else if (const auto id_col = checkAndGetColumnConst<ColumnVector<UInt64>>(id_col_untyped))
             executeDispatch(block, arguments, result, dict, attr_name, id_col);
         else
-            throw Exception{
-                "Third argument of function " + getName() + " must be UInt64",
-                ErrorCodes::ILLEGAL_COLUMN};
+            throw Exception{"Third argument of function " + getName() + " must be UInt64", ErrorCodes::ILLEGAL_COLUMN};
 
         return true;
     }
@@ -1213,9 +1067,7 @@ private:
             block.getByPosition(result).column = std::move(out);
         }
         else
-            throw Exception{
-                "Fourth argument of function " + getName() + " must be " + String(DataType{}.getFamilyName()),
-                ErrorCodes::ILLEGAL_COLUMN};
+            throw Exception{"Fourth argument of function " + getName() + " must be " + String(DataType{}.getFamilyName()), ErrorCodes::ILLEGAL_COLUMN};
     }
 
     template <typename DictionaryType>
@@ -1247,9 +1099,7 @@ private:
             block.getByPosition(result).column = DataTypeNumber<Type>().createColumnConst(id_col->size(), toField(data.front()));
         }
         else
-            throw Exception{
-                "Fourth argument of function " + getName() + " must be " + String(DataType{}.getFamilyName()),
-                ErrorCodes::ILLEGAL_COLUMN};
+            throw Exception{"Fourth argument of function " + getName() + " must be " + String(DataType{}.getFamilyName()), ErrorCodes::ILLEGAL_COLUMN};
     }
 
     template <typename DictionaryType>
@@ -1262,9 +1112,7 @@ private:
 
         const auto attr_name_col = checkAndGetColumnConst<ColumnString>(block.getByPosition(arguments[1]).column.get());
         if (!attr_name_col)
-            throw Exception{
-                "Second argument of function " + getName() + " must be a constant string",
-                ErrorCodes::ILLEGAL_COLUMN};
+            throw Exception{"Second argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN};
 
         String attr_name = attr_name_col->getValue<String>();
 
@@ -1298,9 +1146,7 @@ private:
             DictGetTraits<DataType>::getOrDefault(dict, attr_name, key_columns, key_types, def, data);
         }
         else
-            throw Exception{
-                "Fourth argument of function " + getName() + " must be " + String(DataType{}.getFamilyName()),
-                ErrorCodes::ILLEGAL_COLUMN};
+            throw Exception{"Fourth argument of function " + getName() + " must be " + String(DataType{}.getFamilyName()), ErrorCodes::ILLEGAL_COLUMN};
 
         block.getByPosition(result).column = std::move(out);
         return true;
@@ -1358,23 +1204,18 @@ private:
     size_t getNumberOfArguments() const override { return 2; }
     bool isInjective(const Block & /*sample_block*/) override { return true; }
 
+    bool useDefaultImplementationForConstants() const final { return true; }
+    ColumnNumbers getArgumentsThatAreAlwaysConstant() const final { return {0}; }
+
     DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
     {
         if (!arguments[0]->isString())
-        {
-            throw Exception{
-                "Illegal type " + arguments[0]->getName() + " of first argument of function " + getName()
-                    + ", expected a string.",
-                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
-        }
+            throw Exception{"Illegal type " + arguments[0]->getName() + " of first argument of function " + getName()
+                + ", expected a string.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
 
         if (!checkDataType<DataTypeUInt64>(arguments[1].get()))
-        {
-            throw Exception{
-                "Illegal type " + arguments[1]->getName() + " of second argument of function " + getName()
-                    + ", must be UInt64.",
-                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
-        }
+            throw Exception{"Illegal type " + arguments[1]->getName() + " of second argument of function " + getName()
+                + ", must be UInt64.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
 
         return std::make_shared<DataTypeArray>(std::make_shared<DataTypeUInt64>());
     }
@@ -1385,9 +1226,7 @@ private:
     {
         const auto dict_name_col = checkAndGetColumnConst<ColumnString>(block.getByPosition(arguments[0]).column.get());
         if (!dict_name_col)
-            throw Exception{
-                "First argument of function " + getName() + " must be a constant string",
-                ErrorCodes::ILLEGAL_COLUMN};
+            throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN};
 
         auto dict = dictionaries.getDictionary(dict_name_col->getValue<String>());
         const auto dict_ptr = dict.get();
@@ -1395,9 +1234,7 @@ private:
         if (!executeDispatch<FlatDictionary>(block, arguments, result, dict_ptr) &&
             !executeDispatch<HashedDictionary>(block, arguments, result, dict_ptr) &&
             !executeDispatch<CacheDictionary>(block, arguments, result, dict_ptr))
-            throw Exception{
-                "Unsupported dictionary type " + dict_ptr->getTypeName(),
-                ErrorCodes::UNKNOWN_TYPE};
+            throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE};
     }
 
     template <typename DictionaryType>
@@ -1409,9 +1246,7 @@ private:
             return false;
 
         if (!dict->hasHierarchy())
-            throw Exception{
-                "Dictionary does not have a hierarchy",
-                ErrorCodes::UNSUPPORTED_METHOD};
+            throw Exception{"Dictionary does not have a hierarchy", ErrorCodes::UNSUPPORTED_METHOD};
 
         const auto get_hierarchies = [&] (const PaddedPODArray<UInt64> & in, PaddedPODArray<UInt64> & out, PaddedPODArray<UInt64> & offsets)
         {
@@ -1492,11 +1327,7 @@ private:
             block.getByPosition(result).column = block.getByPosition(result).type->createColumnConst(id_col->size(), (*array)[0].get<Array>());
         }
         else
-        {
-            throw Exception{
-                "Second argument of function " + getName() + " must be UInt64",
-                ErrorCodes::ILLEGAL_COLUMN};
-        }
+            throw Exception{"Second argument of function " + getName() + " must be UInt64", ErrorCodes::ILLEGAL_COLUMN};
 
         return true;
     }
@@ -1522,31 +1353,22 @@ public:
 private:
     size_t getNumberOfArguments() const override { return 3; }
 
+    bool useDefaultImplementationForConstants() const final { return true; }
+    ColumnNumbers getArgumentsThatAreAlwaysConstant() const final { return {0}; }
+
     DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
     {
         if (!arguments[0]->isString())
-        {
-            throw Exception{
-                "Illegal type " + arguments[0]->getName() + " of first argument of function " + getName()
-                    + ", expected a string.",
-                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
-        }
+            throw Exception{"Illegal type " + arguments[0]->getName() + " of first argument of function " + getName()
+                + ", expected a string.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
 
         if (!checkDataType<DataTypeUInt64>(arguments[1].get()))
-        {
-            throw Exception{
-                "Illegal type " + arguments[1]->getName() + " of second argument of function " + getName()
-                    + ", must be UInt64.",
-                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
-        }
+            throw Exception{"Illegal type " + arguments[1]->getName() + " of second argument of function " + getName()
+                + ", must be UInt64.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
 
         if (!checkDataType<DataTypeUInt64>(arguments[2].get()))
-        {
-            throw Exception{
-                "Illegal type " + arguments[2]->getName() + " of third argument of function " + getName()
-                    + ", must be UInt64.",
-                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
-        }
+            throw Exception{"Illegal type " + arguments[2]->getName() + " of third argument of function " + getName()
+                + ", must be UInt64.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
 
         return std::make_shared<DataTypeUInt8>();
     }
@@ -1557,9 +1379,7 @@ private:
     {
         const auto dict_name_col = checkAndGetColumnConst<ColumnString>(block.getByPosition(arguments[0]).column.get());
         if (!dict_name_col)
-            throw Exception{
-                "First argument of function " + getName() + " must be a constant string",
-                ErrorCodes::ILLEGAL_COLUMN};
+            throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN};
 
         auto dict = dictionaries.getDictionary(dict_name_col->getValue<String>());
         const auto dict_ptr = dict.get();
@@ -1567,9 +1387,7 @@ private:
         if (!executeDispatch<FlatDictionary>(block, arguments, result, dict_ptr)
             && !executeDispatch<HashedDictionary>(block, arguments, result, dict_ptr)
             && !executeDispatch<CacheDictionary>(block, arguments, result, dict_ptr))
-            throw Exception{
-                "Unsupported dictionary type " + dict_ptr->getTypeName(),
-                ErrorCodes::UNKNOWN_TYPE};
+            throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE};
     }
 
     template <typename DictionaryType>
@@ -1581,9 +1399,7 @@ private:
             return false;
 
         if (!dict->hasHierarchy())
-            throw Exception{
-                "Dictionary does not have a hierarchy",
-                ErrorCodes::UNSUPPORTED_METHOD};
+            throw Exception{"Dictionary does not have a hierarchy", ErrorCodes::UNSUPPORTED_METHOD};
 
         const auto child_id_col_untyped = block.getByPosition(arguments[1]).column.get();
         const auto ancestor_id_col_untyped = block.getByPosition(arguments[2]).column.get();
@@ -1593,10 +1409,8 @@ private:
         else if (const auto child_id_col = checkAndGetColumnConst<ColumnVector<UInt64>>(child_id_col_untyped))
             execute(block, result, dict, child_id_col, ancestor_id_col_untyped);
         else
-            throw Exception{
-                "Illegal column " + child_id_col_untyped->getName()
-                    + " of second argument of function " + getName(),
-                ErrorCodes::ILLEGAL_COLUMN};
+            throw Exception{"Illegal column " + child_id_col_untyped->getName()
+                + " of second argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN};
 
         return true;
     }
@@ -1633,10 +1447,8 @@ private:
         }
         else
         {
-            throw Exception{
-                "Illegal column " + ancestor_id_col_untyped->getName()
-                    + " of third argument of function " + getName(),
-                ErrorCodes::ILLEGAL_COLUMN};
+            throw Exception{"Illegal column " + ancestor_id_col_untyped->getName()
+                + " of third argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN};
         }
 
         return true;
@@ -1669,12 +1481,8 @@ private:
             block.getByPosition(result).column = DataTypeUInt8().createColumnConst(child_id_col->size(), UInt64(res));
         }
         else
-        {
-            throw Exception{
-                "Illegal column " + ancestor_id_col_untyped->getName()
-                    + " of third argument of function " + getName(),
-                ErrorCodes::ILLEGAL_COLUMN};
-        }
+            throw Exception{"Illegal column " + ancestor_id_col_untyped->getName()
+                + " of third argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN};
 
         return true;
     }

From 30192b1b42bf272012e40fea228570cb2cd5388c Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 16 Mar 2018 06:13:13 +0300
Subject: [PATCH 124/244] Fixed backward compatibility [#CLICKHOUSE-3650].

---
 .../Functions/FunctionsExternalDictionaries.h | 56 +++++++++++++++++++
 1 file changed, 56 insertions(+)

diff --git a/dbms/src/Functions/FunctionsExternalDictionaries.h b/dbms/src/Functions/FunctionsExternalDictionaries.h
index abb8c518aa6..bbf9b1b5c60 100644
--- a/dbms/src/Functions/FunctionsExternalDictionaries.h
+++ b/dbms/src/Functions/FunctionsExternalDictionaries.h
@@ -101,6 +101,20 @@ private:
         if (!dict_name_col)
             throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN};
 
+        /** Do not require existence of the dictionary if the function is called for empty block.
+          * This is needed to allow successful query analysis on a server,
+          *  that is the initiator of a distributed query,
+          *  in the case when the function will be invoked for real data only at the remote servers.
+          * This feature is controversial and implemented specially
+          *  for backward compatibility with the case in Yandex Banner System.
+          */
+        if (block.rows() == 0)
+        {
+            auto & elem = block.getByPosition(result);
+            elem.column = elem.type->createColumn();
+            return;
+        }
+
         auto dict = dictionaries.getDictionary(dict_name_col->getValue<String>());
         const auto dict_ptr = dict.get();
 
@@ -252,6 +266,13 @@ private:
         if (!dict_name_col)
             throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN};
 
+        if (block.rows() == 0)
+        {
+            auto & elem = block.getByPosition(result);
+            elem.column = elem.type->createColumn();
+            return;
+        }
+
         auto dict = dictionaries.getDictionary(dict_name_col->getValue<String>());
         const auto dict_ptr = dict.get();
 
@@ -475,6 +496,13 @@ private:
         if (!dict_name_col)
             throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN};
 
+        if (block.rows() == 0)
+        {
+            auto & elem = block.getByPosition(result);
+            elem.column = elem.type->createColumn();
+            return;
+        }
+
         auto dict = dictionaries.getDictionary(dict_name_col->getValue<String>());
         const auto dict_ptr = dict.get();
 
@@ -734,6 +762,13 @@ private:
         if (!dict_name_col)
             throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN};
 
+        if (block.rows() == 0)
+        {
+            auto & elem = block.getByPosition(result);
+            elem.column = elem.type->createColumn();
+            return;
+        }
+
         auto dict = dictionaries.getDictionary(dict_name_col->getValue<String>());
         const auto dict_ptr = dict.get();
 
@@ -1002,6 +1037,13 @@ private:
         if (!dict_name_col)
             throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN};
 
+        if (block.rows() == 0)
+        {
+            auto & elem = block.getByPosition(result);
+            elem.column = elem.type->createColumn();
+            return;
+        }
+
         auto dict = dictionaries.getDictionary(dict_name_col->getValue<String>());
         const auto dict_ptr = dict.get();
 
@@ -1228,6 +1270,13 @@ private:
         if (!dict_name_col)
             throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN};
 
+        if (block.rows() == 0)
+        {
+            auto & elem = block.getByPosition(result);
+            elem.column = elem.type->createColumn();
+            return;
+        }
+
         auto dict = dictionaries.getDictionary(dict_name_col->getValue<String>());
         const auto dict_ptr = dict.get();
 
@@ -1381,6 +1430,13 @@ private:
         if (!dict_name_col)
             throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN};
 
+        if (block.rows() == 0)
+        {
+            auto & elem = block.getByPosition(result);
+            elem.column = elem.type->createColumn();
+            return;
+        }
+
         auto dict = dictionaries.getDictionary(dict_name_col->getValue<String>());
         const auto dict_ptr = dict.get();
 

From f4c54f02a5629e18af37b8812d4864b5bf2687c9 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 16 Mar 2018 07:00:00 +0300
Subject: [PATCH 125/244] Removed trashy, obsolete code [#CLICKHOUSE-3649]

---
 dbms/src/Interpreters/ExpressionAnalyzer.cpp | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp
index ca9bc493566..059e557baf7 100644
--- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp
+++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp
@@ -1647,15 +1647,7 @@ void ExpressionAnalyzer::makeExplicitSet(const ASTFunction * node, const Block &
     if (left_arg_tuple && left_arg_tuple->name == "tuple")
     {
         for (const auto & arg : left_arg_tuple->arguments->children)
-        {
-            const auto & data_type = sample_block.getByName(arg->getColumnName()).type;
-
-            /// NOTE prevent crash in query: SELECT (1, [1]) in (1, 1)
-            if (const auto array = typeid_cast<const DataTypeArray * >(data_type.get()))
-                throw Exception("Incorrect element of tuple: " + array->getName(), ErrorCodes::INCORRECT_ELEMENT_OF_SET);
-
-            set_element_types.push_back(data_type);
-        }
+            set_element_types.push_back(sample_block.getByName(arg->getColumnName()).type);
     }
     else
     {

From 83b775fbf53c08ca5b1eefc53c3078266eba152e Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 16 Mar 2018 07:10:38 +0300
Subject: [PATCH 126/244] Attempt to fix bad code, try 1 [#CLICKHOUSE-3649]
 #1830

---
 dbms/src/Interpreters/ExpressionAnalyzer.cpp | 19 +++++--------------
 1 file changed, 5 insertions(+), 14 deletions(-)

diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp
index 059e557baf7..a32ea02754f 100644
--- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp
+++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp
@@ -1504,21 +1504,12 @@ void ExpressionAnalyzer::makeSetsForIndexImpl(const ASTPtr & node, const Block &
             }
             else
             {
-                try
-                {
-                    ExpressionActionsPtr temp_actions = std::make_shared<ExpressionActions>(source_columns, settings);
-                    getRootActions(func->arguments->children.at(0), true, false, temp_actions);
-                    makeExplicitSet(func, temp_actions->getSampleBlock(), true);
-                }
-                catch (const Exception & e)
-                {
-                    /// in `sample_block` there are no columns that are added by `getActions`
-                    if (e.code() != ErrorCodes::NOT_FOUND_COLUMN_IN_BLOCK && e.code() != ErrorCodes::UNKNOWN_IDENTIFIER)
-                        throw;
+                ExpressionActionsPtr temp_actions = std::make_shared<ExpressionActions>(source_columns, settings);
+                getRootActions(func->arguments->children.at(0), true, false, temp_actions);
 
-                    /// TODO: Delete the catch in the next release
-                    tryLogCurrentException(&Poco::Logger::get("ExpressionAnalyzer"));
-                }
+                Block sample_block_with_calculated_columns = temp_actions->getSampleBlock();
+                if (sample_block_with_calculated_columns.has(args.children.at(0)->getColumnName()))
+                    makeExplicitSet(func, sample_block_with_calculated_columns, true);
             }
         }
     }

From eb0095a7b9b19cc8b77b0906f095acfb1d23fede Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 16 Mar 2018 07:13:24 +0300
Subject: [PATCH 127/244] Attempt to fix bad code, try 2 [#CLICKHOUSE-3649]
 #1830

---
 dbms/src/Interpreters/ExpressionAnalyzer.cpp | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp
index a32ea02754f..67fcad625c5 100644
--- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp
+++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp
@@ -1504,12 +1504,10 @@ void ExpressionAnalyzer::makeSetsForIndexImpl(const ASTPtr & node, const Block &
             }
             else
             {
-                ExpressionActionsPtr temp_actions = std::make_shared<ExpressionActions>(source_columns, settings);
+                ExpressionActionsPtr temp_actions = std::make_shared<ExpressionActions>(aggregated_columns, settings);
                 getRootActions(func->arguments->children.at(0), true, false, temp_actions);
-
                 Block sample_block_with_calculated_columns = temp_actions->getSampleBlock();
-                if (sample_block_with_calculated_columns.has(args.children.at(0)->getColumnName()))
-                    makeExplicitSet(func, sample_block_with_calculated_columns, true);
+                makeExplicitSet(func, sample_block_with_calculated_columns, true);
             }
         }
     }

From 73b04a4d278c5f1ad24621a67ad2b455df6e55ef Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 16 Mar 2018 07:37:21 +0300
Subject: [PATCH 128/244] Revert "Attempt to fix bad code, try 2
 [#CLICKHOUSE-3649] #1830"

This reverts commit eb0095a7b9b19cc8b77b0906f095acfb1d23fede.
---
 dbms/src/Interpreters/ExpressionAnalyzer.cpp | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp
index 67fcad625c5..a32ea02754f 100644
--- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp
+++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp
@@ -1504,10 +1504,12 @@ void ExpressionAnalyzer::makeSetsForIndexImpl(const ASTPtr & node, const Block &
             }
             else
             {
-                ExpressionActionsPtr temp_actions = std::make_shared<ExpressionActions>(aggregated_columns, settings);
+                ExpressionActionsPtr temp_actions = std::make_shared<ExpressionActions>(source_columns, settings);
                 getRootActions(func->arguments->children.at(0), true, false, temp_actions);
+
                 Block sample_block_with_calculated_columns = temp_actions->getSampleBlock();
-                makeExplicitSet(func, sample_block_with_calculated_columns, true);
+                if (sample_block_with_calculated_columns.has(args.children.at(0)->getColumnName()))
+                    makeExplicitSet(func, sample_block_with_calculated_columns, true);
             }
         }
     }

From 50ae82d9fbac72bfaf861070127b23133e988d50 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 16 Mar 2018 07:47:00 +0300
Subject: [PATCH 129/244] Removing --ssl from test under pbuilder, because
 nothing works #2051

---
 debian/pbuilder-hooks/B90test-server | 1 -
 1 file changed, 1 deletion(-)

diff --git a/debian/pbuilder-hooks/B90test-server b/debian/pbuilder-hooks/B90test-server
index fc7c0a04167..da929746937 100755
--- a/debian/pbuilder-hooks/B90test-server
+++ b/debian/pbuilder-hooks/B90test-server
@@ -25,7 +25,6 @@ if  [ "${TEST_SSL}" ]; then
 
     openssl dhparam -out /etc/clickhouse-server/dhparam.pem 256
     openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout /etc/clickhouse-server/server.key -out /etc/clickhouse-server/server.crt
-    CLIENT_ADD="--ssl"
 fi
 
 service clickhouse-server start

From ed9517bb0ba6ab97f25e7aeb486b71a876c2ea4d Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 16 Mar 2018 07:49:00 +0300
Subject: [PATCH 130/244] Whitespace #2051

---
 dbms/src/Server/Client.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp
index 1580f45640b..1c838a99569 100644
--- a/dbms/src/Server/Client.cpp
+++ b/dbms/src/Server/Client.cpp
@@ -387,8 +387,8 @@ private:
     void connect()
     {
         auto encryption = config().getBool("ssl", false)
-        ? Protocol::Encryption::Enable
-        : Protocol::Encryption::Disable;
+            ? Protocol::Encryption::Enable
+            : Protocol::Encryption::Disable;
 
         String host = config().getString("host", "localhost");
         UInt16 port = config().getInt("port", config().getInt(static_cast<bool>(encryption) ? "tcp_ssl_port" : "tcp_port", static_cast<bool>(encryption) ? DBMS_DEFAULT_SECURE_PORT : DBMS_DEFAULT_PORT));

From 536a4a0c52921af935645d871e84997e9157cb10 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 16 Mar 2018 07:49:22 +0300
Subject: [PATCH 131/244] More checks [#CLICKHOUSE-2]

---
 dbms/src/Functions/IFunction.cpp | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/dbms/src/Functions/IFunction.cpp b/dbms/src/Functions/IFunction.cpp
index 81ada906a8d..ba05bc34e9d 100644
--- a/dbms/src/Functions/IFunction.cpp
+++ b/dbms/src/Functions/IFunction.cpp
@@ -15,7 +15,8 @@ namespace DB
 
 namespace ErrorCodes
 {
-extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+    extern const int ILLEGAL_COLUMN;
 }
 
 namespace
@@ -127,7 +128,7 @@ NullPresence getNullPresense(const ColumnsWithTypeAndName & args)
 bool allArgumentsAreConstants(const Block & block, const ColumnNumbers & args)
 {
     for (auto arg : args)
-        if (!typeid_cast<const ColumnConst *>(block.getByPosition(arg).column.get()))
+        if (!block.getByPosition(arg).column->isColumnConst())
             return false;
     return true;
 }
@@ -135,11 +136,16 @@ bool allArgumentsAreConstants(const Block & block, const ColumnNumbers & args)
 
 bool PreparedFunctionImpl::defaultImplementationForConstantArguments(Block & block, const ColumnNumbers & args, size_t result)
 {
+    ColumnNumbers arguments_to_remain_constants = getArgumentsThatAreAlwaysConstant();
+
+    /// Check that these arguments are really constant.
+    for (auto arg_num : arguments_to_remain_constants)
+        if (arg_num < args.size() && !block.getByPosition(args[arg_num]).column->isColumnConst())
+            throw Exception("Argument at index " + toString(arg_num) + " for function " + getName() + " must be constant", ErrorCodes::ILLEGAL_COLUMN);
+
     if (args.empty() || !useDefaultImplementationForConstants() || !allArgumentsAreConstants(block, args))
         return false;
 
-    ColumnNumbers arguments_to_remain_constants = getArgumentsThatAreAlwaysConstant();
-
     Block temporary_block;
     bool have_converted_columns = false;
 
@@ -162,7 +168,7 @@ bool PreparedFunctionImpl::defaultImplementationForConstantArguments(Block & blo
       */
     if (!have_converted_columns)
         throw Exception("Number of arguments for function " + getName() + " doesn't match: the function requires more arguments",
-                        ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
+            ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
 
     temporary_block.insert(block.getByPosition(result));
 

From 7633c0bf2b2c1c68ccfe52ba8195332182daeb5f Mon Sep 17 00:00:00 2001
From: robot-metrika-test <robot-metrika-test@yandex-team.ru>
Date: Fri, 16 Mar 2018 07:51:36 +0300
Subject: [PATCH 132/244] Auto version update to [54367]

---
 dbms/cmake/version.cmake | 4 ++--
 debian/changelog         | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake
index 80e9aaf2e57..623d51823fb 100644
--- a/dbms/cmake/version.cmake
+++ b/dbms/cmake/version.cmake
@@ -1,6 +1,6 @@
 # This strings autochanged from release_lib.sh:
-set(VERSION_DESCRIBE v1.1.54366-testing)
-set(VERSION_REVISION 54366)
+set(VERSION_DESCRIBE v1.1.54367-testing)
+set(VERSION_REVISION 54367)
 # end of autochange
 
 set (VERSION_MAJOR 1)
diff --git a/debian/changelog b/debian/changelog
index 6f142ef0c16..0ee6be63a8b 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,5 @@
-clickhouse (1.1.54366) unstable; urgency=low
+clickhouse (1.1.54367) unstable; urgency=low
 
   * Modified source code
 
- --  <robot-metrika-test@yandex-team.ru>  Fri, 16 Mar 2018 00:07:09 +0300
+ --  <robot-metrika-test@yandex-team.ru>  Fri, 16 Mar 2018 07:51:36 +0300

From 2bc42d53f46a3ac476319f33d7947e80bc293201 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 16 Mar 2018 08:18:13 +0300
Subject: [PATCH 133/244] More information on signals [#CLICKHOUSE-2]

---
 libs/libdaemon/src/BaseDaemon.cpp | 146 +++++++++++++++++++++++++++---
 1 file changed, 133 insertions(+), 13 deletions(-)

diff --git a/libs/libdaemon/src/BaseDaemon.cpp b/libs/libdaemon/src/BaseDaemon.cpp
index 29405901206..849a80ff193 100644
--- a/libs/libdaemon/src/BaseDaemon.cpp
+++ b/libs/libdaemon/src/BaseDaemon.cpp
@@ -311,19 +311,6 @@ private:
         LOG_ERROR(log, "(from thread " << thread_num << ") "
             << "Received signal " << strsignal(sig) << " (" << sig << ")" << ".");
 
-        if (sig == SIGSEGV)
-        {
-            /// Print info about address and reason.
-            if (nullptr == info.si_addr)
-                LOG_ERROR(log, "Address: NULL pointer.");
-            else
-                LOG_ERROR(log, "Address: " << info.si_addr
-                    << (info.si_code == SEGV_ACCERR ? ". Attempted access has violated the permissions assigned to the memory area." : ""));
-        }
-
-        if (already_printed_stack_trace)
-            return;
-
         void * caller_address = nullptr;
 
 #if defined(__x86_64__)
@@ -334,11 +321,144 @@ private:
         caller_address = reinterpret_cast<void *>(context.uc_mcontext->__ss.__rip);
         #else
         caller_address = reinterpret_cast<void *>(context.uc_mcontext.gregs[REG_RIP]);
+        auto err_mask = context.uc_mcontext.gregs[REG_ERR];
         #endif
 #elif defined(__aarch64__)
         caller_address = reinterpret_cast<void *>(context.uc_mcontext.pc);
 #endif
 
+        switch (sig)
+        {
+            case SIGSEGV:
+            {
+                /// Print info about address and reason.
+                if (nullptr == info.si_addr)
+                    LOG_ERROR(log, "Address: NULL pointer.");
+                else
+                    LOG_ERROR(log, "Address: " << info.si_addr);
+
+#if defined(__x86_64__) && !defined(__FreeBSD__) && !defined(__APPLE__)
+                if ((err_mask & 0x02))
+                    LOG_ERROR(log, "Access: write.");
+                else
+                    LOG_ERROR(log, "Access: read.");
+#endif
+
+                switch (info.si_code)
+                {
+                    case SEGV_ACCERR:
+                        LOG_ERROR(log, "Attempted access has violated the permissions assigned to the memory area.");
+                        break;
+                    case SEGV_MAPERR:
+                        LOG_ERROR(log, "Address not mapped to object.");
+                        break;
+                    default:
+                        LOG_ERROR(log, "Unknown si_code.");
+                        break;
+                }
+                break;
+            }
+
+            case SIGBUS:
+            {
+                switch (info.si_code)
+                {
+                    case BUS_ADRALN:
+                        LOG_ERROR(log, "Invalid address alignment.");
+                        break;
+                    case BUS_ADRERR:
+                        LOG_ERROR(log, "Non-existant physical address.");
+                        break;
+                    case BUS_OBJERR:
+                        LOG_ERROR(log, "Object specific hardware error.");
+                        break;
+                    case BUS_MCEERR_AR:
+                        LOG_ERROR(log, "Hardware memory error: action required.");
+                        break;
+                    case BUS_MCEERR_AO:
+                        LOG_ERROR(log, "Hardware memory error: action optional.");
+                        break;
+                    default:
+                        LOG_ERROR(log, "Unknown si_code.");
+                        break;
+                }
+                break;
+            }
+
+            case SIGILL:
+            {
+                switch (info.si_code)
+                {
+                    case ILL_ILLOPC:
+                        LOG_ERROR(log, "Illegal opcode.");
+                        break;
+                    case ILL_ILLOPN:
+                        LOG_ERROR(log, "Illegal operand.");
+                        break;
+                    case ILL_ILLADR:
+                        LOG_ERROR(log, "Illegal addressing mode.");
+                        break;
+                    case ILL_ILLTRP:
+                        LOG_ERROR(log, "Illegal trap.");
+                        break;
+                    case ILL_PRVOPC:
+                        LOG_ERROR(log, "Privileged opcode.");
+                        break;
+                    case ILL_PRVREG:
+                        LOG_ERROR(log, "Privileged register.");
+                        break;
+                    case ILL_COPROC:
+                        LOG_ERROR(log, "Coprocessor error.");
+                        break;
+                    case ILL_BADSTK:
+                        LOG_ERROR(log, "Internal stack error.");
+                        break;
+                    default:
+                        LOG_ERROR(log, "Unknown si_code.");
+                        break;
+                }
+                break;
+            }
+
+            case SIGFPE:
+            {
+                switch (info.si_code)
+                {
+                    case FPE_INTDIV:
+                        LOG_ERROR(log, "Integer divide by zero.");
+                        break;
+                    case FPE_INTOVF:
+                        LOG_ERROR(log, "Integer overflow.");
+                        break;
+                    case FPE_FLTDIV:
+                        LOG_ERROR(log, "Floating point divide by zero.");
+                        break;
+                    case FPE_FLTOVF:
+                        LOG_ERROR(log, "Floating point overflow.");
+                        break;
+                    case FPE_FLTUND:
+                        LOG_ERROR(log, "Floating point underflow.");
+                        break;
+                    case FPE_FLTRES:
+                        LOG_ERROR(log, "Floating point inexact result.");
+                        break;
+                    case FPE_FLTINV:
+                        LOG_ERROR(log, "Floating point invalid operation.");
+                        break;
+                    case FPE_FLTSUB:
+                        LOG_ERROR(log, "Subscript out of range.");
+                        break;
+                    default:
+                        LOG_ERROR(log, "Unknown si_code.");
+                        break;
+                }
+                break;
+            }
+        }
+
+        if (already_printed_stack_trace)
+            return;
+
         static const int max_frames = 50;
         void * frames[max_frames];
 

From dc4a9be6fa7d41b499963bf8eb25a449618a076b Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 16 Mar 2018 09:33:23 +0300
Subject: [PATCH 134/244] Fixed error with "makeSetsForIndex" [#CLICKHOUSE-2]

---
 dbms/src/Interpreters/ExpressionAnalyzer.cpp | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp
index a32ea02754f..f9967a8f9e8 100644
--- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp
+++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp
@@ -1458,8 +1458,13 @@ void ExpressionAnalyzer::optimizeLimitBy()
 
 void ExpressionAnalyzer::makeSetsForIndex()
 {
-    if (storage && ast && storage->supportsIndexForIn())
-        makeSetsForIndexImpl(ast, storage->getSampleBlock());
+    if (storage && select_query && storage->supportsIndexForIn())
+    {
+        if (select_query->where_expression)
+            makeSetsForIndexImpl(select_query->where_expression, storage->getSampleBlock());
+        if (select_query->prewhere_expression)
+            makeSetsForIndexImpl(select_query->prewhere_expression, storage->getSampleBlock());
+    }
 }
 
 
@@ -1485,8 +1490,15 @@ void ExpressionAnalyzer::makeSetsForIndexImpl(const ASTPtr & node, const Block &
     for (auto & child : node->children)
     {
         /// Process expression only in current subquery
-        if (!typeid_cast<ASTSubquery *>(child.get()))
-            makeSetsForIndexImpl(child, sample_block);
+        if (typeid_cast<ASTSubquery *>(child.get()))
+            continue;
+
+        /// Don't dive into lambda functions
+        const ASTFunction * func = typeid_cast<const ASTFunction *>(child.get());
+        if (func && func->name == "lambda")
+            continue;
+
+        makeSetsForIndexImpl(child, sample_block);
     }
 
     const ASTFunction * func = typeid_cast<const ASTFunction *>(node.get());

From 2e23b43ed37d89d2e555c7e2b421e3c49c3544a5 Mon Sep 17 00:00:00 2001
From: robot-metrika-test <robot-metrika-test@yandex-team.ru>
Date: Fri, 16 Mar 2018 09:35:34 +0300
Subject: [PATCH 135/244] Auto version update to [54368]

---
 dbms/cmake/version.cmake | 4 ++--
 debian/changelog         | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake
index 623d51823fb..7da4f053dfb 100644
--- a/dbms/cmake/version.cmake
+++ b/dbms/cmake/version.cmake
@@ -1,6 +1,6 @@
 # This strings autochanged from release_lib.sh:
-set(VERSION_DESCRIBE v1.1.54367-testing)
-set(VERSION_REVISION 54367)
+set(VERSION_DESCRIBE v1.1.54368-testing)
+set(VERSION_REVISION 54368)
 # end of autochange
 
 set (VERSION_MAJOR 1)
diff --git a/debian/changelog b/debian/changelog
index 0ee6be63a8b..2d4e333778f 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,5 @@
-clickhouse (1.1.54367) unstable; urgency=low
+clickhouse (1.1.54368) unstable; urgency=low
 
   * Modified source code
 
- --  <robot-metrika-test@yandex-team.ru>  Fri, 16 Mar 2018 07:51:36 +0300
+ --  <robot-metrika-test@yandex-team.ru>  Fri, 16 Mar 2018 09:35:34 +0300

From cf6112fd377bcd306ac85ce979749c6f425de91a Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 16 Mar 2018 09:39:32 +0300
Subject: [PATCH 136/244] Fixed half of bad code [#CLICKHOUSE-2]

---
 dbms/src/Interpreters/ExpressionAnalyzer.cpp | 32 +++++++++++---------
 1 file changed, 18 insertions(+), 14 deletions(-)

diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp
index f9967a8f9e8..abe503137cf 100644
--- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp
+++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp
@@ -1505,23 +1505,27 @@ void ExpressionAnalyzer::makeSetsForIndexImpl(const ASTPtr & node, const Block &
     if (func && functionIsInOperator(func->name))
     {
         const IAST & args = *func->arguments;
-        const ASTPtr & arg = args.children.at(1);
 
-        if (!prepared_sets.count(arg.get())) /// Not already prepared.
+        if (storage && storage->mayBenefitFromIndexForIn(args.children.at(0)))
         {
-            if (typeid_cast<ASTSubquery *>(arg.get()) || typeid_cast<ASTIdentifier *>(arg.get()))
-            {
-                if (settings.use_index_for_in_with_subqueries && storage->mayBenefitFromIndexForIn(args.children.at(0)))
-                    tryMakeSetFromSubquery(arg);
-            }
-            else
-            {
-                ExpressionActionsPtr temp_actions = std::make_shared<ExpressionActions>(source_columns, settings);
-                getRootActions(func->arguments->children.at(0), true, false, temp_actions);
+            const ASTPtr & arg = args.children.at(1);
 
-                Block sample_block_with_calculated_columns = temp_actions->getSampleBlock();
-                if (sample_block_with_calculated_columns.has(args.children.at(0)->getColumnName()))
-                    makeExplicitSet(func, sample_block_with_calculated_columns, true);
+            if (!prepared_sets.count(arg.get())) /// Not already prepared.
+            {
+                if (typeid_cast<ASTSubquery *>(arg.get()) || typeid_cast<ASTIdentifier *>(arg.get()))
+                {
+                    if (settings.use_index_for_in_with_subqueries)
+                        tryMakeSetFromSubquery(arg);
+                }
+                else
+                {
+                    ExpressionActionsPtr temp_actions = std::make_shared<ExpressionActions>(source_columns, settings);
+                    getRootActions(func->arguments->children.at(0), true, false, temp_actions);
+
+                    Block sample_block_with_calculated_columns = temp_actions->getSampleBlock();
+                    if (sample_block_with_calculated_columns.has(args.children.at(0)->getColumnName()))
+                        makeExplicitSet(func, sample_block_with_calculated_columns, true);
+                }
             }
         }
     }

From 02c67e2b13572c76dc6fdd59e541c7d6b8f7ab8e Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 16 Mar 2018 09:51:37 +0300
Subject: [PATCH 137/244] Fixed error [#CLICKHOUSE-2]

---
 dbms/src/Storages/MergeTree/MergeTreeData.cpp | 12 ++++++++----
 dbms/src/Storages/MergeTree/MergeTreeData.h   |  4 ++--
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp
index b5200441383..ef4b8022323 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp
@@ -2194,7 +2194,7 @@ MergeTreeData::DataPartsVector MergeTreeData::Transaction::commit()
     return total_covered_parts;
 }
 
-bool MergeTreeData::isPrimaryKeyColumn(const ASTPtr &node) const
+bool MergeTreeData::isPrimaryKeyColumnPossiblyWrappedInFunctions(const ASTPtr & node) const
 {
     String column_name = node->getColumnName();
 
@@ -2202,6 +2202,10 @@ bool MergeTreeData::isPrimaryKeyColumn(const ASTPtr &node) const
         if (column_name == column.column_name)
             return true;
 
+    if (const ASTFunction * func = typeid_cast<const ASTFunction *>(node.get()))
+        if (func->arguments->children.size() == 1)
+            return isPrimaryKeyColumnPossiblyWrappedInFunctions(func->arguments->children.front());
+
     return false;
 }
 
@@ -2213,16 +2217,16 @@ bool MergeTreeData::mayBenefitFromIndexForIn(const ASTPtr & left_in_operand) con
     if (left_in_operand_tuple && left_in_operand_tuple->name == "tuple")
     {
         for (const auto & item : left_in_operand_tuple->arguments->children)
-            if (!isPrimaryKeyColumn(item))
+            if (!isPrimaryKeyColumnPossiblyWrappedInFunctions(item))
                 /// The tuple itself may be part of the primary key, so check that as a last resort.
-                return isPrimaryKeyColumn(left_in_operand);
+                return isPrimaryKeyColumnPossiblyWrappedInFunctions(left_in_operand);
 
         /// tuple() is invalid but can still be found here since this method may be called before the arguments are validated.
         return !left_in_operand_tuple->arguments->children.empty();
     }
     else
     {
-        return isPrimaryKeyColumn(left_in_operand);
+        return isPrimaryKeyColumnPossiblyWrappedInFunctions(left_in_operand);
     }
 }
 
diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h
index d440d86d62b..47abd959645 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeData.h
+++ b/dbms/src/Storages/MergeTree/MergeTreeData.h
@@ -675,8 +675,8 @@ private:
         DataPartPtr & out_covering_part,
         std::lock_guard<std::mutex> & data_parts_lock) const;
 
-    /// Checks whether the column is in the primary key.
-    bool isPrimaryKeyColumn(const ASTPtr &node) const;
+    /// Checks whether the column is in the primary key, possibly wrapped in a chain of functions with single argument.
+    bool isPrimaryKeyColumnPossiblyWrappedInFunctions(const ASTPtr &node) const;
 };
 
 }

From 796c65a09ff98e93e094a5cd533bbfa0eb1c6f9e Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 16 Mar 2018 10:04:51 +0300
Subject: [PATCH 138/244] Added test [#CLICKHOUSE-2]

---
 .../0_stateless/00607_index_in_in.reference      |  6 ++++++
 .../queries/0_stateless/00607_index_in_in.sql    | 16 ++++++++++++++++
 2 files changed, 22 insertions(+)
 create mode 100644 dbms/tests/queries/0_stateless/00607_index_in_in.reference
 create mode 100644 dbms/tests/queries/0_stateless/00607_index_in_in.sql

diff --git a/dbms/tests/queries/0_stateless/00607_index_in_in.reference b/dbms/tests/queries/0_stateless/00607_index_in_in.reference
new file mode 100644
index 00000000000..a9e2f17562a
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00607_index_in_in.reference
@@ -0,0 +1,6 @@
+1
+1
+1
+1
+1
+1
diff --git a/dbms/tests/queries/0_stateless/00607_index_in_in.sql b/dbms/tests/queries/0_stateless/00607_index_in_in.sql
new file mode 100644
index 00000000000..a1e2efa6a71
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00607_index_in_in.sql
@@ -0,0 +1,16 @@
+DROP TABLE IF EXISTS test.merge_tree;
+CREATE TABLE test.merge_tree (x UInt32) ENGINE = MergeTree ORDER BY x SETTINGS index_granularity = 1;
+INSERT INTO test.merge_tree VALUES (0), (1);
+
+SET force_primary_key = 1;
+SET max_rows_to_read = 1;
+
+SELECT count() FROM test.merge_tree WHERE x = 0;
+SELECT count() FROM test.merge_tree WHERE toUInt32(x) = 0;
+SELECT count() FROM test.merge_tree WHERE toUInt64(x) = 0;
+
+SELECT count() FROM test.merge_tree WHERE x IN (0, 0);
+SELECT count() FROM test.merge_tree WHERE toUInt32(x) IN (0, 0);
+SELECT count() FROM test.merge_tree WHERE toUInt64(x) IN (0, 0);
+
+DROP TABLE test.merge_tree;

From 91266924419b083295fd2821c0c59060de1ea0d0 Mon Sep 17 00:00:00 2001
From: robot-metrika-test <robot-metrika-test@yandex-team.ru>
Date: Fri, 16 Mar 2018 10:31:37 +0300
Subject: [PATCH 139/244] Auto version update to [54369]

---
 dbms/cmake/version.cmake | 4 ++--
 debian/changelog         | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake
index 7da4f053dfb..17e609f794b 100644
--- a/dbms/cmake/version.cmake
+++ b/dbms/cmake/version.cmake
@@ -1,6 +1,6 @@
 # This strings autochanged from release_lib.sh:
-set(VERSION_DESCRIBE v1.1.54368-testing)
-set(VERSION_REVISION 54368)
+set(VERSION_DESCRIBE v1.1.54369-testing)
+set(VERSION_REVISION 54369)
 # end of autochange
 
 set (VERSION_MAJOR 1)
diff --git a/debian/changelog b/debian/changelog
index 2d4e333778f..180d00dfaf3 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,5 @@
-clickhouse (1.1.54368) unstable; urgency=low
+clickhouse (1.1.54369) unstable; urgency=low
 
   * Modified source code
 
- --  <robot-metrika-test@yandex-team.ru>  Fri, 16 Mar 2018 09:35:34 +0300
+ --  <robot-metrika-test@yandex-team.ru>  Fri, 16 Mar 2018 10:31:36 +0300

From a6892b881388b42f7cb04fb6204e9cfd66b84cb2 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 16 Mar 2018 12:00:04 +0300
Subject: [PATCH 140/244] Fixed test [#CLICKHOUSE-2]

---
 dbms/src/Storages/StorageBuffer.cpp | 14 ++++++++++++++
 dbms/src/Storages/StorageBuffer.h   |  2 ++
 dbms/src/Storages/StorageMerge.cpp  | 21 +++++++++++++++++++++
 dbms/src/Storages/StorageMerge.h    |  2 ++
 4 files changed, 39 insertions(+)

diff --git a/dbms/src/Storages/StorageBuffer.cpp b/dbms/src/Storages/StorageBuffer.cpp
index 2072ac72c5c..5c59ffa7c40 100644
--- a/dbms/src/Storages/StorageBuffer.cpp
+++ b/dbms/src/Storages/StorageBuffer.cpp
@@ -323,6 +323,20 @@ BlockOutputStreamPtr StorageBuffer::write(const ASTPtr & /*query*/, const Settin
 }
 
 
+bool StorageBuffer::mayBenefitFromIndexForIn(const ASTPtr & left_in_operand) const
+{
+    if (no_destination)
+        return false;
+
+    auto destination = context.getTable(destination_database, destination_table);
+
+    if (destination.get() == this)
+        throw Exception("Destination table is myself. Read will cause infinite loop.", ErrorCodes::INFINITE_LOOP);
+
+    return destination->mayBenefitFromIndexForIn(left_in_operand);
+}
+
+
 void StorageBuffer::startup()
 {
     if (context.getSettingsRef().readonly)
diff --git a/dbms/src/Storages/StorageBuffer.h b/dbms/src/Storages/StorageBuffer.h
index 02c2ef9b09b..ea8a0562628 100644
--- a/dbms/src/Storages/StorageBuffer.h
+++ b/dbms/src/Storages/StorageBuffer.h
@@ -75,6 +75,8 @@ public:
     bool supportsFinal() const override { return true; }
     bool supportsIndexForIn() const override { return true; }
 
+    bool mayBenefitFromIndexForIn(const ASTPtr & left_in_operand) const override;
+
     /// The structure of the subordinate table is not checked and does not change.
     void alter(const AlterCommands & params, const String & database_name, const String & table_name, const Context & context) override;
 
diff --git a/dbms/src/Storages/StorageMerge.cpp b/dbms/src/Storages/StorageMerge.cpp
index e7090ad66ed..c0f264e7f85 100644
--- a/dbms/src/Storages/StorageMerge.cpp
+++ b/dbms/src/Storages/StorageMerge.cpp
@@ -111,6 +111,27 @@ namespace
 }
 
 
+bool StorageMerge::mayBenefitFromIndexForIn(const ASTPtr & left_in_operand) const
+{
+    /// It's beneficial if it is true for at least one table.
+    StorageListWithLocks selected_tables = getSelectedTables();
+
+    size_t i = 0;
+    for (const auto & table : selected_tables)
+    {
+        if (table.first->mayBenefitFromIndexForIn(left_in_operand))
+            return true;
+
+        ++i;
+        /// For simplicity reasons, check only first ten tables.
+        if (i > 10)
+            break;
+    }
+
+    return false;
+}
+
+
 BlockInputStreams StorageMerge::read(
     const Names & column_names,
     const SelectQueryInfo & query_info,
diff --git a/dbms/src/Storages/StorageMerge.h b/dbms/src/Storages/StorageMerge.h
index 0c37e4e7fb4..e48cfc1d1fc 100644
--- a/dbms/src/Storages/StorageMerge.h
+++ b/dbms/src/Storages/StorageMerge.h
@@ -44,6 +44,8 @@ public:
     /// the structure of sub-tables is not checked
     void alter(const AlterCommands & params, const String & database_name, const String & table_name, const Context & context) override;
 
+    bool mayBenefitFromIndexForIn(const ASTPtr & left_in_operand) const override;
+
 private:
     String name;
     String source_database;

From 86e4099cc23182809c9bcb13819a1b2c75847c31 Mon Sep 17 00:00:00 2001
From: robot-metrika-test <robot-metrika-test@yandex-team.ru>
Date: Fri, 16 Mar 2018 12:01:50 +0300
Subject: [PATCH 141/244] Auto version update to [54370]

---
 dbms/cmake/version.cmake | 4 ++--
 debian/changelog         | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake
index 17e609f794b..ad5ad09cce2 100644
--- a/dbms/cmake/version.cmake
+++ b/dbms/cmake/version.cmake
@@ -1,6 +1,6 @@
 # This strings autochanged from release_lib.sh:
-set(VERSION_DESCRIBE v1.1.54369-testing)
-set(VERSION_REVISION 54369)
+set(VERSION_DESCRIBE v1.1.54370-testing)
+set(VERSION_REVISION 54370)
 # end of autochange
 
 set (VERSION_MAJOR 1)
diff --git a/debian/changelog b/debian/changelog
index 180d00dfaf3..75502e3cc96 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,5 @@
-clickhouse (1.1.54369) unstable; urgency=low
+clickhouse (1.1.54370) unstable; urgency=low
 
   * Modified source code
 
- --  <robot-metrika-test@yandex-team.ru>  Fri, 16 Mar 2018 10:31:36 +0300
+ --  <robot-metrika-test@yandex-team.ru>  Fri, 16 Mar 2018 12:01:50 +0300

From 1f28410490a6bedc23a4b6c123f454ad7b6f0ad8 Mon Sep 17 00:00:00 2001
From: Vitaliy Lyudvichenko <vludv@yandex-team.ru>
Date: Fri, 16 Mar 2018 17:01:00 +0300
Subject: [PATCH 142/244] Fixed false positive error messages for queries with
 IN and ARRAY JOIN. [#CLICKHOUSE-2]

---
 dbms/src/Interpreters/ExpressionAnalyzer.cpp             | 2 +-
 .../0_stateless/00563_complex_in_expression.reference    | 4 ++++
 .../queries/0_stateless/00563_complex_in_expression.sql  | 9 ++++++++-
 3 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp
index abe503137cf..500e4cd775d 100644
--- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp
+++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp
@@ -1489,7 +1489,7 @@ void ExpressionAnalyzer::makeSetsForIndexImpl(const ASTPtr & node, const Block &
 {
     for (auto & child : node->children)
     {
-        /// Process expression only in current subquery
+        /// Don't descent into subqueries.
         if (typeid_cast<ASTSubquery *>(child.get()))
             continue;
 
diff --git a/dbms/tests/queries/0_stateless/00563_complex_in_expression.reference b/dbms/tests/queries/0_stateless/00563_complex_in_expression.reference
index b7d32274dd7..480b2d124fe 100644
--- a/dbms/tests/queries/0_stateless/00563_complex_in_expression.reference
+++ b/dbms/tests/queries/0_stateless/00563_complex_in_expression.reference
@@ -1,3 +1,7 @@
 2018-01-29	100	key
 2018-01-29	100	key
 2
+0	1
+1	0
+2	0
+99	1
diff --git a/dbms/tests/queries/0_stateless/00563_complex_in_expression.sql b/dbms/tests/queries/0_stateless/00563_complex_in_expression.sql
index 522ff0168ed..1a84e560509 100644
--- a/dbms/tests/queries/0_stateless/00563_complex_in_expression.sql
+++ b/dbms/tests/queries/0_stateless/00563_complex_in_expression.sql
@@ -22,4 +22,11 @@ ALL INNER JOIN
     FROM test.join_with_index
     WHERE toUInt64(data) IN (0, 529335254087962442)
 ) USING (key);
-DROP TABLE IF EXISTS test.join_with_index;
\ No newline at end of file
+
+SELECT _uniq, _uniq IN (0, 99)
+FROM test.join_with_index
+ARRAY JOIN
+    [key, data] AS _uniq
+ORDER BY _uniq;
+
+DROP TABLE IF EXISTS test.join_with_index;

From 109b73b94724225afabb93621f1bd8e9d39deba3 Mon Sep 17 00:00:00 2001
From: Kirill Shvakov <shvakov@gmail.com>
Date: Mon, 12 Mar 2018 17:36:33 +0200
Subject: [PATCH 143/244] fix CREATE TEMPORARY TABLE IF NOT EXISTS

---
 dbms/src/Interpreters/InterpreterCreateQuery.cpp              | 4 +++-
 .../00600_create_temporary_table_if_not_exists.reference      | 3 +++
 .../00600_create_temporary_table_if_not_exists.sql            | 4 ++++
 3 files changed, 10 insertions(+), 1 deletion(-)
 create mode 100644 dbms/tests/queries/0_stateless/00600_create_temporary_table_if_not_exists.reference
 create mode 100644 dbms/tests/queries/0_stateless/00600_create_temporary_table_if_not_exists.sql

diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp
index dd594d2993a..3d70fc343b9 100644
--- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp
@@ -516,7 +516,9 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create)
                     throw Exception("Table " + database_name + "." + table_name + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS);
             }
         }
-
+        else if (context.tryGetExternalTable(table_name) && create.if_not_exists)
+             return {};
+             
         res = StorageFactory::instance().get(create,
             data_path,
             table_name,
diff --git a/dbms/tests/queries/0_stateless/00600_create_temporary_table_if_not_exists.reference b/dbms/tests/queries/0_stateless/00600_create_temporary_table_if_not_exists.reference
new file mode 100644
index 00000000000..01e79c32a8c
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00600_create_temporary_table_if_not_exists.reference
@@ -0,0 +1,3 @@
+1
+2
+3
diff --git a/dbms/tests/queries/0_stateless/00600_create_temporary_table_if_not_exists.sql b/dbms/tests/queries/0_stateless/00600_create_temporary_table_if_not_exists.sql
new file mode 100644
index 00000000000..3f4c80ddcc6
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00600_create_temporary_table_if_not_exists.sql
@@ -0,0 +1,4 @@
+CREATE TEMPORARY TABLE IF NOT EXISTS temporary_table (column UInt32) ENGINE = Memory;
+CREATE TEMPORARY TABLE IF NOT EXISTS temporary_table (column UInt32) ENGINE = Memory;
+INSERT INTO temporary_table VALUES (1), (2), (3);
+SELECT column FROM temporary_table ORDER BY column;
\ No newline at end of file

From 02aacc894580add12be2132d7d7dec4ce6070cf6 Mon Sep 17 00:00:00 2001
From: Alexey Zatelepin <ztlpn@yandex-team.ru>
Date: Sat, 17 Mar 2018 02:11:15 +0300
Subject: [PATCH 144/244] fix packages repo URL in Dockerfiles

---
 docker/client/Dockerfile | 2 +-
 docker/server/Dockerfile | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile
index 61f27405b83..9c8f3d087a3 100644
--- a/docker/client/Dockerfile
+++ b/docker/client/Dockerfile
@@ -1,6 +1,6 @@
 FROM ubuntu:16.04
 
-ARG repository="deb https://repo.yandex.ru/clickhouse/xenial/ dists/stable/main/binary-amd64/"
+ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/"
 ARG version=\*
 
 RUN apt-get update && \
diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile
index ddc189877ac..2d4f998eff1 100644
--- a/docker/server/Dockerfile
+++ b/docker/server/Dockerfile
@@ -1,6 +1,6 @@
 FROM ubuntu:16.04
 
-ARG repository="deb https://repo.yandex.ru/clickhouse/xenial/ dists/stable/main/binary-amd64/"
+ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/"
 ARG version=\*
 
 RUN apt-get update && \

From 60ecc03253ef6e517c22d7663c3e9b482bfb661f Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Sat, 17 Mar 2018 20:46:25 +0300
Subject: [PATCH 145/244] Preparation #2026

---
 dbms/src/Functions/FunctionsComparison.h | 179 ++++++++---------------
 1 file changed, 62 insertions(+), 117 deletions(-)

diff --git a/dbms/src/Functions/FunctionsComparison.h b/dbms/src/Functions/FunctionsComparison.h
index 011ecdfb030..5ea8f8e738d 100644
--- a/dbms/src/Functions/FunctionsComparison.h
+++ b/dbms/src/Functions/FunctionsComparison.h
@@ -135,11 +135,6 @@ struct NumComparisonImpl
     {
         NumComparisonImpl<B, A, typename Op::SymmetricOp>::vector_constant(b, a, c);
     }
-
-    static void constant_constant(A a, B b, UInt8 & c)
-    {
-        c = Op::apply(a, b);
-    }
 };
 
 
@@ -338,18 +333,6 @@ struct StringComparisonImpl
     {
         StringComparisonImpl<typename Op::SymmetricOp>::fixed_string_vector_constant(b_data, b_n, a, c);
     }
-
-    static void constant_constant(
-        const std::string & a,
-        const std::string & b,
-        UInt8 & c)
-    {
-        size_t a_n = a.size();
-        size_t b_n = b.size();
-
-        int res = memcmp(a.data(), b.data(), std::min(a_n, b_n));
-        c = Op::apply(res, 0) || (res == 0 && Op::apply(a_n, b_n));
-    }
 };
 
 
@@ -513,14 +496,6 @@ struct StringEqualsImpl
     {
         fixed_string_vector_constant(b_data, b_n, a, c);
     }
-
-    static void constant_constant(
-        const std::string & a,
-        const std::string & b,
-        UInt8 & c)
-    {
-        c = positive == (a == b);
-    }
 };
 
 
@@ -552,11 +527,6 @@ struct GenericComparisonImpl
     {
         GenericComparisonImpl<typename Op::SymmetricOp>::vector_constant(b, a, c);
     }
-
-    static void constant_constant(const IColumn & a, const IColumn & b, UInt8 & c)
-    {
-        c = Op::apply(a.compareAt(0, 0, b, 1), 0);
-    }
 };
 
 
@@ -577,6 +547,8 @@ public:
     static constexpr auto name = Name::name;
     static FunctionPtr create(const Context &) { return std::make_shared<FunctionComparison>(); };
 
+    bool useDefaultImplementationForConstants() const override { return true; }
+
 private:
     template <typename T0, typename T1>
     bool executeNumRightType(Block & block, size_t result, const ColumnVector<T0> * col_left, const IColumn * col_right_untyped)
@@ -621,14 +593,6 @@ private:
             block.getByPosition(result).column = std::move(col_res);
             return true;
         }
-        else if (auto col_right = checkAndGetColumnConst<ColumnVector<T1>>(col_right_untyped))
-        {
-            UInt8 res = 0;
-            NumComparisonImpl<T0, T1, Op<T0, T1>>::constant_constant(col_left->template getValue<T0>(), col_right->template getValue<T1>(), res);
-
-            block.getByPosition(result).column = DataTypeUInt8().createColumnConst(col_left->size(), toField(res));
-            return true;
-        }
 
         return false;
     }
@@ -692,68 +656,58 @@ private:
 
         using StringImpl = StringComparisonImpl<Op<int, int>>;
 
-        if (c0_const && c1_const)
-        {
-            UInt8 res = 0;
-            StringImpl::constant_constant(c0_const->getValue<String>(), c1_const->getValue<String>(), res);
-            block.getByPosition(result).column = block.getByPosition(result).type->createColumnConst(c0_const->size(), toField(res));
-            return true;
-        }
+        auto c_res = ColumnUInt8::create();
+        ColumnUInt8::Container & vec_res = c_res->getData();
+        vec_res.resize(c0->size());
+
+        if (c0_string && c1_string)
+            StringImpl::string_vector_string_vector(
+                c0_string->getChars(), c0_string->getOffsets(),
+                c1_string->getChars(), c1_string->getOffsets(),
+                c_res->getData());
+        else if (c0_string && c1_fixed_string)
+            StringImpl::string_vector_fixed_string_vector(
+                c0_string->getChars(), c0_string->getOffsets(),
+                c1_fixed_string->getChars(), c1_fixed_string->getN(),
+                c_res->getData());
+        else if (c0_string && c1_const)
+            StringImpl::string_vector_constant(
+                c0_string->getChars(), c0_string->getOffsets(),
+                c1_const->getValue<String>(),
+                c_res->getData());
+        else if (c0_fixed_string && c1_string)
+            StringImpl::fixed_string_vector_string_vector(
+                c0_fixed_string->getChars(), c0_fixed_string->getN(),
+                c1_string->getChars(), c1_string->getOffsets(),
+                c_res->getData());
+        else if (c0_fixed_string && c1_fixed_string)
+            StringImpl::fixed_string_vector_fixed_string_vector(
+                c0_fixed_string->getChars(), c0_fixed_string->getN(),
+                c1_fixed_string->getChars(), c1_fixed_string->getN(),
+                c_res->getData());
+        else if (c0_fixed_string && c1_const)
+            StringImpl::fixed_string_vector_constant(
+                c0_fixed_string->getChars(), c0_fixed_string->getN(),
+                c1_const->getValue<String>(),
+                c_res->getData());
+        else if (c0_const && c1_string)
+            StringImpl::constant_string_vector(
+                c0_const->getValue<String>(),
+                c1_string->getChars(), c1_string->getOffsets(),
+                c_res->getData());
+        else if (c0_const && c1_fixed_string)
+            StringImpl::constant_fixed_string_vector(
+                c0_const->getValue<String>(),
+                c1_fixed_string->getChars(), c1_fixed_string->getN(),
+                c_res->getData());
         else
-        {
-            auto c_res = ColumnUInt8::create();
-            ColumnUInt8::Container & vec_res = c_res->getData();
-            vec_res.resize(c0->size());
+            throw Exception("Illegal columns "
+                + c0->getName() + " and " + c1->getName()
+                + " of arguments of function " + getName(),
+                ErrorCodes::ILLEGAL_COLUMN);
 
-            if (c0_string && c1_string)
-                StringImpl::string_vector_string_vector(
-                    c0_string->getChars(), c0_string->getOffsets(),
-                    c1_string->getChars(), c1_string->getOffsets(),
-                    c_res->getData());
-            else if (c0_string && c1_fixed_string)
-                StringImpl::string_vector_fixed_string_vector(
-                    c0_string->getChars(), c0_string->getOffsets(),
-                    c1_fixed_string->getChars(), c1_fixed_string->getN(),
-                    c_res->getData());
-            else if (c0_string && c1_const)
-                StringImpl::string_vector_constant(
-                    c0_string->getChars(), c0_string->getOffsets(),
-                    c1_const->getValue<String>(),
-                    c_res->getData());
-            else if (c0_fixed_string && c1_string)
-                StringImpl::fixed_string_vector_string_vector(
-                    c0_fixed_string->getChars(), c0_fixed_string->getN(),
-                    c1_string->getChars(), c1_string->getOffsets(),
-                    c_res->getData());
-            else if (c0_fixed_string && c1_fixed_string)
-                StringImpl::fixed_string_vector_fixed_string_vector(
-                    c0_fixed_string->getChars(), c0_fixed_string->getN(),
-                    c1_fixed_string->getChars(), c1_fixed_string->getN(),
-                    c_res->getData());
-            else if (c0_fixed_string && c1_const)
-                StringImpl::fixed_string_vector_constant(
-                    c0_fixed_string->getChars(), c0_fixed_string->getN(),
-                    c1_const->getValue<String>(),
-                    c_res->getData());
-            else if (c0_const && c1_string)
-                StringImpl::constant_string_vector(
-                    c0_const->getValue<String>(),
-                    c1_string->getChars(), c1_string->getOffsets(),
-                    c_res->getData());
-            else if (c0_const && c1_fixed_string)
-                StringImpl::constant_fixed_string_vector(
-                    c0_const->getValue<String>(),
-                    c1_fixed_string->getChars(), c1_fixed_string->getN(),
-                    c_res->getData());
-            else
-                throw Exception("Illegal columns "
-                    + c0->getName() + " and " + c1->getName()
-                    + " of arguments of function " + getName(),
-                    ErrorCodes::ILLEGAL_COLUMN);
-
-            block.getByPosition(result).column = std::move(c_res);
-            return true;
-        }
+        block.getByPosition(result).column = std::move(c_res);
+        return true;
     }
 
     void executeDateOrDateTimeOrEnumWithConstString(
@@ -985,27 +939,18 @@ private:
         bool c0_const = c0->isColumnConst();
         bool c1_const = c1->isColumnConst();
 
-        if (c0_const && c1_const)
-        {
-            UInt8 res = 0;
-            GenericComparisonImpl<Op<int, int>>::constant_constant(*c0, *c1, res);
-            block.getByPosition(result).column = DataTypeUInt8().createColumnConst(c0->size(), toField(res));
-        }
+        auto c_res = ColumnUInt8::create();
+        ColumnUInt8::Container & vec_res = c_res->getData();
+        vec_res.resize(c0->size());
+
+        if (c0_const)
+            GenericComparisonImpl<Op<int, int>>::constant_vector(*c0, *c1, vec_res);
+        else if (c1_const)
+            GenericComparisonImpl<Op<int, int>>::vector_constant(*c0, *c1, vec_res);
         else
-        {
-            auto c_res = ColumnUInt8::create();
-            ColumnUInt8::Container & vec_res = c_res->getData();
-            vec_res.resize(c0->size());
+            GenericComparisonImpl<Op<int, int>>::vector_vector(*c0, *c1, vec_res);
 
-            if (c0_const)
-                GenericComparisonImpl<Op<int, int>>::constant_vector(*c0, *c1, vec_res);
-            else if (c1_const)
-                GenericComparisonImpl<Op<int, int>>::vector_constant(*c0, *c1, vec_res);
-            else
-                GenericComparisonImpl<Op<int, int>>::vector_vector(*c0, *c1, vec_res);
-
-            block.getByPosition(result).column = std::move(c_res);
-        }
+        block.getByPosition(result).column = std::move(c_res);
     }
 
 public:

From 60701afa3e93c6753ec4e88068c2ef91e84c3540 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Sat, 17 Mar 2018 21:02:19 +0300
Subject: [PATCH 146/244] Removed unused code #2010

---
 .../AggregateFunctionUniq.h                   | 54 ++++---------------
 1 file changed, 11 insertions(+), 43 deletions(-)

diff --git a/dbms/src/AggregateFunctions/AggregateFunctionUniq.h b/dbms/src/AggregateFunctions/AggregateFunctionUniq.h
index f5949a6d468..288d1fc5c3c 100644
--- a/dbms/src/AggregateFunctions/AggregateFunctionUniq.h
+++ b/dbms/src/AggregateFunctions/AggregateFunctionUniq.h
@@ -122,8 +122,8 @@ struct AggregateFunctionUniqExactData<String>
     static String getName() { return "uniqExact"; }
 };
 
-template <typename T, HyperLogLogMode mode>
-struct BaseUniqCombinedData
+template <typename T>
+struct AggregateFunctionUniqCombinedData
 {
     using Key = UInt32;
     using Set = CombinedCardinalityEstimator<
@@ -135,14 +135,15 @@ struct BaseUniqCombinedData
         TrivialHash,
         UInt32,
         HyperLogLogBiasEstimator<UniqCombinedBiasData>,
-        mode
-    >;
+        HyperLogLogMode::FullFeatured>;
 
     Set set;
+
+    static String getName() { return "uniqCombined"; }
 };
 
-template <HyperLogLogMode mode>
-struct BaseUniqCombinedData<String, mode>
+template <>
+struct AggregateFunctionUniqCombinedData<String>
 {
     using Key = UInt64;
     using Set = CombinedCardinalityEstimator<
@@ -154,44 +155,14 @@ struct BaseUniqCombinedData<String, mode>
         TrivialHash,
         UInt64,
         HyperLogLogBiasEstimator<UniqCombinedBiasData>,
-        mode
-    >;
+        HyperLogLogMode::FullFeatured>;
 
     Set set;
-};
 
-/// Aggregate functions uniqCombinedRaw, uniqCombinedLinearCounting, and uniqCombinedBiasCorrected
-///  are intended for development of new versions of the uniqCombined function.
-/// Users should only use uniqCombined.
-
-template <typename T>
-struct AggregateFunctionUniqCombinedRawData
-    : public BaseUniqCombinedData<T, HyperLogLogMode::Raw>
-{
-    static String getName() { return "uniqCombinedRaw"; }
-};
-
-template <typename T>
-struct AggregateFunctionUniqCombinedLinearCountingData
-    : public BaseUniqCombinedData<T, HyperLogLogMode::LinearCounting>
-{
-    static String getName() { return "uniqCombinedLinearCounting"; }
-};
-
-template <typename T>
-struct AggregateFunctionUniqCombinedBiasCorrectedData
-    : public BaseUniqCombinedData<T, HyperLogLogMode::BiasCorrected>
-{
-    static String getName() { return "uniqCombinedBiasCorrected"; }
-};
-
-template <typename T>
-struct AggregateFunctionUniqCombinedData
-    : public BaseUniqCombinedData<T, HyperLogLogMode::FullFeatured>
-{
     static String getName() { return "uniqCombined"; }
 };
 
+
 namespace detail
 {
 
@@ -288,10 +259,7 @@ struct OneAdder
                 data.set.insert(CityHash_v1_0_2::CityHash64(value.data, value.size));
             }
         }
-        else if constexpr (std::is_same_v<Data, AggregateFunctionUniqCombinedRawData<T>>
-            || std::is_same_v<Data, AggregateFunctionUniqCombinedLinearCountingData<T>>
-            || std::is_same_v<Data, AggregateFunctionUniqCombinedBiasCorrectedData<T>>
-            || std::is_same_v<Data, AggregateFunctionUniqCombinedData<T>>)
+        else if constexpr (std::is_same_v<Data, AggregateFunctionUniqCombinedData<T>>)
         {
             if constexpr (!std::is_same_v<T, String>)
             {
@@ -371,7 +339,7 @@ public:
 
 /** For multiple arguments. To compute, hashes them.
   * You can pass multiple arguments as is; You can also pass one argument - a tuple.
-  * But (for the possibility of effective implementation), you can not pass several arguments, among which there are tuples.
+  * But (for the possibility of efficient implementation), you can not pass several arguments, among which there are tuples.
   */
 template <typename Data, bool argument_is_tuple>
 class AggregateFunctionUniqVariadic final : public IAggregateFunctionDataHelper<Data, AggregateFunctionUniqVariadic<Data, argument_is_tuple>>

From b916f17215fd504399bd6550979bfa26dd559df1 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Sat, 17 Mar 2018 21:14:05 +0300
Subject: [PATCH 147/244] Implemented 'uniq' family of functions for generic
 case including Array argument #2010

---
 .../AggregateFunctionUniq.cpp                 | 34 +++++++++----------
 .../AggregateFunctionUniqUpTo.cpp             | 16 ++++-----
 .../0_stateless/00608_uniq_array.reference    |  3 ++
 .../queries/0_stateless/00608_uniq_array.sql  |  3 ++
 4 files changed, 30 insertions(+), 26 deletions(-)
 create mode 100644 dbms/tests/queries/0_stateless/00608_uniq_array.reference
 create mode 100644 dbms/tests/queries/0_stateless/00608_uniq_array.sql

diff --git a/dbms/src/AggregateFunctions/AggregateFunctionUniq.cpp b/dbms/src/AggregateFunctions/AggregateFunctionUniq.cpp
index e64247796af..d6831ce1c43 100644
--- a/dbms/src/AggregateFunctions/AggregateFunctionUniq.cpp
+++ b/dbms/src/AggregateFunctions/AggregateFunctionUniq.cpp
@@ -33,6 +33,10 @@ AggregateFunctionPtr createAggregateFunctionUniq(const std::string & name, const
 {
     assertNoParameters(name, params);
 
+    if (argument_types.empty())
+        throw Exception("Incorrect number of arguments for aggregate function " + name,
+            ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
+
     if (argument_types.size() == 1)
     {
         const IDataType & argument_type = *argument_types[0];
@@ -51,23 +55,18 @@ AggregateFunctionPtr createAggregateFunctionUniq(const std::string & name, const
             return std::make_shared<AggregateFunctionUniqVariadic<DataForVariadic, true>>(argument_types);
         else if (typeid_cast<const DataTypeUUID *>(&argument_type))
             return std::make_shared<AggregateFunctionUniq<DataTypeUUID::FieldType, Data>>();
-        else
-            throw Exception("Illegal type " + argument_types[0]->getName() + " of argument for aggregate function " + name,
-                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
     }
-    else if (argument_types.size() > 1)
+    else
     {
         /// If there are several arguments, then no tuples allowed among them.
         for (const auto & type : argument_types)
             if (typeid_cast<const DataTypeTuple *>(type.get()))
                 throw Exception("Tuple argument of function " + name + " must be the only argument",
                     ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
-
-        return std::make_shared<AggregateFunctionUniqVariadic<DataForVariadic, false>>(argument_types);
     }
-    else
-        throw Exception("Incorrect number of arguments for aggregate function " + name,
-            ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
+
+    /// "Variadic" method also works as a fallback generic case for single argument.
+    return std::make_shared<AggregateFunctionUniqVariadic<DataForVariadic, false>>(argument_types);
 }
 
 template <template <typename> class Data, typename DataForVariadic>
@@ -75,6 +74,10 @@ AggregateFunctionPtr createAggregateFunctionUniq(const std::string & name, const
 {
     assertNoParameters(name, params);
 
+    if (argument_types.empty())
+        throw Exception("Incorrect number of arguments for aggregate function " + name,
+            ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
+
     if (argument_types.size() == 1)
     {
         const IDataType & argument_type = *argument_types[0];
@@ -93,23 +96,18 @@ AggregateFunctionPtr createAggregateFunctionUniq(const std::string & name, const
             return std::make_shared<AggregateFunctionUniqVariadic<DataForVariadic, true>>(argument_types);
         else if (typeid_cast<const DataTypeUUID *>(&argument_type))
             return std::make_shared<AggregateFunctionUniq<DataTypeUUID::FieldType, Data<DataTypeUUID::FieldType>>>();
-        else
-            throw Exception("Illegal type " + argument_types[0]->getName() + " of argument for aggregate function " + name,
-                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
     }
-    else if (argument_types.size() > 1)
+    else
     {
         /// If there are several arguments, then no tuples allowed among them.
         for (const auto & type : argument_types)
             if (typeid_cast<const DataTypeTuple *>(type.get()))
                 throw Exception("Tuple argument of function " + name + " must be the only argument",
                     ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
-
-        return std::make_shared<AggregateFunctionUniqVariadic<DataForVariadic, false>>(argument_types);
     }
-    else
-        throw Exception("Incorrect number of arguments for aggregate function " + name,
-            ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
+
+    /// "Variadic" method also works as a fallback generic case for single argument.
+    return std::make_shared<AggregateFunctionUniqVariadic<DataForVariadic, false>>(argument_types);
 }
 
 }
diff --git a/dbms/src/AggregateFunctions/AggregateFunctionUniqUpTo.cpp b/dbms/src/AggregateFunctions/AggregateFunctionUniqUpTo.cpp
index 1a3934af88f..d1d9b70f5f9 100644
--- a/dbms/src/AggregateFunctions/AggregateFunctionUniqUpTo.cpp
+++ b/dbms/src/AggregateFunctions/AggregateFunctionUniqUpTo.cpp
@@ -42,6 +42,10 @@ AggregateFunctionPtr createAggregateFunctionUniqUpTo(const std::string & name, c
         threshold = threshold_param;
     }
 
+    if (argument_types.empty())
+        throw Exception("Incorrect number of arguments for aggregate function " + name,
+            ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
+
     if (argument_types.size() == 1)
     {
         const IDataType & argument_type = *argument_types[0];
@@ -60,22 +64,18 @@ AggregateFunctionPtr createAggregateFunctionUniqUpTo(const std::string & name, c
             return std::make_shared<AggregateFunctionUniqUpToVariadic<true>>(argument_types, threshold);
         else if (typeid_cast<const DataTypeUUID *>(&argument_type))
             return std::make_shared<AggregateFunctionUniqUpTo<DataTypeUUID::FieldType>>(threshold);
-        else
-            throw Exception("Illegal type " + argument_types[0]->getName() + " of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
     }
-    else if (argument_types.size() > 1)
+    else
     {
         /// If there are several arguments, then no tuples allowed among them.
         for (const auto & type : argument_types)
             if (typeid_cast<const DataTypeTuple *>(type.get()))
                 throw Exception("Tuple argument of function " + name + " must be the only argument",
                     ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
-
-        return std::make_shared<AggregateFunctionUniqUpToVariadic<false>>(argument_types, threshold);
     }
-    else
-        throw Exception("Incorrect number of arguments for aggregate function " + name,
-            ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
+
+    /// "Variadic" method also works as a fallback generic case for single argument.
+    return std::make_shared<AggregateFunctionUniqUpToVariadic<false>>(argument_types, threshold);
 }
 
 }
diff --git a/dbms/tests/queries/0_stateless/00608_uniq_array.reference b/dbms/tests/queries/0_stateless/00608_uniq_array.reference
new file mode 100644
index 00000000000..1f242fa6f00
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00608_uniq_array.reference
@@ -0,0 +1,3 @@
+3
+3
+3
diff --git a/dbms/tests/queries/0_stateless/00608_uniq_array.sql b/dbms/tests/queries/0_stateless/00608_uniq_array.sql
new file mode 100644
index 00000000000..d4b0efd8fb4
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00608_uniq_array.sql
@@ -0,0 +1,3 @@
+SELECT uniq(x) FROM (SELECT arrayJoin([[1, 2], [1, 2], [1, 2, 3], []]) AS x);
+SELECT uniqExact(x) FROM (SELECT arrayJoin([[1, 2], [1, 2], [1, 2, 3], []]) AS x);
+SELECT uniqUpTo(2)(x) FROM (SELECT arrayJoin([[1, 2], [1, 2], [1, 2, 3], []]) AS x);

From 90aa6374a7e0cd14ddaae96244bbdb43130846df Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Sat, 17 Mar 2018 21:29:50 +0300
Subject: [PATCH 148/244] Revert "Preparation #2026"

This reverts commit 60ecc03253ef6e517c22d7663c3e9b482bfb661f.
---
 dbms/src/Functions/FunctionsComparison.h | 179 +++++++++++++++--------
 1 file changed, 117 insertions(+), 62 deletions(-)

diff --git a/dbms/src/Functions/FunctionsComparison.h b/dbms/src/Functions/FunctionsComparison.h
index 5ea8f8e738d..011ecdfb030 100644
--- a/dbms/src/Functions/FunctionsComparison.h
+++ b/dbms/src/Functions/FunctionsComparison.h
@@ -135,6 +135,11 @@ struct NumComparisonImpl
     {
         NumComparisonImpl<B, A, typename Op::SymmetricOp>::vector_constant(b, a, c);
     }
+
+    static void constant_constant(A a, B b, UInt8 & c)
+    {
+        c = Op::apply(a, b);
+    }
 };
 
 
@@ -333,6 +338,18 @@ struct StringComparisonImpl
     {
         StringComparisonImpl<typename Op::SymmetricOp>::fixed_string_vector_constant(b_data, b_n, a, c);
     }
+
+    static void constant_constant(
+        const std::string & a,
+        const std::string & b,
+        UInt8 & c)
+    {
+        size_t a_n = a.size();
+        size_t b_n = b.size();
+
+        int res = memcmp(a.data(), b.data(), std::min(a_n, b_n));
+        c = Op::apply(res, 0) || (res == 0 && Op::apply(a_n, b_n));
+    }
 };
 
 
@@ -496,6 +513,14 @@ struct StringEqualsImpl
     {
         fixed_string_vector_constant(b_data, b_n, a, c);
     }
+
+    static void constant_constant(
+        const std::string & a,
+        const std::string & b,
+        UInt8 & c)
+    {
+        c = positive == (a == b);
+    }
 };
 
 
@@ -527,6 +552,11 @@ struct GenericComparisonImpl
     {
         GenericComparisonImpl<typename Op::SymmetricOp>::vector_constant(b, a, c);
     }
+
+    static void constant_constant(const IColumn & a, const IColumn & b, UInt8 & c)
+    {
+        c = Op::apply(a.compareAt(0, 0, b, 1), 0);
+    }
 };
 
 
@@ -547,8 +577,6 @@ public:
     static constexpr auto name = Name::name;
     static FunctionPtr create(const Context &) { return std::make_shared<FunctionComparison>(); };
 
-    bool useDefaultImplementationForConstants() const override { return true; }
-
 private:
     template <typename T0, typename T1>
     bool executeNumRightType(Block & block, size_t result, const ColumnVector<T0> * col_left, const IColumn * col_right_untyped)
@@ -593,6 +621,14 @@ private:
             block.getByPosition(result).column = std::move(col_res);
             return true;
         }
+        else if (auto col_right = checkAndGetColumnConst<ColumnVector<T1>>(col_right_untyped))
+        {
+            UInt8 res = 0;
+            NumComparisonImpl<T0, T1, Op<T0, T1>>::constant_constant(col_left->template getValue<T0>(), col_right->template getValue<T1>(), res);
+
+            block.getByPosition(result).column = DataTypeUInt8().createColumnConst(col_left->size(), toField(res));
+            return true;
+        }
 
         return false;
     }
@@ -656,58 +692,68 @@ private:
 
         using StringImpl = StringComparisonImpl<Op<int, int>>;
 
-        auto c_res = ColumnUInt8::create();
-        ColumnUInt8::Container & vec_res = c_res->getData();
-        vec_res.resize(c0->size());
-
-        if (c0_string && c1_string)
-            StringImpl::string_vector_string_vector(
-                c0_string->getChars(), c0_string->getOffsets(),
-                c1_string->getChars(), c1_string->getOffsets(),
-                c_res->getData());
-        else if (c0_string && c1_fixed_string)
-            StringImpl::string_vector_fixed_string_vector(
-                c0_string->getChars(), c0_string->getOffsets(),
-                c1_fixed_string->getChars(), c1_fixed_string->getN(),
-                c_res->getData());
-        else if (c0_string && c1_const)
-            StringImpl::string_vector_constant(
-                c0_string->getChars(), c0_string->getOffsets(),
-                c1_const->getValue<String>(),
-                c_res->getData());
-        else if (c0_fixed_string && c1_string)
-            StringImpl::fixed_string_vector_string_vector(
-                c0_fixed_string->getChars(), c0_fixed_string->getN(),
-                c1_string->getChars(), c1_string->getOffsets(),
-                c_res->getData());
-        else if (c0_fixed_string && c1_fixed_string)
-            StringImpl::fixed_string_vector_fixed_string_vector(
-                c0_fixed_string->getChars(), c0_fixed_string->getN(),
-                c1_fixed_string->getChars(), c1_fixed_string->getN(),
-                c_res->getData());
-        else if (c0_fixed_string && c1_const)
-            StringImpl::fixed_string_vector_constant(
-                c0_fixed_string->getChars(), c0_fixed_string->getN(),
-                c1_const->getValue<String>(),
-                c_res->getData());
-        else if (c0_const && c1_string)
-            StringImpl::constant_string_vector(
-                c0_const->getValue<String>(),
-                c1_string->getChars(), c1_string->getOffsets(),
-                c_res->getData());
-        else if (c0_const && c1_fixed_string)
-            StringImpl::constant_fixed_string_vector(
-                c0_const->getValue<String>(),
-                c1_fixed_string->getChars(), c1_fixed_string->getN(),
-                c_res->getData());
+        if (c0_const && c1_const)
+        {
+            UInt8 res = 0;
+            StringImpl::constant_constant(c0_const->getValue<String>(), c1_const->getValue<String>(), res);
+            block.getByPosition(result).column = block.getByPosition(result).type->createColumnConst(c0_const->size(), toField(res));
+            return true;
+        }
         else
-            throw Exception("Illegal columns "
-                + c0->getName() + " and " + c1->getName()
-                + " of arguments of function " + getName(),
-                ErrorCodes::ILLEGAL_COLUMN);
+        {
+            auto c_res = ColumnUInt8::create();
+            ColumnUInt8::Container & vec_res = c_res->getData();
+            vec_res.resize(c0->size());
 
-        block.getByPosition(result).column = std::move(c_res);
-        return true;
+            if (c0_string && c1_string)
+                StringImpl::string_vector_string_vector(
+                    c0_string->getChars(), c0_string->getOffsets(),
+                    c1_string->getChars(), c1_string->getOffsets(),
+                    c_res->getData());
+            else if (c0_string && c1_fixed_string)
+                StringImpl::string_vector_fixed_string_vector(
+                    c0_string->getChars(), c0_string->getOffsets(),
+                    c1_fixed_string->getChars(), c1_fixed_string->getN(),
+                    c_res->getData());
+            else if (c0_string && c1_const)
+                StringImpl::string_vector_constant(
+                    c0_string->getChars(), c0_string->getOffsets(),
+                    c1_const->getValue<String>(),
+                    c_res->getData());
+            else if (c0_fixed_string && c1_string)
+                StringImpl::fixed_string_vector_string_vector(
+                    c0_fixed_string->getChars(), c0_fixed_string->getN(),
+                    c1_string->getChars(), c1_string->getOffsets(),
+                    c_res->getData());
+            else if (c0_fixed_string && c1_fixed_string)
+                StringImpl::fixed_string_vector_fixed_string_vector(
+                    c0_fixed_string->getChars(), c0_fixed_string->getN(),
+                    c1_fixed_string->getChars(), c1_fixed_string->getN(),
+                    c_res->getData());
+            else if (c0_fixed_string && c1_const)
+                StringImpl::fixed_string_vector_constant(
+                    c0_fixed_string->getChars(), c0_fixed_string->getN(),
+                    c1_const->getValue<String>(),
+                    c_res->getData());
+            else if (c0_const && c1_string)
+                StringImpl::constant_string_vector(
+                    c0_const->getValue<String>(),
+                    c1_string->getChars(), c1_string->getOffsets(),
+                    c_res->getData());
+            else if (c0_const && c1_fixed_string)
+                StringImpl::constant_fixed_string_vector(
+                    c0_const->getValue<String>(),
+                    c1_fixed_string->getChars(), c1_fixed_string->getN(),
+                    c_res->getData());
+            else
+                throw Exception("Illegal columns "
+                    + c0->getName() + " and " + c1->getName()
+                    + " of arguments of function " + getName(),
+                    ErrorCodes::ILLEGAL_COLUMN);
+
+            block.getByPosition(result).column = std::move(c_res);
+            return true;
+        }
     }
 
     void executeDateOrDateTimeOrEnumWithConstString(
@@ -939,18 +985,27 @@ private:
         bool c0_const = c0->isColumnConst();
         bool c1_const = c1->isColumnConst();
 
-        auto c_res = ColumnUInt8::create();
-        ColumnUInt8::Container & vec_res = c_res->getData();
-        vec_res.resize(c0->size());
-
-        if (c0_const)
-            GenericComparisonImpl<Op<int, int>>::constant_vector(*c0, *c1, vec_res);
-        else if (c1_const)
-            GenericComparisonImpl<Op<int, int>>::vector_constant(*c0, *c1, vec_res);
+        if (c0_const && c1_const)
+        {
+            UInt8 res = 0;
+            GenericComparisonImpl<Op<int, int>>::constant_constant(*c0, *c1, res);
+            block.getByPosition(result).column = DataTypeUInt8().createColumnConst(c0->size(), toField(res));
+        }
         else
-            GenericComparisonImpl<Op<int, int>>::vector_vector(*c0, *c1, vec_res);
+        {
+            auto c_res = ColumnUInt8::create();
+            ColumnUInt8::Container & vec_res = c_res->getData();
+            vec_res.resize(c0->size());
 
-        block.getByPosition(result).column = std::move(c_res);
+            if (c0_const)
+                GenericComparisonImpl<Op<int, int>>::constant_vector(*c0, *c1, vec_res);
+            else if (c1_const)
+                GenericComparisonImpl<Op<int, int>>::vector_constant(*c0, *c1, vec_res);
+            else
+                GenericComparisonImpl<Op<int, int>>::vector_vector(*c0, *c1, vec_res);
+
+            block.getByPosition(result).column = std::move(c_res);
+        }
     }
 
 public:

From bbb12e89f06884f8ee41f05e795e66d31bb20deb Mon Sep 17 00:00:00 2001
From: proller <proller@users.noreply.github.com>
Date: Sun, 18 Mar 2018 12:02:29 +0300
Subject: [PATCH 149/244] Better listen_try, pbuilder fix (#2068)

* rename libboost/boost_1_65_0 -> contrib/boost ; make submodule ; update boost to 1.66.0

* Update contrib/boost

* Update contrib/boost

* Add comments

* Fix

* Pbuilder fix

* Better listen_try

* Much better listen_try

* Revert "remove ipv6 listen_host from docker_related_config to allow running in ipv4-only dockers"

This reverts commit 3cfa4c3e650d9d3d6037e3a18454b7f3e04d807b.

* Fix freebsd and macos build
---
 dbms/src/Server/Server.cpp              | 7 +------
 debian/.pbuilderrc                      | 4 ++--
 docker/server/docker_related_config.xml | 2 ++
 libs/libdaemon/src/BaseDaemon.cpp       | 7 +++++++
 4 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/dbms/src/Server/Server.cpp b/dbms/src/Server/Server.cpp
index 12598af18e6..820a593cd09 100644
--- a/dbms/src/Server/Server.cpp
+++ b/dbms/src/Server/Server.cpp
@@ -465,12 +465,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
             }
             catch (const Poco::Net::NetException & e)
             {
-                const auto code = e.code();
-                if (listen_try && (code == POCO_EPROTONOSUPPORT || code == POCO_EADDRNOTAVAIL || code == EAI_FAMILY
-#if defined(EAI_ADDRFAMILY)
-                    || code == EAI_ADDRFAMILY
-#endif
-                ))
+                if (listen_try)
                     LOG_ERROR(log, "Listen [" << listen_host << "]: " << e.code() << ": " << e.what() << ": " << e.message()
                         << "  If it is an IPv6 or IPv4 address and your host has disabled IPv6 or IPv4, then consider to "
                         "specify not disabled IPv4 or IPv6 address to listen in <listen_host> element of configuration "
diff --git a/debian/.pbuilderrc b/debian/.pbuilderrc
index 33abbb1ab89..e3704760ec1 100644
--- a/debian/.pbuilderrc
+++ b/debian/.pbuilderrc
@@ -164,11 +164,11 @@ esac
 case "$DIST" in
     "bionic" )
         EXTRAPACKAGES+=" liblld-6.0-dev libclang-6.0-dev liblld-6.0 "
-        export CMAKE_FLAGS="-DUSE_EMBEDDED_COMPILER=1 -DLLVM_VERSION_POSTFIX=-6.0 $CMAKE_FLAGS"
+        export CMAKE_FLAGS="-DENABLE_EMBEDDED_COMPILER=1 -DLLVM_VERSION_POSTFIX=-6.0 $CMAKE_FLAGS"
         ;;
     "artful" | "experimental" | "unstable" | "testing" )
         EXTRAPACKAGES+=" liblld-5.0-dev libclang-5.0-dev liblld-5.0 "
-        export CMAKE_FLAGS="-DUSE_EMBEDDED_COMPILER=1 $CMAKE_FLAGS"
+        export CMAKE_FLAGS="-DENABLE_EMBEDDED_COMPILER=1 $CMAKE_FLAGS"
         ;;
 esac
 
diff --git a/docker/server/docker_related_config.xml b/docker/server/docker_related_config.xml
index 7171a8b5ff7..ab6f82ad4c4 100644
--- a/docker/server/docker_related_config.xml
+++ b/docker/server/docker_related_config.xml
@@ -1,4 +1,6 @@
 <yandex>
      <!-- Listen wildcard address to allow accepting connections from other containers and host network. -->
     <listen_host>0.0.0.0</listen_host>
+    <listen_host>::</listen_host>
+    <listen_try>1</listen_try>
 </yandex>
diff --git a/libs/libdaemon/src/BaseDaemon.cpp b/libs/libdaemon/src/BaseDaemon.cpp
index 849a80ff193..e262b2df17c 100644
--- a/libs/libdaemon/src/BaseDaemon.cpp
+++ b/libs/libdaemon/src/BaseDaemon.cpp
@@ -372,12 +372,19 @@ private:
                     case BUS_OBJERR:
                         LOG_ERROR(log, "Object specific hardware error.");
                         break;
+
+                    // Linux specific
+#if defined(BUS_MCEERR_AR)
                     case BUS_MCEERR_AR:
                         LOG_ERROR(log, "Hardware memory error: action required.");
                         break;
+#endif
+#if defined(BUS_MCEERR_AO)
                     case BUS_MCEERR_AO:
                         LOG_ERROR(log, "Hardware memory error: action optional.");
                         break;
+#endif
+
                     default:
                         LOG_ERROR(log, "Unknown si_code.");
                         break;

From 590feb02b5907aad4715227e64efc4b773c2a72f Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Mon, 19 Mar 2018 17:29:40 +0300
Subject: [PATCH 150/244] Fixed error; added test from Alexander Zaitsev
 [#CLICKHOUSE-2]

---
 dbms/src/Storages/StorageMaterializedView.h      |  1 +
 dbms/src/Storages/StorageMergeTree.h             |  8 ++++----
 .../0_stateless/00609_mv_index_in_in.reference   |  3 +++
 .../queries/0_stateless/00609_mv_index_in_in.sql | 16 ++++++++++++++++
 4 files changed, 24 insertions(+), 4 deletions(-)
 create mode 100644 dbms/tests/queries/0_stateless/00609_mv_index_in_in.reference
 create mode 100644 dbms/tests/queries/0_stateless/00609_mv_index_in_in.sql

diff --git a/dbms/src/Storages/StorageMaterializedView.h b/dbms/src/Storages/StorageMaterializedView.h
index 82a5b0822b4..21105068f7a 100644
--- a/dbms/src/Storages/StorageMaterializedView.h
+++ b/dbms/src/Storages/StorageMaterializedView.h
@@ -27,6 +27,7 @@ public:
     bool supportsPrewhere() const override { return getTargetTable()->supportsPrewhere(); }
     bool supportsFinal() const override { return getTargetTable()->supportsFinal(); }
     bool supportsIndexForIn() const override { return getTargetTable()->supportsIndexForIn(); }
+    bool mayBenefitFromIndexForIn(const ASTPtr & left_in_operand) const override { return getTargetTable()->mayBenefitFromIndexForIn(left_in_operand); }
 
     BlockOutputStreamPtr write(const ASTPtr & query, const Settings & settings) override;
     void drop() override;
diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h
index acb0e03a60a..a2787a8bb30 100644
--- a/dbms/src/Storages/StorageMergeTree.h
+++ b/dbms/src/Storages/StorageMergeTree.h
@@ -32,9 +32,12 @@ public:
     }
 
     std::string getTableName() const override { return table_name; }
+
     bool supportsSampling() const override { return data.supportsSampling(); }
-    bool supportsFinal() const override { return data.supportsFinal(); }
     bool supportsPrewhere() const override { return data.supportsPrewhere(); }
+    bool supportsFinal() const override { return data.supportsFinal(); }
+    bool supportsIndexForIn() const override { return true; }
+    bool mayBenefitFromIndexForIn(const ASTPtr & left_in_operand) const override { return data.mayBenefitFromIndexForIn(left_in_operand); }
 
     const ColumnsDescription & getColumns() const override { return data.getColumns(); }
     void setColumns(ColumnsDescription columns_) override { return data.setColumns(std::move(columns_)); }
@@ -74,9 +77,6 @@ public:
 
     void alter(const AlterCommands & params, const String & database_name, const String & table_name, const Context & context) override;
 
-    bool supportsIndexForIn() const override { return true; }
-    bool mayBenefitFromIndexForIn(const ASTPtr & left_in_operand) const override { return data.mayBenefitFromIndexForIn(left_in_operand); }
-
     bool checkTableCanBeDropped() const override;
 
     MergeTreeData & getData() { return data; }
diff --git a/dbms/tests/queries/0_stateless/00609_mv_index_in_in.reference b/dbms/tests/queries/0_stateless/00609_mv_index_in_in.reference
new file mode 100644
index 00000000000..49fe65e4358
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00609_mv_index_in_in.reference
@@ -0,0 +1,3 @@
+1	2000-01-01
+1	2000-01-01
+1	2000-01-01
diff --git a/dbms/tests/queries/0_stateless/00609_mv_index_in_in.sql b/dbms/tests/queries/0_stateless/00609_mv_index_in_in.sql
new file mode 100644
index 00000000000..da583a7c2f4
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00609_mv_index_in_in.sql
@@ -0,0 +1,16 @@
+USE test;
+
+DROP TABLE IF EXISTS test;
+DROP TABLE IF EXISTS test_mv;
+
+create table test (a Int8) engine=Memory;
+
+insert into test values (1);
+create materialized view test_mv Engine=MergeTree(date, (a), 8192) populate as select a, toDate('2000-01-01') date from test;
+
+select * from test_mv; -- OK
+select * from test_mv where a in (select a from test_mv); -- EMPTY (bug)
+select * from ".inner.test_mv" where a in (select a from test_mv); -- OK
+
+DROP TABLE test;
+DROP TABLE test_mv;

From 1f4b0184f6e9dd7a7584e4bcfabe6b90bea2c042 Mon Sep 17 00:00:00 2001
From: Vitaliy Lyudvichenko <vludv@yandex-team.ru>
Date: Mon, 19 Mar 2018 17:30:19 +0300
Subject: [PATCH 151/244] Add RU changelog for 1.1.54370 release.
 [#CLICKHOUSE-3]

---
 CHANGELOG.md    |  5 ++++-
 CHANGELOG_RU.md | 30 +++++++++++++++++++++++++++++-
 2 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 847d0506427..dffde42b520 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,7 @@
-# ClickHouse 1.1.54356 Release Candidate, 2018-03-06
+# ClickHouse 1.1.54370 Release Candidate, 2018-03-16
+
+
+# ClickHouse release 1.1.54356, 2018-03-06
 
 ## New features:
 
diff --git a/CHANGELOG_RU.md b/CHANGELOG_RU.md
index 2dfbdccb7a4..d433ffd58bc 100644
--- a/CHANGELOG_RU.md
+++ b/CHANGELOG_RU.md
@@ -1,4 +1,32 @@
-# ClickHouse 1.1.54362 Release Candidate, 2018-03-11
+# ClickHouse 1.1.54370 Release Candidate, 2018-03-16
+
+## Новые возможности:
+
+* Добавлена системная таблица `system.macros` и автоматическое обновление макросов при изменении конфигурационного файла.
+* Добавлен запрос `SYSTEM RELOAD CONFIG`.
+* Добавлена агрегатная функция `maxIntersections(left, right)`, возвращающая максимальное количество интервалов `[left(i); right(i)]`, пересекающихся в одной точке. Функция `intersectionsMaxPos(left, right)` возвращает начало такого "максимального" интервала. (Michael Furmur).
+
+## Улучшения:
+
+* При вставке данных в `Replicated`-таблицу делается меньше обращений к `ZooKeeper` (также из лога `ZooKeeper` исчезло большинство user-level ошибок).
+* Добавлена возможность создавать `ALIAS`-ы для множеств.
+
+## Исправление ошибок:
+
+* Исправлена ошибка `Illegal PREWHERE` при чтении из Merge-таблицы над `Distributed`-таблицами.
+* Добавлены исправления, позволяющие запускать clickhouse-server в IPv4-only docker-контейнерах.
+* Исправлен race condition при чтении из системной таблицы system.parts_columns
+* Убрана двойная буферизация при синхронной вставке в Distributed-таблицу, которая могла приводить к timeout-ам соединений.
+* Исправлена ошибка, вызывающая чрезмерно долгое ожидание недоступной реплики перед началом выполнения `SELECT`.
+* Исправлено некорректное отображение дат в таблице `system.parts`.
+* Исправлена ошибка, приводящая к невозможности вставить данные в `Replicated`-таблицу, если в конфигурации кворума `ZooKeeper` задан непустой `chroot`.
+* Исправлен алгоритм вертикального мержа при пустом ключе `ORDER BY`.
+* Возвращена возможность использовать словари в запросах к удаленным таблицам, даже если этих словарей нет на сервере-инициаторе. Данная функциональность была потеряна в версии 1.1.54362.
+* Восстановлено поведение, при котором в запросах типа `SELECT * FROM remote('server2', default.table) WHERE col IN (SELECT col2 FROM default.table)` в правой части `IN` должна использоваться удаленная таблица `default.table`, а не локальная. данное поведение было нарушено в версии 1.1.54358.
+* Устранено ненужное Error-level логирование `Not found column ... in block`.
+
+
+# Релиз ClickHouse 1.1.54362, 2018-03-11
 
 ## Новые возможности:
 

From 45c2b0ac10573dd06031ec71e701008b56861837 Mon Sep 17 00:00:00 2001
From: Vitaliy Lyudvichenko <ludv1x@yandex.ru>
Date: Mon, 19 Mar 2018 20:39:46 +0300
Subject: [PATCH 152/244] Update CHANGELOG_RU.md

---
 CHANGELOG_RU.md | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG_RU.md b/CHANGELOG_RU.md
index d433ffd58bc..36853ed541f 100644
--- a/CHANGELOG_RU.md
+++ b/CHANGELOG_RU.md
@@ -4,23 +4,23 @@
 
 * Добавлена системная таблица `system.macros` и автоматическое обновление макросов при изменении конфигурационного файла.
 * Добавлен запрос `SYSTEM RELOAD CONFIG`.
-* Добавлена агрегатная функция `maxIntersections(left, right)`, возвращающая максимальное количество интервалов `[left(i); right(i)]`, пересекающихся в одной точке. Функция `intersectionsMaxPos(left, right)` возвращает начало такого "максимального" интервала. (Michael Furmur).
+* Добавлена агрегатная функция `maxIntersections(left_col, right_col)`, возвращающая максимальное количество одновременно пересекающихся интервалов `[left; right]`. Функция `maxIntersectionsPosition(left, right)` возвращает начало такого "максимального" интервала. ([Michael Furmur](https://github.com/yandex/ClickHouse/pull/2012)).
 
 ## Улучшения:
 
 * При вставке данных в `Replicated`-таблицу делается меньше обращений к `ZooKeeper` (также из лога `ZooKeeper` исчезло большинство user-level ошибок).
-* Добавлена возможность создавать `ALIAS`-ы для множеств.
+* Добавлена возможность создавать алиасы для множеств.
 
 ## Исправление ошибок:
 
 * Исправлена ошибка `Illegal PREWHERE` при чтении из Merge-таблицы над `Distributed`-таблицами.
 * Добавлены исправления, позволяющие запускать clickhouse-server в IPv4-only docker-контейнерах.
-* Исправлен race condition при чтении из системной таблицы system.parts_columns
-* Убрана двойная буферизация при синхронной вставке в Distributed-таблицу, которая могла приводить к timeout-ам соединений.
+* Исправлен race condition при чтении из системной таблицы `system.parts_columns`
+* Убрана двойная буферизация при синхронной вставке в `Distributed`-таблицу, которая могла приводить к timeout-ам соединений.
 * Исправлена ошибка, вызывающая чрезмерно долгое ожидание недоступной реплики перед началом выполнения `SELECT`.
 * Исправлено некорректное отображение дат в таблице `system.parts`.
-* Исправлена ошибка, приводящая к невозможности вставить данные в `Replicated`-таблицу, если в конфигурации кворума `ZooKeeper` задан непустой `chroot`.
-* Исправлен алгоритм вертикального мержа при пустом ключе `ORDER BY`.
+* Исправлена ошибка, приводящая к невозможности вставить данные в `Replicated`-таблицу, если в конфигурации кластера `ZooKeeper` задан непустой `chroot`.
+* Исправлен алгоритм вертикального мержа при пустом ключе `ORDER BY` таблицы.
 * Возвращена возможность использовать словари в запросах к удаленным таблицам, даже если этих словарей нет на сервере-инициаторе. Данная функциональность была потеряна в версии 1.1.54362.
 * Восстановлено поведение, при котором в запросах типа `SELECT * FROM remote('server2', default.table) WHERE col IN (SELECT col2 FROM default.table)` в правой части `IN` должна использоваться удаленная таблица `default.table`, а не локальная. данное поведение было нарушено в версии 1.1.54358.
 * Устранено ненужное Error-level логирование `Not found column ... in block`.

From 6b2cab139efbe3f0ae9e1b17a7cab1dc888dae1d Mon Sep 17 00:00:00 2001
From: Vitaliy Lyudvichenko <ludv1x@yandex.ru>
Date: Mon, 19 Mar 2018 20:42:47 +0300
Subject: [PATCH 153/244] Update CHANGELOG_RU.md

---
 CHANGELOG_RU.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG_RU.md b/CHANGELOG_RU.md
index 36853ed541f..4e0b481e718 100644
--- a/CHANGELOG_RU.md
+++ b/CHANGELOG_RU.md
@@ -9,7 +9,7 @@
 ## Улучшения:
 
 * При вставке данных в `Replicated`-таблицу делается меньше обращений к `ZooKeeper` (также из лога `ZooKeeper` исчезло большинство user-level ошибок).
-* Добавлена возможность создавать алиасы для множеств.
+* Добавлена возможность создавать алиасы для множеств. Пример: `WITH (1, 2, 3) AS set SELECT number IN set FROM system.numbers LIMIT 10`.
 
 ## Исправление ошибок:
 

From b1274f7b34bc7c8cd1c228521cfc5bc7bee9dc27 Mon Sep 17 00:00:00 2001
From: Alexey Zatelepin <ztlpn@yandex-team.ru>
Date: Mon, 19 Mar 2018 20:52:48 +0300
Subject: [PATCH 154/244] fix deadlock in StorageKafka::shutdown() in case
 startup() wasn't called #2075

---
 dbms/src/Storages/StorageKafka.cpp | 3 ++-
 dbms/src/Storages/StorageKafka.h   | 7 ++++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/dbms/src/Storages/StorageKafka.cpp b/dbms/src/Storages/StorageKafka.cpp
index a822fe47a92..bfe72aa016e 100644
--- a/dbms/src/Storages/StorageKafka.cpp
+++ b/dbms/src/Storages/StorageKafka.cpp
@@ -288,6 +288,7 @@ void StorageKafka::startup()
 
         // Make consumer available
         pushConsumer(consumer);
+        ++num_created_consumers;
     }
 
     // Start the reader thread
@@ -303,7 +304,7 @@ void StorageKafka::shutdown()
 
     // Unsubscribe from assignments
     LOG_TRACE(log, "Unsubscribing from assignments");
-    for (size_t i = 0; i < num_consumers; ++i)
+    for (size_t i = 0; i < num_created_consumers; ++i)
     {
         auto consumer = claimConsumer();
         consumer->unsubscribe();
diff --git a/dbms/src/Storages/StorageKafka.h b/dbms/src/Storages/StorageKafka.h
index 655b0bfe5e8..45530517e94 100644
--- a/dbms/src/Storages/StorageKafka.h
+++ b/dbms/src/Storages/StorageKafka.h
@@ -76,7 +76,12 @@ private:
     const String group;
     const String format_name;
     const String schema_name;
-    size_t num_consumers; /// Total number of created consumers
+     /// Total number of consumers
+    size_t num_consumers;
+    /// Number of actually created consumers.
+    /// Can differ from num_consumers in case of exception in startup() (or if startup() hasn't been called).
+    /// In this case we still need to be able to shutdown() properly.
+    size_t num_created_consumers = 0;
     Poco::Logger * log;
 
     // Consumer list

From e41b653f1444b89edd56c9df3544a76aa61edc89 Mon Sep 17 00:00:00 2001
From: Vitaliy Lyudvichenko <vludv@yandex-team.ru>
Date: Mon, 19 Mar 2018 21:26:52 +0300
Subject: [PATCH 155/244] Parallelized block split in sync Distributed insert.
 [#CLICKHOUSE-2]

---
 .../DistributedBlockOutputStream.cpp          | 108 ++++++++++++------
 .../DistributedBlockOutputStream.h            |  32 ++++--
 2 files changed, 92 insertions(+), 48 deletions(-)

diff --git a/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp
index 607eb9efb67..98416437497 100644
--- a/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp
+++ b/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp
@@ -89,7 +89,7 @@ void DistributedBlockOutputStream::writeAsync(const Block & block)
         return writeSplitAsync(block);
 
     writeAsyncImpl(block);
-    ++blocks_inserted;
+    ++inserted_blocks;
 }
 
 
@@ -100,17 +100,17 @@ std::string DistributedBlockOutputStream::getCurrentStateDescription()
 
     buffer << "Insertion status:\n";
     for (auto & shard_jobs : per_shard_jobs)
-        for (JobInfo & job : shard_jobs)
+        for (JobReplica & job : shard_jobs.replicas_jobs)
         {
             buffer << "Wrote " << job.blocks_written << " blocks and " << job.rows_written << " rows"
                    << " on shard " << job.shard_index << " replica " << job.replica_index
                    << ", " << addresses[job.shard_index][job.replica_index].readableString();
 
             /// Performance statistics
-            if (job.bloks_started > 0)
+            if (job.blocks_started > 0)
             {
-                buffer << " (average " << job.elapsed_time_ms / job.bloks_started << " ms per block";
-                if (job.bloks_started > 1)
+                buffer << " (average " << job.elapsed_time_ms / job.blocks_started << " ms per block";
+                if (job.blocks_started > 1)
                     buffer << ", the slowest block " << job.max_elapsed_time_for_block_ms << " ms";
                 buffer << ")";
             }
@@ -122,10 +122,11 @@ std::string DistributedBlockOutputStream::getCurrentStateDescription()
 }
 
 
-void DistributedBlockOutputStream::initWritingJobs()
+void DistributedBlockOutputStream::initWritingJobs(const Block & first_block)
 {
     const auto & addresses_with_failovers = cluster->getShardsAddresses();
     const auto & shards_info = cluster->getShardsInfo();
+    size_t num_shards = shards_info.size();
 
     remote_jobs_count = 0;
     local_jobs_count = 0;
@@ -145,7 +146,7 @@ void DistributedBlockOutputStream::initWritingJobs()
             {
                 if (!replicas[replica_index].is_local)
                 {
-                    shard_jobs.emplace_back(shard_index, replica_index, false);
+                    shard_jobs.replicas_jobs.emplace_back(shard_index, replica_index, false, first_block);
                     ++remote_jobs_count;
 
                     if (shard_info.hasInternalReplication())
@@ -156,9 +157,12 @@ void DistributedBlockOutputStream::initWritingJobs()
 
         if (shard_info.isLocal())
         {
-            shard_jobs.emplace_back(shard_index, 0, true);
+            shard_jobs.replicas_jobs.emplace_back(shard_index, 0, true, first_block);
             ++local_jobs_count;
         }
+
+        if (num_shards > 1)
+            shard_jobs.shard_current_block_permuation.reserve(first_block.rows());
     }
 }
 
@@ -184,15 +188,15 @@ void DistributedBlockOutputStream::waitForJobs()
 }
 
 
-ThreadPool::Job DistributedBlockOutputStream::runWritingJob(DistributedBlockOutputStream::JobInfo & job)
+ThreadPool::Job DistributedBlockOutputStream::runWritingJob(DistributedBlockOutputStream::JobReplica & job, const Block & current_block)
 {
     auto memory_tracker = current_memory_tracker;
-    return [this, memory_tracker, &job]()
+    return [this, memory_tracker, &job, &current_block]()
     {
         SCOPE_EXIT({++finished_jobs_count;});
 
         Stopwatch watch;
-        ++job.bloks_started;
+        ++job.blocks_started;
 
         SCOPE_EXIT({
             UInt64 elapsed_time_for_block_ms = watch.elapsedMilliseconds();
@@ -207,8 +211,30 @@ ThreadPool::Job DistributedBlockOutputStream::runWritingJob(DistributedBlockOutp
         }
 
         const auto & shard_info = cluster->getShardsInfo()[job.shard_index];
+        size_t num_shards = cluster->getShardsInfo().size();
+        auto & shard_job = per_shard_jobs[job.shard_index];
         const auto & addresses = cluster->getShardsAddresses();
-        Block & block = current_blocks.at(job.shard_index);
+
+        /// Generate current shard block
+        if (num_shards > 1)
+        {
+            auto & shard_permutation = shard_job.shard_current_block_permuation;
+            size_t num_shard_rows = shard_permutation.size();
+
+            for (size_t j = 0; j < current_block.columns(); ++j)
+            {
+                auto & src_column = current_block.getByPosition(j).column;
+                auto & dst_column = job.current_shard_block.getByPosition(j).column;
+
+                /// Zero permutation size has special meaning in IColumn::permute
+                if (num_shard_rows)
+                    dst_column = src_column->permute(shard_permutation, num_shard_rows);
+                else
+                    dst_column = src_column->cloneEmpty();
+            }
+        }
+
+        const Block & shard_block = (num_shards > 1) ? job.current_shard_block : current_block;
 
         if (!job.is_local_job)
         {
@@ -217,7 +243,7 @@ ThreadPool::Job DistributedBlockOutputStream::runWritingJob(DistributedBlockOutp
                 if (shard_info.hasInternalReplication())
                 {
                     /// Skip replica_index in case of internal replication
-                    if (per_shard_jobs[job.shard_index].size() != 1)
+                    if (shard_job.replicas_jobs.size() != 1)
                         throw Exception("There are several writing job for an automatically replicated shard", ErrorCodes::LOGICAL_ERROR);
 
                     /// TODO: it make sense to rewrite skip_unavailable_shards and max_parallel_replicas here
@@ -248,7 +274,7 @@ ThreadPool::Job DistributedBlockOutputStream::runWritingJob(DistributedBlockOutp
             }
 
             CurrentMetrics::Increment metric_increment{CurrentMetrics::DistributedSend};
-            job.stream->write(block);
+            job.stream->write(shard_block);
         }
         else
         {
@@ -265,21 +291,25 @@ ThreadPool::Job DistributedBlockOutputStream::runWritingJob(DistributedBlockOutp
 
             size_t num_repetitions = shard_info.getLocalNodeCount();
             for (size_t i = 0; i < num_repetitions; ++i)
-                job.stream->write(block);
+                job.stream->write(shard_block);
         }
 
         job.blocks_written += 1;
-        job.rows_written += block.rows();
+        job.rows_written += shard_block.rows();
     };
 }
 
 
 void DistributedBlockOutputStream::writeSync(const Block & block)
 {
+    const auto & shards_info = cluster->getShardsInfo();
+    size_t num_shards = shards_info.size();
+
     if (!pool)
     {
         /// Deferred initialization. Only for sync insertion.
-        initWritingJobs();
+        initWritingJobs(block);
+
         pool.emplace(remote_jobs_count + local_jobs_count);
         query_string = queryToString(query_ast);
 
@@ -292,14 +322,23 @@ void DistributedBlockOutputStream::writeSync(const Block & block)
         watch.restart();
     }
 
-    const auto & shards_info = cluster->getShardsInfo();
-    current_blocks = shards_info.size() > 1 ? splitBlock(block) : Blocks({block});
+    if (num_shards > 1)
+    {
+        auto current_selector = createSelector(block);
+
+        /// Prepare row numbers for each shard
+        for (size_t shard_index : ext::range(0, num_shards))
+            per_shard_jobs[shard_index].shard_current_block_permuation.resize(0);
+
+        for (size_t i = 0; i < block.rows(); ++i)
+            per_shard_jobs[current_selector[i]].shard_current_block_permuation.push_back(i);
+    }
 
     /// Run jobs in parallel for each block and wait them
     finished_jobs_count = 0;
-    for (size_t shard_index : ext::range(0, current_blocks.size()))
-        for (JobInfo & job : per_shard_jobs.at(shard_index))
-            pool->schedule(runWritingJob(job));
+    for (size_t shard_index : ext::range(0, shards_info.size()))
+        for (JobReplica & job : per_shard_jobs[shard_index].replicas_jobs)
+            pool->schedule(runWritingJob(job, block));
 
     try
     {
@@ -311,7 +350,8 @@ void DistributedBlockOutputStream::writeSync(const Block & block)
         throw;
     }
 
-    ++blocks_inserted;
+    inserted_blocks += 1;
+    inserted_rows += block.rows();
 }
 
 
@@ -321,7 +361,7 @@ void DistributedBlockOutputStream::writeSuffix()
     {
         finished_jobs_count = 0;
         for (auto & shard_jobs : per_shard_jobs)
-            for (JobInfo & job : shard_jobs)
+            for (JobReplica & job : shard_jobs.replicas_jobs)
             {
                 if (job.stream)
                     pool->schedule([&job] () { job.stream->writeSuffix(); });
@@ -338,17 +378,19 @@ void DistributedBlockOutputStream::writeSuffix()
         }
 
         double elapsed = watch.elapsedSeconds();
-        LOG_DEBUG(log, "It took " << std::fixed << std::setprecision(1) << elapsed << " sec. to insert " << blocks_inserted << " blocks"
-                       << " (average " << std::fixed << std::setprecision(1) << elapsed / blocks_inserted * 1000 << " ms. per block)"
+        LOG_DEBUG(log, "It took " << std::fixed << std::setprecision(1) << elapsed << " sec. to insert " << inserted_blocks << " blocks"
+                       << ", " << std::fixed << std::setprecision(1) << inserted_rows / elapsed << " rows per second"
                        << ". " << getCurrentStateDescription());
     }
 }
 
 
-IColumn::Selector DistributedBlockOutputStream::createSelector(Block block)
+IColumn::Selector DistributedBlockOutputStream::createSelector(const Block & source_block)
 {
-    storage.getShardingKeyExpr()->execute(block);
-    const auto & key_column = block.getByName(storage.getShardingKeyColumnName());
+    Block current_block_with_sharding_key_expr = source_block;
+    storage.getShardingKeyExpr()->execute(current_block_with_sharding_key_expr);
+
+    const auto & key_column = current_block_with_sharding_key_expr.getByName(storage.getShardingKeyColumnName());
     const auto & slot_to_shard = cluster->getSlotToShard();
 
 #define CREATE_FOR_TYPE(TYPE) \
@@ -372,12 +414,6 @@ IColumn::Selector DistributedBlockOutputStream::createSelector(Block block)
 
 Blocks DistributedBlockOutputStream::splitBlock(const Block & block)
 {
-    const auto num_cols = block.columns();
-    /// cache column pointers for later reuse
-    std::vector<const IColumn *> columns(num_cols);
-    for (size_t i = 0; i < columns.size(); ++i)
-        columns[i] = block.safeGetByPosition(i).column.get();
-
     auto selector = createSelector(block);
 
     /// Split block to num_shard smaller block, using 'selector'.
@@ -409,7 +445,7 @@ void DistributedBlockOutputStream::writeSplitAsync(const Block & block)
         if (splitted_blocks[shard_idx].rows())
             writeAsyncImpl(splitted_blocks[shard_idx], shard_idx);
 
-    ++blocks_inserted;
+    ++inserted_blocks;
 }
 
 
diff --git a/dbms/src/Storages/Distributed/DistributedBlockOutputStream.h b/dbms/src/Storages/Distributed/DistributedBlockOutputStream.h
index eed38adf417..0b37d51c856 100644
--- a/dbms/src/Storages/Distributed/DistributedBlockOutputStream.h
+++ b/dbms/src/Storages/Distributed/DistributedBlockOutputStream.h
@@ -44,7 +44,7 @@ public:
 
 private:
 
-    IColumn::Selector createSelector(Block block);
+    IColumn::Selector createSelector(const Block & source_block);
 
 
     void writeAsync(const Block & block);
@@ -65,10 +65,10 @@ private:
     /// Performs synchronous insertion to remote nodes. If timeout_exceeded flag was set, throws.
     void writeSync(const Block & block);
 
-    void initWritingJobs();
+    void initWritingJobs(const Block & first_block);
 
-    struct JobInfo;
-    ThreadPool::Job runWritingJob(JobInfo & job);
+    struct JobReplica;
+    ThreadPool::Job runWritingJob(JobReplica & job, const Block & current_block);
 
     void waitForJobs();
 
@@ -80,7 +80,8 @@ private:
     ASTPtr query_ast;
     ClusterPtr cluster;
     const Settings & settings;
-    size_t blocks_inserted = 0;
+    size_t inserted_blocks = 0;
+    size_t inserted_rows = 0;
 
     bool insert_sync;
 
@@ -91,16 +92,18 @@ private:
     ThrottlerPtr throttler;
     String query_string;
 
-    struct JobInfo
+    struct JobReplica
     {
-        JobInfo() = default;
-        JobInfo(size_t shard_index, size_t replica_index, bool is_local_job)
-            : shard_index(shard_index), replica_index(replica_index), is_local_job(is_local_job) {}
+        JobReplica() = default;
+        JobReplica(size_t shard_index, size_t replica_index, bool is_local_job, const Block & sample_block)
+            : shard_index(shard_index), replica_index(replica_index), is_local_job(is_local_job), current_shard_block(sample_block.cloneEmpty()) {}
 
         size_t shard_index = 0;
         size_t replica_index = 0;
         bool is_local_job = false;
 
+        Block current_shard_block;
+
         ConnectionPool::Entry connection_entry;
         std::unique_ptr<Context> local_context;
         BlockOutputStreamPtr stream;
@@ -108,13 +111,18 @@ private:
         UInt64 blocks_written = 0;
         UInt64 rows_written = 0;
 
-        UInt64 bloks_started = 0;
+        UInt64 blocks_started = 0;
         UInt64 elapsed_time_ms = 0;
         UInt64 max_elapsed_time_for_block_ms = 0;
     };
 
-    std::vector<std::list<JobInfo>> per_shard_jobs;
-    Blocks current_blocks;
+    struct JobShard
+    {
+        std::list<JobReplica> replicas_jobs;
+        IColumn::Permutation shard_current_block_permuation;
+    };
+
+    std::vector<JobShard> per_shard_jobs;
 
     size_t remote_jobs_count = 0;
     size_t local_jobs_count = 0;

From 1a0833d0e7759a85965781f28ecbfb68469059b1 Mon Sep 17 00:00:00 2001
From: Vitaliy Lyudvichenko <vludv@yandex-team.ru>
Date: Mon, 19 Mar 2018 21:30:49 +0300
Subject: [PATCH 156/244] Fixed small race condition in ClusterCopier.
 [#CLICKHOUSE-2]

---
 dbms/src/Server/ClusterCopier.cpp | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/dbms/src/Server/ClusterCopier.cpp b/dbms/src/Server/ClusterCopier.cpp
index 4233ce70e3f..3bcbad95d76 100644
--- a/dbms/src/Server/ClusterCopier.cpp
+++ b/dbms/src/Server/ClusterCopier.cpp
@@ -1259,8 +1259,10 @@ protected:
         }
 
         /// Remove the locking node
-        cleaner_holder.reset();
-        zookeeper->remove(is_dirty_flag_path);
+        zkutil::Ops ops;
+        ops.emplace_back(new zkutil::Op::Remove(dirt_cleaner_path, -1));
+        ops.emplace_back(new zkutil::Op::Remove(is_dirty_flag_path, -1));
+        zookeeper->multi(ops);
 
         LOG_INFO(log, "Partition " << task_partition.name << " was dropped on cluster " << task_table.cluster_push_name);
         return true;

From 86d53b34a55226715edb592070dd6ca52a83a9a3 Mon Sep 17 00:00:00 2001
From: Alexey Zatelepin <ztlpn@yandex-team.ru>
Date: Tue, 20 Mar 2018 16:42:44 +0300
Subject: [PATCH 157/244] forward ALTER PARTITION statements for materialized
 views [#CLICKHOUSE-3659]

---
 dbms/src/Storages/StorageMaterializedView.cpp | 33 +++++++++++++++++++
 dbms/src/Storages/StorageMaterializedView.h   | 12 ++++++-
 2 files changed, 44 insertions(+), 1 deletion(-)

diff --git a/dbms/src/Storages/StorageMaterializedView.cpp b/dbms/src/Storages/StorageMaterializedView.cpp
index ccdbcf204fc..c008eecb1c4 100644
--- a/dbms/src/Storages/StorageMaterializedView.cpp
+++ b/dbms/src/Storages/StorageMaterializedView.cpp
@@ -204,11 +204,44 @@ void StorageMaterializedView::drop()
     }
 }
 
+void StorageMaterializedView::checkStatementCanBeForwarded() const
+{
+    if (!has_inner_table)
+        throw Exception(
+            "MATERIALIZED VIEW targets existing table " + target_database_name + "." + target_table_name + ". "
+            + "Execute the statement directly on it.", ErrorCodes::INCORRECT_QUERY);
+}
+
 bool StorageMaterializedView::optimize(const ASTPtr & query, const ASTPtr & partition, bool final, bool deduplicate, const Context & context)
 {
+    checkStatementCanBeForwarded();
     return getTargetTable()->optimize(query, partition, final, deduplicate, context);
 }
 
+void StorageMaterializedView::dropPartition(const ASTPtr & query, const ASTPtr & partition, bool detach, const Context & context)
+{
+    checkStatementCanBeForwarded();
+    getTargetTable()->dropPartition(query, partition, detach, context);
+}
+
+void StorageMaterializedView::clearColumnInPartition(const ASTPtr & partition, const Field & column_name, const Context & context)
+{
+    checkStatementCanBeForwarded();
+    getTargetTable()->clearColumnInPartition(partition, column_name, context);
+}
+
+void StorageMaterializedView::attachPartition(const ASTPtr & partition, bool part, const Context & context)
+{
+    checkStatementCanBeForwarded();
+    getTargetTable()->attachPartition(partition, part, context);
+}
+
+void StorageMaterializedView::freezePartition(const ASTPtr & partition, const String & with_name, const Context & context)
+{
+    checkStatementCanBeForwarded();
+    getTargetTable()->freezePartition(partition, with_name, context);
+}
+
 void StorageMaterializedView::shutdown()
 {
     /// Make sure the dependency is removed after DETACH TABLE
diff --git a/dbms/src/Storages/StorageMaterializedView.h b/dbms/src/Storages/StorageMaterializedView.h
index 21105068f7a..7edffe28d76 100644
--- a/dbms/src/Storages/StorageMaterializedView.h
+++ b/dbms/src/Storages/StorageMaterializedView.h
@@ -18,7 +18,6 @@ public:
     std::string getName() const override { return "MaterializedView"; }
     std::string getTableName() const override { return table_name; }
     ASTPtr getInnerQuery() const { return inner_query->clone(); };
-    StoragePtr getTargetTable() const;
 
     NameAndTypePair getColumn(const String & column_name) const override;
     bool hasColumn(const String & column_name) const override;
@@ -31,7 +30,14 @@ public:
 
     BlockOutputStreamPtr write(const ASTPtr & query, const Settings & settings) override;
     void drop() override;
+
     bool optimize(const ASTPtr & query, const ASTPtr & partition, bool final, bool deduplicate, const Context & context) override;
+
+    void dropPartition(const ASTPtr & query, const ASTPtr & partition, bool detach, const Context & context) override;
+    void clearColumnInPartition(const ASTPtr & partition, const Field & column_name, const Context & context) override;
+    void attachPartition(const ASTPtr & partition, bool part, const Context & context) override;
+    void freezePartition(const ASTPtr & partition, const String & with_name, const Context & context) override;
+
     void shutdown() override;
     bool checkTableCanBeDropped() const override;
 
@@ -56,6 +62,10 @@ private:
     Context & global_context;
     bool has_inner_table = false;
 
+    StoragePtr getTargetTable() const;
+
+    void checkStatementCanBeForwarded() const;
+
 protected:
     StorageMaterializedView(
         const String & table_name_,

From 3effbc44117b20d1a3ac949929c07c6bf7ac79a0 Mon Sep 17 00:00:00 2001
From: Alexey Zatelepin <ztlpn@yandex-team.ru>
Date: Tue, 20 Mar 2018 16:43:25 +0300
Subject: [PATCH 158/244] add test [#CLICKHOUSE-3659]

---
 ...orward_alter_partition_statements.reference |  6 ++++++
 ...view_forward_alter_partition_statements.sql | 18 ++++++++++++++++++
 2 files changed, 24 insertions(+)
 create mode 100644 dbms/tests/queries/0_stateless/00610_materialized_view_forward_alter_partition_statements.reference
 create mode 100644 dbms/tests/queries/0_stateless/00610_materialized_view_forward_alter_partition_statements.sql

diff --git a/dbms/tests/queries/0_stateless/00610_materialized_view_forward_alter_partition_statements.reference b/dbms/tests/queries/0_stateless/00610_materialized_view_forward_alter_partition_statements.reference
new file mode 100644
index 00000000000..d989886d07a
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00610_materialized_view_forward_alter_partition_statements.reference
@@ -0,0 +1,6 @@
+-- Before DROP PARTITION --
+2018-01-01	2
+2018-01-01	3
+2018-02-01	4
+-- After DROP PARTITION --
+2018-02-01	4
diff --git a/dbms/tests/queries/0_stateless/00610_materialized_view_forward_alter_partition_statements.sql b/dbms/tests/queries/0_stateless/00610_materialized_view_forward_alter_partition_statements.sql
new file mode 100644
index 00000000000..5469318068e
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00610_materialized_view_forward_alter_partition_statements.sql
@@ -0,0 +1,18 @@
+DROP TABLE IF EXISTS test.tab;
+DROP TABLE IF EXISTS test.mv;
+
+CREATE TABLE test.tab(d Date, x UInt32) ENGINE MergeTree(d, x, 8192);
+CREATE MATERIALIZED VIEW test.mv(d Date, y UInt64) ENGINE MergeTree(d, y, 8192) AS SELECT d, x + 1 AS y FROM test.tab;
+
+INSERT INTO test.tab VALUES ('2018-01-01', 1), ('2018-01-01', 2), ('2018-02-01', 3);
+
+SELECT '-- Before DROP PARTITION --';
+SELECT * FROM test.mv ORDER BY y;
+
+ALTER TABLE test.mv DROP PARTITION 201801;
+
+SELECT '-- After DROP PARTITION --';
+SELECT * FROM test.mv ORDER BY y;
+
+DROP TABLE test.tab;
+DROP TABLE test.mv;

From a399c17f0196c7ae0326676f2a00b0d74cb1a8e4 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Tue, 20 Mar 2018 13:58:16 +0300
Subject: [PATCH 159/244] && - qualified IColumn::mutate

---
 dbms/src/Columns/FilterDescription.cpp         |  3 ++-
 dbms/src/Columns/IColumn.h                     |  4 ++--
 dbms/src/Common/COWPtr.h                       | 18 ++++++++++++++----
 dbms/src/Core/Block.cpp                        |  2 +-
 dbms/src/Core/Block.h                          |  2 +-
 dbms/src/DataStreams/ColumnGathererStream.cpp  |  5 ++---
 .../MergingSortedBlockInputStream.cpp          |  2 +-
 dbms/src/DataStreams/SquashingTransform.cpp    |  2 +-
 .../ComplexKeyHashedDictionary.cpp             |  2 +-
 .../Dictionaries/ExternalResultDescription.cpp |  2 +-
 dbms/src/Dictionaries/FlatDictionary.cpp       |  2 +-
 dbms/src/Dictionaries/HashedDictionary.cpp     |  2 +-
 dbms/src/Functions/FunctionsConditional.h      |  7 ++++---
 dbms/src/Functions/IFunction.cpp               |  2 +-
 dbms/src/Interpreters/NullableUtils.cpp        |  2 +-
 dbms/src/Storages/StorageBuffer.cpp            |  4 ++--
 16 files changed, 36 insertions(+), 25 deletions(-)

diff --git a/dbms/src/Columns/FilterDescription.cpp b/dbms/src/Columns/FilterDescription.cpp
index 1d29cd97435..3d3e3f5ffea 100644
--- a/dbms/src/Columns/FilterDescription.cpp
+++ b/dbms/src/Columns/FilterDescription.cpp
@@ -57,7 +57,8 @@ FilterDescription::FilterDescription(const IColumn & column)
 
     if (const ColumnNullable * nullable_column = typeid_cast<const ColumnNullable *>(&column))
     {
-        MutableColumnPtr mutable_holder = nullable_column->getNestedColumn().mutate();
+        ColumnPtr nested_column = nullable_column->getNestedColumnPtr();
+        MutableColumnPtr mutable_holder = (*std::move(nested_column)).mutate();
 
         ColumnUInt8 * concrete_column = typeid_cast<ColumnUInt8 *>(mutable_holder.get());
         if (!concrete_column)
diff --git a/dbms/src/Columns/IColumn.h b/dbms/src/Columns/IColumn.h
index 9fd98e6f701..3a733e4a025 100644
--- a/dbms/src/Columns/IColumn.h
+++ b/dbms/src/Columns/IColumn.h
@@ -247,10 +247,10 @@ public:
     virtual void forEachSubcolumn(ColumnCallback) {}
 
 
-    MutablePtr mutate() const
+    MutablePtr mutate() const &&
     {
         MutablePtr res = COWPtr<IColumn>::mutate();
-        res->forEachSubcolumn([](Ptr & subcolumn) { subcolumn = subcolumn->mutate(); });
+        res->forEachSubcolumn([](Ptr & subcolumn) { subcolumn = (*std::move(subcolumn)).mutate(); });
         return res;
     }
 
diff --git a/dbms/src/Common/COWPtr.h b/dbms/src/Common/COWPtr.h
index 9f7cf21ad63..39c912f8abc 100644
--- a/dbms/src/Common/COWPtr.h
+++ b/dbms/src/Common/COWPtr.h
@@ -80,12 +80,22 @@ private:
     Derived * derived() { return static_cast<Derived *>(this); }
     const Derived * derived() const { return static_cast<const Derived *>(this); }
 
+    template <typename T>
+    class IntrusivePtr : public boost::intrusive_ptr<T>
+    {
+    public:
+        using boost::intrusive_ptr<T>::intrusive_ptr;
+
+        T & operator*() const & { return boost::intrusive_ptr<T>::operator*(); }
+        T && operator*() const && { return const_cast<typename std::remove_const<T>::type &&>(*boost::intrusive_ptr<T>::get()); }
+    };
+
 protected:
     template <typename T>
-    class mutable_ptr : public boost::intrusive_ptr<T>
+    class mutable_ptr : public IntrusivePtr<T>
     {
     private:
-        using Base = boost::intrusive_ptr<T>;
+        using Base = IntrusivePtr<T>;
 
         template <typename> friend class COWPtr;
         template <typename, typename> friend class COWPtrHelper;
@@ -114,10 +124,10 @@ public:
 
 protected:
     template <typename T>
-    class immutable_ptr : public boost::intrusive_ptr<const T>
+    class immutable_ptr : public IntrusivePtr<const T>
     {
     private:
-        using Base = boost::intrusive_ptr<const T>;
+        using Base = IntrusivePtr<const T>;
 
         template <typename> friend class COWPtr;
         template <typename, typename> friend class COWPtrHelper;
diff --git a/dbms/src/Core/Block.cpp b/dbms/src/Core/Block.cpp
index c969e3256da..1cae4ba252e 100644
--- a/dbms/src/Core/Block.cpp
+++ b/dbms/src/Core/Block.cpp
@@ -312,7 +312,7 @@ MutableColumns Block::mutateColumns() const
     size_t num_columns = data.size();
     MutableColumns columns(num_columns);
     for (size_t i = 0; i < num_columns; ++i)
-        columns[i] = data[i].column ? data[i].column->mutate() : data[i].type->createColumn();
+        columns[i] = data[i].column ? (*std::move(data[i].column)).mutate() : data[i].type->createColumn();
     return columns;
 }
 
diff --git a/dbms/src/Core/Block.h b/dbms/src/Core/Block.h
index 7c836e49532..ea4a4c00cef 100644
--- a/dbms/src/Core/Block.h
+++ b/dbms/src/Core/Block.h
@@ -109,7 +109,7 @@ public:
     /** Get empty columns with the same types as in block. */
     MutableColumns cloneEmptyColumns() const;
 
-    /** Get columns from block for mutation. */
+    /** Get columns from block for mutation. Columns in block will be nullptr. */
     MutableColumns mutateColumns() const;
 
     /** Replace columns in a block */
diff --git a/dbms/src/DataStreams/ColumnGathererStream.cpp b/dbms/src/DataStreams/ColumnGathererStream.cpp
index ba4107f7e8b..baa19ae9130 100644
--- a/dbms/src/DataStreams/ColumnGathererStream.cpp
+++ b/dbms/src/DataStreams/ColumnGathererStream.cpp
@@ -76,10 +76,9 @@ Block ColumnGathererStream::readImpl()
         return Block();
 
     output_block = Block{column.cloneEmpty()};
-    MutableColumnPtr output_column = output_block.getByPosition(0).column->mutate();
+    MutableColumnPtr output_column = (*std::move(output_block.getByPosition(0).column)).mutate();
     output_column->gather(*this);
-    if (!output_column->empty())
-        output_block.getByPosition(0).column = std::move(output_column);
+    output_block.getByPosition(0).column = std::move(output_column);
 
     return output_block;
 }
diff --git a/dbms/src/DataStreams/MergingSortedBlockInputStream.cpp b/dbms/src/DataStreams/MergingSortedBlockInputStream.cpp
index f93197ca123..b60cc072b47 100644
--- a/dbms/src/DataStreams/MergingSortedBlockInputStream.cpp
+++ b/dbms/src/DataStreams/MergingSortedBlockInputStream.cpp
@@ -221,7 +221,7 @@ void MergingSortedBlockInputStream::merge(MutableColumns & merged_columns, std::
                     throw Exception("Logical error in MergingSortedBlockInputStream", ErrorCodes::LOGICAL_ERROR);
 
                 for (size_t i = 0; i < num_columns; ++i)
-                    merged_columns[i] = source_blocks[source_num]->getByPosition(i).column->mutate();
+                    merged_columns[i] = (*std::move(source_blocks[source_num]->getByPosition(i).column)).mutate();
 
     //            std::cerr << "copied columns\n";
 
diff --git a/dbms/src/DataStreams/SquashingTransform.cpp b/dbms/src/DataStreams/SquashingTransform.cpp
index 9e8f590f171..d31ff6f8063 100644
--- a/dbms/src/DataStreams/SquashingTransform.cpp
+++ b/dbms/src/DataStreams/SquashingTransform.cpp
@@ -62,7 +62,7 @@ void SquashingTransform::append(Block && block)
 
     for (size_t i = 0; i < columns; ++i)
     {
-        MutableColumnPtr mutable_column = accumulated_block.getByPosition(i).column->mutate();
+        MutableColumnPtr mutable_column = (*std::move(accumulated_block.getByPosition(i).column)).mutate();
         mutable_column->insertRangeFrom(*block.getByPosition(i).column, 0, rows);
         accumulated_block.getByPosition(i).column = std::move(mutable_column);
     }
diff --git a/dbms/src/Dictionaries/ComplexKeyHashedDictionary.cpp b/dbms/src/Dictionaries/ComplexKeyHashedDictionary.cpp
index 0f8c75ebd89..eaaeeab4e0a 100644
--- a/dbms/src/Dictionaries/ComplexKeyHashedDictionary.cpp
+++ b/dbms/src/Dictionaries/ComplexKeyHashedDictionary.cpp
@@ -289,7 +289,7 @@ void ComplexKeyHashedDictionary::updateData()
             for (const auto attribute_idx : ext::range(0, keys_size + attributes_size))
             {
                 const IColumn & update_column = *block.getByPosition(attribute_idx).column.get();
-                MutableColumnPtr saved_column = saved_block->getByPosition(attribute_idx).column->mutate();
+                MutableColumnPtr saved_column = saved_block->getByPosition(attribute_idx).column->assumeMutable();
                 saved_column->insertRangeFrom(update_column, 0, update_column.size());
             }
         }
diff --git a/dbms/src/Dictionaries/ExternalResultDescription.cpp b/dbms/src/Dictionaries/ExternalResultDescription.cpp
index 4c05fc32555..5eca329e5f2 100644
--- a/dbms/src/Dictionaries/ExternalResultDescription.cpp
+++ b/dbms/src/Dictionaries/ExternalResultDescription.cpp
@@ -66,7 +66,7 @@ void ExternalResultDescription::init(const Block & sample_block_)
         /// If default value for column was not provided, use default from data type.
         if (sample_columns.back()->empty())
         {
-            MutableColumnPtr mutable_column = sample_columns.back()->mutate();
+            MutableColumnPtr mutable_column = (*std::move(sample_columns.back())).mutate();
             column.type->insertDefaultInto(*mutable_column);
             sample_columns.back() = std::move(mutable_column);
         }
diff --git a/dbms/src/Dictionaries/FlatDictionary.cpp b/dbms/src/Dictionaries/FlatDictionary.cpp
index 0fc477725be..e1341907649 100644
--- a/dbms/src/Dictionaries/FlatDictionary.cpp
+++ b/dbms/src/Dictionaries/FlatDictionary.cpp
@@ -316,7 +316,7 @@ void FlatDictionary::updateData()
             for (const auto attribute_idx : ext::range(0, attributes.size() + 1))
             {
                 const IColumn & update_column = *block.getByPosition(attribute_idx).column.get();
-                MutableColumnPtr saved_column = saved_block->getByPosition(attribute_idx).column->mutate();
+                MutableColumnPtr saved_column = saved_block->getByPosition(attribute_idx).column->assumeMutable();
                 saved_column->insertRangeFrom(update_column, 0, update_column.size());
             }
         }
diff --git a/dbms/src/Dictionaries/HashedDictionary.cpp b/dbms/src/Dictionaries/HashedDictionary.cpp
index c0c7f1a8a17..09880cdbeb5 100644
--- a/dbms/src/Dictionaries/HashedDictionary.cpp
+++ b/dbms/src/Dictionaries/HashedDictionary.cpp
@@ -310,7 +310,7 @@ void HashedDictionary::updateData()
             for (const auto attribute_idx : ext::range(0, attributes.size() + 1))
             {
                 const IColumn & update_column = *block.getByPosition(attribute_idx).column.get();
-                MutableColumnPtr saved_column = saved_block->getByPosition(attribute_idx).column->mutate();
+                MutableColumnPtr saved_column = saved_block->getByPosition(attribute_idx).column->assumeMutable();
                 saved_column->insertRangeFrom(update_column, 0, update_column.size());
             }
         }
diff --git a/dbms/src/Functions/FunctionsConditional.h b/dbms/src/Functions/FunctionsConditional.h
index 841295f0643..27567237928 100644
--- a/dbms/src/Functions/FunctionsConditional.h
+++ b/dbms/src/Functions/FunctionsConditional.h
@@ -592,7 +592,7 @@ private:
             const ColumnPtr & result_column = temporary_block.getByPosition(3).column;
             if (result_column->isColumnNullable())
             {
-                MutableColumnPtr mutable_result_column = result_column->mutate();
+                MutableColumnPtr mutable_result_column = (*std::move(result_column)).mutate();
                 static_cast<ColumnNullable &>(*mutable_result_column).applyNullMap(static_cast<const ColumnNullable &>(*arg_cond.column));
                 block.getByPosition(result).column = std::move(mutable_result_column);
                 return true;
@@ -744,7 +744,7 @@ private:
             {
                 if (arg_else.column->isColumnNullable())
                 {
-                    auto result_column = arg_else.column->mutate();
+                    auto result_column = (*std::move(arg_else.column)).mutate();
                     static_cast<ColumnNullable &>(*result_column).applyNullMap(static_cast<const ColumnUInt8 &>(*arg_cond.column));
                     block.getByPosition(result).column = std::move(result_column);
                 }
@@ -785,7 +785,8 @@ private:
 
                 if (arg_then.column->isColumnNullable())
                 {
-                    auto result_column = arg_then.column->mutate();
+                    auto arg_then_column = arg_then.column;
+                    auto result_column = (*std::move(arg_then_column)).mutate();
                     static_cast<ColumnNullable &>(*result_column).applyNegatedNullMap(static_cast<const ColumnUInt8 &>(*arg_cond.column));
                     block.getByPosition(result).column = std::move(result_column);
                 }
diff --git a/dbms/src/Functions/IFunction.cpp b/dbms/src/Functions/IFunction.cpp
index ba05bc34e9d..12e8dfabbd8 100644
--- a/dbms/src/Functions/IFunction.cpp
+++ b/dbms/src/Functions/IFunction.cpp
@@ -63,7 +63,7 @@ ColumnPtr wrapInNullable(const ColumnPtr & src, Block & block, const ColumnNumbe
             }
             else
             {
-                MutableColumnPtr mutable_result_null_map_column = result_null_map_column->mutate();
+                MutableColumnPtr mutable_result_null_map_column = (*std::move(result_null_map_column)).mutate();
 
                 NullMap & result_null_map = static_cast<ColumnUInt8 &>(*mutable_result_null_map_column).getData();
                 const NullMap & src_null_map = static_cast<const ColumnUInt8 &>(*null_map_column).getData();
diff --git a/dbms/src/Interpreters/NullableUtils.cpp b/dbms/src/Interpreters/NullableUtils.cpp
index 972c548e67e..dcb0657aa9d 100644
--- a/dbms/src/Interpreters/NullableUtils.cpp
+++ b/dbms/src/Interpreters/NullableUtils.cpp
@@ -31,7 +31,7 @@ void extractNestedColumnsAndNullMap(ColumnRawPtrs & key_columns, ColumnPtr & nul
                 }
                 else
                 {
-                    MutableColumnPtr mutable_null_map_holder = null_map_holder->mutate();
+                    MutableColumnPtr mutable_null_map_holder = (*std::move(null_map_holder)).mutate();
 
                     PaddedPODArray<UInt8> & mutable_null_map = static_cast<ColumnUInt8 &>(*mutable_null_map_holder).getData();
                     const PaddedPODArray<UInt8> & other_null_map = column_nullable.getNullMapData();
diff --git a/dbms/src/Storages/StorageBuffer.cpp b/dbms/src/Storages/StorageBuffer.cpp
index 5c59ffa7c40..8688d191b9a 100644
--- a/dbms/src/Storages/StorageBuffer.cpp
+++ b/dbms/src/Storages/StorageBuffer.cpp
@@ -164,7 +164,7 @@ static void appendBlock(const Block & from, Block & to)
         for (size_t column_no = 0, columns = to.columns(); column_no < columns; ++column_no)
         {
             const IColumn & col_from = *from.getByPosition(column_no).column.get();
-            MutableColumnPtr col_to = to.getByPosition(column_no).column->mutate();
+            MutableColumnPtr col_to = (*std::move(to.getByPosition(column_no).column)).mutate();
 
             col_to->insertRangeFrom(col_from, 0, rows);
 
@@ -183,7 +183,7 @@ static void appendBlock(const Block & from, Block & to)
             {
                 ColumnPtr & col_to = to.getByPosition(column_no).column;
                 if (col_to->size() != old_rows)
-                    col_to = col_to->mutate()->cut(0, old_rows);
+                    col_to = (*std::move(col_to)).mutate()->cut(0, old_rows);
             }
         }
         catch (...)

From 5209615cf6d13a9aa90c12bba3513b9bc0846e5a Mon Sep 17 00:00:00 2001
From: Vitaliy Lyudvichenko <ludv1x@yandex.ru>
Date: Tue, 20 Mar 2018 19:17:46 +0300
Subject: [PATCH 160/244] Update CHANGELOG.md

---
 CHANGELOG.md | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index dffde42b520..fdb7f5303b1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,29 @@
 # ClickHouse 1.1.54370 Release Candidate, 2018-03-16
 
+## New features:
+
+* Added the `system.macros` table and auto updating of macros when the config file is changed.
+* Added the `SYSTEM RELOAD CONFIG` query.
+* Added the `maxIntersections(left_col, right_col)` aggregate function, which returns the maximum number of simultaneously intersecting intervals `[left; right]`. The `maxIntersectionsPosition(left, right)` function returns the beginning of the "maximum" interval. ([Michael Furmur](https://github.com/yandex/ClickHouse/pull/2012)).
+
+## Improvements:
+
+* When inserting data in a `Replicated` table, fewer requests are made to `ZooKeeper` (and most of the user-level errors have disappeared from the `ZooKeeper` log).
+* Added the ability to create aliases for sets. Example: `WITH (1, 2, 3) AS set SELECT number IN set FROM system.numbers LIMIT 10`.
+
+## Bug fixes:
+
+* Fixed the `Illegal PREWHERE` error when reading from `Merge` tables over `Distributed` tables.
+* Added fixes that allow you to run `clickhouse-server` in IPv4-only Docker containers.
+* Fixed a race condition when reading from system `system.parts_columns` tables.
+* Removed double buffering during a synchronous insert to a `Distributed` table, which could have caused the connection to timeout.
+* Fixed a bug that caused excessively long waits for an unavailable replica before beginning a `SELECT` query.
+* Fixed incorrect dates in the `system.parts` table.
+* Fixed a bug that made it impossible to insert data in a `Replicated` table if `chroot` was non-empty in the configuration of the `ZooKeeper` cluster.
+* Fixed the vertical merging algorithm for an empty `ORDER BY` table.
+* Restored the ability to use dictionaries in queries to remote tables, even if these dictionaries are not present on the requestor server. This functionality was lost in release 1.1.54362.
+* Restored the behavior for queries like `SELECT * FROM remote('server2', default.table) WHERE col IN (SELECT col2 FROM default.table)` when the right side argument of the `IN` should use a remote `default.table` instead of a local one. This behavior was broken in version 1.1.54358.
+* Removed extraneous error-level logging of `Not found column ... in block`.
 
 # ClickHouse release 1.1.54356, 2018-03-06
 

From 1f4db58d533cc4708a9a9a6e965e9d9c27d86166 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Tue, 20 Mar 2018 20:01:13 +0300
Subject: [PATCH 161/244] fix test

---
 dbms/src/DataStreams/ColumnGathererStream.cpp | 5 +++--
 dbms/src/Functions/FunctionsConditional.h     | 3 ++-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/dbms/src/DataStreams/ColumnGathererStream.cpp b/dbms/src/DataStreams/ColumnGathererStream.cpp
index baa19ae9130..5ce00ea8417 100644
--- a/dbms/src/DataStreams/ColumnGathererStream.cpp
+++ b/dbms/src/DataStreams/ColumnGathererStream.cpp
@@ -76,9 +76,10 @@ Block ColumnGathererStream::readImpl()
         return Block();
 
     output_block = Block{column.cloneEmpty()};
-    MutableColumnPtr output_column = (*std::move(output_block.getByPosition(0).column)).mutate();
+    MutableColumnPtr output_column = output_block.getByPosition(0).column->assumeMutable();
     output_column->gather(*this);
-    output_block.getByPosition(0).column = std::move(output_column);
+    if (!output_column->empty())
+        output_block.getByPosition(0).column = std::move(output_column);
 
     return output_block;
 }
diff --git a/dbms/src/Functions/FunctionsConditional.h b/dbms/src/Functions/FunctionsConditional.h
index 27567237928..7c88bca6264 100644
--- a/dbms/src/Functions/FunctionsConditional.h
+++ b/dbms/src/Functions/FunctionsConditional.h
@@ -744,7 +744,8 @@ private:
             {
                 if (arg_else.column->isColumnNullable())
                 {
-                    auto result_column = (*std::move(arg_else.column)).mutate();
+                    auto arg_else_column = arg_else.column;
+                    auto result_column = (*std::move(arg_else_column)).mutate();
                     static_cast<ColumnNullable &>(*result_column).applyNullMap(static_cast<const ColumnUInt8 &>(*arg_cond.column));
                     block.getByPosition(result).column = std::move(result_column);
                 }

From f6f505ea339961cee51c4efcddfbc115bda9aa8b Mon Sep 17 00:00:00 2001
From: proller <proller@users.noreply.github.com>
Date: Tue, 20 Mar 2018 20:24:39 +0300
Subject: [PATCH 162/244] better test in pbuilder (#2080)

* better test in pbuilder
---
 dbms/tests/CMakeLists.txt            |  8 +----
 dbms/tests/clickhouse-test-server    |  2 +-
 debian/pbuilder-hooks/B90test-server | 54 +++++++++++++++++-----------
 3 files changed, 35 insertions(+), 29 deletions(-)

diff --git a/dbms/tests/CMakeLists.txt b/dbms/tests/CMakeLists.txt
index be51bcb8d69..128ffee582f 100644
--- a/dbms/tests/CMakeLists.txt
+++ b/dbms/tests/CMakeLists.txt
@@ -25,12 +25,6 @@ add_subdirectory (external_dictionaries)
 if (ENABLE_TESTS)
     configure_file(${CMAKE_CURRENT_SOURCE_DIR}/CTestCustom.cmake ${CMAKE_BINARY_DIR})
 
-    if ($ENV{TEST_OPT})
-        set(TEST_OPT $ENV{TEST_OPT})
-    else ()
-        set(TEST_OPT "--skip compile")
-    endif ()
-
     # maybe add --no-long ?
-    add_test(NAME with_server COMMAND bash -c "env BUILD_DIR=${ClickHouse_BINARY_DIR} TEST_OPT='${TEST_OPT}' ${CMAKE_CURRENT_SOURCE_DIR}/clickhouse-test-server")
+    add_test(NAME with_server COMMAND bash -c "env BUILD_DIR=${ClickHouse_BINARY_DIR} TEST_OPT0='--skip compile' ${CMAKE_CURRENT_SOURCE_DIR}/clickhouse-test-server")
 endif ()
diff --git a/dbms/tests/clickhouse-test-server b/dbms/tests/clickhouse-test-server
index 778dc29b041..38b6515a746 100755
--- a/dbms/tests/clickhouse-test-server
+++ b/dbms/tests/clickhouse-test-server
@@ -64,5 +64,5 @@ if [ -n "$*" ]; then
 else
     ${BIN_DIR}clickhouse-client --config ${CONFIG_CLIENT} -q 'SELECT * from system.build_options;'
     PATH=$PATH:$BIN_DIR \
-      ${TEST_DIR}clickhouse-test --binary ${BIN_DIR}clickhouse --configclient ${CONFIG_CLIENT} --configserver ${CLICKHOUSE_CONFIG} --tmp $DATA_DIR/tmp --queries ${QUERIES_DIR} $TEST_OPT
+      ${TEST_DIR}clickhouse-test --binary ${BIN_DIR}clickhouse --configclient ${CONFIG_CLIENT} --configserver ${CLICKHOUSE_CONFIG} --tmp $DATA_DIR/tmp --queries ${QUERIES_DIR} $TEST_OPT0 $TEST_OPT
 fi
diff --git a/debian/pbuilder-hooks/B90test-server b/debian/pbuilder-hooks/B90test-server
index da929746937..eac7b5a6374 100755
--- a/debian/pbuilder-hooks/B90test-server
+++ b/debian/pbuilder-hooks/B90test-server
@@ -18,29 +18,41 @@ dpkg-reconfigure -f noninteractive tzdata
 
 mkdir -p /etc/clickhouse-server/config.d /etc/clickhouse-client/config.d
 
-TEST_SSL=${TEST_SSL=1}
-if  [ "${TEST_SSL}" ]; then
-    echo "<yandex><https_port>8443</https_port><tcp_ssl_port>9440</tcp_ssl_port></yandex>" > /etc/clickhouse-server/config.d/ssl.xml
-    echo "<yandex><openSSL><client><verificationMode>none</verificationMode><invalidCertificateHandler><name>AcceptCertificateHandler</name></invalidCertificateHandler></client></openSSL></yandex>" > /etc/clickhouse-client/config.d/ssl.xml
+TEST_CONNECT=${TEST_CONNECT=1}
+if [ "${TEST_CONNECT}" ]; then
+
+    TEST_SSL=${TEST_SSL=1}
+    if  [ "${TEST_SSL}" ]; then
+        echo "<yandex><https_port>8443</https_port><tcp_ssl_port>9440</tcp_ssl_port></yandex>" > /etc/clickhouse-server/config.d/ssl.xml
+        echo "<yandex><openSSL><client><verificationMode>none</verificationMode><invalidCertificateHandler><name>AcceptCertificateHandler</name></invalidCertificateHandler></client></openSSL></yandex>" > /etc/clickhouse-client/config.d/ssl.xml
+
+        openssl dhparam -out /etc/clickhouse-server/dhparam.pem 256
+        openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout /etc/clickhouse-server/server.key -out /etc/clickhouse-server/server.crt
+        CLIENT_ADD="--ssl"
+    fi
+
+    function finish {
+        service clickhouse-server stop
+        tail -n 100 /var/log/clickhouse-server/*
+        sleep 1
+        killall -9 clickhouse-server || true
+    }
+    trap finish EXIT SIGINT SIGQUIT SIGTERM
+
+    service clickhouse-server start
+    sleep 3
+
+    # TODO: remove me or make only on error:
+    tail -n100 /var/log/clickhouse-server/*
+
+    clickhouse-client -q "SELECT * from system.build_options;"
+    clickhouse-client ${CLIENT_ADD} -q "SELECT toDateTime(1);"
+
+    ( [ "${TEST_RUN}" ] && clickhouse-test --queries /usr/share/clickhouse-test/queries --tmp /tmp/clickhouse-test/ ${TEST_OPT} ) || ${TEST_TRUE:=true}
+
+    service clickhouse-server stop
 
-    openssl dhparam -out /etc/clickhouse-server/dhparam.pem 256
-    openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout /etc/clickhouse-server/server.key -out /etc/clickhouse-server/server.crt
 fi
 
-service clickhouse-server start
-sleep 3
-
-# TODO: remove me or make only on error:
-tail -n100 /var/log/clickhouse-server/*
-
-clickhouse-client -q "SELECT * from system.build_options;"
-clickhouse-client ${CLIENT_ADD} -q "SELECT toDateTime(1);"
-
-
-( [ "${TEST_RUN}" ] && clickhouse-test --queries /usr/share/clickhouse-test/queries --tmp /tmp/clickhouse-test/ ${TEST_OPT} ) || ${TEST_TRUE:=true}
-
-
-service clickhouse-server stop
-
 # Test debug symbols
 # gdb -ex quit --args /usr/bin/clickhouse-server

From 9c9e918b789bbeb922f9c66d3050656c492d3fdf Mon Sep 17 00:00:00 2001
From: proller <proller@github.com>
Date: Tue, 20 Mar 2018 22:18:56 +0300
Subject: [PATCH 163/244] Fix flapping test 00371_union_all

---
 dbms/tests/queries/0_stateless/00371_union_all.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dbms/tests/queries/0_stateless/00371_union_all.sql b/dbms/tests/queries/0_stateless/00371_union_all.sql
index 893e100e12b..0a738b6b20c 100644
--- a/dbms/tests/queries/0_stateless/00371_union_all.sql
+++ b/dbms/tests/queries/0_stateless/00371_union_all.sql
@@ -1,2 +1,2 @@
 select toUInt64(1) union all select countIf(n>0) from (select 2 as n);
-select [1,2,3] AS q UNION ALL select groupArray(arrayJoin([4,5,6])) AS q ORDER BY q;
+SELECT q FROM (select [1,2,3] AS q UNION ALL select groupArray(arrayJoin([4,5,6])) AS q ORDER BY q) ORDER BY q;

From 476d4b339c257d1d458237808e429420eefa0249 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Tue, 20 Mar 2018 17:17:09 +0300
Subject: [PATCH 164/244] Immutable construction for columns

---
 dbms/src/Columns/ColumnAggregateFunction.cpp  |  6 +-
 dbms/src/Columns/ColumnAggregateFunction.h    |  6 +-
 dbms/src/Columns/ColumnArray.cpp              | 56 ++++++++++---------
 dbms/src/Columns/ColumnArray.h                | 46 +++++++++------
 dbms/src/Columns/ColumnConst.cpp              |  8 +--
 dbms/src/Columns/ColumnConst.h                | 11 ++--
 dbms/src/Columns/ColumnFixedString.cpp        |  6 +-
 dbms/src/Columns/ColumnFixedString.h          |  6 +-
 dbms/src/Columns/ColumnFunction.cpp           |  8 +--
 dbms/src/Columns/ColumnFunction.h             |  8 +--
 dbms/src/Columns/ColumnNullable.cpp           | 14 ++---
 dbms/src/Columns/ColumnNullable.h             | 15 +++--
 dbms/src/Columns/ColumnString.cpp             |  6 +-
 dbms/src/Columns/ColumnString.h               |  6 +-
 dbms/src/Columns/ColumnTuple.cpp              | 33 ++++++++---
 dbms/src/Columns/ColumnTuple.h                | 10 ++--
 dbms/src/Columns/ColumnVector.cpp             |  6 +-
 dbms/src/Columns/ColumnVector.h               |  6 +-
 dbms/src/Columns/IColumn.h                    | 12 ++--
 dbms/src/Columns/IColumnDummy.h               |  6 +-
 dbms/src/Common/COWPtr.h                      | 42 +++++++++++++-
 .../MergingSortedBlockInputStream.cpp         |  2 +-
 .../SummingSortedBlockInputStream.cpp         |  6 +-
 dbms/src/DataTypes/DataTypeTuple.cpp          |  4 +-
 dbms/src/Functions/FunctionsHigherOrder.h     |  2 +-
 .../Functions/GatherUtils/createArraySink.cpp |  3 +-
 .../Storages/MergeTree/MergeTreeReader.cpp    |  3 +-
 27 files changed, 210 insertions(+), 127 deletions(-)

diff --git a/dbms/src/Columns/ColumnAggregateFunction.cpp b/dbms/src/Columns/ColumnAggregateFunction.cpp
index c901dac36f1..b989c007e56 100644
--- a/dbms/src/Columns/ColumnAggregateFunction.cpp
+++ b/dbms/src/Columns/ColumnAggregateFunction.cpp
@@ -113,7 +113,7 @@ void ColumnAggregateFunction::insertRangeFrom(const IColumn & from, size_t start
 }
 
 
-MutableColumnPtr ColumnAggregateFunction::filter(const Filter & filter, ssize_t result_size_hint) const
+ColumnPtr ColumnAggregateFunction::filter(const Filter & filter, ssize_t result_size_hint) const
 {
     size_t size = getData().size();
     if (size != filter.size())
@@ -140,7 +140,7 @@ MutableColumnPtr ColumnAggregateFunction::filter(const Filter & filter, ssize_t
 }
 
 
-MutableColumnPtr ColumnAggregateFunction::permute(const Permutation & perm, size_t limit) const
+ColumnPtr ColumnAggregateFunction::permute(const Permutation & perm, size_t limit) const
 {
     size_t size = getData().size();
 
@@ -325,7 +325,7 @@ void ColumnAggregateFunction::popBack(size_t n)
     data.resize_assume_reserved(new_size);
 }
 
-MutableColumnPtr ColumnAggregateFunction::replicate(const IColumn::Offsets & offsets) const
+ColumnPtr ColumnAggregateFunction::replicate(const IColumn::Offsets & offsets) const
 {
     size_t size = data.size();
     if (size != offsets.size())
diff --git a/dbms/src/Columns/ColumnAggregateFunction.h b/dbms/src/Columns/ColumnAggregateFunction.h
index a9ec5ed486c..852d76130d5 100644
--- a/dbms/src/Columns/ColumnAggregateFunction.h
+++ b/dbms/src/Columns/ColumnAggregateFunction.h
@@ -152,11 +152,11 @@ public:
 
     void popBack(size_t n) override;
 
-    MutableColumnPtr filter(const Filter & filter, ssize_t result_size_hint) const override;
+    ColumnPtr filter(const Filter & filter, ssize_t result_size_hint) const override;
 
-    MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
+    ColumnPtr permute(const Permutation & perm, size_t limit) const override;
 
-    MutableColumnPtr replicate(const Offsets & offsets) const override;
+    ColumnPtr replicate(const Offsets & offsets) const override;
 
     MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override;
 
diff --git a/dbms/src/Columns/ColumnArray.cpp b/dbms/src/Columns/ColumnArray.cpp
index 4674c57eef6..a943b46eb49 100644
--- a/dbms/src/Columns/ColumnArray.cpp
+++ b/dbms/src/Columns/ColumnArray.cpp
@@ -31,8 +31,8 @@ namespace ErrorCodes
 }
 
 
-ColumnArray::ColumnArray(const ColumnPtr & nested_column, const ColumnPtr & offsets_column)
-    : data(nested_column), offsets(offsets_column)
+ColumnArray::ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column)
+    : data(std::move(nested_column)), offsets(std::move(offsets_column))
 {
     if (!typeid_cast<const ColumnOffsets *>(offsets_column.get()))
         throw Exception("offsets_column must be a ColumnUInt64", ErrorCodes::ILLEGAL_COLUMN);
@@ -43,8 +43,8 @@ ColumnArray::ColumnArray(const ColumnPtr & nested_column, const ColumnPtr & offs
       */
 }
 
-ColumnArray::ColumnArray(const ColumnPtr & nested_column)
-    : data(nested_column)
+ColumnArray::ColumnArray(MutableColumnPtr && nested_column)
+    : data(std::move(nested_column))
 {
     if (!data->empty())
         throw Exception("Not empty data passed to ColumnArray, but no offsets passed", ErrorCodes::ILLEGAL_COLUMN);
@@ -317,7 +317,7 @@ bool ColumnArray::hasEqualOffsets(const ColumnArray & other) const
 }
 
 
-MutableColumnPtr ColumnArray::convertToFullColumnIfConst() const
+ColumnPtr ColumnArray::convertToFullColumnIfConst() const
 {
     ColumnPtr new_data;
 
@@ -391,7 +391,7 @@ void ColumnArray::insertRangeFrom(const IColumn & src, size_t start, size_t leng
 }
 
 
-MutableColumnPtr ColumnArray::filter(const Filter & filt, ssize_t result_size_hint) const
+ColumnPtr ColumnArray::filter(const Filter & filt, ssize_t result_size_hint) const
 {
     if (typeid_cast<const ColumnUInt8 *>(data.get()))      return filterNumber<UInt8>(filt, result_size_hint);
     if (typeid_cast<const ColumnUInt16 *>(data.get()))     return filterNumber<UInt16>(filt, result_size_hint);
@@ -410,7 +410,7 @@ MutableColumnPtr ColumnArray::filter(const Filter & filt, ssize_t result_size_hi
 }
 
 template <typename T>
-MutableColumnPtr ColumnArray::filterNumber(const Filter & filt, ssize_t result_size_hint) const
+ColumnPtr ColumnArray::filterNumber(const Filter & filt, ssize_t result_size_hint) const
 {
     if (getOffsets().size() == 0)
         return ColumnArray::create(data);
@@ -424,7 +424,7 @@ MutableColumnPtr ColumnArray::filterNumber(const Filter & filt, ssize_t result_s
     return std::move(res);
 }
 
-MutableColumnPtr ColumnArray::filterString(const Filter & filt, ssize_t result_size_hint) const
+ColumnPtr ColumnArray::filterString(const Filter & filt, ssize_t result_size_hint) const
 {
     size_t col_size = getOffsets().size();
     if (col_size != filt.size())
@@ -492,7 +492,7 @@ MutableColumnPtr ColumnArray::filterString(const Filter & filt, ssize_t result_s
     return std::move(res);
 }
 
-MutableColumnPtr ColumnArray::filterGeneric(const Filter & filt, ssize_t result_size_hint) const
+ColumnPtr ColumnArray::filterGeneric(const Filter & filt, ssize_t result_size_hint) const
 {
     size_t size = getOffsets().size();
     if (size != filt.size())
@@ -537,7 +537,7 @@ MutableColumnPtr ColumnArray::filterGeneric(const Filter & filt, ssize_t result_
     return std::move(res);
 }
 
-MutableColumnPtr ColumnArray::filterNullable(const Filter & filt, ssize_t result_size_hint) const
+ColumnPtr ColumnArray::filterNullable(const Filter & filt, ssize_t result_size_hint) const
 {
     if (getOffsets().size() == 0)
         return ColumnArray::create(data);
@@ -560,7 +560,7 @@ MutableColumnPtr ColumnArray::filterNullable(const Filter & filt, ssize_t result
         filtered_offsets);
 }
 
-MutableColumnPtr ColumnArray::filterTuple(const Filter & filt, ssize_t result_size_hint) const
+ColumnPtr ColumnArray::filterTuple(const Filter & filt, ssize_t result_size_hint) const
 {
     if (getOffsets().size() == 0)
         return ColumnArray::create(data);
@@ -576,7 +576,8 @@ MutableColumnPtr ColumnArray::filterTuple(const Filter & filt, ssize_t result_si
 
     Columns temporary_arrays(tuple_size);
     for (size_t i = 0; i < tuple_size; ++i)
-        temporary_arrays[i] = ColumnArray(tuple.getColumns()[i], getOffsetsPtr()).filter(filt, result_size_hint);
+        temporary_arrays[i] = ColumnArray(tuple.getColumns()[i]->assumeMutable(), getOffsetsPtr()->assumeMutable())
+                .filter(filt, result_size_hint);
 
     Columns tuple_columns(tuple_size);
     for (size_t i = 0; i < tuple_size; ++i)
@@ -588,7 +589,7 @@ MutableColumnPtr ColumnArray::filterTuple(const Filter & filt, ssize_t result_si
 }
 
 
-MutableColumnPtr ColumnArray::permute(const Permutation & perm, size_t limit) const
+ColumnPtr ColumnArray::permute(const Permutation & perm, size_t limit) const
 {
     size_t size = getOffsets().size();
 
@@ -652,7 +653,7 @@ void ColumnArray::getPermutation(bool reverse, size_t limit, int nan_direction_h
 }
 
 
-MutableColumnPtr ColumnArray::replicate(const Offsets & replicate_offsets) const
+ColumnPtr ColumnArray::replicate(const Offsets & replicate_offsets) const
 {
     if (typeid_cast<const ColumnUInt8 *>(data.get()))     return replicateNumber<UInt8>(replicate_offsets);
     if (typeid_cast<const ColumnUInt16 *>(data.get()))    return replicateNumber<UInt16>(replicate_offsets);
@@ -673,7 +674,7 @@ MutableColumnPtr ColumnArray::replicate(const Offsets & replicate_offsets) const
 
 
 template <typename T>
-MutableColumnPtr ColumnArray::replicateNumber(const Offsets & replicate_offsets) const
+ColumnPtr ColumnArray::replicateNumber(const Offsets & replicate_offsets) const
 {
     size_t col_size = size();
     if (col_size != replicate_offsets.size())
@@ -721,7 +722,7 @@ MutableColumnPtr ColumnArray::replicateNumber(const Offsets & replicate_offsets)
 }
 
 
-MutableColumnPtr ColumnArray::replicateString(const Offsets & replicate_offsets) const
+ColumnPtr ColumnArray::replicateString(const Offsets & replicate_offsets) const
 {
     size_t col_size = size();
     if (col_size != replicate_offsets.size())
@@ -796,7 +797,7 @@ MutableColumnPtr ColumnArray::replicateString(const Offsets & replicate_offsets)
 }
 
 
-MutableColumnPtr ColumnArray::replicateConst(const Offsets & replicate_offsets) const
+ColumnPtr ColumnArray::replicateConst(const Offsets & replicate_offsets) const
 {
     size_t col_size = size();
     if (col_size != replicate_offsets.size())
@@ -834,7 +835,7 @@ MutableColumnPtr ColumnArray::replicateConst(const Offsets & replicate_offsets)
 }
 
 
-MutableColumnPtr ColumnArray::replicateGeneric(const Offsets & replicate_offsets) const
+ColumnPtr ColumnArray::replicateGeneric(const Offsets & replicate_offsets) const
 {
     size_t col_size = size();
     if (col_size != replicate_offsets.size())
@@ -860,25 +861,27 @@ MutableColumnPtr ColumnArray::replicateGeneric(const Offsets & replicate_offsets
 }
 
 
-MutableColumnPtr ColumnArray::replicateNullable(const Offsets & replicate_offsets) const
+ColumnPtr ColumnArray::replicateNullable(const Offsets & replicate_offsets) const
 {
     const ColumnNullable & nullable = static_cast<const ColumnNullable &>(*data);
 
     /// Make temporary arrays for each components of Nullable. Then replicate them independently and collect back to result.
     /// NOTE Offsets are calculated twice and it is redundant.
 
-    auto array_of_nested = ColumnArray(nullable.getNestedColumnPtr(), getOffsetsPtr()).replicate(replicate_offsets);
-    auto array_of_null_map = ColumnArray(nullable.getNullMapColumnPtr(), getOffsetsPtr()).replicate(replicate_offsets);
+    auto array_of_nested = ColumnArray(nullable.getNestedColumnPtr()->assumeMutable(), getOffsetsPtr()->assumeMutable())
+            .replicate(replicate_offsets);
+    auto array_of_null_map = ColumnArray(nullable.getNullMapColumnPtr()->assumeMutable(), getOffsetsPtr()->assumeMutable())
+            .replicate(replicate_offsets);
 
     return ColumnArray::create(
         ColumnNullable::create(
-            static_cast<ColumnArray &>(*array_of_nested).getDataPtr(),
-            static_cast<ColumnArray &>(*array_of_null_map).getDataPtr()),
-        static_cast<ColumnArray &>(*array_of_nested).getOffsetsPtr());
+            static_cast<const ColumnArray &>(*array_of_nested).getDataPtr(),
+            static_cast<const ColumnArray &>(*array_of_null_map).getDataPtr()),
+        static_cast<const ColumnArray &>(*array_of_nested).getOffsetsPtr());
 }
 
 
-MutableColumnPtr ColumnArray::replicateTuple(const Offsets & replicate_offsets) const
+ColumnPtr ColumnArray::replicateTuple(const Offsets & replicate_offsets) const
 {
     const ColumnTuple & tuple = static_cast<const ColumnTuple &>(*data);
 
@@ -891,7 +894,8 @@ MutableColumnPtr ColumnArray::replicateTuple(const Offsets & replicate_offsets)
 
     Columns temporary_arrays(tuple_size);
     for (size_t i = 0; i < tuple_size; ++i)
-        temporary_arrays[i] = ColumnArray(tuple.getColumns()[i], getOffsetsPtr()).replicate(replicate_offsets);
+        temporary_arrays[i] = ColumnArray(tuple.getColumns()[i]->assumeMutable(), getOffsetsPtr()->assumeMutable())
+                .replicate(replicate_offsets);
 
     Columns tuple_columns(tuple_size);
     for (size_t i = 0; i < tuple_size; ++i)
diff --git a/dbms/src/Columns/ColumnArray.h b/dbms/src/Columns/ColumnArray.h
index d1f951eedb7..a2564b20f98 100644
--- a/dbms/src/Columns/ColumnArray.h
+++ b/dbms/src/Columns/ColumnArray.h
@@ -24,10 +24,22 @@ private:
     friend class COWPtrHelper<IColumn, ColumnArray>;
 
     /** Create an array column with specified values and offsets. */
-    ColumnArray(const ColumnPtr & nested_column, const ColumnPtr & offsets_column);
+    ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column);
 
     /** Create an empty column of arrays with the type of values as in the column `nested_column` */
-    ColumnArray(const ColumnPtr & nested_column);
+    ColumnArray(MutableColumnPtr && nested_column);
+
+    using Ptr = COWPtrHelper<IColumn, ColumnArray>::Ptr;
+
+    static Ptr createImmutable(const ColumnPtr & nested_column, const ColumnPtr & offsets_column)
+    {
+        return ColumnArray::create(nested_column->assumeMutable(), offsets_column->assumeMutable());
+    }
+
+    static Ptr createImmutable(const ColumnPtr & nested_column)
+    {
+        return ColumnArray::create(nested_column->assumeMutable());
+    }
 
     ColumnArray(const ColumnArray &) = default;
 
@@ -51,15 +63,15 @@ public:
     void insertFrom(const IColumn & src_, size_t n) override;
     void insertDefault() override;
     void popBack(size_t n) override;
-    MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
-    MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
+    ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
+    ColumnPtr permute(const Permutation & perm, size_t limit) const override;
     int compareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const override;
     void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override;
     void reserve(size_t n) override;
     size_t byteSize() const override;
     size_t allocatedBytes() const override;
-    MutableColumnPtr replicate(const Offsets & replicate_offsets) const override;
-    MutableColumnPtr convertToFullColumnIfConst() const override;
+    ColumnPtr replicate(const Offsets & replicate_offsets) const override;
+    ColumnPtr convertToFullColumnIfConst() const override;
     void getExtremes(Field & min, Field & max) const override;
 
     bool hasEqualOffsets(const ColumnArray & other) const;
@@ -110,33 +122,33 @@ private:
 
     /// Multiply values if the nested column is ColumnVector<T>.
     template <typename T>
-    MutableColumnPtr replicateNumber(const Offsets & replicate_offsets) const;
+    ColumnPtr replicateNumber(const Offsets & replicate_offsets) const;
 
     /// Multiply the values if the nested column is ColumnString. The code is too complicated.
-    MutableColumnPtr replicateString(const Offsets & replicate_offsets) const;
+    ColumnPtr replicateString(const Offsets & replicate_offsets) const;
 
     /** Non-constant arrays of constant values are quite rare.
       * Most functions can not work with them, and does not create such columns as a result.
       * An exception is the function `replicate`(see FunctionsMiscellaneous.h), which has service meaning for the implementation of lambda functions.
       * Only for its sake is the implementation of the `replicate` method for ColumnArray(ColumnConst).
       */
-    MutableColumnPtr replicateConst(const Offsets & replicate_offsets) const;
+    ColumnPtr replicateConst(const Offsets & replicate_offsets) const;
 
     /** The following is done by simply replicating of nested columns.
       */
-    MutableColumnPtr replicateTuple(const Offsets & replicate_offsets) const;
-    MutableColumnPtr replicateNullable(const Offsets & replicate_offsets) const;
-    MutableColumnPtr replicateGeneric(const Offsets & replicate_offsets) const;
+    ColumnPtr replicateTuple(const Offsets & replicate_offsets) const;
+    ColumnPtr replicateNullable(const Offsets & replicate_offsets) const;
+    ColumnPtr replicateGeneric(const Offsets & replicate_offsets) const;
 
 
     /// Specializations for the filter function.
     template <typename T>
-    MutableColumnPtr filterNumber(const Filter & filt, ssize_t result_size_hint) const;
+    ColumnPtr filterNumber(const Filter & filt, ssize_t result_size_hint) const;
 
-    MutableColumnPtr filterString(const Filter & filt, ssize_t result_size_hint) const;
-    MutableColumnPtr filterTuple(const Filter & filt, ssize_t result_size_hint) const;
-    MutableColumnPtr filterNullable(const Filter & filt, ssize_t result_size_hint) const;
-    MutableColumnPtr filterGeneric(const Filter & filt, ssize_t result_size_hint) const;
+    ColumnPtr filterString(const Filter & filt, ssize_t result_size_hint) const;
+    ColumnPtr filterTuple(const Filter & filt, ssize_t result_size_hint) const;
+    ColumnPtr filterNullable(const Filter & filt, ssize_t result_size_hint) const;
+    ColumnPtr filterGeneric(const Filter & filt, ssize_t result_size_hint) const;
 };
 
 
diff --git a/dbms/src/Columns/ColumnConst.cpp b/dbms/src/Columns/ColumnConst.cpp
index 53e43547108..325160e5fc5 100644
--- a/dbms/src/Columns/ColumnConst.cpp
+++ b/dbms/src/Columns/ColumnConst.cpp
@@ -25,12 +25,12 @@ ColumnConst::ColumnConst(const ColumnPtr & data_, size_t s)
             ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
 }
 
-MutableColumnPtr ColumnConst::convertToFullColumn() const
+ColumnPtr ColumnConst::convertToFullColumn() const
 {
     return data->replicate(Offsets(1, s));
 }
 
-MutableColumnPtr ColumnConst::filter(const Filter & filt, ssize_t /*result_size_hint*/) const
+ColumnPtr ColumnConst::filter(const Filter & filt, ssize_t /*result_size_hint*/) const
 {
     if (s != filt.size())
         throw Exception("Size of filter (" + toString(filt.size()) + ") doesn't match size of column (" + toString(s) + ")",
@@ -39,7 +39,7 @@ MutableColumnPtr ColumnConst::filter(const Filter & filt, ssize_t /*result_size_
     return ColumnConst::create(data, countBytesInFilter(filt));
 }
 
-MutableColumnPtr ColumnConst::replicate(const Offsets & offsets) const
+ColumnPtr ColumnConst::replicate(const Offsets & offsets) const
 {
     if (s != offsets.size())
         throw Exception("Size of offsets (" + toString(offsets.size()) + ") doesn't match size of column (" + toString(s) + ")",
@@ -49,7 +49,7 @@ MutableColumnPtr ColumnConst::replicate(const Offsets & offsets) const
     return ColumnConst::create(data, replicated_size);
 }
 
-MutableColumnPtr ColumnConst::permute(const Permutation & perm, size_t limit) const
+ColumnPtr ColumnConst::permute(const Permutation & perm, size_t limit) const
 {
     if (limit == 0)
         limit = s;
diff --git a/dbms/src/Columns/ColumnConst.h b/dbms/src/Columns/ColumnConst.h
index 0ff94d6ec4b..89f4c488940 100644
--- a/dbms/src/Columns/ColumnConst.h
+++ b/dbms/src/Columns/ColumnConst.h
@@ -29,9 +29,9 @@ private:
     ColumnConst(const ColumnConst & src) = default;
 
 public:
-    MutableColumnPtr convertToFullColumn() const;
+    ColumnPtr convertToFullColumn() const;
 
-    MutableColumnPtr convertToFullColumnIfConst() const override
+    ColumnPtr convertToFullColumnIfConst() const override
     {
         return convertToFullColumn();
     }
@@ -133,6 +133,7 @@ public:
 
     const char * deserializeAndInsertFromArena(const char * pos) override
     {
+        data = (*std::move(data)).mutate();
         auto & mutable_data = data->assumeMutableRef();
         auto res = mutable_data.deserializeAndInsertFromArena(pos);
         mutable_data.popBack(1);
@@ -145,9 +146,9 @@ public:
         data->updateHashWithValue(0, hash);
     }
 
-    MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
-    MutableColumnPtr replicate(const Offsets & offsets) const override;
-    MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
+    ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
+    ColumnPtr replicate(const Offsets & offsets) const override;
+    ColumnPtr permute(const Permutation & perm, size_t limit) const override;
     void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override;
 
     size_t byteSize() const override
diff --git a/dbms/src/Columns/ColumnFixedString.cpp b/dbms/src/Columns/ColumnFixedString.cpp
index 61e2fbdd7d6..2da3c28e50a 100644
--- a/dbms/src/Columns/ColumnFixedString.cpp
+++ b/dbms/src/Columns/ColumnFixedString.cpp
@@ -153,7 +153,7 @@ void ColumnFixedString::insertRangeFrom(const IColumn & src, size_t start, size_
     memcpy(&chars[old_size], &src_concrete.chars[start * n], length * n);
 }
 
-MutableColumnPtr ColumnFixedString::filter(const IColumn::Filter & filt, ssize_t result_size_hint) const
+ColumnPtr ColumnFixedString::filter(const IColumn::Filter & filt, ssize_t result_size_hint) const
 {
     size_t col_size = size();
     if (col_size != filt.size())
@@ -230,7 +230,7 @@ MutableColumnPtr ColumnFixedString::filter(const IColumn::Filter & filt, ssize_t
     return std::move(res);
 }
 
-MutableColumnPtr ColumnFixedString::permute(const Permutation & perm, size_t limit) const
+ColumnPtr ColumnFixedString::permute(const Permutation & perm, size_t limit) const
 {
     size_t col_size = size();
 
@@ -258,7 +258,7 @@ MutableColumnPtr ColumnFixedString::permute(const Permutation & perm, size_t lim
     return std::move(res);
 }
 
-MutableColumnPtr ColumnFixedString::replicate(const Offsets & offsets) const
+ColumnPtr ColumnFixedString::replicate(const Offsets & offsets) const
 {
     size_t col_size = size();
     if (col_size != offsets.size())
diff --git a/dbms/src/Columns/ColumnFixedString.h b/dbms/src/Columns/ColumnFixedString.h
index e5ed788aaf0..cd465a1814d 100644
--- a/dbms/src/Columns/ColumnFixedString.h
+++ b/dbms/src/Columns/ColumnFixedString.h
@@ -104,11 +104,11 @@ public:
 
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
 
-    MutableColumnPtr filter(const IColumn::Filter & filt, ssize_t result_size_hint) const override;
+    ColumnPtr filter(const IColumn::Filter & filt, ssize_t result_size_hint) const override;
 
-    MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
+    ColumnPtr permute(const Permutation & perm, size_t limit) const override;
 
-    MutableColumnPtr replicate(const Offsets & offsets) const override;
+    ColumnPtr replicate(const Offsets & offsets) const override;
 
     MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override
     {
diff --git a/dbms/src/Columns/ColumnFunction.cpp b/dbms/src/Columns/ColumnFunction.cpp
index 498bcbc26b3..4675e7915f7 100644
--- a/dbms/src/Columns/ColumnFunction.cpp
+++ b/dbms/src/Columns/ColumnFunction.cpp
@@ -26,7 +26,7 @@ MutableColumnPtr ColumnFunction::cloneResized(size_t size) const
     return ColumnFunction::create(size, function, capture);
 }
 
-MutableColumnPtr ColumnFunction::replicate(const Offsets & offsets) const
+ColumnPtr ColumnFunction::replicate(const Offsets & offsets) const
 {
     if (size_ != offsets.size())
         throw Exception("Size of offsets (" + toString(offsets.size()) + ") doesn't match size of column ("
@@ -40,7 +40,7 @@ MutableColumnPtr ColumnFunction::replicate(const Offsets & offsets) const
     return ColumnFunction::create(replicated_size, function, capture);
 }
 
-MutableColumnPtr ColumnFunction::cut(size_t start, size_t length) const
+ColumnPtr ColumnFunction::cut(size_t start, size_t length) const
 {
     ColumnsWithTypeAndName capture = captured_columns;
     for (auto & column : capture)
@@ -49,7 +49,7 @@ MutableColumnPtr ColumnFunction::cut(size_t start, size_t length) const
     return ColumnFunction::create(length, function, capture);
 }
 
-MutableColumnPtr ColumnFunction::filter(const Filter & filt, ssize_t result_size_hint) const
+ColumnPtr ColumnFunction::filter(const Filter & filt, ssize_t result_size_hint) const
 {
     if (size_ != filt.size())
         throw Exception("Size of filter (" + toString(filt.size()) + ") doesn't match size of column ("
@@ -68,7 +68,7 @@ MutableColumnPtr ColumnFunction::filter(const Filter & filt, ssize_t result_size
     return ColumnFunction::create(filtered_size, function, capture);
 }
 
-MutableColumnPtr ColumnFunction::permute(const Permutation & perm, size_t limit) const
+ColumnPtr ColumnFunction::permute(const Permutation & perm, size_t limit) const
 {
     if (limit == 0)
         limit = size_;
diff --git a/dbms/src/Columns/ColumnFunction.h b/dbms/src/Columns/ColumnFunction.h
index 087086302aa..d10459175cc 100644
--- a/dbms/src/Columns/ColumnFunction.h
+++ b/dbms/src/Columns/ColumnFunction.h
@@ -29,10 +29,10 @@ public:
 
     size_t size() const override { return size_; }
 
-    MutableColumnPtr cut(size_t start, size_t length) const override;
-    MutableColumnPtr replicate(const Offsets & offsets) const override;
-    MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
-    MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
+    ColumnPtr cut(size_t start, size_t length) const override;
+    ColumnPtr replicate(const Offsets & offsets) const override;
+    ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
+    ColumnPtr permute(const Permutation & perm, size_t limit) const override;
     void insertDefault() override;
     void popBack(size_t n) override;
     std::vector<MutableColumnPtr> scatter(IColumn::ColumnIndex num_columns,
diff --git a/dbms/src/Columns/ColumnNullable.cpp b/dbms/src/Columns/ColumnNullable.cpp
index 59f225c962f..557a57b3c6d 100644
--- a/dbms/src/Columns/ColumnNullable.cpp
+++ b/dbms/src/Columns/ColumnNullable.cpp
@@ -18,8 +18,8 @@ namespace ErrorCodes
 }
 
 
-ColumnNullable::ColumnNullable(const ColumnPtr & nested_column_, const ColumnPtr & null_map_)
-    : nested_column{nested_column_}, null_map{null_map_}
+ColumnNullable::ColumnNullable(MutableColumnPtr && nested_column_, MutableColumnPtr && null_map_)
+    : nested_column(std::move(nested_column_)), null_map(std::move(null_map_))
 {
     /// ColumnNullable cannot have constant nested column. But constant argument could be passed. Materialize it.
     if (ColumnPtr nested_column_materialized = getNestedColumn().convertToFullColumnIfConst())
@@ -44,7 +44,7 @@ void ColumnNullable::updateHashWithValue(size_t n, SipHash & hash) const
 
 MutableColumnPtr ColumnNullable::cloneResized(size_t new_size) const
 {
-    ColumnPtr new_nested_col = getNestedColumn().cloneResized(new_size);
+    MutableColumnPtr new_nested_col = getNestedColumn().cloneResized(new_size);
     auto new_null_map = ColumnUInt8::create();
 
     if (new_size > 0)
@@ -59,7 +59,7 @@ MutableColumnPtr ColumnNullable::cloneResized(size_t new_size) const
             memset(&new_null_map->getData()[count], 1, new_size - count);
     }
 
-    return ColumnNullable::create(new_nested_col, std::move(new_null_map));
+    return ColumnNullable::create(std::move(new_nested_col), std::move(new_null_map));
 }
 
 
@@ -152,14 +152,14 @@ void ColumnNullable::popBack(size_t n)
     getNullMapColumn().popBack(n);
 }
 
-MutableColumnPtr ColumnNullable::filter(const Filter & filt, ssize_t result_size_hint) const
+ColumnPtr ColumnNullable::filter(const Filter & filt, ssize_t result_size_hint) const
 {
     ColumnPtr filtered_data = getNestedColumn().filter(filt, result_size_hint);
     ColumnPtr filtered_null_map = getNullMapColumn().filter(filt, result_size_hint);
     return ColumnNullable::create(filtered_data, filtered_null_map);
 }
 
-MutableColumnPtr ColumnNullable::permute(const Permutation & perm, size_t limit) const
+ColumnPtr ColumnNullable::permute(const Permutation & perm, size_t limit) const
 {
     ColumnPtr permuted_data = getNestedColumn().permute(perm, limit);
     ColumnPtr permuted_null_map = getNullMapColumn().permute(perm, limit);
@@ -384,7 +384,7 @@ void ColumnNullable::getExtremes(Field & min, Field & max) const
 }
 
 
-MutableColumnPtr ColumnNullable::replicate(const Offsets & offsets) const
+ColumnPtr ColumnNullable::replicate(const Offsets & offsets) const
 {
     ColumnPtr replicated_data = getNestedColumn().replicate(offsets);
     ColumnPtr replicated_null_map = getNullMapColumn().replicate(offsets);
diff --git a/dbms/src/Columns/ColumnNullable.h b/dbms/src/Columns/ColumnNullable.h
index 6eff31379ff..bc082bbcb8c 100644
--- a/dbms/src/Columns/ColumnNullable.h
+++ b/dbms/src/Columns/ColumnNullable.h
@@ -23,9 +23,16 @@ class ColumnNullable final : public COWPtrHelper<IColumn, ColumnNullable>
 private:
     friend class COWPtrHelper<IColumn, ColumnNullable>;
 
-    ColumnNullable(const ColumnPtr & nested_column_, const ColumnPtr & null_map_);
+    ColumnNullable(MutableColumnPtr && nested_column_, MutableColumnPtr && null_map_);
     ColumnNullable(const ColumnNullable &) = default;
 
+    using Ptr = COWPtrHelper<IColumn, ColumnNullable>::Ptr;
+
+    static Ptr createImmutable(const ColumnPtr & nested_column_, const ColumnPtr & null_map_)
+    {
+        return ColumnNullable::create(nested_column_->assumeMutable(), null_map_->assumeMutable());
+    }
+
 public:
     const char * getFamilyName() const override { return "Nullable"; }
     std::string getName() const override { return "Nullable(" + nested_column->getName() + ")"; }
@@ -50,14 +57,14 @@ public:
     }
 
     void popBack(size_t n) override;
-    MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
-    MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
+    ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
+    ColumnPtr permute(const Permutation & perm, size_t limit) const override;
     int compareAt(size_t n, size_t m, const IColumn & rhs_, int null_direction_hint) const override;
     void getPermutation(bool reverse, size_t limit, int null_direction_hint, Permutation & res) const override;
     void reserve(size_t n) override;
     size_t byteSize() const override;
     size_t allocatedBytes() const override;
-    MutableColumnPtr replicate(const Offsets & replicate_offsets) const override;
+    ColumnPtr replicate(const Offsets & replicate_offsets) const override;
     void updateHashWithValue(size_t n, SipHash & hash) const override;
     void getExtremes(Field & min, Field & max) const override;
 
diff --git a/dbms/src/Columns/ColumnString.cpp b/dbms/src/Columns/ColumnString.cpp
index 654a56590d1..ecb3bc6f62e 100644
--- a/dbms/src/Columns/ColumnString.cpp
+++ b/dbms/src/Columns/ColumnString.cpp
@@ -97,7 +97,7 @@ void ColumnString::insertRangeFrom(const IColumn & src, size_t start, size_t len
 }
 
 
-MutableColumnPtr ColumnString::filter(const Filter & filt, ssize_t result_size_hint) const
+ColumnPtr ColumnString::filter(const Filter & filt, ssize_t result_size_hint) const
 {
     if (offsets.size() == 0)
         return ColumnString::create();
@@ -112,7 +112,7 @@ MutableColumnPtr ColumnString::filter(const Filter & filt, ssize_t result_size_h
 }
 
 
-MutableColumnPtr ColumnString::permute(const Permutation & perm, size_t limit) const
+ColumnPtr ColumnString::permute(const Permutation & perm, size_t limit) const
 {
     size_t size = offsets.size();
 
@@ -208,7 +208,7 @@ void ColumnString::getPermutation(bool reverse, size_t limit, int /*nan_directio
 }
 
 
-MutableColumnPtr ColumnString::replicate(const Offsets & replicate_offsets) const
+ColumnPtr ColumnString::replicate(const Offsets & replicate_offsets) const
 {
     size_t col_size = size();
     if (col_size != replicate_offsets.size())
diff --git a/dbms/src/Columns/ColumnString.h b/dbms/src/Columns/ColumnString.h
index cd60eedfa43..c37fca3ab44 100644
--- a/dbms/src/Columns/ColumnString.h
+++ b/dbms/src/Columns/ColumnString.h
@@ -206,9 +206,9 @@ public:
 
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
 
-    MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
+    ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
 
-    MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
+    ColumnPtr permute(const Permutation & perm, size_t limit) const override;
 
     void insertDefault() override
     {
@@ -239,7 +239,7 @@ public:
     /// Sorting with respect of collation.
     void getPermutationWithCollation(const Collator & collator, bool reverse, size_t limit, Permutation & res) const;
 
-    MutableColumnPtr replicate(const Offsets & replicate_offsets) const override;
+    ColumnPtr replicate(const Offsets & replicate_offsets) const override;
 
     MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override
     {
diff --git a/dbms/src/Columns/ColumnTuple.cpp b/dbms/src/Columns/ColumnTuple.cpp
index 4bb250aa4c9..4bfb6a30c2a 100644
--- a/dbms/src/Columns/ColumnTuple.cpp
+++ b/dbms/src/Columns/ColumnTuple.cpp
@@ -31,21 +31,38 @@ std::string ColumnTuple::getName() const
     return res.str();
 }
 
-ColumnTuple::ColumnTuple(const Columns & columns) : columns(columns)
+ColumnTuple::ColumnTuple(MutableColumns && mutable_columns)
+{
+    columns.reserve(mutable_columns.size());
+    for (auto & column : mutable_columns)
+    {
+        if (column->isColumnConst())
+            throw Exception{"ColumnTuple cannot have ColumnConst as its element", ErrorCodes::ILLEGAL_COLUMN};
+
+        columns.push_back(std::move(column));
+    }
+}
+
+ColumnTuple::Ptr ColumnTuple::createImmutable(const Columns & columns)
 {
     for (const auto & column : columns)
         if (column->isColumnConst())
             throw Exception{"ColumnTuple cannot have ColumnConst as its element", ErrorCodes::ILLEGAL_COLUMN};
+
+    auto column_tuple = ColumnTuple::create(MutableColumns());
+    column_tuple->columns = columns;
+
+    return std::move(column_tuple);
 }
 
 MutableColumnPtr ColumnTuple::cloneEmpty() const
 {
     const size_t tuple_size = columns.size();
-    Columns new_columns(tuple_size);
+    MutableColumns new_columns(tuple_size);
     for (size_t i = 0; i < tuple_size; ++i)
         new_columns[i] = columns[i]->cloneEmpty();
 
-    return ColumnTuple::create(new_columns);
+    return ColumnTuple::create(std::move(new_columns));
 }
 
 Field ColumnTuple::operator[](size_t n) const
@@ -140,7 +157,7 @@ void ColumnTuple::insertRangeFrom(const IColumn & src, size_t start, size_t leng
             start, length);
 }
 
-MutableColumnPtr ColumnTuple::filter(const Filter & filt, ssize_t result_size_hint) const
+ColumnPtr ColumnTuple::filter(const Filter & filt, ssize_t result_size_hint) const
 {
     const size_t tuple_size = columns.size();
     Columns new_columns(tuple_size);
@@ -151,7 +168,7 @@ MutableColumnPtr ColumnTuple::filter(const Filter & filt, ssize_t result_size_hi
     return ColumnTuple::create(new_columns);
 }
 
-MutableColumnPtr ColumnTuple::permute(const Permutation & perm, size_t limit) const
+ColumnPtr ColumnTuple::permute(const Permutation & perm, size_t limit) const
 {
     const size_t tuple_size = columns.size();
     Columns new_columns(tuple_size);
@@ -162,7 +179,7 @@ MutableColumnPtr ColumnTuple::permute(const Permutation & perm, size_t limit) co
     return ColumnTuple::create(new_columns);
 }
 
-MutableColumnPtr ColumnTuple::replicate(const Offsets & offsets) const
+ColumnPtr ColumnTuple::replicate(const Offsets & offsets) const
 {
     const size_t tuple_size = columns.size();
     Columns new_columns(tuple_size);
@@ -185,10 +202,10 @@ MutableColumns ColumnTuple::scatter(ColumnIndex num_columns, const Selector & se
 
     for (size_t scattered_idx = 0; scattered_idx < num_columns; ++scattered_idx)
     {
-        Columns new_columns(tuple_size);
+        MutableColumns new_columns(tuple_size);
         for (size_t tuple_element_idx = 0; tuple_element_idx < tuple_size; ++tuple_element_idx)
             new_columns[tuple_element_idx] = std::move(scattered_tuple_elements[tuple_element_idx][scattered_idx]);
-        res[scattered_idx] = ColumnTuple::create(new_columns);
+        res[scattered_idx] = ColumnTuple::create(std::move(new_columns));
     }
 
     return res;
diff --git a/dbms/src/Columns/ColumnTuple.h b/dbms/src/Columns/ColumnTuple.h
index c2ca2e43ab7..752a649ba28 100644
--- a/dbms/src/Columns/ColumnTuple.h
+++ b/dbms/src/Columns/ColumnTuple.h
@@ -22,9 +22,11 @@ private:
     template <bool positive>
     struct Less;
 
-    ColumnTuple(const Columns & columns);
+    ColumnTuple(MutableColumns && columns);
     ColumnTuple(const ColumnTuple &) = default;
 
+    static Ptr createImmutable(const Columns & columns);
+
 public:
     std::string getName() const override;
     const char * getFamilyName() const override { return "Tuple"; }
@@ -49,9 +51,9 @@ public:
     const char * deserializeAndInsertFromArena(const char * pos) override;
     void updateHashWithValue(size_t n, SipHash & hash) const override;
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
-    MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
-    MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
-    MutableColumnPtr replicate(const Offsets & offsets) const override;
+    ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
+    ColumnPtr permute(const Permutation & perm, size_t limit) const override;
+    ColumnPtr replicate(const Offsets & offsets) const override;
     MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override;
     void gather(ColumnGathererStream & gatherer_stream) override;
     int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
diff --git a/dbms/src/Columns/ColumnVector.cpp b/dbms/src/Columns/ColumnVector.cpp
index b1a3e55cb86..3509c14c3aa 100644
--- a/dbms/src/Columns/ColumnVector.cpp
+++ b/dbms/src/Columns/ColumnVector.cpp
@@ -146,7 +146,7 @@ void ColumnVector<T>::insertRangeFrom(const IColumn & src, size_t start, size_t
 }
 
 template <typename T>
-MutableColumnPtr ColumnVector<T>::filter(const IColumn::Filter & filt, ssize_t result_size_hint) const
+ColumnPtr ColumnVector<T>::filter(const IColumn::Filter & filt, ssize_t result_size_hint) const
 {
     size_t size = data.size();
     if (size != filt.size())
@@ -210,7 +210,7 @@ MutableColumnPtr ColumnVector<T>::filter(const IColumn::Filter & filt, ssize_t r
 }
 
 template <typename T>
-MutableColumnPtr ColumnVector<T>::permute(const IColumn::Permutation & perm, size_t limit) const
+ColumnPtr ColumnVector<T>::permute(const IColumn::Permutation & perm, size_t limit) const
 {
     size_t size = data.size();
 
@@ -231,7 +231,7 @@ MutableColumnPtr ColumnVector<T>::permute(const IColumn::Permutation & perm, siz
 }
 
 template <typename T>
-MutableColumnPtr ColumnVector<T>::replicate(const IColumn::Offsets & offsets) const
+ColumnPtr ColumnVector<T>::replicate(const IColumn::Offsets & offsets) const
 {
     size_t size = data.size();
     if (size != offsets.size())
diff --git a/dbms/src/Columns/ColumnVector.h b/dbms/src/Columns/ColumnVector.h
index 0b75a0c3dfc..5ce33e82028 100644
--- a/dbms/src/Columns/ColumnVector.h
+++ b/dbms/src/Columns/ColumnVector.h
@@ -243,11 +243,11 @@ public:
 
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
 
-    MutableColumnPtr filter(const IColumn::Filter & filt, ssize_t result_size_hint) const override;
+    ColumnPtr filter(const IColumn::Filter & filt, ssize_t result_size_hint) const override;
 
-    MutableColumnPtr permute(const IColumn::Permutation & perm, size_t limit) const override;
+    ColumnPtr permute(const IColumn::Permutation & perm, size_t limit) const override;
 
-    MutableColumnPtr replicate(const IColumn::Offsets & offsets) const override;
+    ColumnPtr replicate(const IColumn::Offsets & offsets) const override;
 
     void getExtremes(Field & min, Field & max) const override;
 
diff --git a/dbms/src/Columns/IColumn.h b/dbms/src/Columns/IColumn.h
index 3a733e4a025..c546f315c0c 100644
--- a/dbms/src/Columns/IColumn.h
+++ b/dbms/src/Columns/IColumn.h
@@ -45,7 +45,7 @@ public:
     /** If column isn't constant, returns nullptr (or itself).
       * If column is constant, transforms constant to full column (if column type allows such tranform) and return it.
       */
-    virtual MutablePtr convertToFullColumnIfConst() const { return {}; }
+    virtual Ptr convertToFullColumnIfConst() const { return {}; }
 
     /// Creates empty column with the same type.
     virtual MutablePtr cloneEmpty() const { return cloneResized(0); }
@@ -104,11 +104,11 @@ public:
 
     /// Removes all elements outside of specified range.
     /// Is used in LIMIT operation, for example.
-    virtual MutablePtr cut(size_t start, size_t length) const
+    virtual Ptr cut(size_t start, size_t length) const
     {
         MutablePtr res = cloneEmpty();
         res->insertRangeFrom(*this, start, length);
-        return res;
+        return std::move(res);
     }
 
     /// Appends new value at the end of column (column's size is increased by 1).
@@ -171,12 +171,12 @@ public:
       *  otherwise (i.e. < 0), makes reserve() using size of source column.
       */
     using Filter = PaddedPODArray<UInt8>;
-    virtual MutablePtr filter(const Filter & filt, ssize_t result_size_hint) const = 0;
+    virtual Ptr filter(const Filter & filt, ssize_t result_size_hint) const = 0;
 
     /// Permutes elements using specified permutation. Is used in sortings.
     /// limit - if it isn't 0, puts only first limit elements in the result.
     using Permutation = PaddedPODArray<size_t>;
-    virtual MutablePtr permute(const Permutation & perm, size_t limit) const = 0;
+    virtual Ptr permute(const Permutation & perm, size_t limit) const = 0;
 
     /** Compares (*this)[n] and rhs[m].
       * Returns negative number, 0, or positive number (*this)[n] is less, equal, greater than rhs[m] respectively.
@@ -205,7 +205,7 @@ public:
       */
     using Offset = UInt64;
     using Offsets = PaddedPODArray<Offset>;
-    virtual MutablePtr replicate(const Offsets & offsets) const = 0;
+    virtual Ptr replicate(const Offsets & offsets) const = 0;
 
     /** Split column to smaller columns. Each value goes to column index, selected by corresponding element of 'selector'.
       * Selector must contain values from 0 to num_columns - 1.
diff --git a/dbms/src/Columns/IColumnDummy.h b/dbms/src/Columns/IColumnDummy.h
index dbf9daea796..c580a2b0cb1 100644
--- a/dbms/src/Columns/IColumnDummy.h
+++ b/dbms/src/Columns/IColumnDummy.h
@@ -74,12 +74,12 @@ public:
         s += length;
     }
 
-    MutableColumnPtr filter(const Filter & filt, ssize_t /*result_size_hint*/) const override
+    ColumnPtr filter(const Filter & filt, ssize_t /*result_size_hint*/) const override
     {
         return cloneDummy(countBytesInFilter(filt));
     }
 
-    MutableColumnPtr permute(const Permutation & perm, size_t limit) const override
+    ColumnPtr permute(const Permutation & perm, size_t limit) const override
     {
         if (s != perm.size())
             throw Exception("Size of permutation doesn't match size of column.", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
@@ -94,7 +94,7 @@ public:
             res[i] = i;
     }
 
-    MutableColumnPtr replicate(const Offsets & offsets) const override
+    ColumnPtr replicate(const Offsets & offsets) const override
     {
         if (s != offsets.size())
             throw Exception("Size of offsets doesn't match size of column.", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
diff --git a/dbms/src/Common/COWPtr.h b/dbms/src/Common/COWPtr.h
index 39c912f8abc..a7fce08ae13 100644
--- a/dbms/src/Common/COWPtr.h
+++ b/dbms/src/Common/COWPtr.h
@@ -226,15 +226,53 @@ private:
     Derived * derived() { return static_cast<Derived *>(this); }
     const Derived * derived() const { return static_cast<const Derived *>(this); }
 
+    template<typename Class, typename... Args>
+    struct HasCreateImmutable
+    {
+    private:
+        template<typename T>
+        static constexpr auto check(decltype(std::declval<T>().createImmutable(std::declval<Args>()... )) *) -> std::true_type;
+
+        template<typename>
+        static constexpr std::false_type check(...);
+
+        typedef decltype(check<Class>(nullptr)) type;
+
+    public:
+        static constexpr bool value = type::value;
+    };
+
+    template<typename Class, typename... Args>
+    struct IsConstructable
+    {
+    private:
+        template<typename T>
+        static constexpr auto check(decltype(T(std::declval<Args>()... )) *) -> std::true_type;
+
+        template<typename>
+        static constexpr std::false_type check(...);
+
+        typedef decltype(check<Class>(nullptr)) type;
+
+    public:
+        static constexpr bool value = type::value;
+    };
+
 public:
     using Ptr = typename Base::template immutable_ptr<Derived>;
     using MutablePtr = typename Base::template mutable_ptr<Derived>;
 
     template <typename... Args>
-    static MutablePtr create(Args &&... args) { return MutablePtr(new Derived(std::forward<Args>(args)...)); }
+    static auto create(Args &&... args)
+    {
+        if constexpr (HasCreateImmutable<Derived, Args ...>::value && !IsConstructable<Derived, Args ...>::value)
+            return Derived::createImmutable(std::forward<Args>(args)...);
+        else
+            return MutablePtr(new Derived(std::forward<Args>(args)...));
+    }
 
     template <typename T>
-    static MutablePtr create(std::initializer_list<T> && arg) { return create(std::forward<std::initializer_list<T>>(arg)); }
+    static auto create(std::initializer_list<T> && arg) { return create(std::forward<std::initializer_list<T>>(arg)); }
 
     typename Base::MutablePtr clone() const override { return typename Base::MutablePtr(new Derived(*derived())); }
 };
diff --git a/dbms/src/DataStreams/MergingSortedBlockInputStream.cpp b/dbms/src/DataStreams/MergingSortedBlockInputStream.cpp
index b60cc072b47..be90a00e4b9 100644
--- a/dbms/src/DataStreams/MergingSortedBlockInputStream.cpp
+++ b/dbms/src/DataStreams/MergingSortedBlockInputStream.cpp
@@ -233,7 +233,7 @@ void MergingSortedBlockInputStream::merge(MutableColumns & merged_columns, std::
                     for (size_t i = 0; i < num_columns; ++i)
                     {
                         auto & column = merged_columns[i];
-                        column = column->cut(0, merged_rows);
+                        column = (*column->cut(0, merged_rows)).mutate();
                     }
 
                     cancel(false);
diff --git a/dbms/src/DataStreams/SummingSortedBlockInputStream.cpp b/dbms/src/DataStreams/SummingSortedBlockInputStream.cpp
index 1102258d348..66817f23037 100644
--- a/dbms/src/DataStreams/SummingSortedBlockInputStream.cpp
+++ b/dbms/src/DataStreams/SummingSortedBlockInputStream.cpp
@@ -259,11 +259,11 @@ Block SummingSortedBlockInputStream::readImpl()
         if (checkDataType<DataTypeTuple>(desc.function->getReturnType().get()))
         {
             size_t tuple_size = desc.column_numbers.size();
-            Columns tuple_columns(tuple_size);
+            MutableColumns tuple_columns(tuple_size);
             for (size_t i = 0; i < tuple_size; ++i)
-                tuple_columns[i] = header.safeGetByPosition(desc.column_numbers[i]).column;
+                tuple_columns[i] = header.safeGetByPosition(desc.column_numbers[i]).column->assumeMutable();
 
-            desc.merged_column = ColumnTuple::create(tuple_columns);
+            desc.merged_column = ColumnTuple::create(std::move(tuple_columns));
         }
         else
             desc.merged_column = header.safeGetByPosition(desc.column_numbers[0]).column->cloneEmpty();
diff --git a/dbms/src/DataTypes/DataTypeTuple.cpp b/dbms/src/DataTypes/DataTypeTuple.cpp
index 6f7d692af5f..2da9f0f6ecf 100644
--- a/dbms/src/DataTypes/DataTypeTuple.cpp
+++ b/dbms/src/DataTypes/DataTypeTuple.cpp
@@ -329,10 +329,10 @@ void DataTypeTuple::deserializeBinaryBulkWithMultipleStreams(
 MutableColumnPtr DataTypeTuple::createColumn() const
 {
     size_t size = elems.size();
-    Columns tuple_columns(size);
+    MutableColumns tuple_columns(size);
     for (size_t i = 0; i < size; ++i)
         tuple_columns[i] = elems[i]->createColumn();
-    return ColumnTuple::create(tuple_columns);
+    return ColumnTuple::create(std::move(tuple_columns));
 }
 
 Field DataTypeTuple::getDefault() const
diff --git a/dbms/src/Functions/FunctionsHigherOrder.h b/dbms/src/Functions/FunctionsHigherOrder.h
index c63dce364b9..47722370777 100644
--- a/dbms/src/Functions/FunctionsHigherOrder.h
+++ b/dbms/src/Functions/FunctionsHigherOrder.h
@@ -936,7 +936,7 @@ public:
             }
 
             /// Put all the necessary columns multiplied by the sizes of arrays into the block.
-            auto replicated_column_function_ptr = column_function->replicate(column_first_array->getOffsets());
+            auto replicated_column_function_ptr = (*column_function->replicate(column_first_array->getOffsets())).mutate();
             auto * replicated_column_function = typeid_cast<ColumnFunction *>(replicated_column_function_ptr.get());
             replicated_column_function->appendArguments(arrays);
 
diff --git a/dbms/src/Functions/GatherUtils/createArraySink.cpp b/dbms/src/Functions/GatherUtils/createArraySink.cpp
index b97bf476919..e4097195e4c 100644
--- a/dbms/src/Functions/GatherUtils/createArraySink.cpp
+++ b/dbms/src/Functions/GatherUtils/createArraySink.cpp
@@ -41,7 +41,8 @@ std::unique_ptr<IArraySink> createArraySink(ColumnArray & col, size_t column_siz
     using Creator = ApplyTypeListForClass<ArraySinkCreator, TypeListNumbers>::Type;
     if (auto column_nullable = typeid_cast<ColumnNullable *>(&col.getData()))
     {
-        auto column = ColumnArray::create(column_nullable->getNestedColumnPtr(), col.getOffsetsPtr());
+        auto column = ColumnArray::create(column_nullable->getNestedColumnPtr()->assumeMutable(),
+                                          col.getOffsetsPtr()->assumeMutable());
         return Creator::create(*column, &column_nullable->getNullMapData(), column_size);
     }
     return Creator::create(col, nullptr, column_size);
diff --git a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp
index e172760b17f..a69b7ca57e5 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp
@@ -111,7 +111,8 @@ size_t MergeTreeReader::readRows(size_t from_mark, bool continue_reading, size_t
 
                 /// share offsets in all elements of nested structure
                 if (!append)
-                    column = ColumnArray::create(type_arr->getNestedType()->createColumn(), it_inserted.first->second);
+                    column = ColumnArray::create(type_arr->getNestedType()->createColumn(),
+                                                 it_inserted.first->second)->assumeMutable();
             }
 
             try

From 5c91eecb2b0900846ea0d8b8030ef2d60c43fb3e Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Wed, 21 Mar 2018 21:43:17 +0300
Subject: [PATCH 165/244] fix test

---
 dbms/src/Columns/ColumnConst.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/dbms/src/Columns/ColumnConst.h b/dbms/src/Columns/ColumnConst.h
index 89f4c488940..2e4a692451f 100644
--- a/dbms/src/Columns/ColumnConst.h
+++ b/dbms/src/Columns/ColumnConst.h
@@ -133,7 +133,6 @@ public:
 
     const char * deserializeAndInsertFromArena(const char * pos) override
     {
-        data = (*std::move(data)).mutate();
         auto & mutable_data = data->assumeMutableRef();
         auto res = mutable_data.deserializeAndInsertFromArena(pos);
         mutable_data.popBack(1);

From f4b742e9fb4ef02208b6702b1edd13bcb2e1aace Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Thu, 22 Mar 2018 18:37:24 +0300
Subject: [PATCH 166/244] Fixed segfault in case of bad argument types for some
 aggregate functions [#CLICKHOUSE-2]

---
 dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp   | 5 ++++-
 .../AggregateFunctionMaxIntersections.cpp                  | 7 ++++++-
 dbms/src/AggregateFunctions/AggregateFunctionSumMap.cpp    | 6 +++++-
 3 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp b/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp
index 5fc341b3095..eca854a031b 100644
--- a/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp
+++ b/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp
@@ -70,7 +70,10 @@ AggregateFunctionPtr AggregateFunctionFactory::get(
         return combinator->transformAggregateFunction(nested_function, argument_types, parameters);
     }
 
-    return getImpl(name, argument_types, parameters, recursion_level);
+    auto res = getImpl(name, argument_types, parameters, recursion_level);
+    if (!res)
+        throw Exception("Logical error: AggregateFunctionFactory returned nullptr", ErrorCodes::LOGICAL_ERROR);
+    return res;
 }
 
 
diff --git a/dbms/src/AggregateFunctions/AggregateFunctionMaxIntersections.cpp b/dbms/src/AggregateFunctions/AggregateFunctionMaxIntersections.cpp
index e81dfe98259..9f63d399fe7 100644
--- a/dbms/src/AggregateFunctions/AggregateFunctionMaxIntersections.cpp
+++ b/dbms/src/AggregateFunctions/AggregateFunctionMaxIntersections.cpp
@@ -16,7 +16,12 @@ namespace
         assertBinary(name, argument_types);
         assertNoParameters(name, parameters);
 
-        return AggregateFunctionPtr{createWithNumericType<AggregateFunctionIntersectionsMax>(*argument_types[0], kind, argument_types)};
+        AggregateFunctionPtr res(createWithNumericType<AggregateFunctionIntersectionsMax>(*argument_types[0], kind, argument_types));
+        if (!res)
+            throw Exception("Illegal types " + argument_types[0]->getName() + " and " + argument_types[1]->getName()
+                + " of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+
+        return res;
     }
 }
 
diff --git a/dbms/src/AggregateFunctions/AggregateFunctionSumMap.cpp b/dbms/src/AggregateFunctions/AggregateFunctionSumMap.cpp
index 9aedeaabe24..7cad7c35092 100644
--- a/dbms/src/AggregateFunctions/AggregateFunctionSumMap.cpp
+++ b/dbms/src/AggregateFunctions/AggregateFunctionSumMap.cpp
@@ -37,7 +37,11 @@ AggregateFunctionPtr createAggregateFunctionSumMap(const std::string & name, con
         values_types.push_back(array_type->getNestedType());
     }
 
-    return AggregateFunctionPtr(createWithNumericType<AggregateFunctionSumMap>(*keys_type, keys_type, std::move(values_types)));
+    AggregateFunctionPtr res(createWithNumericType<AggregateFunctionSumMap>(*keys_type, keys_type, std::move(values_types)));
+    if (!res)
+        throw Exception("Illegal type of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+
+    return res;
 }
 
 }

From 97624d7d78f8b0232552ad666573ab884cc898a3 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Thu, 22 Mar 2018 22:43:00 +0300
Subject: [PATCH 167/244] fix reading with evaluated defaults
 [#CLICKHOUSE-3663]

---
 .../Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp  | 8 ++++----
 dbms/src/Storages/MergeTree/MergeTreeReader.cpp           | 7 +++----
 dbms/src/Storages/MergeTree/MergeTreeReader.h             | 6 +++++-
 3 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp
index b5095e79118..0cd903038ed 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp
@@ -216,7 +216,7 @@ Block MergeTreeBaseBlockInputStream::readFromPart()
             }
 
             progressImpl({ res.rows(), res.bytes() });
-            pre_reader->fillMissingColumns(res, task->ordered_names, task->should_reorder);
+            pre_reader->fillMissingColumns(res, task->ordered_names, task->should_reorder, res.rows());
 
             /// Compute the expression in PREWHERE.
             prewhere_actions->execute(res);
@@ -396,7 +396,7 @@ Block MergeTreeBaseBlockInputStream::readFromPart()
                 ///  as a result of reading components of Nested data structures with no data in filesystem.
                 /// We must fill these arrays to filter them correctly.
 
-                reader->fillMissingColumns(res, task->ordered_names, task->should_reorder);
+                reader->fillMissingColumns(res, task->ordered_names, task->should_reorder, post_filter.size(), true);
 
                 /// Filter the columns related to PREWHERE using pre_filter,
                 ///  other columns - using post_filter.
@@ -422,7 +422,7 @@ Block MergeTreeBaseBlockInputStream::readFromPart()
                 if (task->size_predictor)
                     task->size_predictor->update(res);
 
-                reader->fillMissingColumns(res, task->ordered_names, true);
+                reader->fillMissingColumns(res, task->ordered_names, true, res.rows());
 
                 res.checkNumberOfRows();
             }
@@ -464,7 +464,7 @@ Block MergeTreeBaseBlockInputStream::readFromPart()
             return res;
 
         progressImpl({ res.rows(), res.bytes() });
-        reader->fillMissingColumns(res, task->ordered_names, task->should_reorder);
+        reader->fillMissingColumns(res, task->ordered_names, task->should_reorder, res.rows());
     }
 
     return res;
diff --git a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp
index e172760b17f..37d41749420 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp
@@ -413,7 +413,8 @@ static bool arrayHasNoElementsRead(const IColumn & column)
 }
 
 
-void MergeTreeReader::fillMissingColumns(Block & res, const Names & ordered_names, bool always_reorder)
+void MergeTreeReader::fillMissingColumns(Block & res, const Names & ordered_names, bool always_reorder,
+                                         size_t rows, bool never_evaluate_defaults)
 {
     if (!res)
         throw Exception("Empty block passed to fillMissingColumns", ErrorCodes::LOGICAL_ERROR);
@@ -444,8 +445,6 @@ void MergeTreeReader::fillMissingColumns(Block & res, const Names & ordered_name
         bool should_evaluate_defaults = false;
         bool should_sort = always_reorder;
 
-        size_t rows = res.rows();
-
         /// insert default values only for columns without default expressions
         for (const auto & requested_column : columns)
         {
@@ -497,7 +496,7 @@ void MergeTreeReader::fillMissingColumns(Block & res, const Names & ordered_name
         }
 
         /// evaluate defaulted columns if necessary
-        if (should_evaluate_defaults)
+        if (!never_evaluate_defaults && should_evaluate_defaults)
             evaluateMissingDefaults(res, columns, storage.getColumns().defaults, storage.context);
 
         /// sort columns to ensure consistent order among all blocks
diff --git a/dbms/src/Storages/MergeTree/MergeTreeReader.h b/dbms/src/Storages/MergeTree/MergeTreeReader.h
index c2cd086d343..b76cf19ff93 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeReader.h
+++ b/dbms/src/Storages/MergeTree/MergeTreeReader.h
@@ -44,7 +44,11 @@ public:
     /// Add columns from ordered_names that are not present in the block.
     /// Missing columns are added in the order specified by ordered_names.
     /// If at least one column was added, reorders all columns in the block according to ordered_names.
-    void fillMissingColumns(Block & res, const Names & ordered_names, const bool always_reorder);
+    /// Rows is the number of rows will be used to create default columns. It's res.rows() when all columns in block has the same size.
+    /// This function may be used in order to create default columns in nested structures. In that case block may has
+    /// columns with different size, and default columns can't be evaluated, so never_evaluate_defaults = false is used.
+    void fillMissingColumns(Block & res, const Names & ordered_names, const bool always_reorder,
+                            size_t rows, bool never_evaluate_defaults = false);
 
 private:
     class Stream

From 74211b5bf8e6e26cd948e42e9b1ef3ba78f179b6 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Thu, 22 Mar 2018 22:54:21 +0300
Subject: [PATCH 168/244] added test [#CLICKHOUSE-3663]

---
 .../0_stateless/00609_prewhere_and_default.reference |  0
 .../0_stateless/00609_prewhere_and_default.sql       | 12 ++++++++++++
 2 files changed, 12 insertions(+)
 create mode 100644 dbms/tests/queries/0_stateless/00609_prewhere_and_default.reference
 create mode 100644 dbms/tests/queries/0_stateless/00609_prewhere_and_default.sql

diff --git a/dbms/tests/queries/0_stateless/00609_prewhere_and_default.reference b/dbms/tests/queries/0_stateless/00609_prewhere_and_default.reference
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/dbms/tests/queries/0_stateless/00609_prewhere_and_default.sql b/dbms/tests/queries/0_stateless/00609_prewhere_and_default.sql
new file mode 100644
index 00000000000..915f0c85823
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00609_prewhere_and_default.sql
@@ -0,0 +1,12 @@
+drop table if exists test.table;
+create table test.table (key UInt64, val UInt64) engine = MergeTree order by key settings index_granularity=8192;
+insert into test.table select number, number / 8192 from system.numbers limit 100000; 
+alter table test.table add column def UInt64 default val + 1;
+select * from test.table prewhere val > 2 format Null;
+
+drop table if exists test.table;
+create table test.table (key UInt64, val UInt64) engine = MergeTree order by key settings index_granularity=8192;
+insert into test.table select number, number / 8192 from system.numbers limit 100000; 
+alter table test.table add column def UInt64;
+select * from test.table prewhere val > 2 format Null;
+

From cbfd862fa3df628aa30804107924e083cab8c566 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Thu, 22 Mar 2018 22:43:00 +0300
Subject: [PATCH 169/244] fix reading with evaluated defaults
 [#CLICKHOUSE-3663]

---
 .../Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp  | 8 ++++----
 dbms/src/Storages/MergeTree/MergeTreeReader.cpp           | 7 +++----
 dbms/src/Storages/MergeTree/MergeTreeReader.h             | 6 +++++-
 3 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp
index b5095e79118..0cd903038ed 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp
@@ -216,7 +216,7 @@ Block MergeTreeBaseBlockInputStream::readFromPart()
             }
 
             progressImpl({ res.rows(), res.bytes() });
-            pre_reader->fillMissingColumns(res, task->ordered_names, task->should_reorder);
+            pre_reader->fillMissingColumns(res, task->ordered_names, task->should_reorder, res.rows());
 
             /// Compute the expression in PREWHERE.
             prewhere_actions->execute(res);
@@ -396,7 +396,7 @@ Block MergeTreeBaseBlockInputStream::readFromPart()
                 ///  as a result of reading components of Nested data structures with no data in filesystem.
                 /// We must fill these arrays to filter them correctly.
 
-                reader->fillMissingColumns(res, task->ordered_names, task->should_reorder);
+                reader->fillMissingColumns(res, task->ordered_names, task->should_reorder, post_filter.size(), true);
 
                 /// Filter the columns related to PREWHERE using pre_filter,
                 ///  other columns - using post_filter.
@@ -422,7 +422,7 @@ Block MergeTreeBaseBlockInputStream::readFromPart()
                 if (task->size_predictor)
                     task->size_predictor->update(res);
 
-                reader->fillMissingColumns(res, task->ordered_names, true);
+                reader->fillMissingColumns(res, task->ordered_names, true, res.rows());
 
                 res.checkNumberOfRows();
             }
@@ -464,7 +464,7 @@ Block MergeTreeBaseBlockInputStream::readFromPart()
             return res;
 
         progressImpl({ res.rows(), res.bytes() });
-        reader->fillMissingColumns(res, task->ordered_names, task->should_reorder);
+        reader->fillMissingColumns(res, task->ordered_names, task->should_reorder, res.rows());
     }
 
     return res;
diff --git a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp
index e172760b17f..37d41749420 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp
@@ -413,7 +413,8 @@ static bool arrayHasNoElementsRead(const IColumn & column)
 }
 
 
-void MergeTreeReader::fillMissingColumns(Block & res, const Names & ordered_names, bool always_reorder)
+void MergeTreeReader::fillMissingColumns(Block & res, const Names & ordered_names, bool always_reorder,
+                                         size_t rows, bool never_evaluate_defaults)
 {
     if (!res)
         throw Exception("Empty block passed to fillMissingColumns", ErrorCodes::LOGICAL_ERROR);
@@ -444,8 +445,6 @@ void MergeTreeReader::fillMissingColumns(Block & res, const Names & ordered_name
         bool should_evaluate_defaults = false;
         bool should_sort = always_reorder;
 
-        size_t rows = res.rows();
-
         /// insert default values only for columns without default expressions
         for (const auto & requested_column : columns)
         {
@@ -497,7 +496,7 @@ void MergeTreeReader::fillMissingColumns(Block & res, const Names & ordered_name
         }
 
         /// evaluate defaulted columns if necessary
-        if (should_evaluate_defaults)
+        if (!never_evaluate_defaults && should_evaluate_defaults)
             evaluateMissingDefaults(res, columns, storage.getColumns().defaults, storage.context);
 
         /// sort columns to ensure consistent order among all blocks
diff --git a/dbms/src/Storages/MergeTree/MergeTreeReader.h b/dbms/src/Storages/MergeTree/MergeTreeReader.h
index c2cd086d343..b76cf19ff93 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeReader.h
+++ b/dbms/src/Storages/MergeTree/MergeTreeReader.h
@@ -44,7 +44,11 @@ public:
     /// Add columns from ordered_names that are not present in the block.
     /// Missing columns are added in the order specified by ordered_names.
     /// If at least one column was added, reorders all columns in the block according to ordered_names.
-    void fillMissingColumns(Block & res, const Names & ordered_names, const bool always_reorder);
+    /// Rows is the number of rows will be used to create default columns. It's res.rows() when all columns in block has the same size.
+    /// This function may be used in order to create default columns in nested structures. In that case block may has
+    /// columns with different size, and default columns can't be evaluated, so never_evaluate_defaults = false is used.
+    void fillMissingColumns(Block & res, const Names & ordered_names, const bool always_reorder,
+                            size_t rows, bool never_evaluate_defaults = false);
 
 private:
     class Stream

From 6f60e6ee4f3a2808e73527e3a7fe879c8f95bdb7 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Thu, 22 Mar 2018 22:54:21 +0300
Subject: [PATCH 170/244] added test [#CLICKHOUSE-3663]

---
 .../0_stateless/00609_prewhere_and_default.reference |  0
 .../0_stateless/00609_prewhere_and_default.sql       | 12 ++++++++++++
 2 files changed, 12 insertions(+)
 create mode 100644 dbms/tests/queries/0_stateless/00609_prewhere_and_default.reference
 create mode 100644 dbms/tests/queries/0_stateless/00609_prewhere_and_default.sql

diff --git a/dbms/tests/queries/0_stateless/00609_prewhere_and_default.reference b/dbms/tests/queries/0_stateless/00609_prewhere_and_default.reference
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/dbms/tests/queries/0_stateless/00609_prewhere_and_default.sql b/dbms/tests/queries/0_stateless/00609_prewhere_and_default.sql
new file mode 100644
index 00000000000..915f0c85823
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00609_prewhere_and_default.sql
@@ -0,0 +1,12 @@
+drop table if exists test.table;
+create table test.table (key UInt64, val UInt64) engine = MergeTree order by key settings index_granularity=8192;
+insert into test.table select number, number / 8192 from system.numbers limit 100000; 
+alter table test.table add column def UInt64 default val + 1;
+select * from test.table prewhere val > 2 format Null;
+
+drop table if exists test.table;
+create table test.table (key UInt64, val UInt64) engine = MergeTree order by key settings index_granularity=8192;
+insert into test.table select number, number / 8192 from system.numbers limit 100000; 
+alter table test.table add column def UInt64;
+select * from test.table prewhere val > 2 format Null;
+

From 7c2a99c9b8f3bf4ff3e25eea308c5b0e45db0123 Mon Sep 17 00:00:00 2001
From: Vitaliy Lyudvichenko <vludv@yandex-team.ru>
Date: Wed, 21 Mar 2018 23:21:34 +0300
Subject: [PATCH 171/244] Introduced MinimalisticDataPartChecksums.
 [#CLICKHOUSE-3660]

---
 .../Storages/MergeTree/MergeTreeDataPart.cpp  | 283 +-----------
 .../Storages/MergeTree/MergeTreeDataPart.h    |  70 +--
 .../MergeTree/MergeTreeDataPartChecksum.cpp   | 434 ++++++++++++++++++
 .../MergeTree/MergeTreeDataPartChecksum.h     | 119 +++++
 .../ReplicatedMergeTreeBlockOutputStream.cpp  |   2 +-
 5 files changed, 579 insertions(+), 329 deletions(-)
 create mode 100644 dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp
 create mode 100644 dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h

diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp
index 8ab7699a03b..79f3ddb523f 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp
@@ -30,273 +30,15 @@ namespace DB
 
 namespace ErrorCodes
 {
-    extern const int CHECKSUM_DOESNT_MATCH;
     extern const int FILE_DOESNT_EXIST;
     extern const int NO_FILE_IN_DATA_PART;
     extern const int EXPECTED_END_OF_FILE;
-    extern const int BAD_SIZE_OF_FILE_IN_DATA_PART;
     extern const int CORRUPTED_DATA;
-    extern const int FORMAT_VERSION_TOO_OLD;
-    extern const int UNKNOWN_FORMAT;
-    extern const int UNEXPECTED_FILE_IN_DATA_PART;
     extern const int NOT_FOUND_EXPECTED_DATA_PART;
+    extern const int BAD_SIZE_OF_FILE_IN_DATA_PART;
 }
 
 
-void MergeTreeDataPartChecksum::checkEqual(const MergeTreeDataPartChecksum & rhs, bool have_uncompressed, const String & name) const
-{
-    if (is_compressed && have_uncompressed)
-    {
-        if (!rhs.is_compressed)
-            throw Exception("No uncompressed checksum for file " + name, ErrorCodes::CHECKSUM_DOESNT_MATCH);
-        if (rhs.uncompressed_size != uncompressed_size)
-            throw Exception("Unexpected uncompressed size of file " + name + " in data part", ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART);
-        if (rhs.uncompressed_hash != uncompressed_hash)
-            throw Exception("Checksum mismatch for uncompressed file " + name + " in data part", ErrorCodes::CHECKSUM_DOESNT_MATCH);
-        return;
-    }
-    if (rhs.file_size != file_size)
-        throw Exception("Unexpected size of file " + name + " in data part", ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART);
-    if (rhs.file_hash != file_hash)
-        throw Exception("Checksum mismatch for file " + name + " in data part", ErrorCodes::CHECKSUM_DOESNT_MATCH);
-}
-
-void MergeTreeDataPartChecksum::checkSize(const String & path) const
-{
-    Poco::File file(path);
-    if (!file.exists())
-        throw Exception(path + " doesn't exist", ErrorCodes::FILE_DOESNT_EXIST);
-    UInt64 size = file.getSize();
-    if (size != file_size)
-        throw Exception(path + " has unexpected size: " + toString(size) + " instead of " + toString(file_size),
-            ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART);
-}
-
-
-void MergeTreeDataPartChecksums::checkEqual(const MergeTreeDataPartChecksums & rhs, bool have_uncompressed) const
-{
-    for (const auto & it : rhs.files)
-    {
-        const String & name = it.first;
-
-        if (!files.count(name))
-            throw Exception("Unexpected file " + name + " in data part", ErrorCodes::UNEXPECTED_FILE_IN_DATA_PART);
-    }
-
-    for (const auto & it : files)
-    {
-        const String & name = it.first;
-
-        auto jt = rhs.files.find(name);
-        if (jt == rhs.files.end())
-            throw Exception("No file " + name + " in data part", ErrorCodes::NO_FILE_IN_DATA_PART);
-
-        it.second.checkEqual(jt->second, have_uncompressed, name);
-    }
-}
-
-void MergeTreeDataPartChecksums::checkSizes(const String & path) const
-{
-    for (const auto & it : files)
-    {
-        const String & name = it.first;
-        it.second.checkSize(path + name);
-    }
-}
-
-bool MergeTreeDataPartChecksums::read(ReadBuffer & in)
-{
-    files.clear();
-
-    assertString("checksums format version: ", in);
-    int format_version;
-    readText(format_version, in);
-    assertChar('\n', in);
-
-    switch (format_version)
-    {
-        case 1:
-            return false;
-        case 2:
-            return read_v2(in);
-        case 3:
-            return read_v3(in);
-        case 4:
-            return read_v4(in);
-        default:
-            throw Exception("Bad checksums format version: " + DB::toString(format_version), ErrorCodes::UNKNOWN_FORMAT);
-    }
-}
-
-bool MergeTreeDataPartChecksums::read_v2(ReadBuffer & in)
-{
-    size_t count;
-
-    readText(count, in);
-    assertString(" files:\n", in);
-
-    for (size_t i = 0; i < count; ++i)
-    {
-        String name;
-        Checksum sum;
-
-        readString(name, in);
-        assertString("\n\tsize: ", in);
-        readText(sum.file_size, in);
-        assertString("\n\thash: ", in);
-        readText(sum.file_hash.first, in);
-        assertString(" ", in);
-        readText(sum.file_hash.second, in);
-        assertString("\n\tcompressed: ", in);
-        readText(sum.is_compressed, in);
-        if (sum.is_compressed)
-        {
-            assertString("\n\tuncompressed size: ", in);
-            readText(sum.uncompressed_size, in);
-            assertString("\n\tuncompressed hash: ", in);
-            readText(sum.uncompressed_hash.first, in);
-            assertString(" ", in);
-            readText(sum.uncompressed_hash.second, in);
-        }
-        assertChar('\n', in);
-
-        files.insert(std::make_pair(name, sum));
-    }
-
-    return true;
-}
-
-bool MergeTreeDataPartChecksums::read_v3(ReadBuffer & in)
-{
-    size_t count;
-
-    readVarUInt(count, in);
-
-    for (size_t i = 0; i < count; ++i)
-    {
-        String name;
-        Checksum sum;
-
-        readBinary(name, in);
-        readVarUInt(sum.file_size, in);
-        readPODBinary(sum.file_hash, in);
-        readBinary(sum.is_compressed, in);
-
-        if (sum.is_compressed)
-        {
-            readVarUInt(sum.uncompressed_size, in);
-            readPODBinary(sum.uncompressed_hash, in);
-        }
-
-        files.emplace(std::move(name), sum);
-    }
-
-    return true;
-}
-
-bool MergeTreeDataPartChecksums::read_v4(ReadBuffer & from)
-{
-    CompressedReadBuffer in{from};
-    return read_v3(in);
-}
-
-void MergeTreeDataPartChecksums::write(WriteBuffer & to) const
-{
-    writeString("checksums format version: 4\n", to);
-
-    CompressedWriteBuffer out{to, CompressionSettings(CompressionMethod::LZ4), 1 << 16};
-    writeVarUInt(files.size(), out);
-
-    for (const auto & it : files)
-    {
-        const String & name = it.first;
-        const Checksum & sum = it.second;
-
-        writeBinary(name, out);
-        writeVarUInt(sum.file_size, out);
-        writePODBinary(sum.file_hash, out);
-        writeBinary(sum.is_compressed, out);
-
-        if (sum.is_compressed)
-        {
-            writeVarUInt(sum.uncompressed_size, out);
-            writePODBinary(sum.uncompressed_hash, out);
-        }
-    }
-}
-
-void MergeTreeDataPartChecksums::addFile(const String & file_name, UInt64 file_size, MergeTreeDataPartChecksum::uint128 file_hash)
-{
-    files[file_name] = Checksum(file_size, file_hash);
-}
-
-void MergeTreeDataPartChecksums::add(MergeTreeDataPartChecksums && rhs_checksums)
-{
-    for (auto & checksum : rhs_checksums.files)
-        files[std::move(checksum.first)] = std::move(checksum.second);
-
-    rhs_checksums.files.clear();
-}
-
-/// Checksum computed from the set of control sums of .bin files.
-void MergeTreeDataPartChecksums::summaryDataChecksum(SipHash & hash) const
-{
-    /// We use fact that iteration is in deterministic (lexicographical) order.
-    for (const auto & it : files)
-    {
-        const String & name = it.first;
-        const Checksum & sum = it.second;
-
-        if (!endsWith(name, ".bin"))
-            continue;
-
-        UInt64 len = name.size();
-        hash.update(len);
-        hash.update(name.data(), len);
-        hash.update(sum.uncompressed_size);
-        hash.update(sum.uncompressed_hash);
-    }
-}
-
-String MergeTreeDataPartChecksums::toString() const
-{
-    WriteBufferFromOwnString out;
-    write(out);
-    return out.str();
-}
-
-MergeTreeDataPartChecksums MergeTreeDataPartChecksums::parse(const String & s)
-{
-    ReadBufferFromString in(s);
-    MergeTreeDataPartChecksums res;
-    if (!res.read(in))
-        throw Exception("Checksums format is too old", ErrorCodes::FORMAT_VERSION_TOO_OLD);
-    assertEOF(in);
-    return res;
-}
-
-const MergeTreeDataPartChecksums::Checksum * MergeTreeDataPart::tryGetChecksum(const String & name, const String & ext) const
-{
-    if (checksums.empty())
-        return nullptr;
-
-    const auto & files = checksums.files;
-    const auto file_name = escapeForFileName(name) + ext;
-    auto it = files.find(file_name);
-
-    return (it == files.end()) ? nullptr : &it->second;
-}
-
-const MergeTreeDataPartChecksums::Checksum * MergeTreeDataPart::tryGetBinChecksum(const String & name) const
-{
-    return tryGetChecksum(name, ".bin");
-}
-
-const MergeTreeDataPartChecksums::Checksum * MergeTreeDataPart::tryGetMrkChecksum(const String & name) const
-{
-    return tryGetChecksum(name, ".mrk");
-}
-
 static ReadBufferFromFile openForReading(const String & path)
 {
     return ReadBufferFromFile(path, std::min(static_cast<Poco::File::FileSize>(DBMS_DEFAULT_BUFFER_SIZE), Poco::File(path).getSize()));
@@ -928,6 +670,29 @@ bool MergeTreeDataPart::hasColumnFiles(const String & column) const
 }
 
 
+const MergeTreeDataPartChecksums::Checksum * MergeTreeDataPart::tryGetChecksum(const String & name, const String & ext) const
+{
+    if (checksums.empty())
+        return nullptr;
+
+    const auto & files = checksums.files;
+    const auto file_name = escapeForFileName(name) + ext;
+    auto it = files.find(file_name);
+
+    return (it == files.end()) ? nullptr : &it->second;
+}
+
+const MergeTreeDataPartChecksums::Checksum * MergeTreeDataPart::tryGetBinChecksum(const String & name) const
+{
+    return tryGetChecksum(name, ".bin");
+}
+
+const MergeTreeDataPartChecksums::Checksum * MergeTreeDataPart::tryGetMrkChecksum(const String & name) const
+{
+    return tryGetChecksum(name, ".mrk");
+}
+
+
 UInt64 MergeTreeDataPart::getIndexSizeInBytes() const
 {
     UInt64 res = 0;
diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.h b/dbms/src/Storages/MergeTree/MergeTreeDataPart.h
index 95bed39eaec..a617dfedacd 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.h
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.h
@@ -5,82 +5,14 @@
 #include <Core/NamesAndTypes.h>
 #include <Storages/MergeTree/MergeTreePartInfo.h>
 #include <Storages/MergeTree/MergeTreePartition.h>
+#include <Storages/MergeTree/MergeTreeDataPartChecksum.h>
 #include <Columns/IColumn.h>
 #include <shared_mutex>
 
 
-class SipHash;
-
-
 namespace DB
 {
 
-
-/// Checksum of one file.
-struct MergeTreeDataPartChecksum
-{
-    using uint128 = CityHash_v1_0_2::uint128;
-
-    UInt64 file_size {};
-    uint128 file_hash {};
-
-    bool is_compressed = false;
-    UInt64 uncompressed_size {};
-    uint128 uncompressed_hash {};
-
-    MergeTreeDataPartChecksum() {}
-    MergeTreeDataPartChecksum(UInt64 file_size_, uint128 file_hash_) : file_size(file_size_), file_hash(file_hash_) {}
-    MergeTreeDataPartChecksum(UInt64 file_size_, uint128 file_hash_, UInt64 uncompressed_size_, uint128 uncompressed_hash_)
-        : file_size(file_size_), file_hash(file_hash_), is_compressed(true),
-        uncompressed_size(uncompressed_size_), uncompressed_hash(uncompressed_hash_) {}
-
-    void checkEqual(const MergeTreeDataPartChecksum & rhs, bool have_uncompressed, const String & name) const;
-    void checkSize(const String & path) const;
-};
-
-
-/** Checksums of all non-temporary files.
-  * For compressed files, the check sum and the size of the decompressed data are stored to not depend on the compression method.
-  */
-struct MergeTreeDataPartChecksums
-{
-    using Checksum = MergeTreeDataPartChecksum;
-
-    /// The order is important.
-    using FileChecksums = std::map<String, Checksum>;
-    FileChecksums files;
-
-    void addFile(const String & file_name, UInt64 file_size, Checksum::uint128 file_hash);
-
-    void add(MergeTreeDataPartChecksums && rhs_checksums);
-
-    /// Checks that the set of columns and their checksums are the same. If not, throws an exception.
-    /// If have_uncompressed, for compressed files it compares the checksums of the decompressed data. Otherwise, it compares only the checksums of the files.
-    void checkEqual(const MergeTreeDataPartChecksums & rhs, bool have_uncompressed) const;
-
-    /// Checks that the directory contains all the needed files of the correct size. Does not check the checksum.
-    void checkSizes(const String & path) const;
-
-    /// Serializes and deserializes in human readable form.
-    bool read(ReadBuffer & in); /// Returns false if the checksum is too old.
-    bool read_v2(ReadBuffer & in);
-    bool read_v3(ReadBuffer & in);
-    bool read_v4(ReadBuffer & in);
-    void write(WriteBuffer & out) const;
-
-    bool empty() const
-    {
-        return files.empty();
-    }
-
-    /// Checksum from the set of checksums of .bin files.
-    void summaryDataChecksum(SipHash & hash) const;
-
-    String toString() const;
-    static MergeTreeDataPartChecksums parse(const String & s);
-};
-
-
 class MergeTreeData;
 
 
diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp
new file mode 100644
index 00000000000..b252830d0d1
--- /dev/null
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp
@@ -0,0 +1,434 @@
+#include "MergeTreeDataPartChecksum.h"
+#include <Common/SipHash.h>
+#include <IO/ReadHelpers.h>
+#include <IO/WriteHelpers.h>
+#include <IO/ReadBufferFromString.h>
+#include <IO/WriteBufferFromString.h>
+#include <IO/CompressedReadBuffer.h>
+#include <IO/CompressedWriteBuffer.h>
+#include <Poco/File.h>
+
+
+namespace DB
+{
+
+
+namespace ErrorCodes
+{
+    extern const int CHECKSUM_DOESNT_MATCH;
+    extern const int BAD_SIZE_OF_FILE_IN_DATA_PART;
+    extern const int FORMAT_VERSION_TOO_OLD;
+    extern const int FILE_DOESNT_EXIST;
+    extern const int UNEXPECTED_FILE_IN_DATA_PART;
+    extern const int UNKNOWN_FORMAT;
+    extern const int NO_FILE_IN_DATA_PART;
+}
+
+
+void MergeTreeDataPartChecksum::checkEqual(const MergeTreeDataPartChecksum & rhs, bool have_uncompressed, const String & name) const
+{
+    if (is_compressed && have_uncompressed)
+    {
+        if (!rhs.is_compressed)
+            throw Exception("No uncompressed checksum for file " + name, ErrorCodes::CHECKSUM_DOESNT_MATCH);
+        if (rhs.uncompressed_size != uncompressed_size)
+            throw Exception("Unexpected uncompressed size of file " + name + " in data part", ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART);
+        if (rhs.uncompressed_hash != uncompressed_hash)
+            throw Exception("Checksum mismatch for uncompressed file " + name + " in data part", ErrorCodes::CHECKSUM_DOESNT_MATCH);
+        return;
+    }
+    if (rhs.file_size != file_size)
+        throw Exception("Unexpected size of file " + name + " in data part", ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART);
+    if (rhs.file_hash != file_hash)
+        throw Exception("Checksum mismatch for file " + name + " in data part", ErrorCodes::CHECKSUM_DOESNT_MATCH);
+}
+
+void MergeTreeDataPartChecksum::checkSize(const String & path) const
+{
+    Poco::File file(path);
+    if (!file.exists())
+        throw Exception(path + " doesn't exist", ErrorCodes::FILE_DOESNT_EXIST);
+    UInt64 size = file.getSize();
+    if (size != file_size)
+        throw Exception(path + " has unexpected size: " + toString(size) + " instead of " + toString(file_size),
+            ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART);
+}
+
+
+void MergeTreeDataPartChecksums::checkEqual(const MergeTreeDataPartChecksums & rhs, bool have_uncompressed) const
+{
+    for (const auto & it : rhs.files)
+    {
+        const String & name = it.first;
+
+        if (!files.count(name))
+            throw Exception("Unexpected file " + name + " in data part", ErrorCodes::UNEXPECTED_FILE_IN_DATA_PART);
+    }
+
+    for (const auto & it : files)
+    {
+        const String & name = it.first;
+
+        auto jt = rhs.files.find(name);
+        if (jt == rhs.files.end())
+            throw Exception("No file " + name + " in data part", ErrorCodes::NO_FILE_IN_DATA_PART);
+
+        it.second.checkEqual(jt->second, have_uncompressed, name);
+    }
+}
+
+void MergeTreeDataPartChecksums::checkSizes(const String & path) const
+{
+    for (const auto & it : files)
+    {
+        const String & name = it.first;
+        it.second.checkSize(path + name);
+    }
+}
+
+bool MergeTreeDataPartChecksums::read(ReadBuffer & in, size_t format_version)
+{
+    switch (format_version)
+    {
+        case 1:
+            return false;
+        case 2:
+            return read_v2(in);
+        case 3:
+            return read_v3(in);
+        case 4:
+            return read_v4(in);
+        default:
+            throw Exception("Bad checksums format version: " + DB::toString(format_version), ErrorCodes::UNKNOWN_FORMAT);
+    }
+}
+
+bool MergeTreeDataPartChecksums::read(ReadBuffer & in)
+{
+    files.clear();
+
+    assertString("checksums format version: ", in);
+    size_t format_version;
+    readText(format_version, in);
+    assertChar('\n', in);
+
+    read(in, format_version);
+    return true;
+}
+
+bool MergeTreeDataPartChecksums::read_v2(ReadBuffer & in)
+{
+    size_t count;
+
+    readText(count, in);
+    assertString(" files:\n", in);
+
+    for (size_t i = 0; i < count; ++i)
+    {
+        String name;
+        Checksum sum;
+
+        readString(name, in);
+        assertString("\n\tsize: ", in);
+        readText(sum.file_size, in);
+        assertString("\n\thash: ", in);
+        readText(sum.file_hash.first, in);
+        assertString(" ", in);
+        readText(sum.file_hash.second, in);
+        assertString("\n\tcompressed: ", in);
+        readText(sum.is_compressed, in);
+        if (sum.is_compressed)
+        {
+            assertString("\n\tuncompressed size: ", in);
+            readText(sum.uncompressed_size, in);
+            assertString("\n\tuncompressed hash: ", in);
+            readText(sum.uncompressed_hash.first, in);
+            assertString(" ", in);
+            readText(sum.uncompressed_hash.second, in);
+        }
+        assertChar('\n', in);
+
+        files.insert(std::make_pair(name, sum));
+    }
+
+    return true;
+}
+
+bool MergeTreeDataPartChecksums::read_v3(ReadBuffer & in)
+{
+    size_t count;
+
+    readVarUInt(count, in);
+
+    for (size_t i = 0; i < count; ++i)
+    {
+        String name;
+        Checksum sum;
+
+        readBinary(name, in);
+        readVarUInt(sum.file_size, in);
+        readPODBinary(sum.file_hash, in);
+        readBinary(sum.is_compressed, in);
+
+        if (sum.is_compressed)
+        {
+            readVarUInt(sum.uncompressed_size, in);
+            readPODBinary(sum.uncompressed_hash, in);
+        }
+
+        files.emplace(std::move(name), sum);
+    }
+
+    return true;
+}
+
+bool MergeTreeDataPartChecksums::read_v4(ReadBuffer & from)
+{
+    CompressedReadBuffer in{from};
+    return read_v3(in);
+}
+
+void MergeTreeDataPartChecksums::write(WriteBuffer & to) const
+{
+    writeString("checksums format version: 4\n", to);
+
+    CompressedWriteBuffer out{to, CompressionSettings(CompressionMethod::LZ4), 1 << 16};
+
+    writeVarUInt(files.size(), out);
+
+    for (const auto & it : files)
+    {
+        const String & name = it.first;
+        const Checksum & sum = it.second;
+
+        writeBinary(name, out);
+        writeVarUInt(sum.file_size, out);
+        writePODBinary(sum.file_hash, out);
+        writeBinary(sum.is_compressed, out);
+
+        if (sum.is_compressed)
+        {
+            writeVarUInt(sum.uncompressed_size, out);
+            writePODBinary(sum.uncompressed_hash, out);
+        }
+    }
+}
+
+void MergeTreeDataPartChecksums::addFile(const String & file_name, UInt64 file_size, MergeTreeDataPartChecksum::uint128 file_hash)
+{
+    files[file_name] = Checksum(file_size, file_hash);
+}
+
+void MergeTreeDataPartChecksums::add(MergeTreeDataPartChecksums && rhs_checksums)
+{
+    for (auto & checksum : rhs_checksums.files)
+        files[std::move(checksum.first)] = std::move(checksum.second);
+
+    rhs_checksums.files.clear();
+}
+
+/// Checksum computed from the set of control sums of .bin files.
+void MergeTreeDataPartChecksums::computeTotalChecksumDataOnly(SipHash & hash) const
+{
+    /// We use fact that iteration is in deterministic (lexicographical) order.
+    for (const auto & it : files)
+    {
+        const String & name = it.first;
+        const Checksum & sum = it.second;
+
+        if (!endsWith(name, ".bin"))
+            continue;
+
+        UInt64 len = name.size();
+        hash.update(len);
+        hash.update(name.data(), len);
+        hash.update(sum.uncompressed_size);
+        hash.update(sum.uncompressed_hash);
+    }
+}
+
+String MergeTreeDataPartChecksums::toString() const
+{
+    WriteBufferFromOwnString out;
+    write(out);
+    return out.str();
+}
+
+MergeTreeDataPartChecksums MergeTreeDataPartChecksums::deserializeFrom(const String & s)
+{
+    ReadBufferFromString in(s);
+    MergeTreeDataPartChecksums res;
+    if (!res.read(in))
+        throw Exception("Checksums format is too old", ErrorCodes::FORMAT_VERSION_TOO_OLD);
+    assertEOF(in);
+    return res;
+}
+
+MinimalisticDataPartChecksums::MinimalisticDataPartChecksums(const MergeTreeDataPartChecksums & full_checksums_)
+{
+    full_checksums = std::make_unique<MergeTreeDataPartChecksums>(full_checksums_);
+    computeTotalChecksums(*full_checksums);
+}
+
+void MinimalisticDataPartChecksums::serialize(WriteBuffer & to) const
+{
+    writeString("checksums format version: 5\n", to);
+
+    writeVarUInt(num_compressed_files, to);
+    writeVarUInt(num_uncompressed_files, to);
+
+    writePODBinary(hash_of_all_files, to);
+    writePODBinary(hash_of_uncompressed_files, to);
+    writePODBinary(uncompressed_hash_of_compressed_files, to);
+}
+
+String MinimalisticDataPartChecksums::getSerializedString()
+{
+    WriteBufferFromOwnString wb;
+    serialize(wb);
+    return wb.str();
+}
+
+bool MinimalisticDataPartChecksums::deserialize(ReadBuffer & in)
+{
+    assertString("checksums format version: ", in);
+    size_t format_version;
+    readText(format_version, in);
+    assertChar('\n', in);
+
+    if (format_version < MINIMAL_VERSION_WITH_MINIMALISTIC_CHECKSUMS)
+    {
+        auto full_checksums_ptr = std::make_unique<MergeTreeDataPartChecksums>();
+        if (!full_checksums_ptr->read(in, format_version))
+            return false;
+
+        computeTotalChecksums(*full_checksums_ptr);
+        full_checksums = std::move(full_checksums_ptr);
+        return true;
+    }
+
+    if (format_version > MINIMAL_VERSION_WITH_MINIMALISTIC_CHECKSUMS)
+        throw Exception("Unknown checksums format version: " + DB::toString(format_version), ErrorCodes::UNKNOWN_FORMAT);
+
+    readVarUInt(num_compressed_files, in);
+    readVarUInt(num_uncompressed_files, in);
+
+    readPODBinary(hash_of_all_files, in);
+    readPODBinary(hash_of_uncompressed_files, in);
+    readPODBinary(uncompressed_hash_of_compressed_files, in);
+
+    return true;
+}
+
+void MinimalisticDataPartChecksums::computeTotalChecksums(const MergeTreeDataPartChecksums & full_checksums)
+{
+    num_compressed_files = 0;
+    num_uncompressed_files = 0;
+
+    SipHash hash_of_all_files_;
+    SipHash hash_of_uncompressed_files_;
+    SipHash uncompressed_hash_of_compressed_files_;
+
+    auto update_hash = [] (SipHash & hash, const std::string & data)
+    {
+        UInt64 len = data.size();
+        hash.update(len);
+        hash.update(data.data(), len);
+    };
+
+    for (const auto & elem : full_checksums.files)
+    {
+        const String & name = elem.first;
+        const auto & checksum = elem.second;
+
+        update_hash(hash_of_all_files_, name);
+        hash_of_all_files_.update(checksum.file_hash);
+
+        if (!checksum.is_compressed)
+        {
+            ++num_uncompressed_files;
+            update_hash(hash_of_uncompressed_files_, name);
+            hash_of_uncompressed_files_.update(checksum.file_hash);
+        }
+        else
+        {
+            ++num_compressed_files;
+            update_hash(uncompressed_hash_of_compressed_files_, name);
+            uncompressed_hash_of_compressed_files_.update(checksum.uncompressed_hash);
+        }
+    }
+
+    auto get_hash = [] (SipHash & hash, uint128 & data)
+    {
+        hash.get128(data.first, data.second);
+    };
+
+    get_hash(hash_of_all_files_, hash_of_all_files);
+    get_hash(hash_of_uncompressed_files_, hash_of_uncompressed_files);
+    get_hash(uncompressed_hash_of_compressed_files_, uncompressed_hash_of_compressed_files);
+}
+
+String MinimalisticDataPartChecksums::getSerializedString(const MergeTreeDataPartChecksums & full_checksums, bool minimalistic)
+{
+    if (!minimalistic)
+        return full_checksums.toString();
+
+    MinimalisticDataPartChecksums checksums;
+    checksums.computeTotalChecksums(full_checksums);
+    return checksums.getSerializedString();
+}
+
+void MinimalisticDataPartChecksums::checkEqual(const MinimalisticDataPartChecksums & rhs, bool check_uncompressed_hash_in_compressed_files)
+{
+    if (full_checksums && rhs.full_checksums)
+        full_checksums->checkEqual(*rhs.full_checksums, check_uncompressed_hash_in_compressed_files);
+
+    checkEqualImpl(rhs, check_uncompressed_hash_in_compressed_files);
+}
+
+void MinimalisticDataPartChecksums::checkEqualImpl(const MinimalisticDataPartChecksums & rhs, bool check_uncompressed_hash_in_compressed_files)
+{
+    if (num_compressed_files != rhs.num_compressed_files || num_uncompressed_files != rhs.num_uncompressed_files)
+    {
+        std::stringstream error_msg;
+        error_msg << "Different number of files: " << rhs.num_compressed_files << " compressed (expected " << num_compressed_files << ")"
+            << " and " << rhs.num_uncompressed_files << " uncompressed ones (expected " << num_uncompressed_files << ")";
+
+        throw Exception(error_msg.str(), ErrorCodes::CHECKSUM_DOESNT_MATCH);
+    }
+
+    Strings errors;
+
+    if (hash_of_uncompressed_files != rhs.hash_of_uncompressed_files)
+        errors.emplace_back("hash of uncompressed files doesn't match");
+
+    if (check_uncompressed_hash_in_compressed_files)
+    {
+        if (uncompressed_hash_of_compressed_files != rhs.uncompressed_hash_of_compressed_files)
+            errors.emplace_back("uncompressed hash of compressed files doesn't match");
+    }
+    else
+    {
+        if (hash_of_all_files != rhs.hash_of_all_files)
+            errors.emplace_back("total hash of all files doesn't match");
+    }
+
+    if (!errors.empty())
+    {
+        String error_msg = "Checksums of parts don't match: " + errors.front();
+        for (size_t i = 1; i < errors.size(); ++i)
+            error_msg += ", " + errors[i];
+
+        throw Exception(error_msg, ErrorCodes::CHECKSUM_DOESNT_MATCH);
+    }
+}
+
+MinimalisticDataPartChecksums MinimalisticDataPartChecksums::deserializeFrom(const String & s)
+{
+    MinimalisticDataPartChecksums res;
+    ReadBufferFromString rb(s);
+    res.deserialize(rb);
+    return res;
+}
+
+}
diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h
new file mode 100644
index 00000000000..ac2e034b55e
--- /dev/null
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h
@@ -0,0 +1,119 @@
+#pragma once
+#include <Core/Types.h>
+#include <IO/ReadBuffer.h>
+#include <IO/WriteBuffer.h>
+#include <city.h>
+#include <map>
+#include <memory>
+
+
+class SipHash;
+
+
+namespace DB
+{
+
+/// Checksum of one file.
+struct MergeTreeDataPartChecksum
+{
+    using uint128 = CityHash_v1_0_2::uint128;
+
+    UInt64 file_size {};
+    uint128 file_hash {};
+
+    bool is_compressed = false;
+    UInt64 uncompressed_size {};
+    uint128 uncompressed_hash {};
+
+    MergeTreeDataPartChecksum() {}
+    MergeTreeDataPartChecksum(UInt64 file_size_, uint128 file_hash_) : file_size(file_size_), file_hash(file_hash_) {}
+    MergeTreeDataPartChecksum(UInt64 file_size_, uint128 file_hash_, UInt64 uncompressed_size_, uint128 uncompressed_hash_)
+        : file_size(file_size_), file_hash(file_hash_), is_compressed(true),
+        uncompressed_size(uncompressed_size_), uncompressed_hash(uncompressed_hash_) {}
+
+    void checkEqual(const MergeTreeDataPartChecksum & rhs, bool have_uncompressed, const String & name) const;
+    void checkSize(const String & path) const;
+};
+
+
+/** Checksums of all non-temporary files.
+  * For compressed files, the check sum and the size of the decompressed data are stored to not depend on the compression method.
+  */
+struct MergeTreeDataPartChecksums
+{
+    using Checksum = MergeTreeDataPartChecksum;
+
+    /// The order is important.
+    using FileChecksums = std::map<String, Checksum>;
+    FileChecksums files;
+
+    void addFile(const String & file_name, UInt64 file_size, Checksum::uint128 file_hash);
+
+    void add(MergeTreeDataPartChecksums && rhs_checksums);
+
+    bool empty() const
+    {
+        return files.empty();
+    }
+
+    /// Checks that the set of columns and their checksums are the same. If not, throws an exception.
+    /// If have_uncompressed, for compressed files it compares the checksums of the decompressed data.
+    /// Otherwise, it compares only the checksums of the files.
+    void checkEqual(const MergeTreeDataPartChecksums & rhs, bool have_uncompressed) const;
+
+    /// Checks that the directory contains all the needed files of the correct size. Does not check the checksum.
+    void checkSizes(const String & path) const;
+
+    /// Returns false if the checksum is too old.
+    bool read(ReadBuffer & in);
+    /// Assume that header with version (the first line) is read
+    bool read(ReadBuffer & in, size_t format_version);
+    bool read_v2(ReadBuffer & in);
+    bool read_v3(ReadBuffer & in);
+    bool read_v4(ReadBuffer & in);
+
+    void write(WriteBuffer & out) const;
+    String toString() const;
+
+    /// Checksum from the set of checksums of .bin files (for deduplication).
+    void computeTotalChecksumDataOnly(SipHash & hash) const;
+
+    static MergeTreeDataPartChecksums deserializeFrom(const String & s);
+};
+
+
+/// A kind of MergeTreeDataPartChecksums intended to be stored in ZooKeeper (to save its RAM)
+/// MinimalisticDataPartChecksums and MergeTreeDataPartChecksums hasve the same serialization format
+///  for versions less MINIMAL_VERSION_WITH_MINIMALISTIC_CHECKSUMS.
+struct MinimalisticDataPartChecksums
+{
+    UInt64 num_compressed_files = 0;
+    UInt64 num_uncompressed_files = 0;
+
+    using uint128 = MergeTreeDataPartChecksum::uint128;
+    uint128 hash_of_all_files {};
+    uint128 hash_of_uncompressed_files {};
+    uint128 uncompressed_hash_of_compressed_files {};
+
+    /// Is set only for old formats
+    std::unique_ptr<MergeTreeDataPartChecksums> full_checksums;
+
+    static constexpr size_t MINIMAL_VERSION_WITH_MINIMALISTIC_CHECKSUMS = 5;
+
+    MinimalisticDataPartChecksums() = default;
+    explicit MinimalisticDataPartChecksums(const MergeTreeDataPartChecksums & full_checksums);
+    void computeTotalChecksums(const MergeTreeDataPartChecksums & full_checksums);
+
+    bool deserialize(ReadBuffer & in);
+    static MinimalisticDataPartChecksums deserializeFrom(const String & s);
+
+    void serialize(WriteBuffer & to) const;
+    String getSerializedString();
+    static String getSerializedString(const MergeTreeDataPartChecksums & full_checksums, bool minimalistic);
+
+    void checkEqual(const MinimalisticDataPartChecksums & rhs, bool check_uncompressed_hash_in_compressed_files);
+    void checkEqualImpl(const MinimalisticDataPartChecksums & rhs, bool check_uncompressed_hash_in_compressed_files);
+};
+
+
+}
diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp
index 1d5ec76c652..14020e7b9d2 100644
--- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp
+++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp
@@ -133,7 +133,7 @@ void ReplicatedMergeTreeBlockOutputStream::write(const Block & block)
         if (deduplicate)
         {
             SipHash hash;
-            part->checksums.summaryDataChecksum(hash);
+            part->checksums.computeTotalChecksumDataOnly(hash);
             union
             {
                 char bytes[16];

From fe8dfd303807dfae4d11cffe7966110e2a2db787 Mon Sep 17 00:00:00 2001
From: Vitaliy Lyudvichenko <vludv@yandex-team.ru>
Date: Thu, 22 Mar 2018 02:30:20 +0300
Subject: [PATCH 172/244] Add minimalistic checksums. [#CLICKHOUSE-3660]

---
 .../MergeTree/MergeTreeDataPartChecksum.cpp   |  24 ++-
 .../MergeTree/MergeTreeDataPartChecksum.h     |   6 +-
 .../Storages/MergeTree/MergeTreeSettings.h    |   6 +
 .../ReplicatedMergeTreeAlterThread.cpp        |   4 +-
 .../ReplicatedMergeTreeBlockOutputStream.cpp  |   2 +-
 .../ReplicatedMergeTreePartCheckThread.cpp    |   2 +-
 .../Storages/StorageReplicatedMergeTree.cpp   | 165 +++++++++++-------
 .../src/Storages/StorageReplicatedMergeTree.h |  12 +-
 ...eper_different_checksums_formats.reference |   9 +
 ..._zookeeper_different_checksums_formats.sql |  27 +++
 10 files changed, 183 insertions(+), 74 deletions(-)
 create mode 100644 dbms/tests/queries/0_stateless/00611_zookeeper_different_checksums_formats.reference
 create mode 100644 dbms/tests/queries/0_stateless/00611_zookeeper_different_checksums_formats.sql

diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp
index b252830d0d1..b9be8e3d9f0 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp
@@ -247,7 +247,7 @@ void MergeTreeDataPartChecksums::computeTotalChecksumDataOnly(SipHash & hash) co
     }
 }
 
-String MergeTreeDataPartChecksums::toString() const
+String MergeTreeDataPartChecksums::getSerializedString() const
 {
     WriteBufferFromOwnString out;
     write(out);
@@ -264,10 +264,12 @@ MergeTreeDataPartChecksums MergeTreeDataPartChecksums::deserializeFrom(const Str
     return res;
 }
 
-MinimalisticDataPartChecksums::MinimalisticDataPartChecksums(const MergeTreeDataPartChecksums & full_checksums_)
+bool MergeTreeDataPartChecksums::isBadChecksumsErrorCode(int code)
 {
-    full_checksums = std::make_unique<MergeTreeDataPartChecksums>(full_checksums_);
-    computeTotalChecksums(*full_checksums);
+    return code == ErrorCodes::CHECKSUM_DOESNT_MATCH
+           || code == ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART
+           || code == ErrorCodes::NO_FILE_IN_DATA_PART
+           || code == ErrorCodes::UNEXPECTED_FILE_IN_DATA_PART;
 }
 
 void MinimalisticDataPartChecksums::serialize(WriteBuffer & to) const
@@ -371,7 +373,7 @@ void MinimalisticDataPartChecksums::computeTotalChecksums(const MergeTreeDataPar
 String MinimalisticDataPartChecksums::getSerializedString(const MergeTreeDataPartChecksums & full_checksums, bool minimalistic)
 {
     if (!minimalistic)
-        return full_checksums.toString();
+        return full_checksums.getSerializedString();
 
     MinimalisticDataPartChecksums checksums;
     checksums.computeTotalChecksums(full_checksums);
@@ -383,9 +385,21 @@ void MinimalisticDataPartChecksums::checkEqual(const MinimalisticDataPartChecksu
     if (full_checksums && rhs.full_checksums)
         full_checksums->checkEqual(*rhs.full_checksums, check_uncompressed_hash_in_compressed_files);
 
+    // If full checksums were checked, check total checksums just in case
     checkEqualImpl(rhs, check_uncompressed_hash_in_compressed_files);
 }
 
+void MinimalisticDataPartChecksums::checkEqual(const MergeTreeDataPartChecksums & rhs, bool check_uncompressed_hash_in_compressed_files)
+{
+    if (full_checksums)
+        full_checksums->checkEqual(rhs, check_uncompressed_hash_in_compressed_files);
+
+    // If full checksums were checked, check total checksums just in case
+    MinimalisticDataPartChecksums rhs_minimalistic;
+    rhs_minimalistic.computeTotalChecksums(rhs);
+    checkEqualImpl(rhs_minimalistic, check_uncompressed_hash_in_compressed_files);
+}
+
 void MinimalisticDataPartChecksums::checkEqualImpl(const MinimalisticDataPartChecksums & rhs, bool check_uncompressed_hash_in_compressed_files)
 {
     if (num_compressed_files != rhs.num_compressed_files || num_uncompressed_files != rhs.num_uncompressed_files)
diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h
index ac2e034b55e..797c8929b58 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h
@@ -61,6 +61,8 @@ struct MergeTreeDataPartChecksums
     /// Otherwise, it compares only the checksums of the files.
     void checkEqual(const MergeTreeDataPartChecksums & rhs, bool have_uncompressed) const;
 
+    static bool isBadChecksumsErrorCode(int code);
+
     /// Checks that the directory contains all the needed files of the correct size. Does not check the checksum.
     void checkSizes(const String & path) const;
 
@@ -73,11 +75,11 @@ struct MergeTreeDataPartChecksums
     bool read_v4(ReadBuffer & in);
 
     void write(WriteBuffer & out) const;
-    String toString() const;
 
     /// Checksum from the set of checksums of .bin files (for deduplication).
     void computeTotalChecksumDataOnly(SipHash & hash) const;
 
+    String getSerializedString() const;
     static MergeTreeDataPartChecksums deserializeFrom(const String & s);
 };
 
@@ -101,7 +103,6 @@ struct MinimalisticDataPartChecksums
     static constexpr size_t MINIMAL_VERSION_WITH_MINIMALISTIC_CHECKSUMS = 5;
 
     MinimalisticDataPartChecksums() = default;
-    explicit MinimalisticDataPartChecksums(const MergeTreeDataPartChecksums & full_checksums);
     void computeTotalChecksums(const MergeTreeDataPartChecksums & full_checksums);
 
     bool deserialize(ReadBuffer & in);
@@ -112,6 +113,7 @@ struct MinimalisticDataPartChecksums
     static String getSerializedString(const MergeTreeDataPartChecksums & full_checksums, bool minimalistic);
 
     void checkEqual(const MinimalisticDataPartChecksums & rhs, bool check_uncompressed_hash_in_compressed_files);
+    void checkEqual(const MergeTreeDataPartChecksums & rhs, bool check_uncompressed_hash_in_compressed_files);
     void checkEqualImpl(const MinimalisticDataPartChecksums & rhs, bool check_uncompressed_hash_in_compressed_files);
 };
 
diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.h b/dbms/src/Storages/MergeTree/MergeTreeSettings.h
index f91ca880a66..424c634afaf 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeSettings.h
+++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.h
@@ -131,6 +131,12 @@ struct MergeTreeSettings
       *  for backward compatibility.                                                                          \
       */                                                                                                      \
     M(SettingBool, compatibility_allow_sampling_expression_not_in_primary_key, false)                         \
+                                                                                                              \
+    /** Use small format (dozens bytes) for part checksums in ZooKeeper                                       \
+      *  instead of ordinary ones (dozens KB).                                                                \
+      * Before enabling check that all replicas support new format.                                           \
+      */                                                                                                      \
+    M(SettingBool, use_minimalistic_checksums_in_zookeeper, false)
 
     /// Settings that should not change after the creation of a table.
 #define APPLY_FOR_IMMUTABLE_MERGE_TREE_SETTINGS(M)  \
diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp
index 72d7cd91e3a..f0513c04f9e 100644
--- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp
+++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp
@@ -144,7 +144,9 @@ void ReplicatedMergeTreeAlterThread::run()
                         ops.emplace_back(std::make_shared<zkutil::Op::SetData>(
                             storage.replica_path + "/parts/" + part->name + "/columns", transaction->getNewColumns().toString(), -1));
                         ops.emplace_back(std::make_shared<zkutil::Op::SetData>(
-                            storage.replica_path + "/parts/" + part->name + "/checksums", transaction->getNewChecksums().toString(), -1));
+                            storage.replica_path + "/parts/" + part->name + "/checksums",
+                            storage.getChecksumsForZooKeeper(transaction->getNewChecksums()),
+                            -1));
 
                         try
                         {
diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp
index 14020e7b9d2..52bc1c4be3f 100644
--- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp
+++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp
@@ -306,7 +306,7 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo
         zkutil::CreateMode::Persistent));
     ops.emplace_back(std::make_shared<zkutil::Op::Create>(
         storage.replica_path + "/parts/" + part->name + "/checksums",
-        part->checksums.toString(),
+        storage.getChecksumsForZooKeeper(part->checksums),
         acl,
         zkutil::CreateMode::Persistent));
 
diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp
index f64826f6266..85e58f4551b 100644
--- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp
+++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp
@@ -237,7 +237,7 @@ void ReplicatedMergeTreePartCheckThread::checkPart(const String & part_name)
 
             try
             {
-                auto zk_checksums = MergeTreeData::DataPart::Checksums::parse(
+                auto zk_checksums = MinimalisticDataPartChecksums::deserializeFrom(
                     zookeeper->get(storage.replica_path + "/parts/" + part_name + "/checksums"));
                 zk_checksums.checkEqual(part->checksums, true);
 
diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp
index 70326cddd4a..006c2fd63a1 100644
--- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp
@@ -838,7 +838,7 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks)
         LOG_ERROR(log, "Adding unexpected local part to ZooKeeper: " << part->name);
 
         zkutil::Ops ops;
-        checkPartAndAddToZooKeeper(part, ops);
+        checkPartChecksumsAndAddCommitOps(zookeeper, part, ops);
         zookeeper->multi(ops);
     }
 
@@ -878,11 +878,9 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks)
 }
 
 
-void StorageReplicatedMergeTree::checkPartAndAddToZooKeeper(
-    const MergeTreeData::DataPartPtr & part, zkutil::Ops & ops, String part_name)
+void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil::ZooKeeperPtr & zookeeper,
+    const MergeTreeData::DataPartPtr & part, zkutil::Ops & ops, String part_name, NameSet * absent_replicas_paths)
 {
-    auto zookeeper = getZooKeeper();
-
     if (part_name.empty())
         part_name = part->name;
 
@@ -892,24 +890,34 @@ void StorageReplicatedMergeTree::checkPartAndAddToZooKeeper(
     Strings replicas = zookeeper->getChildren(zookeeper_path + "/replicas");
     std::shuffle(replicas.begin(), replicas.end(), rng);
     String expected_columns_str = part->columns.toString();
+    bool has_been_alredy_added = false;
 
     for (const String & replica : replicas)
     {
         zkutil::Stat stat_before, stat_after;
+        String current_part_path = zookeeper_path + "/replicas/" + replica + "/parts/" + part_name;
+
         String columns_str;
-        if (!zookeeper->tryGet(zookeeper_path + "/replicas/" + replica + "/parts/" + part_name + "/columns", columns_str, &stat_before))
+        if (!zookeeper->tryGet(current_part_path + "/columns", columns_str, &stat_before))
+        {
+            if (absent_replicas_paths)
+                absent_replicas_paths->emplace(current_part_path);
+
             continue;
+        }
+
         if (columns_str != expected_columns_str)
         {
             LOG_INFO(log, "Not checking checksums of part " << part_name << " with replica " << replica
                 << " because columns are different");
             continue;
         }
+
         String checksums_str;
         /// Let's check that the node's version with the columns did not change while we were reading the checksums.
         /// This ensures that the columns and the checksum refer to the same data.
-        if (!zookeeper->tryGet(zookeeper_path + "/replicas/" + replica + "/parts/" + part_name + "/checksums", checksums_str) ||
-            !zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/parts/" + part_name + "/columns", &stat_after) ||
+        if (!zookeeper->tryGet(current_part_path + "/checksums", checksums_str) ||
+            !zookeeper->exists(current_part_path + "/columns", &stat_after) ||
             stat_before.version != stat_after.version)
         {
             LOG_INFO(log, "Not checking checksums of part " << part_name << " with replica " << replica
@@ -917,36 +925,90 @@ void StorageReplicatedMergeTree::checkPartAndAddToZooKeeper(
             continue;
         }
 
-        auto checksums = MergeTreeData::DataPart::Checksums::parse(checksums_str);
-        checksums.checkEqual(part->checksums, true);
+        auto zk_checksums = MinimalisticDataPartChecksums::deserializeFrom(checksums_str);
+        zk_checksums.checkEqual(part->checksums, true);
+
+        if (replica == replica_name)
+            has_been_alredy_added = true;
     }
 
-    if (zookeeper->exists(replica_path + "/parts/" + part_name))
+    if (!has_been_alredy_added)
     {
-        LOG_ERROR(log, "checkPartAndAddToZooKeeper: node " << replica_path + "/parts/" + part_name << " already exists");
-        return;
+        auto acl = zookeeper->getDefaultACL();
+        String part_path = replica_path + "/parts/" + part_name;
+
+        ops.emplace_back(std::make_shared<zkutil::Op::Check>(
+            zookeeper_path + "/columns", expected_columns_version));
+        ops.emplace_back(std::make_shared<zkutil::Op::Create>(
+            part_path, "", acl, zkutil::CreateMode::Persistent));
+        ops.emplace_back(std::make_shared<zkutil::Op::Create>(
+            part_path + "/columns", part->columns.toString(), acl, zkutil::CreateMode::Persistent));
+        ops.emplace_back(std::make_shared<zkutil::Op::Create>(
+            part_path + "/checksums", getChecksumsForZooKeeper(part->checksums), acl, zkutil::CreateMode::Persistent));
     }
+    else
+    {
+        LOG_WARNING(log, "checkPartAndAddToZooKeeper: node " << replica_path + "/parts/" + part_name << " already exists."
+            << " Will not commit any nodes.");
+    }
+}
 
-    auto acl = zookeeper->getDefaultACL();
+MergeTreeData::DataPartsVector StorageReplicatedMergeTree::checkPartChecksumsAndCommit(MergeTreeData::Transaction & transaction,
+    const MergeTreeData::DataPartPtr & part)
+{
+    auto zookeeper = getZooKeeper();
 
-    ops.emplace_back(std::make_shared<zkutil::Op::Check>(
-        zookeeper_path + "/columns",
-        expected_columns_version));
-    ops.emplace_back(std::make_shared<zkutil::Op::Create>(
-        replica_path + "/parts/" + part_name,
-        "",
-        acl,
-        zkutil::CreateMode::Persistent));
-    ops.emplace_back(std::make_shared<zkutil::Op::Create>(
-        replica_path + "/parts/" + part_name + "/columns",
-        part->columns.toString(),
-        acl,
-        zkutil::CreateMode::Persistent));
-    ops.emplace_back(std::make_shared<zkutil::Op::Create>(
-        replica_path + "/parts/" + part_name + "/checksums",
-        part->checksums.toString(),
-        acl,
-        zkutil::CreateMode::Persistent));
+    while (true)
+    {
+        zkutil::Ops ops;
+        NameSet absent_part_paths_on_replicas;
+
+        /// Checksums are checked here and `ops` is filled. In fact, the part is added to ZK just below, when executing `multi`.
+        checkPartChecksumsAndAddCommitOps(zookeeper, part, ops, part->name, &absent_part_paths_on_replicas);
+
+        /// Do not commit if the part is obsolete, we have just briefly checked its checksums
+        if (transaction.isEmpty())
+            return {};
+
+        /// Will check that the part did not suddenly appear on skipped replicas
+        if (!absent_part_paths_on_replicas.empty())
+        {
+            zkutil::Ops new_ops;
+            for (const String & part_path : absent_part_paths_on_replicas)
+            {
+                new_ops.emplace_back(std::make_shared<zkutil::Op::Create>(part_path, "", zookeeper->getDefaultACL(), zkutil::CreateMode::Persistent));
+                new_ops.emplace_back(std::make_shared<zkutil::Op::Remove>(part_path, -1));
+            }
+
+            /// Add check ops at the beginning
+            new_ops.insert(new_ops.end(), ops.begin(), ops.end());
+            ops = std::move(new_ops);
+        }
+
+        try
+        {
+            zookeeper->multi(ops);
+            return transaction.commit();
+        }
+        catch (zkutil::KeeperMultiException & e)
+        {
+            size_t num_check_ops = 2 * absent_part_paths_on_replicas.size();
+            size_t failed_op_index = zkutil::getFailedOpIndex(e.info.op_results, e.info.code);
+
+            if (failed_op_index < num_check_ops && e.info.code == ZNODEEXISTS)
+            {
+                LOG_INFO(log, "The part " << e.info.getFailedOp().describe() << " on a replica suddenly appeared, will recheck checksums");
+            }
+            else
+                throw;
+        }
+    }
+}
+
+String StorageReplicatedMergeTree::getChecksumsForZooKeeper(const MergeTreeDataPartChecksums & checksums)
+{
+    return MinimalisticDataPartChecksums::getSerializedString(checksums,
+                                                              static_cast<bool>(data.settings.use_minimalistic_checksums_in_zookeeper));
 }
 
 
@@ -1171,22 +1233,18 @@ void StorageReplicatedMergeTree::tryExecuteMerge(const StorageReplicatedMergeTre
         part = merger.mergePartsToTemporaryPart(
             future_merged_part, *merge_entry, aio_threshold, entry.create_time, reserved_space.get(), entry.deduplicate);
 
-        zkutil::Ops ops;
+        merger.renameMergedTemporaryPart(part, parts, &transaction);
 
         try
         {
-            /// Checksums are checked here and `ops` is filled. In fact, the part is added to ZK just below, when executing `multi`.
-            checkPartAndAddToZooKeeper(part, ops, entry.new_part_name);
+            checkPartChecksumsAndCommit(transaction, part);
         }
         catch (const Exception & e)
         {
-            if (e.code() == ErrorCodes::CHECKSUM_DOESNT_MATCH
-                || e.code() == ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART
-                || e.code() == ErrorCodes::NO_FILE_IN_DATA_PART
-                || e.code() == ErrorCodes::UNEXPECTED_FILE_IN_DATA_PART)
+            if (MergeTreeDataPartChecksums::isBadChecksumsErrorCode(e.code()))
             {
                 do_fetch = true;
-                part->remove();
+                transaction.rollback();
 
                 ProfileEvents::increment(ProfileEvents::DataAfterMergeDiffersFromReplica);
 
@@ -1210,15 +1268,6 @@ void StorageReplicatedMergeTree::tryExecuteMerge(const StorageReplicatedMergeTre
             throw;
         }
 
-        merger.renameMergedTemporaryPart(part, parts, &transaction);
-
-        /// Do not commit if the part is obsolete
-        if (!transaction.isEmpty())
-        {
-            getZooKeeper()->multi(ops); /// After long merge, get fresh ZK handle, because previous session may be expired.
-            transaction.commit();
-        }
-
         /** Removing old chunks from ZK and from the disk is delayed - see ReplicatedMergeTreeCleanupThread, clearOldParts.
           */
 
@@ -1526,7 +1575,7 @@ void StorageReplicatedMergeTree::executeClearColumnInPartition(const LogEntry &
         ops.emplace_back(std::make_shared<zkutil::Op::SetData>(
             replica_path + "/parts/" + part->name + "/columns", transaction->getNewColumns().toString(), -1));
         ops.emplace_back(std::make_shared<zkutil::Op::SetData>(
-            replica_path + "/parts/" + part->name + "/checksums", transaction->getNewChecksums().toString(), -1));
+            replica_path + "/parts/" + part->name + "/checksums", getChecksumsForZooKeeper(transaction->getNewChecksums()), -1));
 
         zookeeper->multi(ops);
 
@@ -2269,24 +2318,15 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Strin
 
         if (!to_detached)
         {
-            zkutil::Ops ops;
+            MergeTreeData::Transaction transaction;
+            data.renameTempPartAndReplace(part, nullptr, &transaction);
 
             /** NOTE
               * Here, an error occurs if ALTER occurred with a change in the column type or column deletion,
               *  and the part on remote server has not yet been modified.
               * After a while, one of the following attempts to make `fetchPart` succeed.
               */
-            checkPartAndAddToZooKeeper(part, ops, part_name);
-
-            MergeTreeData::Transaction transaction;
-            data.renameTempPartAndReplace(part, nullptr, &transaction);
-
-            /// Do not commit if the part is obsolete
-            if (!transaction.isEmpty())
-            {
-                getZooKeeper()->multi(ops);
-                replaced_parts = transaction.commit();
-            }
+            replaced_parts = checkPartChecksumsAndCommit(transaction, part);
 
             /** If a quorum is tracked for this part, you must update it.
               * If you do not have time, in case of losing the session, when you restart the server - see the `ReplicatedMergeTreeRestartingThread::updateQuorumIfWeHavePart` method.
@@ -3731,5 +3771,4 @@ void StorageReplicatedMergeTree::clearBlocksInPartition(
     LOG_TRACE(log, "Deleted " << to_delete_futures.size() << " deduplication block IDs in partition ID " << partition_id);
 }
 
-
 }
diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h
index 4a2193b05db..b16e2538fe5 100644
--- a/dbms/src/Storages/StorageReplicatedMergeTree.h
+++ b/dbms/src/Storages/StorageReplicatedMergeTree.h
@@ -241,6 +241,9 @@ private:
 
     zkutil::LeaderElectionPtr leader_election;
 
+    /// When activated, replica is initialized and startup() method could exit
+    Poco::Event startup_event;
+
     /// Do I need to complete background threads (except restarting_thread)?
     std::atomic<bool> shutdown_called {false};
     Poco::Event shutdown_event;
@@ -317,7 +320,14 @@ private:
       * Adds actions to `ops` that add data about the part into ZooKeeper.
       * Call under TableStructureLock.
       */
-    void checkPartAndAddToZooKeeper(const MergeTreeData::DataPartPtr & part, zkutil::Ops & ops, String name_override = "");
+    void checkPartChecksumsAndAddCommitOps(const zkutil::ZooKeeperPtr & zookeeper, const MergeTreeData::DataPartPtr & part,
+                                           zkutil::Ops & ops, String part_name = "", NameSet * absent_replicas_paths = nullptr);
+
+    String getChecksumsForZooKeeper(const MergeTreeDataPartChecksums & checksums);
+
+    /// Accepts a PreComitted part, atomically checks its checksums with ones on other replicas and commit the part
+    MergeTreeData::DataPartsVector checkPartChecksumsAndCommit(MergeTreeData::Transaction & transaction,
+                                                               const MergeTreeData::DataPartPtr & part);
 
     /// Adds actions to `ops` that remove a part from ZooKeeper.
     void removePartFromZooKeeper(const String & part_name, zkutil::Ops & ops);
diff --git a/dbms/tests/queries/0_stateless/00611_zookeeper_different_checksums_formats.reference b/dbms/tests/queries/0_stateless/00611_zookeeper_different_checksums_formats.reference
new file mode 100644
index 00000000000..da05f197b7e
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00611_zookeeper_different_checksums_formats.reference
@@ -0,0 +1,9 @@
+1
+1
+0	[]
+1	[]
+0	[]
+1	[]
+DETACH
+0	[]
+1	[]
diff --git a/dbms/tests/queries/0_stateless/00611_zookeeper_different_checksums_formats.sql b/dbms/tests/queries/0_stateless/00611_zookeeper_different_checksums_formats.sql
new file mode 100644
index 00000000000..988d4fbd1b3
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00611_zookeeper_different_checksums_formats.sql
@@ -0,0 +1,27 @@
+DROP TABLE IF EXISTS test.table_old;
+DROP TABLE IF EXISTS test.table_new;
+
+CREATE TABLE test.table_old (k UInt64, d Array(String)) ENGINE = ReplicatedMergeTree('/clickhouse/test/tables/checksums_test', 'old') ORDER BY k SETTINGS use_minimalistic_checksums_in_zookeeper=0;
+CREATE TABLE test.table_new (k UInt64, d Array(String)) ENGINE = ReplicatedMergeTree('/clickhouse/test/tables/checksums_test', 'new') ORDER BY k SETTINGS use_minimalistic_checksums_in_zookeeper=1;
+
+-- Workaround to wait registration in leader election to correctly calculate number of active replicas to be able make quorum INSERT
+OPTIMIZE TABLE test.table_new;
+
+SET insert_quorum=2;
+INSERT INTO test.table_old VALUES (0, []);
+SELECT value LIKE '%checksums format version: 4%' FROM system.zookeeper WHERE path='/clickhouse/test/tables/checksums_test/replicas/old/parts/all_0_0_0' AND name = 'checksums';
+
+INSERT INTO test.table_new VALUES (1, []);
+SELECT value LIKE '%checksums format version: 5%' FROM system.zookeeper WHERE path='/clickhouse/test/tables/checksums_test/replicas/new/parts/all_1_1_0' AND name = 'checksums';
+
+OPTIMIZE TABLE test.table_old;
+SELECT * FROM test.table_old ORDER BY k;
+SELECT * FROM test.table_new ORDER BY k;
+
+SELECT 'DETACH';
+DETACH TABLE test.table_old;
+ATTACH TABLE test.table_old (k UInt64, d Array(String)) ENGINE = ReplicatedMergeTree('/clickhouse/test/tables/checksums_test', 'old') ORDER BY k SETTINGS use_minimalistic_checksums_in_zookeeper=1;
+SELECT * FROM test.table_old ORDER BY k;
+
+DROP TABLE IF EXISTS test.table_old;
+DROP TABLE IF EXISTS test.table_new;
\ No newline at end of file

From 883de7a2a6c28304ee5248bb37c0c4bfc1c8243d Mon Sep 17 00:00:00 2001
From: Vitaliy Lyudvichenko <vludv@yandex-team.ru>
Date: Thu, 22 Mar 2018 22:46:01 +0300
Subject: [PATCH 173/244] startup() waits for initialization of
 ReplicatedMergeTreeRestartingThread. [#CLICKHOUSE-2]

---
 .../MergeTree/ReplicatedMergeTreeRestartingThread.cpp      | 7 +++++++
 dbms/src/Storages/StorageReplicatedMergeTree.cpp           | 3 +++
 .../00611_zookeeper_different_checksums_formats.sql        | 3 ---
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp
index f6686c273ce..f8ea02cff9b 100644
--- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp
+++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp
@@ -94,16 +94,22 @@ void ReplicatedMergeTreeRestartingThread::run()
                         /// The exception when you try to zookeeper_init usually happens if DNS does not work. We will try to do it again.
                         tryLogCurrentException(__PRETTY_FUNCTION__);
 
+                        if (first_time)
+                            storage.startup_event.set();
                         wakeup_event.tryWait(retry_period_ms);
                         continue;
                     }
 
                     if (!need_stop && !tryStartup())
                     {
+                        if (first_time)
+                            storage.startup_event.set();
                         wakeup_event.tryWait(retry_period_ms);
                         continue;
                     }
 
+                    if (first_time)
+                        storage.startup_event.set();
                     break;
                 }
 
@@ -150,6 +156,7 @@ void ReplicatedMergeTreeRestartingThread::run()
         }
         catch (...)
         {
+            storage.startup_event.set();
             tryLogCurrentException(__PRETTY_FUNCTION__);
         }
 
diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp
index 006c2fd63a1..f11d9fe28b8 100644
--- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp
@@ -2386,6 +2386,9 @@ void StorageReplicatedMergeTree::startup()
 
     /// In this thread replica will be activated.
     restarting_thread = std::make_unique<ReplicatedMergeTreeRestartingThread>(*this);
+
+    /// Wait while restarting_thread initializes LeaderElection (and so on) or makes first attmept to do it
+    startup_event.wait();
 }
 
 
diff --git a/dbms/tests/queries/0_stateless/00611_zookeeper_different_checksums_formats.sql b/dbms/tests/queries/0_stateless/00611_zookeeper_different_checksums_formats.sql
index 988d4fbd1b3..9076485cfa1 100644
--- a/dbms/tests/queries/0_stateless/00611_zookeeper_different_checksums_formats.sql
+++ b/dbms/tests/queries/0_stateless/00611_zookeeper_different_checksums_formats.sql
@@ -4,9 +4,6 @@ DROP TABLE IF EXISTS test.table_new;
 CREATE TABLE test.table_old (k UInt64, d Array(String)) ENGINE = ReplicatedMergeTree('/clickhouse/test/tables/checksums_test', 'old') ORDER BY k SETTINGS use_minimalistic_checksums_in_zookeeper=0;
 CREATE TABLE test.table_new (k UInt64, d Array(String)) ENGINE = ReplicatedMergeTree('/clickhouse/test/tables/checksums_test', 'new') ORDER BY k SETTINGS use_minimalistic_checksums_in_zookeeper=1;
 
--- Workaround to wait registration in leader election to correctly calculate number of active replicas to be able make quorum INSERT
-OPTIMIZE TABLE test.table_new;
-
 SET insert_quorum=2;
 INSERT INTO test.table_old VALUES (0, []);
 SELECT value LIKE '%checksums format version: 4%' FROM system.zookeeper WHERE path='/clickhouse/test/tables/checksums_test/replicas/old/parts/all_0_0_0' AND name = 'checksums';

From a75d5525e7fbf7de59804b701c983b5fd3aada28 Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Fri, 23 Mar 2018 00:20:20 +0300
Subject: [PATCH 174/244] Update MergeTreeDataPartChecksum.h

---
 dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h
index 797c8929b58..2471d0ef681 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h
@@ -86,7 +86,7 @@ struct MergeTreeDataPartChecksums
 
 /// A kind of MergeTreeDataPartChecksums intended to be stored in ZooKeeper (to save its RAM)
 /// MinimalisticDataPartChecksums and MergeTreeDataPartChecksums hasve the same serialization format
-///  for versions less MINIMAL_VERSION_WITH_MINIMALISTIC_CHECKSUMS.
+///  for versions less than MINIMAL_VERSION_WITH_MINIMALISTIC_CHECKSUMS.
 struct MinimalisticDataPartChecksums
 {
     UInt64 num_compressed_files = 0;

From 62492450180fa37d018a0f454066765b125580b3 Mon Sep 17 00:00:00 2001
From: robot-metrika-test <robot-metrika-test@yandex-team.ru>
Date: Fri, 23 Mar 2018 01:11:43 +0300
Subject: [PATCH 175/244] Auto version update to [54371]

---
 dbms/cmake/version.cmake | 4 ++--
 debian/changelog         | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake
index ad5ad09cce2..990c638bf46 100644
--- a/dbms/cmake/version.cmake
+++ b/dbms/cmake/version.cmake
@@ -1,6 +1,6 @@
 # This strings autochanged from release_lib.sh:
-set(VERSION_DESCRIBE v1.1.54370-testing)
-set(VERSION_REVISION 54370)
+set(VERSION_DESCRIBE v1.1.54371-testing)
+set(VERSION_REVISION 54371)
 # end of autochange
 
 set (VERSION_MAJOR 1)
diff --git a/debian/changelog b/debian/changelog
index 75502e3cc96..2ff28b59456 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,5 @@
-clickhouse (1.1.54370) unstable; urgency=low
+clickhouse (1.1.54371) unstable; urgency=low
 
   * Modified source code
 
- --  <robot-metrika-test@yandex-team.ru>  Fri, 16 Mar 2018 12:01:50 +0300
+ --  <robot-metrika-test@yandex-team.ru>  Fri, 23 Mar 2018 01:11:42 +0300

From 2f468f1ae1345e52567a291df33e4ab6ac9f810c Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Fri, 23 Mar 2018 13:50:20 +0300
Subject: [PATCH 176/244] fix test [#CLICKHOUSE-3663]

---
 dbms/src/Storages/MergeTree/MergeTreeReader.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp
index 37d41749420..8f29c93511d 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp
@@ -500,7 +500,7 @@ void MergeTreeReader::fillMissingColumns(Block & res, const Names & ordered_name
             evaluateMissingDefaults(res, columns, storage.getColumns().defaults, storage.context);
 
         /// sort columns to ensure consistent order among all blocks
-        if (should_sort)
+        if (!never_evaluate_defaults && should_sort)
         {
             Block ordered_block;
 

From c76d1a008d27d4917bf90fd2413863700d94d095 Mon Sep 17 00:00:00 2001
From: Tobias Adamson <tobias@stargazer.com.sg>
Date: Fri, 23 Mar 2018 22:13:09 +0800
Subject: [PATCH 177/244] Make sure to split the topics args list on ','

---
 dbms/src/Storages/StorageKafka.cpp | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/dbms/src/Storages/StorageKafka.cpp b/dbms/src/Storages/StorageKafka.cpp
index bfe72aa016e..d82b12a6fb2 100644
--- a/dbms/src/Storages/StorageKafka.cpp
+++ b/dbms/src/Storages/StorageKafka.cpp
@@ -4,6 +4,7 @@
 
 #include <thread>
 #include <boost/algorithm/string/replace.hpp>
+#include <boost/algorithm/string/split.hpp>
 #include <Common/Exception.h>
 #include <Common/setThreadName.h>
 #include <Common/typeid_cast.h>
@@ -574,9 +575,12 @@ void registerStorageKafka(StorageFactory & factory)
                 throw Exception("Number of consumers must be a positive integer", ErrorCodes::BAD_ARGUMENTS);
         }
 
-        // Parse topic list and consumer group
+        // Parse topic list
         Names topics;
-        topics.push_back(static_cast<const ASTLiteral &>(*engine_args[1]).value.safeGet<String>());
+        String topicArg = static_cast<const ASTLiteral &>(*engine_args[1]).value.safeGet<String>();
+        boost::split(topics, topics , [](char c){return c == ',';});
+        
+        // Parse consumer group
         String group = static_cast<const ASTLiteral &>(*engine_args[2]).value.safeGet<String>();
 
         // Parse format from string

From b87cfd2924a608b64c1d295c1e94ecdf6a323d38 Mon Sep 17 00:00:00 2001
From: proller <proller@github.com>
Date: Thu, 22 Mar 2018 20:32:11 +0300
Subject: [PATCH 178/244] Add tests

---
 dbms/tests/queries/0_stateless/00534_filimonov.data    |  3 +++
 .../0_stateless/00534_long_functions_bad_arguments7.sh |  2 +-
 .../0_stateless/00534_long_functions_bad_arguments8.sh | 10 ++++++++++
 .../0_stateless/00534_long_functions_bad_arguments9.sh | 10 ++++++++++
 4 files changed, 24 insertions(+), 1 deletion(-)
 create mode 100755 dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments8.sh
 create mode 100755 dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments9.sh

diff --git a/dbms/tests/queries/0_stateless/00534_filimonov.data b/dbms/tests/queries/0_stateless/00534_filimonov.data
index 53c64592d0a..2dd470403c0 100644
--- a/dbms/tests/queries/0_stateless/00534_filimonov.data
+++ b/dbms/tests/queries/0_stateless/00534_filimonov.data
@@ -427,3 +427,6 @@ SELECT COVAR_POPArray([CAST( 0  AS Int8)],arrayPopBack([CAST( 0  AS Int8)]));
 SELECT COVAR_SAMPArray([CAST( 0  AS Int8)],arrayPopBack([CAST( 0  AS Int8)]));
 SELECT medianTimingWeightedArray([CAST( 0  AS Int8)],arrayPopBack([CAST( 0  AS Int8)]));
 SELECT quantilesDeterministicArray([CAST( 0  AS Int8)],arrayPopBack([CAST( 0  AS Int32)]));
+
+SELECT maxIntersections([], [])
+SELECT sumMap([], [])
diff --git a/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments7.sh b/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments7.sh
index 78b9afafe55..2cc422cdf9b 100755
--- a/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments7.sh
+++ b/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments7.sh
@@ -5,4 +5,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 . $CURDIR/00534_long_functions_bad_arguments.lib
 
-test_variant 'SELECT $_('');'
+test_variant 'SELECT $_(\'\');'
diff --git a/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments8.sh b/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments8.sh
new file mode 100755
index 00000000000..f884a926c3b
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments8.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+. $CURDIR/../shell_config.sh
+
+. $CURDIR/00534_long_functions_bad_arguments.lib
+
+test_variant 'SELECT $_(NULL, NULL);'
+test_variant 'SELECT $_([], []);'
+test_variant 'SELECT $_([NULL],[NULL]);'
diff --git a/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments9.sh b/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments9.sh
new file mode 100755
index 00000000000..bc1ceda33fa
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments9.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+. $CURDIR/../shell_config.sh
+
+. $CURDIR/00534_long_functions_bad_arguments.lib
+
+test_variant 'SELECT $_(NULL, NULL, NULL);'
+test_variant 'SELECT $_([], [], []);'
+test_variant 'SELECT $_([NULL], [NULL], [NULL]);'

From 1619189ed5dbbec429fcd18d4e3893d2158722d7 Mon Sep 17 00:00:00 2001
From: Artemeey <artemeey@mail.ru>
Date: Tue, 20 Mar 2018 11:41:26 +0300
Subject: [PATCH 179/244] Update settings.md

---
 docs/ru/operations/settings/settings.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md
index 624dadcf096..92e9c2cccae 100644
--- a/docs/ru/operations/settings/settings.md
+++ b/docs/ru/operations/settings/settings.md
@@ -340,6 +340,10 @@ ClickHouse применяет настройку в том случае, ког
 
 ## strict_insert_defaults
 
+```
+Настройка удалена в версии 1.1.54362
+```
+
 Строгое присвоение значений по умолчанию при добавлении данных.
 
 Если при выполнении запроса [INSERT](../../query_language/queries.md#queries-insert) данные для столбца не заданы, то ClickHouse присваивает полям значения по умолчанию. Значения по умолчанию определяются свойством `DEFAULT` для каждого столбца в настройках таблицы. Если для столбца не определен `DEFAULT`, то когда:

From 2590e3b0a6f68448d3481d3fe5214cdd0e61ea3d Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Fri, 23 Mar 2018 18:37:37 +0300
Subject: [PATCH 180/244] Update settings.md

---
 docs/ru/operations/settings/settings.md | 16 ----------------
 1 file changed, 16 deletions(-)

diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md
index 92e9c2cccae..62d183234a4 100644
--- a/docs/ru/operations/settings/settings.md
+++ b/docs/ru/operations/settings/settings.md
@@ -335,19 +335,3 @@ ClickHouse применяет настройку в том случае, ког
 ## output_format_json_quote_64bit_integers
 
 Если значение истинно, то при использовании JSON\* форматов UInt64 и Int64 числа выводятся в кавычках (из соображений совместимости с большинством реализаций JavaScript), иначе - без кавычек.
-
-<a name="settings-strict_insert_defaults"></a>
-
-## strict_insert_defaults
-
-```
-Настройка удалена в версии 1.1.54362
-```
-
-Строгое присвоение значений по умолчанию при добавлении данных.
-
-Если при выполнении запроса [INSERT](../../query_language/queries.md#queries-insert) данные для столбца не заданы, то ClickHouse присваивает полям значения по умолчанию. Значения по умолчанию определяются свойством `DEFAULT` для каждого столбца в настройках таблицы. Если для столбца не определен `DEFAULT`, то когда:
-
--   `strict_insert_defaults=0` - полям столбца присваиваются нули и пустые строки.
--   `strict_insert_defaults=1` - ClickHouse генерирует исключение и обязывает пользователя передать данные в столбец.
-

From 723141c19b92b9f4e72e9c430c0ad6cb28146414 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 23 Mar 2018 18:39:33 +0300
Subject: [PATCH 181/244] Removed setting "strict_insert_defaults" from the en
 docs #2079

---
 docs/en/operations/settings/settings.md | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index a87328cb06c..e006f302c68 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -338,15 +338,3 @@ It works for JSONEachRow and TSKV formats.
 ## output_format_json_quote_64bit_integers
 
 If the value is true, integers appear in quotes when using JSON\* Int64 and UInt64 formats  (for compatibility with most JavaScript implementations); otherwise, integers are output without the quotes.
-
-<a name="settings-strict_insert_defaults"></a>
-
-## strict_insert_defaults
-
-Strictly assigns default values when adding data.
-
-If data is not specified for a column when running an [ INSERT](../../query_language/queries.md#queries-insert)  query, ClickHouse assigns default values to the fields. Default values are defined in the ` DEFAULT` property for each column in the table settings. If ` DEFAULT` is not defined for a column:
-
-- `If strict_insert_defaults = 0`, column fields are assigned zeros and empty strings.
-- `If strict_insert_defaults = 1`, ClickHouse throws an exception and requires the user to pass data to the column.
-

From a6c2e43bd25016105857c6558bc52e35ac6efeb0 Mon Sep 17 00:00:00 2001
From: Alexander Marshalov <_@marshalov.org>
Date: Fri, 23 Mar 2018 00:31:25 +0700
Subject: [PATCH 182/244] The lost information in the Kafka engine
 documentation has been restored, modified and translated (from commit:
 https://github.com/yandex/ClickHouse/pull/1654/files#diff-c9ba89e2414c83b5dcc579233c5c80b5)

---
 docs/en/table_engines/kafka.md | 23 ++++++++++++++++++++++-
 docs/ru/table_engines/kafka.md | 26 ++++++++++++++++++++++++--
 2 files changed, 46 insertions(+), 3 deletions(-)

diff --git a/docs/en/table_engines/kafka.md b/docs/en/table_engines/kafka.md
index 55d2027309d..85943e44fc5 100644
--- a/docs/en/table_engines/kafka.md
+++ b/docs/en/table_engines/kafka.md
@@ -9,7 +9,7 @@ Kafka lets you:
 - Process streams as they become available.
 
 ```
-Kafka(broker_list, topic_list, group_name, format[, schema])
+Kafka(broker_list, topic_list, group_name, format[, schema, num_consumers])
 ```
 
 Parameters:
@@ -19,6 +19,7 @@ Parameters:
 - `group_name` – A group of Kafka consumers (`group1`). Reading margins are tracked for each group separately. If you don't want messages to be duplicated in the cluster, use the same group name everywhere.
 - `format` – Message format. Uses the same notation as the SQL ` FORMAT` function, such as ` JSONEachRow`. For more information, see the section "Formats".
 - `schema` – An optional parameter that must be used if the format requires a schema definition. For example, [Cap'n Proto](https://capnproto.org/)  requires the path to the schema file and the name of the root ` schema.capnp:Message` object.
+- `num_consumers` - Number of created consumers per engine. By default `1`. Create more consumers if the throughput of a single consumer is insufficient. The total number of consumers shouldn't exceed the number of partitions in given topic, as there can be at most 1 consumers assigned to any single partition.
 
 Example:
 
@@ -77,3 +78,23 @@ To stop receiving topic data or to change the conversion logic, detach the mater
 
 If you want to change the target table by using `ALTER` materialized view, we recommend disabling the material view to avoid discrepancies between the target table and the data from the view.
 
+
+## Configuration
+
+Similarly to GraphiteMergeTree, Kafka engine supports extended configuration through the ClickHouse config file. There are two configuration keys you can use - global (`kafka`), and per-topic (`kafka_topic_*`). The global configuration is applied first, then per-topic configuration (if exists).
+
+```xml
+  <!--  Global configuration options for all tables of Kafka engine type -->
+  <kafka>
+    <debug>cgrp</debug>
+    <auto_offset_reset>smallest</auto_offset_reset>
+  </kafka>
+
+  <!-- Configuration specific for topic "logs" -->
+  <kafka_topic_logs>
+    <retry_backoff_ms>250</retry_backoff_ms>
+    <fetch_min_bytes>100000</fetch_min_bytes>
+  </kafka_topic_logs>
+```
+
+See [librdkafka configuration reference](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md) for the list of possible configuration options. Use underscores instead of dots in the ClickHouse configuration, for example `check.crcs=true` would correspond to `<check_crcs>true</check_crcs>`.
diff --git a/docs/ru/table_engines/kafka.md b/docs/ru/table_engines/kafka.md
index 127820a8ef2..8225192e337 100644
--- a/docs/ru/table_engines/kafka.md
+++ b/docs/ru/table_engines/kafka.md
@@ -9,7 +9,7 @@ Kafka позволяет:
 - Обрабатывать потоки по мере их появления.
 
 ```
-Kafka(broker_list, topic_list, group_name, format[, schema])
+Kafka(broker_list, topic_list, group_name, format[, schema, num_consumers])
 ```
 
 Параметры:
@@ -19,6 +19,7 @@ Kafka(broker_list, topic_list, group_name, format[, schema])
 - `group_name` - Группа потребителя Kafka (`group1`). Отступы для чтения отслеживаются для каждой группы отдельно. Если необходимо, чтобы сообщения не повторялись на кластере, используйте везде одно имя группы.
 - `format` - Формат сообщений. Имеет те же обозначения, что выдает SQL-выражение `FORMAT`, например, `JSONEachRow`. Подробнее смотрите в разделе "Форматы".
 - `schema` - Опциональный параметр, необходимый, если используется формат, требующий определения схемы. Например, [Cap'n Proto](https://capnproto.org/) требует путь к файлу со схемой и название корневого объекта `schema.capnp:Message`.
+- `num_consumers` - Количество потребителей (consumer) на таблицу. По умолчанию `1`. Укажите больше потребителей, если пропускная способность одного потребителя недостаточна. Общее число потребителей не должно превышать количество партиций в топике, так как на одну партицию может быть назначено не более одного потребителя.
 
 Пример:
 
@@ -75,4 +76,25 @@ Kafka(broker_list, topic_list, group_name, format[, schema])
   ATTACH MATERIALIZED VIEW consumer;
 ```
 
-Если необходимо изменить целевую таблицу с помощью `ALTER`, то материализованное представление рекомендуется отключить, чтобы избежать несостыковки между целевой таблицей и данными от представления.
\ No newline at end of file
+Если необходимо изменить целевую таблицу с помощью `ALTER`, то материализованное представление рекомендуется отключить, чтобы избежать несостыковки между целевой таблицей и данными от представления.
+
+
+## Конфигурация
+
+Аналогично GraphiteMergeTree, движок Kafka поддерживает расширенную конфигурацию с помощью конфигурационного файла ClickHouse. Существует два конфигурационных ключа, которые можно использовать - глобальный (`kafka`) и по топикам (`kafka_topic_*`). Сначала применяется глобальная конфигурация, затем конфигурация по топикам (если она существует).
+
+```xml
+  <!--  Global configuration options for all tables of Kafka engine type -->
+  <kafka>
+    <debug>cgrp</debug>
+    <auto_offset_reset>smallest</auto_offset_reset>
+  </kafka>
+
+  <!-- Configuration specific for topic "logs" -->
+  <kafka_topic_logs>
+    <retry_backoff_ms>250</retry_backoff_ms>
+    <fetch_min_bytes>100000</fetch_min_bytes>
+  </kafka_topic_logs>
+```
+
+В документе [librdkafka configuration reference](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md) можно увидеть список возможных опций конфигурации. Используйте подчёркивания (`_`) вместо точек в конфигурации ClickHouse, например, `check.crcs=true` будет соответствовать `<check_crcs>true</check_crcs>`.
\ No newline at end of file

From 06d71a9493dd2a831e297323242c0278a11c34fb Mon Sep 17 00:00:00 2001
From: zhang2014 <coswde@gmail.com>
Date: Thu, 3 May 2018 08:30:24 +0800
Subject: [PATCH 183/244] ISSUES-218 add test for distributed & case when then

---
 ...609_distributed_with_case_when_then.reference |  6 ++++++
 .../00609_distributed_with_case_when_then.sql    | 16 ++++++++++++++++
 2 files changed, 22 insertions(+)
 create mode 100644 dbms/tests/queries/0_stateless/00609_distributed_with_case_when_then.reference
 create mode 100644 dbms/tests/queries/0_stateless/00609_distributed_with_case_when_then.sql

diff --git a/dbms/tests/queries/0_stateless/00609_distributed_with_case_when_then.reference b/dbms/tests/queries/0_stateless/00609_distributed_with_case_when_then.reference
new file mode 100644
index 00000000000..c5bb9a19a34
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00609_distributed_with_case_when_then.reference
@@ -0,0 +1,6 @@
+hello
+world
+2
+hello
+world
+2
diff --git a/dbms/tests/queries/0_stateless/00609_distributed_with_case_when_then.sql b/dbms/tests/queries/0_stateless/00609_distributed_with_case_when_then.sql
new file mode 100644
index 00000000000..8d7568415a2
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00609_distributed_with_case_when_then.sql
@@ -0,0 +1,16 @@
+DROP TABLE IF EXISTS test.mergetree;
+DROP TABLE IF EXISTS test.distributed;
+
+CREATE TABLE test.mergetree (x UInt64, s String) ENGINE = MergeTree ORDER BY x;
+INSERT INTO test.mergetree VALUES (1, 'hello'), (2, 'world');
+
+SELECT CASE x WHEN 1 THEN 'hello' WHEN 2 THEN 'world' ELSE  'unknow' END FROM test.mergetree;
+SELECT count() AS cnt FROM (SELECT CASE x WHEN 1 THEN 'hello' WHEN 2 THEN 'world' ELSE  'unknow' END FROM test.mergetree);
+
+CREATE TABLE test.distributed AS test.mergetree ENGINE = Distributed(test_shard_localhost, test, mergetree);
+
+SELECT CASE x WHEN 1 THEN 'hello' WHEN 2 THEN 'world' ELSE  'unknow' END FROM test.distributed;
+SELECT count() AS cnt FROM (SELECT CASE x WHEN 1 THEN 'hello' WHEN 2 THEN 'world' ELSE  'unknow' END FROM test.distributed);
+
+DROP TABLE test.mergetree;
+DROP TABLE test.distributed;

From caf83a650e88f5c1ac3da33dfc9409c5314f4554 Mon Sep 17 00:00:00 2001
From: proller <proller@users.noreply.github.com>
Date: Fri, 23 Mar 2018 09:05:14 -0700
Subject: [PATCH 184/244] libs: compiling with msvc2017 (#1996)

* Allow use bundled *ssl library

* fix

* Add submodule

* Fixes

* fix

* fixes

* fixes

* fix

* fix

* update poco

* fix warnings

* fix

* fix

* Build fixes

* Build fixes

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* add bat

* no zookeeper in dbms

* update boost

* fixes

* fixes

* fix

* fix

* fix

* fix

* try fix

* try fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* Better

* fix

* dh verbose

* fix

* dh verbose

* fix

* clean

* Update LocalDate.h

* Update LocalDateTime.h
---
 cmake/find_poco.cmake                         |  3 ++-
 contrib/CMakeLists.txt                        | 20 ++++++++++++-------
 contrib/boost                                 |  2 +-
 contrib/libcityhash/src/config.h              |  4 ++++
 contrib/libfarmhash/CMakeLists.txt            |  4 ++++
 contrib/libtcmalloc/src/config.h              |  4 ++++
 dbms/CMakeLists.txt                           |  5 -----
 dbms/src/Common/AIO.h                         |  2 +-
 dbms/src/Common/ShellCommand.cpp              |  3 +--
 dbms/src/Common/ZooKeeper/CMakeLists.txt      |  6 +++++-
 dbms/src/Common/ZooKeeper/ZooKeeper.h         |  2 +-
 .../ZooKeeper/tests/zkutil_test_commands.cpp  |  2 +-
 dbms/src/Common/localBackup.cpp               |  5 +----
 dbms/src/Common/randomSeed.cpp                |  4 +---
 dbms/src/Common/tests/compact_array.cpp       |  4 +---
 dbms/src/Core/Defines.h                       | 14 +++++++++++--
 dbms/src/Core/Field.h                         | 12 +++++------
 .../DataStreams/PrettyBlockOutputStream.cpp   |  3 +--
 dbms/src/IO/ReadBufferAIO.cpp                 |  2 +-
 dbms/src/IO/ReadBufferAIO.h                   |  3 +--
 dbms/src/IO/ReadBufferFromFileDescriptor.cpp  |  4 ----
 dbms/src/IO/ReadBufferFromFileDescriptor.h    |  3 +--
 dbms/src/IO/WriteBufferAIO.cpp                |  2 +-
 dbms/src/IO/WriteBufferAIO.h                  |  2 +-
 dbms/src/IO/WriteBufferFromFileDescriptor.cpp |  2 +-
 dbms/src/IO/createReadBufferFromFileBase.cpp  |  6 +++---
 dbms/src/IO/createWriteBufferFromFileBase.cpp |  6 +++---
 dbms/src/IO/tests/read_buffer_aio.cpp         |  5 ++---
 dbms/src/Interpreters/ClientInfo.cpp          |  3 +--
 dbms/src/Interpreters/tests/users.cpp         |  4 +---
 dbms/src/Server/Benchmark.cpp                 | 13 +-----------
 dbms/src/Server/Client.cpp                    |  6 +-----
 dbms/src/Server/PerformanceTest.cpp           |  7 +------
 .../tests/remove_symlink_directory.cpp        |  3 +--
 debian/.pbuilderrc                            |  2 +-
 debian/rules                                  |  2 +-
 libs/libcommon/cmake/find_gperftools.cmake    |  2 +-
 libs/libcommon/include/common/DateLUT.h       |  5 ++++-
 libs/libcommon/include/common/LocalDate.h     |  2 +-
 libs/libcommon/include/common/LocalDateTime.h |  2 +-
 libs/libcommon/include/common/likely.h        |  9 +++++++--
 libs/libcommon/include/common/mremap.h        |  3 +++
 libs/libcommon/include/port/ssize_t.h         |  8 ++++++++
 libs/libcommon/include/port/unistd.h          | 11 ++++++++++
 libs/libcommon/src/DateLUT.cpp                |  8 ++++----
 libs/libcommon/src/mremap.cpp                 | 11 ++++++++--
 libs/libdaemon/include/daemon/BaseDaemon.h    |  2 +-
 libs/libpocoext/src/ThreadNumber.cpp          | 10 ++++------
 utils/build/build_msvc2017.bat                | 14 +++++++++++++
 utils/compressor/zstd_test.cpp                |  3 ++-
 utils/iotest/iotest.cpp                       |  9 +--------
 utils/iotest/iotest_aio.cpp                   | 10 ++--------
 utils/iotest/iotest_nonblock.cpp              |  8 +-------
 53 files changed, 155 insertions(+), 136 deletions(-)
 create mode 100644 libs/libcommon/include/port/ssize_t.h
 create mode 100644 libs/libcommon/include/port/unistd.h
 create mode 100644 utils/build/build_msvc2017.bat

diff --git a/cmake/find_poco.cmake b/cmake/find_poco.cmake
index 9051d63ae47..315e27b798f 100644
--- a/cmake/find_poco.cmake
+++ b/cmake/find_poco.cmake
@@ -49,7 +49,8 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY)
         set (Poco_DataODBC_INCLUDE_DIRS "${ClickHouse_SOURCE_DIR}/contrib/poco/Data/ODBC/include/")
     endif ()
 
-    if (OPENSSL_FOUND)
+    # TODO! fix internal ssl
+    if (OPENSSL_FOUND AND NOT USE_INTERNAL_SSL_LIBRARY)
         set (Poco_NetSSL_FOUND 1)
         set (Poco_NetSSL_LIBRARY PocoNetSSL)
         set (Poco_Crypto_LIBRARY PocoCrypto)
diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
index e74e9a5b7db..9015f7e7d36 100644
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@@ -100,13 +100,17 @@ if (USE_INTERNAL_RDKAFKA_LIBRARY)
     mark_as_advanced (ZLIB_INCLUDE_DIR)
 
     if (USE_INTERNAL_SSL_LIBRARY)
-        add_library(bundled-ssl ALIAS ${OPENSSL_SSL_LIBRARY})
-        set (WITH_BUNDLED_SSL 1)
+        if (MAKE_STATIC_LIBRARIES)
+            add_library(bundled-ssl ALIAS ${OPENSSL_SSL_LIBRARY})
+            set (WITH_BUNDLED_SSL 1 CACHE INTERNAL "")
+        else ()
+            set (WITH_SSL 0 CACHE INTERNAL "")
+        endif ()
     endif ()
 
     add_subdirectory (librdkafka)
 
-    if (USE_INTERNAL_SSL_LIBRARY)
+    if (USE_INTERNAL_SSL_LIBRARY AND MAKE_STATIC_LIBRARIES)
         target_include_directories(rdkafka PRIVATE BEFORE ${OPENSSL_INCLUDE_DIR})
     endif ()
     target_include_directories(rdkafka PRIVATE BEFORE ${ZLIB_INCLUDE_DIR})
@@ -127,16 +131,18 @@ endif ()
 
 
 if (USE_INTERNAL_POCO_LIBRARY)
-    set (ALLOW_DUPLICATE_CUSTOM_TARGETS 1)
     set (save_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
     set (save_CMAKE_C_FLAGS ${CMAKE_C_FLAGS})
     set (_save ${ENABLE_TESTS})
     set (ENABLE_TESTS 0)
     set (CMAKE_DISABLE_FIND_PACKAGE_ZLIB 1)
     if (USE_INTERNAL_SSL_LIBRARY)
-        set (DISABLE_INTERNAL_OPENSSL 1)
-        set (ENABLE_NETSSL 0) # TODO!
-        set (ENABLE_CRYPTO 0) # TODO!
+        set (DISABLE_INTERNAL_OPENSSL 1 CACHE INTERNAL "")
+        set (ENABLE_NETSSL 0 CACHE INTERNAL "") # TODO!
+        set (ENABLE_CRYPTO 0 CACHE INTERNAL "") # TODO!
+    endif ()
+    if (MSVC)
+        set (ENABLE_DATA_ODBC 0 CACHE INTERNAL "") # TODO (build fail)
     endif ()
     add_subdirectory (poco)
     unset (CMAKE_DISABLE_FIND_PACKAGE_ZLIB)
diff --git a/contrib/boost b/contrib/boost
index eb5943711e8..5121cc9d037 160000
--- a/contrib/boost
+++ b/contrib/boost
@@ -1 +1 @@
-Subproject commit eb5943711e88d1008583e6ae3720a5489313d02e
+Subproject commit 5121cc9d0375c7b81b24b6087a51684e6cd62ded
diff --git a/contrib/libcityhash/src/config.h b/contrib/libcityhash/src/config.h
index c873732dd87..61c75fca818 100644
--- a/contrib/libcityhash/src/config.h
+++ b/contrib/libcityhash/src/config.h
@@ -5,7 +5,11 @@
 /* #undef AC_APPLE_UNIVERSAL_BUILD */
 
 /* Define to 1 if the compiler supports __builtin_expect. */
+#if _MSC_VER
+#define HAVE_BUILTIN_EXPECT 0
+#else
 #define HAVE_BUILTIN_EXPECT 1
+#endif
 
 /* Define to 1 if you have the <dlfcn.h> header file. */
 #define HAVE_DLFCN_H 1
diff --git a/contrib/libfarmhash/CMakeLists.txt b/contrib/libfarmhash/CMakeLists.txt
index 7781028094d..16471da3f2f 100644
--- a/contrib/libfarmhash/CMakeLists.txt
+++ b/contrib/libfarmhash/CMakeLists.txt
@@ -2,4 +2,8 @@ add_library(farmhash
 	farmhash.cc
 	farmhash.h)
 
+if (MSVC)
+    target_compile_definitions (farmhash PRIVATE FARMHASH_NO_BUILTIN_EXPECT=1)
+endif ()
+
 target_include_directories (farmhash PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
diff --git a/contrib/libtcmalloc/src/config.h b/contrib/libtcmalloc/src/config.h
index 3338309073d..9f9a7a259e5 100644
--- a/contrib/libtcmalloc/src/config.h
+++ b/contrib/libtcmalloc/src/config.h
@@ -13,7 +13,11 @@
 /* #undef ENABLE_SIZED_DELETE */
 
 /* Define to 1 if compiler supports __builtin_expect */
+#if _MSC_VER
+#define HAVE_BUILTIN_EXPECT 0
+#else
 #define HAVE_BUILTIN_EXPECT 1
+#endif
 
 /* Define to 1 if compiler supports __builtin_stack_pointer */
 /* #undef HAVE_BUILTIN_STACK_POINTER */
diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt
index 965b9b2bfe1..d740ce5a45c 100644
--- a/dbms/CMakeLists.txt
+++ b/dbms/CMakeLists.txt
@@ -131,7 +131,6 @@ target_link_libraries (clickhouse_common_io
     ${LINK_LIBRARIES_ONLY_ON_X86_64}
     ${LZ4_LIBRARY}
     ${ZSTD_LIBRARY}
-    ${ZOOKEEPER_LIBRARY}
     ${DOUBLE_CONVERSION_LIBRARIES}
     ${Poco_Net_LIBRARY}
     ${Poco_Data_LIBRARY}
@@ -158,10 +157,6 @@ if (NOT USE_INTERNAL_RE2_LIBRARY)
     target_include_directories (dbms BEFORE PRIVATE ${RE2_INCLUDE_DIR})
 endif ()
 
-if (NOT USE_INTERNAL_ZOOKEEPER_LIBRARY)
-    target_include_directories (clickhouse_common_io BEFORE PUBLIC ${ZOOKEEPER_INCLUDE_DIR})
-endif ()
-
 if (NOT USE_INTERNAL_BOOST_LIBRARY)
     target_include_directories (clickhouse_common_io BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
 endif ()
diff --git a/dbms/src/Common/AIO.h b/dbms/src/Common/AIO.h
index e7287c85d3d..254a7f02ecb 100644
--- a/dbms/src/Common/AIO.h
+++ b/dbms/src/Common/AIO.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#if !(defined(__FreeBSD__) || defined(__APPLE__))
+#if !(defined(__FreeBSD__) || defined(__APPLE__) || defined(_MSC_VER))
 
 #include <Common/Exception.h>
 #include <common/logger_useful.h>
diff --git a/dbms/src/Common/ShellCommand.cpp b/dbms/src/Common/ShellCommand.cpp
index b3765874ac4..c9e625810ca 100644
--- a/dbms/src/Common/ShellCommand.cpp
+++ b/dbms/src/Common/ShellCommand.cpp
@@ -1,13 +1,12 @@
 #include <sys/types.h>
 #include <sys/wait.h>
-#include <unistd.h>
 #include <fcntl.h>
 #include <dlfcn.h>
-
 #include <Common/Exception.h>
 #include <Common/ShellCommand.h>
 #include <IO/WriteBufferFromVector.h>
 #include <IO/WriteHelpers.h>
+#include <port/unistd.h>
 
 
 namespace DB
diff --git a/dbms/src/Common/ZooKeeper/CMakeLists.txt b/dbms/src/Common/ZooKeeper/CMakeLists.txt
index 9bd72b6fa54..6a02d5733b1 100644
--- a/dbms/src/Common/ZooKeeper/CMakeLists.txt
+++ b/dbms/src/Common/ZooKeeper/CMakeLists.txt
@@ -5,7 +5,11 @@ add_headers_and_sources(clickhouse_common_zookeeper .)
 
 add_library(clickhouse_common_zookeeper ${SPLIT_SHARED} ${clickhouse_common_zookeeper_headers} ${clickhouse_common_zookeeper_sources})
 
-target_link_libraries (clickhouse_common_zookeeper clickhouse_common_io)
+if (NOT USE_INTERNAL_ZOOKEEPER_LIBRARY)
+    target_include_directories (clickhouse_common_zookeeper BEFORE PUBLIC ${ZOOKEEPER_INCLUDE_DIR})
+endif ()
+
+target_link_libraries (clickhouse_common_zookeeper clickhouse_common_io ${ZOOKEEPER_LIBRARY})
 
 if (ENABLE_TESTS)
     add_subdirectory (tests)
diff --git a/dbms/src/Common/ZooKeeper/ZooKeeper.h b/dbms/src/Common/ZooKeeper/ZooKeeper.h
index 52f1968eba6..dcfb5e9393d 100644
--- a/dbms/src/Common/ZooKeeper/ZooKeeper.h
+++ b/dbms/src/Common/ZooKeeper/ZooKeeper.h
@@ -8,10 +8,10 @@
 #include <memory>
 #include <mutex>
 #include <string>
-#include <unistd.h>
 #include <common/logger_useful.h>
 #include <Common/ProfileEvents.h>
 #include <Common/CurrentMetrics.h>
+#include <port/unistd.h>
 
 
 namespace ProfileEvents
diff --git a/dbms/src/Common/ZooKeeper/tests/zkutil_test_commands.cpp b/dbms/src/Common/ZooKeeper/tests/zkutil_test_commands.cpp
index 3c9aed5113d..9b73f79ad7d 100644
--- a/dbms/src/Common/ZooKeeper/tests/zkutil_test_commands.cpp
+++ b/dbms/src/Common/ZooKeeper/tests/zkutil_test_commands.cpp
@@ -1,6 +1,6 @@
 #include <Common/ZooKeeper/ZooKeeper.h>
 #include <iostream>
-#include <unistd.h>
+#include <port/unistd.h>
 
 using namespace zkutil;
 
diff --git a/dbms/src/Common/localBackup.cpp b/dbms/src/Common/localBackup.cpp
index 3b93e0ee452..eb1f76d8c60 100644
--- a/dbms/src/Common/localBackup.cpp
+++ b/dbms/src/Common/localBackup.cpp
@@ -1,13 +1,10 @@
 #include <sys/stat.h>
-#include <unistd.h>
-
 #include <string>
 #include <iostream>
-
 #include <Poco/DirectoryIterator.h>
 #include <Poco/File.h>
-
 #include <Common/Exception.h>
+#include <port/unistd.h>
 
 
 namespace DB
diff --git a/dbms/src/Common/randomSeed.cpp b/dbms/src/Common/randomSeed.cpp
index 62925f55dd3..7618c41a020 100644
--- a/dbms/src/Common/randomSeed.cpp
+++ b/dbms/src/Common/randomSeed.cpp
@@ -1,12 +1,10 @@
 #include <time.h>
-#include <unistd.h>
+#include <port/unistd.h>
 #include <sys/types.h>
-
 #include <Common/Exception.h>
 #include <Common/randomSeed.h>
 #include <Common/SipHash.h>
 #include <Core/Types.h>
-
 #ifdef __APPLE__
 #include <common/apple_rt.h>
 #endif
diff --git a/dbms/src/Common/tests/compact_array.cpp b/dbms/src/Common/tests/compact_array.cpp
index 3beb8b6db9a..14c6b819c37 100644
--- a/dbms/src/Common/tests/compact_array.cpp
+++ b/dbms/src/Common/tests/compact_array.cpp
@@ -7,15 +7,13 @@
 #include <Common/CompactArray.h>
 #include <IO/WriteBufferFromFile.h>
 #include <IO/ReadBufferFromFile.h>
-
 #include <boost/filesystem.hpp>
-
 #include <string>
 #include <iostream>
 #include <fstream>
 #include <stdexcept>
-#include <unistd.h>
 #include <cstdlib>
+#include <port/unistd.h>
 
 namespace fs = boost::filesystem;
 
diff --git a/dbms/src/Core/Defines.h b/dbms/src/Core/Defines.h
index 8b69d83d69e..db722cbf91a 100644
--- a/dbms/src/Core/Defines.h
+++ b/dbms/src/Core/Defines.h
@@ -74,8 +74,18 @@
 #define DEFAULT_HTTP_READ_BUFFER_TIMEOUT 1800
 #define DEFAULT_HTTP_READ_BUFFER_CONNECTION_TIMEOUT 1
 
-#define ALWAYS_INLINE __attribute__((__always_inline__))
-#define NO_INLINE __attribute__((__noinline__))
+// more aliases: https://mailman.videolan.org/pipermail/x264-devel/2014-May/010660.html
+
+#if defined(_MSC_VER)
+    #define ALWAYS_INLINE __forceinline
+    #define NO_INLINE static __declspec(noinline)
+    #define MAY_ALIAS
+#else
+    #define ALWAYS_INLINE __attribute__((__always_inline__))
+    #define NO_INLINE __attribute__((__noinline__))
+    #define MAY_ALIAS __attribute__((__may_alias__))
+#endif
+
 
 #define PLATFORM_NOT_SUPPORTED "The only supported platforms are x86_64 and AArch64 (work in progress)"
 
diff --git a/dbms/src/Core/Field.h b/dbms/src/Core/Field.h
index f4c2f1ade86..795bc4b7070 100644
--- a/dbms/src/Core/Field.h
+++ b/dbms/src/Core/Field.h
@@ -198,14 +198,14 @@ public:
     template <typename T> T & get()
     {
         using TWithoutRef = std::remove_reference_t<T>;
-        TWithoutRef * __attribute__((__may_alias__)) ptr = reinterpret_cast<TWithoutRef*>(&storage);
+        TWithoutRef * MAY_ALIAS ptr = reinterpret_cast<TWithoutRef*>(&storage);
         return *ptr;
     };
 
     template <typename T> const T & get() const
     {
         using TWithoutRef = std::remove_reference_t<T>;
-        const TWithoutRef * __attribute__((__may_alias__)) ptr = reinterpret_cast<const TWithoutRef*>(&storage);
+        const TWithoutRef * MAY_ALIAS ptr = reinterpret_cast<const TWithoutRef*>(&storage);
         return *ptr;
     };
 
@@ -340,7 +340,7 @@ private:
     void createConcrete(T && x)
     {
         using JustT = std::decay_t<T>;
-        JustT * __attribute__((__may_alias__)) ptr = reinterpret_cast<JustT *>(&storage);
+        JustT * MAY_ALIAS ptr = reinterpret_cast<JustT *>(&storage);
         new (ptr) JustT(std::forward<T>(x));
         which = TypeToEnum<JustT>::value;
     }
@@ -350,7 +350,7 @@ private:
     void assignConcrete(T && x)
     {
         using JustT = std::decay_t<T>;
-        JustT * __attribute__((__may_alias__)) ptr = reinterpret_cast<JustT *>(&storage);
+        JustT * MAY_ALIAS ptr = reinterpret_cast<JustT *>(&storage);
         *ptr = std::forward<T>(x);
     }
 
@@ -398,7 +398,7 @@ private:
 
     void create(const char * data, size_t size)
     {
-        String * __attribute__((__may_alias__)) ptr = reinterpret_cast<String*>(&storage);
+        String * MAY_ALIAS ptr = reinterpret_cast<String*>(&storage);
         new (ptr) String(data, size);
         which = Types::String;
     }
@@ -434,7 +434,7 @@ private:
     template <typename T>
     void destroy()
     {
-        T * __attribute__((__may_alias__)) ptr = reinterpret_cast<T*>(&storage);
+        T * MAY_ALIAS ptr = reinterpret_cast<T*>(&storage);
         ptr->~T();
     }
 };
diff --git a/dbms/src/DataStreams/PrettyBlockOutputStream.cpp b/dbms/src/DataStreams/PrettyBlockOutputStream.cpp
index 58b61664b7c..2bb919d4806 100644
--- a/dbms/src/DataStreams/PrettyBlockOutputStream.cpp
+++ b/dbms/src/DataStreams/PrettyBlockOutputStream.cpp
@@ -1,6 +1,5 @@
 #include <sys/ioctl.h>
-#include <unistd.h>
-
+#include <port/unistd.h>
 #include <DataStreams/PrettyBlockOutputStream.h>
 #include <IO/WriteBuffer.h>
 #include <IO/WriteHelpers.h>
diff --git a/dbms/src/IO/ReadBufferAIO.cpp b/dbms/src/IO/ReadBufferAIO.cpp
index c4680b0e409..b9f760b51da 100644
--- a/dbms/src/IO/ReadBufferAIO.cpp
+++ b/dbms/src/IO/ReadBufferAIO.cpp
@@ -1,4 +1,4 @@
-#if !(defined(__FreeBSD__) || defined(__APPLE__))
+#if !(defined(__FreeBSD__) || defined(__APPLE__) || defined(_MSC_VER))
 
 #include <IO/ReadBufferAIO.h>
 #include <Common/ProfileEvents.h>
diff --git a/dbms/src/IO/ReadBufferAIO.h b/dbms/src/IO/ReadBufferAIO.h
index ccbd7330851..9f8451338ac 100644
--- a/dbms/src/IO/ReadBufferAIO.h
+++ b/dbms/src/IO/ReadBufferAIO.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#if !(defined(__FreeBSD__) || defined(__APPLE__))
+#if !(defined(__FreeBSD__) || defined(__APPLE__) || defined(_MSC_VER))
 
 #include <IO/ReadBufferFromFileBase.h>
 #include <IO/ReadBuffer.h>
@@ -8,7 +8,6 @@
 #include <Core/Defines.h>
 #include <Common/AIO.h>
 #include <Common/CurrentMetrics.h>
-
 #include <string>
 #include <limits>
 #include <unistd.h>
diff --git a/dbms/src/IO/ReadBufferFromFileDescriptor.cpp b/dbms/src/IO/ReadBufferFromFileDescriptor.cpp
index eced314ff40..d2acffaf2e7 100644
--- a/dbms/src/IO/ReadBufferFromFileDescriptor.cpp
+++ b/dbms/src/IO/ReadBufferFromFileDescriptor.cpp
@@ -1,14 +1,10 @@
 #include <errno.h>
 #include <time.h>
-
 #include <optional>
-
 #include <Common/ProfileEvents.h>
 #include <Common/Stopwatch.h>
-
 #include <Common/Exception.h>
 #include <Common/CurrentMetrics.h>
-
 #include <IO/ReadBufferFromFileDescriptor.h>
 #include <IO/WriteHelpers.h>
 
diff --git a/dbms/src/IO/ReadBufferFromFileDescriptor.h b/dbms/src/IO/ReadBufferFromFileDescriptor.h
index 1331878912e..55065931148 100644
--- a/dbms/src/IO/ReadBufferFromFileDescriptor.h
+++ b/dbms/src/IO/ReadBufferFromFileDescriptor.h
@@ -1,7 +1,6 @@
 #pragma once
 
-#include <unistd.h>
-
+#include <port/unistd.h>
 #include <IO/ReadBufferFromFileBase.h>
 #include <IO/ReadBuffer.h>
 
diff --git a/dbms/src/IO/WriteBufferAIO.cpp b/dbms/src/IO/WriteBufferAIO.cpp
index ae1253df263..087717b452d 100644
--- a/dbms/src/IO/WriteBufferAIO.cpp
+++ b/dbms/src/IO/WriteBufferAIO.cpp
@@ -1,4 +1,4 @@
-#if !(defined(__FreeBSD__) || defined(__APPLE__))
+#if !(defined(__FreeBSD__) || defined(__APPLE__) || defined(_MSC_VER))
 
 #include <IO/WriteBufferAIO.h>
 #include <Common/ProfileEvents.h>
diff --git a/dbms/src/IO/WriteBufferAIO.h b/dbms/src/IO/WriteBufferAIO.h
index 0942934e87d..3c99eb40eed 100644
--- a/dbms/src/IO/WriteBufferAIO.h
+++ b/dbms/src/IO/WriteBufferAIO.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#if !(defined(__FreeBSD__) || defined(__APPLE__))
+#if !(defined(__FreeBSD__) || defined(__APPLE__) || defined(_MSC_VER))
 
 #include <IO/WriteBufferFromFileBase.h>
 #include <IO/WriteBuffer.h>
diff --git a/dbms/src/IO/WriteBufferFromFileDescriptor.cpp b/dbms/src/IO/WriteBufferFromFileDescriptor.cpp
index 8f6f2b1b126..d22fe7229b8 100644
--- a/dbms/src/IO/WriteBufferFromFileDescriptor.cpp
+++ b/dbms/src/IO/WriteBufferFromFileDescriptor.cpp
@@ -1,4 +1,4 @@
-#include <unistd.h>
+#include <port/unistd.h>
 #include <errno.h>
 
 #include <Common/Exception.h>
diff --git a/dbms/src/IO/createReadBufferFromFileBase.cpp b/dbms/src/IO/createReadBufferFromFileBase.cpp
index bae7a737d36..beb73eda861 100644
--- a/dbms/src/IO/createReadBufferFromFileBase.cpp
+++ b/dbms/src/IO/createReadBufferFromFileBase.cpp
@@ -1,6 +1,6 @@
 #include <IO/createReadBufferFromFileBase.h>
 #include <IO/ReadBufferFromFile.h>
-#if !defined(__APPLE__) && !defined(__FreeBSD__)
+#if !defined(__APPLE__) && !defined(__FreeBSD__) && !defined(_MSC_VER)
 #include <IO/ReadBufferAIO.h>
 #endif
 #include <Common/ProfileEvents.h>
@@ -14,7 +14,7 @@ namespace ProfileEvents
 
 namespace DB
 {
-#if defined(__APPLE__) || defined(__FreeBSD__)
+#if defined(__APPLE__) || defined(__FreeBSD__) || defined(_MSC_VER)
 namespace ErrorCodes
 {
         extern const int NOT_IMPLEMENTED;
@@ -31,7 +31,7 @@ std::unique_ptr<ReadBufferFromFileBase> createReadBufferFromFileBase(const std::
     }
     else
     {
-#if !defined(__APPLE__) && !defined(__FreeBSD__)
+#if !defined(__APPLE__) && !defined(__FreeBSD__) && !defined(_MSC_VER)
         ProfileEvents::increment(ProfileEvents::CreatedReadBufferAIO);
         return std::make_unique<ReadBufferAIO>(filename_, buffer_size_, flags_, existing_memory_);
 #else
diff --git a/dbms/src/IO/createWriteBufferFromFileBase.cpp b/dbms/src/IO/createWriteBufferFromFileBase.cpp
index 94d16d1d41b..de06e2eb1cb 100644
--- a/dbms/src/IO/createWriteBufferFromFileBase.cpp
+++ b/dbms/src/IO/createWriteBufferFromFileBase.cpp
@@ -1,6 +1,6 @@
 #include <IO/createWriteBufferFromFileBase.h>
 #include <IO/WriteBufferFromFile.h>
-#if !defined(__APPLE__) && !defined(__FreeBSD__)
+#if !defined(__APPLE__) && !defined(__FreeBSD__) && !defined(_MSC_VER)
 #include <IO/WriteBufferAIO.h>
 #endif
 #include <Common/ProfileEvents.h>
@@ -15,7 +15,7 @@ namespace ProfileEvents
 namespace DB
 {
 
-#if defined(__APPLE__) || defined(__FreeBSD__)
+#if defined(__APPLE__) || defined(__FreeBSD__) || defined(_MSC_VER)
 namespace ErrorCodes
 {
         extern const int NOT_IMPLEMENTED;
@@ -33,7 +33,7 @@ WriteBufferFromFileBase * createWriteBufferFromFileBase(const std::string & file
     }
     else
     {
-#if !defined(__APPLE__) && !defined(__FreeBSD__)
+#if !defined(__APPLE__) && !defined(__FreeBSD__) && !defined(_MSC_VER)
         ProfileEvents::increment(ProfileEvents::CreatedWriteBufferAIO);
         return new WriteBufferAIO(filename_, buffer_size_, flags_, mode, existing_memory_);
 #else
diff --git a/dbms/src/IO/tests/read_buffer_aio.cpp b/dbms/src/IO/tests/read_buffer_aio.cpp
index b6352d7a54a..2f641d0cd3c 100644
--- a/dbms/src/IO/tests/read_buffer_aio.cpp
+++ b/dbms/src/IO/tests/read_buffer_aio.cpp
@@ -1,14 +1,13 @@
 #include <IO/ReadBufferAIO.h>
 #include <Core/Defines.h>
-
 #include <boost/filesystem.hpp>
-
 #include <vector>
 #include <iostream>
 #include <fstream>
 #include <functional>
 #include <cstdlib>
-#include <unistd.h>
+#include <port/unistd.h>
+
 
 namespace
 {
diff --git a/dbms/src/Interpreters/ClientInfo.cpp b/dbms/src/Interpreters/ClientInfo.cpp
index 40ebabc60df..06fd5e9c589 100644
--- a/dbms/src/Interpreters/ClientInfo.cpp
+++ b/dbms/src/Interpreters/ClientInfo.cpp
@@ -1,5 +1,3 @@
-#include <unistd.h>
-
 #include <Interpreters/ClientInfo.h>
 #include <IO/ReadBuffer.h>
 #include <IO/WriteBuffer.h>
@@ -8,6 +6,7 @@
 #include <Core/Defines.h>
 #include <Common/getFQDNOrHostName.h>
 #include <Common/ClickHouseRevision.h>
+#include <port/unistd.h>
 
 
 namespace DB
diff --git a/dbms/src/Interpreters/tests/users.cpp b/dbms/src/Interpreters/tests/users.cpp
index 5da4ed26134..f4905398262 100644
--- a/dbms/src/Interpreters/tests/users.cpp
+++ b/dbms/src/Interpreters/tests/users.cpp
@@ -1,8 +1,6 @@
 #include <Common/Config/ConfigProcessor.h>
 #include <Interpreters/SecurityManager.h>
-
 #include <boost/filesystem.hpp>
-
 #include <vector>
 #include <string>
 #include <tuple>
@@ -10,8 +8,8 @@
 #include <fstream>
 #include <sstream>
 #include <stdexcept>
-#include <unistd.h>
 #include <cstdlib>
+#include <port/unistd.h>
 
 namespace
 {
diff --git a/dbms/src/Server/Benchmark.cpp b/dbms/src/Server/Benchmark.cpp
index 78a19a398a1..ee4d2dd8802 100644
--- a/dbms/src/Server/Benchmark.cpp
+++ b/dbms/src/Server/Benchmark.cpp
@@ -1,31 +1,24 @@
-#include <unistd.h>
+#include <port/unistd.h>
 #include <stdlib.h>
 #include <fcntl.h>
 #include <signal.h>
 #include <time.h>
-
 #include <iostream>
 #include <fstream>
 #include <iomanip>
 #include <random>
 #include <pcg_random.hpp>
-
 #include <Poco/File.h>
 #include <Poco/Util/Application.h>
-
 #include <Common/Stopwatch.h>
 #include <common/ThreadPool.h>
 #include <AggregateFunctions/ReservoirSampler.h>
 #include <AggregateFunctions/registerAggregateFunctions.h>
-
 #include <boost/program_options.hpp>
-
 #include <Common/ConcurrentBoundedQueue.h>
-
 #include <Common/Exception.h>
 #include <Common/randomSeed.h>
 #include <Core/Types.h>
-
 #include <IO/ReadBufferFromFileDescriptor.h>
 #include <IO/WriteBufferFromFileDescriptor.h>
 #include <IO/WriteBufferFromFile.h>
@@ -33,13 +26,9 @@
 #include <IO/WriteHelpers.h>
 #include <IO/Operators.h>
 #include <IO/ConnectionTimeouts.h>
-
 #include <DataStreams/RemoteBlockInputStream.h>
-
 #include <Interpreters/Context.h>
-
 #include <Client/Connection.h>
-
 #include "InterruptListener.h"
 
 
diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp
index 1c838a99569..a20662ad053 100644
--- a/dbms/src/Server/Client.cpp
+++ b/dbms/src/Server/Client.cpp
@@ -1,8 +1,7 @@
-#include <unistd.h>
+#include <port/unistd.h>
 #include <stdlib.h>
 #include <fcntl.h>
 #include <signal.h>
-
 #include <iostream>
 #include <fstream>
 #include <iomanip>
@@ -10,13 +9,10 @@
 #include <algorithm>
 #include <optional>
 #include <boost/program_options.hpp>
-
 #include <Poco/File.h>
 #include <Poco/Util/Application.h>
-
 #include <common/readline_use.h>
 #include <common/find_first_symbols.h>
-
 #include <Common/ClickHouseRevision.h>
 #include <Common/Stopwatch.h>
 #include <Common/Exception.h>
diff --git a/dbms/src/Server/PerformanceTest.cpp b/dbms/src/Server/PerformanceTest.cpp
index 81f679cd61e..7c9d4436f3c 100644
--- a/dbms/src/Server/PerformanceTest.cpp
+++ b/dbms/src/Server/PerformanceTest.cpp
@@ -3,14 +3,11 @@
 #include <limits>
 #include <regex>
 #include <thread>
-#include <unistd.h>
-
+#include <port/unistd.h>
 #include <boost/filesystem.hpp>
 #include <boost/program_options.hpp>
 #include <sys/stat.h>
-
 #include <common/DateLUT.h>
-
 #include <AggregateFunctions/ReservoirSampler.h>
 #include <Client/Connection.h>
 #include <Common/ConcurrentBoundedQueue.h>
@@ -27,13 +24,11 @@
 #include <Interpreters/Settings.h>
 #include <common/ThreadPool.h>
 #include <common/getMemoryAmount.h>
-
 #include <Poco/AutoPtr.h>
 #include <Poco/Exception.h>
 #include <Poco/SAX/InputSource.h>
 #include <Poco/Util/XMLConfiguration.h>
 #include <Poco/XML/XMLStream.h>
-
 #include "InterruptListener.h"
 
 /** Tests launcher for ClickHouse.
diff --git a/dbms/src/Storages/tests/remove_symlink_directory.cpp b/dbms/src/Storages/tests/remove_symlink_directory.cpp
index 16b2a1b4750..59011c557ee 100644
--- a/dbms/src/Storages/tests/remove_symlink_directory.cpp
+++ b/dbms/src/Storages/tests/remove_symlink_directory.cpp
@@ -1,6 +1,5 @@
-#include <unistd.h>
+#include <port/unistd.h>
 #include <iostream>
-
 #include <Poco/File.h>
 #include <Poco/Path.h>
 #include <Common/Exception.h>
diff --git a/debian/.pbuilderrc b/debian/.pbuilderrc
index e3704760ec1..ba1cdb2c324 100644
--- a/debian/.pbuilderrc
+++ b/debian/.pbuilderrc
@@ -188,7 +188,7 @@ export DEB_BUILD_OPTIONS=parallel=`nproc`
 
 # Floating bug with permissions:
 [ -n "$CCACHEDIR" ] && sudo mkdir -p $CCACHEDIR
-[ -n "$CCACHEDIR" ] && sudo chmod -R a+rwx $CCACHEDIR
+[ -n "$CCACHEDIR" ] && sudo chmod -R a+rwx $CCACHEDIR || true
 # chown -R $BUILDUSERID:$BUILDUSERID $CCACHEDIR
 
 
diff --git a/debian/rules b/debian/rules
index 8f3818c0478..7990db33b1c 100755
--- a/debian/rules
+++ b/debian/rules
@@ -2,7 +2,7 @@
 # -*- makefile -*-
 
 # Uncomment this to turn on verbose mode.
-#export DH_VERBOSE=1
+export DH_VERBOSE=1
 
 # -pie only for static mode
 export DEB_BUILD_MAINT_OPTIONS=hardening=+all,-pie
diff --git a/libs/libcommon/cmake/find_gperftools.cmake b/libs/libcommon/cmake/find_gperftools.cmake
index b9fcc40c0c6..a311f2ba74e 100644
--- a/libs/libcommon/cmake/find_gperftools.cmake
+++ b/libs/libcommon/cmake/find_gperftools.cmake
@@ -4,7 +4,7 @@ else ()
     option (USE_INTERNAL_GPERFTOOLS_LIBRARY "Set to FALSE to use system gperftools (tcmalloc) library instead of bundled" ${NOT_UNBUNDLED})
 endif ()
 
-if (ARCH_FREEBSD)
+if (ARCH_FREEBSD OR MSVC)
     option (ENABLE_TCMALLOC "Set to TRUE to enable tcmalloc" OFF)
 else ()
     option (ENABLE_TCMALLOC "Set to TRUE to enable tcmalloc" ON)
diff --git a/libs/libcommon/include/common/DateLUT.h b/libs/libcommon/include/common/DateLUT.h
index c86b4010108..75a7457d101 100644
--- a/libs/libcommon/include/common/DateLUT.h
+++ b/libs/libcommon/include/common/DateLUT.h
@@ -2,7 +2,6 @@
 
 #include <common/DateLUTImpl.h>
 #include <ext/singleton.h>
-
 #include <unordered_map>
 #include <atomic>
 #include <mutex>
@@ -10,8 +9,12 @@
 
 // Also defined in Core/Defines.h
 #if !defined(ALWAYS_INLINE)
+#if defined(_MSC_VER)
+    #define ALWAYS_INLINE __forceinline
+#else
     #define ALWAYS_INLINE __attribute__((__always_inline__))
 #endif
+#endif
 
 
 /// This class provides lazy initialization and lookup of singleton DateLUTImpl objects for a given timezone.
diff --git a/libs/libcommon/include/common/LocalDate.h b/libs/libcommon/include/common/LocalDate.h
index 93d045786e9..8eddc6f9115 100644
--- a/libs/libcommon/include/common/LocalDate.h
+++ b/libs/libcommon/include/common/LocalDate.h
@@ -18,7 +18,7 @@
   *
   * packed - for memcmp to work naturally (but because m_year is 2 bytes, on little endian, comparison is correct only before year 2047)
   */
-class __attribute__ ((__packed__)) LocalDate
+class LocalDate
 {
 private:
     unsigned short m_year;
diff --git a/libs/libcommon/include/common/LocalDateTime.h b/libs/libcommon/include/common/LocalDateTime.h
index 133bef46723..d0c208edadc 100644
--- a/libs/libcommon/include/common/LocalDateTime.h
+++ b/libs/libcommon/include/common/LocalDateTime.h
@@ -17,7 +17,7 @@
   *
   * packed - for memcmp to work naturally (but because m_year is 2 bytes, on little endian, comparison is correct only before year 2047)
   */
-class __attribute__ ((__packed__)) LocalDateTime
+class LocalDateTime
 {
 private:
     unsigned short m_year;
diff --git a/libs/libcommon/include/common/likely.h b/libs/libcommon/include/common/likely.h
index 7a0a98fcdc0..787d46c0e67 100644
--- a/libs/libcommon/include/common/likely.h
+++ b/libs/libcommon/include/common/likely.h
@@ -1,4 +1,9 @@
 #pragma once
 
-#define likely(x)    (__builtin_expect(!!(x), 1))
-#define unlikely(x)    (__builtin_expect(!!(x), 0))
+#if _MSC_VER
+#define likely(x)   (x)
+#define unlikely(x) (x)
+#else
+#define likely(x)   (__builtin_expect(!!(x), 1))
+#define unlikely(x) (__builtin_expect(!!(x), 0))
+#endif
diff --git a/libs/libcommon/include/common/mremap.h b/libs/libcommon/include/common/mremap.h
index a6a841adc52..edb62070247 100644
--- a/libs/libcommon/include/common/mremap.h
+++ b/libs/libcommon/include/common/mremap.h
@@ -1,7 +1,10 @@
 #pragma once
 
 #include <cstddef>
+#include <sys/types.h>
+#if !_MSC_VER
 #include <sys/mman.h>
+#endif
 
 #if defined(MREMAP_MAYMOVE)
 // we already have implementation (linux)
diff --git a/libs/libcommon/include/port/ssize_t.h b/libs/libcommon/include/port/ssize_t.h
new file mode 100644
index 00000000000..62199bbe1f0
--- /dev/null
+++ b/libs/libcommon/include/port/ssize_t.h
@@ -0,0 +1,8 @@
+#pragma once
+
+#ifdef _MSC_VER
+    #include <basetsd.h>
+    typedef SSIZE_T ssize_t;
+#else
+    #include <sys/types.h>
+#endif
diff --git a/libs/libcommon/include/port/unistd.h b/libs/libcommon/include/port/unistd.h
new file mode 100644
index 00000000000..2c08bd0bcd2
--- /dev/null
+++ b/libs/libcommon/include/port/unistd.h
@@ -0,0 +1,11 @@
+#pragma once
+
+// https://stackoverflow.com/questions/341817/is-there-a-replacement-for-unistd-h-for-windows-visual-c
+
+#ifdef _MSC_VER
+    #include <io.h>
+#else
+    #include <unistd.h>
+#endif
+
+#include "ssize_t.h"
diff --git a/libs/libcommon/src/DateLUT.cpp b/libs/libcommon/src/DateLUT.cpp
index 40634698dbc..22c50a9f694 100644
--- a/libs/libcommon/src/DateLUT.cpp
+++ b/libs/libcommon/src/DateLUT.cpp
@@ -56,13 +56,13 @@ std::string determineDefaultTimeZone()
         /// then the relative path is the time zone id.
         fs::path relative_path = tz_file_path.lexically_relative(tz_database_path);
         if (!relative_path.empty() && *relative_path.begin() != ".." && *relative_path.begin() != ".")
-            return relative_path.native();
+            return relative_path.string();
 
         /// The file is not inside the tz_database_dir, so we hope that it was copied and
         /// try to find the file with exact same contents in the database.
 
         size_t tzfile_size = fs::file_size(tz_file_path);
-        Poco::SHA1Engine::Digest tzfile_sha1 = calcSHA1(tz_file_path.native());
+        Poco::SHA1Engine::Digest tzfile_sha1 = calcSHA1(tz_file_path.string());
 
         fs::recursive_directory_iterator begin(tz_database_path);
         fs::recursive_directory_iterator end;
@@ -80,8 +80,8 @@ std::string determineDefaultTimeZone()
             if (candidate_it->status().type() != fs::regular_file || path.filename() == "localtime")
                 continue;
 
-            if (fs::file_size(path) == tzfile_size && calcSHA1(path.native()) == tzfile_sha1)
-                return path.lexically_relative(tz_database_path).native();
+            if (fs::file_size(path) == tzfile_size && calcSHA1(path.string()) == tzfile_sha1)
+                return path.lexically_relative(tz_database_path).string();
         }
     }
     catch (const Poco::Exception & ex)
diff --git a/libs/libcommon/src/mremap.cpp b/libs/libcommon/src/mremap.cpp
index e01acc8d5d8..5d1ba5287bc 100644
--- a/libs/libcommon/src/mremap.cpp
+++ b/libs/libcommon/src/mremap.cpp
@@ -1,9 +1,8 @@
+#include <common/mremap.h>
 #include <cstddef>
 #include <cstdlib>
 #include <cstring>
-#include <sys/mman.h>
 #include <errno.h>
-#include <common/mremap.h>
 
 #if defined(MREMAP_FIXED)
 // we already have implementation (linux)
@@ -22,18 +21,26 @@ void * mremap(
         return nullptr;
     }
 
+#if _MSC_VER
+    void * new_address = ::operator new(new_size);
+#else
     void * new_address = mmap(nullptr, new_size, mmap_prot, mmap_flags, mmap_fd, mmap_offset);
     if (MAP_FAILED == new_address)
     {
         return MAP_FAILED;
     }
+#endif
 
     memcpy(new_address, old_address, old_size);
 
+#if _MSC_VER
+    delete old_address;
+#else
     if (munmap(old_address, old_size))
     {
         abort();
     }
+#endif
 
     return new_address;
 }
diff --git a/libs/libdaemon/include/daemon/BaseDaemon.h b/libs/libdaemon/include/daemon/BaseDaemon.h
index 20c38469d16..112502e06f6 100644
--- a/libs/libdaemon/include/daemon/BaseDaemon.h
+++ b/libs/libdaemon/include/daemon/BaseDaemon.h
@@ -1,7 +1,7 @@
 #pragma once
 
 #include <sys/types.h>
-#include <unistd.h>
+#include <port/unistd.h>
 #include <iostream>
 #include <memory>
 #include <functional>
diff --git a/libs/libpocoext/src/ThreadNumber.cpp b/libs/libpocoext/src/ThreadNumber.cpp
index 266627d3168..6458ea8c221 100644
--- a/libs/libpocoext/src/ThreadNumber.cpp
+++ b/libs/libpocoext/src/ThreadNumber.cpp
@@ -1,16 +1,14 @@
-#include <pthread.h>
-#include <common/likely.h>
-
 #include <Poco/Ext/ThreadNumber.h>
-
+#include <common/likely.h>
+#include <atomic>
 
 static thread_local unsigned thread_number = 0;
-static unsigned threads = 0;
+static std::atomic_uint threads{0};
 
 unsigned Poco::ThreadNumber::get()
 {
     if (unlikely(thread_number == 0))
-        thread_number = __sync_add_and_fetch(&threads, 1);
+        thread_number = ++threads;
 
     return thread_number;
 }
diff --git a/utils/build/build_msvc2017.bat b/utils/build/build_msvc2017.bat
new file mode 100644
index 00000000000..880802999ab
--- /dev/null
+++ b/utils/build/build_msvc2017.bat
@@ -0,0 +1,14 @@
+
+:: WINDOWS BUILD NOT SUPPORTED!
+:: Script only for development
+
+cd ../..
+git clone --recursive https://github.com/madler/zlib contrib/zlib
+md build
+cd build
+
+:: Stage 1: try build client
+cmake .. -G "Visual Studio 15 2017 Win64" -DENABLE_CLICKHOUSE_ALL=0 -DENABLE_CLICKHOUSE_CLIENT=1 > cmake.log
+cmake --build . --target clickhouse -- /m > build.log
+:: Stage 2: try build minimal server
+:: Stage 3: enable all possible features (ssl, ...)
diff --git a/utils/compressor/zstd_test.cpp b/utils/compressor/zstd_test.cpp
index 6a8b9cb5be6..01016830e7d 100644
--- a/utils/compressor/zstd_test.cpp
+++ b/utils/compressor/zstd_test.cpp
@@ -1,7 +1,8 @@
-#include <unistd.h>
+#include <port/unistd.h>
 #include <zstd.h>
 #include <vector>
 #include <stdexcept>
+#include <sys/types.h>
 
 
 int main(int argc, char ** argv)
diff --git a/utils/iotest/iotest.cpp b/utils/iotest/iotest.cpp
index 439672fc25c..a4849638e48 100644
--- a/utils/iotest/iotest.cpp
+++ b/utils/iotest/iotest.cpp
@@ -1,28 +1,21 @@
 #include <fcntl.h>
-#include <unistd.h>
+#include <port/unistd.h>
 #include <stdlib.h>
 #include <time.h>
-
 #include <iostream>
 #include <iomanip>
 #include <vector>
 #include <random>
 #include <pcg_random.hpp>
-
 #include <Poco/NumberParser.h>
 #include <Poco/NumberFormatter.h>
 #include <Poco/Exception.h>
-
 #include <Common/Exception.h>
 #include <Common/randomSeed.h>
-
 #include <common/ThreadPool.h>
 #include <Common/Stopwatch.h>
-
 #include <IO/BufferWithOwnMemory.h>
-
 #include <cstdlib>
-
 #ifdef __APPLE__
 #include <common/apple_rt.h>
 #endif
diff --git a/utils/iotest/iotest_aio.cpp b/utils/iotest/iotest_aio.cpp
index f86b4e76e3e..bfab8249d19 100644
--- a/utils/iotest/iotest_aio.cpp
+++ b/utils/iotest/iotest_aio.cpp
@@ -1,28 +1,21 @@
 #include <fcntl.h>
-#include <unistd.h>
+#include <port/unistd.h>
 #include <stdlib.h>
 #include <time.h>
-
 #include <iostream>
 #include <iomanip>
 #include <vector>
-
 #include <Poco/NumberParser.h>
 #include <Poco/NumberFormatter.h>
 #include <Poco/Exception.h>
-
 #include <Common/Exception.h>
-
 #include <common/ThreadPool.h>
 #include <Common/Stopwatch.h>
-
 #include <IO/BufferWithOwnMemory.h>
-
 #include <stdlib.h>
 #if !defined(__APPLE__) && !defined(__FreeBSD__)
 #include <malloc.h>
 #endif
-
 #include <fcntl.h>
 #include <stdlib.h>
 #include <stdio.h>
@@ -33,6 +26,7 @@
 #endif
 #include <sys/syscall.h>
 
+
 using DB::throwFromErrno;
 
 inline int io_setup(unsigned nr, aio_context_t *ctxp)
diff --git a/utils/iotest/iotest_nonblock.cpp b/utils/iotest/iotest_nonblock.cpp
index fdd25860825..8f1ad435bf0 100644
--- a/utils/iotest/iotest_nonblock.cpp
+++ b/utils/iotest/iotest_nonblock.cpp
@@ -1,5 +1,5 @@
 #include <fcntl.h>
-#include <unistd.h>
+#include <port/unistd.h>
 #include <stdlib.h>
 #include <time.h>
 #include <stdlib.h>
@@ -7,24 +7,18 @@
 #include <malloc.h>
 #endif
 #include <poll.h>
-
 #include <iostream>
 #include <iomanip>
 #include <vector>
-
 #include <random>
 #include <pcg_random.hpp>
-
 #include <Poco/NumberParser.h>
 #include <Poco/NumberFormatter.h>
 #include <Poco/Exception.h>
-
 #include <Common/Exception.h>
 #include <Common/randomSeed.h>
-
 #include <common/ThreadPool.h>
 #include <Common/Stopwatch.h>
-
 #ifdef __APPLE__
 #include <common/apple_rt.h>
 #endif

From 7ba82eb359a3b37ef54d819bc54df407222c98f1 Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Fri, 23 Mar 2018 19:18:36 +0300
Subject: [PATCH 185/244] Update Defines.h

---
 dbms/src/Core/Defines.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dbms/src/Core/Defines.h b/dbms/src/Core/Defines.h
index f860bd8c71c..569799bd956 100644
--- a/dbms/src/Core/Defines.h
+++ b/dbms/src/Core/Defines.h
@@ -60,7 +60,7 @@
 #define DBMS_MIN_REVISION_WITH_QUOTA_KEY_IN_CLIENT_INFO 54060
 #define DBMS_MIN_REVISION_WITH_TABLES_STATUS 54226
 #define DBMS_MIN_REVISION_WITH_TIME_ZONE_PARAMETER_IN_DATETIME_DATA_TYPE 54337
-#define DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME 54358
+#define DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME 54372
 
 /// Version of ClickHouse TCP protocol. Set to git tag with latest protocol change.
 #define DBMS_TCP_PROTOCOL_VERSION 54226

From 7cdb6b5de12ec05cc973b615cfdfd668bd30e8cf Mon Sep 17 00:00:00 2001
From: proller <proller@github.com>
Date: Fri, 23 Mar 2018 19:25:02 +0300
Subject: [PATCH 186/244] Fix test

---
 .../queries/0_stateless/00534_long_functions_bad_arguments7.sh  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments7.sh b/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments7.sh
index 2cc422cdf9b..af34af94458 100755
--- a/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments7.sh
+++ b/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments7.sh
@@ -5,4 +5,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 . $CURDIR/00534_long_functions_bad_arguments.lib
 
-test_variant 'SELECT $_(\'\');'
+test_variant "SELECT \$_('');"

From e7a2f10e2a681627779f8d41aa1fb880ddbcc805 Mon Sep 17 00:00:00 2001
From: proller <proller@github.com>
Date: Fri, 23 Mar 2018 19:40:15 +0300
Subject: [PATCH 187/244] Fix test build

---
 utils/compressor/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utils/compressor/CMakeLists.txt b/utils/compressor/CMakeLists.txt
index ae4333337db..ca4dd334fb5 100644
--- a/utils/compressor/CMakeLists.txt
+++ b/utils/compressor/CMakeLists.txt
@@ -7,7 +7,7 @@ set_target_properties(util-clickhouse-compressor PROPERTIES OUTPUT_NAME "clickho
 #install (TARGETS util-clickhouse-compressor RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse-compressor)
 
 add_executable (zstd_test zstd_test.cpp)
-target_link_libraries (zstd_test ${ZSTD_LIBRARY} Threads::Threads)
+target_link_libraries (zstd_test ${ZSTD_LIBRARY} common Threads::Threads)
 if (NOT USE_INTERNAL_ZSTD_LIBRARY)
     target_include_directories (zstd_test BEFORE PRIVATE ${ZSTD_INCLUDE_DIR})
 endif ()

From 046346645f5d7039100ec15db11b887d5be7d7f3 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 23 Mar 2018 19:55:35 +0300
Subject: [PATCH 188/244] Minor modifications #1959

---
 dbms/src/IO/ReadHelpers.h  |  2 ++
 dbms/src/Server/Client.cpp | 26 +++++++++++++-------------
 2 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/dbms/src/IO/ReadHelpers.h b/dbms/src/IO/ReadHelpers.h
index 875515149b6..2d3b405675f 100644
--- a/dbms/src/IO/ReadHelpers.h
+++ b/dbms/src/IO/ReadHelpers.h
@@ -52,6 +52,8 @@ inline char parseEscapeSequence(char c)
             return '\a';
         case 'b':
             return '\b';
+        case 'e':
+            return '\x1B';      /// \e escape sequence is non standard for C and C++ but supported by gcc and clang.
         case 'f':
             return '\f';
         case 'n':
diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp
index 721df841820..d16a5de3b03 100644
--- a/dbms/src/Server/Client.cpp
+++ b/dbms/src/Server/Client.cpp
@@ -33,6 +33,7 @@
 #include <IO/WriteBufferFromFileDescriptor.h>
 #include <IO/WriteBufferFromFile.h>
 #include <IO/ReadBufferFromMemory.h>
+#include <IO/ReadBufferFromString.h>
 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
 #include <DataStreams/AsynchronousBlockInputStream.h>
@@ -340,38 +341,37 @@ private:
 
         Strings keys;
 
-        prompt_by_server_display_name = config().getString("prompt_by_server_display_name.default", "{display_name} :) ");
+        prompt_by_server_display_name = config().getRawString("prompt_by_server_display_name.default", "{display_name} :) ");
 
         config().keys("prompt_by_server_display_name", keys);
 
         for (const String & key : keys)
         {
-            if (key != "default" && server_display_name.find(key) != std::string::npos) 
+            if (key != "default" && server_display_name.find(key) != std::string::npos)
             {
-                prompt_by_server_display_name = config().getString("prompt_by_server_display_name." + key);
+                prompt_by_server_display_name = config().getRawString("prompt_by_server_display_name." + key);
                 break;
             }
         }
 
-        std::map<String, String> terminalCharacters = {
-            { "\\e[",   "\e["  },
-            { "\\33[",  "\33[" },
-            { "\\033[", "\033["},
-            { "\\x1B[", "\x1B["},
-        };
-
-        for (const auto & [key, value]: terminalCharacters)
+        /// Prompt may contain escape sequences including \e[ or \x1b[ sequences to set terminal color.
         {
-            boost::replace_all(prompt_by_server_display_name, key, value);
+            String unescaped_prompt_by_server_display_name;
+            ReadBufferFromString in(prompt_by_server_display_name);
+            readEscapedString(unescaped_prompt_by_server_display_name, in);
+            prompt_by_server_display_name = std::move(unescaped_prompt_by_server_display_name);
         }
 
-        std::map<String, String> environment = {
+        /// Prompt may contain the following substitutions in a form of {name}.
+        std::map<String, String> environment =
+        {
             {"host",         config().getString("host", "localhost")},
             {"port",         config().getString("port", "9000")},
             {"user",         config().getString("user", "default")},
             {"display_name", server_display_name},
         };
 
+        /// Quite suboptimal.
         for (const auto & [key, value]: environment)
         {
             boost::replace_all(prompt_by_server_display_name, "{" + key + "}", value);

From f5896d35b4ce975400b67ac62b3fb20a14f02c52 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 23 Mar 2018 21:20:54 +0300
Subject: [PATCH 189/244] Removed duplicate code that could lead to wrong
 behaviour #1959

---
 dbms/src/Server/Client.cpp | 91 +++++++++++++++++++++++++-------------
 1 file changed, 61 insertions(+), 30 deletions(-)

diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp
index d16a5de3b03..47b632aa28e 100644
--- a/dbms/src/Server/Client.cpp
+++ b/dbms/src/Server/Client.cpp
@@ -169,6 +169,49 @@ private:
     std::list<ExternalTable> external_tables;
 
 
+    struct ConnectionParameters
+    {
+        String host;
+        UInt16 port;
+        String default_database;
+        String user;
+        String password;
+        Protocol::Encryption security;
+        Protocol::Compression compression;
+        ConnectionTimeouts timeouts;
+
+        ConnectionParameters() {}
+
+        ConnectionParameters(const Poco::Util::AbstractConfiguration & config)
+        {
+            bool is_secure = config.getBool("secure", false);
+            security = is_secure
+                ? Protocol::Encryption::Enable
+                : Protocol::Encryption::Disable;
+
+            host = config.getString("host", "localhost");
+            port = config.getInt("port",
+                config.getInt(is_secure ? "tcp_secure_port" : "tcp_port",
+                    is_secure ? DBMS_DEFAULT_SECURE_PORT : DBMS_DEFAULT_PORT));
+
+            default_database = config.getString("database", "");
+            user = config.getString("user", "");
+            password = config.getString("password", "");
+
+            compression = config.getBool("compression", true)
+                ? Protocol::Compression::Enable
+                : Protocol::Compression::Disable;
+
+            timeouts = ConnectionTimeouts(
+                Poco::Timespan(config.getInt("connect_timeout", DBMS_DEFAULT_CONNECT_TIMEOUT_SEC), 0),
+                Poco::Timespan(config.getInt("receive_timeout", DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC), 0),
+                Poco::Timespan(config.getInt("send_timeout", DBMS_DEFAULT_SEND_TIMEOUT_SEC), 0));
+        }
+    };
+
+    ConnectionParameters connection_parameters;
+
+
     void initialize(Poco::Util::Application & self)
     {
         Poco::Util::Application::initialize(self);
@@ -310,6 +353,7 @@ private:
             echo_queries = config().getBool("echo", false);
         }
 
+        connection_parameters = ConnectionParameters(config());
         connect();
 
         /// Initialize DateLUT here to avoid counting time spent here as query execution time.
@@ -363,19 +407,17 @@ private:
         }
 
         /// Prompt may contain the following substitutions in a form of {name}.
-        std::map<String, String> environment =
+        std::map<String, String> prompt_substitutions
         {
-            {"host",         config().getString("host", "localhost")},
-            {"port",         config().getString("port", "9000")},
-            {"user",         config().getString("user", "default")},
+            {"host", connection_parameters.host},
+            {"port", toString(connection_parameters.port)},
+            {"user", connection_parameters.user},
             {"display_name", server_display_name},
         };
 
         /// Quite suboptimal.
-        for (const auto & [key, value]: environment)
-        {
+        for (const auto & [key, value]: prompt_substitutions)
             boost::replace_all(prompt_by_server_display_name, "{" + key + "}", value);
-        }
 
         if (is_interactive)
         {
@@ -425,34 +467,23 @@ private:
 
     void connect()
     {
-        auto encryption = config().getBool("ssl", false)
-            ? Protocol::Encryption::Enable
-            : Protocol::Encryption::Disable;
-
-        String host = config().getString("host", "localhost");
-        UInt16 port = config().getInt("port", config().getInt(static_cast<bool>(encryption) ? "tcp_ssl_port" : "tcp_port", static_cast<bool>(encryption) ? DBMS_DEFAULT_SECURE_PORT : DBMS_DEFAULT_PORT));
-        String default_database = config().getString("database", "");
-        String user = config().getString("user", "");
-        String password = config().getString("password", "");
-
-        auto compression = config().getBool("compression", true)
-            ? Protocol::Compression::Enable
-            : Protocol::Compression::Disable;
-
         if (is_interactive)
             std::cout << "Connecting to "
-                << (!default_database.empty() ? "database " + default_database + " at " : "")
-                << host << ":" << port
-                << (!user.empty() ? " as user " + user : "")
+                << (!connection_parameters.default_database.empty() ? "database " + connection_parameters.default_database + " at " : "")
+                << connection_parameters.host << ":" << connection_parameters.port
+                << (!connection_parameters.user.empty() ? " as user " + connection_parameters.user : "")
                 << "." << std::endl;
 
-        ConnectionTimeouts timeouts(
-            Poco::Timespan(config().getInt("connect_timeout", DBMS_DEFAULT_CONNECT_TIMEOUT_SEC), 0),
-            Poco::Timespan(config().getInt("receive_timeout", DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC), 0),
-            Poco::Timespan(config().getInt("send_timeout", DBMS_DEFAULT_SEND_TIMEOUT_SEC), 0));
-
         connection = std::make_unique<Connection>(
-            host, port, default_database, user, password, timeouts, "client", compression, encryption);
+            connection_parameters.host,
+            connection_parameters.port,
+            connection_parameters.default_database,
+            connection_parameters.user,
+            connection_parameters.password,
+            connection_parameters.timeouts,
+            "client",
+            connection_parameters.compression,
+            connection_parameters.security);
 
         String server_name;
         UInt64 server_version_major = 0;

From cc2c1f5171b4f4586189d130461dd52daa2eb5f2 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Wed, 21 Mar 2018 22:39:14 +0300
Subject: [PATCH 190/244] removed createImmutable

---
 dbms/src/Columns/ColumnArray.cpp  |  2 +-
 dbms/src/Columns/ColumnArray.h    | 32 +++++++++++++----------
 dbms/src/Columns/ColumnNullable.h | 13 +++++++---
 dbms/src/Columns/ColumnTuple.cpp  |  2 +-
 dbms/src/Columns/ColumnTuple.h    | 13 +++++++---
 dbms/src/Columns/IColumn.h        | 12 +++++++++
 dbms/src/Common/COWPtr.h          | 42 ++-----------------------------
 7 files changed, 54 insertions(+), 62 deletions(-)

diff --git a/dbms/src/Columns/ColumnArray.cpp b/dbms/src/Columns/ColumnArray.cpp
index a943b46eb49..010569e40ad 100644
--- a/dbms/src/Columns/ColumnArray.cpp
+++ b/dbms/src/Columns/ColumnArray.cpp
@@ -34,7 +34,7 @@ namespace ErrorCodes
 ColumnArray::ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column)
     : data(std::move(nested_column)), offsets(std::move(offsets_column))
 {
-    if (!typeid_cast<const ColumnOffsets *>(offsets_column.get()))
+    if (!typeid_cast<const ColumnOffsets *>(offsets.get()))
         throw Exception("offsets_column must be a ColumnUInt64", ErrorCodes::ILLEGAL_COLUMN);
 
     /** NOTE
diff --git a/dbms/src/Columns/ColumnArray.h b/dbms/src/Columns/ColumnArray.h
index a2564b20f98..ce7744b1fc1 100644
--- a/dbms/src/Columns/ColumnArray.h
+++ b/dbms/src/Columns/ColumnArray.h
@@ -27,23 +27,29 @@ private:
     ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column);
 
     /** Create an empty column of arrays with the type of values as in the column `nested_column` */
-    ColumnArray(MutableColumnPtr && nested_column);
-
-    using Ptr = COWPtrHelper<IColumn, ColumnArray>::Ptr;
-
-    static Ptr createImmutable(const ColumnPtr & nested_column, const ColumnPtr & offsets_column)
-    {
-        return ColumnArray::create(nested_column->assumeMutable(), offsets_column->assumeMutable());
-    }
-
-    static Ptr createImmutable(const ColumnPtr & nested_column)
-    {
-        return ColumnArray::create(nested_column->assumeMutable());
-    }
+    explicit ColumnArray(MutableColumnPtr && nested_column);
 
     ColumnArray(const ColumnArray &) = default;
 
 public:
+    /** Create immutable column using immutable arguments. This arguments may be shared with other columns.
+      * Use IColumn::mutate in order to make mutable column and mutate shared nested columns.
+      */
+    using Base = COWPtrHelper<IColumn, ColumnArray>;
+
+    static Ptr create(const ColumnPtr & nested_column, const ColumnPtr & offsets_column)
+    {
+        return ColumnArray::create(nested_column->assumeMutable(), offsets_column->assumeMutable());
+    }
+
+    static Ptr create(const ColumnPtr & nested_column)
+    {
+        return ColumnArray::create(nested_column->assumeMutable());
+    }
+
+    template <typename ... Args, typename = typename std::enable_if<IsMutableColumns<Args ...>::value>::type>
+    static MutablePtr create(Args &&... args) { return Base::create(std::forward<Args>(args)...); }
+
     /** On the index i there is an offset to the beginning of the i + 1 -th element. */
     using ColumnOffsets = ColumnVector<Offset>;
 
diff --git a/dbms/src/Columns/ColumnNullable.h b/dbms/src/Columns/ColumnNullable.h
index bc082bbcb8c..4ac0f87b8da 100644
--- a/dbms/src/Columns/ColumnNullable.h
+++ b/dbms/src/Columns/ColumnNullable.h
@@ -26,14 +26,19 @@ private:
     ColumnNullable(MutableColumnPtr && nested_column_, MutableColumnPtr && null_map_);
     ColumnNullable(const ColumnNullable &) = default;
 
-    using Ptr = COWPtrHelper<IColumn, ColumnNullable>::Ptr;
-
-    static Ptr createImmutable(const ColumnPtr & nested_column_, const ColumnPtr & null_map_)
+public:
+    /** Create immutable column using immutable arguments. This arguments may be shared with other columns.
+      * Use IColumn::mutate in order to make mutable column and mutate shared nested columns.
+      */
+    using Base = COWPtrHelper<IColumn, ColumnNullable>;
+    static Ptr create(const ColumnPtr & nested_column_, const ColumnPtr & null_map_)
     {
         return ColumnNullable::create(nested_column_->assumeMutable(), null_map_->assumeMutable());
     }
 
-public:
+    template <typename ... Args, typename = typename std::enable_if<IsMutableColumns<Args ...>::value>::type>
+    static MutablePtr create(Args &&... args) { return Base::create(std::forward<Args>(args)...); }
+
     const char * getFamilyName() const override { return "Nullable"; }
     std::string getName() const override { return "Nullable(" + nested_column->getName() + ")"; }
     MutableColumnPtr cloneResized(size_t size) const override;
diff --git a/dbms/src/Columns/ColumnTuple.cpp b/dbms/src/Columns/ColumnTuple.cpp
index 4bfb6a30c2a..558b1685276 100644
--- a/dbms/src/Columns/ColumnTuple.cpp
+++ b/dbms/src/Columns/ColumnTuple.cpp
@@ -43,7 +43,7 @@ ColumnTuple::ColumnTuple(MutableColumns && mutable_columns)
     }
 }
 
-ColumnTuple::Ptr ColumnTuple::createImmutable(const Columns & columns)
+ColumnTuple::Ptr ColumnTuple::create(const Columns & columns)
 {
     for (const auto & column : columns)
         if (column->isColumnConst())
diff --git a/dbms/src/Columns/ColumnTuple.h b/dbms/src/Columns/ColumnTuple.h
index 752a649ba28..a91cb210212 100644
--- a/dbms/src/Columns/ColumnTuple.h
+++ b/dbms/src/Columns/ColumnTuple.h
@@ -22,12 +22,19 @@ private:
     template <bool positive>
     struct Less;
 
-    ColumnTuple(MutableColumns && columns);
+    explicit ColumnTuple(MutableColumns && columns);
     ColumnTuple(const ColumnTuple &) = default;
 
-    static Ptr createImmutable(const Columns & columns);
-
 public:
+    /** Create immutable column using immutable arguments. This arguments may be shared with other columns.
+      * Use IColumn::mutate in order to make mutable column and mutate shared nested columns.
+      */
+    using Base = COWPtrHelper<IColumn, ColumnTuple>;
+    static Ptr create(const Columns & columns);
+
+    template <typename Arg, typename = typename std::enable_if<std::is_rvalue_reference<Arg &&>::value>::type>
+    static MutablePtr create(Arg && arg) { return Base::create(std::forward<Arg>(arg)); }
+
     std::string getName() const override;
     const char * getFamilyName() const override { return "Tuple"; }
 
diff --git a/dbms/src/Columns/IColumn.h b/dbms/src/Columns/IColumn.h
index c546f315c0c..40577a11d3f 100644
--- a/dbms/src/Columns/IColumn.h
+++ b/dbms/src/Columns/IColumn.h
@@ -360,4 +360,16 @@ using MutableColumns = std::vector<MutableColumnPtr>;
 using ColumnRawPtrs = std::vector<const IColumn *>;
 //using MutableColumnRawPtrs = std::vector<IColumn *>;
 
+template <typename ... Args>
+struct IsMutableColumns;
+
+template <typename Arg, typename ... Args>
+struct IsMutableColumns<Arg, Args ...>
+{
+    static const bool value = std::is_assignable<MutableColumnPtr &&, Arg>::value && IsMutableColumns<Args ...>::value;
+};
+
+template <>
+struct IsMutableColumns<> { static const bool value = true; };
+
 }
diff --git a/dbms/src/Common/COWPtr.h b/dbms/src/Common/COWPtr.h
index a7fce08ae13..39c912f8abc 100644
--- a/dbms/src/Common/COWPtr.h
+++ b/dbms/src/Common/COWPtr.h
@@ -226,53 +226,15 @@ private:
     Derived * derived() { return static_cast<Derived *>(this); }
     const Derived * derived() const { return static_cast<const Derived *>(this); }
 
-    template<typename Class, typename... Args>
-    struct HasCreateImmutable
-    {
-    private:
-        template<typename T>
-        static constexpr auto check(decltype(std::declval<T>().createImmutable(std::declval<Args>()... )) *) -> std::true_type;
-
-        template<typename>
-        static constexpr std::false_type check(...);
-
-        typedef decltype(check<Class>(nullptr)) type;
-
-    public:
-        static constexpr bool value = type::value;
-    };
-
-    template<typename Class, typename... Args>
-    struct IsConstructable
-    {
-    private:
-        template<typename T>
-        static constexpr auto check(decltype(T(std::declval<Args>()... )) *) -> std::true_type;
-
-        template<typename>
-        static constexpr std::false_type check(...);
-
-        typedef decltype(check<Class>(nullptr)) type;
-
-    public:
-        static constexpr bool value = type::value;
-    };
-
 public:
     using Ptr = typename Base::template immutable_ptr<Derived>;
     using MutablePtr = typename Base::template mutable_ptr<Derived>;
 
     template <typename... Args>
-    static auto create(Args &&... args)
-    {
-        if constexpr (HasCreateImmutable<Derived, Args ...>::value && !IsConstructable<Derived, Args ...>::value)
-            return Derived::createImmutable(std::forward<Args>(args)...);
-        else
-            return MutablePtr(new Derived(std::forward<Args>(args)...));
-    }
+    static MutablePtr create(Args &&... args) { return MutablePtr(new Derived(std::forward<Args>(args)...)); }
 
     template <typename T>
-    static auto create(std::initializer_list<T> && arg) { return create(std::forward<std::initializer_list<T>>(arg)); }
+    static MutablePtr create(std::initializer_list<T> && arg) { return create(std::forward<std::initializer_list<T>>(arg)); }
 
     typename Base::MutablePtr clone() const override { return typename Base::MutablePtr(new Derived(*derived())); }
 };

From 405ad39cd42afd76b70538789f5f8ad8c49f0d3b Mon Sep 17 00:00:00 2001
From: proller <proller@github.com>
Date: Fri, 23 Mar 2018 22:45:33 +0300
Subject: [PATCH 191/244] Fix test

---
 .../0_stateless/00416_pocopatch_progress_in_http_headers.sh     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh b/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh
index 12f20a956ad..8e762515da1 100755
--- a/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh
+++ b/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 # This test will fail with external poco (progress not supported)
 
-${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}?max_block_size=1&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0" -d 'SELECT number FROM system.numbers LIMIT 10' 2>&1 | grep -E 'Content-Encoding|X-ClickHouse|^[0-9]'
+${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}?max_block_size=1&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0" -d 'SELECT number FROM system.numbers LIMIT 10' 2>&1 | grep -E 'Content-Encoding|X-ClickHouse-Progress|^[0-9]'
 ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?max_block_size=1&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0&enable_http_compression=1" -H 'Accept-Encoding: gzip' -d 'SELECT number FROM system.numbers LIMIT 10' | gzip -d
 
 # 'send_progress_in_http_headers' is false by default

From cf3c24d07d0d28fd88591f7903d5eb2d0141f4f9 Mon Sep 17 00:00:00 2001
From: Alexey Zatelepin <ztlpn@yandex-team.ru>
Date: Fri, 23 Mar 2018 19:33:51 +0300
Subject: [PATCH 192/244] break cycle of ptrs to StorageReplicatedMergeTree
 [#CLICKHOUSE-3664]

---
 dbms/src/Common/ErrorCodes.cpp                    | 1 -
 dbms/src/Databases/DatabaseDictionary.cpp         | 2 +-
 dbms/src/Databases/DatabaseMemory.cpp             | 2 +-
 dbms/src/Databases/DatabaseOrdinary.cpp           | 1 -
 dbms/src/Databases/DatabasesCommon.h              | 6 +++---
 dbms/src/Interpreters/Context.cpp                 | 1 -
 dbms/src/Interpreters/InterpreterCreateQuery.cpp  | 2 --
 dbms/src/Storages/IStorage.h                      | 1 +
 dbms/src/Storages/MergeTree/DataPartsExchange.cpp | 6 +++++-
 dbms/src/Storages/MergeTree/DataPartsExchange.h   | 5 +++--
 10 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/dbms/src/Common/ErrorCodes.cpp b/dbms/src/Common/ErrorCodes.cpp
index 46b4c3394e5..7ee9d799612 100644
--- a/dbms/src/Common/ErrorCodes.cpp
+++ b/dbms/src/Common/ErrorCodes.cpp
@@ -125,7 +125,6 @@ namespace ErrorCodes
     extern const int UNKNOWN_SETTING = 115;
     extern const int THERE_IS_NO_DEFAULT_VALUE = 116;
     extern const int INCORRECT_DATA = 117;
-    extern const int TABLE_METADATA_DOESNT_EXIST = 118;
     extern const int ENGINE_REQUIRED = 119;
     extern const int CANNOT_INSERT_VALUE_OF_DIFFERENT_SIZE_INTO_TUPLE = 120;
     extern const int UNKNOWN_SET_DATA_VARIANT = 121;
diff --git a/dbms/src/Databases/DatabaseDictionary.cpp b/dbms/src/Databases/DatabaseDictionary.cpp
index 503d0195168..2dc26abca8a 100644
--- a/dbms/src/Databases/DatabaseDictionary.cpp
+++ b/dbms/src/Databases/DatabaseDictionary.cpp
@@ -85,7 +85,7 @@ StoragePtr DatabaseDictionary::tryGetTable(
 
 DatabaseIteratorPtr DatabaseDictionary::getIterator(const Context & /*context*/)
 {
-    return std::make_unique<DatabaseSnaphotIterator>(loadTables());
+    return std::make_unique<DatabaseSnapshotIterator>(loadTables());
 }
 
 bool DatabaseDictionary::empty(const Context & /*context*/) const
diff --git a/dbms/src/Databases/DatabaseMemory.cpp b/dbms/src/Databases/DatabaseMemory.cpp
index 57089862863..b5c39773a3e 100644
--- a/dbms/src/Databases/DatabaseMemory.cpp
+++ b/dbms/src/Databases/DatabaseMemory.cpp
@@ -46,7 +46,7 @@ StoragePtr DatabaseMemory::tryGetTable(
 DatabaseIteratorPtr DatabaseMemory::getIterator(const Context & /*context*/)
 {
     std::lock_guard<std::mutex> lock(mutex);
-    return std::make_unique<DatabaseSnaphotIterator>(tables);
+    return std::make_unique<DatabaseSnapshotIterator>(tables);
 }
 
 bool DatabaseMemory::empty(const Context & /*context*/) const
diff --git a/dbms/src/Databases/DatabaseOrdinary.cpp b/dbms/src/Databases/DatabaseOrdinary.cpp
index 9dd705fbf9a..fc82b29a228 100644
--- a/dbms/src/Databases/DatabaseOrdinary.cpp
+++ b/dbms/src/Databases/DatabaseOrdinary.cpp
@@ -25,7 +25,6 @@ namespace ErrorCodes
 {
     extern const int TABLE_ALREADY_EXISTS;
     extern const int UNKNOWN_TABLE;
-    extern const int TABLE_METADATA_DOESNT_EXIST;
     extern const int CANNOT_CREATE_TABLE_FROM_METADATA;
     extern const int INCORRECT_FILE_NAME;
     extern const int FILE_DOESNT_EXIST;
diff --git a/dbms/src/Databases/DatabasesCommon.h b/dbms/src/Databases/DatabasesCommon.h
index c13eda7c9c9..2a2ccbb26ec 100644
--- a/dbms/src/Databases/DatabasesCommon.h
+++ b/dbms/src/Databases/DatabasesCommon.h
@@ -36,17 +36,17 @@ std::pair<String, StoragePtr> createTableFromDefinition(
 
 
 /// Copies list of tables and iterates through such snapshot.
-class DatabaseSnaphotIterator : public IDatabaseIterator
+class DatabaseSnapshotIterator : public IDatabaseIterator
 {
 private:
     Tables tables;
     Tables::iterator it;
 
 public:
-    DatabaseSnaphotIterator(Tables & tables_)
+    DatabaseSnapshotIterator(Tables & tables_)
         : tables(tables_), it(tables.begin()) {}
 
-    DatabaseSnaphotIterator(Tables && tables_)
+    DatabaseSnapshotIterator(Tables && tables_)
         : tables(tables_), it(tables.begin()) {}
 
     void next() override
diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp
index 8fe51bf5c08..a0b182dbc13 100644
--- a/dbms/src/Interpreters/Context.cpp
+++ b/dbms/src/Interpreters/Context.cpp
@@ -74,7 +74,6 @@ namespace ErrorCodes
     extern const int TABLE_ALREADY_EXISTS;
     extern const int TABLE_WAS_NOT_DROPPED;
     extern const int DATABASE_ALREADY_EXISTS;
-    extern const int TABLE_METADATA_DOESNT_EXIST;
     extern const int THERE_IS_NO_SESSION;
     extern const int THERE_IS_NO_QUERY;
     extern const int NO_ELEMENTS_IN_CONFIG;
diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp
index 3d70fc343b9..951861bd174 100644
--- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp
@@ -46,8 +46,6 @@ namespace DB
 
 namespace ErrorCodes
 {
-    extern const int DIRECTORY_DOESNT_EXIST;
-    extern const int DIRECTORY_ALREADY_EXISTS;
     extern const int TABLE_ALREADY_EXISTS;
     extern const int EMPTY_LIST_OF_COLUMNS_PASSED;
     extern const int INCORRECT_QUERY;
diff --git a/dbms/src/Storages/IStorage.h b/dbms/src/Storages/IStorage.h
index ef9fd8004ce..74db0604147 100644
--- a/dbms/src/Storages/IStorage.h
+++ b/dbms/src/Storages/IStorage.h
@@ -35,6 +35,7 @@ class ASTCreateQuery;
 class IStorage;
 
 using StoragePtr = std::shared_ptr<IStorage>;
+using StorageWeakPtr = std::weak_ptr<IStorage>;
 
 struct Settings;
 
diff --git a/dbms/src/Storages/MergeTree/DataPartsExchange.cpp b/dbms/src/Storages/MergeTree/DataPartsExchange.cpp
index b2ff96e0fe3..f205c35ee7f 100644
--- a/dbms/src/Storages/MergeTree/DataPartsExchange.cpp
+++ b/dbms/src/Storages/MergeTree/DataPartsExchange.cpp
@@ -24,6 +24,7 @@ namespace ErrorCodes
     extern const int ABORTED;
     extern const int BAD_SIZE_OF_FILE_IN_DATA_PART;
     extern const int CANNOT_WRITE_TO_OSTREAM;
+    extern const int UNKNOWN_TABLE;
 }
 
 namespace DataPartsExchange
@@ -68,6 +69,9 @@ void Service::processQuery(const Poco::Net::HTMLForm & params, ReadBuffer & /*bo
     ++data.current_table_sends;
     SCOPE_EXIT({--data.current_table_sends;});
 
+    StoragePtr owned_storage = storage.lock();
+    if (!owned_storage)
+        throw Exception("The table was already dropped", ErrorCodes::UNKNOWN_TABLE);
 
     LOG_TRACE(log, "Sending part " << part_name);
 
@@ -122,7 +126,7 @@ void Service::processQuery(const Poco::Net::HTMLForm & params, ReadBuffer & /*bo
     }
     catch (const NetException & e)
     {
-        /// Network error or error on remote side. No need to enquue part for check.
+        /// Network error or error on remote side. No need to enqueue part for check.
         throw;
     }
     catch (const Exception & e)
diff --git a/dbms/src/Storages/MergeTree/DataPartsExchange.h b/dbms/src/Storages/MergeTree/DataPartsExchange.h
index c498a7f8f0c..a9e58956c5a 100644
--- a/dbms/src/Storages/MergeTree/DataPartsExchange.h
+++ b/dbms/src/Storages/MergeTree/DataPartsExchange.h
@@ -2,6 +2,7 @@
 
 #include <Interpreters/InterserverIOHandler.h>
 #include <Storages/MergeTree/MergeTreeData.h>
+#include <Storages/IStorage.h>
 #include <IO/HashingWriteBuffer.h>
 #include <IO/copyData.h>
 #include <IO/ConnectionTimeouts.h>
@@ -19,7 +20,7 @@ class Service final : public InterserverIOEndpoint
 {
 public:
     Service(MergeTreeData & data_, StoragePtr & storage_) : data(data_),
-        owned_storage(storage_), log(&Logger::get(data.getLogName() + " (Replicated PartsService)")) {}
+        storage(storage_), log(&Logger::get(data.getLogName() + " (Replicated PartsService)")) {}
 
     Service(const Service &) = delete;
     Service & operator=(const Service &) = delete;
@@ -32,7 +33,7 @@ private:
 
 private:
     MergeTreeData & data;
-    StoragePtr owned_storage;
+    StorageWeakPtr storage;
     Logger * log;
 };
 

From dd06b39e5fd52fee48852e2bcae4f9ee7e5db962 Mon Sep 17 00:00:00 2001
From: Tobias Adamson <tobias@stargazer.com.sg>
Date: Sat, 24 Mar 2018 09:16:33 +0800
Subject: [PATCH 193/244] Fix rename of variable, duplication of wrong variable
 name

---
 dbms/src/Storages/StorageKafka.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dbms/src/Storages/StorageKafka.cpp b/dbms/src/Storages/StorageKafka.cpp
index d82b12a6fb2..de6ffeaff05 100644
--- a/dbms/src/Storages/StorageKafka.cpp
+++ b/dbms/src/Storages/StorageKafka.cpp
@@ -578,7 +578,7 @@ void registerStorageKafka(StorageFactory & factory)
         // Parse topic list
         Names topics;
         String topicArg = static_cast<const ASTLiteral &>(*engine_args[1]).value.safeGet<String>();
-        boost::split(topics, topics , [](char c){return c == ',';});
+        boost::split(topics, topicArg , [](char c){return c == ',';});
         
         // Parse consumer group
         String group = static_cast<const ASTLiteral &>(*engine_args[2]).value.safeGet<String>();

From 0af09ae0c7a07b281d76a0787820f67c3da98f24 Mon Sep 17 00:00:00 2001
From: Tobias Adamson <tobias@stargazer.com.sg>
Date: Sat, 24 Mar 2018 14:51:35 +0800
Subject: [PATCH 194/244] Add trim to topic names to be a bit more lenient if
 config is 'topic1, topic2'

---
 dbms/src/Storages/StorageKafka.cpp | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/dbms/src/Storages/StorageKafka.cpp b/dbms/src/Storages/StorageKafka.cpp
index de6ffeaff05..4ec2307ef13 100644
--- a/dbms/src/Storages/StorageKafka.cpp
+++ b/dbms/src/Storages/StorageKafka.cpp
@@ -5,6 +5,7 @@
 #include <thread>
 #include <boost/algorithm/string/replace.hpp>
 #include <boost/algorithm/string/split.hpp>
+#include <boost/algorithm/string/trim.hpp>
 #include <Common/Exception.h>
 #include <Common/setThreadName.h>
 #include <Common/typeid_cast.h>
@@ -579,7 +580,9 @@ void registerStorageKafka(StorageFactory & factory)
         Names topics;
         String topicArg = static_cast<const ASTLiteral &>(*engine_args[1]).value.safeGet<String>();
         boost::split(topics, topicArg , [](char c){return c == ',';});
-        
+        for(String & topic : topics) {
+            boost::trim(topic);
+        }
         // Parse consumer group
         String group = static_cast<const ASTLiteral &>(*engine_args[2]).value.safeGet<String>();
 

From 10b669a8338ab4f9890f281caf4cdc125952e55f Mon Sep 17 00:00:00 2001
From: Alexey Zatelepin <ztlpn@yandex-team.ru>
Date: Fri, 23 Mar 2018 23:46:43 +0300
Subject: [PATCH 195/244] extract common base for DatabaseMemory and
 DatabaseOrdinary [#CLICKHOUSE-3664]

---
 dbms/src/Databases/DatabaseMemory.cpp   | 83 ++-----------------------
 dbms/src/Databases/DatabaseMemory.h     | 35 ++---------
 dbms/src/Databases/DatabaseOrdinary.cpp | 10 +--
 dbms/src/Databases/DatabaseOrdinary.h   | 14 ++---
 dbms/src/Databases/DatabasesCommon.cpp  | 77 +++++++++++++++++++++++
 dbms/src/Databases/DatabasesCommon.h    | 32 ++++++++++
 dbms/src/Server/LocalServer.cpp         |  2 +-
 7 files changed, 133 insertions(+), 120 deletions(-)

diff --git a/dbms/src/Databases/DatabaseMemory.cpp b/dbms/src/Databases/DatabaseMemory.cpp
index b5c39773a3e..7eb5053e9cb 100644
--- a/dbms/src/Databases/DatabaseMemory.cpp
+++ b/dbms/src/Databases/DatabaseMemory.cpp
@@ -8,75 +8,22 @@ namespace DB
 
 namespace ErrorCodes
 {
-    extern const int TABLE_ALREADY_EXISTS;
-    extern const int UNKNOWN_TABLE;
-    extern const int LOGICAL_ERROR;
     extern const int CANNOT_GET_CREATE_TABLE_QUERY;
 }
 
+DatabaseMemory::DatabaseMemory(String name_)
+    : DatabaseWithOwnTablesBase(std::move(name_))
+    , log(&Logger::get("DatabaseMemory(" + name + ")"))
+{}
+
 void DatabaseMemory::loadTables(
     Context & /*context*/,
     ThreadPool * /*thread_pool*/,
     bool /*has_force_restore_data_flag*/)
 {
-    log = &Logger::get("DatabaseMemory(" + name + ")");
-
     /// Nothing to load.
 }
 
-bool DatabaseMemory::isTableExist(
-    const Context & /*context*/,
-    const String & table_name) const
-{
-    std::lock_guard<std::mutex> lock(mutex);
-    return tables.find(table_name) != tables.end();
-}
-
-StoragePtr DatabaseMemory::tryGetTable(
-    const Context & /*context*/,
-    const String & table_name) const
-{
-    std::lock_guard<std::mutex> lock(mutex);
-    auto it = tables.find(table_name);
-    if (it == tables.end())
-        return {};
-    return it->second;
-}
-
-DatabaseIteratorPtr DatabaseMemory::getIterator(const Context & /*context*/)
-{
-    std::lock_guard<std::mutex> lock(mutex);
-    return std::make_unique<DatabaseSnapshotIterator>(tables);
-}
-
-bool DatabaseMemory::empty(const Context & /*context*/) const
-{
-    std::lock_guard<std::mutex> lock(mutex);
-    return tables.empty();
-}
-
-StoragePtr DatabaseMemory::detachTable(const String & table_name)
-{
-    StoragePtr res;
-    {
-        std::lock_guard<std::mutex> lock(mutex);
-        auto it = tables.find(table_name);
-        if (it == tables.end())
-            throw Exception("Table " + name + "." + table_name + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
-        res = it->second;
-        tables.erase(it);
-    }
-
-    return res;
-}
-
-void DatabaseMemory::attachTable(const String & table_name, const StoragePtr & table)
-{
-    std::lock_guard<std::mutex> lock(mutex);
-    if (!tables.emplace(table_name, table).second)
-        throw Exception("Table " + name + "." + table_name + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS);
-}
-
 void DatabaseMemory::createTable(
     const Context & /*context*/,
     const String & table_name,
@@ -125,26 +72,6 @@ ASTPtr DatabaseMemory::getCreateQuery(
     throw Exception("There is no CREATE TABLE query for DatabaseMemory tables", ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY);
 }
 
-void DatabaseMemory::shutdown()
-{
-    /// You can not hold a lock during shutdown.
-    /// Because inside `shutdown` function tables can work with database, and mutex is not recursive.
-
-    Tables tables_snapshot;
-    {
-        std::lock_guard<std::mutex> lock(mutex);
-        tables_snapshot = tables;
-    }
-
-    for (const auto & kv: tables_snapshot)
-    {
-        kv.second->shutdown();
-    }
-
-    std::lock_guard<std::mutex> lock(mutex);
-    tables.clear();
-}
-
 void DatabaseMemory::drop()
 {
     /// Additional actions to delete database are not required.
diff --git a/dbms/src/Databases/DatabaseMemory.h b/dbms/src/Databases/DatabaseMemory.h
index 586dcfbe4da..d4599a5c342 100644
--- a/dbms/src/Databases/DatabaseMemory.h
+++ b/dbms/src/Databases/DatabaseMemory.h
@@ -1,8 +1,6 @@
 #pragma once
 
-#include <mutex>
-#include <Databases/IDatabase.h>
-#include <Storages/IStorage.h>
+#include <Databases/DatabasesCommon.h>
 
 
 namespace Poco { class Logger; }
@@ -16,18 +14,10 @@ namespace DB
   * All tables are created by calling code.
   * TODO: Maybe DatabaseRuntime is more suitable class name.
   */
-class DatabaseMemory : public IDatabase
+class DatabaseMemory : public DatabaseWithOwnTablesBase
 {
-protected:
-    const String name;
-    mutable std::mutex mutex;
-    Tables tables;
-
-    Poco::Logger * log;
-
 public:
-
-    DatabaseMemory(const String & name_) : name(name_) {}
+    DatabaseMemory(String name_);
 
     String getEngineName() const override { return "Memory"; }
 
@@ -36,18 +26,6 @@ public:
         ThreadPool * thread_pool,
         bool has_force_restore_data_flag) override;
 
-    bool empty(const Context & context) const override;
-
-    DatabaseIteratorPtr getIterator(const Context & context) override;
-
-    bool isTableExist(
-        const Context & context,
-        const String & table_name) const override;
-
-    StoragePtr tryGetTable(
-        const Context & context,
-        const String & table_name) const override;
-
     void createTable(
         const Context & context,
         const String & table_name,
@@ -58,9 +36,6 @@ public:
         const Context & context,
         const String & table_name) override;
 
-    void attachTable(const String & table_name, const StoragePtr & table) override;
-    StoragePtr detachTable(const String & table_name) override;
-
     void renameTable(
         const Context & context,
         const String & table_name,
@@ -81,8 +56,10 @@ public:
         const Context & context,
         const String & table_name) const override;
 
-    void shutdown() override;
     void drop() override;
+
+private:
+    Poco::Logger * log;
 };
 
 }
diff --git a/dbms/src/Databases/DatabaseOrdinary.cpp b/dbms/src/Databases/DatabaseOrdinary.cpp
index fc82b29a228..aa424e1118c 100644
--- a/dbms/src/Databases/DatabaseOrdinary.cpp
+++ b/dbms/src/Databases/DatabaseOrdinary.cpp
@@ -93,8 +93,11 @@ static void loadTable(
 }
 
 
-DatabaseOrdinary::DatabaseOrdinary(const String & name_, const String & metadata_path_, const Context & context)
-    : DatabaseMemory(name_), metadata_path(metadata_path_), data_path(context.getPath() + "data/" + escapeForFileName(name_) + "/")
+DatabaseOrdinary::DatabaseOrdinary(String name_, const String & metadata_path_, const Context & context)
+    : DatabaseWithOwnTablesBase(std::move(name_))
+    , metadata_path(metadata_path_)
+    , data_path(context.getPath() + "data/" + escapeForFileName(name) + "/")
+    , log(&Logger::get("DatabaseOrdinary (" + name + ")"))
 {
     Poco::File(data_path).createDirectories();
 }
@@ -105,8 +108,6 @@ void DatabaseOrdinary::loadTables(
     ThreadPool * thread_pool,
     bool has_force_restore_data_flag)
 {
-    log = &Logger::get("DatabaseOrdinary (" + name + ")");
-
     using FileNames = std::vector<std::string>;
     FileNames file_names;
 
@@ -457,7 +458,6 @@ void DatabaseOrdinary::drop()
     /// No additional removal actions are required.
 }
 
-
 void DatabaseOrdinary::alterTable(
     const Context & context,
     const String & name,
diff --git a/dbms/src/Databases/DatabaseOrdinary.h b/dbms/src/Databases/DatabaseOrdinary.h
index 25e99114430..5902ee73a76 100644
--- a/dbms/src/Databases/DatabaseOrdinary.h
+++ b/dbms/src/Databases/DatabaseOrdinary.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#include <Databases/DatabaseMemory.h>
+#include <Databases/DatabasesCommon.h>
 
 
 namespace DB
@@ -10,14 +10,10 @@ namespace DB
   * It stores tables list in filesystem using list of .sql files,
   *  that contain declaration of table represented by SQL ATTACH TABLE query.
   */
-class DatabaseOrdinary : public DatabaseMemory
+class DatabaseOrdinary : public DatabaseWithOwnTablesBase
 {
-protected:
-    const String metadata_path;
-    const String data_path;
-
 public:
-    DatabaseOrdinary(const String & name_, const String & metadata_path_, const Context & context);
+    DatabaseOrdinary(String name_, const String & metadata_path_, const Context & context);
 
     String getEngineName() const override { return "Ordinary"; }
 
@@ -64,6 +60,10 @@ public:
     void drop() override;
 
 private:
+    const String metadata_path;
+    const String data_path;
+    Poco::Logger * log;
+
     void startupTables(ThreadPool * thread_pool);
 };
 
diff --git a/dbms/src/Databases/DatabasesCommon.cpp b/dbms/src/Databases/DatabasesCommon.cpp
index 66d05d11184..04511f7b9ce 100644
--- a/dbms/src/Databases/DatabasesCommon.cpp
+++ b/dbms/src/Databases/DatabasesCommon.cpp
@@ -16,6 +16,9 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int EMPTY_LIST_OF_COLUMNS_PASSED;
+    extern const int TABLE_ALREADY_EXISTS;
+    extern const int UNKNOWN_TABLE;
+    extern const int LOGICAL_ERROR;
 }
 
 
@@ -80,4 +83,78 @@ std::pair<String, StoragePtr> createTableFromDefinition(
     };
 }
 
+
+bool DatabaseWithOwnTablesBase::isTableExist(
+    const Context & /*context*/,
+    const String & table_name) const
+{
+    std::lock_guard<std::mutex> lock(mutex);
+    return tables.find(table_name) != tables.end();
+}
+
+StoragePtr DatabaseWithOwnTablesBase::tryGetTable(
+    const Context & /*context*/,
+    const String & table_name) const
+{
+    std::lock_guard<std::mutex> lock(mutex);
+    auto it = tables.find(table_name);
+    if (it == tables.end())
+        return {};
+    return it->second;
+}
+
+DatabaseIteratorPtr DatabaseWithOwnTablesBase::getIterator(const Context & /*context*/)
+{
+    std::lock_guard<std::mutex> lock(mutex);
+    return std::make_unique<DatabaseSnapshotIterator>(tables);
+}
+
+bool DatabaseWithOwnTablesBase::empty(const Context & /*context*/) const
+{
+    std::lock_guard<std::mutex> lock(mutex);
+    return tables.empty();
+}
+
+StoragePtr DatabaseWithOwnTablesBase::detachTable(const String & table_name)
+{
+    StoragePtr res;
+    {
+        std::lock_guard<std::mutex> lock(mutex);
+        auto it = tables.find(table_name);
+        if (it == tables.end())
+            throw Exception("Table " + name + "." + table_name + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
+        res = it->second;
+        tables.erase(it);
+    }
+
+    return res;
+}
+
+void DatabaseWithOwnTablesBase::attachTable(const String & table_name, const StoragePtr & table)
+{
+    std::lock_guard<std::mutex> lock(mutex);
+    if (!tables.emplace(table_name, table).second)
+        throw Exception("Table " + name + "." + table_name + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS);
+}
+
+void DatabaseWithOwnTablesBase::shutdown()
+{
+    /// You can not hold a lock during shutdown.
+    /// Because inside `shutdown` function tables can work with database, and mutex is not recursive.
+
+    Tables tables_snapshot;
+    {
+        std::lock_guard<std::mutex> lock(mutex);
+        tables_snapshot = tables;
+    }
+
+    for (const auto & kv: tables_snapshot)
+    {
+        kv.second->shutdown();
+    }
+
+    std::lock_guard<std::mutex> lock(mutex);
+    tables.clear();
+}
+
 }
diff --git a/dbms/src/Databases/DatabasesCommon.h b/dbms/src/Databases/DatabasesCommon.h
index 2a2ccbb26ec..3f1d8f9d800 100644
--- a/dbms/src/Databases/DatabasesCommon.h
+++ b/dbms/src/Databases/DatabasesCommon.h
@@ -70,4 +70,36 @@ public:
     }
 };
 
+/// A base class for databases that manage their own list of tables.
+class DatabaseWithOwnTablesBase : public IDatabase
+{
+public:
+    bool isTableExist(
+        const Context & context,
+        const String & table_name) const override;
+
+    StoragePtr tryGetTable(
+        const Context & context,
+        const String & table_name) const override;
+
+    bool empty(const Context & context) const override;
+
+
+    void attachTable(const String & table_name, const StoragePtr & table) override;
+
+    StoragePtr detachTable(const String & table_name) override;
+
+    DatabaseIteratorPtr getIterator(const Context & context) override;
+
+    void shutdown() override;
+
+protected:
+    String name;
+
+    mutable std::mutex mutex;
+    Tables tables;
+
+    DatabaseWithOwnTablesBase(String name_) : name(std::move(name_)) { }
+};
+
 }
diff --git a/dbms/src/Server/LocalServer.cpp b/dbms/src/Server/LocalServer.cpp
index 1196c62f06f..035e08c4d3f 100644
--- a/dbms/src/Server/LocalServer.cpp
+++ b/dbms/src/Server/LocalServer.cpp
@@ -6,7 +6,7 @@
 #include <Poco/String.h>
 #include <Poco/Logger.h>
 #include <Poco/NullChannel.h>
-#include <Databases/DatabaseOrdinary.h>
+#include <Databases/DatabaseMemory.h>
 #include <Storages/System/attachSystemTables.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/ProcessList.h>

From 2a366afbf5b73566f29996c8c9c443e0e8bcc94f Mon Sep 17 00:00:00 2001
From: Alexey Zatelepin <ztlpn@yandex-team.ru>
Date: Fri, 23 Mar 2018 23:56:45 +0300
Subject: [PATCH 196/244] add shutdowns everywhere [#CLICKHOUSE-3664]

---
 dbms/src/Databases/DatabasesCommon.cpp         | 12 ++++++++++++
 dbms/src/Databases/DatabasesCommon.h           |  2 ++
 dbms/src/Interpreters/InterpreterDropQuery.cpp |  3 ++-
 3 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/dbms/src/Databases/DatabasesCommon.cpp b/dbms/src/Databases/DatabasesCommon.cpp
index 04511f7b9ce..efc3d8915eb 100644
--- a/dbms/src/Databases/DatabasesCommon.cpp
+++ b/dbms/src/Databases/DatabasesCommon.cpp
@@ -157,4 +157,16 @@ void DatabaseWithOwnTablesBase::shutdown()
     tables.clear();
 }
 
+DatabaseWithOwnTablesBase::~DatabaseWithOwnTablesBase()
+{
+    try
+    {
+        shutdown();
+    }
+    catch(...)
+    {
+        tryLogCurrentException(__PRETTY_FUNCTION__);
+    }
+}
+
 }
diff --git a/dbms/src/Databases/DatabasesCommon.h b/dbms/src/Databases/DatabasesCommon.h
index 3f1d8f9d800..b1b2bc910a5 100644
--- a/dbms/src/Databases/DatabasesCommon.h
+++ b/dbms/src/Databases/DatabasesCommon.h
@@ -93,6 +93,8 @@ public:
 
     void shutdown() override;
 
+    virtual ~DatabaseWithOwnTablesBase() override;
+
 protected:
     String name;
 
diff --git a/dbms/src/Interpreters/InterpreterDropQuery.cpp b/dbms/src/Interpreters/InterpreterDropQuery.cpp
index d99d9052d6b..0fdf2b1ccf4 100644
--- a/dbms/src/Interpreters/InterpreterDropQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterDropQuery.cpp
@@ -41,7 +41,8 @@ BlockIO InterpreterDropQuery::execute()
 
     if (drop_database && drop.detach)
     {
-        context.detachDatabase(drop.database);
+        auto database = context.detachDatabase(drop.database);
+        database->shutdown();
         return {};
     }
 

From d406454f392ae3ca9ef92a42d4b005070d483d7e Mon Sep 17 00:00:00 2001
From: elBroom <Zarina1458@yandex.ru>
Date: Sun, 25 Mar 2018 02:22:58 +0300
Subject: [PATCH 197/244] Added query_id to client

---
 dbms/src/Server/Client.cpp | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp
index 47239c97c6c..3f923209c5d 100644
--- a/dbms/src/Server/Client.cpp
+++ b/dbms/src/Server/Client.cpp
@@ -731,7 +731,8 @@ private:
     /// Process the query that doesn't require transfering data blocks to the server.
     void processOrdinaryQuery()
     {
-        connection->sendQuery(query, "", QueryProcessingStage::Complete, &context.getSettingsRef(), nullptr, true);
+        String query_id = config().getString("query_id", "");
+        connection->sendQuery(query, query_id, QueryProcessingStage::Complete, &context.getSettingsRef(), nullptr, true);
         sendExternalTables();
         receiveResult();
     }
@@ -740,6 +741,7 @@ private:
     /// Process the query that requires transfering data blocks to the server.
     void processInsertQuery()
     {
+        String query_id = config().getString("query_id", "");
         /// Send part of query without data, because data will be sent separately.
         const ASTInsertQuery & parsed_insert_query = typeid_cast<const ASTInsertQuery &>(*parsed_query);
         String query_without_data = parsed_insert_query.data
@@ -749,7 +751,7 @@ private:
         if (!parsed_insert_query.data && (is_interactive || (stdin_is_not_tty && std_in.eof())))
             throw Exception("No data to insert", ErrorCodes::NO_DATA_TO_INSERT);
 
-        connection->sendQuery(query_without_data, "", QueryProcessingStage::Complete, &context.getSettingsRef(), nullptr, true);
+        connection->sendQuery(query_without_data, query_id, QueryProcessingStage::Complete, &context.getSettingsRef(), nullptr, true);
         sendExternalTables();
 
         /// Receive description of table structure.
@@ -1286,6 +1288,7 @@ public:
             ("ssl,s", "ssl")
             ("user,u", boost::program_options::value<std::string>(), "user")
             ("password", boost::program_options::value<std::string>(), "password")
+            ("query_id", boost::program_options::value<std::string>(), "query_id")
             ("query,q", boost::program_options::value<std::string>(), "query")
             ("database,d", boost::program_options::value<std::string>(), "database")
             ("pager", boost::program_options::value<std::string>(), "pager")
@@ -1377,6 +1380,8 @@ public:
             config().setString("config-file", options["config-file"].as<std::string>());
         if (options.count("host") && !options["host"].defaulted())
             config().setString("host", options["host"].as<std::string>());
+        if (options.count("query_id"))
+            config().setString("query_id", options["query_id"].as<std::string>());
         if (options.count("query"))
             config().setString("query", options["query"].as<std::string>());
         if (options.count("database"))

From b80f5ed8750ed2ac83f45b4db70fc34ba8e0db7c Mon Sep 17 00:00:00 2001
From: elBroom <Zarina1458@yandex.ru>
Date: Sun, 25 Mar 2018 02:50:30 +0300
Subject: [PATCH 198/244] ignoring query_id in interactive mode

---
 dbms/src/Server/Client.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp
index 3f923209c5d..1a8d97af25a 100644
--- a/dbms/src/Server/Client.cpp
+++ b/dbms/src/Server/Client.cpp
@@ -111,6 +111,7 @@ private:
     winsize terminal_size {};            /// Terminal size is needed to render progress bar.
 
     std::unique_ptr<Connection> connection;    /// Connection to DB.
+    String query_id;                     /// Current query_id.
     String query;                        /// Current query.
 
     String format;                       /// Query results output format.
@@ -336,6 +337,7 @@ private:
 
         if (is_interactive)
         {
+            query_id = "";
             if (print_time_to_stderr)
                 throw Exception("time option could be specified only in non-interactive mode", ErrorCodes::BAD_ARGUMENTS);
 
@@ -370,6 +372,7 @@ private:
         }
         else
         {
+            query_id = config().getString("query_id", "");
             nonInteractive();
 
             if (last_exception)
@@ -731,7 +734,6 @@ private:
     /// Process the query that doesn't require transfering data blocks to the server.
     void processOrdinaryQuery()
     {
-        String query_id = config().getString("query_id", "");
         connection->sendQuery(query, query_id, QueryProcessingStage::Complete, &context.getSettingsRef(), nullptr, true);
         sendExternalTables();
         receiveResult();
@@ -741,7 +743,6 @@ private:
     /// Process the query that requires transfering data blocks to the server.
     void processInsertQuery()
     {
-        String query_id = config().getString("query_id", "");
         /// Send part of query without data, because data will be sent separately.
         const ASTInsertQuery & parsed_insert_query = typeid_cast<const ASTInsertQuery &>(*parsed_query);
         String query_without_data = parsed_insert_query.data

From a011a466aba86bc37bcccddeb70eec83ea802e4a Mon Sep 17 00:00:00 2001
From: BayoNet <bayonet@yandex-team.ru>
Date: Sun, 25 Mar 2018 05:04:22 +0300
Subject: [PATCH 199/244] English translation update.

---
 docs/en/agg_functions/combinators.md          |   0
 docs/en/agg_functions/index.md                |   1 +
 docs/en/agg_functions/parametric_functions.md |   0
 docs/en/agg_functions/reference.md            |  34 ++--
 docs/en/data_types/array.md                   |   0
 docs/en/data_types/boolean.md                 |   0
 docs/en/data_types/date.md                    |   2 +-
 docs/en/data_types/datetime.md                |   0
 docs/en/data_types/enum.md                    |   0
 docs/en/data_types/fixedstring.md             |   0
 docs/en/data_types/float.md                   |  10 +-
 docs/en/data_types/index.md                   |   5 +-
 docs/en/data_types/int_uint.md                |   0
 .../aggregatefunction.md                      |   0
 .../nested_data_structures/index.md           |   0
 .../nested_data_structures/nested.md          |   0
 .../special_data_types/expression.md          |   0
 .../en/data_types/special_data_types/index.md |   0
 docs/en/data_types/special_data_types/set.md  |   0
 docs/en/data_types/string.md                  |   0
 docs/en/data_types/tuple.md                   |   0
 docs/en/development/style.md                  |  65 +++----
 docs/en/dicts/external_dicts.md               |   5 +-
 docs/en/dicts/external_dicts_dict.md          |   9 +-
 docs/en/dicts/external_dicts_dict_layout.md   |  99 ++---------
 docs/en/dicts/external_dicts_dict_lifetime.md |   6 +-
 docs/en/dicts/external_dicts_dict_sources.md  |   6 +-
 .../en/dicts/external_dicts_dict_structure.md |  14 +-
 docs/en/dicts/index.md                        |   0
 docs/en/dicts/internal_dicts.md               |   0
 docs/en/formats/capnproto.md                  |  52 +++---
 docs/en/formats/csv.md                        |   0
 docs/en/formats/csvwithnames.md               |   0
 docs/en/formats/index.md                      |   1 +
 docs/en/formats/json.md                       |   0
 docs/en/formats/jsoncompact.md                |   2 +-
 docs/en/formats/jsoneachrow.md                |   0
 docs/en/formats/native.md                     |   0
 docs/en/formats/null.md                       |   0
 docs/en/formats/pretty.md                     |   0
 docs/en/formats/prettycompact.md              |   0
 docs/en/formats/prettycompactmonoblock.md     |   0
 docs/en/formats/prettynoescapes.md            |   0
 docs/en/formats/prettyspace.md                |   0
 docs/en/formats/rowbinary.md                  |   2 +-
 docs/en/formats/tabseparated.md               |   0
 docs/en/formats/tabseparatedraw.md            |   0
 docs/en/formats/tabseparatedwithnames.md      |   0
 .../formats/tabseparatedwithnamesandtypes.md  |   0
 docs/en/formats/tskv.md                       |   0
 docs/en/formats/values.md                     |   2 +-
 docs/en/formats/vertical.md                   |   0
 docs/en/formats/xml.md                        |   2 +-
 docs/en/functions/arithmetic_functions.md     |   0
 docs/en/functions/array_functions.md          |  12 +-
 docs/en/functions/array_join.md               |   1 -
 docs/en/functions/bit_functions.md            |   1 -
 docs/en/functions/comparison_functions.md     |   2 +-
 docs/en/functions/conditional_functions.md    |   0
 docs/en/functions/date_time_functions.md      |   4 +
 docs/en/functions/encoding_functions.md       |   0
 docs/en/functions/ext_dict_functions.md       |   8 +-
 docs/en/functions/hash_functions.md           |   0
 docs/en/functions/higher_order_functions.md   |   3 +-
 docs/en/functions/in_functions.md             |   0
 docs/en/functions/index.md                    |   0
 docs/en/functions/ip_address_functions.md     |   0
 docs/en/functions/json_functions.md           |   9 +-
 docs/en/functions/logical_functions.md        |   1 -
 docs/en/functions/math_functions.md           |   1 -
 docs/en/functions/other_functions.md          |   5 +-
 docs/en/functions/random_functions.md         |   0
 docs/en/functions/rounding_functions.md       |   0
 .../functions/splitting_merging_functions.md  |   0
 docs/en/functions/string_functions.md         |   0
 docs/en/functions/string_replace_functions.md |   1 -
 docs/en/functions/string_search_functions.md  |   0
 .../en/functions/type_conversion_functions.md |   0
 docs/en/functions/url_functions.md            |   0
 docs/en/functions/ym_dict_functions.md        |  12 +-
 .../example_datasets/amplab_benchmark.md      |   0
 .../example_datasets/criteo.md                |   4 +-
 .../example_datasets/nyc_taxi.md              |  15 +-
 .../example_datasets/ontime.md                |   0
 .../example_datasets/star_schema.md           |   3 +-
 .../example_datasets/wikistat.md              |   2 +-
 docs/en/getting_started/index.md              |  14 +-
 docs/en/index.md                              |   2 +-
 docs/en/interfaces/cli.md                     |   7 +-
 docs/en/interfaces/http_interface.md          |  16 +-
 docs/en/interfaces/index.md                   |   3 +-
 docs/en/interfaces/jdbc.md                    |   0
 docs/en/interfaces/tcp.md                     |   0
 .../third-party_client_libraries.md           |   2 +-
 docs/en/interfaces/third-party_gui.md         |   0
 docs/en/introduction/distinctive_features.md  |  20 +--
 .../features_considered_disadvantages.md      |   0
 docs/en/introduction/index.md                 |   1 +
 docs/en/introduction/performance.md           |   0
 .../introduction/possible_silly_questions.md  |   4 +-
 docs/en/introduction/ya_metrika_task.md       |   4 +-
 docs/en/operations/access_rights.md           |  59 +++----
 docs/en/operations/configuration_files.md     |   2 +-
 docs/en/operations/index.md                   |   3 +-
 docs/en/operations/quotas.md                  |  35 ++--
 docs/en/operations/server_settings/index.md   |   1 +
 .../en/operations/server_settings/settings.md |  37 ++--
 docs/en/operations/settings/index.md          |   4 +-
 .../operations/settings/query_complexity.md   |   0
 docs/en/operations/settings/settings.md       |  18 +-
 .../operations/settings/settings_profiles.md  |   2 +-
 docs/en/operations/tips.md                    |   5 +-
 docs/en/operators/index.md                    |   4 +-
 docs/en/query_language/index.md               |   1 +
 docs/en/query_language/queries.md             |  53 +++---
 docs/en/query_language/syntax.md              |   2 +-
 docs/en/roadmap.md                            |  70 ++++----
 docs/en/system_tables/index.md                |   2 +-
 .../system.asynchronous_metrics.md            |   0
 docs/en/system_tables/system.clusters.md      |  17 +-
 docs/en/system_tables/system.columns.md       |   1 -
 docs/en/system_tables/system.databases.md     |   0
 docs/en/system_tables/system.dictionaries.md  |  26 +--
 docs/en/system_tables/system.events.md        |   0
 docs/en/system_tables/system.functions.md     |   2 +-
 docs/en/system_tables/system.merges.md        |   1 -
 docs/en/system_tables/system.numbers.md       |   0
 docs/en/system_tables/system.numbers_mt.md    |   0
 docs/en/system_tables/system.one.md           |   0
 docs/en/system_tables/system.parts.md         |   0
 docs/en/system_tables/system.processes.md     |   6 +-
 docs/en/system_tables/system.replicas.md      |  26 +--
 docs/en/system_tables/system.settings.md      |   4 +-
 docs/en/system_tables/system.tables.md        |   3 +-
 docs/en/system_tables/system.zookeeper.md     |   1 -
 docs/en/table_engines/aggregatingmergetree.md |  31 ++--
 docs/en/table_engines/buffer.md               |   0
 docs/en/table_engines/collapsingmergetree.md  |   0
 .../table_engines/custom_partitioning_key.md  |  14 +-
 docs/en/table_engines/dictionary.md           | 106 ++++++++++++
 docs/en/table_engines/distributed.md          |  46 ++---
 docs/en/table_engines/external_data.md        |   0
 docs/en/table_engines/file.md                 |   0
 docs/en/table_engines/graphitemergetree.md    |   5 +-
 docs/en/table_engines/index.md                |   0
 docs/en/table_engines/join.md                 |   0
 docs/en/table_engines/kafka.md                | 158 +++++++++---------
 docs/en/table_engines/log.md                  |   0
 docs/en/table_engines/materializedview.md     |   2 +-
 docs/en/table_engines/memory.md               |   0
 docs/en/table_engines/merge.md                |   0
 docs/en/table_engines/mergetree.md            |   2 +-
 docs/en/table_engines/null.md                 |   0
 docs/en/table_engines/replacingmergetree.md   |   0
 docs/en/table_engines/replication.md          |   2 +-
 docs/en/table_engines/set.md                  |   0
 docs/en/table_engines/summingmergetree.md     |   0
 docs/en/table_engines/tinylog.md              |   0
 docs/en/table_engines/view.md                 |   0
 docs/en/table_functions/index.md              |   0
 docs/en/table_functions/merge.md              |   0
 docs/en/table_functions/remote.md             |   2 +-
 docs/en/utils/clickhouse-copier.md            | 120 ++++++-------
 docs/en/utils/clickhouse-local.md             |   6 +-
 docs/en/utils/index.md                        |   7 +-
 docs/mkdocs_en.yml                            |   2 +
 docs/ru/dicts/external_dicts_dict_layout.md   |  18 +-
 167 files changed, 727 insertions(+), 653 deletions(-)
 mode change 100644 => 100755 docs/en/agg_functions/combinators.md
 mode change 100644 => 100755 docs/en/agg_functions/index.md
 mode change 100644 => 100755 docs/en/agg_functions/parametric_functions.md
 mode change 100644 => 100755 docs/en/agg_functions/reference.md
 mode change 100644 => 100755 docs/en/data_types/array.md
 mode change 100644 => 100755 docs/en/data_types/boolean.md
 mode change 100644 => 100755 docs/en/data_types/date.md
 mode change 100644 => 100755 docs/en/data_types/datetime.md
 mode change 100644 => 100755 docs/en/data_types/enum.md
 mode change 100644 => 100755 docs/en/data_types/fixedstring.md
 mode change 100644 => 100755 docs/en/data_types/float.md
 mode change 100644 => 100755 docs/en/data_types/index.md
 mode change 100644 => 100755 docs/en/data_types/int_uint.md
 mode change 100644 => 100755 docs/en/data_types/nested_data_structures/aggregatefunction.md
 mode change 100644 => 100755 docs/en/data_types/nested_data_structures/index.md
 mode change 100644 => 100755 docs/en/data_types/nested_data_structures/nested.md
 mode change 100644 => 100755 docs/en/data_types/special_data_types/expression.md
 mode change 100644 => 100755 docs/en/data_types/special_data_types/index.md
 mode change 100644 => 100755 docs/en/data_types/special_data_types/set.md
 mode change 100644 => 100755 docs/en/data_types/string.md
 mode change 100644 => 100755 docs/en/data_types/tuple.md
 mode change 100644 => 100755 docs/en/development/style.md
 mode change 100644 => 100755 docs/en/dicts/external_dicts.md
 mode change 100644 => 100755 docs/en/dicts/external_dicts_dict.md
 mode change 100644 => 100755 docs/en/dicts/external_dicts_dict_layout.md
 mode change 100644 => 100755 docs/en/dicts/external_dicts_dict_lifetime.md
 mode change 100644 => 100755 docs/en/dicts/external_dicts_dict_sources.md
 mode change 100644 => 100755 docs/en/dicts/external_dicts_dict_structure.md
 mode change 100644 => 100755 docs/en/dicts/index.md
 mode change 100644 => 100755 docs/en/dicts/internal_dicts.md
 mode change 100644 => 100755 docs/en/formats/capnproto.md
 mode change 100644 => 100755 docs/en/formats/csv.md
 mode change 100644 => 100755 docs/en/formats/csvwithnames.md
 mode change 100644 => 100755 docs/en/formats/index.md
 mode change 100644 => 100755 docs/en/formats/json.md
 mode change 100644 => 100755 docs/en/formats/jsoncompact.md
 mode change 100644 => 100755 docs/en/formats/jsoneachrow.md
 mode change 100644 => 100755 docs/en/formats/native.md
 mode change 100644 => 100755 docs/en/formats/null.md
 mode change 100644 => 100755 docs/en/formats/pretty.md
 mode change 100644 => 100755 docs/en/formats/prettycompact.md
 mode change 100644 => 100755 docs/en/formats/prettycompactmonoblock.md
 mode change 100644 => 100755 docs/en/formats/prettynoescapes.md
 mode change 100644 => 100755 docs/en/formats/prettyspace.md
 mode change 100644 => 100755 docs/en/formats/rowbinary.md
 mode change 100644 => 100755 docs/en/formats/tabseparated.md
 mode change 100644 => 100755 docs/en/formats/tabseparatedraw.md
 mode change 100644 => 100755 docs/en/formats/tabseparatedwithnames.md
 mode change 100644 => 100755 docs/en/formats/tabseparatedwithnamesandtypes.md
 mode change 100644 => 100755 docs/en/formats/tskv.md
 mode change 100644 => 100755 docs/en/formats/values.md
 mode change 100644 => 100755 docs/en/formats/vertical.md
 mode change 100644 => 100755 docs/en/formats/xml.md
 mode change 100644 => 100755 docs/en/functions/arithmetic_functions.md
 mode change 100644 => 100755 docs/en/functions/array_functions.md
 mode change 100644 => 100755 docs/en/functions/array_join.md
 mode change 100644 => 100755 docs/en/functions/bit_functions.md
 mode change 100644 => 100755 docs/en/functions/comparison_functions.md
 mode change 100644 => 100755 docs/en/functions/conditional_functions.md
 mode change 100644 => 100755 docs/en/functions/date_time_functions.md
 mode change 100644 => 100755 docs/en/functions/encoding_functions.md
 mode change 100644 => 100755 docs/en/functions/ext_dict_functions.md
 mode change 100644 => 100755 docs/en/functions/hash_functions.md
 mode change 100644 => 100755 docs/en/functions/higher_order_functions.md
 mode change 100644 => 100755 docs/en/functions/in_functions.md
 mode change 100644 => 100755 docs/en/functions/index.md
 mode change 100644 => 100755 docs/en/functions/ip_address_functions.md
 mode change 100644 => 100755 docs/en/functions/json_functions.md
 mode change 100644 => 100755 docs/en/functions/logical_functions.md
 mode change 100644 => 100755 docs/en/functions/math_functions.md
 mode change 100644 => 100755 docs/en/functions/other_functions.md
 mode change 100644 => 100755 docs/en/functions/random_functions.md
 mode change 100644 => 100755 docs/en/functions/rounding_functions.md
 mode change 100644 => 100755 docs/en/functions/splitting_merging_functions.md
 mode change 100644 => 100755 docs/en/functions/string_functions.md
 mode change 100644 => 100755 docs/en/functions/string_replace_functions.md
 mode change 100644 => 100755 docs/en/functions/string_search_functions.md
 mode change 100644 => 100755 docs/en/functions/type_conversion_functions.md
 mode change 100644 => 100755 docs/en/functions/url_functions.md
 mode change 100644 => 100755 docs/en/functions/ym_dict_functions.md
 mode change 100644 => 100755 docs/en/getting_started/example_datasets/amplab_benchmark.md
 mode change 100644 => 100755 docs/en/getting_started/example_datasets/criteo.md
 mode change 100644 => 100755 docs/en/getting_started/example_datasets/nyc_taxi.md
 mode change 100644 => 100755 docs/en/getting_started/example_datasets/ontime.md
 mode change 100644 => 100755 docs/en/getting_started/example_datasets/star_schema.md
 mode change 100644 => 100755 docs/en/getting_started/example_datasets/wikistat.md
 mode change 100644 => 100755 docs/en/getting_started/index.md
 mode change 100644 => 100755 docs/en/index.md
 mode change 100644 => 100755 docs/en/interfaces/cli.md
 mode change 100644 => 100755 docs/en/interfaces/http_interface.md
 mode change 100644 => 100755 docs/en/interfaces/index.md
 mode change 100644 => 100755 docs/en/interfaces/jdbc.md
 mode change 100644 => 100755 docs/en/interfaces/tcp.md
 mode change 100644 => 100755 docs/en/interfaces/third-party_client_libraries.md
 mode change 100644 => 100755 docs/en/interfaces/third-party_gui.md
 mode change 100644 => 100755 docs/en/introduction/distinctive_features.md
 mode change 100644 => 100755 docs/en/introduction/features_considered_disadvantages.md
 mode change 100644 => 100755 docs/en/introduction/index.md
 mode change 100644 => 100755 docs/en/introduction/performance.md
 mode change 100644 => 100755 docs/en/introduction/possible_silly_questions.md
 mode change 100644 => 100755 docs/en/introduction/ya_metrika_task.md
 mode change 100644 => 100755 docs/en/operations/access_rights.md
 mode change 100644 => 100755 docs/en/operations/configuration_files.md
 mode change 100644 => 100755 docs/en/operations/index.md
 mode change 100644 => 100755 docs/en/operations/quotas.md
 mode change 100644 => 100755 docs/en/operations/server_settings/index.md
 mode change 100644 => 100755 docs/en/operations/server_settings/settings.md
 mode change 100644 => 100755 docs/en/operations/settings/index.md
 mode change 100644 => 100755 docs/en/operations/settings/query_complexity.md
 mode change 100644 => 100755 docs/en/operations/settings/settings.md
 mode change 100644 => 100755 docs/en/operations/settings/settings_profiles.md
 mode change 100644 => 100755 docs/en/operations/tips.md
 mode change 100644 => 100755 docs/en/operators/index.md
 mode change 100644 => 100755 docs/en/query_language/index.md
 mode change 100644 => 100755 docs/en/query_language/queries.md
 mode change 100644 => 100755 docs/en/query_language/syntax.md
 mode change 100644 => 100755 docs/en/roadmap.md
 mode change 100644 => 100755 docs/en/system_tables/index.md
 mode change 100644 => 100755 docs/en/system_tables/system.asynchronous_metrics.md
 mode change 100644 => 100755 docs/en/system_tables/system.clusters.md
 mode change 100644 => 100755 docs/en/system_tables/system.columns.md
 mode change 100644 => 100755 docs/en/system_tables/system.databases.md
 mode change 100644 => 100755 docs/en/system_tables/system.dictionaries.md
 mode change 100644 => 100755 docs/en/system_tables/system.events.md
 mode change 100644 => 100755 docs/en/system_tables/system.functions.md
 mode change 100644 => 100755 docs/en/system_tables/system.merges.md
 mode change 100644 => 100755 docs/en/system_tables/system.numbers.md
 mode change 100644 => 100755 docs/en/system_tables/system.numbers_mt.md
 mode change 100644 => 100755 docs/en/system_tables/system.one.md
 mode change 100644 => 100755 docs/en/system_tables/system.parts.md
 mode change 100644 => 100755 docs/en/system_tables/system.processes.md
 mode change 100644 => 100755 docs/en/system_tables/system.replicas.md
 mode change 100644 => 100755 docs/en/system_tables/system.settings.md
 mode change 100644 => 100755 docs/en/system_tables/system.tables.md
 mode change 100644 => 100755 docs/en/system_tables/system.zookeeper.md
 mode change 100644 => 100755 docs/en/table_engines/aggregatingmergetree.md
 mode change 100644 => 100755 docs/en/table_engines/buffer.md
 mode change 100644 => 100755 docs/en/table_engines/collapsingmergetree.md
 mode change 100644 => 100755 docs/en/table_engines/custom_partitioning_key.md
 create mode 100755 docs/en/table_engines/dictionary.md
 mode change 100644 => 100755 docs/en/table_engines/distributed.md
 mode change 100644 => 100755 docs/en/table_engines/external_data.md
 mode change 100644 => 100755 docs/en/table_engines/file.md
 mode change 100644 => 100755 docs/en/table_engines/graphitemergetree.md
 mode change 100644 => 100755 docs/en/table_engines/index.md
 mode change 100644 => 100755 docs/en/table_engines/join.md
 mode change 100644 => 100755 docs/en/table_engines/kafka.md
 mode change 100644 => 100755 docs/en/table_engines/log.md
 mode change 100644 => 100755 docs/en/table_engines/materializedview.md
 mode change 100644 => 100755 docs/en/table_engines/memory.md
 mode change 100644 => 100755 docs/en/table_engines/merge.md
 mode change 100644 => 100755 docs/en/table_engines/mergetree.md
 mode change 100644 => 100755 docs/en/table_engines/null.md
 mode change 100644 => 100755 docs/en/table_engines/replacingmergetree.md
 mode change 100644 => 100755 docs/en/table_engines/replication.md
 mode change 100644 => 100755 docs/en/table_engines/set.md
 mode change 100644 => 100755 docs/en/table_engines/summingmergetree.md
 mode change 100644 => 100755 docs/en/table_engines/tinylog.md
 mode change 100644 => 100755 docs/en/table_engines/view.md
 mode change 100644 => 100755 docs/en/table_functions/index.md
 mode change 100644 => 100755 docs/en/table_functions/merge.md
 mode change 100644 => 100755 docs/en/table_functions/remote.md
 mode change 100644 => 100755 docs/en/utils/clickhouse-copier.md
 mode change 100644 => 100755 docs/en/utils/clickhouse-local.md
 mode change 100644 => 100755 docs/en/utils/index.md

diff --git a/docs/en/agg_functions/combinators.md b/docs/en/agg_functions/combinators.md
old mode 100644
new mode 100755
diff --git a/docs/en/agg_functions/index.md b/docs/en/agg_functions/index.md
old mode 100644
new mode 100755
index e87bf4ff833..3864f7271c4
--- a/docs/en/agg_functions/index.md
+++ b/docs/en/agg_functions/index.md
@@ -8,3 +8,4 @@ ClickHouse also supports:
 
 - [Parametric aggregate functions](parametric_functions.md#aggregate_functions_parametric), which accept other parameters in addition to columns.
 - [Combinators](combinators.md#aggregate_functions_combinators), which change the behavior of aggregate functions.
+
diff --git a/docs/en/agg_functions/parametric_functions.md b/docs/en/agg_functions/parametric_functions.md
old mode 100644
new mode 100755
diff --git a/docs/en/agg_functions/reference.md b/docs/en/agg_functions/reference.md
old mode 100644
new mode 100755
index 90ff9da58e7..0eb896e4664
--- a/docs/en/agg_functions/reference.md
+++ b/docs/en/agg_functions/reference.md
@@ -19,7 +19,7 @@ In some cases, you can rely on the order of execution. This applies to cases whe
 
 When a `SELECT` query has the `GROUP BY` clause or at least one aggregate function, ClickHouse (in contrast to MySQL) requires that all expressions in the `SELECT`, `HAVING`, and `ORDER BY` clauses be calculated from keys or from aggregate functions. In other words, each column selected from the table must be used either in keys or inside aggregate functions. To get behavior like in MySQL, you can put the other columns in the `any` aggregate function.
 
-## anyHeavy(x)
+## anyHeavy
 
 Selects a frequently occurring value using the [heavy hitters](http://www.cs.umd.edu/~samir/498/karp.pdf) algorithm. If there is a value that occurs more than in half the cases in each of the query's execution threads, this value is returned. Normally, the result is nondeterministic.
 
@@ -39,7 +39,6 @@ Take the [OnTime](../getting_started/example_datasets/ontime.md#example_datasets
 SELECT anyHeavy(AirlineID) AS res
 FROM ontime
 ```
-
 ```
 ┌───res─┐
 │ 19690 │
@@ -125,11 +124,11 @@ The result is always Float64.
 Calculates the approximate number of different values of the argument. Works for numbers, strings, dates, date-with-time, and for multiple arguments and tuple arguments.
 
 Uses an adaptive sampling algorithm: for the calculation state, it uses a sample of element hash values with a size up to 65536.
-This algorithm is also very accurate for data sets with small cardinality (up to 65536) and very efficient on CPU (when computing not too many of these functions, using `uniq` is almost as fast as using other aggregate functions).
+This algorithm is also very accurate for data sets with low cardinality (up to 65536) and very efficient on CPU (when computing not too many of these functions, using `uniq` is almost as fast as using other aggregate functions).
 
 The result is determinate (it doesn't depend on the order of query processing).
 
-This function provides excellent accuracy even for data sets with huge cardinality (10B+ elements) and is recommended for use by default.
+This function provides excellent accuracy even for data sets with extremely high cardinality (over 10 billion elements). It is recommended for default use.
 
 ## uniqCombined(x)
 
@@ -139,16 +138,16 @@ A combination of three algorithms is used: array, hash table and [HyperLogLog](h
 
 The result is determinate (it doesn't depend on the order of query processing).
 
-The `uniqCombined` function is a good default choice for calculating the number of different values, but the following should be considered: for data sets with large cardinality (200M+) error of estimate will only grow and for data sets with huge cardinality(1B+ elements) it returns result with high inaccuracy.
+The `uniqCombined` function is a good default choice for calculating the number of different values, but keep in mind that the estimation error will increase for high-cardinality data sets (200M+ elements), and the function will return very inaccurate results for data sets with extremely high cardinality (1B+ elements).
 
 ## uniqHLL12(x)
 
 Uses the [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog) algorithm to approximate the number of different values of the argument.
-212 5-bit cells are used. The size of the state is slightly more than 2.5 KB. Result is not very accurate (error up to ~10%) for data sets of small cardinality(<10K elements), but for data sets with large cardinality (10K - 100M) result is quite accurate (error up to ~1.6%) and after that error of estimate will only grow and for data sets with huge cardinality (1B+ elements) it returns result with high inaccuracy.
+212 5-bit cells are used. The size of the state is slightly more than 2.5 KB. The result is not very accurate (up to ~10% error) for small data sets (<10K elements). However, the result is fairly accurate for high-cardinality data sets (10K-100M), with a maximum error of ~1.6%. Starting from 100M, the estimation error increases, and the function will return very inaccurate results for data sets with extremely high cardinality (1B+ elements).
 
 The result is determinate (it doesn't depend on the order of query processing).
 
-This function is not recommended for use, and in most cases, use the  `uniq` or `uniqCombined` function.
+We don't recommend using this function. In most cases, use the  `uniq` or `uniqCombined` function.
 
 ## uniqExact(x)
 
@@ -170,7 +169,7 @@ In some cases, you can still rely on the order of execution. This applies to cas
 
 <a name="agg_functions_groupArrayInsertAt"></a>
 
-## groupArrayInsertAt(x)
+## groupArrayInsertAt
 
 Inserts a value into the array in the specified position.
 
@@ -236,8 +235,8 @@ For its purpose (calculating quantiles of page loading times), using this functi
 
 ## quantileTimingWeighted(level)(x, weight)
 
-Differs from the 'quantileTiming' function in that it has a second argument, "weights". Weight is a non-negative integer.
-The result is calculated as if the 'x' value were passed 'weight' number of times to the 'quantileTiming' function.
+Differs from the `quantileTiming`  function in that it has a second argument, "weights". Weight is a non-negative integer.
+The result is calculated as if the `x`  value were passed `weight` number of times to the `quantileTiming` function.
 
 ## quantileExact(level)(x)
 
@@ -257,7 +256,7 @@ The performance of the function is lower than for ` quantile`, ` quantileTiming`
 
 The result depends on the order of running the query, and is nondeterministic.
 
-## median(x)
+## median
 
 All the quantile functions have corresponding median functions: `median`, `medianDeterministic`, `medianTiming`, `medianTimingWeighted`, `medianExact`, `medianExactWeighted`, `medianTDigest`. They are synonyms and their behavior is identical.
 
@@ -275,7 +274,7 @@ Returns `Float64`. When `n <= 1`, returns `+∞`.
 
 ## varPop(x)
 
-Calculates the amount `Σ((x - x̅)^2) / n`, where `n` is the sample size and `x̅`is the average value of `x`.
+Calculates the amount `Σ((x - x̅)^2) / (n - 1)`, where `n` is the sample size and `x̅`is the average value of `x`.
 
 In other words, dispersion for a set of values. Returns `Float64`.
 
@@ -287,30 +286,33 @@ The result is equal to the square root of `varSamp(x)`.
 
 The result is equal to the square root of `varPop(x)`.
 
-## topK(N)(column)
+## topK
 
 Returns an array of the most frequent values in the specified column. The resulting array is sorted in descending order of frequency of values (not by the values themselves).
 
 Implements the [ Filtered Space-Saving](http://www.l2f.inesc-id.pt/~fmmb/wiki/uploads/Work/misnis.ref0a.pdf)  algorithm for analyzing TopK, based on the reduce-and-combine algorithm from [Parallel Space Saving](https://arxiv.org/pdf/1401.0702.pdf).
 
+```
+topK(N)(column)
+```
+
 This function doesn't provide a guaranteed result. In certain situations, errors might occur and it might return frequent values that aren't the most frequent values.
 
 We recommend using the `N < 10 ` value; performance is reduced with large `N` values. Maximum value of ` N = 65536`.
 
 **Arguments**
 
-- 'N' – The number of values.
+- 'N' is the number of values.
 - ' x ' – The column.
 
 **Example**
 
-Take the [OnTime](../getting_started/example_datasets/ontime.md#example_datasets-ontime)data set and select the three most frequently occurring values in the `AirlineID` column.
+Take the [OnTime](../getting_started/example_datasets/ontime.md#example_datasets-ontime) data set and select the three most frequently occurring values in the `AirlineID` column.
 
 ```sql
 SELECT topK(3)(AirlineID) AS res
 FROM ontime
 ```
-
 ```
 ┌─res─────────────────┐
 │ [19393,19790,19805] │
diff --git a/docs/en/data_types/array.md b/docs/en/data_types/array.md
old mode 100644
new mode 100755
diff --git a/docs/en/data_types/boolean.md b/docs/en/data_types/boolean.md
old mode 100644
new mode 100755
diff --git a/docs/en/data_types/date.md b/docs/en/data_types/date.md
old mode 100644
new mode 100755
index 355e5555bfa..cb179c0d8c4
--- a/docs/en/data_types/date.md
+++ b/docs/en/data_types/date.md
@@ -1,6 +1,6 @@
 # Date
 
-Date. Stored in two bytes as the number of days since 1970-01-01 (unsigned). Allows storing values from just after the beginning of the Unix Epoch to the upper threshold defined by a constant at the compilation stage (currently, this is until the year 2038, but it may be expanded to 2106).
+A date. Stored in two bytes as the number of days since 1970-01-01 (unsigned). Allows storing values from just after the beginning of the Unix Epoch to the upper threshold defined by a constant at the compilation stage (currently, this is until the year 2106, but the final fully-supported year is 2105).
 The minimum value is output as 0000-00-00.
 
 The date is stored without the time zone.
diff --git a/docs/en/data_types/datetime.md b/docs/en/data_types/datetime.md
old mode 100644
new mode 100755
diff --git a/docs/en/data_types/enum.md b/docs/en/data_types/enum.md
old mode 100644
new mode 100755
diff --git a/docs/en/data_types/fixedstring.md b/docs/en/data_types/fixedstring.md
old mode 100644
new mode 100755
diff --git a/docs/en/data_types/float.md b/docs/en/data_types/float.md
old mode 100644
new mode 100755
index c6d12999604..9d5cc2c01bb
--- a/docs/en/data_types/float.md
+++ b/docs/en/data_types/float.md
@@ -4,8 +4,8 @@
 
 Types are equivalent to types of C:
 
-- `Float32` - `float`;
-- `Float64`  - ` double`.
+- `Float32` - `float`
+- `Float64`  - ` double`
 
 We recommend that you store data in integer form whenever possible. For example, convert fixed precision numbers to integer values, such as monetary amounts or page load times in milliseconds.
 
@@ -24,9 +24,7 @@ SELECT 1 - 0.9
 ```
 
 - The result of the calculation depends on the calculation method (the processor type and architecture of the computer system).
-
 - Floating-point calculations might result in numbers such as infinity (`Inf`) and "not-a-number" (`NaN`). This should be taken into account when processing the results of calculations.
-
 - When reading floating point numbers from rows, the result might not be the nearest machine-representable number.
 
 ## NaN and Inf
@@ -44,6 +42,7 @@ SELECT 0.5 / 0
 │            inf │
 └────────────────┘
 ```
+
 - `-Inf` – Negative infinity.
 
 ```sql
@@ -55,6 +54,7 @@ SELECT -0.5 / 0
 │            -inf │
 └─────────────────┘
 ```
+
 - `NaN` – Not a number.
 
 ```
@@ -67,5 +67,5 @@ SELECT 0 / 0
 └──────────────┘
 ```
 
-   See the rules for ` NaN` sorting in the section [ORDER BY clause](../query_language/queries.md#query_language-queries-order_by).
+See the rules for ` NaN` sorting in the section [ORDER BY clause](../query_language/queries.md#query_language-queries-order_by).
 
diff --git a/docs/en/data_types/index.md b/docs/en/data_types/index.md
old mode 100644
new mode 100755
index c17b51c08a2..4008a872161
--- a/docs/en/data_types/index.md
+++ b/docs/en/data_types/index.md
@@ -2,6 +2,7 @@
 
 # Data types
 
-ClickHouse table fields can contain data of different types.
+ClickHouse can store various types of data in table cells.
+
+This section describes the supported data types and special considerations when using and/or implementing them, if any.
 
-The topic contains descriptions of data types supported and specificity of their usage of implementation if exists.
\ No newline at end of file
diff --git a/docs/en/data_types/int_uint.md b/docs/en/data_types/int_uint.md
old mode 100644
new mode 100755
diff --git a/docs/en/data_types/nested_data_structures/aggregatefunction.md b/docs/en/data_types/nested_data_structures/aggregatefunction.md
old mode 100644
new mode 100755
diff --git a/docs/en/data_types/nested_data_structures/index.md b/docs/en/data_types/nested_data_structures/index.md
old mode 100644
new mode 100755
diff --git a/docs/en/data_types/nested_data_structures/nested.md b/docs/en/data_types/nested_data_structures/nested.md
old mode 100644
new mode 100755
diff --git a/docs/en/data_types/special_data_types/expression.md b/docs/en/data_types/special_data_types/expression.md
old mode 100644
new mode 100755
diff --git a/docs/en/data_types/special_data_types/index.md b/docs/en/data_types/special_data_types/index.md
old mode 100644
new mode 100755
diff --git a/docs/en/data_types/special_data_types/set.md b/docs/en/data_types/special_data_types/set.md
old mode 100644
new mode 100755
diff --git a/docs/en/data_types/string.md b/docs/en/data_types/string.md
old mode 100644
new mode 100755
diff --git a/docs/en/data_types/tuple.md b/docs/en/data_types/tuple.md
old mode 100644
new mode 100755
diff --git a/docs/en/development/style.md b/docs/en/development/style.md
old mode 100644
new mode 100755
index ef6490187c8..700fede5373
--- a/docs/en/development/style.md
+++ b/docs/en/development/style.md
@@ -4,7 +4,7 @@
 
 1. The following are recommendations, not requirements.
 2. If you are editing code, it makes sense to follow the formatting of the existing code.
-3. Code style is needed for consistency. Consistency makes it easier to read the code. and it also makes it easier to search the code.
+3. Code style is needed for consistency. Consistency makes it easier to read the code, and it also makes it easier to search the code.
 4. Many of the rules do not have logical reasons; they are dictated by established practices.
 
 ## Formatting
@@ -93,25 +93,25 @@
 14. In classes and structures, public, private, and protected are written on the same level as the class/struct, but all other internal elements should be deeper.
 
    ```cpp
-    template <typename T, typename Ptr = std::shared_ptr<T>>
-    class MultiVersion
-    {
-    public:
-        /// The specific version of the object to use.
-        using Version = Ptr;
-        ...
-    }
+   template <typename T>
+class MultiVersion
+{
+public:
+    /// Version of object for usage. shared_ptr manage lifetime of version.
+    using Version = std::shared_ptr<const T>;
+    ...
+}
    ```
 
 15. If the same namespace is used for the entire file, and there isn't anything else significant, an offset is not necessary  inside namespace.
 
 16. If the block for if, for, while... expressions consists of a single statement, you don't need to use curly brackets. Place the statement on a separate line, instead. The same is true for a nested if, for, while... statement. But if the inner statement contains curly brackets or else, the external block should be written in curly brackets.
 
-    ```cpp
-    /// Finish write.
-    for (auto & stream : streams)
-        stream.second->finalize();
-    ```
+   ```cpp
+   /// Finish write.
+for (auto & stream : streams)
+    stream.second->finalize();
+   ```
 
 17. There should be any spaces at the ends of lines.
 
@@ -218,11 +218,11 @@
    */
    void executeQuery(
        ReadBuffer & istr, /// Where to read the query from (and data for INSERT, if applicable)
-           WriteBuffer & ostr, /// Where to write the result
-           Context & context, /// DB, tables, data types, engines, functions, aggregate functions...
-           BlockInputStreamPtr & query_plan, /// A description of query processing can be included here
-           QueryProcessingStage::Enum stage = QueryProcessingStage::Complete /// The last stage to process the SELECT query to
-           )
+       WriteBuffer & ostr, /// Where to write the result
+       Context & context, /// DB, tables, data types, engines, functions, aggregate functions...
+       BlockInputStreamPtr & query_plan, /// A description of query processing can be included here
+       QueryProcessingStage::Enum stage = QueryProcessingStage::Complete /// The last stage to process the SELECT query to
+       )
    ```
 
 4. Comments should be written in English only.
@@ -252,7 +252,7 @@
     */
    ```
 
-   (Example taken from: [http://home.tamk.fi/~jaalto/course/coding-style/doc/unmaintainable-code/)](http://home.tamk.fi/~jaalto/course/coding-style/doc/unmaintainable-code/)
+   (the example is borrowed from the resource [http://home.tamk.fi/~jaalto/course/coding-style/doc/unmaintainable-code/](http://home.tamk.fi/~jaalto/course/coding-style/doc/unmaintainable-code/)
 
 7. Do not write garbage comments (author, creation date ..) at the beginning of each file.
 
@@ -497,7 +497,15 @@ This is not recommended, but it is allowed.
    You can create a separate code block inside a single function in order to make certain variables local, so that the destructors are called when exiting the block.
 
    ```cpp
-   Block block = data.in->read();{    std::lock_guard<std::mutex> lock(mutex);    data.ready = true;    data.block = block;}ready_any.set();
+   Block block = data.in->read();
+
+   {
+       std::lock_guard<std::mutex> lock(mutex);
+       data.ready = true;
+       data.block = block;
+   }
+
+   ready_any.set();
    ```
 
 7. Multithreading.
@@ -560,13 +568,12 @@ This is not recommended, but it is allowed.
 
    ```cpp
    using AggregateFunctionPtr = std::shared_ptr<IAggregateFunction>;
-
-   /** Creates an aggregate function by name.
-     */
+   
+   /** Creates an aggregate function by name.  */
    class AggregateFunctionFactory
    {
    public:
-       AggregateFunctionFactory();
+       AggregateFunctionFactory();    
        AggregateFunctionPtr get(const String & name, const DataTypes & argument_types) const;
    ```
 
@@ -591,10 +598,10 @@ This is not recommended, but it is allowed.
    If later you’ll need to delay initialization, you can add a default constructor that will create an invalid object. Or, for a small number of objects, you can use shared_ptr/unique_ptr.
 
    ```cpp
-    Loader(DB::Connection * connection_, const std::string & query, size_t max_block_size_);
-
-    /// For delayed initialization
-    Loader() {}
+   Loader(DB::Connection * connection_, const std::string & query, size_t max_block_size_);
+   
+   /// For delayed initialization
+   Loader() {}
    ```
 
 17. Virtual functions.
diff --git a/docs/en/dicts/external_dicts.md b/docs/en/dicts/external_dicts.md
old mode 100644
new mode 100755
index b99b02bbf57..a6af84a313f
--- a/docs/en/dicts/external_dicts.md
+++ b/docs/en/dicts/external_dicts.md
@@ -21,12 +21,11 @@ The dictionary config file has the following format:
 
     <!--Optional element. File name with substitutions-->
     <include_from>/etc/metrika.xml</include_from>
-
     
     <dictionary>
-        <!-- Dictionary configuration -->
+        <!-- Dictionary configuration -->    
     </dictionary>
-
+            
     ...
 
     <dictionary>
diff --git a/docs/en/dicts/external_dicts_dict.md b/docs/en/dicts/external_dicts_dict.md
old mode 100644
new mode 100755
index 4133b036e1f..6d2f4128704
--- a/docs/en/dicts/external_dicts_dict.md
+++ b/docs/en/dicts/external_dicts_dict.md
@@ -27,7 +27,8 @@ The dictionary configuration has the following structure:
 ```
 
 - name – The identifier that can be used to access the dictionary. Use the characters `[a-zA-Z0-9_\-]`.
-- [source](external_dicts_dict_sources.md#dicts-external_dicts_dict_sources) – Source of the dictionary.
-- [layout](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout) – Location of the dictionary in memory.
-- [structure](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure) – Structure of the dictionary. A key and attributes that can be retrieved by this key.
-- [lifetime](external_dicts_dict_lifetime.md#dicts-external_dicts_dict_lifetime) – How frequently to update dictionaries.
+- [source](external_dicts_dict_sources.html/#dicts-external_dicts_dict_sources) — Source of the dictionary .
+- [layout](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout) — Dictionary layout in memory.
+- [source](external_dicts_dict_sources.html/#dicts-external_dicts_dict_sources) — Structure of the dictionary . A key and attributes that can be retrieved by this key.
+- [lifetime](external_dicts_dict_lifetime.md#dicts-external_dicts_dict_lifetime) — Frequency of dictionary updates.
+
diff --git a/docs/en/dicts/external_dicts_dict_layout.md b/docs/en/dicts/external_dicts_dict_layout.md
old mode 100644
new mode 100755
index ad635db94f5..8b7cad24b65
--- a/docs/en/dicts/external_dicts_dict_layout.md
+++ b/docs/en/dicts/external_dicts_dict_layout.md
@@ -2,11 +2,11 @@
 
 # Storing dictionaries in memory
 
-There are [many different ways](external_dicts_dict_layout#dicts-external_dicts_dict_layout-manner) to store dictionaries in memory.
+There are a [variety of ways](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-manner) to store dictionaries in memory.
 
-We recommend [flat](external_dicts_dict_layout#dicts-external_dicts_dict_layout-flat), [hashed](external_dicts_dict_layout#dicts-external_dicts_dict_layout-hashed), and [complex_key_hashed](external_dicts_dict_layout#dicts-external_dicts_dict_layout-complex_key_hashed). which provide optimal processing speed.
+We recommend [flat](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-flat), [hashed](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-hashed)and[complex_key_hashed](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-complex_key_hashed). which provide optimal processing speed.
 
-Caching is not recommended because of potentially poor performance and difficulties in selecting optimal parameters. Read more about this in the "[cache](external_dicts_dict_layout#dicts-external_dicts_dict_layout-cache)" section.
+Caching is not recommended because of potentially poor performance and difficulties in selecting optimal parameters. Read more in the section " [cache](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-cache)".
 
 There are several ways to improve dictionary performance:
 
@@ -46,7 +46,6 @@ The configuration looks like this:
 - [range_hashed](#dicts-external_dicts_dict_layout-range_hashed)
 - [complex_key_hashed](#dicts-external_dicts_dict_layout-complex_key_hashed)
 - [complex_key_cache](#dicts-external_dicts_dict_layout-complex_key_cache)
-- [ip_trie](#dicts-external_dicts_dict_layout-ip_trie)
 
 <a name="dicts-external_dicts_dict_layout-flat"></a>
 
@@ -88,7 +87,7 @@ Configuration example:
 
 ### complex_key_hashed
 
-This type of storage is designed for use with compound [keys](external_dicts_dict_structure#dicts-external_dicts_dict_structure). It is similar to hashed.
+This type is for use with composite [keys](external_dicts_dict_structure.md/#dicts-external_dicts_dict_structure). Similar to `hashed`.
 
 Configuration example:
 
@@ -109,18 +108,18 @@ This storage method works the same way as hashed and allows using date/time rang
 Example: The table contains discounts for each advertiser in the format:
 
 ```
-  +---------------+---------------------+-------------------+--------+
-  | advertiser id | discount start date | discount end date | amount |
-  +===============+=====================+===================+========+
-  | 123           | 2015-01-01          | 2015-01-15        | 0.15   |
-  +---------------+---------------------+-------------------+--------+
-  | 123           | 2015-01-16          | 2015-01-31        | 0.25   |
-  +---------------+---------------------+-------------------+--------+
-  | 456           | 2015-01-01          | 2015-01-15        | 0.05   |
-  +---------------+---------------------+-------------------+--------+
++---------------+---------------------+-------------------+--------+
+| advertiser id | discount start date | discount end date | amount |
++===============+=====================+===================+========+
+| 123           | 2015-01-01          | 2015-01-15        | 0.15   |
++---------------+---------------------+-------------------+--------+
+| 123           | 2015-01-16          | 2015-01-31        | 0.25   |
++---------------+---------------------+-------------------+--------+
+| 456           | 2015-01-01          | 2015-01-15        | 0.05   |
++---------------+---------------------+-------------------+--------+
 ```
 
-To use a sample for date ranges, define `range_min` and `range_max` in [structure](external_dicts_dict_structure#dicts-external_dicts_dict_structure).
+To use a sample for date ranges, define the `range_min` and `range_max` elements in the [structure](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure).
 
 Example:
 
@@ -197,15 +196,15 @@ This is the least effective of all the ways to store dictionaries. The speed of
 
 To improve cache performance, use a subquery with ` LIMIT`, and call the function with the dictionary externally.
 
-Supported [sources](external_dicts_dict_sources#dicts-external_dicts_dict_sources): MySQL, ClickHouse, executable, HTTP.
+Supported [sources](external_dicts_dict_sources.md#dicts-external_dicts_dict_sources): MySQL, ClickHouse, executable, HTTP.
 
 Example of settings:
 
 ```xml
 <layout>
     <cache>
-        <!-- The size of the cache, in number of cells. Rounded up to a power of two. -->
-               <size_in_cells>1000000000</size_in_cells>
+        <!-- The size of the cache, in number of cells. Rounded up to a power of two. -->        
+        <size_in_cells>1000000000</size_in_cells>
     </cache>
 </layout>
 ```
@@ -227,66 +226,4 @@ Do not use ClickHouse as a source, because it is slow to process queries with ra
 
 ### complex_key_cache
 
-This type of storage is designed for use with compound [keys](external_dicts_dict_structure#dicts-external_dicts_dict_structure). Similar to `cache`.
-
-<a name="dicts-external_dicts_dict_layout-ip_trie"></a>
-
-### ip_trie
-
-
-The table stores IP prefixes for each key (IP address), which makes it possible to map IP addresses to metadata such as ASN or threat score.
-
-Example: in the table there are prefixes matches to AS number and country:
-
-```
-  +-----------------+-------+--------+
-  | prefix          | asn   | cca2   |
-  +=================+=======+========+
-  | 202.79.32.0/20  | 17501 | NP     |
-  +-----------------+-------+--------+
-  | 2620:0:870::/48 | 3856  | US     |
-  +-----------------+-------+--------+
-  | 2a02:6b8:1::/48 | 13238 | RU     |
-  +-----------------+-------+--------+
-  | 2001:db8::/32   | 65536 | ZZ     |
-  +-----------------+-------+--------+
-```
-
-When using such a layout, the structure should have the "key" element.
-
-Example:
-
-```xml
-<structure>
-    <key>
-        <attribute>
-            <name>prefix</name>
-            <type>String</type>
-        </attribute>
-    </key>
-    <attribute>
-            <name>asn</name>
-            <type>UInt32</type>
-            <null_value />
-    </attribute>
-    <attribute>
-            <name>cca2</name>
-            <type>String</type>
-            <null_value>??</null_value>
-    </attribute>
-    ...
-```
-
-These key must have only one attribute of type String, containing a valid IP prefix. Other types are not yet supported.
-
-For querying, same functions (dictGetT with tuple) as for complex key dictionaries have to be used:
-
-    dictGetT('dict_name', 'attr_name', tuple(ip))
-
-The function accepts either UInt32 for IPv4 address or FixedString(16) for IPv6 address in wire format:
-
-    dictGetString('prefix', 'asn', tuple(IPv6StringToNum('2001:db8::1')))
-
-No other type is supported. The function returns attribute for a prefix matching the given IP address. If there are overlapping prefixes, the most specific one is returned.
-
-The data is stored currently in a bitwise trie, it has to fit in memory.
+This type of storage is for use with composite [keys](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure). Similar to `cache`.
diff --git a/docs/en/dicts/external_dicts_dict_lifetime.md b/docs/en/dicts/external_dicts_dict_lifetime.md
old mode 100644
new mode 100755
index 6431fb3de48..52ee7a4aa78
--- a/docs/en/dicts/external_dicts_dict_lifetime.md
+++ b/docs/en/dicts/external_dicts_dict_lifetime.md
@@ -36,13 +36,13 @@ Example of settings:
 When upgrading the dictionaries, the ClickHouse server applies different logic depending on the type of [ source](external_dicts_dict_sources.md#dicts-external_dicts_dict_sources):
 
 > - For a text file, it checks the time of modification. If the time differs from the previously recorded time, the dictionary is updated.
-> - For MyISAM tables, the time of modification is checked using a `SHOW TABLE STATUS` query.
-> - Dictionaries from other sources are updated every time by default.
+- For MyISAM tables, the time of modification is checked using a `SHOW TABLE STATUS` query.
+- Dictionaries from other sources are updated every time by default.
 
 For MySQL (InnoDB) and ODBC sources, you can set up a query that will update the dictionaries only if they really changed, rather than each time. To do this, follow these steps:
 
 > - The dictionary table must have a field that always changes when the source data is updated.
-> - The settings of the source must specify a query that retrieves the changing field. The ClickHouse server interprets the query result as a row, and if this row has changed relative to its previous state, the dictionary is updated. The query must be specified in the `<invalidate_query>` field in the [ source](external_dicts_dict_sources.md#dicts-external_dicts_dict_sources) settings.
+- The settings of the source must specify a query that retrieves the changing field. The ClickHouse server interprets the query result as a row, and if this row has changed relative to its previous state, the dictionary is updated. Specify the query in the `<invalidate_query>` field in the settings for the [source](external_dicts_dict_sources.md#dicts-external_dicts_dict_sources).
 
 Example of settings:
 
diff --git a/docs/en/dicts/external_dicts_dict_sources.md b/docs/en/dicts/external_dicts_dict_sources.md
old mode 100644
new mode 100755
index 721302cd556..6cb4e0ea44d
--- a/docs/en/dicts/external_dicts_dict_sources.md
+++ b/docs/en/dicts/external_dicts_dict_sources.md
@@ -80,7 +80,7 @@ Setting fields:
 
 ## HTTP(s)
 
-Working with executable files depends on [how the dictionary is stored in memory](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout). If the dictionary is stored using `cache` and `complex_key_cache`, ClickHouse requests the necessary keys by sending a request via the `POST` method.
+Working with an HTTP(s) server depends on [how the dictionary is stored in memory](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout). If the dictionary is stored using `cache` and `complex_key_cache`, ClickHouse requests the necessary keys by sending a request via the `POST` method.
 
 Example of settings:
 
@@ -135,9 +135,9 @@ Installing unixODBC and the ODBC driver for PostgreSQL:
 Configuring `/etc/odbc.ini` (or `~/.odbc.ini`):
 
 ```
-    [DEFAULT]
+[DEFAULT]
     Driver = myconnection
-    
+
     [myconnection]
     Description         = PostgreSQL connection to my_db
     Driver              = PostgreSQL Unicode
diff --git a/docs/en/dicts/external_dicts_dict_structure.md b/docs/en/dicts/external_dicts_dict_structure.md
old mode 100644
new mode 100755
index 5a6d349b350..2542af00ec6
--- a/docs/en/dicts/external_dicts_dict_structure.md
+++ b/docs/en/dicts/external_dicts_dict_structure.md
@@ -25,8 +25,8 @@ Overall structure:
 
 Columns are described in the structure:
 
-- `<id>` – [Key column](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure-key).
-- `<attribute>` – [Data column](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure-attributes). There can be a large number of columns.
+- `<id>` - [key column](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure-key).
+- `<attribute>` - [data column](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure-attributes). There can be a large number of columns.
 
 <a name="dicts-external_dicts_dict_structure-key"></a>
 
@@ -63,12 +63,10 @@ Configuration fields:
 
 ### Composite key
 
-The key can be a `tuple` from any types of fields. The [ layout](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout)  in this case must be `complex_key_hashed`  or `complex_key_cache`.
+The key can be a `tuple` from any types of fields. The [layout](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout) in this case must be `complex_key_hashed` or `complex_key_cache`.
 
 <div class="admonition tip">
-
-A composite key can also consist of a single element, which makes it possible to use a string as the key, for instance.
-
+A composite key can consist of a single element.
 This makes it possible to use a string as the key, for instance.
 </div>
 
 The key structure is set in the element `<key>`. Key fields are specified in the same format as the dictionary [attributes](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure-attributes). Example:
@@ -119,6 +117,6 @@ Configuration fields:
 - `null_value` – The default value for a non-existing element. In the example, it is an empty string.
 - `expression` – The attribute can be an expression. The tag is not required.
 - `hierarchical` – Hierarchical support. Mirrored to the parent identifier. By default, ` false`.
-- `injective` Whether the `id -> attribute` image is injective. If ` true`, then you can optimize the ` GROUP BY` clause. By default, `false`.
-- `is_object_id` - Used for query mongo documents by ObjectId
+- `injective` – Whether the `id -> attribute` image is injective. If ` true`, then you can optimize the ` GROUP BY` clause. By default, `false`.
+- `is_object_id` – Whether the query is executed for a MongoDB document by `ObjectID`.
 
diff --git a/docs/en/dicts/index.md b/docs/en/dicts/index.md
old mode 100644
new mode 100755
diff --git a/docs/en/dicts/internal_dicts.md b/docs/en/dicts/internal_dicts.md
old mode 100644
new mode 100755
diff --git a/docs/en/formats/capnproto.md b/docs/en/formats/capnproto.md
old mode 100644
new mode 100755
index 0d482e20887..918197b2bd9
--- a/docs/en/formats/capnproto.md
+++ b/docs/en/formats/capnproto.md
@@ -1,26 +1,26 @@
-<a name="format_capnproto"></a>
-
-# CapnProto
-
-Cap'n Proto is a binary message format similar to Protocol Buffers and Thrift, but not like JSON or MessagePack.
-
-Cap'n Proto messages are strictly typed and not self-describing, meaning they need an external schema description. The schema is applied on the fly and cached for each query.
-
-```sql
-SELECT SearchPhrase, count() AS c FROM test.hits
-       GROUP BY SearchPhrase FORMAT CapnProto SETTINGS schema = 'schema:Message'
-```
-
-Where `schema.capnp` looks like this:
-
-```
-struct Message {
-  SearchPhrase @0 :Text;
-  c @1 :Uint64;
-}
-```
-
-Schema files are in the file that is located in the directory specified in [ format_schema_path](../operations/server_settings/settings.md#server_settings-format_schema_path) in the server configuration.
-
-Deserialization is effective and usually doesn't increase the system load.
-
+<a name="format_capnproto"></a>
+
+# CapnProto
+
+Cap'n Proto is a binary message format similar to Protocol Buffers and Thrift, but not like JSON or MessagePack.
+
+Cap'n Proto messages are strictly typed and not self-describing, meaning they need an external schema description. The schema is applied on the fly and cached for each query.
+
+```sql
+SELECT SearchPhrase, count() AS c FROM test.hits
+       GROUP BY SearchPhrase FORMAT CapnProto SETTINGS schema = 'schema:Message'
+```
+
+Where `schema.capnp` looks like this:
+
+```
+struct Message {
+  SearchPhrase @0 :Text;
+  c @1 :Uint64;
+}
+```
+
+Schema files are in the file that is located in the directory specified in [ format_schema_path](../operations/server_settings/settings.md#server_settings-format_schema_path) in the server configuration.
+
+Deserialization is effective and usually doesn't increase the system load.
+
diff --git a/docs/en/formats/csv.md b/docs/en/formats/csv.md
old mode 100644
new mode 100755
diff --git a/docs/en/formats/csvwithnames.md b/docs/en/formats/csvwithnames.md
old mode 100644
new mode 100755
diff --git a/docs/en/formats/index.md b/docs/en/formats/index.md
old mode 100644
new mode 100755
index 815a2d060cb..112a13ff5e5
--- a/docs/en/formats/index.md
+++ b/docs/en/formats/index.md
@@ -3,3 +3,4 @@
 # Formats
 
 The format determines how data is returned to you after SELECTs (how it is written and formatted by the server), and how it is accepted for INSERTs (how it is read and parsed by the server).
+
diff --git a/docs/en/formats/json.md b/docs/en/formats/json.md
old mode 100644
new mode 100755
diff --git a/docs/en/formats/jsoncompact.md b/docs/en/formats/jsoncompact.md
old mode 100644
new mode 100755
index e4ce0867bc2..d870b6dff08
--- a/docs/en/formats/jsoncompact.md
+++ b/docs/en/formats/jsoncompact.md
@@ -24,7 +24,7 @@ Example:
                 ["bathroom interior design", "2166"],
                 ["yandex", "1655"],
                 ["spring 2014 fashion", "1549"],
-                ["freeform photos", "1480"]
+                ["freeform photo", "1480"]
         ],
 
         "totals": ["","8873898"],
diff --git a/docs/en/formats/jsoneachrow.md b/docs/en/formats/jsoneachrow.md
old mode 100644
new mode 100755
diff --git a/docs/en/formats/native.md b/docs/en/formats/native.md
old mode 100644
new mode 100755
diff --git a/docs/en/formats/null.md b/docs/en/formats/null.md
old mode 100644
new mode 100755
diff --git a/docs/en/formats/pretty.md b/docs/en/formats/pretty.md
old mode 100644
new mode 100755
diff --git a/docs/en/formats/prettycompact.md b/docs/en/formats/prettycompact.md
old mode 100644
new mode 100755
diff --git a/docs/en/formats/prettycompactmonoblock.md b/docs/en/formats/prettycompactmonoblock.md
old mode 100644
new mode 100755
diff --git a/docs/en/formats/prettynoescapes.md b/docs/en/formats/prettynoescapes.md
old mode 100644
new mode 100755
diff --git a/docs/en/formats/prettyspace.md b/docs/en/formats/prettyspace.md
old mode 100644
new mode 100755
diff --git a/docs/en/formats/rowbinary.md b/docs/en/formats/rowbinary.md
old mode 100644
new mode 100755
index bc8479332ba..aeb3df4c8a8
--- a/docs/en/formats/rowbinary.md
+++ b/docs/en/formats/rowbinary.md
@@ -9,5 +9,5 @@ Date is represented as a UInt16 object that contains the number of days since 19
 String is represented as a varint length (unsigned [LEB128](https://en.wikipedia.org/wiki/LEB128)), followed by the bytes of the string.
 FixedString is represented simply as a sequence of bytes.
 
-Arrays are represented as a varint length (unsigned [LEB128](https://en.wikipedia.org/wiki/LEB128)), followed by the array elements in order.
+Array is represented as a varint length (unsigned [LEB128](https://en.wikipedia.org/wiki/LEB128)), followed by successive elements of the array.
 
diff --git a/docs/en/formats/tabseparated.md b/docs/en/formats/tabseparated.md
old mode 100644
new mode 100755
diff --git a/docs/en/formats/tabseparatedraw.md b/docs/en/formats/tabseparatedraw.md
old mode 100644
new mode 100755
diff --git a/docs/en/formats/tabseparatedwithnames.md b/docs/en/formats/tabseparatedwithnames.md
old mode 100644
new mode 100755
diff --git a/docs/en/formats/tabseparatedwithnamesandtypes.md b/docs/en/formats/tabseparatedwithnamesandtypes.md
old mode 100644
new mode 100755
diff --git a/docs/en/formats/tskv.md b/docs/en/formats/tskv.md
old mode 100644
new mode 100755
diff --git a/docs/en/formats/values.md b/docs/en/formats/values.md
old mode 100644
new mode 100755
index 2e929369848..a672723f33d
--- a/docs/en/formats/values.md
+++ b/docs/en/formats/values.md
@@ -4,5 +4,5 @@ Prints every row in brackets. Rows are separated by commas. There is no comma af
 
 The minimum set of characters that you need to escape when passing data in Values ​​format: single quotes and backslashes.
 
-This is the format that is used in `INSERT INTO t VALUES ...` but you can also use it for query result.
+This is the format that is used in `INSERT INTO t VALUES ...`, but you can also use it for formatting query results.
 
diff --git a/docs/en/formats/vertical.md b/docs/en/formats/vertical.md
old mode 100644
new mode 100755
diff --git a/docs/en/formats/xml.md b/docs/en/formats/xml.md
old mode 100644
new mode 100755
index f91adec9356..0da55875cc3
--- a/docs/en/formats/xml.md
+++ b/docs/en/formats/xml.md
@@ -35,7 +35,7 @@ XML format is suitable only for output, not for parsing. Example:
                         <field>1549</field>
                 </row>
                 <row>
-                        <SearchPhrase>freeform photos</SearchPhrase>
+                        <SearchPhrase>freeform photo</SearchPhrase>
                         <field>1480</field>
                 </row>
                 <row>
diff --git a/docs/en/functions/arithmetic_functions.md b/docs/en/functions/arithmetic_functions.md
old mode 100644
new mode 100755
diff --git a/docs/en/functions/array_functions.md b/docs/en/functions/array_functions.md
old mode 100644
new mode 100755
index 493a465ac82..232f6a20427
--- a/docs/en/functions/array_functions.md
+++ b/docs/en/functions/array_functions.md
@@ -39,7 +39,7 @@ Accepts an empty array and returns a one-element array that is equal to the defa
 Returns an array of numbers from 0 to N-1.
 Just in case, an exception is thrown if arrays with a total length of more than 100,000,000 elements are created in a data block.
 
-## array(x1, ...), operator \[x1, ...\]
+## array(x1, ...), оператор \[x1, ...\]
 
 Creates an array from the function arguments.
 The arguments must be constants and have types that have the smallest common type. At least one argument must be passed, because otherwise it isn't clear which type of array to create. That is, you can't use this function to create an empty array (to do that, use the 'emptyArray\*' function described above).
@@ -62,7 +62,6 @@ arrayConcat(arrays)
 ```sql
 SELECT arrayConcat([1, 2], [3, 4], [5, 6]) AS res
 ```
-
 ```
 ┌─res───────────┐
 │ [1,2,3,4,5,6] │
@@ -203,7 +202,6 @@ arrayPopBack(array)
 ```sql
 SELECT arrayPopBack([1, 2, 3]) AS res
 ```
-
 ```
 ┌─res───┐
 │ [1,2] │
@@ -245,14 +243,13 @@ arrayPushBack(array, single_value)
 **Arguments**
 
 - `array` – Array.
-- `single_value` – A single value. Only numbers can be added to an array with numbers, and only strings can be added to an array of strings. When adding numbers, ClickHouse automatically sets the `single_value` type for the data type of the array. For more information about the types of data in ClickHouse, see "[Data types](../data_types/index.md#data_types)".
+- `single_value` – A single value. Only numbers can be added to an array with numbers, and only strings can be added to an array of strings. When adding numbers, ClickHouse automatically sets the `single_value` type for the data type of the array. For more information about ClickHouse data types, read the section "[Data types](../data_types/index.md#data_types)".
 
 **Example**
 
 ```sql
 SELECT arrayPushBack(['a'], 'b') AS res
 ```
-
 ```
 ┌─res───────┐
 │ ['a','b'] │
@@ -270,7 +267,7 @@ arrayPushFront(array, single_value)
 **Arguments**
 
 - `array` – Array.
-- `single_value` – A single value.  Only numbers can be added to an array with numbers, and only strings can be added to an array of strings. When adding numbers, ClickHouse automatically sets the `single_value` type for the data type of the array.  For more information about the types of data in ClickHouse, see "[Data types](../data_types/index.md#data_types)".
+- `single_value` – A single value.  Only numbers can be added to an array with numbers, and only strings can be added to an array of strings. When adding numbers, ClickHouse automatically sets the `single_value` type for the data type of the array.  For more information about ClickHouse data types, read the section "[Data types](../data_types/index.md#data_types)".
 
 **Example**
 
@@ -295,7 +292,7 @@ arraySlice(array, offset[, length])
 **Arguments**
 
 - `array` –  Array of data.
-- `offset` – Offset from the edge of the array. A positive value indicates an offset on the left, and a negative value is an indent on the right. Numbering of the array items begins with 1.
+- `offset` – Indent from the edge of the array. A positive value indicates an offset on the left, and a negative value is an indent on the right. Numbering of the array items begins with 1.
 - `length` - The length of the required slice. If you specify a negative value, the function returns an open slice `[offset, array_length - length)`. If you omit the value, the function returns the slice `[offset, the_end_of_array]`.
 
 **Example**
@@ -303,7 +300,6 @@ arraySlice(array, offset[, length])
 ```sql
 SELECT arraySlice([1, 2, 3, 4, 5], 2, 3) AS res
 ```
-
 ```
 ┌─res─────┐
 │ [2,3,4] │
diff --git a/docs/en/functions/array_join.md b/docs/en/functions/array_join.md
old mode 100644
new mode 100755
index 6e18f8203c0..f94b2707f52
--- a/docs/en/functions/array_join.md
+++ b/docs/en/functions/array_join.md
@@ -28,4 +28,3 @@ SELECT arrayJoin([1, 2, 3] AS src) AS dst, 'Hello', src
 │   3 │ Hello     │ [1,2,3] │
 └─────┴───────────┴─────────┘
 ```
-
diff --git a/docs/en/functions/bit_functions.md b/docs/en/functions/bit_functions.md
old mode 100644
new mode 100755
index 523413f200a..c5a032aa5d6
--- a/docs/en/functions/bit_functions.md
+++ b/docs/en/functions/bit_functions.md
@@ -15,4 +15,3 @@ The result type is an integer with bits equal to the maximum bits of its argumen
 ## bitShiftLeft(a, b)
 
 ## bitShiftRight(a, b)
-
diff --git a/docs/en/functions/comparison_functions.md b/docs/en/functions/comparison_functions.md
old mode 100644
new mode 100755
index e37642d42ed..9b95966ba84
--- a/docs/en/functions/comparison_functions.md
+++ b/docs/en/functions/comparison_functions.md
@@ -15,7 +15,7 @@ For example, you can't compare a date with a string. You have to use a function
 
 Strings are compared by bytes. A shorter string is smaller than all strings that start with it and that contain at least one more character.
 
-Note: Up until version 1.1.54134, signed and unsigned numbers were compared the same way as in C++. In other words, you could get an incorrect result in cases like SELECT 9223372036854775807 &gt; -1. This behavior changed in version 1.1.54134 and is now mathematically correct.
+Note. Up until version 1.1.54134, signed and unsigned numbers were compared the same way as in C++. In other words, you could get an incorrect result in cases like SELECT 9223372036854775807 &gt; -1. This behavior changed in version 1.1.54134 and is now mathematically correct.
 
 ## equals, a = b and a == b operator
 
diff --git a/docs/en/functions/conditional_functions.md b/docs/en/functions/conditional_functions.md
old mode 100644
new mode 100755
diff --git a/docs/en/functions/date_time_functions.md b/docs/en/functions/date_time_functions.md
old mode 100644
new mode 100755
index da6ad9b4a7c..a7529e5f0e1
--- a/docs/en/functions/date_time_functions.md
+++ b/docs/en/functions/date_time_functions.md
@@ -79,6 +79,10 @@ Rounds down a date with time to the start of the minute.
 
 Rounds down a date with time to the start of the hour.
 
+## toStartOfFifteenMinutes
+
+Rounds down the date with time to the start of the fifteen-minute interval.
+
 Note: If you need to round a date with time to any other number of seconds, minutes, or hours, you can convert it into a number by using the toUInt32 function, then round the number using intDiv and multiplication, and convert it back using the toDateTime function.
 
 ## toStartOfHour
diff --git a/docs/en/functions/encoding_functions.md b/docs/en/functions/encoding_functions.md
old mode 100644
new mode 100755
diff --git a/docs/en/functions/ext_dict_functions.md b/docs/en/functions/ext_dict_functions.md
old mode 100644
new mode 100755
index 949e805d9ab..002e2f55845
--- a/docs/en/functions/ext_dict_functions.md
+++ b/docs/en/functions/ext_dict_functions.md
@@ -18,18 +18,20 @@ For information on connecting and configuring external dictionaries, see "[Exter
 
 `dictGetT('dict_name', 'attr_name', id)`
 
-- Get the value of the attr_name attribute  from the dict_name dictionary using the 'id' key.`dict_name`  and `attr_name`  are constant strings.`id`must be UInt64.
+- Get the value of the attr_name attribute  from the dict_name dictionary using the 'id' key.
+`dict_name`  and `attr_name`  are constant strings.
+`id`must be UInt64.
 If there is no `id` key in the dictionary, it returns the default value specified in the dictionary description.
 
 ## dictGetTOrDefault
 
 `dictGetT('dict_name', 'attr_name', id, default)`
 
-Similar to the functions dictGetT, but the default value is taken from the last argument of the function.
+The same as the `dictGetT` functions, but the default value is taken from the function's last argument.
 
 ## dictIsIn
 
-`dictIsIn ('dict_name', child_id, ancestor_id)`
+`dictIsIn('dict_name', child_id, ancestor_id)`
 
 - For the 'dict_name' hierarchical dictionary, finds out whether the 'child_id' key is located inside 'ancestor_id' (or matches 'ancestor_id'). Returns UInt8.
 
diff --git a/docs/en/functions/hash_functions.md b/docs/en/functions/hash_functions.md
old mode 100644
new mode 100755
diff --git a/docs/en/functions/higher_order_functions.md b/docs/en/functions/higher_order_functions.md
old mode 100644
new mode 100755
index ab9bdc50786..15cc40dead1
--- a/docs/en/functions/higher_order_functions.md
+++ b/docs/en/functions/higher_order_functions.md
@@ -73,7 +73,7 @@ Returns the index of the first element in the 'arr1' array for which 'func' retu
 
 ### arrayCumSum(\[func,\] arr1, ...)
 
-Returns the cumulative sum of the array obtained from the original application of the 'func' function to each element in the 'arr' array.
+Returns an array of partial sums of elements in the source array (a running sum). If the `func` function is specified, then the values of the array elements are converted by this function before summing.
 
 Example:
 
@@ -86,4 +86,3 @@ SELECT arrayCumSum([1, 1, 1, 1]) AS res
 │ [1, 2, 3, 4] │
 └──────────────┘
 ```
-
diff --git a/docs/en/functions/in_functions.md b/docs/en/functions/in_functions.md
old mode 100644
new mode 100755
diff --git a/docs/en/functions/index.md b/docs/en/functions/index.md
old mode 100644
new mode 100755
diff --git a/docs/en/functions/ip_address_functions.md b/docs/en/functions/ip_address_functions.md
old mode 100644
new mode 100755
diff --git a/docs/en/functions/json_functions.md b/docs/en/functions/json_functions.md
old mode 100644
new mode 100755
index 1bf10e9cf0c..90a2ddc47dd
--- a/docs/en/functions/json_functions.md
+++ b/docs/en/functions/json_functions.md
@@ -1,11 +1,11 @@
-# Functions for working with JSON.
+# Functions for working with JSON
 
 In Yandex.Metrica, JSON is transmitted by users as session parameters. There are some special functions for working with this JSON. (Although in most of the cases, the JSONs are additionally pre-processed, and the resulting values are put in separate columns in their processed format.) All these functions are based on strong assumptions about what the JSON can be, but they try to do as little as possible to get the job done.
 
 The following assumptions are made:
 
 1. The field name (function argument) must be a constant.
-2. The field name is somehow canonically encoded in JSON. For example: `visitParamHas('{"abc":"def"}', 'abc') = 1`, but `visitParamHas('{"\\u0061\\u0062\\u0063":"def"}', 'abc') = 0`
+2. The field name is somehow canonically encoded in JSON. For example: `visitParamHas('{"abc":"def"}', 'abc') = 1`, но `visitParamHas('{"\\u0061\\u0062\\u0063":"def"}', 'abc') = 0`
 3. Fields are searched for on any nesting level, indiscriminately. If there are multiple matching fields, the first occurrence is used.
 4. The JSON doesn't have space characters outside of string literals.
 
@@ -47,7 +47,10 @@ Parses the string in double quotes. The value is unescaped. If unescaping failed
 Examples:
 
 ```text
-visitParamExtractString('{"abc":"\\n\\u0000"}', 'abc') = '\n\0'visitParamExtractString('{"abc":"\\u263a"}', 'abc') = '☺'visitParamExtractString('{"abc":"\\u263"}', 'abc') = ''visitParamExtractString('{"abc":"hello}', 'abc') = ''
+visitParamExtractString('{"abc":"\\n\\u0000"}', 'abc') = '\n\0'
+visitParamExtractString('{"abc":"\\u263a"}', 'abc') = '☺'
+visitParamExtractString('{"abc":"\\u263"}', 'abc') = ''
+visitParamExtractString('{"abc":"hello}', 'abc') = ''
 ```
 
 There is currently no support for code points in the format `\uXXXX\uYYYY` that are not from the basic multilingual plane (they are converted to CESU-8 instead of UTF-8).
diff --git a/docs/en/functions/logical_functions.md b/docs/en/functions/logical_functions.md
old mode 100644
new mode 100755
index 4ef0fe5fd32..d396640a49d
--- a/docs/en/functions/logical_functions.md
+++ b/docs/en/functions/logical_functions.md
@@ -11,4 +11,3 @@ Zero as an argument is considered "false," while any non-zero value is considere
 ## not, NOT operator
 
 ## xor
-
diff --git a/docs/en/functions/math_functions.md b/docs/en/functions/math_functions.md
old mode 100644
new mode 100755
index d606c87a509..42e3f3e8018
--- a/docs/en/functions/math_functions.md
+++ b/docs/en/functions/math_functions.md
@@ -97,4 +97,3 @@ The arc tangent.
 ## pow(x, y)
 
 xy.
-
diff --git a/docs/en/functions/other_functions.md b/docs/en/functions/other_functions.md
old mode 100644
new mode 100755
index befd94ecd4e..8a0063750fe
--- a/docs/en/functions/other_functions.md
+++ b/docs/en/functions/other_functions.md
@@ -59,7 +59,8 @@ For elements in a nested data structure, the function checks for the existence o
 
 Allows building a unicode-art diagram.
 
-`bar (x, min, max, width)` – Draws a band with a width proportional to (x - min) and equal to 'width' characters when x == max.`min, max` – Integer constants. The value must fit in Int64.`width` – Constant, positive number, may be a fraction.
+`bar (x, min, max, width)` – Draws a band with a width proportional to (x - min) and equal to 'width' characters when x == max.
+`min, max` – Integer constants. The value must fit in Int64.`width` – Constant, positive number, may be a fraction.
 
 The band is drawn with accuracy to one eighth of a symbol.
 
@@ -137,7 +138,7 @@ Example:
 
 ```sql
 SELECT
-    transform(SearchEngineID, [2, 3], ['Yandex', 'Google'], 'Other' AS title,
+    transform(SearchEngineID, [2, 3], ['Yandex', 'Google'], 'Other') AS title,
     count() AS c
 FROM test.hits
 WHERE SearchEngineID != 0
diff --git a/docs/en/functions/random_functions.md b/docs/en/functions/random_functions.md
old mode 100644
new mode 100755
diff --git a/docs/en/functions/rounding_functions.md b/docs/en/functions/rounding_functions.md
old mode 100644
new mode 100755
diff --git a/docs/en/functions/splitting_merging_functions.md b/docs/en/functions/splitting_merging_functions.md
old mode 100644
new mode 100755
diff --git a/docs/en/functions/string_functions.md b/docs/en/functions/string_functions.md
old mode 100644
new mode 100755
diff --git a/docs/en/functions/string_replace_functions.md b/docs/en/functions/string_replace_functions.md
old mode 100644
new mode 100755
index d3773504278..d70d8f404de
--- a/docs/en/functions/string_replace_functions.md
+++ b/docs/en/functions/string_replace_functions.md
@@ -76,4 +76,3 @@ SELECT replaceRegexpAll('Hello, World!', '^', 'here: ') AS res
 │ here: Hello, World! │
 └─────────────────────┘
 ```
-
diff --git a/docs/en/functions/string_search_functions.md b/docs/en/functions/string_search_functions.md
old mode 100644
new mode 100755
diff --git a/docs/en/functions/type_conversion_functions.md b/docs/en/functions/type_conversion_functions.md
old mode 100644
new mode 100755
diff --git a/docs/en/functions/url_functions.md b/docs/en/functions/url_functions.md
old mode 100644
new mode 100755
diff --git a/docs/en/functions/ym_dict_functions.md b/docs/en/functions/ym_dict_functions.md
old mode 100644
new mode 100755
index 540b5dd601a..7ba7e7012cf
--- a/docs/en/functions/ym_dict_functions.md
+++ b/docs/en/functions/ym_dict_functions.md
@@ -21,7 +21,9 @@ All functions for working with regions have an optional argument at the end –
 Example:
 
 ```text
-regionToCountry(RegionID) – Uses the default dictionary: /opt/geo/regions_hierarchy.txtregionToCountry(RegionID, '') – Uses the default dictionary: /opt/geo/regions_hierarchy.txtregionToCountry(RegionID, 'ua') – Uses the dictionary for the 'ua' key: /opt/geo/regions_hierarchy_ua.txt
+regionToCountry(RegionID) – Uses the default dictionary: /opt/geo/regions_hierarchy.txt
+regionToCountry(RegionID, '') – Uses the default dictionary: /opt/geo/regions_hierarchy.txt
+regionToCountry(RegionID, 'ua') – Uses the dictionary for the 'ua' key: /opt/geo/regions_hierarchy_ua.txt
 ```
 
 ### regionToCity(id[, geobase])
@@ -33,7 +35,9 @@ Accepts a UInt32 number – the region ID from the Yandex geobase. If this regio
 Converts a region to an area (type 5 in the geobase). In every other way, this function is the same as 'regionToCity'.
 
 ```sql
-SELECT DISTINCT regionToName(regionToArea(toUInt32(number), 'ua'))FROM system.numbersLIMIT 15
+SELECT DISTINCT regionToName(regionToArea(toUInt32(number), 'ua'))
+FROM system.numbers
+LIMIT 15
 ```
 
 ```text
@@ -61,7 +65,9 @@ SELECT DISTINCT regionToName(regionToArea(toUInt32(number), 'ua'))FROM system.nu
 Converts a region to a federal district (type 4 in the geobase). In every other way, this function is the same as 'regionToCity'.
 
 ```sql
-SELECT DISTINCT regionToName(regionToDistrict(toUInt32(number), 'ua'))FROM system.numbersLIMIT 15
+SELECT DISTINCT regionToName(regionToDistrict(toUInt32(number), 'ua'))
+FROM system.numbers
+LIMIT 15
 ```
 
 ```text
diff --git a/docs/en/getting_started/example_datasets/amplab_benchmark.md b/docs/en/getting_started/example_datasets/amplab_benchmark.md
old mode 100644
new mode 100755
diff --git a/docs/en/getting_started/example_datasets/criteo.md b/docs/en/getting_started/example_datasets/criteo.md
old mode 100644
new mode 100755
index 9b59d6e5f3d..3c60a68f430
--- a/docs/en/getting_started/example_datasets/criteo.md
+++ b/docs/en/getting_started/example_datasets/criteo.md
@@ -66,6 +66,8 @@ CREATE TABLE criteo
 Transform data from the raw log and put it in the second table:
 
 ```sql
-INSERT INTO criteo SELECT date, clicked, int1, int2, int3, int4, int5, int6, int7, int8, int9, int10, int11, int12, int13, reinterpretAsUInt32(unhex(cat1)) AS icat1, reinterpretAsUInt32(unhex(cat2)) AS icat2, reinterpretAsUInt32(unhex(cat3)) AS icat3, reinterpretAsUInt32(unhex(cat4)) AS icat4, reinterpretAsUInt32(unhex(cat5)) AS icat5, reinterpretAsUInt32(unhex(cat6)) AS icat6, reinterpretAsUInt32(unhex(cat7)) AS icat7, reinterpretAsUInt32(unhex(cat8)) AS icat8, reinterpretAsUInt32(unhex(cat9)) AS icat9, reinterpretAsUInt32(unhex(cat10)) AS icat10, reinterpretAsUInt32(unhex(cat11)) AS icat11, reinterpretAsUInt32(unhex(cat12)) AS icat12, reinterpretAsUInt32(unhex(cat13)) AS icat13, reinterpretAsUInt32(unhex(cat14)) AS icat14, reinterpretAsUInt32(unhex(cat15)) AS icat15, reinterpretAsUInt32(unhex(cat16)) AS icat16, reinterpretAsUInt32(unhex(cat17)) AS icat17, reinterpretAsUInt32(unhex(cat18)) AS icat18, reinterpretAsUInt32(unhex(cat19)) AS icat19, reinterpretAsUInt32(unhex(cat20)) AS icat20, reinterpretAsUInt32(unhex(cat21)) AS icat21, reinterpretAsUInt32(unhex(cat22)) AS icat22, reinterpretAsUInt32(unhex(cat23)) AS icat23, reinterpretAsUInt32(unhex(cat24)) AS icat24, reinterpretAsUInt32(unhex(cat25)) AS icat25, reinterpretAsUInt32(unhex(cat26)) AS icat26 FROM criteo_log;DROP TABLE criteo_log;
+INSERT INTO criteo SELECT date, clicked, int1, int2, int3, int4, int5, int6, int7, int8, int9, int10, int11, int12, int13, reinterpretAsUInt32(unhex(cat1)) AS icat1, reinterpretAsUInt32(unhex(cat2)) AS icat2, reinterpretAsUInt32(unhex(cat3)) AS icat3, reinterpretAsUInt32(unhex(cat4)) AS icat4, reinterpretAsUInt32(unhex(cat5)) AS icat5, reinterpretAsUInt32(unhex(cat6)) AS icat6, reinterpretAsUInt32(unhex(cat7)) AS icat7, reinterpretAsUInt32(unhex(cat8)) AS icat8, reinterpretAsUInt32(unhex(cat9)) AS icat9, reinterpretAsUInt32(unhex(cat10)) AS icat10, reinterpretAsUInt32(unhex(cat11)) AS icat11, reinterpretAsUInt32(unhex(cat12)) AS icat12, reinterpretAsUInt32(unhex(cat13)) AS icat13, reinterpretAsUInt32(unhex(cat14)) AS icat14, reinterpretAsUInt32(unhex(cat15)) AS icat15, reinterpretAsUInt32(unhex(cat16)) AS icat16, reinterpretAsUInt32(unhex(cat17)) AS icat17, reinterpretAsUInt32(unhex(cat18)) AS icat18, reinterpretAsUInt32(unhex(cat19)) AS icat19, reinterpretAsUInt32(unhex(cat20)) AS icat20, reinterpretAsUInt32(unhex(cat21)) AS icat21, reinterpretAsUInt32(unhex(cat22)) AS icat22, reinterpretAsUInt32(unhex(cat23)) AS icat23, reinterpretAsUInt32(unhex(cat24)) AS icat24, reinterpretAsUInt32(unhex(cat25)) AS icat25, reinterpretAsUInt32(unhex(cat26)) AS icat26 FROM criteo_log;
+
+DROP TABLE criteo_log;
 ```
 
diff --git a/docs/en/getting_started/example_datasets/nyc_taxi.md b/docs/en/getting_started/example_datasets/nyc_taxi.md
old mode 100644
new mode 100755
index 11ed81d1a43..a9f04f595d1
--- a/docs/en/getting_started/example_datasets/nyc_taxi.md
+++ b/docs/en/getting_started/example_datasets/nyc_taxi.md
@@ -1,8 +1,8 @@
-# Data about New York taxis
+# New York Taxi data
 
-## How to import raw data
+## How to import the raw data
 
-See <https://github.com/toddwschneider/nyc-taxi-data> and <http://tech.marksblogg.com/billion-nyc-taxi-rides-redshift.html> for description of the dataset and loading instructions.
+See <https://github.com/toddwschneider/nyc-taxi-data> and <http://tech.marksblogg.com/billion-nyc-taxi-rides-redshift.html> for the description of the dataset and instructions for downloading.
 
 Downloading will result in about 227 GB of uncompressed data in CSV files. The download takes about an hour over a 1 Gbit connection (parallel downloading from s3.amazonaws.com recovers at least half of a 1 Gbit channel).
 Some of the files might not download fully. Check the file sizes and re-download any that seem doubtful.
@@ -301,19 +301,14 @@ SELECT passenger_count, toYear(pickup_date) AS year, count(*) FROM trips_mergetr
 Q4:
 
 ```sql
-SELECT passenger_count, toYear(pickup_date) AS year, round(trip_distance) AS distance, count(*)
-FROM trips_mergetree
-GROUP BY passenger_count, year, distance
-ORDER BY year, count(*) DESC
+SELECT passenger_count, toYear(pickup_date) AS year, round(trip_distance) AS distance, count(*)FROM trips_mergetreeGROUP BY passenger_count, year, distanceORDER BY year, count(*) DESC
 ```
 
 3.593 seconds.
 
 The following server was used:
 
-Two Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz, 16 physical kernels total,
-128 GiB RAM,
-8x6 TB HD on hardware RAID-5
+Two Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz, 16 physical kernels total,128 GiB RAM,8x6 TB HD on hardware RAID-5
 
 Execution time is the best of three runsBut starting from the second run, queries read data from the file system cache. No further caching occurs: the data is read out and processed in each run.
 
diff --git a/docs/en/getting_started/example_datasets/ontime.md b/docs/en/getting_started/example_datasets/ontime.md
old mode 100644
new mode 100755
diff --git a/docs/en/getting_started/example_datasets/star_schema.md b/docs/en/getting_started/example_datasets/star_schema.md
old mode 100644
new mode 100755
index 8807de3e670..664ba59f48c
--- a/docs/en/getting_started/example_datasets/star_schema.md
+++ b/docs/en/getting_started/example_datasets/star_schema.md
@@ -1,4 +1,4 @@
-# Star scheme
+# Star Schema Benchmark
 
 Compiling dbgen: <https://github.com/vadimtk/ssb-dbgen>
 
@@ -82,3 +82,4 @@ Downloading data (change 'customer' to 'customerd' in the distributed version):
 cat customer.tbl | sed 's/$/2000-01-01/' | clickhouse-client --query "INSERT INTO customer FORMAT CSV"
 cat lineorder.tbl | clickhouse-client --query "INSERT INTO lineorder FORMAT CSV"
 ```
+
diff --git a/docs/en/getting_started/example_datasets/wikistat.md b/docs/en/getting_started/example_datasets/wikistat.md
old mode 100644
new mode 100755
index 6cbc3b15561..fee0a56b52c
--- a/docs/en/getting_started/example_datasets/wikistat.md
+++ b/docs/en/getting_started/example_datasets/wikistat.md
@@ -20,7 +20,7 @@ CREATE TABLE wikistat
 Loading data:
 
 ```bash
-for i in {2007..2016}; do for j in {01..12}; do echo $i-$j >&2; curl -sS "http://dumps.wikimedia.org/other/pagecounts-raw/$i/$i-$j/" | grep -oE 'pagecounts-[0-9]+-[0-9]+\.gz'; done; done | sort | uniq | tee links.txt
+for i in {2007..2016}; do for j in {01..12}; do echo $i-$j >&2; curl -sSL "http://dumps.wikimedia.org/other/pagecounts-raw/$i/$i-$j/" | grep -oE 'pagecounts-[0-9]+-[0-9]+\.gz'; done; done | sort | uniq | tee links.txt
 cat links.txt | while read link; do wget http://dumps.wikimedia.org/other/pagecounts-raw/$(echo $link | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})[0-9]{2}-[0-9]+\.gz/\1/')/$(echo $link | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})[0-9]{2}-[0-9]+\.gz/\1-\2/')/$link; done
 ls -1 /opt/wikistat/ | grep gz | while read i; do echo $i; gzip -cd /opt/wikistat/$i | ./wikistat-loader --time="$(echo -n $i | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})([0-9]{2})-([0-9]{2})([0-9]{2})([0-9]{2})\.gz/\1-\2-\3 \4-00-00/')" | clickhouse-client --query="INSERT INTO wikistat FORMAT TabSeparated"; done
 ```
diff --git a/docs/en/getting_started/index.md b/docs/en/getting_started/index.md
old mode 100644
new mode 100755
index 42fc1c75551..07d0d91a224
--- a/docs/en/getting_started/index.md
+++ b/docs/en/getting_started/index.md
@@ -16,14 +16,15 @@ The terminal must use UTF-8 encoding (the default in Ubuntu).
 
 For testing and development, the system can be installed on a single server or on a desktop computer.
 
-### Installing from packages Debian/Ubuntu
+### Installing from packages
 
 In `/etc/apt/sources.list` (or in a separate `/etc/apt/sources.list.d/clickhouse.list` file), add the repository:
 
 ```text
-deb http://repo.yandex.ru/clickhouse/deb/stable/ main/
+deb http://repo.yandex.ru/clickhouse/trusty stable main
 ```
 
+On other versions of Ubuntu, replace `trusty` with `xenial` or `precise`.
 If you want to use the most recent test version, replace 'stable' with 'testing'.
 
 Then run:
@@ -34,7 +35,10 @@ sudo apt-get update
 sudo apt-get install clickhouse-client clickhouse-server-common
 ```
 
-You can also download and install packages manually from here: <https://repo.yandex.ru/clickhouse/deb/stable/main/>
+You can also download and install packages manually from here:
+<http://repo.yandex.ru/clickhouse/trusty/pool/main/c/clickhouse/>
+<http://repo.yandex.ru/clickhouse/xenial/pool/main/c/clickhouse/>
+<http://repo.yandex.ru/clickhouse/precise/pool/main/c/clickhouse/>
 
 ClickHouse contains access restriction settings. They are located in the 'users.xml' file (next to 'config.xml').
 By default, access is allowed from anywhere for the 'default' user, without a password. See 'user/default/networks'.
@@ -100,7 +104,8 @@ clickhouse-client
 ```
 
 The default parameters indicate connecting with localhost:9000 on behalf of the user 'default' without a password.
-The client can be used for connecting to a remote server. Example:
+The client can be used for connecting to a remote server.
+Example:
 
 ```bash
 clickhouse-client --host=example.com
@@ -132,3 +137,4 @@ SELECT 1
 **Congratulations, the system works!**
 
 To continue experimenting, you can try to download from the test data sets.
+
diff --git a/docs/en/index.md b/docs/en/index.md
old mode 100644
new mode 100755
index 586c18297a8..72efa70802b
--- a/docs/en/index.md
+++ b/docs/en/index.md
@@ -39,7 +39,7 @@ We'll say that the following is true for the OLAP (online analytical processing)
 - Data is updated in fairly large batches (> 1000 rows), not by single rows; or it is not updated at all.
 - Data is added to the DB but is not modified.
 - For reads, quite a large number of rows are extracted from the DB, but only a small subset of columns.
-- Tables are "wide", meaning they contain a large number of columns.
+- Tables are "wide," meaning they contain a large number of columns.
 - Queries are relatively rare (usually hundreds of queries per server or less per second).
 - For simple queries, latencies around 50 ms are allowed.
 - Column values are fairly small: numbers and short strings (for example, 60 bytes per URL).
diff --git a/docs/en/interfaces/cli.md b/docs/en/interfaces/cli.md
old mode 100644
new mode 100755
index 4fd998fed66..76549b46b36
--- a/docs/en/interfaces/cli.md
+++ b/docs/en/interfaces/cli.md
@@ -6,7 +6,9 @@ To work from the command line, you can use ` clickhouse-client`:
 $ clickhouse-client
 ClickHouse client version 0.0.26176.
 Connecting to localhost:9000.
-Connected to ClickHouse server version 0.0.26176.:)
+Connected to ClickHouse server version 0.0.26176.
+
+:)
 ```
 
 The client supports command-line options and configuration files. For more information, see "[Configuring](#interfaces_cli_configuration)".
@@ -29,7 +31,6 @@ _EOF
 
 cat file.csv | clickhouse-client --database=test --query="INSERT INTO test FORMAT CSV";
 ```
-In batch mode, the default data format is TabSeparated. You can set the format in the FORMAT clause of the query.
 
 By default, you can only process a single query in batch mode. To make multiple queries from a "script," use the --multiquery parameter. This works for all queries except INSERT. Query results are output consecutively without additional separators.
 Similarly, to process a large number of queries, you can run 'clickhouse-client' for each query. Note that it may take tens of milliseconds to launch the 'clickhouse-client' program.
@@ -64,7 +65,7 @@ The command-line client allows passing external data (external temporary tables)
 
 <a name="interfaces_cli_configuration"></a>
 
-## Configure
+## Configuring
 
 You can pass parameters to `clickhouse-client` (all parameters have a default value) using:
 
diff --git a/docs/en/interfaces/http_interface.md b/docs/en/interfaces/http_interface.md
old mode 100644
new mode 100755
index 5c989a59d65..38a70feef46
--- a/docs/en/interfaces/http_interface.md
+++ b/docs/en/interfaces/http_interface.md
@@ -37,7 +37,8 @@ Date: Fri, 16 Nov 2012 19:21:50 GMT
 1
 ```
 
-As you can see, curl is somewhat inconvenient in that spaces must be URL escaped.Although wget escapes everything itself, we don't recommend using it because it doesn't work well over HTTP 1.1 when using keep-alive and Transfer-Encoding: chunked.
+As you can see, curl is somewhat inconvenient in that spaces must be URL escaped.
+Although wget escapes everything itself, we don't recommend using it because it doesn't work well over HTTP 1.1 when using keep-alive and Transfer-Encoding: chunked.
 
 ```bash
 $ echo 'SELECT 1' | curl 'http://localhost:8123/' --data-binary @-
@@ -130,15 +131,11 @@ POST 'http://localhost:8123/?query=DROP TABLE t'
 
 For successful requests that don't return a data table, an empty response body is returned.
 
-You can use compression when transmitting data.
+You can use compression when transmitting data. The compressed data has a non-standard format, and you will need to use the special compressor program to work with it (sudo apt-get install compressor-metrika-yandex).
 
-For using ClickHouse internal compression format, and you will need to use the special clickhouse-compressor program to work with it (installed as a part of clickhouse-client package).
 If you specified 'compress=1' in the URL, the server will compress the data it sends you.
 If you specified 'decompress=1' in the URL, the server will decompress the same data that you pass in the POST method.
 
-Also standard gzip-based HTTP compression can be used. To send gzip compressed POST data just add `Content-Encoding: gzip` to request headers, and gzip POST body.
-To get response compressed, you need to add `Accept-Encoding: gzip` to request headers, and turn on ClickHouse setting called `enable_http_compression`.
-
 You can use this to reduce network traffic when transmitting a large amount of data, or for creating dumps that are immediately compressed.
 
 You can use the 'database' URL parameter to specify the default database.
@@ -194,11 +191,7 @@ $ echo 'SELECT number FROM system.numbers LIMIT 10' | curl 'http://localhost:812
 
 For information about other parameters, see the section "SET".
 
-You can use ClickHouse sessions in the HTTP protocol. To do this, you need to specify the `session_id` GET parameter in HTTP request. You can use any alphanumeric string as a session_id. By default session will be timed out after 60 seconds of inactivity. You can change that by setting `default_session_timeout` in server config file, or by adding GET parameter `session_timeout`. You can also check the status of the session by using GET parameter `session_check=1`. When using sessions you can't run 2 queries with the same session_id simultaneously.
-
-You can get the progress of query execution in X-ClickHouse-Progress headers, by enabling setting send_progress_in_http_headers.
-
-Running query are not aborted automatically after closing HTTP connection. Parsing and data formatting are performed on the server side, and using the network might be ineffective.
+In contrast to the native interface, the HTTP interface does not support the concept of sessions or session settings, does not allow aborting a query (to be exact, it allows this in only a few cases),  and does not show the progress of query processing. Parsing and data formatting are performed on the server side, and using the network might be ineffective.
 The optional 'query_id' parameter can be passed as the query ID (any string). For more information, see the section "Settings, replace_running_query".
 
 The optional 'quota_key' parameter can be passed as the quota key (any string). For more information, see the section "Quotas".
@@ -220,3 +213,4 @@ curl -sS 'http://localhost:8123/?max_result_bytes=4000000&buffer_size=3000000&wa
 ```
 
 Use buffering to avoid situations where a query processing error occurred after the response code and HTTP headers were sent to the client. In this situation, an error message is written at the end of the response body, and on the client side, the error can only be detected at the parsing stage.
+
diff --git a/docs/en/interfaces/index.md b/docs/en/interfaces/index.md
old mode 100644
new mode 100755
index 3e3e3df4853..e43f4474271
--- a/docs/en/interfaces/index.md
+++ b/docs/en/interfaces/index.md
@@ -2,4 +2,5 @@
 
 # Interfaces
 
-To explore the system's capabilities, download data to tables, or make manual queries, use the clickhouse-client program.
\ No newline at end of file
+To explore the system's capabilities, download data to tables, or make manual queries, use the clickhouse-client program.
+
diff --git a/docs/en/interfaces/jdbc.md b/docs/en/interfaces/jdbc.md
old mode 100644
new mode 100755
diff --git a/docs/en/interfaces/tcp.md b/docs/en/interfaces/tcp.md
old mode 100644
new mode 100755
diff --git a/docs/en/interfaces/third-party_client_libraries.md b/docs/en/interfaces/third-party_client_libraries.md
old mode 100644
new mode 100755
index cc8ff1f4307..8437be23b99
--- a/docs/en/interfaces/third-party_client_libraries.md
+++ b/docs/en/interfaces/third-party_client_libraries.md
@@ -2,7 +2,7 @@
 
 There are libraries for working with ClickHouse for:
 
-- Python:
+- Python
    - [infi.clickhouse_orm](https://github.com/Infinidat/infi.clickhouse_orm)
    - [sqlalchemy-clickhouse](https://github.com/cloudflare/sqlalchemy-clickhouse)
    - [clickhouse-driver](https://github.com/mymarilyn/clickhouse-driver)
diff --git a/docs/en/interfaces/third-party_gui.md b/docs/en/interfaces/third-party_gui.md
old mode 100644
new mode 100755
diff --git a/docs/en/introduction/distinctive_features.md b/docs/en/introduction/distinctive_features.md
old mode 100644
new mode 100755
index 3927405579f..59853b8e202
--- a/docs/en/introduction/distinctive_features.md
+++ b/docs/en/introduction/distinctive_features.md
@@ -1,10 +1,10 @@
 # Distinctive features of ClickHouse
 
-## True column-oriented DBMS.
+## True column-oriented DBMS
 
-In a true column-oriented DBMS, there isn't any "garbage" stored with the values. For example, constant-length values must be supported, to avoid storing their length "number" next to the values. As an example, a billion UInt8-type values should actually consume around 1 GB uncompressed, or this will strongly affect the CPU use. It is very important to store data compactly (without any "garbage") even when uncompressed, since the speed of decompression (CPU usage) depends mainly on the volume of uncompressed data.
+In a true column-oriented DBMS, there isn't any "garbage" stored with the values. Among other things, this means that constant-length values must be supported, to avoid storing their length "number" next to the values. As an example, a billion UInt8-type values should actually consume around 1 GB uncompressed, or this will strongly affect the CPU use. It is very important to store data compactly (without any "garbage") even when uncompressed, since the speed of decompression (CPU usage) depends mainly on the volume of uncompressed data.
 
-This is worth noting because there are systems that can store values of separate columns separately, but that can't effectively process analytical queries due to their optimization for other scenarios. Example are HBase, BigTable, Cassandra, and HyperTable. In these systems, you will get throughput around a hundred thousand rows per second, but not hundreds of millions of rows per second.
+This is worth noting because there are systems that can store values of separate columns separately, but that can't effectively process analytical queries due to their optimization for other scenarios. Examples are HBase, BigTable, Cassandra, and HyperTable. In these systems, you will get throughput around a hundred thousand rows per second, but not hundreds of millions of rows per second.
 
 Also note that ClickHouse is a DBMS, not a single database. ClickHouse allows creating tables and databases in runtime, loading data, and running queries without reconfiguring and restarting the server.
 
@@ -12,15 +12,15 @@ Also note that ClickHouse is a DBMS, not a single database. ClickHouse allows cr
 
 Some column-oriented DBMSs (InfiniDB CE and MonetDB) do not use data compression. However, data compression really improves performance.
 
-## Disk storage of data.
+## Disk storage of data
 
 Many column-oriented DBMSs (such as SAP HANA and Google PowerDrill) can only work in RAM. But even on thousands of servers, the RAM is too small for storing all the pageviews and sessions in Yandex.Metrica.
 
-## Parallel processing on multiple cores.
+## Parallel processing on multiple cores
 
 Large queries are parallelized in a natural way.
 
-## Distributed processing on multiple servers.
+## Distributed processing on multiple servers
 
 Almost none of the columnar DBMSs listed above have support for distributed processing.
 In ClickHouse, data can reside on different shards. Each shard can be a group of replicas that are used for fault tolerance. The query is processed on all the shards in parallel. This is transparent for the user.
@@ -30,12 +30,12 @@ In ClickHouse, data can reside on different shards. Each shard can be a group of
 If you are familiar with standard SQL, we can't really talk about SQL support.
 All the functions have different names.
 However, this is a declarative query language based on SQL that can't be differentiated from SQL in many instances.
-Support for JOINs. Subqueries are supported in FROM, IN, and JOIN clauses, as well as scalar subqueries.
+JOINs are supported. Subqueries are supported in FROM, IN, and JOIN clauses, as well as scalar subqueries.
 Dependent subqueries are not supported.
 
 ## Vector engine
 
-Data is not only stored by columns, but is processed by vectors – parts of columns. This allows us to achieve high CPU performance.
+Data is not only stored by columns, but is processed by vectors (parts of columns). This allows us to achieve high CPU performance.
 
 ## Real-time data updates
 
@@ -43,13 +43,13 @@ ClickHouse supports primary key tables. In order to quickly perform queries on t
 
 ## Indexes
 
-Having a primary key allows, for example, extracting data for specific clients (Metrica counters) for a specific time range, with low latency less than several dozen milliseconds.
+Having a primary key makes it possible to extract data for specific clients (for instance, Yandex.Metrica tracking tags) for a specific time range, with low latency less than several dozen milliseconds.
 
 ## Suitable for online queries
 
 This lets us use the system as the back-end for a web interface. Low latency means queries can be processed without delay, while the Yandex.Metrica interface page is loading. In other words, in online mode.
 
-## Support for approximated calculations.
+## Support for approximated calculations
 
 1. The system contains aggregate functions for approximated calculation of the number of various values, medians, and quantiles.
 2. Supports running a query based on a part (sample) of data and getting an approximated result. In this case, proportionally less data is retrieved from the disk.
diff --git a/docs/en/introduction/features_considered_disadvantages.md b/docs/en/introduction/features_considered_disadvantages.md
old mode 100644
new mode 100755
diff --git a/docs/en/introduction/index.md b/docs/en/introduction/index.md
old mode 100644
new mode 100755
index e10b99d0138..3d07efe555d
--- a/docs/en/introduction/index.md
+++ b/docs/en/introduction/index.md
@@ -1 +1,2 @@
 # Introduction
+
diff --git a/docs/en/introduction/performance.md b/docs/en/introduction/performance.md
old mode 100644
new mode 100755
diff --git a/docs/en/introduction/possible_silly_questions.md b/docs/en/introduction/possible_silly_questions.md
old mode 100644
new mode 100755
index 36363ebe247..cf7b2c48032
--- a/docs/en/introduction/possible_silly_questions.md
+++ b/docs/en/introduction/possible_silly_questions.md
@@ -1,8 +1,8 @@
-# Everything you were afraid to ask
+# Questions you were afraid to ask
 
 ## Why not use something like MapReduce?
 
-We can refer to systems like map-reduce as distributed computing systems in which the reduce operation is based on distributed sorting. In this sense, they include Hadoop and YT (Yandex proprietary technology).
+We can refer to systems like map-reduce as distributed computing systems in which the reduce operation is based on distributed sorting. In this sense, they include Hadoop, and YT (YT is developed at Yandex for internal use).
 
 These systems aren't appropriate for online queries due to their high latency. In other words, they can't be used as the back-end for a web interface.
 These types of systems aren't useful for real-time data updates.
diff --git a/docs/en/introduction/ya_metrika_task.md b/docs/en/introduction/ya_metrika_task.md
old mode 100644
new mode 100755
index 6a488be9b5f..10f45f061d6
--- a/docs/en/introduction/ya_metrika_task.md
+++ b/docs/en/introduction/ya_metrika_task.md
@@ -1,6 +1,6 @@
-# The Yandex.Metrica task
+# Yandex.Metrica use case
 
-ClickHouse currently powers [ Yandex.Metrica](https://metrika.yandex.ru/), [ the second largest platform in the world](http://w3techs.com/technologies/overview/traffic_analysis/all) for Web Analytics. With more than 13 trillion records in the database and more than 20 billion events daily, ClickHouse allows you generating custom reports on the fly directly from non-aggregated data.
+ClickHouse currently powers [Yandex.Metrica](https://metrika.yandex.ru/), [the second largest web analytics platform in the world](http://w3techs.com/technologies/overview/traffic_analysis/all). With more than 13 trillion records in the database and more than 20 billion events daily, ClickHouse allows you generating custom reports on the fly directly from non-aggregated data.
 
 We need to get custom reports based on hits and sessions, with custom segments set by the user. Data for the reports is updated in real-time. Queries must be run immediately (in online mode). We must be able to build reports for any time period. Complex aggregates must be calculated, such as the number of unique visitors.
 At this time (April 2014), Yandex.Metrica receives approximately 12 billion events (pageviews and mouse clicks) daily. All these events must be stored in order to build custom reports. A single query may require scanning hundreds of millions of rows over a few seconds, or millions of rows in no more than a few hundred milliseconds.
diff --git a/docs/en/operations/access_rights.md b/docs/en/operations/access_rights.md
old mode 100644
new mode 100755
index 9879dab9a99..1c72bf13b3e
--- a/docs/en/operations/access_rights.md
+++ b/docs/en/operations/access_rights.md
@@ -2,14 +2,14 @@
 
 Users and access rights are set up in the user config. This is usually `users.xml`.
 
-Users are recorded in the 'users' section. We'll look at a fragment of the `users.xml` file:
+Users are recorded in the 'users' section. Here is a fragment of the `users.xml` file:
 
 ```xml
 <!-- Users and ACL. -->
 <users>
     <!-- If the user name is not specified, the 'default' user is used. -->
-        <default>
-            <!-- Password could be specified in plaintext or in SHA256 (in hex format).
+    <default>
+                <!-- Password could be specified in plaintext or in SHA256 (in hex format).
 
              If you want to specify the password in plain text (not recommended), place it in the 'password' element.
              Example: <password>qwerty</password>.
@@ -17,13 +17,14 @@ Users are recorded in the 'users' section. We'll look at a fragment of the `user
 
              If you want to specify SHA256, place it in the 'password_sha256_hex' element.
              Example: <password_sha256_hex>65e84be33532fb784c48129675f9eff3a682b27168c0ea744b2cf58ee02337c5</password_sha256_hex>
-             How to generate a decent password:
-             Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha256sum | tr -d '-'
-             The first line has the password and the second line has the corresponding SHA256.
-        -->
-        <password></password>        
+                          How to generate a decent password:             Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha256sum | tr -d '-'             
+                          The first line has the password and the second line has the corresponding SHA256.
+        -->        
+        <password></password>
+                
         <!-- A list of networks that access is allowed from.
-            Each list item has one of the following forms:<ip>IP address or subnet mask. For example: 198.51.100.0/24 or 2001:DB8::/32.
+            Each list item has one of the following forms:
+            <ip>IP address or subnet mask. For example: 198.51.100.0/24 or 2001:DB8::/32.
             <host> Host name. For example: example01. A DNS query is made for verification, and all addresses obtained are compared with the address of the customer.
             <host_regexp> Regular expression for host names. For example: ^example\d\d-\d\d-\d\.yandex\.ru$                
             For verification, a DNS PTR query is made for the customer's address and a regular expression is applied to the result.
@@ -31,34 +32,34 @@ Users are recorded in the 'users' section. We'll look at a fragment of the `user
             We strongly recommend that the regex ends with \.yandex\.ru$.
 
             If you are installing ClickHouse yourself, enter:
-                            <networks>                        
-                                    <ip>::/0</ip>
-                            </networks>        
-          -->        
-          <networks incl="networks" />
-  
-          <!-- Settings profile for the user. -->
-          <profile>default</profile>
-  
+                 <networks>                        
+                         <ip>::/0</ip>
+                </networks>
+         -->        
+         <networks incl="networks" />        
+         
+         <!-- Settings profile for the user. -->        
+         <profile>default</profile>        
+         
          <!-- Quota for the user. -->        
          <quota>default</quota>    
-     </default>    
-     
-     <!-- For requests from the Yandex.Metrica user interface via the API for data on specific counters. -->    
-     <web>        
-         <password></password>        
-         <networks incl="networks" />        
-         <profile>web</profile>        
-         <quota>default</quota>        
-         <allow_databases>           
-            <database>test</database>
+         </default>    
+         
+         <!-- For requests from the Yandex.Metrica user interface via the API for data on specific counters. -->    
+         <web>        
+             <password></password>        
+             <networks incl="networks" />        
+             <profile>web</profile>        
+             <quota>default</quota>        
+             <allow_databases>           
+             <database>test</database>
         </allow_databases>
     </web>
 ```
 
 You can see a declaration from two users: `default`and`web`. We added the `web` user separately.
 
-The `default` user is chosen in cases when the username is not passed. The `default` user is also used for distributed query processing, if the configuration of the server or cluster doesn't specify `user` and `password` (see the section on the [Distributed](../table_engines/distributed.md#distributed_distributed) engine).
+The `default` user is chosen in cases when the username is not passed. The `default` user is also used for distributed query processing, if the configuration of the server or cluster doesn't specify the `user` and `password` (see the section on the [Distributed](../table_engines/distributed.md#table_engines-distributed) engine).
 
 The user that is used for exchanging information between servers combined in a cluster must not have substantial restrictions or quotas – otherwise, distributed queries will fail.
 
diff --git a/docs/en/operations/configuration_files.md b/docs/en/operations/configuration_files.md
old mode 100644
new mode 100755
index c3122617bf1..52e9e10ffea
--- a/docs/en/operations/configuration_files.md
+++ b/docs/en/operations/configuration_files.md
@@ -14,7 +14,7 @@ If `replace` is specified, it replaces the entire element with the specified one
 
 If ` remove` is specified, it deletes the element.
 
-The config can also define "substitutions". If an element has the `incl` attribute, the corresponding substitution from the file will be used as the value. By default, the path to the file with substitutions is `/etc/metrika.xml`. This can be changed in the config in the [include_from](server_settings/settings.md#server_settings-include_from) element. The substitution values are specified in  `/yandex/substitution_name` elements in this file. If a substitution specified in ` incl`  does not exist, it is recorded in the log. To prevent ClickHouse from logging missing substitutions, specify the  `optional = "true"` attribute (for example, settings for [ macros](server_settings/settings.md#server_settings-macros)).
+The config can also define "substitutions". If an element has the `incl` attribute, the corresponding substitution from the file will be used as the value. By default, the path to the file with substitutions is `/etc/metrika.xml`. This can be changed in the [include_from](server_settings/settings.md#server_settings-include_from) element in the server config. The substitution values are specified in  `/yandex/substitution_name` elements in this file. If a substitution specified in ` incl`  does not exist, it is recorded in the log. To prevent ClickHouse from logging missing substitutions, specify the  `optional="true"` attribute (for example, settings for [macros]()server_settings/settings.md#server_settings-macros)).
 
 Substitutions can also be performed from ZooKeeper. To do this, specify the attribute `from_zk = "/path/to/node"`. The element value is replaced with the contents of the node at ` /path/to/node` in ZooKeeper. You can also put an entire XML subtree on the ZooKeeper node and it will be fully inserted into the source element.
 
diff --git a/docs/en/operations/index.md b/docs/en/operations/index.md
old mode 100644
new mode 100755
index eb90f937cff..0ff38af8086
--- a/docs/en/operations/index.md
+++ b/docs/en/operations/index.md
@@ -1 +1,2 @@
-# Operation
+# Usage
+
diff --git a/docs/en/operations/quotas.md b/docs/en/operations/quotas.md
old mode 100644
new mode 100755
index d7b1a61ce7f..fb1238b257d
--- a/docs/en/operations/quotas.md
+++ b/docs/en/operations/quotas.md
@@ -18,10 +18,10 @@ Let's look at the section of the 'users.xml' file that defines quotas.
     <!-- Quota name. -->    
     <default>
         <!-- Restrictions for a time period. You can set many intervals with different restrictions. -->        
-        <interval>
+        <interval>            
             <!-- Length of the interval. -->            
-            <duration>3600</duration>            
-            
+            <duration>3600</duration>
+
             <!-- Unlimited. Just collect data for the specified time interval. -->            
             <queries>0</queries>            
             <errors>0</errors>            
@@ -39,21 +39,19 @@ The resource consumption calculated for each interval is output to the server lo
 <statbox>
     <!-- Restrictions for a time period. You can set many intervals with different restrictions. -->    
     <interval>        
-        <!-- Length of the interval. -->
-        <duration>3600</duration>
+        <!-- Length of the interval. -->        
+        <duration>3600</duration>        
+        <queries>1000</queries>        
+        <errors>100</errors>        
+        <result_rows>1000000000</result_rows>        
+        <read_rows>100000000000</read_rows>        
+        <execution_time>900</execution_time>    
+    </interval>    
 
-        <queries>1000</queries>
-        <errors>100</errors>
-        <result_rows>1000000000</result_rows>
-        <read_rows>100000000000</read_rows>
-        <execution_time>900</execution_time>
-    </interval>
-
-    <interval>
-        <duration>86400</duration>
-
-        <queries>10000</queries>
-        <errors>1000</errors>
+    <interval>        
+        <duration>86400</duration>        
+        <queries>10000</queries>        
+        <errors>1000</errors>        
         <result_rows>5000000000</result_rows>
         <read_rows>500000000000</read_rows>
         <execution_time>7200</execution_time>
@@ -89,7 +87,7 @@ Quotas can use the "quota key" feature in order to report on resources for multi
         Using keys makes sense only if quota_key is transmitted by the program, not by a user.
 
         You can also write <keyed_by_ip /> so the IP address is used as the quota key.(But keep in mind that users can change the IPv6 address fairly easily.)
-         --> 
+    -->
     <keyed />
 ```
 
@@ -98,4 +96,3 @@ The quota is assigned to users in the 'users' section of the config. See the sec
 For distributed query processing, the accumulated amounts are stored on the requestor server. So if the user goes to another server, the quota there will "start over".
 
 When the server is restarted, quotas are reset.
-
diff --git a/docs/en/operations/server_settings/index.md b/docs/en/operations/server_settings/index.md
old mode 100644
new mode 100755
index 2293e86f5c7..208deec710c
--- a/docs/en/operations/server_settings/index.md
+++ b/docs/en/operations/server_settings/index.md
@@ -9,3 +9,4 @@ These settings are stored in the ` config.xml` file on the ClickHouse server.
 Other settings are described in the "[Settings](../settings/index.md#settings)" section.
 
 Before studying the settings, read the [Configuration files](../configuration_files.md#configuration_files) section and note the use of substitutions (the `incl` and `optional` attributes).
+
diff --git a/docs/en/operations/server_settings/settings.md b/docs/en/operations/server_settings/settings.md
old mode 100644
new mode 100755
index 8818f8ec932..e1575df2f88
--- a/docs/en/operations/server_settings/settings.md
+++ b/docs/en/operations/server_settings/settings.md
@@ -67,7 +67,7 @@ ClickHouse checks ` min_part_size`  and ` min_part_size_ratio`  and processes th
 
 The default database.
 
-Use a [ SHOW DATABASES](../../query_language/queries.md#query_language_queries_show_databases) query to get a list of databases.
+To get a list of databases, use the [ SHOW DATABASES]( query./../query_language/queries.md#query_language_queries_show_databases).
 
 **Example**
 
@@ -81,7 +81,7 @@ Use a [ SHOW DATABASES](../../query_language/queries.md#query_language_queries_s
 
 Default settings profile.
 
-Settings profiles are located in the file specified in the [user_config](#server_settings-users_config) parameter.
+Settings profiles are located in the file specified in the parameter [user_config](#server_settings-users_config).
 
 **Example**
 
@@ -100,7 +100,7 @@ Path:
 - Specify the absolute path or the path relative to the server config file.
 - The path can contain wildcards \* and ?.
 
-See also "[External dictionaries](../../dicts/external_dicts.md#dicts-external_dicts)".
+See also "[External dictionaries]("./../dicts/external_dicts.md#dicts-external_dicts)".
 
 **Example**
 
@@ -130,12 +130,12 @@ The default is ` true`.
 
 ## format_schema_path
 
-The path to the directory with the schemas for the input data, such as schemas for the [ CapnProto](../../formats/capnproto.md#format_capnproto) format.
+The path to the directory with the schemes for the input data, such as schemas for the [CapnProto](../../formats/capnproto.md#format_capnproto) format.
 
 **Example**
 
 ```xml
-<!-- Directory containing schema files for various input formats. -->
+  <!-- Directory containing schema files for various input formats. -->
   <format_schema_path>format_schemas/</format_schema_path>
 ```
 
@@ -179,7 +179,7 @@ You can configure multiple `<graphite>` clauses. For instance, you can use this
 
 Settings for thinning data for Graphite.
 
-For more details, see [ GraphiteMergeTree](../../table_engines/graphitemergetree.md#table_engines-graphitemergetree).
+For more information, see [GraphiteMergeTree](../../table_engines/graphitemergetree.md#table_engines-graphitemergetree).
 
 **Example**
 
@@ -241,7 +241,7 @@ Opens `https://tabix.io/` when accessing ` http://localhost: http_port`.
 
 The path to the file with substitutions.
 
-For details, see the section "[Configuration files](../configuration_files.md#configuration_files)".
+For more information, see the section "[Configuration files](../configuration_files.md#configuration_files)".
 
 **Example**
 
@@ -298,7 +298,8 @@ Restriction on hosts that requests can come from. If you want the server to answ
 Examples:
 
 ```xml
-<listen_host>::1</listen_host><listen_host>127.0.0.1</listen_host>
+<listen_host>::1</listen_host>
+<listen_host>127.0.0.1</listen_host>
 ```
 
 <a name="server_settings-logger"></a>
@@ -347,7 +348,7 @@ For more information, see the section "[Creating replicated tables](../../table_
 
 ## mark_cache_size
 
-Approximate size (in bytes) of the cache of "marks" used by [ MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) engines.
+Approximate size (in bytes) of the cache of "marks" used by [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) family.
 
 The cache is shared for the server and memory is allocated as needed. The cache size must be at least 5368709120.
 
@@ -403,7 +404,7 @@ We recommend using this option in Mac OS X, since the ` getrlimit()` function re
 
 Restriction on deleting tables.
 
-If the size of a [ MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) type table  exceeds `max_table_size_to_drop` (in bytes), you can't delete it using a DROP query.
+If the size of a [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) type table exceeds `max_table_size_to_drop` (in bytes), you can't delete it using a DROP query.
 
 If you still need to delete the table without restarting the ClickHouse server, create the ` <clickhouse-path>/flags/force_drop_table` file and run the DROP query.
 
@@ -439,17 +440,17 @@ For more information, see the MergeTreeSettings.h header file.
 
 SSL client/server configuration.
 
-Support for SSL is provided by the `` libpoco`` library. The description of the interface is in the [ SSLManager.h file.](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h)
+Support for SSL is provided by the `` libpoco`` library. The interface is described in the file [SSLManager.h](https://github.com/yandex/ClickHouse/blob/master/contrib/libpoco/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h)
 
 Keys for server/client settings:
 
 - privateKeyFile – The path to the file with the secret key of the PEM certificate. The file may contain a key and certificate at the same time.
 - certificateFile – The path to the client/server certificate file in PEM format. You can omit it if `` privateKeyFile`` contains the certificate.
 - caConfig – The path to the file or directory that contains trusted root certificates.
-- verificationMode – The method for checking the node's certificates. Details are in the description of the [Context](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/Context.h) class. Acceptable values: ``none``, ``relaxed``, ``strict``, ``once``.
+- verificationMode – The method for checking the node's certificates. Details are in the description of the [Context](https://github.com/yandex/ClickHouse/blob/master/contrib/libpoco/NetSSL_OpenSSL/include/Poco/Net/Context.h) class. Possible values: ``none``, ``relaxed``, ``strict``, ``once``.
 - verificationDepth – The maximum length of the verification chain. Verification will fail if the certificate chain length exceeds the set value.
 - loadDefaultCAFile – Indicates that built-in CA certificates for OpenSSL will be used. Acceptable values: `` true``, `` false``.  |
-- cipherList - Supported OpenSSL-ciphers. For example: `` ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH``.
+- cipherList - Поддерживаемые OpenSSL-шифры. For example: `` ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH``.
 - cacheSessions – Enables or disables caching sessions. Must be used in combination with ``sessionIdContext``. Acceptable values: `` true``, `` false``.
 - sessionIdContext – A unique set of random characters that the server appends to each generated identifier. The length of the string must not exceed ``SSL_MAX_SSL_SESSION_ID_LENGTH``. This parameter is always recommended, since it helps avoid problems both if the server caches the session and if the client requested caching. Default value: ``${application.name}``.
 - sessionCacheSize – The maximum number of sessions that the server caches. Default value: 1024\*20. 0 – Unlimited sessions.
@@ -498,7 +499,7 @@ Keys for server/client settings:
 
 ## part_log
 
-Logging events that are associated with the [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) data type. For instance, adding or merging data. You can use the log to simulate merge algorithms and compare their characteristics. You can visualize the merge process.
+Logging events that are associated with [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) data. For instance, adding or merging data. You can use the log to simulate merge algorithms and compare their characteristics. You can visualize the merge process.
 
 Queries are logged in the ClickHouse table, not in a separate file.
 
@@ -518,7 +519,7 @@ Use the following parameters to configure logging:
 
 - database – Name of the database.
 - table – Name of the table.
-- partition_by - Sets the [custom partition key](../../table_engines/custom_partitioning_key.md#custom-partitioning-key).
+- partition_by – Sets a [custom partitioning key](../../table_engines/custom_partitioning_key.md#custom-partitioning-key).
 - flush_interval_milliseconds – Interval for flushing data from memory to the disk.
 
 **Example**
@@ -562,7 +563,7 @@ Use the following parameters to configure logging:
 
 - database – Name of the database.
 - table – Name of the table.
-- partition_by - Sets the [custom partition key](../../table_engines/custom_partitioning_key.md#custom-partitioning-key).
+- partition_by – Sets a [custom partitioning key](../../table_engines/custom_partitioning_key.md#custom-partitioning-key).
 - flush_interval_milliseconds – Interval for flushing data from memory to the disk.
 
 If the table doesn't exist, ClickHouse will create it. If the structure of the query log changed when the ClickHouse server was updated, the table with the old structure is renamed, and a new table is created automatically.
@@ -584,7 +585,7 @@ If the table doesn't exist, ClickHouse will create it. If the structure of the q
 
 Configuration of clusters used by the Distributed table engine.
 
-For more information, see the section "[Duplicated table engine](../../table_engines/distributed.md#table_engines-distributed)".
+For more information, see the section "[Table engines/Distributed](../../table_engines/distributed.md#table_engines-distributed)".
 
 **Example**
 
@@ -644,7 +645,7 @@ The end slash is mandatory.
 
 ## uncompressed_cache_size
 
-Cache size (in bytes) for uncompressed data used by table engines from the [ MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) family.
+Cache size (in bytes) for uncompressed data used by table engines from the [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) family.
 
 There is one shared cache for the server. Memory is allocated on demand. The cache is used if the option [use_uncompressed_cache](../settings/settings.md#settings-use_uncompressed_cache) is enabled.
 
diff --git a/docs/en/operations/settings/index.md b/docs/en/operations/settings/index.md
old mode 100644
new mode 100755
index 0e967a4c081..0c5ca5d5171
--- a/docs/en/operations/settings/index.md
+++ b/docs/en/operations/settings/index.md
@@ -9,9 +9,9 @@ Ways to configure settings, in order of priority:
 
 - Settings in the server config file.
 
-   Set via user profiles.
+   Settings from user profiles.
 
-- For the session.
+- Session settings.
 
    Send ` SET setting=value` from the ClickHouse console client in interactive mode.
 Similarly, you can use ClickHouse sessions in the HTTP protocol. To do this, you need to specify the `session_id` HTTP parameter.
diff --git a/docs/en/operations/settings/query_complexity.md b/docs/en/operations/settings/query_complexity.md
old mode 100644
new mode 100755
diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
old mode 100644
new mode 100755
index a87328cb06c..7d95b9609c7
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -4,7 +4,7 @@
 
 ## distributed_product_mode
 
-Alters the behavior of [distributed subqueries](../../query_language/queries.md#queries-distributed-subrequests), i.e. in cases when the query contains the product of distributed tables.
+Changes the behavior of [distributed subqueries](../../query_language/queries.md#queries-distributed-subrequests), i.e. in cases when the query contains the product of distributed tables.
 
 ClickHouse applies the configuration if the subqueries on any level have a distributed table that exists on the local server and has more than one shard.
 
@@ -12,9 +12,9 @@ Restrictions:
 
 - Only applied for IN and JOIN subqueries.
 - Used only if a distributed table is used in the FROM clause.
-- Not used for a table-valued [ remote](../../table_functions/remote.md#table_functions-remote)function.
+- Not used for a table-valued [ remote](../../table_functions/remote.md#table_functions-remote) function.
 
-Possible values:
+The possible values ​​are:
 
 <a name="settings-settings-fallback_to_stale_replicas_for_distributed_queries"></a>
 
@@ -36,7 +36,7 @@ Disables query execution if the index can't be used by date.
 
 Works with tables in the MergeTree family.
 
-If `force_index_by_date=1`,  ClickHouse checks whether the query has a date key condition that can be used for restricting data ranges. If there is no suitable condition, it throws an exception. However, it does not check whether the condition actually reduces the amount of data to read. For example, the condition `Date != ' 2000-01-01 '` is acceptable even when it matches all the data in the table (i.e., running the query requires a full scan). For more information about ranges of data in MergeTree tables, see [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree)".
+If `force_index_by_date=1`,  ClickHouse checks whether the query has a date key condition that can be used for restricting data ranges. If there is no suitable condition, it throws an exception. However, it does not check whether the condition actually reduces the amount of data to read. For example, the condition `Date != ' 2000-01-01 '` is acceptable even when it matches all the data in the table (i.e., running the query requires a full scan). For more information about ranges of data in MergeTree tables, see "[MergeTree](../../table_engines/mergetree.md#table_engines-mergetree)".
 
 <a name="settings-settings-force_primary_key"></a>
 
@@ -46,7 +46,7 @@ Disables query execution if indexing by the primary key is not possible.
 
 Works with tables in the MergeTree family.
 
-If `force_primary_key=1`,  ClickHouse checks to see if the query has a primary key condition that can be used for restricting data ranges. If there is no suitable condition, it throws an exception. However, it does not check whether the condition actually reduces the amount of data to read. For more information about ranges of data in MergeTree tables, see [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree)".
+If `force_primary_key=1`,  ClickHouse checks to see if the query has a primary key condition that can be used for restricting data ranges. If there is no suitable condition, it throws an exception. However, it does not check whether the condition actually reduces the amount of data to read. For more information about data ranges in MergeTree tables, see "[MergeTree](../../table_engines/mergetree.md#table_engines-mergetree)".
 
 <a name="settings_settings_fsync_metadata"></a>
 
@@ -158,7 +158,7 @@ Don't confuse blocks for compression (a chunk of memory consisting of bytes) and
 
 ## min_compress_block_size
 
-For [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) tables. In order to reduce latency when processing queries, a block is compressed when writing the next mark if its size is at least 'min_compress_block_size'. By default, 65,536.
+For [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree)" tables. In order to reduce latency when processing queries, a block is compressed when writing the next mark if its size is at least 'min_compress_block_size'. By default, 65,536.
 
 The actual size of the block, if the uncompressed data is less than 'max_compress_block_size', is no less than this value and no less than the volume of data for one mark.
 
@@ -253,13 +253,13 @@ Yandex.Metrica uses this parameter set to 1 for implementing suggestions for seg
 
 ## schema
 
-This parameter is useful when you are using formats that require a schema definition, such as [ Cap'n Proto](https://capnproto.org/). The value depends on the format.
+This parameter is useful when you are using formats that require a schema definition, such as [Cap'n Proto](https://capnproto.org/). The value depends on the format.
 
 <a name="settings-settings_stream_flush_interval_ms"></a>
 
 ## stream_flush_interval_ms
 
-Works for tables with streaming in the case of a timeout, or when a thread generates [ max_insert_block_size](#settings-settings-max_insert_block_size)  rows.
+Works for tables with streaming in the case of a timeout, or when a thread generates[max_insert_block_size](#settings-settings-max_insert_block_size) rows.
 
 The default value is 7500.
 
@@ -345,7 +345,7 @@ If the value is true, integers appear in quotes when using JSON\* Int64 and UInt
 
 Strictly assigns default values when adding data.
 
-If data is not specified for a column when running an [ INSERT](../../query_language/queries.md#queries-insert)  query, ClickHouse assigns default values to the fields. Default values are defined in the ` DEFAULT` property for each column in the table settings. If ` DEFAULT` is not defined for a column:
+If data is not specified for a column when running an [ INSERT](../../query_language/queries.md#queries-insert) query, ClickHouse assigns default values to the fields. Default values are defined in the ` DEFAULT` property for each column in the table settings. If ` DEFAULT` is not defined for a column:
 
 - `If strict_insert_defaults = 0`, column fields are assigned zeros and empty strings.
 - `If strict_insert_defaults = 1`, ClickHouse throws an exception and requires the user to pass data to the column.
diff --git a/docs/en/operations/settings/settings_profiles.md b/docs/en/operations/settings/settings_profiles.md
old mode 100644
new mode 100755
index c978c599bd5..f1fce41ba75
--- a/docs/en/operations/settings/settings_profiles.md
+++ b/docs/en/operations/settings/settings_profiles.md
@@ -17,7 +17,7 @@ Example:
 <profiles>
     <!-- Default settings -->
     <default>
-    <!-- The maximum number of threads when running a single query. -->
+        <!-- The maximum number of threads when running a single query. -->
         <max_threads>8</max_threads>
     </default>
 
diff --git a/docs/en/operations/tips.md b/docs/en/operations/tips.md
old mode 100644
new mode 100755
index 11fc8f6da11..652698fe24c
--- a/docs/en/operations/tips.md
+++ b/docs/en/operations/tips.md
@@ -105,7 +105,7 @@ Use at least a 10 GB network, if possible. 1 Gb will also work, but it will be m
 
 You are probably already using ZooKeeper for other purposes. You can use the same installation of ZooKeeper, if it isn't already overloaded.
 
-It's best to use a fresh version of ZooKeeper – 3.5 or later. The version in stable Linux distributions may be outdated.
+It's best to use a fresh version of ZooKeeper – 3.4.9 or later. The version in stable Linux distributions may be outdated.
 
 With the default settings, ZooKeeper is a time bomb:
 
@@ -174,8 +174,7 @@ dynamicConfigFile=/etc/zookeeper-{{ cluster['name'] }}/conf/zoo.cfg.dynamic
 Java version:
 
 ```text
-Java(TM) SE Runtime Environment (build 1.8.0_25-b17)
-Java HotSpot(TM) 64-Bit Server VM (build 25.25-b02, mixed mode)
+Java(TM) SE Runtime Environment (build 1.8.0_25-b17)Java HotSpot(TM) 64-Bit Server VM (build 25.25-b02, mixed mode)
 ```
 
 JVM parameters:
diff --git a/docs/en/operators/index.md b/docs/en/operators/index.md
old mode 100644
new mode 100755
index 779f3bf3843..411cde34b50
--- a/docs/en/operators/index.md
+++ b/docs/en/operators/index.md
@@ -67,11 +67,11 @@ Groups of operators are listed in order of priority (the higher it is in the lis
 
 `NOT a` The `not(a) function.`
 
-## Logical 'AND' operator
+## Logical AND operator
 
 `a AND b` – The`and(a, b) function.`
 
-## Logical 'OR' operator
+## Logical OR operator
 
 `a OR b` – The `or(a, b) function.`
 
diff --git a/docs/en/query_language/index.md b/docs/en/query_language/index.md
old mode 100644
new mode 100755
index 247d76fc6ed..769d94eb4fd
--- a/docs/en/query_language/index.md
+++ b/docs/en/query_language/index.md
@@ -1 +1,2 @@
 # Query language
+
diff --git a/docs/en/query_language/queries.md b/docs/en/query_language/queries.md
old mode 100644
new mode 100755
index 743706a551f..d235945a646
--- a/docs/en/query_language/queries.md
+++ b/docs/en/query_language/queries.md
@@ -11,7 +11,6 @@ CREATE DATABASE [IF NOT EXISTS] db_name
 `A database` is just a directory for tables.
 If `IF NOT EXISTS` is included, the query won't return an error if the database already exists.
 
-
 <a name="query_language-queries-create_table"></a>
 
 ## CREATE TABLE
@@ -184,7 +183,7 @@ Deletes all tables inside the 'db' database, then deletes the 'db' database itse
 If `IF EXISTS` is specified, it doesn't return an error if the database doesn't exist.
 
 ```sql
-DROP [TEMPORARY] TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster]
+DROP TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster]
 ```
 
 Deletes the table.
@@ -308,8 +307,7 @@ SELECT * FROM system.parts WHERE active
 `active` – Only count active parts. Inactive parts are, for example, source parts remaining after merging to a larger part – these parts are deleted approximately 10 minutes after merging.
 
 Another way to view a set of parts and partitions is to go into the directory with table data.
-Data directory: `/var/lib/clickhouse/data/database/table/`,
-where `/var/lib/clickhouse/` is the path to the ClickHouse data, 'database' is the database name, and 'table' is the table name. Example:
+Data directory: `/var/lib/clickhouse/data/database/table/`,where `/var/lib/clickhouse/` is the path to the ClickHouse data, 'database' is the database name, and 'table' is the table name. Example:
 
 ```bash
 $ ls -l /var/lib/clickhouse/data/test/visits/
@@ -325,7 +323,7 @@ Here, `20140317_20140323_2_2_0` and ` 20140317_20140323_4_4_0` are the directori
 Let's break down the name of the first part: `20140317_20140323_2_2_0`.
 
 - `20140317` is the minimum date of the data in the chunk.
-- `20140323` is the maximum date of the data in the chunk.
+- `20140323` is the maximum data of the data in the chunk.
 - `2` is the minimum number of the data block.
 - `2` is the maximum number of the data block.
 - `0` is the chunk level (the depth of the merge tree it is formed from).
@@ -452,7 +450,7 @@ See also the section "Formats".
 ## SHOW TABLES
 
 ```sql
-SHOW [TEMPORARY] TABLES [FROM db] [LIKE 'pattern'] [INTO OUTFILE filename] [FORMAT format]
+SHOW TABLES [FROM db] [LIKE 'pattern'] [INTO OUTFILE filename] [FORMAT format]
 ```
 
 Displays a list of tables
@@ -462,7 +460,7 @@ Displays a list of tables
 
 This query is identical to: `SELECT name FROM system.tables WHERE database = 'db' [AND name LIKE 'pattern'] [INTO OUTFILE filename] [FORMAT format]`.
 
-See the section "LIKE operator" also.
+See also the section "LIKE operator".
 
 ## SHOW PROCESSLIST
 
@@ -486,7 +484,7 @@ Prints a table containing the columns:
 
 **query** – The query itself. In INSERT queries, the data for insertion is not output.
 
-**query_id** - The query identifier. Non-empty only if it was explicitly defined by the user. For distributed processing, the query ID is not passed to remote servers.
+**query_id** – The query identifier. Non-empty only if it was explicitly defined by the user. For distributed processing, the query ID is not passed to remote servers.
 
 This query is identical to: `SELECT * FROM system.processes [INTO OUTFILE filename] [FORMAT format]`.
 
@@ -499,7 +497,7 @@ watch -n1 "clickhouse-client --query='SHOW PROCESSLIST'"
 ## SHOW CREATE TABLE
 
 ```sql
-SHOW CREATE [TEMPORARY] TABLE [db.]table [INTO OUTFILE filename] [FORMAT format]
+SHOW CREATE TABLE [db.]table [INTO OUTFILE filename] [FORMAT format]
 ```
 
 Returns a single `String`-type 'statement' column, which contains a single value – the `CREATE` query used for creating the specified table.
@@ -517,7 +515,7 @@ Nested data structures are output in "expanded" format. Each column is shown sep
 ## EXISTS
 
 ```sql
-EXISTS [TEMPORARY] TABLE [db.]name [INTO OUTFILE filename] [FORMAT format]
+EXISTS TABLE [db.]name [INTO OUTFILE filename] [FORMAT format]
 ```
 
 Returns a single `UInt8`-type column, which contains the single value `0` if the table or database doesn't exist, or `1` if the table exists in the specified database.
@@ -573,9 +571,9 @@ The query can specify a list of columns to insert `[(c1, c2, c3)]`. In this case
 - The values calculated from the `DEFAULT`  expressions specified in the table definition.
 - Zeros and empty strings, if `DEFAULT` expressions are not defined.
 
-If [strict_insert_defaults=1](../operations/settings/settings.md#settings-strict_insert_defaults), columns that do not have ` DEFAULT` defined must be listed in the query.
+If [strict_insert_defaults=1](../operations/settings/settings.md#settings-strict_insert_defaults), columns that do not have `DEFAULT` defined must be listed in the query.
 
-The INSERT can pass data in any [format](../formats/index.md#formats) supported by ClickHouse. The format must be specified explicitly in the query:
+Data can be passed to the INSERT in any [format](../formats/index.md#formats) supported by ClickHouse. The format must be specified explicitly in the query:
 
 ```sql
 INSERT INTO [db.]table [(c1, c2, c3)] FORMAT format_name data_set
@@ -974,8 +972,7 @@ All columns that are not needed for the JOIN are deleted from the subquery.
 
 There are several types of JOINs:
 
-`INNER` or `LEFT` type:
-If INNER is specified, the result will contain only those rows that have a matching row in the right table.
+`INNER` or `LEFT` type:If INNER is specified, the result will contain only those rows that have a matching row in the right table.
 If LEFT is specified, any rows in the left table that don't have matching rows in the right table will be assigned the default value - zeros or empty rows. LEFT OUTER may be written instead of LEFT; the word OUTER does not affect anything.
 
 `ANY` or `ALL` stringency:If `ANY` is specified and the right table has several matching rows, only the first one found is joined.
@@ -1106,7 +1103,7 @@ Example:
 SELECT
     domainWithoutWWW(URL) AS domain,
     count(),
-    any(Title) AS title -- getting the first occurred page header for each domain.
+    any(Title) AS title -- getting the first occurring page header for each domain.
 FROM hits
 GROUP BY domain
 ```
@@ -1351,7 +1348,7 @@ There are two options for IN-s with subqueries (similar to JOINs): normal `IN`
 
 <div class="admonition attention">
 
-Remember that the algorithms described below may work differently depending on the [](../operations/settings/settings.md#settings-distributed_product_mode) `distributed_product_mode` setting.
+Remember that the algorithms described below may work differently depending on the [settings](../operations/settings/settings.md#settings-distributed_product_mode) `distributed_product_mode` setting.
 
 </div>
 
@@ -1479,34 +1476,34 @@ In all other cases, we don't recommend using the asterisk, since it only gives y
 ## KILL QUERY
 
 ```sql
-KILL QUERY
-  WHERE <where expression to SELECT FROM system.processes query>
-  [SYNC|ASYNC|TEST]
-  [FORMAT format]
+KILL QUERY WHERE <where expression to SELECT FROM system.processes query> [SYNC|ASYNC|TEST] [FORMAT format]
 ```
 
 Attempts to terminate queries currently running.
-The queries to terminate are selected from the system.processes table for which `WHERE` expression is true.
+The queries to terminate are selected from the system.processes table for which expression_for_system.processes is true.
 
 Examples:
 
 ```sql
--- Terminates all queries with the specified query_id.
 KILL QUERY WHERE query_id='2-857d-4a57-9ee0-327da5d60a90'
+```
 
--- Synchronously terminates all queries run by `username`.
+Terminates all queries with the specified query_id.
+
+```sql
 KILL QUERY WHERE user='username' SYNC
 ```
 
+Synchronously terminates all queries run by `username`.
+
 Readonly-users can only terminate their own requests.
-
-By default, the asynchronous version of queries is used (`ASYNC`), which doesn't wait for query termination.
-
-The synchronous version (`SYNC`) waits for all queries to be killed and displays information about each process as it terminates.
+By default, the asynchronous version of queries is used (`ASYNC`), which terminates without waiting for queries to complete.
+The synchronous version (`SYNC`) waits for all queries to be completed and displays information about each process as it terminates.
 The response contains the `kill_status` column, which can take the following values:
 
-1. 'finished' – The query terminated successfully.
+1. 'finished' – The query completed successfully.
 2. 'waiting' – Waiting for the query to finish after sending it a signal to terminate.
 3. The other values ​​explain why the query can't be terminated.
 
 A test query (`TEST`) only checks the user's rights and displays a list of queries to terminate.
+
diff --git a/docs/en/query_language/syntax.md b/docs/en/query_language/syntax.md
old mode 100644
new mode 100755
index e151d2ee3d9..4928f2d4a12
--- a/docs/en/query_language/syntax.md
+++ b/docs/en/query_language/syntax.md
@@ -46,7 +46,7 @@ There are numeric literals, string literals, and compound literals.
 
 A numeric literal tries to be parsed:
 
-- first as a 64-bit signed number, using the 'strtoull' function.
+- First as a 64-bit signed number, using the 'strtoull' function.
 - If unsuccessful, as a 64-bit unsigned number, using the 'strtoll' function.
 - If unsuccessful, as a floating-point number using the 'strtod' function.
 - Otherwise, an error is returned.
diff --git a/docs/en/roadmap.md b/docs/en/roadmap.md
old mode 100644
new mode 100755
index 46b08a89607..8241b0a65ae
--- a/docs/en/roadmap.md
+++ b/docs/en/roadmap.md
@@ -3,11 +3,13 @@
 ## Q1 2018
 
 ### New functionality
-- Initial support for `UPDATE` and `DELETE`.
-- Multi-dimensional and nested arrays.
-    
-    It may look like this:
-    
+
+- Support for `UPDATE` and `DELETE`.
+
+- Multidimensional and nested arrays.
+
+   It can look something like this:
+
 ```sql
 CREATE TABLE t
 (
@@ -21,7 +23,7 @@ ENGINE = MergeTree ORDER BY x
 
 - External MySQL and ODBC tables.
 
-    External tables can be integrated to ClickHouse using external dictionaries. This will be an alternative and a more convenient way to do so.
+   External tables can be integrated into ClickHouse using external dictionaries. This new functionality is a convenient alternative to connecting external tables.
 
 ```sql
 SELECT ... 
@@ -30,66 +32,68 @@ FROM mysql('host:port', 'db', 'table', 'user', 'password')`
 
 ### Improvements
 
-- Efficient data copy between ClickHouse clusters.
+- Effective data copying between ClickHouse clusters.
 
-    Currently, it is possible to copy data using remote() function, e.g.: `
+   Now you can copy data with the remote() function. For example: `
 INSERT INTO t SELECT * FROM remote(...) `.
 
-    The performance of this will be improved by proper distributed execution.
+   This operation will have improved performance.
 
 - O_DIRECT for merges.
 
-    Should improve OS cache performance and correspondingly query performance for 'hot' queries.
-
+   This will improve the performance of the OS cache and "hot" queries.
 
 ## Q2 2018
 
 ### New functionality
 
-- UPDATE/DELETE in order to comply with Europe GDPR.
-- Protobuf and Parquet input/output formats.
-- Create dictionaries by DDL queries.
+- UPDATE/DELETE conform to the EU GDPR.
+- Protobuf and Parquet input and output formats.
+- Creating dictionaries using DDL queries.
 
-    Currently, it is inconvenient and confusing that dictionaries are defined in external XML files while being a part of DB schema. The new approach will fix that.
+   Currently, dictionaries that are part of the database schema are defined in external XML files. This is inconvenient and counter-intuitive. The new approach should fix it.
+
+- Integration with LDAP.
 
-- LDAP integration.
 - WITH ROLLUP and WITH CUBE for GROUP BY.
-- Custom encoding/compression for columns.
 
-    Currently, ClickHouse support LZ4 and ZSTD compressions for columns, and compressions settings are global (see our article [Compression in ClickHouse](https://www.altinity.com/blog/2017/11/21/compression-in-clickhouse)) for more details). Column level encoding (e.g. delta encoding) and compression will allow more efficient data storage and therefore faster queries.
+- Custom encoding and compression for each column individually.
 
-- Store data at multiple disk volumes of a single server.
+   As of now, ClickHouse supports LZ4 and ZSTD compression of columns, and compression settings are global (see the article [Compression in ClickHouse](https://www.altinity.com/blog/2017/11/21/compression-in-clickhouse)). Per-column compression and encoding will provide more efficient data storage, which in turn will speed up queries.
 
-    That will make it easier to extend disk system as well as use different disk systems for different DBs or tables. Currently, users have to use symlinks if DB/table needs to be stored in another volume.
+- Storing data on multiple disks on the same server.
+
+   This functionality will make it easier to extend the disk space, since different disk systems can be used for different databases or tables. Currently, users are forced to use symbolic links if the databases and tables must be stored on a different disk.
 
 ### Improvements
 
-A lot of enhancements and fixes are planned for query execution. In particular:
+Many improvements and fixes are planned for the query execution system. For example:
 
-- Using index for ‘in (subquery)’.
+- Using an index for `in (subquery)`.
 
-    Currently, index is not used for such queries resulting in lower performance.
+   The index is not used right now, which reduces performance.
 
-- Predicate pushdown from ‘where’ into subqueries and Predicate pushdown for views.
+- Passing predicates from `where` to subqueries, and passing predicates to views.
 
-    These two are related since view is replaced by subquery. Currently, performance of filter conditions for views is significantly degraded, views can not use primary key of the underlying table, that makes views on big tables pretty much useless.
+   The predicates must be passed, since the view is changed by the subquery. Performance is still low for view filters, and views can't use the primary key of the original table, which makes views useless for large tables.
 
-- Short-circuit expressions evaluation (ternary operator, if, multiIf).
+- Optimizing branching operations (ternary operator, if, multiIf).
 
-    Currently, ClickHouse evaluates all branches even if the first one needs to be returned due to logical condition result.
+   ClickHouse currently performs all branches, even if they aren't necessary.
 
-- Using primary key for GROUP BY and ORDER BY.
+- Using a primary key for GROUP BY and ORDER BY.
 
-    This may speed up certain types of queries since data is already partially pre-sorted.
+   This will speed up certain types of queries with partially sorted data.
 
 ## Q3-Q4 2018
 
-Longer term plans are not yet finalized. There are two major projects on the list so far.
+We don't have any set plans yet, but the main projects will be:
 
-- Resource pools for query execution.
+- Resource pools for executing queries.
 
-    That will allow managing workloads more efficiently.
+   This will make load management more efficient.
 
 - ANSI SQL JOIN syntax.
 
-    That will make ClickHouse more friendly for numerous SQL tools.
+   Improve ClickHouse compatibility with many SQL tools.
+
diff --git a/docs/en/system_tables/index.md b/docs/en/system_tables/index.md
old mode 100644
new mode 100755
index 614ce4020ec..240105a684b
--- a/docs/en/system_tables/index.md
+++ b/docs/en/system_tables/index.md
@@ -4,5 +4,5 @@ System tables are used for implementing part of the system's functionality, and
 You can't delete a system table (but you can perform DETACH).
 System tables don't have files with data on the disk or files with metadata. The server creates all the system tables when it starts.
 System tables are read-only.
-System tables are located in the 'system' database.
+They are located in the 'system' database.
 
diff --git a/docs/en/system_tables/system.asynchronous_metrics.md b/docs/en/system_tables/system.asynchronous_metrics.md
old mode 100644
new mode 100755
diff --git a/docs/en/system_tables/system.clusters.md b/docs/en/system_tables/system.clusters.md
old mode 100644
new mode 100755
index c0bc3dd13fa..bc8dab86b3c
--- a/docs/en/system_tables/system.clusters.md
+++ b/docs/en/system_tables/system.clusters.md
@@ -4,13 +4,12 @@ Contains information about clusters available in the config file and the servers
 Columns:
 
 ```text
-cluster String      – Cluster name.
-shard_num UInt32    – Number of a shard in the cluster, starting from 1.
-shard_weight UInt32 – Relative weight of a shard when writing data.
-replica_num UInt32  – Number of a replica in the shard, starting from 1.
-host_name String    – Host name as specified in the config.
-host_address String – Host's IP address obtained from DNS.
-port UInt16        – The port used to access the server.
-user String        – The username to use for connecting to the server.
+cluster String      - Cluster name.
+shard_num UInt32    - Number of a shard in the cluster, starting from 1.
+shard_weight UInt32 - Relative weight of a shard when writing data.
+replica_num UInt32  - Number of a replica in the shard, starting from 1.
+host_name String    - Host name as specified in the config.
+host_address String - Host's IP address obtained from DNS.
+port UInt16         - The port used to access the server.
+user String         - The username to use for connecting to the server.
 ```
-
diff --git a/docs/en/system_tables/system.columns.md b/docs/en/system_tables/system.columns.md
old mode 100644
new mode 100755
index 975b84fe9d4..bf05616fbef
--- a/docs/en/system_tables/system.columns.md
+++ b/docs/en/system_tables/system.columns.md
@@ -11,4 +11,3 @@ type String               - Column type.
 default_type String       - Expression type (DEFAULT, MATERIALIZED, ALIAS) for the default value, or an empty string if it is not defined.
 default_expression String - Expression for the default value, or an empty string if it is not defined.
 ```
-
diff --git a/docs/en/system_tables/system.databases.md b/docs/en/system_tables/system.databases.md
old mode 100644
new mode 100755
diff --git a/docs/en/system_tables/system.dictionaries.md b/docs/en/system_tables/system.dictionaries.md
old mode 100644
new mode 100755
index f3c9929d38e..4ef0d7707b8
--- a/docs/en/system_tables/system.dictionaries.md
+++ b/docs/en/system_tables/system.dictionaries.md
@@ -5,19 +5,19 @@ Contains information about external dictionaries.
 Columns:
 
 ```text
-name String                  – Dictionary name.
-type String                  – Dictionary type: Flat, Hashed, Cache.
-origin String                – Path to the config file where the dictionary is described.attribute.
-names Array(String) – Array of attribute names provided by the dictionary.
-attribute.types Array(String) – Corresponding array of attribute types provided by the dictionary.
-has_hierarchy UInt8          – Whether the dictionary is hierarchical.
-bytes_allocated UInt64        – The amount of RAM used by the dictionary.
-hit_rate Float64              – For cache dictionaries, the percent of usage for which the value was in the cache.
-element_count UInt64          – The number of items stored in the dictionary.
-load_factor Float64          – The filled percentage of the dictionary (for a hashed dictionary, it is the filled percentage of the hash table).
-creation_time DateTime        – Time spent for the creation or last successful reload of the dictionary.
-last_exception String        – Text of an error that occurred when creating or reloading the dictionary, if the dictionary couldn't be created.
-source String                – Text describing the data source for the dictionary.
+name String                   - Dictionary name.
+type String                   - Dictionary type: Flat, Hashed, Cache.
+origin String                 - Path to the config file where the dictionary is described.
+attribute.names Array(String) - Array of attribute names provided by the dictionary.
+attribute.types Array(String) - Corresponding array of attribute types provided by the dictionary.
+has_hierarchy UInt8           - Whether the dictionary is hierarchical.
+bytes_allocated UInt64        - The amount of RAM used by the dictionary.
+hit_rate Float64              - For cache dictionaries, the percent of usage for which the value was in the cache.
+element_count UInt64          - The number of items stored in the dictionary.
+load_factor Float64           - The filled percentage of the dictionary (for a hashed dictionary, it is the filled percentage of the hash table).
+creation_time DateTime        - Time spent for the creation or last successful reload of the dictionary.
+last_exception String         - Text of an error that occurred when creating or reloading the dictionary, if the dictionary couldn't be created.
+source String                 - Text describing the data source for the dictionary.
 ```
 
 Note that the amount of memory used by the dictionary is not proportional to the number of items stored in it. So for flat and cached dictionaries, all the memory cells are pre-assigned, regardless of how full the dictionary actually is.
diff --git a/docs/en/system_tables/system.events.md b/docs/en/system_tables/system.events.md
old mode 100644
new mode 100755
diff --git a/docs/en/system_tables/system.functions.md b/docs/en/system_tables/system.functions.md
old mode 100644
new mode 100755
index ac550acc14b..a1022a5e557
--- a/docs/en/system_tables/system.functions.md
+++ b/docs/en/system_tables/system.functions.md
@@ -6,6 +6,6 @@ Columns:
 
 ```text
 name String          – Function name.
-is_aggregate UInt8    – Whether it is an aggregate function.
+is_aggregate UInt8   – Whether it is an aggregate function.
 ```
 
diff --git a/docs/en/system_tables/system.merges.md b/docs/en/system_tables/system.merges.md
old mode 100644
new mode 100755
index 0a10e4a5a8c..59870922ea5
--- a/docs/en/system_tables/system.merges.md
+++ b/docs/en/system_tables/system.merges.md
@@ -18,4 +18,3 @@ rows_read UInt64                   - Number of rows read.
 bytes_written_uncompressed UInt64  - Amount of bytes written, uncompressed.
 rows_written UInt64                - Number of rows written.
 ```
-
diff --git a/docs/en/system_tables/system.numbers.md b/docs/en/system_tables/system.numbers.md
old mode 100644
new mode 100755
diff --git a/docs/en/system_tables/system.numbers_mt.md b/docs/en/system_tables/system.numbers_mt.md
old mode 100644
new mode 100755
diff --git a/docs/en/system_tables/system.one.md b/docs/en/system_tables/system.one.md
old mode 100644
new mode 100755
diff --git a/docs/en/system_tables/system.parts.md b/docs/en/system_tables/system.parts.md
old mode 100644
new mode 100755
diff --git a/docs/en/system_tables/system.processes.md b/docs/en/system_tables/system.processes.md
old mode 100644
new mode 100755
index b9ad6a44e81..0802e555648
--- a/docs/en/system_tables/system.processes.md
+++ b/docs/en/system_tables/system.processes.md
@@ -14,12 +14,12 @@ rows_read UInt64        – The number of rows read from the table. For distribu
 
 bytes_read UInt64 – The number of uncompressed bytes read from the table. For distributed processing, on the requestor server, this is the total for all remote servers.
 
-total_rows_approx UInt64 – The approximate total number of rows that must be read. For distributed processing, on the requestor server, this is the total for all remote servers. It can be updated during request processing, when new sources to process become known.
+UInt64 total_rows_approx – The approximate total number of rows that must be read. For distributed processing, on the requestor server, this is the total for all remote servers. It can be updated during request processing, when new sources to process become known.
 
 memory_usage UInt64 – Memory consumption by the query. It might not include some types of dedicated memory.
 
-query String – The query text. For INSERT, it doesn't include the data to insert.
+Query String – The query text. For INSERT, it doesn't include the data to insert.
 
-query_id String - The query ID, if defined.
+query_id – Query ID, if defined.
 ```
 
diff --git a/docs/en/system_tables/system.replicas.md b/docs/en/system_tables/system.replicas.md
old mode 100644
new mode 100755
index ec1341198dc..75cd8e34340
--- a/docs/en/system_tables/system.replicas.md
+++ b/docs/en/system_tables/system.replicas.md
@@ -42,37 +42,41 @@ Columns:
 database:           database name
 table:              table name
 engine:             table engine name
+
 is_leader:          whether the replica is the leader
 
 Only one replica at a time can be the leader. The leader is responsible for selecting background merges to perform.
 Note that writes can be performed to any replica that is available and has a session in ZK, regardless of whether it is a leader.
 
-is_readonly:        Whether the replica is in read-only mode.This mode is turned on if the config doesn't have sections with ZK, if an unknown error occurred when reinitializing sessions in ZK, and during session reinitialization in ZK.
+is_readonly:        Whether the replica is in read-only mode.
+This mode is turned on if the config doesn't have sections with ZK, if an unknown error occurred when reinitializing sessions in ZK, and during session reinitialization in ZK.
 
 is_session_expired: Whether the ZK session expired.
 Basically, the same thing as is_readonly.
 
-future_parts: The number of data parts that will appear as the result of INSERTs or merges that haven't been done yet. 
+future_parts:       The number of data parts that will appear as the result of INSERTs or merges that haven't been done yet. 
 
-parts_to_check: The number of data parts in the queue for verification. A part is put in the verification queue if there is suspicion that it might be damaged.
+parts_to_check:     The number of data parts in the queue for verification. 
+A part is put in the verification queue if there is suspicion that it might be damaged.
 
-zookeeper_path: The path to the table data in ZK. 
+zookeeper_path:     The path to the table data in ZK. 
 replica_name: Name of the replica in ZK. Different replicas of the same table have different names. 
-replica_path: The path to the replica data in ZK. The same as concatenating zookeeper_path/replicas/replica_path.
+replica_path:       The path to the replica data in ZK. The same as concatenating zookeeper_path/replicas/replica_path.
 
-columns_version: Version number of the table structure. Indicates how many times ALTER was performed. If replicas have different versions, it means some replicas haven't made all of the ALTERs yet.
+columns_version:    Version number of the table structure. Indicates how many times ALTER was performed. If replicas have different versions, it means some replicas haven't made all of the ALTERs yet.
 
-queue_size:         Size of the queue for operations waiting to be performed. Operations include inserting blocks of data, merges, and certain other actions.
+queue_size:         Size of the queue for operations waiting to be performed. 
+Operations include inserting blocks of data, merges, and certain other actions.
 Normally coincides with future_parts.
 
-inserts_in_queue: Number of inserts of blocks of data that need to be made. Insertions are usually replicated fairly quickly. If the number is high, something is wrong.
+inserts_in_queue:   Number of inserts of blocks of data that need to be made. Insertions are usually replicated fairly quickly. If the number is high, something is wrong.
 
-merges_in_queue: The number of merges waiting to be made. Sometimes merges are lengthy, so this value may be greater than zero for a long time.
+merges_in_queue:    The number of merges waiting to be made. Sometimes merges are lengthy, so this value may be greater than zero for a long time.
 
 The next 4 columns have a non-null value only if the ZK session is active.
 
-log_max_index:     Maximum entry number in the log of general activity.
-log_pointer:        Maximum entry number in the log of general activity that the replica copied to its execution queue, plus one. If log_pointer is much smaller than log_max_index, something is wrong.
+log_max_index:      Maximum entry number in the log of general activity. log_pointer:        Maximum entry number in the log of general activity that the replica copied to its execution queue, plus one.
+If log_pointer is much smaller than log_max_index, something is wrong.
 
 total_replicas:     Total number of known replicas of this table.
 active_replicas:    Number of replicas of this table that have a ZK session (the number of active replicas).
diff --git a/docs/en/system_tables/system.settings.md b/docs/en/system_tables/system.settings.md
old mode 100644
new mode 100755
index a055135ebcf..90a392bcc24
--- a/docs/en/system_tables/system.settings.md
+++ b/docs/en/system_tables/system.settings.md
@@ -6,9 +6,9 @@ I.e. used for executing the query you are using to read from the system.settings
 Columns:
 
 ```text
-name String  – Setting name.
+name String   – Setting name.
 value String  – Setting value.
-changed UInt8 -–Whether the setting was explicitly defined in the config or explicitly changed.
+changed UInt8 - Whether the setting was explicitly defined in the config or explicitly changed.
 ```
 
 Example:
diff --git a/docs/en/system_tables/system.tables.md b/docs/en/system_tables/system.tables.md
old mode 100644
new mode 100755
index fabddf4dbb1..5757a8ac3da
--- a/docs/en/system_tables/system.tables.md
+++ b/docs/en/system_tables/system.tables.md
@@ -1,6 +1,7 @@
 # system.tables
 
 This table contains the String columns 'database', 'name', and 'engine'.
-Also, the table has three virtual columns: metadata_modification_time of type DateTime, create_table_query and engine_full of type String.
+The table also contains three virtual columns: metadata_modification_time (DateTime type), create_table_query, and engine_full (String type).
 Each table that the server knows about is entered in the 'system.tables' table.
 This system table is used for implementing SHOW TABLES queries.
+
diff --git a/docs/en/system_tables/system.zookeeper.md b/docs/en/system_tables/system.zookeeper.md
old mode 100644
new mode 100755
index c456e7d9207..46b40e7a08f
--- a/docs/en/system_tables/system.zookeeper.md
+++ b/docs/en/system_tables/system.zookeeper.md
@@ -70,4 +70,3 @@ numChildren:    7
 pzxid:          987021252247
 path:           /clickhouse/tables/01-08/visits/replicas
 ```
-
diff --git a/docs/en/table_engines/aggregatingmergetree.md b/docs/en/table_engines/aggregatingmergetree.md
old mode 100644
new mode 100755
index d75d8353e6d..987c102508b
--- a/docs/en/table_engines/aggregatingmergetree.md
+++ b/docs/en/table_engines/aggregatingmergetree.md
@@ -1,10 +1,11 @@
 # AggregatingMergeTree
 
-This engine differs from MergeTree in that the merge combines the states of aggregate functions stored in the table for rows with the same primary key value.
+This engine differs from `MergeTree` in that the merge combines the states of aggregate functions stored in the table for rows with the same primary key value.
 
-In order for this to work, it uses the AggregateFunction data type and the -State and -Merge modifiers for aggregate functions. Let's examine it more closely.
+For this to work, it uses the `AggregateFunction` data type, as well as `-State` and `-Merge` modifiers for aggregate functions. Let's examine it more closely.
+
+There is an `AggregateFunction` data type. It is a parametric data type. As parameters, the name of the aggregate function is passed, then the types of its arguments.
 
-There is an AggregateFunction data type. It is a parametric data type. As parameters, the name of the aggregate function is passed, then the types of its arguments.
 Examples:
 
 ```sql
@@ -19,12 +20,16 @@ CREATE TABLE t
 This type of column stores the state of an aggregate function.
 
 To get this type of value, use aggregate functions with the `State` suffix.
-Example: `uniqState(UserID), quantilesState(0.5, 0.9)(SendTiming)` – in contrast to the corresponding 'uniq' and 'quantiles' functions, these functions return the state, rather than the prepared value. In other words, they return an AggregateFunction type value.
 
-An AggregateFunction type value can't be output in Pretty formats. In other formats, these types of values are output as implementation-specific binary data. The AggregateFunction type values are not intended for output or saving in a dump.
+Example:
+`uniqState(UserID), quantilesState(0.5, 0.9)(SendTiming)`
 
-The only useful thing you can do with AggregateFunction type values is combine the states and get a result, which essentially means to finish aggregation. Aggregate functions with the 'Merge' suffix are used for this purpose.
-Example: `uniqMerge(UserIDState), where UserIDState has the AggregateFunction type`.
+In contrast to the corresponding `uniq` and `quantiles` functions, these functions return the state, rather than the prepared value. In other words, they return an `AggregateFunction` type value.
+
+An `AggregateFunction` type value can't be output in Pretty formats. In other formats, these types of values are output as implementation-specific binary data. The `AggregateFunction` type values are not intended for output or saving in a dump.
+
+The only useful thing you can do with `AggregateFunction` type values is combine the states and get a result, which essentially means to finish aggregation. Aggregate functions with the 'Merge' suffix are used for this purpose.
+Example: `uniqMerge(UserIDState), where UserIDState has the AggregateFunction` type.
 
 In other words, an aggregate function with the 'Merge' suffix takes a set of states, combines them, and returns the result.
 As an example, these two queries return the same result:
@@ -37,15 +42,15 @@ SELECT uniqMerge(state) FROM (SELECT uniqState(UserID) AS state FROM table GROUP
 
 There is an ` AggregatingMergeTree` engine. Its job during a merge is to combine the states of aggregate functions from different table rows with the same primary key value.
 
-You can't use a normal INSERT to insert a row in a table containing AggregateFunction columns, because you can't explicitly define the AggregateFunction value. Instead, use INSERT SELECT with '-State' aggregate functions for inserting data.
+You can't use a normal INSERT to insert a row in a table containing `AggregateFunction` columns, because you can't explicitly define the `AggregateFunction` value. Instead, use `INSERT SELECT` with `-State` aggregate functions for inserting data.
 
-With SELECT from an AggregatingMergeTree table, use GROUP BY and aggregate functions with the '-Merge' modifier in order to complete data aggregation.
+With SELECT from an `AggregatingMergeTree` table, use GROUP BY and aggregate functions with the '-Merge' modifier in order to complete data aggregation.
 
-You can use AggregatingMergeTree tables for incremental data aggregation, including for aggregated materialized views.
+You can use `AggregatingMergeTree` tables for incremental data aggregation, including for aggregated materialized views.
 
 Example:
 
-Creating a materialized AggregatingMergeTree view that tracks the 'test.visits' table:
+Create an `AggregatingMergeTree` materialized view that watches the `test.visits` table:
 
 ```sql
 CREATE MATERIALIZED VIEW test.basic
@@ -59,13 +64,13 @@ FROM test.visits
 GROUP BY CounterID, StartDate;
 ```
 
-Inserting data in the 'test.visits' table. Data will also be inserted in the view, where it will be aggregated:
+Insert data in the `test.visits` table. Data will also be inserted in the view, where it will be aggregated:
 
 ```sql
 INSERT INTO test.visits ...
 ```
 
-Performing SELECT from the view using GROUP BY to finish data aggregation:
+Perform `SELECT` from the view using `GROUP BY` in order to complete data aggregation:
 
 ```sql
 SELECT
diff --git a/docs/en/table_engines/buffer.md b/docs/en/table_engines/buffer.md
old mode 100644
new mode 100755
diff --git a/docs/en/table_engines/collapsingmergetree.md b/docs/en/table_engines/collapsingmergetree.md
old mode 100644
new mode 100755
diff --git a/docs/en/table_engines/custom_partitioning_key.md b/docs/en/table_engines/custom_partitioning_key.md
old mode 100644
new mode 100755
index 6a468a1137d..bfcb3c2c545
--- a/docs/en/table_engines/custom_partitioning_key.md
+++ b/docs/en/table_engines/custom_partitioning_key.md
@@ -2,7 +2,7 @@
 
 # Custom partitioning key
 
-Starting with version 1.1.54310, you can create tables in the MergeTree family with any partition expression (not only partitioning by month).
+Starting with version 1.1.54310, you can create tables in the MergeTree family with any partitioning expression (not only partitioning by month).
 
 The partition key can be an expression from the table columns, or a tuple of such expressions (similar to the primary key). The partition key can be omitted. When creating a table, specify the partition key in the ENGINE description with the new syntax:
 
@@ -10,7 +10,7 @@ The partition key can be an expression from the table columns, or a tuple of suc
 ENGINE [=] Name(...) [PARTITION BY expr] [ORDER BY expr] [SAMPLE BY expr] [SETTINGS name=value, ...]
 ```
 
-For MergeTree tables, the partition expression is specified after `PARTITION BY`, the primary key after `ORDER BY`, the sampling key after `SAMPLE BY`, and `SETTINGS` can specify `index_granularity` (optional; the default value is 8192), as well as other settings from [MergeTreeSettings.h](https://github.com/yandex/ClickHouse/blob/master/dbms/src/Storages/MergeTree/MergeTreeSettings.h). Example:
+For MergeTree tables, the partition expression is specified after `PARTITION BY`, the primary key after `ORDER BY`, the sampling key after `SAMPLE BY`, and `SETTINGS` can specify `index_granularity` (optional; the default value is 8192), as well as other settings from [MergeTreeSettings.h](https://github.com/yandex/ClickHouse/blob/master/dbms/src/Storages/MergeTree/MergeTreeSettings.h). The other engine parameters are specified in parentheses after the engine name, as previously. Example:
 
 ```sql
 ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/tables/name', 'replica1', Sign)
@@ -23,7 +23,7 @@ The traditional partitioning by month is expressed as `toYYYYMM(date_column)`.
 
 You can't convert an old-style table to a table with custom partitions (only via INSERT SELECT).
 
-After this table is created, merge will only work for data parts that have the same value for the partition expression. Note: This means that you shouldn't make overly granular partitions (more than about a thousand partitions), or SELECT will perform poorly.
+After this table is created, merge will only work for data parts that have the same value for the partitioning expression. Note: This means that you shouldn't make overly granular partitions (more than about a thousand partitions), or SELECT will perform poorly.
 
 To specify a partition in ALTER PARTITION commands, specify the value of the partition expression (or a tuple). Constants and constant expressions are supported. Example:
 
@@ -35,13 +35,13 @@ Deletes the partition for the current week with event type 1. The same is true f
 
 Note: For old-style tables, the partition can be specified either as a number `201710` or a string `'201710'`. The syntax for the new style of tables is stricter with types (similar to the parser for the VALUES input format). In addition, ALTER TABLE FREEZE PARTITION uses exact match for new-style tables (not prefix match).
 
-In the `system.parts` table, the `partition` column should specify the value of the partition expression to use in ALTER queries (if quotas are removed). The `name` column should specify the name of the data part that has a new format.
+In the `system.parts` table, the `partition` column specifies the value of the partition expression to use in ALTER queries (if quotas are removed). The `name` column should specify the name of the data part that has a new format.
 
-Before: `20140317_20140323_2_2_0` (minimal data - maximal data - number of minimal block - number of maximal block - level).
+Was: `20140317_20140323_2_2_0` (minimum date - maximum date - minimum block number - maximum block number - level).
 
-After: `201403_2_2_0` (partition ID - number of minimal block - number of maximal block - level).
+Now: `201403_2_2_0`  (partition ID -  minimum block number - maximum block number - level).
 
 The partition ID is its string identifier (human-readable, if possible) that is used for the names of data parts in the file system and in ZooKeeper. You can specify it in ALTER queries in place of the partition key. Example: Partition key `toYYYYMM(EventDate)`; ALTER can specify either `PARTITION 201710` or `PARTITION ID '201710'`.
 
-There are more examples in the tests [`00502_custom_partitioning_local`](https://github.com/yandex/ClickHouse/blob/master/dbms/tests/queries/0_stateless/00502_custom_partitioning_local.sql) and [`00502_custom_partitioning_replicated_zookeeper`](https://github.com/yandex/ClickHouse/blob/master/dbms/tests/queries/0_stateless/00502_custom_partitioning_replicated_zookeeper.sql).
+For more examples, see the tests [`00502_custom_partitioning_local`](https://github.com/yandex/ClickHouse/blob/master/dbms/tests/queries/0_stateless/00502_custom_partitioning_local.sql) and [`00502_custom_partitioning_replicated_zookeeper`](https://github.com/yandex/ClickHouse/blob/master/dbms/tests/queries/0_stateless/00502_custom_partitioning_replicated_zookeeper.sql).
 
diff --git a/docs/en/table_engines/dictionary.md b/docs/en/table_engines/dictionary.md
new file mode 100755
index 00000000000..ae8cca90d7c
--- /dev/null
+++ b/docs/en/table_engines/dictionary.md
@@ -0,0 +1,106 @@
+<a name="table_engines-dictionary"></a>
+
+# Dictionary
+
+The `Dictionary` engine displays the dictionary data as a ClickHouse table.
+
+As an example, consider a dictionary of `products` with the following configuration:
+
+```xml
+<dictionaries>
+<dictionary>
+        <name>products</name>
+        <source>
+            <odbc>
+                <table>products</table>
+                <connection_string>DSN=some-db-server</connection_string>
+            </odbc>
+        </source>
+        <lifetime>
+            <min>300</min>
+            <max>360</max>
+        </lifetime>
+        <layout>
+            <flat/>
+        </layout>
+        <structure>
+            <id>
+                <name>product_id</name>
+            </id>
+            <attribute>
+                <name>title</name>
+                <type>String</type>
+                <null_value></null_value>
+            </attribute>
+        </structure>
+</dictionary>
+</dictionaries>
+```
+
+Query the dictionary data:
+
+```sql
+select name, type, key, attribute.names, attribute.types, bytes_allocated, element_count,source from system.dictionaries where name = 'products';                     
+
+SELECT 
+    name, 
+    type, 
+    key, 
+    attribute.names, 
+    attribute.types, 
+    bytes_allocated, 
+    element_count, 
+    source
+FROM system.dictionaries 
+WHERE name = 'products'
+```
+```
+┌─name─────┬─type─┬─key────┬─attribute.names─┬─attribute.types─┬─bytes_allocated─┬─element_count─┬─source──────────┐
+│ products │ Flat │ UInt64 │ ['title']       │ ['String']      │        23065376 │        175032 │ ODBC: .products │
+└──────────┴──────┴────────┴─────────────────┴─────────────────┴─────────────────┴───────────────┴─────────────────┘
+```
+
+You can use the [dictGet*](../functions/ext_dict_functions.md#ext_dict_functions) function to get the dictionary data in this format.
+
+This view isn't helpful when you need to get raw data, or when performing a `JOIN` operation. For these cases, you can use the `Dictionary` engine, which displays the dictionary data in a table.
+
+Syntax:
+
+```
+CREATE TABLE %table_name% (%fields%) engine = Dictionary(%dictionary_name%)`
+```
+
+Usage example:
+
+```sql
+create table products (product_id UInt64, title String) Engine = Dictionary(products);
+
+CREATE TABLE products
+(
+    product_id UInt64, 
+    title String, 
+)
+ENGINE = Dictionary(products)
+```
+```
+Ok.
+
+0 rows in set. Elapsed: 0.004 sec.
+```
+
+Take a look at what's in the table.
+
+```sql
+select * from products limit 1;
+
+SELECT *
+FROM products 
+LIMIT 1
+```
+```
+┌────product_id─┬─title───────────┐
+│        152689 │ Некоторый товар │
+└───────────────┴─────────────────┘
+
+1 rows in set. Elapsed: 0.006 sec.
+```
diff --git a/docs/en/table_engines/distributed.md b/docs/en/table_engines/distributed.md
old mode 100644
new mode 100755
index b8643461fbb..dd2ffe27fe5
--- a/docs/en/table_engines/distributed.md
+++ b/docs/en/table_engines/distributed.md
@@ -25,29 +25,29 @@ Clusters are set like this:
 <remote_servers>
     <logs>
         <shard>
-             <!-- Optional. Shard weight when writing data. Default: 1. -->
-             <weight>1</weight>
-             <!-- Optional. Whether to write data to just one of the replicas. Default: false (write data to all replicas). -->
-            <internal_replication>false</internal_replication>
-            <replica>
-                <host>example01-01-1</host>
-                <port>9000</port>
-            </replica>
-            <replica>
-                <host>example01-01-2</host>
-                <port>9000</port>
-            </replica>
-        </shard>
-        <shard>
-            <weight>2</weight>
-            <internal_replication>false</internal_replication>
-            <replica>
-                <host>example01-02-1</host>
-                <port>9000</port>
-            </replica>
-            <replica>
-                <host>example01-02-2</host>
-                <port>9000</port>
+            <!-- Optional. Shard weight when writing data. Default: 1. -->
+            <weight>1</weight>            
+            <!-- Optional. Whether to write data to just one of the replicas. Default: false (write data to all replicas). -->            
+            <internal_replication>false</internal_replication>            
+            <replica>                
+                <host>example01-01-1</host>                
+                <port>9000</port>            
+            </replica>            
+            <replica>                
+                 <host>example01-01-2</host>                
+                 <port>9000</port>            
+            </replica>        
+        </shard>        
+        <shard>            
+            <weight>2</weight>            
+            <internal_replication>false</internal_replication>            
+            <replica>                
+                <host>example01-02-1</host>                
+                <port>9000</port>            
+            </replica>            
+            <replica>                
+            <host>example01-02-2</host>                
+            <port>9000</port>
             </replica>
         </shard>
     </logs>
diff --git a/docs/en/table_engines/external_data.md b/docs/en/table_engines/external_data.md
old mode 100644
new mode 100755
diff --git a/docs/en/table_engines/file.md b/docs/en/table_engines/file.md
old mode 100644
new mode 100755
diff --git a/docs/en/table_engines/graphitemergetree.md b/docs/en/table_engines/graphitemergetree.md
old mode 100644
new mode 100755
index 6452377ac15..a4b62424954
--- a/docs/en/table_engines/graphitemergetree.md
+++ b/docs/en/table_engines/graphitemergetree.md
@@ -2,13 +2,13 @@
 
 # GraphiteMergeTree
 
-This engine is designed for rollup (thinning and aggregating/averaging) [ Graphite](http://graphite.readthedocs.io/en/latest/index.html) data. It may be helpful to developers who want to use ClickHouse as a data store for Graphite.
+This engine is designed for rollup (thinning and aggregating/averaging) [Graphite](http://graphite.readthedocs.io/en/latest/index.html) data. It may be helpful to developers who want to use ClickHouse as a data store for Graphite.
 
 Graphite stores full data in ClickHouse, and data can be retrieved in the following ways:
 
 - Without thinning.
 
-   Using the [MergeTree](mergetree.md#table_engines-mergetree) engine.
+   Uses the [MergeTree](mergetree.md#table_engines-mergetree) engine.
 
 - With thinning.
 
@@ -83,4 +83,3 @@ Example of settings:
     </default>
 </graphite_rollup>
 ```
-
diff --git a/docs/en/table_engines/index.md b/docs/en/table_engines/index.md
old mode 100644
new mode 100755
diff --git a/docs/en/table_engines/join.md b/docs/en/table_engines/join.md
old mode 100644
new mode 100755
diff --git a/docs/en/table_engines/kafka.md b/docs/en/table_engines/kafka.md
old mode 100644
new mode 100755
index 55d2027309d..e69c06ae9f9
--- a/docs/en/table_engines/kafka.md
+++ b/docs/en/table_engines/kafka.md
@@ -1,79 +1,79 @@
-# Kafka
-
-The engine works with [Apache Kafka](http://kafka.apache.org/).
-
-Kafka lets you:
-
-- Publish or subscribe to data flows.
-- Organize fault-tolerant storage.
-- Process streams as they become available.
-
-```
-Kafka(broker_list, topic_list, group_name, format[, schema])
-```
-
-Parameters:
-
-- `broker_list` – A comma-separated list of brokers (`localhost:9092`).
-- `topic_list` – A list of Kafka topics (`my_topic`).
-- `group_name` – A group of Kafka consumers (`group1`). Reading margins are tracked for each group separately. If you don't want messages to be duplicated in the cluster, use the same group name everywhere.
-- `format` – Message format. Uses the same notation as the SQL ` FORMAT` function, such as ` JSONEachRow`. For more information, see the section "Formats".
-- `schema` – An optional parameter that must be used if the format requires a schema definition. For example, [Cap'n Proto](https://capnproto.org/)  requires the path to the schema file and the name of the root ` schema.capnp:Message` object.
-
-Example:
-
-```sql
-  CREATE TABLE queue (
-    timestamp UInt64,
-    level String,
-    message String
-  ) ENGINE = Kafka('localhost:9092', 'topic', 'group1', 'JSONEachRow');
-
-  SELECT * FROM queue LIMIT 5;
-```
-
-The delivered messages are tracked automatically, so each message in a group is only counted once. If you want to get the data twice, then create a copy of the table with another group name.
-
-Groups are flexible and synced on the cluster. For instance, if you have 10 topics and 5 copies of a table in a cluster, then each copy gets 2 topics. If the number of copies changes, the topics are redistributed across the copies automatically. For more information, see [http://kafka.apache.org/intro](http://kafka.apache.org/intro).
-
-`SELECT` is not particularly useful for reading messages (except for debugging), because each message can be read only once. It is more practical to create real-time threads using materialized views. For this purpose, the following was done:
-
-1. Use the engine to create a Kafka consumer and consider it a data stream.
-2. Create a table with the desired structure.
-3. Create a materialized view that converts data from the engine and puts it into a previously created table.
-
-When the `MATERIALIZED VIEW` joins the engine, it starts collecting data in the background. This allows you to continually receive messages from Kafka and convert them to the required format using `SELECT`
-
-Example:
-
-```sql
-  CREATE TABLE queue (
-    timestamp UInt64,
-    level String,
-    message String
-  ) ENGINE = Kafka('localhost:9092', 'topic', 'group1', 'JSONEachRow');
-
-  CREATE TABLE daily (
-    day Date,
-    level String,
-    total UInt64
-  ) ENGINE = SummingMergeTree(day, (day, level), 8192);
-  
-  CREATE MATERIALIZED VIEW consumer TO daily
-    AS SELECT toDate(toDateTime(timestamp)) AS day, level, count() as total
-    FROM queue GROUP BY day, level;
-
-  SELECT level, sum(total) FROM daily GROUP BY level;
-```
-
-To improve performance, received messages are grouped into blocks the size of [max_block_size](../operations/settings/settings.md#settings-settings-max_insert_block_size). If the block wasn't formed within [ stream_flush_interval_ms](../operations/settings/settings.md#settings-settings_stream_flush_interval_ms) milliseconds, the data will be flushed to the table regardless of the completeness of the block.
-
-To stop receiving topic data or to change the conversion logic, detach the materialized view:
-
-```
-  DETACH TABLE consumer;
-  ATTACH MATERIALIZED VIEW consumer;
-```
-
-If you want to change the target table by using `ALTER` materialized view, we recommend disabling the material view to avoid discrepancies between the target table and the data from the view.
-
+# Kafka
+
+This engine works with [Apache Kafka](http://kafka.apache.org/).
+
+Kafka lets you:
+
+- Publish or subscribe to data flows.
+- Organize fault-tolerant storage.
+- Process streams as they become available.
+
+```
+Kafka(broker_list, topic_list, group_name, format[, schema])
+```
+
+Parameters:
+
+- `broker_list` – A comma-separated list of brokers (`localhost:9092`).
+- `topic_list` – A list of Kafka topics (`my_topic`).
+- `group_name` – A group of Kafka consumers (`group1`). Reading margins are tracked for each group separately. If you don't want messages to be duplicated in the cluster, use the same group name everywhere.
+- `--format` – Message format. Uses the same notation as the SQL ` FORMAT` function, such as ` JSONEachRow`.
+- `schema` – An optional parameter that must be used if the format requires a schema definition. For example, [Cap'n Proto](https://capnproto.org/)  requires the path to the schema file and the name of the root `schema.capnp:Message` object.
+
+Example:
+
+```sql
+CREATE TABLE queue (
+    timestamp UInt64,
+    level String,
+    message String
+  ) ENGINE = Kafka('localhost:9092', 'topic', 'group1', 'JSONEachRow');
+
+  SELECT * FROM queue LIMIT 5;
+```
+
+The delivered messages are tracked automatically, so each message in a group is only counted once. If you want to get the data twice, then create a copy of the table with another group name.
+
+Groups are flexible and synced on the cluster. For instance, if you have 10 topics and 5 copies of a table in a cluster, then each copy gets 2 topics. If the number of copies changes, the topics are redistributed across the copies automatically. Read more about this at [http://kafka.apache.org/intro](http://kafka.apache.org/intro).
+
+`SELECT` is not particularly useful for reading messages (except for debugging), because each message can be read only once. It is more practical to create real-time threads using materialized views. To do this:
+
+1. Use the engine to create a Kafka consumer and consider it a data stream.
+2. Create a table with the desired structure.
+3. Create a materialized view that converts data from the engine and puts it into a previously created table.
+
+When the `MATERIALIZED VIEW` joins the engine, it starts collecting data in the background. This allows you to continually receive messages from Kafka and convert them to the required format using `SELECT`
+
+Example:
+
+```sql
+CREATE TABLE queue (
+    timestamp UInt64,
+    level String,
+    message String
+  ) ENGINE = Kafka('localhost:9092', 'topic', 'group1', 'JSONEachRow');
+
+  CREATE TABLE daily (
+    day Date,
+    level String,
+    total UInt64
+  ) ENGINE = SummingMergeTree(day, (day, level), 8192);
+  
+  CREATE MATERIALIZED VIEW consumer TO daily
+    AS SELECT toDate(toDateTime(timestamp)) AS day, level, count() as total
+    FROM queue GROUP BY day, level;
+
+SELECT level, sum(total) FROM daily GROUP BY level;
+```
+
+To improve performance, received messages are grouped into blocks the size of [max_insert_block_size](../operations/settings/settings.md#settings-settings-max_insert_block_size). If the block wasn't formed within [stream_flush_interval_ms](../operations/settings/settings.md#settings-settings_stream_flush_interval_ms) milliseconds, the data will be flushed to the table regardless of the completeness of the block.
+
+To stop receiving topic data or to change the conversion logic, detach the materialized view:
+
+```
+DETACH TABLE consumer;
+ATTACH MATERIALIZED VIEW consumer;
+```
+
+If you want to change the target table by using ` ALTER`materialized view, we recommend disabling the material view to avoid discrepancies between the target table and the data from the view.
+
diff --git a/docs/en/table_engines/log.md b/docs/en/table_engines/log.md
old mode 100644
new mode 100755
diff --git a/docs/en/table_engines/materializedview.md b/docs/en/table_engines/materializedview.md
old mode 100644
new mode 100755
index 00f70bd72bd..5e2741c6aa1
--- a/docs/en/table_engines/materializedview.md
+++ b/docs/en/table_engines/materializedview.md
@@ -1,4 +1,4 @@
 # MaterializedView
 
-Used for implementing materialized views (for more information, see [CREATE TABLE](../query_language/queries.md#query_language-queries-create_table)). For storing data, it uses a different engine that was specified when creating the view. When reading from a table, it just uses this engine.
+Used for implementing materialized views (for more information, see the [CREATE TABLE](../query_language/queries.md#query_language-queries-create_table)) query. For storing data, it uses a different engine that was specified when creating the view. When reading from a table, it just uses this engine.
 
diff --git a/docs/en/table_engines/memory.md b/docs/en/table_engines/memory.md
old mode 100644
new mode 100755
diff --git a/docs/en/table_engines/merge.md b/docs/en/table_engines/merge.md
old mode 100644
new mode 100755
diff --git a/docs/en/table_engines/mergetree.md b/docs/en/table_engines/mergetree.md
old mode 100644
new mode 100755
index 71197f21b34..fea02e01d72
--- a/docs/en/table_engines/mergetree.md
+++ b/docs/en/table_engines/mergetree.md
@@ -56,7 +56,7 @@ In this example, the index can't be used:
 SELECT count() FROM table WHERE CounterID = 34 OR URL LIKE '%upyachka%'
 ```
 
-To check whether ClickHouse can use the index when executing the query, use the settings [ force_index_by_date](../operations/settings/settings.md#settings-settings-force_index_by_date)  and [ force_primary_key](../operations/settings/settings.md#settings-settings-force_primary_key).
+To check whether ClickHouse can use the index when executing the query, use the settings  [force_index_by_date](../operations/settings/settings.md#settings-settings-force_index_by_date)and[force_primary_key](../operations/settings/settings.md#settings-settings-force_primary_key).
 
 The index by date only allows reading those parts that contain dates from the desired range. However, a data part may contain data for many dates (up to an entire month), while within a single part the data is ordered by the primary key, which might not contain the date as the first column. Because of this, using a query with only a date condition that does not specify the primary key prefix will cause more data to be read than for a single date.
 
diff --git a/docs/en/table_engines/null.md b/docs/en/table_engines/null.md
old mode 100644
new mode 100755
diff --git a/docs/en/table_engines/replacingmergetree.md b/docs/en/table_engines/replacingmergetree.md
old mode 100644
new mode 100755
diff --git a/docs/en/table_engines/replication.md b/docs/en/table_engines/replication.md
old mode 100644
new mode 100755
index 1e58878c34e..20dd17e444f
--- a/docs/en/table_engines/replication.md
+++ b/docs/en/table_engines/replication.md
@@ -46,7 +46,7 @@ You can specify any existing ZooKeeper cluster and the system will use a directo
 
 If ZooKeeper isn't set in the config file, you can't create replicated tables, and any existing replicated tables will be read-only.
 
-ZooKeeper isn't used for SELECT queries. In other words, replication doesn't affect the productivity of SELECT queries – they work just as fast as for non-replicated tables. When querying distributed replicated tables, ClickHouse behavior is controlled by the settings [max_replica_delay_for_distributed_queries](../operations/settings/settings.md#settings_settings_max_replica_delay_for_distributed_queries)  and [fallback_to_stale_replicas_for_distributed_queries](../operations/settings/settings.md#settings-settings-fallback_to_stale_replicas_for_distributed_queries).
+ZooKeeper isn't used for SELECT queries. In other words, replication doesn't affect the productivity of SELECT queries – they work just as fast as for non-replicated tables. When querying distributed replicated tables, ClickHouse behavior is controlled by the settings [max_replica_delay_for_distributed_queries](../operations/settings/settings.md#settings_settings_max_replica_delay_for_distributed_queries) and [fallback_to_stale_replicas_for_distributed_queries](../operations/settings/settings.md#settings-settings-fallback_to_stale_replicas_for_distributed_queries).
 
 For each INSERT query (more precisely, for each inserted block of data; the INSERT query contains a single block, or per block for every max_insert_block_size = 1048576 rows), approximately ten entries are made in ZooKeeper in several transactions. This leads to slightly longer latencies for INSERT compared to non-replicated tables. But if you follow the recommendations to insert data in batches of no more than one INSERT per second, it doesn't create any problems. The entire ClickHouse cluster used for coordinating one ZooKeeper cluster has a total of several hundred INSERTs per second. The throughput on data inserts (the number of rows per second) is just as high as for non-replicated data.
 
diff --git a/docs/en/table_engines/set.md b/docs/en/table_engines/set.md
old mode 100644
new mode 100755
diff --git a/docs/en/table_engines/summingmergetree.md b/docs/en/table_engines/summingmergetree.md
old mode 100644
new mode 100755
diff --git a/docs/en/table_engines/tinylog.md b/docs/en/table_engines/tinylog.md
old mode 100644
new mode 100755
diff --git a/docs/en/table_engines/view.md b/docs/en/table_engines/view.md
old mode 100644
new mode 100755
diff --git a/docs/en/table_functions/index.md b/docs/en/table_functions/index.md
old mode 100644
new mode 100755
diff --git a/docs/en/table_functions/merge.md b/docs/en/table_functions/merge.md
old mode 100644
new mode 100755
diff --git a/docs/en/table_functions/remote.md b/docs/en/table_functions/remote.md
old mode 100644
new mode 100755
index 99b0c7bb116..e26e245207b
--- a/docs/en/table_functions/remote.md
+++ b/docs/en/table_functions/remote.md
@@ -52,7 +52,7 @@ example01-{01..02}-1
 
 If you have multiple pairs of curly brackets, it generates the direct product of the corresponding sets.
 
-Addresses and parts of addresses in curly brackets can be separated by the pipe symbol (|). In this case, the corresponding sets of addresses are interpreted as replicas, and the query will be sent to the first healthy replica. The replicas are evaluated in the order currently set in the [load_balancing](../operations/settings/settings.md#settings-load_balancing) setting.
+Addresses and parts of addresses in curly brackets can be separated by the pipe symbol (|). In this case, the corresponding sets of addresses are interpreted as replicas, and the query will be sent to the first healthy replica. However, the replicas are iterated in the order currently set in the [load_balancing](../operations/settings/settings.md#settings-load_balancing) setting.
 
 Example:
 
diff --git a/docs/en/utils/clickhouse-copier.md b/docs/en/utils/clickhouse-copier.md
old mode 100644
new mode 100755
index 25d22f19222..9d15053fe06
--- a/docs/en/utils/clickhouse-copier.md
+++ b/docs/en/utils/clickhouse-copier.md
@@ -1,40 +1,54 @@
-# clickhouse-copier util
+<a name="utils-clickhouse-copier"></a>
 
-The util copies tables data from one cluster to new tables of other (possibly the same) cluster in distributed and fault-tolerant manner.
+# clickhouse-copier
 
-Configuration of copying tasks is set in special ZooKeeper node (called the `/description` node).
-A ZooKeeper path to the description node is specified via `--task-path </task/path>` parameter.
-So, node `/task/path/description` should contain special XML content describing copying tasks.
+Copies data from the tables in one cluster to tables in another (or the same) cluster.
 
-Simultaneously many `clickhouse-copier` processes located on any servers could execute the same task.
-ZooKeeper node `/task/path/` is used by the processes to coordinate their work.
-You must not add additional child nodes to `/task/path/`.
+You can run multiple `clickhouse-copier` instances on different servers to perform the same job. ZooKeeper is used for syncing the processes.
 
-Currently you are responsible for manual launching of all `cluster-copier` processes.
-You can launch as many processes as you want, whenever and wherever you want.
-Each process try to select the nearest available shard of source cluster and copy some part of data (partition) from it to the whole
-destination cluster (with resharding).
-Therefore it makes sense to launch cluster-copier processes on the source cluster nodes to reduce the network usage.
+After starting, `clickhouse-copier`:
 
-Since the workers coordinate their work via ZooKeeper, in addition to `--task-path </task/path>` you have to specify ZooKeeper
-cluster configuration via `--config-file <zookeeper.xml>` parameter. Example of `zookeeper.xml`:
+- Connects to ZooKeeper and receives:
+   - Copying jobs.
+   - The state of the copying jobs.
+
+- It performs the jobs.
+
+   Each running process chooses the "closest" shard of the source cluster and copies the data into the destination cluster, resharding the data if necessary.
+
+`clickhouse-copier` tracks the changes in ZooKeeper and applies them on the fly.
+
+To reduce network traffic, we recommend running `clickhouse-copier` on the same server where the source data is located.
+
+## Running clickhouse-copier
+
+The utility should be run manually:
+
+```bash
+clickhouse-copier copier --daemon --config zookeeper.xml --task-path /task/path --base-dir /path/to/dir
+```
+
+Parameters:
+
+- `daemon` — Starts `clickhouse-copier` in daemon mode.
+- `config` — The path to the `zookeeper.xml` file with the parameters for the connection to ZooKeeper.
+- `task-path` — The path to the ZooKeeper node. This node is used for syncing `clickhouse-copier` processes and storing tasks. Tasks are stored in `$task-path/description`.
+- `base-dir` — The path to logs and auxiliary files. When it starts, `clickhouse-copier` creates `clickhouse-copier_YYYYMMHHSS_<PID>` subdirectories in `$base-dir`. If this parameter is omitted, the directories are created in the directory where `clickhouse-copier` was launched.
+
+## Format of zookeeper.xml
 
 ```xml
-   <yandex>
+<yandex>
     <zookeeper>
         <node index="1">
             <host>127.0.0.1</host>
             <port>2181</port>
         </node>
     </zookeeper>
-   </yandex>
+</yandex>
 ```
 
-When you run `clickhouse-copier --config-file <zookeeper.xml> --task-path </task/path>` the process connects to ZooKeeper cluster, reads tasks config from `/task/path/description` and executes them.
-
-## Format of task config
-
-Here is an example of `/task/path/description` content:
+## Configuration of copying tasks
 
 ```xml
 <yandex>
@@ -69,62 +83,62 @@ Here is an example of `/task/path/description` content:
         <readonly>0</readonly>
     </settings_push>
 
-    <!-- Common setting for fetch (pull) and insert (push) operations. Also, copier process context uses it.
+    <!-- Common setting for fetch (pull) and insert (push) operations. The copier process context also uses it.
          They are overlaid by <settings_pull/> and <settings_push/> respectively. -->
     <settings>
         <connect_timeout>3</connect_timeout>
-        <!-- Sync insert is set forcibly, leave it here just in case. -->
+        <!-- Sync insert is set forcibly, but leave it here just in case. -->
         <insert_distributed_sync>1</insert_distributed_sync>
     </settings>
 
-    <!-- Copying tasks description.
-         You could specify several table task in the same task description (in the same ZooKeeper node), they will be performed
+    <!-- Copying task descriptions.
+         You can specify several table tasks in the same task description (in the same ZooKeeper node), and they will be performed
          sequentially.
     -->
     <tables>
-        <!-- A table task, copies one table. -->
+        <!-- A table task that copies one table. -->
         <table_hits>
-            <!-- Source cluster name (from <remote_servers/> section) and tables in it that should be copied -->
+            <!-- Source cluster name (from the <remote_servers/> section) and tables in it that should be copied. -->
             <cluster_pull>source_cluster</cluster_pull>
             <database_pull>test</database_pull>
             <table_pull>hits</table_pull>
 
-            <!-- Destination cluster name and tables in which the data should be inserted -->
+            <!-- Destination cluster name and tables where the data should be inserted. -->
             <cluster_push>destination_cluster</cluster_push>
             <database_push>test</database_push>
             <table_push>hits2</table_push>
 
-            <!-- Engine of destination tables.
-                 If destination tables have not be created, workers create them using columns definition from source tables and engine
+            <!-- Engine for destination tables.
+                 If the destination tables have not been created yet, workers create them using column definitions from source tables and the engine
                  definition from here.
 
-                 NOTE: If the first worker starts insert data and detects that destination partition is not empty then the partition will
-                 be dropped and refilled, take it into account if you already have some data in destination tables. You could directly
-                 specify partitions that should be copied in <enabled_partitions/>, they should be in quoted format like partition column of
+                 NOTE: If the first worker starts to insert data and detects that the destination partition is not empty, then the partition will
+                 be dropped and refilled. Take this into account if you already have some data in destination tables. You can directly
+                 specify partitions that should be copied in <enabled_partitions/>. They should be in quoted format like the partition column in the
                  system.parts table.
             -->
-            <engine>ENGINE=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/hits2', '{replica}')
+            <engine>
+            ENGINE=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/hits2', '{replica}')
             PARTITION BY toMonday(date)
             ORDER BY (CounterID, EventDate)
             </engine>
 
-            <!-- Sharding key used to insert data to destination cluster -->
+            <!-- The sharding key used to insert data to the destination cluster. -->
             <sharding_key>jumpConsistentHash(intHash64(UserID), 2)</sharding_key>
 
-            <!-- Optional expression that filter data while pull them from source servers -->
+            <!-- Optional expression that filters data while pulling it from source servers. -->
             <where_condition>CounterID != 0</where_condition>
 
-            <!-- This section specifies partitions that should be copied, other partition will be ignored.
+            <!-- This section specifies partitions that should be copied. Other partitions will be ignored.
                  Partition names should have the same format as
-                 partition column of system.parts table (i.e. a quoted text).
-                 Since partition key of source and destination cluster could be different,
-                 these partition names specify destination partitions.
+                the partition column in the system.parts table (i.e. quoted text).
+                 Since the partition key might be different for the source and destination cluster,
+                 these partition names specify the destination partitions.
 
-                 NOTE: In spite of this section is optional (if it is not specified, all partitions will be copied),
-                 it is strictly recommended to specify them explicitly.
-                 If you already have some ready paritions on destination cluster they
-                 will be removed at the start of the copying since they will be interpeted
-                 as unfinished data from the previous copying!!!
+                 Note: Although this section is optional (if it omitted, all partitions will be copied), it is strongly recommended to specify the partitions explicitly.
+                 If you already have some partitions ready on the destination cluster, they
+                 will be removed at the start of the copying, because they will be interpreted
+                 as unfinished data from the previous copying.
             -->
             <enabled_partitions>
                 <partition>'2018-02-26'</partition>
@@ -133,7 +147,7 @@ Here is an example of `/task/path/description` content:
             </enabled_partitions>
         </table_hits>
 
-        <!-- Next table to copy. It is not copied until previous table is copying. -->
+        <!-- Next table to copy. It is not copied until the previous table is copying. -->
         </table_visits>
         ...
         </table_visits>
@@ -142,15 +156,5 @@ Here is an example of `/task/path/description` content:
 </yandex>
 ```
 
-cluster-copier processes watch for `/task/path/description` node update.
-So, if you modify the config settings or `max_workers` params, they will be updated.
+`clickhouse-copier` tracks the changes in `/task/path/description` and applies them on the fly. For instance, if you change the value of `max_workers`, the number of processes running tasks will also change.
 
-## Example
-
-```bash
-clickhouse-copier copier --daemon --config /path/to/copier/zookeeper.xml --task-path /clickhouse-copier/cluster1_tables_hits --base-dir /path/to/copier_logs
-```
-
-`--base-dir /path/to/copier_logs` specifies where auxilary and log files of the copier process will be saved.
-In this case it will create `/path/to/copier_logs/clickhouse-copier_YYYYMMHHSS_<PID>/` dir with log and status-files.
-If it is not specified it will use current dir (`/clickhouse-copier_YYYYMMHHSS_<PID>/` if it is run as a `--daemon`).
diff --git a/docs/en/utils/clickhouse-local.md b/docs/en/utils/clickhouse-local.md
old mode 100644
new mode 100755
index d18cc200320..d5fba56271f
--- a/docs/en/utils/clickhouse-local.md
+++ b/docs/en/utils/clickhouse-local.md
@@ -1,4 +1,6 @@
-# The clickhouse-local program
+<a name="utils-clickhouse-local"></a>
 
-The `clickhouse-local`  program enables you to perform fast processing on local files that store tables, without having to deploy and configure clickhouse-server.
+#clickhouse-local
+
+The `clickhouse-local`  program enables you to perform fast processing on local files that store tables, without having to deploy and configure the ClickHouse server.
 
diff --git a/docs/en/utils/index.md b/docs/en/utils/index.md
old mode 100644
new mode 100755
index 7a8c5ee5138..cf541cda895
--- a/docs/en/utils/index.md
+++ b/docs/en/utils/index.md
@@ -1,6 +1,5 @@
-# ClickHouse utilites
+# ClickHouse utility
 
-There are several ClickHouse utilites that are separate executable files:
+* [clickhouse-local](clickhouse-local.md#utils-clickhouse-local) — Allows running SQL queries on data without stopping the ClickHouse server, similar to how `awk` does this.
+* [clickhouse-copier](clickhouse-copier.md#utils-clickhouse-copier) — Copies (and reshards) data from one cluster to another cluster.
 
-* `clickhouse-local` allows to execute SQL queries on a local data like `awk`
-* `clickhouse-copier` copies (and reshards) immutable data from one cluster to another in a fault-tolerant manner.
diff --git a/docs/mkdocs_en.yml b/docs/mkdocs_en.yml
index d5fadb3f1e1..012d498f3e2 100644
--- a/docs/mkdocs_en.yml
+++ b/docs/mkdocs_en.yml
@@ -86,6 +86,7 @@ pages:
   - 'GraphiteMergeTree': 'table_engines/graphitemergetree.md'
   - 'Data replication': 'table_engines/replication.md'
   - 'Distributed': 'table_engines/distributed.md'
+  - 'Dictionary': 'table_engines/dictionary.md'
   - 'Merge': 'table_engines/merge.md'
   - 'Buffer': 'table_engines/buffer.md'
   - 'File': 'table_engines/file.md'
@@ -95,6 +96,7 @@ pages:
   - 'View': 'table_engines/view.md'
   - 'MaterializedView': 'table_engines/materializedview.md'
   - 'Kafka': 'table_engines/kafka.md'
+  - 'MySQL': 'table_engines/mysql.md'
   - 'External data for query processing': 'table_engines/external_data.md'
 
 - 'System tables':
diff --git a/docs/ru/dicts/external_dicts_dict_layout.md b/docs/ru/dicts/external_dicts_dict_layout.md
index ff1b9c0cdd5..defb0605c0f 100644
--- a/docs/ru/dicts/external_dicts_dict_layout.md
+++ b/docs/ru/dicts/external_dicts_dict_layout.md
@@ -109,15 +109,15 @@
 Пример: таблица содержит скидки для каждого рекламодателя в виде:
 
 ```
-  +------------------+-----------------------------+------------+----------+
-  | id рекламодателя | дата начала действия скидки | дата конца | величина |
-  +==================+=============================+============+==========+
-  | 123              | 2015-01-01                  | 2015-01-15 | 0.15     |
-  +------------------+-----------------------------+------------+----------+
-  | 123              | 2015-01-16                  | 2015-01-31 | 0.25     |
-  +------------------+-----------------------------+------------+----------+
-  | 456              | 2015-01-01                  | 2015-01-15 | 0.05     |
-  +------------------+-----------------------------+------------+----------+
++---------------+---------------------+-------------------+--------+
+| advertiser id | discount start date | discount end date | amount |
++===============+=====================+===================+========+
+| 123           | 2015-01-01          | 2015-01-15        | 0.15   |
++---------------+---------------------+-------------------+--------+
+| 123           | 2015-01-16          | 2015-01-31        | 0.25   |
++---------------+---------------------+-------------------+--------+
+| 456           | 2015-01-01          | 2015-01-15        | 0.05   |
++---------------+---------------------+-------------------+--------+
 ```
 
 Чтобы использовать выборку по диапазонам дат, необходимо в [structure](external_dicts_dict_structure#dicts-external_dicts_dict_structure) определить элементы `range_min`, `range_max`.

From cd59ff0d54039aac2493d819566ea1465e952d8b Mon Sep 17 00:00:00 2001
From: BayoNet <bayonet@yandex-team.ru>
Date: Sun, 25 Mar 2018 05:16:13 +0300
Subject: [PATCH 200/244] mysql.md is not translated yet

---
 docs/mkdocs_en.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docs/mkdocs_en.yml b/docs/mkdocs_en.yml
index 012d498f3e2..eeedc71a79b 100644
--- a/docs/mkdocs_en.yml
+++ b/docs/mkdocs_en.yml
@@ -96,7 +96,6 @@ pages:
   - 'View': 'table_engines/view.md'
   - 'MaterializedView': 'table_engines/materializedview.md'
   - 'Kafka': 'table_engines/kafka.md'
-  - 'MySQL': 'table_engines/mysql.md'
   - 'External data for query processing': 'table_engines/external_data.md'
 
 - 'System tables':

From ec86ecbcdc10c7d17323cb43038bd179fa6176b4 Mon Sep 17 00:00:00 2001
From: BayoNet <bayonet@yandex-team.ru>
Date: Sun, 25 Mar 2018 05:37:07 +0300
Subject: [PATCH 201/244] Some typos are fixed.

---
 docs/ru/introduction/ya_metrika_task.md |  2 +-
 docs/ru/roadmap.md                      | 24 ++++++++++++------------
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/docs/ru/introduction/ya_metrika_task.md b/docs/ru/introduction/ya_metrika_task.md
index 24e595b2c49..765c0450890 100644
--- a/docs/ru/introduction/ya_metrika_task.md
+++ b/docs/ru/introduction/ya_metrika_task.md
@@ -1,6 +1,6 @@
 # Постановка задачи в Яндекс.Метрике
 
-ClickHouse на данный момент обеспечивает рабту [Яндекс.Метрики](https://metrika.yandex.ru/), [второй крупнейшей в мире](http://w3techs.com/technologies/overview/traffic_analysis/all) платформы для веб аналитики. При более 13 триллионах записей в базе данных и более 20 миллиардах событий в сутки, ClickHouse позволяет генерировать индивидуально настроенные отчёты на лету напрямую из неагрегированных данных.
+ClickHouse на данный момент обеспечивает работу [Яндекс.Метрики](https://metrika.yandex.ru/), [второй крупнейшей в мире](http://w3techs.com/technologies/overview/traffic_analysis/all) платформы для веб аналитики. При более 13 триллионах записей в базе данных и более 20 миллиардах событий в сутки, ClickHouse позволяет генерировать индивидуально настроенные отчёты на лету напрямую из неагрегированных данных.
 
 Нужно получать произвольные отчёты на основе хитов и визитов, с произвольными сегментами, задаваемыми пользователем. Данные для отчётов обновляются в реальном времени. Запросы должны выполняться сразу (в режиме онлайн). Отчёты должно быть возможно строить за произвольный период. Требуется вычислять сложные агрегаты типа количества уникальных посетителей.
 На данный момент (апрель 2014), каждый день в Яндекс.Метрику поступает около 12 миллиардов событий (хитов и кликов мыши). Все эти события должны быть сохранены для возможности строить произвольные отчёты. Один запрос может потребовать просканировать сотни миллионов строк за время не более нескольких секунд, или миллионы строк за время не более нескольких сотен миллисекунд.
diff --git a/docs/ru/roadmap.md b/docs/ru/roadmap.md
index 13c2b60c094..bbdc740bf01 100644
--- a/docs/ru/roadmap.md
+++ b/docs/ru/roadmap.md
@@ -5,15 +5,15 @@
 ### Новая функциональность
 - Поддержка `UPDATE` и `DELETE`.
 - Многомерные и вложенные массивы.
-    
+
     Это может выглядеть например так:
-    
+
 ```sql
 CREATE TABLE t
 (
-    x Array(Array(String)), 
+    x Array(Array(String)),
     z Nested(
-        x Array(String), 
+        x Array(String),
         y Nested(...))
 )
 ENGINE = MergeTree ORDER BY x
@@ -24,7 +24,7 @@ ENGINE = MergeTree ORDER BY x
     Внешние таблицы можно интрегрировать в ClickHouse с помощью внешних словарей. Новая функциональность станет более удобной альтернативой для подключения внешних таблиц.
 
 ```sql
-SELECT ... 
+SELECT ...
 FROM mysql('host:port', 'db', 'table', 'user', 'password')`
 ```
 
@@ -40,7 +40,7 @@ INSERT INTO t SELECT * FROM remote(...) `.
 - O_DIRECT for merges.
 
     Улучшит производительность кэша операционной системы, а также производительность 'горячих' запросов.
-    
+
 ## Q2 2018
 
 ### Новая функциональность
@@ -56,23 +56,23 @@ INSERT INTO t SELECT * FROM remote(...) `.
 - Настраиваемые кодировки и сжатие для каждого столбца в отдельности.
 
     Сейчас, ClickHouse поддерживает сжатие столбцов с помощью LZ4 и ZSTD, и настройки сжатия глобальные (смотрите статью [Compression in ClickHouse](https://www.altinity.com/blog/2017/11/21/compression-in-clickhouse)). Поколоночное сжатие и кодирование обеспечит более эффективное хранение данных, что в свою очередь ускорит выполнение запросов.
-    
+
 - Хранение данных на нескольких дисках на одном сервере.
 
     Реализация это функциональности упростит расширение дискового пространства, поскольку можно будет использовать различные дисковые системы для разных баз данных или таблиц. Сейчас, пользователи вынуждены использовать символические ссылки, если базы данных и таблицы должны храниться на другом диске.
-    
+
 ### Улучшения
 
 Планируется множество улучшений и исправлений в системе выполнения запросов. Например:
 
 - Использование индекса для `in (subquery)`.
 
-    Сейчас, индекс не используется, что приводит с снижению производительности.
-    
+    Сейчас, индекс не используется, что приводит к снижению производительности.
+
 - Передача предикатов из `where` в подзапросы, а также передача предикатов в представления.
 
     Передача предикатов необходима, поскольку представление изменяется поздапросом. Сейчас производительность фильтров для представлений низкая, представления не могут использовать первичный ключ оригинальной таблицы, что делает представления для больших таблиц бесполезными.
-    
+
 - Оптимизация операций с ветвлением (тернарный оператор, if, multiIf).
 
     Сейчас, ClickHouse выполняет все ветви, даже если в этом нет необходимости.
@@ -88,7 +88,7 @@ INSERT INTO t SELECT * FROM remote(...) `.
 - Пулы ресурсов для выполнения запросов.
 
     Позволят более эффективно управлять нагрузкой.
-    
+
 - Синтаксис ANSI SQL JOIN.
 
     Улучшит совместимость ClickHouse со множеством SQL-инструментов.

From e07cde4363d830b69adc295f78682b049ea337e9 Mon Sep 17 00:00:00 2001
From: Jonatas Freitas <jonatasfreitasv@gmail.com>
Date: Sun, 25 Mar 2018 01:43:12 -0300
Subject: [PATCH 202/244] Added missing slash on graphite monitoring section

---
 dbms/src/Server/config.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dbms/src/Server/config.xml b/dbms/src/Server/config.xml
index 0eb96cdcf6e..8cf1fd6b519 100644
--- a/dbms/src/Server/config.xml
+++ b/dbms/src/Server/config.xml
@@ -199,7 +199,7 @@
         <timeout>0.1</timeout>
         <interval>60</interval>
         <root_path>one_min</root_path>
-        <hostname_in_path>true<hostname_in_path>
+        <hostname_in_path>true</hostname_in_path>
 
         <metrics>true</metrics>
         <events>true</events>

From 237c530b4e1564301a2bcde9e6d1ccf2abb00b62 Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Mon, 26 Mar 2018 01:42:36 +0300
Subject: [PATCH 203/244] Update StorageKafka.cpp

---
 dbms/src/Storages/StorageKafka.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/dbms/src/Storages/StorageKafka.cpp b/dbms/src/Storages/StorageKafka.cpp
index 4ec2307ef13..80e4942839e 100644
--- a/dbms/src/Storages/StorageKafka.cpp
+++ b/dbms/src/Storages/StorageKafka.cpp
@@ -578,11 +578,11 @@ void registerStorageKafka(StorageFactory & factory)
 
         // Parse topic list
         Names topics;
-        String topicArg = static_cast<const ASTLiteral &>(*engine_args[1]).value.safeGet<String>();
-        boost::split(topics, topicArg , [](char c){return c == ',';});
-        for(String & topic : topics) {
+        String topic_arg = static_cast<const ASTLiteral &>(*engine_args[1]).value.safeGet<String>();
+        boost::split(topics, topic_arg , [](char c){ return c == ','; });
+        for(String & topic : topics)
             boost::trim(topic);
-        }
+
         // Parse consumer group
         String group = static_cast<const ASTLiteral &>(*engine_args[2]).value.safeGet<String>();
 

From 9ba28666e1dff9625fcf85c40fd2138059f99a25 Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Mon, 26 Mar 2018 01:44:54 +0300
Subject: [PATCH 204/244] Update Client.cpp

---
 dbms/src/Server/Client.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp
index 1a8d97af25a..c53a2c1b873 100644
--- a/dbms/src/Server/Client.cpp
+++ b/dbms/src/Server/Client.cpp
@@ -337,7 +337,8 @@ private:
 
         if (is_interactive)
         {
-            query_id = "";
+            if (!query_id.empty())
+                throw Exception("query_id could be specified only in non-interactive mode", ErrorCodes::BAD_ARGUMENTS);
             if (print_time_to_stderr)
                 throw Exception("time option could be specified only in non-interactive mode", ErrorCodes::BAD_ARGUMENTS);
 

From 26df2f7a858fd0b6a334490fb013125696768781 Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Mon, 26 Mar 2018 17:00:55 +0300
Subject: [PATCH 205/244] Revert "English translation is updated"

---
 docs/en/agg_functions/combinators.md          |   0
 docs/en/agg_functions/index.md                |   1 -
 docs/en/agg_functions/parametric_functions.md |   0
 docs/en/agg_functions/reference.md            |  34 +++--
 docs/en/data_types/array.md                   |   0
 docs/en/data_types/boolean.md                 |   0
 docs/en/data_types/date.md                    |   2 +-
 docs/en/data_types/datetime.md                |   0
 docs/en/data_types/enum.md                    |   0
 docs/en/data_types/fixedstring.md             |   0
 docs/en/data_types/float.md                   |  10 +-
 docs/en/data_types/index.md                   |   5 +-
 docs/en/data_types/int_uint.md                |   0
 .../aggregatefunction.md                      |   0
 .../nested_data_structures/index.md           |   0
 .../nested_data_structures/nested.md          |   0
 .../special_data_types/expression.md          |   0
 .../en/data_types/special_data_types/index.md |   0
 docs/en/data_types/special_data_types/set.md  |   0
 docs/en/data_types/string.md                  |   0
 docs/en/data_types/tuple.md                   |   0
 docs/en/development/style.md                  |  65 +++++-----
 docs/en/dicts/external_dicts.md               |   5 +-
 docs/en/dicts/external_dicts_dict.md          |   9 +-
 docs/en/dicts/external_dicts_dict_layout.md   |  99 ++++++++++++---
 docs/en/dicts/external_dicts_dict_lifetime.md |   6 +-
 docs/en/dicts/external_dicts_dict_sources.md  |   6 +-
 .../en/dicts/external_dicts_dict_structure.md |  14 +-
 docs/en/dicts/index.md                        |   0
 docs/en/dicts/internal_dicts.md               |   0
 docs/en/formats/capnproto.md                  |  52 ++++----
 docs/en/formats/csv.md                        |   0
 docs/en/formats/csvwithnames.md               |   0
 docs/en/formats/index.md                      |   1 -
 docs/en/formats/json.md                       |   0
 docs/en/formats/jsoncompact.md                |   2 +-
 docs/en/formats/jsoneachrow.md                |   0
 docs/en/formats/native.md                     |   0
 docs/en/formats/null.md                       |   0
 docs/en/formats/pretty.md                     |   0
 docs/en/formats/prettycompact.md              |   0
 docs/en/formats/prettycompactmonoblock.md     |   0
 docs/en/formats/prettynoescapes.md            |   0
 docs/en/formats/prettyspace.md                |   0
 docs/en/formats/rowbinary.md                  |   2 +-
 docs/en/formats/tabseparated.md               |   0
 docs/en/formats/tabseparatedraw.md            |   0
 docs/en/formats/tabseparatedwithnames.md      |   0
 .../formats/tabseparatedwithnamesandtypes.md  |   0
 docs/en/formats/tskv.md                       |   0
 docs/en/formats/values.md                     |   2 +-
 docs/en/formats/vertical.md                   |   0
 docs/en/formats/xml.md                        |   2 +-
 docs/en/functions/arithmetic_functions.md     |   0
 docs/en/functions/array_functions.md          |  12 +-
 docs/en/functions/array_join.md               |   1 +
 docs/en/functions/bit_functions.md            |   1 +
 docs/en/functions/comparison_functions.md     |   2 +-
 docs/en/functions/conditional_functions.md    |   0
 docs/en/functions/date_time_functions.md      |   4 -
 docs/en/functions/encoding_functions.md       |   0
 docs/en/functions/ext_dict_functions.md       |   8 +-
 docs/en/functions/hash_functions.md           |   0
 docs/en/functions/higher_order_functions.md   |   3 +-
 docs/en/functions/in_functions.md             |   0
 docs/en/functions/index.md                    |   0
 docs/en/functions/ip_address_functions.md     |   0
 docs/en/functions/json_functions.md           |   9 +-
 docs/en/functions/logical_functions.md        |   1 +
 docs/en/functions/math_functions.md           |   1 +
 docs/en/functions/other_functions.md          |   5 +-
 docs/en/functions/random_functions.md         |   0
 docs/en/functions/rounding_functions.md       |   0
 .../functions/splitting_merging_functions.md  |   0
 docs/en/functions/string_functions.md         |   0
 docs/en/functions/string_replace_functions.md |   1 +
 docs/en/functions/string_search_functions.md  |   0
 .../en/functions/type_conversion_functions.md |   0
 docs/en/functions/url_functions.md            |   0
 docs/en/functions/ym_dict_functions.md        |  12 +-
 .../example_datasets/amplab_benchmark.md      |   0
 .../example_datasets/criteo.md                |   4 +-
 .../example_datasets/nyc_taxi.md              |  15 ++-
 .../example_datasets/ontime.md                |   0
 .../example_datasets/star_schema.md           |   3 +-
 .../example_datasets/wikistat.md              |   2 +-
 docs/en/getting_started/index.md              |  14 +-
 docs/en/index.md                              |   2 +-
 docs/en/interfaces/cli.md                     |   7 +-
 docs/en/interfaces/http_interface.md          |  16 ++-
 docs/en/interfaces/index.md                   |   3 +-
 docs/en/interfaces/jdbc.md                    |   0
 docs/en/interfaces/tcp.md                     |   0
 .../third-party_client_libraries.md           |   2 +-
 docs/en/interfaces/third-party_gui.md         |   0
 docs/en/introduction/distinctive_features.md  |  20 +--
 .../features_considered_disadvantages.md      |   0
 docs/en/introduction/index.md                 |   1 -
 docs/en/introduction/performance.md           |   0
 .../introduction/possible_silly_questions.md  |   4 +-
 docs/en/introduction/ya_metrika_task.md       |   4 +-
 docs/en/operations/access_rights.md           |  59 +++++----
 docs/en/operations/configuration_files.md     |   2 +-
 docs/en/operations/index.md                   |   3 +-
 docs/en/operations/quotas.md                  |  35 ++---
 docs/en/operations/server_settings/index.md   |   1 -
 .../en/operations/server_settings/settings.md |  37 +++---
 docs/en/operations/settings/index.md          |   4 +-
 .../operations/settings/query_complexity.md   |   0
 docs/en/operations/settings/settings.md       |  16 +--
 .../operations/settings/settings_profiles.md  |   2 +-
 docs/en/operations/tips.md                    |   5 +-
 docs/en/operators/index.md                    |   4 +-
 docs/en/query_language/index.md               |   1 -
 docs/en/query_language/queries.md             |  53 ++++----
 docs/en/query_language/syntax.md              |   2 +-
 docs/en/roadmap.md                            |  70 +++++-----
 docs/en/system_tables/index.md                |   2 +-
 .../system.asynchronous_metrics.md            |   0
 docs/en/system_tables/system.clusters.md      |  17 +--
 docs/en/system_tables/system.columns.md       |   1 +
 docs/en/system_tables/system.databases.md     |   0
 docs/en/system_tables/system.dictionaries.md  |  26 ++--
 docs/en/system_tables/system.events.md        |   0
 docs/en/system_tables/system.functions.md     |   2 +-
 docs/en/system_tables/system.merges.md        |   1 +
 docs/en/system_tables/system.numbers.md       |   0
 docs/en/system_tables/system.numbers_mt.md    |   0
 docs/en/system_tables/system.one.md           |   0
 docs/en/system_tables/system.parts.md         |   0
 docs/en/system_tables/system.processes.md     |   6 +-
 docs/en/system_tables/system.replicas.md      |  26 ++--
 docs/en/system_tables/system.settings.md      |   4 +-
 docs/en/system_tables/system.tables.md        |   3 +-
 docs/en/system_tables/system.zookeeper.md     |   1 +
 docs/en/table_engines/aggregatingmergetree.md |  31 ++---
 docs/en/table_engines/buffer.md               |   0
 docs/en/table_engines/collapsingmergetree.md  |   0
 .../table_engines/custom_partitioning_key.md  |  14 +-
 docs/en/table_engines/dictionary.md           | 106 ----------------
 docs/en/table_engines/distributed.md          |  46 +++----
 docs/en/table_engines/external_data.md        |   0
 docs/en/table_engines/file.md                 |   0
 docs/en/table_engines/graphitemergetree.md    |   5 +-
 docs/en/table_engines/index.md                |   0
 docs/en/table_engines/join.md                 |   0
 docs/en/table_engines/kafka.md                |   2 +-
 docs/en/table_engines/log.md                  |   0
 docs/en/table_engines/materializedview.md     |   2 +-
 docs/en/table_engines/memory.md               |   0
 docs/en/table_engines/merge.md                |   0
 docs/en/table_engines/mergetree.md            |   2 +-
 docs/en/table_engines/null.md                 |   0
 docs/en/table_engines/replacingmergetree.md   |   0
 docs/en/table_engines/replication.md          |   2 +-
 docs/en/table_engines/set.md                  |   0
 docs/en/table_engines/summingmergetree.md     |   0
 docs/en/table_engines/tinylog.md              |   0
 docs/en/table_engines/view.md                 |   0
 docs/en/table_functions/index.md              |   0
 docs/en/table_functions/merge.md              |   0
 docs/en/table_functions/remote.md             |   2 +-
 docs/en/utils/clickhouse-copier.md            | 120 +++++++++---------
 docs/en/utils/clickhouse-local.md             |   6 +-
 docs/en/utils/index.md                        |   7 +-
 docs/mkdocs_en.yml                            |   1 -
 docs/ru/dicts/external_dicts_dict_layout.md   |  18 +--
 docs/ru/introduction/ya_metrika_task.md       |   2 +-
 docs/ru/roadmap.md                            |  24 ++--
 169 files changed, 587 insertions(+), 660 deletions(-)
 mode change 100755 => 100644 docs/en/agg_functions/combinators.md
 mode change 100755 => 100644 docs/en/agg_functions/index.md
 mode change 100755 => 100644 docs/en/agg_functions/parametric_functions.md
 mode change 100755 => 100644 docs/en/agg_functions/reference.md
 mode change 100755 => 100644 docs/en/data_types/array.md
 mode change 100755 => 100644 docs/en/data_types/boolean.md
 mode change 100755 => 100644 docs/en/data_types/date.md
 mode change 100755 => 100644 docs/en/data_types/datetime.md
 mode change 100755 => 100644 docs/en/data_types/enum.md
 mode change 100755 => 100644 docs/en/data_types/fixedstring.md
 mode change 100755 => 100644 docs/en/data_types/float.md
 mode change 100755 => 100644 docs/en/data_types/index.md
 mode change 100755 => 100644 docs/en/data_types/int_uint.md
 mode change 100755 => 100644 docs/en/data_types/nested_data_structures/aggregatefunction.md
 mode change 100755 => 100644 docs/en/data_types/nested_data_structures/index.md
 mode change 100755 => 100644 docs/en/data_types/nested_data_structures/nested.md
 mode change 100755 => 100644 docs/en/data_types/special_data_types/expression.md
 mode change 100755 => 100644 docs/en/data_types/special_data_types/index.md
 mode change 100755 => 100644 docs/en/data_types/special_data_types/set.md
 mode change 100755 => 100644 docs/en/data_types/string.md
 mode change 100755 => 100644 docs/en/data_types/tuple.md
 mode change 100755 => 100644 docs/en/development/style.md
 mode change 100755 => 100644 docs/en/dicts/external_dicts.md
 mode change 100755 => 100644 docs/en/dicts/external_dicts_dict.md
 mode change 100755 => 100644 docs/en/dicts/external_dicts_dict_layout.md
 mode change 100755 => 100644 docs/en/dicts/external_dicts_dict_lifetime.md
 mode change 100755 => 100644 docs/en/dicts/external_dicts_dict_sources.md
 mode change 100755 => 100644 docs/en/dicts/external_dicts_dict_structure.md
 mode change 100755 => 100644 docs/en/dicts/index.md
 mode change 100755 => 100644 docs/en/dicts/internal_dicts.md
 mode change 100755 => 100644 docs/en/formats/capnproto.md
 mode change 100755 => 100644 docs/en/formats/csv.md
 mode change 100755 => 100644 docs/en/formats/csvwithnames.md
 mode change 100755 => 100644 docs/en/formats/index.md
 mode change 100755 => 100644 docs/en/formats/json.md
 mode change 100755 => 100644 docs/en/formats/jsoncompact.md
 mode change 100755 => 100644 docs/en/formats/jsoneachrow.md
 mode change 100755 => 100644 docs/en/formats/native.md
 mode change 100755 => 100644 docs/en/formats/null.md
 mode change 100755 => 100644 docs/en/formats/pretty.md
 mode change 100755 => 100644 docs/en/formats/prettycompact.md
 mode change 100755 => 100644 docs/en/formats/prettycompactmonoblock.md
 mode change 100755 => 100644 docs/en/formats/prettynoescapes.md
 mode change 100755 => 100644 docs/en/formats/prettyspace.md
 mode change 100755 => 100644 docs/en/formats/rowbinary.md
 mode change 100755 => 100644 docs/en/formats/tabseparated.md
 mode change 100755 => 100644 docs/en/formats/tabseparatedraw.md
 mode change 100755 => 100644 docs/en/formats/tabseparatedwithnames.md
 mode change 100755 => 100644 docs/en/formats/tabseparatedwithnamesandtypes.md
 mode change 100755 => 100644 docs/en/formats/tskv.md
 mode change 100755 => 100644 docs/en/formats/values.md
 mode change 100755 => 100644 docs/en/formats/vertical.md
 mode change 100755 => 100644 docs/en/formats/xml.md
 mode change 100755 => 100644 docs/en/functions/arithmetic_functions.md
 mode change 100755 => 100644 docs/en/functions/array_functions.md
 mode change 100755 => 100644 docs/en/functions/array_join.md
 mode change 100755 => 100644 docs/en/functions/bit_functions.md
 mode change 100755 => 100644 docs/en/functions/comparison_functions.md
 mode change 100755 => 100644 docs/en/functions/conditional_functions.md
 mode change 100755 => 100644 docs/en/functions/date_time_functions.md
 mode change 100755 => 100644 docs/en/functions/encoding_functions.md
 mode change 100755 => 100644 docs/en/functions/ext_dict_functions.md
 mode change 100755 => 100644 docs/en/functions/hash_functions.md
 mode change 100755 => 100644 docs/en/functions/higher_order_functions.md
 mode change 100755 => 100644 docs/en/functions/in_functions.md
 mode change 100755 => 100644 docs/en/functions/index.md
 mode change 100755 => 100644 docs/en/functions/ip_address_functions.md
 mode change 100755 => 100644 docs/en/functions/json_functions.md
 mode change 100755 => 100644 docs/en/functions/logical_functions.md
 mode change 100755 => 100644 docs/en/functions/math_functions.md
 mode change 100755 => 100644 docs/en/functions/other_functions.md
 mode change 100755 => 100644 docs/en/functions/random_functions.md
 mode change 100755 => 100644 docs/en/functions/rounding_functions.md
 mode change 100755 => 100644 docs/en/functions/splitting_merging_functions.md
 mode change 100755 => 100644 docs/en/functions/string_functions.md
 mode change 100755 => 100644 docs/en/functions/string_replace_functions.md
 mode change 100755 => 100644 docs/en/functions/string_search_functions.md
 mode change 100755 => 100644 docs/en/functions/type_conversion_functions.md
 mode change 100755 => 100644 docs/en/functions/url_functions.md
 mode change 100755 => 100644 docs/en/functions/ym_dict_functions.md
 mode change 100755 => 100644 docs/en/getting_started/example_datasets/amplab_benchmark.md
 mode change 100755 => 100644 docs/en/getting_started/example_datasets/criteo.md
 mode change 100755 => 100644 docs/en/getting_started/example_datasets/nyc_taxi.md
 mode change 100755 => 100644 docs/en/getting_started/example_datasets/ontime.md
 mode change 100755 => 100644 docs/en/getting_started/example_datasets/star_schema.md
 mode change 100755 => 100644 docs/en/getting_started/example_datasets/wikistat.md
 mode change 100755 => 100644 docs/en/getting_started/index.md
 mode change 100755 => 100644 docs/en/index.md
 mode change 100755 => 100644 docs/en/interfaces/cli.md
 mode change 100755 => 100644 docs/en/interfaces/http_interface.md
 mode change 100755 => 100644 docs/en/interfaces/index.md
 mode change 100755 => 100644 docs/en/interfaces/jdbc.md
 mode change 100755 => 100644 docs/en/interfaces/tcp.md
 mode change 100755 => 100644 docs/en/interfaces/third-party_client_libraries.md
 mode change 100755 => 100644 docs/en/interfaces/third-party_gui.md
 mode change 100755 => 100644 docs/en/introduction/distinctive_features.md
 mode change 100755 => 100644 docs/en/introduction/features_considered_disadvantages.md
 mode change 100755 => 100644 docs/en/introduction/index.md
 mode change 100755 => 100644 docs/en/introduction/performance.md
 mode change 100755 => 100644 docs/en/introduction/possible_silly_questions.md
 mode change 100755 => 100644 docs/en/introduction/ya_metrika_task.md
 mode change 100755 => 100644 docs/en/operations/access_rights.md
 mode change 100755 => 100644 docs/en/operations/configuration_files.md
 mode change 100755 => 100644 docs/en/operations/index.md
 mode change 100755 => 100644 docs/en/operations/quotas.md
 mode change 100755 => 100644 docs/en/operations/server_settings/index.md
 mode change 100755 => 100644 docs/en/operations/server_settings/settings.md
 mode change 100755 => 100644 docs/en/operations/settings/index.md
 mode change 100755 => 100644 docs/en/operations/settings/query_complexity.md
 mode change 100755 => 100644 docs/en/operations/settings/settings.md
 mode change 100755 => 100644 docs/en/operations/settings/settings_profiles.md
 mode change 100755 => 100644 docs/en/operations/tips.md
 mode change 100755 => 100644 docs/en/operators/index.md
 mode change 100755 => 100644 docs/en/query_language/index.md
 mode change 100755 => 100644 docs/en/query_language/queries.md
 mode change 100755 => 100644 docs/en/query_language/syntax.md
 mode change 100755 => 100644 docs/en/roadmap.md
 mode change 100755 => 100644 docs/en/system_tables/index.md
 mode change 100755 => 100644 docs/en/system_tables/system.asynchronous_metrics.md
 mode change 100755 => 100644 docs/en/system_tables/system.clusters.md
 mode change 100755 => 100644 docs/en/system_tables/system.columns.md
 mode change 100755 => 100644 docs/en/system_tables/system.databases.md
 mode change 100755 => 100644 docs/en/system_tables/system.dictionaries.md
 mode change 100755 => 100644 docs/en/system_tables/system.events.md
 mode change 100755 => 100644 docs/en/system_tables/system.functions.md
 mode change 100755 => 100644 docs/en/system_tables/system.merges.md
 mode change 100755 => 100644 docs/en/system_tables/system.numbers.md
 mode change 100755 => 100644 docs/en/system_tables/system.numbers_mt.md
 mode change 100755 => 100644 docs/en/system_tables/system.one.md
 mode change 100755 => 100644 docs/en/system_tables/system.parts.md
 mode change 100755 => 100644 docs/en/system_tables/system.processes.md
 mode change 100755 => 100644 docs/en/system_tables/system.replicas.md
 mode change 100755 => 100644 docs/en/system_tables/system.settings.md
 mode change 100755 => 100644 docs/en/system_tables/system.tables.md
 mode change 100755 => 100644 docs/en/system_tables/system.zookeeper.md
 mode change 100755 => 100644 docs/en/table_engines/aggregatingmergetree.md
 mode change 100755 => 100644 docs/en/table_engines/buffer.md
 mode change 100755 => 100644 docs/en/table_engines/collapsingmergetree.md
 mode change 100755 => 100644 docs/en/table_engines/custom_partitioning_key.md
 delete mode 100755 docs/en/table_engines/dictionary.md
 mode change 100755 => 100644 docs/en/table_engines/distributed.md
 mode change 100755 => 100644 docs/en/table_engines/external_data.md
 mode change 100755 => 100644 docs/en/table_engines/file.md
 mode change 100755 => 100644 docs/en/table_engines/graphitemergetree.md
 mode change 100755 => 100644 docs/en/table_engines/index.md
 mode change 100755 => 100644 docs/en/table_engines/join.md
 mode change 100755 => 100644 docs/en/table_engines/kafka.md
 mode change 100755 => 100644 docs/en/table_engines/log.md
 mode change 100755 => 100644 docs/en/table_engines/materializedview.md
 mode change 100755 => 100644 docs/en/table_engines/memory.md
 mode change 100755 => 100644 docs/en/table_engines/merge.md
 mode change 100755 => 100644 docs/en/table_engines/mergetree.md
 mode change 100755 => 100644 docs/en/table_engines/null.md
 mode change 100755 => 100644 docs/en/table_engines/replacingmergetree.md
 mode change 100755 => 100644 docs/en/table_engines/replication.md
 mode change 100755 => 100644 docs/en/table_engines/set.md
 mode change 100755 => 100644 docs/en/table_engines/summingmergetree.md
 mode change 100755 => 100644 docs/en/table_engines/tinylog.md
 mode change 100755 => 100644 docs/en/table_engines/view.md
 mode change 100755 => 100644 docs/en/table_functions/index.md
 mode change 100755 => 100644 docs/en/table_functions/merge.md
 mode change 100755 => 100644 docs/en/table_functions/remote.md
 mode change 100755 => 100644 docs/en/utils/clickhouse-copier.md
 mode change 100755 => 100644 docs/en/utils/clickhouse-local.md
 mode change 100755 => 100644 docs/en/utils/index.md

diff --git a/docs/en/agg_functions/combinators.md b/docs/en/agg_functions/combinators.md
old mode 100755
new mode 100644
diff --git a/docs/en/agg_functions/index.md b/docs/en/agg_functions/index.md
old mode 100755
new mode 100644
index 3864f7271c4..e87bf4ff833
--- a/docs/en/agg_functions/index.md
+++ b/docs/en/agg_functions/index.md
@@ -8,4 +8,3 @@ ClickHouse also supports:
 
 - [Parametric aggregate functions](parametric_functions.md#aggregate_functions_parametric), which accept other parameters in addition to columns.
 - [Combinators](combinators.md#aggregate_functions_combinators), which change the behavior of aggregate functions.
-
diff --git a/docs/en/agg_functions/parametric_functions.md b/docs/en/agg_functions/parametric_functions.md
old mode 100755
new mode 100644
diff --git a/docs/en/agg_functions/reference.md b/docs/en/agg_functions/reference.md
old mode 100755
new mode 100644
index 0eb896e4664..90ff9da58e7
--- a/docs/en/agg_functions/reference.md
+++ b/docs/en/agg_functions/reference.md
@@ -19,7 +19,7 @@ In some cases, you can rely on the order of execution. This applies to cases whe
 
 When a `SELECT` query has the `GROUP BY` clause or at least one aggregate function, ClickHouse (in contrast to MySQL) requires that all expressions in the `SELECT`, `HAVING`, and `ORDER BY` clauses be calculated from keys or from aggregate functions. In other words, each column selected from the table must be used either in keys or inside aggregate functions. To get behavior like in MySQL, you can put the other columns in the `any` aggregate function.
 
-## anyHeavy
+## anyHeavy(x)
 
 Selects a frequently occurring value using the [heavy hitters](http://www.cs.umd.edu/~samir/498/karp.pdf) algorithm. If there is a value that occurs more than in half the cases in each of the query's execution threads, this value is returned. Normally, the result is nondeterministic.
 
@@ -39,6 +39,7 @@ Take the [OnTime](../getting_started/example_datasets/ontime.md#example_datasets
 SELECT anyHeavy(AirlineID) AS res
 FROM ontime
 ```
+
 ```
 ┌───res─┐
 │ 19690 │
@@ -124,11 +125,11 @@ The result is always Float64.
 Calculates the approximate number of different values of the argument. Works for numbers, strings, dates, date-with-time, and for multiple arguments and tuple arguments.
 
 Uses an adaptive sampling algorithm: for the calculation state, it uses a sample of element hash values with a size up to 65536.
-This algorithm is also very accurate for data sets with low cardinality (up to 65536) and very efficient on CPU (when computing not too many of these functions, using `uniq` is almost as fast as using other aggregate functions).
+This algorithm is also very accurate for data sets with small cardinality (up to 65536) and very efficient on CPU (when computing not too many of these functions, using `uniq` is almost as fast as using other aggregate functions).
 
 The result is determinate (it doesn't depend on the order of query processing).
 
-This function provides excellent accuracy even for data sets with extremely high cardinality (over 10 billion elements). It is recommended for default use.
+This function provides excellent accuracy even for data sets with huge cardinality (10B+ elements) and is recommended for use by default.
 
 ## uniqCombined(x)
 
@@ -138,16 +139,16 @@ A combination of three algorithms is used: array, hash table and [HyperLogLog](h
 
 The result is determinate (it doesn't depend on the order of query processing).
 
-The `uniqCombined` function is a good default choice for calculating the number of different values, but keep in mind that the estimation error will increase for high-cardinality data sets (200M+ elements), and the function will return very inaccurate results for data sets with extremely high cardinality (1B+ elements).
+The `uniqCombined` function is a good default choice for calculating the number of different values, but the following should be considered: for data sets with large cardinality (200M+) error of estimate will only grow and for data sets with huge cardinality(1B+ elements) it returns result with high inaccuracy.
 
 ## uniqHLL12(x)
 
 Uses the [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog) algorithm to approximate the number of different values of the argument.
-212 5-bit cells are used. The size of the state is slightly more than 2.5 KB. The result is not very accurate (up to ~10% error) for small data sets (<10K elements). However, the result is fairly accurate for high-cardinality data sets (10K-100M), with a maximum error of ~1.6%. Starting from 100M, the estimation error increases, and the function will return very inaccurate results for data sets with extremely high cardinality (1B+ elements).
+212 5-bit cells are used. The size of the state is slightly more than 2.5 KB. Result is not very accurate (error up to ~10%) for data sets of small cardinality(<10K elements), but for data sets with large cardinality (10K - 100M) result is quite accurate (error up to ~1.6%) and after that error of estimate will only grow and for data sets with huge cardinality (1B+ elements) it returns result with high inaccuracy.
 
 The result is determinate (it doesn't depend on the order of query processing).
 
-We don't recommend using this function. In most cases, use the  `uniq` or `uniqCombined` function.
+This function is not recommended for use, and in most cases, use the  `uniq` or `uniqCombined` function.
 
 ## uniqExact(x)
 
@@ -169,7 +170,7 @@ In some cases, you can still rely on the order of execution. This applies to cas
 
 <a name="agg_functions_groupArrayInsertAt"></a>
 
-## groupArrayInsertAt
+## groupArrayInsertAt(x)
 
 Inserts a value into the array in the specified position.
 
@@ -235,8 +236,8 @@ For its purpose (calculating quantiles of page loading times), using this functi
 
 ## quantileTimingWeighted(level)(x, weight)
 
-Differs from the `quantileTiming`  function in that it has a second argument, "weights". Weight is a non-negative integer.
-The result is calculated as if the `x`  value were passed `weight` number of times to the `quantileTiming` function.
+Differs from the 'quantileTiming' function in that it has a second argument, "weights". Weight is a non-negative integer.
+The result is calculated as if the 'x' value were passed 'weight' number of times to the 'quantileTiming' function.
 
 ## quantileExact(level)(x)
 
@@ -256,7 +257,7 @@ The performance of the function is lower than for ` quantile`, ` quantileTiming`
 
 The result depends on the order of running the query, and is nondeterministic.
 
-## median
+## median(x)
 
 All the quantile functions have corresponding median functions: `median`, `medianDeterministic`, `medianTiming`, `medianTimingWeighted`, `medianExact`, `medianExactWeighted`, `medianTDigest`. They are synonyms and their behavior is identical.
 
@@ -274,7 +275,7 @@ Returns `Float64`. When `n <= 1`, returns `+∞`.
 
 ## varPop(x)
 
-Calculates the amount `Σ((x - x̅)^2) / (n - 1)`, where `n` is the sample size and `x̅`is the average value of `x`.
+Calculates the amount `Σ((x - x̅)^2) / n`, where `n` is the sample size and `x̅`is the average value of `x`.
 
 In other words, dispersion for a set of values. Returns `Float64`.
 
@@ -286,33 +287,30 @@ The result is equal to the square root of `varSamp(x)`.
 
 The result is equal to the square root of `varPop(x)`.
 
-## topK
+## topK(N)(column)
 
 Returns an array of the most frequent values in the specified column. The resulting array is sorted in descending order of frequency of values (not by the values themselves).
 
 Implements the [ Filtered Space-Saving](http://www.l2f.inesc-id.pt/~fmmb/wiki/uploads/Work/misnis.ref0a.pdf)  algorithm for analyzing TopK, based on the reduce-and-combine algorithm from [Parallel Space Saving](https://arxiv.org/pdf/1401.0702.pdf).
 
-```
-topK(N)(column)
-```
-
 This function doesn't provide a guaranteed result. In certain situations, errors might occur and it might return frequent values that aren't the most frequent values.
 
 We recommend using the `N < 10 ` value; performance is reduced with large `N` values. Maximum value of ` N = 65536`.
 
 **Arguments**
 
-- 'N' is the number of values.
+- 'N' – The number of values.
 - ' x ' – The column.
 
 **Example**
 
-Take the [OnTime](../getting_started/example_datasets/ontime.md#example_datasets-ontime) data set and select the three most frequently occurring values in the `AirlineID` column.
+Take the [OnTime](../getting_started/example_datasets/ontime.md#example_datasets-ontime)data set and select the three most frequently occurring values in the `AirlineID` column.
 
 ```sql
 SELECT topK(3)(AirlineID) AS res
 FROM ontime
 ```
+
 ```
 ┌─res─────────────────┐
 │ [19393,19790,19805] │
diff --git a/docs/en/data_types/array.md b/docs/en/data_types/array.md
old mode 100755
new mode 100644
diff --git a/docs/en/data_types/boolean.md b/docs/en/data_types/boolean.md
old mode 100755
new mode 100644
diff --git a/docs/en/data_types/date.md b/docs/en/data_types/date.md
old mode 100755
new mode 100644
index cb179c0d8c4..355e5555bfa
--- a/docs/en/data_types/date.md
+++ b/docs/en/data_types/date.md
@@ -1,6 +1,6 @@
 # Date
 
-A date. Stored in two bytes as the number of days since 1970-01-01 (unsigned). Allows storing values from just after the beginning of the Unix Epoch to the upper threshold defined by a constant at the compilation stage (currently, this is until the year 2106, but the final fully-supported year is 2105).
+Date. Stored in two bytes as the number of days since 1970-01-01 (unsigned). Allows storing values from just after the beginning of the Unix Epoch to the upper threshold defined by a constant at the compilation stage (currently, this is until the year 2038, but it may be expanded to 2106).
 The minimum value is output as 0000-00-00.
 
 The date is stored without the time zone.
diff --git a/docs/en/data_types/datetime.md b/docs/en/data_types/datetime.md
old mode 100755
new mode 100644
diff --git a/docs/en/data_types/enum.md b/docs/en/data_types/enum.md
old mode 100755
new mode 100644
diff --git a/docs/en/data_types/fixedstring.md b/docs/en/data_types/fixedstring.md
old mode 100755
new mode 100644
diff --git a/docs/en/data_types/float.md b/docs/en/data_types/float.md
old mode 100755
new mode 100644
index 9d5cc2c01bb..c6d12999604
--- a/docs/en/data_types/float.md
+++ b/docs/en/data_types/float.md
@@ -4,8 +4,8 @@
 
 Types are equivalent to types of C:
 
-- `Float32` - `float`
-- `Float64`  - ` double`
+- `Float32` - `float`;
+- `Float64`  - ` double`.
 
 We recommend that you store data in integer form whenever possible. For example, convert fixed precision numbers to integer values, such as monetary amounts or page load times in milliseconds.
 
@@ -24,7 +24,9 @@ SELECT 1 - 0.9
 ```
 
 - The result of the calculation depends on the calculation method (the processor type and architecture of the computer system).
+
 - Floating-point calculations might result in numbers such as infinity (`Inf`) and "not-a-number" (`NaN`). This should be taken into account when processing the results of calculations.
+
 - When reading floating point numbers from rows, the result might not be the nearest machine-representable number.
 
 ## NaN and Inf
@@ -42,7 +44,6 @@ SELECT 0.5 / 0
 │            inf │
 └────────────────┘
 ```
-
 - `-Inf` – Negative infinity.
 
 ```sql
@@ -54,7 +55,6 @@ SELECT -0.5 / 0
 │            -inf │
 └─────────────────┘
 ```
-
 - `NaN` – Not a number.
 
 ```
@@ -67,5 +67,5 @@ SELECT 0 / 0
 └──────────────┘
 ```
 
-See the rules for ` NaN` sorting in the section [ORDER BY clause](../query_language/queries.md#query_language-queries-order_by).
+   See the rules for ` NaN` sorting in the section [ORDER BY clause](../query_language/queries.md#query_language-queries-order_by).
 
diff --git a/docs/en/data_types/index.md b/docs/en/data_types/index.md
old mode 100755
new mode 100644
index 4008a872161..c17b51c08a2
--- a/docs/en/data_types/index.md
+++ b/docs/en/data_types/index.md
@@ -2,7 +2,6 @@
 
 # Data types
 
-ClickHouse can store various types of data in table cells.
-
-This section describes the supported data types and special considerations when using and/or implementing them, if any.
+ClickHouse table fields can contain data of different types.
 
+The topic contains descriptions of data types supported and specificity of their usage of implementation if exists.
\ No newline at end of file
diff --git a/docs/en/data_types/int_uint.md b/docs/en/data_types/int_uint.md
old mode 100755
new mode 100644
diff --git a/docs/en/data_types/nested_data_structures/aggregatefunction.md b/docs/en/data_types/nested_data_structures/aggregatefunction.md
old mode 100755
new mode 100644
diff --git a/docs/en/data_types/nested_data_structures/index.md b/docs/en/data_types/nested_data_structures/index.md
old mode 100755
new mode 100644
diff --git a/docs/en/data_types/nested_data_structures/nested.md b/docs/en/data_types/nested_data_structures/nested.md
old mode 100755
new mode 100644
diff --git a/docs/en/data_types/special_data_types/expression.md b/docs/en/data_types/special_data_types/expression.md
old mode 100755
new mode 100644
diff --git a/docs/en/data_types/special_data_types/index.md b/docs/en/data_types/special_data_types/index.md
old mode 100755
new mode 100644
diff --git a/docs/en/data_types/special_data_types/set.md b/docs/en/data_types/special_data_types/set.md
old mode 100755
new mode 100644
diff --git a/docs/en/data_types/string.md b/docs/en/data_types/string.md
old mode 100755
new mode 100644
diff --git a/docs/en/data_types/tuple.md b/docs/en/data_types/tuple.md
old mode 100755
new mode 100644
diff --git a/docs/en/development/style.md b/docs/en/development/style.md
old mode 100755
new mode 100644
index 700fede5373..ef6490187c8
--- a/docs/en/development/style.md
+++ b/docs/en/development/style.md
@@ -4,7 +4,7 @@
 
 1. The following are recommendations, not requirements.
 2. If you are editing code, it makes sense to follow the formatting of the existing code.
-3. Code style is needed for consistency. Consistency makes it easier to read the code, and it also makes it easier to search the code.
+3. Code style is needed for consistency. Consistency makes it easier to read the code. and it also makes it easier to search the code.
 4. Many of the rules do not have logical reasons; they are dictated by established practices.
 
 ## Formatting
@@ -93,25 +93,25 @@
 14. In classes and structures, public, private, and protected are written on the same level as the class/struct, but all other internal elements should be deeper.
 
    ```cpp
-   template <typename T>
-class MultiVersion
-{
-public:
-    /// Version of object for usage. shared_ptr manage lifetime of version.
-    using Version = std::shared_ptr<const T>;
-    ...
-}
+    template <typename T, typename Ptr = std::shared_ptr<T>>
+    class MultiVersion
+    {
+    public:
+        /// The specific version of the object to use.
+        using Version = Ptr;
+        ...
+    }
    ```
 
 15. If the same namespace is used for the entire file, and there isn't anything else significant, an offset is not necessary  inside namespace.
 
 16. If the block for if, for, while... expressions consists of a single statement, you don't need to use curly brackets. Place the statement on a separate line, instead. The same is true for a nested if, for, while... statement. But if the inner statement contains curly brackets or else, the external block should be written in curly brackets.
 
-   ```cpp
-   /// Finish write.
-for (auto & stream : streams)
-    stream.second->finalize();
-   ```
+    ```cpp
+    /// Finish write.
+    for (auto & stream : streams)
+        stream.second->finalize();
+    ```
 
 17. There should be any spaces at the ends of lines.
 
@@ -218,11 +218,11 @@ for (auto & stream : streams)
    */
    void executeQuery(
        ReadBuffer & istr, /// Where to read the query from (and data for INSERT, if applicable)
-       WriteBuffer & ostr, /// Where to write the result
-       Context & context, /// DB, tables, data types, engines, functions, aggregate functions...
-       BlockInputStreamPtr & query_plan, /// A description of query processing can be included here
-       QueryProcessingStage::Enum stage = QueryProcessingStage::Complete /// The last stage to process the SELECT query to
-       )
+           WriteBuffer & ostr, /// Where to write the result
+           Context & context, /// DB, tables, data types, engines, functions, aggregate functions...
+           BlockInputStreamPtr & query_plan, /// A description of query processing can be included here
+           QueryProcessingStage::Enum stage = QueryProcessingStage::Complete /// The last stage to process the SELECT query to
+           )
    ```
 
 4. Comments should be written in English only.
@@ -252,7 +252,7 @@ for (auto & stream : streams)
     */
    ```
 
-   (the example is borrowed from the resource [http://home.tamk.fi/~jaalto/course/coding-style/doc/unmaintainable-code/](http://home.tamk.fi/~jaalto/course/coding-style/doc/unmaintainable-code/)
+   (Example taken from: [http://home.tamk.fi/~jaalto/course/coding-style/doc/unmaintainable-code/)](http://home.tamk.fi/~jaalto/course/coding-style/doc/unmaintainable-code/)
 
 7. Do not write garbage comments (author, creation date ..) at the beginning of each file.
 
@@ -497,15 +497,7 @@ This is not recommended, but it is allowed.
    You can create a separate code block inside a single function in order to make certain variables local, so that the destructors are called when exiting the block.
 
    ```cpp
-   Block block = data.in->read();
-
-   {
-       std::lock_guard<std::mutex> lock(mutex);
-       data.ready = true;
-       data.block = block;
-   }
-
-   ready_any.set();
+   Block block = data.in->read();{    std::lock_guard<std::mutex> lock(mutex);    data.ready = true;    data.block = block;}ready_any.set();
    ```
 
 7. Multithreading.
@@ -568,12 +560,13 @@ This is not recommended, but it is allowed.
 
    ```cpp
    using AggregateFunctionPtr = std::shared_ptr<IAggregateFunction>;
-   
-   /** Creates an aggregate function by name.  */
+
+   /** Creates an aggregate function by name.
+     */
    class AggregateFunctionFactory
    {
    public:
-       AggregateFunctionFactory();    
+       AggregateFunctionFactory();
        AggregateFunctionPtr get(const String & name, const DataTypes & argument_types) const;
    ```
 
@@ -598,10 +591,10 @@ This is not recommended, but it is allowed.
    If later you’ll need to delay initialization, you can add a default constructor that will create an invalid object. Or, for a small number of objects, you can use shared_ptr/unique_ptr.
 
    ```cpp
-   Loader(DB::Connection * connection_, const std::string & query, size_t max_block_size_);
-   
-   /// For delayed initialization
-   Loader() {}
+    Loader(DB::Connection * connection_, const std::string & query, size_t max_block_size_);
+
+    /// For delayed initialization
+    Loader() {}
    ```
 
 17. Virtual functions.
diff --git a/docs/en/dicts/external_dicts.md b/docs/en/dicts/external_dicts.md
old mode 100755
new mode 100644
index a6af84a313f..b99b02bbf57
--- a/docs/en/dicts/external_dicts.md
+++ b/docs/en/dicts/external_dicts.md
@@ -21,11 +21,12 @@ The dictionary config file has the following format:
 
     <!--Optional element. File name with substitutions-->
     <include_from>/etc/metrika.xml</include_from>
+
     
     <dictionary>
-        <!-- Dictionary configuration -->    
+        <!-- Dictionary configuration -->
     </dictionary>
-            
+
     ...
 
     <dictionary>
diff --git a/docs/en/dicts/external_dicts_dict.md b/docs/en/dicts/external_dicts_dict.md
old mode 100755
new mode 100644
index 6d2f4128704..4133b036e1f
--- a/docs/en/dicts/external_dicts_dict.md
+++ b/docs/en/dicts/external_dicts_dict.md
@@ -27,8 +27,7 @@ The dictionary configuration has the following structure:
 ```
 
 - name – The identifier that can be used to access the dictionary. Use the characters `[a-zA-Z0-9_\-]`.
-- [source](external_dicts_dict_sources.html/#dicts-external_dicts_dict_sources) — Source of the dictionary .
-- [layout](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout) — Dictionary layout in memory.
-- [source](external_dicts_dict_sources.html/#dicts-external_dicts_dict_sources) — Structure of the dictionary . A key and attributes that can be retrieved by this key.
-- [lifetime](external_dicts_dict_lifetime.md#dicts-external_dicts_dict_lifetime) — Frequency of dictionary updates.
-
+- [source](external_dicts_dict_sources.md#dicts-external_dicts_dict_sources) – Source of the dictionary.
+- [layout](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout) – Location of the dictionary in memory.
+- [structure](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure) – Structure of the dictionary. A key and attributes that can be retrieved by this key.
+- [lifetime](external_dicts_dict_lifetime.md#dicts-external_dicts_dict_lifetime) – How frequently to update dictionaries.
diff --git a/docs/en/dicts/external_dicts_dict_layout.md b/docs/en/dicts/external_dicts_dict_layout.md
old mode 100755
new mode 100644
index 8b7cad24b65..ad635db94f5
--- a/docs/en/dicts/external_dicts_dict_layout.md
+++ b/docs/en/dicts/external_dicts_dict_layout.md
@@ -2,11 +2,11 @@
 
 # Storing dictionaries in memory
 
-There are a [variety of ways](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-manner) to store dictionaries in memory.
+There are [many different ways](external_dicts_dict_layout#dicts-external_dicts_dict_layout-manner) to store dictionaries in memory.
 
-We recommend [flat](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-flat), [hashed](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-hashed)and[complex_key_hashed](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-complex_key_hashed). which provide optimal processing speed.
+We recommend [flat](external_dicts_dict_layout#dicts-external_dicts_dict_layout-flat), [hashed](external_dicts_dict_layout#dicts-external_dicts_dict_layout-hashed), and [complex_key_hashed](external_dicts_dict_layout#dicts-external_dicts_dict_layout-complex_key_hashed). which provide optimal processing speed.
 
-Caching is not recommended because of potentially poor performance and difficulties in selecting optimal parameters. Read more in the section " [cache](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-cache)".
+Caching is not recommended because of potentially poor performance and difficulties in selecting optimal parameters. Read more about this in the "[cache](external_dicts_dict_layout#dicts-external_dicts_dict_layout-cache)" section.
 
 There are several ways to improve dictionary performance:
 
@@ -46,6 +46,7 @@ The configuration looks like this:
 - [range_hashed](#dicts-external_dicts_dict_layout-range_hashed)
 - [complex_key_hashed](#dicts-external_dicts_dict_layout-complex_key_hashed)
 - [complex_key_cache](#dicts-external_dicts_dict_layout-complex_key_cache)
+- [ip_trie](#dicts-external_dicts_dict_layout-ip_trie)
 
 <a name="dicts-external_dicts_dict_layout-flat"></a>
 
@@ -87,7 +88,7 @@ Configuration example:
 
 ### complex_key_hashed
 
-This type is for use with composite [keys](external_dicts_dict_structure.md/#dicts-external_dicts_dict_structure). Similar to `hashed`.
+This type of storage is designed for use with compound [keys](external_dicts_dict_structure#dicts-external_dicts_dict_structure). It is similar to hashed.
 
 Configuration example:
 
@@ -108,18 +109,18 @@ This storage method works the same way as hashed and allows using date/time rang
 Example: The table contains discounts for each advertiser in the format:
 
 ```
-+---------------+---------------------+-------------------+--------+
-| advertiser id | discount start date | discount end date | amount |
-+===============+=====================+===================+========+
-| 123           | 2015-01-01          | 2015-01-15        | 0.15   |
-+---------------+---------------------+-------------------+--------+
-| 123           | 2015-01-16          | 2015-01-31        | 0.25   |
-+---------------+---------------------+-------------------+--------+
-| 456           | 2015-01-01          | 2015-01-15        | 0.05   |
-+---------------+---------------------+-------------------+--------+
+  +---------------+---------------------+-------------------+--------+
+  | advertiser id | discount start date | discount end date | amount |
+  +===============+=====================+===================+========+
+  | 123           | 2015-01-01          | 2015-01-15        | 0.15   |
+  +---------------+---------------------+-------------------+--------+
+  | 123           | 2015-01-16          | 2015-01-31        | 0.25   |
+  +---------------+---------------------+-------------------+--------+
+  | 456           | 2015-01-01          | 2015-01-15        | 0.05   |
+  +---------------+---------------------+-------------------+--------+
 ```
 
-To use a sample for date ranges, define the `range_min` and `range_max` elements in the [structure](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure).
+To use a sample for date ranges, define `range_min` and `range_max` in [structure](external_dicts_dict_structure#dicts-external_dicts_dict_structure).
 
 Example:
 
@@ -196,15 +197,15 @@ This is the least effective of all the ways to store dictionaries. The speed of
 
 To improve cache performance, use a subquery with ` LIMIT`, and call the function with the dictionary externally.
 
-Supported [sources](external_dicts_dict_sources.md#dicts-external_dicts_dict_sources): MySQL, ClickHouse, executable, HTTP.
+Supported [sources](external_dicts_dict_sources#dicts-external_dicts_dict_sources): MySQL, ClickHouse, executable, HTTP.
 
 Example of settings:
 
 ```xml
 <layout>
     <cache>
-        <!-- The size of the cache, in number of cells. Rounded up to a power of two. -->        
-        <size_in_cells>1000000000</size_in_cells>
+        <!-- The size of the cache, in number of cells. Rounded up to a power of two. -->
+               <size_in_cells>1000000000</size_in_cells>
     </cache>
 </layout>
 ```
@@ -226,4 +227,66 @@ Do not use ClickHouse as a source, because it is slow to process queries with ra
 
 ### complex_key_cache
 
-This type of storage is for use with composite [keys](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure). Similar to `cache`.
+This type of storage is designed for use with compound [keys](external_dicts_dict_structure#dicts-external_dicts_dict_structure). Similar to `cache`.
+
+<a name="dicts-external_dicts_dict_layout-ip_trie"></a>
+
+### ip_trie
+
+
+The table stores IP prefixes for each key (IP address), which makes it possible to map IP addresses to metadata such as ASN or threat score.
+
+Example: in the table there are prefixes matches to AS number and country:
+
+```
+  +-----------------+-------+--------+
+  | prefix          | asn   | cca2   |
+  +=================+=======+========+
+  | 202.79.32.0/20  | 17501 | NP     |
+  +-----------------+-------+--------+
+  | 2620:0:870::/48 | 3856  | US     |
+  +-----------------+-------+--------+
+  | 2a02:6b8:1::/48 | 13238 | RU     |
+  +-----------------+-------+--------+
+  | 2001:db8::/32   | 65536 | ZZ     |
+  +-----------------+-------+--------+
+```
+
+When using such a layout, the structure should have the "key" element.
+
+Example:
+
+```xml
+<structure>
+    <key>
+        <attribute>
+            <name>prefix</name>
+            <type>String</type>
+        </attribute>
+    </key>
+    <attribute>
+            <name>asn</name>
+            <type>UInt32</type>
+            <null_value />
+    </attribute>
+    <attribute>
+            <name>cca2</name>
+            <type>String</type>
+            <null_value>??</null_value>
+    </attribute>
+    ...
+```
+
+These key must have only one attribute of type String, containing a valid IP prefix. Other types are not yet supported.
+
+For querying, same functions (dictGetT with tuple) as for complex key dictionaries have to be used:
+
+    dictGetT('dict_name', 'attr_name', tuple(ip))
+
+The function accepts either UInt32 for IPv4 address or FixedString(16) for IPv6 address in wire format:
+
+    dictGetString('prefix', 'asn', tuple(IPv6StringToNum('2001:db8::1')))
+
+No other type is supported. The function returns attribute for a prefix matching the given IP address. If there are overlapping prefixes, the most specific one is returned.
+
+The data is stored currently in a bitwise trie, it has to fit in memory.
diff --git a/docs/en/dicts/external_dicts_dict_lifetime.md b/docs/en/dicts/external_dicts_dict_lifetime.md
old mode 100755
new mode 100644
index 52ee7a4aa78..6431fb3de48
--- a/docs/en/dicts/external_dicts_dict_lifetime.md
+++ b/docs/en/dicts/external_dicts_dict_lifetime.md
@@ -36,13 +36,13 @@ Example of settings:
 When upgrading the dictionaries, the ClickHouse server applies different logic depending on the type of [ source](external_dicts_dict_sources.md#dicts-external_dicts_dict_sources):
 
 > - For a text file, it checks the time of modification. If the time differs from the previously recorded time, the dictionary is updated.
-- For MyISAM tables, the time of modification is checked using a `SHOW TABLE STATUS` query.
-- Dictionaries from other sources are updated every time by default.
+> - For MyISAM tables, the time of modification is checked using a `SHOW TABLE STATUS` query.
+> - Dictionaries from other sources are updated every time by default.
 
 For MySQL (InnoDB) and ODBC sources, you can set up a query that will update the dictionaries only if they really changed, rather than each time. To do this, follow these steps:
 
 > - The dictionary table must have a field that always changes when the source data is updated.
-- The settings of the source must specify a query that retrieves the changing field. The ClickHouse server interprets the query result as a row, and if this row has changed relative to its previous state, the dictionary is updated. Specify the query in the `<invalidate_query>` field in the settings for the [source](external_dicts_dict_sources.md#dicts-external_dicts_dict_sources).
+> - The settings of the source must specify a query that retrieves the changing field. The ClickHouse server interprets the query result as a row, and if this row has changed relative to its previous state, the dictionary is updated. The query must be specified in the `<invalidate_query>` field in the [ source](external_dicts_dict_sources.md#dicts-external_dicts_dict_sources) settings.
 
 Example of settings:
 
diff --git a/docs/en/dicts/external_dicts_dict_sources.md b/docs/en/dicts/external_dicts_dict_sources.md
old mode 100755
new mode 100644
index 6cb4e0ea44d..721302cd556
--- a/docs/en/dicts/external_dicts_dict_sources.md
+++ b/docs/en/dicts/external_dicts_dict_sources.md
@@ -80,7 +80,7 @@ Setting fields:
 
 ## HTTP(s)
 
-Working with an HTTP(s) server depends on [how the dictionary is stored in memory](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout). If the dictionary is stored using `cache` and `complex_key_cache`, ClickHouse requests the necessary keys by sending a request via the `POST` method.
+Working with executable files depends on [how the dictionary is stored in memory](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout). If the dictionary is stored using `cache` and `complex_key_cache`, ClickHouse requests the necessary keys by sending a request via the `POST` method.
 
 Example of settings:
 
@@ -135,9 +135,9 @@ Installing unixODBC and the ODBC driver for PostgreSQL:
 Configuring `/etc/odbc.ini` (or `~/.odbc.ini`):
 
 ```
-[DEFAULT]
+    [DEFAULT]
     Driver = myconnection
-
+    
     [myconnection]
     Description         = PostgreSQL connection to my_db
     Driver              = PostgreSQL Unicode
diff --git a/docs/en/dicts/external_dicts_dict_structure.md b/docs/en/dicts/external_dicts_dict_structure.md
old mode 100755
new mode 100644
index 2542af00ec6..5a6d349b350
--- a/docs/en/dicts/external_dicts_dict_structure.md
+++ b/docs/en/dicts/external_dicts_dict_structure.md
@@ -25,8 +25,8 @@ Overall structure:
 
 Columns are described in the structure:
 
-- `<id>` - [key column](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure-key).
-- `<attribute>` - [data column](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure-attributes). There can be a large number of columns.
+- `<id>` – [Key column](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure-key).
+- `<attribute>` – [Data column](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure-attributes). There can be a large number of columns.
 
 <a name="dicts-external_dicts_dict_structure-key"></a>
 
@@ -63,10 +63,12 @@ Configuration fields:
 
 ### Composite key
 
-The key can be a `tuple` from any types of fields. The [layout](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout) in this case must be `complex_key_hashed` or `complex_key_cache`.
+The key can be a `tuple` from any types of fields. The [ layout](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout)  in this case must be `complex_key_hashed`  or `complex_key_cache`.
 
 <div class="admonition tip">
-A composite key can consist of a single element.
 This makes it possible to use a string as the key, for instance.
+
+A composite key can also consist of a single element, which makes it possible to use a string as the key, for instance.
+
 </div>
 
 The key structure is set in the element `<key>`. Key fields are specified in the same format as the dictionary [attributes](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure-attributes). Example:
@@ -117,6 +119,6 @@ Configuration fields:
 - `null_value` – The default value for a non-existing element. In the example, it is an empty string.
 - `expression` – The attribute can be an expression. The tag is not required.
 - `hierarchical` – Hierarchical support. Mirrored to the parent identifier. By default, ` false`.
-- `injective` – Whether the `id -> attribute` image is injective. If ` true`, then you can optimize the ` GROUP BY` clause. By default, `false`.
-- `is_object_id` – Whether the query is executed for a MongoDB document by `ObjectID`.
+- `injective` Whether the `id -> attribute` image is injective. If ` true`, then you can optimize the ` GROUP BY` clause. By default, `false`.
+- `is_object_id` - Used for query mongo documents by ObjectId
 
diff --git a/docs/en/dicts/index.md b/docs/en/dicts/index.md
old mode 100755
new mode 100644
diff --git a/docs/en/dicts/internal_dicts.md b/docs/en/dicts/internal_dicts.md
old mode 100755
new mode 100644
diff --git a/docs/en/formats/capnproto.md b/docs/en/formats/capnproto.md
old mode 100755
new mode 100644
index 918197b2bd9..0d482e20887
--- a/docs/en/formats/capnproto.md
+++ b/docs/en/formats/capnproto.md
@@ -1,26 +1,26 @@
-<a name="format_capnproto"></a>
-
-# CapnProto
-
-Cap'n Proto is a binary message format similar to Protocol Buffers and Thrift, but not like JSON or MessagePack.
-
-Cap'n Proto messages are strictly typed and not self-describing, meaning they need an external schema description. The schema is applied on the fly and cached for each query.
-
-```sql
-SELECT SearchPhrase, count() AS c FROM test.hits
-       GROUP BY SearchPhrase FORMAT CapnProto SETTINGS schema = 'schema:Message'
-```
-
-Where `schema.capnp` looks like this:
-
-```
-struct Message {
-  SearchPhrase @0 :Text;
-  c @1 :Uint64;
-}
-```
-
-Schema files are in the file that is located in the directory specified in [ format_schema_path](../operations/server_settings/settings.md#server_settings-format_schema_path) in the server configuration.
-
-Deserialization is effective and usually doesn't increase the system load.
-
+<a name="format_capnproto"></a>
+
+# CapnProto
+
+Cap'n Proto is a binary message format similar to Protocol Buffers and Thrift, but not like JSON or MessagePack.
+
+Cap'n Proto messages are strictly typed and not self-describing, meaning they need an external schema description. The schema is applied on the fly and cached for each query.
+
+```sql
+SELECT SearchPhrase, count() AS c FROM test.hits
+       GROUP BY SearchPhrase FORMAT CapnProto SETTINGS schema = 'schema:Message'
+```
+
+Where `schema.capnp` looks like this:
+
+```
+struct Message {
+  SearchPhrase @0 :Text;
+  c @1 :Uint64;
+}
+```
+
+Schema files are in the file that is located in the directory specified in [ format_schema_path](../operations/server_settings/settings.md#server_settings-format_schema_path) in the server configuration.
+
+Deserialization is effective and usually doesn't increase the system load.
+
diff --git a/docs/en/formats/csv.md b/docs/en/formats/csv.md
old mode 100755
new mode 100644
diff --git a/docs/en/formats/csvwithnames.md b/docs/en/formats/csvwithnames.md
old mode 100755
new mode 100644
diff --git a/docs/en/formats/index.md b/docs/en/formats/index.md
old mode 100755
new mode 100644
index 112a13ff5e5..815a2d060cb
--- a/docs/en/formats/index.md
+++ b/docs/en/formats/index.md
@@ -3,4 +3,3 @@
 # Formats
 
 The format determines how data is returned to you after SELECTs (how it is written and formatted by the server), and how it is accepted for INSERTs (how it is read and parsed by the server).
-
diff --git a/docs/en/formats/json.md b/docs/en/formats/json.md
old mode 100755
new mode 100644
diff --git a/docs/en/formats/jsoncompact.md b/docs/en/formats/jsoncompact.md
old mode 100755
new mode 100644
index d870b6dff08..e4ce0867bc2
--- a/docs/en/formats/jsoncompact.md
+++ b/docs/en/formats/jsoncompact.md
@@ -24,7 +24,7 @@ Example:
                 ["bathroom interior design", "2166"],
                 ["yandex", "1655"],
                 ["spring 2014 fashion", "1549"],
-                ["freeform photo", "1480"]
+                ["freeform photos", "1480"]
         ],
 
         "totals": ["","8873898"],
diff --git a/docs/en/formats/jsoneachrow.md b/docs/en/formats/jsoneachrow.md
old mode 100755
new mode 100644
diff --git a/docs/en/formats/native.md b/docs/en/formats/native.md
old mode 100755
new mode 100644
diff --git a/docs/en/formats/null.md b/docs/en/formats/null.md
old mode 100755
new mode 100644
diff --git a/docs/en/formats/pretty.md b/docs/en/formats/pretty.md
old mode 100755
new mode 100644
diff --git a/docs/en/formats/prettycompact.md b/docs/en/formats/prettycompact.md
old mode 100755
new mode 100644
diff --git a/docs/en/formats/prettycompactmonoblock.md b/docs/en/formats/prettycompactmonoblock.md
old mode 100755
new mode 100644
diff --git a/docs/en/formats/prettynoescapes.md b/docs/en/formats/prettynoescapes.md
old mode 100755
new mode 100644
diff --git a/docs/en/formats/prettyspace.md b/docs/en/formats/prettyspace.md
old mode 100755
new mode 100644
diff --git a/docs/en/formats/rowbinary.md b/docs/en/formats/rowbinary.md
old mode 100755
new mode 100644
index aeb3df4c8a8..bc8479332ba
--- a/docs/en/formats/rowbinary.md
+++ b/docs/en/formats/rowbinary.md
@@ -9,5 +9,5 @@ Date is represented as a UInt16 object that contains the number of days since 19
 String is represented as a varint length (unsigned [LEB128](https://en.wikipedia.org/wiki/LEB128)), followed by the bytes of the string.
 FixedString is represented simply as a sequence of bytes.
 
-Array is represented as a varint length (unsigned [LEB128](https://en.wikipedia.org/wiki/LEB128)), followed by successive elements of the array.
+Arrays are represented as a varint length (unsigned [LEB128](https://en.wikipedia.org/wiki/LEB128)), followed by the array elements in order.
 
diff --git a/docs/en/formats/tabseparated.md b/docs/en/formats/tabseparated.md
old mode 100755
new mode 100644
diff --git a/docs/en/formats/tabseparatedraw.md b/docs/en/formats/tabseparatedraw.md
old mode 100755
new mode 100644
diff --git a/docs/en/formats/tabseparatedwithnames.md b/docs/en/formats/tabseparatedwithnames.md
old mode 100755
new mode 100644
diff --git a/docs/en/formats/tabseparatedwithnamesandtypes.md b/docs/en/formats/tabseparatedwithnamesandtypes.md
old mode 100755
new mode 100644
diff --git a/docs/en/formats/tskv.md b/docs/en/formats/tskv.md
old mode 100755
new mode 100644
diff --git a/docs/en/formats/values.md b/docs/en/formats/values.md
old mode 100755
new mode 100644
index a672723f33d..2e929369848
--- a/docs/en/formats/values.md
+++ b/docs/en/formats/values.md
@@ -4,5 +4,5 @@ Prints every row in brackets. Rows are separated by commas. There is no comma af
 
 The minimum set of characters that you need to escape when passing data in Values ​​format: single quotes and backslashes.
 
-This is the format that is used in `INSERT INTO t VALUES ...`, but you can also use it for formatting query results.
+This is the format that is used in `INSERT INTO t VALUES ...` but you can also use it for query result.
 
diff --git a/docs/en/formats/vertical.md b/docs/en/formats/vertical.md
old mode 100755
new mode 100644
diff --git a/docs/en/formats/xml.md b/docs/en/formats/xml.md
old mode 100755
new mode 100644
index 0da55875cc3..f91adec9356
--- a/docs/en/formats/xml.md
+++ b/docs/en/formats/xml.md
@@ -35,7 +35,7 @@ XML format is suitable only for output, not for parsing. Example:
                         <field>1549</field>
                 </row>
                 <row>
-                        <SearchPhrase>freeform photo</SearchPhrase>
+                        <SearchPhrase>freeform photos</SearchPhrase>
                         <field>1480</field>
                 </row>
                 <row>
diff --git a/docs/en/functions/arithmetic_functions.md b/docs/en/functions/arithmetic_functions.md
old mode 100755
new mode 100644
diff --git a/docs/en/functions/array_functions.md b/docs/en/functions/array_functions.md
old mode 100755
new mode 100644
index 232f6a20427..493a465ac82
--- a/docs/en/functions/array_functions.md
+++ b/docs/en/functions/array_functions.md
@@ -39,7 +39,7 @@ Accepts an empty array and returns a one-element array that is equal to the defa
 Returns an array of numbers from 0 to N-1.
 Just in case, an exception is thrown if arrays with a total length of more than 100,000,000 elements are created in a data block.
 
-## array(x1, ...), оператор \[x1, ...\]
+## array(x1, ...), operator \[x1, ...\]
 
 Creates an array from the function arguments.
 The arguments must be constants and have types that have the smallest common type. At least one argument must be passed, because otherwise it isn't clear which type of array to create. That is, you can't use this function to create an empty array (to do that, use the 'emptyArray\*' function described above).
@@ -62,6 +62,7 @@ arrayConcat(arrays)
 ```sql
 SELECT arrayConcat([1, 2], [3, 4], [5, 6]) AS res
 ```
+
 ```
 ┌─res───────────┐
 │ [1,2,3,4,5,6] │
@@ -202,6 +203,7 @@ arrayPopBack(array)
 ```sql
 SELECT arrayPopBack([1, 2, 3]) AS res
 ```
+
 ```
 ┌─res───┐
 │ [1,2] │
@@ -243,13 +245,14 @@ arrayPushBack(array, single_value)
 **Arguments**
 
 - `array` – Array.
-- `single_value` – A single value. Only numbers can be added to an array with numbers, and only strings can be added to an array of strings. When adding numbers, ClickHouse automatically sets the `single_value` type for the data type of the array. For more information about ClickHouse data types, read the section "[Data types](../data_types/index.md#data_types)".
+- `single_value` – A single value. Only numbers can be added to an array with numbers, and only strings can be added to an array of strings. When adding numbers, ClickHouse automatically sets the `single_value` type for the data type of the array. For more information about the types of data in ClickHouse, see "[Data types](../data_types/index.md#data_types)".
 
 **Example**
 
 ```sql
 SELECT arrayPushBack(['a'], 'b') AS res
 ```
+
 ```
 ┌─res───────┐
 │ ['a','b'] │
@@ -267,7 +270,7 @@ arrayPushFront(array, single_value)
 **Arguments**
 
 - `array` – Array.
-- `single_value` – A single value.  Only numbers can be added to an array with numbers, and only strings can be added to an array of strings. When adding numbers, ClickHouse automatically sets the `single_value` type for the data type of the array.  For more information about ClickHouse data types, read the section "[Data types](../data_types/index.md#data_types)".
+- `single_value` – A single value.  Only numbers can be added to an array with numbers, and only strings can be added to an array of strings. When adding numbers, ClickHouse automatically sets the `single_value` type for the data type of the array.  For more information about the types of data in ClickHouse, see "[Data types](../data_types/index.md#data_types)".
 
 **Example**
 
@@ -292,7 +295,7 @@ arraySlice(array, offset[, length])
 **Arguments**
 
 - `array` –  Array of data.
-- `offset` – Indent from the edge of the array. A positive value indicates an offset on the left, and a negative value is an indent on the right. Numbering of the array items begins with 1.
+- `offset` – Offset from the edge of the array. A positive value indicates an offset on the left, and a negative value is an indent on the right. Numbering of the array items begins with 1.
 - `length` - The length of the required slice. If you specify a negative value, the function returns an open slice `[offset, array_length - length)`. If you omit the value, the function returns the slice `[offset, the_end_of_array]`.
 
 **Example**
@@ -300,6 +303,7 @@ arraySlice(array, offset[, length])
 ```sql
 SELECT arraySlice([1, 2, 3, 4, 5], 2, 3) AS res
 ```
+
 ```
 ┌─res─────┐
 │ [2,3,4] │
diff --git a/docs/en/functions/array_join.md b/docs/en/functions/array_join.md
old mode 100755
new mode 100644
index f94b2707f52..6e18f8203c0
--- a/docs/en/functions/array_join.md
+++ b/docs/en/functions/array_join.md
@@ -28,3 +28,4 @@ SELECT arrayJoin([1, 2, 3] AS src) AS dst, 'Hello', src
 │   3 │ Hello     │ [1,2,3] │
 └─────┴───────────┴─────────┘
 ```
+
diff --git a/docs/en/functions/bit_functions.md b/docs/en/functions/bit_functions.md
old mode 100755
new mode 100644
index c5a032aa5d6..523413f200a
--- a/docs/en/functions/bit_functions.md
+++ b/docs/en/functions/bit_functions.md
@@ -15,3 +15,4 @@ The result type is an integer with bits equal to the maximum bits of its argumen
 ## bitShiftLeft(a, b)
 
 ## bitShiftRight(a, b)
+
diff --git a/docs/en/functions/comparison_functions.md b/docs/en/functions/comparison_functions.md
old mode 100755
new mode 100644
index 9b95966ba84..e37642d42ed
--- a/docs/en/functions/comparison_functions.md
+++ b/docs/en/functions/comparison_functions.md
@@ -15,7 +15,7 @@ For example, you can't compare a date with a string. You have to use a function
 
 Strings are compared by bytes. A shorter string is smaller than all strings that start with it and that contain at least one more character.
 
-Note. Up until version 1.1.54134, signed and unsigned numbers were compared the same way as in C++. In other words, you could get an incorrect result in cases like SELECT 9223372036854775807 &gt; -1. This behavior changed in version 1.1.54134 and is now mathematically correct.
+Note: Up until version 1.1.54134, signed and unsigned numbers were compared the same way as in C++. In other words, you could get an incorrect result in cases like SELECT 9223372036854775807 &gt; -1. This behavior changed in version 1.1.54134 and is now mathematically correct.
 
 ## equals, a = b and a == b operator
 
diff --git a/docs/en/functions/conditional_functions.md b/docs/en/functions/conditional_functions.md
old mode 100755
new mode 100644
diff --git a/docs/en/functions/date_time_functions.md b/docs/en/functions/date_time_functions.md
old mode 100755
new mode 100644
index a7529e5f0e1..da6ad9b4a7c
--- a/docs/en/functions/date_time_functions.md
+++ b/docs/en/functions/date_time_functions.md
@@ -79,10 +79,6 @@ Rounds down a date with time to the start of the minute.
 
 Rounds down a date with time to the start of the hour.
 
-## toStartOfFifteenMinutes
-
-Rounds down the date with time to the start of the fifteen-minute interval.
-
 Note: If you need to round a date with time to any other number of seconds, minutes, or hours, you can convert it into a number by using the toUInt32 function, then round the number using intDiv and multiplication, and convert it back using the toDateTime function.
 
 ## toStartOfHour
diff --git a/docs/en/functions/encoding_functions.md b/docs/en/functions/encoding_functions.md
old mode 100755
new mode 100644
diff --git a/docs/en/functions/ext_dict_functions.md b/docs/en/functions/ext_dict_functions.md
old mode 100755
new mode 100644
index 002e2f55845..949e805d9ab
--- a/docs/en/functions/ext_dict_functions.md
+++ b/docs/en/functions/ext_dict_functions.md
@@ -18,20 +18,18 @@ For information on connecting and configuring external dictionaries, see "[Exter
 
 `dictGetT('dict_name', 'attr_name', id)`
 
-- Get the value of the attr_name attribute  from the dict_name dictionary using the 'id' key.
-`dict_name`  and `attr_name`  are constant strings.
-`id`must be UInt64.
+- Get the value of the attr_name attribute  from the dict_name dictionary using the 'id' key.`dict_name`  and `attr_name`  are constant strings.`id`must be UInt64.
 If there is no `id` key in the dictionary, it returns the default value specified in the dictionary description.
 
 ## dictGetTOrDefault
 
 `dictGetT('dict_name', 'attr_name', id, default)`
 
-The same as the `dictGetT` functions, but the default value is taken from the function's last argument.
+Similar to the functions dictGetT, but the default value is taken from the last argument of the function.
 
 ## dictIsIn
 
-`dictIsIn('dict_name', child_id, ancestor_id)`
+`dictIsIn ('dict_name', child_id, ancestor_id)`
 
 - For the 'dict_name' hierarchical dictionary, finds out whether the 'child_id' key is located inside 'ancestor_id' (or matches 'ancestor_id'). Returns UInt8.
 
diff --git a/docs/en/functions/hash_functions.md b/docs/en/functions/hash_functions.md
old mode 100755
new mode 100644
diff --git a/docs/en/functions/higher_order_functions.md b/docs/en/functions/higher_order_functions.md
old mode 100755
new mode 100644
index 15cc40dead1..ab9bdc50786
--- a/docs/en/functions/higher_order_functions.md
+++ b/docs/en/functions/higher_order_functions.md
@@ -73,7 +73,7 @@ Returns the index of the first element in the 'arr1' array for which 'func' retu
 
 ### arrayCumSum(\[func,\] arr1, ...)
 
-Returns an array of partial sums of elements in the source array (a running sum). If the `func` function is specified, then the values of the array elements are converted by this function before summing.
+Returns the cumulative sum of the array obtained from the original application of the 'func' function to each element in the 'arr' array.
 
 Example:
 
@@ -86,3 +86,4 @@ SELECT arrayCumSum([1, 1, 1, 1]) AS res
 │ [1, 2, 3, 4] │
 └──────────────┘
 ```
+
diff --git a/docs/en/functions/in_functions.md b/docs/en/functions/in_functions.md
old mode 100755
new mode 100644
diff --git a/docs/en/functions/index.md b/docs/en/functions/index.md
old mode 100755
new mode 100644
diff --git a/docs/en/functions/ip_address_functions.md b/docs/en/functions/ip_address_functions.md
old mode 100755
new mode 100644
diff --git a/docs/en/functions/json_functions.md b/docs/en/functions/json_functions.md
old mode 100755
new mode 100644
index 90a2ddc47dd..1bf10e9cf0c
--- a/docs/en/functions/json_functions.md
+++ b/docs/en/functions/json_functions.md
@@ -1,11 +1,11 @@
-# Functions for working with JSON
+# Functions for working with JSON.
 
 In Yandex.Metrica, JSON is transmitted by users as session parameters. There are some special functions for working with this JSON. (Although in most of the cases, the JSONs are additionally pre-processed, and the resulting values are put in separate columns in their processed format.) All these functions are based on strong assumptions about what the JSON can be, but they try to do as little as possible to get the job done.
 
 The following assumptions are made:
 
 1. The field name (function argument) must be a constant.
-2. The field name is somehow canonically encoded in JSON. For example: `visitParamHas('{"abc":"def"}', 'abc') = 1`, но `visitParamHas('{"\\u0061\\u0062\\u0063":"def"}', 'abc') = 0`
+2. The field name is somehow canonically encoded in JSON. For example: `visitParamHas('{"abc":"def"}', 'abc') = 1`, but `visitParamHas('{"\\u0061\\u0062\\u0063":"def"}', 'abc') = 0`
 3. Fields are searched for on any nesting level, indiscriminately. If there are multiple matching fields, the first occurrence is used.
 4. The JSON doesn't have space characters outside of string literals.
 
@@ -47,10 +47,7 @@ Parses the string in double quotes. The value is unescaped. If unescaping failed
 Examples:
 
 ```text
-visitParamExtractString('{"abc":"\\n\\u0000"}', 'abc') = '\n\0'
-visitParamExtractString('{"abc":"\\u263a"}', 'abc') = '☺'
-visitParamExtractString('{"abc":"\\u263"}', 'abc') = ''
-visitParamExtractString('{"abc":"hello}', 'abc') = ''
+visitParamExtractString('{"abc":"\\n\\u0000"}', 'abc') = '\n\0'visitParamExtractString('{"abc":"\\u263a"}', 'abc') = '☺'visitParamExtractString('{"abc":"\\u263"}', 'abc') = ''visitParamExtractString('{"abc":"hello}', 'abc') = ''
 ```
 
 There is currently no support for code points in the format `\uXXXX\uYYYY` that are not from the basic multilingual plane (they are converted to CESU-8 instead of UTF-8).
diff --git a/docs/en/functions/logical_functions.md b/docs/en/functions/logical_functions.md
old mode 100755
new mode 100644
index d396640a49d..4ef0fe5fd32
--- a/docs/en/functions/logical_functions.md
+++ b/docs/en/functions/logical_functions.md
@@ -11,3 +11,4 @@ Zero as an argument is considered "false," while any non-zero value is considere
 ## not, NOT operator
 
 ## xor
+
diff --git a/docs/en/functions/math_functions.md b/docs/en/functions/math_functions.md
old mode 100755
new mode 100644
index 42e3f3e8018..d606c87a509
--- a/docs/en/functions/math_functions.md
+++ b/docs/en/functions/math_functions.md
@@ -97,3 +97,4 @@ The arc tangent.
 ## pow(x, y)
 
 xy.
+
diff --git a/docs/en/functions/other_functions.md b/docs/en/functions/other_functions.md
old mode 100755
new mode 100644
index 8a0063750fe..befd94ecd4e
--- a/docs/en/functions/other_functions.md
+++ b/docs/en/functions/other_functions.md
@@ -59,8 +59,7 @@ For elements in a nested data structure, the function checks for the existence o
 
 Allows building a unicode-art diagram.
 
-`bar (x, min, max, width)` – Draws a band with a width proportional to (x - min) and equal to 'width' characters when x == max.
-`min, max` – Integer constants. The value must fit in Int64.`width` – Constant, positive number, may be a fraction.
+`bar (x, min, max, width)` – Draws a band with a width proportional to (x - min) and equal to 'width' characters when x == max.`min, max` – Integer constants. The value must fit in Int64.`width` – Constant, positive number, may be a fraction.
 
 The band is drawn with accuracy to one eighth of a symbol.
 
@@ -138,7 +137,7 @@ Example:
 
 ```sql
 SELECT
-    transform(SearchEngineID, [2, 3], ['Yandex', 'Google'], 'Other') AS title,
+    transform(SearchEngineID, [2, 3], ['Yandex', 'Google'], 'Other' AS title,
     count() AS c
 FROM test.hits
 WHERE SearchEngineID != 0
diff --git a/docs/en/functions/random_functions.md b/docs/en/functions/random_functions.md
old mode 100755
new mode 100644
diff --git a/docs/en/functions/rounding_functions.md b/docs/en/functions/rounding_functions.md
old mode 100755
new mode 100644
diff --git a/docs/en/functions/splitting_merging_functions.md b/docs/en/functions/splitting_merging_functions.md
old mode 100755
new mode 100644
diff --git a/docs/en/functions/string_functions.md b/docs/en/functions/string_functions.md
old mode 100755
new mode 100644
diff --git a/docs/en/functions/string_replace_functions.md b/docs/en/functions/string_replace_functions.md
old mode 100755
new mode 100644
index d70d8f404de..d3773504278
--- a/docs/en/functions/string_replace_functions.md
+++ b/docs/en/functions/string_replace_functions.md
@@ -76,3 +76,4 @@ SELECT replaceRegexpAll('Hello, World!', '^', 'here: ') AS res
 │ here: Hello, World! │
 └─────────────────────┘
 ```
+
diff --git a/docs/en/functions/string_search_functions.md b/docs/en/functions/string_search_functions.md
old mode 100755
new mode 100644
diff --git a/docs/en/functions/type_conversion_functions.md b/docs/en/functions/type_conversion_functions.md
old mode 100755
new mode 100644
diff --git a/docs/en/functions/url_functions.md b/docs/en/functions/url_functions.md
old mode 100755
new mode 100644
diff --git a/docs/en/functions/ym_dict_functions.md b/docs/en/functions/ym_dict_functions.md
old mode 100755
new mode 100644
index 7ba7e7012cf..540b5dd601a
--- a/docs/en/functions/ym_dict_functions.md
+++ b/docs/en/functions/ym_dict_functions.md
@@ -21,9 +21,7 @@ All functions for working with regions have an optional argument at the end –
 Example:
 
 ```text
-regionToCountry(RegionID) – Uses the default dictionary: /opt/geo/regions_hierarchy.txt
-regionToCountry(RegionID, '') – Uses the default dictionary: /opt/geo/regions_hierarchy.txt
-regionToCountry(RegionID, 'ua') – Uses the dictionary for the 'ua' key: /opt/geo/regions_hierarchy_ua.txt
+regionToCountry(RegionID) – Uses the default dictionary: /opt/geo/regions_hierarchy.txtregionToCountry(RegionID, '') – Uses the default dictionary: /opt/geo/regions_hierarchy.txtregionToCountry(RegionID, 'ua') – Uses the dictionary for the 'ua' key: /opt/geo/regions_hierarchy_ua.txt
 ```
 
 ### regionToCity(id[, geobase])
@@ -35,9 +33,7 @@ Accepts a UInt32 number – the region ID from the Yandex geobase. If this regio
 Converts a region to an area (type 5 in the geobase). In every other way, this function is the same as 'regionToCity'.
 
 ```sql
-SELECT DISTINCT regionToName(regionToArea(toUInt32(number), 'ua'))
-FROM system.numbers
-LIMIT 15
+SELECT DISTINCT regionToName(regionToArea(toUInt32(number), 'ua'))FROM system.numbersLIMIT 15
 ```
 
 ```text
@@ -65,9 +61,7 @@ LIMIT 15
 Converts a region to a federal district (type 4 in the geobase). In every other way, this function is the same as 'regionToCity'.
 
 ```sql
-SELECT DISTINCT regionToName(regionToDistrict(toUInt32(number), 'ua'))
-FROM system.numbers
-LIMIT 15
+SELECT DISTINCT regionToName(regionToDistrict(toUInt32(number), 'ua'))FROM system.numbersLIMIT 15
 ```
 
 ```text
diff --git a/docs/en/getting_started/example_datasets/amplab_benchmark.md b/docs/en/getting_started/example_datasets/amplab_benchmark.md
old mode 100755
new mode 100644
diff --git a/docs/en/getting_started/example_datasets/criteo.md b/docs/en/getting_started/example_datasets/criteo.md
old mode 100755
new mode 100644
index 3c60a68f430..9b59d6e5f3d
--- a/docs/en/getting_started/example_datasets/criteo.md
+++ b/docs/en/getting_started/example_datasets/criteo.md
@@ -66,8 +66,6 @@ CREATE TABLE criteo
 Transform data from the raw log and put it in the second table:
 
 ```sql
-INSERT INTO criteo SELECT date, clicked, int1, int2, int3, int4, int5, int6, int7, int8, int9, int10, int11, int12, int13, reinterpretAsUInt32(unhex(cat1)) AS icat1, reinterpretAsUInt32(unhex(cat2)) AS icat2, reinterpretAsUInt32(unhex(cat3)) AS icat3, reinterpretAsUInt32(unhex(cat4)) AS icat4, reinterpretAsUInt32(unhex(cat5)) AS icat5, reinterpretAsUInt32(unhex(cat6)) AS icat6, reinterpretAsUInt32(unhex(cat7)) AS icat7, reinterpretAsUInt32(unhex(cat8)) AS icat8, reinterpretAsUInt32(unhex(cat9)) AS icat9, reinterpretAsUInt32(unhex(cat10)) AS icat10, reinterpretAsUInt32(unhex(cat11)) AS icat11, reinterpretAsUInt32(unhex(cat12)) AS icat12, reinterpretAsUInt32(unhex(cat13)) AS icat13, reinterpretAsUInt32(unhex(cat14)) AS icat14, reinterpretAsUInt32(unhex(cat15)) AS icat15, reinterpretAsUInt32(unhex(cat16)) AS icat16, reinterpretAsUInt32(unhex(cat17)) AS icat17, reinterpretAsUInt32(unhex(cat18)) AS icat18, reinterpretAsUInt32(unhex(cat19)) AS icat19, reinterpretAsUInt32(unhex(cat20)) AS icat20, reinterpretAsUInt32(unhex(cat21)) AS icat21, reinterpretAsUInt32(unhex(cat22)) AS icat22, reinterpretAsUInt32(unhex(cat23)) AS icat23, reinterpretAsUInt32(unhex(cat24)) AS icat24, reinterpretAsUInt32(unhex(cat25)) AS icat25, reinterpretAsUInt32(unhex(cat26)) AS icat26 FROM criteo_log;
-
-DROP TABLE criteo_log;
+INSERT INTO criteo SELECT date, clicked, int1, int2, int3, int4, int5, int6, int7, int8, int9, int10, int11, int12, int13, reinterpretAsUInt32(unhex(cat1)) AS icat1, reinterpretAsUInt32(unhex(cat2)) AS icat2, reinterpretAsUInt32(unhex(cat3)) AS icat3, reinterpretAsUInt32(unhex(cat4)) AS icat4, reinterpretAsUInt32(unhex(cat5)) AS icat5, reinterpretAsUInt32(unhex(cat6)) AS icat6, reinterpretAsUInt32(unhex(cat7)) AS icat7, reinterpretAsUInt32(unhex(cat8)) AS icat8, reinterpretAsUInt32(unhex(cat9)) AS icat9, reinterpretAsUInt32(unhex(cat10)) AS icat10, reinterpretAsUInt32(unhex(cat11)) AS icat11, reinterpretAsUInt32(unhex(cat12)) AS icat12, reinterpretAsUInt32(unhex(cat13)) AS icat13, reinterpretAsUInt32(unhex(cat14)) AS icat14, reinterpretAsUInt32(unhex(cat15)) AS icat15, reinterpretAsUInt32(unhex(cat16)) AS icat16, reinterpretAsUInt32(unhex(cat17)) AS icat17, reinterpretAsUInt32(unhex(cat18)) AS icat18, reinterpretAsUInt32(unhex(cat19)) AS icat19, reinterpretAsUInt32(unhex(cat20)) AS icat20, reinterpretAsUInt32(unhex(cat21)) AS icat21, reinterpretAsUInt32(unhex(cat22)) AS icat22, reinterpretAsUInt32(unhex(cat23)) AS icat23, reinterpretAsUInt32(unhex(cat24)) AS icat24, reinterpretAsUInt32(unhex(cat25)) AS icat25, reinterpretAsUInt32(unhex(cat26)) AS icat26 FROM criteo_log;DROP TABLE criteo_log;
 ```
 
diff --git a/docs/en/getting_started/example_datasets/nyc_taxi.md b/docs/en/getting_started/example_datasets/nyc_taxi.md
old mode 100755
new mode 100644
index a9f04f595d1..11ed81d1a43
--- a/docs/en/getting_started/example_datasets/nyc_taxi.md
+++ b/docs/en/getting_started/example_datasets/nyc_taxi.md
@@ -1,8 +1,8 @@
-# New York Taxi data
+# Data about New York taxis
 
-## How to import the raw data
+## How to import raw data
 
-See <https://github.com/toddwschneider/nyc-taxi-data> and <http://tech.marksblogg.com/billion-nyc-taxi-rides-redshift.html> for the description of the dataset and instructions for downloading.
+See <https://github.com/toddwschneider/nyc-taxi-data> and <http://tech.marksblogg.com/billion-nyc-taxi-rides-redshift.html> for description of the dataset and loading instructions.
 
 Downloading will result in about 227 GB of uncompressed data in CSV files. The download takes about an hour over a 1 Gbit connection (parallel downloading from s3.amazonaws.com recovers at least half of a 1 Gbit channel).
 Some of the files might not download fully. Check the file sizes and re-download any that seem doubtful.
@@ -301,14 +301,19 @@ SELECT passenger_count, toYear(pickup_date) AS year, count(*) FROM trips_mergetr
 Q4:
 
 ```sql
-SELECT passenger_count, toYear(pickup_date) AS year, round(trip_distance) AS distance, count(*)FROM trips_mergetreeGROUP BY passenger_count, year, distanceORDER BY year, count(*) DESC
+SELECT passenger_count, toYear(pickup_date) AS year, round(trip_distance) AS distance, count(*)
+FROM trips_mergetree
+GROUP BY passenger_count, year, distance
+ORDER BY year, count(*) DESC
 ```
 
 3.593 seconds.
 
 The following server was used:
 
-Two Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz, 16 physical kernels total,128 GiB RAM,8x6 TB HD on hardware RAID-5
+Two Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz, 16 physical kernels total,
+128 GiB RAM,
+8x6 TB HD on hardware RAID-5
 
 Execution time is the best of three runsBut starting from the second run, queries read data from the file system cache. No further caching occurs: the data is read out and processed in each run.
 
diff --git a/docs/en/getting_started/example_datasets/ontime.md b/docs/en/getting_started/example_datasets/ontime.md
old mode 100755
new mode 100644
diff --git a/docs/en/getting_started/example_datasets/star_schema.md b/docs/en/getting_started/example_datasets/star_schema.md
old mode 100755
new mode 100644
index 664ba59f48c..8807de3e670
--- a/docs/en/getting_started/example_datasets/star_schema.md
+++ b/docs/en/getting_started/example_datasets/star_schema.md
@@ -1,4 +1,4 @@
-# Star Schema Benchmark
+# Star scheme
 
 Compiling dbgen: <https://github.com/vadimtk/ssb-dbgen>
 
@@ -82,4 +82,3 @@ Downloading data (change 'customer' to 'customerd' in the distributed version):
 cat customer.tbl | sed 's/$/2000-01-01/' | clickhouse-client --query "INSERT INTO customer FORMAT CSV"
 cat lineorder.tbl | clickhouse-client --query "INSERT INTO lineorder FORMAT CSV"
 ```
-
diff --git a/docs/en/getting_started/example_datasets/wikistat.md b/docs/en/getting_started/example_datasets/wikistat.md
old mode 100755
new mode 100644
index fee0a56b52c..6cbc3b15561
--- a/docs/en/getting_started/example_datasets/wikistat.md
+++ b/docs/en/getting_started/example_datasets/wikistat.md
@@ -20,7 +20,7 @@ CREATE TABLE wikistat
 Loading data:
 
 ```bash
-for i in {2007..2016}; do for j in {01..12}; do echo $i-$j >&2; curl -sSL "http://dumps.wikimedia.org/other/pagecounts-raw/$i/$i-$j/" | grep -oE 'pagecounts-[0-9]+-[0-9]+\.gz'; done; done | sort | uniq | tee links.txt
+for i in {2007..2016}; do for j in {01..12}; do echo $i-$j >&2; curl -sS "http://dumps.wikimedia.org/other/pagecounts-raw/$i/$i-$j/" | grep -oE 'pagecounts-[0-9]+-[0-9]+\.gz'; done; done | sort | uniq | tee links.txt
 cat links.txt | while read link; do wget http://dumps.wikimedia.org/other/pagecounts-raw/$(echo $link | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})[0-9]{2}-[0-9]+\.gz/\1/')/$(echo $link | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})[0-9]{2}-[0-9]+\.gz/\1-\2/')/$link; done
 ls -1 /opt/wikistat/ | grep gz | while read i; do echo $i; gzip -cd /opt/wikistat/$i | ./wikistat-loader --time="$(echo -n $i | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})([0-9]{2})-([0-9]{2})([0-9]{2})([0-9]{2})\.gz/\1-\2-\3 \4-00-00/')" | clickhouse-client --query="INSERT INTO wikistat FORMAT TabSeparated"; done
 ```
diff --git a/docs/en/getting_started/index.md b/docs/en/getting_started/index.md
old mode 100755
new mode 100644
index 07d0d91a224..42fc1c75551
--- a/docs/en/getting_started/index.md
+++ b/docs/en/getting_started/index.md
@@ -16,15 +16,14 @@ The terminal must use UTF-8 encoding (the default in Ubuntu).
 
 For testing and development, the system can be installed on a single server or on a desktop computer.
 
-### Installing from packages
+### Installing from packages Debian/Ubuntu
 
 In `/etc/apt/sources.list` (or in a separate `/etc/apt/sources.list.d/clickhouse.list` file), add the repository:
 
 ```text
-deb http://repo.yandex.ru/clickhouse/trusty stable main
+deb http://repo.yandex.ru/clickhouse/deb/stable/ main/
 ```
 
-On other versions of Ubuntu, replace `trusty` with `xenial` or `precise`.
 If you want to use the most recent test version, replace 'stable' with 'testing'.
 
 Then run:
@@ -35,10 +34,7 @@ sudo apt-get update
 sudo apt-get install clickhouse-client clickhouse-server-common
 ```
 
-You can also download and install packages manually from here:
-<http://repo.yandex.ru/clickhouse/trusty/pool/main/c/clickhouse/>
-<http://repo.yandex.ru/clickhouse/xenial/pool/main/c/clickhouse/>
-<http://repo.yandex.ru/clickhouse/precise/pool/main/c/clickhouse/>
+You can also download and install packages manually from here: <https://repo.yandex.ru/clickhouse/deb/stable/main/>
 
 ClickHouse contains access restriction settings. They are located in the 'users.xml' file (next to 'config.xml').
 By default, access is allowed from anywhere for the 'default' user, without a password. See 'user/default/networks'.
@@ -104,8 +100,7 @@ clickhouse-client
 ```
 
 The default parameters indicate connecting with localhost:9000 on behalf of the user 'default' without a password.
-The client can be used for connecting to a remote server.
-Example:
+The client can be used for connecting to a remote server. Example:
 
 ```bash
 clickhouse-client --host=example.com
@@ -137,4 +132,3 @@ SELECT 1
 **Congratulations, the system works!**
 
 To continue experimenting, you can try to download from the test data sets.
-
diff --git a/docs/en/index.md b/docs/en/index.md
old mode 100755
new mode 100644
index 72efa70802b..586c18297a8
--- a/docs/en/index.md
+++ b/docs/en/index.md
@@ -39,7 +39,7 @@ We'll say that the following is true for the OLAP (online analytical processing)
 - Data is updated in fairly large batches (> 1000 rows), not by single rows; or it is not updated at all.
 - Data is added to the DB but is not modified.
 - For reads, quite a large number of rows are extracted from the DB, but only a small subset of columns.
-- Tables are "wide," meaning they contain a large number of columns.
+- Tables are "wide", meaning they contain a large number of columns.
 - Queries are relatively rare (usually hundreds of queries per server or less per second).
 - For simple queries, latencies around 50 ms are allowed.
 - Column values are fairly small: numbers and short strings (for example, 60 bytes per URL).
diff --git a/docs/en/interfaces/cli.md b/docs/en/interfaces/cli.md
old mode 100755
new mode 100644
index 76549b46b36..4fd998fed66
--- a/docs/en/interfaces/cli.md
+++ b/docs/en/interfaces/cli.md
@@ -6,9 +6,7 @@ To work from the command line, you can use ` clickhouse-client`:
 $ clickhouse-client
 ClickHouse client version 0.0.26176.
 Connecting to localhost:9000.
-Connected to ClickHouse server version 0.0.26176.
-
-:)
+Connected to ClickHouse server version 0.0.26176.:)
 ```
 
 The client supports command-line options and configuration files. For more information, see "[Configuring](#interfaces_cli_configuration)".
@@ -31,6 +29,7 @@ _EOF
 
 cat file.csv | clickhouse-client --database=test --query="INSERT INTO test FORMAT CSV";
 ```
+In batch mode, the default data format is TabSeparated. You can set the format in the FORMAT clause of the query.
 
 By default, you can only process a single query in batch mode. To make multiple queries from a "script," use the --multiquery parameter. This works for all queries except INSERT. Query results are output consecutively without additional separators.
 Similarly, to process a large number of queries, you can run 'clickhouse-client' for each query. Note that it may take tens of milliseconds to launch the 'clickhouse-client' program.
@@ -65,7 +64,7 @@ The command-line client allows passing external data (external temporary tables)
 
 <a name="interfaces_cli_configuration"></a>
 
-## Configuring
+## Configure
 
 You can pass parameters to `clickhouse-client` (all parameters have a default value) using:
 
diff --git a/docs/en/interfaces/http_interface.md b/docs/en/interfaces/http_interface.md
old mode 100755
new mode 100644
index 38a70feef46..5c989a59d65
--- a/docs/en/interfaces/http_interface.md
+++ b/docs/en/interfaces/http_interface.md
@@ -37,8 +37,7 @@ Date: Fri, 16 Nov 2012 19:21:50 GMT
 1
 ```
 
-As you can see, curl is somewhat inconvenient in that spaces must be URL escaped.
-Although wget escapes everything itself, we don't recommend using it because it doesn't work well over HTTP 1.1 when using keep-alive and Transfer-Encoding: chunked.
+As you can see, curl is somewhat inconvenient in that spaces must be URL escaped.Although wget escapes everything itself, we don't recommend using it because it doesn't work well over HTTP 1.1 when using keep-alive and Transfer-Encoding: chunked.
 
 ```bash
 $ echo 'SELECT 1' | curl 'http://localhost:8123/' --data-binary @-
@@ -131,11 +130,15 @@ POST 'http://localhost:8123/?query=DROP TABLE t'
 
 For successful requests that don't return a data table, an empty response body is returned.
 
-You can use compression when transmitting data. The compressed data has a non-standard format, and you will need to use the special compressor program to work with it (sudo apt-get install compressor-metrika-yandex).
+You can use compression when transmitting data.
 
+For using ClickHouse internal compression format, and you will need to use the special clickhouse-compressor program to work with it (installed as a part of clickhouse-client package).
 If you specified 'compress=1' in the URL, the server will compress the data it sends you.
 If you specified 'decompress=1' in the URL, the server will decompress the same data that you pass in the POST method.
 
+Also standard gzip-based HTTP compression can be used. To send gzip compressed POST data just add `Content-Encoding: gzip` to request headers, and gzip POST body.
+To get response compressed, you need to add `Accept-Encoding: gzip` to request headers, and turn on ClickHouse setting called `enable_http_compression`.
+
 You can use this to reduce network traffic when transmitting a large amount of data, or for creating dumps that are immediately compressed.
 
 You can use the 'database' URL parameter to specify the default database.
@@ -191,7 +194,11 @@ $ echo 'SELECT number FROM system.numbers LIMIT 10' | curl 'http://localhost:812
 
 For information about other parameters, see the section "SET".
 
-In contrast to the native interface, the HTTP interface does not support the concept of sessions or session settings, does not allow aborting a query (to be exact, it allows this in only a few cases),  and does not show the progress of query processing. Parsing and data formatting are performed on the server side, and using the network might be ineffective.
+You can use ClickHouse sessions in the HTTP protocol. To do this, you need to specify the `session_id` GET parameter in HTTP request. You can use any alphanumeric string as a session_id. By default session will be timed out after 60 seconds of inactivity. You can change that by setting `default_session_timeout` in server config file, or by adding GET parameter `session_timeout`. You can also check the status of the session by using GET parameter `session_check=1`. When using sessions you can't run 2 queries with the same session_id simultaneously.
+
+You can get the progress of query execution in X-ClickHouse-Progress headers, by enabling setting send_progress_in_http_headers.
+
+Running query are not aborted automatically after closing HTTP connection. Parsing and data formatting are performed on the server side, and using the network might be ineffective.
 The optional 'query_id' parameter can be passed as the query ID (any string). For more information, see the section "Settings, replace_running_query".
 
 The optional 'quota_key' parameter can be passed as the quota key (any string). For more information, see the section "Quotas".
@@ -213,4 +220,3 @@ curl -sS 'http://localhost:8123/?max_result_bytes=4000000&buffer_size=3000000&wa
 ```
 
 Use buffering to avoid situations where a query processing error occurred after the response code and HTTP headers were sent to the client. In this situation, an error message is written at the end of the response body, and on the client side, the error can only be detected at the parsing stage.
-
diff --git a/docs/en/interfaces/index.md b/docs/en/interfaces/index.md
old mode 100755
new mode 100644
index e43f4474271..3e3e3df4853
--- a/docs/en/interfaces/index.md
+++ b/docs/en/interfaces/index.md
@@ -2,5 +2,4 @@
 
 # Interfaces
 
-To explore the system's capabilities, download data to tables, or make manual queries, use the clickhouse-client program.
-
+To explore the system's capabilities, download data to tables, or make manual queries, use the clickhouse-client program.
\ No newline at end of file
diff --git a/docs/en/interfaces/jdbc.md b/docs/en/interfaces/jdbc.md
old mode 100755
new mode 100644
diff --git a/docs/en/interfaces/tcp.md b/docs/en/interfaces/tcp.md
old mode 100755
new mode 100644
diff --git a/docs/en/interfaces/third-party_client_libraries.md b/docs/en/interfaces/third-party_client_libraries.md
old mode 100755
new mode 100644
index 8437be23b99..cc8ff1f4307
--- a/docs/en/interfaces/third-party_client_libraries.md
+++ b/docs/en/interfaces/third-party_client_libraries.md
@@ -2,7 +2,7 @@
 
 There are libraries for working with ClickHouse for:
 
-- Python
+- Python:
    - [infi.clickhouse_orm](https://github.com/Infinidat/infi.clickhouse_orm)
    - [sqlalchemy-clickhouse](https://github.com/cloudflare/sqlalchemy-clickhouse)
    - [clickhouse-driver](https://github.com/mymarilyn/clickhouse-driver)
diff --git a/docs/en/interfaces/third-party_gui.md b/docs/en/interfaces/third-party_gui.md
old mode 100755
new mode 100644
diff --git a/docs/en/introduction/distinctive_features.md b/docs/en/introduction/distinctive_features.md
old mode 100755
new mode 100644
index 59853b8e202..3927405579f
--- a/docs/en/introduction/distinctive_features.md
+++ b/docs/en/introduction/distinctive_features.md
@@ -1,10 +1,10 @@
 # Distinctive features of ClickHouse
 
-## True column-oriented DBMS
+## True column-oriented DBMS.
 
-In a true column-oriented DBMS, there isn't any "garbage" stored with the values. Among other things, this means that constant-length values must be supported, to avoid storing their length "number" next to the values. As an example, a billion UInt8-type values should actually consume around 1 GB uncompressed, or this will strongly affect the CPU use. It is very important to store data compactly (without any "garbage") even when uncompressed, since the speed of decompression (CPU usage) depends mainly on the volume of uncompressed data.
+In a true column-oriented DBMS, there isn't any "garbage" stored with the values. For example, constant-length values must be supported, to avoid storing their length "number" next to the values. As an example, a billion UInt8-type values should actually consume around 1 GB uncompressed, or this will strongly affect the CPU use. It is very important to store data compactly (without any "garbage") even when uncompressed, since the speed of decompression (CPU usage) depends mainly on the volume of uncompressed data.
 
-This is worth noting because there are systems that can store values of separate columns separately, but that can't effectively process analytical queries due to their optimization for other scenarios. Examples are HBase, BigTable, Cassandra, and HyperTable. In these systems, you will get throughput around a hundred thousand rows per second, but not hundreds of millions of rows per second.
+This is worth noting because there are systems that can store values of separate columns separately, but that can't effectively process analytical queries due to their optimization for other scenarios. Example are HBase, BigTable, Cassandra, and HyperTable. In these systems, you will get throughput around a hundred thousand rows per second, but not hundreds of millions of rows per second.
 
 Also note that ClickHouse is a DBMS, not a single database. ClickHouse allows creating tables and databases in runtime, loading data, and running queries without reconfiguring and restarting the server.
 
@@ -12,15 +12,15 @@ Also note that ClickHouse is a DBMS, not a single database. ClickHouse allows cr
 
 Some column-oriented DBMSs (InfiniDB CE and MonetDB) do not use data compression. However, data compression really improves performance.
 
-## Disk storage of data
+## Disk storage of data.
 
 Many column-oriented DBMSs (such as SAP HANA and Google PowerDrill) can only work in RAM. But even on thousands of servers, the RAM is too small for storing all the pageviews and sessions in Yandex.Metrica.
 
-## Parallel processing on multiple cores
+## Parallel processing on multiple cores.
 
 Large queries are parallelized in a natural way.
 
-## Distributed processing on multiple servers
+## Distributed processing on multiple servers.
 
 Almost none of the columnar DBMSs listed above have support for distributed processing.
 In ClickHouse, data can reside on different shards. Each shard can be a group of replicas that are used for fault tolerance. The query is processed on all the shards in parallel. This is transparent for the user.
@@ -30,12 +30,12 @@ In ClickHouse, data can reside on different shards. Each shard can be a group of
 If you are familiar with standard SQL, we can't really talk about SQL support.
 All the functions have different names.
 However, this is a declarative query language based on SQL that can't be differentiated from SQL in many instances.
-JOINs are supported. Subqueries are supported in FROM, IN, and JOIN clauses, as well as scalar subqueries.
+Support for JOINs. Subqueries are supported in FROM, IN, and JOIN clauses, as well as scalar subqueries.
 Dependent subqueries are not supported.
 
 ## Vector engine
 
-Data is not only stored by columns, but is processed by vectors (parts of columns). This allows us to achieve high CPU performance.
+Data is not only stored by columns, but is processed by vectors – parts of columns. This allows us to achieve high CPU performance.
 
 ## Real-time data updates
 
@@ -43,13 +43,13 @@ ClickHouse supports primary key tables. In order to quickly perform queries on t
 
 ## Indexes
 
-Having a primary key makes it possible to extract data for specific clients (for instance, Yandex.Metrica tracking tags) for a specific time range, with low latency less than several dozen milliseconds.
+Having a primary key allows, for example, extracting data for specific clients (Metrica counters) for a specific time range, with low latency less than several dozen milliseconds.
 
 ## Suitable for online queries
 
 This lets us use the system as the back-end for a web interface. Low latency means queries can be processed without delay, while the Yandex.Metrica interface page is loading. In other words, in online mode.
 
-## Support for approximated calculations
+## Support for approximated calculations.
 
 1. The system contains aggregate functions for approximated calculation of the number of various values, medians, and quantiles.
 2. Supports running a query based on a part (sample) of data and getting an approximated result. In this case, proportionally less data is retrieved from the disk.
diff --git a/docs/en/introduction/features_considered_disadvantages.md b/docs/en/introduction/features_considered_disadvantages.md
old mode 100755
new mode 100644
diff --git a/docs/en/introduction/index.md b/docs/en/introduction/index.md
old mode 100755
new mode 100644
index 3d07efe555d..e10b99d0138
--- a/docs/en/introduction/index.md
+++ b/docs/en/introduction/index.md
@@ -1,2 +1 @@
 # Introduction
-
diff --git a/docs/en/introduction/performance.md b/docs/en/introduction/performance.md
old mode 100755
new mode 100644
diff --git a/docs/en/introduction/possible_silly_questions.md b/docs/en/introduction/possible_silly_questions.md
old mode 100755
new mode 100644
index cf7b2c48032..36363ebe247
--- a/docs/en/introduction/possible_silly_questions.md
+++ b/docs/en/introduction/possible_silly_questions.md
@@ -1,8 +1,8 @@
-# Questions you were afraid to ask
+# Everything you were afraid to ask
 
 ## Why not use something like MapReduce?
 
-We can refer to systems like map-reduce as distributed computing systems in which the reduce operation is based on distributed sorting. In this sense, they include Hadoop, and YT (YT is developed at Yandex for internal use).
+We can refer to systems like map-reduce as distributed computing systems in which the reduce operation is based on distributed sorting. In this sense, they include Hadoop and YT (Yandex proprietary technology).
 
 These systems aren't appropriate for online queries due to their high latency. In other words, they can't be used as the back-end for a web interface.
 These types of systems aren't useful for real-time data updates.
diff --git a/docs/en/introduction/ya_metrika_task.md b/docs/en/introduction/ya_metrika_task.md
old mode 100755
new mode 100644
index 10f45f061d6..6a488be9b5f
--- a/docs/en/introduction/ya_metrika_task.md
+++ b/docs/en/introduction/ya_metrika_task.md
@@ -1,6 +1,6 @@
-# Yandex.Metrica use case
+# The Yandex.Metrica task
 
-ClickHouse currently powers [Yandex.Metrica](https://metrika.yandex.ru/), [the second largest web analytics platform in the world](http://w3techs.com/technologies/overview/traffic_analysis/all). With more than 13 trillion records in the database and more than 20 billion events daily, ClickHouse allows you generating custom reports on the fly directly from non-aggregated data.
+ClickHouse currently powers [ Yandex.Metrica](https://metrika.yandex.ru/), [ the second largest platform in the world](http://w3techs.com/technologies/overview/traffic_analysis/all) for Web Analytics. With more than 13 trillion records in the database and more than 20 billion events daily, ClickHouse allows you generating custom reports on the fly directly from non-aggregated data.
 
 We need to get custom reports based on hits and sessions, with custom segments set by the user. Data for the reports is updated in real-time. Queries must be run immediately (in online mode). We must be able to build reports for any time period. Complex aggregates must be calculated, such as the number of unique visitors.
 At this time (April 2014), Yandex.Metrica receives approximately 12 billion events (pageviews and mouse clicks) daily. All these events must be stored in order to build custom reports. A single query may require scanning hundreds of millions of rows over a few seconds, or millions of rows in no more than a few hundred milliseconds.
diff --git a/docs/en/operations/access_rights.md b/docs/en/operations/access_rights.md
old mode 100755
new mode 100644
index 1c72bf13b3e..9879dab9a99
--- a/docs/en/operations/access_rights.md
+++ b/docs/en/operations/access_rights.md
@@ -2,14 +2,14 @@
 
 Users and access rights are set up in the user config. This is usually `users.xml`.
 
-Users are recorded in the 'users' section. Here is a fragment of the `users.xml` file:
+Users are recorded in the 'users' section. We'll look at a fragment of the `users.xml` file:
 
 ```xml
 <!-- Users and ACL. -->
 <users>
     <!-- If the user name is not specified, the 'default' user is used. -->
-    <default>
-                <!-- Password could be specified in plaintext or in SHA256 (in hex format).
+        <default>
+            <!-- Password could be specified in plaintext or in SHA256 (in hex format).
 
              If you want to specify the password in plain text (not recommended), place it in the 'password' element.
              Example: <password>qwerty</password>.
@@ -17,14 +17,13 @@ Users are recorded in the 'users' section. Here is a fragment of the `users.xml`
 
              If you want to specify SHA256, place it in the 'password_sha256_hex' element.
              Example: <password_sha256_hex>65e84be33532fb784c48129675f9eff3a682b27168c0ea744b2cf58ee02337c5</password_sha256_hex>
-                          How to generate a decent password:             Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha256sum | tr -d '-'             
-                          The first line has the password and the second line has the corresponding SHA256.
-        -->        
-        <password></password>
-                
+             How to generate a decent password:
+             Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha256sum | tr -d '-'
+             The first line has the password and the second line has the corresponding SHA256.
+        -->
+        <password></password>        
         <!-- A list of networks that access is allowed from.
-            Each list item has one of the following forms:
-            <ip>IP address or subnet mask. For example: 198.51.100.0/24 or 2001:DB8::/32.
+            Each list item has one of the following forms:<ip>IP address or subnet mask. For example: 198.51.100.0/24 or 2001:DB8::/32.
             <host> Host name. For example: example01. A DNS query is made for verification, and all addresses obtained are compared with the address of the customer.
             <host_regexp> Regular expression for host names. For example: ^example\d\d-\d\d-\d\.yandex\.ru$                
             For verification, a DNS PTR query is made for the customer's address and a regular expression is applied to the result.
@@ -32,34 +31,34 @@ Users are recorded in the 'users' section. Here is a fragment of the `users.xml`
             We strongly recommend that the regex ends with \.yandex\.ru$.
 
             If you are installing ClickHouse yourself, enter:
-                 <networks>                        
-                         <ip>::/0</ip>
-                </networks>
-         -->        
-         <networks incl="networks" />        
-         
-         <!-- Settings profile for the user. -->        
-         <profile>default</profile>        
-         
+                            <networks>                        
+                                    <ip>::/0</ip>
+                            </networks>        
+          -->        
+          <networks incl="networks" />
+  
+          <!-- Settings profile for the user. -->
+          <profile>default</profile>
+  
          <!-- Quota for the user. -->        
          <quota>default</quota>    
-         </default>    
-         
-         <!-- For requests from the Yandex.Metrica user interface via the API for data on specific counters. -->    
-         <web>        
-             <password></password>        
-             <networks incl="networks" />        
-             <profile>web</profile>        
-             <quota>default</quota>        
-             <allow_databases>           
-             <database>test</database>
+     </default>    
+     
+     <!-- For requests from the Yandex.Metrica user interface via the API for data on specific counters. -->    
+     <web>        
+         <password></password>        
+         <networks incl="networks" />        
+         <profile>web</profile>        
+         <quota>default</quota>        
+         <allow_databases>           
+            <database>test</database>
         </allow_databases>
     </web>
 ```
 
 You can see a declaration from two users: `default`and`web`. We added the `web` user separately.
 
-The `default` user is chosen in cases when the username is not passed. The `default` user is also used for distributed query processing, if the configuration of the server or cluster doesn't specify the `user` and `password` (see the section on the [Distributed](../table_engines/distributed.md#table_engines-distributed) engine).
+The `default` user is chosen in cases when the username is not passed. The `default` user is also used for distributed query processing, if the configuration of the server or cluster doesn't specify `user` and `password` (see the section on the [Distributed](../table_engines/distributed.md#distributed_distributed) engine).
 
 The user that is used for exchanging information between servers combined in a cluster must not have substantial restrictions or quotas – otherwise, distributed queries will fail.
 
diff --git a/docs/en/operations/configuration_files.md b/docs/en/operations/configuration_files.md
old mode 100755
new mode 100644
index 52e9e10ffea..c3122617bf1
--- a/docs/en/operations/configuration_files.md
+++ b/docs/en/operations/configuration_files.md
@@ -14,7 +14,7 @@ If `replace` is specified, it replaces the entire element with the specified one
 
 If ` remove` is specified, it deletes the element.
 
-The config can also define "substitutions". If an element has the `incl` attribute, the corresponding substitution from the file will be used as the value. By default, the path to the file with substitutions is `/etc/metrika.xml`. This can be changed in the [include_from](server_settings/settings.md#server_settings-include_from) element in the server config. The substitution values are specified in  `/yandex/substitution_name` elements in this file. If a substitution specified in ` incl`  does not exist, it is recorded in the log. To prevent ClickHouse from logging missing substitutions, specify the  `optional="true"` attribute (for example, settings for [macros]()server_settings/settings.md#server_settings-macros)).
+The config can also define "substitutions". If an element has the `incl` attribute, the corresponding substitution from the file will be used as the value. By default, the path to the file with substitutions is `/etc/metrika.xml`. This can be changed in the config in the [include_from](server_settings/settings.md#server_settings-include_from) element. The substitution values are specified in  `/yandex/substitution_name` elements in this file. If a substitution specified in ` incl`  does not exist, it is recorded in the log. To prevent ClickHouse from logging missing substitutions, specify the  `optional = "true"` attribute (for example, settings for [ macros](server_settings/settings.md#server_settings-macros)).
 
 Substitutions can also be performed from ZooKeeper. To do this, specify the attribute `from_zk = "/path/to/node"`. The element value is replaced with the contents of the node at ` /path/to/node` in ZooKeeper. You can also put an entire XML subtree on the ZooKeeper node and it will be fully inserted into the source element.
 
diff --git a/docs/en/operations/index.md b/docs/en/operations/index.md
old mode 100755
new mode 100644
index 0ff38af8086..eb90f937cff
--- a/docs/en/operations/index.md
+++ b/docs/en/operations/index.md
@@ -1,2 +1 @@
-# Usage
-
+# Operation
diff --git a/docs/en/operations/quotas.md b/docs/en/operations/quotas.md
old mode 100755
new mode 100644
index fb1238b257d..d7b1a61ce7f
--- a/docs/en/operations/quotas.md
+++ b/docs/en/operations/quotas.md
@@ -18,10 +18,10 @@ Let's look at the section of the 'users.xml' file that defines quotas.
     <!-- Quota name. -->    
     <default>
         <!-- Restrictions for a time period. You can set many intervals with different restrictions. -->        
-        <interval>            
+        <interval>
             <!-- Length of the interval. -->            
-            <duration>3600</duration>
-
+            <duration>3600</duration>            
+            
             <!-- Unlimited. Just collect data for the specified time interval. -->            
             <queries>0</queries>            
             <errors>0</errors>            
@@ -39,19 +39,21 @@ The resource consumption calculated for each interval is output to the server lo
 <statbox>
     <!-- Restrictions for a time period. You can set many intervals with different restrictions. -->    
     <interval>        
-        <!-- Length of the interval. -->        
-        <duration>3600</duration>        
-        <queries>1000</queries>        
-        <errors>100</errors>        
-        <result_rows>1000000000</result_rows>        
-        <read_rows>100000000000</read_rows>        
-        <execution_time>900</execution_time>    
-    </interval>    
+        <!-- Length of the interval. -->
+        <duration>3600</duration>
 
-    <interval>        
-        <duration>86400</duration>        
-        <queries>10000</queries>        
-        <errors>1000</errors>        
+        <queries>1000</queries>
+        <errors>100</errors>
+        <result_rows>1000000000</result_rows>
+        <read_rows>100000000000</read_rows>
+        <execution_time>900</execution_time>
+    </interval>
+
+    <interval>
+        <duration>86400</duration>
+
+        <queries>10000</queries>
+        <errors>1000</errors>
         <result_rows>5000000000</result_rows>
         <read_rows>500000000000</read_rows>
         <execution_time>7200</execution_time>
@@ -87,7 +89,7 @@ Quotas can use the "quota key" feature in order to report on resources for multi
         Using keys makes sense only if quota_key is transmitted by the program, not by a user.
 
         You can also write <keyed_by_ip /> so the IP address is used as the quota key.(But keep in mind that users can change the IPv6 address fairly easily.)
-    -->
+         --> 
     <keyed />
 ```
 
@@ -96,3 +98,4 @@ The quota is assigned to users in the 'users' section of the config. See the sec
 For distributed query processing, the accumulated amounts are stored on the requestor server. So if the user goes to another server, the quota there will "start over".
 
 When the server is restarted, quotas are reset.
+
diff --git a/docs/en/operations/server_settings/index.md b/docs/en/operations/server_settings/index.md
old mode 100755
new mode 100644
index 208deec710c..2293e86f5c7
--- a/docs/en/operations/server_settings/index.md
+++ b/docs/en/operations/server_settings/index.md
@@ -9,4 +9,3 @@ These settings are stored in the ` config.xml` file on the ClickHouse server.
 Other settings are described in the "[Settings](../settings/index.md#settings)" section.
 
 Before studying the settings, read the [Configuration files](../configuration_files.md#configuration_files) section and note the use of substitutions (the `incl` and `optional` attributes).
-
diff --git a/docs/en/operations/server_settings/settings.md b/docs/en/operations/server_settings/settings.md
old mode 100755
new mode 100644
index e1575df2f88..8818f8ec932
--- a/docs/en/operations/server_settings/settings.md
+++ b/docs/en/operations/server_settings/settings.md
@@ -67,7 +67,7 @@ ClickHouse checks ` min_part_size`  and ` min_part_size_ratio`  and processes th
 
 The default database.
 
-To get a list of databases, use the [ SHOW DATABASES]( query./../query_language/queries.md#query_language_queries_show_databases).
+Use a [ SHOW DATABASES](../../query_language/queries.md#query_language_queries_show_databases) query to get a list of databases.
 
 **Example**
 
@@ -81,7 +81,7 @@ To get a list of databases, use the [ SHOW DATABASES]( query./../query_language/
 
 Default settings profile.
 
-Settings profiles are located in the file specified in the parameter [user_config](#server_settings-users_config).
+Settings profiles are located in the file specified in the [user_config](#server_settings-users_config) parameter.
 
 **Example**
 
@@ -100,7 +100,7 @@ Path:
 - Specify the absolute path or the path relative to the server config file.
 - The path can contain wildcards \* and ?.
 
-See also "[External dictionaries]("./../dicts/external_dicts.md#dicts-external_dicts)".
+See also "[External dictionaries](../../dicts/external_dicts.md#dicts-external_dicts)".
 
 **Example**
 
@@ -130,12 +130,12 @@ The default is ` true`.
 
 ## format_schema_path
 
-The path to the directory with the schemes for the input data, such as schemas for the [CapnProto](../../formats/capnproto.md#format_capnproto) format.
+The path to the directory with the schemas for the input data, such as schemas for the [ CapnProto](../../formats/capnproto.md#format_capnproto) format.
 
 **Example**
 
 ```xml
-  <!-- Directory containing schema files for various input formats. -->
+<!-- Directory containing schema files for various input formats. -->
   <format_schema_path>format_schemas/</format_schema_path>
 ```
 
@@ -179,7 +179,7 @@ You can configure multiple `<graphite>` clauses. For instance, you can use this
 
 Settings for thinning data for Graphite.
 
-For more information, see [GraphiteMergeTree](../../table_engines/graphitemergetree.md#table_engines-graphitemergetree).
+For more details, see [ GraphiteMergeTree](../../table_engines/graphitemergetree.md#table_engines-graphitemergetree).
 
 **Example**
 
@@ -241,7 +241,7 @@ Opens `https://tabix.io/` when accessing ` http://localhost: http_port`.
 
 The path to the file with substitutions.
 
-For more information, see the section "[Configuration files](../configuration_files.md#configuration_files)".
+For details, see the section "[Configuration files](../configuration_files.md#configuration_files)".
 
 **Example**
 
@@ -298,8 +298,7 @@ Restriction on hosts that requests can come from. If you want the server to answ
 Examples:
 
 ```xml
-<listen_host>::1</listen_host>
-<listen_host>127.0.0.1</listen_host>
+<listen_host>::1</listen_host><listen_host>127.0.0.1</listen_host>
 ```
 
 <a name="server_settings-logger"></a>
@@ -348,7 +347,7 @@ For more information, see the section "[Creating replicated tables](../../table_
 
 ## mark_cache_size
 
-Approximate size (in bytes) of the cache of "marks" used by [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) family.
+Approximate size (in bytes) of the cache of "marks" used by [ MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) engines.
 
 The cache is shared for the server and memory is allocated as needed. The cache size must be at least 5368709120.
 
@@ -404,7 +403,7 @@ We recommend using this option in Mac OS X, since the ` getrlimit()` function re
 
 Restriction on deleting tables.
 
-If the size of a [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) type table exceeds `max_table_size_to_drop` (in bytes), you can't delete it using a DROP query.
+If the size of a [ MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) type table  exceeds `max_table_size_to_drop` (in bytes), you can't delete it using a DROP query.
 
 If you still need to delete the table without restarting the ClickHouse server, create the ` <clickhouse-path>/flags/force_drop_table` file and run the DROP query.
 
@@ -440,17 +439,17 @@ For more information, see the MergeTreeSettings.h header file.
 
 SSL client/server configuration.
 
-Support for SSL is provided by the `` libpoco`` library. The interface is described in the file [SSLManager.h](https://github.com/yandex/ClickHouse/blob/master/contrib/libpoco/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h)
+Support for SSL is provided by the `` libpoco`` library. The description of the interface is in the [ SSLManager.h file.](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h)
 
 Keys for server/client settings:
 
 - privateKeyFile – The path to the file with the secret key of the PEM certificate. The file may contain a key and certificate at the same time.
 - certificateFile – The path to the client/server certificate file in PEM format. You can omit it if `` privateKeyFile`` contains the certificate.
 - caConfig – The path to the file or directory that contains trusted root certificates.
-- verificationMode – The method for checking the node's certificates. Details are in the description of the [Context](https://github.com/yandex/ClickHouse/blob/master/contrib/libpoco/NetSSL_OpenSSL/include/Poco/Net/Context.h) class. Possible values: ``none``, ``relaxed``, ``strict``, ``once``.
+- verificationMode – The method for checking the node's certificates. Details are in the description of the [Context](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/Context.h) class. Acceptable values: ``none``, ``relaxed``, ``strict``, ``once``.
 - verificationDepth – The maximum length of the verification chain. Verification will fail if the certificate chain length exceeds the set value.
 - loadDefaultCAFile – Indicates that built-in CA certificates for OpenSSL will be used. Acceptable values: `` true``, `` false``.  |
-- cipherList - Поддерживаемые OpenSSL-шифры. For example: `` ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH``.
+- cipherList - Supported OpenSSL-ciphers. For example: `` ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH``.
 - cacheSessions – Enables or disables caching sessions. Must be used in combination with ``sessionIdContext``. Acceptable values: `` true``, `` false``.
 - sessionIdContext – A unique set of random characters that the server appends to each generated identifier. The length of the string must not exceed ``SSL_MAX_SSL_SESSION_ID_LENGTH``. This parameter is always recommended, since it helps avoid problems both if the server caches the session and if the client requested caching. Default value: ``${application.name}``.
 - sessionCacheSize – The maximum number of sessions that the server caches. Default value: 1024\*20. 0 – Unlimited sessions.
@@ -499,7 +498,7 @@ Keys for server/client settings:
 
 ## part_log
 
-Logging events that are associated with [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) data. For instance, adding or merging data. You can use the log to simulate merge algorithms and compare their characteristics. You can visualize the merge process.
+Logging events that are associated with the [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) data type. For instance, adding or merging data. You can use the log to simulate merge algorithms and compare their characteristics. You can visualize the merge process.
 
 Queries are logged in the ClickHouse table, not in a separate file.
 
@@ -519,7 +518,7 @@ Use the following parameters to configure logging:
 
 - database – Name of the database.
 - table – Name of the table.
-- partition_by – Sets a [custom partitioning key](../../table_engines/custom_partitioning_key.md#custom-partitioning-key).
+- partition_by - Sets the [custom partition key](../../table_engines/custom_partitioning_key.md#custom-partitioning-key).
 - flush_interval_milliseconds – Interval for flushing data from memory to the disk.
 
 **Example**
@@ -563,7 +562,7 @@ Use the following parameters to configure logging:
 
 - database – Name of the database.
 - table – Name of the table.
-- partition_by – Sets a [custom partitioning key](../../table_engines/custom_partitioning_key.md#custom-partitioning-key).
+- partition_by - Sets the [custom partition key](../../table_engines/custom_partitioning_key.md#custom-partitioning-key).
 - flush_interval_milliseconds – Interval for flushing data from memory to the disk.
 
 If the table doesn't exist, ClickHouse will create it. If the structure of the query log changed when the ClickHouse server was updated, the table with the old structure is renamed, and a new table is created automatically.
@@ -585,7 +584,7 @@ If the table doesn't exist, ClickHouse will create it. If the structure of the q
 
 Configuration of clusters used by the Distributed table engine.
 
-For more information, see the section "[Table engines/Distributed](../../table_engines/distributed.md#table_engines-distributed)".
+For more information, see the section "[Duplicated table engine](../../table_engines/distributed.md#table_engines-distributed)".
 
 **Example**
 
@@ -645,7 +644,7 @@ The end slash is mandatory.
 
 ## uncompressed_cache_size
 
-Cache size (in bytes) for uncompressed data used by table engines from the [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) family.
+Cache size (in bytes) for uncompressed data used by table engines from the [ MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) family.
 
 There is one shared cache for the server. Memory is allocated on demand. The cache is used if the option [use_uncompressed_cache](../settings/settings.md#settings-use_uncompressed_cache) is enabled.
 
diff --git a/docs/en/operations/settings/index.md b/docs/en/operations/settings/index.md
old mode 100755
new mode 100644
index 0c5ca5d5171..0e967a4c081
--- a/docs/en/operations/settings/index.md
+++ b/docs/en/operations/settings/index.md
@@ -9,9 +9,9 @@ Ways to configure settings, in order of priority:
 
 - Settings in the server config file.
 
-   Settings from user profiles.
+   Set via user profiles.
 
-- Session settings.
+- For the session.
 
    Send ` SET setting=value` from the ClickHouse console client in interactive mode.
 Similarly, you can use ClickHouse sessions in the HTTP protocol. To do this, you need to specify the `session_id` HTTP parameter.
diff --git a/docs/en/operations/settings/query_complexity.md b/docs/en/operations/settings/query_complexity.md
old mode 100755
new mode 100644
diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
old mode 100755
new mode 100644
index 25c804b0035..e006f302c68
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -4,7 +4,7 @@
 
 ## distributed_product_mode
 
-Changes the behavior of [distributed subqueries](../../query_language/queries.md#queries-distributed-subrequests), i.e. in cases when the query contains the product of distributed tables.
+Alters the behavior of [distributed subqueries](../../query_language/queries.md#queries-distributed-subrequests), i.e. in cases when the query contains the product of distributed tables.
 
 ClickHouse applies the configuration if the subqueries on any level have a distributed table that exists on the local server and has more than one shard.
 
@@ -12,9 +12,9 @@ Restrictions:
 
 - Only applied for IN and JOIN subqueries.
 - Used only if a distributed table is used in the FROM clause.
-- Not used for a table-valued [ remote](../../table_functions/remote.md#table_functions-remote) function.
+- Not used for a table-valued [ remote](../../table_functions/remote.md#table_functions-remote)function.
 
-The possible values ​​are:
+Possible values:
 
 <a name="settings-settings-fallback_to_stale_replicas_for_distributed_queries"></a>
 
@@ -36,7 +36,7 @@ Disables query execution if the index can't be used by date.
 
 Works with tables in the MergeTree family.
 
-If `force_index_by_date=1`,  ClickHouse checks whether the query has a date key condition that can be used for restricting data ranges. If there is no suitable condition, it throws an exception. However, it does not check whether the condition actually reduces the amount of data to read. For example, the condition `Date != ' 2000-01-01 '` is acceptable even when it matches all the data in the table (i.e., running the query requires a full scan). For more information about ranges of data in MergeTree tables, see "[MergeTree](../../table_engines/mergetree.md#table_engines-mergetree)".
+If `force_index_by_date=1`,  ClickHouse checks whether the query has a date key condition that can be used for restricting data ranges. If there is no suitable condition, it throws an exception. However, it does not check whether the condition actually reduces the amount of data to read. For example, the condition `Date != ' 2000-01-01 '` is acceptable even when it matches all the data in the table (i.e., running the query requires a full scan). For more information about ranges of data in MergeTree tables, see [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree)".
 
 <a name="settings-settings-force_primary_key"></a>
 
@@ -46,7 +46,7 @@ Disables query execution if indexing by the primary key is not possible.
 
 Works with tables in the MergeTree family.
 
-If `force_primary_key=1`,  ClickHouse checks to see if the query has a primary key condition that can be used for restricting data ranges. If there is no suitable condition, it throws an exception. However, it does not check whether the condition actually reduces the amount of data to read. For more information about data ranges in MergeTree tables, see "[MergeTree](../../table_engines/mergetree.md#table_engines-mergetree)".
+If `force_primary_key=1`,  ClickHouse checks to see if the query has a primary key condition that can be used for restricting data ranges. If there is no suitable condition, it throws an exception. However, it does not check whether the condition actually reduces the amount of data to read. For more information about ranges of data in MergeTree tables, see [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree)".
 
 <a name="settings_settings_fsync_metadata"></a>
 
@@ -158,7 +158,7 @@ Don't confuse blocks for compression (a chunk of memory consisting of bytes) and
 
 ## min_compress_block_size
 
-For [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree)" tables. In order to reduce latency when processing queries, a block is compressed when writing the next mark if its size is at least 'min_compress_block_size'. By default, 65,536.
+For [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) tables. In order to reduce latency when processing queries, a block is compressed when writing the next mark if its size is at least 'min_compress_block_size'. By default, 65,536.
 
 The actual size of the block, if the uncompressed data is less than 'max_compress_block_size', is no less than this value and no less than the volume of data for one mark.
 
@@ -253,13 +253,13 @@ Yandex.Metrica uses this parameter set to 1 for implementing suggestions for seg
 
 ## schema
 
-This parameter is useful when you are using formats that require a schema definition, such as [Cap'n Proto](https://capnproto.org/). The value depends on the format.
+This parameter is useful when you are using formats that require a schema definition, such as [ Cap'n Proto](https://capnproto.org/). The value depends on the format.
 
 <a name="settings-settings_stream_flush_interval_ms"></a>
 
 ## stream_flush_interval_ms
 
-Works for tables with streaming in the case of a timeout, or when a thread generates[max_insert_block_size](#settings-settings-max_insert_block_size) rows.
+Works for tables with streaming in the case of a timeout, or when a thread generates [ max_insert_block_size](#settings-settings-max_insert_block_size)  rows.
 
 The default value is 7500.
 
diff --git a/docs/en/operations/settings/settings_profiles.md b/docs/en/operations/settings/settings_profiles.md
old mode 100755
new mode 100644
index f1fce41ba75..c978c599bd5
--- a/docs/en/operations/settings/settings_profiles.md
+++ b/docs/en/operations/settings/settings_profiles.md
@@ -17,7 +17,7 @@ Example:
 <profiles>
     <!-- Default settings -->
     <default>
-        <!-- The maximum number of threads when running a single query. -->
+    <!-- The maximum number of threads when running a single query. -->
         <max_threads>8</max_threads>
     </default>
 
diff --git a/docs/en/operations/tips.md b/docs/en/operations/tips.md
old mode 100755
new mode 100644
index 652698fe24c..11fc8f6da11
--- a/docs/en/operations/tips.md
+++ b/docs/en/operations/tips.md
@@ -105,7 +105,7 @@ Use at least a 10 GB network, if possible. 1 Gb will also work, but it will be m
 
 You are probably already using ZooKeeper for other purposes. You can use the same installation of ZooKeeper, if it isn't already overloaded.
 
-It's best to use a fresh version of ZooKeeper – 3.4.9 or later. The version in stable Linux distributions may be outdated.
+It's best to use a fresh version of ZooKeeper – 3.5 or later. The version in stable Linux distributions may be outdated.
 
 With the default settings, ZooKeeper is a time bomb:
 
@@ -174,7 +174,8 @@ dynamicConfigFile=/etc/zookeeper-{{ cluster['name'] }}/conf/zoo.cfg.dynamic
 Java version:
 
 ```text
-Java(TM) SE Runtime Environment (build 1.8.0_25-b17)Java HotSpot(TM) 64-Bit Server VM (build 25.25-b02, mixed mode)
+Java(TM) SE Runtime Environment (build 1.8.0_25-b17)
+Java HotSpot(TM) 64-Bit Server VM (build 25.25-b02, mixed mode)
 ```
 
 JVM parameters:
diff --git a/docs/en/operators/index.md b/docs/en/operators/index.md
old mode 100755
new mode 100644
index 411cde34b50..779f3bf3843
--- a/docs/en/operators/index.md
+++ b/docs/en/operators/index.md
@@ -67,11 +67,11 @@ Groups of operators are listed in order of priority (the higher it is in the lis
 
 `NOT a` The `not(a) function.`
 
-## Logical AND operator
+## Logical 'AND' operator
 
 `a AND b` – The`and(a, b) function.`
 
-## Logical OR operator
+## Logical 'OR' operator
 
 `a OR b` – The `or(a, b) function.`
 
diff --git a/docs/en/query_language/index.md b/docs/en/query_language/index.md
old mode 100755
new mode 100644
index 769d94eb4fd..247d76fc6ed
--- a/docs/en/query_language/index.md
+++ b/docs/en/query_language/index.md
@@ -1,2 +1 @@
 # Query language
-
diff --git a/docs/en/query_language/queries.md b/docs/en/query_language/queries.md
old mode 100755
new mode 100644
index d235945a646..743706a551f
--- a/docs/en/query_language/queries.md
+++ b/docs/en/query_language/queries.md
@@ -11,6 +11,7 @@ CREATE DATABASE [IF NOT EXISTS] db_name
 `A database` is just a directory for tables.
 If `IF NOT EXISTS` is included, the query won't return an error if the database already exists.
 
+
 <a name="query_language-queries-create_table"></a>
 
 ## CREATE TABLE
@@ -183,7 +184,7 @@ Deletes all tables inside the 'db' database, then deletes the 'db' database itse
 If `IF EXISTS` is specified, it doesn't return an error if the database doesn't exist.
 
 ```sql
-DROP TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster]
+DROP [TEMPORARY] TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster]
 ```
 
 Deletes the table.
@@ -307,7 +308,8 @@ SELECT * FROM system.parts WHERE active
 `active` – Only count active parts. Inactive parts are, for example, source parts remaining after merging to a larger part – these parts are deleted approximately 10 minutes after merging.
 
 Another way to view a set of parts and partitions is to go into the directory with table data.
-Data directory: `/var/lib/clickhouse/data/database/table/`,where `/var/lib/clickhouse/` is the path to the ClickHouse data, 'database' is the database name, and 'table' is the table name. Example:
+Data directory: `/var/lib/clickhouse/data/database/table/`,
+where `/var/lib/clickhouse/` is the path to the ClickHouse data, 'database' is the database name, and 'table' is the table name. Example:
 
 ```bash
 $ ls -l /var/lib/clickhouse/data/test/visits/
@@ -323,7 +325,7 @@ Here, `20140317_20140323_2_2_0` and ` 20140317_20140323_4_4_0` are the directori
 Let's break down the name of the first part: `20140317_20140323_2_2_0`.
 
 - `20140317` is the minimum date of the data in the chunk.
-- `20140323` is the maximum data of the data in the chunk.
+- `20140323` is the maximum date of the data in the chunk.
 - `2` is the minimum number of the data block.
 - `2` is the maximum number of the data block.
 - `0` is the chunk level (the depth of the merge tree it is formed from).
@@ -450,7 +452,7 @@ See also the section "Formats".
 ## SHOW TABLES
 
 ```sql
-SHOW TABLES [FROM db] [LIKE 'pattern'] [INTO OUTFILE filename] [FORMAT format]
+SHOW [TEMPORARY] TABLES [FROM db] [LIKE 'pattern'] [INTO OUTFILE filename] [FORMAT format]
 ```
 
 Displays a list of tables
@@ -460,7 +462,7 @@ Displays a list of tables
 
 This query is identical to: `SELECT name FROM system.tables WHERE database = 'db' [AND name LIKE 'pattern'] [INTO OUTFILE filename] [FORMAT format]`.
 
-See also the section "LIKE operator".
+See the section "LIKE operator" also.
 
 ## SHOW PROCESSLIST
 
@@ -484,7 +486,7 @@ Prints a table containing the columns:
 
 **query** – The query itself. In INSERT queries, the data for insertion is not output.
 
-**query_id** – The query identifier. Non-empty only if it was explicitly defined by the user. For distributed processing, the query ID is not passed to remote servers.
+**query_id** - The query identifier. Non-empty only if it was explicitly defined by the user. For distributed processing, the query ID is not passed to remote servers.
 
 This query is identical to: `SELECT * FROM system.processes [INTO OUTFILE filename] [FORMAT format]`.
 
@@ -497,7 +499,7 @@ watch -n1 "clickhouse-client --query='SHOW PROCESSLIST'"
 ## SHOW CREATE TABLE
 
 ```sql
-SHOW CREATE TABLE [db.]table [INTO OUTFILE filename] [FORMAT format]
+SHOW CREATE [TEMPORARY] TABLE [db.]table [INTO OUTFILE filename] [FORMAT format]
 ```
 
 Returns a single `String`-type 'statement' column, which contains a single value – the `CREATE` query used for creating the specified table.
@@ -515,7 +517,7 @@ Nested data structures are output in "expanded" format. Each column is shown sep
 ## EXISTS
 
 ```sql
-EXISTS TABLE [db.]name [INTO OUTFILE filename] [FORMAT format]
+EXISTS [TEMPORARY] TABLE [db.]name [INTO OUTFILE filename] [FORMAT format]
 ```
 
 Returns a single `UInt8`-type column, which contains the single value `0` if the table or database doesn't exist, or `1` if the table exists in the specified database.
@@ -571,9 +573,9 @@ The query can specify a list of columns to insert `[(c1, c2, c3)]`. In this case
 - The values calculated from the `DEFAULT`  expressions specified in the table definition.
 - Zeros and empty strings, if `DEFAULT` expressions are not defined.
 
-If [strict_insert_defaults=1](../operations/settings/settings.md#settings-strict_insert_defaults), columns that do not have `DEFAULT` defined must be listed in the query.
+If [strict_insert_defaults=1](../operations/settings/settings.md#settings-strict_insert_defaults), columns that do not have ` DEFAULT` defined must be listed in the query.
 
-Data can be passed to the INSERT in any [format](../formats/index.md#formats) supported by ClickHouse. The format must be specified explicitly in the query:
+The INSERT can pass data in any [format](../formats/index.md#formats) supported by ClickHouse. The format must be specified explicitly in the query:
 
 ```sql
 INSERT INTO [db.]table [(c1, c2, c3)] FORMAT format_name data_set
@@ -972,7 +974,8 @@ All columns that are not needed for the JOIN are deleted from the subquery.
 
 There are several types of JOINs:
 
-`INNER` or `LEFT` type:If INNER is specified, the result will contain only those rows that have a matching row in the right table.
+`INNER` or `LEFT` type:
+If INNER is specified, the result will contain only those rows that have a matching row in the right table.
 If LEFT is specified, any rows in the left table that don't have matching rows in the right table will be assigned the default value - zeros or empty rows. LEFT OUTER may be written instead of LEFT; the word OUTER does not affect anything.
 
 `ANY` or `ALL` stringency:If `ANY` is specified and the right table has several matching rows, only the first one found is joined.
@@ -1103,7 +1106,7 @@ Example:
 SELECT
     domainWithoutWWW(URL) AS domain,
     count(),
-    any(Title) AS title -- getting the first occurring page header for each domain.
+    any(Title) AS title -- getting the first occurred page header for each domain.
 FROM hits
 GROUP BY domain
 ```
@@ -1348,7 +1351,7 @@ There are two options for IN-s with subqueries (similar to JOINs): normal `IN`
 
 <div class="admonition attention">
 
-Remember that the algorithms described below may work differently depending on the [settings](../operations/settings/settings.md#settings-distributed_product_mode) `distributed_product_mode` setting.
+Remember that the algorithms described below may work differently depending on the [](../operations/settings/settings.md#settings-distributed_product_mode) `distributed_product_mode` setting.
 
 </div>
 
@@ -1476,34 +1479,34 @@ In all other cases, we don't recommend using the asterisk, since it only gives y
 ## KILL QUERY
 
 ```sql
-KILL QUERY WHERE <where expression to SELECT FROM system.processes query> [SYNC|ASYNC|TEST] [FORMAT format]
+KILL QUERY
+  WHERE <where expression to SELECT FROM system.processes query>
+  [SYNC|ASYNC|TEST]
+  [FORMAT format]
 ```
 
 Attempts to terminate queries currently running.
-The queries to terminate are selected from the system.processes table for which expression_for_system.processes is true.
+The queries to terminate are selected from the system.processes table for which `WHERE` expression is true.
 
 Examples:
 
 ```sql
+-- Terminates all queries with the specified query_id.
 KILL QUERY WHERE query_id='2-857d-4a57-9ee0-327da5d60a90'
-```
 
-Terminates all queries with the specified query_id.
-
-```sql
+-- Synchronously terminates all queries run by `username`.
 KILL QUERY WHERE user='username' SYNC
 ```
 
-Synchronously terminates all queries run by `username`.
-
 Readonly-users can only terminate their own requests.
-By default, the asynchronous version of queries is used (`ASYNC`), which terminates without waiting for queries to complete.
-The synchronous version (`SYNC`) waits for all queries to be completed and displays information about each process as it terminates.
+
+By default, the asynchronous version of queries is used (`ASYNC`), which doesn't wait for query termination.
+
+The synchronous version (`SYNC`) waits for all queries to be killed and displays information about each process as it terminates.
 The response contains the `kill_status` column, which can take the following values:
 
-1. 'finished' – The query completed successfully.
+1. 'finished' – The query terminated successfully.
 2. 'waiting' – Waiting for the query to finish after sending it a signal to terminate.
 3. The other values ​​explain why the query can't be terminated.
 
 A test query (`TEST`) only checks the user's rights and displays a list of queries to terminate.
-
diff --git a/docs/en/query_language/syntax.md b/docs/en/query_language/syntax.md
old mode 100755
new mode 100644
index 4928f2d4a12..e151d2ee3d9
--- a/docs/en/query_language/syntax.md
+++ b/docs/en/query_language/syntax.md
@@ -46,7 +46,7 @@ There are numeric literals, string literals, and compound literals.
 
 A numeric literal tries to be parsed:
 
-- First as a 64-bit signed number, using the 'strtoull' function.
+- first as a 64-bit signed number, using the 'strtoull' function.
 - If unsuccessful, as a 64-bit unsigned number, using the 'strtoll' function.
 - If unsuccessful, as a floating-point number using the 'strtod' function.
 - Otherwise, an error is returned.
diff --git a/docs/en/roadmap.md b/docs/en/roadmap.md
old mode 100755
new mode 100644
index 8241b0a65ae..46b08a89607
--- a/docs/en/roadmap.md
+++ b/docs/en/roadmap.md
@@ -3,13 +3,11 @@
 ## Q1 2018
 
 ### New functionality
-
-- Support for `UPDATE` and `DELETE`.
-
-- Multidimensional and nested arrays.
-
-   It can look something like this:
-
+- Initial support for `UPDATE` and `DELETE`.
+- Multi-dimensional and nested arrays.
+    
+    It may look like this:
+    
 ```sql
 CREATE TABLE t
 (
@@ -23,7 +21,7 @@ ENGINE = MergeTree ORDER BY x
 
 - External MySQL and ODBC tables.
 
-   External tables can be integrated into ClickHouse using external dictionaries. This new functionality is a convenient alternative to connecting external tables.
+    External tables can be integrated to ClickHouse using external dictionaries. This will be an alternative and a more convenient way to do so.
 
 ```sql
 SELECT ... 
@@ -32,68 +30,66 @@ FROM mysql('host:port', 'db', 'table', 'user', 'password')`
 
 ### Improvements
 
-- Effective data copying between ClickHouse clusters.
+- Efficient data copy between ClickHouse clusters.
 
-   Now you can copy data with the remote() function. For example: `
+    Currently, it is possible to copy data using remote() function, e.g.: `
 INSERT INTO t SELECT * FROM remote(...) `.
 
-   This operation will have improved performance.
+    The performance of this will be improved by proper distributed execution.
 
 - O_DIRECT for merges.
 
-   This will improve the performance of the OS cache and "hot" queries.
+    Should improve OS cache performance and correspondingly query performance for 'hot' queries.
+
 
 ## Q2 2018
 
 ### New functionality
 
-- UPDATE/DELETE conform to the EU GDPR.
-- Protobuf and Parquet input and output formats.
-- Creating dictionaries using DDL queries.
+- UPDATE/DELETE in order to comply with Europe GDPR.
+- Protobuf and Parquet input/output formats.
+- Create dictionaries by DDL queries.
 
-   Currently, dictionaries that are part of the database schema are defined in external XML files. This is inconvenient and counter-intuitive. The new approach should fix it.
-
-- Integration with LDAP.
+    Currently, it is inconvenient and confusing that dictionaries are defined in external XML files while being a part of DB schema. The new approach will fix that.
 
+- LDAP integration.
 - WITH ROLLUP and WITH CUBE for GROUP BY.
+- Custom encoding/compression for columns.
 
-- Custom encoding and compression for each column individually.
+    Currently, ClickHouse support LZ4 and ZSTD compressions for columns, and compressions settings are global (see our article [Compression in ClickHouse](https://www.altinity.com/blog/2017/11/21/compression-in-clickhouse)) for more details). Column level encoding (e.g. delta encoding) and compression will allow more efficient data storage and therefore faster queries.
 
-   As of now, ClickHouse supports LZ4 and ZSTD compression of columns, and compression settings are global (see the article [Compression in ClickHouse](https://www.altinity.com/blog/2017/11/21/compression-in-clickhouse)). Per-column compression and encoding will provide more efficient data storage, which in turn will speed up queries.
+- Store data at multiple disk volumes of a single server.
 
-- Storing data on multiple disks on the same server.
-
-   This functionality will make it easier to extend the disk space, since different disk systems can be used for different databases or tables. Currently, users are forced to use symbolic links if the databases and tables must be stored on a different disk.
+    That will make it easier to extend disk system as well as use different disk systems for different DBs or tables. Currently, users have to use symlinks if DB/table needs to be stored in another volume.
 
 ### Improvements
 
-Many improvements and fixes are planned for the query execution system. For example:
+A lot of enhancements and fixes are planned for query execution. In particular:
 
-- Using an index for `in (subquery)`.
+- Using index for ‘in (subquery)’.
 
-   The index is not used right now, which reduces performance.
+    Currently, index is not used for such queries resulting in lower performance.
 
-- Passing predicates from `where` to subqueries, and passing predicates to views.
+- Predicate pushdown from ‘where’ into subqueries and Predicate pushdown for views.
 
-   The predicates must be passed, since the view is changed by the subquery. Performance is still low for view filters, and views can't use the primary key of the original table, which makes views useless for large tables.
+    These two are related since view is replaced by subquery. Currently, performance of filter conditions for views is significantly degraded, views can not use primary key of the underlying table, that makes views on big tables pretty much useless.
 
-- Optimizing branching operations (ternary operator, if, multiIf).
+- Short-circuit expressions evaluation (ternary operator, if, multiIf).
 
-   ClickHouse currently performs all branches, even if they aren't necessary.
+    Currently, ClickHouse evaluates all branches even if the first one needs to be returned due to logical condition result.
 
-- Using a primary key for GROUP BY and ORDER BY.
+- Using primary key for GROUP BY and ORDER BY.
 
-   This will speed up certain types of queries with partially sorted data.
+    This may speed up certain types of queries since data is already partially pre-sorted.
 
 ## Q3-Q4 2018
 
-We don't have any set plans yet, but the main projects will be:
+Longer term plans are not yet finalized. There are two major projects on the list so far.
 
-- Resource pools for executing queries.
+- Resource pools for query execution.
 
-   This will make load management more efficient.
+    That will allow managing workloads more efficiently.
 
 - ANSI SQL JOIN syntax.
 
-   Improve ClickHouse compatibility with many SQL tools.
-
+    That will make ClickHouse more friendly for numerous SQL tools.
diff --git a/docs/en/system_tables/index.md b/docs/en/system_tables/index.md
old mode 100755
new mode 100644
index 240105a684b..614ce4020ec
--- a/docs/en/system_tables/index.md
+++ b/docs/en/system_tables/index.md
@@ -4,5 +4,5 @@ System tables are used for implementing part of the system's functionality, and
 You can't delete a system table (but you can perform DETACH).
 System tables don't have files with data on the disk or files with metadata. The server creates all the system tables when it starts.
 System tables are read-only.
-They are located in the 'system' database.
+System tables are located in the 'system' database.
 
diff --git a/docs/en/system_tables/system.asynchronous_metrics.md b/docs/en/system_tables/system.asynchronous_metrics.md
old mode 100755
new mode 100644
diff --git a/docs/en/system_tables/system.clusters.md b/docs/en/system_tables/system.clusters.md
old mode 100755
new mode 100644
index bc8dab86b3c..c0bc3dd13fa
--- a/docs/en/system_tables/system.clusters.md
+++ b/docs/en/system_tables/system.clusters.md
@@ -4,12 +4,13 @@ Contains information about clusters available in the config file and the servers
 Columns:
 
 ```text
-cluster String      - Cluster name.
-shard_num UInt32    - Number of a shard in the cluster, starting from 1.
-shard_weight UInt32 - Relative weight of a shard when writing data.
-replica_num UInt32  - Number of a replica in the shard, starting from 1.
-host_name String    - Host name as specified in the config.
-host_address String - Host's IP address obtained from DNS.
-port UInt16         - The port used to access the server.
-user String         - The username to use for connecting to the server.
+cluster String      – Cluster name.
+shard_num UInt32    – Number of a shard in the cluster, starting from 1.
+shard_weight UInt32 – Relative weight of a shard when writing data.
+replica_num UInt32  – Number of a replica in the shard, starting from 1.
+host_name String    – Host name as specified in the config.
+host_address String – Host's IP address obtained from DNS.
+port UInt16        – The port used to access the server.
+user String        – The username to use for connecting to the server.
 ```
+
diff --git a/docs/en/system_tables/system.columns.md b/docs/en/system_tables/system.columns.md
old mode 100755
new mode 100644
index bf05616fbef..975b84fe9d4
--- a/docs/en/system_tables/system.columns.md
+++ b/docs/en/system_tables/system.columns.md
@@ -11,3 +11,4 @@ type String               - Column type.
 default_type String       - Expression type (DEFAULT, MATERIALIZED, ALIAS) for the default value, or an empty string if it is not defined.
 default_expression String - Expression for the default value, or an empty string if it is not defined.
 ```
+
diff --git a/docs/en/system_tables/system.databases.md b/docs/en/system_tables/system.databases.md
old mode 100755
new mode 100644
diff --git a/docs/en/system_tables/system.dictionaries.md b/docs/en/system_tables/system.dictionaries.md
old mode 100755
new mode 100644
index 4ef0d7707b8..f3c9929d38e
--- a/docs/en/system_tables/system.dictionaries.md
+++ b/docs/en/system_tables/system.dictionaries.md
@@ -5,19 +5,19 @@ Contains information about external dictionaries.
 Columns:
 
 ```text
-name String                   - Dictionary name.
-type String                   - Dictionary type: Flat, Hashed, Cache.
-origin String                 - Path to the config file where the dictionary is described.
-attribute.names Array(String) - Array of attribute names provided by the dictionary.
-attribute.types Array(String) - Corresponding array of attribute types provided by the dictionary.
-has_hierarchy UInt8           - Whether the dictionary is hierarchical.
-bytes_allocated UInt64        - The amount of RAM used by the dictionary.
-hit_rate Float64              - For cache dictionaries, the percent of usage for which the value was in the cache.
-element_count UInt64          - The number of items stored in the dictionary.
-load_factor Float64           - The filled percentage of the dictionary (for a hashed dictionary, it is the filled percentage of the hash table).
-creation_time DateTime        - Time spent for the creation or last successful reload of the dictionary.
-last_exception String         - Text of an error that occurred when creating or reloading the dictionary, if the dictionary couldn't be created.
-source String                 - Text describing the data source for the dictionary.
+name String                  – Dictionary name.
+type String                  – Dictionary type: Flat, Hashed, Cache.
+origin String                – Path to the config file where the dictionary is described.attribute.
+names Array(String) – Array of attribute names provided by the dictionary.
+attribute.types Array(String) – Corresponding array of attribute types provided by the dictionary.
+has_hierarchy UInt8          – Whether the dictionary is hierarchical.
+bytes_allocated UInt64        – The amount of RAM used by the dictionary.
+hit_rate Float64              – For cache dictionaries, the percent of usage for which the value was in the cache.
+element_count UInt64          – The number of items stored in the dictionary.
+load_factor Float64          – The filled percentage of the dictionary (for a hashed dictionary, it is the filled percentage of the hash table).
+creation_time DateTime        – Time spent for the creation or last successful reload of the dictionary.
+last_exception String        – Text of an error that occurred when creating or reloading the dictionary, if the dictionary couldn't be created.
+source String                – Text describing the data source for the dictionary.
 ```
 
 Note that the amount of memory used by the dictionary is not proportional to the number of items stored in it. So for flat and cached dictionaries, all the memory cells are pre-assigned, regardless of how full the dictionary actually is.
diff --git a/docs/en/system_tables/system.events.md b/docs/en/system_tables/system.events.md
old mode 100755
new mode 100644
diff --git a/docs/en/system_tables/system.functions.md b/docs/en/system_tables/system.functions.md
old mode 100755
new mode 100644
index a1022a5e557..ac550acc14b
--- a/docs/en/system_tables/system.functions.md
+++ b/docs/en/system_tables/system.functions.md
@@ -6,6 +6,6 @@ Columns:
 
 ```text
 name String          – Function name.
-is_aggregate UInt8   – Whether it is an aggregate function.
+is_aggregate UInt8    – Whether it is an aggregate function.
 ```
 
diff --git a/docs/en/system_tables/system.merges.md b/docs/en/system_tables/system.merges.md
old mode 100755
new mode 100644
index 59870922ea5..0a10e4a5a8c
--- a/docs/en/system_tables/system.merges.md
+++ b/docs/en/system_tables/system.merges.md
@@ -18,3 +18,4 @@ rows_read UInt64                   - Number of rows read.
 bytes_written_uncompressed UInt64  - Amount of bytes written, uncompressed.
 rows_written UInt64                - Number of rows written.
 ```
+
diff --git a/docs/en/system_tables/system.numbers.md b/docs/en/system_tables/system.numbers.md
old mode 100755
new mode 100644
diff --git a/docs/en/system_tables/system.numbers_mt.md b/docs/en/system_tables/system.numbers_mt.md
old mode 100755
new mode 100644
diff --git a/docs/en/system_tables/system.one.md b/docs/en/system_tables/system.one.md
old mode 100755
new mode 100644
diff --git a/docs/en/system_tables/system.parts.md b/docs/en/system_tables/system.parts.md
old mode 100755
new mode 100644
diff --git a/docs/en/system_tables/system.processes.md b/docs/en/system_tables/system.processes.md
old mode 100755
new mode 100644
index 0802e555648..b9ad6a44e81
--- a/docs/en/system_tables/system.processes.md
+++ b/docs/en/system_tables/system.processes.md
@@ -14,12 +14,12 @@ rows_read UInt64        – The number of rows read from the table. For distribu
 
 bytes_read UInt64 – The number of uncompressed bytes read from the table. For distributed processing, on the requestor server, this is the total for all remote servers.
 
-UInt64 total_rows_approx – The approximate total number of rows that must be read. For distributed processing, on the requestor server, this is the total for all remote servers. It can be updated during request processing, when new sources to process become known.
+total_rows_approx UInt64 – The approximate total number of rows that must be read. For distributed processing, on the requestor server, this is the total for all remote servers. It can be updated during request processing, when new sources to process become known.
 
 memory_usage UInt64 – Memory consumption by the query. It might not include some types of dedicated memory.
 
-Query String – The query text. For INSERT, it doesn't include the data to insert.
+query String – The query text. For INSERT, it doesn't include the data to insert.
 
-query_id – Query ID, if defined.
+query_id String - The query ID, if defined.
 ```
 
diff --git a/docs/en/system_tables/system.replicas.md b/docs/en/system_tables/system.replicas.md
old mode 100755
new mode 100644
index 75cd8e34340..ec1341198dc
--- a/docs/en/system_tables/system.replicas.md
+++ b/docs/en/system_tables/system.replicas.md
@@ -42,41 +42,37 @@ Columns:
 database:           database name
 table:              table name
 engine:             table engine name
-
 is_leader:          whether the replica is the leader
 
 Only one replica at a time can be the leader. The leader is responsible for selecting background merges to perform.
 Note that writes can be performed to any replica that is available and has a session in ZK, regardless of whether it is a leader.
 
-is_readonly:        Whether the replica is in read-only mode.
-This mode is turned on if the config doesn't have sections with ZK, if an unknown error occurred when reinitializing sessions in ZK, and during session reinitialization in ZK.
+is_readonly:        Whether the replica is in read-only mode.This mode is turned on if the config doesn't have sections with ZK, if an unknown error occurred when reinitializing sessions in ZK, and during session reinitialization in ZK.
 
 is_session_expired: Whether the ZK session expired.
 Basically, the same thing as is_readonly.
 
-future_parts:       The number of data parts that will appear as the result of INSERTs or merges that haven't been done yet. 
+future_parts: The number of data parts that will appear as the result of INSERTs or merges that haven't been done yet. 
 
-parts_to_check:     The number of data parts in the queue for verification. 
-A part is put in the verification queue if there is suspicion that it might be damaged.
+parts_to_check: The number of data parts in the queue for verification. A part is put in the verification queue if there is suspicion that it might be damaged.
 
-zookeeper_path:     The path to the table data in ZK. 
+zookeeper_path: The path to the table data in ZK. 
 replica_name: Name of the replica in ZK. Different replicas of the same table have different names. 
-replica_path:       The path to the replica data in ZK. The same as concatenating zookeeper_path/replicas/replica_path.
+replica_path: The path to the replica data in ZK. The same as concatenating zookeeper_path/replicas/replica_path.
 
-columns_version:    Version number of the table structure. Indicates how many times ALTER was performed. If replicas have different versions, it means some replicas haven't made all of the ALTERs yet.
+columns_version: Version number of the table structure. Indicates how many times ALTER was performed. If replicas have different versions, it means some replicas haven't made all of the ALTERs yet.
 
-queue_size:         Size of the queue for operations waiting to be performed. 
-Operations include inserting blocks of data, merges, and certain other actions.
+queue_size:         Size of the queue for operations waiting to be performed. Operations include inserting blocks of data, merges, and certain other actions.
 Normally coincides with future_parts.
 
-inserts_in_queue:   Number of inserts of blocks of data that need to be made. Insertions are usually replicated fairly quickly. If the number is high, something is wrong.
+inserts_in_queue: Number of inserts of blocks of data that need to be made. Insertions are usually replicated fairly quickly. If the number is high, something is wrong.
 
-merges_in_queue:    The number of merges waiting to be made. Sometimes merges are lengthy, so this value may be greater than zero for a long time.
+merges_in_queue: The number of merges waiting to be made. Sometimes merges are lengthy, so this value may be greater than zero for a long time.
 
 The next 4 columns have a non-null value only if the ZK session is active.
 
-log_max_index:      Maximum entry number in the log of general activity. log_pointer:        Maximum entry number in the log of general activity that the replica copied to its execution queue, plus one.
-If log_pointer is much smaller than log_max_index, something is wrong.
+log_max_index:     Maximum entry number in the log of general activity.
+log_pointer:        Maximum entry number in the log of general activity that the replica copied to its execution queue, plus one. If log_pointer is much smaller than log_max_index, something is wrong.
 
 total_replicas:     Total number of known replicas of this table.
 active_replicas:    Number of replicas of this table that have a ZK session (the number of active replicas).
diff --git a/docs/en/system_tables/system.settings.md b/docs/en/system_tables/system.settings.md
old mode 100755
new mode 100644
index 90a392bcc24..a055135ebcf
--- a/docs/en/system_tables/system.settings.md
+++ b/docs/en/system_tables/system.settings.md
@@ -6,9 +6,9 @@ I.e. used for executing the query you are using to read from the system.settings
 Columns:
 
 ```text
-name String   – Setting name.
+name String  – Setting name.
 value String  – Setting value.
-changed UInt8 - Whether the setting was explicitly defined in the config or explicitly changed.
+changed UInt8 -–Whether the setting was explicitly defined in the config or explicitly changed.
 ```
 
 Example:
diff --git a/docs/en/system_tables/system.tables.md b/docs/en/system_tables/system.tables.md
old mode 100755
new mode 100644
index 5757a8ac3da..fabddf4dbb1
--- a/docs/en/system_tables/system.tables.md
+++ b/docs/en/system_tables/system.tables.md
@@ -1,7 +1,6 @@
 # system.tables
 
 This table contains the String columns 'database', 'name', and 'engine'.
-The table also contains three virtual columns: metadata_modification_time (DateTime type), create_table_query, and engine_full (String type).
+Also, the table has three virtual columns: metadata_modification_time of type DateTime, create_table_query and engine_full of type String.
 Each table that the server knows about is entered in the 'system.tables' table.
 This system table is used for implementing SHOW TABLES queries.
-
diff --git a/docs/en/system_tables/system.zookeeper.md b/docs/en/system_tables/system.zookeeper.md
old mode 100755
new mode 100644
index 46b40e7a08f..c456e7d9207
--- a/docs/en/system_tables/system.zookeeper.md
+++ b/docs/en/system_tables/system.zookeeper.md
@@ -70,3 +70,4 @@ numChildren:    7
 pzxid:          987021252247
 path:           /clickhouse/tables/01-08/visits/replicas
 ```
+
diff --git a/docs/en/table_engines/aggregatingmergetree.md b/docs/en/table_engines/aggregatingmergetree.md
old mode 100755
new mode 100644
index 987c102508b..d75d8353e6d
--- a/docs/en/table_engines/aggregatingmergetree.md
+++ b/docs/en/table_engines/aggregatingmergetree.md
@@ -1,11 +1,10 @@
 # AggregatingMergeTree
 
-This engine differs from `MergeTree` in that the merge combines the states of aggregate functions stored in the table for rows with the same primary key value.
+This engine differs from MergeTree in that the merge combines the states of aggregate functions stored in the table for rows with the same primary key value.
 
-For this to work, it uses the `AggregateFunction` data type, as well as `-State` and `-Merge` modifiers for aggregate functions. Let's examine it more closely.
-
-There is an `AggregateFunction` data type. It is a parametric data type. As parameters, the name of the aggregate function is passed, then the types of its arguments.
+In order for this to work, it uses the AggregateFunction data type and the -State and -Merge modifiers for aggregate functions. Let's examine it more closely.
 
+There is an AggregateFunction data type. It is a parametric data type. As parameters, the name of the aggregate function is passed, then the types of its arguments.
 Examples:
 
 ```sql
@@ -20,16 +19,12 @@ CREATE TABLE t
 This type of column stores the state of an aggregate function.
 
 To get this type of value, use aggregate functions with the `State` suffix.
+Example: `uniqState(UserID), quantilesState(0.5, 0.9)(SendTiming)` – in contrast to the corresponding 'uniq' and 'quantiles' functions, these functions return the state, rather than the prepared value. In other words, they return an AggregateFunction type value.
 
-Example:
-`uniqState(UserID), quantilesState(0.5, 0.9)(SendTiming)`
+An AggregateFunction type value can't be output in Pretty formats. In other formats, these types of values are output as implementation-specific binary data. The AggregateFunction type values are not intended for output or saving in a dump.
 
-In contrast to the corresponding `uniq` and `quantiles` functions, these functions return the state, rather than the prepared value. In other words, they return an `AggregateFunction` type value.
-
-An `AggregateFunction` type value can't be output in Pretty formats. In other formats, these types of values are output as implementation-specific binary data. The `AggregateFunction` type values are not intended for output or saving in a dump.
-
-The only useful thing you can do with `AggregateFunction` type values is combine the states and get a result, which essentially means to finish aggregation. Aggregate functions with the 'Merge' suffix are used for this purpose.
-Example: `uniqMerge(UserIDState), where UserIDState has the AggregateFunction` type.
+The only useful thing you can do with AggregateFunction type values is combine the states and get a result, which essentially means to finish aggregation. Aggregate functions with the 'Merge' suffix are used for this purpose.
+Example: `uniqMerge(UserIDState), where UserIDState has the AggregateFunction type`.
 
 In other words, an aggregate function with the 'Merge' suffix takes a set of states, combines them, and returns the result.
 As an example, these two queries return the same result:
@@ -42,15 +37,15 @@ SELECT uniqMerge(state) FROM (SELECT uniqState(UserID) AS state FROM table GROUP
 
 There is an ` AggregatingMergeTree` engine. Its job during a merge is to combine the states of aggregate functions from different table rows with the same primary key value.
 
-You can't use a normal INSERT to insert a row in a table containing `AggregateFunction` columns, because you can't explicitly define the `AggregateFunction` value. Instead, use `INSERT SELECT` with `-State` aggregate functions for inserting data.
+You can't use a normal INSERT to insert a row in a table containing AggregateFunction columns, because you can't explicitly define the AggregateFunction value. Instead, use INSERT SELECT with '-State' aggregate functions for inserting data.
 
-With SELECT from an `AggregatingMergeTree` table, use GROUP BY and aggregate functions with the '-Merge' modifier in order to complete data aggregation.
+With SELECT from an AggregatingMergeTree table, use GROUP BY and aggregate functions with the '-Merge' modifier in order to complete data aggregation.
 
-You can use `AggregatingMergeTree` tables for incremental data aggregation, including for aggregated materialized views.
+You can use AggregatingMergeTree tables for incremental data aggregation, including for aggregated materialized views.
 
 Example:
 
-Create an `AggregatingMergeTree` materialized view that watches the `test.visits` table:
+Creating a materialized AggregatingMergeTree view that tracks the 'test.visits' table:
 
 ```sql
 CREATE MATERIALIZED VIEW test.basic
@@ -64,13 +59,13 @@ FROM test.visits
 GROUP BY CounterID, StartDate;
 ```
 
-Insert data in the `test.visits` table. Data will also be inserted in the view, where it will be aggregated:
+Inserting data in the 'test.visits' table. Data will also be inserted in the view, where it will be aggregated:
 
 ```sql
 INSERT INTO test.visits ...
 ```
 
-Perform `SELECT` from the view using `GROUP BY` in order to complete data aggregation:
+Performing SELECT from the view using GROUP BY to finish data aggregation:
 
 ```sql
 SELECT
diff --git a/docs/en/table_engines/buffer.md b/docs/en/table_engines/buffer.md
old mode 100755
new mode 100644
diff --git a/docs/en/table_engines/collapsingmergetree.md b/docs/en/table_engines/collapsingmergetree.md
old mode 100755
new mode 100644
diff --git a/docs/en/table_engines/custom_partitioning_key.md b/docs/en/table_engines/custom_partitioning_key.md
old mode 100755
new mode 100644
index bfcb3c2c545..6a468a1137d
--- a/docs/en/table_engines/custom_partitioning_key.md
+++ b/docs/en/table_engines/custom_partitioning_key.md
@@ -2,7 +2,7 @@
 
 # Custom partitioning key
 
-Starting with version 1.1.54310, you can create tables in the MergeTree family with any partitioning expression (not only partitioning by month).
+Starting with version 1.1.54310, you can create tables in the MergeTree family with any partition expression (not only partitioning by month).
 
 The partition key can be an expression from the table columns, or a tuple of such expressions (similar to the primary key). The partition key can be omitted. When creating a table, specify the partition key in the ENGINE description with the new syntax:
 
@@ -10,7 +10,7 @@ The partition key can be an expression from the table columns, or a tuple of suc
 ENGINE [=] Name(...) [PARTITION BY expr] [ORDER BY expr] [SAMPLE BY expr] [SETTINGS name=value, ...]
 ```
 
-For MergeTree tables, the partition expression is specified after `PARTITION BY`, the primary key after `ORDER BY`, the sampling key after `SAMPLE BY`, and `SETTINGS` can specify `index_granularity` (optional; the default value is 8192), as well as other settings from [MergeTreeSettings.h](https://github.com/yandex/ClickHouse/blob/master/dbms/src/Storages/MergeTree/MergeTreeSettings.h). The other engine parameters are specified in parentheses after the engine name, as previously. Example:
+For MergeTree tables, the partition expression is specified after `PARTITION BY`, the primary key after `ORDER BY`, the sampling key after `SAMPLE BY`, and `SETTINGS` can specify `index_granularity` (optional; the default value is 8192), as well as other settings from [MergeTreeSettings.h](https://github.com/yandex/ClickHouse/blob/master/dbms/src/Storages/MergeTree/MergeTreeSettings.h). Example:
 
 ```sql
 ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/tables/name', 'replica1', Sign)
@@ -23,7 +23,7 @@ The traditional partitioning by month is expressed as `toYYYYMM(date_column)`.
 
 You can't convert an old-style table to a table with custom partitions (only via INSERT SELECT).
 
-After this table is created, merge will only work for data parts that have the same value for the partitioning expression. Note: This means that you shouldn't make overly granular partitions (more than about a thousand partitions), or SELECT will perform poorly.
+After this table is created, merge will only work for data parts that have the same value for the partition expression. Note: This means that you shouldn't make overly granular partitions (more than about a thousand partitions), or SELECT will perform poorly.
 
 To specify a partition in ALTER PARTITION commands, specify the value of the partition expression (or a tuple). Constants and constant expressions are supported. Example:
 
@@ -35,13 +35,13 @@ Deletes the partition for the current week with event type 1. The same is true f
 
 Note: For old-style tables, the partition can be specified either as a number `201710` or a string `'201710'`. The syntax for the new style of tables is stricter with types (similar to the parser for the VALUES input format). In addition, ALTER TABLE FREEZE PARTITION uses exact match for new-style tables (not prefix match).
 
-In the `system.parts` table, the `partition` column specifies the value of the partition expression to use in ALTER queries (if quotas are removed). The `name` column should specify the name of the data part that has a new format.
+In the `system.parts` table, the `partition` column should specify the value of the partition expression to use in ALTER queries (if quotas are removed). The `name` column should specify the name of the data part that has a new format.
 
-Was: `20140317_20140323_2_2_0` (minimum date - maximum date - minimum block number - maximum block number - level).
+Before: `20140317_20140323_2_2_0` (minimal data - maximal data - number of minimal block - number of maximal block - level).
 
-Now: `201403_2_2_0`  (partition ID -  minimum block number - maximum block number - level).
+After: `201403_2_2_0` (partition ID - number of minimal block - number of maximal block - level).
 
 The partition ID is its string identifier (human-readable, if possible) that is used for the names of data parts in the file system and in ZooKeeper. You can specify it in ALTER queries in place of the partition key. Example: Partition key `toYYYYMM(EventDate)`; ALTER can specify either `PARTITION 201710` or `PARTITION ID '201710'`.
 
-For more examples, see the tests [`00502_custom_partitioning_local`](https://github.com/yandex/ClickHouse/blob/master/dbms/tests/queries/0_stateless/00502_custom_partitioning_local.sql) and [`00502_custom_partitioning_replicated_zookeeper`](https://github.com/yandex/ClickHouse/blob/master/dbms/tests/queries/0_stateless/00502_custom_partitioning_replicated_zookeeper.sql).
+There are more examples in the tests [`00502_custom_partitioning_local`](https://github.com/yandex/ClickHouse/blob/master/dbms/tests/queries/0_stateless/00502_custom_partitioning_local.sql) and [`00502_custom_partitioning_replicated_zookeeper`](https://github.com/yandex/ClickHouse/blob/master/dbms/tests/queries/0_stateless/00502_custom_partitioning_replicated_zookeeper.sql).
 
diff --git a/docs/en/table_engines/dictionary.md b/docs/en/table_engines/dictionary.md
deleted file mode 100755
index ae8cca90d7c..00000000000
--- a/docs/en/table_engines/dictionary.md
+++ /dev/null
@@ -1,106 +0,0 @@
-<a name="table_engines-dictionary"></a>
-
-# Dictionary
-
-The `Dictionary` engine displays the dictionary data as a ClickHouse table.
-
-As an example, consider a dictionary of `products` with the following configuration:
-
-```xml
-<dictionaries>
-<dictionary>
-        <name>products</name>
-        <source>
-            <odbc>
-                <table>products</table>
-                <connection_string>DSN=some-db-server</connection_string>
-            </odbc>
-        </source>
-        <lifetime>
-            <min>300</min>
-            <max>360</max>
-        </lifetime>
-        <layout>
-            <flat/>
-        </layout>
-        <structure>
-            <id>
-                <name>product_id</name>
-            </id>
-            <attribute>
-                <name>title</name>
-                <type>String</type>
-                <null_value></null_value>
-            </attribute>
-        </structure>
-</dictionary>
-</dictionaries>
-```
-
-Query the dictionary data:
-
-```sql
-select name, type, key, attribute.names, attribute.types, bytes_allocated, element_count,source from system.dictionaries where name = 'products';                     
-
-SELECT 
-    name, 
-    type, 
-    key, 
-    attribute.names, 
-    attribute.types, 
-    bytes_allocated, 
-    element_count, 
-    source
-FROM system.dictionaries 
-WHERE name = 'products'
-```
-```
-┌─name─────┬─type─┬─key────┬─attribute.names─┬─attribute.types─┬─bytes_allocated─┬─element_count─┬─source──────────┐
-│ products │ Flat │ UInt64 │ ['title']       │ ['String']      │        23065376 │        175032 │ ODBC: .products │
-└──────────┴──────┴────────┴─────────────────┴─────────────────┴─────────────────┴───────────────┴─────────────────┘
-```
-
-You can use the [dictGet*](../functions/ext_dict_functions.md#ext_dict_functions) function to get the dictionary data in this format.
-
-This view isn't helpful when you need to get raw data, or when performing a `JOIN` operation. For these cases, you can use the `Dictionary` engine, which displays the dictionary data in a table.
-
-Syntax:
-
-```
-CREATE TABLE %table_name% (%fields%) engine = Dictionary(%dictionary_name%)`
-```
-
-Usage example:
-
-```sql
-create table products (product_id UInt64, title String) Engine = Dictionary(products);
-
-CREATE TABLE products
-(
-    product_id UInt64, 
-    title String, 
-)
-ENGINE = Dictionary(products)
-```
-```
-Ok.
-
-0 rows in set. Elapsed: 0.004 sec.
-```
-
-Take a look at what's in the table.
-
-```sql
-select * from products limit 1;
-
-SELECT *
-FROM products 
-LIMIT 1
-```
-```
-┌────product_id─┬─title───────────┐
-│        152689 │ Некоторый товар │
-└───────────────┴─────────────────┘
-
-1 rows in set. Elapsed: 0.006 sec.
-```
diff --git a/docs/en/table_engines/distributed.md b/docs/en/table_engines/distributed.md
old mode 100755
new mode 100644
index dd2ffe27fe5..b8643461fbb
--- a/docs/en/table_engines/distributed.md
+++ b/docs/en/table_engines/distributed.md
@@ -25,29 +25,29 @@ Clusters are set like this:
 <remote_servers>
     <logs>
         <shard>
-            <!-- Optional. Shard weight when writing data. Default: 1. -->
-            <weight>1</weight>            
-            <!-- Optional. Whether to write data to just one of the replicas. Default: false (write data to all replicas). -->            
-            <internal_replication>false</internal_replication>            
-            <replica>                
-                <host>example01-01-1</host>                
-                <port>9000</port>            
-            </replica>            
-            <replica>                
-                 <host>example01-01-2</host>                
-                 <port>9000</port>            
-            </replica>        
-        </shard>        
-        <shard>            
-            <weight>2</weight>            
-            <internal_replication>false</internal_replication>            
-            <replica>                
-                <host>example01-02-1</host>                
-                <port>9000</port>            
-            </replica>            
-            <replica>                
-            <host>example01-02-2</host>                
-            <port>9000</port>
+             <!-- Optional. Shard weight when writing data. Default: 1. -->
+             <weight>1</weight>
+             <!-- Optional. Whether to write data to just one of the replicas. Default: false (write data to all replicas). -->
+            <internal_replication>false</internal_replication>
+            <replica>
+                <host>example01-01-1</host>
+                <port>9000</port>
+            </replica>
+            <replica>
+                <host>example01-01-2</host>
+                <port>9000</port>
+            </replica>
+        </shard>
+        <shard>
+            <weight>2</weight>
+            <internal_replication>false</internal_replication>
+            <replica>
+                <host>example01-02-1</host>
+                <port>9000</port>
+            </replica>
+            <replica>
+                <host>example01-02-2</host>
+                <port>9000</port>
             </replica>
         </shard>
     </logs>
diff --git a/docs/en/table_engines/external_data.md b/docs/en/table_engines/external_data.md
old mode 100755
new mode 100644
diff --git a/docs/en/table_engines/file.md b/docs/en/table_engines/file.md
old mode 100755
new mode 100644
diff --git a/docs/en/table_engines/graphitemergetree.md b/docs/en/table_engines/graphitemergetree.md
old mode 100755
new mode 100644
index a4b62424954..6452377ac15
--- a/docs/en/table_engines/graphitemergetree.md
+++ b/docs/en/table_engines/graphitemergetree.md
@@ -2,13 +2,13 @@
 
 # GraphiteMergeTree
 
-This engine is designed for rollup (thinning and aggregating/averaging) [Graphite](http://graphite.readthedocs.io/en/latest/index.html) data. It may be helpful to developers who want to use ClickHouse as a data store for Graphite.
+This engine is designed for rollup (thinning and aggregating/averaging) [ Graphite](http://graphite.readthedocs.io/en/latest/index.html) data. It may be helpful to developers who want to use ClickHouse as a data store for Graphite.
 
 Graphite stores full data in ClickHouse, and data can be retrieved in the following ways:
 
 - Without thinning.
 
-   Uses the [MergeTree](mergetree.md#table_engines-mergetree) engine.
+   Using the [MergeTree](mergetree.md#table_engines-mergetree) engine.
 
 - With thinning.
 
@@ -83,3 +83,4 @@ Example of settings:
     </default>
 </graphite_rollup>
 ```
+
diff --git a/docs/en/table_engines/index.md b/docs/en/table_engines/index.md
old mode 100755
new mode 100644
diff --git a/docs/en/table_engines/join.md b/docs/en/table_engines/join.md
old mode 100755
new mode 100644
diff --git a/docs/en/table_engines/kafka.md b/docs/en/table_engines/kafka.md
old mode 100755
new mode 100644
index 4f10e55d029..85943e44fc5
--- a/docs/en/table_engines/kafka.md
+++ b/docs/en/table_engines/kafka.md
@@ -59,7 +59,7 @@ Example:
     level String,
     total UInt64
   ) ENGINE = SummingMergeTree(day, (day, level), 8192);
-
+  
   CREATE MATERIALIZED VIEW consumer TO daily
     AS SELECT toDate(toDateTime(timestamp)) AS day, level, count() as total
     FROM queue GROUP BY day, level;
diff --git a/docs/en/table_engines/log.md b/docs/en/table_engines/log.md
old mode 100755
new mode 100644
diff --git a/docs/en/table_engines/materializedview.md b/docs/en/table_engines/materializedview.md
old mode 100755
new mode 100644
index 5e2741c6aa1..00f70bd72bd
--- a/docs/en/table_engines/materializedview.md
+++ b/docs/en/table_engines/materializedview.md
@@ -1,4 +1,4 @@
 # MaterializedView
 
-Used for implementing materialized views (for more information, see the [CREATE TABLE](../query_language/queries.md#query_language-queries-create_table)) query. For storing data, it uses a different engine that was specified when creating the view. When reading from a table, it just uses this engine.
+Used for implementing materialized views (for more information, see [CREATE TABLE](../query_language/queries.md#query_language-queries-create_table)). For storing data, it uses a different engine that was specified when creating the view. When reading from a table, it just uses this engine.
 
diff --git a/docs/en/table_engines/memory.md b/docs/en/table_engines/memory.md
old mode 100755
new mode 100644
diff --git a/docs/en/table_engines/merge.md b/docs/en/table_engines/merge.md
old mode 100755
new mode 100644
diff --git a/docs/en/table_engines/mergetree.md b/docs/en/table_engines/mergetree.md
old mode 100755
new mode 100644
index fea02e01d72..71197f21b34
--- a/docs/en/table_engines/mergetree.md
+++ b/docs/en/table_engines/mergetree.md
@@ -56,7 +56,7 @@ In this example, the index can't be used:
 SELECT count() FROM table WHERE CounterID = 34 OR URL LIKE '%upyachka%'
 ```
 
-To check whether ClickHouse can use the index when executing the query, use the settings  [force_index_by_date](../operations/settings/settings.md#settings-settings-force_index_by_date)and[force_primary_key](../operations/settings/settings.md#settings-settings-force_primary_key).
+To check whether ClickHouse can use the index when executing the query, use the settings [ force_index_by_date](../operations/settings/settings.md#settings-settings-force_index_by_date)  and [ force_primary_key](../operations/settings/settings.md#settings-settings-force_primary_key).
 
 The index by date only allows reading those parts that contain dates from the desired range. However, a data part may contain data for many dates (up to an entire month), while within a single part the data is ordered by the primary key, which might not contain the date as the first column. Because of this, using a query with only a date condition that does not specify the primary key prefix will cause more data to be read than for a single date.
 
diff --git a/docs/en/table_engines/null.md b/docs/en/table_engines/null.md
old mode 100755
new mode 100644
diff --git a/docs/en/table_engines/replacingmergetree.md b/docs/en/table_engines/replacingmergetree.md
old mode 100755
new mode 100644
diff --git a/docs/en/table_engines/replication.md b/docs/en/table_engines/replication.md
old mode 100755
new mode 100644
index 20dd17e444f..1e58878c34e
--- a/docs/en/table_engines/replication.md
+++ b/docs/en/table_engines/replication.md
@@ -46,7 +46,7 @@ You can specify any existing ZooKeeper cluster and the system will use a directo
 
 If ZooKeeper isn't set in the config file, you can't create replicated tables, and any existing replicated tables will be read-only.
 
-ZooKeeper isn't used for SELECT queries. In other words, replication doesn't affect the productivity of SELECT queries – they work just as fast as for non-replicated tables. When querying distributed replicated tables, ClickHouse behavior is controlled by the settings [max_replica_delay_for_distributed_queries](../operations/settings/settings.md#settings_settings_max_replica_delay_for_distributed_queries) and [fallback_to_stale_replicas_for_distributed_queries](../operations/settings/settings.md#settings-settings-fallback_to_stale_replicas_for_distributed_queries).
+ZooKeeper isn't used for SELECT queries. In other words, replication doesn't affect the productivity of SELECT queries – they work just as fast as for non-replicated tables. When querying distributed replicated tables, ClickHouse behavior is controlled by the settings [max_replica_delay_for_distributed_queries](../operations/settings/settings.md#settings_settings_max_replica_delay_for_distributed_queries)  and [fallback_to_stale_replicas_for_distributed_queries](../operations/settings/settings.md#settings-settings-fallback_to_stale_replicas_for_distributed_queries).
 
 For each INSERT query (more precisely, for each inserted block of data; the INSERT query contains a single block, or per block for every max_insert_block_size = 1048576 rows), approximately ten entries are made in ZooKeeper in several transactions. This leads to slightly longer latencies for INSERT compared to non-replicated tables. But if you follow the recommendations to insert data in batches of no more than one INSERT per second, it doesn't create any problems. The entire ClickHouse cluster used for coordinating one ZooKeeper cluster has a total of several hundred INSERTs per second. The throughput on data inserts (the number of rows per second) is just as high as for non-replicated data.
 
diff --git a/docs/en/table_engines/set.md b/docs/en/table_engines/set.md
old mode 100755
new mode 100644
diff --git a/docs/en/table_engines/summingmergetree.md b/docs/en/table_engines/summingmergetree.md
old mode 100755
new mode 100644
diff --git a/docs/en/table_engines/tinylog.md b/docs/en/table_engines/tinylog.md
old mode 100755
new mode 100644
diff --git a/docs/en/table_engines/view.md b/docs/en/table_engines/view.md
old mode 100755
new mode 100644
diff --git a/docs/en/table_functions/index.md b/docs/en/table_functions/index.md
old mode 100755
new mode 100644
diff --git a/docs/en/table_functions/merge.md b/docs/en/table_functions/merge.md
old mode 100755
new mode 100644
diff --git a/docs/en/table_functions/remote.md b/docs/en/table_functions/remote.md
old mode 100755
new mode 100644
index e26e245207b..99b0c7bb116
--- a/docs/en/table_functions/remote.md
+++ b/docs/en/table_functions/remote.md
@@ -52,7 +52,7 @@ example01-{01..02}-1
 
 If you have multiple pairs of curly brackets, it generates the direct product of the corresponding sets.
 
-Addresses and parts of addresses in curly brackets can be separated by the pipe symbol (|). In this case, the corresponding sets of addresses are interpreted as replicas, and the query will be sent to the first healthy replica. However, the replicas are iterated in the order currently set in the [load_balancing](../operations/settings/settings.md#settings-load_balancing) setting.
+Addresses and parts of addresses in curly brackets can be separated by the pipe symbol (|). In this case, the corresponding sets of addresses are interpreted as replicas, and the query will be sent to the first healthy replica. The replicas are evaluated in the order currently set in the [load_balancing](../operations/settings/settings.md#settings-load_balancing) setting.
 
 Example:
 
diff --git a/docs/en/utils/clickhouse-copier.md b/docs/en/utils/clickhouse-copier.md
old mode 100755
new mode 100644
index 9d15053fe06..25d22f19222
--- a/docs/en/utils/clickhouse-copier.md
+++ b/docs/en/utils/clickhouse-copier.md
@@ -1,54 +1,40 @@
-<a name="utils-clickhouse-copier"></a>
+# clickhouse-copier util
 
-# clickhouse-copier
+The util copies tables data from one cluster to new tables of other (possibly the same) cluster in distributed and fault-tolerant manner.
 
-Copies data from the tables in one cluster to tables in another (or the same) cluster.
+Configuration of copying tasks is set in special ZooKeeper node (called the `/description` node).
+A ZooKeeper path to the description node is specified via `--task-path </task/path>` parameter.
+So, node `/task/path/description` should contain special XML content describing copying tasks.
 
-You can run multiple `clickhouse-copier` instances on different servers to perform the same job. ZooKeeper is used for syncing the processes.
+Simultaneously many `clickhouse-copier` processes located on any servers could execute the same task.
+ZooKeeper node `/task/path/` is used by the processes to coordinate their work.
+You must not add additional child nodes to `/task/path/`.
 
-After starting, `clickhouse-copier`:
+Currently you are responsible for manual launching of all `cluster-copier` processes.
+You can launch as many processes as you want, whenever and wherever you want.
+Each process try to select the nearest available shard of source cluster and copy some part of data (partition) from it to the whole
+destination cluster (with resharding).
+Therefore it makes sense to launch cluster-copier processes on the source cluster nodes to reduce the network usage.
 
-- Connects to ZooKeeper and receives:
-   - Copying jobs.
-   - The state of the copying jobs.
-
-- It performs the jobs.
-
-   Each running process chooses the "closest" shard of the source cluster and copies the data into the destination cluster, resharding the data if necessary.
-
-`clickhouse-copier` tracks the changes in ZooKeeper and applies them on the fly.
-
-To reduce network traffic, we recommend running `clickhouse-copier` on the same server where the source data is located.
-
-## Running clickhouse-copier
-
-The utility should be run manually:
-
-```bash
-clickhouse-copier copier --daemon --config zookeeper.xml --task-path /task/path --base-dir /path/to/dir
-```
-
-Parameters:
-
-- `daemon` — Starts `clickhouse-copier` in daemon mode.
-- `config` — The path to the `zookeeper.xml` file with the parameters for the connection to ZooKeeper.
-- `task-path` — The path to the ZooKeeper node. This node is used for syncing `clickhouse-copier` processes and storing tasks. Tasks are stored in `$task-path/description`.
-- `base-dir` — The path to logs and auxiliary files. When it starts, `clickhouse-copier` creates `clickhouse-copier_YYYYMMHHSS_<PID>` subdirectories in `$base-dir`. If this parameter is omitted, the directories are created in the directory where `clickhouse-copier` was launched.
-
-## Format of zookeeper.xml
+Since the workers coordinate their work via ZooKeeper, in addition to `--task-path </task/path>` you have to specify ZooKeeper
+cluster configuration via `--config-file <zookeeper.xml>` parameter. Example of `zookeeper.xml`:
 
 ```xml
-<yandex>
+   <yandex>
     <zookeeper>
         <node index="1">
             <host>127.0.0.1</host>
             <port>2181</port>
         </node>
     </zookeeper>
-</yandex>
+   </yandex>
 ```
 
-## Configuration of copying tasks
+When you run `clickhouse-copier --config-file <zookeeper.xml> --task-path </task/path>` the process connects to ZooKeeper cluster, reads tasks config from `/task/path/description` and executes them.
+
+## Format of task config
+
+Here is an example of `/task/path/description` content:
 
 ```xml
 <yandex>
@@ -83,62 +69,62 @@ Parameters:
         <readonly>0</readonly>
     </settings_push>
 
-    <!-- Common setting for fetch (pull) and insert (push) operations. The copier process context also uses it.
+    <!-- Common setting for fetch (pull) and insert (push) operations. Also, copier process context uses it.
          They are overlaid by <settings_pull/> and <settings_push/> respectively. -->
     <settings>
         <connect_timeout>3</connect_timeout>
-        <!-- Sync insert is set forcibly, but leave it here just in case. -->
+        <!-- Sync insert is set forcibly, leave it here just in case. -->
         <insert_distributed_sync>1</insert_distributed_sync>
     </settings>
 
-    <!-- Copying task descriptions.
-         You can specify several table tasks in the same task description (in the same ZooKeeper node), and they will be performed
+    <!-- Copying tasks description.
+         You could specify several table task in the same task description (in the same ZooKeeper node), they will be performed
          sequentially.
     -->
     <tables>
-        <!-- A table task that copies one table. -->
+        <!-- A table task, copies one table. -->
         <table_hits>
-            <!-- Source cluster name (from the <remote_servers/> section) and tables in it that should be copied. -->
+            <!-- Source cluster name (from <remote_servers/> section) and tables in it that should be copied -->
             <cluster_pull>source_cluster</cluster_pull>
             <database_pull>test</database_pull>
             <table_pull>hits</table_pull>
 
-            <!-- Destination cluster name and tables where the data should be inserted. -->
+            <!-- Destination cluster name and tables in which the data should be inserted -->
             <cluster_push>destination_cluster</cluster_push>
             <database_push>test</database_push>
             <table_push>hits2</table_push>
 
-            <!-- Engine for destination tables.
-                 If the destination tables have not been created yet, workers create them using column definitions from source tables and the engine
+            <!-- Engine of destination tables.
+                 If destination tables have not be created, workers create them using columns definition from source tables and engine
                  definition from here.
 
-                 NOTE: If the first worker starts to insert data and detects that the destination partition is not empty, then the partition will
-                 be dropped and refilled. Take this into account if you already have some data in destination tables. You can directly
-                 specify partitions that should be copied in <enabled_partitions/>. They should be in quoted format like the partition column in the
+                 NOTE: If the first worker starts insert data and detects that destination partition is not empty then the partition will
+                 be dropped and refilled, take it into account if you already have some data in destination tables. You could directly
+                 specify partitions that should be copied in <enabled_partitions/>, they should be in quoted format like partition column of
                  system.parts table.
             -->
-            <engine>
-            ENGINE=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/hits2', '{replica}')
+            <engine>ENGINE=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/hits2', '{replica}')
             PARTITION BY toMonday(date)
             ORDER BY (CounterID, EventDate)
             </engine>
 
-            <!-- The sharding key used to insert data to the destination cluster. -->
+            <!-- Sharding key used to insert data to destination cluster -->
             <sharding_key>jumpConsistentHash(intHash64(UserID), 2)</sharding_key>
 
-            <!-- Optional expression that filters data while pulling it from source servers. -->
+            <!-- Optional expression that filter data while pull them from source servers -->
             <where_condition>CounterID != 0</where_condition>
 
-            <!-- This section specifies partitions that should be copied. Other partitions will be ignored.
+            <!-- This section specifies partitions that should be copied, other partition will be ignored.
                  Partition names should have the same format as
-                the partition column in the system.parts table (i.e. quoted text).
-                 Since the partition key might be different for the source and destination cluster,
-                 these partition names specify the destination partitions.
+                 partition column of system.parts table (i.e. a quoted text).
+                 Since partition key of source and destination cluster could be different,
+                 these partition names specify destination partitions.
 
-                 Note: Although this section is optional (if it omitted, all partitions will be copied), it is strongly recommended to specify the partitions explicitly.
-                 If you already have some partitions ready on the destination cluster, they
-                 will be removed at the start of the copying, because they will be interpreted
-                 as unfinished data from the previous copying.
+                 NOTE: In spite of this section is optional (if it is not specified, all partitions will be copied),
+                 it is strictly recommended to specify them explicitly.
+                 If you already have some ready paritions on destination cluster they
+                 will be removed at the start of the copying since they will be interpeted
+                 as unfinished data from the previous copying!!!
             -->
             <enabled_partitions>
                 <partition>'2018-02-26'</partition>
@@ -147,7 +133,7 @@ Parameters:
             </enabled_partitions>
         </table_hits>
 
-        <!-- Next table to copy. It is not copied until the previous table is copying. -->
+        <!-- Next table to copy. It is not copied until previous table is copying. -->
         </table_visits>
         ...
         </table_visits>
@@ -156,5 +142,15 @@ Parameters:
 </yandex>
 ```
 
-`clickhouse-copier` tracks the changes in `/task/path/description` and applies them on the fly. For instance, if you change the value of `max_workers`, the number of processes running tasks will also change.
+cluster-copier processes watch for `/task/path/description` node update.
+So, if you modify the config settings or `max_workers` params, they will be updated.
 
+## Example
+
+```bash
+clickhouse-copier copier --daemon --config /path/to/copier/zookeeper.xml --task-path /clickhouse-copier/cluster1_tables_hits --base-dir /path/to/copier_logs
+```
+
+`--base-dir /path/to/copier_logs` specifies where auxilary and log files of the copier process will be saved.
+In this case it will create `/path/to/copier_logs/clickhouse-copier_YYYYMMHHSS_<PID>/` dir with log and status-files.
+If it is not specified it will use current dir (`/clickhouse-copier_YYYYMMHHSS_<PID>/` if it is run as a `--daemon`).
diff --git a/docs/en/utils/clickhouse-local.md b/docs/en/utils/clickhouse-local.md
old mode 100755
new mode 100644
index d5fba56271f..d18cc200320
--- a/docs/en/utils/clickhouse-local.md
+++ b/docs/en/utils/clickhouse-local.md
@@ -1,6 +1,4 @@
-<a name="utils-clickhouse-local"></a>
+# The clickhouse-local program
 
-#clickhouse-local
-
-The `clickhouse-local`  program enables you to perform fast processing on local files that store tables, without having to deploy and configure the ClickHouse server.
+The `clickhouse-local`  program enables you to perform fast processing on local files that store tables, without having to deploy and configure clickhouse-server.
 
diff --git a/docs/en/utils/index.md b/docs/en/utils/index.md
old mode 100755
new mode 100644
index cf541cda895..7a8c5ee5138
--- a/docs/en/utils/index.md
+++ b/docs/en/utils/index.md
@@ -1,5 +1,6 @@
-# ClickHouse utility
+# ClickHouse utilites
 
-* [clickhouse-local](clickhouse-local.md#utils-clickhouse-local) — Allows running SQL queries on data without stopping the ClickHouse server, similar to how `awk` does this.
-* [clickhouse-copier](clickhouse-copier.md#utils-clickhouse-copier) — Copies (and reshards) data from one cluster to another cluster.
+There are several ClickHouse utilites that are separate executable files:
 
+* `clickhouse-local` allows to execute SQL queries on a local data like `awk`
+* `clickhouse-copier` copies (and reshards) immutable data from one cluster to another in a fault-tolerant manner.
diff --git a/docs/mkdocs_en.yml b/docs/mkdocs_en.yml
index eeedc71a79b..d5fadb3f1e1 100644
--- a/docs/mkdocs_en.yml
+++ b/docs/mkdocs_en.yml
@@ -86,7 +86,6 @@ pages:
   - 'GraphiteMergeTree': 'table_engines/graphitemergetree.md'
   - 'Data replication': 'table_engines/replication.md'
   - 'Distributed': 'table_engines/distributed.md'
-  - 'Dictionary': 'table_engines/dictionary.md'
   - 'Merge': 'table_engines/merge.md'
   - 'Buffer': 'table_engines/buffer.md'
   - 'File': 'table_engines/file.md'
diff --git a/docs/ru/dicts/external_dicts_dict_layout.md b/docs/ru/dicts/external_dicts_dict_layout.md
index defb0605c0f..ff1b9c0cdd5 100644
--- a/docs/ru/dicts/external_dicts_dict_layout.md
+++ b/docs/ru/dicts/external_dicts_dict_layout.md
@@ -109,15 +109,15 @@
 Пример: таблица содержит скидки для каждого рекламодателя в виде:
 
 ```
-+---------------+---------------------+-------------------+--------+
-| advertiser id | discount start date | discount end date | amount |
-+===============+=====================+===================+========+
-| 123           | 2015-01-01          | 2015-01-15        | 0.15   |
-+---------------+---------------------+-------------------+--------+
-| 123           | 2015-01-16          | 2015-01-31        | 0.25   |
-+---------------+---------------------+-------------------+--------+
-| 456           | 2015-01-01          | 2015-01-15        | 0.05   |
-+---------------+---------------------+-------------------+--------+
+  +------------------+-----------------------------+------------+----------+
+  | id рекламодателя | дата начала действия скидки | дата конца | величина |
+  +==================+=============================+============+==========+
+  | 123              | 2015-01-01                  | 2015-01-15 | 0.15     |
+  +------------------+-----------------------------+------------+----------+
+  | 123              | 2015-01-16                  | 2015-01-31 | 0.25     |
+  +------------------+-----------------------------+------------+----------+
+  | 456              | 2015-01-01                  | 2015-01-15 | 0.05     |
+  +------------------+-----------------------------+------------+----------+
 ```
 
 Чтобы использовать выборку по диапазонам дат, необходимо в [structure](external_dicts_dict_structure#dicts-external_dicts_dict_structure) определить элементы `range_min`, `range_max`.
diff --git a/docs/ru/introduction/ya_metrika_task.md b/docs/ru/introduction/ya_metrika_task.md
index 765c0450890..24e595b2c49 100644
--- a/docs/ru/introduction/ya_metrika_task.md
+++ b/docs/ru/introduction/ya_metrika_task.md
@@ -1,6 +1,6 @@
 # Постановка задачи в Яндекс.Метрике
 
-ClickHouse на данный момент обеспечивает работу [Яндекс.Метрики](https://metrika.yandex.ru/), [второй крупнейшей в мире](http://w3techs.com/technologies/overview/traffic_analysis/all) платформы для веб аналитики. При более 13 триллионах записей в базе данных и более 20 миллиардах событий в сутки, ClickHouse позволяет генерировать индивидуально настроенные отчёты на лету напрямую из неагрегированных данных.
+ClickHouse на данный момент обеспечивает рабту [Яндекс.Метрики](https://metrika.yandex.ru/), [второй крупнейшей в мире](http://w3techs.com/technologies/overview/traffic_analysis/all) платформы для веб аналитики. При более 13 триллионах записей в базе данных и более 20 миллиардах событий в сутки, ClickHouse позволяет генерировать индивидуально настроенные отчёты на лету напрямую из неагрегированных данных.
 
 Нужно получать произвольные отчёты на основе хитов и визитов, с произвольными сегментами, задаваемыми пользователем. Данные для отчётов обновляются в реальном времени. Запросы должны выполняться сразу (в режиме онлайн). Отчёты должно быть возможно строить за произвольный период. Требуется вычислять сложные агрегаты типа количества уникальных посетителей.
 На данный момент (апрель 2014), каждый день в Яндекс.Метрику поступает около 12 миллиардов событий (хитов и кликов мыши). Все эти события должны быть сохранены для возможности строить произвольные отчёты. Один запрос может потребовать просканировать сотни миллионов строк за время не более нескольких секунд, или миллионы строк за время не более нескольких сотен миллисекунд.
diff --git a/docs/ru/roadmap.md b/docs/ru/roadmap.md
index bbdc740bf01..13c2b60c094 100644
--- a/docs/ru/roadmap.md
+++ b/docs/ru/roadmap.md
@@ -5,15 +5,15 @@
 ### Новая функциональность
 - Поддержка `UPDATE` и `DELETE`.
 - Многомерные и вложенные массивы.
-
+    
     Это может выглядеть например так:
-
+    
 ```sql
 CREATE TABLE t
 (
-    x Array(Array(String)),
+    x Array(Array(String)), 
     z Nested(
-        x Array(String),
+        x Array(String), 
         y Nested(...))
 )
 ENGINE = MergeTree ORDER BY x
@@ -24,7 +24,7 @@ ENGINE = MergeTree ORDER BY x
     Внешние таблицы можно интрегрировать в ClickHouse с помощью внешних словарей. Новая функциональность станет более удобной альтернативой для подключения внешних таблиц.
 
 ```sql
-SELECT ...
+SELECT ... 
 FROM mysql('host:port', 'db', 'table', 'user', 'password')`
 ```
 
@@ -40,7 +40,7 @@ INSERT INTO t SELECT * FROM remote(...) `.
 - O_DIRECT for merges.
 
     Улучшит производительность кэша операционной системы, а также производительность 'горячих' запросов.
-
+    
 ## Q2 2018
 
 ### Новая функциональность
@@ -56,23 +56,23 @@ INSERT INTO t SELECT * FROM remote(...) `.
 - Настраиваемые кодировки и сжатие для каждого столбца в отдельности.
 
     Сейчас, ClickHouse поддерживает сжатие столбцов с помощью LZ4 и ZSTD, и настройки сжатия глобальные (смотрите статью [Compression in ClickHouse](https://www.altinity.com/blog/2017/11/21/compression-in-clickhouse)). Поколоночное сжатие и кодирование обеспечит более эффективное хранение данных, что в свою очередь ускорит выполнение запросов.
-
+    
 - Хранение данных на нескольких дисках на одном сервере.
 
     Реализация это функциональности упростит расширение дискового пространства, поскольку можно будет использовать различные дисковые системы для разных баз данных или таблиц. Сейчас, пользователи вынуждены использовать символические ссылки, если базы данных и таблицы должны храниться на другом диске.
-
+    
 ### Улучшения
 
 Планируется множество улучшений и исправлений в системе выполнения запросов. Например:
 
 - Использование индекса для `in (subquery)`.
 
-    Сейчас, индекс не используется, что приводит к снижению производительности.
-
+    Сейчас, индекс не используется, что приводит с снижению производительности.
+    
 - Передача предикатов из `where` в подзапросы, а также передача предикатов в представления.
 
     Передача предикатов необходима, поскольку представление изменяется поздапросом. Сейчас производительность фильтров для представлений низкая, представления не могут использовать первичный ключ оригинальной таблицы, что делает представления для больших таблиц бесполезными.
-
+    
 - Оптимизация операций с ветвлением (тернарный оператор, if, multiIf).
 
     Сейчас, ClickHouse выполняет все ветви, даже если в этом нет необходимости.
@@ -88,7 +88,7 @@ INSERT INTO t SELECT * FROM remote(...) `.
 - Пулы ресурсов для выполнения запросов.
 
     Позволят более эффективно управлять нагрузкой.
-
+    
 - Синтаксис ANSI SQL JOIN.
 
     Улучшит совместимость ClickHouse со множеством SQL-инструментов.

From c5b04f7f17e40402d70cabc27328a945a3656757 Mon Sep 17 00:00:00 2001
From: Alex Zatelepin <ztlpn@users.noreply.github.com>
Date: Mon, 26 Mar 2018 17:18:04 +0300
Subject: [PATCH 206/244] Add info about total uncompressed part size to system
 tables (#2052)

* column size calculating routines now take into account multiple streams [#CLICKHOUSE-2831]

* rename for clarity

* use more appropriate method for calculating table size

* add info about total uncompressed part size to system.parts and system.parts_columns [#CLICKHOUSE-2831]

* rename columns for clarity [#CLICKHOUSE-2831]

* count shared substreams only once [#CLICKHOUSE-2831]
---
 dbms/src/Interpreters/PartLog.cpp             |   2 +-
 .../MergeTree/MergeTreeBlockReadUtils.cpp     |   9 +-
 dbms/src/Storages/MergeTree/MergeTreeData.cpp |  70 ++++--------
 dbms/src/Storages/MergeTree/MergeTreeData.h   |  30 +----
 .../MergeTree/MergeTreeDataMerger.cpp         |   8 +-
 .../Storages/MergeTree/MergeTreeDataPart.cpp  | 107 ++++++++----------
 .../Storages/MergeTree/MergeTreeDataPart.h    |  43 ++++---
 .../MergeTree/MergeTreeDataWriter.cpp         |   2 +-
 .../MergeTree/MergedBlockOutputStream.cpp     |   2 +-
 .../MergeTree/ReplicatedMergeTreeQueue.cpp    |   2 +-
 dbms/src/Storages/StorageMergeTree.cpp        |   4 +-
 .../Storages/StorageReplicatedMergeTree.cpp   |  12 +-
 .../Storages/System/StorageSystemColumns.cpp  |   2 +-
 .../Storages/System/StorageSystemParts.cpp    |  51 +++++----
 .../System/StorageSystemPartsBase.cpp         |  29 ++++-
 .../System/StorageSystemPartsColumns.cpp      |  84 +++++++-------
 16 files changed, 225 insertions(+), 232 deletions(-)

diff --git a/dbms/src/Interpreters/PartLog.cpp b/dbms/src/Interpreters/PartLog.cpp
index 8043f451402..646aae527b1 100644
--- a/dbms/src/Interpreters/PartLog.cpp
+++ b/dbms/src/Interpreters/PartLog.cpp
@@ -108,7 +108,7 @@ bool PartLog::addNewPartToTheLog(Context & context, const MergeTreeDataPart & pa
         elem.table_name = part.storage.getTableName();
         elem.part_name = part.name;
 
-        elem.bytes_compressed_on_disk = part.size_in_bytes;
+        elem.bytes_compressed_on_disk = part.bytes_on_disk;
         elem.rows = part.rows_count;
 
         elem.error = static_cast<UInt16>(execution_status.code);
diff --git a/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp
index 5d52eedfda1..a2bfffabeb2 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp
@@ -102,8 +102,6 @@ void MergeTreeBlockSizePredictor::initialize(const Block & sample_block, const N
         const String & column_name = column_with_type_and_name.name;
         const ColumnPtr & column_data = column_with_type_and_name.column;
 
-        const auto column_checksum = data_part->tryGetBinChecksum(column_name);
-
         if (!from_update && !names_set.count(column_name))
             continue;
 
@@ -122,8 +120,11 @@ void MergeTreeBlockSizePredictor::initialize(const Block & sample_block, const N
             ColumnInfo info;
             info.name = column_name;
             /// If column isn't fixed and doesn't have checksum, than take first
-            info.bytes_per_row_global = column_checksum
-                ? column_checksum->uncompressed_size / number_of_rows_in_part
+            MergeTreeDataPart::ColumnSize column_size = data_part->getColumnSize(
+                column_name, *column_with_type_and_name.type);
+
+            info.bytes_per_row_global = column_size.data_uncompressed
+                ? column_size.data_uncompressed / number_of_rows_in_part
                 : column_data->byteSize() / std::max<size_t>(1, column_data->size());
 
             dynamic_columns_infos.emplace_back(info);
diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp
index ef4b8022323..5b5a6b87c8e 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp
@@ -747,7 +747,7 @@ void MergeTreeData::removePartsFinally(const MergeTreeData::DataPartsVector & pa
         for (auto & part : parts)
         {
             part_log_elem.part_name = part->name;
-            part_log_elem.bytes_compressed_on_disk = part->size_in_bytes;
+            part_log_elem.bytes_compressed_on_disk = part->bytes_on_disk;
             part_log_elem.rows = part->rows_count;
 
             part_log->add(part_log_elem);
@@ -1211,8 +1211,8 @@ MergeTreeData::AlterDataPartTransactionPtr MergeTreeData::alterDataPart(
             false, nullptr, "", false, 0, DBMS_DEFAULT_BUFFER_SIZE, false);
 
         auto compression_settings = this->context.chooseCompressionSettings(
-            part->size_in_bytes,
-            static_cast<double>(part->size_in_bytes) / this->getTotalActiveSizeInBytes());
+            part->bytes_on_disk,
+            static_cast<double>(part->bytes_on_disk) / this->getTotalActiveSizeInBytes());
         ExpressionBlockInputStream in(part_in, expression);
 
         /** Don't write offsets for arrays, because ALTER never change them
@@ -1314,7 +1314,7 @@ void MergeTreeData::AlterDataPartTransaction::commit()
                 file.remove();
         }
 
-        mutable_part.size_in_bytes = MergeTreeData::DataPart::calculateTotalSize(path);
+        mutable_part.bytes_on_disk = MergeTreeData::DataPart::calculateTotalSizeOnDisk(path);
 
         /// TODO: we can skip resetting caches when the column is added.
         data_part->storage.context.dropCaches();
@@ -1674,7 +1674,7 @@ size_t MergeTreeData::getTotalActiveSizeInBytes() const
         std::lock_guard<std::mutex> lock(data_parts_mutex);
 
         for (auto & part : getDataPartsStateRange(DataPartState::Committed))
-            res += part->size_in_bytes;
+            res += part->bytes_on_disk;
     }
 
     return res;
@@ -1832,59 +1832,37 @@ void MergeTreeData::calculateColumnSizesImpl()
 void MergeTreeData::addPartContributionToColumnSizes(const DataPartPtr & part)
 {
     std::shared_lock<std::shared_mutex> lock(part->columns_lock);
-    const auto & files = part->checksums.files;
 
-    /// TODO This method doesn't take into account columns with multiple files.
-    for (const auto & column : getColumns().getAllPhysical())
+    for (const auto & column : part->columns)
     {
-        const auto escaped_name = escapeForFileName(column.name);
-        const auto bin_file_name = escaped_name + ".bin";
-        const auto mrk_file_name = escaped_name + ".mrk";
-
-        ColumnSize & column_size = column_sizes[column.name];
-
-        if (files.count(bin_file_name))
-        {
-            const auto & bin_file_checksums = files.at(bin_file_name);
-            column_size.data_compressed += bin_file_checksums.file_size;
-            column_size.data_uncompressed += bin_file_checksums.uncompressed_size;
-        }
-
-        if (files.count(mrk_file_name))
-            column_size.marks += files.at(mrk_file_name).file_size;
+        DataPart::ColumnSize & total_column_size = column_sizes[column.name];
+        DataPart::ColumnSize part_column_size = part->getColumnSize(column.name, *column.type);
+        total_column_size.add(part_column_size);
     }
 }
 
-static inline void logSubtract(size_t & from, size_t value, Logger * log, const String & variable)
-{
-    if (value > from)
-        LOG_ERROR(log, "Possibly incorrect subtraction: " << from << " - " << value << " = " << from - value << ", variable " << variable);
-
-    from -= value;
-}
-
 void MergeTreeData::removePartContributionToColumnSizes(const DataPartPtr & part)
 {
-    const auto & files = part->checksums.files;
+    std::shared_lock<std::shared_mutex> lock(part->columns_lock);
 
-    /// TODO This method doesn't take into account columns with multiple files.
-    for (const auto & column : getColumns().getAllPhysical())
+    for (const auto & column : part->columns)
     {
-        const auto escaped_name = escapeForFileName(column.name);
-        const auto bin_file_name = escaped_name + ".bin";
-        const auto mrk_file_name = escaped_name + ".mrk";
+        DataPart::ColumnSize & total_column_size = column_sizes[column.name];
+        DataPart::ColumnSize part_column_size = part->getColumnSize(column.name, *column.type);
 
-        auto & column_size = column_sizes[column.name];
-
-        if (files.count(bin_file_name))
+        auto log_subtract = [&](size_t & from, size_t value, const char * field)
         {
-            const auto & bin_file_checksums = files.at(bin_file_name);
-            logSubtract(column_size.data_compressed, bin_file_checksums.file_size, log, bin_file_name + ".file_size");
-            logSubtract(column_size.data_uncompressed, bin_file_checksums.uncompressed_size, log, bin_file_name + ".uncompressed_size");
-        }
+            if (value > from)
+                LOG_ERROR(log, "Possibly incorrect column size subtraction: "
+                    << from << " - " << value << " = " << from - value
+                    << ", column: " << column.name << ", field: " << field);
 
-        if (files.count(mrk_file_name))
-            logSubtract(column_size.marks, files.at(mrk_file_name).file_size, log, mrk_file_name + ".file_size");
+            from -= value;
+        };
+
+        log_subtract(total_column_size.data_compressed, part_column_size.data_compressed, ".data_compressed");
+        log_subtract(total_column_size.data_uncompressed, part_column_size.data_uncompressed, ".data_uncompressed");
+        log_subtract(total_column_size.marks, part_column_size.marks, ".marks");
     }
 }
 
diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h
index 47abd959645..f412419459c 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeData.h
+++ b/dbms/src/Storages/MergeTree/MergeTreeData.h
@@ -452,18 +452,6 @@ public:
     /// Returns the size of partition in bytes.
     size_t getPartitionSize(const std::string & partition_id) const;
 
-    struct ColumnSize
-    {
-        size_t marks = 0;
-        size_t data_compressed = 0;
-        size_t data_uncompressed = 0;
-
-        size_t getTotalCompressedSize() const
-        {
-            return marks + data_compressed;
-        }
-    };
-
     size_t getColumnCompressedSize(const std::string & name) const
     {
         std::lock_guard<std::mutex> lock{data_parts_mutex};
@@ -472,25 +460,13 @@ public:
         return it == std::end(column_sizes) ? 0 : it->second.data_compressed;
     }
 
-    using ColumnSizes = std::unordered_map<std::string, ColumnSize>;
-    ColumnSizes getColumnSizes() const
+    using ColumnSizeByName = std::unordered_map<std::string, DataPart::ColumnSize>;
+    ColumnSizeByName getColumnSizes() const
     {
         std::lock_guard<std::mutex> lock{data_parts_mutex};
         return column_sizes;
     }
 
-    /// NOTE Could be off after DROPped and MODIFYed columns in ALTER. Doesn't include primary.idx.
-    size_t getTotalCompressedSize() const
-    {
-        std::lock_guard<std::mutex> lock{data_parts_mutex};
-        size_t total_size = 0;
-
-        for (const auto & col : column_sizes)
-            total_size += col.second.getTotalCompressedSize();
-
-        return total_size;
-    }
-
     /// Calculates column sizes in compressed form for the current state of data_parts.
     void recalculateColumnSizes()
     {
@@ -555,7 +531,7 @@ private:
     String full_path;
 
     /// Current column sizes in compressed and uncompressed form.
-    ColumnSizes column_sizes;
+    ColumnSizeByName column_sizes;
 
     /// Engine-specific methods
     BrokenPartCallback broken_part_callback;
diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp
index ae8677c57be..f2d6e773be1 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp
@@ -180,7 +180,7 @@ bool MergeTreeDataMerger::selectPartsToMerge(
         }
 
         IMergeSelector::Part part_info;
-        part_info.size = part->size_in_bytes;
+        part_info.size = part->bytes_on_disk;
         part_info.age = current_time - part->modification_time;
         part_info.level = part->info.level;
         part_info.data = &part;
@@ -266,7 +266,7 @@ bool MergeTreeDataMerger::selectAllPartsToMergeWithinPartition(
             return false;
         }
 
-        sum_bytes += (*it)->size_in_bytes;
+        sum_bytes += (*it)->bytes_on_disk;
 
         prev_it = it;
         ++it;
@@ -532,7 +532,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMerger::mergePartsToTemporaryPart
     {
         std::shared_lock<std::shared_mutex> part_lock(part->columns_lock);
 
-        merge_entry->total_size_bytes_compressed += part->size_in_bytes;
+        merge_entry->total_size_bytes_compressed += part->bytes_on_disk;
         merge_entry->total_size_marks += part->marks_count;
     }
 
@@ -883,7 +883,7 @@ size_t MergeTreeDataMerger::estimateDiskSpaceForMerge(const MergeTreeData::DataP
 {
     size_t res = 0;
     for (const MergeTreeData::DataPartPtr & part : parts)
-        res += part->size_in_bytes;
+        res += part->bytes_on_disk;
 
     return static_cast<size_t>(res * DISK_USAGE_COEFFICIENT_TO_RESERVE);
 }
diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp
index 79f3ddb523f..26f7c2c4493 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp
@@ -134,26 +134,51 @@ MergeTreeDataPart::MergeTreeDataPart(MergeTreeData & storage_, const String & na
 {
 }
 
-/// Returns the size of .bin file for column `name` if found, zero otherwise.
-UInt64 MergeTreeDataPart::getColumnCompressedSize(const String & name) const
+/// Takes into account the fact that several columns can e.g. share their .size substreams.
+/// When calculating totals these should be counted only once.
+MergeTreeDataPart::ColumnSize MergeTreeDataPart::getColumnSizeImpl(const String & name, const IDataType & type, std::unordered_set<String> * processed_substreams) const
 {
-    const Checksum * checksum = tryGetBinChecksum(name);
+    ColumnSize size;
+    if (checksums.empty())
+        return size;
 
-    /// Probably a logic error, not sure if this can ever happen if checksums are not empty
-    return checksum ? checksum->file_size : 0;
+    type.enumerateStreams([&](const IDataType::SubstreamPath & substream_path)
+    {
+        String file_name = IDataType::getFileNameForStream(name, substream_path);
+
+        if (processed_substreams && !processed_substreams->insert(file_name).second)
+            return;
+
+        auto bin_checksum = checksums.files.find(file_name + ".bin");
+        if (bin_checksum != checksums.files.end())
+        {
+            size.data_compressed += bin_checksum->second.file_size;
+            size.data_uncompressed += bin_checksum->second.uncompressed_size;
+        }
+
+        auto mrk_checksum = checksums.files.find(file_name + ".mrk");
+        if (mrk_checksum != checksums.files.end())
+            size.marks += mrk_checksum->second.file_size;
+    }, {});
+
+    return size;
 }
 
-UInt64 MergeTreeDataPart::getColumnUncompressedSize(const String & name) const
+MergeTreeDataPart::ColumnSize MergeTreeDataPart::getColumnSize(const String & name, const IDataType & type) const
 {
-    const Checksum * checksum = tryGetBinChecksum(name);
-    return checksum ? checksum->uncompressed_size : 0;
+    return getColumnSizeImpl(name, type, nullptr);
 }
 
-
-UInt64 MergeTreeDataPart::getColumnMrkSize(const String & name) const
+MergeTreeDataPart::ColumnSize MergeTreeDataPart::getTotalColumnsSize() const
 {
-    const Checksum * checksum = tryGetMrkChecksum(name);
-    return checksum ? checksum->file_size : 0;
+    ColumnSize totals;
+    std::unordered_set<String> processed_substreams;
+    for (const NameAndTypePair & column : columns)
+    {
+        ColumnSize size = getColumnSizeImpl(column.name, *column.type, &processed_substreams);
+        totals.add(size);
+    }
+    return totals;
 }
 
 
@@ -171,7 +196,7 @@ String MergeTreeDataPart::getColumnNameWithMinumumCompressedSize() const
         if (!hasColumnFiles(column.name))
             continue;
 
-        const auto size = getColumnCompressedSize(column.name);
+        const auto size = getColumnSize(column.name, *column.type).data_compressed;
         if (size < minimum_size)
         {
             minimum_size = size;
@@ -251,7 +276,7 @@ MergeTreeDataPart::~MergeTreeDataPart()
     }
 }
 
-UInt64 MergeTreeDataPart::calculateTotalSize(const String & from)
+UInt64 MergeTreeDataPart::calculateTotalSizeOnDisk(const String & from)
 {
     Poco::File cur(from);
     if (cur.isFile())
@@ -260,7 +285,7 @@ UInt64 MergeTreeDataPart::calculateTotalSize(const String & from)
     cur.list(files);
     UInt64 res = 0;
     for (const auto & file : files)
-        res += calculateTotalSize(from + file);
+        res += calculateTotalSizeOnDisk(from + file);
     return res;
 }
 
@@ -420,7 +445,7 @@ void MergeTreeDataPart::loadIndex()
         index.assign(std::make_move_iterator(loaded_index.begin()), std::make_move_iterator(loaded_index.end()));
     }
 
-    size_in_bytes = calculateTotalSize(getFullPath());
+    bytes_on_disk = calculateTotalSizeOnDisk(getFullPath());
 }
 
 void MergeTreeDataPart::loadPartitionAndMinMaxIndex()
@@ -484,20 +509,21 @@ void MergeTreeDataPart::loadRowsCount()
         for (const NameAndTypePair & column : columns)
         {
             ColumnPtr column_col = column.type->createColumn();
-            const auto checksum = tryGetBinChecksum(column.name);
+            if (!column_col->isFixedAndContiguous())
+                continue;
 
-            /// Should be fixed non-nullable column
-            if (!checksum || !column_col->isFixedAndContiguous())
+            size_t column_size = getColumnSize(column.name, *column.type).data_uncompressed;
+            if (!column_size)
                 continue;
 
             size_t sizeof_field = column_col->sizeOfValueIfFixed();
-            rows_count = checksum->uncompressed_size / sizeof_field;
+            rows_count = column_size / sizeof_field;
 
-            if (checksum->uncompressed_size % sizeof_field != 0)
+            if (column_size % sizeof_field != 0)
             {
                 throw Exception(
-                    "Column " + column.name + " has indivisible uncompressed size " + toString(checksum->uncompressed_size)
-                    + ", sizeof " + toString(sizeof_field),
+                    "Uncompressed size of column " + column.name + "(" + toString(column_size)
+                    + ") is not divisible by the size of value (" + toString(sizeof_field) + ")",
                     ErrorCodes::LOGICAL_ERROR);
             }
 
@@ -670,29 +696,6 @@ bool MergeTreeDataPart::hasColumnFiles(const String & column) const
 }
 
 
-const MergeTreeDataPartChecksums::Checksum * MergeTreeDataPart::tryGetChecksum(const String & name, const String & ext) const
-{
-    if (checksums.empty())
-        return nullptr;
-
-    const auto & files = checksums.files;
-    const auto file_name = escapeForFileName(name) + ext;
-    auto it = files.find(file_name);
-
-    return (it == files.end()) ? nullptr : &it->second;
-}
-
-const MergeTreeDataPartChecksums::Checksum * MergeTreeDataPart::tryGetBinChecksum(const String & name) const
-{
-    return tryGetChecksum(name, ".bin");
-}
-
-const MergeTreeDataPartChecksums::Checksum * MergeTreeDataPart::tryGetMrkChecksum(const String & name) const
-{
-    return tryGetChecksum(name, ".mrk");
-}
-
-
 UInt64 MergeTreeDataPart::getIndexSizeInBytes() const
 {
     UInt64 res = 0;
@@ -709,18 +712,6 @@ UInt64 MergeTreeDataPart::getIndexSizeInAllocatedBytes() const
     return res;
 }
 
-UInt64 MergeTreeDataPart::getTotalMrkSizeInBytes() const
-{
-    UInt64 res = 0;
-    for (const NameAndTypePair & it : columns)
-    {
-        const Checksum * checksum = tryGetMrkChecksum(it.name);
-        if (checksum)
-            res += checksum->file_size;
-    }
-    return res;
-}
-
 String MergeTreeDataPart::stateToString(MergeTreeDataPart::State state)
 {
     switch (state)
diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.h b/dbms/src/Storages/MergeTree/MergeTreeDataPart.h
index a617dfedacd..c1c1274f7b9 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.h
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.h
@@ -29,22 +29,31 @@ struct MergeTreeDataPart
 
     MergeTreeDataPart(MergeTreeData & storage_, const String & name_);
 
-    const Checksum * tryGetChecksum(const String & name, const String & ext) const;
-    /// Returns checksum of column's binary file.
-    const Checksum * tryGetBinChecksum(const String & name) const;
-    /// Returns checksum of column's mrk file.
-    const Checksum * tryGetMrkChecksum(const String & name) const;
-
-    /// Returns the size of .bin file for column `name` if found, zero otherwise
-    UInt64 getColumnCompressedSize(const String & name) const;
-    UInt64 getColumnUncompressedSize(const String & name) const;
-    /// Returns the size of .mrk file for column `name` if found, zero otherwise
-    UInt64 getColumnMrkSize(const String & name) const;
-
     /// Returns the name of a column with minimum compressed size (as returned by getColumnSize()).
     /// If no checksums are present returns the name of the first physically existing column.
     String getColumnNameWithMinumumCompressedSize() const;
 
+    struct ColumnSize
+    {
+        size_t marks = 0;
+        size_t data_compressed = 0;
+        size_t data_uncompressed = 0;
+
+        void add(const ColumnSize & other)
+        {
+            marks += other.marks;
+            data_compressed += other.data_compressed;
+            data_uncompressed += other.data_uncompressed;
+        }
+    };
+
+    /// NOTE: Returns zeros if column files are not found in checksums.
+    /// NOTE: You must ensure that no ALTERs are in progress when calculating ColumnSizes.
+    ///   (either by locking columns_lock, or by locking table structure).
+    ColumnSize getColumnSize(const String & name, const IDataType & type) const;
+
+    ColumnSize getTotalColumnsSize() const;
+
     /// Returns full path to part dir
     String getFullPath() const;
 
@@ -68,8 +77,8 @@ struct MergeTreeDataPart
 
     size_t rows_count = 0;
     size_t marks_count = 0;
-    std::atomic<UInt64> size_in_bytes {0};  /// size in bytes, 0 - if not counted;
-                                            ///  is used from several threads without locks (it is changed with ALTER).
+    std::atomic<UInt64> bytes_on_disk {0};  /// 0 - if not counted;
+                                            /// Is used from several threads without locks (it is changed with ALTER).
     time_t modification_time = 0;
     /// When the part is removed from the working set. Changes once.
     mutable std::atomic<time_t> remove_time { std::numeric_limits<time_t>::max() };
@@ -208,7 +217,7 @@ struct MergeTreeDataPart
     ~MergeTreeDataPart();
 
     /// Calculate the total size of the entire directory with all the files
-    static UInt64 calculateTotalSize(const String & from);
+    static UInt64 calculateTotalSizeOnDisk(const String & from);
 
     void remove() const;
 
@@ -232,8 +241,6 @@ struct MergeTreeDataPart
     /// For data in RAM ('index')
     UInt64 getIndexSizeInBytes() const;
     UInt64 getIndexSizeInAllocatedBytes() const;
-    /// Total size of *.mrk files
-    UInt64 getTotalMrkSizeInBytes() const;
 
 private:
     /// Reads columns names and types from columns.txt
@@ -252,6 +259,8 @@ private:
     void loadPartitionAndMinMaxIndex();
 
     void checkConsistency(bool require_part_metadata);
+
+    ColumnSize getColumnSizeImpl(const String & name, const IDataType & type, std::unordered_set<String> * processed_substreams) const;
 };
 
 
diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp
index e85efa435fd..9d6d41fa9d5 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp
@@ -214,7 +214,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa
 
     ProfileEvents::increment(ProfileEvents::MergeTreeDataWriterRows, block.rows());
     ProfileEvents::increment(ProfileEvents::MergeTreeDataWriterUncompressedBytes, block.bytes());
-    ProfileEvents::increment(ProfileEvents::MergeTreeDataWriterCompressedBytes, new_data_part->size_in_bytes);
+    ProfileEvents::increment(ProfileEvents::MergeTreeDataWriterCompressedBytes, new_data_part->bytes_on_disk);
 
     return new_data_part;
 }
diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp
index d3162810a1c..6c9409be833 100644
--- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp
+++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp
@@ -347,7 +347,7 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart(
     new_part->columns = *total_column_list;
     new_part->index.assign(std::make_move_iterator(index_columns.begin()), std::make_move_iterator(index_columns.end()));
     new_part->checksums = checksums;
-    new_part->size_in_bytes = MergeTreeData::DataPart::calculateTotalSize(new_part->getFullPath());
+    new_part->bytes_on_disk = MergeTreeData::DataPart::calculateTotalSizeOnDisk(new_part->getFullPath());
 }
 
 void MergedBlockOutputStream::init()
diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp
index c5d4a41d496..32d7b263f80 100644
--- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp
+++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp
@@ -634,7 +634,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry(
 
             auto part = data.getPartIfExists(name, {MergeTreeDataPartState::PreCommitted, MergeTreeDataPartState::Committed, MergeTreeDataPartState::Outdated});
             if (part)
-                sum_parts_size_in_bytes += part->size_in_bytes;
+                sum_parts_size_in_bytes += part->bytes_on_disk;
         }
 
         if (merger.merges_blocker.isCancelled())
diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp
index 293c3254a8f..da205e9a293 100644
--- a/dbms/src/Storages/StorageMergeTree.cpp
+++ b/dbms/src/Storages/StorageMergeTree.cpp
@@ -116,7 +116,7 @@ BlockOutputStreamPtr StorageMergeTree::write(const ASTPtr & /*query*/, const Set
 bool StorageMergeTree::checkTableCanBeDropped() const
 {
     const_cast<MergeTreeData &>(getData()).recalculateColumnSizes();
-    context.checkTableCanBeDropped(database_name, table_name, getData().getTotalCompressedSize());
+    context.checkTableCanBeDropped(database_name, table_name, getData().getTotalActiveSizeInBytes());
     return true;
 }
 
@@ -334,7 +334,7 @@ bool StorageMergeTree::merge(
             part_log_elem.part_name = future_part.name;
 
             if (new_part)
-                part_log_elem.bytes_compressed_on_disk = new_part->size_in_bytes;
+                part_log_elem.bytes_compressed_on_disk = new_part->bytes_on_disk;
 
             part_log_elem.source_part_names.reserve(future_part.parts.size());
             for (const auto & source_part : future_part.parts)
diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp
index f11d9fe28b8..13d50500d09 100644
--- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp
@@ -1142,11 +1142,11 @@ void StorageReplicatedMergeTree::tryExecuteMerge(const StorageReplicatedMergeTre
         /// If entry is old enough, and have enough size, and part are exists in any replica,
         ///  then prefer fetching of merged part from replica.
 
-        size_t sum_parts_size_in_bytes = 0;
+        size_t sum_parts_bytes_on_disk = 0;
         for (const auto & part : parts)
-            sum_parts_size_in_bytes += part->size_in_bytes;
+            sum_parts_bytes_on_disk += part->bytes_on_disk;
 
-        if (sum_parts_size_in_bytes >= data.settings.prefer_fetch_merged_part_size_threshold)
+        if (sum_parts_bytes_on_disk >= data.settings.prefer_fetch_merged_part_size_threshold)
         {
             String replica = findReplicaHavingPart(entry.new_part_name, true);    /// NOTE excessive ZK requests for same data later, may remove.
             if (!replica.empty())
@@ -1205,7 +1205,7 @@ void StorageReplicatedMergeTree::tryExecuteMerge(const StorageReplicatedMergeTre
             part_log_elem.part_name = entry.new_part_name;
 
             if (part)
-                part_log_elem.bytes_compressed_on_disk = part->size_in_bytes;
+                part_log_elem.bytes_compressed_on_disk = part->bytes_on_disk;
 
             part_log_elem.source_part_names.reserve(parts.size());
             for (const auto & source_part : parts)
@@ -2293,7 +2293,7 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Strin
 
             if (part)
             {
-                part_log_elem.bytes_compressed_on_disk = part->size_in_bytes;
+                part_log_elem.bytes_compressed_on_disk = part->bytes_on_disk;
                 part_log_elem.rows = part->rows_count; /// Could be approximate (?)
             }
 
@@ -2949,7 +2949,7 @@ bool StorageReplicatedMergeTree::checkTableCanBeDropped() const
 {
     /// Consider only synchronized data
     const_cast<MergeTreeData &>(getData()).recalculateColumnSizes();
-    context.checkTableCanBeDropped(database_name, table_name, getData().getTotalCompressedSize());
+    context.checkTableCanBeDropped(database_name, table_name, getData().getTotalActiveSizeInBytes());
     return true;
 }
 
diff --git a/dbms/src/Storages/System/StorageSystemColumns.cpp b/dbms/src/Storages/System/StorageSystemColumns.cpp
index 18598b1c1dc..d42d8a80394 100644
--- a/dbms/src/Storages/System/StorageSystemColumns.cpp
+++ b/dbms/src/Storages/System/StorageSystemColumns.cpp
@@ -114,7 +114,7 @@ BlockInputStreams StorageSystemColumns::read(
 
         NamesAndTypesList columns;
         ColumnDefaults column_defaults;
-        MergeTreeData::ColumnSizes column_sizes;
+        MergeTreeData::ColumnSizeByName column_sizes;
 
         {
             StoragePtr storage = storages.at(std::make_pair(database_name, table_name));
diff --git a/dbms/src/Storages/System/StorageSystemParts.cpp b/dbms/src/Storages/System/StorageSystemParts.cpp
index a07e8bc26dd..b2f3d2208c7 100644
--- a/dbms/src/Storages/System/StorageSystemParts.cpp
+++ b/dbms/src/Storages/System/StorageSystemParts.cpp
@@ -17,28 +17,30 @@ namespace DB
 StorageSystemParts::StorageSystemParts(const std::string & name)
     : StorageSystemPartsBase(name,
     {
-        {"partition",           std::make_shared<DataTypeString>()},
-        {"name",                std::make_shared<DataTypeString>()},
-        {"active",              std::make_shared<DataTypeUInt8>()},
-        {"marks",               std::make_shared<DataTypeUInt64>()},
-        {"marks_size",          std::make_shared<DataTypeUInt64>()},
-        {"rows",                std::make_shared<DataTypeUInt64>()},
-        {"bytes",               std::make_shared<DataTypeUInt64>()},
-        {"modification_time",   std::make_shared<DataTypeDateTime>()},
-        {"remove_time",         std::make_shared<DataTypeDateTime>()},
-        {"refcount",            std::make_shared<DataTypeUInt32>()},
-        {"min_date",            std::make_shared<DataTypeDate>()},
-        {"max_date",            std::make_shared<DataTypeDate>()},
-        {"min_block_number",    std::make_shared<DataTypeInt64>()},
-        {"max_block_number",    std::make_shared<DataTypeInt64>()},
-        {"level",               std::make_shared<DataTypeUInt32>()},
-        {"primary_key_bytes_in_memory", std::make_shared<DataTypeUInt64>()},
-        {"primary_key_bytes_in_memory_allocated", std::make_shared<DataTypeUInt64>()},
+        {"partition",                                  std::make_shared<DataTypeString>()},
+        {"name",                                       std::make_shared<DataTypeString>()},
+        {"active",                                     std::make_shared<DataTypeUInt8>()},
+        {"marks",                                      std::make_shared<DataTypeUInt64>()},
+        {"rows",                                       std::make_shared<DataTypeUInt64>()},
+        {"bytes_on_disk",                              std::make_shared<DataTypeUInt64>()},
+        {"data_compressed_bytes",                      std::make_shared<DataTypeUInt64>()},
+        {"data_uncompressed_bytes",                    std::make_shared<DataTypeUInt64>()},
+        {"marks_bytes",                                std::make_shared<DataTypeUInt64>()},
+        {"modification_time",                          std::make_shared<DataTypeDateTime>()},
+        {"remove_time",                                std::make_shared<DataTypeDateTime>()},
+        {"refcount",                                   std::make_shared<DataTypeUInt32>()},
+        {"min_date",                                   std::make_shared<DataTypeDate>()},
+        {"max_date",                                   std::make_shared<DataTypeDate>()},
+        {"min_block_number",                           std::make_shared<DataTypeInt64>()},
+        {"max_block_number",                           std::make_shared<DataTypeInt64>()},
+        {"level",                                      std::make_shared<DataTypeUInt32>()},
+        {"primary_key_bytes_in_memory",                std::make_shared<DataTypeUInt64>()},
+        {"primary_key_bytes_in_memory_allocated",      std::make_shared<DataTypeUInt64>()},
 
-        {"database",            std::make_shared<DataTypeString>()},
-        {"table",               std::make_shared<DataTypeString>()},
-        {"engine",              std::make_shared<DataTypeString>()},
-        {"path",                std::make_shared<DataTypeString>()}
+        {"database",                                   std::make_shared<DataTypeString>()},
+        {"table",                                      std::make_shared<DataTypeString>()},
+        {"engine",                                     std::make_shared<DataTypeString>()},
+        {"path",                                       std::make_shared<DataTypeString>()},
     }
     )
 {
@@ -52,6 +54,7 @@ void StorageSystemParts::processNextStorage(MutableColumns & columns, const Stor
     {
         const auto & part = info.all_parts[part_number];
         auto part_state = info.all_parts_state[part_number];
+        MergeTreeDataPart::ColumnSize columns_size = part->getTotalColumnsSize();
 
         size_t i = 0;
         {
@@ -62,9 +65,11 @@ void StorageSystemParts::processNextStorage(MutableColumns & columns, const Stor
         columns[i++]->insert(part->name);
         columns[i++]->insert(static_cast<UInt64>(part_state == State::Committed));
         columns[i++]->insert(static_cast<UInt64>(part->marks_count));
-        columns[i++]->insert(static_cast<UInt64>(part->getTotalMrkSizeInBytes()));
         columns[i++]->insert(static_cast<UInt64>(part->rows_count));
-        columns[i++]->insert(static_cast<UInt64>(part->size_in_bytes));
+        columns[i++]->insert(static_cast<UInt64>(part->bytes_on_disk));
+        columns[i++]->insert(static_cast<UInt64>(columns_size.data_compressed));
+        columns[i++]->insert(static_cast<UInt64>(columns_size.data_uncompressed));
+        columns[i++]->insert(static_cast<UInt64>(columns_size.marks));
         columns[i++]->insert(static_cast<UInt64>(part->modification_time));
 
         time_t remove_time = part->remove_time.load(std::memory_order_relaxed);
diff --git a/dbms/src/Storages/System/StorageSystemPartsBase.cpp b/dbms/src/Storages/System/StorageSystemPartsBase.cpp
index 9d94655fc18..9a2538cbe6f 100644
--- a/dbms/src/Storages/System/StorageSystemPartsBase.cpp
+++ b/dbms/src/Storages/System/StorageSystemPartsBase.cpp
@@ -11,6 +11,7 @@
 #include <Storages/VirtualColumnUtils.h>
 #include <Databases/IDatabase.h>
 #include <Parsers/queryToString.h>
+#include <Parsers/ASTIdentifier.h>
 
 
 namespace DB
@@ -160,7 +161,7 @@ public:
 
             try
             {
-                /// For table not to be dropped.
+                /// For table not to be dropped and set of columns to remain constant.
                 info.table_lock = info.storage->lockStructure(false, __PRETTY_FUNCTION__);
             }
             catch (const Exception & e)
@@ -280,7 +281,31 @@ bool StorageSystemPartsBase::hasColumn(const String & column_name) const
 StorageSystemPartsBase::StorageSystemPartsBase(std::string name_, NamesAndTypesList && columns_)
     : name(std::move(name_))
 {
-    setColumns(ColumnsDescription(std::move(columns_)));
+    NamesAndTypesList aliases;
+    ColumnDefaults defaults;
+    auto add_alias = [&](const String & alias_name, const String & column_name)
+    {
+        DataTypePtr type;
+        for (const NameAndTypePair & col : columns_)
+        {
+            if (col.name == column_name)
+            {
+                type = col.type;
+                break;
+            }
+        }
+        if (!type)
+            throw Exception("No column " + column_name + " in table system." + name, ErrorCodes::LOGICAL_ERROR);
+
+        aliases.push_back({alias_name, type});
+        defaults[alias_name] = ColumnDefault{ColumnDefaultKind::Alias, std::make_shared<ASTIdentifier>(column_name)};
+    };
+
+    /// Add aliases for old column names for backwards compatibility.
+    add_alias("bytes", "bytes_on_disk");
+    add_alias("marks_size", "marks_bytes");
+
+    setColumns(ColumnsDescription(std::move(columns_), {}, std::move(aliases), std::move(defaults)));
 }
 
 }
diff --git a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp
index 8e55625c5cd..921b32b6c0a 100644
--- a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp
+++ b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp
@@ -16,39 +16,43 @@ namespace DB
 
 
 StorageSystemPartsColumns::StorageSystemPartsColumns(const std::string & name)
-        : StorageSystemPartsBase(name,
-              {
-                      {"partition",           std::make_shared<DataTypeString>()},
-                      {"name",                std::make_shared<DataTypeString>()},
-                      {"active",              std::make_shared<DataTypeUInt8>()},
-                      {"marks",               std::make_shared<DataTypeUInt64>()},
-                      {"marks_bytes_in_block", std::make_shared<DataTypeUInt64>()},
-                      {"rows",                std::make_shared<DataTypeUInt64>()},
-                      {"bytes",               std::make_shared<DataTypeUInt64>()},
-                      {"modification_time",   std::make_shared<DataTypeDateTime>()},
-                      {"remove_time",         std::make_shared<DataTypeDateTime>()},
-                      {"refcount",            std::make_shared<DataTypeUInt32>()},
-                      {"min_date",            std::make_shared<DataTypeDate>()},
-                      {"max_date",            std::make_shared<DataTypeDate>()},
-                      {"min_block_number",    std::make_shared<DataTypeInt64>()},
-                      {"max_block_number",    std::make_shared<DataTypeInt64>()},
-                      {"level",               std::make_shared<DataTypeUInt32>()},
-                      {"primary_key_bytes_in_memory", std::make_shared<DataTypeUInt64>()},
-                      {"primary_key_bytes_in_memory_allocated", std::make_shared<DataTypeUInt64>()},
+    : StorageSystemPartsBase(name,
+    {
+        {"partition",                                  std::make_shared<DataTypeString>()},
+        {"name",                                       std::make_shared<DataTypeString>()},
+        {"active",                                     std::make_shared<DataTypeUInt8>()},
+        {"marks",                                      std::make_shared<DataTypeUInt64>()},
+        {"rows",                                       std::make_shared<DataTypeUInt64>()},
+        {"bytes_on_disk",                              std::make_shared<DataTypeUInt64>()},
+        {"data_compressed_bytes",                      std::make_shared<DataTypeUInt64>()},
+        {"data_uncompressed_bytes",                    std::make_shared<DataTypeUInt64>()},
+        {"marks_bytes",                                std::make_shared<DataTypeUInt64>()},
+        {"modification_time",                          std::make_shared<DataTypeDateTime>()},
+        {"remove_time",                                std::make_shared<DataTypeDateTime>()},
+        {"refcount",                                   std::make_shared<DataTypeUInt32>()},
+        {"min_date",                                   std::make_shared<DataTypeDate>()},
+        {"max_date",                                   std::make_shared<DataTypeDate>()},
+        {"min_block_number",                           std::make_shared<DataTypeInt64>()},
+        {"max_block_number",                           std::make_shared<DataTypeInt64>()},
+        {"level",                                      std::make_shared<DataTypeUInt32>()},
+        {"primary_key_bytes_in_memory",                std::make_shared<DataTypeUInt64>()},
+        {"primary_key_bytes_in_memory_allocated",      std::make_shared<DataTypeUInt64>()},
 
-                      {"database",            std::make_shared<DataTypeString>()},
-                      {"table",               std::make_shared<DataTypeString>()},
-                      {"engine",              std::make_shared<DataTypeString>()},
-                      {"path",                std::make_shared<DataTypeString>()},
-                      {"column",              std::make_shared<DataTypeString>()},
-                      {"type",                std::make_shared<DataTypeString>() },
-                      {"default_kind",        std::make_shared<DataTypeString>() },
-                      {"default_expression",  std::make_shared<DataTypeString>() },
-                      {"data_compressed_bytes",   std::make_shared<DataTypeUInt64>() },
-                      {"data_uncompressed_bytes", std::make_shared<DataTypeUInt64>() },
-                      {"marks_bytes_in_column",   std::make_shared<DataTypeUInt64>() },
-              }
-        )
+        {"database",                                   std::make_shared<DataTypeString>()},
+        {"table",                                      std::make_shared<DataTypeString>()},
+        {"engine",                                     std::make_shared<DataTypeString>()},
+        {"path",                                       std::make_shared<DataTypeString>()},
+
+        {"column",                                     std::make_shared<DataTypeString>()},
+        {"type",                                       std::make_shared<DataTypeString>() },
+        {"default_kind",                               std::make_shared<DataTypeString>() },
+        {"default_expression",                         std::make_shared<DataTypeString>() },
+        {"column_bytes_on_disk",                       std::make_shared<DataTypeUInt64>() },
+        {"column_data_compressed_bytes",               std::make_shared<DataTypeUInt64>() },
+        {"column_data_uncompressed_bytes",             std::make_shared<DataTypeUInt64>() },
+        {"column_marks_bytes",                         std::make_shared<DataTypeUInt64>() },
+    }
+    )
 {
 }
 
@@ -84,8 +88,8 @@ void StorageSystemPartsColumns::processNextStorage(MutableColumns & columns, con
     {
         const auto & part = info.all_parts[part_number];
         auto part_state = info.all_parts_state[part_number];
+        auto columns_size = part->getTotalColumnsSize();
 
-        auto total_mrk_size_in_bytes = part->getTotalMrkSizeInBytes();
         /// For convenience, in returned refcount, don't add references that was due to local variables in this method: all_parts, active_parts.
         auto use_count = part.use_count() - 1;
         auto min_date = part->getMinDate();
@@ -106,10 +110,12 @@ void StorageSystemPartsColumns::processNextStorage(MutableColumns & columns, con
             columns[j++]->insert(part->name);
             columns[j++]->insert(static_cast<UInt64>(part_state == State::Committed));
             columns[j++]->insert(static_cast<UInt64>(part->marks_count));
-            columns[j++]->insert(static_cast<UInt64>(total_mrk_size_in_bytes));
 
             columns[j++]->insert(static_cast<UInt64>(part->rows_count));
-            columns[j++]->insert(static_cast<UInt64>(part->size_in_bytes));
+            columns[j++]->insert(static_cast<UInt64>(part->bytes_on_disk));
+            columns[j++]->insert(static_cast<UInt64>(columns_size.data_compressed));
+            columns[j++]->insert(static_cast<UInt64>(columns_size.data_uncompressed));
+            columns[j++]->insert(static_cast<UInt64>(columns_size.marks));
             columns[j++]->insert(static_cast<UInt64>(part->modification_time));
             columns[j++]->insert(static_cast<UInt64>(part->remove_time.load(std::memory_order_relaxed)));
 
@@ -142,9 +148,11 @@ void StorageSystemPartsColumns::processNextStorage(MutableColumns & columns, con
                 columns[j++]->insertDefault();
             }
 
-            columns[j++]->insert(static_cast<UInt64>(part->getColumnCompressedSize(column.name)));
-            columns[j++]->insert(static_cast<UInt64>(part->getColumnUncompressedSize(column.name)));
-            columns[j++]->insert(static_cast<UInt64>(part->getColumnMrkSize(column.name)));
+            MergeTreeDataPart::ColumnSize column_size = part->getColumnSize(column.name, *column.type);
+            columns[j++]->insert(static_cast<UInt64>(column_size.data_compressed + column_size.marks));
+            columns[j++]->insert(static_cast<UInt64>(column_size.data_compressed));
+            columns[j++]->insert(static_cast<UInt64>(column_size.data_uncompressed));
+            columns[j++]->insert(static_cast<UInt64>(column_size.marks));
 
             if (has_state_column)
                 columns[j++]->insert(part->stateString());

From d63d9a4ffbfebb7132e4ea5894b6af27e4b4a221 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Date: Fri, 23 Mar 2018 22:56:24 +0300
Subject: [PATCH 207/244] fix test [#CLICKHOUSE-3590]

---
 dbms/src/Databases/DatabaseDictionary.cpp    | 36 ++++++++++++++------
 dbms/src/Databases/DatabaseDictionary.h      | 26 +++++++-------
 dbms/src/Databases/DatabaseMemory.cpp        |  7 ----
 dbms/src/Databases/DatabaseMemory.h          |  9 ++---
 dbms/src/Databases/DatabaseOrdinary.cpp      | 34 ++++++++++--------
 dbms/src/Databases/DatabaseOrdinary.h        |  7 ++--
 dbms/src/Databases/IDatabase.h               | 12 +++----
 dbms/src/Interpreters/Context.cpp            |  1 +
 dbms/src/Interpreters/ExternalDictionaries.h |  5 +++
 dbms/src/Interpreters/ExternalLoader.cpp     | 24 ++++++++++---
 dbms/src/Interpreters/ExternalLoader.h       |  3 ++
 11 files changed, 99 insertions(+), 65 deletions(-)

diff --git a/dbms/src/Databases/DatabaseDictionary.cpp b/dbms/src/Databases/DatabaseDictionary.cpp
index c68c801e4cb..3979f71f602 100644
--- a/dbms/src/Databases/DatabaseDictionary.cpp
+++ b/dbms/src/Databases/DatabaseDictionary.cpp
@@ -13,10 +13,11 @@ namespace DB
 {
 namespace ErrorCodes
 {
-extern const int TABLE_ALREADY_EXISTS;
-extern const int UNKNOWN_TABLE;
-extern const int LOGICAL_ERROR;
-extern const int CANNOT_GET_CREATE_TABLE_QUERY;
+    extern const int TABLE_ALREADY_EXISTS;
+    extern const int UNKNOWN_TABLE;
+    extern const int LOGICAL_ERROR;
+    extern const int CANNOT_GET_CREATE_TABLE_QUERY;
+    extern const int SYNTAX_ERROR;
 }
 
 DatabaseDictionary::DatabaseDictionary(const String & name_, const Context & context)
@@ -163,15 +164,17 @@ time_t DatabaseDictionary::getTableMetadataModificationTime(
     return static_cast<time_t>(0);
 }
 
-ASTPtr DatabaseDictionary::getCreateTableQuery(
-    const Context & context,
-    const String & table_name) const
+ASTPtr DatabaseDictionary::getCreateTableQueryImpl(const Context & context,
+                                                   const String & table_name, bool throw_on_error) const
 {
     String query;
     {
         WriteBufferFromString buffer(query);
 
-        auto dictionary = context.getExternalDictionaries().getDictionary(table_name);
+        const auto & dictionaries = context.getExternalDictionaries();
+        auto dictionary = throw_on_error ? dictionaries.getDictionary(table_name)
+                                         : dictionaries.tryGetDictionary(table_name);
+
         auto names_and_types = StorageDictionary::getNamesAndTypes(dictionary->getStructure());
         buffer << "CREATE TABLE " << backQuoteIfNeed(name) << '.' << backQuoteIfNeed(table_name) << " (";
         buffer << StorageDictionary::generateNamesAndTypesDescription(names_and_types.begin(), names_and_types.end());
@@ -179,12 +182,25 @@ ASTPtr DatabaseDictionary::getCreateTableQuery(
     }
 
     ParserCreateQuery parser;
-    return parseQuery(parser, query.data(), query.data() + query.size(), "");
+    const char * pos = query.data();
+    std::string error_message;
+    auto ast = tryParseQuery(parser, pos, pos + query.size(), error_message,
+            /* hilite = */ false, "", /* allow_multi_statements = */ false);
+
+    if (!ast && throw_on_error)
+        throw Exception(error_message, ErrorCodes::SYNTAX_ERROR);
+
+    return ast;
+}
+
+ASTPtr DatabaseDictionary::getCreateTableQuery(const Context & context, const String & table_name) const
+{
+    return getCreateTableQueryImpl(context, table_name, true);
 }
 
 ASTPtr DatabaseDictionary::tryGetCreateTableQuery(const Context & context, const String & table_name) const
 {
-    return getCreateTableQuery(context, table_name);
+    return getCreateTableQueryImpl(context, table_name, false);
 }
 
 ASTPtr DatabaseDictionary::getCreateDatabaseQuery(const Context & /*context*/) const
diff --git a/dbms/src/Databases/DatabaseDictionary.h b/dbms/src/Databases/DatabaseDictionary.h
index e2a036362b0..a234694b768 100644
--- a/dbms/src/Databases/DatabaseDictionary.h
+++ b/dbms/src/Databases/DatabaseDictionary.h
@@ -22,16 +22,6 @@ class ExternalDictionaries;
  */
 class DatabaseDictionary : public IDatabase
 {
-private:
-    const String name;
-    mutable std::mutex mutex;
-    const ExternalDictionaries & external_dictionaries;
-    std::unordered_set<String> deleted_tables;
-
-    Poco::Logger * log;
-
-    Tables loadTables();
-
 public:
     DatabaseDictionary(const String & name_, const Context & context);
 
@@ -94,13 +84,25 @@ public:
         const String & table_name) const override;
 
     ASTPtr tryGetCreateTableQuery(
-        const Context & context,
-        const String & table_name) const override;
+            const Context & context,
+            const String & table_name) const override;
 
     ASTPtr getCreateDatabaseQuery(const Context & context) const override;
 
     void shutdown() override;
     void drop() override;
+
+private:
+    const String name;
+    mutable std::mutex mutex;
+    const ExternalDictionaries & external_dictionaries;
+    std::unordered_set<String> deleted_tables;
+
+    Poco::Logger * log;
+
+    Tables loadTables();
+
+    ASTPtr getCreateTableQueryImpl(const Context & context, const String & table_name, bool throw_on_error) const;
 };
 
 }
diff --git a/dbms/src/Databases/DatabaseMemory.cpp b/dbms/src/Databases/DatabaseMemory.cpp
index 8eacc7f856f..b50fb78468e 100644
--- a/dbms/src/Databases/DatabaseMemory.cpp
+++ b/dbms/src/Databases/DatabaseMemory.cpp
@@ -128,13 +128,6 @@ ASTPtr DatabaseMemory::getCreateTableQuery(
     throw Exception("There is no CREATE TABLE query for DatabaseMemory tables", ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY);
 }
 
-ASTPtr DatabaseMemory::tryGetCreateTableQuery(
-        const Context &,
-        const String &) const
-{
-    throw Exception("There is no CREATE TABLE query for DatabaseMemory tables", ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY);
-}
-
 ASTPtr DatabaseMemory::getCreateDatabaseQuery(const Context &) const
 {
     throw Exception("There is no CREATE DATABASE query for DatabaseMemory", ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY);
diff --git a/dbms/src/Databases/DatabaseMemory.h b/dbms/src/Databases/DatabaseMemory.h
index 1b5f0a20c62..2c68e0a093b 100644
--- a/dbms/src/Databases/DatabaseMemory.h
+++ b/dbms/src/Databases/DatabaseMemory.h
@@ -80,13 +80,8 @@ public:
         const Context & context,
         const String & table_name) override;
 
-    ASTPtr getCreateTableQuery(
-            const Context & context,
-            const String & table_name) const override;
-
-    ASTPtr tryGetCreateTableQuery(
-            const Context & context,
-            const String & table_name) const override;
+    ASTPtr getCreateTableQuery(const Context & context, const String & table_name) const override;
+    ASTPtr tryGetCreateTableQuery(const Context &, const String &) const override { return nullptr; }
 
     ASTPtr getCreateDatabaseQuery(const Context & context) const override;
 
diff --git a/dbms/src/Databases/DatabaseOrdinary.cpp b/dbms/src/Databases/DatabaseOrdinary.cpp
index 4f63834d396..027ce07148d 100644
--- a/dbms/src/Databases/DatabaseOrdinary.cpp
+++ b/dbms/src/Databases/DatabaseOrdinary.cpp
@@ -31,6 +31,7 @@ namespace ErrorCodes
     extern const int FILE_DOESNT_EXIST;
     extern const int LOGICAL_ERROR;
     extern const int CANNOT_GET_CREATE_TABLE_QUERY;
+    extern const int SYNTAX_ERROR;
 }
 
 
@@ -335,7 +336,7 @@ void DatabaseOrdinary::removeTable(
     }
 }
 
-static ASTPtr getQueryFromMetadata(const String & metadata_path)
+static ASTPtr getQueryFromMetadata(const String & metadata_path, bool throw_on_error = true)
 {
     if (!Poco::File(metadata_path).exists())
         return nullptr;
@@ -348,12 +349,20 @@ static ASTPtr getQueryFromMetadata(const String & metadata_path)
     }
 
     ParserCreateQuery parser;
-    return parseQuery(parser, query.data(), query.data() + query.size(), "in file " + metadata_path);
+    const char * pos = query.data();
+    std::string error_message;
+    auto ast = tryParseQuery(parser, pos, pos + query.size(), error_message, /* hilite = */ false,
+                             "in file " + metadata_path, /* allow_multi_statements = */ false);
+
+    if (!ast && throw_on_error)
+        throw Exception(error_message, ErrorCodes::SYNTAX_ERROR);
+
+    return ast;
 }
 
-static ASTPtr getCreateQueryFromMetadata(const String & metadata_path, const String & database)
+static ASTPtr getCreateQueryFromMetadata(const String & metadata_path, const String & database, bool throw_on_error)
 {
-    ASTPtr ast = getQueryFromMetadata(metadata_path);
+    ASTPtr ast = getQueryFromMetadata(metadata_path, throw_on_error);
 
     if (ast)
     {
@@ -428,21 +437,18 @@ time_t DatabaseOrdinary::getTableMetadataModificationTime(
     }
 }
 
-
-ASTPtr DatabaseOrdinary::getCreateTableQueryImpl(const Context & context, const String & table_name, bool try_get) const
+ASTPtr DatabaseOrdinary::getCreateTableQueryImpl(const Context & context,
+                                                 const String & table_name, bool throw_on_error) const
 {
     ASTPtr ast;
 
     auto table_metadata_path = detail::getTableMetadataPath(metadata_path, table_name);
-    ast = getCreateQueryFromMetadata(table_metadata_path, name);
-    if (!ast)
+    ast = getCreateQueryFromMetadata(table_metadata_path, name, throw_on_error);
+    if (!ast && throw_on_error)
     {
         /// Handle system.* tables for which there are no table.sql files.
         bool has_table = tryGetTable(context, table_name) != nullptr;
 
-        if (has_table && try_get)
-            return nullptr;
-
         auto msg = has_table
                    ? "There is no CREATE TABLE query for table "
                    : "There is no metadata file for table ";
@@ -455,12 +461,12 @@ ASTPtr DatabaseOrdinary::getCreateTableQueryImpl(const Context & context, const
 
 ASTPtr DatabaseOrdinary::getCreateTableQuery(const Context & context, const String & table_name) const
 {
-    return getCreateTableQueryImpl(context, table_name, false);
+    return getCreateTableQueryImpl(context, table_name, true);
 }
 
 ASTPtr DatabaseOrdinary::tryGetCreateTableQuery(const Context & context, const String & table_name) const
 {
-    return getCreateTableQueryImpl(context, table_name, true);
+    return getCreateTableQueryImpl(context, table_name, false);
 }
 
 ASTPtr DatabaseOrdinary::getCreateDatabaseQuery(const Context & /*context*/) const
@@ -468,7 +474,7 @@ ASTPtr DatabaseOrdinary::getCreateDatabaseQuery(const Context & /*context*/) con
     ASTPtr ast;
 
     auto database_metadata_path = detail::getDatabaseMetadataPath(metadata_path);
-    ast = getCreateQueryFromMetadata(database_metadata_path, name);
+    ast = getCreateQueryFromMetadata(database_metadata_path, name, true);
     if (!ast)
     {
         /// Handle databases (such as default) for which there are no database.sql files.
diff --git a/dbms/src/Databases/DatabaseOrdinary.h b/dbms/src/Databases/DatabaseOrdinary.h
index cae6686d5ba..86279eff4c7 100644
--- a/dbms/src/Databases/DatabaseOrdinary.h
+++ b/dbms/src/Databases/DatabaseOrdinary.h
@@ -60,8 +60,8 @@ public:
         const String & table_name) const override;
 
     ASTPtr tryGetCreateTableQuery(
-            const Context & context,
-            const String & table_name) const override;
+        const Context & context,
+        const String & table_name) const override;
 
     ASTPtr getCreateDatabaseQuery(const Context & context) const override;
 
@@ -74,7 +74,8 @@ public:
 
 private:
     void startupTables(ThreadPool * thread_pool);
-    ASTPtr getCreateTableQueryImpl(const Context & context, const String & table_name, bool try_get) const;
+
+    ASTPtr getCreateTableQueryImpl(const Context & context, const String & table_name, bool throw_on_error) const;
 };
 
 }
diff --git a/dbms/src/Databases/IDatabase.h b/dbms/src/Databases/IDatabase.h
index 6b84f3fa165..7416fe70fb1 100644
--- a/dbms/src/Databases/IDatabase.h
+++ b/dbms/src/Databases/IDatabase.h
@@ -125,14 +125,12 @@ public:
         const String & name) = 0;
 
     /// Get the CREATE TABLE query for the table. It can also provide information for detached tables for which there is metadata.
-    virtual ASTPtr getCreateTableQuery(
-        const Context & context,
-        const String & name) const = 0;
+    virtual ASTPtr tryGetCreateTableQuery(const Context & context, const String & name) const = 0;
 
-    /// Like getCreateTableQuery, but don't throw for existing tables without metadata.
-    virtual ASTPtr tryGetCreateTableQuery(
-        const Context & context,
-        const String & name) const = 0;
+    virtual ASTPtr getCreateTableQuery(const Context & context, const String & name) const
+    {
+        return tryGetCreateTableQuery(context, name);
+    }
 
     /// Get the CREATE DATABASE query for current database.
     virtual ASTPtr getCreateDatabaseQuery(const Context & context) const = 0;
diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp
index e3e0e268f98..b011a1a3d3c 100644
--- a/dbms/src/Interpreters/Context.cpp
+++ b/dbms/src/Interpreters/Context.cpp
@@ -82,6 +82,7 @@ namespace ErrorCodes
     extern const int TABLE_SIZE_EXCEEDS_MAX_DROP_SIZE_LIMIT;
     extern const int SESSION_NOT_FOUND;
     extern const int SESSION_IS_LOCKED;
+    extern const int CANNOT_GET_CREATE_TABLE_QUERY;
 }
 
 
diff --git a/dbms/src/Interpreters/ExternalDictionaries.h b/dbms/src/Interpreters/ExternalDictionaries.h
index cc03c9b9389..0b00ab9f632 100644
--- a/dbms/src/Interpreters/ExternalDictionaries.h
+++ b/dbms/src/Interpreters/ExternalDictionaries.h
@@ -31,6 +31,11 @@ public:
         return std::static_pointer_cast<IDictionaryBase>(getLoadable(name));
     }
 
+    DictPtr tryGetDictionary(const std::string & name) const
+    {
+        return std::static_pointer_cast<IDictionaryBase>(tryGetLoadable(name));
+    }
+
 protected:
 
     std::unique_ptr<IExternalLoadable> create(const std::string & name, const Configuration & config,
diff --git a/dbms/src/Interpreters/ExternalLoader.cpp b/dbms/src/Interpreters/ExternalLoader.cpp
index 527c3d495ea..fc25327558e 100644
--- a/dbms/src/Interpreters/ExternalLoader.cpp
+++ b/dbms/src/Interpreters/ExternalLoader.cpp
@@ -385,21 +385,35 @@ void ExternalLoader::reload(const std::string & name)
         throw Exception("Failed to load " + object_name + " '" + name + "' during the reload process", ErrorCodes::BAD_ARGUMENTS);
 }
 
-ExternalLoader::LoadablePtr ExternalLoader::getLoadable(const std::string & name) const
+ExternalLoader::LoadablePtr ExternalLoader::getLoadableImpl(const std::string & name, bool throw_on_error) const
 {
     const std::lock_guard<std::mutex> lock{map_mutex};
 
     const auto it = loadable_objects.find(name);
     if (it == std::end(loadable_objects))
-        throw Exception("No such " + object_name + ": " + name, ErrorCodes::BAD_ARGUMENTS);
+    {
+        if (throw_on_error)
+            throw Exception("No such " + object_name + ": " + name, ErrorCodes::BAD_ARGUMENTS);
+        return nullptr;
+    }
 
-    if (!it->second.loadable)
-        it->second.exception ? std::rethrow_exception(it->second.exception) :
-        throw Exception{object_name + " '" + name + "' is not loaded", ErrorCodes::LOGICAL_ERROR};
+    if (!it->second.loadable && throw_on_error)
+        it->second.exception ? std::rethrow_exception(it->second.exception)
+                             : throw Exception{object_name + " '" + name + "' is not loaded", ErrorCodes::LOGICAL_ERROR};
 
     return it->second.loadable;
 }
 
+ExternalLoader::LoadablePtr ExternalLoader::getLoadable(const std::string & name) const
+{
+    return getLoadableImpl(name, true);
+}
+
+ExternalLoader::LoadablePtr ExternalLoader::tryGetLoadable(const std::string & name) const
+{
+    return getLoadableImpl(name, false);
+}
+
 ExternalLoader::LockedObjectsMap ExternalLoader::getObjectsMap() const
 {
     return LockedObjectsMap(map_mutex, loadable_objects);
diff --git a/dbms/src/Interpreters/ExternalLoader.h b/dbms/src/Interpreters/ExternalLoader.h
index a2f64b61dc7..2cf42942bc9 100644
--- a/dbms/src/Interpreters/ExternalLoader.h
+++ b/dbms/src/Interpreters/ExternalLoader.h
@@ -104,6 +104,7 @@ public:
     void reload(const std::string & name);
 
     LoadablePtr getLoadable(const std::string & name) const;
+    LoadablePtr tryGetLoadable(const std::string & name) const;
 
 protected:
     virtual std::unique_ptr<IExternalLoadable> create(const std::string & name, const Configuration & config,
@@ -172,6 +173,8 @@ private:
     void reloadAndUpdate(bool throw_on_error = false);
 
     void reloadPeriodically();
+
+    LoadablePtr getLoadableImpl(const std::string & name, bool throw_on_error) const;
 };
 
 }

From 190964f98e5f6db860bc2c185654cdfca8d718bd Mon Sep 17 00:00:00 2001
From: Vitaliy Lyudvichenko <vludv@yandex-team.ru>
Date: Mon, 26 Mar 2018 21:39:28 +0300
Subject: [PATCH 208/244] Avoid extra waits in copier. [#CLICKHOUSE-2]

---
 dbms/src/Server/ClusterCopier.cpp             | 36 +++++++++++--------
 .../DistributedBlockOutputStream.cpp          |  9 ++---
 .../DistributedBlockOutputStream.h            |  1 +
 3 files changed, 28 insertions(+), 18 deletions(-)

diff --git a/dbms/src/Server/ClusterCopier.cpp b/dbms/src/Server/ClusterCopier.cpp
index 3bcbad95d76..74d5eb5753d 100644
--- a/dbms/src/Server/ClusterCopier.cpp
+++ b/dbms/src/Server/ClusterCopier.cpp
@@ -1128,7 +1128,7 @@ protected:
         catch (const zkutil::KeeperException & e)
         {
             LOG_INFO(log, "A ZooKeeper error occurred while checking partition " << partition_name
-                          << ". Will recheck the partition. Error: " << e.what());
+                          << ". Will recheck the partition. Error: " << e.displayText());
             return false;
         }
 
@@ -1285,6 +1285,7 @@ protected:
             Stopwatch watch;
             TasksShard expected_shards;
             size_t num_failed_shards = 0;
+            bool previous_shard_is_instantly_finished = false;
 
             ++cluster_partition.total_tries;
 
@@ -1330,15 +1331,20 @@ protected:
 
                 expected_shards.emplace_back(shard);
 
+                /// Do not sleep if there is a sequence of already processed shards to increase startup
+                bool sleep_before_execution = !previous_shard_is_instantly_finished && shard->priority.is_remote;
                 PartitionTaskStatus task_status = PartitionTaskStatus::Error;
+                bool was_error = false;
                 for (size_t try_num = 0; try_num < max_shard_partition_tries; ++try_num)
                 {
-                    task_status = tryProcessPartitionTask(partition);
+                    task_status = tryProcessPartitionTask(partition, sleep_before_execution);
 
                     /// Exit if success
                     if (task_status == PartitionTaskStatus::Finished)
                         break;
 
+                    was_error = true;
+
                     /// Skip if the task is being processed by someone
                     if (task_status == PartitionTaskStatus::Active)
                         break;
@@ -1349,6 +1355,8 @@ protected:
 
                 if (task_status == PartitionTaskStatus::Error)
                     ++num_failed_shards;
+
+                previous_shard_is_instantly_finished = !was_error;
             }
 
             cluster_partition.elapsed_time_seconds += watch.elapsedSeconds();
@@ -1415,13 +1423,13 @@ protected:
         Error,
     };
 
-    PartitionTaskStatus tryProcessPartitionTask(ShardPartition & task_partition)
+    PartitionTaskStatus tryProcessPartitionTask(ShardPartition & task_partition, bool sleep_before_execution)
     {
         PartitionTaskStatus res;
 
         try
         {
-            res = processPartitionTaskImpl(task_partition);
+            res = processPartitionTaskImpl(task_partition, sleep_before_execution);
         }
         catch (...)
         {
@@ -1442,7 +1450,7 @@ protected:
         return res;
     }
 
-    PartitionTaskStatus processPartitionTaskImpl(ShardPartition & task_partition)
+    PartitionTaskStatus processPartitionTaskImpl(ShardPartition & task_partition, bool sleep_before_execution)
     {
         TaskShard & task_shard = task_partition.task_shard;
         TaskTable & task_table = task_shard.task_table;
@@ -1482,7 +1490,7 @@ protected:
         };
 
         /// Load balancing
-        auto worker_node_holder = createTaskWorkerNodeAndWaitIfNeed(zookeeper, current_task_status_path, task_shard.priority.is_remote);
+        auto worker_node_holder = createTaskWorkerNodeAndWaitIfNeed(zookeeper, current_task_status_path, sleep_before_execution);
 
         LOG_DEBUG(log, "Processing " << current_task_status_path);
 
@@ -1656,7 +1664,7 @@ protected:
                 }
 
                 using ExistsFuture = zkutil::ZooKeeper::ExistsFuture;
-                auto future_is_dirty_checker = std::make_unique<ExistsFuture>(zookeeper->asyncExists(is_dirty_flag_path));
+                std::unique_ptr<ExistsFuture> future_is_dirty_checker;
 
                 Stopwatch watch(CLOCK_MONOTONIC_COARSE);
                 constexpr size_t check_period_milliseconds = 500;
@@ -1667,9 +1675,15 @@ protected:
                     if (zookeeper->expired())
                         throw Exception("ZooKeeper session is expired, cancel INSERT SELECT", ErrorCodes::UNFINISHED);
 
-                    if (future_is_dirty_checker != nullptr)
+                    if (!future_is_dirty_checker)
+                        future_is_dirty_checker = std::make_unique<ExistsFuture>(zookeeper->asyncExists(is_dirty_flag_path));
+
+                    /// check_period_milliseconds should less than average insert time of single block
+                    /// Otherwise, the insertion will slow a little bit
+                    if (watch.elapsedMilliseconds() >= check_period_milliseconds)
                     {
                         zkutil::ZooKeeper::StatAndExists status;
+
                         try
                         {
                             status = future_is_dirty_checker->get();
@@ -1689,12 +1703,6 @@ protected:
                             throw Exception("Partition is dirty, cancel INSERT SELECT", ErrorCodes::UNFINISHED);
                     }
 
-                    if (watch.elapsedMilliseconds() >= check_period_milliseconds)
-                    {
-                        watch.restart();
-                        future_is_dirty_checker = std::make_unique<ExistsFuture>(zookeeper->asyncExists(is_dirty_flag_path));
-                    }
-
                     return false;
                 };
 
diff --git a/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp
index 98416437497..d739badba88 100644
--- a/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp
+++ b/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp
@@ -193,13 +193,12 @@ ThreadPool::Job DistributedBlockOutputStream::runWritingJob(DistributedBlockOutp
     auto memory_tracker = current_memory_tracker;
     return [this, memory_tracker, &job, &current_block]()
     {
-        SCOPE_EXIT({++finished_jobs_count;});
-
-        Stopwatch watch;
         ++job.blocks_started;
 
         SCOPE_EXIT({
-            UInt64 elapsed_time_for_block_ms = watch.elapsedMilliseconds();
+            ++finished_jobs_count;
+
+            UInt64 elapsed_time_for_block_ms = watch_current_block.elapsedMilliseconds();
             job.elapsed_time_ms += elapsed_time_for_block_ms;
             job.max_elapsed_time_for_block_ms = std::max(job.max_elapsed_time_for_block_ms, elapsed_time_for_block_ms);
         });
@@ -322,6 +321,8 @@ void DistributedBlockOutputStream::writeSync(const Block & block)
         watch.restart();
     }
 
+    watch_current_block.restart();
+
     if (num_shards > 1)
     {
         auto current_selector = createSelector(block);
diff --git a/dbms/src/Storages/Distributed/DistributedBlockOutputStream.h b/dbms/src/Storages/Distributed/DistributedBlockOutputStream.h
index 0b37d51c856..fe9e765021b 100644
--- a/dbms/src/Storages/Distributed/DistributedBlockOutputStream.h
+++ b/dbms/src/Storages/Distributed/DistributedBlockOutputStream.h
@@ -88,6 +88,7 @@ private:
     /// Sync-related stuff
     UInt64 insert_timeout; // in seconds
     Stopwatch watch;
+    Stopwatch watch_current_block;
     std::optional<ThreadPool> pool;
     ThrottlerPtr throttler;
     String query_string;

From 348b7c3bda816630a17004fe4bbb0e1e0dd3e2fb Mon Sep 17 00:00:00 2001
From: proller <proller@github.com>
Date: Tue, 27 Mar 2018 18:08:01 +0300
Subject: [PATCH 209/244] clickhouse-test: Fix run without server config

---
 dbms/tests/clickhouse-test | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/dbms/tests/clickhouse-test b/dbms/tests/clickhouse-test
index 4e9886f8b70..b88eab93c7c 100755
--- a/dbms/tests/clickhouse-test
+++ b/dbms/tests/clickhouse-test
@@ -81,9 +81,12 @@ def main(args):
     # TODO ! use clickhouse-extract-from-config here:
     if args.zookeeper is None:
         code, out = commands.getstatusoutput(args.binary + "-extract-from-config --try --config " + args.configserver + ' --key zookeeper | grep . | wc -l')
-        if int(out) > 0:
-            args.zookeeper = True
-        else:
+        try:
+            if int(out) > 0:
+                args.zookeeper = True
+            else:
+                args.zookeeper = False
+        except ValueError:
             args.zookeeper = False
 
     if args.shard is None:

From d58483b08415037baf8e7e7bd4aea193b8a95fa7 Mon Sep 17 00:00:00 2001
From: Alexey Zatelepin <ztlpn@yandex-team.ru>
Date: Tue, 27 Mar 2018 20:08:41 +0300
Subject: [PATCH 210/244] squash blocks when inserting into materialized views
 #2084 [#CLICKHOUSE-3540]

---
 dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp
index b1f0dc8a9cd..815b8ebf303 100644
--- a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp
+++ b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp
@@ -1,4 +1,5 @@
 #include <DataStreams/PushingToViewsBlockOutputStream.h>
+#include <DataStreams/SquashingBlockOutputStream.h>
 #include <Interpreters/InterpreterSelectQuery.h>
 #include <Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.h>
 
@@ -35,8 +36,14 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream(
             auto & materialized_view = dynamic_cast<const StorageMaterializedView &>(*dependent_table);
 
             auto query = materialized_view.getInnerQuery();
-            auto out = std::make_shared<PushingToViewsBlockOutputStream>(
+            BlockOutputStreamPtr out;
+            out = std::make_shared<PushingToViewsBlockOutputStream>(
                 database_table.first, database_table.second, dependent_table, *views_context, ASTPtr());
+            /// Squashing is needed here because the materialized view query can generate a lot of blocks
+            /// even when only one block is inserted into the parent table (e.g. if the query is a GROUP BY
+            /// and two-level aggregation is triggered).
+            out = std::make_shared<SquashingBlockOutputStream>(
+                out, context.getSettingsRef().min_insert_block_size_rows, context.getSettingsRef().min_insert_block_size_bytes);
             views.emplace_back(ViewInfo{std::move(query), database_table.first, database_table.second, std::move(out)});
         }
     }

From 9744ddf70f7c29e87cb4bc3664bf0189533fc979 Mon Sep 17 00:00:00 2001
From: proller <proller@github.com>
Date: Tue, 27 Mar 2018 21:36:50 +0300
Subject: [PATCH 211/244] Use ninja for package build

---
 debian/control | 1 +
 debian/rules   | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/debian/control b/debian/control
index 79666ace2e5..5edac5f66a8 100644
--- a/debian/control
+++ b/debian/control
@@ -4,6 +4,7 @@ Priority: optional
 Maintainer: Alexey Milovidov <milovidov@yandex-team.ru>
 Build-Depends: debhelper (>= 9),
                cmake3 | cmake,
+               ninja-build,
                gcc-7, g++-7,
                libc6-dev,
                libmariadbclient-dev | default-libmysqlclient-dev | libmysqlclient-dev,
diff --git a/debian/rules b/debian/rules
index 7990db33b1c..4cef8702e05 100755
--- a/debian/rules
+++ b/debian/rules
@@ -41,6 +41,8 @@ endif
 endif
 
 CMAKE_FLAGS += -DCMAKE_CXX_COMPILER=`which $(CXX)` -DCMAKE_C_COMPILER=`which $(CC)`
+CMAKE_FLAGS += -GNinja
+export MAKE=ninja
 
 ifndef DH_VERBOSE
     CMAKE_FLAGS += -DCMAKE_VERBOSE_MAKEFILE=0

From 88a9b7de91e2356a40130c88ba6b1d5466c0903e Mon Sep 17 00:00:00 2001
From: Alexey Zatelepin <ztlpn@yandex-team.ru>
Date: Tue, 27 Mar 2018 23:04:40 +0300
Subject: [PATCH 212/244] fix calls to writeSuffix() in
 PushingToViewsBlockOutputStream #2084 [#CLICKHOUSE-3540]

don't use copyData because it calls writeSuffix() prematurely.
---
 dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp | 3 ++-
 dbms/src/DataStreams/PushingToViewsBlockOutputStream.h   | 9 +++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp
index 815b8ebf303..743c24cc6ca 100644
--- a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp
+++ b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp
@@ -74,7 +74,8 @@ void PushingToViewsBlockOutputStream::write(const Block & block)
             BlockInputStreamPtr from = std::make_shared<OneBlockInputStream>(block);
             InterpreterSelectQuery select(view.query, *views_context, {}, QueryProcessingStage::Complete, 0, from);
             BlockInputStreamPtr data = std::make_shared<MaterializingBlockInputStream>(select.execute().in);
-            copyData(*data, *view.out);
+            while (Block result_block = data->read())
+                view.out->write(result_block);
         }
         catch (Exception & ex)
         {
diff --git a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.h b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.h
index 7703b81dc4e..4ff953fd265 100644
--- a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.h
+++ b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.h
@@ -29,18 +29,27 @@ public:
     {
         if (output)
             output->flush();
+
+        for (auto & view : views)
+            view.out->flush();
     }
 
     void writePrefix() override
     {
         if (output)
             output->writePrefix();
+
+        for (auto & view : views)
+            view.out->writePrefix();
     }
 
     void writeSuffix() override
     {
         if (output)
             output->writeSuffix();
+
+        for (auto & view : views)
+            view.out->writeSuffix();
     }
 
 private:

From 1be7c0970273ece874f3df5186c13675f78d0ad1 Mon Sep 17 00:00:00 2001
From: Alex Zatelepin <mvzp10@gmail.com>
Date: Tue, 27 Mar 2018 22:25:50 +0000
Subject: [PATCH 213/244] add readPrefix()/readSuffix() calls for completeness
 #2084 [#CLICKHOUSE-3540]

---
 dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp
index 743c24cc6ca..dc6e2ef2ea7 100644
--- a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp
+++ b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp
@@ -74,8 +74,13 @@ void PushingToViewsBlockOutputStream::write(const Block & block)
             BlockInputStreamPtr from = std::make_shared<OneBlockInputStream>(block);
             InterpreterSelectQuery select(view.query, *views_context, {}, QueryProcessingStage::Complete, 0, from);
             BlockInputStreamPtr data = std::make_shared<MaterializingBlockInputStream>(select.execute().in);
+
+            data->readPrefix();
+
             while (Block result_block = data->read())
                 view.out->write(result_block);
+
+            data->readSuffix();
         }
         catch (Exception & ex)
         {

From f48caff3606c28dd5d9f4891d8b30dfe1c4b96be Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Wed, 28 Mar 2018 05:27:25 +0300
Subject: [PATCH 214/244] Update COWPtr.h

---
 dbms/src/Common/COWPtr.h | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/dbms/src/Common/COWPtr.h b/dbms/src/Common/COWPtr.h
index 39c912f8abc..525f2372c6c 100644
--- a/dbms/src/Common/COWPtr.h
+++ b/dbms/src/Common/COWPtr.h
@@ -68,10 +68,9 @@
   * of this shared state.
   *
   * Caveats:
-  * - after a call to 'mutate' method, you can still have a reference to immutable ptr somewhere
-  *   and it can still become shared. Also it would be better to make 'mutate' method rvalue-qualified.
+  * - after a call to 'mutate' method, you can still have a reference to immutable ptr somewhere.
   * - as 'mutable_ptr' should be unique, it's refcount is redundant - probably it would be better
-  *   to use std::unique_ptr for it, but see above.
+  *   to use std::unique_ptr for it somehow.
   */
 template <typename Derived>
 class COWPtr : public boost::intrusive_ref_counter<Derived>

From 5877498c4ae94457c7ae49205737c4b39c7d968b Mon Sep 17 00:00:00 2001
From: proller <proller@users.noreply.github.com>
Date: Tue, 27 Mar 2018 19:32:54 -0700
Subject: [PATCH 215/244] Add clickhouse-report script to server package
 (#2123)

* Add clickhouse-report script to server package

* fix

* fix

* fix

* Revert "fix"

This reverts commit ef3ded092028ce3666e099c844e30fe686a7bb33.
---
 debian/clickhouse-server-base.install |  1 +
 utils/CMakeLists.txt                  |  1 +
 utils/report/CMakeLists.txt           |  1 +
 utils/report/clickhouse-report        | 30 +++++++++++++++++++++++++++
 utils/report/clickhouse_report.sh     | 23 --------------------
 5 files changed, 33 insertions(+), 23 deletions(-)
 create mode 100644 utils/report/CMakeLists.txt
 create mode 100755 utils/report/clickhouse-report
 delete mode 100755 utils/report/clickhouse_report.sh

diff --git a/debian/clickhouse-server-base.install b/debian/clickhouse-server-base.install
index 9d3d470b040..573f34042bb 100644
--- a/debian/clickhouse-server-base.install
+++ b/debian/clickhouse-server-base.install
@@ -3,6 +3,7 @@
 /usr/bin/clickhouse-clang
 /usr/bin/clickhouse-lld
 /usr/bin/clickhouse-copier
+/usr/bin/clickhouse-report
 /etc/systemd/system/clickhouse-server.service
 /etc/init.d/clickhouse-server
 /etc/cron.d/clickhouse-server
diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt
index bd055efc791..010e38ebfe0 100644
--- a/utils/CMakeLists.txt
+++ b/utils/CMakeLists.txt
@@ -15,3 +15,4 @@ add_subdirectory (wikistat-loader)
 add_subdirectory (fill-factor)
 add_subdirectory (check-marks)
 add_subdirectory (test-data-generator)
+add_subdirectory (report)
diff --git a/utils/report/CMakeLists.txt b/utils/report/CMakeLists.txt
new file mode 100644
index 00000000000..e39dd155b15
--- /dev/null
+++ b/utils/report/CMakeLists.txt
@@ -0,0 +1 @@
+install (PROGRAMS clickhouse-report DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
diff --git a/utils/report/clickhouse-report b/utils/report/clickhouse-report
new file mode 100755
index 00000000000..1cd212febcb
--- /dev/null
+++ b/utils/report/clickhouse-report
@@ -0,0 +1,30 @@
+#!/bin/sh -x
+# Usages:
+# sh -x clickhouse-report > ch.`hostname`.`date '+%Y%M%''d%H%M%''S'`.dmp 2>&1
+# curl https://raw.githubusercontent.com/yandex/ClickHouse/master/utils/report/clickhouse-report | sh -x > ch.`hostname`.`date '+%Y%M%''d%H%M%''S'`.dmp 2>&1
+
+# Also dump some system info (can contain some private data)
+# sh -x clickhouse-report system > ch.`hostname`.`date '+%Y%M%''d%H%M%''S'`.dmp 2>&1
+# curl https://raw.githubusercontent.com/yandex/ClickHouse/master/utils/report/clickhouse-report system | sh -x > ch.`hostname`.`date '+%Y%M%''d%H%M%''S'`.dmp 2>&1
+
+clickhouse-client -q 'SELECT * FROM system.events FORMAT PrettyCompactNoEscapes'
+clickhouse-client -q 'SELECT * FROM system.metrics FORMAT PrettyCompactNoEscapes'
+clickhouse-client -q 'SELECT * FROM system.asynchronous_metrics FORMAT PrettyCompactNoEscapes'
+clickhouse-client -q 'SELECT * FROM system.build_options FORMAT PrettyCompactNoEscapes'
+clickhouse-client -q 'SELECT * FROM system.processes FORMAT PrettyCompactNoEscapes'
+clickhouse-client -q 'SELECT * FROM system.merges FORMAT PrettyCompactNoEscapes'
+clickhouse-client -q 'SELECT * FROM system.parts FORMAT PrettyCompactNoEscapes'
+clickhouse-client -q 'SELECT * FROM system.replication_queue FORMAT PrettyCompactNoEscapes'
+clickhouse-client -q 'SELECT * FROM system.replicas FORMAT PrettyCompactNoEscapes'
+clickhouse-client -q 'SELECT * FROM system.dictionaries FORMAT PrettyCompactNoEscapes'
+tail -n200 /var/log/clickhouse-server/clickhouse-server.err.log
+tail -n200 /var/log/clickhouse-server/clickhouse-server.log
+tail -n100 /var/log/clickhouse-server/stderr
+
+if [ "$1" = "system" ]; then
+    uname -a
+    cat /etc/lsb-release
+    df -h
+    top -bn1
+    ps auxw
+fi
diff --git a/utils/report/clickhouse_report.sh b/utils/report/clickhouse_report.sh
deleted file mode 100755
index ab1ec396e80..00000000000
--- a/utils/report/clickhouse_report.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/bin/sh -x
-# Usages:
-# sh clickhouse_report.sh > ch.`hostname`.`date '+%Y%M%''d%H%M%''S'`.dmp 2>&1
-# curl https://raw.githubusercontent.com/yandex/ClickHouse/master/utils/report/clickhouse_report.sh | sh > ch.`hostname`.`date '+%Y%M%''d%H%M%''S'`.dmp 2>&1
-
-clickhouse --client -q 'SELECT * FROM system.events FORMAT PrettyCompactNoEscapes'
-clickhouse --client -q 'SELECT * FROM system.metrics FORMAT PrettyCompactNoEscapes'
-clickhouse --client -q 'SELECT * FROM system.asynchronous_metrics FORMAT PrettyCompactNoEscapes'
-clickhouse --client -q 'SELECT * FROM system.build_options FORMAT PrettyCompactNoEscapes'
-clickhouse --client -q 'SELECT * FROM system.processes FORMAT PrettyCompactNoEscapes'
-clickhouse --client -q 'SELECT * FROM system.merges FORMAT PrettyCompactNoEscapes'
-clickhouse --client -q 'SELECT * FROM system.parts FORMAT PrettyCompactNoEscapes'
-clickhouse --client -q 'SELECT * FROM system.replication_queue FORMAT PrettyCompactNoEscapes'
-clickhouse --client -q 'SELECT * FROM system.replicas FORMAT PrettyCompactNoEscapes'
-clickhouse --client -q 'SELECT * FROM system.dictionaries FORMAT PrettyCompactNoEscapes'
-ps auxw
-df -h
-top -bn1
-tail -n200 /var/log/clickhouse-server/clickhouse-server.err.log
-tail -n200 /var/log/clickhouse-server/clickhouse-server.log
-tail -n100 /var/log/clickhouse-server/stderr
-cat /etc/lsb-release
-uname -a

From c0aa85436cacf316d9669b20a1c36ad9b1410d3f Mon Sep 17 00:00:00 2001
From: proller <proller@github.com>
Date: Tue, 27 Mar 2018 14:29:11 +0300
Subject: [PATCH 216/244] METRIQA-2349 centos package: fix adduser:
 unrecognized option '--disabled-login'

---
 debian/clickhouse-server-base.postinst | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/debian/clickhouse-server-base.postinst b/debian/clickhouse-server-base.postinst
index c7aa47b5ef9..3bcfa9e22bc 100644
--- a/debian/clickhouse-server-base.postinst
+++ b/debian/clickhouse-server-base.postinst
@@ -27,8 +27,13 @@ if [ "$1" = configure ]; then
 
     # Make sure the administrative user exists
     if ! getent passwd ${CLICKHOUSE_USER} > /dev/null; then
-        adduser --system --disabled-login --no-create-home --home /nonexistent \
-            --shell /bin/false --group --gecos "Clickhouse server" clickhouse > /dev/null
+        if [ "$OS" == "rhel" ] || [ "$OS" == "centos" ] || [ "$OS" == "fedora" ]; then
+            adduser --system --no-create-home --home /nonexistent \
+                --shell /bin/false clickhouse > /dev/null
+        else
+            adduser --system --disabled-login --no-create-home --home /nonexistent \
+                --shell /bin/false --group --gecos "Clickhouse server" clickhouse > /dev/null
+        fi
     fi
 
     # if the user was created manually, make sure the group is there as well

From 0f9501b21f5e73780ec498b0027c81456b5b67af Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Wed, 28 Mar 2018 05:38:07 +0300
Subject: [PATCH 217/244] Update clickhouse-server-base.postinst

---
 debian/clickhouse-server-base.postinst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/clickhouse-server-base.postinst b/debian/clickhouse-server-base.postinst
index 3bcfa9e22bc..6946f8b2728 100644
--- a/debian/clickhouse-server-base.postinst
+++ b/debian/clickhouse-server-base.postinst
@@ -32,7 +32,7 @@ if [ "$1" = configure ]; then
                 --shell /bin/false clickhouse > /dev/null
         else
             adduser --system --disabled-login --no-create-home --home /nonexistent \
-                --shell /bin/false --group --gecos "Clickhouse server" clickhouse > /dev/null
+                --shell /bin/false --group --gecos "ClickHouse server" clickhouse > /dev/null
         fi
     fi
 

From f2db72a5e5b0846de3bc3fa412dd050406453a2e Mon Sep 17 00:00:00 2001
From: Alexey Zatelepin <ztlpn@yandex-team.ru>
Date: Wed, 28 Mar 2018 14:38:06 +0300
Subject: [PATCH 218/244] move squashing to read side #2084 [#CLICKHOUSE-3540]

This way there is a clear correspondence between blocks inserted into the parent table
and blocks inserted into the materialized view.
---
 .../PushingToViewsBlockOutputStream.cpp       | 23 +++++++++----------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp
index dc6e2ef2ea7..b3dd05aee9c 100644
--- a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp
+++ b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp
@@ -1,5 +1,5 @@
 #include <DataStreams/PushingToViewsBlockOutputStream.h>
-#include <DataStreams/SquashingBlockOutputStream.h>
+#include <DataStreams/SquashingBlockInputStream.h>
 #include <Interpreters/InterpreterSelectQuery.h>
 #include <Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.h>
 
@@ -36,14 +36,8 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream(
             auto & materialized_view = dynamic_cast<const StorageMaterializedView &>(*dependent_table);
 
             auto query = materialized_view.getInnerQuery();
-            BlockOutputStreamPtr out;
-            out = std::make_shared<PushingToViewsBlockOutputStream>(
+            BlockOutputStreamPtr out = std::make_shared<PushingToViewsBlockOutputStream>(
                 database_table.first, database_table.second, dependent_table, *views_context, ASTPtr());
-            /// Squashing is needed here because the materialized view query can generate a lot of blocks
-            /// even when only one block is inserted into the parent table (e.g. if the query is a GROUP BY
-            /// and two-level aggregation is triggered).
-            out = std::make_shared<SquashingBlockOutputStream>(
-                out, context.getSettingsRef().min_insert_block_size_rows, context.getSettingsRef().min_insert_block_size_bytes);
             views.emplace_back(ViewInfo{std::move(query), database_table.first, database_table.second, std::move(out)});
         }
     }
@@ -73,14 +67,19 @@ void PushingToViewsBlockOutputStream::write(const Block & block)
         {
             BlockInputStreamPtr from = std::make_shared<OneBlockInputStream>(block);
             InterpreterSelectQuery select(view.query, *views_context, {}, QueryProcessingStage::Complete, 0, from);
-            BlockInputStreamPtr data = std::make_shared<MaterializingBlockInputStream>(select.execute().in);
+            BlockInputStreamPtr in = std::make_shared<MaterializingBlockInputStream>(select.execute().in);
+            /// Squashing is needed here because the materialized view query can generate a lot of blocks
+            /// even when only one block is inserted into the parent table (e.g. if the query is a GROUP BY
+            /// and two-level aggregation is triggered).
+            in = std::make_shared<SquashingBlockInputStream>(
+                in, context.getSettingsRef().min_insert_block_size_rows, context.getSettingsRef().min_insert_block_size_bytes);
 
-            data->readPrefix();
+            in->readPrefix();
 
-            while (Block result_block = data->read())
+            while (Block result_block = in->read())
                 view.out->write(result_block);
 
-            data->readSuffix();
+            in->readSuffix();
         }
         catch (Exception & ex)
         {

From 0d70314eadad1e28d7dc88c6de11c0557a35b4cb Mon Sep 17 00:00:00 2001
From: proller <proller@users.noreply.github.com>
Date: Wed, 28 Mar 2018 05:17:23 -0700
Subject: [PATCH 219/244] Fix package make with ninja (#2129)

* Fix package make with ninja
---
 debian/rules | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/debian/rules b/debian/rules
index 4cef8702e05..1cb7de10224 100755
--- a/debian/rules
+++ b/debian/rules
@@ -9,7 +9,8 @@ export DEB_BUILD_MAINT_OPTIONS=hardening=+all,-pie
 
 # because copy_headers.sh have hardcoded path to build/include_directories.txt
 BUILDDIR = build
-DESTDIR = debian/tmp
+CURDIR = $(shell pwd)
+DESTDIR = $(CURDIR)/debian/tmp
 
 DEB_HOST_MULTIARCH ?= $(shell dpkg-architecture -qDEB_HOST_MULTIARCH)
 
@@ -94,12 +95,15 @@ override_dh_install:
 	CLANG=$(DESTDIR)/usr/bin/clickhouse-clang ./copy_headers.sh . $(DESTDIR)/usr/share/clickhouse/headers
 
 	# fake metrika files when private dir is empty
-	mkdir -p debian/tmp/etc/clickhouse-server/metrika
-	touch debian/tmp/etc/clickhouse-server/metrika/config.xml
-	touch debian/tmp/etc/clickhouse-server/metrika/users.xml
+	mkdir -p $(DESTDIR)/etc/clickhouse-server/metrika
+	touch $(DESTDIR)/etc/clickhouse-server/metrika/config.xml
+	touch $(DESTDIR)/etc/clickhouse-server/metrika/users.xml
 
 	dh_install --list-missing --sourcedir=$(DESTDIR)
 
+override_dh_auto_install:
+	env DESTDIR=$(DESTDIR) ninja -C $(BUILDDIR) install
+
 override_dh_shlibdeps:
 	true # We depend only on libc and dh_shlibdeps gives us wrong (too strict) dependency.
 

From 239f0974fa503ca20da8794b0665cccd10c04828 Mon Sep 17 00:00:00 2001
From: proller <proller@github.com>
Date: Wed, 28 Mar 2018 15:20:45 +0300
Subject: [PATCH 220/244] CLICKHOUSE-3630: Fix race on logger reload, enable
 reload

---
 contrib/poco               | 2 +-
 dbms/src/Server/Server.cpp | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/contrib/poco b/contrib/poco
index 8238852d7ab..3dae7c46e81 160000
--- a/contrib/poco
+++ b/contrib/poco
@@ -1 +1 @@
-Subproject commit 8238852d7ab2a4abdf87adff233b3b83686f4fe4
+Subproject commit 3dae7c46e813d884d15ff509f6dcc1693417646f
diff --git a/dbms/src/Server/Server.cpp b/dbms/src/Server/Server.cpp
index 820a593cd09..9f362ec0bf0 100644
--- a/dbms/src/Server/Server.cpp
+++ b/dbms/src/Server/Server.cpp
@@ -230,6 +230,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
         std::move(main_config_zk_node_cache),
         [&](ConfigurationPtr config)
         {
+            buildLoggers(*config);
             global_context->setClustersConfig(config);
             global_context->setMacros(std::make_unique<Macros>(*config, "macros"));
         },

From 00da8a477c0fd46ff553dd434fa2429b65c72bcf Mon Sep 17 00:00:00 2001
From: Alexey Zatelepin <ztlpn@yandex-team.ru>
Date: Tue, 27 Mar 2018 20:46:53 +0300
Subject: [PATCH 221/244] report broken files more accurately
 [#CLICKHOUSE-3594]

---
 .../Storages/Distributed/DirectoryMonitor.cpp | 22 +++++++++----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp
index efbd217a608..6ca240d7fff 100644
--- a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp
+++ b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp
@@ -333,6 +333,7 @@ struct StorageDistributedDirectoryMonitor::Batch
         file_indices.clear();
         total_rows = 0;
         total_bytes = 0;
+        recovered = false;
 
         Poco::File{parent.current_batch_file_path}.setSize(0);
     }
@@ -381,21 +382,20 @@ void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map
         if (file_indices_to_skip.count(file_idx))
             continue;
 
-        ReadBufferFromFile in{file_path};
-        String insert_query;
-        readStringBinary(insert_query, in);
-
-        CompressedReadBuffer decompressing_in(in);
-        NativeBlockInputStream block_in(decompressing_in, ClickHouseRevision::get());
-
         size_t total_rows = 0;
         size_t total_bytes = 0;
         Block sample_block;
-
-        block_in.readPrefix();
+        String insert_query;
         try
         {
-            /// Determine size of the current file and check if it is not broken.
+            /// Determine metadata of the current file and check if it is not broken.
+            ReadBufferFromFile in{file_path};
+            readStringBinary(insert_query, in);
+
+            CompressedReadBuffer decompressing_in(in);
+            NativeBlockInputStream block_in(decompressing_in, ClickHouseRevision::get());
+            block_in.readPrefix();
+
             while (Block block = block_in.read())
             {
                 total_rows += block.rows();
@@ -404,6 +404,7 @@ void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map
                 if (!sample_block)
                     sample_block = block.cloneEmpty();
             }
+            block_in.readSuffix();
         }
         catch (const Exception & e)
         {
@@ -415,7 +416,6 @@ void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map
             else
                 throw;
         }
-        block_in.readSuffix();
 
         BatchHeader batch_header(std::move(insert_query), std::move(sample_block));
         Batch & batch = header_to_batch.try_emplace(batch_header, *this).first->second;

From aa4f4e62fe05977a19db02be6a9ded38c48c2d82 Mon Sep 17 00:00:00 2001
From: Alexey Zatelepin <ztlpn@yandex-team.ru>
Date: Tue, 27 Mar 2018 21:59:53 +0300
Subject: [PATCH 222/244] mark failed batches as broken [#CLICKHOUSE-3594]

---
 .../Storages/Distributed/DirectoryMonitor.cpp | 133 ++++++++++++------
 .../Storages/Distributed/DirectoryMonitor.h   |   6 +-
 2 files changed, 97 insertions(+), 42 deletions(-)

diff --git a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp
index 6ca240d7fff..9f08fd50a7c 100644
--- a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp
+++ b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp
@@ -274,10 +274,16 @@ struct StorageDistributedDirectoryMonitor::Batch
     std::vector<UInt64> file_indices;
     size_t total_rows = 0;
     size_t total_bytes = 0;
+    bool recovered = false;
 
     StorageDistributedDirectoryMonitor & parent;
+    const std::map<UInt64, String> & file_index_to_path;
 
-    explicit Batch(StorageDistributedDirectoryMonitor & parent_) : parent(parent_) {}
+    Batch(
+        StorageDistributedDirectoryMonitor & parent_,
+        const std::map<UInt64, String> & file_index_to_path_)
+        : parent(parent_), file_index_to_path(file_index_to_path_)
+    {}
 
     bool isEnoughSize() const
     {
@@ -286,14 +292,14 @@ struct StorageDistributedDirectoryMonitor::Batch
             || (parent.min_batched_block_size_bytes && total_bytes >= parent.min_batched_block_size_bytes);
     }
 
-    void send(bool save, const std::map<UInt64, String> & file_index_to_path)
+    void send()
     {
         if (file_indices.empty())
             return;
 
         CurrentMetrics::Increment metric_increment{CurrentMetrics::DistributedSend};
 
-        if (save)
+        if (!recovered)
         {
             /// For deduplication in Replicated tables to work, in case of error
             /// we must try to re-send exactly the same batches.
@@ -305,31 +311,68 @@ struct StorageDistributedDirectoryMonitor::Batch
 
         auto connection = parent.pool->get();
 
-        String insert_query;
-        std::unique_ptr<RemoteBlockOutputStream> remote;
-        bool first = true;
-
-        for (UInt64 file_idx : file_indices)
+        bool batch_broken = false;
+        try
         {
-            ReadBufferFromFile in{file_index_to_path.at(file_idx)};
-            readStringBinary(insert_query, in); /// NOTE: all files must have the same insert_query
+            String insert_query;
+            std::unique_ptr<RemoteBlockOutputStream> remote;
+            bool first = true;
 
-            if (first)
+            for (UInt64 file_idx : file_indices)
             {
-                first = false;
-                remote = std::make_unique<RemoteBlockOutputStream>(*connection, insert_query);
-                remote->writePrefix();
+                auto file_path = file_index_to_path.find(file_idx);
+                if (file_path == file_index_to_path.end())
+                {
+                    LOG_ERROR(parent.log, "Failed to send batch: file with index " << file_idx << " is absent");
+                    batch_broken = true;
+                    break;
+                }
+
+                ReadBufferFromFile in(file_path->second);
+                readStringBinary(insert_query, in); /// NOTE: all files must have the same insert_query
+
+                if (first)
+                {
+                    first = false;
+                    remote = std::make_unique<RemoteBlockOutputStream>(*connection, insert_query);
+                    remote->writePrefix();
+                }
+
+                remote->writePrepared(in);
             }
 
-            remote->writePrepared(in);
+            remote->writeSuffix();
+        }
+        catch (const Exception & e)
+        {
+            if (isFileBrokenErrorCode(e.code()))
+            {
+                tryLogCurrentException(parent.log, "Failed to send batch due to");
+                batch_broken = true;
+            }
+            else
+                throw;
         }
 
-        remote->writeSuffix();
+        if (!batch_broken)
+        {
+            LOG_TRACE(parent.log, "Sent a batch of " << file_indices.size() << " files.");
 
-        LOG_TRACE(parent.log, "Sent a batch of " << file_indices.size() << " files.");
+            for (UInt64 file_index : file_indices)
+                Poco::File{file_index_to_path.at(file_index)}.remove();
+        }
+        else
+        {
+            LOG_ERROR(parent.log, "Marking a batch of " << file_indices.size() << " files as broken.");
+
+            for (UInt64 file_idx : file_indices)
+            {
+                auto file_path = file_index_to_path.find(file_idx);
+                if (file_path != file_index_to_path.end())
+                    parent.markAsBroken(file_path->second);
+            }
+        }
 
-        for (UInt64 file_index : file_indices)
-            Poco::File{file_index_to_path.at(file_index)}.remove();
         file_indices.clear();
         total_rows = 0;
         total_bytes = 0;
@@ -352,6 +395,7 @@ struct StorageDistributedDirectoryMonitor::Batch
             in >> idx >> "\n";
             file_indices.push_back(idx);
         }
+        recovered = true;
     }
 };
 
@@ -362,11 +406,11 @@ void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map
     if (Poco::File{current_batch_file_path}.exists())
     {
         /// Possibly, we failed to send a batch on the previous iteration. Try to send exactly the same batch.
-        Batch batch{*this};
+        Batch batch(*this, files);
         ReadBufferFromFile in{current_batch_file_path};
         batch.readText(in);
         file_indices_to_skip.insert(batch.file_indices.begin(), batch.file_indices.end());
-        batch.send(/* save = */ false, files);
+        batch.send();
     }
 
     std::unordered_map<BatchHeader, Batch, BatchHeader::Hash> header_to_batch;
@@ -418,47 +462,54 @@ void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map
         }
 
         BatchHeader batch_header(std::move(insert_query), std::move(sample_block));
-        Batch & batch = header_to_batch.try_emplace(batch_header, *this).first->second;
+        Batch & batch = header_to_batch.try_emplace(batch_header, *this, files).first->second;
 
         batch.file_indices.push_back(file_idx);
         batch.total_rows += total_rows;
         batch.total_bytes += total_bytes;
 
         if (batch.isEnoughSize())
-            batch.send(/* save = */ true, files);
+            batch.send();
     }
 
     for (auto & kv : header_to_batch)
     {
         Batch & batch = kv.second;
-        batch.send(/* save = */ true, files);
+        batch.send();
     }
 
     Poco::File{current_batch_file_path}.remove();
 }
 
 
-bool StorageDistributedDirectoryMonitor::maybeMarkAsBroken(const std::string & file_path, const Exception &e) const
+bool StorageDistributedDirectoryMonitor::isFileBrokenErrorCode(int code)
 {
-    const auto code = e.code();
-
-    /// mark file as broken if necessary
-    if (code == ErrorCodes::CHECKSUM_DOESNT_MATCH
+    return code == ErrorCodes::CHECKSUM_DOESNT_MATCH
         || code == ErrorCodes::TOO_LARGE_SIZE_COMPRESSED
         || code == ErrorCodes::CANNOT_READ_ALL_DATA
-        || code == ErrorCodes::ATTEMPT_TO_READ_AFTER_EOF)
+        || code == ErrorCodes::ATTEMPT_TO_READ_AFTER_EOF;
+}
+
+void StorageDistributedDirectoryMonitor::markAsBroken(const std::string & file_path) const
+{
+    const auto last_path_separator_pos = file_path.rfind('/');
+    const auto & path = file_path.substr(0, last_path_separator_pos + 1);
+    const auto & file_name = file_path.substr(last_path_separator_pos + 1);
+    const auto & broken_path = path + "broken/";
+    const auto & broken_file_path = broken_path + file_name;
+
+    Poco::File{broken_path}.createDirectory();
+    Poco::File{file_path}.renameTo(broken_file_path);
+
+    LOG_ERROR(log, "Renamed `" << file_path << "` to `" << broken_file_path << '`');
+}
+
+bool StorageDistributedDirectoryMonitor::maybeMarkAsBroken(const std::string & file_path, const Exception & e) const
+{
+    /// mark file as broken if necessary
+    if (isFileBrokenErrorCode(e.code()))
     {
-        const auto last_path_separator_pos = file_path.rfind('/');
-        const auto & path = file_path.substr(0, last_path_separator_pos + 1);
-        const auto & file_name = file_path.substr(last_path_separator_pos + 1);
-        const auto & broken_path = path + "broken/";
-        const auto & broken_file_path = broken_path + file_name;
-
-        Poco::File{broken_path}.createDirectory();
-        Poco::File{file_path}.renameTo(broken_file_path);
-
-        LOG_ERROR(log, "Renamed `" << file_path << "` to `" << broken_file_path << '`');
-
+        markAsBroken(file_path);
         return true;
     }
     else
diff --git a/dbms/src/Storages/Distributed/DirectoryMonitor.h b/dbms/src/Storages/Distributed/DirectoryMonitor.h
index f1b6546c195..2781de1b512 100644
--- a/dbms/src/Storages/Distributed/DirectoryMonitor.h
+++ b/dbms/src/Storages/Distributed/DirectoryMonitor.h
@@ -27,7 +27,11 @@ private:
     bool findFiles();
     void processFile(const std::string & file_path);
     void processFilesWithBatching(const std::map<UInt64, std::string> & files);
-    bool maybeMarkAsBroken(const std::string & file_path, const Exception &e) const;
+
+    static bool isFileBrokenErrorCode(int code);
+    void markAsBroken(const std::string & file_path) const;
+    bool maybeMarkAsBroken(const std::string & file_path, const Exception & e) const;
+
     std::string getLoggerName() const;
 
     StorageDistributed & storage;

From 701b58b4ff394d9bad48ce0790b63fb1f7b1d1aa Mon Sep 17 00:00:00 2001
From: Vitaliy Lyudvichenko <vludv@yandex-team.ru>
Date: Wed, 28 Mar 2018 15:48:40 +0300
Subject: [PATCH 223/244] Faster DROP PARTITION. [#CLICKHOUSE-2]

---
 dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp
index 32d7b263f80..854a1f16413 100644
--- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp
+++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp
@@ -94,7 +94,12 @@ void ReplicatedMergeTreeQueue::initialize(
 void ReplicatedMergeTreeQueue::insertUnlocked(LogEntryPtr & entry, std::optional<time_t> & min_unprocessed_insert_time_changed, std::lock_guard<std::mutex> &)
 {
     virtual_parts.add(entry->new_part_name);
-    queue.push_back(entry);
+
+    /// Put 'DROP PARTITION' entries at the beginning of the queue not to make superfluous fetches of parts that will be eventually deleted
+    if (entry->type != LogEntry::DROP_RANGE)
+        queue.push_back(entry);
+    else
+        queue.push_front(entry);
 
     if (entry->type == LogEntry::GET_PART)
     {

From 7f60bb0a9b62c991510457efac3386dc573baafb Mon Sep 17 00:00:00 2001
From: Kirill Shvakov <shvakov@gmail.com>
Date: Wed, 28 Mar 2018 16:45:48 +0300
Subject: [PATCH 224/244] SystemLog: execute prepareTable()  on each flush

---
 dbms/src/Interpreters/SystemLog.h | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/dbms/src/Interpreters/SystemLog.h b/dbms/src/Interpreters/SystemLog.h
index 5c6071e647d..64796bc13db 100644
--- a/dbms/src/Interpreters/SystemLog.h
+++ b/dbms/src/Interpreters/SystemLog.h
@@ -127,11 +127,6 @@ protected:
     void threadFunction();
     void flush();
 
-    /** Creates new table if it does not exist.
-      * Renames old table if its structure is not suitable.
-      * This cannot be done in constructor to avoid deadlock while renaming a table under locked Context when SystemLog object is created.
-      */
-    bool is_prepared = false;
     void prepareTable();
 };
 
@@ -233,8 +228,7 @@ void SystemLog<LogElement>::flush()
     {
         LOG_TRACE(log, "Flushing system log");
 
-        if (!is_prepared)    /// BTW, flush method is called from single thread.
-            prepareTable();
+        prepareTable();
 
         Block block = LogElement::createBlock();
         for (const LogElement & elem : data)
@@ -267,7 +261,10 @@ void SystemLog<LogElement>::flush()
     }
 }
 
-
+/** Creates new table if it does not exist.
+ * Renames old table if its structure is not suitable.
+ * This cannot be done in constructor to avoid deadlock while renaming a table under locked Context when SystemLog object is created.
+ */
 template <typename LogElement>
 void SystemLog<LogElement>::prepareTable()
 {
@@ -341,7 +338,6 @@ void SystemLog<LogElement>::prepareTable()
         table = context.getTable(database_name, table_name);
     }
 
-    is_prepared = true;
 }
 
 

From de3791dbdaf4199233a9242e7e7794ec43da7571 Mon Sep 17 00:00:00 2001
From: Vitaliy Lyudvichenko <vludv@yandex-team.ru>
Date: Wed, 28 Mar 2018 17:07:28 +0300
Subject: [PATCH 225/244] Accpet client-side send and recieve timeouts.
 [#CLICKHOUSE-2]

---
 dbms/src/Client/Connection.cpp  | 27 +------------------
 dbms/src/Client/TimeoutSetter.h | 46 +++++++++++++++++++++++++++++++++
 dbms/src/Server/TCPHandler.cpp  | 11 +++++++-
 dbms/src/Server/TCPHandler.h    |  4 +++
 4 files changed, 61 insertions(+), 27 deletions(-)
 create mode 100644 dbms/src/Client/TimeoutSetter.h

diff --git a/dbms/src/Client/Connection.cpp b/dbms/src/Client/Connection.cpp
index 0960ac8e77d..8015b708920 100644
--- a/dbms/src/Client/Connection.cpp
+++ b/dbms/src/Client/Connection.cpp
@@ -12,6 +12,7 @@
 #include <DataStreams/NativeBlockInputStream.h>
 #include <DataStreams/NativeBlockOutputStream.h>
 #include <Client/Connection.h>
+#include <Client/TimeoutSetter.h>
 #include <Common/ClickHouseRevision.h>
 #include <Common/Exception.h>
 #include <Common/NetException.h>
@@ -228,32 +229,6 @@ void Connection::forceConnected()
     }
 }
 
-struct TimeoutSetter
-{
-    TimeoutSetter(Poco::Net::StreamSocket & socket_, const Poco::Timespan & timeout_)
-        : socket(socket_), timeout(timeout_)
-    {
-        old_send_timeout = socket.getSendTimeout();
-        old_receive_timeout = socket.getReceiveTimeout();
-
-        if (old_send_timeout > timeout)
-            socket.setSendTimeout(timeout);
-        if (old_receive_timeout > timeout)
-            socket.setReceiveTimeout(timeout);
-    }
-
-    ~TimeoutSetter()
-    {
-        socket.setSendTimeout(old_send_timeout);
-        socket.setReceiveTimeout(old_receive_timeout);
-    }
-
-    Poco::Net::StreamSocket & socket;
-    Poco::Timespan timeout;
-    Poco::Timespan old_send_timeout;
-    Poco::Timespan old_receive_timeout;
-};
-
 bool Connection::ping()
 {
     // LOG_TRACE(log_wrapper.get(), "Ping");
diff --git a/dbms/src/Client/TimeoutSetter.h b/dbms/src/Client/TimeoutSetter.h
new file mode 100644
index 00000000000..0e908c1bdac
--- /dev/null
+++ b/dbms/src/Client/TimeoutSetter.h
@@ -0,0 +1,46 @@
+#pragma once
+
+#include <Poco/Timespan.h>
+#include <Poco/Net/StreamSocket.h>
+
+
+namespace DB
+{
+
+/// Temporarily overrides socket send/recieve timeouts and reset them back into destructor
+/// Timeouts could be only decreased
+struct TimeoutSetter
+{
+    TimeoutSetter(Poco::Net::StreamSocket & socket_, const Poco::Timespan & send_timeout_, const Poco::Timespan & recieve_timeout_)
+        : socket(socket_), send_timeout(send_timeout_), recieve_timeout(recieve_timeout_)
+    {
+        old_send_timeout = socket.getSendTimeout();
+        old_receive_timeout = socket.getReceiveTimeout();
+
+        if (old_send_timeout > send_timeout)
+            socket.setSendTimeout(send_timeout);
+
+        if (old_receive_timeout > recieve_timeout)
+            socket.setReceiveTimeout(recieve_timeout);
+    }
+
+    TimeoutSetter(Poco::Net::StreamSocket & socket_, const Poco::Timespan & timeout_)
+        : TimeoutSetter(socket_, timeout_, timeout_) {}
+
+    ~TimeoutSetter()
+    {
+        socket.setSendTimeout(old_send_timeout);
+        socket.setReceiveTimeout(old_receive_timeout);
+    }
+
+    Poco::Net::StreamSocket & socket;
+
+    Poco::Timespan send_timeout;
+    Poco::Timespan recieve_timeout;
+
+    Poco::Timespan old_send_timeout;
+    Poco::Timespan old_receive_timeout;
+};
+
+
+}
diff --git a/dbms/src/Server/TCPHandler.cpp b/dbms/src/Server/TCPHandler.cpp
index faf39dded67..425f9974df2 100644
--- a/dbms/src/Server/TCPHandler.cpp
+++ b/dbms/src/Server/TCPHandler.cpp
@@ -31,6 +31,7 @@
 #include "TCPHandler.h"
 
 #include <Common/NetException.h>
+#include <ext/scope_guard.h>
 
 
 namespace DB
@@ -139,6 +140,9 @@ void TCPHandler::runImpl()
             /// Restore context of request.
             query_context = connection_context;
 
+            /// If a user passed query-local timeouts, reset socket to initial state at the end of the query
+            SCOPE_EXIT({state.timeout_setter.reset();});
+
             /** If Query - process it. If Ping or Cancel - go back to the beginning.
              *  There may come settings for a separate query that modify `query_context`.
              */
@@ -600,7 +604,12 @@ void TCPHandler::receiveQuery()
     }
 
     /// Per query settings.
-    query_context.getSettingsRef().deserialize(*in);
+    Settings & settings = query_context.getSettingsRef();
+    settings.deserialize(*in);
+
+    /// Sync timeouts on client and server during current query to avoid dangling queries on server
+    /// NOTE: these settings are applied only for current connection (not for distributed tables' connections)
+    state.timeout_setter = std::make_unique<TimeoutSetter>(socket(), settings.send_timeout, settings.receive_timeout);
 
     readVarUInt(stage, *in);
     state.stage = QueryProcessingStage::Enum(stage);
diff --git a/dbms/src/Server/TCPHandler.h b/dbms/src/Server/TCPHandler.h
index f53850ee487..7202067dab8 100644
--- a/dbms/src/Server/TCPHandler.h
+++ b/dbms/src/Server/TCPHandler.h
@@ -11,6 +11,7 @@
 #include <DataStreams/BlockIO.h>
 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
+#include <Client/TimeoutSetter.h>
 
 #include "IServer.h"
 
@@ -59,6 +60,9 @@ struct QueryState
     /// To output progress, the difference after the previous sending of progress.
     Progress progress;
 
+    /// Timeouts setter for current query
+    std::unique_ptr<TimeoutSetter> timeout_setter;
+
 
     void reset()
     {

From a6f9df105c83d3f4824edcbe9aafe6a5c4153316 Mon Sep 17 00:00:00 2001
From: proller <proller@users.noreply.github.com>
Date: Wed, 28 Mar 2018 18:41:06 -0700
Subject: [PATCH 226/244] Allow use encryption for distributed tables (#2065)

* Allow use encryption for distributed tables

* wip

* fix

* rename encryption to secure

* Fix DirectoryMonitor with +secure

* fix

* fix

* Update DirectoryMonitor.cpp

* Update DirectoryMonitor.cpp
---
 dbms/src/Client/Connection.cpp                |  7 +++---
 dbms/src/Client/Connection.h                  | 10 ++++-----
 dbms/src/Client/ConnectionPool.h              | 14 +++++++-----
 dbms/src/Core/Protocol.h                      |  2 +-
 .../ClickHouseDictionarySource.cpp            | 15 ++++++++-----
 .../Dictionaries/ClickHouseDictionarySource.h |  1 +
 dbms/src/Interpreters/Cluster.cpp             | 11 ++++++----
 dbms/src/Interpreters/Cluster.h               |  4 +++-
 dbms/src/Server/Client.cpp                    | 12 +++++-----
 dbms/src/Server/Server.cpp                    |  6 ++---
 dbms/src/Server/config.xml                    | 13 +++++++++--
 .../Storages/Distributed/DirectoryMonitor.cpp | 15 +++++++++++--
 dbms/tests/client-test.xml                    |  2 +-
 .../0_stateless/00505_distributed_secure.data | 22 +++++++++++++++++++
 .../0_stateless/00505_secure.reference        | 17 ++++++++++++++
 .../tests/queries/0_stateless/00505_secure.sh | 22 +++++++++++++++++++
 .../0_stateless/00505_tcp_ssl.reference       |  3 ---
 .../queries/0_stateless/00505_tcp_ssl.sh      | 18 ---------------
 dbms/tests/queries/shell_config.sh            |  4 ++--
 dbms/tests/server-test.xml                    | 12 +++++++++-
 debian/pbuilder-hooks/B90test-server          |  4 ++--
 21 files changed, 149 insertions(+), 65 deletions(-)
 create mode 100644 dbms/tests/queries/0_stateless/00505_distributed_secure.data
 create mode 100644 dbms/tests/queries/0_stateless/00505_secure.reference
 create mode 100755 dbms/tests/queries/0_stateless/00505_secure.sh
 delete mode 100644 dbms/tests/queries/0_stateless/00505_tcp_ssl.reference
 delete mode 100755 dbms/tests/queries/0_stateless/00505_tcp_ssl.sh

diff --git a/dbms/src/Client/Connection.cpp b/dbms/src/Client/Connection.cpp
index 8015b708920..42cdfd59ffc 100644
--- a/dbms/src/Client/Connection.cpp
+++ b/dbms/src/Client/Connection.cpp
@@ -51,14 +51,15 @@ void Connection::connect()
         if (connected)
             disconnect();
 
-        LOG_TRACE(log_wrapper.get(), "Connecting. Database: " << (default_database.empty() ? "(not specified)" : default_database) << ". User: " << user);
+        LOG_TRACE(log_wrapper.get(), "Connecting. Database: " << (default_database.empty() ? "(not specified)" : default_database) << ". User: " << user
+        << (static_cast<bool>(secure) ? ". Secure" : "") << (static_cast<bool>(compression) ? "" : ". Uncompressed") );
 
-        if (static_cast<bool>(encryption))
+        if (static_cast<bool>(secure))
         {
 #if Poco_NetSSL_FOUND
             socket = std::make_unique<Poco::Net::SecureStreamSocket>();
 #else
-            throw Exception{"tcp_ssl protocol is disabled because poco library was built without NetSSL support.", ErrorCodes::SUPPORT_IS_DISABLED};
+            throw Exception{"tcp_secure protocol is disabled because poco library was built without NetSSL support.", ErrorCodes::SUPPORT_IS_DISABLED};
 #endif
         }
         else
diff --git a/dbms/src/Client/Connection.h b/dbms/src/Client/Connection.h
index cac188f12c8..539c8245aeb 100644
--- a/dbms/src/Client/Connection.h
+++ b/dbms/src/Client/Connection.h
@@ -58,14 +58,14 @@ public:
         const ConnectionTimeouts & timeouts_,
         const String & client_name_ = "client",
         Protocol::Compression compression_ = Protocol::Compression::Enable,
-        Protocol::Encryption encryption_ = Protocol::Encryption::Disable,
+        Protocol::Secure secure_ = Protocol::Secure::Disable,
         Poco::Timespan sync_request_timeout_ = Poco::Timespan(DBMS_DEFAULT_SYNC_REQUEST_TIMEOUT_SEC, 0))
         :
         host(host_), port(port_), default_database(default_database_),
         user(user_), password(password_), resolved_address(host, port),
         client_name(client_name_),
         compression(compression_),
-        encryption(encryption_),
+        secure(secure_),
         timeouts(timeouts_),
         sync_request_timeout(sync_request_timeout_),
         log_wrapper(*this)
@@ -84,7 +84,7 @@ public:
         const ConnectionTimeouts & timeouts_,
         const String & client_name_ = "client",
         Protocol::Compression compression_ = Protocol::Compression::Enable,
-        Protocol::Encryption encryption_ = Protocol::Encryption::Disable,
+        Protocol::Secure secure_ = Protocol::Secure::Disable,
         Poco::Timespan sync_request_timeout_ = Poco::Timespan(DBMS_DEFAULT_SYNC_REQUEST_TIMEOUT_SEC, 0))
         :
         host(host_), port(port_),
@@ -93,7 +93,7 @@ public:
         resolved_address(resolved_address_),
         client_name(client_name_),
         compression(compression_),
-        encryption(encryption_),
+        secure(secure_),
         timeouts(timeouts_),
         sync_request_timeout(sync_request_timeout_),
         log_wrapper(*this)
@@ -222,7 +222,7 @@ private:
 
     String query_id;
     Protocol::Compression compression;        /// Enable data compression for communication.
-    Protocol::Encryption encryption;             /// Enable data encryption for communication.
+    Protocol::Secure secure;             /// Enable data encryption for communication.
 
     /// What compression settings to use while sending data for INSERT queries and external tables.
     CompressionSettings compression_settings;
diff --git a/dbms/src/Client/ConnectionPool.h b/dbms/src/Client/ConnectionPool.h
index eb71a7069c9..c07396fe842 100644
--- a/dbms/src/Client/ConnectionPool.h
+++ b/dbms/src/Client/ConnectionPool.h
@@ -51,11 +51,12 @@ public:
             const ConnectionTimeouts & timeouts,
             const String & client_name_ = "client",
             Protocol::Compression compression_ = Protocol::Compression::Enable,
-            Protocol::Encryption encryption_ = Protocol::Encryption::Disable)
+            Protocol::Secure secure_ = Protocol::Secure::Disable)
        : Base(max_connections_, &Logger::get("ConnectionPool (" + host_ + ":" + toString(port_) + ")")),
         host(host_), port(port_), default_database(default_database_),
         user(user_), password(password_), resolved_address(host_, port_),
-        client_name(client_name_), compression(compression_), encryption(encryption_),
+        client_name(client_name_), compression(compression_),
+        secure{secure_},
         timeouts(timeouts)
     {
     }
@@ -67,11 +68,12 @@ public:
             const ConnectionTimeouts & timeouts,
             const String & client_name_ = "client",
             Protocol::Compression compression_ = Protocol::Compression::Enable,
-            Protocol::Encryption encryption_ = Protocol::Encryption::Disable)
+            Protocol::Secure secure_ = Protocol::Secure::Disable)
         : Base(max_connections_, &Logger::get("ConnectionPool (" + host_ + ":" + toString(port_) + ")")),
         host(host_), port(port_), default_database(default_database_),
         user(user_), password(password_), resolved_address(resolved_address_),
-        client_name(client_name_), compression(compression_), encryption(encryption_),
+        client_name(client_name_), compression(compression_),
+        secure{secure_},
         timeouts(timeouts)
     {
     }
@@ -102,7 +104,7 @@ protected:
         return std::make_shared<Connection>(
             host, port, resolved_address,
             default_database, user, password, timeouts,
-            client_name, compression, encryption);
+            client_name, compression, secure);
     }
 
 private:
@@ -119,7 +121,7 @@ private:
 
     String client_name;
     Protocol::Compression compression;        /// Whether to compress data when interacting with the server.
-    Protocol::Encryption encryption;          /// Whether to encrypt data when interacting with the server.
+    Protocol::Secure secure;          /// Whether to encrypt data when interacting with the server.
 
     ConnectionTimeouts timeouts;
 };
diff --git a/dbms/src/Core/Protocol.h b/dbms/src/Core/Protocol.h
index 6a3b2050921..cd5456cca34 100644
--- a/dbms/src/Core/Protocol.h
+++ b/dbms/src/Core/Protocol.h
@@ -116,7 +116,7 @@ namespace Protocol
     };
 
     /// Whether the ssl must be used.
-    enum class Encryption
+    enum class Secure
     {
         Disable = 0,
         Enable = 1,
diff --git a/dbms/src/Dictionaries/ClickHouseDictionarySource.cpp b/dbms/src/Dictionaries/ClickHouseDictionarySource.cpp
index cdf1d7c5c46..47377bb6519 100644
--- a/dbms/src/Dictionaries/ClickHouseDictionarySource.cpp
+++ b/dbms/src/Dictionaries/ClickHouseDictionarySource.cpp
@@ -21,13 +21,15 @@ namespace ErrorCodes
 static const size_t MAX_CONNECTIONS = 16;
 
 static ConnectionPoolWithFailoverPtr createPool(
-        const std::string & host, UInt16 port, const std::string & db,
+        const std::string & host, UInt16 port, bool secure, const std::string & db,
         const std::string & user, const std::string & password, const Context & context)
 {
     auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(context.getSettingsRef());
     ConnectionPoolPtrs pools;
     pools.emplace_back(std::make_shared<ConnectionPool>(
-            MAX_CONNECTIONS, host, port, db, user, password, timeouts, "ClickHouseDictionarySource"));
+            MAX_CONNECTIONS, host, port, db, user, password, timeouts, "ClickHouseDictionarySource",
+            Protocol::Compression::Enable,
+            secure ? Protocol::Secure::Enable : Protocol::Secure::Disable));
     return std::make_shared<ConnectionPoolWithFailover>(pools, LoadBalancing::RANDOM);
 }
 
@@ -41,6 +43,7 @@ ClickHouseDictionarySource::ClickHouseDictionarySource(
         dict_struct{dict_struct_},
         host{config.getString(config_prefix + ".host")},
         port(config.getInt(config_prefix + ".port")),
+        secure(config.getBool(config_prefix + ".secure", false)),
         user{config.getString(config_prefix + ".user", "")},
         password{config.getString(config_prefix + ".password", "")},
         db{config.getString(config_prefix + ".db", "")},
@@ -50,7 +53,7 @@ ClickHouseDictionarySource::ClickHouseDictionarySource(
         query_builder{dict_struct, db, table, where, ExternalQueryBuilder::Backticks},
         sample_block{sample_block}, context(context),
         is_local{isLocalAddress({ host, port }, config.getInt("tcp_port", 0))},
-        pool{is_local ? nullptr : createPool(host, port, db, user, password, context)},
+        pool{is_local ? nullptr : createPool(host, port, secure, db, user, password, context)},
         load_all_query{query_builder.composeLoadAllQuery()}
 {}
 
@@ -58,14 +61,16 @@ ClickHouseDictionarySource::ClickHouseDictionarySource(
 ClickHouseDictionarySource::ClickHouseDictionarySource(const ClickHouseDictionarySource & other)
     : update_time{other.update_time},
         dict_struct{other.dict_struct},
-        host{other.host}, port{other.port}, user{other.user}, password{other.password},
+        host{other.host}, port{other.port},
+        secure{other.secure},
+        user{other.user}, password{other.password},
         db{other.db}, table{other.table},
         where{other.where},
         update_field{other.update_field},
         query_builder{dict_struct, db, table, where, ExternalQueryBuilder::Backticks},
         sample_block{other.sample_block}, context(other.context),
         is_local{other.is_local},
-        pool{is_local ? nullptr : createPool(host, port, db, user, password, context)},
+        pool{is_local ? nullptr : createPool(host, port, secure, db, user, password, context)},
         load_all_query{other.load_all_query}
 {}
 
diff --git a/dbms/src/Dictionaries/ClickHouseDictionarySource.h b/dbms/src/Dictionaries/ClickHouseDictionarySource.h
index 6fe90ff5030..60b4f00093c 100644
--- a/dbms/src/Dictionaries/ClickHouseDictionarySource.h
+++ b/dbms/src/Dictionaries/ClickHouseDictionarySource.h
@@ -53,6 +53,7 @@ private:
     const DictionaryStructure dict_struct;
     const std::string host;
     const UInt16 port;
+    const bool secure;
     const std::string user;
     const std::string password;
     const std::string db;
diff --git a/dbms/src/Interpreters/Cluster.cpp b/dbms/src/Interpreters/Cluster.cpp
index 919fd8605ca..4f76505fdd9 100644
--- a/dbms/src/Interpreters/Cluster.cpp
+++ b/dbms/src/Interpreters/Cluster.cpp
@@ -65,6 +65,8 @@ Cluster::Address::Address(Poco::Util::AbstractConfiguration & config, const Stri
     password = config.getString(config_prefix + ".password", "");
     default_database = config.getString(config_prefix + ".default_database", "");
     is_local = isLocal(*this, clickhouse_port);
+    secure = config.getBool(config_prefix + ".secure", false) ? Protocol::Secure::Enable : Protocol::Secure::Disable;
+    compression = config.getBool(config_prefix + ".compression", true) ? Protocol::Compression::Enable : Protocol::Compression::Disable;
 }
 
 
@@ -113,7 +115,8 @@ String Cluster::Address::toStringFull() const
         (password.empty() ? "" : (':' + escapeForFileName(password))) + '@' +
         escapeForFileName(resolved_address.host().toString()) + ':' +
         std::to_string(resolved_address.port()) +
-        (default_database.empty() ? "" : ('#' + escapeForFileName(default_database)));
+        (default_database.empty() ? "" : ('#' + escapeForFileName(default_database)))
+        + ((secure == Protocol::Secure::Enable) ? "+secure" : "");
 }
 
 
@@ -220,7 +223,7 @@ Cluster::Cluster(Poco::Util::AbstractConfiguration & config, const Settings & se
                     address.host_name, address.port, address.resolved_address,
                     address.default_database, address.user, address.password,
                     ConnectionTimeouts::getTCPTimeoutsWithoutFailover(settings).getSaturated(settings.max_execution_time),
-                    "server", Protocol::Compression::Enable, Protocol::Encryption::Disable);
+                    "server", address.compression, address.secure);
 
                 info.pool = std::make_shared<ConnectionPoolWithFailover>(
                     ConnectionPoolPtrs{pool}, settings.load_balancing, settings.connections_with_failover_max_tries);
@@ -303,7 +306,7 @@ Cluster::Cluster(Poco::Util::AbstractConfiguration & config, const Settings & se
                         replica.host_name, replica.port, replica.resolved_address,
                         replica.default_database, replica.user, replica.password,
                         ConnectionTimeouts::getTCPTimeoutsWithFailover(settings).getSaturated(settings.max_execution_time),
-                        "server", Protocol::Compression::Enable, Protocol::Encryption::Disable);
+                        "server", replica.compression, replica.secure);
 
                     remote_replicas_pools.emplace_back(replica_pool);
                     all_replicas_pools.emplace_back(replica_pool);
@@ -367,7 +370,7 @@ Cluster::Cluster(const Settings & settings, const std::vector<std::vector<String
                         replica.host_name, replica.port, replica.resolved_address,
                         replica.default_database, replica.user, replica.password,
                         ConnectionTimeouts::getTCPTimeoutsWithFailover(settings).getSaturated(settings.max_execution_time),
-                        "server", Protocol::Compression::Enable, Protocol::Encryption::Disable);
+                        "server", replica.compression, replica.secure);
                 all_replicas.emplace_back(replica_pool);
                 remote_replicas.emplace_back(replica_pool);
             }
diff --git a/dbms/src/Interpreters/Cluster.h b/dbms/src/Interpreters/Cluster.h
index b58eb63ed7b..d15b7eb3223 100644
--- a/dbms/src/Interpreters/Cluster.h
+++ b/dbms/src/Interpreters/Cluster.h
@@ -48,7 +48,7 @@ public:
         *     <replica>
         *         <host>example01-01-1</host>
         *         <port>9000</port>
-        *         <!-- <user>, <password>, <default_database> if needed -->
+        *         <!-- <user>, <password>, <default_database>. <secure> if needed -->
         *    </replica>
         * </shard>
         */
@@ -60,6 +60,8 @@ public:
         String default_database;    /// this database is selected when no database is specified for Distributed table
         UInt32 replica_num;
         bool is_local;
+        Protocol::Compression compression = Protocol::Compression::Enable;
+        Protocol::Secure secure = Protocol::Secure::Disable;
 
         Address() = default;
         Address(Poco::Util::AbstractConfiguration & config, const String & config_prefix);
diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp
index eb0e4482a37..ff258a6c879 100644
--- a/dbms/src/Server/Client.cpp
+++ b/dbms/src/Server/Client.cpp
@@ -177,7 +177,7 @@ private:
         String default_database;
         String user;
         String password;
-        Protocol::Encryption security;
+        Protocol::Secure security;
         Protocol::Compression compression;
         ConnectionTimeouts timeouts;
 
@@ -187,8 +187,8 @@ private:
         {
             bool is_secure = config.getBool("secure", false);
             security = is_secure
-                ? Protocol::Encryption::Enable
-                : Protocol::Encryption::Disable;
+                ? Protocol::Secure::Enable
+                : Protocol::Secure::Disable;
 
             host = config.getString("host", "localhost");
             port = config.getInt("port",
@@ -1371,7 +1371,7 @@ public:
             ("config-file,c", boost::program_options::value<std::string>(), "config-file path")
             ("host,h", boost::program_options::value<std::string>()->default_value("localhost"), "server host")
             ("port", boost::program_options::value<int>()->default_value(9000), "server port")
-            ("ssl,s", "ssl")
+            ("secure,s", "secure")
             ("user,u", boost::program_options::value<std::string>(), "user")
             ("password", boost::program_options::value<std::string>(), "password")
             ("query_id", boost::program_options::value<std::string>(), "query_id")
@@ -1474,8 +1474,8 @@ public:
 
         if (options.count("port") && !options["port"].defaulted())
             config().setInt("port", options["port"].as<int>());
-        if (options.count("ssl"))
-            config().setBool("ssl", true);
+        if (options.count("secure"))
+            config().setBool("secure", true);
         if (options.count("user"))
             config().setString("user", options["user"].as<std::string>());
         if (options.count("password"))
diff --git a/dbms/src/Server/Server.cpp b/dbms/src/Server/Server.cpp
index 9f362ec0bf0..916cee00afd 100644
--- a/dbms/src/Server/Server.cpp
+++ b/dbms/src/Server/Server.cpp
@@ -425,10 +425,10 @@ int Server::main(const std::vector<std::string> & /*args*/)
                 }
 
                 /// TCP with SSL
-                if (config().has("tcp_ssl_port"))
+                if (config().has("tcp_port_secure"))
                 {
 #if Poco_NetSSL_FOUND
-                    Poco::Net::SocketAddress tcp_address = make_socket_address(listen_host, config().getInt("tcp_ssl_port"));
+                    Poco::Net::SocketAddress tcp_address = make_socket_address(listen_host, config().getInt("tcp_port_secure"));
                     Poco::Net::SecureServerSocket tcp_socket(tcp_address);
                     tcp_socket.setReceiveTimeout(settings.receive_timeout);
                     tcp_socket.setSendTimeout(settings.send_timeout);
@@ -437,7 +437,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
                                                                   server_pool,
                                                                   tcp_socket,
                                                                   new Poco::Net::TCPServerParams));
-                    LOG_INFO(log, "Listening tcp_ssl: " + tcp_address.toString());
+                    LOG_INFO(log, "Listening tcp_secure: " + tcp_address.toString());
 #else
                     throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support.",
                         ErrorCodes::SUPPORT_IS_DISABLED};
diff --git a/dbms/src/Server/config.xml b/dbms/src/Server/config.xml
index 8cf1fd6b519..b4fddbbb739 100644
--- a/dbms/src/Server/config.xml
+++ b/dbms/src/Server/config.xml
@@ -16,10 +16,10 @@
     <!-- For HTTPS and SSL over native protocol. -->
     <!--
     <https_port>8443</https_port>
-    <tcp_ssl_port>9440</tcp_ssl_port>
+    <tcp_port_secure>9440</tcp_port_secure>
     -->
 
-    <!-- Used with https_port and tcp_ssl_port. Full ssl options list: https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h#L71 -->
+    <!-- Used with https_port and tcp_port_secure. Full ssl options list: https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h#L71 -->
     <openSSL>
         <server> <!-- Used for https server AND secure tcp port -->
             <!-- openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout /etc/clickhouse-server/server.key -out /etc/clickhouse-server/server.crt -->
@@ -150,6 +150,15 @@
                 </replica>
             </shard>
         </test_shard_localhost>
+        <test_shard_localhost_secure>
+            <shard>
+                <replica>
+                    <host>localhost</host>
+                    <port>9440</port>
+                    <secure>1</secure>
+                </replica>
+            </shard>
+        </test_shard_localhost_secure>
     </remote_servers>
 
 
diff --git a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp
index 9f08fd50a7c..76e03979c29 100644
--- a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp
+++ b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp
@@ -52,6 +52,14 @@ namespace
             const char * address_begin = static_cast<const char*>(address.data());
             const char * address_end = address_begin + address.size();
 
+            Protocol::Secure secure = Protocol::Secure::Disable;
+            const char * secure_tag = "+secure";
+            if (endsWith(address, secure_tag))
+            {
+                address_end -= strlen(secure_tag);
+                secure = Protocol::Secure::Enable;
+            }
+
             const char * user_pw_end = strchr(address.data(), '@');
             const char * colon = strchr(address.data(), ':');
             if (!user_pw_end || !colon)
@@ -76,7 +84,7 @@ namespace
             const auto database = has_db ? unescapeForFileName(std::string(has_db + 1, address_end))
                                          : std::string();
 
-            pools.emplace_back(factory(host, port, user, password, database));
+            pools.emplace_back(factory(host, port, secure, user, password, database));
         }
 
         return pools;
@@ -152,13 +160,16 @@ ConnectionPoolPtr StorageDistributedDirectoryMonitor::createPool(const std::stri
 {
     auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(storage.context.getSettingsRef());
     const auto pool_factory = [&storage, &name, &timeouts] (const std::string & host, const UInt16 port,
+                                                 const Protocol::Secure secure,
                                                  const std::string & user, const std::string & password,
                                                  const std::string & default_database)
     {
         return std::make_shared<ConnectionPool>(
             1, host, port, default_database,
             user, password, timeouts,
-            storage.getName() + '_' + name);
+            storage.getName() + '_' + name,
+            Protocol::Compression::Enable,
+            secure);
     };
 
     auto pools = createPoolsForAddresses(name, pool_factory);
diff --git a/dbms/tests/client-test.xml b/dbms/tests/client-test.xml
index 8747c773a66..936068a0923 100644
--- a/dbms/tests/client-test.xml
+++ b/dbms/tests/client-test.xml
@@ -1,7 +1,7 @@
 <!-- Config for connecting to test server -->
 <yandex>
     <tcp_port>59000</tcp_port>
-    <tcp_ssl_port>59440</tcp_ssl_port>
+    <tcp_port_secure>59440</tcp_port_secure>
     <openSSL>
         <client>
             <loadDefaultCAFile>true</loadDefaultCAFile>
diff --git a/dbms/tests/queries/0_stateless/00505_distributed_secure.data b/dbms/tests/queries/0_stateless/00505_distributed_secure.data
new file mode 100644
index 00000000000..59c9d4a9ad3
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00505_distributed_secure.data
@@ -0,0 +1,22 @@
+DROP TABLE IF EXISTS test.secure1;
+DROP TABLE IF EXISTS test.secure2;
+DROP TABLE IF EXISTS test.secure3;
+
+CREATE TABLE test.secure1 ( date Date, a Int32, b Int32, c Int32, d Int32) ENGINE = MergeTree(date, (a, date), 8192);
+CREATE TABLE test.secure2 ( date Date, a Int32, b Int32, c Int32, d Int32) ENGINE = Distributed(test_shard_localhost_secure, 'test', 'secure1');
+CREATE TABLE test.secure3 ( date Date, a Int32, b Int32, c Int32, d Int32) ENGINE = Distributed(test_shard_localhost_secure, 'test', 'secure2');
+
+INSERT INTO test.secure1 VALUES (1,  2, 3, 4, 5);
+INSERT INTO test.secure1 VALUES (11,12,13,14,15);
+INSERT INTO test.secure2 VALUES (21,22,23,24,25);
+INSERT INTO test.secure3 VALUES (31,32,33,34,35);
+
+SELECT sleep(1);
+
+SELECT * FROM test.secure1 ORDER BY a;
+SELECT * FROM test.secure2 ORDER BY a;
+SELECT * FROM test.secure3 ORDER BY a;
+
+DROP TABLE test.secure1;
+DROP TABLE test.secure2;
+DROP TABLE test.secure3;
diff --git a/dbms/tests/queries/0_stateless/00505_secure.reference b/dbms/tests/queries/0_stateless/00505_secure.reference
new file mode 100644
index 00000000000..bc4e8bf4b35
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00505_secure.reference
@@ -0,0 +1,17 @@
+1
+2
+3
+1
+0
+1970-01-02	2	3	4	5
+1970-01-12	12	13	14	15
+1970-01-22	22	23	24	25
+1970-02-01	32	33	34	35
+1970-01-02	2	3	4	5
+1970-01-12	12	13	14	15
+1970-01-22	22	23	24	25
+1970-02-01	32	33	34	35
+1970-01-02	2	3	4	5
+1970-01-12	12	13	14	15
+1970-01-22	22	23	24	25
+1970-02-01	32	33	34	35
diff --git a/dbms/tests/queries/0_stateless/00505_secure.sh b/dbms/tests/queries/0_stateless/00505_secure.sh
new file mode 100755
index 00000000000..b68ff4fcf34
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00505_secure.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+. $CURDIR/../shell_config.sh
+
+# Not default server config needed
+
+tcp_port_secure=`${CLICKHOUSE_EXTRACT_CONFIG} -k tcp_port_secure 2>/dev/null`
+if [ -z ${tcp_port_secure} ]; then
+    # Secure port disabled. Fake result
+    cat $CURDIR/00505_secure.reference
+else
+    # Auto port detect
+    ${CLICKHOUSE_CLIENT} --secure -q "SELECT 1";
+    ${CLICKHOUSE_CLIENT} --secure --port=${CLICKHOUSE_PORT_TCP_SECURE} -q "SELECT 2";
+
+    ${CLICKHOUSE_CURL} -sS --insecure ${CLICKHOUSE_URL_HTTPS}?query=SELECT%203
+
+    ${CLICKHOUSE_CLIENT} --secure -q "SELECT 1";
+
+    cat $CURDIR/00505_distributed_secure.data | $CLICKHOUSE_CLIENT --secure -n -m
+fi
diff --git a/dbms/tests/queries/0_stateless/00505_tcp_ssl.reference b/dbms/tests/queries/0_stateless/00505_tcp_ssl.reference
deleted file mode 100644
index 01e79c32a8c..00000000000
--- a/dbms/tests/queries/0_stateless/00505_tcp_ssl.reference
+++ /dev/null
@@ -1,3 +0,0 @@
-1
-2
-3
diff --git a/dbms/tests/queries/0_stateless/00505_tcp_ssl.sh b/dbms/tests/queries/0_stateless/00505_tcp_ssl.sh
deleted file mode 100755
index 47f3c3b35da..00000000000
--- a/dbms/tests/queries/0_stateless/00505_tcp_ssl.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/usr/bin/env bash
-
-CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-. $CURDIR/../shell_config.sh
-
-# Not default server config needed
-
-tcp_ssl_port=`${CLICKHOUSE_EXTRACT_CONFIG} -k tcp_ssl_port 2>/dev/null`
-if [ -z ${tcp_ssl_port} ]; then
-    # Secure port disabled. Fake result
-    cat $CURDIR/00505_tcp_ssl.reference
-else
-    # Auto port detect
-    ${CLICKHOUSE_CLIENT} --ssl -q "SELECT 1";
-    ${CLICKHOUSE_CLIENT} --ssl --port=${CLICKHOUSE_PORT_TCP_SSL} -q "SELECT 2";
-
-    ${CLICKHOUSE_CURL} -sS --insecure ${CLICKHOUSE_URL_HTTPS}?query=SELECT%203
-fi
diff --git a/dbms/tests/queries/shell_config.sh b/dbms/tests/queries/shell_config.sh
index 0aec2aab89e..f554105b5bf 100644
--- a/dbms/tests/queries/shell_config.sh
+++ b/dbms/tests/queries/shell_config.sh
@@ -10,8 +10,8 @@ export CLICKHOUSE_CONFIG_GREP=${CLICKHOUSE_CONFIG_GREP:="/etc/clickhouse-server/
 export CLICKHOUSE_HOST=${CLICKHOUSE_HOST:="localhost"}
 export CLICKHOUSE_PORT_TCP=${CLICKHOUSE_PORT_TCP:=`${CLICKHOUSE_EXTRACT_CONFIG} --try --key=tcp_port 2>/dev/null`} 2>/dev/null
 export CLICKHOUSE_PORT_TCP=${CLICKHOUSE_PORT_TCP:="9000"}
-export CLICKHOUSE_PORT_TCP_SSL=${CLICKHOUSE_PORT_TCP_SSL:=`${CLICKHOUSE_EXTRACT_CONFIG} --try --key=tcp_ssl_port 2>/dev/null`} 2>/dev/null
-export CLICKHOUSE_PORT_TCP_SSL=${CLICKHOUSE_PORT_TCP_SSL:="9440"}
+export CLICKHOUSE_PORT_TCP_SECURE=${CLICKHOUSE_PORT_TCP_SECURE:=`${CLICKHOUSE_EXTRACT_CONFIG} --try --key=tcp_port_secure 2>/dev/null`} 2>/dev/null
+export CLICKHOUSE_PORT_TCP_SECURE=${CLICKHOUSE_PORT_TCP_SECURE:="9440"}
 export CLICKHOUSE_PORT_HTTP=${CLICKHOUSE_PORT_HTTP:=`${CLICKHOUSE_EXTRACT_CONFIG} --key=http_port 2>/dev/null`}
 export CLICKHOUSE_PORT_HTTP=${CLICKHOUSE_PORT_HTTP:="8123"}
 export CLICKHOUSE_PORT_HTTPS=${CLICKHOUSE_PORT_HTTPS:=`${CLICKHOUSE_EXTRACT_CONFIG} --try --key=https_port 2>/dev/null`} 2>/dev/null
diff --git a/dbms/tests/server-test.xml b/dbms/tests/server-test.xml
index 78e53e576e2..f921178f162 100644
--- a/dbms/tests/server-test.xml
+++ b/dbms/tests/server-test.xml
@@ -7,6 +7,7 @@
         <errorlog>/tmp/clickhouse/log/clickhouse-server.err.log</errorlog>
         <size>10M</size>
         <count>1</count>
+        <compress>0</compress>
     </logger>
     <listen_host>0.0.0.0</listen_host>
     <listen_host>::</listen_host>
@@ -14,7 +15,7 @@
     <http_port>58123</http_port>
     <tcp_port>59000</tcp_port>
     <https_port>58443</https_port>
-    <tcp_ssl_port>59440</tcp_ssl_port>
+    <tcp_port_secure>59440</tcp_port_secure>
     <interserver_http_port>59009</interserver_http_port>
     <openSSL>
         <server> <!-- Used for https server AND secure tcp port -->
@@ -60,6 +61,15 @@
                 </replica>
             </shard>
         </test_shard_localhost>
+        <test_shard_localhost_secure>
+            <shard>
+                <replica>
+                    <host>localhost</host>
+                    <port>59440</port>
+                    <secure>1</secure>
+                </replica>
+            </shard>
+        </test_shard_localhost_secure>
     </remote_servers>
     <zookeeper incl="zookeeper-servers" optional="true" />
     <macros incl="macros" optional="true" />
diff --git a/debian/pbuilder-hooks/B90test-server b/debian/pbuilder-hooks/B90test-server
index eac7b5a6374..39a79e5f529 100755
--- a/debian/pbuilder-hooks/B90test-server
+++ b/debian/pbuilder-hooks/B90test-server
@@ -23,12 +23,12 @@ if [ "${TEST_CONNECT}" ]; then
 
     TEST_SSL=${TEST_SSL=1}
     if  [ "${TEST_SSL}" ]; then
-        echo "<yandex><https_port>8443</https_port><tcp_ssl_port>9440</tcp_ssl_port></yandex>" > /etc/clickhouse-server/config.d/ssl.xml
+        echo "<yandex><https_port>8443</https_port><tcp_port_secure>9440</tcp_port_secure></yandex>" > /etc/clickhouse-server/config.d/ssl.xml
         echo "<yandex><openSSL><client><verificationMode>none</verificationMode><invalidCertificateHandler><name>AcceptCertificateHandler</name></invalidCertificateHandler></client></openSSL></yandex>" > /etc/clickhouse-client/config.d/ssl.xml
 
         openssl dhparam -out /etc/clickhouse-server/dhparam.pem 256
         openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout /etc/clickhouse-server/server.key -out /etc/clickhouse-server/server.crt
-        CLIENT_ADD="--ssl"
+        CLIENT_ADD="--secure"
     fi
 
     function finish {

From d78a46d958cffeabf9543cb0e35e744c7e141bd6 Mon Sep 17 00:00:00 2001
From: Kirill Shvakov <shvakov@gmail.com>
Date: Thu, 29 Mar 2018 08:16:58 +0300
Subject: [PATCH 227/244] #2131 deleted excessive logging

---
 dbms/src/Interpreters/SystemLog.h | 2 --
 1 file changed, 2 deletions(-)

diff --git a/dbms/src/Interpreters/SystemLog.h b/dbms/src/Interpreters/SystemLog.h
index 64796bc13db..f99b2f684cb 100644
--- a/dbms/src/Interpreters/SystemLog.h
+++ b/dbms/src/Interpreters/SystemLog.h
@@ -308,8 +308,6 @@ void SystemLog<LogElement>::prepareTable()
             /// The required table will be created.
             table = nullptr;
         }
-        else
-            LOG_DEBUG(log, "Will use existing table " << description << " for " + LogElement::name());
     }
 
     if (!table)

From 62b27b539af552c41457cd9c16a18a0833476759 Mon Sep 17 00:00:00 2001
From: proller <proller@github.com>
Date: Thu, 29 Mar 2018 15:57:23 +0300
Subject: [PATCH 228/244] Debian make: Do not use ninja if not installed

---
 debian/rules | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/debian/rules b/debian/rules
index 1cb7de10224..c9d419b151b 100755
--- a/debian/rules
+++ b/debian/rules
@@ -42,8 +42,12 @@ endif
 endif
 
 CMAKE_FLAGS += -DCMAKE_CXX_COMPILER=`which $(CXX)` -DCMAKE_C_COMPILER=`which $(CC)`
-CMAKE_FLAGS += -GNinja
-export MAKE=ninja
+
+NINJA=$(shell which ninja)
+ifneq ($(NINJA),)
+    CMAKE_FLAGS += -GNinja
+    export MAKE=$(NINJA)
+endif
 
 ifndef DH_VERBOSE
     CMAKE_FLAGS += -DCMAKE_VERBOSE_MAKEFILE=0
@@ -102,7 +106,7 @@ override_dh_install:
 	dh_install --list-missing --sourcedir=$(DESTDIR)
 
 override_dh_auto_install:
-	env DESTDIR=$(DESTDIR) ninja -C $(BUILDDIR) install
+	env DESTDIR=$(DESTDIR) $(MAKE) -C $(BUILDDIR) install
 
 override_dh_shlibdeps:
 	true # We depend only on libc and dh_shlibdeps gives us wrong (too strict) dependency.

From 3b09d2dab3c8bc2557b25a359a39dc62e4ac164b Mon Sep 17 00:00:00 2001
From: Kirill Shvakov <shvakov@gmail.com>
Date: Thu, 29 Mar 2018 15:59:30 +0300
Subject: [PATCH 229/244] Docs: add Ninja build to the required libraries
 (#2136)

* Added Ninja build to the required libraries

* remove ninja from required libraries
---
 docs/en/development/build.md | 2 +-
 docs/ru/development/build.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/en/development/build.md b/docs/en/development/build.md
index fe8424c0e0d..2caf33fd593 100644
--- a/docs/en/development/build.md
+++ b/docs/en/development/build.md
@@ -52,7 +52,7 @@ export CXX=g++-7
 ## Install required libraries from packages
 
 ```bash
-sudo apt-get install libicu-dev libreadline-dev libmysqlclient-dev libssl-dev unixodbc-dev
+sudo apt-get install libicu-dev libreadline-dev libmysqlclient-dev libssl-dev unixodbc-dev ninja-build
 ```
 
 ## Checkout ClickHouse sources
diff --git a/docs/ru/development/build.md b/docs/ru/development/build.md
index 673192a14f7..73554285fb3 100644
--- a/docs/ru/development/build.md
+++ b/docs/ru/development/build.md
@@ -52,7 +52,7 @@ export CXX=g++-7
 ## Install required libraries from packages
 
 ```bash
-sudo apt-get install libicu-dev libreadline-dev libmysqlclient-dev libssl-dev unixodbc-dev
+sudo apt-get install libicu-dev libreadline-dev libmysqlclient-dev libssl-dev unixodbc-dev ninja-build
 ```
 
 ## Checkout ClickHouse sources

From 63eee94319882d99861cabd9d51808ebbdda66ab Mon Sep 17 00:00:00 2001
From: Vitaliy Lyudvichenko <ludv1x@yandex.ru>
Date: Thu, 29 Mar 2018 16:24:36 +0300
Subject: [PATCH 230/244] Add max_network_bandwidth_for_all_users setting.
 [#CLICKHOUSE-2] (#2137)

---
 dbms/src/Common/Throttler.h                        |  9 +++++++++
 .../src/Interpreters/ClusterProxy/executeQuery.cpp |  7 +++----
 dbms/src/Interpreters/ProcessList.cpp              | 14 +++++++++++---
 dbms/src/Interpreters/ProcessList.h                |  5 ++++-
 dbms/src/Interpreters/Settings.h                   |  5 +++--
 5 files changed, 30 insertions(+), 10 deletions(-)

diff --git a/dbms/src/Common/Throttler.h b/dbms/src/Common/Throttler.h
index abf7be122cc..22aa102cd1b 100644
--- a/dbms/src/Common/Throttler.h
+++ b/dbms/src/Common/Throttler.h
@@ -27,6 +27,9 @@ namespace ErrorCodes
 class Throttler
 {
 public:
+    Throttler(size_t max_speed_, const std::shared_ptr<Throttler> & parent = nullptr)
+            : max_speed(max_speed_), limit_exceeded_exception_message(""), parent(parent) {}
+
     Throttler(size_t max_speed_, size_t limit_, const char * limit_exceeded_exception_message_,
               const std::shared_ptr<Throttler> & parent = nullptr)
         : max_speed(max_speed_), limit(limit_), limit_exceeded_exception_message(limit_exceeded_exception_message_), parent(parent) {}
@@ -76,6 +79,12 @@ public:
             parent->add(amount);
     }
 
+    /// Not thread safe
+    void setParent(const std::shared_ptr<Throttler> & parent_)
+    {
+        parent = parent_;
+    }
+
     void reset()
     {
         std::lock_guard lock(mutex);
diff --git a/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp b/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp
index 7cf50db62eb..27b7d8338af 100644
--- a/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp
+++ b/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp
@@ -40,9 +40,8 @@ BlockInputStreams executeQuery(
     new_context.setSettings(new_settings);
 
     ThrottlerPtr user_level_throttler;
-    if (settings.max_network_bandwidth_for_user)
-        if (auto process_list_element = context.getProcessListElement())
-            user_level_throttler = process_list_element->getUserNetworkThrottler();
+    if (auto process_list_element = context.getProcessListElement())
+        user_level_throttler = process_list_element->getUserNetworkThrottler();
 
     /// Network bandwidth limit, if needed.
     ThrottlerPtr throttler;
@@ -54,7 +53,7 @@ BlockInputStreams executeQuery(
                 "Limit for bytes to send or receive over network exceeded.",
                 user_level_throttler);
     }
-    else if (settings.max_network_bandwidth_for_user)
+    else
         throttler = user_level_throttler;
 
     for (const auto & shard_info : cluster->getShardsInfo())
diff --git a/dbms/src/Interpreters/ProcessList.cpp b/dbms/src/Interpreters/ProcessList.cpp
index 208373cddba..e6e77c85180 100644
--- a/dbms/src/Interpreters/ProcessList.cpp
+++ b/dbms/src/Interpreters/ProcessList.cpp
@@ -101,10 +101,17 @@ ProcessList::EntryPtr ProcessList::insert(
             user_process_list.user_memory_tracker.setNext(&total_memory_tracker);
         }
 
-        if (settings.max_network_bandwidth_for_user && !user_process_list.user_throttler)
+        if (!total_network_throttler && settings.max_network_bandwidth_for_all_users)
         {
-            user_process_list.user_throttler = std::make_shared<Throttler>(settings.max_network_bandwidth_for_user, 0,
-                "Network bandwidth limit for a user exceeded.");
+            total_network_throttler = std::make_shared<Throttler>(settings.max_network_bandwidth_for_all_users);
+        }
+
+        if (!user_process_list.user_throttler)
+        {
+            if (settings.max_network_bandwidth_for_user)
+                user_process_list.user_throttler = std::make_shared<Throttler>(settings.max_network_bandwidth_for_user, total_network_throttler);
+            else if (settings.max_network_bandwidth_for_all_users)
+                user_process_list.user_throttler = total_network_throttler;
         }
 
         res->get().user_process_list = &user_process_list;
@@ -173,6 +180,7 @@ ProcessListEntry::~ProcessListEntry()
         /// Reset MemoryTracker, similarly (see above).
         parent.total_memory_tracker.logPeakMemoryUsage();
         parent.total_memory_tracker.reset();
+        parent.total_network_throttler.reset();
     }
 }
 
diff --git a/dbms/src/Interpreters/ProcessList.h b/dbms/src/Interpreters/ProcessList.h
index c27204a63c4..a76e886414d 100644
--- a/dbms/src/Interpreters/ProcessList.h
+++ b/dbms/src/Interpreters/ProcessList.h
@@ -204,7 +204,7 @@ struct ProcessListForUser
     {
         user_memory_tracker.reset();
         if (user_throttler)
-            user_throttler->reset();
+            user_throttler.reset();
     }
 };
 
@@ -265,6 +265,9 @@ private:
     /// Limit and counter for memory of all simultaneously running queries.
     MemoryTracker total_memory_tracker;
 
+    /// Limit network bandwidth for all users
+    ThrottlerPtr total_network_throttler;
+
     /// Call under lock. Finds process with specified current_user and current_query_id.
     ProcessListElement * tryGetProcessListElement(const String & current_query_id, const String & current_user);
 
diff --git a/dbms/src/Interpreters/Settings.h b/dbms/src/Interpreters/Settings.h
index 79d76850088..f7621469042 100644
--- a/dbms/src/Interpreters/Settings.h
+++ b/dbms/src/Interpreters/Settings.h
@@ -244,9 +244,10 @@ struct Settings
     M(SettingUInt64, max_memory_usage_for_user, 0, "Maximum memory usage for processing all concurrently running queries for the user. Zero means unlimited.") \
     M(SettingUInt64, max_memory_usage_for_all_queries, 0, "Maximum memory usage for processing all concurrently running queries on the server. Zero means unlimited.") \
     \
-    M(SettingUInt64, max_network_bandwidth, 0, "The maximum speed of data exchange over the network in bytes per second. Zero means unlimited.") \
+    M(SettingUInt64, max_network_bandwidth, 0, "The maximum speed of data exchange over the network in bytes per second for a query. Zero means unlimited.") \
     M(SettingUInt64, max_network_bytes, 0, "The maximum number of bytes (compressed) to receive or transmit over the network for execution of the query.") \
-    M(SettingUInt64, max_network_bandwidth_for_user, 0, "The maximum speed of data exchange over the network in bytes per second for all concurrently running queries for the user. Zero means unlimited.")
+    M(SettingUInt64, max_network_bandwidth_for_user, 0, "The maximum speed of data exchange over the network in bytes per second for all concurrently running user queries. Zero means unlimited.")\
+    M(SettingUInt64, max_network_bandwidth_for_all_users, 0, "The maximum speed of data exchange over the network in bytes per second for all concurrently running queries. Zero means unlimited.")
 
 #define DECLARE(TYPE, NAME, DEFAULT, DESCRIPTION) \
     TYPE NAME {DEFAULT};

From 134efcd7edfe387b752ac44cac1bc219f2c6f5c1 Mon Sep 17 00:00:00 2001
From: Vitaliy Lyudvichenko <vludv@yandex-team.ru>
Date: Thu, 29 Mar 2018 16:33:48 +0300
Subject: [PATCH 231/244] Swapped send and receive timeouts settings from
 client. [#CLICKHOUSE-2]

---
 dbms/src/Server/TCPHandler.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/dbms/src/Server/TCPHandler.cpp b/dbms/src/Server/TCPHandler.cpp
index 425f9974df2..72caa5ffe53 100644
--- a/dbms/src/Server/TCPHandler.cpp
+++ b/dbms/src/Server/TCPHandler.cpp
@@ -608,8 +608,10 @@ void TCPHandler::receiveQuery()
     settings.deserialize(*in);
 
     /// Sync timeouts on client and server during current query to avoid dangling queries on server
+    /// NOTE: We use settings.send_timeout for the receive timeout and vice versa (change arguments ordering in TimeoutSetter),
+    ///  because settings.send_timeout is client-side setting which has opposite meaning on the server side.
     /// NOTE: these settings are applied only for current connection (not for distributed tables' connections)
-    state.timeout_setter = std::make_unique<TimeoutSetter>(socket(), settings.send_timeout, settings.receive_timeout);
+    state.timeout_setter = std::make_unique<TimeoutSetter>(socket(), settings.receive_timeout, settings.send_timeout);
 
     readVarUInt(stage, *in);
     state.stage = QueryProcessingStage::Enum(stage);

From 042a197e91dfda1197dad69b6d5ac1e80952a803 Mon Sep 17 00:00:00 2001
From: Kirill Shvakov <shvakov@gmail.com>
Date: Thu, 29 Mar 2018 16:43:24 +0300
Subject: [PATCH 232/244] Revert "#2131 deleted excessive logging"

This reverts commit d78a46d958cffeabf9543cb0e35e744c7e141bd6.
---
 dbms/src/Interpreters/SystemLog.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/dbms/src/Interpreters/SystemLog.h b/dbms/src/Interpreters/SystemLog.h
index f99b2f684cb..64796bc13db 100644
--- a/dbms/src/Interpreters/SystemLog.h
+++ b/dbms/src/Interpreters/SystemLog.h
@@ -308,6 +308,8 @@ void SystemLog<LogElement>::prepareTable()
             /// The required table will be created.
             table = nullptr;
         }
+        else
+            LOG_DEBUG(log, "Will use existing table " << description << " for " + LogElement::name());
     }
 
     if (!table)

From d095f69ae6c8ed596e5e68ec21397dde89d67a67 Mon Sep 17 00:00:00 2001
From: Kirill Shvakov <shvakov@gmail.com>
Date: Thu, 29 Mar 2018 16:46:42 +0300
Subject: [PATCH 233/244] Revert "SystemLog: execute prepareTable()  on each
 flush"

This reverts commit 7f60bb0a9b62c991510457efac3386dc573baafb.
---
 dbms/src/Interpreters/SystemLog.h | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/dbms/src/Interpreters/SystemLog.h b/dbms/src/Interpreters/SystemLog.h
index 64796bc13db..5c6071e647d 100644
--- a/dbms/src/Interpreters/SystemLog.h
+++ b/dbms/src/Interpreters/SystemLog.h
@@ -127,6 +127,11 @@ protected:
     void threadFunction();
     void flush();
 
+    /** Creates new table if it does not exist.
+      * Renames old table if its structure is not suitable.
+      * This cannot be done in constructor to avoid deadlock while renaming a table under locked Context when SystemLog object is created.
+      */
+    bool is_prepared = false;
     void prepareTable();
 };
 
@@ -228,7 +233,8 @@ void SystemLog<LogElement>::flush()
     {
         LOG_TRACE(log, "Flushing system log");
 
-        prepareTable();
+        if (!is_prepared)    /// BTW, flush method is called from single thread.
+            prepareTable();
 
         Block block = LogElement::createBlock();
         for (const LogElement & elem : data)
@@ -261,10 +267,7 @@ void SystemLog<LogElement>::flush()
     }
 }
 
-/** Creates new table if it does not exist.
- * Renames old table if its structure is not suitable.
- * This cannot be done in constructor to avoid deadlock while renaming a table under locked Context when SystemLog object is created.
- */
+
 template <typename LogElement>
 void SystemLog<LogElement>::prepareTable()
 {
@@ -338,6 +341,7 @@ void SystemLog<LogElement>::prepareTable()
         table = context.getTable(database_name, table_name);
     }
 
+    is_prepared = true;
 }
 
 

From 44fb6eb66d0f3e91f0c5a6e5ce87f1e33c5cf799 Mon Sep 17 00:00:00 2001
From: Kirill Shvakov <shvakov@gmail.com>
Date: Thu, 29 Mar 2018 16:54:50 +0300
Subject: [PATCH 234/244] SystemLog: execute prepareTable() on each flush

---
 dbms/src/Interpreters/SystemLog.h | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/dbms/src/Interpreters/SystemLog.h b/dbms/src/Interpreters/SystemLog.h
index 5c6071e647d..8e04076dc14 100644
--- a/dbms/src/Interpreters/SystemLog.h
+++ b/dbms/src/Interpreters/SystemLog.h
@@ -233,8 +233,7 @@ void SystemLog<LogElement>::flush()
     {
         LOG_TRACE(log, "Flushing system log");
 
-        if (!is_prepared)    /// BTW, flush method is called from single thread.
-            prepareTable();
+        prepareTable();
 
         Block block = LogElement::createBlock();
         for (const LogElement & elem : data)
@@ -311,7 +310,7 @@ void SystemLog<LogElement>::prepareTable()
             /// The required table will be created.
             table = nullptr;
         }
-        else
+        else if (!is_prepared)
             LOG_DEBUG(log, "Will use existing table " << description << " for " + LogElement::name());
     }
 

From b7429087319f9c6307499a29c8f1b8800f103c69 Mon Sep 17 00:00:00 2001
From: Kirill Shvakov <shvakov@gmail.com>
Date: Thu, 29 Mar 2018 16:57:16 +0300
Subject: [PATCH 235/244] SystemLog: execute prepareTable()  on each flush

Merge remote-tracking branch 'upstream/master' into system-log-prepare-table-on-each-flush

#2131 deleted excessive logging

Revert "#2131 deleted excessive logging"

This reverts commit d78a46d958cffeabf9543cb0e35e744c7e141bd6.

Revert "SystemLog: execute prepareTable()  on each flush"

This reverts commit 7f60bb0a9b62c991510457efac3386dc573baafb.

SystemLog: execute prepareTable() on each flush
---
 cmake/find_poco.cmake                         |   3 +-
 contrib/CMakeLists.txt                        |  20 +-
 contrib/boost                                 |   2 +-
 contrib/libcityhash/src/config.h              |   4 +
 contrib/libfarmhash/CMakeLists.txt            |   4 +
 contrib/libtcmalloc/src/config.h              |   4 +
 dbms/CMakeLists.txt                           |   5 -
 dbms/cmake/version.cmake                      |   4 +-
 .../AggregateFunctionFactory.cpp              |   5 +-
 .../AggregateFunctionMaxIntersections.cpp     |   7 +-
 .../AggregateFunctionSumMap.cpp               |   6 +-
 dbms/src/Client/Connection.cpp                |  12 +
 dbms/src/Client/Connection.h                  |   2 +
 dbms/src/Columns/ColumnAggregateFunction.cpp  |   6 +-
 dbms/src/Columns/ColumnAggregateFunction.h    |   6 +-
 dbms/src/Columns/ColumnArray.cpp              |  58 +--
 dbms/src/Columns/ColumnArray.h                |  52 +-
 dbms/src/Columns/ColumnConst.cpp              |   8 +-
 dbms/src/Columns/ColumnConst.h                |  10 +-
 dbms/src/Columns/ColumnFixedString.cpp        |   6 +-
 dbms/src/Columns/ColumnFixedString.h          |   6 +-
 dbms/src/Columns/ColumnFunction.cpp           |   8 +-
 dbms/src/Columns/ColumnFunction.h             |   8 +-
 dbms/src/Columns/ColumnNullable.cpp           |  14 +-
 dbms/src/Columns/ColumnNullable.h             |  20 +-
 dbms/src/Columns/ColumnString.cpp             |   6 +-
 dbms/src/Columns/ColumnString.h               |   6 +-
 dbms/src/Columns/ColumnTuple.cpp              |  33 +-
 dbms/src/Columns/ColumnTuple.h                |  17 +-
 dbms/src/Columns/ColumnVector.cpp             |   6 +-
 dbms/src/Columns/ColumnVector.h               |   6 +-
 dbms/src/Columns/FilterDescription.cpp        |   3 +-
 dbms/src/Columns/IColumn.h                    |  28 +-
 dbms/src/Columns/IColumnDummy.h               |   6 +-
 dbms/src/Common/AIO.h                         |   2 +-
 dbms/src/Common/COWPtr.h                      |  23 +-
 dbms/src/Common/ShellCommand.cpp              |   3 +-
 dbms/src/Common/UTF8Helpers.h                 |  27 +-
 dbms/src/Common/ZooKeeper/CMakeLists.txt      |   6 +-
 dbms/src/Common/ZooKeeper/ZooKeeper.h         |   2 +-
 .../ZooKeeper/tests/zkutil_test_commands.cpp  |   2 +-
 dbms/src/Common/localBackup.cpp               |   5 +-
 dbms/src/Common/randomSeed.cpp                |   4 +-
 dbms/src/Common/tests/compact_array.cpp       |   4 +-
 dbms/src/Core/Block.cpp                       |   2 +-
 dbms/src/Core/Block.h                         |   2 +-
 dbms/src/Core/Defines.h                       |  15 +-
 dbms/src/Core/Field.h                         |  12 +-
 dbms/src/DataStreams/ColumnGathererStream.cpp |   2 +-
 .../MergingSortedBlockInputStream.cpp         |   4 +-
 .../DataStreams/PrettyBlockOutputStream.cpp   |   3 +-
 .../PushingToViewsBlockOutputStream.cpp       |  18 +-
 .../PushingToViewsBlockOutputStream.h         |   9 +
 dbms/src/DataStreams/SquashingTransform.cpp   |   2 +-
 .../SummingSortedBlockInputStream.cpp         |   6 +-
 dbms/src/DataTypes/DataTypeTuple.cpp          |   4 +-
 dbms/src/Databases/DatabaseDictionary.cpp     |  65 ++-
 dbms/src/Databases/DatabaseDictionary.h       |  30 +-
 dbms/src/Databases/DatabaseMemory.cpp         |   7 +-
 dbms/src/Databases/DatabaseMemory.h           |   7 +-
 dbms/src/Databases/DatabaseOrdinary.cpp       | 101 +++-
 dbms/src/Databases/DatabaseOrdinary.h         |  10 +-
 dbms/src/Databases/IDatabase.h                |  12 +-
 .../ComplexKeyHashedDictionary.cpp            |   2 +-
 .../ExternalResultDescription.cpp             |   2 +-
 dbms/src/Dictionaries/FlatDictionary.cpp      |   2 +-
 dbms/src/Dictionaries/HashedDictionary.cpp    |   2 +-
 dbms/src/Functions/FunctionsConditional.h     |   8 +-
 dbms/src/Functions/FunctionsHigherOrder.h     |   2 +-
 .../Functions/GatherUtils/createArraySink.cpp |   3 +-
 dbms/src/Functions/IFunction.cpp              |   2 +-
 dbms/src/IO/ReadBufferAIO.cpp                 |   2 +-
 dbms/src/IO/ReadBufferAIO.h                   |   3 +-
 dbms/src/IO/ReadBufferFromFileDescriptor.cpp  |   4 -
 dbms/src/IO/ReadBufferFromFileDescriptor.h    |   3 +-
 dbms/src/IO/ReadHelpers.h                     |   2 +
 dbms/src/IO/WriteBufferAIO.cpp                |   2 +-
 dbms/src/IO/WriteBufferAIO.h                  |   2 +-
 dbms/src/IO/WriteBufferFromFileDescriptor.cpp |   2 +-
 dbms/src/IO/createReadBufferFromFileBase.cpp  |   6 +-
 dbms/src/IO/createWriteBufferFromFileBase.cpp |   6 +-
 dbms/src/IO/tests/read_buffer_aio.cpp         |   5 +-
 dbms/src/Interpreters/ClientInfo.cpp          |   3 +-
 dbms/src/Interpreters/Context.cpp             |  16 +-
 dbms/src/Interpreters/Context.h               |   5 +-
 dbms/src/Interpreters/ExternalDictionaries.h  |   5 +
 dbms/src/Interpreters/ExternalLoader.cpp      |  24 +-
 dbms/src/Interpreters/ExternalLoader.h        |   3 +
 .../Interpreters/InterpreterCreateQuery.cpp   |   4 +-
 dbms/src/Interpreters/InterpreterFactory.cpp  |   6 +-
 .../InterpreterShowCreateQuery.cpp            |  11 +-
 dbms/src/Interpreters/NullableUtils.cpp       |   2 +-
 dbms/src/Interpreters/PartLog.cpp             |   2 +-
 dbms/src/Interpreters/SystemLog.h             |   5 +-
 dbms/src/Interpreters/tests/users.cpp         |   4 +-
 .../Parsers/ParserTablePropertiesQuery.cpp    |  39 +-
 dbms/src/Parsers/TablePropertiesQueriesASTs.h |  22 +-
 dbms/src/Server/Benchmark.cpp                 |  13 +-
 dbms/src/Server/Client.cpp                    | 152 ++++--
 dbms/src/Server/ClusterCopier.cpp             |  42 +-
 dbms/src/Server/HTTPHandler.cpp               |   5 +-
 dbms/src/Server/HTTPHandler.h                 |   3 +
 dbms/src/Server/PerformanceTest.cpp           |   7 +-
 dbms/src/Server/TCPHandler.cpp                |   4 +
 dbms/src/Server/TCPHandler.h                  |   6 +-
 dbms/src/Server/clickhouse-client.xml         |  15 +
 dbms/src/Server/config.xml                    |   4 +-
 .../DistributedBlockOutputStream.cpp          | 117 +++--
 .../DistributedBlockOutputStream.h            |  33 +-
 .../MergeTreeBaseBlockInputStream.cpp         |   8 +-
 .../MergeTree/MergeTreeBlockReadUtils.cpp     |   9 +-
 dbms/src/Storages/MergeTree/MergeTreeData.cpp |  70 +--
 dbms/src/Storages/MergeTree/MergeTreeData.h   |  30 +-
 .../MergeTree/MergeTreeDataMerger.cpp         |   8 +-
 .../Storages/MergeTree/MergeTreeDataPart.cpp  | 344 ++------------
 .../Storages/MergeTree/MergeTreeDataPart.h    | 113 ++---
 .../MergeTree/MergeTreeDataPartChecksum.cpp   | 448 ++++++++++++++++++
 .../MergeTree/MergeTreeDataPartChecksum.h     | 121 +++++
 .../MergeTree/MergeTreeDataWriter.cpp         |   2 +-
 .../Storages/MergeTree/MergeTreeReader.cpp    |  12 +-
 dbms/src/Storages/MergeTree/MergeTreeReader.h |   6 +-
 .../Storages/MergeTree/MergeTreeSettings.h    |   6 +
 .../MergeTree/MergedBlockOutputStream.cpp     |   2 +-
 .../ReplicatedMergeTreeAlterThread.cpp        |   4 +-
 .../ReplicatedMergeTreeBlockOutputStream.cpp  |   4 +-
 .../ReplicatedMergeTreePartCheckThread.cpp    |   2 +-
 .../MergeTree/ReplicatedMergeTreeQueue.cpp    |   2 +-
 .../ReplicatedMergeTreeRestartingThread.cpp   |   7 +
 dbms/src/Storages/StorageBuffer.cpp           |   4 +-
 dbms/src/Storages/StorageDictionary.cpp       |   1 -
 dbms/src/Storages/StorageDictionary.h         |  40 +-
 dbms/src/Storages/StorageKafka.cpp            |  11 +-
 dbms/src/Storages/StorageMergeTree.cpp        |   4 +-
 .../Storages/StorageReplicatedMergeTree.cpp   | 180 ++++---
 .../src/Storages/StorageReplicatedMergeTree.h |  12 +-
 .../Storages/System/StorageSystemColumns.cpp  |   2 +-
 .../Storages/System/StorageSystemParts.cpp    |  51 +-
 .../System/StorageSystemPartsBase.cpp         |  29 +-
 .../System/StorageSystemPartsColumns.cpp      |  84 ++--
 .../Storages/System/StorageSystemTables.cpp   |  12 +-
 .../tests/remove_symlink_directory.cpp        |   3 +-
 dbms/tests/clickhouse-test                    |   9 +-
 .../functions_length/functions_length.xml     |  38 ++
 ...0416_pocopatch_progress_in_http_headers.sh |   2 +-
 .../queries/0_stateless/00501_http_head.sh    |   2 +-
 .../queries/0_stateless/00534_filimonov.data  |   3 +
 .../00534_long_functions_bad_arguments7.sh    |   2 +-
 .../00534_long_functions_bad_arguments8.sh    |  10 +
 .../00534_long_functions_bad_arguments9.sh    |  10 +
 .../00604_show_create_database.reference      |   1 +
 .../00604_show_create_database.sql            |   2 +
 ..._distributed_with_case_when_then.reference |   6 +
 .../00609_distributed_with_case_when_then.sql |  16 +
 .../00609_prewhere_and_default.reference      |   0
 .../00609_prewhere_and_default.sql            |  12 +
 ...eper_different_checksums_formats.reference |   9 +
 ..._zookeeper_different_checksums_formats.sql |  24 +
 debian/.pbuilderrc                            |   2 +-
 debian/changelog                              |   4 +-
 debian/clickhouse-server-base.install         |   1 +
 debian/clickhouse-server-base.postinst        |   9 +-
 debian/control                                |   1 +
 debian/rules                                  |  16 +-
 docs/en/operations/settings/settings.md       |  12 -
 docs/en/table_engines/kafka.md                |  23 +-
 docs/ru/operations/settings/settings.md       |  12 -
 docs/ru/table_engines/kafka.md                |  26 +-
 libs/libcommon/cmake/find_gperftools.cmake    |   2 +-
 libs/libcommon/include/common/DateLUT.h       |   5 +-
 libs/libcommon/include/common/LocalDate.h     |   2 +-
 libs/libcommon/include/common/LocalDateTime.h |   2 +-
 libs/libcommon/include/common/likely.h        |   9 +-
 libs/libcommon/include/common/mremap.h        |   3 +
 libs/libcommon/include/port/ssize_t.h         |   8 +
 libs/libcommon/include/port/unistd.h          |  11 +
 libs/libcommon/src/DateLUT.cpp                |   8 +-
 libs/libcommon/src/mremap.cpp                 |  11 +-
 libs/libdaemon/include/daemon/BaseDaemon.h    |   2 +-
 libs/libpocoext/src/ThreadNumber.cpp          |  10 +-
 utils/CMakeLists.txt                          |   1 +
 utils/build/build_msvc2017.bat                |  14 +
 utils/compressor/CMakeLists.txt               |   2 +-
 utils/compressor/zstd_test.cpp                |   3 +-
 utils/iotest/iotest.cpp                       |   9 +-
 utils/iotest/iotest_aio.cpp                   |  10 +-
 utils/iotest/iotest_nonblock.cpp              |   8 +-
 utils/report/CMakeLists.txt                   |   1 +
 utils/report/clickhouse-report                |  30 ++
 utils/report/clickhouse_report.sh             |  23 -
 189 files changed, 2256 insertions(+), 1193 deletions(-)
 create mode 100644 dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp
 create mode 100644 dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h
 create mode 100644 dbms/tests/performance/functions_length/functions_length.xml
 create mode 100755 dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments8.sh
 create mode 100755 dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments9.sh
 create mode 100644 dbms/tests/queries/0_stateless/00604_show_create_database.reference
 create mode 100644 dbms/tests/queries/0_stateless/00604_show_create_database.sql
 create mode 100644 dbms/tests/queries/0_stateless/00609_distributed_with_case_when_then.reference
 create mode 100644 dbms/tests/queries/0_stateless/00609_distributed_with_case_when_then.sql
 create mode 100644 dbms/tests/queries/0_stateless/00609_prewhere_and_default.reference
 create mode 100644 dbms/tests/queries/0_stateless/00609_prewhere_and_default.sql
 create mode 100644 dbms/tests/queries/0_stateless/00611_zookeeper_different_checksums_formats.reference
 create mode 100644 dbms/tests/queries/0_stateless/00611_zookeeper_different_checksums_formats.sql
 create mode 100644 libs/libcommon/include/port/ssize_t.h
 create mode 100644 libs/libcommon/include/port/unistd.h
 create mode 100644 utils/build/build_msvc2017.bat
 create mode 100644 utils/report/CMakeLists.txt
 create mode 100755 utils/report/clickhouse-report
 delete mode 100755 utils/report/clickhouse_report.sh

diff --git a/cmake/find_poco.cmake b/cmake/find_poco.cmake
index 9051d63ae47..315e27b798f 100644
--- a/cmake/find_poco.cmake
+++ b/cmake/find_poco.cmake
@@ -49,7 +49,8 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY)
         set (Poco_DataODBC_INCLUDE_DIRS "${ClickHouse_SOURCE_DIR}/contrib/poco/Data/ODBC/include/")
     endif ()
 
-    if (OPENSSL_FOUND)
+    # TODO! fix internal ssl
+    if (OPENSSL_FOUND AND NOT USE_INTERNAL_SSL_LIBRARY)
         set (Poco_NetSSL_FOUND 1)
         set (Poco_NetSSL_LIBRARY PocoNetSSL)
         set (Poco_Crypto_LIBRARY PocoCrypto)
diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
index e74e9a5b7db..9015f7e7d36 100644
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@@ -100,13 +100,17 @@ if (USE_INTERNAL_RDKAFKA_LIBRARY)
     mark_as_advanced (ZLIB_INCLUDE_DIR)
 
     if (USE_INTERNAL_SSL_LIBRARY)
-        add_library(bundled-ssl ALIAS ${OPENSSL_SSL_LIBRARY})
-        set (WITH_BUNDLED_SSL 1)
+        if (MAKE_STATIC_LIBRARIES)
+            add_library(bundled-ssl ALIAS ${OPENSSL_SSL_LIBRARY})
+            set (WITH_BUNDLED_SSL 1 CACHE INTERNAL "")
+        else ()
+            set (WITH_SSL 0 CACHE INTERNAL "")
+        endif ()
     endif ()
 
     add_subdirectory (librdkafka)
 
-    if (USE_INTERNAL_SSL_LIBRARY)
+    if (USE_INTERNAL_SSL_LIBRARY AND MAKE_STATIC_LIBRARIES)
         target_include_directories(rdkafka PRIVATE BEFORE ${OPENSSL_INCLUDE_DIR})
     endif ()
     target_include_directories(rdkafka PRIVATE BEFORE ${ZLIB_INCLUDE_DIR})
@@ -127,16 +131,18 @@ endif ()
 
 
 if (USE_INTERNAL_POCO_LIBRARY)
-    set (ALLOW_DUPLICATE_CUSTOM_TARGETS 1)
     set (save_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
     set (save_CMAKE_C_FLAGS ${CMAKE_C_FLAGS})
     set (_save ${ENABLE_TESTS})
     set (ENABLE_TESTS 0)
     set (CMAKE_DISABLE_FIND_PACKAGE_ZLIB 1)
     if (USE_INTERNAL_SSL_LIBRARY)
-        set (DISABLE_INTERNAL_OPENSSL 1)
-        set (ENABLE_NETSSL 0) # TODO!
-        set (ENABLE_CRYPTO 0) # TODO!
+        set (DISABLE_INTERNAL_OPENSSL 1 CACHE INTERNAL "")
+        set (ENABLE_NETSSL 0 CACHE INTERNAL "") # TODO!
+        set (ENABLE_CRYPTO 0 CACHE INTERNAL "") # TODO!
+    endif ()
+    if (MSVC)
+        set (ENABLE_DATA_ODBC 0 CACHE INTERNAL "") # TODO (build fail)
     endif ()
     add_subdirectory (poco)
     unset (CMAKE_DISABLE_FIND_PACKAGE_ZLIB)
diff --git a/contrib/boost b/contrib/boost
index eb5943711e8..5121cc9d037 160000
--- a/contrib/boost
+++ b/contrib/boost
@@ -1 +1 @@
-Subproject commit eb5943711e88d1008583e6ae3720a5489313d02e
+Subproject commit 5121cc9d0375c7b81b24b6087a51684e6cd62ded
diff --git a/contrib/libcityhash/src/config.h b/contrib/libcityhash/src/config.h
index c873732dd87..61c75fca818 100644
--- a/contrib/libcityhash/src/config.h
+++ b/contrib/libcityhash/src/config.h
@@ -5,7 +5,11 @@
 /* #undef AC_APPLE_UNIVERSAL_BUILD */
 
 /* Define to 1 if the compiler supports __builtin_expect. */
+#if _MSC_VER
+#define HAVE_BUILTIN_EXPECT 0
+#else
 #define HAVE_BUILTIN_EXPECT 1
+#endif
 
 /* Define to 1 if you have the <dlfcn.h> header file. */
 #define HAVE_DLFCN_H 1
diff --git a/contrib/libfarmhash/CMakeLists.txt b/contrib/libfarmhash/CMakeLists.txt
index 7781028094d..16471da3f2f 100644
--- a/contrib/libfarmhash/CMakeLists.txt
+++ b/contrib/libfarmhash/CMakeLists.txt
@@ -2,4 +2,8 @@ add_library(farmhash
 	farmhash.cc
 	farmhash.h)
 
+if (MSVC)
+    target_compile_definitions (farmhash PRIVATE FARMHASH_NO_BUILTIN_EXPECT=1)
+endif ()
+
 target_include_directories (farmhash PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
diff --git a/contrib/libtcmalloc/src/config.h b/contrib/libtcmalloc/src/config.h
index 3338309073d..9f9a7a259e5 100644
--- a/contrib/libtcmalloc/src/config.h
+++ b/contrib/libtcmalloc/src/config.h
@@ -13,7 +13,11 @@
 /* #undef ENABLE_SIZED_DELETE */
 
 /* Define to 1 if compiler supports __builtin_expect */
+#if _MSC_VER
+#define HAVE_BUILTIN_EXPECT 0
+#else
 #define HAVE_BUILTIN_EXPECT 1
+#endif
 
 /* Define to 1 if compiler supports __builtin_stack_pointer */
 /* #undef HAVE_BUILTIN_STACK_POINTER */
diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt
index 965b9b2bfe1..d740ce5a45c 100644
--- a/dbms/CMakeLists.txt
+++ b/dbms/CMakeLists.txt
@@ -131,7 +131,6 @@ target_link_libraries (clickhouse_common_io
     ${LINK_LIBRARIES_ONLY_ON_X86_64}
     ${LZ4_LIBRARY}
     ${ZSTD_LIBRARY}
-    ${ZOOKEEPER_LIBRARY}
     ${DOUBLE_CONVERSION_LIBRARIES}
     ${Poco_Net_LIBRARY}
     ${Poco_Data_LIBRARY}
@@ -158,10 +157,6 @@ if (NOT USE_INTERNAL_RE2_LIBRARY)
     target_include_directories (dbms BEFORE PRIVATE ${RE2_INCLUDE_DIR})
 endif ()
 
-if (NOT USE_INTERNAL_ZOOKEEPER_LIBRARY)
-    target_include_directories (clickhouse_common_io BEFORE PUBLIC ${ZOOKEEPER_INCLUDE_DIR})
-endif ()
-
 if (NOT USE_INTERNAL_BOOST_LIBRARY)
     target_include_directories (clickhouse_common_io BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
 endif ()
diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake
index ad5ad09cce2..990c638bf46 100644
--- a/dbms/cmake/version.cmake
+++ b/dbms/cmake/version.cmake
@@ -1,6 +1,6 @@
 # This strings autochanged from release_lib.sh:
-set(VERSION_DESCRIBE v1.1.54370-testing)
-set(VERSION_REVISION 54370)
+set(VERSION_DESCRIBE v1.1.54371-testing)
+set(VERSION_REVISION 54371)
 # end of autochange
 
 set (VERSION_MAJOR 1)
diff --git a/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp b/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp
index 5fc341b3095..eca854a031b 100644
--- a/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp
+++ b/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp
@@ -70,7 +70,10 @@ AggregateFunctionPtr AggregateFunctionFactory::get(
         return combinator->transformAggregateFunction(nested_function, argument_types, parameters);
     }
 
-    return getImpl(name, argument_types, parameters, recursion_level);
+    auto res = getImpl(name, argument_types, parameters, recursion_level);
+    if (!res)
+        throw Exception("Logical error: AggregateFunctionFactory returned nullptr", ErrorCodes::LOGICAL_ERROR);
+    return res;
 }
 
 
diff --git a/dbms/src/AggregateFunctions/AggregateFunctionMaxIntersections.cpp b/dbms/src/AggregateFunctions/AggregateFunctionMaxIntersections.cpp
index e81dfe98259..9f63d399fe7 100644
--- a/dbms/src/AggregateFunctions/AggregateFunctionMaxIntersections.cpp
+++ b/dbms/src/AggregateFunctions/AggregateFunctionMaxIntersections.cpp
@@ -16,7 +16,12 @@ namespace
         assertBinary(name, argument_types);
         assertNoParameters(name, parameters);
 
-        return AggregateFunctionPtr{createWithNumericType<AggregateFunctionIntersectionsMax>(*argument_types[0], kind, argument_types)};
+        AggregateFunctionPtr res(createWithNumericType<AggregateFunctionIntersectionsMax>(*argument_types[0], kind, argument_types));
+        if (!res)
+            throw Exception("Illegal types " + argument_types[0]->getName() + " and " + argument_types[1]->getName()
+                + " of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+
+        return res;
     }
 }
 
diff --git a/dbms/src/AggregateFunctions/AggregateFunctionSumMap.cpp b/dbms/src/AggregateFunctions/AggregateFunctionSumMap.cpp
index 9aedeaabe24..7cad7c35092 100644
--- a/dbms/src/AggregateFunctions/AggregateFunctionSumMap.cpp
+++ b/dbms/src/AggregateFunctions/AggregateFunctionSumMap.cpp
@@ -37,7 +37,11 @@ AggregateFunctionPtr createAggregateFunctionSumMap(const std::string & name, con
         values_types.push_back(array_type->getNestedType());
     }
 
-    return AggregateFunctionPtr(createWithNumericType<AggregateFunctionSumMap>(*keys_type, keys_type, std::move(values_types)));
+    AggregateFunctionPtr res(createWithNumericType<AggregateFunctionSumMap>(*keys_type, keys_type, std::move(values_types)));
+    if (!res)
+        throw Exception("Illegal type of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+
+    return res;
 }
 
 }
diff --git a/dbms/src/Client/Connection.cpp b/dbms/src/Client/Connection.cpp
index 3c6c8d75d0e..0960ac8e77d 100644
--- a/dbms/src/Client/Connection.cpp
+++ b/dbms/src/Client/Connection.cpp
@@ -148,6 +148,10 @@ void Connection::receiveHello()
         {
             readStringBinary(server_timezone, *in);
         }
+        if (server_revision >= DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME)
+        {
+            readStringBinary(server_display_name, *in);
+        }
     }
     else if (packet_type == Protocol::Server::Exception)
         receiveException()->rethrow();
@@ -203,6 +207,14 @@ const String & Connection::getServerTimezone()
     return server_timezone;
 }
 
+const String & Connection::getServerDisplayName()
+{
+    if (!connected)
+        connect();
+
+    return server_display_name;
+}
+
 void Connection::forceConnected()
 {
     if (!connected)
diff --git a/dbms/src/Client/Connection.h b/dbms/src/Client/Connection.h
index c9b401c91ee..cac188f12c8 100644
--- a/dbms/src/Client/Connection.h
+++ b/dbms/src/Client/Connection.h
@@ -134,6 +134,7 @@ public:
     void getServerVersion(String & name, UInt64 & version_major, UInt64 & version_minor, UInt64 & revision);
 
     const String & getServerTimezone();
+    const String & getServerDisplayName();
 
     /// For log and exception messages.
     const String & getDescription() const;
@@ -213,6 +214,7 @@ private:
     UInt64 server_version_minor = 0;
     UInt64 server_revision = 0;
     String server_timezone;
+    String server_display_name;
 
     std::unique_ptr<Poco::Net::StreamSocket> socket;
     std::shared_ptr<ReadBuffer> in;
diff --git a/dbms/src/Columns/ColumnAggregateFunction.cpp b/dbms/src/Columns/ColumnAggregateFunction.cpp
index c901dac36f1..b989c007e56 100644
--- a/dbms/src/Columns/ColumnAggregateFunction.cpp
+++ b/dbms/src/Columns/ColumnAggregateFunction.cpp
@@ -113,7 +113,7 @@ void ColumnAggregateFunction::insertRangeFrom(const IColumn & from, size_t start
 }
 
 
-MutableColumnPtr ColumnAggregateFunction::filter(const Filter & filter, ssize_t result_size_hint) const
+ColumnPtr ColumnAggregateFunction::filter(const Filter & filter, ssize_t result_size_hint) const
 {
     size_t size = getData().size();
     if (size != filter.size())
@@ -140,7 +140,7 @@ MutableColumnPtr ColumnAggregateFunction::filter(const Filter & filter, ssize_t
 }
 
 
-MutableColumnPtr ColumnAggregateFunction::permute(const Permutation & perm, size_t limit) const
+ColumnPtr ColumnAggregateFunction::permute(const Permutation & perm, size_t limit) const
 {
     size_t size = getData().size();
 
@@ -325,7 +325,7 @@ void ColumnAggregateFunction::popBack(size_t n)
     data.resize_assume_reserved(new_size);
 }
 
-MutableColumnPtr ColumnAggregateFunction::replicate(const IColumn::Offsets & offsets) const
+ColumnPtr ColumnAggregateFunction::replicate(const IColumn::Offsets & offsets) const
 {
     size_t size = data.size();
     if (size != offsets.size())
diff --git a/dbms/src/Columns/ColumnAggregateFunction.h b/dbms/src/Columns/ColumnAggregateFunction.h
index a9ec5ed486c..852d76130d5 100644
--- a/dbms/src/Columns/ColumnAggregateFunction.h
+++ b/dbms/src/Columns/ColumnAggregateFunction.h
@@ -152,11 +152,11 @@ public:
 
     void popBack(size_t n) override;
 
-    MutableColumnPtr filter(const Filter & filter, ssize_t result_size_hint) const override;
+    ColumnPtr filter(const Filter & filter, ssize_t result_size_hint) const override;
 
-    MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
+    ColumnPtr permute(const Permutation & perm, size_t limit) const override;
 
-    MutableColumnPtr replicate(const Offsets & offsets) const override;
+    ColumnPtr replicate(const Offsets & offsets) const override;
 
     MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override;
 
diff --git a/dbms/src/Columns/ColumnArray.cpp b/dbms/src/Columns/ColumnArray.cpp
index 4674c57eef6..010569e40ad 100644
--- a/dbms/src/Columns/ColumnArray.cpp
+++ b/dbms/src/Columns/ColumnArray.cpp
@@ -31,10 +31,10 @@ namespace ErrorCodes
 }
 
 
-ColumnArray::ColumnArray(const ColumnPtr & nested_column, const ColumnPtr & offsets_column)
-    : data(nested_column), offsets(offsets_column)
+ColumnArray::ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column)
+    : data(std::move(nested_column)), offsets(std::move(offsets_column))
 {
-    if (!typeid_cast<const ColumnOffsets *>(offsets_column.get()))
+    if (!typeid_cast<const ColumnOffsets *>(offsets.get()))
         throw Exception("offsets_column must be a ColumnUInt64", ErrorCodes::ILLEGAL_COLUMN);
 
     /** NOTE
@@ -43,8 +43,8 @@ ColumnArray::ColumnArray(const ColumnPtr & nested_column, const ColumnPtr & offs
       */
 }
 
-ColumnArray::ColumnArray(const ColumnPtr & nested_column)
-    : data(nested_column)
+ColumnArray::ColumnArray(MutableColumnPtr && nested_column)
+    : data(std::move(nested_column))
 {
     if (!data->empty())
         throw Exception("Not empty data passed to ColumnArray, but no offsets passed", ErrorCodes::ILLEGAL_COLUMN);
@@ -317,7 +317,7 @@ bool ColumnArray::hasEqualOffsets(const ColumnArray & other) const
 }
 
 
-MutableColumnPtr ColumnArray::convertToFullColumnIfConst() const
+ColumnPtr ColumnArray::convertToFullColumnIfConst() const
 {
     ColumnPtr new_data;
 
@@ -391,7 +391,7 @@ void ColumnArray::insertRangeFrom(const IColumn & src, size_t start, size_t leng
 }
 
 
-MutableColumnPtr ColumnArray::filter(const Filter & filt, ssize_t result_size_hint) const
+ColumnPtr ColumnArray::filter(const Filter & filt, ssize_t result_size_hint) const
 {
     if (typeid_cast<const ColumnUInt8 *>(data.get()))      return filterNumber<UInt8>(filt, result_size_hint);
     if (typeid_cast<const ColumnUInt16 *>(data.get()))     return filterNumber<UInt16>(filt, result_size_hint);
@@ -410,7 +410,7 @@ MutableColumnPtr ColumnArray::filter(const Filter & filt, ssize_t result_size_hi
 }
 
 template <typename T>
-MutableColumnPtr ColumnArray::filterNumber(const Filter & filt, ssize_t result_size_hint) const
+ColumnPtr ColumnArray::filterNumber(const Filter & filt, ssize_t result_size_hint) const
 {
     if (getOffsets().size() == 0)
         return ColumnArray::create(data);
@@ -424,7 +424,7 @@ MutableColumnPtr ColumnArray::filterNumber(const Filter & filt, ssize_t result_s
     return std::move(res);
 }
 
-MutableColumnPtr ColumnArray::filterString(const Filter & filt, ssize_t result_size_hint) const
+ColumnPtr ColumnArray::filterString(const Filter & filt, ssize_t result_size_hint) const
 {
     size_t col_size = getOffsets().size();
     if (col_size != filt.size())
@@ -492,7 +492,7 @@ MutableColumnPtr ColumnArray::filterString(const Filter & filt, ssize_t result_s
     return std::move(res);
 }
 
-MutableColumnPtr ColumnArray::filterGeneric(const Filter & filt, ssize_t result_size_hint) const
+ColumnPtr ColumnArray::filterGeneric(const Filter & filt, ssize_t result_size_hint) const
 {
     size_t size = getOffsets().size();
     if (size != filt.size())
@@ -537,7 +537,7 @@ MutableColumnPtr ColumnArray::filterGeneric(const Filter & filt, ssize_t result_
     return std::move(res);
 }
 
-MutableColumnPtr ColumnArray::filterNullable(const Filter & filt, ssize_t result_size_hint) const
+ColumnPtr ColumnArray::filterNullable(const Filter & filt, ssize_t result_size_hint) const
 {
     if (getOffsets().size() == 0)
         return ColumnArray::create(data);
@@ -560,7 +560,7 @@ MutableColumnPtr ColumnArray::filterNullable(const Filter & filt, ssize_t result
         filtered_offsets);
 }
 
-MutableColumnPtr ColumnArray::filterTuple(const Filter & filt, ssize_t result_size_hint) const
+ColumnPtr ColumnArray::filterTuple(const Filter & filt, ssize_t result_size_hint) const
 {
     if (getOffsets().size() == 0)
         return ColumnArray::create(data);
@@ -576,7 +576,8 @@ MutableColumnPtr ColumnArray::filterTuple(const Filter & filt, ssize_t result_si
 
     Columns temporary_arrays(tuple_size);
     for (size_t i = 0; i < tuple_size; ++i)
-        temporary_arrays[i] = ColumnArray(tuple.getColumns()[i], getOffsetsPtr()).filter(filt, result_size_hint);
+        temporary_arrays[i] = ColumnArray(tuple.getColumns()[i]->assumeMutable(), getOffsetsPtr()->assumeMutable())
+                .filter(filt, result_size_hint);
 
     Columns tuple_columns(tuple_size);
     for (size_t i = 0; i < tuple_size; ++i)
@@ -588,7 +589,7 @@ MutableColumnPtr ColumnArray::filterTuple(const Filter & filt, ssize_t result_si
 }
 
 
-MutableColumnPtr ColumnArray::permute(const Permutation & perm, size_t limit) const
+ColumnPtr ColumnArray::permute(const Permutation & perm, size_t limit) const
 {
     size_t size = getOffsets().size();
 
@@ -652,7 +653,7 @@ void ColumnArray::getPermutation(bool reverse, size_t limit, int nan_direction_h
 }
 
 
-MutableColumnPtr ColumnArray::replicate(const Offsets & replicate_offsets) const
+ColumnPtr ColumnArray::replicate(const Offsets & replicate_offsets) const
 {
     if (typeid_cast<const ColumnUInt8 *>(data.get()))     return replicateNumber<UInt8>(replicate_offsets);
     if (typeid_cast<const ColumnUInt16 *>(data.get()))    return replicateNumber<UInt16>(replicate_offsets);
@@ -673,7 +674,7 @@ MutableColumnPtr ColumnArray::replicate(const Offsets & replicate_offsets) const
 
 
 template <typename T>
-MutableColumnPtr ColumnArray::replicateNumber(const Offsets & replicate_offsets) const
+ColumnPtr ColumnArray::replicateNumber(const Offsets & replicate_offsets) const
 {
     size_t col_size = size();
     if (col_size != replicate_offsets.size())
@@ -721,7 +722,7 @@ MutableColumnPtr ColumnArray::replicateNumber(const Offsets & replicate_offsets)
 }
 
 
-MutableColumnPtr ColumnArray::replicateString(const Offsets & replicate_offsets) const
+ColumnPtr ColumnArray::replicateString(const Offsets & replicate_offsets) const
 {
     size_t col_size = size();
     if (col_size != replicate_offsets.size())
@@ -796,7 +797,7 @@ MutableColumnPtr ColumnArray::replicateString(const Offsets & replicate_offsets)
 }
 
 
-MutableColumnPtr ColumnArray::replicateConst(const Offsets & replicate_offsets) const
+ColumnPtr ColumnArray::replicateConst(const Offsets & replicate_offsets) const
 {
     size_t col_size = size();
     if (col_size != replicate_offsets.size())
@@ -834,7 +835,7 @@ MutableColumnPtr ColumnArray::replicateConst(const Offsets & replicate_offsets)
 }
 
 
-MutableColumnPtr ColumnArray::replicateGeneric(const Offsets & replicate_offsets) const
+ColumnPtr ColumnArray::replicateGeneric(const Offsets & replicate_offsets) const
 {
     size_t col_size = size();
     if (col_size != replicate_offsets.size())
@@ -860,25 +861,27 @@ MutableColumnPtr ColumnArray::replicateGeneric(const Offsets & replicate_offsets
 }
 
 
-MutableColumnPtr ColumnArray::replicateNullable(const Offsets & replicate_offsets) const
+ColumnPtr ColumnArray::replicateNullable(const Offsets & replicate_offsets) const
 {
     const ColumnNullable & nullable = static_cast<const ColumnNullable &>(*data);
 
     /// Make temporary arrays for each components of Nullable. Then replicate them independently and collect back to result.
     /// NOTE Offsets are calculated twice and it is redundant.
 
-    auto array_of_nested = ColumnArray(nullable.getNestedColumnPtr(), getOffsetsPtr()).replicate(replicate_offsets);
-    auto array_of_null_map = ColumnArray(nullable.getNullMapColumnPtr(), getOffsetsPtr()).replicate(replicate_offsets);
+    auto array_of_nested = ColumnArray(nullable.getNestedColumnPtr()->assumeMutable(), getOffsetsPtr()->assumeMutable())
+            .replicate(replicate_offsets);
+    auto array_of_null_map = ColumnArray(nullable.getNullMapColumnPtr()->assumeMutable(), getOffsetsPtr()->assumeMutable())
+            .replicate(replicate_offsets);
 
     return ColumnArray::create(
         ColumnNullable::create(
-            static_cast<ColumnArray &>(*array_of_nested).getDataPtr(),
-            static_cast<ColumnArray &>(*array_of_null_map).getDataPtr()),
-        static_cast<ColumnArray &>(*array_of_nested).getOffsetsPtr());
+            static_cast<const ColumnArray &>(*array_of_nested).getDataPtr(),
+            static_cast<const ColumnArray &>(*array_of_null_map).getDataPtr()),
+        static_cast<const ColumnArray &>(*array_of_nested).getOffsetsPtr());
 }
 
 
-MutableColumnPtr ColumnArray::replicateTuple(const Offsets & replicate_offsets) const
+ColumnPtr ColumnArray::replicateTuple(const Offsets & replicate_offsets) const
 {
     const ColumnTuple & tuple = static_cast<const ColumnTuple &>(*data);
 
@@ -891,7 +894,8 @@ MutableColumnPtr ColumnArray::replicateTuple(const Offsets & replicate_offsets)
 
     Columns temporary_arrays(tuple_size);
     for (size_t i = 0; i < tuple_size; ++i)
-        temporary_arrays[i] = ColumnArray(tuple.getColumns()[i], getOffsetsPtr()).replicate(replicate_offsets);
+        temporary_arrays[i] = ColumnArray(tuple.getColumns()[i]->assumeMutable(), getOffsetsPtr()->assumeMutable())
+                .replicate(replicate_offsets);
 
     Columns tuple_columns(tuple_size);
     for (size_t i = 0; i < tuple_size; ++i)
diff --git a/dbms/src/Columns/ColumnArray.h b/dbms/src/Columns/ColumnArray.h
index d1f951eedb7..ce7744b1fc1 100644
--- a/dbms/src/Columns/ColumnArray.h
+++ b/dbms/src/Columns/ColumnArray.h
@@ -24,14 +24,32 @@ private:
     friend class COWPtrHelper<IColumn, ColumnArray>;
 
     /** Create an array column with specified values and offsets. */
-    ColumnArray(const ColumnPtr & nested_column, const ColumnPtr & offsets_column);
+    ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column);
 
     /** Create an empty column of arrays with the type of values as in the column `nested_column` */
-    ColumnArray(const ColumnPtr & nested_column);
+    explicit ColumnArray(MutableColumnPtr && nested_column);
 
     ColumnArray(const ColumnArray &) = default;
 
 public:
+    /** Create immutable column using immutable arguments. This arguments may be shared with other columns.
+      * Use IColumn::mutate in order to make mutable column and mutate shared nested columns.
+      */
+    using Base = COWPtrHelper<IColumn, ColumnArray>;
+
+    static Ptr create(const ColumnPtr & nested_column, const ColumnPtr & offsets_column)
+    {
+        return ColumnArray::create(nested_column->assumeMutable(), offsets_column->assumeMutable());
+    }
+
+    static Ptr create(const ColumnPtr & nested_column)
+    {
+        return ColumnArray::create(nested_column->assumeMutable());
+    }
+
+    template <typename ... Args, typename = typename std::enable_if<IsMutableColumns<Args ...>::value>::type>
+    static MutablePtr create(Args &&... args) { return Base::create(std::forward<Args>(args)...); }
+
     /** On the index i there is an offset to the beginning of the i + 1 -th element. */
     using ColumnOffsets = ColumnVector<Offset>;
 
@@ -51,15 +69,15 @@ public:
     void insertFrom(const IColumn & src_, size_t n) override;
     void insertDefault() override;
     void popBack(size_t n) override;
-    MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
-    MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
+    ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
+    ColumnPtr permute(const Permutation & perm, size_t limit) const override;
     int compareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const override;
     void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override;
     void reserve(size_t n) override;
     size_t byteSize() const override;
     size_t allocatedBytes() const override;
-    MutableColumnPtr replicate(const Offsets & replicate_offsets) const override;
-    MutableColumnPtr convertToFullColumnIfConst() const override;
+    ColumnPtr replicate(const Offsets & replicate_offsets) const override;
+    ColumnPtr convertToFullColumnIfConst() const override;
     void getExtremes(Field & min, Field & max) const override;
 
     bool hasEqualOffsets(const ColumnArray & other) const;
@@ -110,33 +128,33 @@ private:
 
     /// Multiply values if the nested column is ColumnVector<T>.
     template <typename T>
-    MutableColumnPtr replicateNumber(const Offsets & replicate_offsets) const;
+    ColumnPtr replicateNumber(const Offsets & replicate_offsets) const;
 
     /// Multiply the values if the nested column is ColumnString. The code is too complicated.
-    MutableColumnPtr replicateString(const Offsets & replicate_offsets) const;
+    ColumnPtr replicateString(const Offsets & replicate_offsets) const;
 
     /** Non-constant arrays of constant values are quite rare.
       * Most functions can not work with them, and does not create such columns as a result.
       * An exception is the function `replicate`(see FunctionsMiscellaneous.h), which has service meaning for the implementation of lambda functions.
       * Only for its sake is the implementation of the `replicate` method for ColumnArray(ColumnConst).
       */
-    MutableColumnPtr replicateConst(const Offsets & replicate_offsets) const;
+    ColumnPtr replicateConst(const Offsets & replicate_offsets) const;
 
     /** The following is done by simply replicating of nested columns.
       */
-    MutableColumnPtr replicateTuple(const Offsets & replicate_offsets) const;
-    MutableColumnPtr replicateNullable(const Offsets & replicate_offsets) const;
-    MutableColumnPtr replicateGeneric(const Offsets & replicate_offsets) const;
+    ColumnPtr replicateTuple(const Offsets & replicate_offsets) const;
+    ColumnPtr replicateNullable(const Offsets & replicate_offsets) const;
+    ColumnPtr replicateGeneric(const Offsets & replicate_offsets) const;
 
 
     /// Specializations for the filter function.
     template <typename T>
-    MutableColumnPtr filterNumber(const Filter & filt, ssize_t result_size_hint) const;
+    ColumnPtr filterNumber(const Filter & filt, ssize_t result_size_hint) const;
 
-    MutableColumnPtr filterString(const Filter & filt, ssize_t result_size_hint) const;
-    MutableColumnPtr filterTuple(const Filter & filt, ssize_t result_size_hint) const;
-    MutableColumnPtr filterNullable(const Filter & filt, ssize_t result_size_hint) const;
-    MutableColumnPtr filterGeneric(const Filter & filt, ssize_t result_size_hint) const;
+    ColumnPtr filterString(const Filter & filt, ssize_t result_size_hint) const;
+    ColumnPtr filterTuple(const Filter & filt, ssize_t result_size_hint) const;
+    ColumnPtr filterNullable(const Filter & filt, ssize_t result_size_hint) const;
+    ColumnPtr filterGeneric(const Filter & filt, ssize_t result_size_hint) const;
 };
 
 
diff --git a/dbms/src/Columns/ColumnConst.cpp b/dbms/src/Columns/ColumnConst.cpp
index 53e43547108..325160e5fc5 100644
--- a/dbms/src/Columns/ColumnConst.cpp
+++ b/dbms/src/Columns/ColumnConst.cpp
@@ -25,12 +25,12 @@ ColumnConst::ColumnConst(const ColumnPtr & data_, size_t s)
             ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
 }
 
-MutableColumnPtr ColumnConst::convertToFullColumn() const
+ColumnPtr ColumnConst::convertToFullColumn() const
 {
     return data->replicate(Offsets(1, s));
 }
 
-MutableColumnPtr ColumnConst::filter(const Filter & filt, ssize_t /*result_size_hint*/) const
+ColumnPtr ColumnConst::filter(const Filter & filt, ssize_t /*result_size_hint*/) const
 {
     if (s != filt.size())
         throw Exception("Size of filter (" + toString(filt.size()) + ") doesn't match size of column (" + toString(s) + ")",
@@ -39,7 +39,7 @@ MutableColumnPtr ColumnConst::filter(const Filter & filt, ssize_t /*result_size_
     return ColumnConst::create(data, countBytesInFilter(filt));
 }
 
-MutableColumnPtr ColumnConst::replicate(const Offsets & offsets) const
+ColumnPtr ColumnConst::replicate(const Offsets & offsets) const
 {
     if (s != offsets.size())
         throw Exception("Size of offsets (" + toString(offsets.size()) + ") doesn't match size of column (" + toString(s) + ")",
@@ -49,7 +49,7 @@ MutableColumnPtr ColumnConst::replicate(const Offsets & offsets) const
     return ColumnConst::create(data, replicated_size);
 }
 
-MutableColumnPtr ColumnConst::permute(const Permutation & perm, size_t limit) const
+ColumnPtr ColumnConst::permute(const Permutation & perm, size_t limit) const
 {
     if (limit == 0)
         limit = s;
diff --git a/dbms/src/Columns/ColumnConst.h b/dbms/src/Columns/ColumnConst.h
index 0ff94d6ec4b..2e4a692451f 100644
--- a/dbms/src/Columns/ColumnConst.h
+++ b/dbms/src/Columns/ColumnConst.h
@@ -29,9 +29,9 @@ private:
     ColumnConst(const ColumnConst & src) = default;
 
 public:
-    MutableColumnPtr convertToFullColumn() const;
+    ColumnPtr convertToFullColumn() const;
 
-    MutableColumnPtr convertToFullColumnIfConst() const override
+    ColumnPtr convertToFullColumnIfConst() const override
     {
         return convertToFullColumn();
     }
@@ -145,9 +145,9 @@ public:
         data->updateHashWithValue(0, hash);
     }
 
-    MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
-    MutableColumnPtr replicate(const Offsets & offsets) const override;
-    MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
+    ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
+    ColumnPtr replicate(const Offsets & offsets) const override;
+    ColumnPtr permute(const Permutation & perm, size_t limit) const override;
     void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override;
 
     size_t byteSize() const override
diff --git a/dbms/src/Columns/ColumnFixedString.cpp b/dbms/src/Columns/ColumnFixedString.cpp
index 61e2fbdd7d6..2da3c28e50a 100644
--- a/dbms/src/Columns/ColumnFixedString.cpp
+++ b/dbms/src/Columns/ColumnFixedString.cpp
@@ -153,7 +153,7 @@ void ColumnFixedString::insertRangeFrom(const IColumn & src, size_t start, size_
     memcpy(&chars[old_size], &src_concrete.chars[start * n], length * n);
 }
 
-MutableColumnPtr ColumnFixedString::filter(const IColumn::Filter & filt, ssize_t result_size_hint) const
+ColumnPtr ColumnFixedString::filter(const IColumn::Filter & filt, ssize_t result_size_hint) const
 {
     size_t col_size = size();
     if (col_size != filt.size())
@@ -230,7 +230,7 @@ MutableColumnPtr ColumnFixedString::filter(const IColumn::Filter & filt, ssize_t
     return std::move(res);
 }
 
-MutableColumnPtr ColumnFixedString::permute(const Permutation & perm, size_t limit) const
+ColumnPtr ColumnFixedString::permute(const Permutation & perm, size_t limit) const
 {
     size_t col_size = size();
 
@@ -258,7 +258,7 @@ MutableColumnPtr ColumnFixedString::permute(const Permutation & perm, size_t lim
     return std::move(res);
 }
 
-MutableColumnPtr ColumnFixedString::replicate(const Offsets & offsets) const
+ColumnPtr ColumnFixedString::replicate(const Offsets & offsets) const
 {
     size_t col_size = size();
     if (col_size != offsets.size())
diff --git a/dbms/src/Columns/ColumnFixedString.h b/dbms/src/Columns/ColumnFixedString.h
index e5ed788aaf0..cd465a1814d 100644
--- a/dbms/src/Columns/ColumnFixedString.h
+++ b/dbms/src/Columns/ColumnFixedString.h
@@ -104,11 +104,11 @@ public:
 
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
 
-    MutableColumnPtr filter(const IColumn::Filter & filt, ssize_t result_size_hint) const override;
+    ColumnPtr filter(const IColumn::Filter & filt, ssize_t result_size_hint) const override;
 
-    MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
+    ColumnPtr permute(const Permutation & perm, size_t limit) const override;
 
-    MutableColumnPtr replicate(const Offsets & offsets) const override;
+    ColumnPtr replicate(const Offsets & offsets) const override;
 
     MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override
     {
diff --git a/dbms/src/Columns/ColumnFunction.cpp b/dbms/src/Columns/ColumnFunction.cpp
index 498bcbc26b3..4675e7915f7 100644
--- a/dbms/src/Columns/ColumnFunction.cpp
+++ b/dbms/src/Columns/ColumnFunction.cpp
@@ -26,7 +26,7 @@ MutableColumnPtr ColumnFunction::cloneResized(size_t size) const
     return ColumnFunction::create(size, function, capture);
 }
 
-MutableColumnPtr ColumnFunction::replicate(const Offsets & offsets) const
+ColumnPtr ColumnFunction::replicate(const Offsets & offsets) const
 {
     if (size_ != offsets.size())
         throw Exception("Size of offsets (" + toString(offsets.size()) + ") doesn't match size of column ("
@@ -40,7 +40,7 @@ MutableColumnPtr ColumnFunction::replicate(const Offsets & offsets) const
     return ColumnFunction::create(replicated_size, function, capture);
 }
 
-MutableColumnPtr ColumnFunction::cut(size_t start, size_t length) const
+ColumnPtr ColumnFunction::cut(size_t start, size_t length) const
 {
     ColumnsWithTypeAndName capture = captured_columns;
     for (auto & column : capture)
@@ -49,7 +49,7 @@ MutableColumnPtr ColumnFunction::cut(size_t start, size_t length) const
     return ColumnFunction::create(length, function, capture);
 }
 
-MutableColumnPtr ColumnFunction::filter(const Filter & filt, ssize_t result_size_hint) const
+ColumnPtr ColumnFunction::filter(const Filter & filt, ssize_t result_size_hint) const
 {
     if (size_ != filt.size())
         throw Exception("Size of filter (" + toString(filt.size()) + ") doesn't match size of column ("
@@ -68,7 +68,7 @@ MutableColumnPtr ColumnFunction::filter(const Filter & filt, ssize_t result_size
     return ColumnFunction::create(filtered_size, function, capture);
 }
 
-MutableColumnPtr ColumnFunction::permute(const Permutation & perm, size_t limit) const
+ColumnPtr ColumnFunction::permute(const Permutation & perm, size_t limit) const
 {
     if (limit == 0)
         limit = size_;
diff --git a/dbms/src/Columns/ColumnFunction.h b/dbms/src/Columns/ColumnFunction.h
index 087086302aa..d10459175cc 100644
--- a/dbms/src/Columns/ColumnFunction.h
+++ b/dbms/src/Columns/ColumnFunction.h
@@ -29,10 +29,10 @@ public:
 
     size_t size() const override { return size_; }
 
-    MutableColumnPtr cut(size_t start, size_t length) const override;
-    MutableColumnPtr replicate(const Offsets & offsets) const override;
-    MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
-    MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
+    ColumnPtr cut(size_t start, size_t length) const override;
+    ColumnPtr replicate(const Offsets & offsets) const override;
+    ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
+    ColumnPtr permute(const Permutation & perm, size_t limit) const override;
     void insertDefault() override;
     void popBack(size_t n) override;
     std::vector<MutableColumnPtr> scatter(IColumn::ColumnIndex num_columns,
diff --git a/dbms/src/Columns/ColumnNullable.cpp b/dbms/src/Columns/ColumnNullable.cpp
index 59f225c962f..557a57b3c6d 100644
--- a/dbms/src/Columns/ColumnNullable.cpp
+++ b/dbms/src/Columns/ColumnNullable.cpp
@@ -18,8 +18,8 @@ namespace ErrorCodes
 }
 
 
-ColumnNullable::ColumnNullable(const ColumnPtr & nested_column_, const ColumnPtr & null_map_)
-    : nested_column{nested_column_}, null_map{null_map_}
+ColumnNullable::ColumnNullable(MutableColumnPtr && nested_column_, MutableColumnPtr && null_map_)
+    : nested_column(std::move(nested_column_)), null_map(std::move(null_map_))
 {
     /// ColumnNullable cannot have constant nested column. But constant argument could be passed. Materialize it.
     if (ColumnPtr nested_column_materialized = getNestedColumn().convertToFullColumnIfConst())
@@ -44,7 +44,7 @@ void ColumnNullable::updateHashWithValue(size_t n, SipHash & hash) const
 
 MutableColumnPtr ColumnNullable::cloneResized(size_t new_size) const
 {
-    ColumnPtr new_nested_col = getNestedColumn().cloneResized(new_size);
+    MutableColumnPtr new_nested_col = getNestedColumn().cloneResized(new_size);
     auto new_null_map = ColumnUInt8::create();
 
     if (new_size > 0)
@@ -59,7 +59,7 @@ MutableColumnPtr ColumnNullable::cloneResized(size_t new_size) const
             memset(&new_null_map->getData()[count], 1, new_size - count);
     }
 
-    return ColumnNullable::create(new_nested_col, std::move(new_null_map));
+    return ColumnNullable::create(std::move(new_nested_col), std::move(new_null_map));
 }
 
 
@@ -152,14 +152,14 @@ void ColumnNullable::popBack(size_t n)
     getNullMapColumn().popBack(n);
 }
 
-MutableColumnPtr ColumnNullable::filter(const Filter & filt, ssize_t result_size_hint) const
+ColumnPtr ColumnNullable::filter(const Filter & filt, ssize_t result_size_hint) const
 {
     ColumnPtr filtered_data = getNestedColumn().filter(filt, result_size_hint);
     ColumnPtr filtered_null_map = getNullMapColumn().filter(filt, result_size_hint);
     return ColumnNullable::create(filtered_data, filtered_null_map);
 }
 
-MutableColumnPtr ColumnNullable::permute(const Permutation & perm, size_t limit) const
+ColumnPtr ColumnNullable::permute(const Permutation & perm, size_t limit) const
 {
     ColumnPtr permuted_data = getNestedColumn().permute(perm, limit);
     ColumnPtr permuted_null_map = getNullMapColumn().permute(perm, limit);
@@ -384,7 +384,7 @@ void ColumnNullable::getExtremes(Field & min, Field & max) const
 }
 
 
-MutableColumnPtr ColumnNullable::replicate(const Offsets & offsets) const
+ColumnPtr ColumnNullable::replicate(const Offsets & offsets) const
 {
     ColumnPtr replicated_data = getNestedColumn().replicate(offsets);
     ColumnPtr replicated_null_map = getNullMapColumn().replicate(offsets);
diff --git a/dbms/src/Columns/ColumnNullable.h b/dbms/src/Columns/ColumnNullable.h
index 6eff31379ff..4ac0f87b8da 100644
--- a/dbms/src/Columns/ColumnNullable.h
+++ b/dbms/src/Columns/ColumnNullable.h
@@ -23,10 +23,22 @@ class ColumnNullable final : public COWPtrHelper<IColumn, ColumnNullable>
 private:
     friend class COWPtrHelper<IColumn, ColumnNullable>;
 
-    ColumnNullable(const ColumnPtr & nested_column_, const ColumnPtr & null_map_);
+    ColumnNullable(MutableColumnPtr && nested_column_, MutableColumnPtr && null_map_);
     ColumnNullable(const ColumnNullable &) = default;
 
 public:
+    /** Create immutable column using immutable arguments. This arguments may be shared with other columns.
+      * Use IColumn::mutate in order to make mutable column and mutate shared nested columns.
+      */
+    using Base = COWPtrHelper<IColumn, ColumnNullable>;
+    static Ptr create(const ColumnPtr & nested_column_, const ColumnPtr & null_map_)
+    {
+        return ColumnNullable::create(nested_column_->assumeMutable(), null_map_->assumeMutable());
+    }
+
+    template <typename ... Args, typename = typename std::enable_if<IsMutableColumns<Args ...>::value>::type>
+    static MutablePtr create(Args &&... args) { return Base::create(std::forward<Args>(args)...); }
+
     const char * getFamilyName() const override { return "Nullable"; }
     std::string getName() const override { return "Nullable(" + nested_column->getName() + ")"; }
     MutableColumnPtr cloneResized(size_t size) const override;
@@ -50,14 +62,14 @@ public:
     }
 
     void popBack(size_t n) override;
-    MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
-    MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
+    ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
+    ColumnPtr permute(const Permutation & perm, size_t limit) const override;
     int compareAt(size_t n, size_t m, const IColumn & rhs_, int null_direction_hint) const override;
     void getPermutation(bool reverse, size_t limit, int null_direction_hint, Permutation & res) const override;
     void reserve(size_t n) override;
     size_t byteSize() const override;
     size_t allocatedBytes() const override;
-    MutableColumnPtr replicate(const Offsets & replicate_offsets) const override;
+    ColumnPtr replicate(const Offsets & replicate_offsets) const override;
     void updateHashWithValue(size_t n, SipHash & hash) const override;
     void getExtremes(Field & min, Field & max) const override;
 
diff --git a/dbms/src/Columns/ColumnString.cpp b/dbms/src/Columns/ColumnString.cpp
index 654a56590d1..ecb3bc6f62e 100644
--- a/dbms/src/Columns/ColumnString.cpp
+++ b/dbms/src/Columns/ColumnString.cpp
@@ -97,7 +97,7 @@ void ColumnString::insertRangeFrom(const IColumn & src, size_t start, size_t len
 }
 
 
-MutableColumnPtr ColumnString::filter(const Filter & filt, ssize_t result_size_hint) const
+ColumnPtr ColumnString::filter(const Filter & filt, ssize_t result_size_hint) const
 {
     if (offsets.size() == 0)
         return ColumnString::create();
@@ -112,7 +112,7 @@ MutableColumnPtr ColumnString::filter(const Filter & filt, ssize_t result_size_h
 }
 
 
-MutableColumnPtr ColumnString::permute(const Permutation & perm, size_t limit) const
+ColumnPtr ColumnString::permute(const Permutation & perm, size_t limit) const
 {
     size_t size = offsets.size();
 
@@ -208,7 +208,7 @@ void ColumnString::getPermutation(bool reverse, size_t limit, int /*nan_directio
 }
 
 
-MutableColumnPtr ColumnString::replicate(const Offsets & replicate_offsets) const
+ColumnPtr ColumnString::replicate(const Offsets & replicate_offsets) const
 {
     size_t col_size = size();
     if (col_size != replicate_offsets.size())
diff --git a/dbms/src/Columns/ColumnString.h b/dbms/src/Columns/ColumnString.h
index cd60eedfa43..c37fca3ab44 100644
--- a/dbms/src/Columns/ColumnString.h
+++ b/dbms/src/Columns/ColumnString.h
@@ -206,9 +206,9 @@ public:
 
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
 
-    MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
+    ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
 
-    MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
+    ColumnPtr permute(const Permutation & perm, size_t limit) const override;
 
     void insertDefault() override
     {
@@ -239,7 +239,7 @@ public:
     /// Sorting with respect of collation.
     void getPermutationWithCollation(const Collator & collator, bool reverse, size_t limit, Permutation & res) const;
 
-    MutableColumnPtr replicate(const Offsets & replicate_offsets) const override;
+    ColumnPtr replicate(const Offsets & replicate_offsets) const override;
 
     MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override
     {
diff --git a/dbms/src/Columns/ColumnTuple.cpp b/dbms/src/Columns/ColumnTuple.cpp
index 4bb250aa4c9..558b1685276 100644
--- a/dbms/src/Columns/ColumnTuple.cpp
+++ b/dbms/src/Columns/ColumnTuple.cpp
@@ -31,21 +31,38 @@ std::string ColumnTuple::getName() const
     return res.str();
 }
 
-ColumnTuple::ColumnTuple(const Columns & columns) : columns(columns)
+ColumnTuple::ColumnTuple(MutableColumns && mutable_columns)
+{
+    columns.reserve(mutable_columns.size());
+    for (auto & column : mutable_columns)
+    {
+        if (column->isColumnConst())
+            throw Exception{"ColumnTuple cannot have ColumnConst as its element", ErrorCodes::ILLEGAL_COLUMN};
+
+        columns.push_back(std::move(column));
+    }
+}
+
+ColumnTuple::Ptr ColumnTuple::create(const Columns & columns)
 {
     for (const auto & column : columns)
         if (column->isColumnConst())
             throw Exception{"ColumnTuple cannot have ColumnConst as its element", ErrorCodes::ILLEGAL_COLUMN};
+
+    auto column_tuple = ColumnTuple::create(MutableColumns());
+    column_tuple->columns = columns;
+
+    return std::move(column_tuple);
 }
 
 MutableColumnPtr ColumnTuple::cloneEmpty() const
 {
     const size_t tuple_size = columns.size();
-    Columns new_columns(tuple_size);
+    MutableColumns new_columns(tuple_size);
     for (size_t i = 0; i < tuple_size; ++i)
         new_columns[i] = columns[i]->cloneEmpty();
 
-    return ColumnTuple::create(new_columns);
+    return ColumnTuple::create(std::move(new_columns));
 }
 
 Field ColumnTuple::operator[](size_t n) const
@@ -140,7 +157,7 @@ void ColumnTuple::insertRangeFrom(const IColumn & src, size_t start, size_t leng
             start, length);
 }
 
-MutableColumnPtr ColumnTuple::filter(const Filter & filt, ssize_t result_size_hint) const
+ColumnPtr ColumnTuple::filter(const Filter & filt, ssize_t result_size_hint) const
 {
     const size_t tuple_size = columns.size();
     Columns new_columns(tuple_size);
@@ -151,7 +168,7 @@ MutableColumnPtr ColumnTuple::filter(const Filter & filt, ssize_t result_size_hi
     return ColumnTuple::create(new_columns);
 }
 
-MutableColumnPtr ColumnTuple::permute(const Permutation & perm, size_t limit) const
+ColumnPtr ColumnTuple::permute(const Permutation & perm, size_t limit) const
 {
     const size_t tuple_size = columns.size();
     Columns new_columns(tuple_size);
@@ -162,7 +179,7 @@ MutableColumnPtr ColumnTuple::permute(const Permutation & perm, size_t limit) co
     return ColumnTuple::create(new_columns);
 }
 
-MutableColumnPtr ColumnTuple::replicate(const Offsets & offsets) const
+ColumnPtr ColumnTuple::replicate(const Offsets & offsets) const
 {
     const size_t tuple_size = columns.size();
     Columns new_columns(tuple_size);
@@ -185,10 +202,10 @@ MutableColumns ColumnTuple::scatter(ColumnIndex num_columns, const Selector & se
 
     for (size_t scattered_idx = 0; scattered_idx < num_columns; ++scattered_idx)
     {
-        Columns new_columns(tuple_size);
+        MutableColumns new_columns(tuple_size);
         for (size_t tuple_element_idx = 0; tuple_element_idx < tuple_size; ++tuple_element_idx)
             new_columns[tuple_element_idx] = std::move(scattered_tuple_elements[tuple_element_idx][scattered_idx]);
-        res[scattered_idx] = ColumnTuple::create(new_columns);
+        res[scattered_idx] = ColumnTuple::create(std::move(new_columns));
     }
 
     return res;
diff --git a/dbms/src/Columns/ColumnTuple.h b/dbms/src/Columns/ColumnTuple.h
index c2ca2e43ab7..a91cb210212 100644
--- a/dbms/src/Columns/ColumnTuple.h
+++ b/dbms/src/Columns/ColumnTuple.h
@@ -22,10 +22,19 @@ private:
     template <bool positive>
     struct Less;
 
-    ColumnTuple(const Columns & columns);
+    explicit ColumnTuple(MutableColumns && columns);
     ColumnTuple(const ColumnTuple &) = default;
 
 public:
+    /** Create immutable column using immutable arguments. This arguments may be shared with other columns.
+      * Use IColumn::mutate in order to make mutable column and mutate shared nested columns.
+      */
+    using Base = COWPtrHelper<IColumn, ColumnTuple>;
+    static Ptr create(const Columns & columns);
+
+    template <typename Arg, typename = typename std::enable_if<std::is_rvalue_reference<Arg &&>::value>::type>
+    static MutablePtr create(Arg && arg) { return Base::create(std::forward<Arg>(arg)); }
+
     std::string getName() const override;
     const char * getFamilyName() const override { return "Tuple"; }
 
@@ -49,9 +58,9 @@ public:
     const char * deserializeAndInsertFromArena(const char * pos) override;
     void updateHashWithValue(size_t n, SipHash & hash) const override;
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
-    MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
-    MutableColumnPtr permute(const Permutation & perm, size_t limit) const override;
-    MutableColumnPtr replicate(const Offsets & offsets) const override;
+    ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
+    ColumnPtr permute(const Permutation & perm, size_t limit) const override;
+    ColumnPtr replicate(const Offsets & offsets) const override;
     MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override;
     void gather(ColumnGathererStream & gatherer_stream) override;
     int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
diff --git a/dbms/src/Columns/ColumnVector.cpp b/dbms/src/Columns/ColumnVector.cpp
index b1a3e55cb86..3509c14c3aa 100644
--- a/dbms/src/Columns/ColumnVector.cpp
+++ b/dbms/src/Columns/ColumnVector.cpp
@@ -146,7 +146,7 @@ void ColumnVector<T>::insertRangeFrom(const IColumn & src, size_t start, size_t
 }
 
 template <typename T>
-MutableColumnPtr ColumnVector<T>::filter(const IColumn::Filter & filt, ssize_t result_size_hint) const
+ColumnPtr ColumnVector<T>::filter(const IColumn::Filter & filt, ssize_t result_size_hint) const
 {
     size_t size = data.size();
     if (size != filt.size())
@@ -210,7 +210,7 @@ MutableColumnPtr ColumnVector<T>::filter(const IColumn::Filter & filt, ssize_t r
 }
 
 template <typename T>
-MutableColumnPtr ColumnVector<T>::permute(const IColumn::Permutation & perm, size_t limit) const
+ColumnPtr ColumnVector<T>::permute(const IColumn::Permutation & perm, size_t limit) const
 {
     size_t size = data.size();
 
@@ -231,7 +231,7 @@ MutableColumnPtr ColumnVector<T>::permute(const IColumn::Permutation & perm, siz
 }
 
 template <typename T>
-MutableColumnPtr ColumnVector<T>::replicate(const IColumn::Offsets & offsets) const
+ColumnPtr ColumnVector<T>::replicate(const IColumn::Offsets & offsets) const
 {
     size_t size = data.size();
     if (size != offsets.size())
diff --git a/dbms/src/Columns/ColumnVector.h b/dbms/src/Columns/ColumnVector.h
index 0b75a0c3dfc..5ce33e82028 100644
--- a/dbms/src/Columns/ColumnVector.h
+++ b/dbms/src/Columns/ColumnVector.h
@@ -243,11 +243,11 @@ public:
 
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
 
-    MutableColumnPtr filter(const IColumn::Filter & filt, ssize_t result_size_hint) const override;
+    ColumnPtr filter(const IColumn::Filter & filt, ssize_t result_size_hint) const override;
 
-    MutableColumnPtr permute(const IColumn::Permutation & perm, size_t limit) const override;
+    ColumnPtr permute(const IColumn::Permutation & perm, size_t limit) const override;
 
-    MutableColumnPtr replicate(const IColumn::Offsets & offsets) const override;
+    ColumnPtr replicate(const IColumn::Offsets & offsets) const override;
 
     void getExtremes(Field & min, Field & max) const override;
 
diff --git a/dbms/src/Columns/FilterDescription.cpp b/dbms/src/Columns/FilterDescription.cpp
index 1d29cd97435..3d3e3f5ffea 100644
--- a/dbms/src/Columns/FilterDescription.cpp
+++ b/dbms/src/Columns/FilterDescription.cpp
@@ -57,7 +57,8 @@ FilterDescription::FilterDescription(const IColumn & column)
 
     if (const ColumnNullable * nullable_column = typeid_cast<const ColumnNullable *>(&column))
     {
-        MutableColumnPtr mutable_holder = nullable_column->getNestedColumn().mutate();
+        ColumnPtr nested_column = nullable_column->getNestedColumnPtr();
+        MutableColumnPtr mutable_holder = (*std::move(nested_column)).mutate();
 
         ColumnUInt8 * concrete_column = typeid_cast<ColumnUInt8 *>(mutable_holder.get());
         if (!concrete_column)
diff --git a/dbms/src/Columns/IColumn.h b/dbms/src/Columns/IColumn.h
index 9fd98e6f701..40577a11d3f 100644
--- a/dbms/src/Columns/IColumn.h
+++ b/dbms/src/Columns/IColumn.h
@@ -45,7 +45,7 @@ public:
     /** If column isn't constant, returns nullptr (or itself).
       * If column is constant, transforms constant to full column (if column type allows such tranform) and return it.
       */
-    virtual MutablePtr convertToFullColumnIfConst() const { return {}; }
+    virtual Ptr convertToFullColumnIfConst() const { return {}; }
 
     /// Creates empty column with the same type.
     virtual MutablePtr cloneEmpty() const { return cloneResized(0); }
@@ -104,11 +104,11 @@ public:
 
     /// Removes all elements outside of specified range.
     /// Is used in LIMIT operation, for example.
-    virtual MutablePtr cut(size_t start, size_t length) const
+    virtual Ptr cut(size_t start, size_t length) const
     {
         MutablePtr res = cloneEmpty();
         res->insertRangeFrom(*this, start, length);
-        return res;
+        return std::move(res);
     }
 
     /// Appends new value at the end of column (column's size is increased by 1).
@@ -171,12 +171,12 @@ public:
       *  otherwise (i.e. < 0), makes reserve() using size of source column.
       */
     using Filter = PaddedPODArray<UInt8>;
-    virtual MutablePtr filter(const Filter & filt, ssize_t result_size_hint) const = 0;
+    virtual Ptr filter(const Filter & filt, ssize_t result_size_hint) const = 0;
 
     /// Permutes elements using specified permutation. Is used in sortings.
     /// limit - if it isn't 0, puts only first limit elements in the result.
     using Permutation = PaddedPODArray<size_t>;
-    virtual MutablePtr permute(const Permutation & perm, size_t limit) const = 0;
+    virtual Ptr permute(const Permutation & perm, size_t limit) const = 0;
 
     /** Compares (*this)[n] and rhs[m].
       * Returns negative number, 0, or positive number (*this)[n] is less, equal, greater than rhs[m] respectively.
@@ -205,7 +205,7 @@ public:
       */
     using Offset = UInt64;
     using Offsets = PaddedPODArray<Offset>;
-    virtual MutablePtr replicate(const Offsets & offsets) const = 0;
+    virtual Ptr replicate(const Offsets & offsets) const = 0;
 
     /** Split column to smaller columns. Each value goes to column index, selected by corresponding element of 'selector'.
       * Selector must contain values from 0 to num_columns - 1.
@@ -247,10 +247,10 @@ public:
     virtual void forEachSubcolumn(ColumnCallback) {}
 
 
-    MutablePtr mutate() const
+    MutablePtr mutate() const &&
     {
         MutablePtr res = COWPtr<IColumn>::mutate();
-        res->forEachSubcolumn([](Ptr & subcolumn) { subcolumn = subcolumn->mutate(); });
+        res->forEachSubcolumn([](Ptr & subcolumn) { subcolumn = (*std::move(subcolumn)).mutate(); });
         return res;
     }
 
@@ -360,4 +360,16 @@ using MutableColumns = std::vector<MutableColumnPtr>;
 using ColumnRawPtrs = std::vector<const IColumn *>;
 //using MutableColumnRawPtrs = std::vector<IColumn *>;
 
+template <typename ... Args>
+struct IsMutableColumns;
+
+template <typename Arg, typename ... Args>
+struct IsMutableColumns<Arg, Args ...>
+{
+    static const bool value = std::is_assignable<MutableColumnPtr &&, Arg>::value && IsMutableColumns<Args ...>::value;
+};
+
+template <>
+struct IsMutableColumns<> { static const bool value = true; };
+
 }
diff --git a/dbms/src/Columns/IColumnDummy.h b/dbms/src/Columns/IColumnDummy.h
index dbf9daea796..c580a2b0cb1 100644
--- a/dbms/src/Columns/IColumnDummy.h
+++ b/dbms/src/Columns/IColumnDummy.h
@@ -74,12 +74,12 @@ public:
         s += length;
     }
 
-    MutableColumnPtr filter(const Filter & filt, ssize_t /*result_size_hint*/) const override
+    ColumnPtr filter(const Filter & filt, ssize_t /*result_size_hint*/) const override
     {
         return cloneDummy(countBytesInFilter(filt));
     }
 
-    MutableColumnPtr permute(const Permutation & perm, size_t limit) const override
+    ColumnPtr permute(const Permutation & perm, size_t limit) const override
     {
         if (s != perm.size())
             throw Exception("Size of permutation doesn't match size of column.", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
@@ -94,7 +94,7 @@ public:
             res[i] = i;
     }
 
-    MutableColumnPtr replicate(const Offsets & offsets) const override
+    ColumnPtr replicate(const Offsets & offsets) const override
     {
         if (s != offsets.size())
             throw Exception("Size of offsets doesn't match size of column.", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
diff --git a/dbms/src/Common/AIO.h b/dbms/src/Common/AIO.h
index e7287c85d3d..254a7f02ecb 100644
--- a/dbms/src/Common/AIO.h
+++ b/dbms/src/Common/AIO.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#if !(defined(__FreeBSD__) || defined(__APPLE__))
+#if !(defined(__FreeBSD__) || defined(__APPLE__) || defined(_MSC_VER))
 
 #include <Common/Exception.h>
 #include <common/logger_useful.h>
diff --git a/dbms/src/Common/COWPtr.h b/dbms/src/Common/COWPtr.h
index 9f7cf21ad63..525f2372c6c 100644
--- a/dbms/src/Common/COWPtr.h
+++ b/dbms/src/Common/COWPtr.h
@@ -68,10 +68,9 @@
   * of this shared state.
   *
   * Caveats:
-  * - after a call to 'mutate' method, you can still have a reference to immutable ptr somewhere
-  *   and it can still become shared. Also it would be better to make 'mutate' method rvalue-qualified.
+  * - after a call to 'mutate' method, you can still have a reference to immutable ptr somewhere.
   * - as 'mutable_ptr' should be unique, it's refcount is redundant - probably it would be better
-  *   to use std::unique_ptr for it, but see above.
+  *   to use std::unique_ptr for it somehow.
   */
 template <typename Derived>
 class COWPtr : public boost::intrusive_ref_counter<Derived>
@@ -80,12 +79,22 @@ private:
     Derived * derived() { return static_cast<Derived *>(this); }
     const Derived * derived() const { return static_cast<const Derived *>(this); }
 
+    template <typename T>
+    class IntrusivePtr : public boost::intrusive_ptr<T>
+    {
+    public:
+        using boost::intrusive_ptr<T>::intrusive_ptr;
+
+        T & operator*() const & { return boost::intrusive_ptr<T>::operator*(); }
+        T && operator*() const && { return const_cast<typename std::remove_const<T>::type &&>(*boost::intrusive_ptr<T>::get()); }
+    };
+
 protected:
     template <typename T>
-    class mutable_ptr : public boost::intrusive_ptr<T>
+    class mutable_ptr : public IntrusivePtr<T>
     {
     private:
-        using Base = boost::intrusive_ptr<T>;
+        using Base = IntrusivePtr<T>;
 
         template <typename> friend class COWPtr;
         template <typename, typename> friend class COWPtrHelper;
@@ -114,10 +123,10 @@ public:
 
 protected:
     template <typename T>
-    class immutable_ptr : public boost::intrusive_ptr<const T>
+    class immutable_ptr : public IntrusivePtr<const T>
     {
     private:
-        using Base = boost::intrusive_ptr<const T>;
+        using Base = IntrusivePtr<const T>;
 
         template <typename> friend class COWPtr;
         template <typename, typename> friend class COWPtrHelper;
diff --git a/dbms/src/Common/ShellCommand.cpp b/dbms/src/Common/ShellCommand.cpp
index b3765874ac4..c9e625810ca 100644
--- a/dbms/src/Common/ShellCommand.cpp
+++ b/dbms/src/Common/ShellCommand.cpp
@@ -1,13 +1,12 @@
 #include <sys/types.h>
 #include <sys/wait.h>
-#include <unistd.h>
 #include <fcntl.h>
 #include <dlfcn.h>
-
 #include <Common/Exception.h>
 #include <Common/ShellCommand.h>
 #include <IO/WriteBufferFromVector.h>
 #include <IO/WriteHelpers.h>
+#include <port/unistd.h>
 
 
 namespace DB
diff --git a/dbms/src/Common/UTF8Helpers.h b/dbms/src/Common/UTF8Helpers.h
index 1ce31426e85..0237b6f036c 100644
--- a/dbms/src/Common/UTF8Helpers.h
+++ b/dbms/src/Common/UTF8Helpers.h
@@ -3,6 +3,9 @@
 #include <Core/Types.h>
 #include <Common/BitHelpers.h>
 
+#if __SSE2__
+#include <emmintrin.h>
+#endif
 
 namespace DB
 {
@@ -49,9 +52,29 @@ inline size_t seqLength(const UInt8 first_octet)
 inline size_t countCodePoints(const UInt8 * data, size_t size)
 {
     size_t res = 0;
+    const auto end = data + size;
 
-    /// TODO SIMD implementation looks quite simple.
-    for (auto end = data + size; data < end; ++data) /// Skip UTF-8 continuation bytes.
+#if __SSE2__
+    const auto bytes_sse = sizeof(__m128i);
+    const auto src_end_sse = (data + size) - (size % bytes_sse);
+
+    const auto align_sse = _mm_set1_epi8(0x40);
+    const auto upper_bound = _mm_set1_epi8(0xBF);
+
+    for (; data < src_end_sse; data += bytes_sse)
+    {
+        const auto chars = _mm_loadu_si128(reinterpret_cast<const __m128i *>(data));
+
+        ///Align to zero for the solve two case
+        const auto align_res = _mm_adds_epu8(chars, align_sse);
+        const auto less_than_and_equals = _mm_cmpeq_epi8(_mm_min_epu8(align_res, upper_bound), align_res);
+
+        res += __builtin_popcount(_mm_movemask_epi8(less_than_and_equals));
+    }
+
+#endif
+
+    for (; data < end; ++data) /// Skip UTF-8 continuation bytes.
         res += (*data <= 0x7F || *data >= 0xC0);
 
     return res;
diff --git a/dbms/src/Common/ZooKeeper/CMakeLists.txt b/dbms/src/Common/ZooKeeper/CMakeLists.txt
index 9bd72b6fa54..6a02d5733b1 100644
--- a/dbms/src/Common/ZooKeeper/CMakeLists.txt
+++ b/dbms/src/Common/ZooKeeper/CMakeLists.txt
@@ -5,7 +5,11 @@ add_headers_and_sources(clickhouse_common_zookeeper .)
 
 add_library(clickhouse_common_zookeeper ${SPLIT_SHARED} ${clickhouse_common_zookeeper_headers} ${clickhouse_common_zookeeper_sources})
 
-target_link_libraries (clickhouse_common_zookeeper clickhouse_common_io)
+if (NOT USE_INTERNAL_ZOOKEEPER_LIBRARY)
+    target_include_directories (clickhouse_common_zookeeper BEFORE PUBLIC ${ZOOKEEPER_INCLUDE_DIR})
+endif ()
+
+target_link_libraries (clickhouse_common_zookeeper clickhouse_common_io ${ZOOKEEPER_LIBRARY})
 
 if (ENABLE_TESTS)
     add_subdirectory (tests)
diff --git a/dbms/src/Common/ZooKeeper/ZooKeeper.h b/dbms/src/Common/ZooKeeper/ZooKeeper.h
index 52f1968eba6..dcfb5e9393d 100644
--- a/dbms/src/Common/ZooKeeper/ZooKeeper.h
+++ b/dbms/src/Common/ZooKeeper/ZooKeeper.h
@@ -8,10 +8,10 @@
 #include <memory>
 #include <mutex>
 #include <string>
-#include <unistd.h>
 #include <common/logger_useful.h>
 #include <Common/ProfileEvents.h>
 #include <Common/CurrentMetrics.h>
+#include <port/unistd.h>
 
 
 namespace ProfileEvents
diff --git a/dbms/src/Common/ZooKeeper/tests/zkutil_test_commands.cpp b/dbms/src/Common/ZooKeeper/tests/zkutil_test_commands.cpp
index 3c9aed5113d..9b73f79ad7d 100644
--- a/dbms/src/Common/ZooKeeper/tests/zkutil_test_commands.cpp
+++ b/dbms/src/Common/ZooKeeper/tests/zkutil_test_commands.cpp
@@ -1,6 +1,6 @@
 #include <Common/ZooKeeper/ZooKeeper.h>
 #include <iostream>
-#include <unistd.h>
+#include <port/unistd.h>
 
 using namespace zkutil;
 
diff --git a/dbms/src/Common/localBackup.cpp b/dbms/src/Common/localBackup.cpp
index 3b93e0ee452..eb1f76d8c60 100644
--- a/dbms/src/Common/localBackup.cpp
+++ b/dbms/src/Common/localBackup.cpp
@@ -1,13 +1,10 @@
 #include <sys/stat.h>
-#include <unistd.h>
-
 #include <string>
 #include <iostream>
-
 #include <Poco/DirectoryIterator.h>
 #include <Poco/File.h>
-
 #include <Common/Exception.h>
+#include <port/unistd.h>
 
 
 namespace DB
diff --git a/dbms/src/Common/randomSeed.cpp b/dbms/src/Common/randomSeed.cpp
index 62925f55dd3..7618c41a020 100644
--- a/dbms/src/Common/randomSeed.cpp
+++ b/dbms/src/Common/randomSeed.cpp
@@ -1,12 +1,10 @@
 #include <time.h>
-#include <unistd.h>
+#include <port/unistd.h>
 #include <sys/types.h>
-
 #include <Common/Exception.h>
 #include <Common/randomSeed.h>
 #include <Common/SipHash.h>
 #include <Core/Types.h>
-
 #ifdef __APPLE__
 #include <common/apple_rt.h>
 #endif
diff --git a/dbms/src/Common/tests/compact_array.cpp b/dbms/src/Common/tests/compact_array.cpp
index 3beb8b6db9a..14c6b819c37 100644
--- a/dbms/src/Common/tests/compact_array.cpp
+++ b/dbms/src/Common/tests/compact_array.cpp
@@ -7,15 +7,13 @@
 #include <Common/CompactArray.h>
 #include <IO/WriteBufferFromFile.h>
 #include <IO/ReadBufferFromFile.h>
-
 #include <boost/filesystem.hpp>
-
 #include <string>
 #include <iostream>
 #include <fstream>
 #include <stdexcept>
-#include <unistd.h>
 #include <cstdlib>
+#include <port/unistd.h>
 
 namespace fs = boost::filesystem;
 
diff --git a/dbms/src/Core/Block.cpp b/dbms/src/Core/Block.cpp
index c969e3256da..1cae4ba252e 100644
--- a/dbms/src/Core/Block.cpp
+++ b/dbms/src/Core/Block.cpp
@@ -312,7 +312,7 @@ MutableColumns Block::mutateColumns() const
     size_t num_columns = data.size();
     MutableColumns columns(num_columns);
     for (size_t i = 0; i < num_columns; ++i)
-        columns[i] = data[i].column ? data[i].column->mutate() : data[i].type->createColumn();
+        columns[i] = data[i].column ? (*std::move(data[i].column)).mutate() : data[i].type->createColumn();
     return columns;
 }
 
diff --git a/dbms/src/Core/Block.h b/dbms/src/Core/Block.h
index 7c836e49532..ea4a4c00cef 100644
--- a/dbms/src/Core/Block.h
+++ b/dbms/src/Core/Block.h
@@ -109,7 +109,7 @@ public:
     /** Get empty columns with the same types as in block. */
     MutableColumns cloneEmptyColumns() const;
 
-    /** Get columns from block for mutation. */
+    /** Get columns from block for mutation. Columns in block will be nullptr. */
     MutableColumns mutateColumns() const;
 
     /** Replace columns in a block */
diff --git a/dbms/src/Core/Defines.h b/dbms/src/Core/Defines.h
index 8b69d83d69e..7f3a37613dc 100644
--- a/dbms/src/Core/Defines.h
+++ b/dbms/src/Core/Defines.h
@@ -60,6 +60,7 @@
 #define DBMS_MIN_REVISION_WITH_QUOTA_KEY_IN_CLIENT_INFO 54060
 #define DBMS_MIN_REVISION_WITH_TABLES_STATUS 54226
 #define DBMS_MIN_REVISION_WITH_TIME_ZONE_PARAMETER_IN_DATETIME_DATA_TYPE 54337
+#define DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME 54372
 
 /// Version of ClickHouse TCP protocol. Set to git tag with latest protocol change.
 #define DBMS_TCP_PROTOCOL_VERSION 54226
@@ -74,8 +75,18 @@
 #define DEFAULT_HTTP_READ_BUFFER_TIMEOUT 1800
 #define DEFAULT_HTTP_READ_BUFFER_CONNECTION_TIMEOUT 1
 
-#define ALWAYS_INLINE __attribute__((__always_inline__))
-#define NO_INLINE __attribute__((__noinline__))
+// more aliases: https://mailman.videolan.org/pipermail/x264-devel/2014-May/010660.html
+
+#if defined(_MSC_VER)
+    #define ALWAYS_INLINE __forceinline
+    #define NO_INLINE static __declspec(noinline)
+    #define MAY_ALIAS
+#else
+    #define ALWAYS_INLINE __attribute__((__always_inline__))
+    #define NO_INLINE __attribute__((__noinline__))
+    #define MAY_ALIAS __attribute__((__may_alias__))
+#endif
+
 
 #define PLATFORM_NOT_SUPPORTED "The only supported platforms are x86_64 and AArch64 (work in progress)"
 
diff --git a/dbms/src/Core/Field.h b/dbms/src/Core/Field.h
index f4c2f1ade86..795bc4b7070 100644
--- a/dbms/src/Core/Field.h
+++ b/dbms/src/Core/Field.h
@@ -198,14 +198,14 @@ public:
     template <typename T> T & get()
     {
         using TWithoutRef = std::remove_reference_t<T>;
-        TWithoutRef * __attribute__((__may_alias__)) ptr = reinterpret_cast<TWithoutRef*>(&storage);
+        TWithoutRef * MAY_ALIAS ptr = reinterpret_cast<TWithoutRef*>(&storage);
         return *ptr;
     };
 
     template <typename T> const T & get() const
     {
         using TWithoutRef = std::remove_reference_t<T>;
-        const TWithoutRef * __attribute__((__may_alias__)) ptr = reinterpret_cast<const TWithoutRef*>(&storage);
+        const TWithoutRef * MAY_ALIAS ptr = reinterpret_cast<const TWithoutRef*>(&storage);
         return *ptr;
     };
 
@@ -340,7 +340,7 @@ private:
     void createConcrete(T && x)
     {
         using JustT = std::decay_t<T>;
-        JustT * __attribute__((__may_alias__)) ptr = reinterpret_cast<JustT *>(&storage);
+        JustT * MAY_ALIAS ptr = reinterpret_cast<JustT *>(&storage);
         new (ptr) JustT(std::forward<T>(x));
         which = TypeToEnum<JustT>::value;
     }
@@ -350,7 +350,7 @@ private:
     void assignConcrete(T && x)
     {
         using JustT = std::decay_t<T>;
-        JustT * __attribute__((__may_alias__)) ptr = reinterpret_cast<JustT *>(&storage);
+        JustT * MAY_ALIAS ptr = reinterpret_cast<JustT *>(&storage);
         *ptr = std::forward<T>(x);
     }
 
@@ -398,7 +398,7 @@ private:
 
     void create(const char * data, size_t size)
     {
-        String * __attribute__((__may_alias__)) ptr = reinterpret_cast<String*>(&storage);
+        String * MAY_ALIAS ptr = reinterpret_cast<String*>(&storage);
         new (ptr) String(data, size);
         which = Types::String;
     }
@@ -434,7 +434,7 @@ private:
     template <typename T>
     void destroy()
     {
-        T * __attribute__((__may_alias__)) ptr = reinterpret_cast<T*>(&storage);
+        T * MAY_ALIAS ptr = reinterpret_cast<T*>(&storage);
         ptr->~T();
     }
 };
diff --git a/dbms/src/DataStreams/ColumnGathererStream.cpp b/dbms/src/DataStreams/ColumnGathererStream.cpp
index ba4107f7e8b..5ce00ea8417 100644
--- a/dbms/src/DataStreams/ColumnGathererStream.cpp
+++ b/dbms/src/DataStreams/ColumnGathererStream.cpp
@@ -76,7 +76,7 @@ Block ColumnGathererStream::readImpl()
         return Block();
 
     output_block = Block{column.cloneEmpty()};
-    MutableColumnPtr output_column = output_block.getByPosition(0).column->mutate();
+    MutableColumnPtr output_column = output_block.getByPosition(0).column->assumeMutable();
     output_column->gather(*this);
     if (!output_column->empty())
         output_block.getByPosition(0).column = std::move(output_column);
diff --git a/dbms/src/DataStreams/MergingSortedBlockInputStream.cpp b/dbms/src/DataStreams/MergingSortedBlockInputStream.cpp
index f93197ca123..be90a00e4b9 100644
--- a/dbms/src/DataStreams/MergingSortedBlockInputStream.cpp
+++ b/dbms/src/DataStreams/MergingSortedBlockInputStream.cpp
@@ -221,7 +221,7 @@ void MergingSortedBlockInputStream::merge(MutableColumns & merged_columns, std::
                     throw Exception("Logical error in MergingSortedBlockInputStream", ErrorCodes::LOGICAL_ERROR);
 
                 for (size_t i = 0; i < num_columns; ++i)
-                    merged_columns[i] = source_blocks[source_num]->getByPosition(i).column->mutate();
+                    merged_columns[i] = (*std::move(source_blocks[source_num]->getByPosition(i).column)).mutate();
 
     //            std::cerr << "copied columns\n";
 
@@ -233,7 +233,7 @@ void MergingSortedBlockInputStream::merge(MutableColumns & merged_columns, std::
                     for (size_t i = 0; i < num_columns; ++i)
                     {
                         auto & column = merged_columns[i];
-                        column = column->cut(0, merged_rows);
+                        column = (*column->cut(0, merged_rows)).mutate();
                     }
 
                     cancel(false);
diff --git a/dbms/src/DataStreams/PrettyBlockOutputStream.cpp b/dbms/src/DataStreams/PrettyBlockOutputStream.cpp
index 58b61664b7c..2bb919d4806 100644
--- a/dbms/src/DataStreams/PrettyBlockOutputStream.cpp
+++ b/dbms/src/DataStreams/PrettyBlockOutputStream.cpp
@@ -1,6 +1,5 @@
 #include <sys/ioctl.h>
-#include <unistd.h>
-
+#include <port/unistd.h>
 #include <DataStreams/PrettyBlockOutputStream.h>
 #include <IO/WriteBuffer.h>
 #include <IO/WriteHelpers.h>
diff --git a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp
index b1f0dc8a9cd..b3dd05aee9c 100644
--- a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp
+++ b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp
@@ -1,4 +1,5 @@
 #include <DataStreams/PushingToViewsBlockOutputStream.h>
+#include <DataStreams/SquashingBlockInputStream.h>
 #include <Interpreters/InterpreterSelectQuery.h>
 #include <Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.h>
 
@@ -35,7 +36,7 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream(
             auto & materialized_view = dynamic_cast<const StorageMaterializedView &>(*dependent_table);
 
             auto query = materialized_view.getInnerQuery();
-            auto out = std::make_shared<PushingToViewsBlockOutputStream>(
+            BlockOutputStreamPtr out = std::make_shared<PushingToViewsBlockOutputStream>(
                 database_table.first, database_table.second, dependent_table, *views_context, ASTPtr());
             views.emplace_back(ViewInfo{std::move(query), database_table.first, database_table.second, std::move(out)});
         }
@@ -66,8 +67,19 @@ void PushingToViewsBlockOutputStream::write(const Block & block)
         {
             BlockInputStreamPtr from = std::make_shared<OneBlockInputStream>(block);
             InterpreterSelectQuery select(view.query, *views_context, {}, QueryProcessingStage::Complete, 0, from);
-            BlockInputStreamPtr data = std::make_shared<MaterializingBlockInputStream>(select.execute().in);
-            copyData(*data, *view.out);
+            BlockInputStreamPtr in = std::make_shared<MaterializingBlockInputStream>(select.execute().in);
+            /// Squashing is needed here because the materialized view query can generate a lot of blocks
+            /// even when only one block is inserted into the parent table (e.g. if the query is a GROUP BY
+            /// and two-level aggregation is triggered).
+            in = std::make_shared<SquashingBlockInputStream>(
+                in, context.getSettingsRef().min_insert_block_size_rows, context.getSettingsRef().min_insert_block_size_bytes);
+
+            in->readPrefix();
+
+            while (Block result_block = in->read())
+                view.out->write(result_block);
+
+            in->readSuffix();
         }
         catch (Exception & ex)
         {
diff --git a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.h b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.h
index 7703b81dc4e..4ff953fd265 100644
--- a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.h
+++ b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.h
@@ -29,18 +29,27 @@ public:
     {
         if (output)
             output->flush();
+
+        for (auto & view : views)
+            view.out->flush();
     }
 
     void writePrefix() override
     {
         if (output)
             output->writePrefix();
+
+        for (auto & view : views)
+            view.out->writePrefix();
     }
 
     void writeSuffix() override
     {
         if (output)
             output->writeSuffix();
+
+        for (auto & view : views)
+            view.out->writeSuffix();
     }
 
 private:
diff --git a/dbms/src/DataStreams/SquashingTransform.cpp b/dbms/src/DataStreams/SquashingTransform.cpp
index 9e8f590f171..d31ff6f8063 100644
--- a/dbms/src/DataStreams/SquashingTransform.cpp
+++ b/dbms/src/DataStreams/SquashingTransform.cpp
@@ -62,7 +62,7 @@ void SquashingTransform::append(Block && block)
 
     for (size_t i = 0; i < columns; ++i)
     {
-        MutableColumnPtr mutable_column = accumulated_block.getByPosition(i).column->mutate();
+        MutableColumnPtr mutable_column = (*std::move(accumulated_block.getByPosition(i).column)).mutate();
         mutable_column->insertRangeFrom(*block.getByPosition(i).column, 0, rows);
         accumulated_block.getByPosition(i).column = std::move(mutable_column);
     }
diff --git a/dbms/src/DataStreams/SummingSortedBlockInputStream.cpp b/dbms/src/DataStreams/SummingSortedBlockInputStream.cpp
index 1102258d348..66817f23037 100644
--- a/dbms/src/DataStreams/SummingSortedBlockInputStream.cpp
+++ b/dbms/src/DataStreams/SummingSortedBlockInputStream.cpp
@@ -259,11 +259,11 @@ Block SummingSortedBlockInputStream::readImpl()
         if (checkDataType<DataTypeTuple>(desc.function->getReturnType().get()))
         {
             size_t tuple_size = desc.column_numbers.size();
-            Columns tuple_columns(tuple_size);
+            MutableColumns tuple_columns(tuple_size);
             for (size_t i = 0; i < tuple_size; ++i)
-                tuple_columns[i] = header.safeGetByPosition(desc.column_numbers[i]).column;
+                tuple_columns[i] = header.safeGetByPosition(desc.column_numbers[i]).column->assumeMutable();
 
-            desc.merged_column = ColumnTuple::create(tuple_columns);
+            desc.merged_column = ColumnTuple::create(std::move(tuple_columns));
         }
         else
             desc.merged_column = header.safeGetByPosition(desc.column_numbers[0]).column->cloneEmpty();
diff --git a/dbms/src/DataTypes/DataTypeTuple.cpp b/dbms/src/DataTypes/DataTypeTuple.cpp
index 6f7d692af5f..2da9f0f6ecf 100644
--- a/dbms/src/DataTypes/DataTypeTuple.cpp
+++ b/dbms/src/DataTypes/DataTypeTuple.cpp
@@ -329,10 +329,10 @@ void DataTypeTuple::deserializeBinaryBulkWithMultipleStreams(
 MutableColumnPtr DataTypeTuple::createColumn() const
 {
     size_t size = elems.size();
-    Columns tuple_columns(size);
+    MutableColumns tuple_columns(size);
     for (size_t i = 0; i < size; ++i)
         tuple_columns[i] = elems[i]->createColumn();
-    return ColumnTuple::create(tuple_columns);
+    return ColumnTuple::create(std::move(tuple_columns));
 }
 
 Field DataTypeTuple::getDefault() const
diff --git a/dbms/src/Databases/DatabaseDictionary.cpp b/dbms/src/Databases/DatabaseDictionary.cpp
index 503d0195168..dd674f3698d 100644
--- a/dbms/src/Databases/DatabaseDictionary.cpp
+++ b/dbms/src/Databases/DatabaseDictionary.cpp
@@ -3,15 +3,21 @@
 #include <Interpreters/ExternalDictionaries.h>
 #include <Storages/StorageDictionary.h>
 #include <common/logger_useful.h>
+#include <Parsers/IAST.h>
+#include <IO/WriteBufferFromString.h>
+#include <IO/Operators.h>
+#include <Parsers/ParserCreateQuery.h>
+#include <Parsers/parseQuery.h>
 
 namespace DB
 {
 namespace ErrorCodes
 {
-extern const int TABLE_ALREADY_EXISTS;
-extern const int UNKNOWN_TABLE;
-extern const int LOGICAL_ERROR;
-extern const int CANNOT_GET_CREATE_TABLE_QUERY;
+    extern const int TABLE_ALREADY_EXISTS;
+    extern const int UNKNOWN_TABLE;
+    extern const int LOGICAL_ERROR;
+    extern const int CANNOT_GET_CREATE_TABLE_QUERY;
+    extern const int SYNTAX_ERROR;
 }
 
 DatabaseDictionary::DatabaseDictionary(const String & name_, const Context & context)
@@ -153,11 +159,54 @@ time_t DatabaseDictionary::getTableMetadataModificationTime(
     return static_cast<time_t>(0);
 }
 
-ASTPtr DatabaseDictionary::getCreateQuery(
-    const Context &,
-    const String &) const
+ASTPtr DatabaseDictionary::getCreateTableQueryImpl(const Context & context,
+                                                   const String & table_name, bool throw_on_error) const
 {
-    throw Exception("There is no CREATE TABLE query for DatabaseDictionary tables", ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY);
+    String query;
+    {
+        WriteBufferFromString buffer(query);
+
+        const auto & dictionaries = context.getExternalDictionaries();
+        auto dictionary = throw_on_error ? dictionaries.getDictionary(table_name)
+                                         : dictionaries.tryGetDictionary(table_name);
+
+        auto names_and_types = StorageDictionary::getNamesAndTypes(dictionary->getStructure());
+        buffer << "CREATE TABLE " << backQuoteIfNeed(name) << '.' << backQuoteIfNeed(table_name) << " (";
+        buffer << StorageDictionary::generateNamesAndTypesDescription(names_and_types.begin(), names_and_types.end());
+        buffer << ") Engine = Dictionary(" << backQuoteIfNeed(table_name) << ")";
+    }
+
+    ParserCreateQuery parser;
+    const char * pos = query.data();
+    std::string error_message;
+    auto ast = tryParseQuery(parser, pos, pos + query.size(), error_message,
+            /* hilite = */ false, "", /* allow_multi_statements = */ false);
+
+    if (!ast && throw_on_error)
+        throw Exception(error_message, ErrorCodes::SYNTAX_ERROR);
+
+    return ast;
+}
+
+ASTPtr DatabaseDictionary::getCreateTableQuery(const Context & context, const String & table_name) const
+{
+    return getCreateTableQueryImpl(context, table_name, true);
+}
+
+ASTPtr DatabaseDictionary::tryGetCreateTableQuery(const Context & context, const String & table_name) const
+{
+    return getCreateTableQueryImpl(context, table_name, false);
+}
+
+ASTPtr DatabaseDictionary::getCreateDatabaseQuery(const Context & /*context*/) const
+{
+    String query;
+    {
+        WriteBufferFromString buffer(query);
+        buffer << "CREATE DATABASE " << backQuoteIfNeed(name) << " ENGINE = Dictionary";
+    }
+    ParserCreateQuery parser;
+    return parseQuery(parser, query.data(), query.data() + query.size(), "");
 }
 
 void DatabaseDictionary::shutdown()
diff --git a/dbms/src/Databases/DatabaseDictionary.h b/dbms/src/Databases/DatabaseDictionary.h
index aba93faa5b1..73279f243df 100644
--- a/dbms/src/Databases/DatabaseDictionary.h
+++ b/dbms/src/Databases/DatabaseDictionary.h
@@ -22,16 +22,6 @@ class ExternalDictionaries;
  */
 class DatabaseDictionary : public IDatabase
 {
-private:
-    const String name;
-    mutable std::mutex mutex;
-    const ExternalDictionaries & external_dictionaries;
-    std::unordered_set<String> deleted_tables;
-
-    Poco::Logger * log;
-
-    Tables loadTables();
-
 public:
     DatabaseDictionary(const String & name_, const Context & context);
 
@@ -86,12 +76,30 @@ public:
         const Context & context,
         const String & table_name) override;
 
-    ASTPtr getCreateQuery(
+    ASTPtr getCreateTableQuery(
         const Context & context,
         const String & table_name) const override;
 
+    ASTPtr tryGetCreateTableQuery(
+            const Context & context,
+            const String & table_name) const override;
+
+    ASTPtr getCreateDatabaseQuery(const Context & context) const override;
+
     void shutdown() override;
     void drop() override;
+
+private:
+    const String name;
+    mutable std::mutex mutex;
+    const ExternalDictionaries & external_dictionaries;
+    std::unordered_set<String> deleted_tables;
+
+    Poco::Logger * log;
+
+    Tables loadTables();
+
+    ASTPtr getCreateTableQueryImpl(const Context & context, const String & table_name, bool throw_on_error) const;
 };
 
 }
diff --git a/dbms/src/Databases/DatabaseMemory.cpp b/dbms/src/Databases/DatabaseMemory.cpp
index 57089862863..25df1d8e1cb 100644
--- a/dbms/src/Databases/DatabaseMemory.cpp
+++ b/dbms/src/Databases/DatabaseMemory.cpp
@@ -118,13 +118,18 @@ time_t DatabaseMemory::getTableMetadataModificationTime(
     return static_cast<time_t>(0);
 }
 
-ASTPtr DatabaseMemory::getCreateQuery(
+ASTPtr DatabaseMemory::getCreateTableQuery(
     const Context &,
     const String &) const
 {
     throw Exception("There is no CREATE TABLE query for DatabaseMemory tables", ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY);
 }
 
+ASTPtr DatabaseMemory::getCreateDatabaseQuery(const Context &) const
+{
+    throw Exception("There is no CREATE DATABASE query for DatabaseMemory", ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY);
+}
+
 void DatabaseMemory::shutdown()
 {
     /// You can not hold a lock during shutdown.
diff --git a/dbms/src/Databases/DatabaseMemory.h b/dbms/src/Databases/DatabaseMemory.h
index 586dcfbe4da..faf384cb7f6 100644
--- a/dbms/src/Databases/DatabaseMemory.h
+++ b/dbms/src/Databases/DatabaseMemory.h
@@ -77,9 +77,10 @@ public:
         const Context & context,
         const String & table_name) override;
 
-    ASTPtr getCreateQuery(
-        const Context & context,
-        const String & table_name) const override;
+    ASTPtr getCreateTableQuery(const Context & context, const String & table_name) const override;
+    ASTPtr tryGetCreateTableQuery(const Context &, const String &) const override { return nullptr; }
+
+    ASTPtr getCreateDatabaseQuery(const Context & context) const override;
 
     void shutdown() override;
     void drop() override;
diff --git a/dbms/src/Databases/DatabaseOrdinary.cpp b/dbms/src/Databases/DatabaseOrdinary.cpp
index 9dd705fbf9a..608ec107bd5 100644
--- a/dbms/src/Databases/DatabaseOrdinary.cpp
+++ b/dbms/src/Databases/DatabaseOrdinary.cpp
@@ -31,6 +31,7 @@ namespace ErrorCodes
     extern const int FILE_DOESNT_EXIST;
     extern const int LOGICAL_ERROR;
     extern const int CANNOT_GET_CREATE_TABLE_QUERY;
+    extern const int SYNTAX_ERROR;
 }
 
 
@@ -45,6 +46,12 @@ namespace detail
     {
         return base_path + (endsWith(base_path, "/") ? "" : "/") + escapeForFileName(table_name) + ".sql";
     }
+
+    String getDatabaseMetadataPath(const String & base_path)
+    {
+        return (endsWith(base_path, "/") ? base_path.substr(0, base_path.size() - 1) : base_path) + ".sql";
+    }
+
 }
 
 static void loadTable(
@@ -329,19 +336,42 @@ void DatabaseOrdinary::removeTable(
     }
 }
 
-
-static ASTPtr getCreateQueryImpl(const String & path, const String & table_name)
+static ASTPtr getQueryFromMetadata(const String & metadata_path, bool throw_on_error = true)
 {
-    String table_metadata_path = detail::getTableMetadataPath(path, table_name);
+    if (!Poco::File(metadata_path).exists())
+        return nullptr;
 
     String query;
+
     {
-        ReadBufferFromFile in(table_metadata_path, 4096);
+        ReadBufferFromFile in(metadata_path, 4096);
         readStringUntilEOF(query, in);
     }
 
     ParserCreateQuery parser;
-    return parseQuery(parser, query.data(), query.data() + query.size(), "in file " + table_metadata_path);
+    const char * pos = query.data();
+    std::string error_message;
+    auto ast = tryParseQuery(parser, pos, pos + query.size(), error_message, /* hilite = */ false,
+                             "in file " + metadata_path, /* allow_multi_statements = */ false);
+
+    if (!ast && throw_on_error)
+        throw Exception(error_message, ErrorCodes::SYNTAX_ERROR);
+
+    return ast;
+}
+
+static ASTPtr getCreateQueryFromMetadata(const String & metadata_path, const String & database, bool throw_on_error)
+{
+    ASTPtr ast = getQueryFromMetadata(metadata_path, throw_on_error);
+
+    if (ast)
+    {
+        ASTCreateQuery & ast_create_query = typeid_cast<ASTCreateQuery &>(*ast);
+        ast_create_query.attach = false;
+        ast_create_query.database = database;
+    }
+
+    return ast;
 }
 
 
@@ -378,7 +408,9 @@ void DatabaseOrdinary::renameTable(
         throw Exception{e};
     }
 
-    ASTPtr ast = getCreateQueryImpl(metadata_path, table_name);
+    ASTPtr ast = getQueryFromMetadata(detail::getTableMetadataPath(metadata_path, table_name));
+    if (!ast)
+        throw Exception("There is no metadata file for table " + table_name, ErrorCodes::FILE_DOESNT_EXIST);
     ASTCreateQuery & ast_create_query = typeid_cast<ASTCreateQuery &>(*ast);
     ast_create_query.table = to_table_name;
 
@@ -405,28 +437,51 @@ time_t DatabaseOrdinary::getTableMetadataModificationTime(
     }
 }
 
-
-ASTPtr DatabaseOrdinary::getCreateQuery(
-    const Context & context,
-    const String & table_name) const
+ASTPtr DatabaseOrdinary::getCreateTableQueryImpl(const Context & context,
+                                                 const String & table_name, bool throw_on_error) const
 {
     ASTPtr ast;
-    try
-    {
-        ast = getCreateQueryImpl(metadata_path, table_name);
-    }
-    catch (const Exception & e)
-    {
-        /// Handle system.* tables for which there are no table.sql files
-        if (e.code() == ErrorCodes::FILE_DOESNT_EXIST && tryGetTable(context, table_name) != nullptr)
-            throw Exception("There is no CREATE TABLE query for table " + table_name, ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY);
 
-        throw;
+    auto table_metadata_path = detail::getTableMetadataPath(metadata_path, table_name);
+    ast = getCreateQueryFromMetadata(table_metadata_path, name, throw_on_error);
+    if (!ast && throw_on_error)
+    {
+        /// Handle system.* tables for which there are no table.sql files.
+        bool has_table = tryGetTable(context, table_name) != nullptr;
+
+        auto msg = has_table
+                   ? "There is no CREATE TABLE query for table "
+                   : "There is no metadata file for table ";
+
+        throw Exception(msg + table_name, ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY);
     }
 
-    ASTCreateQuery & ast_create_query = typeid_cast<ASTCreateQuery &>(*ast);
-    ast_create_query.attach = false;
-    ast_create_query.database = name;
+    return ast;
+}
+
+ASTPtr DatabaseOrdinary::getCreateTableQuery(const Context & context, const String & table_name) const
+{
+    return getCreateTableQueryImpl(context, table_name, true);
+}
+
+ASTPtr DatabaseOrdinary::tryGetCreateTableQuery(const Context & context, const String & table_name) const
+{
+    return getCreateTableQueryImpl(context, table_name, false);
+}
+
+ASTPtr DatabaseOrdinary::getCreateDatabaseQuery(const Context & /*context*/) const
+{
+    ASTPtr ast;
+
+    auto database_metadata_path = detail::getDatabaseMetadataPath(metadata_path);
+    ast = getCreateQueryFromMetadata(database_metadata_path, name, true);
+    if (!ast)
+    {
+        /// Handle databases (such as default) for which there are no database.sql files.
+        String query = "CREATE DATABASE " + backQuoteIfNeed(name) + " ENGINE = Ordinary";
+        ParserCreateQuery parser;
+        ast = parseQuery(parser, query.data(), query.data() + query.size(), "");
+    }
 
     return ast;
 }
diff --git a/dbms/src/Databases/DatabaseOrdinary.h b/dbms/src/Databases/DatabaseOrdinary.h
index 25e99114430..756dde07b6e 100644
--- a/dbms/src/Databases/DatabaseOrdinary.h
+++ b/dbms/src/Databases/DatabaseOrdinary.h
@@ -52,10 +52,16 @@ public:
         const Context & context,
         const String & table_name) override;
 
-    ASTPtr getCreateQuery(
+    ASTPtr getCreateTableQuery(
         const Context & context,
         const String & table_name) const override;
 
+    ASTPtr tryGetCreateTableQuery(
+        const Context & context,
+        const String & table_name) const override;
+
+    ASTPtr getCreateDatabaseQuery(const Context & context) const override;
+
     String getDataPath() const override;
     String getMetadataPath() const override;
     String getTableMetadataPath(const String & table_name) const override;
@@ -65,6 +71,8 @@ public:
 
 private:
     void startupTables(ThreadPool * thread_pool);
+
+    ASTPtr getCreateTableQueryImpl(const Context & context, const String & table_name, bool throw_on_error) const;
 };
 
 }
diff --git a/dbms/src/Databases/IDatabase.h b/dbms/src/Databases/IDatabase.h
index f7b8af632ca..638c9d982d6 100644
--- a/dbms/src/Databases/IDatabase.h
+++ b/dbms/src/Databases/IDatabase.h
@@ -122,9 +122,15 @@ public:
         const String & name) = 0;
 
     /// Get the CREATE TABLE query for the table. It can also provide information for detached tables for which there is metadata.
-    virtual ASTPtr getCreateQuery(
-        const Context & context,
-        const String & name) const = 0;
+    virtual ASTPtr tryGetCreateTableQuery(const Context & context, const String & name) const = 0;
+
+    virtual ASTPtr getCreateTableQuery(const Context & context, const String & name) const
+    {
+        return tryGetCreateTableQuery(context, name);
+    }
+
+    /// Get the CREATE DATABASE query for current database.
+    virtual ASTPtr getCreateDatabaseQuery(const Context & context) const = 0;
 
     /// Returns path for persistent data storage if the database supports it, empty string otherwise
     virtual String getDataPath() const { return {}; }
diff --git a/dbms/src/Dictionaries/ComplexKeyHashedDictionary.cpp b/dbms/src/Dictionaries/ComplexKeyHashedDictionary.cpp
index 0f8c75ebd89..eaaeeab4e0a 100644
--- a/dbms/src/Dictionaries/ComplexKeyHashedDictionary.cpp
+++ b/dbms/src/Dictionaries/ComplexKeyHashedDictionary.cpp
@@ -289,7 +289,7 @@ void ComplexKeyHashedDictionary::updateData()
             for (const auto attribute_idx : ext::range(0, keys_size + attributes_size))
             {
                 const IColumn & update_column = *block.getByPosition(attribute_idx).column.get();
-                MutableColumnPtr saved_column = saved_block->getByPosition(attribute_idx).column->mutate();
+                MutableColumnPtr saved_column = saved_block->getByPosition(attribute_idx).column->assumeMutable();
                 saved_column->insertRangeFrom(update_column, 0, update_column.size());
             }
         }
diff --git a/dbms/src/Dictionaries/ExternalResultDescription.cpp b/dbms/src/Dictionaries/ExternalResultDescription.cpp
index 4c05fc32555..5eca329e5f2 100644
--- a/dbms/src/Dictionaries/ExternalResultDescription.cpp
+++ b/dbms/src/Dictionaries/ExternalResultDescription.cpp
@@ -66,7 +66,7 @@ void ExternalResultDescription::init(const Block & sample_block_)
         /// If default value for column was not provided, use default from data type.
         if (sample_columns.back()->empty())
         {
-            MutableColumnPtr mutable_column = sample_columns.back()->mutate();
+            MutableColumnPtr mutable_column = (*std::move(sample_columns.back())).mutate();
             column.type->insertDefaultInto(*mutable_column);
             sample_columns.back() = std::move(mutable_column);
         }
diff --git a/dbms/src/Dictionaries/FlatDictionary.cpp b/dbms/src/Dictionaries/FlatDictionary.cpp
index 0fc477725be..e1341907649 100644
--- a/dbms/src/Dictionaries/FlatDictionary.cpp
+++ b/dbms/src/Dictionaries/FlatDictionary.cpp
@@ -316,7 +316,7 @@ void FlatDictionary::updateData()
             for (const auto attribute_idx : ext::range(0, attributes.size() + 1))
             {
                 const IColumn & update_column = *block.getByPosition(attribute_idx).column.get();
-                MutableColumnPtr saved_column = saved_block->getByPosition(attribute_idx).column->mutate();
+                MutableColumnPtr saved_column = saved_block->getByPosition(attribute_idx).column->assumeMutable();
                 saved_column->insertRangeFrom(update_column, 0, update_column.size());
             }
         }
diff --git a/dbms/src/Dictionaries/HashedDictionary.cpp b/dbms/src/Dictionaries/HashedDictionary.cpp
index c0c7f1a8a17..09880cdbeb5 100644
--- a/dbms/src/Dictionaries/HashedDictionary.cpp
+++ b/dbms/src/Dictionaries/HashedDictionary.cpp
@@ -310,7 +310,7 @@ void HashedDictionary::updateData()
             for (const auto attribute_idx : ext::range(0, attributes.size() + 1))
             {
                 const IColumn & update_column = *block.getByPosition(attribute_idx).column.get();
-                MutableColumnPtr saved_column = saved_block->getByPosition(attribute_idx).column->mutate();
+                MutableColumnPtr saved_column = saved_block->getByPosition(attribute_idx).column->assumeMutable();
                 saved_column->insertRangeFrom(update_column, 0, update_column.size());
             }
         }
diff --git a/dbms/src/Functions/FunctionsConditional.h b/dbms/src/Functions/FunctionsConditional.h
index 841295f0643..7c88bca6264 100644
--- a/dbms/src/Functions/FunctionsConditional.h
+++ b/dbms/src/Functions/FunctionsConditional.h
@@ -592,7 +592,7 @@ private:
             const ColumnPtr & result_column = temporary_block.getByPosition(3).column;
             if (result_column->isColumnNullable())
             {
-                MutableColumnPtr mutable_result_column = result_column->mutate();
+                MutableColumnPtr mutable_result_column = (*std::move(result_column)).mutate();
                 static_cast<ColumnNullable &>(*mutable_result_column).applyNullMap(static_cast<const ColumnNullable &>(*arg_cond.column));
                 block.getByPosition(result).column = std::move(mutable_result_column);
                 return true;
@@ -744,7 +744,8 @@ private:
             {
                 if (arg_else.column->isColumnNullable())
                 {
-                    auto result_column = arg_else.column->mutate();
+                    auto arg_else_column = arg_else.column;
+                    auto result_column = (*std::move(arg_else_column)).mutate();
                     static_cast<ColumnNullable &>(*result_column).applyNullMap(static_cast<const ColumnUInt8 &>(*arg_cond.column));
                     block.getByPosition(result).column = std::move(result_column);
                 }
@@ -785,7 +786,8 @@ private:
 
                 if (arg_then.column->isColumnNullable())
                 {
-                    auto result_column = arg_then.column->mutate();
+                    auto arg_then_column = arg_then.column;
+                    auto result_column = (*std::move(arg_then_column)).mutate();
                     static_cast<ColumnNullable &>(*result_column).applyNegatedNullMap(static_cast<const ColumnUInt8 &>(*arg_cond.column));
                     block.getByPosition(result).column = std::move(result_column);
                 }
diff --git a/dbms/src/Functions/FunctionsHigherOrder.h b/dbms/src/Functions/FunctionsHigherOrder.h
index c63dce364b9..47722370777 100644
--- a/dbms/src/Functions/FunctionsHigherOrder.h
+++ b/dbms/src/Functions/FunctionsHigherOrder.h
@@ -936,7 +936,7 @@ public:
             }
 
             /// Put all the necessary columns multiplied by the sizes of arrays into the block.
-            auto replicated_column_function_ptr = column_function->replicate(column_first_array->getOffsets());
+            auto replicated_column_function_ptr = (*column_function->replicate(column_first_array->getOffsets())).mutate();
             auto * replicated_column_function = typeid_cast<ColumnFunction *>(replicated_column_function_ptr.get());
             replicated_column_function->appendArguments(arrays);
 
diff --git a/dbms/src/Functions/GatherUtils/createArraySink.cpp b/dbms/src/Functions/GatherUtils/createArraySink.cpp
index b97bf476919..e4097195e4c 100644
--- a/dbms/src/Functions/GatherUtils/createArraySink.cpp
+++ b/dbms/src/Functions/GatherUtils/createArraySink.cpp
@@ -41,7 +41,8 @@ std::unique_ptr<IArraySink> createArraySink(ColumnArray & col, size_t column_siz
     using Creator = ApplyTypeListForClass<ArraySinkCreator, TypeListNumbers>::Type;
     if (auto column_nullable = typeid_cast<ColumnNullable *>(&col.getData()))
     {
-        auto column = ColumnArray::create(column_nullable->getNestedColumnPtr(), col.getOffsetsPtr());
+        auto column = ColumnArray::create(column_nullable->getNestedColumnPtr()->assumeMutable(),
+                                          col.getOffsetsPtr()->assumeMutable());
         return Creator::create(*column, &column_nullable->getNullMapData(), column_size);
     }
     return Creator::create(col, nullptr, column_size);
diff --git a/dbms/src/Functions/IFunction.cpp b/dbms/src/Functions/IFunction.cpp
index ba05bc34e9d..12e8dfabbd8 100644
--- a/dbms/src/Functions/IFunction.cpp
+++ b/dbms/src/Functions/IFunction.cpp
@@ -63,7 +63,7 @@ ColumnPtr wrapInNullable(const ColumnPtr & src, Block & block, const ColumnNumbe
             }
             else
             {
-                MutableColumnPtr mutable_result_null_map_column = result_null_map_column->mutate();
+                MutableColumnPtr mutable_result_null_map_column = (*std::move(result_null_map_column)).mutate();
 
                 NullMap & result_null_map = static_cast<ColumnUInt8 &>(*mutable_result_null_map_column).getData();
                 const NullMap & src_null_map = static_cast<const ColumnUInt8 &>(*null_map_column).getData();
diff --git a/dbms/src/IO/ReadBufferAIO.cpp b/dbms/src/IO/ReadBufferAIO.cpp
index c4680b0e409..b9f760b51da 100644
--- a/dbms/src/IO/ReadBufferAIO.cpp
+++ b/dbms/src/IO/ReadBufferAIO.cpp
@@ -1,4 +1,4 @@
-#if !(defined(__FreeBSD__) || defined(__APPLE__))
+#if !(defined(__FreeBSD__) || defined(__APPLE__) || defined(_MSC_VER))
 
 #include <IO/ReadBufferAIO.h>
 #include <Common/ProfileEvents.h>
diff --git a/dbms/src/IO/ReadBufferAIO.h b/dbms/src/IO/ReadBufferAIO.h
index ccbd7330851..9f8451338ac 100644
--- a/dbms/src/IO/ReadBufferAIO.h
+++ b/dbms/src/IO/ReadBufferAIO.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#if !(defined(__FreeBSD__) || defined(__APPLE__))
+#if !(defined(__FreeBSD__) || defined(__APPLE__) || defined(_MSC_VER))
 
 #include <IO/ReadBufferFromFileBase.h>
 #include <IO/ReadBuffer.h>
@@ -8,7 +8,6 @@
 #include <Core/Defines.h>
 #include <Common/AIO.h>
 #include <Common/CurrentMetrics.h>
-
 #include <string>
 #include <limits>
 #include <unistd.h>
diff --git a/dbms/src/IO/ReadBufferFromFileDescriptor.cpp b/dbms/src/IO/ReadBufferFromFileDescriptor.cpp
index eced314ff40..d2acffaf2e7 100644
--- a/dbms/src/IO/ReadBufferFromFileDescriptor.cpp
+++ b/dbms/src/IO/ReadBufferFromFileDescriptor.cpp
@@ -1,14 +1,10 @@
 #include <errno.h>
 #include <time.h>
-
 #include <optional>
-
 #include <Common/ProfileEvents.h>
 #include <Common/Stopwatch.h>
-
 #include <Common/Exception.h>
 #include <Common/CurrentMetrics.h>
-
 #include <IO/ReadBufferFromFileDescriptor.h>
 #include <IO/WriteHelpers.h>
 
diff --git a/dbms/src/IO/ReadBufferFromFileDescriptor.h b/dbms/src/IO/ReadBufferFromFileDescriptor.h
index 1331878912e..55065931148 100644
--- a/dbms/src/IO/ReadBufferFromFileDescriptor.h
+++ b/dbms/src/IO/ReadBufferFromFileDescriptor.h
@@ -1,7 +1,6 @@
 #pragma once
 
-#include <unistd.h>
-
+#include <port/unistd.h>
 #include <IO/ReadBufferFromFileBase.h>
 #include <IO/ReadBuffer.h>
 
diff --git a/dbms/src/IO/ReadHelpers.h b/dbms/src/IO/ReadHelpers.h
index 875515149b6..2d3b405675f 100644
--- a/dbms/src/IO/ReadHelpers.h
+++ b/dbms/src/IO/ReadHelpers.h
@@ -52,6 +52,8 @@ inline char parseEscapeSequence(char c)
             return '\a';
         case 'b':
             return '\b';
+        case 'e':
+            return '\x1B';      /// \e escape sequence is non standard for C and C++ but supported by gcc and clang.
         case 'f':
             return '\f';
         case 'n':
diff --git a/dbms/src/IO/WriteBufferAIO.cpp b/dbms/src/IO/WriteBufferAIO.cpp
index ae1253df263..087717b452d 100644
--- a/dbms/src/IO/WriteBufferAIO.cpp
+++ b/dbms/src/IO/WriteBufferAIO.cpp
@@ -1,4 +1,4 @@
-#if !(defined(__FreeBSD__) || defined(__APPLE__))
+#if !(defined(__FreeBSD__) || defined(__APPLE__) || defined(_MSC_VER))
 
 #include <IO/WriteBufferAIO.h>
 #include <Common/ProfileEvents.h>
diff --git a/dbms/src/IO/WriteBufferAIO.h b/dbms/src/IO/WriteBufferAIO.h
index 0942934e87d..3c99eb40eed 100644
--- a/dbms/src/IO/WriteBufferAIO.h
+++ b/dbms/src/IO/WriteBufferAIO.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#if !(defined(__FreeBSD__) || defined(__APPLE__))
+#if !(defined(__FreeBSD__) || defined(__APPLE__) || defined(_MSC_VER))
 
 #include <IO/WriteBufferFromFileBase.h>
 #include <IO/WriteBuffer.h>
diff --git a/dbms/src/IO/WriteBufferFromFileDescriptor.cpp b/dbms/src/IO/WriteBufferFromFileDescriptor.cpp
index 8f6f2b1b126..d22fe7229b8 100644
--- a/dbms/src/IO/WriteBufferFromFileDescriptor.cpp
+++ b/dbms/src/IO/WriteBufferFromFileDescriptor.cpp
@@ -1,4 +1,4 @@
-#include <unistd.h>
+#include <port/unistd.h>
 #include <errno.h>
 
 #include <Common/Exception.h>
diff --git a/dbms/src/IO/createReadBufferFromFileBase.cpp b/dbms/src/IO/createReadBufferFromFileBase.cpp
index bae7a737d36..beb73eda861 100644
--- a/dbms/src/IO/createReadBufferFromFileBase.cpp
+++ b/dbms/src/IO/createReadBufferFromFileBase.cpp
@@ -1,6 +1,6 @@
 #include <IO/createReadBufferFromFileBase.h>
 #include <IO/ReadBufferFromFile.h>
-#if !defined(__APPLE__) && !defined(__FreeBSD__)
+#if !defined(__APPLE__) && !defined(__FreeBSD__) && !defined(_MSC_VER)
 #include <IO/ReadBufferAIO.h>
 #endif
 #include <Common/ProfileEvents.h>
@@ -14,7 +14,7 @@ namespace ProfileEvents
 
 namespace DB
 {
-#if defined(__APPLE__) || defined(__FreeBSD__)
+#if defined(__APPLE__) || defined(__FreeBSD__) || defined(_MSC_VER)
 namespace ErrorCodes
 {
         extern const int NOT_IMPLEMENTED;
@@ -31,7 +31,7 @@ std::unique_ptr<ReadBufferFromFileBase> createReadBufferFromFileBase(const std::
     }
     else
     {
-#if !defined(__APPLE__) && !defined(__FreeBSD__)
+#if !defined(__APPLE__) && !defined(__FreeBSD__) && !defined(_MSC_VER)
         ProfileEvents::increment(ProfileEvents::CreatedReadBufferAIO);
         return std::make_unique<ReadBufferAIO>(filename_, buffer_size_, flags_, existing_memory_);
 #else
diff --git a/dbms/src/IO/createWriteBufferFromFileBase.cpp b/dbms/src/IO/createWriteBufferFromFileBase.cpp
index 94d16d1d41b..de06e2eb1cb 100644
--- a/dbms/src/IO/createWriteBufferFromFileBase.cpp
+++ b/dbms/src/IO/createWriteBufferFromFileBase.cpp
@@ -1,6 +1,6 @@
 #include <IO/createWriteBufferFromFileBase.h>
 #include <IO/WriteBufferFromFile.h>
-#if !defined(__APPLE__) && !defined(__FreeBSD__)
+#if !defined(__APPLE__) && !defined(__FreeBSD__) && !defined(_MSC_VER)
 #include <IO/WriteBufferAIO.h>
 #endif
 #include <Common/ProfileEvents.h>
@@ -15,7 +15,7 @@ namespace ProfileEvents
 namespace DB
 {
 
-#if defined(__APPLE__) || defined(__FreeBSD__)
+#if defined(__APPLE__) || defined(__FreeBSD__) || defined(_MSC_VER)
 namespace ErrorCodes
 {
         extern const int NOT_IMPLEMENTED;
@@ -33,7 +33,7 @@ WriteBufferFromFileBase * createWriteBufferFromFileBase(const std::string & file
     }
     else
     {
-#if !defined(__APPLE__) && !defined(__FreeBSD__)
+#if !defined(__APPLE__) && !defined(__FreeBSD__) && !defined(_MSC_VER)
         ProfileEvents::increment(ProfileEvents::CreatedWriteBufferAIO);
         return new WriteBufferAIO(filename_, buffer_size_, flags_, mode, existing_memory_);
 #else
diff --git a/dbms/src/IO/tests/read_buffer_aio.cpp b/dbms/src/IO/tests/read_buffer_aio.cpp
index b6352d7a54a..2f641d0cd3c 100644
--- a/dbms/src/IO/tests/read_buffer_aio.cpp
+++ b/dbms/src/IO/tests/read_buffer_aio.cpp
@@ -1,14 +1,13 @@
 #include <IO/ReadBufferAIO.h>
 #include <Core/Defines.h>
-
 #include <boost/filesystem.hpp>
-
 #include <vector>
 #include <iostream>
 #include <fstream>
 #include <functional>
 #include <cstdlib>
-#include <unistd.h>
+#include <port/unistd.h>
+
 
 namespace
 {
diff --git a/dbms/src/Interpreters/ClientInfo.cpp b/dbms/src/Interpreters/ClientInfo.cpp
index 40ebabc60df..06fd5e9c589 100644
--- a/dbms/src/Interpreters/ClientInfo.cpp
+++ b/dbms/src/Interpreters/ClientInfo.cpp
@@ -1,5 +1,3 @@
-#include <unistd.h>
-
 #include <Interpreters/ClientInfo.h>
 #include <IO/ReadBuffer.h>
 #include <IO/WriteBuffer.h>
@@ -8,6 +6,7 @@
 #include <Core/Defines.h>
 #include <Common/getFQDNOrHostName.h>
 #include <Common/ClickHouseRevision.h>
+#include <port/unistd.h>
 
 
 namespace DB
diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp
index 8fe51bf5c08..e40f0a4eb00 100644
--- a/dbms/src/Interpreters/Context.cpp
+++ b/dbms/src/Interpreters/Context.cpp
@@ -82,6 +82,7 @@ namespace ErrorCodes
     extern const int TABLE_SIZE_EXCEEDS_MAX_DROP_SIZE_LIMIT;
     extern const int SESSION_NOT_FOUND;
     extern const int SESSION_IS_LOCKED;
+    extern const int CANNOT_GET_CREATE_TABLE_QUERY;
 }
 
 
@@ -912,17 +913,17 @@ DatabasePtr Context::detachDatabase(const String & database_name)
 }
 
 
-ASTPtr Context::getCreateQuery(const String & database_name, const String & table_name) const
+ASTPtr Context::getCreateTableQuery(const String & database_name, const String & table_name) const
 {
     auto lock = getLock();
 
     String db = resolveDatabase(database_name, current_database);
     assertDatabaseExists(db);
 
-    return shared->databases[db]->getCreateQuery(*this, table_name);
+    return shared->databases[db]->getCreateTableQuery(*this, table_name);
 }
 
-ASTPtr Context::getCreateExternalQuery(const String & table_name) const
+ASTPtr Context::getCreateExternalTableQuery(const String & table_name) const
 {
     TableAndCreateASTs::const_iterator jt = external_tables.find(table_name);
     if (external_tables.end() == jt)
@@ -931,6 +932,15 @@ ASTPtr Context::getCreateExternalQuery(const String & table_name) const
     return jt->second.second;
 }
 
+ASTPtr Context::getCreateDatabaseQuery(const String & database_name) const
+{
+    auto lock = getLock();
+
+    String db = resolveDatabase(database_name, current_database);
+    assertDatabaseExists(db);
+
+    return shared->databases[db]->getCreateDatabaseQuery(*this);
+}
 
 Settings Context::getSettings() const
 {
diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h
index 4f714842b62..73ddb5dce64 100644
--- a/dbms/src/Interpreters/Context.h
+++ b/dbms/src/Interpreters/Context.h
@@ -241,8 +241,9 @@ public:
     UInt16 getTCPPort() const;
 
     /// Get query for the CREATE table.
-    ASTPtr getCreateQuery(const String & database_name, const String & table_name) const;
-    ASTPtr getCreateExternalQuery(const String & table_name) const;
+    ASTPtr getCreateTableQuery(const String & database_name, const String & table_name) const;
+    ASTPtr getCreateExternalTableQuery(const String & table_name) const;
+    ASTPtr getCreateDatabaseQuery(const String & database_name) const;
 
     const DatabasePtr getDatabase(const String & database_name) const;
     DatabasePtr getDatabase(const String & database_name);
diff --git a/dbms/src/Interpreters/ExternalDictionaries.h b/dbms/src/Interpreters/ExternalDictionaries.h
index cc03c9b9389..0b00ab9f632 100644
--- a/dbms/src/Interpreters/ExternalDictionaries.h
+++ b/dbms/src/Interpreters/ExternalDictionaries.h
@@ -31,6 +31,11 @@ public:
         return std::static_pointer_cast<IDictionaryBase>(getLoadable(name));
     }
 
+    DictPtr tryGetDictionary(const std::string & name) const
+    {
+        return std::static_pointer_cast<IDictionaryBase>(tryGetLoadable(name));
+    }
+
 protected:
 
     std::unique_ptr<IExternalLoadable> create(const std::string & name, const Configuration & config,
diff --git a/dbms/src/Interpreters/ExternalLoader.cpp b/dbms/src/Interpreters/ExternalLoader.cpp
index 527c3d495ea..fc25327558e 100644
--- a/dbms/src/Interpreters/ExternalLoader.cpp
+++ b/dbms/src/Interpreters/ExternalLoader.cpp
@@ -385,21 +385,35 @@ void ExternalLoader::reload(const std::string & name)
         throw Exception("Failed to load " + object_name + " '" + name + "' during the reload process", ErrorCodes::BAD_ARGUMENTS);
 }
 
-ExternalLoader::LoadablePtr ExternalLoader::getLoadable(const std::string & name) const
+ExternalLoader::LoadablePtr ExternalLoader::getLoadableImpl(const std::string & name, bool throw_on_error) const
 {
     const std::lock_guard<std::mutex> lock{map_mutex};
 
     const auto it = loadable_objects.find(name);
     if (it == std::end(loadable_objects))
-        throw Exception("No such " + object_name + ": " + name, ErrorCodes::BAD_ARGUMENTS);
+    {
+        if (throw_on_error)
+            throw Exception("No such " + object_name + ": " + name, ErrorCodes::BAD_ARGUMENTS);
+        return nullptr;
+    }
 
-    if (!it->second.loadable)
-        it->second.exception ? std::rethrow_exception(it->second.exception) :
-        throw Exception{object_name + " '" + name + "' is not loaded", ErrorCodes::LOGICAL_ERROR};
+    if (!it->second.loadable && throw_on_error)
+        it->second.exception ? std::rethrow_exception(it->second.exception)
+                             : throw Exception{object_name + " '" + name + "' is not loaded", ErrorCodes::LOGICAL_ERROR};
 
     return it->second.loadable;
 }
 
+ExternalLoader::LoadablePtr ExternalLoader::getLoadable(const std::string & name) const
+{
+    return getLoadableImpl(name, true);
+}
+
+ExternalLoader::LoadablePtr ExternalLoader::tryGetLoadable(const std::string & name) const
+{
+    return getLoadableImpl(name, false);
+}
+
 ExternalLoader::LockedObjectsMap ExternalLoader::getObjectsMap() const
 {
     return LockedObjectsMap(map_mutex, loadable_objects);
diff --git a/dbms/src/Interpreters/ExternalLoader.h b/dbms/src/Interpreters/ExternalLoader.h
index a2f64b61dc7..2cf42942bc9 100644
--- a/dbms/src/Interpreters/ExternalLoader.h
+++ b/dbms/src/Interpreters/ExternalLoader.h
@@ -104,6 +104,7 @@ public:
     void reload(const std::string & name);
 
     LoadablePtr getLoadable(const std::string & name) const;
+    LoadablePtr tryGetLoadable(const std::string & name) const;
 
 protected:
     virtual std::unique_ptr<IExternalLoadable> create(const std::string & name, const Configuration & config,
@@ -172,6 +173,8 @@ private:
     void reloadAndUpdate(bool throw_on_error = false);
 
     void reloadPeriodically();
+
+    LoadablePtr getLoadableImpl(const std::string & name, bool throw_on_error) const;
 };
 
 }
diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp
index 3d70fc343b9..ef255b319a8 100644
--- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp
@@ -425,7 +425,7 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const
         String as_database_name = create.as_database.empty() ? context.getCurrentDatabase() : create.as_database;
         String as_table_name = create.as_table;
 
-        ASTPtr as_create_ptr = context.getCreateQuery(as_database_name, as_table_name);
+        ASTPtr as_create_ptr = context.getCreateTableQuery(as_database_name, as_table_name);
         const auto & as_create = typeid_cast<const ASTCreateQuery &>(*as_create_ptr);
 
         if (as_create.is_view)
@@ -454,7 +454,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create)
     if (create.attach && !create.storage && !create.columns)
     {
         // Table SQL definition is available even if the table is detached
-        auto query = context.getCreateQuery(database_name, table_name);
+        auto query = context.getCreateTableQuery(database_name, table_name);
         auto & as_create = typeid_cast<const ASTCreateQuery &>(*query);
         create = as_create; // Copy the saved create query, but use ATTACH instead of CREATE
         create.attach = true;
diff --git a/dbms/src/Interpreters/InterpreterFactory.cpp b/dbms/src/Interpreters/InterpreterFactory.cpp
index 9d30c586129..2e43efbb12e 100644
--- a/dbms/src/Interpreters/InterpreterFactory.cpp
+++ b/dbms/src/Interpreters/InterpreterFactory.cpp
@@ -107,7 +107,11 @@ std::unique_ptr<IInterpreter> InterpreterFactory::get(ASTPtr & query, Context &
     {
         return std::make_unique<InterpreterExistsQuery>(query, context);
     }
-    else if (typeid_cast<ASTShowCreateQuery *>(query.get()))
+    else if (typeid_cast<ASTShowCreateTableQuery *>(query.get()))
+    {
+        return std::make_unique<InterpreterShowCreateQuery>(query, context);
+    }
+    else if (typeid_cast<ASTShowCreateDatabaseQuery *>(query.get()))
     {
         return std::make_unique<InterpreterShowCreateQuery>(query, context);
     }
diff --git a/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp b/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp
index d4bd116bcf7..a4c3bd7224c 100644
--- a/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp
@@ -39,13 +39,18 @@ Block InterpreterShowCreateQuery::getSampleBlock()
 
 BlockInputStreamPtr InterpreterShowCreateQuery::executeImpl()
 {
-    const ASTShowCreateQuery & ast = typeid_cast<const ASTShowCreateQuery &>(*query_ptr);
+    const auto & ast = dynamic_cast<const ASTQueryWithTableAndOutput &>(*query_ptr);
 
     if (ast.temporary && !ast.database.empty())
         throw Exception("Temporary databases are not possible.", ErrorCodes::SYNTAX_ERROR);
 
-    ASTPtr create_query = (ast.temporary ? context.getCreateExternalQuery(ast.table) :
-                          context.getCreateQuery(ast.database, ast.table));
+    ASTPtr create_query;
+    if (ast.temporary)
+        create_query = context.getCreateExternalTableQuery(ast.table);
+    else if (ast.table.empty())
+        create_query = context.getCreateDatabaseQuery(ast.database);
+    else
+        create_query = context.getCreateTableQuery(ast.database, ast.table);
 
     if (!create_query && ast.temporary)
         throw Exception("Unable to show the create query of " + ast.table + ". Maybe it was created by the system.", ErrorCodes::THERE_IS_NO_QUERY);
diff --git a/dbms/src/Interpreters/NullableUtils.cpp b/dbms/src/Interpreters/NullableUtils.cpp
index 972c548e67e..dcb0657aa9d 100644
--- a/dbms/src/Interpreters/NullableUtils.cpp
+++ b/dbms/src/Interpreters/NullableUtils.cpp
@@ -31,7 +31,7 @@ void extractNestedColumnsAndNullMap(ColumnRawPtrs & key_columns, ColumnPtr & nul
                 }
                 else
                 {
-                    MutableColumnPtr mutable_null_map_holder = null_map_holder->mutate();
+                    MutableColumnPtr mutable_null_map_holder = (*std::move(null_map_holder)).mutate();
 
                     PaddedPODArray<UInt8> & mutable_null_map = static_cast<ColumnUInt8 &>(*mutable_null_map_holder).getData();
                     const PaddedPODArray<UInt8> & other_null_map = column_nullable.getNullMapData();
diff --git a/dbms/src/Interpreters/PartLog.cpp b/dbms/src/Interpreters/PartLog.cpp
index 8043f451402..646aae527b1 100644
--- a/dbms/src/Interpreters/PartLog.cpp
+++ b/dbms/src/Interpreters/PartLog.cpp
@@ -108,7 +108,7 @@ bool PartLog::addNewPartToTheLog(Context & context, const MergeTreeDataPart & pa
         elem.table_name = part.storage.getTableName();
         elem.part_name = part.name;
 
-        elem.bytes_compressed_on_disk = part.size_in_bytes;
+        elem.bytes_compressed_on_disk = part.bytes_on_disk;
         elem.rows = part.rows_count;
 
         elem.error = static_cast<UInt16>(execution_status.code);
diff --git a/dbms/src/Interpreters/SystemLog.h b/dbms/src/Interpreters/SystemLog.h
index 5c6071e647d..8e04076dc14 100644
--- a/dbms/src/Interpreters/SystemLog.h
+++ b/dbms/src/Interpreters/SystemLog.h
@@ -233,8 +233,7 @@ void SystemLog<LogElement>::flush()
     {
         LOG_TRACE(log, "Flushing system log");
 
-        if (!is_prepared)    /// BTW, flush method is called from single thread.
-            prepareTable();
+        prepareTable();
 
         Block block = LogElement::createBlock();
         for (const LogElement & elem : data)
@@ -311,7 +310,7 @@ void SystemLog<LogElement>::prepareTable()
             /// The required table will be created.
             table = nullptr;
         }
-        else
+        else if (!is_prepared)
             LOG_DEBUG(log, "Will use existing table " << description << " for " + LogElement::name());
     }
 
diff --git a/dbms/src/Interpreters/tests/users.cpp b/dbms/src/Interpreters/tests/users.cpp
index 5da4ed26134..f4905398262 100644
--- a/dbms/src/Interpreters/tests/users.cpp
+++ b/dbms/src/Interpreters/tests/users.cpp
@@ -1,8 +1,6 @@
 #include <Common/Config/ConfigProcessor.h>
 #include <Interpreters/SecurityManager.h>
-
 #include <boost/filesystem.hpp>
-
 #include <vector>
 #include <string>
 #include <tuple>
@@ -10,8 +8,8 @@
 #include <fstream>
 #include <sstream>
 #include <stdexcept>
-#include <unistd.h>
 #include <cstdlib>
+#include <port/unistd.h>
 
 namespace
 {
diff --git a/dbms/src/Parsers/ParserTablePropertiesQuery.cpp b/dbms/src/Parsers/ParserTablePropertiesQuery.cpp
index 570597b0fa6..45e92f9e181 100644
--- a/dbms/src/Parsers/ParserTablePropertiesQuery.cpp
+++ b/dbms/src/Parsers/ParserTablePropertiesQuery.cpp
@@ -19,6 +19,7 @@ bool ParserTablePropertiesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected &
     ParserKeyword s_desc("DESC");
     ParserKeyword s_show("SHOW");
     ParserKeyword s_create("CREATE");
+    ParserKeyword s_database("DATABASE");
     ParserKeyword s_table("TABLE");
     ParserToken s_dot(TokenType::Dot);
     ParserIdentifier name_p;
@@ -27,6 +28,8 @@ bool ParserTablePropertiesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected &
     ASTPtr table;
     std::shared_ptr<ASTQueryWithTableAndOutput> query;
 
+    bool parse_only_database_name = false;
+
     if (s_exists.ignore(pos, expected))
     {
         query = std::make_shared<ASTExistsQuery>();
@@ -36,26 +39,40 @@ bool ParserTablePropertiesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected &
         if (!s_create.ignore(pos, expected))
             return false;
 
-        query = std::make_shared<ASTShowCreateQuery>();
+        if (s_database.ignore(pos, expected))
+        {
+            parse_only_database_name = true;
+            query = std::make_shared<ASTShowCreateDatabaseQuery>();
+        }
+        else
+            query = std::make_shared<ASTShowCreateTableQuery>();
     }
     else
     {
         return false;
     }
 
-    if (s_temporary.ignore(pos, expected))
-        query->temporary = true;
-
-    s_table.ignore(pos, expected);
-
-    if (!name_p.parse(pos, table, expected))
-        return false;
-
-    if (s_dot.ignore(pos, expected))
+    if (parse_only_database_name)
     {
-        database = table;
+        if (!name_p.parse(pos, database, expected))
+            return false;
+    }
+    else
+    {
+        if (s_temporary.ignore(pos, expected))
+            query->temporary = true;
+
+        s_table.ignore(pos, expected);
+
         if (!name_p.parse(pos, table, expected))
             return false;
+
+        if (s_dot.ignore(pos, expected))
+        {
+            database = table;
+            if (!name_p.parse(pos, table, expected))
+                return false;
+        }
     }
 
     if (database)
diff --git a/dbms/src/Parsers/TablePropertiesQueriesASTs.h b/dbms/src/Parsers/TablePropertiesQueriesASTs.h
index 312b112f494..15a1d397fa4 100644
--- a/dbms/src/Parsers/TablePropertiesQueriesASTs.h
+++ b/dbms/src/Parsers/TablePropertiesQueriesASTs.h
@@ -12,12 +12,18 @@ struct ASTExistsQueryIDAndQueryNames
     static constexpr auto Query = "EXISTS TABLE";
 };
 
-struct ASTShowCreateQueryIDAndQueryNames
+struct ASTShowCreateTableQueryIDAndQueryNames
 {
-    static constexpr auto ID = "ShowCreateQuery";
+    static constexpr auto ID = "ShowCreateTableQuery";
     static constexpr auto Query = "SHOW CREATE TABLE";
 };
 
+struct ASTShowCreateDatabaseQueryIDAndQueryNames
+{
+    static constexpr auto ID = "ShowCreateDatabaseQuery";
+    static constexpr auto Query = "SHOW CREATE DATABASE";
+};
+
 struct ASTDescribeQueryExistsQueryIDAndQueryNames
 {
     static constexpr auto ID = "DescribeQuery";
@@ -25,7 +31,17 @@ struct ASTDescribeQueryExistsQueryIDAndQueryNames
 };
 
 using ASTExistsQuery = ASTQueryWithTableAndOutputImpl<ASTExistsQueryIDAndQueryNames>;
-using ASTShowCreateQuery = ASTQueryWithTableAndOutputImpl<ASTShowCreateQueryIDAndQueryNames>;
+using ASTShowCreateTableQuery = ASTQueryWithTableAndOutputImpl<ASTShowCreateTableQueryIDAndQueryNames>;
+
+class ASTShowCreateDatabaseQuery : public ASTQueryWithTableAndOutputImpl<ASTShowCreateDatabaseQueryIDAndQueryNames>
+{
+protected:
+    void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override
+    {
+        settings.ostr << (settings.hilite ? hilite_keyword : "") << ASTShowCreateDatabaseQueryIDAndQueryNames::Query
+                      << " " << (settings.hilite ? hilite_none : "") << backQuoteIfNeed(database);
+    }
+};
 
 class ASTDescribeQuery : public ASTQueryWithOutput
 {
diff --git a/dbms/src/Server/Benchmark.cpp b/dbms/src/Server/Benchmark.cpp
index 78a19a398a1..ee4d2dd8802 100644
--- a/dbms/src/Server/Benchmark.cpp
+++ b/dbms/src/Server/Benchmark.cpp
@@ -1,31 +1,24 @@
-#include <unistd.h>
+#include <port/unistd.h>
 #include <stdlib.h>
 #include <fcntl.h>
 #include <signal.h>
 #include <time.h>
-
 #include <iostream>
 #include <fstream>
 #include <iomanip>
 #include <random>
 #include <pcg_random.hpp>
-
 #include <Poco/File.h>
 #include <Poco/Util/Application.h>
-
 #include <Common/Stopwatch.h>
 #include <common/ThreadPool.h>
 #include <AggregateFunctions/ReservoirSampler.h>
 #include <AggregateFunctions/registerAggregateFunctions.h>
-
 #include <boost/program_options.hpp>
-
 #include <Common/ConcurrentBoundedQueue.h>
-
 #include <Common/Exception.h>
 #include <Common/randomSeed.h>
 #include <Core/Types.h>
-
 #include <IO/ReadBufferFromFileDescriptor.h>
 #include <IO/WriteBufferFromFileDescriptor.h>
 #include <IO/WriteBufferFromFile.h>
@@ -33,13 +26,9 @@
 #include <IO/WriteHelpers.h>
 #include <IO/Operators.h>
 #include <IO/ConnectionTimeouts.h>
-
 #include <DataStreams/RemoteBlockInputStream.h>
-
 #include <Interpreters/Context.h>
-
 #include <Client/Connection.h>
-
 #include "InterruptListener.h"
 
 
diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp
index 1c838a99569..eb0e4482a37 100644
--- a/dbms/src/Server/Client.cpp
+++ b/dbms/src/Server/Client.cpp
@@ -1,8 +1,8 @@
-#include <unistd.h>
+#include <port/unistd.h>
 #include <stdlib.h>
 #include <fcntl.h>
 #include <signal.h>
-
+#include <map>
 #include <iostream>
 #include <fstream>
 #include <iomanip>
@@ -10,13 +10,11 @@
 #include <algorithm>
 #include <optional>
 #include <boost/program_options.hpp>
-
+#include <boost/algorithm/string/replace.hpp>
 #include <Poco/File.h>
 #include <Poco/Util/Application.h>
-
 #include <common/readline_use.h>
 #include <common/find_first_symbols.h>
-
 #include <Common/ClickHouseRevision.h>
 #include <Common/Stopwatch.h>
 #include <Common/Exception.h>
@@ -35,6 +33,7 @@
 #include <IO/WriteBufferFromFileDescriptor.h>
 #include <IO/WriteBufferFromFile.h>
 #include <IO/ReadBufferFromMemory.h>
+#include <IO/ReadBufferFromString.h>
 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
 #include <DataStreams/AsynchronousBlockInputStream.h>
@@ -102,7 +101,6 @@ private:
         "учшеж", "йгшеж", "дщпщгеж",
         "q", "й", "\\q", "\\Q", "\\й", "\\Й", ":q", "Жй"
     };
-
     bool is_interactive = true;          /// Use either readline interface or batch mode.
     bool need_render_progress = true;    /// Render query execution progress.
     bool echo_queries = false;           /// Print queries before execution in batch mode.
@@ -112,6 +110,7 @@ private:
     winsize terminal_size {};            /// Terminal size is needed to render progress bar.
 
     std::unique_ptr<Connection> connection;    /// Connection to DB.
+    String query_id;                     /// Current query_id.
     String query;                        /// Current query.
 
     String format;                       /// Query results output format.
@@ -139,6 +138,8 @@ private:
 
     String current_profile;
 
+    String prompt_by_server_display_name;
+
     /// Path to a file containing command history.
     String history_file;
 
@@ -154,6 +155,7 @@ private:
     /// If the last query resulted in exception.
     bool got_exception = false;
     String server_version;
+    String server_display_name;
 
     Stopwatch watch;
 
@@ -168,6 +170,49 @@ private:
     std::list<ExternalTable> external_tables;
 
 
+    struct ConnectionParameters
+    {
+        String host;
+        UInt16 port;
+        String default_database;
+        String user;
+        String password;
+        Protocol::Encryption security;
+        Protocol::Compression compression;
+        ConnectionTimeouts timeouts;
+
+        ConnectionParameters() {}
+
+        ConnectionParameters(const Poco::Util::AbstractConfiguration & config)
+        {
+            bool is_secure = config.getBool("secure", false);
+            security = is_secure
+                ? Protocol::Encryption::Enable
+                : Protocol::Encryption::Disable;
+
+            host = config.getString("host", "localhost");
+            port = config.getInt("port",
+                config.getInt(is_secure ? "tcp_secure_port" : "tcp_port",
+                    is_secure ? DBMS_DEFAULT_SECURE_PORT : DBMS_DEFAULT_PORT));
+
+            default_database = config.getString("database", "");
+            user = config.getString("user", "");
+            password = config.getString("password", "");
+
+            compression = config.getBool("compression", true)
+                ? Protocol::Compression::Enable
+                : Protocol::Compression::Disable;
+
+            timeouts = ConnectionTimeouts(
+                Poco::Timespan(config.getInt("connect_timeout", DBMS_DEFAULT_CONNECT_TIMEOUT_SEC), 0),
+                Poco::Timespan(config.getInt("receive_timeout", DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC), 0),
+                Poco::Timespan(config.getInt("send_timeout", DBMS_DEFAULT_SEND_TIMEOUT_SEC), 0));
+        }
+    };
+
+    ConnectionParameters connection_parameters;
+
+
     void initialize(Poco::Util::Application & self)
     {
         Poco::Util::Application::initialize(self);
@@ -309,6 +354,7 @@ private:
             echo_queries = config().getBool("echo", false);
         }
 
+        connection_parameters = ConnectionParameters(config());
         connect();
 
         /// Initialize DateLUT here to avoid counting time spent here as query execution time.
@@ -338,8 +384,46 @@ private:
             }
         }
 
+        Strings keys;
+
+        prompt_by_server_display_name = config().getRawString("prompt_by_server_display_name.default", "{display_name} :) ");
+
+        config().keys("prompt_by_server_display_name", keys);
+
+        for (const String & key : keys)
+        {
+            if (key != "default" && server_display_name.find(key) != std::string::npos)
+            {
+                prompt_by_server_display_name = config().getRawString("prompt_by_server_display_name." + key);
+                break;
+            }
+        }
+
+        /// Prompt may contain escape sequences including \e[ or \x1b[ sequences to set terminal color.
+        {
+            String unescaped_prompt_by_server_display_name;
+            ReadBufferFromString in(prompt_by_server_display_name);
+            readEscapedString(unescaped_prompt_by_server_display_name, in);
+            prompt_by_server_display_name = std::move(unescaped_prompt_by_server_display_name);
+        }
+
+        /// Prompt may contain the following substitutions in a form of {name}.
+        std::map<String, String> prompt_substitutions
+        {
+            {"host", connection_parameters.host},
+            {"port", toString(connection_parameters.port)},
+            {"user", connection_parameters.user},
+            {"display_name", server_display_name},
+        };
+
+        /// Quite suboptimal.
+        for (const auto & [key, value]: prompt_substitutions)
+            boost::replace_all(prompt_by_server_display_name, "{" + key + "}", value);
+
         if (is_interactive)
         {
+            if (!query_id.empty())
+                throw Exception("query_id could be specified only in non-interactive mode", ErrorCodes::BAD_ARGUMENTS);
             if (print_time_to_stderr)
                 throw Exception("time option could be specified only in non-interactive mode", ErrorCodes::BAD_ARGUMENTS);
 
@@ -374,6 +458,7 @@ private:
         }
         else
         {
+            query_id = config().getString("query_id", "");
             nonInteractive();
 
             if (last_exception)
@@ -386,34 +471,23 @@ private:
 
     void connect()
     {
-        auto encryption = config().getBool("ssl", false)
-            ? Protocol::Encryption::Enable
-            : Protocol::Encryption::Disable;
-
-        String host = config().getString("host", "localhost");
-        UInt16 port = config().getInt("port", config().getInt(static_cast<bool>(encryption) ? "tcp_ssl_port" : "tcp_port", static_cast<bool>(encryption) ? DBMS_DEFAULT_SECURE_PORT : DBMS_DEFAULT_PORT));
-        String default_database = config().getString("database", "");
-        String user = config().getString("user", "");
-        String password = config().getString("password", "");
-
-        auto compression = config().getBool("compression", true)
-            ? Protocol::Compression::Enable
-            : Protocol::Compression::Disable;
-
         if (is_interactive)
             std::cout << "Connecting to "
-                << (!default_database.empty() ? "database " + default_database + " at " : "")
-                << host << ":" << port
-                << (!user.empty() ? " as user " + user : "")
+                << (!connection_parameters.default_database.empty() ? "database " + connection_parameters.default_database + " at " : "")
+                << connection_parameters.host << ":" << connection_parameters.port
+                << (!connection_parameters.user.empty() ? " as user " + connection_parameters.user : "")
                 << "." << std::endl;
 
-        ConnectionTimeouts timeouts(
-            Poco::Timespan(config().getInt("connect_timeout", DBMS_DEFAULT_CONNECT_TIMEOUT_SEC), 0),
-            Poco::Timespan(config().getInt("receive_timeout", DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC), 0),
-            Poco::Timespan(config().getInt("send_timeout", DBMS_DEFAULT_SEND_TIMEOUT_SEC), 0));
-
         connection = std::make_unique<Connection>(
-            host, port, default_database, user, password, timeouts, "client", compression, encryption);
+            connection_parameters.host,
+            connection_parameters.port,
+            connection_parameters.default_database,
+            connection_parameters.user,
+            connection_parameters.password,
+            connection_parameters.timeouts,
+            "client",
+            connection_parameters.compression,
+            connection_parameters.security);
 
         String server_name;
         UInt64 server_version_major = 0;
@@ -429,6 +503,12 @@ private:
         connection->getServerVersion(server_name, server_version_major, server_version_minor, server_revision);
 
         server_version = toString(server_version_major) + "." + toString(server_version_minor) + "." + toString(server_revision);
+
+        if (server_display_name = connection->getServerDisplayName(); server_display_name.length() == 0)
+        {
+            server_display_name = config().getString("host", "localhost");
+        }
+
         if (is_interactive)
         {
             std::cout << "Connected to " << server_name
@@ -449,12 +529,17 @@ private:
         return select(1, &fds, 0, 0, &timeout) == 1;
     }
 
+    inline const String prompt() const
+    {
+        return boost::replace_all_copy(prompt_by_server_display_name, "{database}", config().getString("database", "default"));
+    }
 
     void loop()
     {
         String query;
         String prev_query;
-        while (char * line_ = readline(query.empty() ? ":) " : ":-] "))
+
+        while (char * line_ = readline(query.empty() ? prompt().c_str() : ":-] "))
         {
             String line = line_;
             free(line_);
@@ -735,7 +820,7 @@ private:
     /// Process the query that doesn't require transfering data blocks to the server.
     void processOrdinaryQuery()
     {
-        connection->sendQuery(query, "", QueryProcessingStage::Complete, &context.getSettingsRef(), nullptr, true);
+        connection->sendQuery(query, query_id, QueryProcessingStage::Complete, &context.getSettingsRef(), nullptr, true);
         sendExternalTables();
         receiveResult();
     }
@@ -753,7 +838,7 @@ private:
         if (!parsed_insert_query.data && (is_interactive || (stdin_is_not_tty && std_in.eof())))
             throw Exception("No data to insert", ErrorCodes::NO_DATA_TO_INSERT);
 
-        connection->sendQuery(query_without_data, "", QueryProcessingStage::Complete, &context.getSettingsRef(), nullptr, true);
+        connection->sendQuery(query_without_data, query_id, QueryProcessingStage::Complete, &context.getSettingsRef(), nullptr, true);
         sendExternalTables();
 
         /// Receive description of table structure.
@@ -1289,6 +1374,7 @@ public:
             ("ssl,s", "ssl")
             ("user,u", boost::program_options::value<std::string>(), "user")
             ("password", boost::program_options::value<std::string>(), "password")
+            ("query_id", boost::program_options::value<std::string>(), "query_id")
             ("query,q", boost::program_options::value<std::string>(), "query")
             ("database,d", boost::program_options::value<std::string>(), "database")
             ("pager", boost::program_options::value<std::string>(), "pager")
@@ -1377,6 +1463,8 @@ public:
             config().setString("config-file", options["config-file"].as<std::string>());
         if (options.count("host") && !options["host"].defaulted())
             config().setString("host", options["host"].as<std::string>());
+        if (options.count("query_id"))
+            config().setString("query_id", options["query_id"].as<std::string>());
         if (options.count("query"))
             config().setString("query", options["query"].as<std::string>());
         if (options.count("database"))
diff --git a/dbms/src/Server/ClusterCopier.cpp b/dbms/src/Server/ClusterCopier.cpp
index 4233ce70e3f..74d5eb5753d 100644
--- a/dbms/src/Server/ClusterCopier.cpp
+++ b/dbms/src/Server/ClusterCopier.cpp
@@ -1128,7 +1128,7 @@ protected:
         catch (const zkutil::KeeperException & e)
         {
             LOG_INFO(log, "A ZooKeeper error occurred while checking partition " << partition_name
-                          << ". Will recheck the partition. Error: " << e.what());
+                          << ". Will recheck the partition. Error: " << e.displayText());
             return false;
         }
 
@@ -1259,8 +1259,10 @@ protected:
         }
 
         /// Remove the locking node
-        cleaner_holder.reset();
-        zookeeper->remove(is_dirty_flag_path);
+        zkutil::Ops ops;
+        ops.emplace_back(new zkutil::Op::Remove(dirt_cleaner_path, -1));
+        ops.emplace_back(new zkutil::Op::Remove(is_dirty_flag_path, -1));
+        zookeeper->multi(ops);
 
         LOG_INFO(log, "Partition " << task_partition.name << " was dropped on cluster " << task_table.cluster_push_name);
         return true;
@@ -1283,6 +1285,7 @@ protected:
             Stopwatch watch;
             TasksShard expected_shards;
             size_t num_failed_shards = 0;
+            bool previous_shard_is_instantly_finished = false;
 
             ++cluster_partition.total_tries;
 
@@ -1328,15 +1331,20 @@ protected:
 
                 expected_shards.emplace_back(shard);
 
+                /// Do not sleep if there is a sequence of already processed shards to increase startup
+                bool sleep_before_execution = !previous_shard_is_instantly_finished && shard->priority.is_remote;
                 PartitionTaskStatus task_status = PartitionTaskStatus::Error;
+                bool was_error = false;
                 for (size_t try_num = 0; try_num < max_shard_partition_tries; ++try_num)
                 {
-                    task_status = tryProcessPartitionTask(partition);
+                    task_status = tryProcessPartitionTask(partition, sleep_before_execution);
 
                     /// Exit if success
                     if (task_status == PartitionTaskStatus::Finished)
                         break;
 
+                    was_error = true;
+
                     /// Skip if the task is being processed by someone
                     if (task_status == PartitionTaskStatus::Active)
                         break;
@@ -1347,6 +1355,8 @@ protected:
 
                 if (task_status == PartitionTaskStatus::Error)
                     ++num_failed_shards;
+
+                previous_shard_is_instantly_finished = !was_error;
             }
 
             cluster_partition.elapsed_time_seconds += watch.elapsedSeconds();
@@ -1413,13 +1423,13 @@ protected:
         Error,
     };
 
-    PartitionTaskStatus tryProcessPartitionTask(ShardPartition & task_partition)
+    PartitionTaskStatus tryProcessPartitionTask(ShardPartition & task_partition, bool sleep_before_execution)
     {
         PartitionTaskStatus res;
 
         try
         {
-            res = processPartitionTaskImpl(task_partition);
+            res = processPartitionTaskImpl(task_partition, sleep_before_execution);
         }
         catch (...)
         {
@@ -1440,7 +1450,7 @@ protected:
         return res;
     }
 
-    PartitionTaskStatus processPartitionTaskImpl(ShardPartition & task_partition)
+    PartitionTaskStatus processPartitionTaskImpl(ShardPartition & task_partition, bool sleep_before_execution)
     {
         TaskShard & task_shard = task_partition.task_shard;
         TaskTable & task_table = task_shard.task_table;
@@ -1480,7 +1490,7 @@ protected:
         };
 
         /// Load balancing
-        auto worker_node_holder = createTaskWorkerNodeAndWaitIfNeed(zookeeper, current_task_status_path, task_shard.priority.is_remote);
+        auto worker_node_holder = createTaskWorkerNodeAndWaitIfNeed(zookeeper, current_task_status_path, sleep_before_execution);
 
         LOG_DEBUG(log, "Processing " << current_task_status_path);
 
@@ -1654,7 +1664,7 @@ protected:
                 }
 
                 using ExistsFuture = zkutil::ZooKeeper::ExistsFuture;
-                auto future_is_dirty_checker = std::make_unique<ExistsFuture>(zookeeper->asyncExists(is_dirty_flag_path));
+                std::unique_ptr<ExistsFuture> future_is_dirty_checker;
 
                 Stopwatch watch(CLOCK_MONOTONIC_COARSE);
                 constexpr size_t check_period_milliseconds = 500;
@@ -1665,9 +1675,15 @@ protected:
                     if (zookeeper->expired())
                         throw Exception("ZooKeeper session is expired, cancel INSERT SELECT", ErrorCodes::UNFINISHED);
 
-                    if (future_is_dirty_checker != nullptr)
+                    if (!future_is_dirty_checker)
+                        future_is_dirty_checker = std::make_unique<ExistsFuture>(zookeeper->asyncExists(is_dirty_flag_path));
+
+                    /// check_period_milliseconds should less than average insert time of single block
+                    /// Otherwise, the insertion will slow a little bit
+                    if (watch.elapsedMilliseconds() >= check_period_milliseconds)
                     {
                         zkutil::ZooKeeper::StatAndExists status;
+
                         try
                         {
                             status = future_is_dirty_checker->get();
@@ -1687,12 +1703,6 @@ protected:
                             throw Exception("Partition is dirty, cancel INSERT SELECT", ErrorCodes::UNFINISHED);
                     }
 
-                    if (watch.elapsedMilliseconds() >= check_period_milliseconds)
-                    {
-                        watch.restart();
-                        future_is_dirty_checker = std::make_unique<ExistsFuture>(zookeeper->asyncExists(is_dirty_flag_path));
-                    }
-
                     return false;
                 };
 
diff --git a/dbms/src/Server/HTTPHandler.cpp b/dbms/src/Server/HTTPHandler.cpp
index e06a3f0b885..fba60c5c12b 100644
--- a/dbms/src/Server/HTTPHandler.cpp
+++ b/dbms/src/Server/HTTPHandler.cpp
@@ -12,7 +12,7 @@
 #include <Common/ExternalTable.h>
 #include <Common/StringUtils/StringUtils.h>
 #include <Common/escapeForFileName.h>
-
+#include <Common/getFQDNOrHostName.h>
 #include <IO/ReadBufferFromIStream.h>
 #include <IO/ZlibInflatingReadBuffer.h>
 #include <IO/ReadBufferFromString.h>
@@ -198,6 +198,7 @@ HTTPHandler::HTTPHandler(IServer & server_)
     : server(server_)
     , log(&Logger::get("HTTPHandler"))
 {
+    server_display_name = server.config().getString("display_name", getFQDNOrHostName());
 }
 
 
@@ -629,7 +630,7 @@ void HTTPHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne
     try
     {
         response.setContentType("text/plain; charset=UTF-8");
-
+        response.set("X-ClickHouse-Server-Display-Name", server_display_name);
         /// For keep-alive to work.
         if (request.getVersion() == Poco::Net::HTTPServerRequest::HTTP_1_1)
             response.setChunkedTransferEncoding(true);
diff --git a/dbms/src/Server/HTTPHandler.h b/dbms/src/Server/HTTPHandler.h
index b0b7164852e..72606afa827 100644
--- a/dbms/src/Server/HTTPHandler.h
+++ b/dbms/src/Server/HTTPHandler.h
@@ -55,6 +55,9 @@ private:
     IServer & server;
     Poco::Logger * log;
 
+    /// It is the name of the server that will be sent in an http-header X-ClickHouse-Server-Display-Name. 
+    String server_display_name;
+
     CurrentMetrics::Increment metric_increment{CurrentMetrics::HTTPConnection};
 
     /// Also initializes 'used_output'.
diff --git a/dbms/src/Server/PerformanceTest.cpp b/dbms/src/Server/PerformanceTest.cpp
index 81f679cd61e..7c9d4436f3c 100644
--- a/dbms/src/Server/PerformanceTest.cpp
+++ b/dbms/src/Server/PerformanceTest.cpp
@@ -3,14 +3,11 @@
 #include <limits>
 #include <regex>
 #include <thread>
-#include <unistd.h>
-
+#include <port/unistd.h>
 #include <boost/filesystem.hpp>
 #include <boost/program_options.hpp>
 #include <sys/stat.h>
-
 #include <common/DateLUT.h>
-
 #include <AggregateFunctions/ReservoirSampler.h>
 #include <Client/Connection.h>
 #include <Common/ConcurrentBoundedQueue.h>
@@ -27,13 +24,11 @@
 #include <Interpreters/Settings.h>
 #include <common/ThreadPool.h>
 #include <common/getMemoryAmount.h>
-
 #include <Poco/AutoPtr.h>
 #include <Poco/Exception.h>
 #include <Poco/SAX/InputSource.h>
 #include <Poco/Util/XMLConfiguration.h>
 #include <Poco/XML/XMLStream.h>
-
 #include "InterruptListener.h"
 
 /** Tests launcher for ClickHouse.
diff --git a/dbms/src/Server/TCPHandler.cpp b/dbms/src/Server/TCPHandler.cpp
index 76a79d09722..faf39dded67 100644
--- a/dbms/src/Server/TCPHandler.cpp
+++ b/dbms/src/Server/TCPHandler.cpp
@@ -508,6 +508,10 @@ void TCPHandler::sendHello()
     {
         writeStringBinary(DateLUT::instance().getTimeZone(), *out);
     }
+    if (client_revision >= DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME)
+    {
+        writeStringBinary(server_display_name, *out);
+    }
     out->next();
 }
 
diff --git a/dbms/src/Server/TCPHandler.h b/dbms/src/Server/TCPHandler.h
index 444d8eb4990..f53850ee487 100644
--- a/dbms/src/Server/TCPHandler.h
+++ b/dbms/src/Server/TCPHandler.h
@@ -2,6 +2,7 @@
 
 #include <Poco/Net/TCPServerConnection.h>
 
+#include <Common/getFQDNOrHostName.h>
 #include <Common/CurrentMetrics.h>
 #include <Common/Stopwatch.h>
 #include <IO/Progress.h>
@@ -81,6 +82,7 @@ public:
         , connection_context(server.context())
         , query_context(server.context())
     {
+        server_display_name =  server.config().getString("display_name", getFQDNOrHostName());
     }
 
     void run();
@@ -111,7 +113,9 @@ private:
     QueryState state;
 
     CurrentMetrics::Increment metric_increment{CurrentMetrics::TCPConnection};
-
+    
+    /// It is the name of the server that will be sent to the client. 
+    String server_display_name;
 
     void runImpl();
 
diff --git a/dbms/src/Server/clickhouse-client.xml b/dbms/src/Server/clickhouse-client.xml
index c19606c518b..3cd05d7fec9 100644
--- a/dbms/src/Server/clickhouse-client.xml
+++ b/dbms/src/Server/clickhouse-client.xml
@@ -13,4 +13,19 @@
             </invalidCertificateHandler>
         </client>
     </openSSL>
+    <!--
+        It's a custom prompt settings for the clickhouse-client
+        Possible macros: 
+            {host}
+            {port}
+            {user}
+            {database} 
+            {display_name}
+        Terminal colors: https://misc.flogisoft.com/bash/tip_colors_and_formatting
+    -->
+    <prompt_by_server_display_name>
+        <default>{display_name} :) </default>
+        <test>{display_name} \e[1;32m:)\e[0m  </test> <!-- if it matched to the substring "test" in the server display name - -->
+        <production>{display_name} \e[1;31m:)\e[0m </production> <!-- if it matched to the substring "production" in the server display name -->
+    </prompt_by_server_display_name>
 </config>
diff --git a/dbms/src/Server/config.xml b/dbms/src/Server/config.xml
index ca783df32ce..8cf1fd6b519 100644
--- a/dbms/src/Server/config.xml
+++ b/dbms/src/Server/config.xml
@@ -9,7 +9,7 @@
         <count>10</count>
         <!-- <console>1</console> --> <!-- Default behavior is autodetection (log to console if not daemon mode and is tty) -->
     </logger>
-
+    <!--display_name>production</display_name--> <!-- It is the name that will be shown in the client -->
     <http_port>8123</http_port>
     <tcp_port>9000</tcp_port>
 
@@ -199,7 +199,7 @@
         <timeout>0.1</timeout>
         <interval>60</interval>
         <root_path>one_min</root_path>
-        <hostname_in_path>true<hostname_in_path>
+        <hostname_in_path>true</hostname_in_path>
 
         <metrics>true</metrics>
         <events>true</events>
diff --git a/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp
index 607eb9efb67..d739badba88 100644
--- a/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp
+++ b/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp
@@ -89,7 +89,7 @@ void DistributedBlockOutputStream::writeAsync(const Block & block)
         return writeSplitAsync(block);
 
     writeAsyncImpl(block);
-    ++blocks_inserted;
+    ++inserted_blocks;
 }
 
 
@@ -100,17 +100,17 @@ std::string DistributedBlockOutputStream::getCurrentStateDescription()
 
     buffer << "Insertion status:\n";
     for (auto & shard_jobs : per_shard_jobs)
-        for (JobInfo & job : shard_jobs)
+        for (JobReplica & job : shard_jobs.replicas_jobs)
         {
             buffer << "Wrote " << job.blocks_written << " blocks and " << job.rows_written << " rows"
                    << " on shard " << job.shard_index << " replica " << job.replica_index
                    << ", " << addresses[job.shard_index][job.replica_index].readableString();
 
             /// Performance statistics
-            if (job.bloks_started > 0)
+            if (job.blocks_started > 0)
             {
-                buffer << " (average " << job.elapsed_time_ms / job.bloks_started << " ms per block";
-                if (job.bloks_started > 1)
+                buffer << " (average " << job.elapsed_time_ms / job.blocks_started << " ms per block";
+                if (job.blocks_started > 1)
                     buffer << ", the slowest block " << job.max_elapsed_time_for_block_ms << " ms";
                 buffer << ")";
             }
@@ -122,10 +122,11 @@ std::string DistributedBlockOutputStream::getCurrentStateDescription()
 }
 
 
-void DistributedBlockOutputStream::initWritingJobs()
+void DistributedBlockOutputStream::initWritingJobs(const Block & first_block)
 {
     const auto & addresses_with_failovers = cluster->getShardsAddresses();
     const auto & shards_info = cluster->getShardsInfo();
+    size_t num_shards = shards_info.size();
 
     remote_jobs_count = 0;
     local_jobs_count = 0;
@@ -145,7 +146,7 @@ void DistributedBlockOutputStream::initWritingJobs()
             {
                 if (!replicas[replica_index].is_local)
                 {
-                    shard_jobs.emplace_back(shard_index, replica_index, false);
+                    shard_jobs.replicas_jobs.emplace_back(shard_index, replica_index, false, first_block);
                     ++remote_jobs_count;
 
                     if (shard_info.hasInternalReplication())
@@ -156,9 +157,12 @@ void DistributedBlockOutputStream::initWritingJobs()
 
         if (shard_info.isLocal())
         {
-            shard_jobs.emplace_back(shard_index, 0, true);
+            shard_jobs.replicas_jobs.emplace_back(shard_index, 0, true, first_block);
             ++local_jobs_count;
         }
+
+        if (num_shards > 1)
+            shard_jobs.shard_current_block_permuation.reserve(first_block.rows());
     }
 }
 
@@ -184,18 +188,17 @@ void DistributedBlockOutputStream::waitForJobs()
 }
 
 
-ThreadPool::Job DistributedBlockOutputStream::runWritingJob(DistributedBlockOutputStream::JobInfo & job)
+ThreadPool::Job DistributedBlockOutputStream::runWritingJob(DistributedBlockOutputStream::JobReplica & job, const Block & current_block)
 {
     auto memory_tracker = current_memory_tracker;
-    return [this, memory_tracker, &job]()
+    return [this, memory_tracker, &job, &current_block]()
     {
-        SCOPE_EXIT({++finished_jobs_count;});
-
-        Stopwatch watch;
-        ++job.bloks_started;
+        ++job.blocks_started;
 
         SCOPE_EXIT({
-            UInt64 elapsed_time_for_block_ms = watch.elapsedMilliseconds();
+            ++finished_jobs_count;
+
+            UInt64 elapsed_time_for_block_ms = watch_current_block.elapsedMilliseconds();
             job.elapsed_time_ms += elapsed_time_for_block_ms;
             job.max_elapsed_time_for_block_ms = std::max(job.max_elapsed_time_for_block_ms, elapsed_time_for_block_ms);
         });
@@ -207,8 +210,30 @@ ThreadPool::Job DistributedBlockOutputStream::runWritingJob(DistributedBlockOutp
         }
 
         const auto & shard_info = cluster->getShardsInfo()[job.shard_index];
+        size_t num_shards = cluster->getShardsInfo().size();
+        auto & shard_job = per_shard_jobs[job.shard_index];
         const auto & addresses = cluster->getShardsAddresses();
-        Block & block = current_blocks.at(job.shard_index);
+
+        /// Generate current shard block
+        if (num_shards > 1)
+        {
+            auto & shard_permutation = shard_job.shard_current_block_permuation;
+            size_t num_shard_rows = shard_permutation.size();
+
+            for (size_t j = 0; j < current_block.columns(); ++j)
+            {
+                auto & src_column = current_block.getByPosition(j).column;
+                auto & dst_column = job.current_shard_block.getByPosition(j).column;
+
+                /// Zero permutation size has special meaning in IColumn::permute
+                if (num_shard_rows)
+                    dst_column = src_column->permute(shard_permutation, num_shard_rows);
+                else
+                    dst_column = src_column->cloneEmpty();
+            }
+        }
+
+        const Block & shard_block = (num_shards > 1) ? job.current_shard_block : current_block;
 
         if (!job.is_local_job)
         {
@@ -217,7 +242,7 @@ ThreadPool::Job DistributedBlockOutputStream::runWritingJob(DistributedBlockOutp
                 if (shard_info.hasInternalReplication())
                 {
                     /// Skip replica_index in case of internal replication
-                    if (per_shard_jobs[job.shard_index].size() != 1)
+                    if (shard_job.replicas_jobs.size() != 1)
                         throw Exception("There are several writing job for an automatically replicated shard", ErrorCodes::LOGICAL_ERROR);
 
                     /// TODO: it make sense to rewrite skip_unavailable_shards and max_parallel_replicas here
@@ -248,7 +273,7 @@ ThreadPool::Job DistributedBlockOutputStream::runWritingJob(DistributedBlockOutp
             }
 
             CurrentMetrics::Increment metric_increment{CurrentMetrics::DistributedSend};
-            job.stream->write(block);
+            job.stream->write(shard_block);
         }
         else
         {
@@ -265,21 +290,25 @@ ThreadPool::Job DistributedBlockOutputStream::runWritingJob(DistributedBlockOutp
 
             size_t num_repetitions = shard_info.getLocalNodeCount();
             for (size_t i = 0; i < num_repetitions; ++i)
-                job.stream->write(block);
+                job.stream->write(shard_block);
         }
 
         job.blocks_written += 1;
-        job.rows_written += block.rows();
+        job.rows_written += shard_block.rows();
     };
 }
 
 
 void DistributedBlockOutputStream::writeSync(const Block & block)
 {
+    const auto & shards_info = cluster->getShardsInfo();
+    size_t num_shards = shards_info.size();
+
     if (!pool)
     {
         /// Deferred initialization. Only for sync insertion.
-        initWritingJobs();
+        initWritingJobs(block);
+
         pool.emplace(remote_jobs_count + local_jobs_count);
         query_string = queryToString(query_ast);
 
@@ -292,14 +321,25 @@ void DistributedBlockOutputStream::writeSync(const Block & block)
         watch.restart();
     }
 
-    const auto & shards_info = cluster->getShardsInfo();
-    current_blocks = shards_info.size() > 1 ? splitBlock(block) : Blocks({block});
+    watch_current_block.restart();
+
+    if (num_shards > 1)
+    {
+        auto current_selector = createSelector(block);
+
+        /// Prepare row numbers for each shard
+        for (size_t shard_index : ext::range(0, num_shards))
+            per_shard_jobs[shard_index].shard_current_block_permuation.resize(0);
+
+        for (size_t i = 0; i < block.rows(); ++i)
+            per_shard_jobs[current_selector[i]].shard_current_block_permuation.push_back(i);
+    }
 
     /// Run jobs in parallel for each block and wait them
     finished_jobs_count = 0;
-    for (size_t shard_index : ext::range(0, current_blocks.size()))
-        for (JobInfo & job : per_shard_jobs.at(shard_index))
-            pool->schedule(runWritingJob(job));
+    for (size_t shard_index : ext::range(0, shards_info.size()))
+        for (JobReplica & job : per_shard_jobs[shard_index].replicas_jobs)
+            pool->schedule(runWritingJob(job, block));
 
     try
     {
@@ -311,7 +351,8 @@ void DistributedBlockOutputStream::writeSync(const Block & block)
         throw;
     }
 
-    ++blocks_inserted;
+    inserted_blocks += 1;
+    inserted_rows += block.rows();
 }
 
 
@@ -321,7 +362,7 @@ void DistributedBlockOutputStream::writeSuffix()
     {
         finished_jobs_count = 0;
         for (auto & shard_jobs : per_shard_jobs)
-            for (JobInfo & job : shard_jobs)
+            for (JobReplica & job : shard_jobs.replicas_jobs)
             {
                 if (job.stream)
                     pool->schedule([&job] () { job.stream->writeSuffix(); });
@@ -338,17 +379,19 @@ void DistributedBlockOutputStream::writeSuffix()
         }
 
         double elapsed = watch.elapsedSeconds();
-        LOG_DEBUG(log, "It took " << std::fixed << std::setprecision(1) << elapsed << " sec. to insert " << blocks_inserted << " blocks"
-                       << " (average " << std::fixed << std::setprecision(1) << elapsed / blocks_inserted * 1000 << " ms. per block)"
+        LOG_DEBUG(log, "It took " << std::fixed << std::setprecision(1) << elapsed << " sec. to insert " << inserted_blocks << " blocks"
+                       << ", " << std::fixed << std::setprecision(1) << inserted_rows / elapsed << " rows per second"
                        << ". " << getCurrentStateDescription());
     }
 }
 
 
-IColumn::Selector DistributedBlockOutputStream::createSelector(Block block)
+IColumn::Selector DistributedBlockOutputStream::createSelector(const Block & source_block)
 {
-    storage.getShardingKeyExpr()->execute(block);
-    const auto & key_column = block.getByName(storage.getShardingKeyColumnName());
+    Block current_block_with_sharding_key_expr = source_block;
+    storage.getShardingKeyExpr()->execute(current_block_with_sharding_key_expr);
+
+    const auto & key_column = current_block_with_sharding_key_expr.getByName(storage.getShardingKeyColumnName());
     const auto & slot_to_shard = cluster->getSlotToShard();
 
 #define CREATE_FOR_TYPE(TYPE) \
@@ -372,12 +415,6 @@ IColumn::Selector DistributedBlockOutputStream::createSelector(Block block)
 
 Blocks DistributedBlockOutputStream::splitBlock(const Block & block)
 {
-    const auto num_cols = block.columns();
-    /// cache column pointers for later reuse
-    std::vector<const IColumn *> columns(num_cols);
-    for (size_t i = 0; i < columns.size(); ++i)
-        columns[i] = block.safeGetByPosition(i).column.get();
-
     auto selector = createSelector(block);
 
     /// Split block to num_shard smaller block, using 'selector'.
@@ -409,7 +446,7 @@ void DistributedBlockOutputStream::writeSplitAsync(const Block & block)
         if (splitted_blocks[shard_idx].rows())
             writeAsyncImpl(splitted_blocks[shard_idx], shard_idx);
 
-    ++blocks_inserted;
+    ++inserted_blocks;
 }
 
 
diff --git a/dbms/src/Storages/Distributed/DistributedBlockOutputStream.h b/dbms/src/Storages/Distributed/DistributedBlockOutputStream.h
index eed38adf417..fe9e765021b 100644
--- a/dbms/src/Storages/Distributed/DistributedBlockOutputStream.h
+++ b/dbms/src/Storages/Distributed/DistributedBlockOutputStream.h
@@ -44,7 +44,7 @@ public:
 
 private:
 
-    IColumn::Selector createSelector(Block block);
+    IColumn::Selector createSelector(const Block & source_block);
 
 
     void writeAsync(const Block & block);
@@ -65,10 +65,10 @@ private:
     /// Performs synchronous insertion to remote nodes. If timeout_exceeded flag was set, throws.
     void writeSync(const Block & block);
 
-    void initWritingJobs();
+    void initWritingJobs(const Block & first_block);
 
-    struct JobInfo;
-    ThreadPool::Job runWritingJob(JobInfo & job);
+    struct JobReplica;
+    ThreadPool::Job runWritingJob(JobReplica & job, const Block & current_block);
 
     void waitForJobs();
 
@@ -80,27 +80,31 @@ private:
     ASTPtr query_ast;
     ClusterPtr cluster;
     const Settings & settings;
-    size_t blocks_inserted = 0;
+    size_t inserted_blocks = 0;
+    size_t inserted_rows = 0;
 
     bool insert_sync;
 
     /// Sync-related stuff
     UInt64 insert_timeout; // in seconds
     Stopwatch watch;
+    Stopwatch watch_current_block;
     std::optional<ThreadPool> pool;
     ThrottlerPtr throttler;
     String query_string;
 
-    struct JobInfo
+    struct JobReplica
     {
-        JobInfo() = default;
-        JobInfo(size_t shard_index, size_t replica_index, bool is_local_job)
-            : shard_index(shard_index), replica_index(replica_index), is_local_job(is_local_job) {}
+        JobReplica() = default;
+        JobReplica(size_t shard_index, size_t replica_index, bool is_local_job, const Block & sample_block)
+            : shard_index(shard_index), replica_index(replica_index), is_local_job(is_local_job), current_shard_block(sample_block.cloneEmpty()) {}
 
         size_t shard_index = 0;
         size_t replica_index = 0;
         bool is_local_job = false;
 
+        Block current_shard_block;
+
         ConnectionPool::Entry connection_entry;
         std::unique_ptr<Context> local_context;
         BlockOutputStreamPtr stream;
@@ -108,13 +112,18 @@ private:
         UInt64 blocks_written = 0;
         UInt64 rows_written = 0;
 
-        UInt64 bloks_started = 0;
+        UInt64 blocks_started = 0;
         UInt64 elapsed_time_ms = 0;
         UInt64 max_elapsed_time_for_block_ms = 0;
     };
 
-    std::vector<std::list<JobInfo>> per_shard_jobs;
-    Blocks current_blocks;
+    struct JobShard
+    {
+        std::list<JobReplica> replicas_jobs;
+        IColumn::Permutation shard_current_block_permuation;
+    };
+
+    std::vector<JobShard> per_shard_jobs;
 
     size_t remote_jobs_count = 0;
     size_t local_jobs_count = 0;
diff --git a/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp
index b5095e79118..0cd903038ed 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp
@@ -216,7 +216,7 @@ Block MergeTreeBaseBlockInputStream::readFromPart()
             }
 
             progressImpl({ res.rows(), res.bytes() });
-            pre_reader->fillMissingColumns(res, task->ordered_names, task->should_reorder);
+            pre_reader->fillMissingColumns(res, task->ordered_names, task->should_reorder, res.rows());
 
             /// Compute the expression in PREWHERE.
             prewhere_actions->execute(res);
@@ -396,7 +396,7 @@ Block MergeTreeBaseBlockInputStream::readFromPart()
                 ///  as a result of reading components of Nested data structures with no data in filesystem.
                 /// We must fill these arrays to filter them correctly.
 
-                reader->fillMissingColumns(res, task->ordered_names, task->should_reorder);
+                reader->fillMissingColumns(res, task->ordered_names, task->should_reorder, post_filter.size(), true);
 
                 /// Filter the columns related to PREWHERE using pre_filter,
                 ///  other columns - using post_filter.
@@ -422,7 +422,7 @@ Block MergeTreeBaseBlockInputStream::readFromPart()
                 if (task->size_predictor)
                     task->size_predictor->update(res);
 
-                reader->fillMissingColumns(res, task->ordered_names, true);
+                reader->fillMissingColumns(res, task->ordered_names, true, res.rows());
 
                 res.checkNumberOfRows();
             }
@@ -464,7 +464,7 @@ Block MergeTreeBaseBlockInputStream::readFromPart()
             return res;
 
         progressImpl({ res.rows(), res.bytes() });
-        reader->fillMissingColumns(res, task->ordered_names, task->should_reorder);
+        reader->fillMissingColumns(res, task->ordered_names, task->should_reorder, res.rows());
     }
 
     return res;
diff --git a/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp
index 5d52eedfda1..a2bfffabeb2 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp
@@ -102,8 +102,6 @@ void MergeTreeBlockSizePredictor::initialize(const Block & sample_block, const N
         const String & column_name = column_with_type_and_name.name;
         const ColumnPtr & column_data = column_with_type_and_name.column;
 
-        const auto column_checksum = data_part->tryGetBinChecksum(column_name);
-
         if (!from_update && !names_set.count(column_name))
             continue;
 
@@ -122,8 +120,11 @@ void MergeTreeBlockSizePredictor::initialize(const Block & sample_block, const N
             ColumnInfo info;
             info.name = column_name;
             /// If column isn't fixed and doesn't have checksum, than take first
-            info.bytes_per_row_global = column_checksum
-                ? column_checksum->uncompressed_size / number_of_rows_in_part
+            MergeTreeDataPart::ColumnSize column_size = data_part->getColumnSize(
+                column_name, *column_with_type_and_name.type);
+
+            info.bytes_per_row_global = column_size.data_uncompressed
+                ? column_size.data_uncompressed / number_of_rows_in_part
                 : column_data->byteSize() / std::max<size_t>(1, column_data->size());
 
             dynamic_columns_infos.emplace_back(info);
diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp
index ef4b8022323..5b5a6b87c8e 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp
@@ -747,7 +747,7 @@ void MergeTreeData::removePartsFinally(const MergeTreeData::DataPartsVector & pa
         for (auto & part : parts)
         {
             part_log_elem.part_name = part->name;
-            part_log_elem.bytes_compressed_on_disk = part->size_in_bytes;
+            part_log_elem.bytes_compressed_on_disk = part->bytes_on_disk;
             part_log_elem.rows = part->rows_count;
 
             part_log->add(part_log_elem);
@@ -1211,8 +1211,8 @@ MergeTreeData::AlterDataPartTransactionPtr MergeTreeData::alterDataPart(
             false, nullptr, "", false, 0, DBMS_DEFAULT_BUFFER_SIZE, false);
 
         auto compression_settings = this->context.chooseCompressionSettings(
-            part->size_in_bytes,
-            static_cast<double>(part->size_in_bytes) / this->getTotalActiveSizeInBytes());
+            part->bytes_on_disk,
+            static_cast<double>(part->bytes_on_disk) / this->getTotalActiveSizeInBytes());
         ExpressionBlockInputStream in(part_in, expression);
 
         /** Don't write offsets for arrays, because ALTER never change them
@@ -1314,7 +1314,7 @@ void MergeTreeData::AlterDataPartTransaction::commit()
                 file.remove();
         }
 
-        mutable_part.size_in_bytes = MergeTreeData::DataPart::calculateTotalSize(path);
+        mutable_part.bytes_on_disk = MergeTreeData::DataPart::calculateTotalSizeOnDisk(path);
 
         /// TODO: we can skip resetting caches when the column is added.
         data_part->storage.context.dropCaches();
@@ -1674,7 +1674,7 @@ size_t MergeTreeData::getTotalActiveSizeInBytes() const
         std::lock_guard<std::mutex> lock(data_parts_mutex);
 
         for (auto & part : getDataPartsStateRange(DataPartState::Committed))
-            res += part->size_in_bytes;
+            res += part->bytes_on_disk;
     }
 
     return res;
@@ -1832,59 +1832,37 @@ void MergeTreeData::calculateColumnSizesImpl()
 void MergeTreeData::addPartContributionToColumnSizes(const DataPartPtr & part)
 {
     std::shared_lock<std::shared_mutex> lock(part->columns_lock);
-    const auto & files = part->checksums.files;
 
-    /// TODO This method doesn't take into account columns with multiple files.
-    for (const auto & column : getColumns().getAllPhysical())
+    for (const auto & column : part->columns)
     {
-        const auto escaped_name = escapeForFileName(column.name);
-        const auto bin_file_name = escaped_name + ".bin";
-        const auto mrk_file_name = escaped_name + ".mrk";
-
-        ColumnSize & column_size = column_sizes[column.name];
-
-        if (files.count(bin_file_name))
-        {
-            const auto & bin_file_checksums = files.at(bin_file_name);
-            column_size.data_compressed += bin_file_checksums.file_size;
-            column_size.data_uncompressed += bin_file_checksums.uncompressed_size;
-        }
-
-        if (files.count(mrk_file_name))
-            column_size.marks += files.at(mrk_file_name).file_size;
+        DataPart::ColumnSize & total_column_size = column_sizes[column.name];
+        DataPart::ColumnSize part_column_size = part->getColumnSize(column.name, *column.type);
+        total_column_size.add(part_column_size);
     }
 }
 
-static inline void logSubtract(size_t & from, size_t value, Logger * log, const String & variable)
-{
-    if (value > from)
-        LOG_ERROR(log, "Possibly incorrect subtraction: " << from << " - " << value << " = " << from - value << ", variable " << variable);
-
-    from -= value;
-}
-
 void MergeTreeData::removePartContributionToColumnSizes(const DataPartPtr & part)
 {
-    const auto & files = part->checksums.files;
+    std::shared_lock<std::shared_mutex> lock(part->columns_lock);
 
-    /// TODO This method doesn't take into account columns with multiple files.
-    for (const auto & column : getColumns().getAllPhysical())
+    for (const auto & column : part->columns)
     {
-        const auto escaped_name = escapeForFileName(column.name);
-        const auto bin_file_name = escaped_name + ".bin";
-        const auto mrk_file_name = escaped_name + ".mrk";
+        DataPart::ColumnSize & total_column_size = column_sizes[column.name];
+        DataPart::ColumnSize part_column_size = part->getColumnSize(column.name, *column.type);
 
-        auto & column_size = column_sizes[column.name];
-
-        if (files.count(bin_file_name))
+        auto log_subtract = [&](size_t & from, size_t value, const char * field)
         {
-            const auto & bin_file_checksums = files.at(bin_file_name);
-            logSubtract(column_size.data_compressed, bin_file_checksums.file_size, log, bin_file_name + ".file_size");
-            logSubtract(column_size.data_uncompressed, bin_file_checksums.uncompressed_size, log, bin_file_name + ".uncompressed_size");
-        }
+            if (value > from)
+                LOG_ERROR(log, "Possibly incorrect column size subtraction: "
+                    << from << " - " << value << " = " << from - value
+                    << ", column: " << column.name << ", field: " << field);
 
-        if (files.count(mrk_file_name))
-            logSubtract(column_size.marks, files.at(mrk_file_name).file_size, log, mrk_file_name + ".file_size");
+            from -= value;
+        };
+
+        log_subtract(total_column_size.data_compressed, part_column_size.data_compressed, ".data_compressed");
+        log_subtract(total_column_size.data_uncompressed, part_column_size.data_uncompressed, ".data_uncompressed");
+        log_subtract(total_column_size.marks, part_column_size.marks, ".marks");
     }
 }
 
diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h
index 47abd959645..f412419459c 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeData.h
+++ b/dbms/src/Storages/MergeTree/MergeTreeData.h
@@ -452,18 +452,6 @@ public:
     /// Returns the size of partition in bytes.
     size_t getPartitionSize(const std::string & partition_id) const;
 
-    struct ColumnSize
-    {
-        size_t marks = 0;
-        size_t data_compressed = 0;
-        size_t data_uncompressed = 0;
-
-        size_t getTotalCompressedSize() const
-        {
-            return marks + data_compressed;
-        }
-    };
-
     size_t getColumnCompressedSize(const std::string & name) const
     {
         std::lock_guard<std::mutex> lock{data_parts_mutex};
@@ -472,25 +460,13 @@ public:
         return it == std::end(column_sizes) ? 0 : it->second.data_compressed;
     }
 
-    using ColumnSizes = std::unordered_map<std::string, ColumnSize>;
-    ColumnSizes getColumnSizes() const
+    using ColumnSizeByName = std::unordered_map<std::string, DataPart::ColumnSize>;
+    ColumnSizeByName getColumnSizes() const
     {
         std::lock_guard<std::mutex> lock{data_parts_mutex};
         return column_sizes;
     }
 
-    /// NOTE Could be off after DROPped and MODIFYed columns in ALTER. Doesn't include primary.idx.
-    size_t getTotalCompressedSize() const
-    {
-        std::lock_guard<std::mutex> lock{data_parts_mutex};
-        size_t total_size = 0;
-
-        for (const auto & col : column_sizes)
-            total_size += col.second.getTotalCompressedSize();
-
-        return total_size;
-    }
-
     /// Calculates column sizes in compressed form for the current state of data_parts.
     void recalculateColumnSizes()
     {
@@ -555,7 +531,7 @@ private:
     String full_path;
 
     /// Current column sizes in compressed and uncompressed form.
-    ColumnSizes column_sizes;
+    ColumnSizeByName column_sizes;
 
     /// Engine-specific methods
     BrokenPartCallback broken_part_callback;
diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp
index ae8677c57be..f2d6e773be1 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp
@@ -180,7 +180,7 @@ bool MergeTreeDataMerger::selectPartsToMerge(
         }
 
         IMergeSelector::Part part_info;
-        part_info.size = part->size_in_bytes;
+        part_info.size = part->bytes_on_disk;
         part_info.age = current_time - part->modification_time;
         part_info.level = part->info.level;
         part_info.data = &part;
@@ -266,7 +266,7 @@ bool MergeTreeDataMerger::selectAllPartsToMergeWithinPartition(
             return false;
         }
 
-        sum_bytes += (*it)->size_in_bytes;
+        sum_bytes += (*it)->bytes_on_disk;
 
         prev_it = it;
         ++it;
@@ -532,7 +532,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMerger::mergePartsToTemporaryPart
     {
         std::shared_lock<std::shared_mutex> part_lock(part->columns_lock);
 
-        merge_entry->total_size_bytes_compressed += part->size_in_bytes;
+        merge_entry->total_size_bytes_compressed += part->bytes_on_disk;
         merge_entry->total_size_marks += part->marks_count;
     }
 
@@ -883,7 +883,7 @@ size_t MergeTreeDataMerger::estimateDiskSpaceForMerge(const MergeTreeData::DataP
 {
     size_t res = 0;
     for (const MergeTreeData::DataPartPtr & part : parts)
-        res += part->size_in_bytes;
+        res += part->bytes_on_disk;
 
     return static_cast<size_t>(res * DISK_USAGE_COEFFICIENT_TO_RESERVE);
 }
diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp
index 8ab7699a03b..26f7c2c4493 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp
@@ -30,273 +30,15 @@ namespace DB
 
 namespace ErrorCodes
 {
-    extern const int CHECKSUM_DOESNT_MATCH;
     extern const int FILE_DOESNT_EXIST;
     extern const int NO_FILE_IN_DATA_PART;
     extern const int EXPECTED_END_OF_FILE;
-    extern const int BAD_SIZE_OF_FILE_IN_DATA_PART;
     extern const int CORRUPTED_DATA;
-    extern const int FORMAT_VERSION_TOO_OLD;
-    extern const int UNKNOWN_FORMAT;
-    extern const int UNEXPECTED_FILE_IN_DATA_PART;
     extern const int NOT_FOUND_EXPECTED_DATA_PART;
+    extern const int BAD_SIZE_OF_FILE_IN_DATA_PART;
 }
 
 
-void MergeTreeDataPartChecksum::checkEqual(const MergeTreeDataPartChecksum & rhs, bool have_uncompressed, const String & name) const
-{
-    if (is_compressed && have_uncompressed)
-    {
-        if (!rhs.is_compressed)
-            throw Exception("No uncompressed checksum for file " + name, ErrorCodes::CHECKSUM_DOESNT_MATCH);
-        if (rhs.uncompressed_size != uncompressed_size)
-            throw Exception("Unexpected uncompressed size of file " + name + " in data part", ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART);
-        if (rhs.uncompressed_hash != uncompressed_hash)
-            throw Exception("Checksum mismatch for uncompressed file " + name + " in data part", ErrorCodes::CHECKSUM_DOESNT_MATCH);
-        return;
-    }
-    if (rhs.file_size != file_size)
-        throw Exception("Unexpected size of file " + name + " in data part", ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART);
-    if (rhs.file_hash != file_hash)
-        throw Exception("Checksum mismatch for file " + name + " in data part", ErrorCodes::CHECKSUM_DOESNT_MATCH);
-}
-
-void MergeTreeDataPartChecksum::checkSize(const String & path) const
-{
-    Poco::File file(path);
-    if (!file.exists())
-        throw Exception(path + " doesn't exist", ErrorCodes::FILE_DOESNT_EXIST);
-    UInt64 size = file.getSize();
-    if (size != file_size)
-        throw Exception(path + " has unexpected size: " + toString(size) + " instead of " + toString(file_size),
-            ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART);
-}
-
-
-void MergeTreeDataPartChecksums::checkEqual(const MergeTreeDataPartChecksums & rhs, bool have_uncompressed) const
-{
-    for (const auto & it : rhs.files)
-    {
-        const String & name = it.first;
-
-        if (!files.count(name))
-            throw Exception("Unexpected file " + name + " in data part", ErrorCodes::UNEXPECTED_FILE_IN_DATA_PART);
-    }
-
-    for (const auto & it : files)
-    {
-        const String & name = it.first;
-
-        auto jt = rhs.files.find(name);
-        if (jt == rhs.files.end())
-            throw Exception("No file " + name + " in data part", ErrorCodes::NO_FILE_IN_DATA_PART);
-
-        it.second.checkEqual(jt->second, have_uncompressed, name);
-    }
-}
-
-void MergeTreeDataPartChecksums::checkSizes(const String & path) const
-{
-    for (const auto & it : files)
-    {
-        const String & name = it.first;
-        it.second.checkSize(path + name);
-    }
-}
-
-bool MergeTreeDataPartChecksums::read(ReadBuffer & in)
-{
-    files.clear();
-
-    assertString("checksums format version: ", in);
-    int format_version;
-    readText(format_version, in);
-    assertChar('\n', in);
-
-    switch (format_version)
-    {
-        case 1:
-            return false;
-        case 2:
-            return read_v2(in);
-        case 3:
-            return read_v3(in);
-        case 4:
-            return read_v4(in);
-        default:
-            throw Exception("Bad checksums format version: " + DB::toString(format_version), ErrorCodes::UNKNOWN_FORMAT);
-    }
-}
-
-bool MergeTreeDataPartChecksums::read_v2(ReadBuffer & in)
-{
-    size_t count;
-
-    readText(count, in);
-    assertString(" files:\n", in);
-
-    for (size_t i = 0; i < count; ++i)
-    {
-        String name;
-        Checksum sum;
-
-        readString(name, in);
-        assertString("\n\tsize: ", in);
-        readText(sum.file_size, in);
-        assertString("\n\thash: ", in);
-        readText(sum.file_hash.first, in);
-        assertString(" ", in);
-        readText(sum.file_hash.second, in);
-        assertString("\n\tcompressed: ", in);
-        readText(sum.is_compressed, in);
-        if (sum.is_compressed)
-        {
-            assertString("\n\tuncompressed size: ", in);
-            readText(sum.uncompressed_size, in);
-            assertString("\n\tuncompressed hash: ", in);
-            readText(sum.uncompressed_hash.first, in);
-            assertString(" ", in);
-            readText(sum.uncompressed_hash.second, in);
-        }
-        assertChar('\n', in);
-
-        files.insert(std::make_pair(name, sum));
-    }
-
-    return true;
-}
-
-bool MergeTreeDataPartChecksums::read_v3(ReadBuffer & in)
-{
-    size_t count;
-
-    readVarUInt(count, in);
-
-    for (size_t i = 0; i < count; ++i)
-    {
-        String name;
-        Checksum sum;
-
-        readBinary(name, in);
-        readVarUInt(sum.file_size, in);
-        readPODBinary(sum.file_hash, in);
-        readBinary(sum.is_compressed, in);
-
-        if (sum.is_compressed)
-        {
-            readVarUInt(sum.uncompressed_size, in);
-            readPODBinary(sum.uncompressed_hash, in);
-        }
-
-        files.emplace(std::move(name), sum);
-    }
-
-    return true;
-}
-
-bool MergeTreeDataPartChecksums::read_v4(ReadBuffer & from)
-{
-    CompressedReadBuffer in{from};
-    return read_v3(in);
-}
-
-void MergeTreeDataPartChecksums::write(WriteBuffer & to) const
-{
-    writeString("checksums format version: 4\n", to);
-
-    CompressedWriteBuffer out{to, CompressionSettings(CompressionMethod::LZ4), 1 << 16};
-    writeVarUInt(files.size(), out);
-
-    for (const auto & it : files)
-    {
-        const String & name = it.first;
-        const Checksum & sum = it.second;
-
-        writeBinary(name, out);
-        writeVarUInt(sum.file_size, out);
-        writePODBinary(sum.file_hash, out);
-        writeBinary(sum.is_compressed, out);
-
-        if (sum.is_compressed)
-        {
-            writeVarUInt(sum.uncompressed_size, out);
-            writePODBinary(sum.uncompressed_hash, out);
-        }
-    }
-}
-
-void MergeTreeDataPartChecksums::addFile(const String & file_name, UInt64 file_size, MergeTreeDataPartChecksum::uint128 file_hash)
-{
-    files[file_name] = Checksum(file_size, file_hash);
-}
-
-void MergeTreeDataPartChecksums::add(MergeTreeDataPartChecksums && rhs_checksums)
-{
-    for (auto & checksum : rhs_checksums.files)
-        files[std::move(checksum.first)] = std::move(checksum.second);
-
-    rhs_checksums.files.clear();
-}
-
-/// Checksum computed from the set of control sums of .bin files.
-void MergeTreeDataPartChecksums::summaryDataChecksum(SipHash & hash) const
-{
-    /// We use fact that iteration is in deterministic (lexicographical) order.
-    for (const auto & it : files)
-    {
-        const String & name = it.first;
-        const Checksum & sum = it.second;
-
-        if (!endsWith(name, ".bin"))
-            continue;
-
-        UInt64 len = name.size();
-        hash.update(len);
-        hash.update(name.data(), len);
-        hash.update(sum.uncompressed_size);
-        hash.update(sum.uncompressed_hash);
-    }
-}
-
-String MergeTreeDataPartChecksums::toString() const
-{
-    WriteBufferFromOwnString out;
-    write(out);
-    return out.str();
-}
-
-MergeTreeDataPartChecksums MergeTreeDataPartChecksums::parse(const String & s)
-{
-    ReadBufferFromString in(s);
-    MergeTreeDataPartChecksums res;
-    if (!res.read(in))
-        throw Exception("Checksums format is too old", ErrorCodes::FORMAT_VERSION_TOO_OLD);
-    assertEOF(in);
-    return res;
-}
-
-const MergeTreeDataPartChecksums::Checksum * MergeTreeDataPart::tryGetChecksum(const String & name, const String & ext) const
-{
-    if (checksums.empty())
-        return nullptr;
-
-    const auto & files = checksums.files;
-    const auto file_name = escapeForFileName(name) + ext;
-    auto it = files.find(file_name);
-
-    return (it == files.end()) ? nullptr : &it->second;
-}
-
-const MergeTreeDataPartChecksums::Checksum * MergeTreeDataPart::tryGetBinChecksum(const String & name) const
-{
-    return tryGetChecksum(name, ".bin");
-}
-
-const MergeTreeDataPartChecksums::Checksum * MergeTreeDataPart::tryGetMrkChecksum(const String & name) const
-{
-    return tryGetChecksum(name, ".mrk");
-}
-
 static ReadBufferFromFile openForReading(const String & path)
 {
     return ReadBufferFromFile(path, std::min(static_cast<Poco::File::FileSize>(DBMS_DEFAULT_BUFFER_SIZE), Poco::File(path).getSize()));
@@ -392,26 +134,51 @@ MergeTreeDataPart::MergeTreeDataPart(MergeTreeData & storage_, const String & na
 {
 }
 
-/// Returns the size of .bin file for column `name` if found, zero otherwise.
-UInt64 MergeTreeDataPart::getColumnCompressedSize(const String & name) const
+/// Takes into account the fact that several columns can e.g. share their .size substreams.
+/// When calculating totals these should be counted only once.
+MergeTreeDataPart::ColumnSize MergeTreeDataPart::getColumnSizeImpl(const String & name, const IDataType & type, std::unordered_set<String> * processed_substreams) const
 {
-    const Checksum * checksum = tryGetBinChecksum(name);
+    ColumnSize size;
+    if (checksums.empty())
+        return size;
 
-    /// Probably a logic error, not sure if this can ever happen if checksums are not empty
-    return checksum ? checksum->file_size : 0;
+    type.enumerateStreams([&](const IDataType::SubstreamPath & substream_path)
+    {
+        String file_name = IDataType::getFileNameForStream(name, substream_path);
+
+        if (processed_substreams && !processed_substreams->insert(file_name).second)
+            return;
+
+        auto bin_checksum = checksums.files.find(file_name + ".bin");
+        if (bin_checksum != checksums.files.end())
+        {
+            size.data_compressed += bin_checksum->second.file_size;
+            size.data_uncompressed += bin_checksum->second.uncompressed_size;
+        }
+
+        auto mrk_checksum = checksums.files.find(file_name + ".mrk");
+        if (mrk_checksum != checksums.files.end())
+            size.marks += mrk_checksum->second.file_size;
+    }, {});
+
+    return size;
 }
 
-UInt64 MergeTreeDataPart::getColumnUncompressedSize(const String & name) const
+MergeTreeDataPart::ColumnSize MergeTreeDataPart::getColumnSize(const String & name, const IDataType & type) const
 {
-    const Checksum * checksum = tryGetBinChecksum(name);
-    return checksum ? checksum->uncompressed_size : 0;
+    return getColumnSizeImpl(name, type, nullptr);
 }
 
-
-UInt64 MergeTreeDataPart::getColumnMrkSize(const String & name) const
+MergeTreeDataPart::ColumnSize MergeTreeDataPart::getTotalColumnsSize() const
 {
-    const Checksum * checksum = tryGetMrkChecksum(name);
-    return checksum ? checksum->file_size : 0;
+    ColumnSize totals;
+    std::unordered_set<String> processed_substreams;
+    for (const NameAndTypePair & column : columns)
+    {
+        ColumnSize size = getColumnSizeImpl(column.name, *column.type, &processed_substreams);
+        totals.add(size);
+    }
+    return totals;
 }
 
 
@@ -429,7 +196,7 @@ String MergeTreeDataPart::getColumnNameWithMinumumCompressedSize() const
         if (!hasColumnFiles(column.name))
             continue;
 
-        const auto size = getColumnCompressedSize(column.name);
+        const auto size = getColumnSize(column.name, *column.type).data_compressed;
         if (size < minimum_size)
         {
             minimum_size = size;
@@ -509,7 +276,7 @@ MergeTreeDataPart::~MergeTreeDataPart()
     }
 }
 
-UInt64 MergeTreeDataPart::calculateTotalSize(const String & from)
+UInt64 MergeTreeDataPart::calculateTotalSizeOnDisk(const String & from)
 {
     Poco::File cur(from);
     if (cur.isFile())
@@ -518,7 +285,7 @@ UInt64 MergeTreeDataPart::calculateTotalSize(const String & from)
     cur.list(files);
     UInt64 res = 0;
     for (const auto & file : files)
-        res += calculateTotalSize(from + file);
+        res += calculateTotalSizeOnDisk(from + file);
     return res;
 }
 
@@ -678,7 +445,7 @@ void MergeTreeDataPart::loadIndex()
         index.assign(std::make_move_iterator(loaded_index.begin()), std::make_move_iterator(loaded_index.end()));
     }
 
-    size_in_bytes = calculateTotalSize(getFullPath());
+    bytes_on_disk = calculateTotalSizeOnDisk(getFullPath());
 }
 
 void MergeTreeDataPart::loadPartitionAndMinMaxIndex()
@@ -742,20 +509,21 @@ void MergeTreeDataPart::loadRowsCount()
         for (const NameAndTypePair & column : columns)
         {
             ColumnPtr column_col = column.type->createColumn();
-            const auto checksum = tryGetBinChecksum(column.name);
+            if (!column_col->isFixedAndContiguous())
+                continue;
 
-            /// Should be fixed non-nullable column
-            if (!checksum || !column_col->isFixedAndContiguous())
+            size_t column_size = getColumnSize(column.name, *column.type).data_uncompressed;
+            if (!column_size)
                 continue;
 
             size_t sizeof_field = column_col->sizeOfValueIfFixed();
-            rows_count = checksum->uncompressed_size / sizeof_field;
+            rows_count = column_size / sizeof_field;
 
-            if (checksum->uncompressed_size % sizeof_field != 0)
+            if (column_size % sizeof_field != 0)
             {
                 throw Exception(
-                    "Column " + column.name + " has indivisible uncompressed size " + toString(checksum->uncompressed_size)
-                    + ", sizeof " + toString(sizeof_field),
+                    "Uncompressed size of column " + column.name + "(" + toString(column_size)
+                    + ") is not divisible by the size of value (" + toString(sizeof_field) + ")",
                     ErrorCodes::LOGICAL_ERROR);
             }
 
@@ -944,18 +712,6 @@ UInt64 MergeTreeDataPart::getIndexSizeInAllocatedBytes() const
     return res;
 }
 
-UInt64 MergeTreeDataPart::getTotalMrkSizeInBytes() const
-{
-    UInt64 res = 0;
-    for (const NameAndTypePair & it : columns)
-    {
-        const Checksum * checksum = tryGetMrkChecksum(it.name);
-        if (checksum)
-            res += checksum->file_size;
-    }
-    return res;
-}
-
 String MergeTreeDataPart::stateToString(MergeTreeDataPart::State state)
 {
     switch (state)
diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.h b/dbms/src/Storages/MergeTree/MergeTreeDataPart.h
index 95bed39eaec..c1c1274f7b9 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.h
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.h
@@ -5,82 +5,14 @@
 #include <Core/NamesAndTypes.h>
 #include <Storages/MergeTree/MergeTreePartInfo.h>
 #include <Storages/MergeTree/MergeTreePartition.h>
+#include <Storages/MergeTree/MergeTreeDataPartChecksum.h>
 #include <Columns/IColumn.h>
 #include <shared_mutex>
 
 
-class SipHash;
-
-
 namespace DB
 {
 
-
-/// Checksum of one file.
-struct MergeTreeDataPartChecksum
-{
-    using uint128 = CityHash_v1_0_2::uint128;
-
-    UInt64 file_size {};
-    uint128 file_hash {};
-
-    bool is_compressed = false;
-    UInt64 uncompressed_size {};
-    uint128 uncompressed_hash {};
-
-    MergeTreeDataPartChecksum() {}
-    MergeTreeDataPartChecksum(UInt64 file_size_, uint128 file_hash_) : file_size(file_size_), file_hash(file_hash_) {}
-    MergeTreeDataPartChecksum(UInt64 file_size_, uint128 file_hash_, UInt64 uncompressed_size_, uint128 uncompressed_hash_)
-        : file_size(file_size_), file_hash(file_hash_), is_compressed(true),
-        uncompressed_size(uncompressed_size_), uncompressed_hash(uncompressed_hash_) {}
-
-    void checkEqual(const MergeTreeDataPartChecksum & rhs, bool have_uncompressed, const String & name) const;
-    void checkSize(const String & path) const;
-};
-
-
-/** Checksums of all non-temporary files.
-  * For compressed files, the check sum and the size of the decompressed data are stored to not depend on the compression method.
-  */
-struct MergeTreeDataPartChecksums
-{
-    using Checksum = MergeTreeDataPartChecksum;
-
-    /// The order is important.
-    using FileChecksums = std::map<String, Checksum>;
-    FileChecksums files;
-
-    void addFile(const String & file_name, UInt64 file_size, Checksum::uint128 file_hash);
-
-    void add(MergeTreeDataPartChecksums && rhs_checksums);
-
-    /// Checks that the set of columns and their checksums are the same. If not, throws an exception.
-    /// If have_uncompressed, for compressed files it compares the checksums of the decompressed data. Otherwise, it compares only the checksums of the files.
-    void checkEqual(const MergeTreeDataPartChecksums & rhs, bool have_uncompressed) const;
-
-    /// Checks that the directory contains all the needed files of the correct size. Does not check the checksum.
-    void checkSizes(const String & path) const;
-
-    /// Serializes and deserializes in human readable form.
-    bool read(ReadBuffer & in); /// Returns false if the checksum is too old.
-    bool read_v2(ReadBuffer & in);
-    bool read_v3(ReadBuffer & in);
-    bool read_v4(ReadBuffer & in);
-    void write(WriteBuffer & out) const;
-
-    bool empty() const
-    {
-        return files.empty();
-    }
-
-    /// Checksum from the set of checksums of .bin files.
-    void summaryDataChecksum(SipHash & hash) const;
-
-    String toString() const;
-    static MergeTreeDataPartChecksums parse(const String & s);
-};
-
-
 class MergeTreeData;
 
 
@@ -97,22 +29,31 @@ struct MergeTreeDataPart
 
     MergeTreeDataPart(MergeTreeData & storage_, const String & name_);
 
-    const Checksum * tryGetChecksum(const String & name, const String & ext) const;
-    /// Returns checksum of column's binary file.
-    const Checksum * tryGetBinChecksum(const String & name) const;
-    /// Returns checksum of column's mrk file.
-    const Checksum * tryGetMrkChecksum(const String & name) const;
-
-    /// Returns the size of .bin file for column `name` if found, zero otherwise
-    UInt64 getColumnCompressedSize(const String & name) const;
-    UInt64 getColumnUncompressedSize(const String & name) const;
-    /// Returns the size of .mrk file for column `name` if found, zero otherwise
-    UInt64 getColumnMrkSize(const String & name) const;
-
     /// Returns the name of a column with minimum compressed size (as returned by getColumnSize()).
     /// If no checksums are present returns the name of the first physically existing column.
     String getColumnNameWithMinumumCompressedSize() const;
 
+    struct ColumnSize
+    {
+        size_t marks = 0;
+        size_t data_compressed = 0;
+        size_t data_uncompressed = 0;
+
+        void add(const ColumnSize & other)
+        {
+            marks += other.marks;
+            data_compressed += other.data_compressed;
+            data_uncompressed += other.data_uncompressed;
+        }
+    };
+
+    /// NOTE: Returns zeros if column files are not found in checksums.
+    /// NOTE: You must ensure that no ALTERs are in progress when calculating ColumnSizes.
+    ///   (either by locking columns_lock, or by locking table structure).
+    ColumnSize getColumnSize(const String & name, const IDataType & type) const;
+
+    ColumnSize getTotalColumnsSize() const;
+
     /// Returns full path to part dir
     String getFullPath() const;
 
@@ -136,8 +77,8 @@ struct MergeTreeDataPart
 
     size_t rows_count = 0;
     size_t marks_count = 0;
-    std::atomic<UInt64> size_in_bytes {0};  /// size in bytes, 0 - if not counted;
-                                            ///  is used from several threads without locks (it is changed with ALTER).
+    std::atomic<UInt64> bytes_on_disk {0};  /// 0 - if not counted;
+                                            /// Is used from several threads without locks (it is changed with ALTER).
     time_t modification_time = 0;
     /// When the part is removed from the working set. Changes once.
     mutable std::atomic<time_t> remove_time { std::numeric_limits<time_t>::max() };
@@ -276,7 +217,7 @@ struct MergeTreeDataPart
     ~MergeTreeDataPart();
 
     /// Calculate the total size of the entire directory with all the files
-    static UInt64 calculateTotalSize(const String & from);
+    static UInt64 calculateTotalSizeOnDisk(const String & from);
 
     void remove() const;
 
@@ -300,8 +241,6 @@ struct MergeTreeDataPart
     /// For data in RAM ('index')
     UInt64 getIndexSizeInBytes() const;
     UInt64 getIndexSizeInAllocatedBytes() const;
-    /// Total size of *.mrk files
-    UInt64 getTotalMrkSizeInBytes() const;
 
 private:
     /// Reads columns names and types from columns.txt
@@ -320,6 +259,8 @@ private:
     void loadPartitionAndMinMaxIndex();
 
     void checkConsistency(bool require_part_metadata);
+
+    ColumnSize getColumnSizeImpl(const String & name, const IDataType & type, std::unordered_set<String> * processed_substreams) const;
 };
 
 
diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp
new file mode 100644
index 00000000000..b9be8e3d9f0
--- /dev/null
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp
@@ -0,0 +1,448 @@
+#include "MergeTreeDataPartChecksum.h"
+#include <Common/SipHash.h>
+#include <IO/ReadHelpers.h>
+#include <IO/WriteHelpers.h>
+#include <IO/ReadBufferFromString.h>
+#include <IO/WriteBufferFromString.h>
+#include <IO/CompressedReadBuffer.h>
+#include <IO/CompressedWriteBuffer.h>
+#include <Poco/File.h>
+
+
+namespace DB
+{
+
+
+namespace ErrorCodes
+{
+    extern const int CHECKSUM_DOESNT_MATCH;
+    extern const int BAD_SIZE_OF_FILE_IN_DATA_PART;
+    extern const int FORMAT_VERSION_TOO_OLD;
+    extern const int FILE_DOESNT_EXIST;
+    extern const int UNEXPECTED_FILE_IN_DATA_PART;
+    extern const int UNKNOWN_FORMAT;
+    extern const int NO_FILE_IN_DATA_PART;
+}
+
+
+void MergeTreeDataPartChecksum::checkEqual(const MergeTreeDataPartChecksum & rhs, bool have_uncompressed, const String & name) const
+{
+    if (is_compressed && have_uncompressed)
+    {
+        if (!rhs.is_compressed)
+            throw Exception("No uncompressed checksum for file " + name, ErrorCodes::CHECKSUM_DOESNT_MATCH);
+        if (rhs.uncompressed_size != uncompressed_size)
+            throw Exception("Unexpected uncompressed size of file " + name + " in data part", ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART);
+        if (rhs.uncompressed_hash != uncompressed_hash)
+            throw Exception("Checksum mismatch for uncompressed file " + name + " in data part", ErrorCodes::CHECKSUM_DOESNT_MATCH);
+        return;
+    }
+    if (rhs.file_size != file_size)
+        throw Exception("Unexpected size of file " + name + " in data part", ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART);
+    if (rhs.file_hash != file_hash)
+        throw Exception("Checksum mismatch for file " + name + " in data part", ErrorCodes::CHECKSUM_DOESNT_MATCH);
+}
+
+void MergeTreeDataPartChecksum::checkSize(const String & path) const
+{
+    Poco::File file(path);
+    if (!file.exists())
+        throw Exception(path + " doesn't exist", ErrorCodes::FILE_DOESNT_EXIST);
+    UInt64 size = file.getSize();
+    if (size != file_size)
+        throw Exception(path + " has unexpected size: " + toString(size) + " instead of " + toString(file_size),
+            ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART);
+}
+
+
+void MergeTreeDataPartChecksums::checkEqual(const MergeTreeDataPartChecksums & rhs, bool have_uncompressed) const
+{
+    for (const auto & it : rhs.files)
+    {
+        const String & name = it.first;
+
+        if (!files.count(name))
+            throw Exception("Unexpected file " + name + " in data part", ErrorCodes::UNEXPECTED_FILE_IN_DATA_PART);
+    }
+
+    for (const auto & it : files)
+    {
+        const String & name = it.first;
+
+        auto jt = rhs.files.find(name);
+        if (jt == rhs.files.end())
+            throw Exception("No file " + name + " in data part", ErrorCodes::NO_FILE_IN_DATA_PART);
+
+        it.second.checkEqual(jt->second, have_uncompressed, name);
+    }
+}
+
+void MergeTreeDataPartChecksums::checkSizes(const String & path) const
+{
+    for (const auto & it : files)
+    {
+        const String & name = it.first;
+        it.second.checkSize(path + name);
+    }
+}
+
+bool MergeTreeDataPartChecksums::read(ReadBuffer & in, size_t format_version)
+{
+    switch (format_version)
+    {
+        case 1:
+            return false;
+        case 2:
+            return read_v2(in);
+        case 3:
+            return read_v3(in);
+        case 4:
+            return read_v4(in);
+        default:
+            throw Exception("Bad checksums format version: " + DB::toString(format_version), ErrorCodes::UNKNOWN_FORMAT);
+    }
+}
+
+bool MergeTreeDataPartChecksums::read(ReadBuffer & in)
+{
+    files.clear();
+
+    assertString("checksums format version: ", in);
+    size_t format_version;
+    readText(format_version, in);
+    assertChar('\n', in);
+
+    read(in, format_version);
+    return true;
+}
+
+bool MergeTreeDataPartChecksums::read_v2(ReadBuffer & in)
+{
+    size_t count;
+
+    readText(count, in);
+    assertString(" files:\n", in);
+
+    for (size_t i = 0; i < count; ++i)
+    {
+        String name;
+        Checksum sum;
+
+        readString(name, in);
+        assertString("\n\tsize: ", in);
+        readText(sum.file_size, in);
+        assertString("\n\thash: ", in);
+        readText(sum.file_hash.first, in);
+        assertString(" ", in);
+        readText(sum.file_hash.second, in);
+        assertString("\n\tcompressed: ", in);
+        readText(sum.is_compressed, in);
+        if (sum.is_compressed)
+        {
+            assertString("\n\tuncompressed size: ", in);
+            readText(sum.uncompressed_size, in);
+            assertString("\n\tuncompressed hash: ", in);
+            readText(sum.uncompressed_hash.first, in);
+            assertString(" ", in);
+            readText(sum.uncompressed_hash.second, in);
+        }
+        assertChar('\n', in);
+
+        files.insert(std::make_pair(name, sum));
+    }
+
+    return true;
+}
+
+bool MergeTreeDataPartChecksums::read_v3(ReadBuffer & in)
+{
+    size_t count;
+
+    readVarUInt(count, in);
+
+    for (size_t i = 0; i < count; ++i)
+    {
+        String name;
+        Checksum sum;
+
+        readBinary(name, in);
+        readVarUInt(sum.file_size, in);
+        readPODBinary(sum.file_hash, in);
+        readBinary(sum.is_compressed, in);
+
+        if (sum.is_compressed)
+        {
+            readVarUInt(sum.uncompressed_size, in);
+            readPODBinary(sum.uncompressed_hash, in);
+        }
+
+        files.emplace(std::move(name), sum);
+    }
+
+    return true;
+}
+
+bool MergeTreeDataPartChecksums::read_v4(ReadBuffer & from)
+{
+    CompressedReadBuffer in{from};
+    return read_v3(in);
+}
+
+void MergeTreeDataPartChecksums::write(WriteBuffer & to) const
+{
+    writeString("checksums format version: 4\n", to);
+
+    CompressedWriteBuffer out{to, CompressionSettings(CompressionMethod::LZ4), 1 << 16};
+
+    writeVarUInt(files.size(), out);
+
+    for (const auto & it : files)
+    {
+        const String & name = it.first;
+        const Checksum & sum = it.second;
+
+        writeBinary(name, out);
+        writeVarUInt(sum.file_size, out);
+        writePODBinary(sum.file_hash, out);
+        writeBinary(sum.is_compressed, out);
+
+        if (sum.is_compressed)
+        {
+            writeVarUInt(sum.uncompressed_size, out);
+            writePODBinary(sum.uncompressed_hash, out);
+        }
+    }
+}
+
+void MergeTreeDataPartChecksums::addFile(const String & file_name, UInt64 file_size, MergeTreeDataPartChecksum::uint128 file_hash)
+{
+    files[file_name] = Checksum(file_size, file_hash);
+}
+
+void MergeTreeDataPartChecksums::add(MergeTreeDataPartChecksums && rhs_checksums)
+{
+    for (auto & checksum : rhs_checksums.files)
+        files[std::move(checksum.first)] = std::move(checksum.second);
+
+    rhs_checksums.files.clear();
+}
+
+/// Checksum computed from the set of control sums of .bin files.
+void MergeTreeDataPartChecksums::computeTotalChecksumDataOnly(SipHash & hash) const
+{
+    /// We use fact that iteration is in deterministic (lexicographical) order.
+    for (const auto & it : files)
+    {
+        const String & name = it.first;
+        const Checksum & sum = it.second;
+
+        if (!endsWith(name, ".bin"))
+            continue;
+
+        UInt64 len = name.size();
+        hash.update(len);
+        hash.update(name.data(), len);
+        hash.update(sum.uncompressed_size);
+        hash.update(sum.uncompressed_hash);
+    }
+}
+
+String MergeTreeDataPartChecksums::getSerializedString() const
+{
+    WriteBufferFromOwnString out;
+    write(out);
+    return out.str();
+}
+
+MergeTreeDataPartChecksums MergeTreeDataPartChecksums::deserializeFrom(const String & s)
+{
+    ReadBufferFromString in(s);
+    MergeTreeDataPartChecksums res;
+    if (!res.read(in))
+        throw Exception("Checksums format is too old", ErrorCodes::FORMAT_VERSION_TOO_OLD);
+    assertEOF(in);
+    return res;
+}
+
+bool MergeTreeDataPartChecksums::isBadChecksumsErrorCode(int code)
+{
+    return code == ErrorCodes::CHECKSUM_DOESNT_MATCH
+           || code == ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART
+           || code == ErrorCodes::NO_FILE_IN_DATA_PART
+           || code == ErrorCodes::UNEXPECTED_FILE_IN_DATA_PART;
+}
+
+void MinimalisticDataPartChecksums::serialize(WriteBuffer & to) const
+{
+    writeString("checksums format version: 5\n", to);
+
+    writeVarUInt(num_compressed_files, to);
+    writeVarUInt(num_uncompressed_files, to);
+
+    writePODBinary(hash_of_all_files, to);
+    writePODBinary(hash_of_uncompressed_files, to);
+    writePODBinary(uncompressed_hash_of_compressed_files, to);
+}
+
+String MinimalisticDataPartChecksums::getSerializedString()
+{
+    WriteBufferFromOwnString wb;
+    serialize(wb);
+    return wb.str();
+}
+
+bool MinimalisticDataPartChecksums::deserialize(ReadBuffer & in)
+{
+    assertString("checksums format version: ", in);
+    size_t format_version;
+    readText(format_version, in);
+    assertChar('\n', in);
+
+    if (format_version < MINIMAL_VERSION_WITH_MINIMALISTIC_CHECKSUMS)
+    {
+        auto full_checksums_ptr = std::make_unique<MergeTreeDataPartChecksums>();
+        if (!full_checksums_ptr->read(in, format_version))
+            return false;
+
+        computeTotalChecksums(*full_checksums_ptr);
+        full_checksums = std::move(full_checksums_ptr);
+        return true;
+    }
+
+    if (format_version > MINIMAL_VERSION_WITH_MINIMALISTIC_CHECKSUMS)
+        throw Exception("Unknown checksums format version: " + DB::toString(format_version), ErrorCodes::UNKNOWN_FORMAT);
+
+    readVarUInt(num_compressed_files, in);
+    readVarUInt(num_uncompressed_files, in);
+
+    readPODBinary(hash_of_all_files, in);
+    readPODBinary(hash_of_uncompressed_files, in);
+    readPODBinary(uncompressed_hash_of_compressed_files, in);
+
+    return true;
+}
+
+void MinimalisticDataPartChecksums::computeTotalChecksums(const MergeTreeDataPartChecksums & full_checksums)
+{
+    num_compressed_files = 0;
+    num_uncompressed_files = 0;
+
+    SipHash hash_of_all_files_;
+    SipHash hash_of_uncompressed_files_;
+    SipHash uncompressed_hash_of_compressed_files_;
+
+    auto update_hash = [] (SipHash & hash, const std::string & data)
+    {
+        UInt64 len = data.size();
+        hash.update(len);
+        hash.update(data.data(), len);
+    };
+
+    for (const auto & elem : full_checksums.files)
+    {
+        const String & name = elem.first;
+        const auto & checksum = elem.second;
+
+        update_hash(hash_of_all_files_, name);
+        hash_of_all_files_.update(checksum.file_hash);
+
+        if (!checksum.is_compressed)
+        {
+            ++num_uncompressed_files;
+            update_hash(hash_of_uncompressed_files_, name);
+            hash_of_uncompressed_files_.update(checksum.file_hash);
+        }
+        else
+        {
+            ++num_compressed_files;
+            update_hash(uncompressed_hash_of_compressed_files_, name);
+            uncompressed_hash_of_compressed_files_.update(checksum.uncompressed_hash);
+        }
+    }
+
+    auto get_hash = [] (SipHash & hash, uint128 & data)
+    {
+        hash.get128(data.first, data.second);
+    };
+
+    get_hash(hash_of_all_files_, hash_of_all_files);
+    get_hash(hash_of_uncompressed_files_, hash_of_uncompressed_files);
+    get_hash(uncompressed_hash_of_compressed_files_, uncompressed_hash_of_compressed_files);
+}
+
+String MinimalisticDataPartChecksums::getSerializedString(const MergeTreeDataPartChecksums & full_checksums, bool minimalistic)
+{
+    if (!minimalistic)
+        return full_checksums.getSerializedString();
+
+    MinimalisticDataPartChecksums checksums;
+    checksums.computeTotalChecksums(full_checksums);
+    return checksums.getSerializedString();
+}
+
+void MinimalisticDataPartChecksums::checkEqual(const MinimalisticDataPartChecksums & rhs, bool check_uncompressed_hash_in_compressed_files)
+{
+    if (full_checksums && rhs.full_checksums)
+        full_checksums->checkEqual(*rhs.full_checksums, check_uncompressed_hash_in_compressed_files);
+
+    // If full checksums were checked, check total checksums just in case
+    checkEqualImpl(rhs, check_uncompressed_hash_in_compressed_files);
+}
+
+void MinimalisticDataPartChecksums::checkEqual(const MergeTreeDataPartChecksums & rhs, bool check_uncompressed_hash_in_compressed_files)
+{
+    if (full_checksums)
+        full_checksums->checkEqual(rhs, check_uncompressed_hash_in_compressed_files);
+
+    // If full checksums were checked, check total checksums just in case
+    MinimalisticDataPartChecksums rhs_minimalistic;
+    rhs_minimalistic.computeTotalChecksums(rhs);
+    checkEqualImpl(rhs_minimalistic, check_uncompressed_hash_in_compressed_files);
+}
+
+void MinimalisticDataPartChecksums::checkEqualImpl(const MinimalisticDataPartChecksums & rhs, bool check_uncompressed_hash_in_compressed_files)
+{
+    if (num_compressed_files != rhs.num_compressed_files || num_uncompressed_files != rhs.num_uncompressed_files)
+    {
+        std::stringstream error_msg;
+        error_msg << "Different number of files: " << rhs.num_compressed_files << " compressed (expected " << num_compressed_files << ")"
+            << " and " << rhs.num_uncompressed_files << " uncompressed ones (expected " << num_uncompressed_files << ")";
+
+        throw Exception(error_msg.str(), ErrorCodes::CHECKSUM_DOESNT_MATCH);
+    }
+
+    Strings errors;
+
+    if (hash_of_uncompressed_files != rhs.hash_of_uncompressed_files)
+        errors.emplace_back("hash of uncompressed files doesn't match");
+
+    if (check_uncompressed_hash_in_compressed_files)
+    {
+        if (uncompressed_hash_of_compressed_files != rhs.uncompressed_hash_of_compressed_files)
+            errors.emplace_back("uncompressed hash of compressed files doesn't match");
+    }
+    else
+    {
+        if (hash_of_all_files != rhs.hash_of_all_files)
+            errors.emplace_back("total hash of all files doesn't match");
+    }
+
+    if (!errors.empty())
+    {
+        String error_msg = "Checksums of parts don't match: " + errors.front();
+        for (size_t i = 1; i < errors.size(); ++i)
+            error_msg += ", " + errors[i];
+
+        throw Exception(error_msg, ErrorCodes::CHECKSUM_DOESNT_MATCH);
+    }
+}
+
+MinimalisticDataPartChecksums MinimalisticDataPartChecksums::deserializeFrom(const String & s)
+{
+    MinimalisticDataPartChecksums res;
+    ReadBufferFromString rb(s);
+    res.deserialize(rb);
+    return res;
+}
+
+}
diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h
new file mode 100644
index 00000000000..2471d0ef681
--- /dev/null
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h
@@ -0,0 +1,121 @@
+#pragma once
+#include <Core/Types.h>
+#include <IO/ReadBuffer.h>
+#include <IO/WriteBuffer.h>
+#include <city.h>
+#include <map>
+#include <memory>
+
+
+class SipHash;
+
+
+namespace DB
+{
+
+/// Checksum of one file.
+struct MergeTreeDataPartChecksum
+{
+    using uint128 = CityHash_v1_0_2::uint128;
+
+    UInt64 file_size {};
+    uint128 file_hash {};
+
+    bool is_compressed = false;
+    UInt64 uncompressed_size {};
+    uint128 uncompressed_hash {};
+
+    MergeTreeDataPartChecksum() {}
+    MergeTreeDataPartChecksum(UInt64 file_size_, uint128 file_hash_) : file_size(file_size_), file_hash(file_hash_) {}
+    MergeTreeDataPartChecksum(UInt64 file_size_, uint128 file_hash_, UInt64 uncompressed_size_, uint128 uncompressed_hash_)
+        : file_size(file_size_), file_hash(file_hash_), is_compressed(true),
+        uncompressed_size(uncompressed_size_), uncompressed_hash(uncompressed_hash_) {}
+
+    void checkEqual(const MergeTreeDataPartChecksum & rhs, bool have_uncompressed, const String & name) const;
+    void checkSize(const String & path) const;
+};
+
+
+/** Checksums of all non-temporary files.
+  * For compressed files, the check sum and the size of the decompressed data are stored to not depend on the compression method.
+  */
+struct MergeTreeDataPartChecksums
+{
+    using Checksum = MergeTreeDataPartChecksum;
+
+    /// The order is important.
+    using FileChecksums = std::map<String, Checksum>;
+    FileChecksums files;
+
+    void addFile(const String & file_name, UInt64 file_size, Checksum::uint128 file_hash);
+
+    void add(MergeTreeDataPartChecksums && rhs_checksums);
+
+    bool empty() const
+    {
+        return files.empty();
+    }
+
+    /// Checks that the set of columns and their checksums are the same. If not, throws an exception.
+    /// If have_uncompressed, for compressed files it compares the checksums of the decompressed data.
+    /// Otherwise, it compares only the checksums of the files.
+    void checkEqual(const MergeTreeDataPartChecksums & rhs, bool have_uncompressed) const;
+
+    static bool isBadChecksumsErrorCode(int code);
+
+    /// Checks that the directory contains all the needed files of the correct size. Does not check the checksum.
+    void checkSizes(const String & path) const;
+
+    /// Returns false if the checksum is too old.
+    bool read(ReadBuffer & in);
+    /// Assume that header with version (the first line) is read
+    bool read(ReadBuffer & in, size_t format_version);
+    bool read_v2(ReadBuffer & in);
+    bool read_v3(ReadBuffer & in);
+    bool read_v4(ReadBuffer & in);
+
+    void write(WriteBuffer & out) const;
+
+    /// Checksum from the set of checksums of .bin files (for deduplication).
+    void computeTotalChecksumDataOnly(SipHash & hash) const;
+
+    String getSerializedString() const;
+    static MergeTreeDataPartChecksums deserializeFrom(const String & s);
+};
+
+
+/// A kind of MergeTreeDataPartChecksums intended to be stored in ZooKeeper (to save its RAM)
+/// MinimalisticDataPartChecksums and MergeTreeDataPartChecksums hasve the same serialization format
+///  for versions less than MINIMAL_VERSION_WITH_MINIMALISTIC_CHECKSUMS.
+struct MinimalisticDataPartChecksums
+{
+    UInt64 num_compressed_files = 0;
+    UInt64 num_uncompressed_files = 0;
+
+    using uint128 = MergeTreeDataPartChecksum::uint128;
+    uint128 hash_of_all_files {};
+    uint128 hash_of_uncompressed_files {};
+    uint128 uncompressed_hash_of_compressed_files {};
+
+    /// Is set only for old formats
+    std::unique_ptr<MergeTreeDataPartChecksums> full_checksums;
+
+    static constexpr size_t MINIMAL_VERSION_WITH_MINIMALISTIC_CHECKSUMS = 5;
+
+    MinimalisticDataPartChecksums() = default;
+    void computeTotalChecksums(const MergeTreeDataPartChecksums & full_checksums);
+
+    bool deserialize(ReadBuffer & in);
+    static MinimalisticDataPartChecksums deserializeFrom(const String & s);
+
+    void serialize(WriteBuffer & to) const;
+    String getSerializedString();
+    static String getSerializedString(const MergeTreeDataPartChecksums & full_checksums, bool minimalistic);
+
+    void checkEqual(const MinimalisticDataPartChecksums & rhs, bool check_uncompressed_hash_in_compressed_files);
+    void checkEqual(const MergeTreeDataPartChecksums & rhs, bool check_uncompressed_hash_in_compressed_files);
+    void checkEqualImpl(const MinimalisticDataPartChecksums & rhs, bool check_uncompressed_hash_in_compressed_files);
+};
+
+
+}
diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp
index e85efa435fd..9d6d41fa9d5 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp
@@ -214,7 +214,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa
 
     ProfileEvents::increment(ProfileEvents::MergeTreeDataWriterRows, block.rows());
     ProfileEvents::increment(ProfileEvents::MergeTreeDataWriterUncompressedBytes, block.bytes());
-    ProfileEvents::increment(ProfileEvents::MergeTreeDataWriterCompressedBytes, new_data_part->size_in_bytes);
+    ProfileEvents::increment(ProfileEvents::MergeTreeDataWriterCompressedBytes, new_data_part->bytes_on_disk);
 
     return new_data_part;
 }
diff --git a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp
index e172760b17f..86066762769 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp
@@ -111,7 +111,8 @@ size_t MergeTreeReader::readRows(size_t from_mark, bool continue_reading, size_t
 
                 /// share offsets in all elements of nested structure
                 if (!append)
-                    column = ColumnArray::create(type_arr->getNestedType()->createColumn(), it_inserted.first->second);
+                    column = ColumnArray::create(type_arr->getNestedType()->createColumn(),
+                                                 it_inserted.first->second)->assumeMutable();
             }
 
             try
@@ -413,7 +414,8 @@ static bool arrayHasNoElementsRead(const IColumn & column)
 }
 
 
-void MergeTreeReader::fillMissingColumns(Block & res, const Names & ordered_names, bool always_reorder)
+void MergeTreeReader::fillMissingColumns(Block & res, const Names & ordered_names, bool always_reorder,
+                                         size_t rows, bool never_evaluate_defaults)
 {
     if (!res)
         throw Exception("Empty block passed to fillMissingColumns", ErrorCodes::LOGICAL_ERROR);
@@ -444,8 +446,6 @@ void MergeTreeReader::fillMissingColumns(Block & res, const Names & ordered_name
         bool should_evaluate_defaults = false;
         bool should_sort = always_reorder;
 
-        size_t rows = res.rows();
-
         /// insert default values only for columns without default expressions
         for (const auto & requested_column : columns)
         {
@@ -497,11 +497,11 @@ void MergeTreeReader::fillMissingColumns(Block & res, const Names & ordered_name
         }
 
         /// evaluate defaulted columns if necessary
-        if (should_evaluate_defaults)
+        if (!never_evaluate_defaults && should_evaluate_defaults)
             evaluateMissingDefaults(res, columns, storage.getColumns().defaults, storage.context);
 
         /// sort columns to ensure consistent order among all blocks
-        if (should_sort)
+        if (!never_evaluate_defaults && should_sort)
         {
             Block ordered_block;
 
diff --git a/dbms/src/Storages/MergeTree/MergeTreeReader.h b/dbms/src/Storages/MergeTree/MergeTreeReader.h
index c2cd086d343..b76cf19ff93 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeReader.h
+++ b/dbms/src/Storages/MergeTree/MergeTreeReader.h
@@ -44,7 +44,11 @@ public:
     /// Add columns from ordered_names that are not present in the block.
     /// Missing columns are added in the order specified by ordered_names.
     /// If at least one column was added, reorders all columns in the block according to ordered_names.
-    void fillMissingColumns(Block & res, const Names & ordered_names, const bool always_reorder);
+    /// Rows is the number of rows will be used to create default columns. It's res.rows() when all columns in block has the same size.
+    /// This function may be used in order to create default columns in nested structures. In that case block may has
+    /// columns with different size, and default columns can't be evaluated, so never_evaluate_defaults = false is used.
+    void fillMissingColumns(Block & res, const Names & ordered_names, const bool always_reorder,
+                            size_t rows, bool never_evaluate_defaults = false);
 
 private:
     class Stream
diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.h b/dbms/src/Storages/MergeTree/MergeTreeSettings.h
index f91ca880a66..424c634afaf 100644
--- a/dbms/src/Storages/MergeTree/MergeTreeSettings.h
+++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.h
@@ -131,6 +131,12 @@ struct MergeTreeSettings
       *  for backward compatibility.                                                                          \
       */                                                                                                      \
     M(SettingBool, compatibility_allow_sampling_expression_not_in_primary_key, false)                         \
+                                                                                                              \
+    /** Use small format (dozens bytes) for part checksums in ZooKeeper                                       \
+      *  instead of ordinary ones (dozens KB).                                                                \
+      * Before enabling check that all replicas support new format.                                           \
+      */                                                                                                      \
+    M(SettingBool, use_minimalistic_checksums_in_zookeeper, false)
 
     /// Settings that should not change after the creation of a table.
 #define APPLY_FOR_IMMUTABLE_MERGE_TREE_SETTINGS(M)  \
diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp
index d3162810a1c..6c9409be833 100644
--- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp
+++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp
@@ -347,7 +347,7 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart(
     new_part->columns = *total_column_list;
     new_part->index.assign(std::make_move_iterator(index_columns.begin()), std::make_move_iterator(index_columns.end()));
     new_part->checksums = checksums;
-    new_part->size_in_bytes = MergeTreeData::DataPart::calculateTotalSize(new_part->getFullPath());
+    new_part->bytes_on_disk = MergeTreeData::DataPart::calculateTotalSizeOnDisk(new_part->getFullPath());
 }
 
 void MergedBlockOutputStream::init()
diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp
index 72d7cd91e3a..f0513c04f9e 100644
--- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp
+++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp
@@ -144,7 +144,9 @@ void ReplicatedMergeTreeAlterThread::run()
                         ops.emplace_back(std::make_shared<zkutil::Op::SetData>(
                             storage.replica_path + "/parts/" + part->name + "/columns", transaction->getNewColumns().toString(), -1));
                         ops.emplace_back(std::make_shared<zkutil::Op::SetData>(
-                            storage.replica_path + "/parts/" + part->name + "/checksums", transaction->getNewChecksums().toString(), -1));
+                            storage.replica_path + "/parts/" + part->name + "/checksums",
+                            storage.getChecksumsForZooKeeper(transaction->getNewChecksums()),
+                            -1));
 
                         try
                         {
diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp
index 1d5ec76c652..52bc1c4be3f 100644
--- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp
+++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp
@@ -133,7 +133,7 @@ void ReplicatedMergeTreeBlockOutputStream::write(const Block & block)
         if (deduplicate)
         {
             SipHash hash;
-            part->checksums.summaryDataChecksum(hash);
+            part->checksums.computeTotalChecksumDataOnly(hash);
             union
             {
                 char bytes[16];
@@ -306,7 +306,7 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo
         zkutil::CreateMode::Persistent));
     ops.emplace_back(std::make_shared<zkutil::Op::Create>(
         storage.replica_path + "/parts/" + part->name + "/checksums",
-        part->checksums.toString(),
+        storage.getChecksumsForZooKeeper(part->checksums),
         acl,
         zkutil::CreateMode::Persistent));
 
diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp
index f64826f6266..85e58f4551b 100644
--- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp
+++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp
@@ -237,7 +237,7 @@ void ReplicatedMergeTreePartCheckThread::checkPart(const String & part_name)
 
             try
             {
-                auto zk_checksums = MergeTreeData::DataPart::Checksums::parse(
+                auto zk_checksums = MinimalisticDataPartChecksums::deserializeFrom(
                     zookeeper->get(storage.replica_path + "/parts/" + part_name + "/checksums"));
                 zk_checksums.checkEqual(part->checksums, true);
 
diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp
index c5d4a41d496..32d7b263f80 100644
--- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp
+++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp
@@ -634,7 +634,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry(
 
             auto part = data.getPartIfExists(name, {MergeTreeDataPartState::PreCommitted, MergeTreeDataPartState::Committed, MergeTreeDataPartState::Outdated});
             if (part)
-                sum_parts_size_in_bytes += part->size_in_bytes;
+                sum_parts_size_in_bytes += part->bytes_on_disk;
         }
 
         if (merger.merges_blocker.isCancelled())
diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp
index f6686c273ce..f8ea02cff9b 100644
--- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp
+++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp
@@ -94,16 +94,22 @@ void ReplicatedMergeTreeRestartingThread::run()
                         /// The exception when you try to zookeeper_init usually happens if DNS does not work. We will try to do it again.
                         tryLogCurrentException(__PRETTY_FUNCTION__);
 
+                        if (first_time)
+                            storage.startup_event.set();
                         wakeup_event.tryWait(retry_period_ms);
                         continue;
                     }
 
                     if (!need_stop && !tryStartup())
                     {
+                        if (first_time)
+                            storage.startup_event.set();
                         wakeup_event.tryWait(retry_period_ms);
                         continue;
                     }
 
+                    if (first_time)
+                        storage.startup_event.set();
                     break;
                 }
 
@@ -150,6 +156,7 @@ void ReplicatedMergeTreeRestartingThread::run()
         }
         catch (...)
         {
+            storage.startup_event.set();
             tryLogCurrentException(__PRETTY_FUNCTION__);
         }
 
diff --git a/dbms/src/Storages/StorageBuffer.cpp b/dbms/src/Storages/StorageBuffer.cpp
index 5c59ffa7c40..8688d191b9a 100644
--- a/dbms/src/Storages/StorageBuffer.cpp
+++ b/dbms/src/Storages/StorageBuffer.cpp
@@ -164,7 +164,7 @@ static void appendBlock(const Block & from, Block & to)
         for (size_t column_no = 0, columns = to.columns(); column_no < columns; ++column_no)
         {
             const IColumn & col_from = *from.getByPosition(column_no).column.get();
-            MutableColumnPtr col_to = to.getByPosition(column_no).column->mutate();
+            MutableColumnPtr col_to = (*std::move(to.getByPosition(column_no).column)).mutate();
 
             col_to->insertRangeFrom(col_from, 0, rows);
 
@@ -183,7 +183,7 @@ static void appendBlock(const Block & from, Block & to)
             {
                 ColumnPtr & col_to = to.getByPosition(column_no).column;
                 if (col_to->size() != old_rows)
-                    col_to = col_to->mutate()->cut(0, old_rows);
+                    col_to = (*std::move(col_to)).mutate()->cut(0, old_rows);
             }
         }
         catch (...)
diff --git a/dbms/src/Storages/StorageDictionary.cpp b/dbms/src/Storages/StorageDictionary.cpp
index b3dbf39e9c2..e4f2a154474 100644
--- a/dbms/src/Storages/StorageDictionary.cpp
+++ b/dbms/src/Storages/StorageDictionary.cpp
@@ -86,7 +86,6 @@ void StorageDictionary::checkNamesAndTypesCompatibleWithDictionary(const Diction
     }
 }
 
-
 void registerStorageDictionary(StorageFactory & factory)
 {
     factory.registerStorage("Dictionary", [](const StorageFactory::Arguments & args)
diff --git a/dbms/src/Storages/StorageDictionary.h b/dbms/src/Storages/StorageDictionary.h
index 0320a514839..2e47c5fbe0d 100644
--- a/dbms/src/Storages/StorageDictionary.h
+++ b/dbms/src/Storages/StorageDictionary.h
@@ -4,6 +4,8 @@
 #include <Core/Defines.h>
 #include <common/MultiVersion.h>
 #include <ext/shared_ptr_helper.h>
+#include <IO/WriteBufferFromString.h>
+#include <IO/Operators.h>
 
 
 namespace Poco
@@ -32,6 +34,26 @@ public:
     void drop() override {}
     static NamesAndTypesList getNamesAndTypes(const DictionaryStructure & dictionary_structure);
 
+    template <typename ForwardIterator>
+    static std::string generateNamesAndTypesDescription(ForwardIterator begin, ForwardIterator end)
+    {
+        std::string description;
+        {
+            WriteBufferFromString buffer(description);
+            bool first = true;
+            for (; begin != end; ++begin)
+            {
+                if (!first)
+                    buffer << ", ";
+                first = false;
+
+                buffer << begin->name << ' ' << begin->type->getName();
+            }
+        }
+
+        return description;
+    }
+
 private:
     using Ptr = MultiVersion<IDictionaryBase>::Version;
 
@@ -41,24 +63,6 @@ private:
 
     void checkNamesAndTypesCompatibleWithDictionary(const DictionaryStructure & dictionary_structure) const;
 
-    template <typename ForwardIterator>
-    std::string generateNamesAndTypesDescription(ForwardIterator begin, ForwardIterator end) const
-    {
-        if (begin == end)
-        {
-            return "";
-        }
-        std::string description;
-        for (; begin != end; ++begin)
-        {
-            description += ", ";
-            description += begin->name;
-            description += ' ';
-            description += begin->type->getName();
-        }
-        return description.substr(2, description.size());
-    }
-
 protected:
     StorageDictionary(const String & table_name_,
         const ColumnsDescription & columns_,
diff --git a/dbms/src/Storages/StorageKafka.cpp b/dbms/src/Storages/StorageKafka.cpp
index bfe72aa016e..80e4942839e 100644
--- a/dbms/src/Storages/StorageKafka.cpp
+++ b/dbms/src/Storages/StorageKafka.cpp
@@ -4,6 +4,8 @@
 
 #include <thread>
 #include <boost/algorithm/string/replace.hpp>
+#include <boost/algorithm/string/split.hpp>
+#include <boost/algorithm/string/trim.hpp>
 #include <Common/Exception.h>
 #include <Common/setThreadName.h>
 #include <Common/typeid_cast.h>
@@ -574,9 +576,14 @@ void registerStorageKafka(StorageFactory & factory)
                 throw Exception("Number of consumers must be a positive integer", ErrorCodes::BAD_ARGUMENTS);
         }
 
-        // Parse topic list and consumer group
+        // Parse topic list
         Names topics;
-        topics.push_back(static_cast<const ASTLiteral &>(*engine_args[1]).value.safeGet<String>());
+        String topic_arg = static_cast<const ASTLiteral &>(*engine_args[1]).value.safeGet<String>();
+        boost::split(topics, topic_arg , [](char c){ return c == ','; });
+        for(String & topic : topics)
+            boost::trim(topic);
+
+        // Parse consumer group
         String group = static_cast<const ASTLiteral &>(*engine_args[2]).value.safeGet<String>();
 
         // Parse format from string
diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp
index 293c3254a8f..da205e9a293 100644
--- a/dbms/src/Storages/StorageMergeTree.cpp
+++ b/dbms/src/Storages/StorageMergeTree.cpp
@@ -116,7 +116,7 @@ BlockOutputStreamPtr StorageMergeTree::write(const ASTPtr & /*query*/, const Set
 bool StorageMergeTree::checkTableCanBeDropped() const
 {
     const_cast<MergeTreeData &>(getData()).recalculateColumnSizes();
-    context.checkTableCanBeDropped(database_name, table_name, getData().getTotalCompressedSize());
+    context.checkTableCanBeDropped(database_name, table_name, getData().getTotalActiveSizeInBytes());
     return true;
 }
 
@@ -334,7 +334,7 @@ bool StorageMergeTree::merge(
             part_log_elem.part_name = future_part.name;
 
             if (new_part)
-                part_log_elem.bytes_compressed_on_disk = new_part->size_in_bytes;
+                part_log_elem.bytes_compressed_on_disk = new_part->bytes_on_disk;
 
             part_log_elem.source_part_names.reserve(future_part.parts.size());
             for (const auto & source_part : future_part.parts)
diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp
index 70326cddd4a..13d50500d09 100644
--- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp
@@ -838,7 +838,7 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks)
         LOG_ERROR(log, "Adding unexpected local part to ZooKeeper: " << part->name);
 
         zkutil::Ops ops;
-        checkPartAndAddToZooKeeper(part, ops);
+        checkPartChecksumsAndAddCommitOps(zookeeper, part, ops);
         zookeeper->multi(ops);
     }
 
@@ -878,11 +878,9 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks)
 }
 
 
-void StorageReplicatedMergeTree::checkPartAndAddToZooKeeper(
-    const MergeTreeData::DataPartPtr & part, zkutil::Ops & ops, String part_name)
+void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil::ZooKeeperPtr & zookeeper,
+    const MergeTreeData::DataPartPtr & part, zkutil::Ops & ops, String part_name, NameSet * absent_replicas_paths)
 {
-    auto zookeeper = getZooKeeper();
-
     if (part_name.empty())
         part_name = part->name;
 
@@ -892,24 +890,34 @@ void StorageReplicatedMergeTree::checkPartAndAddToZooKeeper(
     Strings replicas = zookeeper->getChildren(zookeeper_path + "/replicas");
     std::shuffle(replicas.begin(), replicas.end(), rng);
     String expected_columns_str = part->columns.toString();
+    bool has_been_alredy_added = false;
 
     for (const String & replica : replicas)
     {
         zkutil::Stat stat_before, stat_after;
+        String current_part_path = zookeeper_path + "/replicas/" + replica + "/parts/" + part_name;
+
         String columns_str;
-        if (!zookeeper->tryGet(zookeeper_path + "/replicas/" + replica + "/parts/" + part_name + "/columns", columns_str, &stat_before))
+        if (!zookeeper->tryGet(current_part_path + "/columns", columns_str, &stat_before))
+        {
+            if (absent_replicas_paths)
+                absent_replicas_paths->emplace(current_part_path);
+
             continue;
+        }
+
         if (columns_str != expected_columns_str)
         {
             LOG_INFO(log, "Not checking checksums of part " << part_name << " with replica " << replica
                 << " because columns are different");
             continue;
         }
+
         String checksums_str;
         /// Let's check that the node's version with the columns did not change while we were reading the checksums.
         /// This ensures that the columns and the checksum refer to the same data.
-        if (!zookeeper->tryGet(zookeeper_path + "/replicas/" + replica + "/parts/" + part_name + "/checksums", checksums_str) ||
-            !zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/parts/" + part_name + "/columns", &stat_after) ||
+        if (!zookeeper->tryGet(current_part_path + "/checksums", checksums_str) ||
+            !zookeeper->exists(current_part_path + "/columns", &stat_after) ||
             stat_before.version != stat_after.version)
         {
             LOG_INFO(log, "Not checking checksums of part " << part_name << " with replica " << replica
@@ -917,36 +925,90 @@ void StorageReplicatedMergeTree::checkPartAndAddToZooKeeper(
             continue;
         }
 
-        auto checksums = MergeTreeData::DataPart::Checksums::parse(checksums_str);
-        checksums.checkEqual(part->checksums, true);
+        auto zk_checksums = MinimalisticDataPartChecksums::deserializeFrom(checksums_str);
+        zk_checksums.checkEqual(part->checksums, true);
+
+        if (replica == replica_name)
+            has_been_alredy_added = true;
     }
 
-    if (zookeeper->exists(replica_path + "/parts/" + part_name))
+    if (!has_been_alredy_added)
     {
-        LOG_ERROR(log, "checkPartAndAddToZooKeeper: node " << replica_path + "/parts/" + part_name << " already exists");
-        return;
+        auto acl = zookeeper->getDefaultACL();
+        String part_path = replica_path + "/parts/" + part_name;
+
+        ops.emplace_back(std::make_shared<zkutil::Op::Check>(
+            zookeeper_path + "/columns", expected_columns_version));
+        ops.emplace_back(std::make_shared<zkutil::Op::Create>(
+            part_path, "", acl, zkutil::CreateMode::Persistent));
+        ops.emplace_back(std::make_shared<zkutil::Op::Create>(
+            part_path + "/columns", part->columns.toString(), acl, zkutil::CreateMode::Persistent));
+        ops.emplace_back(std::make_shared<zkutil::Op::Create>(
+            part_path + "/checksums", getChecksumsForZooKeeper(part->checksums), acl, zkutil::CreateMode::Persistent));
     }
+    else
+    {
+        LOG_WARNING(log, "checkPartAndAddToZooKeeper: node " << replica_path + "/parts/" + part_name << " already exists."
+            << " Will not commit any nodes.");
+    }
+}
 
-    auto acl = zookeeper->getDefaultACL();
+MergeTreeData::DataPartsVector StorageReplicatedMergeTree::checkPartChecksumsAndCommit(MergeTreeData::Transaction & transaction,
+    const MergeTreeData::DataPartPtr & part)
+{
+    auto zookeeper = getZooKeeper();
 
-    ops.emplace_back(std::make_shared<zkutil::Op::Check>(
-        zookeeper_path + "/columns",
-        expected_columns_version));
-    ops.emplace_back(std::make_shared<zkutil::Op::Create>(
-        replica_path + "/parts/" + part_name,
-        "",
-        acl,
-        zkutil::CreateMode::Persistent));
-    ops.emplace_back(std::make_shared<zkutil::Op::Create>(
-        replica_path + "/parts/" + part_name + "/columns",
-        part->columns.toString(),
-        acl,
-        zkutil::CreateMode::Persistent));
-    ops.emplace_back(std::make_shared<zkutil::Op::Create>(
-        replica_path + "/parts/" + part_name + "/checksums",
-        part->checksums.toString(),
-        acl,
-        zkutil::CreateMode::Persistent));
+    while (true)
+    {
+        zkutil::Ops ops;
+        NameSet absent_part_paths_on_replicas;
+
+        /// Checksums are checked here and `ops` is filled. In fact, the part is added to ZK just below, when executing `multi`.
+        checkPartChecksumsAndAddCommitOps(zookeeper, part, ops, part->name, &absent_part_paths_on_replicas);
+
+        /// Do not commit if the part is obsolete, we have just briefly checked its checksums
+        if (transaction.isEmpty())
+            return {};
+
+        /// Will check that the part did not suddenly appear on skipped replicas
+        if (!absent_part_paths_on_replicas.empty())
+        {
+            zkutil::Ops new_ops;
+            for (const String & part_path : absent_part_paths_on_replicas)
+            {
+                new_ops.emplace_back(std::make_shared<zkutil::Op::Create>(part_path, "", zookeeper->getDefaultACL(), zkutil::CreateMode::Persistent));
+                new_ops.emplace_back(std::make_shared<zkutil::Op::Remove>(part_path, -1));
+            }
+
+            /// Add check ops at the beginning
+            new_ops.insert(new_ops.end(), ops.begin(), ops.end());
+            ops = std::move(new_ops);
+        }
+
+        try
+        {
+            zookeeper->multi(ops);
+            return transaction.commit();
+        }
+        catch (zkutil::KeeperMultiException & e)
+        {
+            size_t num_check_ops = 2 * absent_part_paths_on_replicas.size();
+            size_t failed_op_index = zkutil::getFailedOpIndex(e.info.op_results, e.info.code);
+
+            if (failed_op_index < num_check_ops && e.info.code == ZNODEEXISTS)
+            {
+                LOG_INFO(log, "The part " << e.info.getFailedOp().describe() << " on a replica suddenly appeared, will recheck checksums");
+            }
+            else
+                throw;
+        }
+    }
+}
+
+String StorageReplicatedMergeTree::getChecksumsForZooKeeper(const MergeTreeDataPartChecksums & checksums)
+{
+    return MinimalisticDataPartChecksums::getSerializedString(checksums,
+                                                              static_cast<bool>(data.settings.use_minimalistic_checksums_in_zookeeper));
 }
 
 
@@ -1080,11 +1142,11 @@ void StorageReplicatedMergeTree::tryExecuteMerge(const StorageReplicatedMergeTre
         /// If entry is old enough, and have enough size, and part are exists in any replica,
         ///  then prefer fetching of merged part from replica.
 
-        size_t sum_parts_size_in_bytes = 0;
+        size_t sum_parts_bytes_on_disk = 0;
         for (const auto & part : parts)
-            sum_parts_size_in_bytes += part->size_in_bytes;
+            sum_parts_bytes_on_disk += part->bytes_on_disk;
 
-        if (sum_parts_size_in_bytes >= data.settings.prefer_fetch_merged_part_size_threshold)
+        if (sum_parts_bytes_on_disk >= data.settings.prefer_fetch_merged_part_size_threshold)
         {
             String replica = findReplicaHavingPart(entry.new_part_name, true);    /// NOTE excessive ZK requests for same data later, may remove.
             if (!replica.empty())
@@ -1143,7 +1205,7 @@ void StorageReplicatedMergeTree::tryExecuteMerge(const StorageReplicatedMergeTre
             part_log_elem.part_name = entry.new_part_name;
 
             if (part)
-                part_log_elem.bytes_compressed_on_disk = part->size_in_bytes;
+                part_log_elem.bytes_compressed_on_disk = part->bytes_on_disk;
 
             part_log_elem.source_part_names.reserve(parts.size());
             for (const auto & source_part : parts)
@@ -1171,22 +1233,18 @@ void StorageReplicatedMergeTree::tryExecuteMerge(const StorageReplicatedMergeTre
         part = merger.mergePartsToTemporaryPart(
             future_merged_part, *merge_entry, aio_threshold, entry.create_time, reserved_space.get(), entry.deduplicate);
 
-        zkutil::Ops ops;
+        merger.renameMergedTemporaryPart(part, parts, &transaction);
 
         try
         {
-            /// Checksums are checked here and `ops` is filled. In fact, the part is added to ZK just below, when executing `multi`.
-            checkPartAndAddToZooKeeper(part, ops, entry.new_part_name);
+            checkPartChecksumsAndCommit(transaction, part);
         }
         catch (const Exception & e)
         {
-            if (e.code() == ErrorCodes::CHECKSUM_DOESNT_MATCH
-                || e.code() == ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART
-                || e.code() == ErrorCodes::NO_FILE_IN_DATA_PART
-                || e.code() == ErrorCodes::UNEXPECTED_FILE_IN_DATA_PART)
+            if (MergeTreeDataPartChecksums::isBadChecksumsErrorCode(e.code()))
             {
                 do_fetch = true;
-                part->remove();
+                transaction.rollback();
 
                 ProfileEvents::increment(ProfileEvents::DataAfterMergeDiffersFromReplica);
 
@@ -1210,15 +1268,6 @@ void StorageReplicatedMergeTree::tryExecuteMerge(const StorageReplicatedMergeTre
             throw;
         }
 
-        merger.renameMergedTemporaryPart(part, parts, &transaction);
-
-        /// Do not commit if the part is obsolete
-        if (!transaction.isEmpty())
-        {
-            getZooKeeper()->multi(ops); /// After long merge, get fresh ZK handle, because previous session may be expired.
-            transaction.commit();
-        }
-
         /** Removing old chunks from ZK and from the disk is delayed - see ReplicatedMergeTreeCleanupThread, clearOldParts.
           */
 
@@ -1526,7 +1575,7 @@ void StorageReplicatedMergeTree::executeClearColumnInPartition(const LogEntry &
         ops.emplace_back(std::make_shared<zkutil::Op::SetData>(
             replica_path + "/parts/" + part->name + "/columns", transaction->getNewColumns().toString(), -1));
         ops.emplace_back(std::make_shared<zkutil::Op::SetData>(
-            replica_path + "/parts/" + part->name + "/checksums", transaction->getNewChecksums().toString(), -1));
+            replica_path + "/parts/" + part->name + "/checksums", getChecksumsForZooKeeper(transaction->getNewChecksums()), -1));
 
         zookeeper->multi(ops);
 
@@ -2244,7 +2293,7 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Strin
 
             if (part)
             {
-                part_log_elem.bytes_compressed_on_disk = part->size_in_bytes;
+                part_log_elem.bytes_compressed_on_disk = part->bytes_on_disk;
                 part_log_elem.rows = part->rows_count; /// Could be approximate (?)
             }
 
@@ -2269,24 +2318,15 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Strin
 
         if (!to_detached)
         {
-            zkutil::Ops ops;
+            MergeTreeData::Transaction transaction;
+            data.renameTempPartAndReplace(part, nullptr, &transaction);
 
             /** NOTE
               * Here, an error occurs if ALTER occurred with a change in the column type or column deletion,
               *  and the part on remote server has not yet been modified.
               * After a while, one of the following attempts to make `fetchPart` succeed.
               */
-            checkPartAndAddToZooKeeper(part, ops, part_name);
-
-            MergeTreeData::Transaction transaction;
-            data.renameTempPartAndReplace(part, nullptr, &transaction);
-
-            /// Do not commit if the part is obsolete
-            if (!transaction.isEmpty())
-            {
-                getZooKeeper()->multi(ops);
-                replaced_parts = transaction.commit();
-            }
+            replaced_parts = checkPartChecksumsAndCommit(transaction, part);
 
             /** If a quorum is tracked for this part, you must update it.
               * If you do not have time, in case of losing the session, when you restart the server - see the `ReplicatedMergeTreeRestartingThread::updateQuorumIfWeHavePart` method.
@@ -2346,6 +2386,9 @@ void StorageReplicatedMergeTree::startup()
 
     /// In this thread replica will be activated.
     restarting_thread = std::make_unique<ReplicatedMergeTreeRestartingThread>(*this);
+
+    /// Wait while restarting_thread initializes LeaderElection (and so on) or makes first attmept to do it
+    startup_event.wait();
 }
 
 
@@ -2906,7 +2949,7 @@ bool StorageReplicatedMergeTree::checkTableCanBeDropped() const
 {
     /// Consider only synchronized data
     const_cast<MergeTreeData &>(getData()).recalculateColumnSizes();
-    context.checkTableCanBeDropped(database_name, table_name, getData().getTotalCompressedSize());
+    context.checkTableCanBeDropped(database_name, table_name, getData().getTotalActiveSizeInBytes());
     return true;
 }
 
@@ -3731,5 +3774,4 @@ void StorageReplicatedMergeTree::clearBlocksInPartition(
     LOG_TRACE(log, "Deleted " << to_delete_futures.size() << " deduplication block IDs in partition ID " << partition_id);
 }
 
-
 }
diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h
index 4a2193b05db..b16e2538fe5 100644
--- a/dbms/src/Storages/StorageReplicatedMergeTree.h
+++ b/dbms/src/Storages/StorageReplicatedMergeTree.h
@@ -241,6 +241,9 @@ private:
 
     zkutil::LeaderElectionPtr leader_election;
 
+    /// When activated, replica is initialized and startup() method could exit
+    Poco::Event startup_event;
+
     /// Do I need to complete background threads (except restarting_thread)?
     std::atomic<bool> shutdown_called {false};
     Poco::Event shutdown_event;
@@ -317,7 +320,14 @@ private:
       * Adds actions to `ops` that add data about the part into ZooKeeper.
       * Call under TableStructureLock.
       */
-    void checkPartAndAddToZooKeeper(const MergeTreeData::DataPartPtr & part, zkutil::Ops & ops, String name_override = "");
+    void checkPartChecksumsAndAddCommitOps(const zkutil::ZooKeeperPtr & zookeeper, const MergeTreeData::DataPartPtr & part,
+                                           zkutil::Ops & ops, String part_name = "", NameSet * absent_replicas_paths = nullptr);
+
+    String getChecksumsForZooKeeper(const MergeTreeDataPartChecksums & checksums);
+
+    /// Accepts a PreComitted part, atomically checks its checksums with ones on other replicas and commit the part
+    MergeTreeData::DataPartsVector checkPartChecksumsAndCommit(MergeTreeData::Transaction & transaction,
+                                                               const MergeTreeData::DataPartPtr & part);
 
     /// Adds actions to `ops` that remove a part from ZooKeeper.
     void removePartFromZooKeeper(const String & part_name, zkutil::Ops & ops);
diff --git a/dbms/src/Storages/System/StorageSystemColumns.cpp b/dbms/src/Storages/System/StorageSystemColumns.cpp
index 18598b1c1dc..d42d8a80394 100644
--- a/dbms/src/Storages/System/StorageSystemColumns.cpp
+++ b/dbms/src/Storages/System/StorageSystemColumns.cpp
@@ -114,7 +114,7 @@ BlockInputStreams StorageSystemColumns::read(
 
         NamesAndTypesList columns;
         ColumnDefaults column_defaults;
-        MergeTreeData::ColumnSizes column_sizes;
+        MergeTreeData::ColumnSizeByName column_sizes;
 
         {
             StoragePtr storage = storages.at(std::make_pair(database_name, table_name));
diff --git a/dbms/src/Storages/System/StorageSystemParts.cpp b/dbms/src/Storages/System/StorageSystemParts.cpp
index a07e8bc26dd..b2f3d2208c7 100644
--- a/dbms/src/Storages/System/StorageSystemParts.cpp
+++ b/dbms/src/Storages/System/StorageSystemParts.cpp
@@ -17,28 +17,30 @@ namespace DB
 StorageSystemParts::StorageSystemParts(const std::string & name)
     : StorageSystemPartsBase(name,
     {
-        {"partition",           std::make_shared<DataTypeString>()},
-        {"name",                std::make_shared<DataTypeString>()},
-        {"active",              std::make_shared<DataTypeUInt8>()},
-        {"marks",               std::make_shared<DataTypeUInt64>()},
-        {"marks_size",          std::make_shared<DataTypeUInt64>()},
-        {"rows",                std::make_shared<DataTypeUInt64>()},
-        {"bytes",               std::make_shared<DataTypeUInt64>()},
-        {"modification_time",   std::make_shared<DataTypeDateTime>()},
-        {"remove_time",         std::make_shared<DataTypeDateTime>()},
-        {"refcount",            std::make_shared<DataTypeUInt32>()},
-        {"min_date",            std::make_shared<DataTypeDate>()},
-        {"max_date",            std::make_shared<DataTypeDate>()},
-        {"min_block_number",    std::make_shared<DataTypeInt64>()},
-        {"max_block_number",    std::make_shared<DataTypeInt64>()},
-        {"level",               std::make_shared<DataTypeUInt32>()},
-        {"primary_key_bytes_in_memory", std::make_shared<DataTypeUInt64>()},
-        {"primary_key_bytes_in_memory_allocated", std::make_shared<DataTypeUInt64>()},
+        {"partition",                                  std::make_shared<DataTypeString>()},
+        {"name",                                       std::make_shared<DataTypeString>()},
+        {"active",                                     std::make_shared<DataTypeUInt8>()},
+        {"marks",                                      std::make_shared<DataTypeUInt64>()},
+        {"rows",                                       std::make_shared<DataTypeUInt64>()},
+        {"bytes_on_disk",                              std::make_shared<DataTypeUInt64>()},
+        {"data_compressed_bytes",                      std::make_shared<DataTypeUInt64>()},
+        {"data_uncompressed_bytes",                    std::make_shared<DataTypeUInt64>()},
+        {"marks_bytes",                                std::make_shared<DataTypeUInt64>()},
+        {"modification_time",                          std::make_shared<DataTypeDateTime>()},
+        {"remove_time",                                std::make_shared<DataTypeDateTime>()},
+        {"refcount",                                   std::make_shared<DataTypeUInt32>()},
+        {"min_date",                                   std::make_shared<DataTypeDate>()},
+        {"max_date",                                   std::make_shared<DataTypeDate>()},
+        {"min_block_number",                           std::make_shared<DataTypeInt64>()},
+        {"max_block_number",                           std::make_shared<DataTypeInt64>()},
+        {"level",                                      std::make_shared<DataTypeUInt32>()},
+        {"primary_key_bytes_in_memory",                std::make_shared<DataTypeUInt64>()},
+        {"primary_key_bytes_in_memory_allocated",      std::make_shared<DataTypeUInt64>()},
 
-        {"database",            std::make_shared<DataTypeString>()},
-        {"table",               std::make_shared<DataTypeString>()},
-        {"engine",              std::make_shared<DataTypeString>()},
-        {"path",                std::make_shared<DataTypeString>()}
+        {"database",                                   std::make_shared<DataTypeString>()},
+        {"table",                                      std::make_shared<DataTypeString>()},
+        {"engine",                                     std::make_shared<DataTypeString>()},
+        {"path",                                       std::make_shared<DataTypeString>()},
     }
     )
 {
@@ -52,6 +54,7 @@ void StorageSystemParts::processNextStorage(MutableColumns & columns, const Stor
     {
         const auto & part = info.all_parts[part_number];
         auto part_state = info.all_parts_state[part_number];
+        MergeTreeDataPart::ColumnSize columns_size = part->getTotalColumnsSize();
 
         size_t i = 0;
         {
@@ -62,9 +65,11 @@ void StorageSystemParts::processNextStorage(MutableColumns & columns, const Stor
         columns[i++]->insert(part->name);
         columns[i++]->insert(static_cast<UInt64>(part_state == State::Committed));
         columns[i++]->insert(static_cast<UInt64>(part->marks_count));
-        columns[i++]->insert(static_cast<UInt64>(part->getTotalMrkSizeInBytes()));
         columns[i++]->insert(static_cast<UInt64>(part->rows_count));
-        columns[i++]->insert(static_cast<UInt64>(part->size_in_bytes));
+        columns[i++]->insert(static_cast<UInt64>(part->bytes_on_disk));
+        columns[i++]->insert(static_cast<UInt64>(columns_size.data_compressed));
+        columns[i++]->insert(static_cast<UInt64>(columns_size.data_uncompressed));
+        columns[i++]->insert(static_cast<UInt64>(columns_size.marks));
         columns[i++]->insert(static_cast<UInt64>(part->modification_time));
 
         time_t remove_time = part->remove_time.load(std::memory_order_relaxed);
diff --git a/dbms/src/Storages/System/StorageSystemPartsBase.cpp b/dbms/src/Storages/System/StorageSystemPartsBase.cpp
index 9d94655fc18..9a2538cbe6f 100644
--- a/dbms/src/Storages/System/StorageSystemPartsBase.cpp
+++ b/dbms/src/Storages/System/StorageSystemPartsBase.cpp
@@ -11,6 +11,7 @@
 #include <Storages/VirtualColumnUtils.h>
 #include <Databases/IDatabase.h>
 #include <Parsers/queryToString.h>
+#include <Parsers/ASTIdentifier.h>
 
 
 namespace DB
@@ -160,7 +161,7 @@ public:
 
             try
             {
-                /// For table not to be dropped.
+                /// For table not to be dropped and set of columns to remain constant.
                 info.table_lock = info.storage->lockStructure(false, __PRETTY_FUNCTION__);
             }
             catch (const Exception & e)
@@ -280,7 +281,31 @@ bool StorageSystemPartsBase::hasColumn(const String & column_name) const
 StorageSystemPartsBase::StorageSystemPartsBase(std::string name_, NamesAndTypesList && columns_)
     : name(std::move(name_))
 {
-    setColumns(ColumnsDescription(std::move(columns_)));
+    NamesAndTypesList aliases;
+    ColumnDefaults defaults;
+    auto add_alias = [&](const String & alias_name, const String & column_name)
+    {
+        DataTypePtr type;
+        for (const NameAndTypePair & col : columns_)
+        {
+            if (col.name == column_name)
+            {
+                type = col.type;
+                break;
+            }
+        }
+        if (!type)
+            throw Exception("No column " + column_name + " in table system." + name, ErrorCodes::LOGICAL_ERROR);
+
+        aliases.push_back({alias_name, type});
+        defaults[alias_name] = ColumnDefault{ColumnDefaultKind::Alias, std::make_shared<ASTIdentifier>(column_name)};
+    };
+
+    /// Add aliases for old column names for backwards compatibility.
+    add_alias("bytes", "bytes_on_disk");
+    add_alias("marks_size", "marks_bytes");
+
+    setColumns(ColumnsDescription(std::move(columns_), {}, std::move(aliases), std::move(defaults)));
 }
 
 }
diff --git a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp
index 8e55625c5cd..921b32b6c0a 100644
--- a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp
+++ b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp
@@ -16,39 +16,43 @@ namespace DB
 
 
 StorageSystemPartsColumns::StorageSystemPartsColumns(const std::string & name)
-        : StorageSystemPartsBase(name,
-              {
-                      {"partition",           std::make_shared<DataTypeString>()},
-                      {"name",                std::make_shared<DataTypeString>()},
-                      {"active",              std::make_shared<DataTypeUInt8>()},
-                      {"marks",               std::make_shared<DataTypeUInt64>()},
-                      {"marks_bytes_in_block", std::make_shared<DataTypeUInt64>()},
-                      {"rows",                std::make_shared<DataTypeUInt64>()},
-                      {"bytes",               std::make_shared<DataTypeUInt64>()},
-                      {"modification_time",   std::make_shared<DataTypeDateTime>()},
-                      {"remove_time",         std::make_shared<DataTypeDateTime>()},
-                      {"refcount",            std::make_shared<DataTypeUInt32>()},
-                      {"min_date",            std::make_shared<DataTypeDate>()},
-                      {"max_date",            std::make_shared<DataTypeDate>()},
-                      {"min_block_number",    std::make_shared<DataTypeInt64>()},
-                      {"max_block_number",    std::make_shared<DataTypeInt64>()},
-                      {"level",               std::make_shared<DataTypeUInt32>()},
-                      {"primary_key_bytes_in_memory", std::make_shared<DataTypeUInt64>()},
-                      {"primary_key_bytes_in_memory_allocated", std::make_shared<DataTypeUInt64>()},
+    : StorageSystemPartsBase(name,
+    {
+        {"partition",                                  std::make_shared<DataTypeString>()},
+        {"name",                                       std::make_shared<DataTypeString>()},
+        {"active",                                     std::make_shared<DataTypeUInt8>()},
+        {"marks",                                      std::make_shared<DataTypeUInt64>()},
+        {"rows",                                       std::make_shared<DataTypeUInt64>()},
+        {"bytes_on_disk",                              std::make_shared<DataTypeUInt64>()},
+        {"data_compressed_bytes",                      std::make_shared<DataTypeUInt64>()},
+        {"data_uncompressed_bytes",                    std::make_shared<DataTypeUInt64>()},
+        {"marks_bytes",                                std::make_shared<DataTypeUInt64>()},
+        {"modification_time",                          std::make_shared<DataTypeDateTime>()},
+        {"remove_time",                                std::make_shared<DataTypeDateTime>()},
+        {"refcount",                                   std::make_shared<DataTypeUInt32>()},
+        {"min_date",                                   std::make_shared<DataTypeDate>()},
+        {"max_date",                                   std::make_shared<DataTypeDate>()},
+        {"min_block_number",                           std::make_shared<DataTypeInt64>()},
+        {"max_block_number",                           std::make_shared<DataTypeInt64>()},
+        {"level",                                      std::make_shared<DataTypeUInt32>()},
+        {"primary_key_bytes_in_memory",                std::make_shared<DataTypeUInt64>()},
+        {"primary_key_bytes_in_memory_allocated",      std::make_shared<DataTypeUInt64>()},
 
-                      {"database",            std::make_shared<DataTypeString>()},
-                      {"table",               std::make_shared<DataTypeString>()},
-                      {"engine",              std::make_shared<DataTypeString>()},
-                      {"path",                std::make_shared<DataTypeString>()},
-                      {"column",              std::make_shared<DataTypeString>()},
-                      {"type",                std::make_shared<DataTypeString>() },
-                      {"default_kind",        std::make_shared<DataTypeString>() },
-                      {"default_expression",  std::make_shared<DataTypeString>() },
-                      {"data_compressed_bytes",   std::make_shared<DataTypeUInt64>() },
-                      {"data_uncompressed_bytes", std::make_shared<DataTypeUInt64>() },
-                      {"marks_bytes_in_column",   std::make_shared<DataTypeUInt64>() },
-              }
-        )
+        {"database",                                   std::make_shared<DataTypeString>()},
+        {"table",                                      std::make_shared<DataTypeString>()},
+        {"engine",                                     std::make_shared<DataTypeString>()},
+        {"path",                                       std::make_shared<DataTypeString>()},
+
+        {"column",                                     std::make_shared<DataTypeString>()},
+        {"type",                                       std::make_shared<DataTypeString>() },
+        {"default_kind",                               std::make_shared<DataTypeString>() },
+        {"default_expression",                         std::make_shared<DataTypeString>() },
+        {"column_bytes_on_disk",                       std::make_shared<DataTypeUInt64>() },
+        {"column_data_compressed_bytes",               std::make_shared<DataTypeUInt64>() },
+        {"column_data_uncompressed_bytes",             std::make_shared<DataTypeUInt64>() },
+        {"column_marks_bytes",                         std::make_shared<DataTypeUInt64>() },
+    }
+    )
 {
 }
 
@@ -84,8 +88,8 @@ void StorageSystemPartsColumns::processNextStorage(MutableColumns & columns, con
     {
         const auto & part = info.all_parts[part_number];
         auto part_state = info.all_parts_state[part_number];
+        auto columns_size = part->getTotalColumnsSize();
 
-        auto total_mrk_size_in_bytes = part->getTotalMrkSizeInBytes();
         /// For convenience, in returned refcount, don't add references that was due to local variables in this method: all_parts, active_parts.
         auto use_count = part.use_count() - 1;
         auto min_date = part->getMinDate();
@@ -106,10 +110,12 @@ void StorageSystemPartsColumns::processNextStorage(MutableColumns & columns, con
             columns[j++]->insert(part->name);
             columns[j++]->insert(static_cast<UInt64>(part_state == State::Committed));
             columns[j++]->insert(static_cast<UInt64>(part->marks_count));
-            columns[j++]->insert(static_cast<UInt64>(total_mrk_size_in_bytes));
 
             columns[j++]->insert(static_cast<UInt64>(part->rows_count));
-            columns[j++]->insert(static_cast<UInt64>(part->size_in_bytes));
+            columns[j++]->insert(static_cast<UInt64>(part->bytes_on_disk));
+            columns[j++]->insert(static_cast<UInt64>(columns_size.data_compressed));
+            columns[j++]->insert(static_cast<UInt64>(columns_size.data_uncompressed));
+            columns[j++]->insert(static_cast<UInt64>(columns_size.marks));
             columns[j++]->insert(static_cast<UInt64>(part->modification_time));
             columns[j++]->insert(static_cast<UInt64>(part->remove_time.load(std::memory_order_relaxed)));
 
@@ -142,9 +148,11 @@ void StorageSystemPartsColumns::processNextStorage(MutableColumns & columns, con
                 columns[j++]->insertDefault();
             }
 
-            columns[j++]->insert(static_cast<UInt64>(part->getColumnCompressedSize(column.name)));
-            columns[j++]->insert(static_cast<UInt64>(part->getColumnUncompressedSize(column.name)));
-            columns[j++]->insert(static_cast<UInt64>(part->getColumnMrkSize(column.name)));
+            MergeTreeDataPart::ColumnSize column_size = part->getColumnSize(column.name, *column.type);
+            columns[j++]->insert(static_cast<UInt64>(column_size.data_compressed + column_size.marks));
+            columns[j++]->insert(static_cast<UInt64>(column_size.data_compressed));
+            columns[j++]->insert(static_cast<UInt64>(column_size.data_uncompressed));
+            columns[j++]->insert(static_cast<UInt64>(column_size.marks));
 
             if (has_state_column)
                 columns[j++]->insert(part->stateString());
diff --git a/dbms/src/Storages/System/StorageSystemTables.cpp b/dbms/src/Storages/System/StorageSystemTables.cpp
index 99ed89739f9..9f9903d8eb2 100644
--- a/dbms/src/Storages/System/StorageSystemTables.cpp
+++ b/dbms/src/Storages/System/StorageSystemTables.cpp
@@ -200,17 +200,7 @@ BlockInputStreams StorageSystemTables::read(
 
             if (has_create_table_query || has_engine_full)
             {
-                ASTPtr ast;
-
-                try
-                {
-                    ast = database->getCreateQuery(context, table_name);
-                }
-                catch (const Exception & e)
-                {
-                    if (e.code() != ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY)
-                        throw;
-                }
+                ASTPtr ast = database->tryGetCreateTableQuery(context, table_name);
 
                 if (has_create_table_query)
                     res_columns[j++]->insert(ast ? queryToString(ast) : "");
diff --git a/dbms/src/Storages/tests/remove_symlink_directory.cpp b/dbms/src/Storages/tests/remove_symlink_directory.cpp
index 16b2a1b4750..59011c557ee 100644
--- a/dbms/src/Storages/tests/remove_symlink_directory.cpp
+++ b/dbms/src/Storages/tests/remove_symlink_directory.cpp
@@ -1,6 +1,5 @@
-#include <unistd.h>
+#include <port/unistd.h>
 #include <iostream>
-
 #include <Poco/File.h>
 #include <Poco/Path.h>
 #include <Common/Exception.h>
diff --git a/dbms/tests/clickhouse-test b/dbms/tests/clickhouse-test
index 4e9886f8b70..b88eab93c7c 100755
--- a/dbms/tests/clickhouse-test
+++ b/dbms/tests/clickhouse-test
@@ -81,9 +81,12 @@ def main(args):
     # TODO ! use clickhouse-extract-from-config here:
     if args.zookeeper is None:
         code, out = commands.getstatusoutput(args.binary + "-extract-from-config --try --config " + args.configserver + ' --key zookeeper | grep . | wc -l')
-        if int(out) > 0:
-            args.zookeeper = True
-        else:
+        try:
+            if int(out) > 0:
+                args.zookeeper = True
+            else:
+                args.zookeeper = False
+        except ValueError:
             args.zookeeper = False
 
     if args.shard is None:
diff --git a/dbms/tests/performance/functions_length/functions_length.xml b/dbms/tests/performance/functions_length/functions_length.xml
new file mode 100644
index 00000000000..d285cd2422f
--- /dev/null
+++ b/dbms/tests/performance/functions_length/functions_length.xml
@@ -0,0 +1,38 @@
+<test>
+    <name>functions_length</name>
+    <type>once</type>
+
+    <stop_conditions>
+        <all_of>
+            <total_time_ms>10000</total_time_ms>
+        </all_of>
+        <any_of>
+            <average_speed_not_changing_for_ms>5000</average_speed_not_changing_for_ms>
+            <total_time_ms>20000</total_time_ms>
+        </any_of>
+    </stop_conditions>
+
+    <main_metric>
+        <avg_rows_per_second/>
+    </main_metric>
+
+    <substitutions>
+        <substitution>
+            <name>string</name>
+            <values>
+                <value>materialize('')</value>
+                <value>materialize('Hello, world')</value>
+                <value>toString(number)</value>
+                <value>reinterpretAsString(number)</value>
+                <value>materialize('中文测试字符串')</value>
+                <value>materialize('https://github.com/yandex/ClickHouse/pull/1882')</value>
+                <value>materialize('https://zh.wikipedia.org/wiki/%E4%B8%AD%E6%97%A5%E9%9F%93%E7%B5%B1%E4%B8%80%E8%A1%A8%E6%84%8F%E6%96%87%E5%AD%97%E6%93%B4%E5%B1%95%E5%8D%80F')</value>
+                <value>concat('中文测试字符串 ', toString(number), ' Привет, мир!')</value>
+                <value>concat(concat('中文测试字符串 ', toString(number), ' Привет, мир!') AS x, x, x, x, x, x, x, x, x, x)</value>
+                <value>convertCharset(concat(reinterpretAsString(rand64(1)), reinterpretAsString(rand64(2)), reinterpretAsString(rand64(3)), reinterpretAsString(rand64(4)), reinterpretAsString(rand64(5)), reinterpretAsString(rand64(6)), reinterpretAsString(rand64(7)), reinterpretAsString(rand64(8)), reinterpretAsString(rand64(9)), reinterpretAsString(rand64(10))), 'UTF-16', 'UTF-8')</value>
+            </values>
+        </substitution>
+    </substitutions>
+
+    <query>SELECT count() FROM system.numbers WHERE NOT ignore(lengthUTF8({string}))</query>
+</test>
diff --git a/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh b/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh
index 12f20a956ad..8e762515da1 100755
--- a/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh
+++ b/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 # This test will fail with external poco (progress not supported)
 
-${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}?max_block_size=1&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0" -d 'SELECT number FROM system.numbers LIMIT 10' 2>&1 | grep -E 'Content-Encoding|X-ClickHouse|^[0-9]'
+${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}?max_block_size=1&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0" -d 'SELECT number FROM system.numbers LIMIT 10' 2>&1 | grep -E 'Content-Encoding|X-ClickHouse-Progress|^[0-9]'
 ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?max_block_size=1&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0&enable_http_compression=1" -H 'Accept-Encoding: gzip' -d 'SELECT number FROM system.numbers LIMIT 10' | gzip -d
 
 # 'send_progress_in_http_headers' is false by default
diff --git a/dbms/tests/queries/0_stateless/00501_http_head.sh b/dbms/tests/queries/0_stateless/00501_http_head.sh
index c15b8679fab..578e27da751 100755
--- a/dbms/tests/queries/0_stateless/00501_http_head.sh
+++ b/dbms/tests/queries/0_stateless/00501_http_head.sh
@@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . $CURDIR/../shell_config.sh
 
 ( ${CLICKHOUSE_CURL} -s --head "${CLICKHOUSE_URL}?query=SELECT%201";
-  ${CLICKHOUSE_CURL} -s --head "${CLICKHOUSE_URL}?query=select+*+from+system.numbers+limit+1000000" ) | grep -v "Date:"
+  ${CLICKHOUSE_CURL} -s --head "${CLICKHOUSE_URL}?query=select+*+from+system.numbers+limit+1000000" ) | grep -v "Date:" | grep -v "X-ClickHouse-Server-Display-Name:"
 
 if [[ `${CLICKHOUSE_CURL} -sS -X POST -I "${CLICKHOUSE_URL}?query=SELECT+1" | grep -c '411 Length Required'` -ne 1 ]]; then
     echo FAIL
diff --git a/dbms/tests/queries/0_stateless/00534_filimonov.data b/dbms/tests/queries/0_stateless/00534_filimonov.data
index 53c64592d0a..2dd470403c0 100644
--- a/dbms/tests/queries/0_stateless/00534_filimonov.data
+++ b/dbms/tests/queries/0_stateless/00534_filimonov.data
@@ -427,3 +427,6 @@ SELECT COVAR_POPArray([CAST( 0  AS Int8)],arrayPopBack([CAST( 0  AS Int8)]));
 SELECT COVAR_SAMPArray([CAST( 0  AS Int8)],arrayPopBack([CAST( 0  AS Int8)]));
 SELECT medianTimingWeightedArray([CAST( 0  AS Int8)],arrayPopBack([CAST( 0  AS Int8)]));
 SELECT quantilesDeterministicArray([CAST( 0  AS Int8)],arrayPopBack([CAST( 0  AS Int32)]));
+
+SELECT maxIntersections([], [])
+SELECT sumMap([], [])
diff --git a/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments7.sh b/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments7.sh
index 78b9afafe55..af34af94458 100755
--- a/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments7.sh
+++ b/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments7.sh
@@ -5,4 +5,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 . $CURDIR/00534_long_functions_bad_arguments.lib
 
-test_variant 'SELECT $_('');'
+test_variant "SELECT \$_('');"
diff --git a/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments8.sh b/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments8.sh
new file mode 100755
index 00000000000..f884a926c3b
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments8.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+. $CURDIR/../shell_config.sh
+
+. $CURDIR/00534_long_functions_bad_arguments.lib
+
+test_variant 'SELECT $_(NULL, NULL);'
+test_variant 'SELECT $_([], []);'
+test_variant 'SELECT $_([NULL],[NULL]);'
diff --git a/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments9.sh b/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments9.sh
new file mode 100755
index 00000000000..bc1ceda33fa
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments9.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+. $CURDIR/../shell_config.sh
+
+. $CURDIR/00534_long_functions_bad_arguments.lib
+
+test_variant 'SELECT $_(NULL, NULL, NULL);'
+test_variant 'SELECT $_([], [], []);'
+test_variant 'SELECT $_([NULL], [NULL], [NULL]);'
diff --git a/dbms/tests/queries/0_stateless/00604_show_create_database.reference b/dbms/tests/queries/0_stateless/00604_show_create_database.reference
new file mode 100644
index 00000000000..ab242ee47fd
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00604_show_create_database.reference
@@ -0,0 +1 @@
+CREATE DATABASE test ENGINE = Ordinary
diff --git a/dbms/tests/queries/0_stateless/00604_show_create_database.sql b/dbms/tests/queries/0_stateless/00604_show_create_database.sql
new file mode 100644
index 00000000000..b5671b3a984
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00604_show_create_database.sql
@@ -0,0 +1,2 @@
+create database if not exists test;
+show create database test;
diff --git a/dbms/tests/queries/0_stateless/00609_distributed_with_case_when_then.reference b/dbms/tests/queries/0_stateless/00609_distributed_with_case_when_then.reference
new file mode 100644
index 00000000000..c5bb9a19a34
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00609_distributed_with_case_when_then.reference
@@ -0,0 +1,6 @@
+hello
+world
+2
+hello
+world
+2
diff --git a/dbms/tests/queries/0_stateless/00609_distributed_with_case_when_then.sql b/dbms/tests/queries/0_stateless/00609_distributed_with_case_when_then.sql
new file mode 100644
index 00000000000..8d7568415a2
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00609_distributed_with_case_when_then.sql
@@ -0,0 +1,16 @@
+DROP TABLE IF EXISTS test.mergetree;
+DROP TABLE IF EXISTS test.distributed;
+
+CREATE TABLE test.mergetree (x UInt64, s String) ENGINE = MergeTree ORDER BY x;
+INSERT INTO test.mergetree VALUES (1, 'hello'), (2, 'world');
+
+SELECT CASE x WHEN 1 THEN 'hello' WHEN 2 THEN 'world' ELSE  'unknow' END FROM test.mergetree;
+SELECT count() AS cnt FROM (SELECT CASE x WHEN 1 THEN 'hello' WHEN 2 THEN 'world' ELSE  'unknow' END FROM test.mergetree);
+
+CREATE TABLE test.distributed AS test.mergetree ENGINE = Distributed(test_shard_localhost, test, mergetree);
+
+SELECT CASE x WHEN 1 THEN 'hello' WHEN 2 THEN 'world' ELSE  'unknow' END FROM test.distributed;
+SELECT count() AS cnt FROM (SELECT CASE x WHEN 1 THEN 'hello' WHEN 2 THEN 'world' ELSE  'unknow' END FROM test.distributed);
+
+DROP TABLE test.mergetree;
+DROP TABLE test.distributed;
diff --git a/dbms/tests/queries/0_stateless/00609_prewhere_and_default.reference b/dbms/tests/queries/0_stateless/00609_prewhere_and_default.reference
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/dbms/tests/queries/0_stateless/00609_prewhere_and_default.sql b/dbms/tests/queries/0_stateless/00609_prewhere_and_default.sql
new file mode 100644
index 00000000000..915f0c85823
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00609_prewhere_and_default.sql
@@ -0,0 +1,12 @@
+drop table if exists test.table;
+create table test.table (key UInt64, val UInt64) engine = MergeTree order by key settings index_granularity=8192;
+insert into test.table select number, number / 8192 from system.numbers limit 100000; 
+alter table test.table add column def UInt64 default val + 1;
+select * from test.table prewhere val > 2 format Null;
+
+drop table if exists test.table;
+create table test.table (key UInt64, val UInt64) engine = MergeTree order by key settings index_granularity=8192;
+insert into test.table select number, number / 8192 from system.numbers limit 100000; 
+alter table test.table add column def UInt64;
+select * from test.table prewhere val > 2 format Null;
+
diff --git a/dbms/tests/queries/0_stateless/00611_zookeeper_different_checksums_formats.reference b/dbms/tests/queries/0_stateless/00611_zookeeper_different_checksums_formats.reference
new file mode 100644
index 00000000000..da05f197b7e
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00611_zookeeper_different_checksums_formats.reference
@@ -0,0 +1,9 @@
+1
+1
+0	[]
+1	[]
+0	[]
+1	[]
+DETACH
+0	[]
+1	[]
diff --git a/dbms/tests/queries/0_stateless/00611_zookeeper_different_checksums_formats.sql b/dbms/tests/queries/0_stateless/00611_zookeeper_different_checksums_formats.sql
new file mode 100644
index 00000000000..9076485cfa1
--- /dev/null
+++ b/dbms/tests/queries/0_stateless/00611_zookeeper_different_checksums_formats.sql
@@ -0,0 +1,24 @@
+DROP TABLE IF EXISTS test.table_old;
+DROP TABLE IF EXISTS test.table_new;
+
+CREATE TABLE test.table_old (k UInt64, d Array(String)) ENGINE = ReplicatedMergeTree('/clickhouse/test/tables/checksums_test', 'old') ORDER BY k SETTINGS use_minimalistic_checksums_in_zookeeper=0;
+CREATE TABLE test.table_new (k UInt64, d Array(String)) ENGINE = ReplicatedMergeTree('/clickhouse/test/tables/checksums_test', 'new') ORDER BY k SETTINGS use_minimalistic_checksums_in_zookeeper=1;
+
+SET insert_quorum=2;
+INSERT INTO test.table_old VALUES (0, []);
+SELECT value LIKE '%checksums format version: 4%' FROM system.zookeeper WHERE path='/clickhouse/test/tables/checksums_test/replicas/old/parts/all_0_0_0' AND name = 'checksums';
+
+INSERT INTO test.table_new VALUES (1, []);
+SELECT value LIKE '%checksums format version: 5%' FROM system.zookeeper WHERE path='/clickhouse/test/tables/checksums_test/replicas/new/parts/all_1_1_0' AND name = 'checksums';
+
+OPTIMIZE TABLE test.table_old;
+SELECT * FROM test.table_old ORDER BY k;
+SELECT * FROM test.table_new ORDER BY k;
+
+SELECT 'DETACH';
+DETACH TABLE test.table_old;
+ATTACH TABLE test.table_old (k UInt64, d Array(String)) ENGINE = ReplicatedMergeTree('/clickhouse/test/tables/checksums_test', 'old') ORDER BY k SETTINGS use_minimalistic_checksums_in_zookeeper=1;
+SELECT * FROM test.table_old ORDER BY k;
+
+DROP TABLE IF EXISTS test.table_old;
+DROP TABLE IF EXISTS test.table_new;
\ No newline at end of file
diff --git a/debian/.pbuilderrc b/debian/.pbuilderrc
index e3704760ec1..ba1cdb2c324 100644
--- a/debian/.pbuilderrc
+++ b/debian/.pbuilderrc
@@ -188,7 +188,7 @@ export DEB_BUILD_OPTIONS=parallel=`nproc`
 
 # Floating bug with permissions:
 [ -n "$CCACHEDIR" ] && sudo mkdir -p $CCACHEDIR
-[ -n "$CCACHEDIR" ] && sudo chmod -R a+rwx $CCACHEDIR
+[ -n "$CCACHEDIR" ] && sudo chmod -R a+rwx $CCACHEDIR || true
 # chown -R $BUILDUSERID:$BUILDUSERID $CCACHEDIR
 
 
diff --git a/debian/changelog b/debian/changelog
index 75502e3cc96..2ff28b59456 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,5 @@
-clickhouse (1.1.54370) unstable; urgency=low
+clickhouse (1.1.54371) unstable; urgency=low
 
   * Modified source code
 
- --  <robot-metrika-test@yandex-team.ru>  Fri, 16 Mar 2018 12:01:50 +0300
+ --  <robot-metrika-test@yandex-team.ru>  Fri, 23 Mar 2018 01:11:42 +0300
diff --git a/debian/clickhouse-server-base.install b/debian/clickhouse-server-base.install
index 9d3d470b040..573f34042bb 100644
--- a/debian/clickhouse-server-base.install
+++ b/debian/clickhouse-server-base.install
@@ -3,6 +3,7 @@
 /usr/bin/clickhouse-clang
 /usr/bin/clickhouse-lld
 /usr/bin/clickhouse-copier
+/usr/bin/clickhouse-report
 /etc/systemd/system/clickhouse-server.service
 /etc/init.d/clickhouse-server
 /etc/cron.d/clickhouse-server
diff --git a/debian/clickhouse-server-base.postinst b/debian/clickhouse-server-base.postinst
index c7aa47b5ef9..6946f8b2728 100644
--- a/debian/clickhouse-server-base.postinst
+++ b/debian/clickhouse-server-base.postinst
@@ -27,8 +27,13 @@ if [ "$1" = configure ]; then
 
     # Make sure the administrative user exists
     if ! getent passwd ${CLICKHOUSE_USER} > /dev/null; then
-        adduser --system --disabled-login --no-create-home --home /nonexistent \
-            --shell /bin/false --group --gecos "Clickhouse server" clickhouse > /dev/null
+        if [ "$OS" == "rhel" ] || [ "$OS" == "centos" ] || [ "$OS" == "fedora" ]; then
+            adduser --system --no-create-home --home /nonexistent \
+                --shell /bin/false clickhouse > /dev/null
+        else
+            adduser --system --disabled-login --no-create-home --home /nonexistent \
+                --shell /bin/false --group --gecos "ClickHouse server" clickhouse > /dev/null
+        fi
     fi
 
     # if the user was created manually, make sure the group is there as well
diff --git a/debian/control b/debian/control
index 79666ace2e5..5edac5f66a8 100644
--- a/debian/control
+++ b/debian/control
@@ -4,6 +4,7 @@ Priority: optional
 Maintainer: Alexey Milovidov <milovidov@yandex-team.ru>
 Build-Depends: debhelper (>= 9),
                cmake3 | cmake,
+               ninja-build,
                gcc-7, g++-7,
                libc6-dev,
                libmariadbclient-dev | default-libmysqlclient-dev | libmysqlclient-dev,
diff --git a/debian/rules b/debian/rules
index 8f3818c0478..1cb7de10224 100755
--- a/debian/rules
+++ b/debian/rules
@@ -2,14 +2,15 @@
 # -*- makefile -*-
 
 # Uncomment this to turn on verbose mode.
-#export DH_VERBOSE=1
+export DH_VERBOSE=1
 
 # -pie only for static mode
 export DEB_BUILD_MAINT_OPTIONS=hardening=+all,-pie
 
 # because copy_headers.sh have hardcoded path to build/include_directories.txt
 BUILDDIR = build
-DESTDIR = debian/tmp
+CURDIR = $(shell pwd)
+DESTDIR = $(CURDIR)/debian/tmp
 
 DEB_HOST_MULTIARCH ?= $(shell dpkg-architecture -qDEB_HOST_MULTIARCH)
 
@@ -41,6 +42,8 @@ endif
 endif
 
 CMAKE_FLAGS += -DCMAKE_CXX_COMPILER=`which $(CXX)` -DCMAKE_C_COMPILER=`which $(CC)`
+CMAKE_FLAGS += -GNinja
+export MAKE=ninja
 
 ifndef DH_VERBOSE
     CMAKE_FLAGS += -DCMAKE_VERBOSE_MAKEFILE=0
@@ -92,12 +95,15 @@ override_dh_install:
 	CLANG=$(DESTDIR)/usr/bin/clickhouse-clang ./copy_headers.sh . $(DESTDIR)/usr/share/clickhouse/headers
 
 	# fake metrika files when private dir is empty
-	mkdir -p debian/tmp/etc/clickhouse-server/metrika
-	touch debian/tmp/etc/clickhouse-server/metrika/config.xml
-	touch debian/tmp/etc/clickhouse-server/metrika/users.xml
+	mkdir -p $(DESTDIR)/etc/clickhouse-server/metrika
+	touch $(DESTDIR)/etc/clickhouse-server/metrika/config.xml
+	touch $(DESTDIR)/etc/clickhouse-server/metrika/users.xml
 
 	dh_install --list-missing --sourcedir=$(DESTDIR)
 
+override_dh_auto_install:
+	env DESTDIR=$(DESTDIR) ninja -C $(BUILDDIR) install
+
 override_dh_shlibdeps:
 	true # We depend only on libc and dh_shlibdeps gives us wrong (too strict) dependency.
 
diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index a87328cb06c..e006f302c68 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -338,15 +338,3 @@ It works for JSONEachRow and TSKV formats.
 ## output_format_json_quote_64bit_integers
 
 If the value is true, integers appear in quotes when using JSON\* Int64 and UInt64 formats  (for compatibility with most JavaScript implementations); otherwise, integers are output without the quotes.
-
-<a name="settings-strict_insert_defaults"></a>
-
-## strict_insert_defaults
-
-Strictly assigns default values when adding data.
-
-If data is not specified for a column when running an [ INSERT](../../query_language/queries.md#queries-insert)  query, ClickHouse assigns default values to the fields. Default values are defined in the ` DEFAULT` property for each column in the table settings. If ` DEFAULT` is not defined for a column:
-
-- `If strict_insert_defaults = 0`, column fields are assigned zeros and empty strings.
-- `If strict_insert_defaults = 1`, ClickHouse throws an exception and requires the user to pass data to the column.
-
diff --git a/docs/en/table_engines/kafka.md b/docs/en/table_engines/kafka.md
index 55d2027309d..85943e44fc5 100644
--- a/docs/en/table_engines/kafka.md
+++ b/docs/en/table_engines/kafka.md
@@ -9,7 +9,7 @@ Kafka lets you:
 - Process streams as they become available.
 
 ```
-Kafka(broker_list, topic_list, group_name, format[, schema])
+Kafka(broker_list, topic_list, group_name, format[, schema, num_consumers])
 ```
 
 Parameters:
@@ -19,6 +19,7 @@ Parameters:
 - `group_name` – A group of Kafka consumers (`group1`). Reading margins are tracked for each group separately. If you don't want messages to be duplicated in the cluster, use the same group name everywhere.
 - `format` – Message format. Uses the same notation as the SQL ` FORMAT` function, such as ` JSONEachRow`. For more information, see the section "Formats".
 - `schema` – An optional parameter that must be used if the format requires a schema definition. For example, [Cap'n Proto](https://capnproto.org/)  requires the path to the schema file and the name of the root ` schema.capnp:Message` object.
+- `num_consumers` - Number of created consumers per engine. By default `1`. Create more consumers if the throughput of a single consumer is insufficient. The total number of consumers shouldn't exceed the number of partitions in given topic, as there can be at most 1 consumers assigned to any single partition.
 
 Example:
 
@@ -77,3 +78,23 @@ To stop receiving topic data or to change the conversion logic, detach the mater
 
 If you want to change the target table by using `ALTER` materialized view, we recommend disabling the material view to avoid discrepancies between the target table and the data from the view.
 
+
+## Configuration
+
+Similarly to GraphiteMergeTree, Kafka engine supports extended configuration through the ClickHouse config file. There are two configuration keys you can use - global (`kafka`), and per-topic (`kafka_topic_*`). The global configuration is applied first, then per-topic configuration (if exists).
+
+```xml
+  <!--  Global configuration options for all tables of Kafka engine type -->
+  <kafka>
+    <debug>cgrp</debug>
+    <auto_offset_reset>smallest</auto_offset_reset>
+  </kafka>
+
+  <!-- Configuration specific for topic "logs" -->
+  <kafka_topic_logs>
+    <retry_backoff_ms>250</retry_backoff_ms>
+    <fetch_min_bytes>100000</fetch_min_bytes>
+  </kafka_topic_logs>
+```
+
+See [librdkafka configuration reference](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md) for the list of possible configuration options. Use underscores instead of dots in the ClickHouse configuration, for example `check.crcs=true` would correspond to `<check_crcs>true</check_crcs>`.
diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md
index 624dadcf096..62d183234a4 100644
--- a/docs/ru/operations/settings/settings.md
+++ b/docs/ru/operations/settings/settings.md
@@ -335,15 +335,3 @@ ClickHouse применяет настройку в том случае, ког
 ## output_format_json_quote_64bit_integers
 
 Если значение истинно, то при использовании JSON\* форматов UInt64 и Int64 числа выводятся в кавычках (из соображений совместимости с большинством реализаций JavaScript), иначе - без кавычек.
-
-<a name="settings-strict_insert_defaults"></a>
-
-## strict_insert_defaults
-
-Строгое присвоение значений по умолчанию при добавлении данных.
-
-Если при выполнении запроса [INSERT](../../query_language/queries.md#queries-insert) данные для столбца не заданы, то ClickHouse присваивает полям значения по умолчанию. Значения по умолчанию определяются свойством `DEFAULT` для каждого столбца в настройках таблицы. Если для столбца не определен `DEFAULT`, то когда:
-
--   `strict_insert_defaults=0` - полям столбца присваиваются нули и пустые строки.
--   `strict_insert_defaults=1` - ClickHouse генерирует исключение и обязывает пользователя передать данные в столбец.
-
diff --git a/docs/ru/table_engines/kafka.md b/docs/ru/table_engines/kafka.md
index 127820a8ef2..8225192e337 100644
--- a/docs/ru/table_engines/kafka.md
+++ b/docs/ru/table_engines/kafka.md
@@ -9,7 +9,7 @@ Kafka позволяет:
 - Обрабатывать потоки по мере их появления.
 
 ```
-Kafka(broker_list, topic_list, group_name, format[, schema])
+Kafka(broker_list, topic_list, group_name, format[, schema, num_consumers])
 ```
 
 Параметры:
@@ -19,6 +19,7 @@ Kafka(broker_list, topic_list, group_name, format[, schema])
 - `group_name` - Группа потребителя Kafka (`group1`). Отступы для чтения отслеживаются для каждой группы отдельно. Если необходимо, чтобы сообщения не повторялись на кластере, используйте везде одно имя группы.
 - `format` - Формат сообщений. Имеет те же обозначения, что выдает SQL-выражение `FORMAT`, например, `JSONEachRow`. Подробнее смотрите в разделе "Форматы".
 - `schema` - Опциональный параметр, необходимый, если используется формат, требующий определения схемы. Например, [Cap'n Proto](https://capnproto.org/) требует путь к файлу со схемой и название корневого объекта `schema.capnp:Message`.
+- `num_consumers` - Количество потребителей (consumer) на таблицу. По умолчанию `1`. Укажите больше потребителей, если пропускная способность одного потребителя недостаточна. Общее число потребителей не должно превышать количество партиций в топике, так как на одну партицию может быть назначено не более одного потребителя.
 
 Пример:
 
@@ -75,4 +76,25 @@ Kafka(broker_list, topic_list, group_name, format[, schema])
   ATTACH MATERIALIZED VIEW consumer;
 ```
 
-Если необходимо изменить целевую таблицу с помощью `ALTER`, то материализованное представление рекомендуется отключить, чтобы избежать несостыковки между целевой таблицей и данными от представления.
\ No newline at end of file
+Если необходимо изменить целевую таблицу с помощью `ALTER`, то материализованное представление рекомендуется отключить, чтобы избежать несостыковки между целевой таблицей и данными от представления.
+
+
+## Конфигурация
+
+Аналогично GraphiteMergeTree, движок Kafka поддерживает расширенную конфигурацию с помощью конфигурационного файла ClickHouse. Существует два конфигурационных ключа, которые можно использовать - глобальный (`kafka`) и по топикам (`kafka_topic_*`). Сначала применяется глобальная конфигурация, затем конфигурация по топикам (если она существует).
+
+```xml
+  <!--  Global configuration options for all tables of Kafka engine type -->
+  <kafka>
+    <debug>cgrp</debug>
+    <auto_offset_reset>smallest</auto_offset_reset>
+  </kafka>
+
+  <!-- Configuration specific for topic "logs" -->
+  <kafka_topic_logs>
+    <retry_backoff_ms>250</retry_backoff_ms>
+    <fetch_min_bytes>100000</fetch_min_bytes>
+  </kafka_topic_logs>
+```
+
+В документе [librdkafka configuration reference](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md) можно увидеть список возможных опций конфигурации. Используйте подчёркивания (`_`) вместо точек в конфигурации ClickHouse, например, `check.crcs=true` будет соответствовать `<check_crcs>true</check_crcs>`.
\ No newline at end of file
diff --git a/libs/libcommon/cmake/find_gperftools.cmake b/libs/libcommon/cmake/find_gperftools.cmake
index b9fcc40c0c6..a311f2ba74e 100644
--- a/libs/libcommon/cmake/find_gperftools.cmake
+++ b/libs/libcommon/cmake/find_gperftools.cmake
@@ -4,7 +4,7 @@ else ()
     option (USE_INTERNAL_GPERFTOOLS_LIBRARY "Set to FALSE to use system gperftools (tcmalloc) library instead of bundled" ${NOT_UNBUNDLED})
 endif ()
 
-if (ARCH_FREEBSD)
+if (ARCH_FREEBSD OR MSVC)
     option (ENABLE_TCMALLOC "Set to TRUE to enable tcmalloc" OFF)
 else ()
     option (ENABLE_TCMALLOC "Set to TRUE to enable tcmalloc" ON)
diff --git a/libs/libcommon/include/common/DateLUT.h b/libs/libcommon/include/common/DateLUT.h
index c86b4010108..75a7457d101 100644
--- a/libs/libcommon/include/common/DateLUT.h
+++ b/libs/libcommon/include/common/DateLUT.h
@@ -2,7 +2,6 @@
 
 #include <common/DateLUTImpl.h>
 #include <ext/singleton.h>
-
 #include <unordered_map>
 #include <atomic>
 #include <mutex>
@@ -10,8 +9,12 @@
 
 // Also defined in Core/Defines.h
 #if !defined(ALWAYS_INLINE)
+#if defined(_MSC_VER)
+    #define ALWAYS_INLINE __forceinline
+#else
     #define ALWAYS_INLINE __attribute__((__always_inline__))
 #endif
+#endif
 
 
 /// This class provides lazy initialization and lookup of singleton DateLUTImpl objects for a given timezone.
diff --git a/libs/libcommon/include/common/LocalDate.h b/libs/libcommon/include/common/LocalDate.h
index 93d045786e9..8eddc6f9115 100644
--- a/libs/libcommon/include/common/LocalDate.h
+++ b/libs/libcommon/include/common/LocalDate.h
@@ -18,7 +18,7 @@
   *
   * packed - for memcmp to work naturally (but because m_year is 2 bytes, on little endian, comparison is correct only before year 2047)
   */
-class __attribute__ ((__packed__)) LocalDate
+class LocalDate
 {
 private:
     unsigned short m_year;
diff --git a/libs/libcommon/include/common/LocalDateTime.h b/libs/libcommon/include/common/LocalDateTime.h
index 133bef46723..d0c208edadc 100644
--- a/libs/libcommon/include/common/LocalDateTime.h
+++ b/libs/libcommon/include/common/LocalDateTime.h
@@ -17,7 +17,7 @@
   *
   * packed - for memcmp to work naturally (but because m_year is 2 bytes, on little endian, comparison is correct only before year 2047)
   */
-class __attribute__ ((__packed__)) LocalDateTime
+class LocalDateTime
 {
 private:
     unsigned short m_year;
diff --git a/libs/libcommon/include/common/likely.h b/libs/libcommon/include/common/likely.h
index 7a0a98fcdc0..787d46c0e67 100644
--- a/libs/libcommon/include/common/likely.h
+++ b/libs/libcommon/include/common/likely.h
@@ -1,4 +1,9 @@
 #pragma once
 
-#define likely(x)    (__builtin_expect(!!(x), 1))
-#define unlikely(x)    (__builtin_expect(!!(x), 0))
+#if _MSC_VER
+#define likely(x)   (x)
+#define unlikely(x) (x)
+#else
+#define likely(x)   (__builtin_expect(!!(x), 1))
+#define unlikely(x) (__builtin_expect(!!(x), 0))
+#endif
diff --git a/libs/libcommon/include/common/mremap.h b/libs/libcommon/include/common/mremap.h
index a6a841adc52..edb62070247 100644
--- a/libs/libcommon/include/common/mremap.h
+++ b/libs/libcommon/include/common/mremap.h
@@ -1,7 +1,10 @@
 #pragma once
 
 #include <cstddef>
+#include <sys/types.h>
+#if !_MSC_VER
 #include <sys/mman.h>
+#endif
 
 #if defined(MREMAP_MAYMOVE)
 // we already have implementation (linux)
diff --git a/libs/libcommon/include/port/ssize_t.h b/libs/libcommon/include/port/ssize_t.h
new file mode 100644
index 00000000000..62199bbe1f0
--- /dev/null
+++ b/libs/libcommon/include/port/ssize_t.h
@@ -0,0 +1,8 @@
+#pragma once
+
+#ifdef _MSC_VER
+    #include <basetsd.h>
+    typedef SSIZE_T ssize_t;
+#else
+    #include <sys/types.h>
+#endif
diff --git a/libs/libcommon/include/port/unistd.h b/libs/libcommon/include/port/unistd.h
new file mode 100644
index 00000000000..2c08bd0bcd2
--- /dev/null
+++ b/libs/libcommon/include/port/unistd.h
@@ -0,0 +1,11 @@
+#pragma once
+
+// https://stackoverflow.com/questions/341817/is-there-a-replacement-for-unistd-h-for-windows-visual-c
+
+#ifdef _MSC_VER
+    #include <io.h>
+#else
+    #include <unistd.h>
+#endif
+
+#include "ssize_t.h"
diff --git a/libs/libcommon/src/DateLUT.cpp b/libs/libcommon/src/DateLUT.cpp
index 40634698dbc..22c50a9f694 100644
--- a/libs/libcommon/src/DateLUT.cpp
+++ b/libs/libcommon/src/DateLUT.cpp
@@ -56,13 +56,13 @@ std::string determineDefaultTimeZone()
         /// then the relative path is the time zone id.
         fs::path relative_path = tz_file_path.lexically_relative(tz_database_path);
         if (!relative_path.empty() && *relative_path.begin() != ".." && *relative_path.begin() != ".")
-            return relative_path.native();
+            return relative_path.string();
 
         /// The file is not inside the tz_database_dir, so we hope that it was copied and
         /// try to find the file with exact same contents in the database.
 
         size_t tzfile_size = fs::file_size(tz_file_path);
-        Poco::SHA1Engine::Digest tzfile_sha1 = calcSHA1(tz_file_path.native());
+        Poco::SHA1Engine::Digest tzfile_sha1 = calcSHA1(tz_file_path.string());
 
         fs::recursive_directory_iterator begin(tz_database_path);
         fs::recursive_directory_iterator end;
@@ -80,8 +80,8 @@ std::string determineDefaultTimeZone()
             if (candidate_it->status().type() != fs::regular_file || path.filename() == "localtime")
                 continue;
 
-            if (fs::file_size(path) == tzfile_size && calcSHA1(path.native()) == tzfile_sha1)
-                return path.lexically_relative(tz_database_path).native();
+            if (fs::file_size(path) == tzfile_size && calcSHA1(path.string()) == tzfile_sha1)
+                return path.lexically_relative(tz_database_path).string();
         }
     }
     catch (const Poco::Exception & ex)
diff --git a/libs/libcommon/src/mremap.cpp b/libs/libcommon/src/mremap.cpp
index e01acc8d5d8..5d1ba5287bc 100644
--- a/libs/libcommon/src/mremap.cpp
+++ b/libs/libcommon/src/mremap.cpp
@@ -1,9 +1,8 @@
+#include <common/mremap.h>
 #include <cstddef>
 #include <cstdlib>
 #include <cstring>
-#include <sys/mman.h>
 #include <errno.h>
-#include <common/mremap.h>
 
 #if defined(MREMAP_FIXED)
 // we already have implementation (linux)
@@ -22,18 +21,26 @@ void * mremap(
         return nullptr;
     }
 
+#if _MSC_VER
+    void * new_address = ::operator new(new_size);
+#else
     void * new_address = mmap(nullptr, new_size, mmap_prot, mmap_flags, mmap_fd, mmap_offset);
     if (MAP_FAILED == new_address)
     {
         return MAP_FAILED;
     }
+#endif
 
     memcpy(new_address, old_address, old_size);
 
+#if _MSC_VER
+    delete old_address;
+#else
     if (munmap(old_address, old_size))
     {
         abort();
     }
+#endif
 
     return new_address;
 }
diff --git a/libs/libdaemon/include/daemon/BaseDaemon.h b/libs/libdaemon/include/daemon/BaseDaemon.h
index 20c38469d16..112502e06f6 100644
--- a/libs/libdaemon/include/daemon/BaseDaemon.h
+++ b/libs/libdaemon/include/daemon/BaseDaemon.h
@@ -1,7 +1,7 @@
 #pragma once
 
 #include <sys/types.h>
-#include <unistd.h>
+#include <port/unistd.h>
 #include <iostream>
 #include <memory>
 #include <functional>
diff --git a/libs/libpocoext/src/ThreadNumber.cpp b/libs/libpocoext/src/ThreadNumber.cpp
index 266627d3168..6458ea8c221 100644
--- a/libs/libpocoext/src/ThreadNumber.cpp
+++ b/libs/libpocoext/src/ThreadNumber.cpp
@@ -1,16 +1,14 @@
-#include <pthread.h>
-#include <common/likely.h>
-
 #include <Poco/Ext/ThreadNumber.h>
-
+#include <common/likely.h>
+#include <atomic>
 
 static thread_local unsigned thread_number = 0;
-static unsigned threads = 0;
+static std::atomic_uint threads{0};
 
 unsigned Poco::ThreadNumber::get()
 {
     if (unlikely(thread_number == 0))
-        thread_number = __sync_add_and_fetch(&threads, 1);
+        thread_number = ++threads;
 
     return thread_number;
 }
diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt
index bd055efc791..010e38ebfe0 100644
--- a/utils/CMakeLists.txt
+++ b/utils/CMakeLists.txt
@@ -15,3 +15,4 @@ add_subdirectory (wikistat-loader)
 add_subdirectory (fill-factor)
 add_subdirectory (check-marks)
 add_subdirectory (test-data-generator)
+add_subdirectory (report)
diff --git a/utils/build/build_msvc2017.bat b/utils/build/build_msvc2017.bat
new file mode 100644
index 00000000000..880802999ab
--- /dev/null
+++ b/utils/build/build_msvc2017.bat
@@ -0,0 +1,14 @@
+
+:: WINDOWS BUILD NOT SUPPORTED!
+:: Script only for development
+
+cd ../..
+git clone --recursive https://github.com/madler/zlib contrib/zlib
+md build
+cd build
+
+:: Stage 1: try build client
+cmake .. -G "Visual Studio 15 2017 Win64" -DENABLE_CLICKHOUSE_ALL=0 -DENABLE_CLICKHOUSE_CLIENT=1 > cmake.log
+cmake --build . --target clickhouse -- /m > build.log
+:: Stage 2: try build minimal server
+:: Stage 3: enable all possible features (ssl, ...)
diff --git a/utils/compressor/CMakeLists.txt b/utils/compressor/CMakeLists.txt
index ae4333337db..ca4dd334fb5 100644
--- a/utils/compressor/CMakeLists.txt
+++ b/utils/compressor/CMakeLists.txt
@@ -7,7 +7,7 @@ set_target_properties(util-clickhouse-compressor PROPERTIES OUTPUT_NAME "clickho
 #install (TARGETS util-clickhouse-compressor RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse-compressor)
 
 add_executable (zstd_test zstd_test.cpp)
-target_link_libraries (zstd_test ${ZSTD_LIBRARY} Threads::Threads)
+target_link_libraries (zstd_test ${ZSTD_LIBRARY} common Threads::Threads)
 if (NOT USE_INTERNAL_ZSTD_LIBRARY)
     target_include_directories (zstd_test BEFORE PRIVATE ${ZSTD_INCLUDE_DIR})
 endif ()
diff --git a/utils/compressor/zstd_test.cpp b/utils/compressor/zstd_test.cpp
index 6a8b9cb5be6..01016830e7d 100644
--- a/utils/compressor/zstd_test.cpp
+++ b/utils/compressor/zstd_test.cpp
@@ -1,7 +1,8 @@
-#include <unistd.h>
+#include <port/unistd.h>
 #include <zstd.h>
 #include <vector>
 #include <stdexcept>
+#include <sys/types.h>
 
 
 int main(int argc, char ** argv)
diff --git a/utils/iotest/iotest.cpp b/utils/iotest/iotest.cpp
index 439672fc25c..a4849638e48 100644
--- a/utils/iotest/iotest.cpp
+++ b/utils/iotest/iotest.cpp
@@ -1,28 +1,21 @@
 #include <fcntl.h>
-#include <unistd.h>
+#include <port/unistd.h>
 #include <stdlib.h>
 #include <time.h>
-
 #include <iostream>
 #include <iomanip>
 #include <vector>
 #include <random>
 #include <pcg_random.hpp>
-
 #include <Poco/NumberParser.h>
 #include <Poco/NumberFormatter.h>
 #include <Poco/Exception.h>
-
 #include <Common/Exception.h>
 #include <Common/randomSeed.h>
-
 #include <common/ThreadPool.h>
 #include <Common/Stopwatch.h>
-
 #include <IO/BufferWithOwnMemory.h>
-
 #include <cstdlib>
-
 #ifdef __APPLE__
 #include <common/apple_rt.h>
 #endif
diff --git a/utils/iotest/iotest_aio.cpp b/utils/iotest/iotest_aio.cpp
index f86b4e76e3e..bfab8249d19 100644
--- a/utils/iotest/iotest_aio.cpp
+++ b/utils/iotest/iotest_aio.cpp
@@ -1,28 +1,21 @@
 #include <fcntl.h>
-#include <unistd.h>
+#include <port/unistd.h>
 #include <stdlib.h>
 #include <time.h>
-
 #include <iostream>
 #include <iomanip>
 #include <vector>
-
 #include <Poco/NumberParser.h>
 #include <Poco/NumberFormatter.h>
 #include <Poco/Exception.h>
-
 #include <Common/Exception.h>
-
 #include <common/ThreadPool.h>
 #include <Common/Stopwatch.h>
-
 #include <IO/BufferWithOwnMemory.h>
-
 #include <stdlib.h>
 #if !defined(__APPLE__) && !defined(__FreeBSD__)
 #include <malloc.h>
 #endif
-
 #include <fcntl.h>
 #include <stdlib.h>
 #include <stdio.h>
@@ -33,6 +26,7 @@
 #endif
 #include <sys/syscall.h>
 
+
 using DB::throwFromErrno;
 
 inline int io_setup(unsigned nr, aio_context_t *ctxp)
diff --git a/utils/iotest/iotest_nonblock.cpp b/utils/iotest/iotest_nonblock.cpp
index fdd25860825..8f1ad435bf0 100644
--- a/utils/iotest/iotest_nonblock.cpp
+++ b/utils/iotest/iotest_nonblock.cpp
@@ -1,5 +1,5 @@
 #include <fcntl.h>
-#include <unistd.h>
+#include <port/unistd.h>
 #include <stdlib.h>
 #include <time.h>
 #include <stdlib.h>
@@ -7,24 +7,18 @@
 #include <malloc.h>
 #endif
 #include <poll.h>
-
 #include <iostream>
 #include <iomanip>
 #include <vector>
-
 #include <random>
 #include <pcg_random.hpp>
-
 #include <Poco/NumberParser.h>
 #include <Poco/NumberFormatter.h>
 #include <Poco/Exception.h>
-
 #include <Common/Exception.h>
 #include <Common/randomSeed.h>
-
 #include <common/ThreadPool.h>
 #include <Common/Stopwatch.h>
-
 #ifdef __APPLE__
 #include <common/apple_rt.h>
 #endif
diff --git a/utils/report/CMakeLists.txt b/utils/report/CMakeLists.txt
new file mode 100644
index 00000000000..e39dd155b15
--- /dev/null
+++ b/utils/report/CMakeLists.txt
@@ -0,0 +1 @@
+install (PROGRAMS clickhouse-report DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
diff --git a/utils/report/clickhouse-report b/utils/report/clickhouse-report
new file mode 100755
index 00000000000..1cd212febcb
--- /dev/null
+++ b/utils/report/clickhouse-report
@@ -0,0 +1,30 @@
+#!/bin/sh -x
+# Usages:
+# sh -x clickhouse-report > ch.`hostname`.`date '+%Y%M%''d%H%M%''S'`.dmp 2>&1
+# curl https://raw.githubusercontent.com/yandex/ClickHouse/master/utils/report/clickhouse-report | sh -x > ch.`hostname`.`date '+%Y%M%''d%H%M%''S'`.dmp 2>&1
+
+# Also dump some system info (can contain some private data)
+# sh -x clickhouse-report system > ch.`hostname`.`date '+%Y%M%''d%H%M%''S'`.dmp 2>&1
+# curl https://raw.githubusercontent.com/yandex/ClickHouse/master/utils/report/clickhouse-report system | sh -x > ch.`hostname`.`date '+%Y%M%''d%H%M%''S'`.dmp 2>&1
+
+clickhouse-client -q 'SELECT * FROM system.events FORMAT PrettyCompactNoEscapes'
+clickhouse-client -q 'SELECT * FROM system.metrics FORMAT PrettyCompactNoEscapes'
+clickhouse-client -q 'SELECT * FROM system.asynchronous_metrics FORMAT PrettyCompactNoEscapes'
+clickhouse-client -q 'SELECT * FROM system.build_options FORMAT PrettyCompactNoEscapes'
+clickhouse-client -q 'SELECT * FROM system.processes FORMAT PrettyCompactNoEscapes'
+clickhouse-client -q 'SELECT * FROM system.merges FORMAT PrettyCompactNoEscapes'
+clickhouse-client -q 'SELECT * FROM system.parts FORMAT PrettyCompactNoEscapes'
+clickhouse-client -q 'SELECT * FROM system.replication_queue FORMAT PrettyCompactNoEscapes'
+clickhouse-client -q 'SELECT * FROM system.replicas FORMAT PrettyCompactNoEscapes'
+clickhouse-client -q 'SELECT * FROM system.dictionaries FORMAT PrettyCompactNoEscapes'
+tail -n200 /var/log/clickhouse-server/clickhouse-server.err.log
+tail -n200 /var/log/clickhouse-server/clickhouse-server.log
+tail -n100 /var/log/clickhouse-server/stderr
+
+if [ "$1" = "system" ]; then
+    uname -a
+    cat /etc/lsb-release
+    df -h
+    top -bn1
+    ps auxw
+fi
diff --git a/utils/report/clickhouse_report.sh b/utils/report/clickhouse_report.sh
deleted file mode 100755
index ab1ec396e80..00000000000
--- a/utils/report/clickhouse_report.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/bin/sh -x
-# Usages:
-# sh clickhouse_report.sh > ch.`hostname`.`date '+%Y%M%''d%H%M%''S'`.dmp 2>&1
-# curl https://raw.githubusercontent.com/yandex/ClickHouse/master/utils/report/clickhouse_report.sh | sh > ch.`hostname`.`date '+%Y%M%''d%H%M%''S'`.dmp 2>&1
-
-clickhouse --client -q 'SELECT * FROM system.events FORMAT PrettyCompactNoEscapes'
-clickhouse --client -q 'SELECT * FROM system.metrics FORMAT PrettyCompactNoEscapes'
-clickhouse --client -q 'SELECT * FROM system.asynchronous_metrics FORMAT PrettyCompactNoEscapes'
-clickhouse --client -q 'SELECT * FROM system.build_options FORMAT PrettyCompactNoEscapes'
-clickhouse --client -q 'SELECT * FROM system.processes FORMAT PrettyCompactNoEscapes'
-clickhouse --client -q 'SELECT * FROM system.merges FORMAT PrettyCompactNoEscapes'
-clickhouse --client -q 'SELECT * FROM system.parts FORMAT PrettyCompactNoEscapes'
-clickhouse --client -q 'SELECT * FROM system.replication_queue FORMAT PrettyCompactNoEscapes'
-clickhouse --client -q 'SELECT * FROM system.replicas FORMAT PrettyCompactNoEscapes'
-clickhouse --client -q 'SELECT * FROM system.dictionaries FORMAT PrettyCompactNoEscapes'
-ps auxw
-df -h
-top -bn1
-tail -n200 /var/log/clickhouse-server/clickhouse-server.err.log
-tail -n200 /var/log/clickhouse-server/clickhouse-server.log
-tail -n100 /var/log/clickhouse-server/stderr
-cat /etc/lsb-release
-uname -a

From b99783028997e19653fea0aab92579f754302416 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Thu, 29 Mar 2018 18:31:59 +0300
Subject: [PATCH 236/244] Fixed error found by Artemeey Gavryushin
 [#CLICKHOUSE-3]

---
 dbms/src/Functions/FunctionsStringArray.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dbms/src/Functions/FunctionsStringArray.h b/dbms/src/Functions/FunctionsStringArray.h
index 1723dea7adf..cd12145d5df 100644
--- a/dbms/src/Functions/FunctionsStringArray.h
+++ b/dbms/src/Functions/FunctionsStringArray.h
@@ -128,7 +128,7 @@ public:
             throw Exception("Illegal type " + arguments[0]->getName() + " of first argument of function " + getName() + ". Must be String.",
                 ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
 
-        if (!arguments[0]->isString())
+        if (!arguments[1]->isString())
             throw Exception("Illegal type " + arguments[1]->getName() + " of second argument of function " + getName() + ". Must be String.",
                 ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
     }

From fef91af96f1a121701326f18ac7bbec8b85d06af Mon Sep 17 00:00:00 2001
From: Brett Hoerner <brett@bretthoerner.com>
Date: Thu, 29 Mar 2018 12:01:47 -0500
Subject: [PATCH 237/244] Fix build when using GCC 7.3.1: -Wmaybe-uninitialized
 error
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Before adding this ignore you would receive the following when using GCC
7.3.1:

[ 82%] Building CXX object dbms/src/Storages/System/CMakeFiles/clickhouse_storages_system.dir/StorageSystemAsynchronousMetrics.cpp.o
In file included from /ClickHouse/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.cpp:5:0:
/ClickHouse/dbms/src/Columns/ColumnString.h: In member function ‘virtual DB::BlockInputStreams DB::StorageSystemAsynchronousMetrics::read(const Names&, const DB::SelectQueryInfo&, const DB::Context&, DB::QueryProcessingStage::Enum&, size_t, unsigned int)’:
/ClickHouse/dbms/src/Columns/ColumnString.h:96:45: error: ‘*((void*)&<anonymous> +8)’ may be used uninitialized in this function [-Werror=maybe-uninitialized]
         const size_t size_to_append = s.size() + 1;
                                       ~~~~~~^~
cc1plus: all warnings being treated as errors
make[2]: *** [dbms/src/Storages/System/CMakeFiles/clickhouse_storages_system.dir/build.make:63: dbms/src/Storages/System/CMakeFiles/clickhouse_storages_system.dir/StorageSystemAsynchronousMetrics.cpp.o] Error 1
make[1]: *** [CMakeFiles/Makefile2:9105: dbms/src/Storages/System/CMakeFiles/clickhouse_storages_system.dir/all] Error 2
make: *** [Makefile:163: all] Error 2
---
 dbms/src/Columns/ColumnString.h | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/dbms/src/Columns/ColumnString.h b/dbms/src/Columns/ColumnString.h
index c37fca3ab44..c61fb3ee8fb 100644
--- a/dbms/src/Columns/ColumnString.h
+++ b/dbms/src/Columns/ColumnString.h
@@ -89,7 +89,13 @@ public:
         return StringRef(&chars[offsetAt(n)], sizeAt(n));
     }
 
-    void insert(const Field & x) override
+/// Suppress gcc7 warnings: '*((void*)&<anonymous> +8)' may be used uninitialized in this function
+#if !__clang__
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
+#endif
+
+        void insert(const Field & x) override
     {
         const String & s = DB::get<const String &>(x);
         const size_t old_size = chars.size();
@@ -101,7 +107,11 @@ public:
         offsets.push_back(new_size);
     }
 
-    void insertFrom(const IColumn & src_, size_t n) override
+#if !__clang__
+#pragma GCC diagnostic pop
+#endif
+
+        void insertFrom(const IColumn & src_, size_t n) override
     {
         const ColumnString & src = static_cast<const ColumnString &>(src_);
 

From 037cf6223053aa513976c5b069c2f8c7a939084a Mon Sep 17 00:00:00 2001
From: proller <proller@github.com>
Date: Thu, 29 Mar 2018 20:24:46 +0300
Subject: [PATCH 238/244] CLICKHOUSE-3644: Server: do not use SO_REUSEPORT

---
 contrib/poco               |  2 +-
 dbms/src/Server/Server.cpp | 80 +++++++++++++++++++++++---------------
 2 files changed, 49 insertions(+), 33 deletions(-)

diff --git a/contrib/poco b/contrib/poco
index 3dae7c46e81..a107b0c9cee 160000
--- a/contrib/poco
+++ b/contrib/poco
@@ -1 +1 @@
-Subproject commit 3dae7c46e813d884d15ff509f6dcc1693417646f
+Subproject commit a107b0c9cee109fe0abfbf509df3c78a1e0c05fa
diff --git a/dbms/src/Server/Server.cpp b/dbms/src/Server/Server.cpp
index 916cee00afd..0cba1117e60 100644
--- a/dbms/src/Server/Server.cpp
+++ b/dbms/src/Server/Server.cpp
@@ -326,7 +326,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
 
         std::vector<std::string> listen_hosts = DB::getMultipleValuesFromConfig(config(), "", "listen_host");
 
-        bool listen_try = config().getUInt("listen_try", false);
+        bool listen_try = config().getBool("listen_try", false);
         if (listen_hosts.empty())
         {
             listen_hosts.emplace_back("::1");
@@ -362,6 +362,23 @@ int Server::main(const std::vector<std::string> & /*args*/)
             return socket_address;
         };
 
+        auto socket_bind_listen = [&](auto & socket, const std::string & host, UInt16 port, bool secure = 0)
+        {
+               auto address = make_socket_address(host, port);
+#if !POCO_CLICKHOUSE_PATCH || POCO_VERSION <= 0x02000000 // TODO: fill correct version
+               if (secure)
+                   /// Bug in old poco, listen() after bind() with reusePort param will fail because have no implementation in SecureServerSocketImpl
+                   /// https://github.com/pocoproject/poco/pull/2257
+                   socket.bind(address, true);
+               else
+#endif
+                   socket.bind(address, /* reuseAddress = */ true, /* reusePort = */ config().getBool("listen_reuse_port", false));
+
+               socket.listen(/* backlog = */ config().getUInt("listen_backlog", 64));
+
+               return address;
+        };
+
         for (const auto & listen_host : listen_hosts)
         {
             /// For testing purposes, user may omit tcp_port or http_port or https_port in configuration file.
@@ -370,18 +387,17 @@ int Server::main(const std::vector<std::string> & /*args*/)
                 /// HTTP
                 if (config().has("http_port"))
                 {
-                    Poco::Net::SocketAddress http_socket_address = make_socket_address(listen_host, config().getInt("http_port"));
-                    Poco::Net::ServerSocket http_socket(http_socket_address);
-                    http_socket.setReceiveTimeout(settings.http_receive_timeout);
-                    http_socket.setSendTimeout(settings.http_send_timeout);
-
+                    Poco::Net::ServerSocket socket;
+                    auto address = socket_bind_listen(socket, listen_host, config().getInt("http_port"));
+                    socket.setReceiveTimeout(settings.http_receive_timeout);
+                    socket.setSendTimeout(settings.http_send_timeout);
                     servers.emplace_back(new Poco::Net::HTTPServer(
                         new HTTPHandlerFactory(*this, "HTTPHandler-factory"),
                         server_pool,
-                        http_socket,
+                        socket,
                         http_params));
 
-                    LOG_INFO(log, "Listening http://" + http_socket_address.toString());
+                    LOG_INFO(log, "Listening http://" + address.toString());
                 }
 
                 /// HTTPS
@@ -389,18 +405,18 @@ int Server::main(const std::vector<std::string> & /*args*/)
                 {
 #if Poco_NetSSL_FOUND
                     std::call_once(ssl_init_once, SSLInit);
-                    Poco::Net::SocketAddress http_socket_address = make_socket_address(listen_host, config().getInt("https_port"));
-                    Poco::Net::SecureServerSocket http_socket(http_socket_address);
-                    http_socket.setReceiveTimeout(settings.http_receive_timeout);
-                    http_socket.setSendTimeout(settings.http_send_timeout);
 
+                    Poco::Net::SecureServerSocket socket;
+                    auto address = socket_bind_listen(socket, listen_host, config().getInt("https_port"), /* secure = */ true);
+                    socket.setReceiveTimeout(settings.http_receive_timeout);
+                    socket.setSendTimeout(settings.http_send_timeout);
                     servers.emplace_back(new Poco::Net::HTTPServer(
                         new HTTPHandlerFactory(*this, "HTTPSHandler-factory"),
                         server_pool,
-                        http_socket,
+                        socket,
                         http_params));
 
-                    LOG_INFO(log, "Listening https://" + http_socket_address.toString());
+                    LOG_INFO(log, "Listening https://" + address.toString());
 #else
                     throw Exception{"HTTPS protocol is disabled because Poco library was built without NetSSL support.",
                         ErrorCodes::SUPPORT_IS_DISABLED};
@@ -411,33 +427,33 @@ int Server::main(const std::vector<std::string> & /*args*/)
                 if (config().has("tcp_port"))
                 {
                     std::call_once(ssl_init_once, SSLInit);
-                    Poco::Net::SocketAddress tcp_address = make_socket_address(listen_host, config().getInt("tcp_port"));
-                    Poco::Net::ServerSocket tcp_socket(tcp_address);
-                    tcp_socket.setReceiveTimeout(settings.receive_timeout);
-                    tcp_socket.setSendTimeout(settings.send_timeout);
+                    Poco::Net::ServerSocket socket;
+                    auto address = socket_bind_listen(socket, listen_host, config().getInt("tcp_port"));
+                    socket.setReceiveTimeout(settings.receive_timeout);
+                    socket.setSendTimeout(settings.send_timeout);
                     servers.emplace_back(new Poco::Net::TCPServer(
                         new TCPHandlerFactory(*this),
                         server_pool,
-                        tcp_socket,
+                        socket,
                         new Poco::Net::TCPServerParams));
 
-                    LOG_INFO(log, "Listening tcp: " + tcp_address.toString());
+                    LOG_INFO(log, "Listening tcp: " + address.toString());
                 }
 
                 /// TCP with SSL
                 if (config().has("tcp_port_secure"))
                 {
 #if Poco_NetSSL_FOUND
-                    Poco::Net::SocketAddress tcp_address = make_socket_address(listen_host, config().getInt("tcp_port_secure"));
-                    Poco::Net::SecureServerSocket tcp_socket(tcp_address);
-                    tcp_socket.setReceiveTimeout(settings.receive_timeout);
-                    tcp_socket.setSendTimeout(settings.send_timeout);
+                    Poco::Net::SecureServerSocket socket;
+                    auto address = socket_bind_listen(socket, listen_host, config().getInt("tcp_port_secure"), /* secure = */ true);
+                    socket.setReceiveTimeout(settings.receive_timeout);
+                    socket.setSendTimeout(settings.send_timeout);
                     servers.emplace_back(new Poco::Net::TCPServer(
                         new TCPHandlerFactory(*this, /* secure= */ true ),
                                                                   server_pool,
-                                                                  tcp_socket,
+                                                                  socket,
                                                                   new Poco::Net::TCPServerParams));
-                    LOG_INFO(log, "Listening tcp_secure: " + tcp_address.toString());
+                    LOG_INFO(log, "Listening tcp_secure: " + address.toString());
 #else
                     throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support.",
                         ErrorCodes::SUPPORT_IS_DISABLED};
@@ -451,17 +467,17 @@ int Server::main(const std::vector<std::string> & /*args*/)
                 /// Interserver IO HTTP
                 if (config().has("interserver_http_port"))
                 {
-                    Poco::Net::SocketAddress interserver_address = make_socket_address(listen_host, config().getInt("interserver_http_port"));
-                    Poco::Net::ServerSocket interserver_io_http_socket(interserver_address);
-                    interserver_io_http_socket.setReceiveTimeout(settings.http_receive_timeout);
-                    interserver_io_http_socket.setSendTimeout(settings.http_send_timeout);
+                    Poco::Net::ServerSocket socket;
+                    auto address = socket_bind_listen(socket, listen_host, config().getInt("interserver_http_port"));
+                    socket.setReceiveTimeout(settings.http_receive_timeout);
+                    socket.setSendTimeout(settings.http_send_timeout);
                     servers.emplace_back(new Poco::Net::HTTPServer(
                         new InterserverIOHTTPHandlerFactory(*this, "InterserverIOHTTPHandler-factory"),
                         server_pool,
-                        interserver_io_http_socket,
+                        socket,
                         http_params));
 
-                    LOG_INFO(log, "Listening interserver: " + interserver_address.toString());
+                    LOG_INFO(log, "Listening interserver http: " + address.toString());
                 }
             }
             catch (const Poco::Net::NetException & e)

From 450efef4569d8a5417d211182c5980c3804dfcc5 Mon Sep 17 00:00:00 2001
From: proller <proller@github.com>
Date: Thu, 29 Mar 2018 22:12:24 +0300
Subject: [PATCH 239/244] Maybe fix test 00429_long_http_bufferization (remove
 curl timeout)

---
 .../tests/queries/0_stateless/00429_long_http_bufferization.sh | 2 +-
 dbms/tests/queries/shell_config.sh                             | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/dbms/tests/queries/0_stateless/00429_long_http_bufferization.sh b/dbms/tests/queries/0_stateless/00429_long_http_bufferization.sh
index e94db0d951f..06e2abdae53 100755
--- a/dbms/tests/queries/0_stateless/00429_long_http_bufferization.sh
+++ b/dbms/tests/queries/0_stateless/00429_long_http_bufferization.sh
@@ -14,7 +14,7 @@ function query {
 }
 
 function ch_url() {
-    ${CLICKHOUSE_CURL} -sS "$URL?max_block_size=$max_block_size&$1" -d "`query $2`"
+    ${CLICKHOUSE_CURL_COMMAND} -sS "$URL?max_block_size=$max_block_size&$1" -d "`query $2`"
 }
 
 
diff --git a/dbms/tests/queries/shell_config.sh b/dbms/tests/queries/shell_config.sh
index f554105b5bf..1eae3159870 100644
--- a/dbms/tests/queries/shell_config.sh
+++ b/dbms/tests/queries/shell_config.sh
@@ -19,6 +19,7 @@ export CLICKHOUSE_PORT_HTTPS=${CLICKHOUSE_PORT_HTTPS:="8443"}
 export CLICKHOUSE_PORT_HTTP_PROTO=${CLICKHOUSE_PORT_HTTP_PROTO:="http"}
 export CLICKHOUSE_URL=${CLICKHOUSE_URL:="${CLICKHOUSE_PORT_HTTP_PROTO}://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/"}
 export CLICKHOUSE_URL_HTTPS=${CLICKHOUSE_URL_HTTPS:="https://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTPS}/"}
-export CLICKHOUSE_CURL=${CLICKHOUSE_CURL:="curl --max-time 10"}
+export CLICKHOUSE_CURL_COMMAND=${CLICKHOUSE_CURL_COMMAND:="curl"}
+export CLICKHOUSE_CURL=${CLICKHOUSE_CURL:="${CLICKHOUSE_CURL_COMMAND} --max-time 10"}
 export CLICKHOUSE_TMP=${CLICKHOUSE_TMP:="."}
 mkdir -p ${CLICKHOUSE_TMP}

From ccedfbc4eade3489fb1a9b724ce42c00db7826a5 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Thu, 29 Mar 2018 23:11:07 +0300
Subject: [PATCH 240/244] Fixed error with unneeded dependency on libc version
 [#CLICKHOUSE-2]

---
 contrib/CMakeLists.txt                 |   2 +-
 contrib/zookeeper-cmake/CMakeLists.txt | 200 +++++++++++++++++++++++++
 2 files changed, 201 insertions(+), 1 deletion(-)
 create mode 100644 contrib/zookeeper-cmake/CMakeLists.txt

diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
index 9015f7e7d36..1314bfe57ad 100644
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@@ -26,7 +26,7 @@ if (USE_INTERNAL_DOUBLE_CONVERSION_LIBRARY)
 endif ()
 
 if (USE_INTERNAL_ZOOKEEPER_LIBRARY)
-    add_subdirectory (zookeeper/src/c)
+    add_subdirectory (zookeeper-cmake)
 endif ()
 
 if (USE_INTERNAL_CITYHASH_LIBRARY)
diff --git a/contrib/zookeeper-cmake/CMakeLists.txt b/contrib/zookeeper-cmake/CMakeLists.txt
new file mode 100644
index 00000000000..1211f1f8e14
--- /dev/null
+++ b/contrib/zookeeper-cmake/CMakeLists.txt
@@ -0,0 +1,200 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cmake_minimum_required(VERSION 3.0)
+
+# Modified version of CMakeLists.txt for ClickHouse. Doesn't link the library to libm.
+# Otherwise we have extra dependency when compiling with the most fresh libc version.
+# How to check:
+# readelf -s ./clickhouse | grep -F 2.23
+
+SET(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/zookeeper/src/c)
+
+project(zookeeper VERSION 3.5.3)
+set(email user@zookeeper.apache.org)
+set(description "zookeeper C client")
+
+# general options
+if(UNIX)
+  add_compile_options(-Wall -fPIC)
+elseif(WIN32)
+  add_compile_options(/W3)
+endif()
+add_definitions(-DUSE_STATIC_LIB)
+
+# TODO: Enable /WX and /W4 on Windows. Currently there are ~1000 warnings.
+# TODO: Add Solaris support.
+# TODO: Add a shared library option.
+# TODO: Specify symbols to export.
+# TODO: Generate doxygen documentation.
+
+# Sync API option
+option(WANT_SYNCAPI "Enables Sync API support" ON)
+if(WANT_SYNCAPI)
+  add_definitions(-DTHREADED)
+  if(WIN32)
+    # Note that the generator expression ensures that `/MTd` is used when Debug
+    # configurations are built.
+    add_compile_options(/MT$<$<CONFIG:Debug>:d>)
+  endif()
+endif()
+
+# CppUnit option
+if(WIN32 OR APPLE)
+  # The tests do not yet compile on Windows or macOS,
+  # so we set this to off by default.
+  #
+  # Note that CMake does not have expressions except in conditionals,
+  # so we're left with this if/else/endif pattern.
+  set(DEFAULT_WANT_CPPUNIT OFF)
+else()
+  set(DEFAULT_WANT_CPPUNIT ON)
+endif()
+option(WANT_CPPUNIT "Enables CppUnit and tests" ${DEFAULT_WANT_CPPUNIT})
+
+# The function `to_have(in out)` converts a header name like `arpa/inet.h`
+# into an Autotools style preprocessor definition `HAVE_ARPA_INET_H`.
+# This is then set or unset in `configure_file()` step.
+#
+# Note that CMake functions do not have return values; instead an "out"
+# variable must be passed, and explicitly set with parent scope.
+function(to_have in out)
+  string(TOUPPER ${in} str)
+  string(REGEX REPLACE "/|\\." "_" str ${str})
+  set(${out} "HAVE_${str}" PARENT_SCOPE)
+endfunction()
+
+# include file checks
+foreach(f generated/zookeeper.jute.h generated/zookeeper.jute.c)
+  if(EXISTS "${LIBRARY_DIR}/${f}")
+    to_have(${f} name)
+    set(${name} 1)
+  else()
+    message(FATAL_ERROR
+      "jute files are missing!\n"
+      "Please run 'ant compile_jute' while in the ZooKeeper top level directory.")
+  endif()
+endforeach()
+
+# header checks
+include(CheckIncludeFile)
+set(check_headers
+  arpa/inet.h
+  dlfcn.h
+  fcntl.h
+  inttypes.h
+  memory.h
+  netdb.h
+  netinet/in.h
+  stdint.h
+  stdlib.h
+  string.h
+  strings.h
+  sys/socket.h
+  sys/stat.h
+  sys/time.h
+  sys/types.h
+  unistd.h
+  sys/utsname.h)
+
+foreach(f ${check_headers})
+  to_have(${f} name)
+  check_include_file(${f} ${name})
+endforeach()
+
+# function checks
+include(CheckFunctionExists)
+set(check_functions
+  getcwd
+  gethostbyname
+  gethostname
+  getlogin
+  getpwuid_r
+  gettimeofday
+  getuid
+  memmove
+  memset
+  poll
+  socket
+  strchr
+  strdup
+  strerror
+  strtol)
+
+foreach(fn ${check_functions})
+  to_have(${fn} name)
+  check_function_exists(${fn} ${name})
+endforeach()
+
+# library checks
+set(check_libraries rt m pthread)
+foreach(lib ${check_libraries})
+  to_have("lib${lib}" name)
+  find_library(${name} ${lib})
+endforeach()
+
+# IPv6 check
+include(CheckStructHasMember)
+check_struct_has_member("struct sockaddr_in6" sin6_addr "netinet/in.h" ZOO_IPV6_ENABLED)
+
+# configure
+configure_file(${LIBRARY_DIR}/cmake_config.h.in ${CMAKE_CURRENT_BINARY_DIR}/include/config.h)
+
+# hashtable library
+set(hashtable_sources ${LIBRARY_DIR}/src/hashtable/hashtable_itr.c ${LIBRARY_DIR}/src/hashtable/hashtable.c)
+add_library(hashtable STATIC ${hashtable_sources})
+
+# zookeeper library
+set(zookeeper_sources
+  ${LIBRARY_DIR}/src/zookeeper.c
+  ${LIBRARY_DIR}/src/recordio.c
+  ${LIBRARY_DIR}/generated/zookeeper.jute.c
+  ${LIBRARY_DIR}/src/zk_log.c
+  ${LIBRARY_DIR}/src/zk_hashtable.c
+)
+#  src/addrvec.c
+
+if(WANT_SYNCAPI)
+  list(APPEND zookeeper_sources ${LIBRARY_DIR}/src/mt_adaptor.c)
+else()
+  list(APPEND zookeeper_sources ${LIBRARY_DIR}/src/st_adaptor.c)
+endif()
+
+if(WIN32)
+  list(APPEND zookeeper_sources ${LIBRARY_DIR}/src/winport.c)
+endif()
+
+add_library(zookeeper STATIC ${zookeeper_sources})
+target_include_directories(zookeeper BEFORE PUBLIC ${LIBRARY_DIR}/include ${CMAKE_CURRENT_BINARY_DIR}/include ${LIBRARY_DIR}/generated)
+target_link_libraries(zookeeper PUBLIC
+  hashtable
+  $<$<PLATFORM_ID:Linux>:rt> # clock_gettime
+  $<$<PLATFORM_ID:Windows>:ws2_32>) # Winsock 2.0
+
+if(WANT_SYNCAPI AND NOT WIN32)
+  find_package(Threads REQUIRED)
+  target_link_libraries(zookeeper PUBLIC Threads::Threads)
+endif()
+
+# cli executable
+add_executable(cli ${LIBRARY_DIR}/src/cli.c)
+target_link_libraries(cli zookeeper)
+
+# load_gen executable
+if(WANT_SYNCAPI AND NOT WIN32)
+  add_executable(load_gen ${LIBRARY_DIR}/src/load_gen.c)
+  target_link_libraries(load_gen zookeeper)
+endif()

From db531908541018cc2e7b470af5207c836aa422ae Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Thu, 29 Mar 2018 23:21:40 +0300
Subject: [PATCH 241/244] Update ColumnString.h

---
 dbms/src/Columns/ColumnString.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dbms/src/Columns/ColumnString.h b/dbms/src/Columns/ColumnString.h
index c61fb3ee8fb..11272fae140 100644
--- a/dbms/src/Columns/ColumnString.h
+++ b/dbms/src/Columns/ColumnString.h
@@ -95,7 +95,7 @@ public:
 #pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
 #endif
 
-        void insert(const Field & x) override
+    void insert(const Field & x) override
     {
         const String & s = DB::get<const String &>(x);
         const size_t old_size = chars.size();

From 30838f4d9f5e5248aa46e87eada813cd23686d60 Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Thu, 29 Mar 2018 23:54:57 +0300
Subject: [PATCH 242/244] Update SystemLog.h

---
 dbms/src/Interpreters/SystemLog.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/dbms/src/Interpreters/SystemLog.h b/dbms/src/Interpreters/SystemLog.h
index 8e04076dc14..521d18e644d 100644
--- a/dbms/src/Interpreters/SystemLog.h
+++ b/dbms/src/Interpreters/SystemLog.h
@@ -233,6 +233,9 @@ void SystemLog<LogElement>::flush()
     {
         LOG_TRACE(log, "Flushing system log");
 
+        /// We check for existence of the table and create it as needed at every flush.
+        /// This is done to allow user to drop the table at any moment (new empty table will be created automatically).
+        /// BTW, flush method is called from single thread.
         prepareTable();
 
         Block block = LogElement::createBlock();

From 761bb666997785a738b86e0e12bf9890fd439ce3 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@yandex-team.ru>
Date: Fri, 30 Mar 2018 00:11:56 +0300
Subject: [PATCH 243/244] Added links to the meetups [#CLICKHOUSE-3]

---
 website/index.css  | 2 +-
 website/index.html | 6 +++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/website/index.css b/website/index.css
index 47afa8bfd01..2127a3ad484 100644
--- a/website/index.css
+++ b/website/index.css
@@ -215,11 +215,11 @@ a:hover, a:active {
 #announcement {
     margin: 0 0 60px 0;
     padding: 20px 0;
+    font: 400 150%/133% 'Yandex Sans Display Web',Arial,sans-serif;
 }
 #announcement-link {
     color: #000;
     text-decoration: underline;
-    font: 400 150%/133% 'Yandex Sans Display Web',Arial,sans-serif;
     margin: 1em 0;
 }
 #announcement-link:hover {
diff --git a/website/index.html b/website/index.html
index 7a91035999c..e315b78199d 100644
--- a/website/index.html
+++ b/website/index.html
@@ -90,7 +90,11 @@
         <div class="clear"></div>
     </div>
 </div>
-
+<div id="announcement" class="colored-block">
+    <div class="page">
+        ClickHouse Meetup in <a id="announcement-link" href="https://www.meetup.com/San-Francisco-Bay-Area-ClickHouse-Meetup/events/248898966/" rel="external nofollow" target="_blank">Sunnyvale</a> &amp; <a id="announcement-link" href="https://www.meetup.com/San-Francisco-Bay-Area-ClickHouse-Meetup/events/249162518/" rel="external nofollow" target="_blank">San Francisco</a>, April 23-27</a>
+    </div>
+</div>
 <div class="page">
     <h2 id="slogan">ClickHouse. Just makes you think faster.</h2>
 

From f358dc4125f123b78e315004113b9bc10e71f678 Mon Sep 17 00:00:00 2001
From: alexey-milovidov <milovidov@yandex-team.ru>
Date: Fri, 30 Mar 2018 00:15:14 +0300
Subject: [PATCH 244/244] Update ColumnString.h

---
 dbms/src/Columns/ColumnString.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dbms/src/Columns/ColumnString.h b/dbms/src/Columns/ColumnString.h
index 11272fae140..0eda083f22e 100644
--- a/dbms/src/Columns/ColumnString.h
+++ b/dbms/src/Columns/ColumnString.h
@@ -89,7 +89,7 @@ public:
         return StringRef(&chars[offsetAt(n)], sizeAt(n));
     }
 
-/// Suppress gcc7 warnings: '*((void*)&<anonymous> +8)' may be used uninitialized in this function
+/// Suppress gcc 7.3.1 warning: '*((void*)&<anonymous> +8)' may be used uninitialized in this function
 #if !__clang__
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wmaybe-uninitialized"