Merge branch 'master' into remap-executable

README.md

@@ -15,7 +15,7 @@ ClickHouse is an open-source column-oriented database management system that all
 * [Contacts](https://clickhouse.tech/#contacts) can help to get your questions answered if there are any.
 * You can also [fill this form](https://clickhouse.tech/#meet) to meet Yandex ClickHouse team in person.
 
-## Upcoming Events		
+## Upcoming Events
 
-* [ClickHouse Data Integration Virtual Meetup](https://www.eventbrite.com/e/clickhouse-september-virtual-meetup-data-integration-tickets-117421895049) on September 10, 2020.
+* [eBay migrating from Druid](https://us02web.zoom.us/webinar/register/tZMkfu6rpjItHtaQ1DXcgPWcSOnmM73HLGKL) on September 23, 2020.
-* [ClickHouse talk at Ya.Subbotnik (in Russian)](https://ya.cc/t/cIBI-3yECj5JF) on September 12, 2020.
+* [ClickHouse for Edge Analytics](https://ones2020.sched.com/event/bWPs) on September 29, 2020.

base/common/throwError.h

@@ -0,0 +1,13 @@
+#pragma once
+#include <stdexcept>
+
+/// Throw DB::Exception-like exception before its definition.
+/// DB::Exception derived from Poco::Exception derived from std::exception.
+/// DB::Exception generally cought as Poco::Exception. std::exception generally has other catch blocks and could lead to other outcomes.
+/// DB::Exception is not defined yet. It'd better to throw Poco::Exception but we do not want to include any big header here, even <string>.
+/// So we throw some std::exception instead in the hope its catch block is the same as DB::Exception one.
+template <typename T>
+inline void throwError(const T & err)
+{
+    throw std::runtime_error(err);
+}

base/common/types.h

@@ -1,8 +1,6 @@
 #pragma once
 
-#include <algorithm>
 #include <cstdint>
-#include <cstdlib>
 #include <string>
 #include <type_traits>
 
@@ -25,8 +23,8 @@ using UInt64 = uint64_t;
 
 using Int128 = __int128;
 
-using wInt256 = std::wide_integer<256, signed>;
+using wInt256 = wide::integer<256, signed>;
-using wUInt256 = std::wide_integer<256, unsigned>;
+using wUInt256 = wide::integer<256, unsigned>;
 
 static_assert(sizeof(wInt256) == 32);
 static_assert(sizeof(wUInt256) == 32);
@@ -121,12 +119,6 @@ template <> struct is_big_int<wUInt256> { static constexpr bool value = true; };
 template <typename T>
 inline constexpr bool is_big_int_v = is_big_int<T>::value;
 
-template <typename T>
-inline std::string bigintToString(const T & x)
-{
-    return to_string(x);
-}
-
 template <typename To, typename From>
 inline To bigint_cast(const From & x [[maybe_unused]])
 {

base/common/wide_integer.h

@@ -22,79 +22,87 @@
  * without express or implied warranty.
  */
 
-#include <climits> // CHAR_BIT
-#include <cmath>
 #include <cstdint>
 #include <limits>
 #include <type_traits>
+#include <initializer_list>
+
+namespace wide
+{
+template <size_t Bits, typename Signed>
+class integer;
+}
 
 namespace std
 {
-template <size_t Bits, typename Signed>
-class wide_integer;
 
 template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
-struct common_type<wide_integer<Bits, Signed>, wide_integer<Bits2, Signed2>>;
+struct common_type<wide::integer<Bits, Signed>, wide::integer<Bits2, Signed2>>;
 
 template <size_t Bits, typename Signed, typename Arithmetic>
-struct common_type<wide_integer<Bits, Signed>, Arithmetic>;
+struct common_type<wide::integer<Bits, Signed>, Arithmetic>;
 
 template <typename Arithmetic, size_t Bits, typename Signed>
-struct common_type<Arithmetic, wide_integer<Bits, Signed>>;
+struct common_type<Arithmetic, wide::integer<Bits, Signed>>;
+
+}
+
+namespace wide
+{
 
 template <size_t Bits, typename Signed>
-class wide_integer
+class integer
 {
 public:
     using base_type = uint8_t;
     using signed_base_type = int8_t;
 
     // ctors
-    wide_integer() = default;
+    integer() = default;
 
     template <typename T>
-    constexpr wide_integer(T rhs) noexcept;
+    constexpr integer(T rhs) noexcept;
     template <typename T>
-    constexpr wide_integer(std::initializer_list<T> il) noexcept;
+    constexpr integer(std::initializer_list<T> il) noexcept;
 
     // assignment
     template <size_t Bits2, typename Signed2>
-    constexpr wide_integer<Bits, Signed> & operator=(const wide_integer<Bits2, Signed2> & rhs) noexcept;
+    constexpr integer<Bits, Signed> & operator=(const integer<Bits2, Signed2> & rhs) noexcept;
 
     template <typename Arithmetic>
-    constexpr wide_integer<Bits, Signed> & operator=(Arithmetic rhs) noexcept;
+    constexpr integer<Bits, Signed> & operator=(Arithmetic rhs) noexcept;
 
     template <typename Arithmetic>
-    constexpr wide_integer<Bits, Signed> & operator*=(const Arithmetic & rhs);
+    constexpr integer<Bits, Signed> & operator*=(const Arithmetic & rhs);
 
     template <typename Arithmetic>
-    constexpr wide_integer<Bits, Signed> & operator/=(const Arithmetic & rhs);
+    constexpr integer<Bits, Signed> & operator/=(const Arithmetic & rhs);
 
     template <typename Arithmetic>
-    constexpr wide_integer<Bits, Signed> & operator+=(const Arithmetic & rhs) noexcept(is_same<Signed, unsigned>::value);
+    constexpr integer<Bits, Signed> & operator+=(const Arithmetic & rhs) noexcept(std::is_same_v<Signed, unsigned>);
 
     template <typename Arithmetic>
-    constexpr wide_integer<Bits, Signed> & operator-=(const Arithmetic & rhs) noexcept(is_same<Signed, unsigned>::value);
+    constexpr integer<Bits, Signed> & operator-=(const Arithmetic & rhs) noexcept(std::is_same_v<Signed, unsigned>);
 
     template <typename Integral>
-    constexpr wide_integer<Bits, Signed> & operator%=(const Integral & rhs);
+    constexpr integer<Bits, Signed> & operator%=(const Integral & rhs);
 
     template <typename Integral>
-    constexpr wide_integer<Bits, Signed> & operator&=(const Integral & rhs) noexcept;
+    constexpr integer<Bits, Signed> & operator&=(const Integral & rhs) noexcept;
 
     template <typename Integral>
-    constexpr wide_integer<Bits, Signed> & operator|=(const Integral & rhs) noexcept;
+    constexpr integer<Bits, Signed> & operator|=(const Integral & rhs) noexcept;
 
     template <typename Integral>
-    constexpr wide_integer<Bits, Signed> & operator^=(const Integral & rhs) noexcept;
+    constexpr integer<Bits, Signed> & operator^=(const Integral & rhs) noexcept;
 
-    constexpr wide_integer<Bits, Signed> & operator<<=(int n);
+    constexpr integer<Bits, Signed> & operator<<=(int n) noexcept;
-    constexpr wide_integer<Bits, Signed> & operator>>=(int n) noexcept;
+    constexpr integer<Bits, Signed> & operator>>=(int n) noexcept;
 
-    constexpr wide_integer<Bits, Signed> & operator++() noexcept(is_same<Signed, unsigned>::value);
+    constexpr integer<Bits, Signed> & operator++() noexcept(std::is_same_v<Signed, unsigned>);
-    constexpr wide_integer<Bits, Signed> operator++(int) noexcept(is_same<Signed, unsigned>::value);
+    constexpr integer<Bits, Signed> operator++(int) noexcept(std::is_same_v<Signed, unsigned>);
-    constexpr wide_integer<Bits, Signed> & operator--() noexcept(is_same<Signed, unsigned>::value);
+    constexpr integer<Bits, Signed> & operator--() noexcept(std::is_same_v<Signed, unsigned>);
-    constexpr wide_integer<Bits, Signed> operator--(int) noexcept(is_same<Signed, unsigned>::value);
+    constexpr integer<Bits, Signed> operator--(int) noexcept(std::is_same_v<Signed, unsigned>);
 
     // observers
 
@@ -114,10 +122,10 @@ public:
 
 private:
     template <size_t Bits2, typename Signed2>
-    friend class wide_integer;
+    friend class integer;
 
-    friend class numeric_limits<wide_integer<Bits, signed>>;
+    friend class std::numeric_limits<integer<Bits, signed>>;
-    friend class numeric_limits<wide_integer<Bits, unsigned>>;
+    friend class std::numeric_limits<integer<Bits, unsigned>>;
 
     base_type m_arr[_impl::arr_size];
 };
@@ -134,115 +142,117 @@ using __only_integer = typename std::enable_if<IntegralConcept<T>() && IntegralC
 
 // Unary operators
 template <size_t Bits, typename Signed>
-constexpr wide_integer<Bits, Signed> operator~(const wide_integer<Bits, Signed> & lhs) noexcept;
+constexpr integer<Bits, Signed> operator~(const integer<Bits, Signed> & lhs) noexcept;
 
 template <size_t Bits, typename Signed>
-constexpr wide_integer<Bits, Signed> operator-(const wide_integer<Bits, Signed> & lhs) noexcept(is_same<Signed, unsigned>::value);
+constexpr integer<Bits, Signed> operator-(const integer<Bits, Signed> & lhs) noexcept(std::is_same_v<Signed, unsigned>);
 
 template <size_t Bits, typename Signed>
-constexpr wide_integer<Bits, Signed> operator+(const wide_integer<Bits, Signed> & lhs) noexcept(is_same<Signed, unsigned>::value);
+constexpr integer<Bits, Signed> operator+(const integer<Bits, Signed> & lhs) noexcept(std::is_same_v<Signed, unsigned>);
 
 // Binary operators
 template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
-std::common_type_t<wide_integer<Bits, Signed>, wide_integer<Bits2, Signed2>> constexpr
+std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
-operator*(const wide_integer<Bits, Signed> & lhs, const wide_integer<Bits2, Signed2> & rhs);
+operator*(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
 template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
 std::common_type_t<Arithmetic, Arithmetic2> constexpr operator*(const Arithmetic & rhs, const Arithmetic2 & lhs);
 
 template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
-std::common_type_t<wide_integer<Bits, Signed>, wide_integer<Bits2, Signed2>> constexpr
+std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
-operator/(const wide_integer<Bits, Signed> & lhs, const wide_integer<Bits2, Signed2> & rhs);
+operator/(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
 template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
 std::common_type_t<Arithmetic, Arithmetic2> constexpr operator/(const Arithmetic & rhs, const Arithmetic2 & lhs);
 
 template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
-std::common_type_t<wide_integer<Bits, Signed>, wide_integer<Bits2, Signed2>> constexpr
+std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
-operator+(const wide_integer<Bits, Signed> & lhs, const wide_integer<Bits2, Signed2> & rhs);
+operator+(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
 template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
 std::common_type_t<Arithmetic, Arithmetic2> constexpr operator+(const Arithmetic & rhs, const Arithmetic2 & lhs);
 
 template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
-std::common_type_t<wide_integer<Bits, Signed>, wide_integer<Bits2, Signed2>> constexpr
+std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
-operator-(const wide_integer<Bits, Signed> & lhs, const wide_integer<Bits2, Signed2> & rhs);
+operator-(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
 template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
 std::common_type_t<Arithmetic, Arithmetic2> constexpr operator-(const Arithmetic & rhs, const Arithmetic2 & lhs);
 
 template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
-std::common_type_t<wide_integer<Bits, Signed>, wide_integer<Bits2, Signed2>> constexpr
+std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
-operator%(const wide_integer<Bits, Signed> & lhs, const wide_integer<Bits2, Signed2> & rhs);
+operator%(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
 template <typename Integral, typename Integral2, class = __only_integer<Integral, Integral2>>
 std::common_type_t<Integral, Integral2> constexpr operator%(const Integral & rhs, const Integral2 & lhs);
 
 template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
-std::common_type_t<wide_integer<Bits, Signed>, wide_integer<Bits2, Signed2>> constexpr
+std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
-operator&(const wide_integer<Bits, Signed> & lhs, const wide_integer<Bits2, Signed2> & rhs);
+operator&(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
 template <typename Integral, typename Integral2, class = __only_integer<Integral, Integral2>>
 std::common_type_t<Integral, Integral2> constexpr operator&(const Integral & rhs, const Integral2 & lhs);
 
 template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
-std::common_type_t<wide_integer<Bits, Signed>, wide_integer<Bits2, Signed2>> constexpr
+std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
-operator|(const wide_integer<Bits, Signed> & lhs, const wide_integer<Bits2, Signed2> & rhs);
+operator|(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
 template <typename Integral, typename Integral2, class = __only_integer<Integral, Integral2>>
 std::common_type_t<Integral, Integral2> constexpr operator|(const Integral & rhs, const Integral2 & lhs);
 
 template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
-std::common_type_t<wide_integer<Bits, Signed>, wide_integer<Bits2, Signed2>> constexpr
+std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
-operator^(const wide_integer<Bits, Signed> & lhs, const wide_integer<Bits2, Signed2> & rhs);
+operator^(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
 template <typename Integral, typename Integral2, class = __only_integer<Integral, Integral2>>
 std::common_type_t<Integral, Integral2> constexpr operator^(const Integral & rhs, const Integral2 & lhs);
 
 // TODO: Integral
 template <size_t Bits, typename Signed>
-constexpr wide_integer<Bits, Signed> operator<<(const wide_integer<Bits, Signed> & lhs, int n) noexcept;
+constexpr integer<Bits, Signed> operator<<(const integer<Bits, Signed> & lhs, int n) noexcept;
 template <size_t Bits, typename Signed>
-constexpr wide_integer<Bits, Signed> operator>>(const wide_integer<Bits, Signed> & lhs, int n) noexcept;
+constexpr integer<Bits, Signed> operator>>(const integer<Bits, Signed> & lhs, int n) noexcept;
 
 template <size_t Bits, typename Signed, typename Int, typename = std::enable_if_t<!std::is_same_v<Int, int>>>
-constexpr wide_integer<Bits, Signed> operator<<(const wide_integer<Bits, Signed> & lhs, Int n) noexcept
+constexpr integer<Bits, Signed> operator<<(const integer<Bits, Signed> & lhs, Int n) noexcept
 {
     return lhs << int(n);
 }
 template <size_t Bits, typename Signed, typename Int, typename = std::enable_if_t<!std::is_same_v<Int, int>>>
-constexpr wide_integer<Bits, Signed> operator>>(const wide_integer<Bits, Signed> & lhs, Int n) noexcept
+constexpr integer<Bits, Signed> operator>>(const integer<Bits, Signed> & lhs, Int n) noexcept
 {
     return lhs >> int(n);
 }
 
 template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
-constexpr bool operator<(const wide_integer<Bits, Signed> & lhs, const wide_integer<Bits2, Signed2> & rhs);
+constexpr bool operator<(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
 template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
 constexpr bool operator<(const Arithmetic & rhs, const Arithmetic2 & lhs);
 
 template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
-constexpr bool operator>(const wide_integer<Bits, Signed> & lhs, const wide_integer<Bits2, Signed2> & rhs);
+constexpr bool operator>(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
 template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
 constexpr bool operator>(const Arithmetic & rhs, const Arithmetic2 & lhs);
 
 template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
-constexpr bool operator<=(const wide_integer<Bits, Signed> & lhs, const wide_integer<Bits2, Signed2> & rhs);
+constexpr bool operator<=(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
 template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
 constexpr bool operator<=(const Arithmetic & rhs, const Arithmetic2 & lhs);
 
 template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
-constexpr bool operator>=(const wide_integer<Bits, Signed> & lhs, const wide_integer<Bits2, Signed2> & rhs);
+constexpr bool operator>=(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
 template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
 constexpr bool operator>=(const Arithmetic & rhs, const Arithmetic2 & lhs);
 
 template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
-constexpr bool operator==(const wide_integer<Bits, Signed> & lhs, const wide_integer<Bits2, Signed2> & rhs);
+constexpr bool operator==(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
 template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
 constexpr bool operator==(const Arithmetic & rhs, const Arithmetic2 & lhs);
 
 template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
-constexpr bool operator!=(const wide_integer<Bits, Signed> & lhs, const wide_integer<Bits2, Signed2> & rhs);
+constexpr bool operator!=(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
 template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
 constexpr bool operator!=(const Arithmetic & rhs, const Arithmetic2 & lhs);
 
-template <size_t Bits, typename Signed>
+}
-std::string to_string(const wide_integer<Bits, Signed> & n);
+
+namespace std
+{
 
 template <size_t Bits, typename Signed>
-struct hash<wide_integer<Bits, Signed>>;
+struct hash<wide::integer<Bits, Signed>>;
 
 }
 

base/common/wide_integer_to_string.h

@@ -0,0 +1,35 @@
+#pragma once
+
+#include <string>
+
+#include "wide_integer.h"
+
+namespace wide
+{
+
+template <size_t Bits, typename Signed>
+inline std::string to_string(const integer<Bits, Signed> & n)
+{
+    std::string res;
+    if (integer<Bits, Signed>::_impl::operator_eq(n, 0U))
+        return "0";
+
+    integer<Bits, unsigned> t;
+    bool is_neg = integer<Bits, Signed>::_impl::is_negative(n);
+    if (is_neg)
+        t = integer<Bits, Signed>::_impl::operator_unary_minus(n);
+    else
+        t = n;
+
+    while (!integer<Bits, unsigned>::_impl::operator_eq(t, 0U))
+    {
+        res.insert(res.begin(), '0' + char(integer<Bits, unsigned>::_impl::operator_percent(t, 10U)));
+        t = integer<Bits, unsigned>::_impl::operator_slash(t, 10U);
+    }
+
+    if (is_neg)
+        res.insert(res.begin(), '-');
+    return res;
+}
+
+}

cmake/sanitize.cmake

@@ -36,7 +36,15 @@ if (SANITIZE)
         endif ()
 
     elseif (SANITIZE STREQUAL "thread")
-        set (TSAN_FLAGS "-fsanitize=thread -fsanitize-blacklist=${CMAKE_SOURCE_DIR}/tests/tsan_suppressions.txt")
+        set (TSAN_FLAGS "-fsanitize=thread")
+        if (COMPILER_CLANG)
+            set (TSAN_FLAGS "${TSAN_FLAGS} -fsanitize-blacklist=${CMAKE_SOURCE_DIR}/tests/tsan_suppressions.txt")
+        else()
+            message (WARNING "TSAN suppressions was not passed to the compiler (since the compiler is not clang)")
+            message (WARNING "Use the following command to pass them manually:")
+            message (WARNING "    export TSAN_OPTIONS=\"$TSAN_OPTIONS suppressions=${CMAKE_SOURCE_DIR}/tests/tsan_suppressions.txt\"")
+        endif()
+
 
         set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${TSAN_FLAGS}")
         set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} ${TSAN_FLAGS}")

docker/test/fasttest/run.sh

@@ -10,7 +10,7 @@ stage=${stage:-}
 
 # A variable to pass additional flags to CMake.
 # Here we explicitly default it to nothing so that bash doesn't complain about
-# it being undefined. Also read it as array so that we can pass an empty list 
+# it being undefined. Also read it as array so that we can pass an empty list
 # of additional variable to cmake properly, and it doesn't generate an extra
 # empty parameter.
 read -ra FASTTEST_CMAKE_FLAGS <<< "${FASTTEST_CMAKE_FLAGS:-}"
@@ -127,6 +127,7 @@ ln -s /usr/share/clickhouse-test/config/access_management.xml /etc/clickhouse-se
 ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/
 ln -s /usr/share/clickhouse-test/config/strings_dictionary.xml /etc/clickhouse-server/
 ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/
+ln -s /usr/share/clickhouse-test/config/executable_dictionary.xml /etc/clickhouse-server/
 ln -s /usr/share/clickhouse-test/config/macros.xml /etc/clickhouse-server/config.d/
 ln -s /usr/share/clickhouse-test/config/disks.xml /etc/clickhouse-server/config.d/
 #ln -s /usr/share/clickhouse-test/config/secure_ports.xml /etc/clickhouse-server/config.d/

docker/test/performance-comparison/compare.sh

@@ -394,12 +394,24 @@ create table query_run_metrics_denorm engine File(TSV, 'analyze/query-run-metric
     order by test, query_index, metric_names, version, query_id
     ;
 
+-- Filter out tests that don't have an even number of runs, to avoid breaking
+-- the further calculations. This may happen if there was an error during the
+-- test runs, e.g. the server died. It will be reported in test errors, so we
+-- don't have to report it again.
+create view broken_queries as
+    select test, query_index
+    from query_runs
+    group by test, query_index
+    having count(*) % 2 != 0
+    ;
+
 -- This is for statistical processing with eqmed.sql
 create table query_run_metrics_for_stats engine File(
         TSV, -- do not add header -- will parse with grep
         'analyze/query-run-metrics-for-stats.tsv')
     as select test, query_index, 0 run, version, metric_values
     from query_run_metric_arrays
+    where (test, query_index) not in broken_queries
     order by test, query_index, run, version
     ;
 
@@ -915,13 +927,15 @@ done
 
 function report_metrics
 {
+build_log_column_definitions
+
 rm -rf metrics ||:
 mkdir metrics
 
 clickhouse-local --query "
 create view right_async_metric_log as
     select * from file('right-async-metric-log.tsv', TSVWithNamesAndTypes,
-        'event_date Date, event_time DateTime, name String, value Float64')
+        '$(cat right-async-metric-log.tsv.columns)')
     ;
 
 -- Use the right log as time reference because it may have higher precision.
@@ -930,7 +944,7 @@ create table metrics engine File(TSV, 'metrics/metrics.tsv') as
     select name metric, r.event_time - min_time event_time, l.value as left, r.value as right
     from right_async_metric_log r
     asof join file('left-async-metric-log.tsv', TSVWithNamesAndTypes,
-        'event_date Date, event_time DateTime, name String, value Float64') l
+        '$(cat left-async-metric-log.tsv.columns)') l
     on l.name = r.name and r.event_time <= l.event_time
     order by metric, event_time
     ;

docker/test/performance-comparison/eqmed.sql

@@ -8,7 +8,7 @@ select
 from
    (
       -- quantiles of randomization distributions
-      select quantileExactForEach(0.999)(
+      select quantileExactForEach(0.99)(
         arrayMap(x, y -> abs(x - y), metrics_by_label[1], metrics_by_label[2]) as d
       ) threshold
       ---- uncomment to see what the distribution is really like
@@ -33,7 +33,7 @@ from
                                 -- strip the query away before the join -- it might be several kB long;
                                 (select metrics, run, version from table) no_query,
                                 -- duplicate input measurements into many virtual runs
-                                numbers(1, 100000) nn
+                                numbers(1, 10000) nn
                               -- for each virtual run, randomly reorder measurements
                               order by virtual_run, rand()
                            ) virtual_runs

docker/test/performance-comparison/perf.py

@@ -20,7 +20,7 @@ parser = argparse.ArgumentParser(description='Run performance test.')
 parser.add_argument('file', metavar='FILE', type=argparse.FileType('r', encoding='utf-8'), nargs=1, help='test description file')
 parser.add_argument('--host', nargs='*', default=['localhost'], help="Server hostname(s). Corresponds to '--port' options.")
 parser.add_argument('--port', nargs='*', default=[9000], help="Server port(s). Corresponds to '--host' options.")
-parser.add_argument('--runs', type=int, default=int(os.environ.get('CHPC_RUNS', 13)), help='Number of query runs per server. Defaults to CHPC_RUNS environment variable.')
+parser.add_argument('--runs', type=int, default=int(os.environ.get('CHPC_RUNS', 7)), help='Number of query runs per server. Defaults to CHPC_RUNS environment variable.')
 parser.add_argument('--long', action='store_true', help='Do not skip the tests tagged as long.')
 parser.add_argument('--print-queries', action='store_true', help='Print test queries and exit.')
 parser.add_argument('--print-settings', action='store_true', help='Print test settings and exit.')

docker/test/performance-comparison/report.py

@@ -372,7 +372,7 @@ if args.report == 'main':
             'New, s',                                          # 1
             'Ratio of speedup (-) or slowdown (+)',                 # 2
             'Relative difference (new − old) / old',   # 3
-            'p&nbsp;<&nbsp;0.001 threshold',                   # 4
+            'p&nbsp;<&nbsp;0.01 threshold',                   # 4
             # Failed                                           # 5
             'Test',                                            # 6
             '#',                                               # 7
@@ -416,7 +416,7 @@ if args.report == 'main':
             'Old,&nbsp;s', #0
             'New,&nbsp;s', #1
             'Relative difference (new&nbsp;-&nbsp;old)/old', #2
-            'p&nbsp;&lt;&nbsp;0.001 threshold', #3
+            'p&nbsp;&lt;&nbsp;0.01 threshold', #3
             # Failed #4
             'Test', #5
             '#',    #6
@@ -470,12 +470,13 @@ if args.report == 'main':
         text = tableStart('Test times')
         text += tableHeader(columns)
 
-        nominal_runs = 13  # FIXME pass this as an argument
+        nominal_runs = 7  # FIXME pass this as an argument
         total_runs = (nominal_runs + 1) * 2  # one prewarm run, two servers
+        allowed_average_run_time = allowed_single_run_time + 60 / total_runs; # some allowance for fill/create queries
         attrs = ['' for c in columns]
         for r in rows:
             anchor = f'{currentTableAnchor()}.{r[0]}'
-            if float(r[6]) > 1.5 * total_runs:
+            if float(r[6]) > allowed_average_run_time * total_runs:
                 # FIXME should be 15s max -- investigate parallel_insert
                 slow_average_tests += 1
                 attrs[6] = f'style="background: {color_bad}"'
@@ -649,7 +650,7 @@ elif args.report == 'all-queries':
             'New,&nbsp;s', #3
             'Ratio of speedup&nbsp;(-) or slowdown&nbsp;(+)',                 #4
             'Relative difference (new&nbsp;&minus;&nbsp;old) / old', #5
-            'p&nbsp;&lt;&nbsp;0.001 threshold',          #6
+            'p&nbsp;&lt;&nbsp;0.01 threshold',          #6
             'Test',                                   #7
             '#',                                      #8
             'Query',                                  #9

docker/test/stateless/run.sh

@@ -24,6 +24,7 @@ ln -s /usr/share/clickhouse-test/config/access_management.xml /etc/clickhouse-se
 ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/
 ln -s /usr/share/clickhouse-test/config/strings_dictionary.xml /etc/clickhouse-server/
 ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/
+ln -s /usr/share/clickhouse-test/config/executable_dictionary.xml /etc/clickhouse-server/
 ln -s /usr/share/clickhouse-test/config/macros.xml /etc/clickhouse-server/config.d/
 ln -s /usr/share/clickhouse-test/config/disks.xml /etc/clickhouse-server/config.d/
 ln -s /usr/share/clickhouse-test/config/secure_ports.xml /etc/clickhouse-server/config.d/

docker/test/stateless_unbundled/run.sh

@@ -24,6 +24,7 @@ ln -s /usr/share/clickhouse-test/config/access_management.xml /etc/clickhouse-se
 ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/
 ln -s /usr/share/clickhouse-test/config/strings_dictionary.xml /etc/clickhouse-server/
 ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/
+ln -s /usr/share/clickhouse-test/config/executable_dictionary.xml /etc/clickhouse-server/
 ln -s /usr/share/clickhouse-test/config/macros.xml /etc/clickhouse-server/config.d/
 ln -s /usr/share/clickhouse-test/config/disks.xml /etc/clickhouse-server/config.d/
 ln -s /usr/share/clickhouse-test/config/secure_ports.xml /etc/clickhouse-server/config.d/

docker/test/stateless_with_coverage/run.sh

@@ -57,6 +57,7 @@ ln -s /usr/share/clickhouse-test/config/access_management.xml /etc/clickhouse-se
 ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/
 ln -s /usr/share/clickhouse-test/config/strings_dictionary.xml /etc/clickhouse-server/
 ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/
+ln -s /usr/share/clickhouse-test/config/executable_dictionary.xml /etc/clickhouse-server/
 ln -s /usr/share/clickhouse-test/config/macros.xml /etc/clickhouse-server/config.d/
 ln -s /usr/share/clickhouse-test/config/disks.xml /etc/clickhouse-server/config.d/
 ln -s /usr/share/clickhouse-test/config/secure_ports.xml /etc/clickhouse-server/config.d/

docs/en/interfaces/formats.md

@@ -10,42 +10,51 @@ results of a `SELECT`, and to perform `INSERT`s into a file-backed table.
 
 The supported formats are:
 
-| Format                                                          | Input | Output |
+| Format                                                                                  | Input | Output |
-|-----------------------------------------------------------------|-------|--------|
+|-----------------------------------------------------------------------------------------|-------|--------|
-| [TabSeparated](#tabseparated)                                   | ✔     | ✔      |
+| [TabSeparated](#tabseparated)                                                           | ✔     | ✔      |
-| [TabSeparatedRaw](#tabseparatedraw)                             | ✔     | ✔      |
+| [TabSeparatedRaw](#tabseparatedraw)                                                     | ✔     | ✔      |
-| [TabSeparatedWithNames](#tabseparatedwithnames)                 | ✔     | ✔      |
+| [TabSeparatedWithNames](#tabseparatedwithnames)                                         | ✔     | ✔      |
-| [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes) | ✔     | ✔      |
+| [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes)                         | ✔     | ✔      |
-| [Template](#format-template)                                    | ✔     | ✔      |
+| [Template](#format-template)                                                            | ✔     | ✔      |
-| [TemplateIgnoreSpaces](#templateignorespaces)                   | ✔     | ✗      |
+| [TemplateIgnoreSpaces](#templateignorespaces)                                           | ✔     | ✗      |
-| [CSV](#csv)                                                     | ✔     | ✔      |
+| [CSV](#csv)                                                                             | ✔     | ✔      |
-| [CSVWithNames](#csvwithnames)                                   | ✔     | ✔      |
+| [CSVWithNames](#csvwithnames)                                                           | ✔     | ✔      |
-| [CustomSeparated](#format-customseparated)                      | ✔     | ✔      |
+| [CustomSeparated](#format-customseparated)                                              | ✔     | ✔      |
-| [Values](#data-format-values)                                   | ✔     | ✔      |
+| [Values](#data-format-values)                                                           | ✔     | ✔      |
-| [Vertical](#vertical)                                           | ✗     | ✔      |
+| [Vertical](#vertical)                                                                   | ✗     | ✔      |
-| [VerticalRaw](#verticalraw)                                     | ✗     | ✔      |
+| [VerticalRaw](#verticalraw)                                                             | ✗     | ✔      |
-| [JSON](#json)                                                   | ✗     | ✔      |
+| [JSON](#json)                                                                           | ✗     | ✔      |
-| [JSONCompact](#jsoncompact)                                     | ✗     | ✔      |
+| [JSONString](#jsonstring)                                                               | ✗     | ✔      |
-| [JSONEachRow](#jsoneachrow)                                     | ✔     | ✔      |
+| [JSONCompact](#jsoncompact)                                                             | ✗     | ✔      |
-| [TSKV](#tskv)                                                   | ✔     | ✔      |
+| [JSONCompactString](#jsoncompactstring)                                                 | ✗     | ✔      |
-| [Pretty](#pretty)                                               | ✗     | ✔      |
+| [JSONEachRow](#jsoneachrow)                                                             | ✔     | ✔      |
-| [PrettyCompact](#prettycompact)                                 | ✗     | ✔      |
+| [JSONEachRowWithProgress](#jsoneachrowwithprogress)                                     | ✗     | ✔      |
-| [PrettyCompactMonoBlock](#prettycompactmonoblock)               | ✗     | ✔      |
+| [JSONStringEachRow](#jsonstringeachrow)                                                 | ✔     | ✔      |
-| [PrettyNoEscapes](#prettynoescapes)                             | ✗     | ✔      |
+| [JSONStringEachRowWithProgress](#jsonstringeachrowwithprogress)                         | ✗     | ✔      |
-| [PrettySpace](#prettyspace)                                     | ✗     | ✔      |
+| [JSONCompactEachRow](#jsoncompacteachrow)                                               | ✔     | ✔      |
-| [Protobuf](#protobuf)                                           | ✔     | ✔      |
+| [JSONCompactEachRowWithNamesAndTypes](#jsoncompacteachrowwithnamesandtypes)             | ✔     | ✔      |
-| [Avro](#data-format-avro)                                       | ✔     | ✔      |
+| [JSONCompactStringEachRow](#jsoncompactstringeachrow)                                   | ✔     | ✔      |
-| [AvroConfluent](#data-format-avro-confluent)                    | ✔     | ✗      |
+| [JSONCompactStringEachRowWithNamesAndTypes](#jsoncompactstringeachrowwithnamesandtypes) | ✔     | ✔      |
-| [Parquet](#data-format-parquet)                                 | ✔     | ✔      |
+| [TSKV](#tskv)                                                                           | ✔     | ✔      |
-| [Arrow](#data-format-arrow)                                     | ✔     | ✔      |
+| [Pretty](#pretty)                                                                       | ✗     | ✔      |
-| [ArrowStream](#data-format-arrow-stream)                        | ✔     | ✔      |
+| [PrettyCompact](#prettycompact)                                                         | ✗     | ✔      |
-| [ORC](#data-format-orc)                                         | ✔     | ✗      |
+| [PrettyCompactMonoBlock](#prettycompactmonoblock)                                       | ✗     | ✔      |
-| [RowBinary](#rowbinary)                                         | ✔     | ✔      |
+| [PrettyNoEscapes](#prettynoescapes)                                                     | ✗     | ✔      |
-| [RowBinaryWithNamesAndTypes](#rowbinarywithnamesandtypes)       | ✔     | ✔      |
+| [PrettySpace](#prettyspace)                                                             | ✗     | ✔      |
-| [Native](#native)                                               | ✔     | ✔      |
+| [Protobuf](#protobuf)                                                                   | ✔     | ✔      |
-| [Null](#null)                                                   | ✗     | ✔      |
+| [Avro](#data-format-avro)                                                               | ✔     | ✔      |
-| [XML](#xml)                                                     | ✗     | ✔      |
+| [AvroConfluent](#data-format-avro-confluent)                                            | ✔     | ✗      |
-| [CapnProto](#capnproto)                                         | ✔     | ✗      |
+| [Parquet](#data-format-parquet)                                                         | ✔     | ✔      |
+| [Arrow](#data-format-arrow)                                                             | ✔     | ✔      |
+| [ArrowStream](#data-format-arrow-stream)                                                | ✔     | ✔      |
+| [ORC](#data-format-orc)                                                                 | ✔     | ✗      |
+| [RowBinary](#rowbinary)                                                                 | ✔     | ✔      |
+| [RowBinaryWithNamesAndTypes](#rowbinarywithnamesandtypes)                               | ✔     | ✔      |
+| [Native](#native)                                                                       | ✔     | ✔      |
+| [Null](#null)                                                                           | ✗     | ✔      |
+| [XML](#xml)                                                                             | ✗     | ✔      |
+| [CapnProto](#capnproto)                                                                 | ✔     | ✗      |
 
 You can control some format processing parameters with the ClickHouse settings. For more information read the [Settings](../operations/settings/settings.md) section.
 
@@ -392,62 +401,41 @@ SELECT SearchPhrase, count() AS c FROM test.hits GROUP BY SearchPhrase WITH TOTA
         "meta":
         [
                 {
-                        "name": "SearchPhrase",
+                        "name": "'hello'",
                         "type": "String"
                 },
                 {
-                        "name": "c",
+                        "name": "multiply(42, number)",
                         "type": "UInt64"
+                },
+                {
+                        "name": "range(5)",
+                        "type": "Array(UInt8)"
                 }
         ],
 
         "data":
         [
                 {
-                        "SearchPhrase": "",
+                        "'hello'": "hello",
-                        "c": "8267016"
+                        "multiply(42, number)": "0",
+                        "range(5)": [0,1,2,3,4]
                 },
                 {
-                        "SearchPhrase": "bathroom interior design",
+                        "'hello'": "hello",
-                        "c": "2166"
+                        "multiply(42, number)": "42",
+                        "range(5)": [0,1,2,3,4]
                 },
                 {
-                        "SearchPhrase": "yandex",
+                        "'hello'": "hello",
-                        "c": "1655"
+                        "multiply(42, number)": "84",
-                },
+                        "range(5)": [0,1,2,3,4]
-                {
-                        "SearchPhrase": "spring 2014 fashion",
-                        "c": "1549"
-                },
-                {
-                        "SearchPhrase": "freeform photos",
-                        "c": "1480"
                 }
         ],
 
-        "totals":
+        "rows": 3,
-        {
-                "SearchPhrase": "",
-                "c": "8873898"
-        },
 
-        "extremes":
+        "rows_before_limit_at_least": 3
-        {
-                "min":
-                {
-                        "SearchPhrase": "",
-                        "c": "1480"
-                },
-                "max":
-                {
-                        "SearchPhrase": "",
-                        "c": "8267016"
-                }
-        },
-
-        "rows": 5,
-
-        "rows_before_limit_at_least": 141137
 }
 ```
 
@@ -468,63 +456,165 @@ ClickHouse supports [NULL](../sql-reference/syntax.md), which is displayed as `n
 
 See also the [JSONEachRow](#jsoneachrow) format.
 
+## JSONString {#jsonstring}
+
+Differs from JSON only in that data fields are output in strings, not in typed json values.
+
+Example:
+
+```json
+{
+        "meta":
+        [
+                {
+                        "name": "'hello'",
+                        "type": "String"
+                },
+                {
+                        "name": "multiply(42, number)",
+                        "type": "UInt64"
+                },
+                {
+                        "name": "range(5)",
+                        "type": "Array(UInt8)"
+                }
+        ],
+
+        "data":
+        [
+                {
+                        "'hello'": "hello",
+                        "multiply(42, number)": "0",
+                        "range(5)": "[0,1,2,3,4]"
+                },
+                {
+                        "'hello'": "hello",
+                        "multiply(42, number)": "42",
+                        "range(5)": "[0,1,2,3,4]"
+                },
+                {
+                        "'hello'": "hello",
+                        "multiply(42, number)": "84",
+                        "range(5)": "[0,1,2,3,4]"
+                }
+        ],
+
+        "rows": 3,
+
+        "rows_before_limit_at_least": 3
+}
+```
+
 ## JSONCompact {#jsoncompact}
+## JSONCompactString {#jsoncompactstring}
 
 Differs from JSON only in that data rows are output in arrays, not in objects.
 
 Example:
 
 ``` json
+// JSONCompact
 {
         "meta":
         [
                 {
-                        "name": "SearchPhrase",
+                        "name": "'hello'",
                         "type": "String"
                 },
                 {
-                        "name": "c",
+                        "name": "multiply(42, number)",
                         "type": "UInt64"
+                },
+                {
+                        "name": "range(5)",
+                        "type": "Array(UInt8)"
                 }
         ],
 
         "data":
         [
-                ["", "8267016"],
+                ["hello", "0", [0,1,2,3,4]],
-                ["bathroom interior design", "2166"],
+                ["hello", "42", [0,1,2,3,4]],
-                ["yandex", "1655"],
+                ["hello", "84", [0,1,2,3,4]]
-                ["fashion trends spring 2014", "1549"],
-                ["freeform photo", "1480"]
         ],
 
-        "totals": ["","8873898"],
+        "rows": 3,
 
-        "extremes":
+        "rows_before_limit_at_least": 3
-        {
-                "min": ["","1480"],
-                "max": ["","8267016"]
-        },
-
-        "rows": 5,
-
-        "rows_before_limit_at_least": 141137
 }
 ```
 
-This format is only appropriate for outputting a query result, but not for parsing (retrieving data to insert in a table).
+```json
-See also the `JSONEachRow` format.
+// JSONCompactString
+{
+        "meta":
+        [
+                {
+                        "name": "'hello'",
+                        "type": "String"
+                },
+                {
+                        "name": "multiply(42, number)",
+                        "type": "UInt64"
+                },
+                {
+                        "name": "range(5)",
+                        "type": "Array(UInt8)"
+                }
+        ],
 
-## JSONEachRow {#jsoneachrow}
+        "data":
+        [
+                ["hello", "0", "[0,1,2,3,4]"],
+                ["hello", "42", "[0,1,2,3,4]"],
+                ["hello", "84", "[0,1,2,3,4]"]
+        ],
 
-When using this format, ClickHouse outputs rows as separated, newline-delimited JSON objects, but the data as a whole is not valid JSON.
+        "rows": 3,
 
-``` json
+        "rows_before_limit_at_least": 3
-{"SearchPhrase":"curtain designs","count()":"1064"}
+}
-{"SearchPhrase":"baku","count()":"1000"}
-{"SearchPhrase":"","count()":"8267016"}
 ```
 
-When inserting the data, you should provide a separate JSON object for each row.
+## JSONEachRow {#jsoneachrow}
+## JSONStringEachRow {#jsonstringeachrow}
+## JSONCompactEachRow {#jsoncompacteachrow}
+## JSONCompactStringEachRow {#jsoncompactstringeachrow}
+
+When using these formats, ClickHouse outputs rows as separated, newline-delimited JSON values, but the data as a whole is not valid JSON.
+
+``` json
+{"some_int":42,"some_str":"hello","some_tuple":[1,"a"]} // JSONEachRow
+[42,"hello",[1,"a"]] // JSONCompactEachRow
+["42","hello","(2,'a')"] // JSONCompactStringsEachRow
+```
+
+When inserting the data, you should provide a separate JSON value for each row.
+
+## JSONEachRowWithProgress {#jsoneachrowwithprogress}
+## JSONStringEachRowWithProgress {#jsonstringeachrowwithprogress}
+
+Differs from JSONEachRow/JSONStringEachRow in that ClickHouse will also yield progress information as JSON objects.
+
+```json
+{"row":{"'hello'":"hello","multiply(42, number)":"0","range(5)":[0,1,2,3,4]}}
+{"row":{"'hello'":"hello","multiply(42, number)":"42","range(5)":[0,1,2,3,4]}}
+{"row":{"'hello'":"hello","multiply(42, number)":"84","range(5)":[0,1,2,3,4]}}
+{"progress":{"read_rows":"3","read_bytes":"24","written_rows":"0","written_bytes":"0","total_rows_to_read":"3"}}
+```
+
+## JSONCompactEachRowWithNamesAndTypes {#jsoncompacteachrowwithnamesandtypes}
+## JSONCompactStringEachRowWithNamesAndTypes {#jsoncompactstringeachrowwithnamesandtypes}
+
+Differs from JSONCompactEachRow/JSONCompactStringEachRow in that the column names and types are written as the first two rows.
+
+```json
+["'hello'", "multiply(42, number)", "range(5)"]
+["String", "UInt64", "Array(UInt8)"]
+["hello", "0", [0,1,2,3,4]]
+["hello", "42", [0,1,2,3,4]]
+["hello", "84", [0,1,2,3,4]]
+```
 
 ### Inserting Data {#inserting-data}
 

docs/en/operations/system-tables/asynchronous_metric_log.md

@@ -6,6 +6,7 @@ Columns:
 
 -   `event_date` ([Date](../../sql-reference/data-types/date.md)) — Event date.
 -   `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Event time.
+-   `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — Event time with microseconds resolution.
 -   `name` ([String](../../sql-reference/data-types/string.md)) — Metric name.
 -   `value` ([Float64](../../sql-reference/data-types/float.md)) — Metric value.
 
@@ -16,18 +17,18 @@ SELECT * FROM system.asynchronous_metric_log LIMIT 10
 ```
 
 ``` text
-┌─event_date─┬──────────event_time─┬─name─────────────────────────────────────┬────value─┐
+┌─event_date─┬──────────event_time─┬────event_time_microseconds─┬─name─────────────────────────────────────┬─────value─┐
-│ 2020-06-22 │ 2020-06-22 06:57:30 │ jemalloc.arenas.all.pmuzzy               │        0 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ CPUFrequencyMHz_0                        │    2120.9 │
-│ 2020-06-22 │ 2020-06-22 06:57:30 │ jemalloc.arenas.all.pdirty               │     4214 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.arenas.all.pmuzzy               │       743 │
-│ 2020-06-22 │ 2020-06-22 06:57:30 │ jemalloc.background_thread.run_intervals │        0 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.arenas.all.pdirty               │     26288 │
-│ 2020-06-22 │ 2020-06-22 06:57:30 │ jemalloc.background_thread.num_runs      │        0 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.background_thread.run_intervals │         0 │
-│ 2020-06-22 │ 2020-06-22 06:57:30 │ jemalloc.retained                        │ 17657856 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.background_thread.num_runs      │         0 │
-│ 2020-06-22 │ 2020-06-22 06:57:30 │ jemalloc.mapped                          │ 71471104 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.retained                        │  60694528 │
-│ 2020-06-22 │ 2020-06-22 06:57:30 │ jemalloc.resident                        │ 61538304 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.mapped                          │ 303161344 │
-│ 2020-06-22 │ 2020-06-22 06:57:30 │ jemalloc.metadata                        │  6199264 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.resident                        │ 260931584 │
-│ 2020-06-22 │ 2020-06-22 06:57:30 │ jemalloc.allocated                       │ 38074336 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.metadata                        │  12079488 │
-│ 2020-06-22 │ 2020-06-22 06:57:30 │ jemalloc.epoch                           │        2 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.allocated                       │ 133756128 │
-└────────────┴─────────────────────┴──────────────────────────────────────────┴──────────┘
+└────────────┴─────────────────────┴────────────────────────────┴──────────────────────────────────────────┴───────────┘
 ```
 
 **See Also**

docs/en/operations/system-tables/merges.md

@@ -10,12 +10,16 @@ Columns:
 -   `progress` (Float64) — The percentage of completed work from 0 to 1.
 -   `num_parts` (UInt64) — The number of pieces to be merged.
 -   `result_part_name` (String) — The name of the part that will be formed as the result of merging.
--   `is_mutation` (UInt8) - 1 if this process is a part mutation.
+-   `is_mutation` (UInt8) — 1 if this process is a part mutation.
 -   `total_size_bytes_compressed` (UInt64) — The total size of the compressed data in the merged chunks.
 -   `total_size_marks` (UInt64) — The total number of marks in the merged parts.
 -   `bytes_read_uncompressed` (UInt64) — Number of bytes read, uncompressed.
 -   `rows_read` (UInt64) — Number of rows read.
 -   `bytes_written_uncompressed` (UInt64) — Number of bytes written, uncompressed.
 -   `rows_written` (UInt64) — Number of rows written.
+-   `memory_usage` (UInt64) — Memory consumption of the merge process.
+-   `thread_id` (UInt64) — Thread ID of the merge process.
+-   `merge_type` — The type of current merge. Empty if it's an mutation.
+-   `merge_algorithm` — The algorithm used in current merge. Empty if it's an mutation.
 
 [Original article](https://clickhouse.tech/docs/en/operations/system_tables/merges) <!--hide-->

docs/en/operations/system-tables/metric_log.md

@@ -23,28 +23,28 @@ SELECT * FROM system.metric_log LIMIT 1 FORMAT Vertical;
 ``` text
 Row 1:
 ──────
-event_date:                                                 2020-02-18
+event_date:                                                      2020-09-05
-event_time:                                                 2020-02-18 07:15:33
+event_time:                                                      2020-09-05 16:22:33
-milliseconds:                                               554
+event_time_microseconds:                                         2020-09-05 16:22:33.196807
-ProfileEvent_Query:                                         0
+milliseconds:                                                    196
-ProfileEvent_SelectQuery:                                   0
+ProfileEvent_Query:                                              0
-ProfileEvent_InsertQuery:                                   0
+ProfileEvent_SelectQuery:                                        0
-ProfileEvent_FileOpen:                                      0
+ProfileEvent_InsertQuery:                                        0
-ProfileEvent_Seek:                                          0
+ProfileEvent_FailedQuery:                                        0
-ProfileEvent_ReadBufferFromFileDescriptorRead:              1
+ProfileEvent_FailedSelectQuery:                                  0
-ProfileEvent_ReadBufferFromFileDescriptorReadFailed:        0
-ProfileEvent_ReadBufferFromFileDescriptorReadBytes:         0
-ProfileEvent_WriteBufferFromFileDescriptorWrite:            1
-ProfileEvent_WriteBufferFromFileDescriptorWriteFailed:      0
-ProfileEvent_WriteBufferFromFileDescriptorWriteBytes:       56
 ...
-CurrentMetric_Query:                                        0
-CurrentMetric_Merge:                                        0
-CurrentMetric_PartMutation:                                 0
-CurrentMetric_ReplicatedFetch:                              0
-CurrentMetric_ReplicatedSend:                               0
-CurrentMetric_ReplicatedChecks:                             0
 ...
+CurrentMetric_Revision:                                          54439
+CurrentMetric_VersionInteger:                                    20009001
+CurrentMetric_RWLockWaitingReaders:                              0
+CurrentMetric_RWLockWaitingWriters:                              0
+CurrentMetric_RWLockActiveReaders:                               0
+CurrentMetric_RWLockActiveWriters:                               0
+CurrentMetric_GlobalThread:                                      74
+CurrentMetric_GlobalThreadActive:                                26
+CurrentMetric_LocalThread:                                       0
+CurrentMetric_LocalThreadActive:                                 0
+CurrentMetric_DistributedFilesToInsert:                          0
 ```
 
 **See also**

docs/en/sql-reference/functions/other-functions.md

@@ -515,6 +515,29 @@ SELECT
 └────────────────┴────────────┘
 ```
 
+## formatReadableQuantity(x) {#formatreadablequantityx}
+
+Accepts the number. Returns a rounded number with a suffix (thousand, million, billion, etc.) as a string.
+
+It is useful for reading big numbers by human.
+
+Example:
+
+``` sql
+SELECT
+    arrayJoin([1024, 1234 * 1000, (4567 * 1000) * 1000, 98765432101234]) AS number,
+    formatReadableQuantity(number) AS number_for_humans
+```
+
+``` text
+┌─────────number─┬─number_for_humans─┐
+│           1024 │ 1.02 thousand     │
+│        1234000 │ 1.23 million      │
+│     4567000000 │ 4.57 billion      │
+│ 98765432101234 │ 98.77 trillion    │
+└────────────────┴───────────────────┘
+```
+
 ## least(a, b) {#leasta-b}
 
 Returns the smallest value from a and b.

docs/en/sql-reference/functions/tuple-map-functions.md

@@ -46,3 +46,25 @@ SELECT mapSubtract(([toUInt8(1), 2], [toInt32(1), 1]), ([toUInt8(1), 2], [toInt3
 │ ([1,2],[-1,0]) │ Tuple(Array(UInt8), Array(Int64)) │
 └────────────────┴───────────────────────────────────┘
 ````
+
+## mapPopulateSeries {#function-mappopulateseries}
+
+Syntax: `mapPopulateSeries((keys : Array(<IntegerType>), values : Array(<IntegerType>)[, max : <IntegerType>])`
+
+Generates a map, where keys are a series of numbers, from minimum to maximum keys (or `max` argument if it specified) taken from `keys` array with step size of one,
+and corresponding values taken from `values` array. If the value is not specified for the key, then it uses default value in the resulting map.
+For repeated keys only the first value (in order of appearing) gets associated with the key.
+
+The number of elements in `keys` and `values` must be the same for each row.
+
+Returns a tuple of two arrays: keys in sorted order, and values the corresponding keys.
+
+``` sql
+select mapPopulateSeries([1,2,4], [11,22,44], 5) as res, toTypeName(res) as type;
+```
+
+``` text
+┌─res──────────────────────────┬─type──────────────────────────────┐
+│ ([1,2,3,4,5],[11,22,0,44,0]) │ Tuple(Array(UInt8), Array(UInt8)) │
+└──────────────────────────────┴───────────────────────────────────┘
+```

docs/es/operations/backup.md

@@ -1,20 +1,18 @@
 ---
-machine_translated: true
-machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd
 toc_priority: 49
 toc_title: Copia de seguridad de datos
 ---
 
 # Copia de seguridad de datos {#data-backup}
 
-Mientras [replicación](../engines/table-engines/mergetree-family/replication.md) provides protection from hardware failures, it does not protect against human errors: accidental deletion of data, deletion of the wrong table or a table on the wrong cluster, and software bugs that result in incorrect data processing or data corruption. In many cases mistakes like these will affect all replicas. ClickHouse has built-in safeguards to prevent some types of mistakes — for example, by default [no puede simplemente eliminar tablas con un motor similar a MergeTree que contenga más de 50 Gb de datos](https://github.com/ClickHouse/ClickHouse/blob/v18.14.18-stable/programs/server/config.xml#L322-L330). Sin embargo, estas garantías no cubren todos los casos posibles y pueden eludirse.
+Mientras que la [replicación](../engines/table-engines/mergetree-family/replication.md) proporciona protección contra fallos de hardware, no protege de errores humanos: el borrado accidental de datos, elminar la tabla equivocada o una tabla en el clúster equivocado, y bugs de software que dan como resultado un procesado incorrecto de los datos o la corrupción de los datos. En muchos casos, errores como estos afectarán a todas las réplicas. ClickHouse dispone de salvaguardas para prevenir algunos tipos de errores — por ejemplo, por defecto [no se puede simplemente eliminar tablas con un motor similar a MergeTree que contenga más de 50 Gb de datos](https://github.com/ClickHouse/ClickHouse/blob/v18.14.18-stable/programs/server/config.xml#L322-L330). Sin embargo, estas salvaguardas no cubren todos los casos posibles y pueden eludirse.
 
 Para mitigar eficazmente los posibles errores humanos, debe preparar cuidadosamente una estrategia para realizar copias de seguridad y restaurar sus datos **previamente**.
 
-Cada empresa tiene diferentes recursos disponibles y requisitos comerciales, por lo que no existe una solución universal para las copias de seguridad y restauraciones de ClickHouse que se adapten a cada situación. Lo que funciona para un gigabyte de datos probablemente no funcionará para decenas de petabytes. Hay una variedad de posibles enfoques con sus propios pros y contras, que se discutirán a continuación. Es una buena idea utilizar varios enfoques en lugar de solo uno para compensar sus diversas deficiencias.
+Cada empresa tiene diferentes recursos disponibles y requisitos comerciales, por lo que no existe una solución universal para las copias de seguridad y restauraciones de ClickHouse que se adapten a cada situación. Lo que funciona para un gigabyte de datos probablemente no funcionará para decenas de petabytes. Hay una variedad de posibles enfoques con sus propios pros y contras, que se discutirán a continuación. Es una buena idea utilizar varios enfoques en lugar de uno solo para compensar sus diversas deficiencias.
 
 !!! note "Nota"
-    Tenga en cuenta que si realizó una copia de seguridad de algo y nunca intentó restaurarlo, es probable que la restauración no funcione correctamente cuando realmente la necesite (o al menos tomará más tiempo de lo que las empresas pueden tolerar). Por lo tanto, cualquiera que sea el enfoque de copia de seguridad que elija, asegúrese de automatizar el proceso de restauración también y practicarlo en un clúster de ClickHouse de repuesto regularmente.
+    Tenga en cuenta que si realizó una copia de seguridad de algo y nunca intentó restaurarlo, es probable que la restauración no funcione correctamente cuando realmente la necesite (o al menos tomará más tiempo de lo que las empresas pueden tolerar). Por lo tanto, cualquiera que sea el enfoque de copia de seguridad que elija, asegúrese de automatizar el proceso de restauración también y ponerlo en practica en un clúster de ClickHouse de repuesto regularmente.
 
 ## Duplicar datos de origen en otro lugar {#duplicating-source-data-somewhere-else}
 
@@ -32,7 +30,7 @@ Para volúmenes de datos más pequeños, un simple `INSERT INTO ... SELECT ...`
 
 ## Manipulaciones con piezas {#manipulations-with-parts}
 
-ClickHouse permite usar el `ALTER TABLE ... FREEZE PARTITION ...` consulta para crear una copia local de particiones de tabla. Esto se implementa utilizando enlaces duros al `/var/lib/clickhouse/shadow/` carpeta, por lo que generalmente no consume espacio adicional en disco para datos antiguos. Las copias creadas de archivos no son manejadas por el servidor ClickHouse, por lo que puede dejarlas allí: tendrá una copia de seguridad simple que no requiere ningún sistema externo adicional, pero seguirá siendo propenso a problemas de hardware. Por esta razón, es mejor copiarlos de forma remota en otra ubicación y luego eliminar las copias locales. Los sistemas de archivos distribuidos y los almacenes de objetos siguen siendo una buena opción para esto, pero los servidores de archivos conectados normales con una capacidad lo suficientemente grande podrían funcionar también (en este caso, la transferencia ocurrirá a través del sistema de archivos de red o tal vez [rsync](https://en.wikipedia.org/wiki/Rsync)).
+ClickHouse permite usar la consulta `ALTER TABLE ... FREEZE PARTITION ...` para crear una copia local de particiones de tabla. Esto se implementa utilizando enlaces duros a la carpeta `/var/lib/clickhouse/shadow/`, por lo que generalmente no consume espacio adicional en disco para datos antiguos. Las copias creadas de archivos no son manejadas por el servidor ClickHouse, por lo que puede dejarlas allí: tendrá una copia de seguridad simple que no requiere ningún sistema externo adicional, pero seguirá siendo propenso a problemas de hardware. Por esta razón, es mejor copiarlos de forma remota en otra ubicación y luego eliminar las copias locales. Los sistemas de archivos distribuidos y los almacenes de objetos siguen siendo una buena opción para esto, pero los servidores de archivos conectados normales con una capacidad lo suficientemente grande podrían funcionar también (en este caso, la transferencia ocurrirá a través del sistema de archivos de red o tal vez [rsync](https://en.wikipedia.org/wiki/Rsync)).
 
 Para obtener más información sobre las consultas relacionadas con las manipulaciones de particiones, consulte [Documentación de ALTER](../sql-reference/statements/alter.md#alter_manipulations-with-partitions).
 

docs/ru/interfaces/formats.md

@@ -28,6 +28,8 @@ ClickHouse может принимать (`INSERT`) и отдавать (`SELECT
 | [PrettySpace](#prettyspace)                                     | ✗      | ✔      |
 | [Protobuf](#protobuf)                                           | ✔      | ✔      |
 | [Parquet](#data-format-parquet)                                 | ✔      | ✔      |
+| [Arrow](#data-format-arrow)                                     | ✔      | ✔      |
+| [ArrowStream](#data-format-arrow-stream)                        | ✔      | ✔      |
 | [ORC](#data-format-orc)                                         | ✔      | ✗      |
 | [RowBinary](#rowbinary)                                         | ✔      | ✔      |
 | [RowBinaryWithNamesAndTypes](#rowbinarywithnamesandtypes)       | ✔      | ✔      |
@@ -947,6 +949,12 @@ ClickHouse пишет и читает сообщения `Protocol Buffers` в
 
 ## Avro {#data-format-avro}
 
+[Apache Avro](https://avro.apache.org/) — это ориентированный на строки фреймворк для сериализации данных. Разработан в рамках проекта Apache Hadoop.
+
+В ClickHouse формат Avro поддерживает чтение и запись [файлов данных Avro](https://avro.apache.org/docs/current/spec.html#Object+Container+Files).
+
+[Логические типы Avro](https://avro.apache.org/docs/current/spec.html#Logical+Types)
+
 ## AvroConfluent {#data-format-avro-confluent}
 
 Для формата `AvroConfluent` ClickHouse поддерживает декодирование сообщений `Avro` с одним объектом. Такие сообщения используются с [Kafka] (http://kafka.apache.org/) и  реестром схем [Confluent](https://docs.confluent.io/current/schema-registry/index.html). 
@@ -996,7 +1004,7 @@ SELECT * FROM topic1_stream;
 
 ## Parquet {#data-format-parquet}
 
-[Apache Parquet](http://parquet.apache.org/) — формат поколоночного хранения данных, который распространён в экосистеме Hadoop. Для формата `Parquet` ClickHouse поддерживает операции чтения и записи.
+[Apache Parquet](https://parquet.apache.org/) — формат поколоночного хранения данных, который распространён в экосистеме Hadoop. Для формата `Parquet` ClickHouse поддерживает операции чтения и записи.
 
 ### Соответствие типов данных {#sootvetstvie-tipov-dannykh}
 
@@ -1042,6 +1050,16 @@ $ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Parquet" > {some_
 
 Для обмена данными с экосистемой Hadoop можно использовать движки таблиц [HDFS](../engines/table-engines/integrations/hdfs.md).
 
+## Arrow {data-format-arrow}
+
+[Apache Arrow](https://arrow.apache.org/) поставляется с двумя встроенными поколоночнами форматами хранения. ClickHouse поддерживает операции чтения и записи для этих форматов.
+
+`Arrow` — это Apache Arrow's "file mode" формат. Он предназначен для произвольного доступа в памяти.
+
+## ArrowStream {data-format-arrow-stream}
+
+`ArrowStream` — это Apache Arrow's "stream mode" формат. Он предназначен для обработки потоков в памяти.
+
 ## ORC {#data-format-orc}
 
 [Apache ORC](https://orc.apache.org/) - это column-oriented формат данных, распространённый в экосистеме Hadoop. Вы можете только вставлять данные этого формата в ClickHouse.

docs/ru/sql-reference/functions/other-functions.md

@@ -508,6 +508,29 @@ SELECT
 └────────────────┴────────────┘
 ```
 
+## formatReadableQuantity(x) {#formatreadablequantityx}
+
+Принимает число. Возвращает округленное число с суффиксом (thousand, million, billion и т.д.) в виде строки.
+
+Облегчает визуальное восприятие больших чисел живым человеком.
+
+Пример:
+
+``` sql
+SELECT
+    arrayJoin([1024, 1234 * 1000, (4567 * 1000) * 1000, 98765432101234]) AS number,
+    formatReadableQuantity(number) AS number_for_humans
+```
+
+``` text
+┌─────────number─┬─number_for_humans─┐
+│           1024 │ 1.02 thousand     │
+│        1234000 │ 1.23 million      │
+│     4567000000 │ 4.57 billion      │
+│ 98765432101234 │ 98.77 trillion    │
+└────────────────┴───────────────────┘
+```
+
 ## least(a, b) {#leasta-b}
 
 Возвращает наименьшее значение из a и b.

docs/ru/sql-reference/functions/random-functions.md

@@ -55,4 +55,50 @@ FROM numbers(3)
 └────────────┴────────────┴──────────────┴────────────────┴─────────────────┴──────────────────────┘
 ```
 
+# Случайные функции для работы со строками {#random-functions-for-working-with-strings}
+
+## randomString {#random-string}
+
+## randomFixedString {#random-fixed-string}
+
+## randomPrintableASCII {#random-printable-ascii}
+
+## randomStringUTF8 {#random-string-utf8}
+
+## fuzzBits {#fuzzbits}
+
+**Синтаксис**
+
+``` sql
+fuzzBits([s], [prob])
+```
+Инвертирует каждый бит `s` с вероятностью `prob`.
+
+**Параметры**
+
+- `s` — `String` or `FixedString`
+- `prob` — constant `Float32/64`
+
+**Возвращаемое значение**
+
+Измененная случайным образом строка с тем же типом, что и `s`.
+
+**Пример**
+
+Запрос:
+
+``` sql
+SELECT fuzzBits(materialize('abacaba'), 0.1)
+FROM numbers(3)
+```
+
+Результат:
+
+``` text
+┌─fuzzBits(materialize('abacaba'), 0.1)─┐
+│ abaaaja                               │
+│ a*cjab+                               │
+│ aeca2A                                │
+└───────────────────────────────────────┘
+
 [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/random_functions/) <!--hide-->

docs/ru/sql-reference/functions/type-conversion-functions.md

@@ -513,4 +513,95 @@ SELECT parseDateTimeBestEffort('10 20:19')
 -   [toDate](#todate)
 -   [toDateTime](#todatetime)
 
+## toUnixTimestamp64Milli
+## toUnixTimestamp64Micro
+## toUnixTimestamp64Nano
+
+Преобразует значение `DateTime64` в значение `Int64` с фиксированной точностью менее одной секунды. 
+Входное значение округляется соответствующим образом вверх или вниз в зависимости от его точности. Обратите внимание, что возвращаемое значение - это временная метка в UTC, а не в часовом поясе `DateTime64`.
+
+**Синтаксис**
+
+``` sql
+toUnixTimestamp64Milli(value)
+```
+
+**Параметры**
+
+-   `value` — значение `DateTime64` с любой точностью.
+
+**Возвращаемое значение**
+
+-   Значение `value`, преобразованное в тип данных `Int64`.
+
+**Примеры**
+
+Запрос:
+
+``` sql
+WITH toDateTime64('2019-09-16 19:20:12.345678910', 6) AS dt64
+SELECT toUnixTimestamp64Milli(dt64)
+```
+
+Ответ:
+
+``` text
+┌─toUnixTimestamp64Milli(dt64)─┐
+│                1568650812345 │
+└──────────────────────────────┘
+```
+
+Запрос: 
+
+``` sql
+WITH toDateTime64('2019-09-16 19:20:12.345678910', 6) AS dt64
+SELECT toUnixTimestamp64Nano(dt64)
+```
+
+Ответ:
+
+``` text
+┌─toUnixTimestamp64Nano(dt64)─┐
+│         1568650812345678000 │
+└─────────────────────────────┘
+```
+
+## fromUnixTimestamp64Milli
+## fromUnixTimestamp64Micro
+## fromUnixTimestamp64Nano
+
+Преобразует значение `Int64` в значение `DateTime64` с фиксированной точностью менее одной секунды и дополнительным часовым поясом. Входное значение округляется соответствующим образом вверх или вниз в зависимости от его точности. Обратите внимание, что входное значение обрабатывается как метка времени UTC, а не метка времени в заданном (или неявном) часовом поясе.
+
+**Синтаксис**
+
+``` sql
+fromUnixTimestamp64Milli(value [, ti])
+```
+
+**Параметры**
+
+-   `value` — значение типы `Int64` с любой точностью.
+-   `timezone` — (не обязательный параметр) часовой пояс в формате `String` для возвращаемого результата.
+
+**Возвращаемое значение**
+
+-   Значение `value`, преобразованное в тип данных `DateTime64`.
+
+**Пример**
+
+Запрос:
+
+``` sql
+WITH CAST(1234567891011, 'Int64') AS i64
+SELECT fromUnixTimestamp64Milli(i64, 'UTC')
+```
+
+Ответ:
+
+``` text
+┌─fromUnixTimestamp64Milli(i64, 'UTC')─┐
+│              2009-02-13 23:31:31.011 │
+└──────────────────────────────────────┘
+```
+
 [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/type_conversion_functions/) <!--hide-->

docs/ru/sql-reference/statements/create/view.md

@@ -5,13 +5,15 @@ toc_title: Представление
 
 # CREATE VIEW {#create-view}
 
-``` sql
-CREATE [MATERIALIZED] VIEW [IF NOT EXISTS] [db.]table_name [TO[db.]name] [ENGINE = engine] [POPULATE] AS SELECT ...
-```
-
 Создаёт представление. Представления бывают двух видов - обычные и материализованные (MATERIALIZED).
 
-Обычные представления не хранят никаких данных, а всего лишь производят чтение из другой таблицы. То есть, обычное представление - не более чем сохранённый запрос. При чтении из представления, этот сохранённый запрос, используется в качестве подзапроса в секции FROM.
+## Обычные представления {#normal}
+
+``` sql
+CREATE [OR REPLACE] VIEW [IF NOT EXISTS] [db.]table_name [ON CLUSTER] AS SELECT ...
+```
+
+Normal views don’t store any data, they just perform a read from another table on each access. In other words, a normal view is nothing more than a saved query. When reading from a view, this saved query is used as a subquery in the [FROM](../../../sql-reference/statements/select/from.md) clause.
 
 Для примера, пусть вы создали представление:
 
@@ -31,15 +33,24 @@ SELECT a, b, c FROM view
 SELECT a, b, c FROM (SELECT ...)
 ```
 
-Материализованные (MATERIALIZED) представления хранят данные, преобразованные соответствующим запросом SELECT.
+## Материализованные представления {#materialized}
 
-При создании материализованного представления без использования `TO [db].[table]`, нужно обязательно указать ENGINE - движок таблицы для хранения данных.
+``` sql
+CREATE MATERIALIZED VIEW [IF NOT EXISTS] [db.]table_name [ON CLUSTER] [TO[db.]name] [ENGINE = engine] [POPULATE] AS SELECT ...
+```
+
+Материализованные (MATERIALIZED) представления хранят данные, преобразованные соответствующим запросом [SELECT](../../../sql-reference/statements/select/index.md).
+
+При создании материализованного представления без использования `TO [db].[table]`, нужно обязательно указать `ENGINE` - движок таблицы для хранения данных.
 
 При создании материализованного представления с испольованием `TO [db].[table]`, нельзя указывать `POPULATE`
 
 Материализованное представление устроено следующим образом: при вставке данных в таблицу, указанную в SELECT-е, кусок вставляемых данных преобразуется этим запросом SELECT, и полученный результат вставляется в представление.
 
-Если указано POPULATE, то при создании представления, в него будут вставлены имеющиеся данные таблицы, как если бы был сделан запрос `CREATE TABLE ... AS SELECT ...` . Иначе, представление будет содержать только данные, вставляемые в таблицу после создания представления. Не рекомендуется использовать POPULATE, так как вставляемые в таблицу данные во время создания представления, не попадут в него.
+!!! important "Важно"
+    Материализованные представлени в ClickHouse больше похожи на `after insert` триггеры. Если в запросе материализованного представления есть агрегирование, оно применяется только к вставляемому блоку записей. Любые изменения существующих данных исходной таблицы (например обновление, удаление, удаление раздела и т.д.) не изменяют материализованное представление.
+
+Если указано `POPULATE`, то при создании представления, в него будут вставлены имеющиеся данные таблицы, как если бы был сделан запрос `CREATE TABLE ... AS SELECT ...` . Иначе, представление будет содержать только данные, вставляемые в таблицу после создания представления. Не рекомендуется использовать POPULATE, так как вставляемые в таблицу данные во время создания представления, не попадут в него.
 
 Запрос `SELECT` может содержать `DISTINCT`, `GROUP BY`, `ORDER BY`, `LIMIT`… Следует иметь ввиду, что соответствующие преобразования будут выполняться независимо, на каждый блок вставляемых данных. Например, при наличии `GROUP BY`, данные будут агрегироваться при вставке, но только в рамках одной пачки вставляемых данных. Далее, данные не будут доагрегированы. Исключение - использование ENGINE, производящего агрегацию данных самостоятельно, например, `SummingMergeTree`.
 
@@ -50,4 +61,4 @@ SELECT a, b, c FROM (SELECT ...)
 Отсутствует отдельный запрос для удаления представлений. Чтобы удалить представление, следует использовать `DROP TABLE`.
 
 [Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/create/view) 
-<!--hide-->
+<!--hide-->

docs/ru/sql-reference/statements/drop.md

@@ -5,18 +5,35 @@ toc_title: DROP
 
 # DROP {#drop}
 
-Запрос имеет два вида: `DROP DATABASE` и `DROP TABLE`.
+Удаляет существующий объект. 
+Если указано `IF EXISTS` - не выдавать ошибку, если объекта не существует.
+
+## DROP DATABASE {#drop-database}
 
 ``` sql
 DROP DATABASE [IF EXISTS] db [ON CLUSTER cluster]
 ```
 
+Удаляет все таблицы в базе данных db, затем удаляет саму базу данных db.
+
+
+## DROP TABLE {#drop-table}
+
 ``` sql
 DROP [TEMPORARY] TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster]
 ```
 
 Удаляет таблицу.
-Если указано `IF EXISTS` - не выдавать ошибку, если таблица не существует или база данных не существует.
+
+
+## DROP DICTIONARY {#drop-dictionary}
+
+``` sql
+DROP DICTIONARY [IF EXISTS] [db.]name
+```
+
+Удаляет словарь.
+
 
 ## DROP USER {#drop-user-statement}
 
@@ -41,6 +58,7 @@ DROP USER [IF EXISTS] name [,...] [ON CLUSTER cluster_name]
 DROP ROLE [IF EXISTS] name [,...] [ON CLUSTER cluster_name]
 ```
 
+
 ## DROP ROW POLICY {#drop-row-policy-statement}
 
 Удаляет политику доступа к строкам.
@@ -80,5 +98,13 @@ DROP [SETTINGS] PROFILE [IF EXISTS] name [,...] [ON CLUSTER cluster_name]
 ```
 
 
+## DROP VIEW {#drop-view}
 
-[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/drop/) <!--hide-->
+``` sql
+DROP VIEW [IF EXISTS] [db.]name [ON CLUSTER cluster]
+```
+
+Удаляет представление. Представления могут быть удалены и командой `DROP TABLE`, но команда `DROP VIEW` проверяет, что `[db.]name` является представлением.
+
+
+[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/drop/) <!--hide-->

programs/CMakeLists.txt

@@ -16,6 +16,7 @@ option (ENABLE_CLICKHOUSE_COMPRESSOR "Enable clickhouse-compressor" ${ENABLE_CLI
 option (ENABLE_CLICKHOUSE_COPIER "Enable clickhouse-copier" ${ENABLE_CLICKHOUSE_ALL})
 option (ENABLE_CLICKHOUSE_FORMAT "Enable clickhouse-format" ${ENABLE_CLICKHOUSE_ALL})
 option (ENABLE_CLICKHOUSE_OBFUSCATOR "Enable clickhouse-obfuscator" ${ENABLE_CLICKHOUSE_ALL})
+option (ENABLE_CLICKHOUSE_GIT_IMPORT "Enable clickhouse-git-import" ${ENABLE_CLICKHOUSE_ALL})
 option (ENABLE_CLICKHOUSE_ODBC_BRIDGE "Enable clickhouse-odbc-bridge" ${ENABLE_CLICKHOUSE_ALL})
 
 if (CLICKHOUSE_SPLIT_BINARY)
@@ -91,21 +92,22 @@ add_subdirectory (copier)
 add_subdirectory (format)
 add_subdirectory (obfuscator)
 add_subdirectory (install)
+add_subdirectory (git-import)
 
 if (ENABLE_CLICKHOUSE_ODBC_BRIDGE)
     add_subdirectory (odbc-bridge)
 endif ()
 
 if (CLICKHOUSE_ONE_SHARED)
-    add_library(clickhouse-lib SHARED ${CLICKHOUSE_SERVER_SOURCES} ${CLICKHOUSE_CLIENT_SOURCES} ${CLICKHOUSE_LOCAL_SOURCES} ${CLICKHOUSE_BENCHMARK_SOURCES} ${CLICKHOUSE_COPIER_SOURCES} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_SOURCES} ${CLICKHOUSE_COMPRESSOR_SOURCES} ${CLICKHOUSE_FORMAT_SOURCES} ${CLICKHOUSE_OBFUSCATOR_SOURCES} ${CLICKHOUSE_ODBC_BRIDGE_SOURCES})
+    add_library(clickhouse-lib SHARED ${CLICKHOUSE_SERVER_SOURCES} ${CLICKHOUSE_CLIENT_SOURCES} ${CLICKHOUSE_LOCAL_SOURCES} ${CLICKHOUSE_BENCHMARK_SOURCES} ${CLICKHOUSE_COPIER_SOURCES} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_SOURCES} ${CLICKHOUSE_COMPRESSOR_SOURCES} ${CLICKHOUSE_FORMAT_SOURCES} ${CLICKHOUSE_OBFUSCATOR_SOURCES} ${CLICKHOUSE_GIT_IMPORT_SOURCES} ${CLICKHOUSE_ODBC_BRIDGE_SOURCES})
-    target_link_libraries(clickhouse-lib ${CLICKHOUSE_SERVER_LINK} ${CLICKHOUSE_CLIENT_LINK} ${CLICKHOUSE_LOCAL_LINK} ${CLICKHOUSE_BENCHMARK_LINK} ${CLICKHOUSE_COPIER_LINK} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_LINK} ${CLICKHOUSE_COMPRESSOR_LINK} ${CLICKHOUSE_FORMAT_LINK} ${CLICKHOUSE_OBFUSCATOR_LINK} ${CLICKHOUSE_ODBC_BRIDGE_LINK})
+    target_link_libraries(clickhouse-lib ${CLICKHOUSE_SERVER_LINK} ${CLICKHOUSE_CLIENT_LINK} ${CLICKHOUSE_LOCAL_LINK} ${CLICKHOUSE_BENCHMARK_LINK} ${CLICKHOUSE_COPIER_LINK} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_LINK} ${CLICKHOUSE_COMPRESSOR_LINK} ${CLICKHOUSE_FORMAT_LINK} ${CLICKHOUSE_OBFUSCATOR_LINK} ${CLICKHOUSE_GIT_IMPORT_LINK} ${CLICKHOUSE_ODBC_BRIDGE_LINK})
-    target_include_directories(clickhouse-lib ${CLICKHOUSE_SERVER_INCLUDE} ${CLICKHOUSE_CLIENT_INCLUDE} ${CLICKHOUSE_LOCAL_INCLUDE} ${CLICKHOUSE_BENCHMARK_INCLUDE} ${CLICKHOUSE_COPIER_INCLUDE} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_INCLUDE} ${CLICKHOUSE_COMPRESSOR_INCLUDE} ${CLICKHOUSE_FORMAT_INCLUDE} ${CLICKHOUSE_OBFUSCATOR_INCLUDE} ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE})
+    target_include_directories(clickhouse-lib ${CLICKHOUSE_SERVER_INCLUDE} ${CLICKHOUSE_CLIENT_INCLUDE} ${CLICKHOUSE_LOCAL_INCLUDE} ${CLICKHOUSE_BENCHMARK_INCLUDE} ${CLICKHOUSE_COPIER_INCLUDE} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_INCLUDE} ${CLICKHOUSE_COMPRESSOR_INCLUDE} ${CLICKHOUSE_FORMAT_INCLUDE} ${CLICKHOUSE_OBFUSCATOR_INCLUDE} ${CLICKHOUSE_GIT_IMPORT_INCLUDE} ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE})
     set_target_properties(clickhouse-lib PROPERTIES SOVERSION ${VERSION_MAJOR}.${VERSION_MINOR} VERSION ${VERSION_SO} OUTPUT_NAME clickhouse DEBUG_POSTFIX "")
     install (TARGETS clickhouse-lib LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT clickhouse)
 endif()
 
 if (CLICKHOUSE_SPLIT_BINARY)
-    set (CLICKHOUSE_ALL_TARGETS clickhouse-server clickhouse-client clickhouse-local clickhouse-benchmark clickhouse-extract-from-config clickhouse-compressor clickhouse-format clickhouse-obfuscator clickhouse-copier)
+    set (CLICKHOUSE_ALL_TARGETS clickhouse-server clickhouse-client clickhouse-local clickhouse-benchmark clickhouse-extract-from-config clickhouse-compressor clickhouse-format clickhouse-obfuscator clickhouse-git-import clickhouse-copier)
 
     if (ENABLE_CLICKHOUSE_ODBC_BRIDGE)
         list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-odbc-bridge)
@@ -149,6 +151,9 @@ else ()
     if (ENABLE_CLICKHOUSE_OBFUSCATOR)
         clickhouse_target_link_split_lib(clickhouse obfuscator)
     endif ()
+    if (ENABLE_CLICKHOUSE_GIT_IMPORT)
+        clickhouse_target_link_split_lib(clickhouse git-import)
+    endif ()
     if (ENABLE_CLICKHOUSE_INSTALL)
         clickhouse_target_link_split_lib(clickhouse install)
     endif ()
@@ -199,6 +204,11 @@ else ()
         install (FILES ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-obfuscator DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
         list(APPEND CLICKHOUSE_BUNDLE clickhouse-obfuscator)
     endif ()
+    if (ENABLE_CLICKHOUSE_GIT_IMPORT)
+        add_custom_target (clickhouse-git-import ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-git-import DEPENDS clickhouse)
+        install (FILES ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-git-import DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
+        list(APPEND CLICKHOUSE_BUNDLE clickhouse-git-import)
+    endif ()
     if(ENABLE_CLICKHOUSE_ODBC_BRIDGE)
         list(APPEND CLICKHOUSE_BUNDLE clickhouse-odbc-bridge)
     endif()

programs/client/Client.cpp

@@ -866,6 +866,8 @@ private:
             // will exit. The ping() would be the best match here, but it's
             // private, probably for a good reason that the protocol doesn't allow
             // pings at any possible moment.
+            // Don't forget to reset the default database which might have changed.
+            connection->setDefaultDatabase("");
             connection->forceConnected(connection_parameters.timeouts);
 
             if (text.size() > 4 * 1024)
@@ -900,74 +902,127 @@ private:
         return processMultiQuery(text);
     }
 
-    bool processMultiQuery(const String & text)
+    bool processMultiQuery(const String & all_queries_text)
     {
         const bool test_mode = config().has("testmode");
 
         {   /// disable logs if expects errors
-            TestHint test_hint(test_mode, text);
+            TestHint test_hint(test_mode, all_queries_text);
             if (test_hint.clientError() || test_hint.serverError())
                 processTextAsSingleQuery("SET send_logs_level = 'none'");
         }
 
         /// Several queries separated by ';'.
         /// INSERT data is ended by the end of line, not ';'.
+        /// An exception is VALUES format where we also support semicolon in
+        /// addition to end of line.
 
-        const char * begin = text.data();
+        const char * this_query_begin = all_queries_text.data();
-        const char * end = begin + text.size();
+        const char * all_queries_end = all_queries_text.data() + all_queries_text.size();
 
-        while (begin < end)
+        while (this_query_begin < all_queries_end)
         {
-            const char * pos = begin;
+            // Use the token iterator to skip any whitespace, semicolons and
-            ASTPtr orig_ast = parseQuery(pos, end, true);
+            // comments at the beginning of the query. An example from regression
+            // tests:
+            //      insert into table t values ('invalid'); -- { serverError 469 }
+            //      select 1
+            // Here the test hint comment gets parsed as a part of second query.
+            // We parse the `INSERT VALUES` up to the semicolon, and the rest
+            // looks like a two-line query:
+            //      -- { serverError 469 }
+            //      select 1
+            // and we expect it to fail with error 469, but this hint is actually
+            // for the previous query. Test hints should go after the query, so
+            // we can fix this by skipping leading comments. Token iterator skips
+            // comments and whitespace by itself, so we only have to check for
+            // semicolons.
+            // The code block is to limit visibility of `tokens` because we have
+            // another such variable further down the code, and get warnings for
+            // that.
+            {
+                Tokens tokens(this_query_begin, all_queries_end);
+                IParser::Pos token_iterator(tokens,
+                    context.getSettingsRef().max_parser_depth);
+                while (token_iterator->type == TokenType::Semicolon
+                        && token_iterator.isValid())
+                {
+                    ++token_iterator;
+                }
+                this_query_begin = token_iterator->begin;
+                if (this_query_begin >= all_queries_end)
+                {
+                    break;
+                }
+            }
 
-            if (!orig_ast)
+            // Try to parse the query.
+            const char * this_query_end = this_query_begin;
+            parsed_query = parseQuery(this_query_end, all_queries_end, true);
+
+            if (!parsed_query)
             {
                 if (ignore_error)
                 {
-                    Tokens tokens(begin, end);
+                    Tokens tokens(this_query_begin, all_queries_end);
                     IParser::Pos token_iterator(tokens, context.getSettingsRef().max_parser_depth);
                     while (token_iterator->type != TokenType::Semicolon && token_iterator.isValid())
                         ++token_iterator;
-                    begin = token_iterator->end;
+                    this_query_begin = token_iterator->end;
 
                     continue;
                 }
                 return true;
             }
 
-            auto * insert = orig_ast->as<ASTInsertQuery>();
+            // INSERT queries may have the inserted data in the query text
-
+            // that follow the query itself, e.g. "insert into t format CSV 1;2".
-            if (insert && insert->data)
+            // They need special handling. First of all, here we find where the
+            // inserted data ends. In multy-query mode, it is delimited by a
+            // newline.
+            // The VALUES format needs even more handling -- we also allow the
+            // data to be delimited by semicolon. This case is handled later by
+            // the format parser itself.
+            auto * insert_ast = parsed_query->as<ASTInsertQuery>();
+            if (insert_ast && insert_ast->data)
             {
-                pos = find_first_symbols<'\n'>(insert->data, end);
+                this_query_end = find_first_symbols<'\n'>(insert_ast->data, all_queries_end);
-                insert->end = pos;
+                insert_ast->end = this_query_end;
+                query_to_send = all_queries_text.substr(
+                    this_query_begin - all_queries_text.data(),
+                    insert_ast->data - this_query_begin);
+            }
+            else
+            {
+                query_to_send = all_queries_text.substr(
+                    this_query_begin - all_queries_text.data(),
+                    this_query_end - this_query_begin);
             }
 
-            String str = text.substr(begin - text.data(), pos - begin);
+            // full_query is the query + inline INSERT data.
+            full_query = all_queries_text.substr(
+                this_query_begin - all_queries_text.data(),
+                this_query_end - this_query_begin);
 
-            begin = pos;
+            // Look for the hint in the text of query + insert data, if any.
-            while (isWhitespaceASCII(*begin) || *begin == ';')
+            // e.g. insert into t format CSV 'a' -- { serverError 123 }.
-                ++begin;
+            TestHint test_hint(test_mode, full_query);
-
-            TestHint test_hint(test_mode, str);
             expected_client_error = test_hint.clientError();
             expected_server_error = test_hint.serverError();
 
             try
             {
-                auto ast_to_process = orig_ast;
+                processParsedSingleQuery();
-                if (insert && insert->data)
+
+                if (insert_ast && insert_ast->data)
                 {
-                    ast_to_process = nullptr;
+                    // For VALUES format: use the end of inline data as reported
-                    processTextAsSingleQuery(str);
+                    // by the format parser (it is saved in sendData()). This
-                }
+                    // allows us to handle queries like:
-                else
+                    //   insert into t values (1); select 1
-                {
+                    //, where the inline data is delimited by semicolon and not
-                    parsed_query = ast_to_process;
+                    // by a newline.
-                    full_query = str;
+                    this_query_end = parsed_query->as<ASTInsertQuery>()->end;
-                    query_to_send = str;
-                    processParsedSingleQuery();
                 }
             }
             catch (...)
@@ -975,7 +1030,7 @@ private:
                 last_exception_received_from_server = std::make_unique<Exception>(getCurrentExceptionMessage(true), getCurrentExceptionCode());
                 actual_client_error = last_exception_received_from_server->code();
                 if (!ignore_error && (!actual_client_error || actual_client_error != expected_client_error))
-                    std::cerr << "Error on processing query: " << str << std::endl << last_exception_received_from_server->message();
+                    std::cerr << "Error on processing query: " << full_query << std::endl << last_exception_received_from_server->message();
                 received_exception_from_server = true;
             }
 
@@ -989,6 +1044,8 @@ private:
                 else
                     return false;
             }
+
+            this_query_begin = this_query_end;
         }
 
         return true;
@@ -1103,7 +1160,9 @@ private:
                 {
                     last_exception_received_from_server = std::make_unique<Exception>(getCurrentExceptionMessage(true), getCurrentExceptionCode());
                     received_exception_from_server = true;
-                    std::cerr << "Error on processing query: " << ast_to_process->formatForErrorMessage() << std::endl << last_exception_received_from_server->message();
+                    fmt::print(stderr, "Error on processing query '{}': {}\n",
+                        ast_to_process->formatForErrorMessage(),
+                        last_exception_received_from_server->message());
                 }
 
                 if (!connection->isConnected())
@@ -1411,7 +1470,7 @@ private:
     void sendData(Block & sample, const ColumnsDescription & columns_description)
     {
         /// If INSERT data must be sent.
-        const auto * parsed_insert_query = parsed_query->as<ASTInsertQuery>();
+        auto * parsed_insert_query = parsed_query->as<ASTInsertQuery>();
         if (!parsed_insert_query)
             return;
 
@@ -1420,6 +1479,9 @@ private:
             /// Send data contained in the query.
             ReadBufferFromMemory data_in(parsed_insert_query->data, parsed_insert_query->end - parsed_insert_query->data);
             sendDataFrom(data_in, sample, columns_description);
+            // Remember where the data ended. We use this info later to determine
+            // where the next query begins.
+            parsed_insert_query->end = data_in.buffer().begin() + data_in.count();
         }
         else if (!is_interactive)
         {

programs/config_tools.h.in

@@ -12,5 +12,6 @@
 #cmakedefine01 ENABLE_CLICKHOUSE_COMPRESSOR
 #cmakedefine01 ENABLE_CLICKHOUSE_FORMAT
 #cmakedefine01 ENABLE_CLICKHOUSE_OBFUSCATOR
+#cmakedefine01 ENABLE_CLICKHOUSE_GIT_IMPORT
 #cmakedefine01 ENABLE_CLICKHOUSE_INSTALL
 #cmakedefine01 ENABLE_CLICKHOUSE_ODBC_BRIDGE

programs/git-import/CMakeLists.txt

@@ -0,0 +1,10 @@
+set (CLICKHOUSE_GIT_IMPORT_SOURCES git-import.cpp)
+
+set (CLICKHOUSE_GIT_IMPORT_LINK
+    PRIVATE
+        boost::program_options
+        dbms
+)
+
+clickhouse_program_add(git-import)
+

programs/git-import/clickhouse-git-import.cpp

@@ -0,0 +1,2 @@
+int mainEntryClickHouseGitImport(int argc, char ** argv);
+int main(int argc_, char ** argv_) { return mainEntryClickHouseGitImport(argc_, argv_); }

programs/install/Install.cpp

@@ -205,6 +205,7 @@ int mainEntryClickHouseInstall(int argc, char ** argv)
             "clickhouse-benchmark",
             "clickhouse-copier",
             "clickhouse-obfuscator",
+            "clickhouse-git-import",
             "clickhouse-compressor",
             "clickhouse-format",
             "clickhouse-extract-from-config"

programs/main.cpp

@@ -46,6 +46,9 @@ int mainEntryClickHouseClusterCopier(int argc, char ** argv);
 #if ENABLE_CLICKHOUSE_OBFUSCATOR
 int mainEntryClickHouseObfuscator(int argc, char ** argv);
 #endif
+#if ENABLE_CLICKHOUSE_GIT_IMPORT
+int mainEntryClickHouseGitImport(int argc, char ** argv);
+#endif
 #if ENABLE_CLICKHOUSE_INSTALL
 int mainEntryClickHouseInstall(int argc, char ** argv);
 int mainEntryClickHouseStart(int argc, char ** argv);
@@ -91,6 +94,9 @@ std::pair<const char *, MainFunc> clickhouse_applications[] =
 #if ENABLE_CLICKHOUSE_OBFUSCATOR
     {"obfuscator", mainEntryClickHouseObfuscator},
 #endif
+#if ENABLE_CLICKHOUSE_GIT_IMPORT
+    {"git-import", mainEntryClickHouseGitImport},
+#endif
 #if ENABLE_CLICKHOUSE_INSTALL
     {"install", mainEntryClickHouseInstall},
     {"start", mainEntryClickHouseStart},

programs/server/config.xml

@@ -212,8 +212,17 @@
     <!-- Directory with user provided files that are accessible by 'file' table function. -->
     <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
 
-    <!-- Path to folder where users and roles created by SQL commands are stored. -->
+    <!-- Sources to read users, roles, access rights, profiles of settings, quotas. -->
-    <access_control_path>/var/lib/clickhouse/access/</access_control_path>
+    <user_directories>
+        <users_xml>
+            <!-- Path to configuration file with predefined users. -->
+            <path>users.xml</path>
+        </users_xml>
+        <local_directory>
+            <!-- Path to folder where users created by SQL commands are stored. -->
+            <path>/var/lib/clickhouse/access/</path>
+        </local_directory>
+    </user_directories>
 
     <!-- External user directories (LDAP). -->
     <ldap_servers>
@@ -256,9 +265,6 @@
         -->
     </ldap_servers>
 
-    <!-- Path to configuration file with users, access rights, profiles of settings, quotas. -->
-    <users_config>users.xml</users_config>
-
     <!-- Default profile of settings. -->
     <default_profile>default</default_profile>
 

src/Access/AccessControlManager.cpp

@@ -181,6 +181,15 @@ void AccessControlManager::addUsersConfigStorage(
     const String & preprocessed_dir_,
     const zkutil::GetZooKeeper & get_zookeeper_function_)
 {
+    auto storages = getStoragesPtr();
+    for (const auto & storage : *storages)
+    {
+        if (auto users_config_storage = typeid_cast<std::shared_ptr<UsersConfigAccessStorage>>(storage))
+        {
+            if (users_config_storage->getStoragePath() == users_config_path_)
+                return;
+        }
+    }
     auto check_setting_name_function = [this](const std::string_view & setting_name) { checkSettingNameIsAllowed(setting_name); };
     auto new_storage = std::make_shared<UsersConfigAccessStorage>(storage_name_, check_setting_name_function);
     new_storage->load(users_config_path_, include_from_path_, preprocessed_dir_, get_zookeeper_function_);
@@ -210,17 +219,36 @@ void AccessControlManager::startPeriodicReloadingUsersConfigs()
 
 void AccessControlManager::addDiskStorage(const String & directory_, bool readonly_)
 {
-    addStorage(std::make_shared<DiskAccessStorage>(directory_, readonly_));
+    addDiskStorage(DiskAccessStorage::STORAGE_TYPE, directory_, readonly_);
 }
 
 void AccessControlManager::addDiskStorage(const String & storage_name_, const String & directory_, bool readonly_)
 {
+    auto storages = getStoragesPtr();
+    for (const auto & storage : *storages)
+    {
+        if (auto disk_storage = typeid_cast<std::shared_ptr<DiskAccessStorage>>(storage))
+        {
+            if (disk_storage->isStoragePathEqual(directory_))
+            {
+                if (readonly_)
+                    disk_storage->setReadOnly(readonly_);
+                return;
+            }
+        }
+    }
     addStorage(std::make_shared<DiskAccessStorage>(storage_name_, directory_, readonly_));
 }
 
 
 void AccessControlManager::addMemoryStorage(const String & storage_name_)
 {
+    auto storages = getStoragesPtr();
+    for (const auto & storage : *storages)
+    {
+        if (auto memory_storage = typeid_cast<std::shared_ptr<MemoryAccessStorage>>(storage))
+            return;
+    }
     addStorage(std::make_shared<MemoryAccessStorage>(storage_name_));
 }
 

src/Access/DiskAccessStorage.cpp

@@ -218,6 +218,16 @@ namespace
     }
 
 
+    /// Converts a path to an absolute path and append it with a separator.
+    String makeDirectoryPathCanonical(const String & directory_path)
+    {
+        auto canonical_directory_path = std::filesystem::weakly_canonical(directory_path);
+        if (canonical_directory_path.has_filename())
+            canonical_directory_path += std::filesystem::path::preferred_separator;
+        return canonical_directory_path;
+    }
+
+
     /// Calculates the path to a file named <id>.sql for saving an access entity.
     String getEntityFilePath(const String & directory_path, const UUID & id)
     {
@@ -298,22 +308,17 @@ DiskAccessStorage::DiskAccessStorage(const String & directory_path_, bool readon
 {
 }
 
-
 DiskAccessStorage::DiskAccessStorage(const String & storage_name_, const String & directory_path_, bool readonly_)
     : IAccessStorage(storage_name_)
 {
-    auto canonical_directory_path = std::filesystem::weakly_canonical(directory_path_);
+    directory_path = makeDirectoryPathCanonical(directory_path_);
-    if (canonical_directory_path.has_filename())
+    readonly = readonly_;
-        canonical_directory_path += std::filesystem::path::preferred_separator;
 
     std::error_code create_dir_error_code;
-    std::filesystem::create_directories(canonical_directory_path, create_dir_error_code);
+    std::filesystem::create_directories(directory_path, create_dir_error_code);
 
-    if (!std::filesystem::exists(canonical_directory_path) || !std::filesystem::is_directory(canonical_directory_path) || create_dir_error_code)
+    if (!std::filesystem::exists(directory_path) || !std::filesystem::is_directory(directory_path) || create_dir_error_code)
-        throw Exception("Couldn't create directory " + canonical_directory_path.string() + " reason: '" + create_dir_error_code.message() + "'", ErrorCodes::DIRECTORY_DOESNT_EXIST);
+        throw Exception("Couldn't create directory " + directory_path + " reason: '" + create_dir_error_code.message() + "'", ErrorCodes::DIRECTORY_DOESNT_EXIST);
-
-    directory_path = canonical_directory_path;
-    readonly = readonly_;
 
     bool should_rebuild_lists = std::filesystem::exists(getNeedRebuildListsMarkFilePath(directory_path));
     if (!should_rebuild_lists)
@@ -337,6 +342,12 @@ DiskAccessStorage::~DiskAccessStorage()
 }
 
 
+bool DiskAccessStorage::isStoragePathEqual(const String & directory_path_) const
+{
+    return getStoragePath() == makeDirectoryPathCanonical(directory_path_);
+}
+
+
 void DiskAccessStorage::clear()
 {
     entries_by_id.clear();
@@ -426,33 +437,41 @@ bool DiskAccessStorage::writeLists()
 void DiskAccessStorage::scheduleWriteLists(EntityType type)
 {
     if (failed_to_write_lists)
-        return;
+        return; /// We don't try to write list files after the first fail.
+                /// The next restart of the server will invoke rebuilding of the list files.
 
-    bool already_scheduled = !types_of_lists_to_write.empty();
     types_of_lists_to_write.insert(type);
 
-    if (already_scheduled)
+    if (lists_writing_thread_is_waiting)
-        return;
+        return; /// If the lists' writing thread is still waiting we can update `types_of_lists_to_write` easily,
+                /// without restarting that thread.
+
+    if (lists_writing_thread.joinable())
+        lists_writing_thread.join();
 
     /// Create the 'need_rebuild_lists.mark' file.
     /// This file will be used later to find out if writing lists is successful or not.
     std::ofstream{getNeedRebuildListsMarkFilePath(directory_path)};
 
-    startListsWritingThread();
+    lists_writing_thread = ThreadFromGlobalPool{&DiskAccessStorage::listsWritingThreadFunc, this};
+    lists_writing_thread_is_waiting = true;
 }
 
 
-void DiskAccessStorage::startListsWritingThread()
+void DiskAccessStorage::listsWritingThreadFunc()
 {
-    if (lists_writing_thread.joinable())
+    std::unique_lock lock{mutex};
+
     {
-        if (!lists_writing_thread_exited)
+        /// It's better not to write the lists files too often, that's why we need
-            return;
+        /// the following timeout.
-        lists_writing_thread.detach();
+        const auto timeout = std::chrono::minutes(1);
+        SCOPE_EXIT({ lists_writing_thread_is_waiting = false; });
+        if (lists_writing_thread_should_exit.wait_for(lock, timeout) != std::cv_status::timeout)
+            return; /// The destructor requires us to exit.
     }
 
-    lists_writing_thread_exited = false;
+    writeLists();
-    lists_writing_thread = ThreadFromGlobalPool{&DiskAccessStorage::listsWritingThreadFunc, this};
 }
 
 
@@ -466,21 +485,6 @@ void DiskAccessStorage::stopListsWritingThread()
 }
 
 
-void DiskAccessStorage::listsWritingThreadFunc()
-{
-    std::unique_lock lock{mutex};
-    SCOPE_EXIT({ lists_writing_thread_exited = true; });
-
-    /// It's better not to write the lists files too often, that's why we need
-    /// the following timeout.
-    const auto timeout = std::chrono::minutes(1);
-    if (lists_writing_thread_should_exit.wait_for(lock, timeout) != std::cv_status::timeout)
-        return; /// The destructor requires us to exit.
-
-    writeLists();
-}
-
-
 /// Reads and parses all the "<id>.sql" files from a specified directory
 /// and then saves the files "users.list", "roles.list", etc. to the same directory.
 bool DiskAccessStorage::rebuildLists()

src/Access/DiskAccessStorage.h

@@ -18,7 +18,11 @@ public:
     ~DiskAccessStorage() override;
 
     const char * getStorageType() const override { return STORAGE_TYPE; }
+
     String getStoragePath() const override { return directory_path; }
+    bool isStoragePathEqual(const String & directory_path_) const;
+
+    void setReadOnly(bool readonly_) { readonly = readonly_; }
     bool isStorageReadOnly() const override { return readonly; }
 
 private:
@@ -42,9 +46,8 @@ private:
     void scheduleWriteLists(EntityType type);
     bool rebuildLists();
 
-    void startListsWritingThread();
-    void stopListsWritingThread();
     void listsWritingThreadFunc();
+    void stopListsWritingThread();
 
     void insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, Notifications & notifications);
     void removeNoLock(const UUID & id, Notifications & notifications);
@@ -67,14 +70,14 @@ private:
     void prepareNotifications(const UUID & id, const Entry & entry, bool remove, Notifications & notifications) const;
 
     String directory_path;
-    bool readonly;
+    std::atomic<bool> readonly;
     std::unordered_map<UUID, Entry> entries_by_id;
     std::unordered_map<std::string_view, Entry *> entries_by_name_and_type[static_cast<size_t>(EntityType::MAX)];
     boost::container::flat_set<EntityType> types_of_lists_to_write;
     bool failed_to_write_lists = false;                          /// Whether writing of the list files has been failed since the recent restart of the server.
     ThreadFromGlobalPool lists_writing_thread;                   /// List files are written in a separate thread.
     std::condition_variable lists_writing_thread_should_exit;    /// Signals `lists_writing_thread` to exit.
-    std::atomic<bool> lists_writing_thread_exited = false;
+    bool lists_writing_thread_is_waiting = false;
     mutable std::list<OnChangedHandler> handlers_by_type[static_cast<size_t>(EntityType::MAX)];
     mutable std::mutex mutex;
 };

src/Columns/ColumnVector.h

@@ -7,6 +7,7 @@
 #include <common/unaligned.h>
 #include <Core/Field.h>
 #include <Core/BigInt.h>
+#include <Common/assert_cast.h>
 
 
 namespace DB
@@ -130,7 +131,7 @@ public:
 
     void insertFrom(const IColumn & src, size_t n) override
     {
-        data.push_back(static_cast<const Self &>(src).getData()[n]);
+        data.push_back(assert_cast<const Self &>(src).getData()[n]);
     }
 
     void insertData(const char * pos, size_t) override
@@ -205,14 +206,14 @@ public:
     /// This method implemented in header because it could be possibly devirtualized.
     int compareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const override
     {
-        return CompareHelper<T>::compare(data[n], static_cast<const Self &>(rhs_).data[m], nan_direction_hint);
+        return CompareHelper<T>::compare(data[n], assert_cast<const Self &>(rhs_).data[m], nan_direction_hint);
     }
 
     void compareColumn(const IColumn & rhs, size_t rhs_row_num,
                        PaddedPODArray<UInt64> * row_indexes, PaddedPODArray<Int8> & compare_results,
                        int direction, int nan_direction_hint) const override
     {
-        return this->template doCompareColumn<Self>(static_cast<const Self &>(rhs), rhs_row_num, row_indexes,
+        return this->template doCompareColumn<Self>(assert_cast<const Self &>(rhs), rhs_row_num, row_indexes,
                                                     compare_results, direction, nan_direction_hint);
     }
 

src/Columns/ya.make

@@ -2,8 +2,6 @@
 LIBRARY()
 
 ADDINCL(
-    contrib/libs/icu/common
-    contrib/libs/icu/i18n
     contrib/libs/pdqsort
 )
 

src/Common/Macros.cpp

@@ -68,8 +68,14 @@ String Macros::expand(const String & s,
             res += database_name;
         else if (macro_name == "table" && !table_name.empty())
             res += table_name;
-        else if (macro_name == "uuid" && uuid != UUIDHelpers::Nil)
+        else if (macro_name == "uuid")
+        {
+            if (uuid == UUIDHelpers::Nil)
+                throw Exception("Macro 'uuid' and empty arguments of ReplicatedMergeTree "
+                                "are supported only for ON CLUSTER queries with Atomic database engine",
+                                ErrorCodes::SYNTAX_ERROR);
             res += toString(uuid);
+        }
         else
             throw Exception("No macro '" + macro_name +
                 "' in config while processing substitutions in '" + s + "' at '"

src/Common/MemoryTracker.cpp

@@ -6,6 +6,7 @@
 #include <Common/Exception.h>
 #include <Common/formatReadable.h>
 #include <common/logger_useful.h>
+#include <Common/ProfileEvents.h>
 
 #include <atomic>
 #include <cmath>
@@ -22,6 +23,10 @@ namespace DB
     }
 }
 
+namespace ProfileEvents
+{
+    extern const Event QueryMemoryLimitExceeded;
+}
 
 static constexpr size_t log_peak_memory_usage_every = 1ULL << 30;
 
@@ -104,6 +109,7 @@ void MemoryTracker::alloc(Int64 size)
         /// Prevent recursion. Exception::ctor -> std::string -> new[] -> MemoryTracker::alloc
         auto untrack_lock = blocker.cancel(); // NOLINT
 
+        ProfileEvents::increment(ProfileEvents::QueryMemoryLimitExceeded);
         std::stringstream message;
         message << "Memory tracker";
         if (const auto * description = description_ptr.load(std::memory_order_relaxed))
@@ -136,6 +142,7 @@ void MemoryTracker::alloc(Int64 size)
         /// Prevent recursion. Exception::ctor -> std::string -> new[] -> MemoryTracker::alloc
         auto no_track = blocker.cancel(); // NOLINT
 
+        ProfileEvents::increment(ProfileEvents::QueryMemoryLimitExceeded);
         std::stringstream message;
         message << "Memory limit";
         if (const auto * description = description_ptr.load(std::memory_order_relaxed))

src/Common/ProfileEvents.cpp

@@ -233,6 +233,7 @@
     M(S3WriteRequestsErrors, "Number of non-throttling errors in POST, DELETE, PUT and PATCH requests to S3 storage.") \
     M(S3WriteRequestsThrottling, "Number of 429 and 503 errors in POST, DELETE, PUT and PATCH requests to S3 storage.") \
     M(S3WriteRequestsRedirects, "Number of redirects in POST, DELETE, PUT and PATCH requests to S3 storage.") \
+    M(QueryMemoryLimitExceeded, "Number of times when memory limit exceeded for query.") \
 
 
 namespace ProfileEvents

src/Common/ShellCommand.cpp

@@ -57,7 +57,16 @@ ShellCommand::~ShellCommand()
             LOG_WARNING(getLogger(), "Cannot kill shell command pid {} errno '{}'", pid, errnoToString(retcode));
     }
     else if (!wait_called)
-        tryWait();
+    {
+        try
+        {
+            tryWait();
+        }
+        catch (...)
+        {
+            tryLogCurrentException(getLogger());
+        }
+    }
 }
 
 void ShellCommand::logCommand(const char * filename, char * const argv[])
@@ -74,7 +83,8 @@ void ShellCommand::logCommand(const char * filename, char * const argv[])
     LOG_TRACE(ShellCommand::getLogger(), "Will start shell command '{}' with arguments {}", filename, args.str());
 }
 
-std::unique_ptr<ShellCommand> ShellCommand::executeImpl(const char * filename, char * const argv[], bool pipe_stdin_only, bool terminate_in_destructor)
+std::unique_ptr<ShellCommand> ShellCommand::executeImpl(
+    const char * filename, char * const argv[], bool pipe_stdin_only, bool terminate_in_destructor)
 {
     logCommand(filename, argv);
 
@@ -130,7 +140,8 @@ std::unique_ptr<ShellCommand> ShellCommand::executeImpl(const char * filename, c
         _exit(int(ReturnCodes::CANNOT_EXEC));
     }
 
-    std::unique_ptr<ShellCommand> res(new ShellCommand(pid, pipe_stdin.fds_rw[1], pipe_stdout.fds_rw[0], pipe_stderr.fds_rw[0], terminate_in_destructor));
+    std::unique_ptr<ShellCommand> res(new ShellCommand(
+        pid, pipe_stdin.fds_rw[1], pipe_stdout.fds_rw[0], pipe_stderr.fds_rw[0], terminate_in_destructor));
 
     LOG_TRACE(getLogger(), "Started shell command '{}' with pid {}", filename, pid);
 
@@ -143,7 +154,8 @@ std::unique_ptr<ShellCommand> ShellCommand::executeImpl(const char * filename, c
 }
 
 
-std::unique_ptr<ShellCommand> ShellCommand::execute(const std::string & command, bool pipe_stdin_only, bool terminate_in_destructor)
+std::unique_ptr<ShellCommand> ShellCommand::execute(
+    const std::string & command, bool pipe_stdin_only, bool terminate_in_destructor)
 {
     /// Arguments in non-constant chunks of memory (as required for `execv`).
     /// Moreover, their copying must be done before calling `vfork`, so after `vfork` do a minimum of things.
@@ -157,7 +169,8 @@ std::unique_ptr<ShellCommand> ShellCommand::execute(const std::string & command,
 }
 
 
-std::unique_ptr<ShellCommand> ShellCommand::executeDirect(const std::string & path, const std::vector<std::string> & arguments, bool terminate_in_destructor)
+std::unique_ptr<ShellCommand> ShellCommand::executeDirect(
+    const std::string & path, const std::vector<std::string> & arguments, bool terminate_in_destructor)
 {
     size_t argv_sum_size = path.size() + 1;
     for (const auto & arg : arguments)
@@ -186,6 +199,10 @@ int ShellCommand::tryWait()
 {
     wait_called = true;
 
+    in.close();
+    out.close();
+    err.close();
+
     LOG_TRACE(getLogger(), "Will wait for shell command pid {}", pid);
 
     int status = 0;

src/Common/tests/CMakeLists.txt

@@ -84,3 +84,6 @@ target_link_libraries (procfs_metrics_provider_perf PRIVATE clickhouse_common_io
 
 add_executable (average average.cpp)
 target_link_libraries (average PRIVATE clickhouse_common_io)
+
+add_executable (shell_command_inout shell_command_inout.cpp)
+target_link_libraries (shell_command_inout PRIVATE clickhouse_common_io)

src/Common/tests/shell_command_inout.cpp

@@ -0,0 +1,47 @@
+#include <thread>
+
+#include <Common/ShellCommand.h>
+#include <Common/Exception.h>
+
+#include <IO/ReadBufferFromFileDescriptor.h>
+#include <IO/WriteBufferFromFileDescriptor.h>
+#include <IO/copyData.h>
+
+/** This example shows how we can proxy stdin to ShellCommand and obtain stdout in streaming fashion. */
+
+int main(int argc, char ** argv)
+try
+{
+    using namespace DB;
+
+    if (argc < 2)
+    {
+        std::cerr << "Usage: shell_command_inout 'command...' < in > out\n";
+        return 1;
+    }
+
+    auto command = ShellCommand::execute(argv[1]);
+
+    ReadBufferFromFileDescriptor in(STDIN_FILENO);
+    WriteBufferFromFileDescriptor out(STDOUT_FILENO);
+    WriteBufferFromFileDescriptor err(STDERR_FILENO);
+
+    /// Background thread sends data and foreground thread receives result.
+
+    std::thread thread([&]
+    {
+        copyData(in, command->in);
+        command->in.close();
+    });
+
+    copyData(command->out, out);
+    copyData(command->err, err);
+
+    thread.join();
+    return 0;
+}
+catch (...)
+{
+    std::cerr << DB::getCurrentExceptionMessage(true) << '\n';
+    throw;
+}

src/Core/MySQL/IMySQLReadPacket.cpp

@@ -50,21 +50,22 @@ uint64_t readLengthEncodedNumber(ReadBuffer & buffer)
     uint64_t buf = 0;
     buffer.readStrict(c);
     auto cc = static_cast<uint8_t>(c);
-    if (cc < 0xfc)
+    switch (cc)
     {
-        return cc;
+        /// NULL
-    }
+        case 0xfb:
-    else if (cc < 0xfd)
+            break;
-    {
+        case 0xfc:
-        buffer.readStrict(reinterpret_cast<char *>(&buf), 2);
+            buffer.readStrict(reinterpret_cast<char *>(&buf), 2);
-    }
+            break;
-    else if (cc < 0xfe)
+        case 0xfd:
-    {
+            buffer.readStrict(reinterpret_cast<char *>(&buf), 3);
-        buffer.readStrict(reinterpret_cast<char *>(&buf), 3);
+            break;
-    }
+        case 0xfe:
-    else
+            buffer.readStrict(reinterpret_cast<char *>(&buf), 8);
-    {
+            break;
-        buffer.readStrict(reinterpret_cast<char *>(&buf), 8);
+        default:
+            return cc;
     }
     return buf;
 }

src/Core/MySQL/MySQLReplication.cpp

@@ -171,7 +171,7 @@ namespace MySQLReplication
 
         /// Ignore MySQL 8.0 optional metadata fields.
         /// https://mysqlhighavailability.com/more-metadata-is-written-into-binary-log/
-        payload.ignore(payload.available() - CHECKSUM_CRC32_SIGNATURE_LENGTH);
+        payload.ignoreAll();
     }
 
     /// Types that do not used in the binlog event:
@@ -221,6 +221,7 @@ namespace MySQLReplication
                 }
                 case MYSQL_TYPE_NEWDECIMAL:
                 case MYSQL_TYPE_STRING: {
+                    /// Big-Endian
                     auto b0 = UInt16(meta[pos] << 8);
                     auto b1 = UInt8(meta[pos + 1]);
                     column_meta.emplace_back(UInt16(b0 + b1));
@@ -231,6 +232,7 @@ namespace MySQLReplication
                 case MYSQL_TYPE_BIT:
                 case MYSQL_TYPE_VARCHAR:
                 case MYSQL_TYPE_VAR_STRING: {
+                    /// Little-Endian
                     auto b0 = UInt8(meta[pos]);
                     auto b1 = UInt16(meta[pos + 1] << 8);
                     column_meta.emplace_back(UInt16(b0 + b1));
@@ -911,7 +913,7 @@ namespace MySQLReplication
                 break;
             }
         }
-        payload.tryIgnore(CHECKSUM_CRC32_SIGNATURE_LENGTH);
+        payload.ignoreAll();
     }
 }
 

src/Core/tests/mysql_protocol.cpp

@@ -283,6 +283,7 @@ int main(int argc, char ** argv)
     }
 
     {
+        /// mysql_protocol --host=172.17.0.3 --user=root --password=123 --db=sbtest
         try
         {
             boost::program_options::options_description desc("Allowed options");

src/DataTypes/DataTypeNullable.cpp

@@ -308,16 +308,30 @@ ReturnType DataTypeNullable::deserializeTextQuoted(IColumn & column, ReadBuffer
                                                    const DataTypePtr & nested_data_type)
 {
     return safeDeserialize<ReturnType>(column, *nested_data_type,
-        [&istr] { return checkStringByFirstCharacterAndAssertTheRestCaseInsensitive("NULL", istr); },
+        [&istr]
+        {
+            return checkStringByFirstCharacterAndAssertTheRestCaseInsensitive("NULL", istr);
+        },
         [&nested_data_type, &istr, &settings] (IColumn & nested) { nested_data_type->deserializeAsTextQuoted(nested, istr, settings); });
 }
 
 
 void DataTypeNullable::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
-    safeDeserialize(column, *nested_data_type,
+    deserializeWholeText<void>(column, istr, settings, nested_data_type);
-        [&istr] { return checkStringByFirstCharacterAndAssertTheRestCaseInsensitive("NULL", istr); },
+}
-        [this, &istr, &settings] (IColumn & nested) { nested_data_type->deserializeAsWholeText(nested, istr, settings); });
+
+template <typename ReturnType>
+ReturnType DataTypeNullable::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings,
+                                                  const DataTypePtr & nested_data_type)
+{
+    return safeDeserialize<ReturnType>(column, *nested_data_type,
+        [&istr]
+        {
+            return checkStringByFirstCharacterAndAssertTheRestCaseInsensitive("NULL", istr)
+                || checkStringByFirstCharacterAndAssertTheRest("ᴺᵁᴸᴸ", istr);
+        },
+        [&nested_data_type, &istr, &settings] (IColumn & nested) { nested_data_type->deserializeAsWholeText(nested, istr, settings); });
 }
 
 
@@ -544,6 +558,7 @@ DataTypePtr removeNullable(const DataTypePtr & type)
 }
 
 
+template bool DataTypeNullable::deserializeWholeText<bool>(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, const DataTypePtr & nested);
 template bool DataTypeNullable::deserializeTextEscaped<bool>(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, const DataTypePtr & nested);
 template bool DataTypeNullable::deserializeTextQuoted<bool>(IColumn & column, ReadBuffer & istr, const FormatSettings &, const DataTypePtr & nested);
 template bool DataTypeNullable::deserializeTextCSV<bool>(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, const DataTypePtr & nested);

src/DataTypes/DataTypeNullable.h

@@ -103,6 +103,8 @@ public:
     /// If ReturnType is bool, check for NULL and deserialize value into non-nullable column (and return true) or insert default value of nested type (and return false)
     /// If ReturnType is void, deserialize Nullable(T)
     template <typename ReturnType = bool>
+    static ReturnType deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, const DataTypePtr & nested);
+    template <typename ReturnType = bool>
     static ReturnType deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, const DataTypePtr & nested);
     template <typename ReturnType = bool>
     static ReturnType deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &, const DataTypePtr & nested);

src/Databases/MySQL/MaterializeMySQLSyncThread.cpp

@@ -195,6 +195,7 @@ void MaterializeMySQLSyncThread::synchronization(const String & mysql_version)
     }
     catch (...)
     {
+        client.disconnect();
         tryLogCurrentException(log);
         getDatabase(database_name).setException(std::current_exception());
     }
@@ -206,6 +207,7 @@ void MaterializeMySQLSyncThread::stopSynchronization()
     {
         sync_quit = true;
         background_thread_pool->join();
+        client.disconnect();
     }
 }
 

src/Dictionaries/ExecutableDictionarySource.cpp

@@ -1,12 +1,13 @@
 #include "ExecutableDictionarySource.h"
 
-#include <future>
+#include <functional>
-#include <thread>
 #include <ext/scope_guard.h>
 #include <DataStreams/IBlockOutputStream.h>
 #include <DataStreams/OwningBlockInputStream.h>
 #include <Interpreters/Context.h>
 #include <IO/WriteHelpers.h>
+#include <IO/ReadHelpers.h>
+#include <IO/copyData.h>
 #include <Common/ShellCommand.h>
 #include <Common/ThreadPool.h>
 #include <common/logger_useful.h>
@@ -16,6 +17,7 @@
 #include "DictionaryStructure.h"
 #include "registerDictionaries.h"
 
+
 namespace DB
 {
 static const UInt64 max_block_size = 8192;
@@ -31,15 +33,23 @@ namespace
     /// Owns ShellCommand and calls wait for it.
     class ShellCommandOwningBlockInputStream : public OwningBlockInputStream<ShellCommand>
     {
+    private:
+        Poco::Logger * log;
     public:
-        ShellCommandOwningBlockInputStream(const BlockInputStreamPtr & impl, std::unique_ptr<ShellCommand> own_)
+        ShellCommandOwningBlockInputStream(Poco::Logger * log_, const BlockInputStreamPtr & impl, std::unique_ptr<ShellCommand> command_)
-            : OwningBlockInputStream(std::move(impl), std::move(own_))
+            : OwningBlockInputStream(std::move(impl), std::move(command_)), log(log_)
         {
         }
 
         void readSuffix() override
         {
             OwningBlockInputStream<ShellCommand>::readSuffix();
+
+            std::string err;
+            readStringUntilEOF(err, own->err);
+            if (!err.empty())
+                LOG_ERROR(log, "Having stderr: {}", err);
+
             own->wait();
         }
     };
@@ -80,7 +90,7 @@ BlockInputStreamPtr ExecutableDictionarySource::loadAll()
     LOG_TRACE(log, "loadAll {}", toString());
     auto process = ShellCommand::execute(command);
     auto input_stream = context.getInputFormat(format, process->out, sample_block, max_block_size);
-    return std::make_shared<ShellCommandOwningBlockInputStream>(input_stream, std::move(process));
+    return std::make_shared<ShellCommandOwningBlockInputStream>(log, input_stream, std::move(process));
 }
 
 BlockInputStreamPtr ExecutableDictionarySource::loadUpdatedAll()
@@ -95,67 +105,73 @@ BlockInputStreamPtr ExecutableDictionarySource::loadUpdatedAll()
     LOG_TRACE(log, "loadUpdatedAll {}", command_with_update_field);
     auto process = ShellCommand::execute(command_with_update_field);
     auto input_stream = context.getInputFormat(format, process->out, sample_block, max_block_size);
-    return std::make_shared<ShellCommandOwningBlockInputStream>(input_stream, std::move(process));
+    return std::make_shared<ShellCommandOwningBlockInputStream>(log, input_stream, std::move(process));
 }
 
 namespace
 {
-    /** A stream, that also runs and waits for background thread
+    /** A stream, that runs child process and sends data to its stdin in background thread,
-      * (that will feed data into pipe to be read from the other side of the pipe).
+      *  and receives data from its stdout.
       */
     class BlockInputStreamWithBackgroundThread final : public IBlockInputStream
     {
     public:
         BlockInputStreamWithBackgroundThread(
-            const BlockInputStreamPtr & stream_, std::unique_ptr<ShellCommand> && command_, std::packaged_task<void()> && task_)
+            const Context & context,
-            : stream{stream_}, command{std::move(command_)}, task(std::move(task_)), thread([this] {
+            const std::string & format,
-                task();
+            const Block & sample_block,
-                command->in.close();
+            const std::string & command_str,
-            })
+            Poco::Logger * log_,
+            std::function<void(WriteBufferFromFile &)> && send_data_)
+            : log(log_),
+            command(ShellCommand::execute(command_str)),
+            send_data(std::move(send_data_)),
+            thread([this] { send_data(command->in); })
         {
-            children.push_back(stream);
+            stream = context.getInputFormat(format, command->out, sample_block, max_block_size);
         }
 
         ~BlockInputStreamWithBackgroundThread() override
         {
             if (thread.joinable())
-            {
+                thread.join();
-                try
-                {
-                    readSuffix();
-                }
-                catch (...)
-                {
-                    tryLogCurrentException(__PRETTY_FUNCTION__);
-                }
-            }
         }
 
-        Block getHeader() const override { return stream->getHeader(); }
+        Block getHeader() const override
+        {
+            return stream->getHeader();
+        }
 
     private:
-        Block readImpl() override { return stream->read(); }
+        Block readImpl() override
+        {
+            return stream->read();
+        }
+
+        void readPrefix() override
+        {
+            stream->readPrefix();
+        }
 
         void readSuffix() override
         {
-            IBlockInputStream::readSuffix();
+            stream->readSuffix();
-            if (!wait_called)
+
-            {
+            std::string err;
-                wait_called = true;
+            readStringUntilEOF(err, command->err);
-                command->wait();
+            if (!err.empty())
-            }
+                LOG_ERROR(log, "Having stderr: {}", err);
-            thread.join();
+
-            /// To rethrow an exception, if any.
+            command->wait();
-            task.get_future().get();
         }
 
         String getName() const override { return "WithBackgroundThread"; }
 
+        Poco::Logger * log;
         BlockInputStreamPtr stream;
         std::unique_ptr<ShellCommand> command;
-        std::packaged_task<void()> task;
+        std::function<void(WriteBufferFromFile &)> send_data;
         ThreadFromGlobalPool thread;
-        bool wait_called = false;
     };
 
 }
@@ -164,28 +180,29 @@ namespace
 BlockInputStreamPtr ExecutableDictionarySource::loadIds(const std::vector<UInt64> & ids)
 {
     LOG_TRACE(log, "loadIds {} size = {}", toString(), ids.size());
-    auto process = ShellCommand::execute(command);
-
-    auto output_stream = context.getOutputFormat(format, process->in, sample_block);
-    auto input_stream = context.getInputFormat(format, process->out, sample_block, max_block_size);
 
     return std::make_shared<BlockInputStreamWithBackgroundThread>(
-        input_stream, std::move(process), std::packaged_task<void()>([output_stream, &ids]() mutable { formatIDs(output_stream, ids); }));
+        context, format, sample_block, command, log,
+        [&ids, this](WriteBufferFromFile & out) mutable
+        {
+            auto output_stream = context.getOutputFormat(format, out, sample_block);
+            formatIDs(output_stream, ids);
+            out.close();
+        });
 }
 
 BlockInputStreamPtr ExecutableDictionarySource::loadKeys(const Columns & key_columns, const std::vector<size_t> & requested_rows)
 {
     LOG_TRACE(log, "loadKeys {} size = {}", toString(), requested_rows.size());
-    auto process = ShellCommand::execute(command);
-
-    auto output_stream = context.getOutputFormat(format, process->in, sample_block);
-    auto input_stream = context.getInputFormat(format, process->out, sample_block, max_block_size);
 
     return std::make_shared<BlockInputStreamWithBackgroundThread>(
-        input_stream, std::move(process), std::packaged_task<void()>([output_stream, key_columns, &requested_rows, this]() mutable
+        context, format, sample_block, command, log,
+        [key_columns, &requested_rows, this](WriteBufferFromFile & out) mutable
         {
+            auto output_stream = context.getOutputFormat(format, out, sample_block);
             formatKeys(dict_struct, output_stream, key_columns, requested_rows);
-        }));
+            out.close();
+        });
 }
 
 bool ExecutableDictionarySource::isModified() const

src/Formats/FormatFactory.cpp

@@ -324,13 +324,86 @@ void FormatFactory::registerFileSegmentationEngine(const String & name, FileSegm
     target = std::move(file_segmentation_engine);
 }
 
+/// File Segmentation Engines for parallel reading
+
+void registerFileSegmentationEngineTabSeparated(FormatFactory & factory);
+void registerFileSegmentationEngineCSV(FormatFactory & factory);
+void registerFileSegmentationEngineJSONEachRow(FormatFactory & factory);
+void registerFileSegmentationEngineRegexp(FormatFactory & factory);
+void registerFileSegmentationEngineJSONAsString(FormatFactory & factory);
+
+/// Formats for both input/output.
+
+void registerInputFormatNative(FormatFactory & factory);
+void registerOutputFormatNative(FormatFactory & factory);
+
+void registerInputFormatProcessorNative(FormatFactory & factory);
+void registerOutputFormatProcessorNative(FormatFactory & factory);
+void registerInputFormatProcessorRowBinary(FormatFactory & factory);
+void registerOutputFormatProcessorRowBinary(FormatFactory & factory);
+void registerInputFormatProcessorTabSeparated(FormatFactory & factory);
+void registerOutputFormatProcessorTabSeparated(FormatFactory & factory);
+void registerInputFormatProcessorValues(FormatFactory & factory);
+void registerOutputFormatProcessorValues(FormatFactory & factory);
+void registerInputFormatProcessorCSV(FormatFactory & factory);
+void registerOutputFormatProcessorCSV(FormatFactory & factory);
+void registerInputFormatProcessorTSKV(FormatFactory & factory);
+void registerOutputFormatProcessorTSKV(FormatFactory & factory);
+void registerInputFormatProcessorJSONEachRow(FormatFactory & factory);
+void registerOutputFormatProcessorJSONEachRow(FormatFactory & factory);
+void registerInputFormatProcessorJSONCompactEachRow(FormatFactory & factory);
+void registerOutputFormatProcessorJSONCompactEachRow(FormatFactory & factory);
+void registerInputFormatProcessorProtobuf(FormatFactory & factory);
+void registerOutputFormatProcessorProtobuf(FormatFactory & factory);
+void registerInputFormatProcessorTemplate(FormatFactory & factory);
+void registerOutputFormatProcessorTemplate(FormatFactory & factory);
+void registerInputFormatProcessorMsgPack(FormatFactory & factory);
+void registerOutputFormatProcessorMsgPack(FormatFactory & factory);
+void registerInputFormatProcessorORC(FormatFactory & factory);
+void registerOutputFormatProcessorORC(FormatFactory & factory);
+void registerInputFormatProcessorParquet(FormatFactory & factory);
+void registerOutputFormatProcessorParquet(FormatFactory & factory);
+void registerInputFormatProcessorArrow(FormatFactory & factory);
+void registerOutputFormatProcessorArrow(FormatFactory & factory);
+void registerInputFormatProcessorAvro(FormatFactory & factory);
+void registerOutputFormatProcessorAvro(FormatFactory & factory);
+
+/// Output only (presentational) formats.
+
+void registerOutputFormatNull(FormatFactory & factory);
+
+void registerOutputFormatProcessorPretty(FormatFactory & factory);
+void registerOutputFormatProcessorPrettyCompact(FormatFactory & factory);
+void registerOutputFormatProcessorPrettySpace(FormatFactory & factory);
+void registerOutputFormatProcessorVertical(FormatFactory & factory);
+void registerOutputFormatProcessorJSON(FormatFactory & factory);
+void registerOutputFormatProcessorJSONCompact(FormatFactory & factory);
+void registerOutputFormatProcessorJSONEachRowWithProgress(FormatFactory & factory);
+void registerOutputFormatProcessorXML(FormatFactory & factory);
+void registerOutputFormatProcessorODBCDriver2(FormatFactory & factory);
+void registerOutputFormatProcessorNull(FormatFactory & factory);
+void registerOutputFormatProcessorMySQLWire(FormatFactory & factory);
+void registerOutputFormatProcessorMarkdown(FormatFactory & factory);
+void registerOutputFormatProcessorPostgreSQLWire(FormatFactory & factory);
+
+/// Input only formats.
+
+void registerInputFormatProcessorRegexp(FormatFactory & factory);
+void registerInputFormatProcessorJSONAsString(FormatFactory & factory);
+void registerInputFormatProcessorLineAsString(FormatFactory & factory);
+void registerInputFormatProcessorCapnProto(FormatFactory & factory);
+
 FormatFactory::FormatFactory()
 {
+    registerFileSegmentationEngineTabSeparated(*this);
+    registerFileSegmentationEngineCSV(*this);
+    registerFileSegmentationEngineJSONEachRow(*this);
+    registerFileSegmentationEngineRegexp(*this);
+    registerFileSegmentationEngineJSONAsString(*this);
+
     registerInputFormatNative(*this);
     registerOutputFormatNative(*this);
 
-    registerOutputFormatProcessorJSONEachRowWithProgress(*this);
-
     registerInputFormatProcessorNative(*this);
     registerOutputFormatProcessorNative(*this);
     registerInputFormatProcessorRowBinary(*this);
@@ -349,8 +422,11 @@ FormatFactory::FormatFactory()
     registerOutputFormatProcessorJSONCompactEachRow(*this);
     registerInputFormatProcessorProtobuf(*this);
     registerOutputFormatProcessorProtobuf(*this);
+    registerInputFormatProcessorTemplate(*this);
+    registerOutputFormatProcessorTemplate(*this);
+    registerInputFormatProcessorMsgPack(*this);
+    registerOutputFormatProcessorMsgPack(*this);
 #if !defined(ARCADIA_BUILD)
-    registerInputFormatProcessorCapnProto(*this);
     registerInputFormatProcessorORC(*this);
     registerOutputFormatProcessorORC(*this);
     registerInputFormatProcessorParquet(*this);
@@ -360,18 +436,6 @@ FormatFactory::FormatFactory()
     registerInputFormatProcessorAvro(*this);
     registerOutputFormatProcessorAvro(*this);
 #endif
-    registerInputFormatProcessorTemplate(*this);
-    registerOutputFormatProcessorTemplate(*this);
-    registerInputFormatProcessorRegexp(*this);
-    registerInputFormatProcessorMsgPack(*this);
-    registerOutputFormatProcessorMsgPack(*this);
-    registerInputFormatProcessorJSONAsString(*this);
-
-    registerFileSegmentationEngineTabSeparated(*this);
-    registerFileSegmentationEngineCSV(*this);
-    registerFileSegmentationEngineJSONEachRow(*this);
-    registerFileSegmentationEngineRegexp(*this);
-    registerFileSegmentationEngineJSONAsString(*this);
 
     registerOutputFormatNull(*this);
 
@@ -381,12 +445,20 @@ FormatFactory::FormatFactory()
     registerOutputFormatProcessorVertical(*this);
     registerOutputFormatProcessorJSON(*this);
     registerOutputFormatProcessorJSONCompact(*this);
+    registerOutputFormatProcessorJSONEachRowWithProgress(*this);
     registerOutputFormatProcessorXML(*this);
     registerOutputFormatProcessorODBCDriver2(*this);
     registerOutputFormatProcessorNull(*this);
     registerOutputFormatProcessorMySQLWire(*this);
     registerOutputFormatProcessorMarkdown(*this);
     registerOutputFormatProcessorPostgreSQLWire(*this);
+
+    registerInputFormatProcessorRegexp(*this);
+    registerInputFormatProcessorJSONAsString(*this);
+    registerInputFormatProcessorLineAsString(*this);
+#if !defined(ARCADIA_BUILD)
+    registerInputFormatProcessorCapnProto(*this);
+#endif
 }
 
 FormatFactory & FormatFactory::instance()

src/Formats/FormatFactory.h

@@ -141,73 +141,4 @@ private:
     const Creators & getCreators(const String & name) const;
 };
 
-/// Formats for both input/output.
-
-void registerInputFormatNative(FormatFactory & factory);
-void registerOutputFormatNative(FormatFactory & factory);
-
-void registerInputFormatProcessorNative(FormatFactory & factory);
-void registerOutputFormatProcessorNative(FormatFactory & factory);
-void registerInputFormatProcessorRowBinary(FormatFactory & factory);
-void registerOutputFormatProcessorRowBinary(FormatFactory & factory);
-void registerInputFormatProcessorTabSeparated(FormatFactory & factory);
-void registerOutputFormatProcessorTabSeparated(FormatFactory & factory);
-void registerInputFormatProcessorValues(FormatFactory & factory);
-void registerOutputFormatProcessorValues(FormatFactory & factory);
-void registerInputFormatProcessorCSV(FormatFactory & factory);
-void registerOutputFormatProcessorCSV(FormatFactory & factory);
-void registerInputFormatProcessorTSKV(FormatFactory & factory);
-void registerOutputFormatProcessorTSKV(FormatFactory & factory);
-void registerInputFormatProcessorJSONEachRow(FormatFactory & factory);
-void registerOutputFormatProcessorJSONEachRow(FormatFactory & factory);
-void registerInputFormatProcessorJSONCompactEachRow(FormatFactory & factory);
-void registerOutputFormatProcessorJSONCompactEachRow(FormatFactory & factory);
-void registerInputFormatProcessorParquet(FormatFactory & factory);
-void registerOutputFormatProcessorParquet(FormatFactory & factory);
-void registerInputFormatProcessorArrow(FormatFactory & factory);
-void registerOutputFormatProcessorArrow(FormatFactory & factory);
-void registerInputFormatProcessorProtobuf(FormatFactory & factory);
-void registerOutputFormatProcessorProtobuf(FormatFactory & factory);
-void registerInputFormatProcessorAvro(FormatFactory & factory);
-void registerOutputFormatProcessorAvro(FormatFactory & factory);
-void registerInputFormatProcessorTemplate(FormatFactory & factory);
-void registerOutputFormatProcessorTemplate(FormatFactory & factory);
-void registerInputFormatProcessorMsgPack(FormatFactory & factory);
-void registerOutputFormatProcessorMsgPack(FormatFactory & factory);
-void registerInputFormatProcessorORC(FormatFactory & factory);
-void registerOutputFormatProcessorORC(FormatFactory & factory);
-
-
-/// File Segmentation Engines for parallel reading
-
-void registerFileSegmentationEngineTabSeparated(FormatFactory & factory);
-void registerFileSegmentationEngineCSV(FormatFactory & factory);
-void registerFileSegmentationEngineJSONEachRow(FormatFactory & factory);
-void registerFileSegmentationEngineRegexp(FormatFactory & factory);
-void registerFileSegmentationEngineJSONAsString(FormatFactory & factory);
-
-/// Output only (presentational) formats.
-
-void registerOutputFormatNull(FormatFactory & factory);
-
-void registerOutputFormatProcessorPretty(FormatFactory & factory);
-void registerOutputFormatProcessorPrettyCompact(FormatFactory & factory);
-void registerOutputFormatProcessorPrettySpace(FormatFactory & factory);
-void registerOutputFormatProcessorPrettyASCII(FormatFactory & factory);
-void registerOutputFormatProcessorVertical(FormatFactory & factory);
-void registerOutputFormatProcessorJSON(FormatFactory & factory);
-void registerOutputFormatProcessorJSONCompact(FormatFactory & factory);
-void registerOutputFormatProcessorJSONEachRowWithProgress(FormatFactory & factory);
-void registerOutputFormatProcessorXML(FormatFactory & factory);
-void registerOutputFormatProcessorODBCDriver2(FormatFactory & factory);
-void registerOutputFormatProcessorNull(FormatFactory & factory);
-void registerOutputFormatProcessorMySQLWire(FormatFactory & factory);
-void registerOutputFormatProcessorMarkdown(FormatFactory & factory);
-void registerOutputFormatProcessorPostgreSQLWire(FormatFactory & factory);
-
-/// Input only formats.
-void registerInputFormatProcessorCapnProto(FormatFactory & factory);
-void registerInputFormatProcessorRegexp(FormatFactory & factory);
-void registerInputFormatProcessorJSONAsString(FormatFactory & factory);
-
 }

src/Functions/CMakeLists.txt

@@ -53,8 +53,28 @@ endif()
 
 target_include_directories(clickhouse_functions SYSTEM PRIVATE ${SPARSEHASH_INCLUDE_DIR})
 
-# Won't generate debug info for files with heavy template instantiation to achieve faster linking and lower size.
+if (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE"
-target_compile_options(clickhouse_functions PRIVATE "-g0")
+    OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO"
+    OR CMAKE_BUILD_TYPE_UC STREQUAL "MINSIZEREL")
+    set (STRIP_DSF_DEFAULT ON)
+else()
+    set (STRIP_DSF_DEFAULT OFF)
+endif()
+
+
+option(STRIP_DEBUG_SYMBOLS_FUNCTIONS
+    "Do not generate debugger info for ClickHouse functions.
+    Provides faster linking and lower binary size.
+    Tradeoff is the inability to debug some source files with e.g. gdb
+    (empty stack frames and no local variables)."
+    ${STRIP_DSF_DEFAULT})
+
+if (STRIP_DEBUG_SYMBOLS_FUNCTIONS)
+    message(WARNING "Not generating debugger info for ClickHouse functions")
+    target_compile_options(clickhouse_functions PRIVATE "-g0")
+else()
+    message(STATUS "Generating debugger info for ClickHouse functions")
+endif()
 
 if (USE_ICU)
     target_link_libraries (clickhouse_functions PRIVATE ${ICU_LIBRARIES})

src/Functions/FunctionBinaryArithmetic.h

@@ -561,6 +561,8 @@ public:
 template <template <typename, typename> class Op, typename Name, bool valid_on_default_arguments = true>
 class FunctionBinaryArithmetic : public IFunction
 {
+    static constexpr const bool is_plus = IsOperation<Op>::plus;
+    static constexpr const bool is_minus = IsOperation<Op>::minus;
     static constexpr const bool is_multiply = IsOperation<Op>::multiply;
     static constexpr const bool is_division = IsOperation<Op>::division;
 
@@ -612,9 +614,7 @@ class FunctionBinaryArithmetic : public IFunction
         /// Special case when the function is plus or minus, one of arguments is Date/DateTime and another is Interval.
         /// We construct another function (example: addMonths) and call it.
 
-        static constexpr bool function_is_plus = IsOperation<Op>::plus;
+        if constexpr (!is_plus && !is_minus)
-        static constexpr bool function_is_minus = IsOperation<Op>::minus;
-        if constexpr (!function_is_plus && !function_is_minus)
             return {};
 
         const DataTypePtr & type_time = first_is_date_or_datetime ? type0 : type1;
@@ -631,21 +631,21 @@ class FunctionBinaryArithmetic : public IFunction
                 return {};
         }
 
-        if (second_is_date_or_datetime && function_is_minus)
+        if (second_is_date_or_datetime && is_minus)
             throw Exception("Wrong order of arguments for function " + getName() + ": argument of type Interval cannot be first.",
                 ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
 
         std::string function_name;
         if (interval_data_type)
         {
-            function_name = String(function_is_plus ? "add" : "subtract") + interval_data_type->getKind().toString() + 's';
+            function_name = String(is_plus ? "add" : "subtract") + interval_data_type->getKind().toString() + 's';
         }
         else
         {
             if (isDate(type_time))
-                function_name = function_is_plus ? "addDays" : "subtractDays";
+                function_name = is_plus ? "addDays" : "subtractDays";
             else
-                function_name = function_is_plus ? "addSeconds" : "subtractSeconds";
+                function_name = is_plus ? "addSeconds" : "subtractSeconds";
         }
 
         return FunctionFactory::instance().get(function_name, context);
@@ -653,7 +653,7 @@ class FunctionBinaryArithmetic : public IFunction
 
     bool isAggregateMultiply(const DataTypePtr & type0, const DataTypePtr & type1) const
     {
-        if constexpr (!IsOperation<Op>::multiply)
+        if constexpr (!is_multiply)
             return false;
 
         WhichDataType which0(type0);
@@ -665,7 +665,7 @@ class FunctionBinaryArithmetic : public IFunction
 
     bool isAggregateAddition(const DataTypePtr & type0, const DataTypePtr & type1) const
     {
-        if constexpr (!IsOperation<Op>::plus)
+        if constexpr (!is_plus)
             return false;
 
         WhichDataType which0(type0);
@@ -994,8 +994,6 @@ public:
 
         if constexpr (!std::is_same_v<ResultDataType, InvalidType>)
         {
-            constexpr bool result_is_decimal = IsDataTypeDecimal<LeftDataType> || IsDataTypeDecimal<RightDataType>;
-
             using T0 = typename LeftDataType::FieldType;
             using T1 = typename RightDataType::FieldType;
             using ResultType = typename ResultDataType::FieldType;
@@ -1003,112 +1001,91 @@ public:
             using ColVecT1 = std::conditional_t<IsDecimalNumber<T1>, ColumnDecimal<T1>, ColumnVector<T1>>;
             using ColVecResult = std::conditional_t<IsDecimalNumber<ResultType>, ColumnDecimal<ResultType>, ColumnVector<ResultType>>;
 
-            /// Decimal operations need scale. Operations are on result type.
-            using OpImpl = std::conditional_t<IsDataTypeDecimal<ResultDataType>,
-                DecimalBinaryOperation<T0, T1, Op, ResultType>,
-                BinaryOperationImpl<T0, T1, Op<T0, T1>, ResultType>>;
-
             auto col_left_raw = block.getByPosition(arguments[0]).column.get();
             auto col_right_raw = block.getByPosition(arguments[1]).column.get();
-            if (auto col_left = checkAndGetColumnConst<ColVecT0>(col_left_raw))
-            {
-                if (auto col_right = checkAndGetColumnConst<ColVecT1>(col_right_raw))
-                {
-                    /// the only case with a non-vector result
-                    if constexpr (result_is_decimal)
-                    {
-                        ResultDataType type = decimalResultType<is_multiply, is_division>(left, right);
-                        typename ResultDataType::FieldType scale_a = type.scaleFactorFor(left, is_multiply);
-                        typename ResultDataType::FieldType scale_b = type.scaleFactorFor(right, is_multiply || is_division);
-                        if constexpr (IsDataTypeDecimal<RightDataType> && is_division)
-                            scale_a = right.getScaleMultiplier();
 
-                        auto res = OpImpl::constantConstant(col_left->template getValue<T0>(), col_right->template getValue<T1>(),
+            auto col_left_const = checkAndGetColumnConst<ColVecT0>(col_left_raw);
-                                                                scale_a, scale_b, check_decimal_overflow);
+            auto col_right_const = checkAndGetColumnConst<ColVecT1>(col_right_raw);
-                        block.getByPosition(result).column =
-                            ResultDataType(type.getPrecision(), type.getScale()).createColumnConst(
-                                col_left->size(), toField(res, type.getScale()));
-
-                    }
-                    else
-                    {
-                        auto res = OpImpl::constantConstant(col_left->template getValue<T0>(), col_right->template getValue<T1>());
-                        block.getByPosition(result).column = ResultDataType().createColumnConst(col_left->size(), toField(res));
-                    }
-                    return true;
-                }
-            }
 
             typename ColVecResult::MutablePtr col_res = nullptr;
-            if constexpr (result_is_decimal)
+
+            auto col_left = checkAndGetColumn<ColVecT0>(col_left_raw);
+            auto col_right = checkAndGetColumn<ColVecT1>(col_right_raw);
+
+            if constexpr (IsDataTypeDecimal<LeftDataType> || IsDataTypeDecimal<RightDataType>)
             {
+                using OpImpl = DecimalBinaryOperation<T0, T1, Op, ResultType>;
+
                 ResultDataType type = decimalResultType<is_multiply, is_division>(left, right);
-                col_res = ColVecResult::create(0, type.getScale());
-            }
-            else
-                col_res = ColVecResult::create();
 
-            auto & vec_res = col_res->getData();
+                typename ResultDataType::FieldType scale_a = type.scaleFactorFor(left, is_multiply);
-            vec_res.resize(block.rows());
+                typename ResultDataType::FieldType scale_b = type.scaleFactorFor(right, is_multiply || is_division);
+                if constexpr (IsDataTypeDecimal<RightDataType> && is_division)
+                    scale_a = right.getScaleMultiplier();
 
-            if (auto col_left_const = checkAndGetColumnConst<ColVecT0>(col_left_raw))
+                /// non-vector result
-            {
+                if (col_left_const && col_right_const)
-                if (auto col_right = checkAndGetColumn<ColVecT1>(col_right_raw))
                 {
-                    if constexpr (result_is_decimal)
+                    auto res = OpImpl::constantConstant(col_left_const->template getValue<T0>(), col_right_const->template getValue<T1>(),
-                    {
+                                                        scale_a, scale_b, check_decimal_overflow);
-                        ResultDataType type = decimalResultType<is_multiply, is_division>(left, right);
 
-                        typename ResultDataType::FieldType scale_a = type.scaleFactorFor(left, is_multiply);
+                    block.getByPosition(result).column = ResultDataType(type.getPrecision(), type.getScale()).createColumnConst(
-                        typename ResultDataType::FieldType scale_b = type.scaleFactorFor(right, is_multiply || is_division);
+                            col_left_const->size(), toField(res, type.getScale()));
-                        if constexpr (IsDataTypeDecimal<RightDataType> && is_division)
+                    return true;
-                            scale_a = right.getScaleMultiplier();
+                }
 
-                        OpImpl::constantVector(col_left_const->template getValue<T0>(), col_right->getData(), vec_res,
+                col_res = ColVecResult::create(0, type.getScale());
-                                                scale_a, scale_b, check_decimal_overflow);
+                auto & vec_res = col_res->getData();
-                    }
+                vec_res.resize(block.rows());
-                    else
+
-                        OpImpl::constantVector(col_left_const->template getValue<T0>(), col_right->getData().data(), vec_res.data(), vec_res.size());
+                if (col_left && col_right)
+                {
+                    OpImpl::vectorVector(col_left->getData(), col_right->getData(), vec_res, scale_a, scale_b, check_decimal_overflow);
+                }
+                else if (col_left_const && col_right)
+                {
+                    OpImpl::constantVector(col_left_const->template getValue<T0>(), col_right->getData(), vec_res,
+                                           scale_a, scale_b, check_decimal_overflow);
+                }
+                else if (col_left && col_right_const)
+                {
+                    OpImpl::vectorConstant(col_left->getData(), col_right_const->template getValue<T1>(), vec_res,
+                                           scale_a, scale_b, check_decimal_overflow);
                 }
                 else
                     return false;
             }
-            else if (auto col_left = checkAndGetColumn<ColVecT0>(col_left_raw))
+            else
             {
-                if constexpr (result_is_decimal)
+                using OpImpl = BinaryOperationImpl<T0, T1, Op<T0, T1>, ResultType>;
+
+                /// non-vector result
+                if (col_left_const && col_right_const)
                 {
-                    ResultDataType type = decimalResultType<is_multiply, is_division>(left, right);
+                    auto res = OpImpl::constantConstant(col_left_const->template getValue<T0>(), col_right_const->template getValue<T1>());
+                    block.getByPosition(result).column = ResultDataType().createColumnConst(col_left_const->size(), toField(res));
+                    return true;
+                }
 
-                    typename ResultDataType::FieldType scale_a = type.scaleFactorFor(left, is_multiply);
+                col_res = ColVecResult::create();
-                    typename ResultDataType::FieldType scale_b = type.scaleFactorFor(right, is_multiply || is_division);
+                auto & vec_res = col_res->getData();
-                    if constexpr (IsDataTypeDecimal<RightDataType> && is_division)
+                vec_res.resize(block.rows());
-                        scale_a = right.getScaleMultiplier();
 
-                    if (auto col_right = checkAndGetColumn<ColVecT1>(col_right_raw))
+                if (col_left && col_right)
-                    {
+                {
-                        OpImpl::vectorVector(col_left->getData(), col_right->getData(), vec_res, scale_a, scale_b,
+                    OpImpl::vectorVector(col_left->getData().data(), col_right->getData().data(), vec_res.data(), vec_res.size());
-                                              check_decimal_overflow);
+                }
-                    }
+                else if (col_left_const && col_right)
-                    else if (auto col_right_const = checkAndGetColumnConst<ColVecT1>(col_right_raw))
+                {
-                    {
+                    OpImpl::constantVector(col_left_const->template getValue<T0>(), col_right->getData().data(), vec_res.data(), vec_res.size());
-                        OpImpl::vectorConstant(col_left->getData(), col_right_const->template getValue<T1>(), vec_res,
+                }
-                                                scale_a, scale_b, check_decimal_overflow);
+                else if (col_left && col_right_const)
-                    }
+                {
-                    else
+                    OpImpl::vectorConstant(col_left->getData().data(), col_right_const->template getValue<T1>(), vec_res.data(), vec_res.size());
-                        return false;
                 }
                 else
-                {
+                    return false;
-                    if (auto col_right = checkAndGetColumn<ColVecT1>(col_right_raw))
-                        OpImpl::vectorVector(col_left->getData().data(), col_right->getData().data(), vec_res.data(), vec_res.size());
-                    else if (auto col_right_const = checkAndGetColumnConst<ColVecT1>(col_right_raw))
-                        OpImpl::vectorConstant(col_left->getData().data(), col_right_const->template getValue<T1>(), vec_res.data(), vec_res.size());
-                    else
-                        return false;
-                }
             }
-            else
-                return false;
 
             block.getByPosition(result).column = std::move(col_res);
             return true;

src/Functions/FunctionJoinGet.cpp

@@ -1,10 +1,10 @@
 #include <Functions/FunctionJoinGet.h>
 
+#include <Columns/ColumnString.h>
 #include <Functions/FunctionFactory.h>
 #include <Functions/FunctionHelpers.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/HashJoin.h>
-#include <Columns/ColumnString.h>
 #include <Storages/StorageJoin.h>
 
 
@@ -16,19 +16,35 @@ namespace ErrorCodes
     extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
 }
 
+template <bool or_null>
+void ExecutableFunctionJoinGet<or_null>::execute(Block & block, const ColumnNumbers & arguments, size_t result, size_t)
+{
+    Block keys;
+    for (size_t i = 2; i < arguments.size(); ++i)
+    {
+        auto key = block.getByPosition(arguments[i]);
+        keys.insert(std::move(key));
+    }
+    block.getByPosition(result) = join->joinGet(keys, result_block);
+}
+
+template <bool or_null>
+ExecutableFunctionImplPtr FunctionJoinGet<or_null>::prepare(const Block &, const ColumnNumbers &, size_t) const
+{
+    return std::make_unique<ExecutableFunctionJoinGet<or_null>>(join, Block{{return_type->createColumn(), return_type, attr_name}});
+}
+
 static auto getJoin(const ColumnsWithTypeAndName & arguments, const Context & context)
 {
-    if (arguments.size() != 3)
-        throw Exception{"Function joinGet takes 3 arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH};
-
     String join_name;
     if (const auto * name_col = checkAndGetColumnConst<ColumnString>(arguments[0].column.get()))
     {
         join_name = name_col->getValue<String>();
     }
     else
-        throw Exception{"Illegal type " + arguments[0].type->getName() + " of first argument of function joinGet, expected a const string.",
+        throw Exception(
-                        ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
+            "Illegal type " + arguments[0].type->getName() + " of first argument of function joinGet, expected a const string.",
+            ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
 
     size_t dot = join_name.find('.');
     String database_name;
@@ -43,10 +59,12 @@ static auto getJoin(const ColumnsWithTypeAndName & arguments, const Context & co
         ++dot;
     }
     String table_name = join_name.substr(dot);
+    if (table_name.empty())
+        throw Exception("joinGet does not allow empty table name", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
     auto table = DatabaseCatalog::instance().getTable({database_name, table_name}, context);
     auto storage_join = std::dynamic_pointer_cast<StorageJoin>(table);
     if (!storage_join)
-        throw Exception{"Table " + join_name + " should have engine StorageJoin", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
+        throw Exception("Table " + join_name + " should have engine StorageJoin", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
 
     String attr_name;
     if (const auto * name_col = checkAndGetColumnConst<ColumnString>(arguments[1].column.get()))
@@ -54,57 +72,30 @@ static auto getJoin(const ColumnsWithTypeAndName & arguments, const Context & co
         attr_name = name_col->getValue<String>();
     }
     else
-        throw Exception{"Illegal type " + arguments[1].type->getName()
+        throw Exception(
-                            + " of second argument of function joinGet, expected a const string.",
+            "Illegal type " + arguments[1].type->getName() + " of second argument of function joinGet, expected a const string.",
-                        ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
+            ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
     return std::make_pair(storage_join, attr_name);
 }
 
 template <bool or_null>
 FunctionBaseImplPtr JoinGetOverloadResolver<or_null>::build(const ColumnsWithTypeAndName & arguments, const DataTypePtr &) const
 {
+    if (arguments.size() < 3)
+        throw Exception(
+            "Number of arguments for function " + getName() + " doesn't match: passed " + toString(arguments.size())
+                + ", should be greater or equal to 3",
+            ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
     auto [storage_join, attr_name] = getJoin(arguments, context);
     auto join = storage_join->getJoin();
-    DataTypes data_types(arguments.size());
+    DataTypes data_types(arguments.size() - 2);
-
+    for (size_t i = 2; i < arguments.size(); ++i)
+        data_types[i - 2] = arguments[i].type;
+    auto return_type = join->joinGetCheckAndGetReturnType(data_types, attr_name, or_null);
     auto table_lock = storage_join->lockForShare(context.getInitialQueryId(), context.getSettingsRef().lock_acquire_timeout);
-    for (size_t i = 0; i < arguments.size(); ++i)
-        data_types[i] = arguments[i].type;
-
-    auto return_type = join->joinGetReturnType(attr_name, or_null);
     return std::make_unique<FunctionJoinGet<or_null>>(table_lock, storage_join, join, attr_name, data_types, return_type);
 }
 
-template <bool or_null>
-DataTypePtr JoinGetOverloadResolver<or_null>::getReturnType(const ColumnsWithTypeAndName & arguments) const
-{
-    auto [storage_join, attr_name] = getJoin(arguments, context);
-    auto join = storage_join->getJoin();
-    return join->joinGetReturnType(attr_name, or_null);
-}
-
-
-template <bool or_null>
-void ExecutableFunctionJoinGet<or_null>::execute(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count)
-{
-    auto ctn = block.getByPosition(arguments[2]);
-    if (isColumnConst(*ctn.column))
-        ctn.column = ctn.column->cloneResized(1);
-    ctn.name = ""; // make sure the key name never collide with the join columns
-    Block key_block = {ctn};
-    join->joinGet(key_block, attr_name, or_null);
-    auto & result_ctn = key_block.getByPosition(1);
-    if (isColumnConst(*ctn.column))
-        result_ctn.column = ColumnConst::create(result_ctn.column, input_rows_count);
-    block.getByPosition(result) = result_ctn;
-}
-
-template <bool or_null>
-ExecutableFunctionImplPtr FunctionJoinGet<or_null>::prepare(const Block &, const ColumnNumbers &, size_t) const
-{
-    return std::make_unique<ExecutableFunctionJoinGet<or_null>>(join, attr_name);
-}
-
 void registerFunctionJoinGet(FunctionFactory & factory)
 {
     // joinGet

src/Functions/FunctionJoinGet.h

@@ -13,14 +13,14 @@ template <bool or_null>
 class ExecutableFunctionJoinGet final : public IExecutableFunctionImpl
 {
 public:
-    ExecutableFunctionJoinGet(HashJoinPtr join_, String attr_name_)
+    ExecutableFunctionJoinGet(HashJoinPtr join_, const Block & result_block_)
-        : join(std::move(join_)), attr_name(std::move(attr_name_)) {}
+        : join(std::move(join_)), result_block(result_block_) {}
 
     static constexpr auto name = or_null ? "joinGetOrNull" : "joinGet";
 
     bool useDefaultImplementationForNulls() const override { return false; }
-    bool useDefaultImplementationForConstants() const override { return true; }
     bool useDefaultImplementationForLowCardinalityColumns() const override { return true; }
+    bool useDefaultImplementationForConstants() const override { return true; }
 
     void execute(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override;
 
@@ -28,7 +28,7 @@ public:
 
 private:
     HashJoinPtr join;
-    const String attr_name;
+    Block result_block;
 };
 
 template <bool or_null>
@@ -77,13 +77,14 @@ public:
     String getName() const override { return name; }
 
     FunctionBaseImplPtr build(const ColumnsWithTypeAndName & arguments, const DataTypePtr &) const override;
-    DataTypePtr getReturnType(const ColumnsWithTypeAndName & arguments) const override;
+    DataTypePtr getReturnType(const ColumnsWithTypeAndName &) const override { return {}; } // Not used
 
     bool useDefaultImplementationForNulls() const override { return false; }
     bool useDefaultImplementationForLowCardinalityColumns() const override { return true; }
 
     bool isVariadic() const override { return true; }
     size_t getNumberOfArguments() const override { return 0; }
+    ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0, 1}; }
 
 private:
     const Context & context;

src/Functions/FunctionsFormatting.cpp

@@ -9,6 +9,7 @@ void registerFunctionsFormatting(FunctionFactory & factory)
 {
     factory.registerFunction<FunctionBitmaskToList>();
     factory.registerFunction<FunctionFormatReadableSize>();
+    factory.registerFunction<FunctionFormatReadableQuantity>();
 }
 
 }

src/Functions/FunctionsFormatting.h

@@ -202,4 +202,80 @@ private:
     }
 };
 
+
+class FunctionFormatReadableQuantity : public IFunction
+{
+public:
+    static constexpr auto name = "formatReadableQuantity";
+    static FunctionPtr create(const Context &) { return std::make_shared<FunctionFormatReadableQuantity>(); }
+
+    String getName() const override
+    {
+        return name;
+    }
+
+    size_t getNumberOfArguments() const override { return 1; }
+
+    DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
+    {
+        const IDataType & type = *arguments[0];
+
+        if (!isNativeNumber(type))
+            throw Exception("Cannot format " + type.getName() + " as quantity", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+
+        return std::make_shared<DataTypeString>();
+    }
+
+    bool useDefaultImplementationForConstants() const override { return true; }
+
+    void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override
+    {
+        if (!(executeType<UInt8>(block, arguments, result)
+            || executeType<UInt16>(block, arguments, result)
+            || executeType<UInt32>(block, arguments, result)
+            || executeType<UInt64>(block, arguments, result)
+            || executeType<Int8>(block, arguments, result)
+            || executeType<Int16>(block, arguments, result)
+            || executeType<Int32>(block, arguments, result)
+            || executeType<Int64>(block, arguments, result)
+            || executeType<Float32>(block, arguments, result)
+            || executeType<Float64>(block, arguments, result)))
+            throw Exception("Illegal column " + block.getByPosition(arguments[0]).column->getName()
+                + " of argument of function " + getName(),
+                ErrorCodes::ILLEGAL_COLUMN);
+    }
+
+private:
+    template <typename T>
+    bool executeType(Block & block, const ColumnNumbers & arguments, size_t result) const
+    {
+        if (const ColumnVector<T> * col_from = checkAndGetColumn<ColumnVector<T>>(block.getByPosition(arguments[0]).column.get()))
+        {
+            auto col_to = ColumnString::create();
+
+            const typename ColumnVector<T>::Container & vec_from = col_from->getData();
+            ColumnString::Chars & data_to = col_to->getChars();
+            ColumnString::Offsets & offsets_to = col_to->getOffsets();
+            size_t size = vec_from.size();
+            data_to.resize(size * 2);
+            offsets_to.resize(size);
+
+            WriteBufferFromVector<ColumnString::Chars> buf_to(data_to);
+
+            for (size_t i = 0; i < size; ++i)
+            {
+                formatReadableQuantity(static_cast<double>(vec_from[i]), buf_to);
+                writeChar(0, buf_to);
+                offsets_to[i] = buf_to.count();
+            }
+
+            buf_to.finalize();
+            block.getByPosition(result).column = std::move(col_to);
+            return true;
+        }
+
+        return false;
+    }
+};
+
 }

src/Functions/GatherUtils/CMakeLists.txt

@@ -3,5 +3,17 @@ add_headers_and_sources(clickhouse_functions_gatherutils .)
 add_library(clickhouse_functions_gatherutils ${clickhouse_functions_gatherutils_sources} ${clickhouse_functions_gatherutils_headers})
 target_link_libraries(clickhouse_functions_gatherutils PRIVATE dbms)
 
-# Won't generate debug info for files with heavy template instantiation to achieve faster linking and lower size.
+check_cxx_compiler_flag("-Wsuggest-override" HAS_SUGGEST_OVERRIDE)
-target_compile_options(clickhouse_functions_gatherutils PRIVATE "-g0")
+check_cxx_compiler_flag("-Wsuggest-destructor-override" HAS_SUGGEST_DESTRUCTOR_OVERRIDE)
+
+if (HAS_SUGGEST_OVERRIDE)
+    target_compile_definitions(clickhouse_functions_gatherutils PUBLIC HAS_SUGGEST_OVERRIDE)
+endif()
+
+if (HAS_SUGGEST_DESTRUCTOR_OVERRIDE)
+    target_compile_definitions(clickhouse_functions_gatherutils PUBLIC HAS_SUGGEST_DESTRUCTOR_OVERRIDE)
+endif()
+
+if (STRIP_DEBUG_SYMBOLS_FUNCTIONS)
+    target_compile_options(clickhouse_functions_gatherutils PRIVATE "-g0")
+endif()

src/Functions/GatherUtils/Sources.h

@@ -129,9 +129,13 @@ struct NumericArraySource : public ArraySourceImpl<NumericArraySource<T>>
     #pragma GCC diagnostic ignored "-Wsuggest-override"
 #elif __clang_major__ >= 11
     #pragma GCC diagnostic push
+#ifdef HAS_SUGGEST_OVERRIDE
     #pragma GCC diagnostic ignored "-Wsuggest-override"
+#endif
+#ifdef HAS_SUGGEST_DESTRUCTOR_OVERRIDE
     #pragma GCC diagnostic ignored "-Wsuggest-destructor-override"
 #endif
+#endif
 
 template <typename Base>
 struct ConstSource : public Base

src/Functions/URL/CMakeLists.txt

@@ -3,8 +3,9 @@ add_headers_and_sources(clickhouse_functions_url .)
 add_library(clickhouse_functions_url ${clickhouse_functions_url_sources} ${clickhouse_functions_url_headers})
 target_link_libraries(clickhouse_functions_url PRIVATE dbms)
 
-# Won't generate debug info for files with heavy template instantiation to achieve faster linking and lower size.
+if (STRIP_DEBUG_SYMBOLS_FUNCTIONS)
-target_compile_options(clickhouse_functions_url PRIVATE "-g0")
+    target_compile_options(clickhouse_functions_url PRIVATE "-g0")
+endif()
 
 # TODO: move Functions/Regexps.h to some lib and use here
 target_link_libraries(clickhouse_functions_url PRIVATE hyperscan)

src/Functions/array/CMakeLists.txt

@@ -3,5 +3,6 @@ add_headers_and_sources(clickhouse_functions_array .)
 add_library(clickhouse_functions_array ${clickhouse_functions_array_sources} ${clickhouse_functions_array_headers})
 target_link_libraries(clickhouse_functions_array PRIVATE dbms clickhouse_functions_gatherutils)
 
-# Won't generate debug info for files with heavy template instantiation to achieve faster linking and lower size.
+if (STRIP_DEBUG_SYMBOLS_FUNCTIONS)
-target_compile_options(clickhouse_functions_array PRIVATE "-g0")
+    target_compile_options(clickhouse_functions_array PRIVATE "-g0")
+endif()

src/Functions/array/mapPopulateSeries.cpp

@@ -0,0 +1,312 @@
+#include <Columns/ColumnArray.h>
+#include <Columns/ColumnTuple.h>
+#include <Columns/ColumnVector.h>
+#include <DataTypes/DataTypeArray.h>
+#include <DataTypes/DataTypeTuple.h>
+#include <Functions/FunctionFactory.h>
+#include <Functions/FunctionHelpers.h>
+#include <Functions/IFunction.h>
+#include "Core/ColumnWithTypeAndName.h"
+#include "DataTypes/IDataType.h"
+
+namespace DB
+{
+namespace ErrorCodes
+{
+    extern const int ILLEGAL_COLUMN;
+    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+}
+
+class FunctionMapPopulateSeries : public IFunction
+{
+public:
+    static constexpr auto name = "mapPopulateSeries";
+    static FunctionPtr create(const Context &) { return std::make_shared<FunctionMapPopulateSeries>(); }
+
+private:
+    String getName() const override { return name; }
+
+    size_t getNumberOfArguments() const override { return 0; }
+    bool isVariadic() const override { return true; }
+    bool useDefaultImplementationForConstants() const override { return true; }
+
+    DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
+    {
+        if (arguments.size() < 2)
+            throw Exception{getName() + " accepts at least two arrays for key and value", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH};
+
+        if (arguments.size() > 3)
+            throw Exception{"too many arguments in " + getName() + " call", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH};
+
+        const DataTypeArray * key_array_type = checkAndGetDataType<DataTypeArray>(arguments[0].get());
+        const DataTypeArray * val_array_type = checkAndGetDataType<DataTypeArray>(arguments[1].get());
+
+        if (!key_array_type || !val_array_type)
+            throw Exception{getName() + " accepts two arrays for key and value", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
+
+        DataTypePtr keys_type = key_array_type->getNestedType();
+        WhichDataType which_key(keys_type);
+        if (!(which_key.isNativeInt() || which_key.isNativeUInt()))
+        {
+            throw Exception(
+                "Keys for " + getName() + " should be of native integer type (signed or unsigned)", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+        }
+
+        if (arguments.size() == 3)
+        {
+            DataTypePtr max_key_type = arguments[2];
+            WhichDataType which_max_key(max_key_type);
+
+            if (which_max_key.isNullable())
+                throw Exception(
+                    "Max key argument in arguments of function " + getName() + " can not be Nullable",
+                    ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+
+            if (keys_type->getTypeId() != max_key_type->getTypeId())
+                throw Exception("Max key type in " + getName() + " should be same as keys type", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+        }
+
+        return std::make_shared<DataTypeTuple>(DataTypes{arguments[0], arguments[1]});
+    }
+
+    template <typename KeyType, typename ValType>
+    void execute2(
+        Block & block, size_t result, ColumnPtr key_column, ColumnPtr val_column, ColumnPtr max_key_column, const DataTypeTuple & res_type)
+        const
+    {
+        MutableColumnPtr res_tuple = res_type.createColumn();
+
+        auto * to_tuple = assert_cast<ColumnTuple *>(res_tuple.get());
+        auto & to_keys_arr = assert_cast<ColumnArray &>(to_tuple->getColumn(0));
+        auto & to_keys_data = to_keys_arr.getData();
+        auto & to_keys_offsets = to_keys_arr.getOffsets();
+
+        auto & to_vals_arr = assert_cast<ColumnArray &>(to_tuple->getColumn(1));
+        auto & to_values_data = to_vals_arr.getData();
+
+        bool max_key_is_const = false, key_is_const = false, val_is_const = false;
+
+        const auto * keys_array = checkAndGetColumn<ColumnArray>(key_column.get());
+        if (!keys_array)
+        {
+            const ColumnConst * const_array = checkAndGetColumnConst<ColumnArray>(key_column.get());
+            if (!const_array)
+                throw Exception("Expected array column, found " + key_column->getName(), ErrorCodes::ILLEGAL_COLUMN);
+
+            keys_array = checkAndGetColumn<ColumnArray>(const_array->getDataColumnPtr().get());
+            key_is_const = true;
+        }
+
+        const auto * values_array = checkAndGetColumn<ColumnArray>(val_column.get());
+        if (!values_array)
+        {
+            const ColumnConst * const_array = checkAndGetColumnConst<ColumnArray>(val_column.get());
+            if (!const_array)
+                throw Exception("Expected array column, found " + val_column->getName(), ErrorCodes::ILLEGAL_COLUMN);
+
+            values_array = checkAndGetColumn<ColumnArray>(const_array->getDataColumnPtr().get());
+            val_is_const = true;
+        }
+
+        if (!keys_array || !values_array)
+            /* something went wrong */
+            throw Exception{"Illegal columns in arguments of function " + getName(), ErrorCodes::ILLEGAL_COLUMN};
+
+        KeyType max_key_const{0};
+
+        if (max_key_column && isColumnConst(*max_key_column))
+        {
+            const auto * column_const = static_cast<const ColumnConst *>(&*max_key_column);
+            max_key_const = column_const->template getValue<KeyType>();
+            max_key_is_const = true;
+        }
+
+        auto & keys_data = assert_cast<const ColumnVector<KeyType> &>(keys_array->getData()).getData();
+        auto & values_data = assert_cast<const ColumnVector<ValType> &>(values_array->getData()).getData();
+
+        // Original offsets
+        const IColumn::Offsets & key_offsets = keys_array->getOffsets();
+        const IColumn::Offsets & val_offsets = values_array->getOffsets();
+
+        IColumn::Offset offset{0};
+        size_t row_count = key_is_const ? values_array->size() : keys_array->size();
+
+        std::map<KeyType, ValType> res_map;
+
+        //Iterate through two arrays and fill result values.
+        for (size_t row = 0; row < row_count; ++row)
+        {
+            size_t key_offset = 0, val_offset = 0, array_size = key_offsets[0], val_array_size = val_offsets[0];
+
+            res_map.clear();
+
+            if (!key_is_const)
+            {
+                key_offset = row > 0 ? key_offsets[row - 1] : 0;
+                array_size = key_offsets[row] - key_offset;
+            }
+
+            if (!val_is_const)
+            {
+                val_offset = row > 0 ? val_offsets[row - 1] : 0;
+                val_array_size = val_offsets[row] - val_offset;
+            }
+
+            if (array_size != val_array_size)
+                throw Exception("Key and value array should have same amount of elements", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
+
+            if (array_size == 0)
+            {
+                to_keys_offsets.push_back(offset);
+                continue;
+            }
+
+            for (size_t i = 0; i < array_size; ++i)
+            {
+                res_map.insert({keys_data[key_offset + i], values_data[val_offset + i]});
+            }
+
+            auto min_key = res_map.begin()->first;
+            auto max_key = res_map.rbegin()->first;
+
+            if (max_key_column)
+            {
+                /* update the current max key if it's not constant */
+                if (max_key_is_const)
+                {
+                    max_key = max_key_const;
+                }
+                else
+                {
+                    max_key = (static_cast<const ColumnVector<KeyType> *>(max_key_column.get()))->getData()[row];
+                }
+
+                /* no need to add anything, max key is less that first key */
+                if (max_key < min_key)
+                {
+                    to_keys_offsets.push_back(offset);
+                    continue;
+                }
+            }
+
+            /* fill the result arrays */
+            KeyType key;
+            for (key = min_key; key <= max_key; ++key)
+            {
+                to_keys_data.insert(key);
+
+                auto it = res_map.find(key);
+                if (it != res_map.end())
+                {
+                    to_values_data.insert(it->second);
+                }
+                else
+                {
+                    to_values_data.insertDefault();
+                }
+
+                ++offset;
+            }
+
+            to_keys_offsets.push_back(offset);
+        }
+
+        to_vals_arr.getOffsets().insert(to_keys_offsets.begin(), to_keys_offsets.end());
+        block.getByPosition(result).column = std::move(res_tuple);
+    }
+
+    template <typename KeyType>
+    void execute1(
+        Block & block, size_t result, ColumnPtr key_column, ColumnPtr val_column, ColumnPtr max_key_column, const DataTypeTuple & res_type)
+        const
+    {
+        const auto & val_type = (assert_cast<const DataTypeArray *>(res_type.getElements()[1].get()))->getNestedType();
+        switch (val_type->getTypeId())
+        {
+            case TypeIndex::Int8:
+                execute2<KeyType, Int8>(block, result, key_column, val_column, max_key_column, res_type);
+                break;
+            case TypeIndex::Int16:
+                execute2<KeyType, Int16>(block, result, key_column, val_column, max_key_column, res_type);
+                break;
+            case TypeIndex::Int32:
+                execute2<KeyType, Int32>(block, result, key_column, val_column, max_key_column, res_type);
+                break;
+            case TypeIndex::Int64:
+                execute2<KeyType, Int64>(block, result, key_column, val_column, max_key_column, res_type);
+                break;
+            case TypeIndex::UInt8:
+                execute2<KeyType, UInt8>(block, result, key_column, val_column, max_key_column, res_type);
+                break;
+            case TypeIndex::UInt16:
+                execute2<KeyType, UInt16>(block, result, key_column, val_column, max_key_column, res_type);
+                break;
+            case TypeIndex::UInt32:
+                execute2<KeyType, UInt32>(block, result, key_column, val_column, max_key_column, res_type);
+                break;
+            case TypeIndex::UInt64:
+                execute2<KeyType, UInt64>(block, result, key_column, val_column, max_key_column, res_type);
+                break;
+            default:
+                throw Exception{"Illegal columns in arguments of function " + getName(), ErrorCodes::ILLEGAL_COLUMN};
+        }
+    }
+
+    void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t) const override
+    {
+        auto col1 = block.safeGetByPosition(arguments[0]), col2 = block.safeGetByPosition(arguments[1]);
+
+        const auto * k = assert_cast<const DataTypeArray *>(col1.type.get());
+        const auto * v = assert_cast<const DataTypeArray *>(col2.type.get());
+
+        /* determine output type */
+        const DataTypeTuple & res_type = DataTypeTuple(
+            DataTypes{std::make_shared<DataTypeArray>(k->getNestedType()), std::make_shared<DataTypeArray>(v->getNestedType())});
+
+        ColumnPtr max_key_column = nullptr;
+
+        if (arguments.size() == 3)
+        {
+            /* max key provided */
+            max_key_column = block.safeGetByPosition(arguments[2]).column;
+        }
+
+        switch (k->getNestedType()->getTypeId())
+        {
+            case TypeIndex::Int8:
+                execute1<Int8>(block, result, col1.column, col2.column, max_key_column, res_type);
+                break;
+            case TypeIndex::Int16:
+                execute1<Int16>(block, result, col1.column, col2.column, max_key_column, res_type);
+                break;
+            case TypeIndex::Int32:
+                execute1<Int32>(block, result, col1.column, col2.column, max_key_column, res_type);
+                break;
+            case TypeIndex::Int64:
+                execute1<Int64>(block, result, col1.column, col2.column, max_key_column, res_type);
+                break;
+            case TypeIndex::UInt8:
+                execute1<UInt8>(block, result, col1.column, col2.column, max_key_column, res_type);
+                break;
+            case TypeIndex::UInt16:
+                execute1<UInt16>(block, result, col1.column, col2.column, max_key_column, res_type);
+                break;
+            case TypeIndex::UInt32:
+                execute1<UInt32>(block, result, col1.column, col2.column, max_key_column, res_type);
+                break;
+            case TypeIndex::UInt64:
+                execute1<UInt64>(block, result, col1.column, col2.column, max_key_column, res_type);
+                break;
+            default:
+                throw Exception{"Illegal columns in arguments of function " + getName(), ErrorCodes::ILLEGAL_COLUMN};
+        }
+    }
+};
+
+void registerFunctionMapPopulateSeries(FunctionFactory & factory)
+{
+    factory.registerFunction<FunctionMapPopulateSeries>();
+}
+
+}

src/Functions/array/registerFunctionsArray.cpp

@@ -36,6 +36,7 @@ void registerFunctionArrayZip(FunctionFactory &);
 void registerFunctionArrayAUC(FunctionFactory &);
 void registerFunctionArrayReduceInRanges(FunctionFactory &);
 void registerFunctionMapOp(FunctionFactory &);
+void registerFunctionMapPopulateSeries(FunctionFactory &);
 
 void registerFunctionsArray(FunctionFactory & factory)
 {
@@ -73,6 +74,7 @@ void registerFunctionsArray(FunctionFactory & factory)
     registerFunctionArrayZip(factory);
     registerFunctionArrayAUC(factory);
     registerFunctionMapOp(factory);
+    registerFunctionMapPopulateSeries(factory);
 }
 
 }

src/Functions/divide.cpp

@@ -13,7 +13,6 @@ template <typename A, typename B>
 struct DivideFloatingImpl
 {
     using ResultType = typename NumberTraits::ResultOfFloatingPointDivision<A, B>::Type;
-    static const constexpr bool allow_decimal = true;
     static const constexpr bool allow_fixed_string = false;
 
     template <typename Result = ResultType>

src/Functions/if.cpp

@@ -604,7 +604,6 @@ private:
         const ColumnUInt8 * cond_col, Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count)
     {
         /// Convert both columns to the common type (if needed).
-
         const ColumnWithTypeAndName & arg1 = block.getByPosition(arguments[1]);
         const ColumnWithTypeAndName & arg2 = block.getByPosition(arguments[2]);
 
@@ -765,10 +764,22 @@ private:
         return ColumnNullable::create(materialized, ColumnUInt8::create(column->size(), 0));
     }
 
-    static ColumnPtr getNestedColumn(const ColumnPtr & column)
+    /// Return nested column recursively removing Nullable, examples:
+    /// Nullable(size = 1, Int32(size = 1), UInt8(size = 1)) -> Int32(size = 1)
+    /// Const(size = 0, Nullable(size = 1, Int32(size = 1), UInt8(size = 1))) ->
+    /// Const(size = 0, Int32(size = 1))
+    static ColumnPtr recursiveGetNestedColumnWithoutNullable(const ColumnPtr & column)
     {
         if (const auto * nullable = checkAndGetColumn<ColumnNullable>(*column))
+        {
+            /// Nullable cannot contain Nullable
             return nullable->getNestedColumnPtr();
+        }
+        else if (const auto * column_const = checkAndGetColumn<ColumnConst>(*column))
+        {
+            /// Save Constant, but remove Nullable
+            return ColumnConst::create(recursiveGetNestedColumnWithoutNullable(column_const->getDataColumnPtr()), column->size());
+        }
 
         return column;
     }
@@ -826,12 +837,12 @@ private:
             {
                 arg_cond,
                 {
-                    getNestedColumn(arg_then.column),
+                    recursiveGetNestedColumnWithoutNullable(arg_then.column),
                     removeNullable(arg_then.type),
                     ""
                 },
                 {
-                    getNestedColumn(arg_else.column),
+                    recursiveGetNestedColumnWithoutNullable(arg_else.column),
                     removeNullable(arg_else.type),
                     ""
                 },

src/Functions/minus.cpp

@@ -9,7 +9,6 @@ template <typename A, typename B>
 struct MinusImpl
 {
     using ResultType = typename NumberTraits::ResultOfSubtraction<A, B>::Type;
-    static const constexpr bool allow_decimal = true;
     static const constexpr bool allow_fixed_string = false;
 
     template <typename Result = ResultType>

src/Functions/multiply.cpp

@@ -9,7 +9,6 @@ template <typename A, typename B>
 struct MultiplyImpl
 {
     using ResultType = typename NumberTraits::ResultOfAdditionMultiplication<A, B>::Type;
-    static const constexpr bool allow_decimal = true;
     static const constexpr bool allow_fixed_string = false;
 
     template <typename Result = ResultType>

src/Functions/plus.cpp

@@ -9,8 +9,8 @@ template <typename A, typename B>
 struct PlusImpl
 {
     using ResultType = typename NumberTraits::ResultOfAdditionMultiplication<A, B>::Type;
-    static const constexpr bool allow_decimal = true;
     static const constexpr bool allow_fixed_string = false;
+    static const constexpr bool is_commutative = true;
 
     template <typename Result = ResultType>
     static inline NO_SANITIZE_UNDEFINED Result apply(A a, B b)

src/Functions/ya.make

@@ -10,7 +10,6 @@ ADDINCL(
     contrib/libs/farmhash
     contrib/libs/h3/h3lib/include
     contrib/libs/hyperscan/src
-    contrib/libs/icu/common
     contrib/libs/libdivide
     contrib/libs/rapidjson/include
     contrib/libs/xxhash
@@ -100,6 +99,7 @@ SRCS(
     array/indexOf.cpp
     array/length.cpp
     array/mapOp.cpp
+    array/mapPopulateSeries.cpp
     array/range.cpp
     array/registerFunctionsArray.cpp
     asin.cpp

src/Functions/ya.make.in

@@ -9,7 +9,6 @@ ADDINCL(
     contrib/libs/farmhash
     contrib/libs/h3/h3lib/include
     contrib/libs/hyperscan/src
-    contrib/libs/icu/common
     contrib/libs/libdivide
     contrib/libs/rapidjson/include
     contrib/libs/xxhash

src/IO/ReadBufferFromFile.cpp

@@ -77,6 +77,9 @@ ReadBufferFromFile::~ReadBufferFromFile()
 
 void ReadBufferFromFile::close()
 {
+    if (fd < 0)
+        return;
+
     if (0 != ::close(fd))
         throw Exception("Cannot close file", ErrorCodes::CANNOT_CLOSE_FILE);
 

src/IO/WriteBufferFromFile.cpp

@@ -92,6 +92,9 @@ WriteBufferFromFile::~WriteBufferFromFile()
 /// Close file before destruction of object.
 void WriteBufferFromFile::close()
 {
+    if (fd < 0)
+        return;
+
     next();
 
     if (0 != ::close(fd))

src/IO/WriteHelpers.h

@@ -11,6 +11,7 @@
 #include <common/LocalDateTime.h>
 #include <common/find_symbols.h>
 #include <common/StringRef.h>
+#include <common/wide_integer_to_string.h>
 
 #include <Core/DecimalFunctions.h>
 #include <Core/Types.h>
@@ -42,6 +43,12 @@ namespace ErrorCodes
     extern const int ILLEGAL_TYPE_OF_ARGUMENT;
 }
 
+template <typename T>
+inline std::string bigintToString(const T & x)
+{
+    return to_string(x);
+}
+
 /// Helper functions for formatted and binary output.
 
 inline void writeChar(char x, WriteBuffer & buf)

src/Interpreters/AsynchronousMetricLog.cpp

@@ -2,6 +2,7 @@
 #include <DataTypes/DataTypesNumber.h>
 #include <DataTypes/DataTypeDate.h>
 #include <DataTypes/DataTypeDateTime.h>
+#include <DataTypes/DataTypeDateTime64.h>
 #include <DataTypes/DataTypeString.h>
 #include <Interpreters/AsynchronousMetrics.h>
 
@@ -13,10 +14,11 @@ Block AsynchronousMetricLogElement::createBlock()
 {
     ColumnsWithTypeAndName columns;
 
-    columns.emplace_back(std::make_shared<DataTypeDate>(),      "event_date");
+    columns.emplace_back(std::make_shared<DataTypeDate>(),          "event_date");
-    columns.emplace_back(std::make_shared<DataTypeDateTime>(),  "event_time");
+    columns.emplace_back(std::make_shared<DataTypeDateTime>(),      "event_time");
-    columns.emplace_back(std::make_shared<DataTypeString>(),    "name");
+    columns.emplace_back(std::make_shared<DataTypeDateTime64>(6),   "event_time_microseconds");
-    columns.emplace_back(std::make_shared<DataTypeFloat64>(),   "value");
+    columns.emplace_back(std::make_shared<DataTypeString>(),        "name");
+    columns.emplace_back(std::make_shared<DataTypeFloat64>(),       "value");
 
     return Block(columns);
 }
@@ -28,6 +30,7 @@ void AsynchronousMetricLogElement::appendToBlock(MutableColumns & columns) const
 
     columns[column_idx++]->insert(event_date);
     columns[column_idx++]->insert(event_time);
+    columns[column_idx++]->insert(event_time_microseconds);
     columns[column_idx++]->insert(metric_name);
     columns[column_idx++]->insert(value);
 }
@@ -38,6 +41,11 @@ inline UInt64 time_in_milliseconds(std::chrono::time_point<std::chrono::system_c
     return std::chrono::duration_cast<std::chrono::milliseconds>(timepoint.time_since_epoch()).count();
 }
 
+inline UInt64 time_in_microseconds(std::chrono::time_point<std::chrono::system_clock> timepoint)
+{
+    return std::chrono::duration_cast<std::chrono::microseconds>(timepoint.time_since_epoch()).count();
+}
+
 
 inline UInt64 time_in_seconds(std::chrono::time_point<std::chrono::system_clock> timepoint)
 {
@@ -50,6 +58,7 @@ void AsynchronousMetricLog::addValues(const AsynchronousMetricValues & values)
 
     const auto now = std::chrono::system_clock::now();
     element.event_time = time_in_seconds(now);
+    element.event_time_microseconds = time_in_microseconds(now);
     element.event_date = DateLUT::instance().toDayNum(element.event_time);
 
     for (const auto & [key, value] : values)

src/Interpreters/AsynchronousMetricLog.h

@@ -22,6 +22,7 @@ struct AsynchronousMetricLogElement
 {
     UInt16 event_date;
     time_t event_time;
+    UInt64 event_time_microseconds;
     std::string metric_name;
     double value;
 

src/Interpreters/Context.cpp

@@ -23,6 +23,7 @@
 #include <Storages/MergeTree/MergeTreeSettings.h>
 #include <Storages/CompressionCodecSelector.h>
 #include <Storages/StorageS3Settings.h>
+#include <Storages/LiveView/TemporaryLiveViewCleaner.h>
 #include <Disks/DiskLocal.h>
 #include <TableFunctions/TableFunctionFactory.h>
 #include <Interpreters/ActionLocksManager.h>
@@ -430,6 +431,7 @@ struct ContextShared
         if (system_logs)
             system_logs->shutdown();
 
+        TemporaryLiveViewCleaner::shutdown();
         DatabaseCatalog::shutdown();
 
         /// Preemptive destruction is important, because these objects may have a refcount to ContextShared (cyclic reference).
@@ -487,6 +489,12 @@ Context Context::createGlobal(ContextShared * shared)
     return res;
 }
 
+void Context::initGlobal()
+{
+    DatabaseCatalog::init(this);
+    TemporaryLiveViewCleaner::init(*this);
+}
+
 SharedContextHolder Context::createShared()
 {
     return SharedContextHolder(std::make_unique<ContextShared>());

src/Interpreters/Context.h

@@ -445,11 +445,7 @@ public:
 
     void makeQueryContext() { query_context = this; }
     void makeSessionContext() { session_context = this; }
-    void makeGlobalContext()
+    void makeGlobalContext() { initGlobal(); global_context = this; }
-    {
-        global_context = this;
-        DatabaseCatalog::init(this);
-    }
 
     const Settings & getSettingsRef() const { return settings; }
 
@@ -625,6 +621,8 @@ public:
 private:
     std::unique_lock<std::recursive_mutex> getLock() const;
 
+    void initGlobal();
+
     /// Compute and set actual user settings, client_info.current_user should be set
     void calculateAccessRights();
 

src/Interpreters/ExternalLoader.cpp

@@ -893,6 +893,8 @@ private:
             cancelLoading(info);
         }
 
+        putBackFinishedThreadsToPool();
+
         /// All loadings have unique loading IDs.
         size_t loading_id = next_id_counter++;
         info.loading_id = loading_id;
@@ -914,6 +916,21 @@ private:
         }
     }
 
+    void putBackFinishedThreadsToPool()
+    {
+        for (auto loading_id : recently_finished_loadings)
+        {
+            auto it = loading_threads.find(loading_id);
+            if (it != loading_threads.end())
+            {
+                auto thread = std::move(it->second);
+                loading_threads.erase(it);
+                thread.join(); /// It's very likely that `thread` has already finished.
+            }
+        }
+        recently_finished_loadings.clear();
+    }
+
     static void cancelLoading(Info & info)
     {
         if (!info.isLoading())
@@ -1095,12 +1112,11 @@ private:
         }
         min_id_to_finish_loading_dependencies.erase(std::this_thread::get_id());
 
-        auto it = loading_threads.find(loading_id);
+        /// Add `loading_id` to the list of recently finished loadings.
-        if (it != loading_threads.end())
+        /// This list is used to later put the threads which finished loading back to the thread pool.
-        {
+        /// (We can't put the loading thread back to the thread pool immediately here because at this point
-            it->second.detach();
+        /// the loading thread is about to finish but it's not finished yet right now.)
-            loading_threads.erase(it);
+        recently_finished_loadings.push_back(loading_id);
-        }
     }
 
     /// Calculate next update time for loaded_object. Can be called without mutex locking,
@@ -1158,6 +1174,7 @@ private:
     bool always_load_everything = false;
     std::atomic<bool> enable_async_loading = false;
     std::unordered_map<size_t, ThreadFromGlobalPool> loading_threads;
+    std::vector<size_t> recently_finished_loadings;
     std::unordered_map<std::thread::id, size_t> min_id_to_finish_loading_dependencies;
     size_t next_id_counter = 1; /// should always be > 0
     mutable pcg64 rnd_engine{randomSeed()};

src/Interpreters/HashJoin.cpp

@@ -41,6 +41,7 @@ namespace ErrorCodes
     extern const int SYNTAX_ERROR;
     extern const int SET_SIZE_LIMIT_EXCEEDED;
     extern const int TYPE_MISMATCH;
+    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
 }
 
 namespace
@@ -1109,27 +1110,34 @@ void HashJoin::joinBlockImplCross(Block & block, ExtraBlockPtr & not_processed)
     block = block.cloneWithColumns(std::move(dst_columns));
 }
 
-static void checkTypeOfKey(const Block & block_left, const Block & block_right)
-{
-    const auto & [c1, left_type_origin, left_name] = block_left.safeGetByPosition(0);
-    const auto & [c2, right_type_origin, right_name] = block_right.safeGetByPosition(0);
-    auto left_type = removeNullable(left_type_origin);
-    auto right_type = removeNullable(right_type_origin);
 
-    if (!left_type->equals(*right_type))
+DataTypePtr HashJoin::joinGetCheckAndGetReturnType(const DataTypes & data_types, const String & column_name, bool or_null) const
-        throw Exception("Type mismatch of columns to joinGet by: "
-            + left_name + " " + left_type->getName() + " at left, "
-            + right_name + " " + right_type->getName() + " at right",
-            ErrorCodes::TYPE_MISMATCH);
-}
-
-
-DataTypePtr HashJoin::joinGetReturnType(const String & column_name, bool or_null) const
 {
     std::shared_lock lock(data->rwlock);
 
+    size_t num_keys = data_types.size();
+    if (right_table_keys.columns() != num_keys)
+        throw Exception(
+            "Number of arguments for function joinGet" + toString(or_null ? "OrNull" : "")
+                + " doesn't match: passed, should be equal to " + toString(num_keys),
+            ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
+
+    for (size_t i = 0; i < num_keys; ++i)
+    {
+        const auto & left_type_origin = data_types[i];
+        const auto & [c2, right_type_origin, right_name] = right_table_keys.safeGetByPosition(i);
+        auto left_type = removeNullable(left_type_origin);
+        auto right_type = removeNullable(right_type_origin);
+        if (!left_type->equals(*right_type))
+            throw Exception(
+                "Type mismatch in joinGet key " + toString(i) + ": found type " + left_type->getName() + ", while the needed type is "
+                    + right_type->getName(),
+                ErrorCodes::TYPE_MISMATCH);
+    }
+
     if (!sample_block_with_columns_to_add.has(column_name))
         throw Exception("StorageJoin doesn't contain column " + column_name, ErrorCodes::NO_SUCH_COLUMN_IN_TABLE);
+
     auto elem = sample_block_with_columns_to_add.getByName(column_name);
     if (or_null)
         elem.type = makeNullable(elem.type);
@@ -1138,34 +1146,33 @@ DataTypePtr HashJoin::joinGetReturnType(const String & column_name, bool or_null
 
 
 template <typename Maps>
-void HashJoin::joinGetImpl(Block & block, const Block & block_with_columns_to_add, const Maps & maps_) const
+ColumnWithTypeAndName HashJoin::joinGetImpl(const Block & block, const Block & block_with_columns_to_add, const Maps & maps_) const
 {
-    joinBlockImpl<ASTTableJoin::Kind::Left, ASTTableJoin::Strictness::RightAny>(
+    // Assemble the key block with correct names.
-        block, {block.getByPosition(0).name}, block_with_columns_to_add, maps_);
+    Block keys;
+    for (size_t i = 0; i < block.columns(); ++i)
+    {
+        auto key = block.getByPosition(i);
+        key.name = key_names_right[i];
+        keys.insert(std::move(key));
+    }
+
+    joinBlockImpl<ASTTableJoin::Kind::Left, ASTTableJoin::Strictness::Any>(
+        keys, key_names_right, block_with_columns_to_add, maps_);
+    return keys.getByPosition(keys.columns() - 1);
 }
 
 
-// TODO: support composite key
 // TODO: return multiple columns as named tuple
 // TODO: return array of values when strictness == ASTTableJoin::Strictness::All
-void HashJoin::joinGet(Block & block, const String & column_name, bool or_null) const
+ColumnWithTypeAndName HashJoin::joinGet(const Block & block, const Block & block_with_columns_to_add) const
 {
     std::shared_lock lock(data->rwlock);
 
-    if (key_names_right.size() != 1)
-        throw Exception("joinGet only supports StorageJoin containing exactly one key", ErrorCodes::UNSUPPORTED_JOIN_KEYS);
-
-    checkTypeOfKey(block, right_table_keys);
-
-    auto elem = sample_block_with_columns_to_add.getByName(column_name);
-    if (or_null)
-        elem.type = makeNullable(elem.type);
-    elem.column = elem.type->createColumn();
-
     if ((strictness == ASTTableJoin::Strictness::Any || strictness == ASTTableJoin::Strictness::RightAny) &&
         kind == ASTTableJoin::Kind::Left)
     {
-        joinGetImpl(block, {elem}, std::get<MapsOne>(data->maps));
+        return joinGetImpl(block, block_with_columns_to_add, std::get<MapsOne>(data->maps));
     }
     else
         throw Exception("joinGet only supports StorageJoin of type Left Any", ErrorCodes::INCOMPATIBLE_TYPE_OF_JOIN);

src/Interpreters/HashJoin.h

@@ -162,11 +162,11 @@ public:
       */
     void joinBlock(Block & block, ExtraBlockPtr & not_processed) override;
 
-    /// Infer the return type for joinGet function
+    /// Check joinGet arguments and infer the return type.
-    DataTypePtr joinGetReturnType(const String & column_name, bool or_null) const;
+    DataTypePtr joinGetCheckAndGetReturnType(const DataTypes & data_types, const String & column_name, bool or_null) const;
 
-    /// Used by joinGet function that turns StorageJoin into a dictionary
+    /// Used by joinGet function that turns StorageJoin into a dictionary.
-    void joinGet(Block & block, const String & column_name, bool or_null) const;
+    ColumnWithTypeAndName joinGet(const Block & block, const Block & block_with_columns_to_add) const;
 
     /** Keep "totals" (separate part of dataset, see WITH TOTALS) to use later.
       */
@@ -389,7 +389,7 @@ private:
     void joinBlockImplCross(Block & block, ExtraBlockPtr & not_processed) const;
 
     template <typename Maps>
-    void joinGetImpl(Block & block, const Block & block_with_columns_to_add, const Maps & maps_) const;
+    ColumnWithTypeAndName joinGetImpl(const Block & block, const Block & block_with_columns_to_add, const Maps & maps_) const;
 
     static Type chooseMethod(const ColumnRawPtrs & key_columns, Sizes & key_sizes);
 };

src/Interpreters/InterpreterDropQuery.h

@@ -10,6 +10,7 @@ namespace DB
 {
 class Context;
 using DatabaseAndTable = std::pair<DatabasePtr, StoragePtr>;
+class AccessRightsElements;
 
 /** Allow to either drop table with all its data (DROP),
   * or remove information about table (just forget) from server (DETACH),

src/Interpreters/InterpreterInsertQuery.cpp

@@ -17,6 +17,7 @@
 #include <Interpreters/InterpreterWatchQuery.h>
 #include <Interpreters/JoinedTables.h>
 #include <Parsers/ASTFunction.h>
+#include <Parsers/ASTIdentifier.h>
 #include <Parsers/ASTInsertQuery.h>
 #include <Parsers/ASTSelectQuery.h>
 #include <Parsers/ASTSelectWithUnionQuery.h>
@@ -29,6 +30,8 @@
 #include <Storages/StorageDistributed.h>
 #include <TableFunctions/TableFunctionFactory.h>
 #include <Common/checkStackSize.h>
+#include <Interpreters/TranslateQualifiedNamesVisitor.h>
+#include <Interpreters/getTableExpressions.h>
 
 namespace
 {
@@ -90,9 +93,32 @@ Block InterpreterInsertQuery::getSampleBlock(
     }
 
     Block table_sample = metadata_snapshot->getSampleBlock();
+
+    /// Process column transformers (e.g. * EXCEPT(a)), asterisks and qualified columns.
+    const auto & columns = metadata_snapshot->getColumns();
+    auto names_and_types = columns.getOrdinary();
+    removeDuplicateColumns(names_and_types);
+    auto table_expr = std::make_shared<ASTTableExpression>();
+    table_expr->database_and_table_name = createTableIdentifier(table->getStorageID());
+    table_expr->children.push_back(table_expr->database_and_table_name);
+    TablesWithColumns tables_with_columns;
+    tables_with_columns.emplace_back(DatabaseAndTableWithAlias(*table_expr, context.getCurrentDatabase()), names_and_types);
+
+    tables_with_columns[0].addHiddenColumns(columns.getMaterialized());
+    tables_with_columns[0].addHiddenColumns(columns.getAliases());
+    tables_with_columns[0].addHiddenColumns(table->getVirtuals());
+
+    NameSet source_columns_set;
+    for (const auto & identifier : query.columns->children)
+        source_columns_set.insert(identifier->getColumnName());
+    TranslateQualifiedNamesVisitor::Data visitor_data(source_columns_set, tables_with_columns);
+    TranslateQualifiedNamesVisitor visitor(visitor_data);
+    auto columns_ast = query.columns->clone();
+    visitor.visit(columns_ast);
+
     /// Form the block based on the column names from the query
     Block res;
-    for (const auto & identifier : query.columns->children)
+    for (const auto & identifier : columns_ast->children)
     {
         std::string current_name = identifier->getColumnName();
 

src/Interpreters/MetricLog.cpp

@@ -2,6 +2,7 @@
 #include <DataTypes/DataTypesNumber.h>
 #include <DataTypes/DataTypeDate.h>
 #include <DataTypes/DataTypeDateTime.h>
+#include <DataTypes/DataTypeDateTime64.h>
 
 
 namespace DB
@@ -11,9 +12,10 @@ Block MetricLogElement::createBlock()
 {
     ColumnsWithTypeAndName columns_with_type_and_name;
 
-    columns_with_type_and_name.emplace_back(std::make_shared<DataTypeDate>(),     "event_date");
+    columns_with_type_and_name.emplace_back(std::make_shared<DataTypeDate>(),           "event_date");
-    columns_with_type_and_name.emplace_back(std::make_shared<DataTypeDateTime>(), "event_time");
+    columns_with_type_and_name.emplace_back(std::make_shared<DataTypeDateTime>(),       "event_time");
-    columns_with_type_and_name.emplace_back(std::make_shared<DataTypeUInt64>(),   "milliseconds");
+    columns_with_type_and_name.emplace_back(std::make_shared<DataTypeDateTime64>(6),    "event_time_microseconds");
+    columns_with_type_and_name.emplace_back(std::make_shared<DataTypeUInt64>(),         "milliseconds");
 
     for (size_t i = 0, end = ProfileEvents::end(); i < end; ++i)
     {
@@ -41,6 +43,7 @@ void MetricLogElement::appendToBlock(MutableColumns & columns) const
 
     columns[column_idx++]->insert(DateLUT::instance().toDayNum(event_time));
     columns[column_idx++]->insert(event_time);
+    columns[column_idx++]->insert(event_time_microseconds);
     columns[column_idx++]->insert(milliseconds);
 
     for (size_t i = 0, end = ProfileEvents::end(); i < end; ++i)
@@ -80,6 +83,10 @@ inline UInt64 time_in_milliseconds(std::chrono::time_point<std::chrono::system_c
     return std::chrono::duration_cast<std::chrono::milliseconds>(timepoint.time_since_epoch()).count();
 }
 
+inline UInt64 time_in_microseconds(std::chrono::time_point<std::chrono::system_clock> timepoint)
+{
+    return std::chrono::duration_cast<std::chrono::microseconds>(timepoint.time_since_epoch()).count();
+}
 
 inline UInt64 time_in_seconds(std::chrono::time_point<std::chrono::system_clock> timepoint)
 {
@@ -102,6 +109,7 @@ void MetricLog::metricThreadFunction()
 
             MetricLogElement elem;
             elem.event_time = std::chrono::system_clock::to_time_t(current_time);
+            elem.event_time_microseconds = time_in_microseconds(current_time);
             elem.milliseconds = time_in_milliseconds(current_time) - time_in_seconds(current_time) * 1000;
 
             elem.profile_events.resize(ProfileEvents::end());

src/Interpreters/MetricLog.h

@@ -18,6 +18,7 @@ namespace DB
 struct MetricLogElement
 {
     time_t event_time{};
+    UInt64 event_time_microseconds{};
     UInt64 milliseconds{};
 
     std::vector<ProfileEvents::Count> profile_events;

src/Interpreters/QueryNormalizer.cpp

@@ -152,7 +152,7 @@ void QueryNormalizer::visitChildren(const ASTPtr & node, Data & data)
 {
     if (const auto * func_node = node->as<ASTFunction>())
     {
-        if (func_node->query)
+        if (func_node->tryGetQueryArgument())
         {
             if (func_node->name != "view")
                 throw Exception("Query argument can only be used in the `view` TableFunction", ErrorCodes::BAD_ARGUMENTS);

src/Interpreters/misc.h

@@ -28,7 +28,7 @@ inline bool functionIsLikeOperator(const std::string & name)
 
 inline bool functionIsJoinGet(const std::string & name)
 {
-    return name == "joinGet" || startsWith(name, "dictGet");
+    return startsWith(name, "joinGet");
 }
 
 inline bool functionIsDictGet(const std::string & name)

src/Parsers/ASTColumnsTransformers.cpp

@@ -110,7 +110,7 @@ void ASTColumnsReplaceTransformer::replaceChildren(ASTPtr & node, const ASTPtr &
         if (const auto * id = child->as<ASTIdentifier>())
         {
             if (id->shortName() == name)
-                child = replacement;
+                child = replacement->clone();
         }
         else
             replaceChildren(child, replacement, name);

src/Parsers/ASTFunction.cpp

@@ -48,7 +48,6 @@ ASTPtr ASTFunction::clone() const
     auto res = std::make_shared<ASTFunction>(*this);
     res->children.clear();
 
-    if (query) { res->query = query->clone(); res->children.push_back(res->query); }
     if (arguments) { res->arguments = arguments->clone(); res->children.push_back(res->arguments); }
     if (parameters) { res->parameters = parameters->clone(); res->children.push_back(res->parameters); }
 
@@ -112,6 +111,16 @@ static bool highlightStringLiteralWithMetacharacters(const ASTPtr & node, const
 }
 
 
+ASTSelectWithUnionQuery * ASTFunction::tryGetQueryArgument() const
+{
+    if (arguments && arguments->children.size() == 1)
+    {
+        return arguments->children[0]->as<ASTSelectWithUnionQuery>();
+    }
+    return nullptr;
+}
+
+
 void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
 {
     frame.expression_list_prepend_whitespace = false;
@@ -120,7 +129,7 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format
     nested_need_parens.need_parens = true;
     nested_dont_need_parens.need_parens = false;
 
-    if (query)
+    if (auto * query = tryGetQueryArgument())
     {
         std::string nl_or_nothing = settings.one_line ? "" : "\n";
         std::string indent_str = settings.one_line ? "" : std::string(4u * frame.indent, ' ');

src/Parsers/ASTFunction.h

@@ -2,6 +2,7 @@
 
 #include <Parsers/ASTWithAlias.h>
 #include <Parsers/ASTExpressionList.h>
+#include <Parsers/ASTSelectWithUnionQuery.h>
 
 
 namespace DB
@@ -13,7 +14,6 @@ class ASTFunction : public ASTWithAlias
 {
 public:
     String name;
-    ASTPtr query; // It's possible for a function to accept a query as its only argument.
     ASTPtr arguments;
     /// parameters - for parametric aggregate function. Example: quantile(0.9)(x) - what in first parens are 'parameters'.
     ASTPtr parameters;
@@ -26,6 +26,8 @@ public:
 
     void updateTreeHashImpl(SipHash & hash_state) const override;
 
+    ASTSelectWithUnionQuery * tryGetQueryArgument() const;
+
 protected:
     void formatImplWithoutAlias(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
     void appendColumnNameImpl(WriteBuffer & ostr) const override;

src/Parsers/ExpressionElementParsers.cpp

@@ -260,8 +260,10 @@ bool ParserFunction::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
                 ++pos;
                 auto function_node = std::make_shared<ASTFunction>();
                 tryGetIdentifierNameInto(identifier, function_node->name);
-                function_node->query = query;
+                auto expr_list_with_single_query = std::make_shared<ASTExpressionList>();
-                function_node->children.push_back(function_node->query);
+                expr_list_with_single_query->children.push_back(query);
+                function_node->arguments = expr_list_with_single_query;
+                function_node->children.push_back(function_node->arguments);
                 node = function_node;
                 return true;
             }

src/Parsers/ParserInsertQuery.cpp

@@ -36,7 +36,7 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
     ParserToken s_lparen(TokenType::OpeningRoundBracket);
     ParserToken s_rparen(TokenType::ClosingRoundBracket);
     ParserIdentifier name_p;
-    ParserList columns_p(std::make_unique<ParserCompoundIdentifier>(), std::make_unique<ParserToken>(TokenType::Comma), false);
+    ParserList columns_p(std::make_unique<ParserInsertElement>(), std::make_unique<ParserToken>(TokenType::Comma), false);
     ParserFunction table_function_p{false};
 
     ASTPtr database;
@@ -189,5 +189,12 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
     return true;
 }
 
+bool ParserInsertElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
+{
+    return ParserColumnsMatcher().parse(pos, node, expected)
+        || ParserQualifiedAsterisk().parse(pos, node, expected)
+        || ParserAsterisk().parse(pos, node, expected)
+        || ParserCompoundIdentifier().parse(pos, node, expected);
+}
 
 }

src/Parsers/ParserInsertQuery.h

@@ -33,4 +33,13 @@ public:
     ParserInsertQuery(const char * end_) : end(end_) {}
 };
 
+/** Insert accepts an identifier and an asterisk with variants.
+  */
+class ParserInsertElement : public IParserBase
+{
+protected:
+    const char * getName() const override { return "insert element"; }
+    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
+};
+
 }

src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.cpp

@@ -1,4 +1,5 @@
 #include <IO/ReadHelpers.h>
+#include <IO/ReadBufferFromString.h>
 
 #include <Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.h>
 #include <Formats/FormatFactory.h>
@@ -19,8 +20,9 @@ JSONCompactEachRowRowInputFormat::JSONCompactEachRowRowInputFormat(ReadBuffer &
         const Block & header_,
         Params params_,
         const FormatSettings & format_settings_,
-        bool with_names_)
+        bool with_names_,
-        : IRowInputFormat(header_, in_, std::move(params_)), format_settings(format_settings_), with_names(with_names_)
+        bool yield_strings_)
+        : IRowInputFormat(header_, in_, std::move(params_)), format_settings(format_settings_), with_names(with_names_), yield_strings(yield_strings_)
 {
     const auto & sample = getPort().getHeader();
     size_t num_columns = sample.columns();
@@ -200,10 +202,26 @@ void JSONCompactEachRowRowInputFormat::readField(size_t index, MutableColumns &
     {
         read_columns[index] = true;
         const auto & type = data_types[index];
-        if (format_settings.null_as_default && !type->isNullable())
+
-            read_columns[index] = DataTypeNullable::deserializeTextJSON(*columns[index], in, format_settings, type);
+        if (yield_strings)
+        {
+            String str;
+            readJSONString(str, in);
+
+            ReadBufferFromString buf(str);
+
+            if (format_settings.null_as_default && !type->isNullable())
+                read_columns[index] = DataTypeNullable::deserializeWholeText(*columns[index], buf, format_settings, type);
+            else
+                type->deserializeAsWholeText(*columns[index], buf, format_settings);
+        }
         else
-            type->deserializeAsTextJSON(*columns[index], in, format_settings);
+        {
+            if (format_settings.null_as_default && !type->isNullable())
+                read_columns[index] = DataTypeNullable::deserializeTextJSON(*columns[index], in, format_settings, type);
+            else
+                type->deserializeAsTextJSON(*columns[index], in, format_settings);
+        }
     }
     catch (Exception & e)
     {
@@ -225,7 +243,7 @@ void registerInputFormatProcessorJSONCompactEachRow(FormatFactory & factory)
             IRowInputFormat::Params params,
             const FormatSettings & settings)
     {
-        return std::make_shared<JSONCompactEachRowRowInputFormat>(buf, sample, std::move(params), settings, false);
+        return std::make_shared<JSONCompactEachRowRowInputFormat>(buf, sample, std::move(params), settings, false, false);
     });
 
     factory.registerInputFormatProcessor("JSONCompactEachRowWithNamesAndTypes", [](
@@ -234,7 +252,25 @@ void registerInputFormatProcessorJSONCompactEachRow(FormatFactory & factory)
             IRowInputFormat::Params params,
             const FormatSettings & settings)
     {
-        return std::make_shared<JSONCompactEachRowRowInputFormat>(buf, sample, std::move(params), settings, true);
+        return std::make_shared<JSONCompactEachRowRowInputFormat>(buf, sample, std::move(params), settings, true, false);
+    });
+
+    factory.registerInputFormatProcessor("JSONCompactStringsEachRow", [](
+            ReadBuffer & buf,
+            const Block & sample,
+            IRowInputFormat::Params params,
+            const FormatSettings & settings)
+    {
+        return std::make_shared<JSONCompactEachRowRowInputFormat>(buf, sample, std::move(params), settings, false, true);
+    });
+
+    factory.registerInputFormatProcessor("JSONCompactStringsEachRowWithNamesAndTypes", [](
+            ReadBuffer & buf,
+            const Block & sample,
+            IRowInputFormat::Params params,
+            const FormatSettings & settings)
+    {
+        return std::make_shared<JSONCompactEachRowRowInputFormat>(buf, sample, std::move(params), settings, true, true);
     });
 }
 

src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.h

@@ -1,7 +1,5 @@
 #pragma once
 
-#pragma once
-
 #include <Core/Block.h>
 #include <Processors/Formats/IRowInputFormat.h>
 #include <Formats/FormatSettings.h>
@@ -12,12 +10,23 @@ namespace DB
 
 class ReadBuffer;
 
-/** A stream for reading data in JSONCompactEachRow and JSONCompactEachRowWithNamesAndTypes formats
+/** A stream for reading data in a bunch of formats:
+ *  - JSONCompactEachRow
+ *  - JSONCompactEachRowWithNamesAndTypes
+ *  - JSONCompactStringsEachRow
+ *  - JSONCompactStringsEachRowWithNamesAndTypes
+ *
 */
 class JSONCompactEachRowRowInputFormat : public IRowInputFormat
 {
 public:
-    JSONCompactEachRowRowInputFormat(ReadBuffer & in_, const Block & header_, Params params_, const FormatSettings & format_settings_, bool with_names_);
+    JSONCompactEachRowRowInputFormat(
+        ReadBuffer & in_,
+        const Block & header_,
+        Params params_,
+        const FormatSettings & format_settings_,
+        bool with_names_,
+        bool yield_strings_);
 
     String getName() const override { return "JSONCompactEachRowRowInputFormat"; }
 
@@ -48,7 +57,10 @@ private:
     /// This is for the correct exceptions in skipping unknown fields.
     std::vector<String> names_of_columns;
 
+    /// For *WithNamesAndTypes formats.
     bool with_names;
+    /// For JSONCompactString* formats.
+    bool yield_strings;
 };
 
 }