Merge branch 'master' of github.com:ClickHouse/ClickHouse into divanik/generate_series_function

CMakeLists.txt

@@ -61,11 +61,16 @@ if (ENABLE_CHECK_HEAVY_BUILDS)
     # set CPU time limit to 1000 seconds
     set (RLIMIT_CPU 1000)
 
-    # -fsanitize=memory and address are too heavy
+    # Sanitizers are too heavy
     if (SANITIZE OR SANITIZE_COVERAGE OR WITH_COVERAGE)
        set (RLIMIT_DATA 10000000000) # 10G
     endif()
 
+    # For some files currently building RISCV64 might be too slow. TODO: Improve compilation times per file
+    if (ARCH_RISCV64)
+        set (RLIMIT_CPU 1800)
+    endif()
+
     set (CMAKE_CXX_COMPILER_LAUNCHER prlimit --as=${RLIMIT_AS} --data=${RLIMIT_DATA} --cpu=${RLIMIT_CPU} ${CMAKE_CXX_COMPILER_LAUNCHER})
 endif ()
 

base/base/CMakeLists.txt

@@ -20,6 +20,7 @@ set (SRCS
     getPageSize.cpp
     getThreadId.cpp
     int8_to_string.cpp
+    itoa.cpp
     JSON.cpp
     mremap.cpp
     phdr_cache.cpp

base/base/IPv4andIPv6.h

@@ -1,8 +1,7 @@
 #pragma once
 
-#include <base/strong_typedef.h>
 #include <base/extended_types.h>
-#include <Common/formatIPv6.h>
+#include <base/strong_typedef.h>
 #include <Common/memcmpSmall.h>
 
 namespace DB
@@ -62,7 +61,8 @@ namespace std
     {
         size_t operator()(const DB::IPv6 & x) const
         {
-            return std::hash<std::string_view>{}(std::string_view(reinterpret_cast<const char*>(&x.toUnderType()), IPV6_BINARY_LENGTH));
+            return std::hash<std::string_view>{}(
+                std::string_view(reinterpret_cast<const char *>(&x.toUnderType()), sizeof(DB::IPv6::UnderlyingType)));
         }
     };
 

base/base/itoa.cpp

@@ -0,0 +1,503 @@
+// Based on https://github.com/amdn/itoa and combined with our optimizations
+//
+//=== itoa.cpp - Fast integer to ascii conversion                 --*- C++ -*-//
+//
+// The MIT License (MIT)
+// Copyright (c) 2016 Arturo Martin-de-Nicolas
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+//     The above copyright notice and this permission notice shall be included
+//     in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+//===----------------------------------------------------------------------===//
+
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <type_traits>
+#include <base/defines.h>
+#include <base/extended_types.h>
+#include <base/itoa.h>
+
+namespace
+{
+template <typename T>
+ALWAYS_INLINE inline constexpr T pow10(size_t x)
+{
+    return x ? 10 * pow10<T>(x - 1) : 1;
+}
+
+// Division by a power of 10 is implemented using a multiplicative inverse.
+// This strength reduction is also done by optimizing compilers, but
+// presently the fastest results are produced by using the values
+// for the multiplication and the shift as given by the algorithm
+// described by Agner Fog in "Optimizing Subroutines in Assembly Language"
+//
+// http://www.agner.org/optimize/optimizing_assembly.pdf
+//
+// "Integer division by a constant (all processors)
+// A floating point number can be divided by a constant by multiplying
+// with the reciprocal. If we want to do the same with integers, we have
+// to scale the reciprocal by 2n and then shift the product to the right
+// by n. There are various algorithms for finding a suitable value of n
+// and compensating for rounding errors. The algorithm described below
+// was invented by Terje Mathisen, Norway, and not published elsewhere."
+
+/// Division by constant is performed by:
+/// 1. Adding 1 if needed;
+/// 2. Multiplying by another constant;
+/// 3. Shifting right by another constant.
+template <typename UInt, bool add_, UInt multiplier_, unsigned shift_>
+struct Division
+{
+    static constexpr bool add{add_};
+    static constexpr UInt multiplier{multiplier_};
+    static constexpr unsigned shift{shift_};
+};
+
+/// Select a type with appropriate number of bytes from the list of types.
+/// First parameter is the number of bytes requested. Then goes a list of types with 1, 2, 4, ... number of bytes.
+/// Example: SelectType<4, uint8_t, uint16_t, uint32_t, uint64_t> will select uint32_t.
+template <size_t N, typename T, typename... Ts>
+struct SelectType
+{
+    using Result = typename SelectType<N / 2, Ts...>::Result;
+};
+
+template <typename T, typename... Ts>
+struct SelectType<1, T, Ts...>
+{
+    using Result = T;
+};
+
+
+/// Division by 10^N where N is the size of the type.
+template <size_t N>
+using DivisionBy10PowN = typename SelectType<
+    N,
+    Division<uint8_t, false, 205U, 11>, /// divide by 10
+    Division<uint16_t, true, 41943U, 22>, /// divide by 100
+    Division<uint32_t, false, 3518437209U, 45>, /// divide by 10000
+    Division<uint64_t, false, 12379400392853802749ULL, 90> /// divide by 100000000
+    >::Result;
+
+template <size_t N>
+using UnsignedOfSize = typename SelectType<N, uint8_t, uint16_t, uint32_t, uint64_t, __uint128_t>::Result;
+
+/// Holds the result of dividing an unsigned N-byte variable by 10^N resulting in
+template <size_t N>
+struct QuotientAndRemainder
+{
+    UnsignedOfSize<N> quotient; // quotient with fewer than 2*N decimal digits
+    UnsignedOfSize<N / 2> remainder; // remainder with at most N decimal digits
+};
+
+template <size_t N>
+QuotientAndRemainder<N> inline split(UnsignedOfSize<N> value)
+{
+    constexpr DivisionBy10PowN<N> division;
+
+    UnsignedOfSize<N> quotient = (division.multiplier * (UnsignedOfSize<2 * N>(value) + division.add)) >> division.shift;
+    UnsignedOfSize<N / 2> remainder = static_cast<UnsignedOfSize<N / 2>>(value - quotient * pow10<UnsignedOfSize<N / 2>>(N));
+
+    return {quotient, remainder};
+}
+
+ALWAYS_INLINE inline char * outDigit(char * p, uint8_t value)
+{
+    *p = '0' + value;
+    ++p;
+    return p;
+}
+
+// Using a lookup table to convert binary numbers from 0 to 99
+// into ascii characters as described by Andrei Alexandrescu in
+// https://www.facebook.com/notes/facebook-engineering/three-optimization-tips-for-c/10151361643253920/
+
+const char digits[201] = "00010203040506070809"
+                         "10111213141516171819"
+                         "20212223242526272829"
+                         "30313233343536373839"
+                         "40414243444546474849"
+                         "50515253545556575859"
+                         "60616263646566676869"
+                         "70717273747576777879"
+                         "80818283848586878889"
+                         "90919293949596979899";
+
+ALWAYS_INLINE inline char * outTwoDigits(char * p, uint8_t value)
+{
+    memcpy(p, &digits[value * 2], 2);
+    p += 2;
+    return p;
+}
+
+namespace convert
+{
+template <typename UInt, size_t N = sizeof(UInt)>
+char * head(char * p, UInt u);
+template <typename UInt, size_t N = sizeof(UInt)>
+char * tail(char * p, UInt u);
+
+//===----------------------------------------------------------===//
+//     head: find most significant digit, skip leading zeros
+//===----------------------------------------------------------===//
+
+// "x" contains quotient and remainder after division by 10^N
+// quotient is less than 10^N
+template <size_t N>
+ALWAYS_INLINE inline char * head(char * p, QuotientAndRemainder<N> x)
+{
+    p = head(p, UnsignedOfSize<N / 2>(x.quotient));
+    p = tail(p, x.remainder);
+    return p;
+}
+
+// "u" is less than 10^2*N
+template <typename UInt, size_t N>
+ALWAYS_INLINE inline char * head(char * p, UInt u)
+{
+    return u < pow10<UnsignedOfSize<N>>(N) ? head(p, UnsignedOfSize<N / 2>(u)) : head<N>(p, split<N>(u));
+}
+
+// recursion base case, selected when "u" is one byte
+template <>
+ALWAYS_INLINE inline char * head<UnsignedOfSize<1>, 1>(char * p, UnsignedOfSize<1> u)
+{
+    return u < 10 ? outDigit(p, u) : outTwoDigits(p, u);
+}
+
+//===----------------------------------------------------------===//
+//     tail: produce all digits including leading zeros
+//===----------------------------------------------------------===//
+
+// recursive step, "u" is less than 10^2*N
+template <typename UInt, size_t N>
+ALWAYS_INLINE inline char * tail(char * p, UInt u)
+{
+    QuotientAndRemainder<N> x = split<N>(u);
+    p = tail(p, UnsignedOfSize<N / 2>(x.quotient));
+    p = tail(p, x.remainder);
+    return p;
+}
+
+// recursion base case, selected when "u" is one byte
+template <>
+ALWAYS_INLINE inline char * tail<UnsignedOfSize<1>, 1>(char * p, UnsignedOfSize<1> u)
+{
+    return outTwoDigits(p, u);
+}
+
+//===----------------------------------------------------------===//
+// large values are >= 10^2*N
+// where x contains quotient and remainder after division by 10^N
+//===----------------------------------------------------------===//
+template <size_t N>
+ALWAYS_INLINE inline char * large(char * p, QuotientAndRemainder<N> x)
+{
+    QuotientAndRemainder<N> y = split<N>(x.quotient);
+    p = head(p, UnsignedOfSize<N / 2>(y.quotient));
+    p = tail(p, y.remainder);
+    p = tail(p, x.remainder);
+    return p;
+}
+
+//===----------------------------------------------------------===//
+// handle values of "u" that might be >= 10^2*N
+// where N is the size of "u" in bytes
+//===----------------------------------------------------------===//
+template <typename UInt, size_t N = sizeof(UInt)>
+ALWAYS_INLINE inline char * uitoa(char * p, UInt u)
+{
+    if (u < pow10<UnsignedOfSize<N>>(N))
+        return head(p, UnsignedOfSize<N / 2>(u));
+    QuotientAndRemainder<N> x = split<N>(u);
+
+    return u < pow10<UnsignedOfSize<N>>(2 * N) ? head<N>(p, x) : large<N>(p, x);
+}
+
+// selected when "u" is one byte
+template <>
+ALWAYS_INLINE inline char * uitoa<UnsignedOfSize<1>, 1>(char * p, UnsignedOfSize<1> u)
+{
+    if (u < 10)
+        return outDigit(p, u);
+    else if (u < 100)
+        return outTwoDigits(p, u);
+    else
+    {
+        p = outDigit(p, u / 100);
+        p = outTwoDigits(p, u % 100);
+        return p;
+    }
+}
+
+//===----------------------------------------------------------===//
+//     handle unsigned and signed integral operands
+//===----------------------------------------------------------===//
+
+// itoa: handle unsigned integral operands (selected by SFINAE)
+template <typename U, std::enable_if_t<!std::is_signed_v<U> && std::is_integral_v<U>> * = nullptr>
+ALWAYS_INLINE inline char * itoa(U u, char * p)
+{
+    return convert::uitoa(p, u);
+}
+
+// itoa: handle signed integral operands (selected by SFINAE)
+template <typename I, size_t N = sizeof(I), std::enable_if_t<std::is_signed_v<I> && std::is_integral_v<I>> * = nullptr>
+ALWAYS_INLINE inline char * itoa(I i, char * p)
+{
+    // Need "mask" to be filled with a copy of the sign bit.
+    // If "i" is a negative value, then the result of "operator >>"
+    // is implementation-defined, though usually it is an arithmetic
+    // right shift that replicates the sign bit.
+    // Use a conditional expression to be portable,
+    // a good optimizing compiler generates an arithmetic right shift
+    // and avoids the conditional branch.
+    UnsignedOfSize<N> mask = i < 0 ? ~UnsignedOfSize<N>(0) : 0;
+    // Now get the absolute value of "i" and cast to unsigned type UnsignedOfSize<N>.
+    // Cannot use std::abs() because the result is undefined
+    // in 2's complement systems for the most-negative value.
+    // Want to avoid conditional branch for performance reasons since
+    // CPU branch prediction will be ineffective when negative values
+    // occur randomly.
+    // Let "u" be "i" cast to unsigned type UnsignedOfSize<N>.
+    // Subtract "u" from 2*u if "i" is positive or 0 if "i" is negative.
+    // This yields the absolute value with the desired type without
+    // using a conditional branch and without invoking undefined or
+    // implementation defined behavior:
+    UnsignedOfSize<N> u = ((2 * UnsignedOfSize<N>(i)) & ~mask) - UnsignedOfSize<N>(i);
+    // Unconditionally store a minus sign when producing digits
+    // in a forward direction and increment the pointer only if
+    // the value is in fact negative.
+    // This avoids a conditional branch and is safe because we will
+    // always produce at least one digit and it will overwrite the
+    // minus sign when the value is not negative.
+    *p = '-';
+    p += (mask & 1);
+    p = convert::uitoa(p, u);
+    return p;
+}
+}
+
+const uint64_t max_multiple_of_hundred_that_fits_in_64_bits = 1'00'00'00'00'00'00'00'00'00ull;
+const int max_multiple_of_hundred_blocks = 9;
+static_assert(max_multiple_of_hundred_that_fits_in_64_bits % 100 == 0);
+
+ALWAYS_INLINE inline char * writeUIntText(UInt128 _x, char * p)
+{
+    /// If we the highest 64bit item is empty, we can print just the lowest item as u64
+    if (_x.items[UInt128::_impl::little(1)] == 0)
+        return convert::itoa(_x.items[UInt128::_impl::little(0)], p);
+
+    /// Doing operations using __int128 is faster and we already rely on this feature
+    using T = unsigned __int128;
+    T x = (T(_x.items[UInt128::_impl::little(1)]) << 64) + T(_x.items[UInt128::_impl::little(0)]);
+
+    /// We are going to accumulate blocks of 2 digits to print until the number is small enough to be printed as u64
+    /// To do this we could do: x / 100, x % 100
+    /// But these would mean doing many iterations with long integers, so instead we divide by a much longer integer
+    /// multiple of 100 (100^9) and then get the blocks out of it (as u64)
+    /// Once we reach u64::max we can stop and use the fast method to print that in the front
+    static const T large_divisor = max_multiple_of_hundred_that_fits_in_64_bits;
+    static const T largest_uint64 = std::numeric_limits<uint64_t>::max();
+    uint8_t two_values[20] = {0}; // 39 Max characters / 2
+
+    int current_block = 0;
+    while (x > largest_uint64)
+    {
+        uint64_t u64_remainder = uint64_t(x % large_divisor);
+        x /= large_divisor;
+
+        int pos = current_block;
+        while (u64_remainder)
+        {
+            two_values[pos] = uint8_t(u64_remainder % 100);
+            pos++;
+            u64_remainder /= 100;
+        }
+        current_block += max_multiple_of_hundred_blocks;
+    }
+
+    char * highest_part_print = convert::itoa(uint64_t(x), p);
+    for (int i = 0; i < current_block; i++)
+    {
+        outTwoDigits(highest_part_print, two_values[current_block - 1 - i]);
+        highest_part_print += 2;
+    }
+
+    return highest_part_print;
+}
+
+ALWAYS_INLINE inline char * writeUIntText(UInt256 _x, char * p)
+{
+    /// If possible, treat it as a smaller integer as they are much faster to print
+    if (_x.items[UInt256::_impl::little(3)] == 0 && _x.items[UInt256::_impl::little(2)] == 0)
+        return writeUIntText(UInt128{_x.items[UInt256::_impl::little(0)], _x.items[UInt256::_impl::little(1)]}, p);
+
+    /// If available (x86) we transform from our custom class to _BitInt(256) which has better support in the compiler
+    /// and produces better code
+    using T =
+#if defined(__x86_64__)
+#    pragma clang diagnostic push
+#    pragma clang diagnostic ignored "-Wbit-int-extension"
+        unsigned _BitInt(256)
+#    pragma clang diagnostic pop
+#else
+        UInt256
+#endif
+        ;
+
+#if defined(__x86_64__)
+    T x = (T(_x.items[UInt256::_impl::little(3)]) << 192) + (T(_x.items[UInt256::_impl::little(2)]) << 128)
+        + (T(_x.items[UInt256::_impl::little(1)]) << 64) + T(_x.items[UInt256::_impl::little(0)]);
+#else
+    T x = _x;
+#endif
+
+    /// Similar to writeUIntText(UInt128) only that in this case we will stop as soon as we reach the largest u128
+    /// and switch to that function
+    uint8_t two_values[39] = {0}; // 78 Max characters / 2
+    int current_pos = 0;
+
+    static const T large_divisor = max_multiple_of_hundred_that_fits_in_64_bits;
+    static const T largest_uint128 = T(std::numeric_limits<uint64_t>::max()) << 64 | T(std::numeric_limits<uint64_t>::max());
+
+    while (x > largest_uint128)
+    {
+        uint64_t u64_remainder = uint64_t(x % large_divisor);
+        x /= large_divisor;
+
+        int pos = current_pos;
+        while (u64_remainder)
+        {
+            two_values[pos] = uint8_t(u64_remainder % 100);
+            pos++;
+            u64_remainder /= 100;
+        }
+        current_pos += max_multiple_of_hundred_blocks;
+    }
+
+#if defined(__x86_64__)
+    UInt128 pending{uint64_t(x), uint64_t(x >> 64)};
+#else
+    UInt128 pending{x.items[UInt256::_impl::little(0)], x.items[UInt256::_impl::little(1)]};
+#endif
+
+    char * highest_part_print = writeUIntText(pending, p);
+    for (int i = 0; i < current_pos; i++)
+    {
+        outTwoDigits(highest_part_print, two_values[current_pos - 1 - i]);
+        highest_part_print += 2;
+    }
+
+    return highest_part_print;
+}
+
+ALWAYS_INLINE inline char * writeLeadingMinus(char * pos)
+{
+    *pos = '-';
+    return pos + 1;
+}
+
+template <typename T>
+ALWAYS_INLINE inline char * writeSIntText(T x, char * pos)
+{
+    static_assert(std::is_same_v<T, Int128> || std::is_same_v<T, Int256>);
+
+    using UnsignedT = make_unsigned_t<T>;
+    static constexpr T min_int = UnsignedT(1) << (sizeof(T) * 8 - 1);
+
+    if (unlikely(x == min_int))
+    {
+        if constexpr (std::is_same_v<T, Int128>)
+        {
+            const char * res = "-170141183460469231731687303715884105728";
+            memcpy(pos, res, strlen(res));
+            return pos + strlen(res);
+        }
+        else if constexpr (std::is_same_v<T, Int256>)
+        {
+            const char * res = "-57896044618658097711785492504343953926634992332820282019728792003956564819968";
+            memcpy(pos, res, strlen(res));
+            return pos + strlen(res);
+        }
+    }
+
+    if (x < 0)
+    {
+        x = -x;
+        pos = writeLeadingMinus(pos);
+    }
+    return writeUIntText(UnsignedT(x), pos);
+}
+}
+
+char * itoa(UInt8 i, char * p)
+{
+    return convert::itoa(uint8_t(i), p);
+}
+
+char * itoa(Int8 i, char * p)
+{
+    return convert::itoa(int8_t(i), p);
+}
+
+char * itoa(UInt128 i, char * p)
+{
+    return writeUIntText(i, p);
+}
+
+char * itoa(Int128 i, char * p)
+{
+    return writeSIntText(i, p);
+}
+
+char * itoa(UInt256 i, char * p)
+{
+    return writeUIntText(i, p);
+}
+
+char * itoa(Int256 i, char * p)
+{
+    return writeSIntText(i, p);
+}
+
+#define DEFAULT_ITOA(T) \
+    char * itoa(T i, char * p) \
+    { \
+        return convert::itoa(i, p); \
+    }
+
+#define FOR_MISSING_INTEGER_TYPES(M) \
+    M(uint8_t) \
+    M(UInt16) \
+    M(UInt32) \
+    M(UInt64) \
+    M(int8_t) \
+    M(Int16) \
+    M(Int32) \
+    M(Int64)
+
+FOR_MISSING_INTEGER_TYPES(DEFAULT_ITOA)
+
+#if defined(OS_DARWIN)
+DEFAULT_ITOA(unsigned long)
+DEFAULT_ITOA(long)
+#endif
+
+#undef FOR_MISSING_INTEGER_TYPES
+#undef DEFAULT_ITOA

base/base/itoa.h

@@ -1,446 +1,30 @@
 #pragma once
 
-// Based on https://github.com/amdn/itoa and combined with our optimizations
-//
-//=== itoa.h - Fast integer to ascii conversion                   --*- C++ -*-//
-//
-// The MIT License (MIT)
-// Copyright (c) 2016 Arturo Martin-de-Nicolas
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-//     The above copyright notice and this permission notice shall be included
-//     in all copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
-//===----------------------------------------------------------------------===//
-
-#include <cstdint>
-#include <cstddef>
-#include <cstring>
-#include <type_traits>
 #include <base/extended_types.h>
 
+#define FOR_INTEGER_TYPES(M) \
+    M(uint8_t) \
+    M(UInt8) \
+    M(UInt16) \
+    M(UInt32) \
+    M(UInt64) \
+    M(UInt128) \
+    M(UInt256) \
+    M(int8_t) \
+    M(Int8) \
+    M(Int16) \
+    M(Int32) \
+    M(Int64) \
+    M(Int128) \
+    M(Int256)
 
-template <typename T>
+#define INSTANTIATION(T) char * itoa(T i, char * p);
-inline int digits10(T x)
+FOR_INTEGER_TYPES(INSTANTIATION)
-{
-    if (x < 10ULL)
-        return 1;
-    if (x < 100ULL)
-        return 2;
-    if (x < 1000ULL)
-        return 3;
 
-    if (x < 1000000000000ULL)
+#if defined(OS_DARWIN)
-    {
+INSTANTIATION(unsigned long)
-        if (x < 100000000ULL)
+INSTANTIATION(long)
-        {
+#endif
-            if (x < 1000000ULL)
-            {
-                if (x < 10000ULL)
-                    return 4;
-                else
-                    return 5 + (x >= 100000ULL);
-            }
 
-            return 7 + (x >= 10000000ULL);
+#undef FOR_INTEGER_TYPES
-        }
+#undef INSTANTIATION
-
-        if (x < 10000000000ULL)
-            return 9 + (x >= 1000000000ULL);
-
-        return 11 + (x >= 100000000000ULL);
-    }
-
-    return 12 + digits10(x / 1000000000000ULL);
-}
-
-
-namespace impl
-{
-
-template <typename T>
-static constexpr T pow10(size_t x)
-{
-    return x ? 10 * pow10<T>(x - 1) : 1;
-}
-
-// Division by a power of 10 is implemented using a multiplicative inverse.
-// This strength reduction is also done by optimizing compilers, but
-// presently the fastest results are produced by using the values
-// for the multiplication and the shift as given by the algorithm
-// described by Agner Fog in "Optimizing Subroutines in Assembly Language"
-//
-// http://www.agner.org/optimize/optimizing_assembly.pdf
-//
-// "Integer division by a constant (all processors)
-// A floating point number can be divided by a constant by multiplying
-// with the reciprocal. If we want to do the same with integers, we have
-// to scale the reciprocal by 2n and then shift the product to the right
-// by n. There are various algorithms for finding a suitable value of n
-// and compensating for rounding errors. The algorithm described below
-// was invented by Terje Mathisen, Norway, and not published elsewhere."
-
-/// Division by constant is performed by:
-/// 1. Adding 1 if needed;
-/// 2. Multiplying by another constant;
-/// 3. Shifting right by another constant.
-template <typename UInt, bool add_, UInt multiplier_, unsigned shift_>
-struct Division
-{
-    static constexpr bool add{add_};
-    static constexpr UInt multiplier{multiplier_};
-    static constexpr unsigned shift{shift_};
-};
-
-/// Select a type with appropriate number of bytes from the list of types.
-/// First parameter is the number of bytes requested. Then goes a list of types with 1, 2, 4, ... number of bytes.
-/// Example: SelectType<4, uint8_t, uint16_t, uint32_t, uint64_t> will select uint32_t.
-template <size_t N, typename T, typename... Ts>
-struct SelectType
-{
-    using Result = typename SelectType<N / 2, Ts...>::Result;
-};
-
-template <typename T, typename... Ts>
-struct SelectType<1, T, Ts...>
-{
-    using Result = T;
-};
-
-
-/// Division by 10^N where N is the size of the type.
-template <size_t N>
-using DivisionBy10PowN = typename SelectType
-<
-    N,
-    Division<uint8_t, false, 205U, 11>,                           /// divide by 10
-    Division<uint16_t, true, 41943U, 22>,                         /// divide by 100
-    Division<uint32_t, false, 3518437209U, 45>,                   /// divide by 10000
-    Division<uint64_t, false, 12379400392853802749ULL, 90>        /// divide by 100000000
->::Result;
-
-template <size_t N>
-using UnsignedOfSize = typename SelectType
-<
-    N,
-    uint8_t,
-    uint16_t,
-    uint32_t,
-    uint64_t,
-    __uint128_t
->::Result;
-
-/// Holds the result of dividing an unsigned N-byte variable by 10^N resulting in
-template <size_t N>
-struct QuotientAndRemainder
-{
-    UnsignedOfSize<N> quotient; // quotient with fewer than 2*N decimal digits
-    UnsignedOfSize<N / 2> remainder; // remainder with at most N decimal digits
-};
-
-template <size_t N>
-QuotientAndRemainder<N> static inline split(UnsignedOfSize<N> value)
-{
-    constexpr DivisionBy10PowN<N> division;
-
-    UnsignedOfSize<N> quotient = (division.multiplier * (UnsignedOfSize<2 * N>(value) + division.add)) >> division.shift;
-    UnsignedOfSize<N / 2> remainder = static_cast<UnsignedOfSize<N / 2>>(value - quotient * pow10<UnsignedOfSize<N / 2>>(N));
-
-    return {quotient, remainder};
-}
-
-
-static inline char * outDigit(char * p, uint8_t value)
-{
-    *p = '0' + value;
-    ++p;
-    return p;
-}
-
-// Using a lookup table to convert binary numbers from 0 to 99
-// into ascii characters as described by Andrei Alexandrescu in
-// https://www.facebook.com/notes/facebook-engineering/three-optimization-tips-for-c/10151361643253920/
-
-static const char digits[201] = "00010203040506070809"
-                                "10111213141516171819"
-                                "20212223242526272829"
-                                "30313233343536373839"
-                                "40414243444546474849"
-                                "50515253545556575859"
-                                "60616263646566676869"
-                                "70717273747576777879"
-                                "80818283848586878889"
-                                "90919293949596979899";
-
-static inline char * outTwoDigits(char * p, uint8_t value)
-{
-    memcpy(p, &digits[value * 2], 2);
-    p += 2;
-    return p;
-}
-
-
-namespace convert
-{
-    template <typename UInt, size_t N = sizeof(UInt)> static char * head(char * p, UInt u);
-    template <typename UInt, size_t N = sizeof(UInt)> static char * tail(char * p, UInt u);
-
-    //===----------------------------------------------------------===//
-    //     head: find most significant digit, skip leading zeros
-    //===----------------------------------------------------------===//
-
-    // "x" contains quotient and remainder after division by 10^N
-    // quotient is less than 10^N
-    template <size_t N>
-    static inline char * head(char * p, QuotientAndRemainder<N> x)
-    {
-        p = head(p, UnsignedOfSize<N / 2>(x.quotient));
-        p = tail(p, x.remainder);
-        return p;
-    }
-
-    // "u" is less than 10^2*N
-    template <typename UInt, size_t N>
-    static inline char * head(char * p, UInt u)
-    {
-        return u < pow10<UnsignedOfSize<N>>(N)
-            ? head(p, UnsignedOfSize<N / 2>(u))
-            : head<N>(p, split<N>(u));
-    }
-
-    // recursion base case, selected when "u" is one byte
-    template <>
-    inline char * head<UnsignedOfSize<1>, 1>(char * p, UnsignedOfSize<1> u)
-    {
-        return u < 10
-            ? outDigit(p, u)
-            : outTwoDigits(p, u);
-    }
-
-    //===----------------------------------------------------------===//
-    //     tail: produce all digits including leading zeros
-    //===----------------------------------------------------------===//
-
-    // recursive step, "u" is less than 10^2*N
-    template <typename UInt, size_t N>
-    static inline char * tail(char * p, UInt u)
-    {
-        QuotientAndRemainder<N> x = split<N>(u);
-        p = tail(p, UnsignedOfSize<N / 2>(x.quotient));
-        p = tail(p, x.remainder);
-        return p;
-    }
-
-    // recursion base case, selected when "u" is one byte
-    template <>
-    inline char * tail<UnsignedOfSize<1>, 1>(char * p, UnsignedOfSize<1> u)
-    {
-        return outTwoDigits(p, u);
-    }
-
-    //===----------------------------------------------------------===//
-    // large values are >= 10^2*N
-    // where x contains quotient and remainder after division by 10^N
-    //===----------------------------------------------------------===//
-
-    template <size_t N>
-    static inline char * large(char * p, QuotientAndRemainder<N> x)
-    {
-        QuotientAndRemainder<N> y = split<N>(x.quotient);
-        p = head(p, UnsignedOfSize<N / 2>(y.quotient));
-        p = tail(p, y.remainder);
-        p = tail(p, x.remainder);
-        return p;
-    }
-
-    //===----------------------------------------------------------===//
-    // handle values of "u" that might be >= 10^2*N
-    // where N is the size of "u" in bytes
-    //===----------------------------------------------------------===//
-
-    template <typename UInt, size_t N = sizeof(UInt)>
-    static inline char * uitoa(char * p, UInt u)
-    {
-        if (u < pow10<UnsignedOfSize<N>>(N))
-            return head(p, UnsignedOfSize<N / 2>(u));
-        QuotientAndRemainder<N> x = split<N>(u);
-
-        return u < pow10<UnsignedOfSize<N>>(2 * N)
-            ? head<N>(p, x)
-            : large<N>(p, x);
-    }
-
-    // selected when "u" is one byte
-    template <>
-    inline char * uitoa<UnsignedOfSize<1>, 1>(char * p, UnsignedOfSize<1> u)
-    {
-        if (u < 10)
-            return outDigit(p, u);
-        else if (u < 100)
-            return outTwoDigits(p, u);
-        else
-        {
-            p = outDigit(p, u / 100);
-            p = outTwoDigits(p, u % 100);
-            return p;
-        }
-    }
-
-    //===----------------------------------------------------------===//
-    //     handle unsigned and signed integral operands
-    //===----------------------------------------------------------===//
-
-    // itoa: handle unsigned integral operands (selected by SFINAE)
-    template <typename U, std::enable_if_t<!std::is_signed_v<U> && std::is_integral_v<U>> * = nullptr>
-    static inline char * itoa(U u, char * p)
-    {
-        return convert::uitoa(p, u);
-    }
-
-    // itoa: handle signed integral operands (selected by SFINAE)
-    template <typename I, size_t N = sizeof(I), std::enable_if_t<std::is_signed_v<I> && std::is_integral_v<I>> * = nullptr>
-    static inline char * itoa(I i, char * p)
-    {
-        // Need "mask" to be filled with a copy of the sign bit.
-        // If "i" is a negative value, then the result of "operator >>"
-        // is implementation-defined, though usually it is an arithmetic
-        // right shift that replicates the sign bit.
-        // Use a conditional expression to be portable,
-        // a good optimizing compiler generates an arithmetic right shift
-        // and avoids the conditional branch.
-        UnsignedOfSize<N> mask = i < 0 ? ~UnsignedOfSize<N>(0) : 0;
-        // Now get the absolute value of "i" and cast to unsigned type UnsignedOfSize<N>.
-        // Cannot use std::abs() because the result is undefined
-        // in 2's complement systems for the most-negative value.
-        // Want to avoid conditional branch for performance reasons since
-        // CPU branch prediction will be ineffective when negative values
-        // occur randomly.
-        // Let "u" be "i" cast to unsigned type UnsignedOfSize<N>.
-        // Subtract "u" from 2*u if "i" is positive or 0 if "i" is negative.
-        // This yields the absolute value with the desired type without
-        // using a conditional branch and without invoking undefined or
-        // implementation defined behavior:
-        UnsignedOfSize<N> u = ((2 * UnsignedOfSize<N>(i)) & ~mask) - UnsignedOfSize<N>(i);
-        // Unconditionally store a minus sign when producing digits
-        // in a forward direction and increment the pointer only if
-        // the value is in fact negative.
-        // This avoids a conditional branch and is safe because we will
-        // always produce at least one digit and it will overwrite the
-        // minus sign when the value is not negative.
-        *p = '-';
-        p += (mask & 1);
-        p = convert::uitoa(p, u);
-        return p;
-    }
-}
-
-
-template <typename T>
-static inline char * writeUIntText(T x, char * p)
-{
-    static_assert(is_unsigned_v<T>);
-
-    int len = digits10(x);
-    auto * pp = p + len;
-    while (x >= 100)
-    {
-        const auto i = x % 100;
-        x /= 100;
-        pp -= 2;
-        outTwoDigits(pp, i);
-    }
-    if (x < 10)
-        *p = '0' + x;
-    else
-        outTwoDigits(p, x);
-    return p + len;
-}
-
-static inline char * writeLeadingMinus(char * pos)
-{
-    *pos = '-';
-    return pos + 1;
-}
-
-template <typename T>
-static inline char * writeSIntText(T x, char * pos)
-{
-    static_assert(std::is_same_v<T, Int128> || std::is_same_v<T, Int256>);
-
-    using UnsignedT = make_unsigned_t<T>;
-    static constexpr T min_int = UnsignedT(1) << (sizeof(T) * 8 - 1);
-
-    if (unlikely(x == min_int))
-    {
-        if constexpr (std::is_same_v<T, Int128>)
-        {
-            const char * res = "-170141183460469231731687303715884105728";
-            memcpy(pos, res, strlen(res));
-            return pos + strlen(res);
-        }
-        else if constexpr (std::is_same_v<T, Int256>)
-        {
-            const char * res = "-57896044618658097711785492504343953926634992332820282019728792003956564819968";
-            memcpy(pos, res, strlen(res));
-            return pos + strlen(res);
-        }
-    }
-
-    if (x < 0)
-    {
-        x = -x;
-        pos = writeLeadingMinus(pos);
-    }
-    return writeUIntText(UnsignedT(x), pos);
-}
-
-}
-
-template <typename I>
-char * itoa(I i, char * p)
-{
-    return impl::convert::itoa(i, p);
-}
-
-template <>
-inline char * itoa(char8_t i, char * p)
-{
-    return impl::convert::itoa(uint8_t(i), p);
-}
-
-template <>
-inline char * itoa(UInt128 i, char * p)
-{
-    return impl::writeUIntText(i, p);
-}
-
-template <>
-inline char * itoa(Int128 i, char * p)
-{
-    return impl::writeSIntText(i, p);
-}
-
-template <>
-inline char * itoa(UInt256 i, char * p)
-{
-    return impl::writeUIntText(i, p);
-}
-
-template <>
-inline char * itoa(Int256 i, char * p)
-{
-    return impl::writeSIntText(i, p);
-}

base/poco/Foundation/include/Poco/FPEnvironment_SUN.h

@@ -0,0 +1,75 @@
+//
+// FPEnvironment_SUN.h
+//
+// Library: Foundation
+// Package: Core
+// Module:  FPEnvironment
+//
+// Definitions of class FPEnvironmentImpl for Solaris.
+//
+// Copyright (c) 2005-2006, Applied Informatics Software Engineering GmbH.
+// and Contributors.
+//
+// SPDX-License-Identifier:	BSL-1.0
+//
+
+
+#ifndef Foundation_FPEnvironment_SUN_INCLUDED
+#define Foundation_FPEnvironment_SUN_INCLUDED
+
+
+#include <ieeefp.h>
+#include "Poco/Foundation.h"
+
+
+namespace Poco
+{
+
+
+class FPEnvironmentImpl
+{
+protected:
+    enum RoundingModeImpl
+    {
+        FP_ROUND_DOWNWARD_IMPL = FP_RM,
+        FP_ROUND_UPWARD_IMPL = FP_RP,
+        FP_ROUND_TONEAREST_IMPL = FP_RN,
+        FP_ROUND_TOWARDZERO_IMPL = FP_RZ
+    };
+    enum FlagImpl
+    {
+        FP_DIVIDE_BY_ZERO_IMPL = FP_X_DZ,
+        FP_INEXACT_IMPL = FP_X_IMP,
+        FP_OVERFLOW_IMPL = FP_X_OFL,
+        FP_UNDERFLOW_IMPL = FP_X_UFL,
+        FP_INVALID_IMPL = FP_X_INV
+    };
+    FPEnvironmentImpl();
+    FPEnvironmentImpl(const FPEnvironmentImpl & env);
+    ~FPEnvironmentImpl();
+    FPEnvironmentImpl & operator=(const FPEnvironmentImpl & env);
+    void keepCurrentImpl();
+    static void clearFlagsImpl();
+    static bool isFlagImpl(FlagImpl flag);
+    static void setRoundingModeImpl(RoundingModeImpl mode);
+    static RoundingModeImpl getRoundingModeImpl();
+    static bool isInfiniteImpl(float value);
+    static bool isInfiniteImpl(double value);
+    static bool isInfiniteImpl(long double value);
+    static bool isNaNImpl(float value);
+    static bool isNaNImpl(double value);
+    static bool isNaNImpl(long double value);
+    static float copySignImpl(float target, float source);
+    static double copySignImpl(double target, double source);
+    static long double copySignImpl(long double target, long double source);
+
+private:
+    fp_rnd _rnd;
+    fp_except _exc;
+};
+
+
+} // namespace Poco
+
+
+#endif // Foundation_FPEnvironment_SUN_INCLUDED

base/poco/Foundation/src/FPEnvironment_SUN.cpp

@@ -0,0 +1,139 @@
+//
+// FPEnvironment_SUN.cpp
+//
+// Library: Foundation
+// Package: Core
+// Module:  FPEnvironment
+//
+// Copyright (c) 2005-2006, Applied Informatics Software Engineering GmbH.
+// and Contributors.
+//
+// SPDX-License-Identifier:	BSL-1.0
+//
+
+
+#include <math.h>
+#include "Poco/FPEnvironment_SUN.h"
+
+
+namespace Poco {
+
+
+FPEnvironmentImpl::FPEnvironmentImpl()
+{
+	_rnd = fpgetround();
+	_exc = fpgetmask();
+}
+
+
+FPEnvironmentImpl::FPEnvironmentImpl(const FPEnvironmentImpl& env)
+{
+	_rnd = env._rnd;
+	_exc = env._exc;
+}
+
+
+FPEnvironmentImpl::~FPEnvironmentImpl()
+{
+	fpsetround(_rnd);
+	fpsetmask(_exc);
+}
+
+
+FPEnvironmentImpl& FPEnvironmentImpl::operator = (const FPEnvironmentImpl& env)
+{
+	_rnd = env._rnd;
+	_exc = env._exc;
+	return *this;
+}
+
+
+bool FPEnvironmentImpl::isInfiniteImpl(float value)
+{
+	int cls = fpclass(value);
+	return cls == FP_PINF || cls == FP_NINF;
+}
+
+
+bool FPEnvironmentImpl::isInfiniteImpl(double value)
+{
+	int cls = fpclass(value);
+	return cls == FP_PINF || cls == FP_NINF;
+}
+
+
+bool FPEnvironmentImpl::isInfiniteImpl(long double value)
+{
+	int cls = fpclass(value);
+	return cls == FP_PINF || cls == FP_NINF;
+}
+
+
+bool FPEnvironmentImpl::isNaNImpl(float value)
+{
+	return isnanf(value) != 0;
+}
+
+
+bool FPEnvironmentImpl::isNaNImpl(double value)
+{
+	return isnan(value) != 0;
+}
+
+
+bool FPEnvironmentImpl::isNaNImpl(long double value)
+{
+	return isnan((double) value) != 0;
+}
+
+
+float FPEnvironmentImpl::copySignImpl(float target, float source)
+{
+	return (float) copysign(target, source);
+}
+
+
+double FPEnvironmentImpl::copySignImpl(double target, double source)
+{
+	return (float) copysign(target, source);
+}
+
+
+long double FPEnvironmentImpl::copySignImpl(long double target, long double source)
+{
+	return (source > 0 && target > 0) || (source < 0 && target < 0) ? target : -target;
+}
+
+
+void FPEnvironmentImpl::keepCurrentImpl()
+{
+	fpsetround(_rnd);
+	fpsetmask(_exc);
+}
+
+
+void FPEnvironmentImpl::clearFlagsImpl()
+{
+	fpsetsticky(0);
+}
+
+
+bool FPEnvironmentImpl::isFlagImpl(FlagImpl flag)
+{
+	return (fpgetsticky() & flag) != 0;
+}
+
+
+void FPEnvironmentImpl::setRoundingModeImpl(RoundingModeImpl mode)
+{
+	fpsetround((fp_rnd) mode);
+}
+
+
+FPEnvironmentImpl::RoundingModeImpl FPEnvironmentImpl::getRoundingModeImpl()
+{
+	return (FPEnvironmentImpl::RoundingModeImpl) fpgetround();
+}
+
+
+} // namespace Poco

docker/test/fuzzer/run-fuzzer.sh

@@ -173,16 +173,15 @@ function fuzz
 
     mkdir -p /var/run/clickhouse-server
 
-    # NOTE: we use process substitution here to preserve keep $! as a pid of clickhouse-server
+    # server.log -> All server logs, including sanitizer
-    # server.log -> CH logs
+    # stderr.log -> Process logs (sanitizer) only
-    # stderr.log -> Process logs (sanitizer)
     clickhouse-server \
         --config-file db/config.xml \
         --pid-file /var/run/clickhouse-server/clickhouse-server.pid \
         --  --path db \
             --logger.console=0 \
-            --logger.log=server.log > stderr.log 2>&1 &
+            --logger.log=server.log 2>&1 | tee -a stderr.log >> server.log 2>&1 &
-    server_pid=$!
+    server_pid=$(pidof clickhouse-server)
 
     kill -0 $server_pid
 
@@ -310,7 +309,7 @@ quit
     if [ "$server_died" == 1 ]
     then
         # The server has died.
-        if ! rg --text -o 'Received signal.*|Logical error.*|Assertion.*failed|Failed assertion.*|.*runtime error: .*|.*is located.*|(SUMMARY|ERROR): [a-zA-Z]+Sanitizer:.*|.*_LIBCPP_ASSERT.*|.*Child process was terminated by signal 9.*' server.log stderr.log > description.txt
+        if ! rg --text -o 'Received signal.*|Logical error.*|Assertion.*failed|Failed assertion.*|.*runtime error: .*|.*is located.*|(SUMMARY|ERROR): [a-zA-Z]+Sanitizer:.*|.*_LIBCPP_ASSERT.*|.*Child process was terminated by signal 9.*' server.log > description.txt
         then
             echo "Lost connection to server. See the logs." > description.txt
         fi

docker/test/integration/runner/Dockerfile

@@ -126,7 +126,6 @@ RUN set -x \
 
 COPY modprobe.sh /usr/local/bin/modprobe
 COPY dockerd-entrypoint.sh /usr/local/bin/
-COPY compose/ /compose/
 COPY misc/ /misc/
 
 

docker/test/style/Dockerfile

@@ -8,20 +8,22 @@ ARG apt_archive="http://archive.ubuntu.com"
 RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
 
 RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \
-    aspell \
+        aspell \
-    curl \
+        curl \
-    git \
+        git \
-    file \
+        file \
-    libxml2-utils \
+        libxml2-utils \
-    moreutils \
+        moreutils \
-    python3-fuzzywuzzy \
+        python3-fuzzywuzzy \
-    python3-pip \
+        python3-pip \
-    yamllint \
+        yamllint \
-    locales \
+        locales \
-    && pip3 install black==23.12.0 boto3 codespell==2.2.1 mypy==1.8.0 PyGithub unidiff pylint==3.1.0 \
-        requests types-requests \
     && apt-get clean \
-    && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* \
+    && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
+
+# python-magic is the same version as in Ubuntu 22.04
+RUN pip3 install black==23.12.0 boto3 codespell==2.2.1 mypy==1.8.0 PyGithub unidiff pylint==3.1.0 \
+        python-magic==0.4.24 requests types-requests \
     && rm -rf /root/.cache/pip
 
 RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen en_US.UTF-8

docs/en/operations/server-configuration-parameters/settings.md

@@ -933,9 +933,9 @@ Hard limit is configured via system tools
 
 ## database_atomic_delay_before_drop_table_sec {#database_atomic_delay_before_drop_table_sec}
 
-The delay before a table data is dropped in seconds. If the `DROP TABLE` query has a `SYNC` modifier, this setting is ignored.
+Sets the delay before remove table data in seconds. If the query has `SYNC` modifier, this setting is ignored.
 
-Default value: `480` (8 minutes).
+Default value: `480` (8 minute).
 
 ## database_catalog_unused_dir_hide_timeout_sec {#database_catalog_unused_dir_hide_timeout_sec}
 

docs/en/sql-reference/statements/undrop.md

@@ -13,6 +13,13 @@ a system table called `system.dropped_tables`.
 
 If you have a materialized view without a `TO` clause associated with the dropped table, then you will also have to UNDROP the inner table of that view.
 
+:::note
+UNDROP TABLE is experimental.  To use it add this setting: 
+```sql
+set allow_experimental_undrop_table_query = 1;
+```
+:::
+
 :::tip
 Also see [DROP TABLE](/docs/en/sql-reference/statements/drop.md)
 :::
@@ -25,53 +32,60 @@ UNDROP TABLE [db.]name [UUID '<uuid>'] [ON CLUSTER cluster]
 
 **Example**
 
+``` sql
+set allow_experimental_undrop_table_query = 1;
+```
+
 ```sql
-CREATE TABLE tab
+CREATE TABLE undropMe
 (
     `id` UInt8
 )
 ENGINE = MergeTree
-ORDER BY id;
+ORDER BY id
-
-DROP TABLE tab;
-
-SELECT *
-FROM system.dropped_tables
-FORMAT Vertical;
 ```
 
+```sql
+DROP TABLE undropMe
+```
+```sql
+SELECT *
+FROM system.dropped_tables
+FORMAT Vertical
+```
 ```response
 Row 1:
 ──────
 index:                 0
 database:              default
-table:                 tab
+table:                 undropMe
 uuid:                  aa696a1a-1d70-4e60-a841-4c80827706cc
 engine:                MergeTree
-metadata_dropped_path: /var/lib/clickhouse/metadata_dropped/default.tab.aa696a1a-1d70-4e60-a841-4c80827706cc.sql
+metadata_dropped_path: /var/lib/clickhouse/metadata_dropped/default.undropMe.aa696a1a-1d70-4e60-a841-4c80827706cc.sql
 table_dropped_time:    2023-04-05 14:12:12
 
 1 row in set. Elapsed: 0.001 sec. 
 ```
-
 ```sql
-UNDROP TABLE tab;
+UNDROP TABLE undropMe
-
+```
+```response
+Ok.
+```
+```sql
 SELECT *
 FROM system.dropped_tables
-FORMAT Vertical;
+FORMAT Vertical
-
+```
 ```response
 Ok.
 
 0 rows in set. Elapsed: 0.001 sec. 
 ```
-
 ```sql
-DESCRIBE TABLE tab
+DESCRIBE TABLE undropMe
-FORMAT Vertical;
+FORMAT Vertical
 ```
-
 ```response
 Row 1:
 ──────

programs/benchmark/Benchmark.cpp

@@ -34,6 +34,7 @@
 #include <Common/StudentTTest.h>
 #include <Common/CurrentMetrics.h>
 #include <Common/ErrorCodes.h>
+#include <Core/BaseSettingsProgramOptions.h>
 
 
 /** A tool for evaluating ClickHouse performance.
@@ -623,7 +624,7 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv)
         ;
 
         Settings settings;
-        settings.addProgramOptions(desc);
+        addProgramOptions(settings, desc);
 
         boost::program_options::variables_map options;
         boost::program_options::store(boost::program_options::parse_command_line(argc, argv, desc), options);

programs/client/Client.cpp

@@ -933,7 +933,7 @@ void Client::addOptions(OptionsDescription & options_description)
         ("config,c", po::value<std::string>(), "config-file path (another shorthand)")
         ("connection", po::value<std::string>(), "connection to use (from the client config), by default connection name is hostname")
         ("secure,s", "Use TLS connection")
-        ("no-secure,s", "Don't use TLS connection")
+        ("no-secure", "Don't use TLS connection")
         ("user,u", po::value<std::string>()->default_value("default"), "user")
         ("password", po::value<std::string>(), "password")
         ("ask-password", "ask-password")

programs/format/Format.cpp

@@ -17,6 +17,7 @@
 #include <Common/ErrorCodes.h>
 #include <Common/StringUtils/StringUtils.h>
 #include <Common/TerminalSize.h>
+#include <Core/BaseSettingsProgramOptions.h>
 
 #include <Interpreters/Context.h>
 #include <Functions/FunctionFactory.h>
@@ -102,7 +103,7 @@ int mainEntryClickHouseFormat(int argc, char ** argv)
         {
             std::string_view name = field.getName();
             if (name == "max_parser_depth" || name == "max_query_size")
-                cmd_settings.addProgramOption(desc, name, field);
+                addProgramOption(cmd_settings, desc, name, field);
         }
 
         boost::program_options::variables_map options;

programs/server/Server.cpp

@@ -36,6 +36,7 @@
 #include <Common/getNumberOfPhysicalCPUCores.h>
 #include <Common/getExecutablePath.h>
 #include <Common/ProfileEvents.h>
+#include <Common/Scheduler/IResourceManager.h>
 #include <Common/ThreadProfileEvents.h>
 #include <Common/ThreadStatus.h>
 #include <Common/getMappedArea.h>

src/Access/Common/QuotaDefs.cpp

@@ -2,6 +2,7 @@
 #include <Common/Exception.h>
 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
+#include <base/range.h>
 
 #include <boost/algorithm/string/case_conv.hpp>
 #include <boost/algorithm/string/classification.hpp>

src/Access/IAccessStorage.h

@@ -13,6 +13,8 @@
 #include <optional>
 #include <vector>
 
+#include <boost/noncopyable.hpp>
+
 
 namespace Poco { class Logger; }
 namespace Poco::Net { class IPAddress; }

src/Access/User.cpp

@@ -1,4 +1,5 @@
 #include <Access/User.h>
+#include <Common/StringUtils/StringUtils.h>
 #include <Core/Protocol.h>
 #include <base/insertAtEnd.h>
 

src/AggregateFunctions/AggregateFunctionRetention.cpp

@@ -1,16 +1,16 @@
 #include <AggregateFunctions/AggregateFunctionFactory.h>
+#include <AggregateFunctions/IAggregateFunction.h>
 #include <AggregateFunctions/FactoryHelpers.h>
-
-#include <unordered_set>
 #include <Columns/ColumnArray.h>
 #include <Common/assert_cast.h>
 #include <DataTypes/DataTypesNumber.h>
 #include <DataTypes/DataTypeArray.h>
 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
-#include <bitset>
+#include <base/range.h>
 
-#include <AggregateFunctions/IAggregateFunction.h>
+#include <bitset>
+#include <unordered_set>
 
 
 namespace DB

src/AggregateFunctions/AggregateFunctionSequenceMatch.cpp

@@ -10,6 +10,8 @@
 #include <Common/assert_cast.h>
 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
+#include <base/range.h>
+
 #include <bitset>
 #include <stack>
 

src/Analyzer/Passes/QueryAnalysisPass.cpp

@@ -48,6 +48,7 @@
 #include <Interpreters/SelectQueryOptions.h>
 #include <Interpreters/Set.h>
 #include <Interpreters/Context.h>
+#include <Interpreters/DatabaseCatalog.h>
 #include <Interpreters/ExternalDictionariesLoader.h>
 #include <Interpreters/InterpreterSelectQueryAnalyzer.h>
 
@@ -7409,18 +7410,40 @@ void QueryAnalyzer::resolveJoin(QueryTreeNodePtr & join_node, IdentifierResolveS
             if (!result_left_table_expression)
                 result_left_table_expression = tryResolveIdentifierFromJoinTreeNode(identifier_lookup, join_node_typed.getLeftTableExpression(), scope);
 
-            /// Here we may try to resolve identifier from projection in case it's not resolved from left table expression
+            /** Here we may try to resolve identifier from projection in case it's not resolved from left table expression
-            /// and analyzer_compatibility_join_using_top_level_identifier is disabled.
+              * and analyzer_compatibility_join_using_top_level_identifier is disabled.
-            /// For now we do not do this, because not all corner cases are clear.
+              * For now we do not do this, because not all corner cases are clear.
+              * But let's at least mention it in error message
+              */
             /// if (!settings.analyzer_compatibility_join_using_top_level_identifier && !result_left_table_expression)
             ///     result_left_table_expression = try_resolve_identifier_from_query_projection(identifier_full_name, join_node_typed.getLeftTableExpression(), scope);
 
             if (!result_left_table_expression)
+            {
+                String extra_message;
+                const QueryNode * query_node = scope.scope_node ? scope.scope_node->as<QueryNode>() : nullptr;
+                if (settings.analyzer_compatibility_join_using_top_level_identifier && query_node)
+                {
+                    for (const auto & projection_node : query_node->getProjection().getNodes())
+                    {
+                        if (projection_node->hasAlias() && identifier_full_name == projection_node->getAlias())
+                        {
+                            extra_message = fmt::format(
+                                ", but alias '{}' is present in SELECT list."
+                                " You may try to SET analyzer_compatibility_join_using_top_level_identifier = 1, to allow to use it in USING clause",
+                                projection_node->formatASTForErrorMessage());
+                            break;
+                        }
+                    }
+                }
+
                 throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER,
-                    "JOIN {} using identifier '{}' cannot be resolved from left table expression. In scope {}",
+                    "JOIN {} using identifier '{}' cannot be resolved from left table expression{}. In scope {}",
                     join_node_typed.formatASTForErrorMessage(),
                     identifier_full_name,
+                    extra_message,
                     scope.scope_node->formatASTForErrorMessage());
+            }
 
             if (result_left_table_expression->getNodeType() != QueryTreeNodeType::COLUMN)
                 throw Exception(ErrorCodes::UNSUPPORTED_METHOD,

src/Analyzer/Passes/SumIfToCountIfPass.cpp

@@ -97,13 +97,15 @@ public:
         if (!if_true_condition_constant_node || !if_false_condition_constant_node)
             return;
 
+        if (auto constant_type = if_true_condition_constant_node->getResultType(); !isNativeInteger(constant_type))
+            return;
+
+        if (auto constant_type = if_false_condition_constant_node->getResultType(); !isNativeInteger(constant_type))
+            return;
+
         const auto & if_true_condition_constant_value_literal = if_true_condition_constant_node->getValue();
         const auto & if_false_condition_constant_value_literal = if_false_condition_constant_node->getValue();
 
-        if (!isInt64OrUInt64FieldType(if_true_condition_constant_value_literal.getType()) ||
-            !isInt64OrUInt64FieldType(if_false_condition_constant_value_literal.getType()))
-            return;
-
         auto if_true_condition_value = if_true_condition_constant_value_literal.get<UInt64>();
         auto if_false_condition_value = if_false_condition_constant_value_literal.get<UInt64>();
 

src/Backups/BackupEntriesCollector.cpp

@@ -6,6 +6,7 @@
 #include <Backups/DDLAdjustingForBackupVisitor.h>
 #include <Databases/IDatabase.h>
 #include <Interpreters/Context.h>
+#include <Interpreters/DatabaseCatalog.h>
 #include <Parsers/ASTCreateQuery.h>
 #include <Parsers/formatAST.h>
 #include <Storages/IStorage.h>

src/Backups/BackupIO_Default.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <Backups/BackupIO.h>
+#include <Common/Logger.h>
 #include <IO/ReadSettings.h>
 #include <IO/WriteSettings.h>
 

src/Backups/BackupIO_Disk.h

@@ -1,7 +1,9 @@
 #pragma once
 
 #include <Backups/BackupIO_Default.h>
+#include <Common/Logger.h>
 #include <Disks/DiskType.h>
+
 #include <filesystem>
 
 

src/Backups/BackupIO_S3.h

@@ -4,6 +4,7 @@
 
 #if USE_AWS_S3
 #include <Backups/BackupIO_Default.h>
+#include <Common/Logger.h>
 #include <Disks/DiskType.h>
 #include <IO/S3Common.h>
 #include <Storages/StorageS3Settings.h>

src/Client/ClientBase.cpp

@@ -9,6 +9,7 @@
 #include <base/safeExit.h>
 #include <base/scope_guard.h>
 #include <Core/Block.h>
+#include <Core/BaseSettingsProgramOptions.h>
 #include <Core/Protocol.h>
 #include <Common/DateLUT.h>
 #include <Common/MemoryTracker.h>
@@ -2714,9 +2715,9 @@ private:
 void ClientBase::parseAndCheckOptions(OptionsDescription & options_description, po::variables_map & options, Arguments & arguments)
 {
     if (allow_repeated_settings)
-        cmd_settings.addProgramOptionsAsMultitokens(options_description.main_description.value());
+        addProgramOptionsAsMultitokens(cmd_settings, options_description.main_description.value());
     else
-        cmd_settings.addProgramOptions(options_description.main_description.value());
+        addProgramOptions(cmd_settings, options_description.main_description.value());
 
     if (allow_merge_tree_settings)
     {
@@ -2737,9 +2738,9 @@ void ClientBase::parseAndCheckOptions(OptionsDescription & options_description,
                     return;
 
                 if (allow_repeated_settings)
-                    cmd_merge_tree_settings.addProgramOptionAsMultitoken(main_options, name, setting);
+                    addProgramOptionAsMultitoken(cmd_merge_tree_settings, main_options, name, setting);
                 else
-                    cmd_merge_tree_settings.addProgramOption(main_options, name, setting);
+                    addProgramOption(cmd_merge_tree_settings, main_options, name, setting);
             };
 
             const auto & setting_name = setting.getName();

src/Client/LocalConnection.cpp

@@ -1,13 +1,14 @@
 #include "LocalConnection.h"
+#include <Core/Protocol.h>
+#include <Interpreters/DatabaseCatalog.h>
 #include <Interpreters/executeQuery.h>
 #include <Processors/Executors/CompletedPipelineExecutor.h>
 #include <Processors/Executors/PullingAsyncPipelineExecutor.h>
-#include <Processors/Executors/PushingPipelineExecutor.h>
 #include <Processors/Executors/PushingAsyncPipelineExecutor.h>
+#include <Processors/Executors/PushingPipelineExecutor.h>
 #include <Storages/IStorage.h>
 #include <Common/ConcurrentBoundedQueue.h>
 #include <Common/CurrentThread.h>
-#include <Core/Protocol.h>
 
 
 namespace DB

src/Common/HTTPConnectionPool.cpp

@@ -10,14 +10,15 @@
 #include <Common/MemoryTrackerSwitcher.h>
 #include <Common/SipHash.h>
 
-#include <Poco/Net/HTTPClientSession.h>
-#include <Poco/Net/HTTPStream.h>
-#include <Poco/Net/HTTPFixedLengthStream.h>
 #include <Poco/Net/HTTPChunkedStream.h>
+#include <Poco/Net/HTTPClientSession.h>
+#include <Poco/Net/HTTPFixedLengthStream.h>
+#include <Poco/Net/HTTPRequest.h>
+#include <Poco/Net/HTTPResponse.h>
+#include <Poco/Net/HTTPStream.h>
 #include <Poco/Timespan.h>
 
-#include <Poco/Net/HTTPResponse.h>
+#include <queue>
-#include <Poco/Net/HTTPRequest.h>
 
 #include "config.h"
 

src/Common/IntervalKind.cpp

@@ -1,6 +1,8 @@
 #include <Common/IntervalKind.h>
 #include <Common/Exception.h>
 
+#include <base/EnumReflection.h>
+
 
 namespace DB
 {
@@ -10,6 +12,11 @@ namespace ErrorCodes
     extern const int BAD_ARGUMENTS;
 }
 
+std::string_view IntervalKind::toString() const
+{
+    return magic_enum::enum_name(kind);
+}
+
 Int64 IntervalKind::toAvgNanoseconds() const
 {
     static constexpr Int64 NANOSECONDS_PER_MICROSECOND = 1000;

src/Common/IntervalKind.h

@@ -1,7 +1,6 @@
 #pragma once
 
 #include <base/types.h>
-#include <base/EnumReflection.h>
 
 namespace DB
 {
@@ -27,7 +26,7 @@ struct IntervalKind
     IntervalKind(Kind kind_ = Kind::Second) : kind(kind_) {} /// NOLINT
     operator Kind() const { return kind; } /// NOLINT
 
-    constexpr std::string_view toString() const { return magic_enum::enum_name(kind); }
+    std::string_view toString() const;
 
     /// Returns number of nanoseconds in one interval.
     /// For `Month`, `Quarter` and `Year` the function returns an average number of nanoseconds.

src/Common/Scheduler/IResourceManager.h

@@ -12,6 +12,9 @@
 namespace DB
 {
 
+class ISchedulerNode;
+using SchedulerNodePtr = std::shared_ptr<ISchedulerNode>;
+
 /*
  * Instance of derived class holds everything required for resource consumption,
  * including resources currently registered at `SchedulerRoot`. This is required to avoid

src/Common/Scheduler/ResouceLink.cpp

@@ -0,0 +1,25 @@
+#include <Common/Scheduler/ISchedulerQueue.h>
+#include <Common/Scheduler/ResourceLink.h>
+#include <Common/Scheduler/ResourceRequest.h>
+
+namespace DB
+{
+void ResourceLink::adjust(ResourceCost estimated_cost, ResourceCost real_cost) const
+{
+    if (queue)
+        queue->adjustBudget(estimated_cost, real_cost);
+}
+
+void ResourceLink::consumed(ResourceCost cost) const
+{
+    if (queue)
+        queue->consumeBudget(cost);
+}
+
+void ResourceLink::accumulate(DB::ResourceCost cost) const
+{
+    if (queue)
+        queue->accumulateBudget(cost);
+}
+}
+

src/Common/Scheduler/ResourceGuard.h

@@ -2,9 +2,10 @@
 
 #include <base/types.h>
 
+#include <Common/Scheduler/ISchedulerConstraint.h>
+#include <Common/Scheduler/ISchedulerQueue.h>
 #include <Common/Scheduler/ResourceRequest.h>
 #include <Common/Scheduler/ResourceLink.h>
-#include <Common/Scheduler/ISchedulerConstraint.h>
 
 #include <condition_variable>
 #include <mutex>

src/Common/Scheduler/ResourceLink.h

@@ -2,12 +2,10 @@
 
 #include <base/types.h>
 
-#include <Common/Scheduler/ResourceRequest.h>
-#include <Common/Scheduler/ISchedulerQueue.h>
-
-
 namespace DB
 {
+class ISchedulerQueue;
+using ResourceCost = Int64;
 
 /*
  * Everything required for resource consumption. Connection to a specific resource queue.
@@ -17,23 +15,11 @@ struct ResourceLink
     ISchedulerQueue * queue = nullptr;
     bool operator==(const ResourceLink &) const = default;
 
-    void adjust(ResourceCost estimated_cost, ResourceCost real_cost) const
+    void adjust(ResourceCost estimated_cost, ResourceCost real_cost) const;
-    {
-        if (queue)
-            queue->adjustBudget(estimated_cost, real_cost);
-    }
 
-    void consumed(ResourceCost cost) const
+    void consumed(ResourceCost cost) const;
-    {
-        if (queue)
-            queue->consumeBudget(cost);
-    }
 
-    void accumulate(ResourceCost cost) const
+    void accumulate(ResourceCost cost) const;
-    {
-        if (queue)
-            queue->accumulateBudget(cost);
-    }
 };
 
 }

src/Common/SymbolIndex.cpp

@@ -2,6 +2,8 @@
 
 #include <Common/SymbolIndex.h>
 #include <Common/MemorySanitizer.h>
+#include <base/hex.h>
+#include <base/sort.h>
 
 #include <algorithm>
 #include <optional>
@@ -11,8 +13,6 @@
 
 #include <filesystem>
 
-#include <base/sort.h>
-
 /**
 
 ELF object can contain three different places with symbol names and addresses:

src/Common/ThreadStatus.h

@@ -1,12 +1,12 @@
 #pragma once
 
-#include <Core/SettingsEnums.h>
+#include <Core/LogsLevel.h>
-#include <Interpreters/Context_fwd.h>
 #include <IO/Progress.h>
+#include <Interpreters/Context_fwd.h>
+#include <base/StringRef.h>
 #include <Common/MemoryTracker.h>
 #include <Common/ProfileEvents.h>
 #include <Common/Stopwatch.h>
-#include <base/StringRef.h>
 
 #include <boost/noncopyable.hpp>
 

src/Common/formatIPv6.h

@@ -1,13 +1,12 @@
 #pragma once
 
-#include <base/types.h>
-#include <cstring>
 #include <algorithm>
+#include <cstring>
 #include <type_traits>
 #include <utility>
-#include <base/range.h>
-#include <base/unaligned.h>
 #include <base/hex.h>
+#include <base/types.h>
+#include <base/unaligned.h>
 #include <Common/StringUtils/StringUtils.h>
 
 constexpr size_t IPV4_BINARY_LENGTH = 4;

src/Common/tests/gtest_proxy_configuration_resolver_provider.cpp

@@ -4,6 +4,8 @@
 #include <Common/tests/gtest_global_context.h>
 #include <Common/tests/gtest_helper_functions.h>
 
+#include <Poco/Util/MapConfiguration.h>
+
 using ConfigurationPtr = Poco::AutoPtr<Poco::Util::AbstractConfiguration>;
 
 class ProxyConfigurationResolverProviderTests : public ::testing::Test

src/Core/BaseSettings.h

@@ -7,7 +7,6 @@
 #include <base/range.h>
 #include <boost/blank.hpp>
 #include <unordered_map>
-#include <boost/program_options/options_description.hpp>
 
 
 namespace boost::program_options
@@ -129,18 +128,6 @@ public:
         std::conditional_t<Traits::allow_custom_settings, const CustomSettingMap::mapped_type*, boost::blank> custom_setting;
     };
 
-    /// Adds program options to set the settings from a command line.
-    /// (Don't forget to call notify() on the `variables_map` after parsing it!)
-    void addProgramOptions(boost::program_options::options_description & options);
-
-    /// Adds program options as to set the settings from a command line.
-    /// Allows to set one setting multiple times, the last value will be used.
-    /// (Don't forget to call notify() on the `variables_map` after parsing it!)
-    void addProgramOptionsAsMultitokens(boost::program_options::options_description & options);
-
-    void addProgramOption(boost::program_options::options_description & options, std::string_view name, const SettingFieldRef & field);
-    void addProgramOptionAsMultitoken(boost::program_options::options_description & options, std::string_view name, const SettingFieldRef & field);
-
     enum SkipFlags
     {
         SKIP_NONE = 0,
@@ -561,57 +548,6 @@ String BaseSettings<TTraits>::toString() const
     return out.str();
 }
 
-template <typename TTraits>
-void BaseSettings<TTraits>::addProgramOptions(boost::program_options::options_description & options)
-{
-    const auto & settings_to_aliases = TTraits::settingsToAliases();
-    for (const auto & field : all())
-    {
-        std::string_view name = field.getName();
-        addProgramOption(options, name, field);
-
-        if (auto it = settings_to_aliases.find(name); it != settings_to_aliases.end())
-        {
-            for (const auto alias : it->second)
-                addProgramOption(options, alias, field);
-        }
-    }
-}
-
-template <typename TTraits>
-void BaseSettings<TTraits>::addProgramOptionsAsMultitokens(boost::program_options::options_description & options)
-{
-    const auto & settings_to_aliases = TTraits::settingsToAliases();
-    for (const auto & field : all())
-    {
-        std::string_view name = field.getName();
-        addProgramOptionAsMultitoken(options, name, field);
-
-        if (auto it = settings_to_aliases.find(name); it != settings_to_aliases.end())
-        {
-            for (const auto alias : it->second)
-                addProgramOptionAsMultitoken(options, alias, field);
-        }
-    }
-}
-
-
-template <typename TTraits>
-void BaseSettings<TTraits>::addProgramOption(boost::program_options::options_description & options, std::string_view name, const SettingFieldRef & field)
-{
-    auto on_program_option = boost::function1<void, const std::string &>([this, name](const std::string & value) { set(name, value); });
-    options.add(boost::shared_ptr<boost::program_options::option_description>(new boost::program_options::option_description(
-        name.data(), boost::program_options::value<std::string>()->composing()->notifier(on_program_option), field.getDescription())));
-}
-
-template <typename TTraits>
-void BaseSettings<TTraits>::addProgramOptionAsMultitoken(boost::program_options::options_description & options, std::string_view name, const SettingFieldRef & field)
-{
-    auto on_program_option = boost::function1<void, const Strings &>([this, name](const Strings & values) { set(name, values.back()); });
-    options.add(boost::shared_ptr<boost::program_options::option_description>(new boost::program_options::option_description(
-        name.data(), boost::program_options::value<Strings>()->multitoken()->composing()->notifier(on_program_option), field.getDescription())));
-}
-
 template <typename TTraits>
 bool operator==(const BaseSettings<TTraits> & left, const BaseSettings<TTraits> & right)
 {

src/Core/BaseSettingsProgramOptions.h

@@ -0,0 +1,60 @@
+#pragma once
+
+#include <Core/Settings.h>
+#include <Core/Types_fwd.h>
+
+#include <boost/program_options.hpp>
+
+namespace DB
+{
+
+template <typename T>
+void addProgramOptionAsMultitoken(T &cmd_settings, boost::program_options::options_description & options, std::string_view name, const typename T::SettingFieldRef & field)
+{
+    auto on_program_option = boost::function1<void, const Strings &>([&cmd_settings, name](const Strings & values) { cmd_settings.set(name, values.back()); });
+    options.add(boost::shared_ptr<boost::program_options::option_description>(new boost::program_options::option_description(
+            name.data(), boost::program_options::value<Strings>()->multitoken()->composing()->notifier(on_program_option), field.getDescription())));
+}
+
+template <typename T>
+void addProgramOptionsAsMultitokens(T &cmd_settings, boost::program_options::options_description & options)
+{
+    const auto & settings_to_aliases = T::Traits::settingsToAliases();
+    for (const auto & field : cmd_settings.all())
+    {
+        std::string_view name = field.getName();
+        addProgramOptionAsMultitoken(cmd_settings, options, name, field);
+
+        if (auto it = settings_to_aliases.find(name); it != settings_to_aliases.end())
+            for (const auto alias : it->second)
+                addProgramOptionAsMultitoken(cmd_settings, options, alias, field);
+    }
+}
+
+/// Adds program options to set the settings from a command line.
+/// (Don't forget to call notify() on the `variables_map` after parsing it!)
+template <typename T>
+void addProgramOption(T &cmd_settings, boost::program_options::options_description & options, std::string_view name, const typename T::SettingFieldRef  & field)
+{
+    auto on_program_option = boost::function1<void, const std::string &>([&cmd_settings, name](const std::string & value) { cmd_settings.set(name, value); });
+    options.add(boost::shared_ptr<boost::program_options::option_description>(new boost::program_options::option_description(
+            name.data(), boost::program_options::value<std::string>()->composing()->notifier(on_program_option), field.getDescription())));
+}
+
+template <typename T>
+void addProgramOptions(T &cmd_settings, boost::program_options::options_description & options)
+{
+    const auto & settings_to_aliases = T::Traits::settingsToAliases();
+    for (const auto & field : cmd_settings.all())
+    {
+        std::string_view name = field.getName();
+        addProgramOption(cmd_settings, options, name, field);
+
+        if (auto it = settings_to_aliases.find(name); it != settings_to_aliases.end())
+            for (const auto alias : it->second)
+                addProgramOption(cmd_settings, options, alias, field);
+    }
+}
+
+
+}

src/Core/LogsLevel.h

@@ -0,0 +1,16 @@
+#pragma once
+
+namespace DB
+{
+enum class LogsLevel
+{
+    none = 0, /// Disable
+    fatal,
+    error,
+    warning,
+    information,
+    debug,
+    trace,
+    test,
+};
+}

src/Core/SettingsChangesHistory.h

@@ -111,7 +111,7 @@ static std::map<ClickHouseVersion, SettingsChangesHistory::SettingsChanges> sett
               {"query_plan_optimize_prewhere", true, true, "Allow to push down filter to PREWHERE expression for supported storages"},
               {"async_insert_max_data_size", 1000000, 10485760, "The previous value appeared to be too small."},
               {"async_insert_poll_timeout_ms", 10, 10, "Timeout in milliseconds for polling data from asynchronous insert queue"},
-              {"async_insert_use_adaptive_busy_timeout", true, true, "Use adaptive asynchronous insert timeout"},
+              {"async_insert_use_adaptive_busy_timeout", false, true, "Use adaptive asynchronous insert timeout"},
               {"async_insert_busy_timeout_min_ms", 50, 50, "The minimum value of the asynchronous insert timeout in milliseconds; it also serves as the initial value, which may be increased later by the adaptive algorithm"},
               {"async_insert_busy_timeout_max_ms", 200, 200, "The minimum value of the asynchronous insert timeout in milliseconds; async_insert_busy_timeout_ms is aliased to async_insert_busy_timeout_max_ms"},
               {"async_insert_busy_timeout_increase_rate", 0.2, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout increases"},

src/Core/SettingsEnums.h

@@ -1,12 +1,13 @@
 #pragma once
 
-#include <Core/SettingsFields.h>
 #include <Core/Joins.h>
-#include <QueryPipeline/SizeLimits.h>
+#include <Core/LogsLevel.h>
+#include <Core/SettingsFields.h>
 #include <Formats/FormatSettings.h>
 #include <IO/ReadSettings.h>
-#include <Common/ShellCommandSettings.h>
 #include <Parsers/ASTSQLSecurity.h>
+#include <QueryPipeline/SizeLimits.h>
+#include <Common/ShellCommandSettings.h>
 
 
 namespace DB
@@ -90,18 +91,6 @@ DECLARE_SETTING_ENUM_WITH_RENAME(IntervalOutputFormat, FormatSettings::IntervalO
 
 DECLARE_SETTING_ENUM_WITH_RENAME(ParquetVersion, FormatSettings::ParquetVersion)
 
-enum class LogsLevel
-{
-    none = 0,    /// Disable
-    fatal,
-    error,
-    warning,
-    information,
-    debug,
-    trace,
-    test,
-};
-
 DECLARE_SETTING_ENUM(LogsLevel)
 
 

src/Daemon/SentryWriter.cpp

@@ -12,6 +12,7 @@
 #include <Common/StackTrace.h>
 #include <Common/getNumberOfPhysicalCPUCores.h>
 #include <Core/ServerUUID.h>
+#include <IO/WriteHelpers.h>
 
 #include "config.h"
 #include <Common/config_version.h>

src/DataTypes/DataTypeDomainBool.cpp

@@ -1,6 +1,7 @@
-#include <DataTypes/Serializations/SerializationBool.h>
-#include <DataTypes/DataTypeFactory.h>
 #include <DataTypes/DataTypeCustom.h>
+#include <DataTypes/DataTypeFactory.h>
+#include <DataTypes/IDataType.h>
+#include <DataTypes/Serializations/SerializationBool.h>
 
 namespace DB
 {

src/DataTypes/DataTypeFactory.h

@@ -1,6 +1,5 @@
 #pragma once
 
-#include <DataTypes/IDataType.h>
 #include <Parsers/IAST_fwd.h>
 #include <Common/IFactoryWithAliases.h>
 #include <DataTypes/DataTypeCustom.h>

src/DataTypes/DataTypeTuple.cpp

@@ -346,7 +346,7 @@ SerializationPtr DataTypeTuple::getSerialization(const SerializationInfo & info)
     return std::make_shared<SerializationTuple>(std::move(serializations), have_explicit_names);
 }
 
-MutableSerializationInfoPtr DataTypeTuple::createSerializationInfo(const SerializationInfo::Settings & settings) const
+MutableSerializationInfoPtr DataTypeTuple::createSerializationInfo(const SerializationInfoSettings & settings) const
 {
     MutableSerializationInfos infos;
     infos.reserve(elems.size());

src/DataTypes/DataTypeTuple.h

@@ -58,7 +58,7 @@ public:
 
     SerializationPtr doGetDefaultSerialization() const override;
     SerializationPtr getSerialization(const SerializationInfo & info) const override;
-    MutableSerializationInfoPtr createSerializationInfo(const SerializationInfo::Settings & settings) const override;
+    MutableSerializationInfoPtr createSerializationInfo(const SerializationInfoSettings & settings) const override;
     SerializationInfoPtr getSerializationInfo(const IColumn & column) const override;
 
     const DataTypePtr & getElement(size_t i) const { return elems[i]; }

src/DataTypes/IDataType.cpp

@@ -202,7 +202,7 @@ void IDataType::setCustomization(DataTypeCustomDescPtr custom_desc_) const
         custom_serialization = std::move(custom_desc_->serialization);
 }
 
-MutableSerializationInfoPtr IDataType::createSerializationInfo(const SerializationInfo::Settings & settings) const
+MutableSerializationInfoPtr IDataType::createSerializationInfo(const SerializationInfoSettings & settings) const
 {
     return std::make_shared<SerializationInfo>(ISerialization::Kind::DEFAULT, settings);
 }

src/DataTypes/IDataType.h

@@ -7,8 +7,6 @@
 #include <Common/COW.h>
 #include <DataTypes/DataTypeCustom.h>
 #include <DataTypes/Serializations/ISerialization.h>
-#include <DataTypes/Serializations/SerializationInfo.h>
-
 
 namespace DB
 {
@@ -38,6 +36,11 @@ struct DataTypeWithConstInfo
 
 using DataTypesWithConstInfo = std::vector<DataTypeWithConstInfo>;
 
+class SerializationInfo;
+using SerializationInfoPtr = std::shared_ptr<const SerializationInfo>;
+using MutableSerializationInfoPtr = std::shared_ptr<SerializationInfo>;
+struct SerializationInfoSettings;
+
 /** Properties of data type.
   *
   * Contains methods for getting serialization instances.
@@ -117,7 +120,7 @@ public:
 
     Names getSubcolumnNames() const;
 
-    virtual MutableSerializationInfoPtr createSerializationInfo(const SerializationInfo::Settings & settings) const;
+    virtual MutableSerializationInfoPtr createSerializationInfo(const SerializationInfoSettings & settings) const;
     virtual SerializationInfoPtr getSerializationInfo(const IColumn & column) const;
 
     /// TODO: support more types.

src/DataTypes/ObjectUtils.h

@@ -13,6 +13,7 @@ namespace DB
 
 struct StorageSnapshot;
 using StorageSnapshotPtr = std::shared_ptr<StorageSnapshot>;
+class ColumnsDescription;
 
 /// Returns number of dimensions in Array type. 0 if type is not array.
 size_t getNumberOfDimensions(const IDataType & type);

src/DataTypes/Serializations/SerializationInfo.h

@@ -2,6 +2,8 @@
 
 #include <Core/Types_fwd.h>
 #include <DataTypes/Serializations/ISerialization.h>
+#include <DataTypes/Serializations/SerializationInfoSettings.h>
+
 #include <Poco/JSON/Object.h>
 
 
@@ -28,6 +30,8 @@ constexpr auto SERIALIZATION_INFO_VERSION = 0;
 class SerializationInfo
 {
 public:
+    using Settings = SerializationInfoSettings;
+
     struct Data
     {
         size_t num_rows = 0;
@@ -38,16 +42,8 @@ public:
         void addDefaults(size_t length);
     };
 
-    struct Settings
+    SerializationInfo(ISerialization::Kind kind_, const SerializationInfoSettings & settings_);
-    {
+    SerializationInfo(ISerialization::Kind kind_, const SerializationInfoSettings & settings_, const Data & data_);
-        const double ratio_of_defaults_for_sparse = 1.0;
-        const bool choose_kind = false;
-
-        bool isAlwaysDefault() const { return ratio_of_defaults_for_sparse >= 1.0; }
-    };
-
-    SerializationInfo(ISerialization::Kind kind_, const Settings & settings_);
-    SerializationInfo(ISerialization::Kind kind_, const Settings & settings_, const Data & data_);
 
     virtual ~SerializationInfo() = default;
 
@@ -64,7 +60,7 @@ public:
     virtual std::shared_ptr<SerializationInfo> createWithType(
         const IDataType & old_type,
         const IDataType & new_type,
-        const Settings & new_settings) const;
+        const SerializationInfoSettings & new_settings) const;
 
     virtual void serialializeKindBinary(WriteBuffer & out) const;
     virtual void deserializeFromKindsBinary(ReadBuffer & in);
@@ -73,14 +69,14 @@ public:
     virtual void fromJSON(const Poco::JSON::Object & object);
 
     void setKind(ISerialization::Kind kind_) { kind = kind_; }
-    const Settings & getSettings() const { return settings; }
+    const SerializationInfoSettings & getSettings() const { return settings; }
     const Data & getData() const { return data; }
     ISerialization::Kind getKind() const { return kind; }
 
-    static ISerialization::Kind chooseKind(const Data & data, const Settings & settings);
+    static ISerialization::Kind chooseKind(const Data & data, const SerializationInfoSettings & settings);
 
 protected:
-    const Settings settings;
+    const SerializationInfoSettings settings;
 
     ISerialization::Kind kind;
     Data data;
@@ -96,7 +92,7 @@ using MutableSerializationInfos = std::vector<MutableSerializationInfoPtr>;
 class SerializationInfoByName : public std::map<String, MutableSerializationInfoPtr>
 {
 public:
-    using Settings = SerializationInfo::Settings;
+    using Settings = SerializationInfoSettings;
 
     SerializationInfoByName() = default;
     SerializationInfoByName(const NamesAndTypesList & columns, const Settings & settings);

src/DataTypes/Serializations/SerializationInfoSettings.h

@@ -0,0 +1,14 @@
+#pragma once
+
+namespace DB
+{
+
+struct SerializationInfoSettings
+{
+    const double ratio_of_defaults_for_sparse = 1.0;
+    const bool choose_kind = false;
+
+    bool isAlwaysDefault() const { return ratio_of_defaults_for_sparse >= 1.0; }
+};
+
+}

src/DataTypes/tests/gtest_DataType_deserializeAsText.cpp

@@ -1,6 +1,7 @@
 #include <Columns/IColumn.h>
 #include <Core/Field.h>
 #include <DataTypes/DataTypeFactory.h>
+#include <DataTypes/IDataType.h>
 #include <Formats/FormatSettings.h>
 #include <IO/ReadBuffer.h>
 

src/Databases/DatabaseAtomic.cpp

@@ -11,6 +11,7 @@
 #include <Common/filesystemHelpers.h>
 #include <Storages/StorageMaterializedView.h>
 #include <Interpreters/Context.h>
+#include <Interpreters/DatabaseCatalog.h>
 #include <Interpreters/ExternalDictionariesLoader.h>
 #include <filesystem>
 #include <Interpreters/DDLTask.h>

src/Databases/DatabaseMemory.cpp

@@ -6,6 +6,7 @@
 #include <Databases/DDLDependencyVisitor.h>
 #include <Databases/DDLLoadingDependencyVisitor.h>
 #include <Interpreters/Context.h>
+#include <Interpreters/DatabaseCatalog.h>
 #include <Parsers/ASTCreateQuery.h>
 #include <Parsers/ASTFunction.h>
 #include <Parsers/formatAST.h>

src/Databases/DatabaseOnDisk.cpp

@@ -5,8 +5,9 @@
 #include <IO/WriteBufferFromFile.h>
 #include <IO/WriteHelpers.h>
 #include <Interpreters/Context.h>
-#include <Interpreters/InterpreterCreateQuery.h>
 #include <Interpreters/ApplyWithSubqueryVisitor.h>
+#include <Interpreters/DatabaseCatalog.h>
+#include <Interpreters/InterpreterCreateQuery.h>
 #include <Parsers/ASTCreateQuery.h>
 #include <Parsers/ASTFunction.h>
 #include <Parsers/ParserCreateQuery.h>

src/Databases/DatabaseReplicated.cpp

@@ -20,6 +20,7 @@
 #include <Databases/TablesDependencyGraph.h>
 #include <Interpreters/Cluster.h>
 #include <Interpreters/Context.h>
+#include <Interpreters/DatabaseCatalog.h>
 #include <Interpreters/DDLTask.h>
 #include <Interpreters/evaluateConstantExpression.h>
 #include <Interpreters/executeDDLQueryOnCluster.h>

src/Databases/DatabasesCommon.cpp

@@ -1,6 +1,7 @@
 #include <Databases/DatabasesCommon.h>
 #include <Interpreters/InterpreterCreateQuery.h>
 #include <Interpreters/Context.h>
+#include <Interpreters/DatabaseCatalog.h>
 #include <Parsers/ASTCreateQuery.h>
 #include <Parsers/ASTSelectWithUnionQuery.h>
 #include <Parsers/ParserCreateQuery.h>

src/Databases/IDatabase.cpp

@@ -1,11 +1,12 @@
 #include <memory>
 #include <Databases/IDatabase.h>
-#include <Storages/IStorage.h>
-#include <Parsers/ASTCreateQuery.h>
-#include <Common/quoteString.h>
 #include <Interpreters/DatabaseCatalog.h>
-#include <Common/NamePrompter.h>
+#include <Interpreters/TableNameHints.h>
+#include <Parsers/ASTCreateQuery.h>
+#include <Storages/IStorage.h>
 #include <Common/CurrentMetrics.h>
+#include <Common/NamePrompter.h>
+#include <Common/quoteString.h>
 
 
 namespace CurrentMetrics

src/Disks/ObjectStorages/DiskObjectStorage.cpp

@@ -9,6 +9,7 @@
 #include <Common/logger_useful.h>
 #include <Common/filesystemHelpers.h>
 #include <Common/CurrentMetrics.h>
+#include <Common/Scheduler/IResourceManager.h>
 #include <Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.h>
 #include <Disks/ObjectStorages/DiskObjectStorageTransaction.h>
 #include <Disks/FakeDiskTransaction.h>

src/Formats/FormatFactory.h

@@ -23,6 +23,7 @@ namespace DB
 class Block;
 struct Settings;
 struct FormatFactorySettings;
+struct ReadSettings;
 
 class ReadBuffer;
 class WriteBuffer;

src/Formats/FormatSchemaInfo.h

@@ -8,6 +8,7 @@
 namespace DB
 {
 class Context;
+class Block;
 
 /// Extracts information about where the format schema file is from passed context and keep it.
 class FormatSchemaInfo

src/Formats/ReadSchemaUtils.cpp

@@ -3,6 +3,7 @@
 #include <IO/PeekableReadBuffer.h>
 #include <IO/WithFileSize.h>
 #include <Interpreters/Context.h>
+#include <Interpreters/DatabaseCatalog.h>
 #include <Processors/Formats/ISchemaReader.h>
 #include <Storages/IStorage.h>
 #include <Common/assert_cast.h>

src/Formats/SchemaInferenceUtils.h

@@ -3,9 +3,15 @@
 #include <DataTypes/IDataType.h>
 #include <IO/ReadBuffer.h>
 
+#include <vector>
+
 namespace DB
 {
 
+class Block;
+class NamesAndTypesList;
+using NamesAndTypesLists = std::vector<NamesAndTypesList>;
+
 /// Struct with some additional information about inferred types for JSON formats.
 struct JSONInferenceInfo
 {

src/Functions/CMakeLists.txt

@@ -37,6 +37,7 @@ list (APPEND PUBLIC_LIBS
         clickhouse_dictionaries_embedded
         clickhouse_parsers
         ch_contrib::consistent_hashing
+        common
         dbms
         ch_contrib::metrohash
         ch_contrib::murmurhash

src/Functions/DateTimeTransforms.h

@@ -11,6 +11,7 @@
 #include <Columns/ColumnsNumber.h>
 #include <Columns/ColumnVector.h>
 #include <Columns/ColumnDecimal.h>
+#include <Formats/FormatSettings.h>
 #include <Functions/FunctionHelpers.h>
 #include <Functions/IFunction.h>
 #include <Functions/extractTimeZoneFromFunctionArguments.h>

src/Functions/FunctionJoinGet.cpp

@@ -1,12 +1,13 @@
 #include <Columns/ColumnString.h>
+#include <Core/Block.h>
 #include <Functions/FunctionFactory.h>
 #include <Functions/FunctionHelpers.h>
-#include <Interpreters/Context.h>
-#include <Interpreters/HashJoin.h>
 #include <Functions/IFunction.h>
+#include <Interpreters/Context.h>
+#include <Interpreters/DatabaseCatalog.h>
+#include <Interpreters/HashJoin.h>
 #include <Storages/StorageJoin.h>
 #include <Storages/TableLockHolder.h>
-#include <Core/Block.h>
 
 
 namespace DB

src/Functions/FunctionsStringHashFixedString.cpp

@@ -274,7 +274,7 @@ public:
             const typename ColumnIPv6::Container & data = col_from_ip->getData();
             const auto size = col_from_ip->size();
             auto & chars_to = col_to->getChars();
-            const auto length = IPV6_BINARY_LENGTH;
+            const auto length = sizeof(IPv6::UnderlyingType);
             chars_to.resize(size * Impl::length);
             for (size_t i = 0; i < size; ++i)
             {

src/Functions/appendTrailingCharIfAbsent.cpp

@@ -1,9 +1,10 @@
 #include <Columns/ColumnString.h>
-#include <Common/assert_cast.h>
 #include <DataTypes/DataTypeString.h>
 #include <Functions/FunctionFactory.h>
 #include <Functions/FunctionHelpers.h>
 #include <Functions/IFunction.h>
+#include <base/range.h>
+#include <Common/assert_cast.h>
 
 
 namespace DB

src/Functions/array/arrayIntersect.cpp

@@ -20,6 +20,7 @@
 #include <Columns/ColumnTuple.h>
 #include <Common/HashTable/ClearableHashMap.h>
 #include <Common/assert_cast.h>
+#include <base/range.h>
 #include <base/TypeLists.h>
 #include <Interpreters/castColumn.h>
 

src/Functions/array/arraySort.cpp

@@ -46,7 +46,10 @@ ColumnPtr ArraySortImpl<positive, is_partial>::execute(
                     ErrorCodes::LOGICAL_ERROR,
                     "Expected fixed arguments to get the limit for partial array sort"
                 );
-            return fixed_arguments[0].column.get()->getUInt(0);
+
+            /// During dryRun the input column might be empty
+            if (!fixed_arguments[0].column->empty())
+                return fixed_arguments[0].column->getUInt(0);
         }
         return 0;
     }();

src/Functions/countDigits.cpp

@@ -20,6 +20,40 @@ namespace ErrorCodes
 namespace
 {
 
+template <typename T>
+int digits10(T x)
+{
+    if (x < 10ULL)
+        return 1;
+    if (x < 100ULL)
+        return 2;
+    if (x < 1000ULL)
+        return 3;
+
+    if (x < 1000000000000ULL)
+    {
+        if (x < 100000000ULL)
+        {
+            if (x < 1000000ULL)
+            {
+                if (x < 10000ULL)
+                    return 4;
+                else
+                    return 5 + (x >= 100000ULL);
+            }
+
+            return 7 + (x >= 10000000ULL);
+        }
+
+        if (x < 10000000000ULL)
+            return 9 + (x >= 1000000000ULL);
+
+        return 11 + (x >= 100000000000ULL);
+    }
+
+    return 12 + digits10(x / 1000000000000ULL);
+}
+
 /// Returns number of decimal digits you need to represent the value.
 /// For Decimal values takes in account their scales: calculates result over underlying int type which is (value * scale).
 /// countDigits(42) = 2, countDigits(42.000) = 5, countDigits(0.04200) = 4.

src/Functions/dateDiff.cpp

@@ -6,7 +6,7 @@
 #include <Columns/ColumnsDateTime.h>
 #include <Columns/ColumnsNumber.h>
 #include <Columns/ColumnDecimal.h>
-
+#include <Formats/FormatSettings.h>
 #include <Functions/IFunction.h>
 #include <Functions/FunctionHelpers.h>
 #include <Functions/FunctionFactory.h>

src/Functions/date_trunc.cpp

@@ -1,9 +1,10 @@
 #include <Columns/ColumnConst.h>
-#include <Columns/ColumnsNumber.h>
 #include <Columns/ColumnString.h>
+#include <Columns/ColumnsNumber.h>
 #include <DataTypes/DataTypeDate.h>
 #include <DataTypes/DataTypeDateTime.h>
 #include <DataTypes/DataTypeInterval.h>
+#include <Formats/FormatSettings.h>
 #include <Functions/DateTimeTransforms.h>
 #include <Functions/FunctionFactory.h>
 

src/Functions/hasColumnInTable.cpp

@@ -7,6 +7,7 @@
 #include <Storages/IStorage.h>
 #include <Interpreters/Cluster.h>
 #include <Interpreters/Context.h>
+#include <Interpreters/DatabaseCatalog.h>
 #include <Storages/getStructureOfRemoteTable.h>
 
 

src/Functions/keyvaluepair/ArgumentExtractor.h

@@ -4,6 +4,7 @@
 #include <Columns/ColumnsNumber.h>
 #include <Core/ColumnsWithTypeAndName.h>
 
+#include <list>
 #include <optional>
 
 namespace DB

src/Functions/tupleConcat.cpp

@@ -4,6 +4,8 @@
 #include <Functions/FunctionHelpers.h>
 #include <Functions/IFunction.h>
 
+#include <base/range.h>
+
 namespace DB
 {
 namespace ErrorCodes

src/IO/CachedInMemoryReadBufferFromFile.h

@@ -1,7 +1,8 @@
 #pragma once
 
-#include <Common/PageCache.h>
 #include <IO/ReadBufferFromFileBase.h>
+#include <IO/ReadSettings.h>
+#include <Common/PageCache.h>
 
 namespace DB
 {

src/IO/ReadHelpers.h

@@ -154,9 +154,12 @@ inline void readIPv6Binary(IPv6 & ip, ReadBuffer & buf)
     size_t size = 0;
     readVarUInt(size, buf);
 
-    if (size != IPV6_BINARY_LENGTH)
+    if (size != sizeof(IPv6::UnderlyingType))
-        throw Exception(ErrorCodes::SIZE_OF_FIXED_STRING_DOESNT_MATCH,
+        throw Exception(
-                        "Size of the string {} doesn't match size of binary IPv6 {}", size, IPV6_BINARY_LENGTH);
+            ErrorCodes::SIZE_OF_FIXED_STRING_DOESNT_MATCH,
+            "Size of the string {} doesn't match size of binary IPv6 {}",
+            size,
+            sizeof(IPv6::UnderlyingType));
 
     buf.readStrict(reinterpret_cast<char*>(&ip.toUnderType()), size);
 }

src/IO/S3/copyS3File.h

@@ -14,6 +14,7 @@
 
 namespace DB
 {
+struct ReadSettings;
 class SeekableReadBuffer;
 
 using CreateReadBuffer = std::function<std::unique_ptr<SeekableReadBuffer>()>;

src/IO/examples/read_buffer_from_hdfs.cpp

@@ -6,6 +6,8 @@
 #include <base/types.h>
 #include <Common/Config/ConfigProcessor.h>
 
+#include <Poco/Util/MapConfiguration.h>
+
 using namespace DB;
 
 int main()

src/Interpreters/ActionLocksManager.cpp

@@ -1,5 +1,6 @@
 #include "ActionLocksManager.h"
 #include <Interpreters/Context.h>
+#include <Interpreters/DatabaseCatalog.h>
 #include <Databases/IDatabase.h>
 #include <Storages/IStorage.h>
 

src/Interpreters/ActionsVisitor.cpp

@@ -44,6 +44,7 @@
 #include <Interpreters/ExpressionActions.h>
 #include <Interpreters/misc.h>
 #include <Interpreters/ActionsVisitor.h>
+#include <Interpreters/DatabaseCatalog.h>
 #include <Interpreters/Set.h>
 #include <Interpreters/evaluateConstantExpression.h>
 #include <Interpreters/convertFieldToType.h>

src/Interpreters/ActionsVisitor.h

@@ -1,13 +1,14 @@
 #pragma once
 
 #include <string_view>
+#include <Core/ColumnNumbers.h>
+#include <Core/ColumnWithTypeAndName.h>
 #include <Core/NamesAndTypes.h>
 #include <Interpreters/Context_fwd.h>
 #include <Interpreters/InDepthNodeVisitor.h>
 #include <Interpreters/PreparedSets.h>
 #include <Parsers/IAST.h>
-#include <Core/ColumnNumbers.h>
+#include <QueryPipeline/SizeLimits.h>
-#include <Core/ColumnWithTypeAndName.h>
 
 namespace DB
 {

src/Interpreters/Aggregator.cpp

@@ -1867,7 +1867,12 @@ Aggregator::convertToBlockImpl(Method & method, Table & data, Arena * arena, Are
     if (data.empty())
     {
         auto && out_cols = prepareOutputBlockColumns(params, aggregate_functions, getHeader(final), aggregates_pools, final, rows);
-        return {finalizeBlock(params, getHeader(final), std::move(out_cols), final, rows)};
+        auto finalized_block = finalizeBlock(params, getHeader(final), std::move(out_cols), final, rows);
+
+        if (return_single_block)
+            return std::move(finalized_block);
+
+        return BlocksList{std::move(finalized_block)};
     }
     ConvertToBlockResVariant res;
     bool use_compiled_functions = false;

src/Interpreters/AsynchronousInsertQueue.cpp

@@ -11,6 +11,7 @@
 #include <IO/copyData.h>
 #include <Interpreters/AsynchronousInsertLog.h>
 #include <Interpreters/Context.h>
+#include <Interpreters/DatabaseCatalog.h>
 #include <Interpreters/InterpreterInsertQuery.h>
 #include <Interpreters/ProcessList.h>
 #include <Interpreters/executeQuery.h>

src/Interpreters/CollectJoinOnKeysVisitor.h

@@ -1,11 +1,12 @@
 #pragma once
 
+#include <Core/Joins.h>
 #include <Core/Names.h>
+#include <Interpreters/Aliases.h>
+#include <Interpreters/DatabaseAndTableWithAlias.h>
+#include <Interpreters/InDepthNodeVisitor.h>
 #include <Parsers/ASTFunction.h>
 #include <Parsers/queryToString.h>
-#include <Interpreters/InDepthNodeVisitor.h>
-#include <Interpreters/DatabaseAndTableWithAlias.h>
-#include <Interpreters/Aliases.h>
 
 
 namespace DB

src/Interpreters/Context.h

@@ -19,9 +19,8 @@
 #include <Disks/IO/getThreadPoolReader.h>
 #include <Interpreters/ClientInfo.h>
 #include <Interpreters/Context_fwd.h>
-#include <Interpreters/DatabaseCatalog.h>
+#include <Interpreters/StorageID.h>
 #include <Interpreters/MergeTreeTransactionHolder.h>
-#include <Common/Scheduler/IResourceManager.h>
 #include <Parsers/IAST_fwd.h>
 #include <Server/HTTP/HTTPContext.h>
 #include <Storages/ColumnsDescription.h>
@@ -149,6 +148,18 @@ template <class Queue>
 class MergeTreeBackgroundExecutor;
 class AsyncLoader;
 
+struct TemporaryTableHolder;
+using TemporaryTablesMapping = std::map<String, std::shared_ptr<TemporaryTableHolder>>;
+
+class LoadTask;
+using LoadTaskPtr = std::shared_ptr<LoadTask>;
+using LoadTaskPtrs = std::vector<LoadTaskPtr>;
+
+class IClassifier;
+using ClassifierPtr = std::shared_ptr<IClassifier>;
+class IResourceManager;
+using ResourceManagerPtr = std::shared_ptr<IResourceManager>;
+
 /// Scheduling policy can be changed using `background_merges_mutations_scheduling_policy` config option.
 /// By default concurrent merges are scheduled using "round_robin" to ensure fair and starvation-free operation.
 /// Previously in heavily overloaded shards big merges could possibly be starved by smaller

src/Interpreters/DDLTask.cpp

@@ -2,6 +2,8 @@
 #include <base/sort.h>
 #include <Common/DNSResolver.h>
 #include <Common/isLocalAddress.h>
+#include <Databases/DatabaseReplicated.h>
+#include <Interpreters/DatabaseCatalog.h>
 #include <IO/WriteHelpers.h>
 #include <IO/ReadHelpers.h>
 #include <IO/Operators.h>
@@ -14,7 +16,6 @@
 #include <Parsers/parseQuery.h>
 #include <Parsers/queryToString.h>
 #include <Parsers/ASTQueryWithTableAndOutput.h>
-#include <Databases/DatabaseReplicated.h>
 
 
 namespace DB

src/Interpreters/DatabaseCatalog.cpp

@@ -2,6 +2,7 @@
 #include <mutex>
 #include <Interpreters/DatabaseCatalog.h>
 #include <Interpreters/Context.h>
+#include <Interpreters/TableNameHints.h>
 #include <Interpreters/loadMetadata.h>
 #include <Interpreters/executeQuery.h>
 #include <Interpreters/InterpreterCreateQuery.h>
@@ -1142,7 +1143,7 @@ void DatabaseCatalog::dequeueDroppedTableCleanup(StorageID table_id)
     TableMarkedAsDropped dropped_table;
     {
         std::lock_guard lock(tables_marked_dropped_mutex);
-        auto latest_drop_time = std::numeric_limits<time_t>::min();
+        time_t latest_drop_time = std::numeric_limits<time_t>::min();
         auto it_dropped_table = tables_marked_dropped.end();
         for (auto it = tables_marked_dropped.begin(); it != tables_marked_dropped.end(); ++it)
         {
@@ -1167,7 +1168,7 @@ void DatabaseCatalog::dequeueDroppedTableCleanup(StorageID table_id)
         }
         if (it_dropped_table == tables_marked_dropped.end())
             throw Exception(ErrorCodes::UNKNOWN_TABLE,
-                "Table {} is being dropped, has been dropped, or the database engine does not support UNDROP",
+                "The drop task of table {} is in progress, has been dropped or the database engine doesn't support it",
                 table_id.getNameForLogs());
         latest_metadata_dropped_path = it_dropped_table->metadata_path;
         String table_metadata_path = getPathForMetadata(it_dropped_table->table_id);
@@ -1705,4 +1706,43 @@ DDLGuard::~DDLGuard()
     releaseTableLock();
 }
 
+std::pair<String, String> TableNameHints::getHintForTable(const String & table_name) const
+{
+    auto results = this->getHints(table_name, getAllRegisteredNames());
+    if (results.empty())
+        return getExtendedHintForTable(table_name);
+    return std::make_pair(database->getDatabaseName(), results[0]);
+}
+
+std::pair<String, String> TableNameHints::getExtendedHintForTable(const String & table_name) const
+{
+    /// load all available databases from the DatabaseCatalog instance
+    auto & database_catalog = DatabaseCatalog::instance();
+    auto all_databases = database_catalog.getDatabases();
+
+    for (const auto & [db_name, db] : all_databases)
+    {
+        /// this case should be covered already by getHintForTable
+        if (db_name == database->getDatabaseName())
+            continue;
+
+        TableNameHints hints(db, context);
+        auto results = hints.getHints(table_name);
+
+        /// if the results are not empty, return the first instance of the table_name
+        /// and the corresponding database_name that was found.
+        if (!results.empty())
+            return std::make_pair(db_name, results[0]);
+    }
+    return {};
+}
+
+Names TableNameHints::getAllRegisteredNames() const
+{
+    Names result;
+    if (database)
+        for (auto table_it = database->getTablesIterator(context); table_it->isValid(); table_it->next())
+            result.emplace_back(table_it->name());
+    return result;
+}
 }

src/Interpreters/DatabaseCatalog.h

@@ -1,13 +1,11 @@
 #pragma once
 
 #include <Core/UUID.h>
-#include <Databases/IDatabase.h>
 #include <Databases/TablesDependencyGraph.h>
 #include <Interpreters/Context_fwd.h>
 #include <Interpreters/StorageID.h>
 #include <Parsers/IAST_fwd.h>
 #include <Storages/IStorage_fwd.h>
-#include <Common/NamePrompter.h>
 #include <Common/SharedMutex.h>
 
 #include <boost/noncopyable.hpp>
@@ -365,68 +363,6 @@ private:
     static constexpr time_t DBMS_DEFAULT_DISK_RELOAD_PERIOD_SEC = 5;
 };
 
-class TableNameHints : public IHints<>
-{
-public:
-    TableNameHints(ConstDatabasePtr database_, ContextPtr context_)
-        : context(context_),
-        database(database_)
-    {
-    }
-
-    /// getHintForTable tries to get a hint for the provided table_name in the provided
-    /// database. If the results are empty, it goes for extended hints for the table
-    /// with getExtendedHintForTable which looks for the table name in every database that's
-    /// available in the database catalog. It finally returns a single hint which is the database
-    /// name and table_name pair which is similar to the table_name provided. Perhaps something to
-    /// consider is should we return more than one pair of hint?
-    std::pair<String, String> getHintForTable(const String & table_name) const
-    {
-        auto results = this->getHints(table_name, getAllRegisteredNames());
-        if (results.empty())
-            return getExtendedHintForTable(table_name);
-        return std::make_pair(database->getDatabaseName(), results[0]);
-    }
-
-    /// getExtendedHintsForTable tries to get hint for the given table_name across all
-    /// the databases that are available in the database catalog.
-    std::pair<String, String> getExtendedHintForTable(const String & table_name) const
-    {
-        /// load all available databases from the DatabaseCatalog instance
-        auto & database_catalog = DatabaseCatalog::instance();
-        auto all_databases = database_catalog.getDatabases();
-
-        for (const auto & [db_name, db] : all_databases)
-        {
-            /// this case should be covered already by getHintForTable
-            if (db_name == database->getDatabaseName())
-                continue;
-
-            TableNameHints hints(db, context);
-            auto results = hints.getHints(table_name);
-
-            /// if the results are not empty, return the first instance of the table_name
-            /// and the corresponding database_name that was found.
-            if (!results.empty())
-                return std::make_pair(db_name, results[0]);
-        }
-        return {};
-    }
-
-    Names getAllRegisteredNames() const override
-    {
-        Names result;
-        if (database)
-            for (auto table_it = database->getTablesIterator(context); table_it->isValid(); table_it->next())
-                result.emplace_back(table_it->name());
-        return result;
-    }
-
-private:
-    ContextPtr context;
-    ConstDatabasePtr database;
-};
-
 
 /// This class is useful when creating a table or database.
 /// Usually we create IStorage/IDatabase object first and then add it to IDatabase/DatabaseCatalog.

src/Interpreters/FullSortingMergeJoin.h

@@ -21,9 +21,11 @@ namespace ErrorCodes
 class FullSortingMergeJoin : public IJoin
 {
 public:
-    explicit FullSortingMergeJoin(std::shared_ptr<TableJoin> table_join_, const Block & right_sample_block_)
+    explicit FullSortingMergeJoin(std::shared_ptr<TableJoin> table_join_, const Block & right_sample_block_,
+                                  int null_direction_ = 1)
         : table_join(table_join_)
         , right_sample_block(right_sample_block_)
+        , null_direction(null_direction_)
     {
         LOG_TRACE(getLogger("FullSortingMergeJoin"), "Will use full sorting merge join");
     }
@@ -31,6 +33,8 @@ public:
     std::string getName() const override { return "FullSortingMergeJoin"; }
     const TableJoin & getTableJoin() const override { return *table_join; }
 
+    int getNullDirection() const { return null_direction; }
+
     bool addBlockToJoin(const Block & /* block */, bool /* check_limits */) override
     {
         throw Exception(ErrorCodes::LOGICAL_ERROR, "FullSortingMergeJoin::addBlockToJoin should not be called");
@@ -119,6 +123,7 @@ private:
     std::shared_ptr<TableJoin> table_join;
     Block right_sample_block;
     Block totals;
+    int null_direction;
 };
 
 }

src/Interpreters/IKeyValueEntity.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <Core/Block.h>
 #include <Core/Names.h>
 #include <Processors/Chunk.h>
 

src/Interpreters/InJoinSubqueriesPreprocessor.cpp

@@ -1,6 +1,7 @@
 #include <Interpreters/InJoinSubqueriesPreprocessor.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/DatabaseAndTableWithAlias.h>
+#include <Interpreters/DatabaseCatalog.h>
 #include <Interpreters/IdentifierSemantic.h>
 #include <Interpreters/InDepthNodeVisitor.h>
 #include <Storages/StorageDistributed.h>

src/Interpreters/InterpreterCheckQuery.cpp

@@ -16,6 +16,7 @@
 #include <DataTypes/DataTypeString.h>
 
 #include <Interpreters/Context.h>
+#include <Interpreters/DatabaseCatalog.h>
 #include <Interpreters/ProcessList.h>
 
 #include <Parsers/ASTCheckQuery.h>

src/Interpreters/InterpreterCreateIndexQuery.cpp

@@ -1,10 +1,11 @@
-#include <Interpreters/InterpreterFactory.h>
-#include <Interpreters/InterpreterCreateIndexQuery.h>
 
 #include <Access/ContextAccess.h>
 #include <Databases/DatabaseReplicated.h>
 #include <Interpreters/Context.h>
+#include <Interpreters/DatabaseCatalog.h>
 #include <Interpreters/executeDDLQueryOnCluster.h>
+#include <Interpreters/InterpreterFactory.h>
+#include <Interpreters/InterpreterCreateIndexQuery.h>
 #include <Interpreters/FunctionNameNormalizer.h>
 #include <Parsers/ASTCreateIndexQuery.h>
 #include <Parsers/ASTIdentifier.h>

src/Interpreters/InterpreterDescribeQuery.cpp

@@ -9,6 +9,7 @@
 #include <Interpreters/InterpreterSelectWithUnionQuery.h>
 #include <Interpreters/InterpreterSelectQueryAnalyzer.h>
 #include <Interpreters/Context.h>
+#include <Interpreters/DatabaseCatalog.h>
 #include <Interpreters/InterpreterFactory.h>
 #include <Interpreters/InterpreterDescribeQuery.h>
 #include <Interpreters/IdentifierSemantic.h>