Merge branch 'ClickHouse:master' into master

.github/workflows/main.yml

@@ -152,7 +152,7 @@ jobs:
           sudo rm -fr $TEMP_PATH
           mkdir -p $TEMP_PATH
           cp -r $GITHUB_WORKSPACE $TEMP_PATH
-          cd $REPO_COPY/tests/ci && python3 compatibility_check.py 0
+          cd $REPO_COPY/tests/ci && python3 compatibility_check.py
       - name: Cleanup
         if: always()
         run: |
@@ -518,6 +518,7 @@ jobs:
       - BuilderDebDebug
       - BuilderDebSplitted
     runs-on: [self-hosted, style-checker]
+    if: always()
     steps:
       - name: Download json reports
         uses: actions/download-artifact@v2
@@ -604,7 +605,9 @@ jobs:
           sudo rm -fr $TEMP_PATH
   FunctionalStatelessTestTsan:
     needs: [BuilderDebTsan]
-    runs-on: [self-hosted, func-tester]
+    # tests can consume more than 60GB of memory,
+    # so use bigger server
+    runs-on: [self-hosted, stress-tester]
     steps:
       - name: Download json reports
         uses: actions/download-artifact@v2
@@ -1297,6 +1300,34 @@ jobs:
           docker kill $(docker ps -q) ||:
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr $TEMP_PATH
+  IntegrationTestsFlakyCheck:
+    needs: [BuilderDebAsan]
+    runs-on: [self-hosted, stress-tester]
+    steps:
+      - name: Download json reports
+        uses: actions/download-artifact@v2
+        with:
+          path: ${{runner.temp}}/reports_dir
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Integration test
+        env:
+          TEMP_PATH: ${{runner.temp}}/integration_tests_asan_flaky_check
+          REPORTS_PATH: ${{runner.temp}}/reports_dir
+          CHECK_NAME: 'Integration tests flaky check (asan, actions)'
+          REPO_COPY: ${{runner.temp}}/integration_tests_asan_flaky_check/ClickHouse
+        run: |
+          sudo rm -fr $TEMP_PATH
+          mkdir -p $TEMP_PATH
+          cp -r $GITHUB_WORKSPACE $TEMP_PATH
+          cd $REPO_COPY/tests/ci
+          python3 integration_test_check.py "$CHECK_NAME"
+      - name: Cleanup
+        if: always()
+        run: |
+          docker kill $(docker ps -q) ||:
+          docker rm -f $(docker ps -a -q) ||:
+          sudo rm -fr $TEMP_PATH
 #############################################################################################
 #################################### UNIT TESTS #############################################
 #############################################################################################
@@ -1481,6 +1512,7 @@ jobs:
       - UnitTestsReleaseClang
       - SplitBuildSmokeTest
       - CompatibilityCheck
+      - IntegrationTestsFlakyCheck
     runs-on: [self-hosted, style-checker]
     steps:
       - name: Check out repository code

CMakeLists.txt

@@ -169,9 +169,7 @@ endif ()
 include (cmake/check_flags.cmake)
 include (cmake/add_warning.cmake)
 
-if (NOT MSVC)
 set (COMMON_WARNING_FLAGS "${COMMON_WARNING_FLAGS} -Wall")    # -Werror and many more is also added inside cmake/warnings.cmake
-endif ()
 
 if (COMPILER_CLANG)
     # clang: warning: argument unused during compilation: '-specs=/usr/share/dpkg/no-pie-compile.specs' [-Wunused-command-line-argument]
@@ -312,6 +310,10 @@ include(cmake/cpu_features.cmake)
 # Enable it explicitly.
 set (COMPILER_FLAGS "${COMPILER_FLAGS} -fasynchronous-unwind-tables")
 
+# Reproducible builds
+set (COMPILER_FLAGS "${COMPILER_FLAGS} -ffile-prefix-map=${CMAKE_SOURCE_DIR}=.")
+set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} -ffile-prefix-map=${CMAKE_SOURCE_DIR}=.")
+
 if (${CMAKE_VERSION} VERSION_LESS "3.12.4")
     # CMake < 3.12 doesn't support setting 20 as a C++ standard version.
     # We will add C++ standard controlling flag in CMAKE_CXX_FLAGS manually for now.

PreLoad.cmake

@@ -18,3 +18,26 @@ if (NOT DEFINED ENV{CLION_IDE} AND NOT DEFINED ENV{XCODE_IDE})
         set(CMAKE_GENERATOR "Ninja" CACHE INTERNAL "" FORCE)
     endif ()
 endif()
+
+
+# Default toolchain - this is needed to avoid dependency on OS files.
+execute_process(COMMAND uname -s OUTPUT_VARIABLE OS)
+execute_process(COMMAND uname -m OUTPUT_VARIABLE ARCH)
+
+if (OS MATCHES "Linux"
+    AND NOT DEFINED CMAKE_TOOLCHAIN_FILE
+    AND NOT DISABLE_HERMETIC_BUILD
+    AND ($ENV{CC} MATCHES ".*clang.*" OR CMAKE_C_COMPILER MATCHES ".*clang.*")
+    AND (USE_STATIC_LIBRARIES OR NOT DEFINED USE_STATIC_LIBRARIES))
+
+    if (ARCH MATCHES "amd64|x86_64")
+        set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-x86_64.cmake" CACHE INTERNAL "" FORCE)
+    elseif (ARCH MATCHES "^(aarch64.*|AARCH64.*|arm64.*|ARM64.*)")
+        set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-aarch64.cmake" CACHE INTERNAL "" FORCE)
+    elseif (ARCH MATCHES "^(ppc64le.*|PPC64LE.*)")
+        set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-ppc64le.cmake" CACHE INTERNAL "" FORCE)
+    else ()
+        message (FATAL_ERROR "Unsupported architecture: ${ARCH}")
+    endif ()
+
+endif()

SECURITY.md

@@ -19,9 +19,11 @@ The following versions of ClickHouse server are currently being supported with s
 | 21.4   | :x: |
 | 21.5   | :x: |
 | 21.6   | :x: |
-| 21.7   | ✅ |
+| 21.7   | :x: |
 | 21.8   | ✅ |
 | 21.9   | ✅ |
+| 21.10   | ✅ |
+| 21.11   | ✅ |
 
 ## Reporting a Vulnerability
 

base/base/CachedFn.h

@@ -18,8 +18,8 @@ struct CachedFn
 {
 private:
     using Traits = FnTraits<decltype(Func)>;
-    using DecayedArgs = TLMap<std::decay_t, typename Traits::Args>;
+    using DecayedArgs = TypeListMap<std::decay_t, typename Traits::Args>;
-    using Key = TLChangeRoot<std::tuple, DecayedArgs>;
+    using Key = TypeListChangeRoot<std::tuple, DecayedArgs>;
     using Result = typename Traits::Ret;
 
     std::map<Key, Result> cache; // Can't use hashmap as tuples are unhashable by default

base/base/FnTraits.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "Typelist.h"
+#include "TypeList.h"
 
 namespace detail
 {
@@ -14,7 +14,7 @@ struct FnTraits<R(A...)>
     static constexpr bool value = std::is_invocable_r_v<R, F, A...>;
 
     using Ret = R;
-    using Args = Typelist<A...>;
+    using Args = TypeList<A...>;
 };
 
 template <class R, class ...A>

base/base/TypeList.h

@@ -0,0 +1,44 @@
+#pragma once
+
+#include <cstddef>
+#include <type_traits>
+#include <utility>
+#include "defines.h"
+#include "TypePair.h"
+
+/// General-purpose typelist. Easy on compilation times as it does not use recursion.
+template <typename ...Args>
+struct TypeList { static constexpr size_t size = sizeof...(Args); };
+
+namespace TypeListUtils /// In some contexts it's more handy to use functions instead of aliases
+{
+    template <typename ...LArgs, typename ...RArgs>
+    constexpr TypeList<LArgs..., RArgs...> concat(TypeList<LArgs...>, TypeList<RArgs...>) { return {}; }
+
+    template <typename T, typename ...Args>
+    constexpr TypeList<T, Args...> prepend(TypeList<Args...>) { return {}; }
+
+    template <typename T, typename ...Args>
+    constexpr TypeList<Args..., T> append(TypeList<Args...>) { return {}; }
+
+    template <template <typename> typename F, typename ...Args>
+    constexpr TypeList<F<Args>...> map(TypeList<Args...>) { return {}; }
+
+    template <template <typename...> typename Root, typename ...Args>
+    constexpr Root<Args...> changeRoot(TypeList<Args...>) { return {}; }
+
+    template <typename F, typename ...Args>
+    constexpr void forEach(TypeList<Args...>, F && f) { (std::forward<F>(f)(Id<Args>{}), ...); }
+}
+
+template <typename TypeListLeft, typename TypeListRight>
+using TypeListConcat = decltype(TypeListUtils::concat(TypeListLeft{}, TypeListRight{}));
+
+template <typename T, typename List> using TypeListPrepend = decltype(TypeListUtils::prepend<T>(List{}));
+template <typename T, typename List> using TypeListAppend = decltype(TypeListUtils::append<T>(List{}));
+
+template <template <typename> typename F, typename List>
+using TypeListMap = decltype(TypeListUtils::map<F>(List{}));
+
+template <template <typename...> typename Root, typename List>
+using TypeListChangeRoot = decltype(TypeListUtils::changeRoot<Root>(List{}));

base/base/TypeLists.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include "TypeList.h"
+#include "extended_types.h"
+#include "Decimal.h"
+#include "UUID.h"
+
+namespace DB
+{
+
+using TypeListNativeInt = TypeList<UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64>;
+using TypeListFloat = TypeList<Float32, Float64>;
+using TypeListNativeNumber = TypeListConcat<TypeListNativeInt, TypeListFloat>;
+using TypeListWideInt = TypeList<UInt128, Int128, UInt256, Int256>;
+using TypeListInt = TypeListConcat<TypeListNativeInt, TypeListWideInt>;
+using TypeListIntAndFloat = TypeListConcat<TypeListInt, TypeListFloat>;
+using TypeListDecimal = TypeList<Decimal32, Decimal64, Decimal128, Decimal256>;
+using TypeListNumber = TypeListConcat<TypeListIntAndFloat, TypeListDecimal>;
+using TypeListNumberWithUUID = TypeListAppend<UUID, TypeListNumber>;
+
+}

base/base/TypePair.h

@@ -1,4 +1,4 @@
 #pragma once
 
-template <class T, class V> struct TypePair { };
+template <typename T, typename V> struct TypePair {};
-template <class T> struct Id { };
+template <typename T> struct Id {};

base/base/Typelist.h

@@ -1,44 +0,0 @@
-#pragma once
-
-#include <cstddef>
-#include <type_traits>
-#include <utility>
-#include "defines.h"
-#include "TypePair.h"
-
-/// General-purpose typelist. Easy on compilation times as it does not use recursion.
-template <class ...Args>
-struct Typelist { static constexpr size_t size = sizeof...(Args); };
-
-namespace TLUtils /// In some contexts it's more handy to use functions instead of aliases
-{
-template <class ...LArgs, class ...RArgs>
-constexpr Typelist<LArgs..., RArgs...> concat(Typelist<LArgs...>, Typelist<RArgs...>) { return {}; }
-
-template <class T, class ...Args>
-constexpr Typelist<T, Args...> prepend(Typelist<Args...>) { return {}; }
-
-template <class T, class ...Args>
-constexpr Typelist<Args..., T> append(Typelist<Args...>) { return {}; }
-
-template <template<class> class F, class ...Args>
-constexpr Typelist<F<Args>...> map(Typelist<Args...>) { return {}; }
-
-template <template<class...> class Root, class ...Args>
-constexpr Root<Args...> changeRoot(Typelist<Args...>) { return {}; }
-
-template <class F, class ...Args>
-constexpr void forEach(Typelist<Args...>, F && f) { (std::forward<F>(f)(Id<Args>{}), ...); }
-}
-
-template <class TLLeft, class TLRight>
-using TLConcat = decltype(TLUtils::concat(TLLeft{}, TLRight{}));
-
-template <class T, class Typelist> using TLPrepend = decltype(TLUtils::prepend<T>(Typelist{}));
-template <class T, class Typelist> using TLAppend = decltype(TLUtils::append<T>(Typelist{}));
-
-template <template<class> class F, class Typelist>
-using TLMap = decltype(TLUtils::map<F>(Typelist{}));
-
-template <template<class...> class Root, class Typelist>
-using TLChangeRoot = decltype(TLUtils::changeRoot<Root>(Typelist{}));

base/base/Typelists.h

@@ -1,18 +0,0 @@
-#pragma once
-
-#include "Typelist.h"
-#include "extended_types.h"
-#include "Decimal.h"
-#include "UUID.h"
-
-namespace DB
-{
-using TLIntegral = Typelist<UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64>;
-using TLExtendedIntegral = Typelist<UInt128, Int128, UInt256, Int256>;
-using TLDecimals = Typelist<Decimal32, Decimal64, Decimal128, Decimal256>;
-
-using TLIntegralWithExtended = TLConcat<TLIntegral, TLExtendedIntegral>;
-
-using TLNumbers = TLConcat<TLIntegralWithExtended, TLDecimals>;
-using TLNumbersWithUUID = TLAppend<UUID, TLNumbers>;
-}

base/base/iostream_debug_helpers.h

@@ -6,6 +6,7 @@
 #include <tuple>
 #include <iomanip>
 #include <iostream>
+#include <magic_enum.hpp>
 
 /** Usage:
   *
@@ -61,6 +62,13 @@ std::enable_if_t<priority == 2, Out> & dumpImpl(Out & out, T && x, std::decay_t<
 }
 
 
+template <int priority, typename Out, typename T>
+std::enable_if_t<priority == 3 && std::is_enum_v<std::decay_t<T>>, Out> &
+dumpImpl(Out & out, T && x)
+{
+    return out << magic_enum::enum_name(x);
+}
+
 /// string and const char * - output not as container or pointer.
 
 template <int priority, typename Out, typename T>
@@ -131,15 +139,26 @@ Out & dumpValue(Out & out, T && x)
 template <typename Out, typename T>
 Out & dump(Out & out, const char * name, T && x)
 {
+    // Dumping string literal, printing name and demangled type is irrelevant.
+    if constexpr (std::is_same_v<const char *, std::decay_t<std::remove_reference_t<T>>>)
+    {
+         const auto name_len = strlen(name);
+         const auto value_len = strlen(x);
+         // `name` is the same as quoted `x`
+         if (name_len > 2 && value_len > 0 && name[0] == '"' && name[name_len - 1] == '"'
+                 && strncmp(name + 1, x, std::min(value_len, name_len) - 1) == 0)
+             return out << x;
+    }
+
     out << demangle(typeid(x).name()) << " " << name << " = ";
-    return dumpValue(out, x);
+    return dumpValue(out, x) << "; ";
 }
 
 #ifdef __clang__
 #pragma clang diagnostic ignored "-Wgnu-zero-variadic-macro-arguments"
 #endif
 
-#define DUMPVAR(VAR) ::dump(std::cerr, #VAR, (VAR)); std::cerr << "; ";
+#define DUMPVAR(VAR) ::dump(std::cerr, #VAR, (VAR));
 #define DUMPHEAD std::cerr << __FILE__ << ':' << __LINE__ << " [ " << getThreadId() << " ] ";
 #define DUMPTAIL std::cerr << '\n';
 

cmake/add_warning.cmake

@@ -12,13 +12,13 @@ macro (add_warning flag)
     if (SUPPORTS_CXXFLAG_${underscored_flag})
         set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W${flag}")
     else ()
-        message (WARNING "Flag -W${flag} is unsupported")
+        message (STATUS "Flag -W${flag} is unsupported")
     endif ()
 
     if (SUPPORTS_CFLAG_${underscored_flag})
         set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -W${flag}")
     else ()
-        message (WARNING "Flag -W${flag} is unsupported")
+        message (STATUS "Flag -W${flag} is unsupported")
     endif ()
 
 endmacro ()
@@ -39,7 +39,7 @@ macro (target_add_warning target flag)
     if (SUPPORTS_CXXFLAG_${underscored_flag})
         target_compile_options (${target} PRIVATE "-W${flag}")
     else ()
-        message (WARNING "Flag -W${flag} is unsupported")
+        message (STATUS "Flag -W${flag} is unsupported")
     endif ()
 endmacro ()
 

cmake/find/parquet.cmake

@@ -1,5 +1,5 @@
 if (Protobuf_PROTOC_EXECUTABLE)
-    option (ENABLE_PARQUET "Enable parquet" ON)
+    option (ENABLE_PARQUET "Enable parquet" ${ENABLE_LIBRARIES})
 elseif(ENABLE_PARQUET OR USE_INTERNAL_PARQUET_LIBRARY)
     message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use parquet without protoc executable")
 endif()

cmake/freebsd/toolchain-aarch64.cmake

@@ -3,7 +3,7 @@ set (CMAKE_SYSTEM_PROCESSOR "aarch64")
 set (CMAKE_C_COMPILER_TARGET "aarch64-unknown-freebsd12")
 set (CMAKE_CXX_COMPILER_TARGET "aarch64-unknown-freebsd12")
 set (CMAKE_ASM_COMPILER_TARGET "aarch64-unknown-freebsd12")
-set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../toolchain/freebsd-aarch64")
+set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/freebsd-aarch64")
 
 set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)  # disable linkage check - it doesn't work in CMake
 

cmake/freebsd/toolchain-x86_64.cmake

@@ -3,7 +3,7 @@ set (CMAKE_SYSTEM_PROCESSOR "x86_64")
 set (CMAKE_C_COMPILER_TARGET "x86_64-pc-freebsd11")
 set (CMAKE_CXX_COMPILER_TARGET "x86_64-pc-freebsd11")
 set (CMAKE_ASM_COMPILER_TARGET "x86_64-pc-freebsd11")
-set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../toolchain/freebsd-x86_64")
+set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/freebsd-x86_64")
 
 set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)  # disable linkage check - it doesn't work in CMake
 

cmake/linux/default_libs.cmake

@@ -5,8 +5,12 @@ set (DEFAULT_LIBS "-nodefaultlibs")
 
 # We need builtins from Clang's RT even without libcxx - for ubsan+int128.
 # See https://bugs.llvm.org/show_bug.cgi?id=16404
-if (COMPILER_CLANG AND NOT CMAKE_CROSSCOMPILING)
+if (COMPILER_CLANG)
-    execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-libgcc-file-name --rtlib=compiler-rt OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE)
+    execute_process (COMMAND ${CMAKE_CXX_COMPILER} --target=${CMAKE_CXX_COMPILER_TARGET} --print-libgcc-file-name --rtlib=compiler-rt OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+    if (NOT EXISTS "${BUILTINS_LIBRARY}")
+        set (BUILTINS_LIBRARY "-lgcc")
+    endif ()
 else ()
     set (BUILTINS_LIBRARY "-lgcc")
 endif ()

cmake/print_flags.cmake

@@ -1,6 +1,12 @@
 set (FULL_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE_UC}}")
 set (FULL_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UC}}")
 set (FULL_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${CMAKE_EXE_LINKER_FLAGS_${CMAKE_BUILD_TYPE_UC}}")
+
 message (STATUS "compiler C   = ${CMAKE_C_COMPILER} ${FULL_C_FLAGS}")
 message (STATUS "compiler CXX = ${CMAKE_CXX_COMPILER} ${FULL_CXX_FLAGS}")
 message (STATUS "LINKER_FLAGS = ${FULL_EXE_LINKER_FLAGS}")
+
+# Reproducible builds
+string (REPLACE "${CMAKE_SOURCE_DIR}" "." FULL_C_FLAGS_NORMALIZED "${FULL_C_FLAGS}")
+string (REPLACE "${CMAKE_SOURCE_DIR}" "." FULL_CXX_FLAGS_NORMALIZED "${FULL_CXX_FLAGS}")
+string (REPLACE "${CMAKE_SOURCE_DIR}" "." FULL_EXE_LINKER_FLAGS_NORMALIZED "${FULL_EXE_LINKER_FLAGS}")

cmake/sanitize.cmake

@@ -78,6 +78,15 @@ if (SANITIZE)
 
     elseif (SANITIZE STREQUAL "undefined")
         set (UBSAN_FLAGS "-fsanitize=undefined -fno-sanitize-recover=all -fno-sanitize=float-divide-by-zero")
+        if (ENABLE_FUZZING)
+            # Unsigned integer overflow is well defined behaviour from a perspective of C++ standard,
+            # compilers or CPU. We use in hash functions like SipHash and many other places in our codebase.
+            # This flag is needed only because fuzzers are run inside oss-fuzz infrastructure
+            # and they have a bunch of flags not halt the program if UIO happend and even to silence that warnings.
+            # But for unknown reason that flags don't work with ClickHouse or we don't understand how to properly use them,
+            # that's why we often receive reports about UIO. The simplest way to avoid this is just  set this flag here.
+            set(UBSAN_FLAGS "${SAN_FLAGS} -fno-sanitize=unsigned-integer-overflow")
+        endif()
         if (COMPILER_CLANG)
             set (UBSAN_FLAGS "${UBSAN_FLAGS} -fsanitize-blacklist=${CMAKE_SOURCE_DIR}/tests/ubsan_suppressions.txt")
         else()

contrib/cctz-cmake/CMakeLists.txt

@@ -82,7 +82,7 @@ if (NOT EXTERNAL_CCTZ_LIBRARY_FOUND OR NOT EXTERNAL_CCTZ_LIBRARY_WORKS)
 
     # each file in that dir (except of tab and localtime) store the info about timezone
     execute_process(COMMAND
-        bash -c "cd ${TZDIR} && find * -type f -and ! -name '*.tab' -and ! -name 'localtime' | sort | paste -sd ';' -"
+        bash -c "cd ${TZDIR} && find * -type f -and ! -name '*.tab' -and ! -name 'localtime' | LC_ALL=C sort | paste -sd ';' -"
         OUTPUT_STRIP_TRAILING_WHITESPACE
         OUTPUT_VARIABLE TIMEZONES)
 

contrib/krb5-cmake/CMakeLists.txt

@@ -500,7 +500,6 @@ function(preprocess_et out_var)
       COMMAND perl "${KRB5_SOURCE_DIR}/util/et/compile_et" -d "${KRB5_SOURCE_DIR}/util/et" ${in_f}
       DEPENDS ${in_f} "${KRB5_SOURCE_DIR}/util/et/compile_et"
       WORKING_DIRECTORY ${ET_PATH}
-      COMMENT "Creating preprocessed file ${F_C}"
       VERBATIM
       )
     list(APPEND result ${F_C})
@@ -526,7 +525,6 @@ add_custom_command(
 add_custom_target(
     ERROR_MAP_H
     DEPENDS "${KRB5_SOURCE_DIR}/lib/gssapi/krb5/error_map.h"
-    COMMENT "generating error_map.h"
     VERBATIM
 )
 
@@ -539,14 +537,12 @@ add_custom_command(
 add_custom_target(
     ERRMAP_H
     DEPENDS "${KRB5_SOURCE_DIR}/lib/gssapi/generic/errmap.h"
-    COMMENT "generating errmap.h"
     VERBATIM
 )
 
 add_custom_target(
     KRB_5_H
     DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/include/krb5/krb5.h"
-    COMMENT "generating krb5.h"
     VERBATIM
 )
 

contrib/libhdfs3

@@ -1 +1 @@
-Subproject commit a8c37ee001af1ae88e5dfa637ae5b31b087c96d3
+Subproject commit 9194af44588633c1b2dae44bf945804401ff883e

contrib/protobuf-cmake/CMakeLists.txt

@@ -110,6 +110,9 @@ set(libprotobuf_files
 )
 
 add_library(libprotobuf ${libprotobuf_lite_files} ${libprotobuf_files})
+if (ENABLE_FUZZING)
+    target_compile_options(libprotobuf PRIVATE "-fsanitize-recover=all")
+endif()
 target_link_libraries(libprotobuf pthread)
 target_link_libraries(libprotobuf ${ZLIB_LIBRARIES})
 if(${CMAKE_SYSTEM_NAME} STREQUAL "Android")
@@ -211,7 +214,9 @@ add_library(protobuf::libprotoc ALIAS libprotoc)
 
 set(protoc_files ${protobuf_source_dir}/src/google/protobuf/compiler/main.cc)
 
-if (NOT CMAKE_CROSSCOMPILING)
+if (CMAKE_HOST_SYSTEM_NAME STREQUAL CMAKE_SYSTEM_NAME
+    AND CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL CMAKE_SYSTEM_PROCESSOR)
+
     add_executable(protoc ${protoc_files})
     target_link_libraries(protoc libprotoc libprotobuf pthread)
     add_executable(protobuf::protoc ALIAS protoc)

contrib/sysroot

@@ -1 +1 @@
-Subproject commit 1a64956aa7c280448be6526251bb2b8e6d380ab1
+Subproject commit 4ef348b7f30f2ad5b02b266268b3c948e51ad457

docker/packager/binary/Dockerfile

@@ -93,9 +93,6 @@ RUN git clone https://github.com/tpoechtrager/cctools-port.git \
 # Download toolchain and SDK for Darwin
 RUN wget -nv https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX11.0.sdk.tar.xz
 
-# Download toolchain for FreeBSD 11.3
-RUN wget -nv https://clickhouse-datasets.s3.yandex.net/toolchains/toolchains/freebsd-11.3-toolchain.tar.xz
-
 # NOTE: Seems like gcc-11 is too new for ubuntu20 repository
 RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \
     && apt-get update \

docker/packager/binary/build.sh

@@ -6,9 +6,6 @@ mkdir -p build/cmake/toolchain/darwin-x86_64
 tar xJf MacOSX11.0.sdk.tar.xz -C build/cmake/toolchain/darwin-x86_64 --strip-components=1
 ln -sf darwin-x86_64 build/cmake/toolchain/darwin-aarch64
 
-mkdir -p build/cmake/toolchain/freebsd-x86_64
-tar xJf freebsd-11.3-toolchain.tar.xz -C build/cmake/toolchain/freebsd-x86_64 --strip-components=1
-
 # Uncomment to debug ccache. Don't put ccache log in /output right away, or it
 # will be confusingly packed into the "performance" package.
 # export CCACHE_LOGFILE=/build/ccache.log

docker/test/base/Dockerfile

@@ -52,14 +52,18 @@ RUN apt-get update \
         --yes --no-install-recommends
 
 # Sanitizer options for services (clickhouse-server)
-RUN echo "TSAN_OPTIONS='verbosity=1000 halt_on_error=1 history_size=7'" >> /etc/environment; \
+# Set resident memory limit for TSAN to 45GiB (46080MiB) to avoid OOMs in Stress tests
+# and MEMORY_LIMIT_EXCEEDED exceptions in Functional tests (total memory limit in Functional tests is ~55.24 GiB).
+# TSAN will flush shadow memory when reaching this limit.
+# It may cause false-negatives, but it's better than OOM.
+RUN echo "TSAN_OPTIONS='verbosity=1000 halt_on_error=1 history_size=7 memory_limit_mb=46080'" >> /etc/environment; \
   echo "UBSAN_OPTIONS='print_stacktrace=1'" >> /etc/environment; \
   echo "MSAN_OPTIONS='abort_on_error=1 poison_in_dtor=1'" >> /etc/environment; \
   echo "LSAN_OPTIONS='suppressions=/usr/share/clickhouse-test/config/lsan_suppressions.txt'" >> /etc/environment; \
   ln -s /usr/lib/llvm-${LLVM_VERSION}/bin/llvm-symbolizer /usr/bin/llvm-symbolizer;
 # Sanitizer options for current shell (not current, but the one that will be spawned on "docker run")
 # (but w/o verbosity for TSAN, otherwise test.reference will not match)
-ENV TSAN_OPTIONS='halt_on_error=1 history_size=7'
+ENV TSAN_OPTIONS='halt_on_error=1 history_size=7 memory_limit_mb=46080'
 ENV UBSAN_OPTIONS='print_stacktrace=1'
 ENV MSAN_OPTIONS='abort_on_error=1 poison_in_dtor=1'
 

docker/test/fasttest/run.sh

@@ -159,6 +159,7 @@ function clone_submodules
         cd "$FASTTEST_SOURCE"
 
         SUBMODULES_TO_UPDATE=(
+            contrib/sysroot
             contrib/magic_enum
             contrib/abseil-cpp
             contrib/boost

docker/test/fuzzer/run-fuzzer.sh

@@ -77,7 +77,7 @@ function configure
 
 function watchdog
 {
-    sleep 3600
+    sleep 1800
 
     echo "Fuzzing run has timed out"
     for _ in {1..10}

docker/test/performance-comparison/report.py

@@ -227,10 +227,20 @@ def tableEnd():
     return '</table>'
 
 def tsvRows(n):
-    result = []
     try:
         with open(n, encoding='utf-8') as fd:
-            return [row for row in csv.reader(fd, delimiter="\t", quotechar='"')]
+            result = []
+            for row in csv.reader(fd, delimiter="\t", quoting=csv.QUOTE_NONE):
+                new_row = []
+                for e in row:
+                    # The first one .encode('utf-8').decode('unicode-escape') decodes the escape characters from the strings.
+                    # The second one (encode('latin1').decode('utf-8')) fixes the changes with unicode vs utf-8 chars, so
+                    # 'Чем зÐ<C2B7>нимаеÑ<C2B5>ЬÑ<C2AC>Ñ<EFBFBD>' is transformed back into 'Чем зАнимаешЬся'.
+
+                    new_row.append(e.encode('utf-8').decode('unicode-escape').encode('latin1').decode('utf-8'))
+                result.append(new_row)
+        return result
+
     except:
         report_errors.append(
             traceback.format_exception_only(

docker/test/pvs/Dockerfile

@@ -40,7 +40,7 @@ RUN set -x \
 ENV CCACHE_DIR=/test_output/ccache
 
 CMD echo "Running PVS version $PKG_VERSION" && mkdir -p $CCACHE_DIR && cd /repo_folder && pvs-studio-analyzer credentials $LICENCE_NAME $LICENCE_KEY -o ./licence.lic  \
-    && cmake . -D"ENABLE_EMBEDDED_COMPILER"=OFF -D"USE_INTERNAL_PROTOBUF_LIBRARY"=OFF -D"USE_INTERNAL_GRPC_LIBRARY"=OFF -DCMAKE_C_COMPILER=clang-13 -DCMAKE_CXX_COMPILER=clang\+\+-13 \
+    && cmake . -D"ENABLE_EMBEDDED_COMPILER"=OFF -D"DISABLE_HERMETIC_BUILD"=ON -DCMAKE_C_COMPILER=clang-13 -DCMAKE_CXX_COMPILER=clang\+\+-13 \
     && ninja re2_st clickhouse_grpc_protos \
     && pvs-studio-analyzer analyze -o pvs-studio.log -e contrib -j 4 -l ./licence.lic; \
     cp /repo_folder/pvs-studio.log /test_output; \

docker/test/stateful/run.sh

@@ -120,8 +120,12 @@ timeout "$MAX_RUN_TIME" bash -c run_tests ||:
 ./process_functional_tests_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv
 
 grep -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server.log ||:
+
 pigz < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.gz ||:
+# FIXME: remove once only github actions will be left
+rm /var/log/clickhouse-server/clickhouse-server.log
 mv /var/log/clickhouse-server/stderr.log /test_output/ ||:
+
 if [[ -n "$WITH_COVERAGE" ]] && [[ "$WITH_COVERAGE" -eq 1 ]]; then
     tar -chf /test_output/clickhouse_coverage.tar.gz /profraw ||:
 fi
@@ -130,6 +134,9 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]
     grep -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server2.log ||:
     pigz < /var/log/clickhouse-server/clickhouse-server1.log > /test_output/clickhouse-server1.log.gz ||:
     pigz < /var/log/clickhouse-server/clickhouse-server2.log > /test_output/clickhouse-server2.log.gz ||:
+    # FIXME: remove once only github actions will be left
+    rm /var/log/clickhouse-server/clickhouse-server1.log
+    rm /var/log/clickhouse-server/clickhouse-server2.log
     mv /var/log/clickhouse-server/stderr1.log /test_output/ ||:
     mv /var/log/clickhouse-server/stderr2.log /test_output/ ||:
 fi

docker/test/stateless/run.sh

@@ -135,6 +135,8 @@ done
 
 wait ||:
 
+# Compressed (FIXME: remove once only github actions will be left)
+rm /var/log/clickhouse-server/clickhouse-server.log
 mv /var/log/clickhouse-server/stderr.log /test_output/ ||:
 if [[ -n "$WITH_COVERAGE" ]] && [[ "$WITH_COVERAGE" -eq 1 ]]; then
     tar -chf /test_output/clickhouse_coverage.tar.gz /profraw ||:
@@ -155,6 +157,9 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]
     grep -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server2.log ||:
     pigz < /var/log/clickhouse-server/clickhouse-server1.log > /test_output/clickhouse-server1.log.gz ||:
     pigz < /var/log/clickhouse-server/clickhouse-server2.log > /test_output/clickhouse-server2.log.gz ||:
+    # FIXME: remove once only github actions will be left
+    rm /var/log/clickhouse-server/clickhouse-server1.log
+    rm /var/log/clickhouse-server/clickhouse-server2.log
     mv /var/log/clickhouse-server/stderr1.log /test_output/ ||:
     mv /var/log/clickhouse-server/stderr2.log /test_output/ ||:
     tar -chf /test_output/zookeeper_log_dump1.tar /var/lib/clickhouse1/data/system/zookeeper_log ||:

docker/test/stress/run.sh

@@ -185,6 +185,8 @@ zgrep -Fa "########################################" /test_output/* > /dev/null
 for log_file in /var/log/clickhouse-server/clickhouse-server.log*
 do
     pigz < "${log_file}" > /test_output/"$(basename ${log_file})".gz
+    # FIXME: remove once only github actions will be left
+    rm "${log_file}"
 done
 
 tar -chf /test_output/coordination.tar /var/lib/clickhouse/coordination ||:

docker/test/stress/stress

@@ -138,7 +138,7 @@ if __name__ == "__main__":
     parser.add_argument("--client-cmd", default='clickhouse-client')
     parser.add_argument("--server-log-folder", default='/var/log/clickhouse-server')
     parser.add_argument("--output-folder")
-    parser.add_argument("--global-time-limit", type=int, default=3600)
+    parser.add_argument("--global-time-limit", type=int, default=1800)
     parser.add_argument("--num-parallel", type=int, default=cpu_count())
     parser.add_argument('--hung-check', action='store_true', default=False)
     # make sense only for hung check

docker/test/style/Dockerfile

@@ -10,7 +10,7 @@ RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \
     python3-pip \
     pylint \
     yamllint \
-    && pip3 install codespell PyGithub boto3 unidiff
+    && pip3 install codespell PyGithub boto3 unidiff dohq-artifactory
 
 COPY run.sh /
 COPY process_style_check_result.py /

docs/en/development/build.md

@@ -74,7 +74,7 @@ The build requires the following components:
 -   Git (is used only to checkout the sources, it’s not needed for the build)
 -   CMake 3.10 or newer
 -   Ninja
--   C++ compiler: clang-11 or newer
+-   C++ compiler: clang-13 or newer
 -   Linker: lld
 -   Python (is only used inside LLVM build and it is optional)
 

docs/en/operations/server-configuration-parameters/settings.md

@@ -588,7 +588,7 @@ For more information, see the section [Creating replicated tables](../../engines
 
 Approximate size (in bytes) of the cache of marks used by table engines of the [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) family.
 
-The cache is shared for the server and memory is allocated as needed. The cache size must be at least 5368709120.
+The cache is shared for the server and memory is allocated as needed.
 
 **Example**
 

docs/en/sql-reference/functions/json-functions.md

@@ -275,7 +275,7 @@ If the part does not exist or isn’t array, an empty array will be returned.
 Example:
 
 ``` sql
-SELECT JSONExtractArrayRaw('{"a": "hello", "b": [-100, 200.0, "hello"]}', 'b') = ['-100', '200.0', '"hello"']';
+SELECT JSONExtractArrayRaw('{"a": "hello", "b": [-100, 200.0, "hello"]}', 'b') = ['-100', '200.0', '"hello"'];
 ```
 
 ## JSONExtractKeysAndValuesRaw {#json-extract-keys-and-values-raw}

docs/en/sql-reference/functions/splitting-merging-functions.md

@@ -290,7 +290,7 @@ ngrams(string, ngramsize)
 
 -   Array with n-grams.
 
-Type: [Array](../../sql-reference/data-types/array.md)([FixedString](../../sql-reference/data-types/fixedstring.md)).
+Type: [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)).
 
 **Example**
 

docs/ru/sql-reference/functions/splitting-merging-functions.md

@@ -252,7 +252,7 @@ ngrams(string, ngramsize)
 
 -   Массив с n-граммами.
 
-Тип: [Array](../../sql-reference/data-types/array.md)([FixedString](../../sql-reference/data-types/fixedstring.md)).
+Тип: [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)).
 
 **Пример**
 

programs/copier/ClusterCopier.cpp

@@ -8,6 +8,7 @@
 #include <Common/setThreadName.h>
 #include <IO/ConnectionTimeoutsContext.h>
 #include <Interpreters/InterpreterInsertQuery.h>
+#include <Parsers/ASTFunction.h>
 #include <Processors/Transforms/ExpressionTransform.h>
 #include <QueryPipeline/QueryPipelineBuilder.h>
 #include <QueryPipeline/Chain.h>

programs/copier/Internals.cpp

@@ -1,8 +1,10 @@
 #include "Internals.h"
-#include <Storages/MergeTree/MergeTreeData.h>
+#include <Parsers/ASTFunction.h>
-#include <Storages/extractKeyExpressionList.h>
+#include <Parsers/ASTIdentifier.h>
 #include <Processors/Executors/PullingPipelineExecutor.h>
 #include <Processors/Transforms/SquashingChunksTransform.h>
+#include <Storages/MergeTree/MergeTreeData.h>
+#include <Storages/extractKeyExpressionList.h>
 
 namespace DB
 {

programs/copier/TaskTableAndShard.h

@@ -5,6 +5,7 @@
 #include "ClusterPartition.h"
 
 #include <Core/Defines.h>
+#include <Parsers/ASTFunction.h>
 
 #include <base/map.h>
 #include <boost/algorithm/string/join.hpp>

programs/odbc-bridge/ODBCBlockInputStream.cpp

@@ -2,6 +2,7 @@
 #include <vector>
 #include <IO/ReadBufferFromString.h>
 #include <DataTypes/DataTypeNullable.h>
+#include <DataTypes/DataTypeDateTime64.h>
 #include <Columns/ColumnNullable.h>
 #include <Columns/ColumnString.h>
 #include <Columns/ColumnsNumber.h>
@@ -141,7 +142,18 @@ void ODBCSource::insertValue(
             assert_cast<ColumnUInt32 &>(column).insertValue(time);
             break;
         }
-        case ValueType::vtDateTime64:[[fallthrough]];
+        case ValueType::vtDateTime64:
+        {
+            auto value = row.get<std::string>(idx);
+            ReadBufferFromString in(value);
+            DateTime64 time = 0;
+            const auto * datetime_type = assert_cast<const DataTypeDateTime64 *>(data_type.get());
+            readDateTime64Text(time, datetime_type->getScale(), in, datetime_type->getTimeZone());
+            if (time < 0)
+                time = 0;
+            assert_cast<DataTypeDateTime64::ColumnType &>(column).insertValue(time);
+            break;
+        }
         case ValueType::vtDecimal32: [[fallthrough]];
         case ValueType::vtDecimal64: [[fallthrough]];
         case ValueType::vtDecimal128: [[fallthrough]];

programs/server/Server.cpp

@@ -85,9 +85,6 @@
 #if !defined(ARCADIA_BUILD)
 #   include "config_core.h"
 #   include "Common/config_version.h"
-#   if USE_OPENCL
-#       include "Common/BitonicSort.h"
-#   endif
 #endif
 
 #if defined(OS_LINUX)

src/Access/AccessEntityIO.cpp

@@ -7,6 +7,7 @@
 #include <Access/SettingsProfile.h>
 #include <Access/User.h>
 #include <Core/Defines.h>
+#include <IO/WriteHelpers.h>
 #include <Interpreters/Access/InterpreterCreateQuotaQuery.h>
 #include <Interpreters/Access/InterpreterCreateRoleQuery.h>
 #include <Interpreters/Access/InterpreterCreateRowPolicyQuery.h>

src/AggregateFunctions/AggregateFunctionSparkbar.h

@@ -102,11 +102,16 @@ private:
     size_t width;
     X min_x;
     X max_x;
+    bool specified_min_max_x;
 
-    String getBar(const UInt8 value) const
+    template <class T>
+    String getBar(const T value) const
     {
+        if (isNaN(value) || value > 8 || value < 1)
+            return " ";
+
         // ▁▂▃▄▅▆▇█
-        switch (value)
+        switch (static_cast<UInt8>(value))
         {
             case 1: return "▁";
             case 2: return "▂";
@@ -136,9 +141,20 @@ private:
         String value;
         if (data.points.empty() || !width)
             return value;
-        X local_min_x = data.min_x;
+
-        X local_max_x = data.max_x;
+        size_t diff_x;
-        size_t diff_x = local_max_x - local_min_x;
+        X min_x_local;
+        if (specified_min_max_x)
+        {
+            diff_x = max_x - min_x;
+            min_x_local = min_x;
+        }
+        else
+        {
+            diff_x = data.max_x - data.min_x;
+            min_x_local = data.min_x;
+        }
+
         if ((diff_x + 1) <= width)
         {
             Y min_y = data.min_y;
@@ -149,15 +165,15 @@ private:
             {
                 for (size_t i = 0; i <= diff_x; ++i)
                 {
-                    auto it = data.points.find(local_min_x + i);
+                    auto it = data.points.find(min_x_local + i);
                     bool found = it != data.points.end();
-                    value += getBar(found ? static_cast<UInt8>(std::round(((it->getMapped() - min_y) / diff_y) * 7) + 1) : 0);
+                    value += getBar(found ? std::round(((it->getMapped() - min_y) / diff_y) * 7) + 1 : 0.0);
                 }
             }
             else
             {
                 for (size_t i = 0; i <= diff_x; ++i)
-                    value += getBar(data.points.has(local_min_x + i) ? 1 : 0);
+                    value += getBar(data.points.has(min_x_local + i) ? 1 : 0);
             }
         }
         else
@@ -186,9 +202,9 @@ private:
                 if (i == bound.first) // is bound
                 {
                     Float64 proportion = bound.second - bound.first;
-                    auto it = data.points.find(local_min_x + i);
+                    auto it = data.points.find(min_x_local + i);
                     bool found = (it != data.points.end());
-                    if (found)
+                    if (found && proportion > 0)
                         new_y = new_y.value_or(0) + it->getMapped() * proportion;
 
                     if (new_y)
@@ -213,7 +229,7 @@ private:
                 }
                 else
                 {
-                    auto it = data.points.find(local_min_x + i);
+                    auto it = data.points.find(min_x_local + i);
                     if (it != data.points.end())
                         new_y = new_y.value_or(0) + it->getMapped();
                 }
@@ -226,7 +242,7 @@ private:
 
             auto getBars = [&] (const std::optional<Float64> & point_y)
             {
-                value += getBar(point_y ? static_cast<UInt8>(std::round(((point_y.value() - min_y.value()) / diff_y) * 7) + 1) : 0);
+                value += getBar(point_y ? std::round(((point_y.value() - min_y.value()) / diff_y) * 7) + 1 : 0);
             };
             auto getBarsForConstant = [&] (const std::optional<Float64> & point_y)
             {
@@ -250,11 +266,13 @@ public:
         width = params.at(0).safeGet<UInt64>();
         if (params.size() == 3)
         {
+            specified_min_max_x = true;
             min_x = params.at(1).safeGet<X>();
             max_x = params.at(2).safeGet<X>();
         }
         else
         {
+            specified_min_max_x = false;
             min_x = std::numeric_limits<X>::min();
             max_x = std::numeric_limits<X>::max();
         }

src/AggregateFunctions/parseAggregateFunctionParameters.cpp

@@ -1,6 +1,6 @@
 #include <AggregateFunctions/parseAggregateFunctionParameters.h>
 
-#include <Parsers/ASTFunction.h>
+#include <Parsers/ASTLiteral.h>
 #include <Parsers/ExpressionListParsers.h>
 #include <Parsers/parseQuery.h>
 

src/AggregateFunctions/parseAggregateFunctionParameters.h

@@ -1,8 +1,8 @@
 #pragma once
 
-#include <Parsers/ASTLiteral.h>
-#include <Parsers/ASTExpressionList.h>
 #include <Interpreters/Context_fwd.h>
+#include <Parsers/ASTExpressionList.h>
+#include <Parsers/IAST_fwd.h>
 
 
 namespace DB

src/Backups/renameInCreateQuery.cpp

@@ -1,12 +1,13 @@
-#include <Backups/renameInCreateQuery.h>
 #include <Backups/BackupRenamingConfig.h>
+#include <Backups/renameInCreateQuery.h>
+#include <Interpreters/InDepthNodeVisitor.h>
+#include <Interpreters/evaluateConstantExpression.h>
 #include <Parsers/ASTCreateQuery.h>
 #include <Parsers/ASTFunction.h>
+#include <Parsers/ASTIdentifier.h>
 #include <Parsers/ASTLiteral.h>
 #include <Parsers/ASTTablesInSelectQuery.h>
 #include <TableFunctions/TableFunctionFactory.h>
-#include <Interpreters/InDepthNodeVisitor.h>
-#include <Interpreters/evaluateConstantExpression.h>
 
 
 namespace DB

src/Client/ClientBase.cpp

@@ -40,7 +40,6 @@
 #include <Parsers/ASTDropQuery.h>
 #include <Parsers/ASTSetQuery.h>
 #include <Parsers/ASTUseQuery.h>
-#include <Parsers/ASTSelectQuery.h>
 #include <Parsers/ASTSelectWithUnionQuery.h>
 #include <Parsers/ASTQueryWithOutput.h>
 #include <Parsers/ASTLiteral.h>

src/Columns/ColumnMap.cpp

@@ -263,7 +263,7 @@ void ColumnMap::getExtremes(Field & min, Field & max) const
 
 void ColumnMap::forEachSubcolumn(ColumnCallback callback)
 {
-    nested->forEachSubcolumn(callback);
+    callback(nested);
 }
 
 bool ColumnMap::structureEquals(const IColumn & rhs) const

src/Common/ErrorCodes.cpp

@@ -598,6 +598,8 @@
     M(628, OFFSET_FETCH_WITHOUT_ORDER_BY) \
     M(629, HTTP_RANGE_NOT_SATISFIABLE) \
     M(630, HAVE_DEPENDENT_OBJECTS) \
+    M(631, UNKNOWN_FILE_SIZE) \
+    M(632, UNEXPECTED_DATA_AFTER_PARSED_VALUE) \
     \
     M(999, KEEPER_EXCEPTION) \
     M(1000, POCO_EXCEPTION) \

src/Common/ProfileEvents.cpp

@@ -261,6 +261,8 @@
     M(RemoteFSUnprefetchedReads, "Number of reads from unprefetched buffer") \
     M(RemoteFSBuffers, "Number of buffers created for asynchronous reading from remote filesystem") \
     \
+    M(ReadBufferSeekCancelConnection, "Number of seeks which lead to new connection (s3, http)") \
+    \
     M(SleepFunctionCalls, "Number of times a sleep function (sleep, sleepEachRow) has been called.") \
     M(SleepFunctionMicroseconds, "Time spent sleeping due to a sleep function call.") \
     \

src/Common/Visitor.h

@@ -1,7 +1,7 @@
 #pragma once
 
 #include <base/demangle.h>
-#include <base/Typelist.h>
+#include <base/TypeList.h>
 #include <Common/Exception.h>
 
 /* Generic utils which are intended for visitor pattern implementation.
@@ -64,7 +64,7 @@ template <>
 class Visitor<>
 {
 public:
-    using List = Typelist<>;
+    using List = TypeList<>;
 
 protected:
     ~Visitor() = default;
@@ -74,7 +74,7 @@ template <typename Type>
 class Visitor<Type> : public Visitor<>
 {
 public:
-    using List = Typelist<Type>;
+    using List = TypeList<Type>;
 
     virtual void visit(Type &) = 0;
 
@@ -86,7 +86,7 @@ template <typename Type, typename ... Types>
 class Visitor<Type, Types ...> : public Visitor<Types ...>
 {
 public:
-    using List = Typelist<Type, Types ...>;
+    using List = TypeList<Type, Types ...>;
     using Visitor<Types ...>::visit;
 
     virtual void visit(Type &) = 0;
@@ -145,10 +145,10 @@ protected:
 
 template <typename Derived, typename VisitorBase>
 class VisitorImpl : public
-    TLChangeRoot<
+    TypeListChangeRoot<
         VisitorImplHelper,
-        TLConcat<
+        TypeListConcat<
-            Typelist<Derived, VisitorBase>,
+            TypeList<Derived, VisitorBase>,
             typename VisitorBase::List
         >
     >

src/Common/renameat2.cpp

@@ -4,13 +4,6 @@
 #include <Poco/Environment.h>
 #include <filesystem>
 
-#if defined(linux) || defined(__linux) || defined(__linux__)
-#include <unistd.h>
-#include <fcntl.h>
-#include <sys/syscall.h>
-#include <linux/fs.h>
-#endif
-
 namespace fs = std::filesystem;
 
 namespace DB
@@ -25,25 +18,56 @@ namespace ErrorCodes
     extern const int FILE_ALREADY_EXISTS;
 }
 
+}
+
+
+#if defined(__linux__)
+
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/syscall.h>
+#include <linux/fs.h>
+
+/// For old versions of libc.
+#if !defined(RENAME_NOREPLACE)
+    #define RENAME_NOREPLACE 1
+#endif
+
+#if !defined(RENAME_EXCHANGE)
+    #define RENAME_EXCHANGE 2
+#endif
+
+#if !defined(__NR_renameat2)
+    #if defined(__x86_64__)
+        #define __NR_renameat2 316
+    #elif defined(__aarch64__)
+        #define __NR_renameat2 276
+    #elif defined(__ppc64__)
+        #define __NR_renameat2 357
+    #elif defined(__riscv)
+        #define __NR_renameat2 276
+    #else
+        #error "Unsupported architecture"
+    #endif
+#endif
+
+
+namespace DB
+{
+
 static bool supportsRenameat2Impl()
 {
-#if defined(__NR_renameat2)
     VersionNumber renameat2_minimal_version(3, 15, 0);
     VersionNumber linux_version(Poco::Environment::osVersion());
     return linux_version >= renameat2_minimal_version;
-#else
-    return false;
-#endif
 }
 
-#if defined(__NR_renameat2)
-
 static bool renameat2(const std::string & old_path, const std::string & new_path, int flags)
 {
     if (!supportsRenameat2())
         return false;
     if (old_path.empty() || new_path.empty())
-        throw Exception("Cannot rename " + old_path + " to " + new_path + ": path is empty", ErrorCodes::LOGICAL_ERROR);
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot rename {} to {}: path is empty", old_path, new_path);
 
     /// int olddirfd (ignored for absolute oldpath), const char *oldpath,
     /// int newdirfd (ignored for absolute newpath), const char *newpath,
@@ -63,28 +87,50 @@ static bool renameat2(const std::string & old_path, const std::string & new_path
         return false;
 
     if (errno == EEXIST)
-        throwFromErrno("Cannot rename " + old_path + " to " + new_path + " because the second path already exists", ErrorCodes::ATOMIC_RENAME_FAIL);
+        throwFromErrno(fmt::format("Cannot rename {} to {} because the second path already exists", old_path, new_path), ErrorCodes::ATOMIC_RENAME_FAIL);
     if (errno == ENOENT)
-        throwFromErrno("Paths cannot be exchanged because " + old_path + " or " + new_path + " does not exist", ErrorCodes::ATOMIC_RENAME_FAIL);
+        throwFromErrno(fmt::format("Paths cannot be exchanged because {} or {} does not exist", old_path, new_path), ErrorCodes::ATOMIC_RENAME_FAIL);
-    throwFromErrnoWithPath("Cannot rename " + old_path + " to " + new_path, new_path, ErrorCodes::SYSTEM_ERROR);
+    throwFromErrnoWithPath(fmt::format("Cannot rename {} to {}", old_path, new_path), new_path, ErrorCodes::SYSTEM_ERROR);
+}
+
+bool supportsRenameat2()
+{
+    static bool supports = supportsRenameat2Impl();
+    return supports;
+}
+
 }
 
 #else
+
 #define RENAME_NOREPLACE -1
 #define RENAME_EXCHANGE -1
 
+namespace DB
+{
+
 static bool renameat2(const std::string &, const std::string &, int)
 {
     return false;
 }
 
+bool supportsRenameat2()
+{
+    return false;
+}
+
+}
+
 #endif
 
+namespace DB
+{
+
 static void renameNoReplaceFallback(const std::string & old_path, const std::string & new_path)
 {
     /// NOTE it's unsafe
     if (fs::exists(new_path))
-        throw Exception("File " + new_path + " exists", ErrorCodes::FILE_ALREADY_EXISTS);
+        throw Exception(ErrorCodes::FILE_ALREADY_EXISTS, "File {} exists", new_path);
     fs::rename(old_path, new_path);
 }
 
@@ -97,13 +143,6 @@ static void renameExchangeFallback(const std::string &, const std::string &)
 }
 #pragma GCC diagnostic pop
 
-
-bool supportsRenameat2()
-{
-    static bool supports = supportsRenameat2Impl();
-    return supports;
-}
-
 void renameNoReplace(const std::string & old_path, const std::string & new_path)
 {
     if (!renameat2(old_path, new_path, RENAME_NOREPLACE))

src/Compression/CachedCompressedReadBuffer.cpp

@@ -87,7 +87,7 @@ CachedCompressedReadBuffer::CachedCompressedReadBuffer(
 void CachedCompressedReadBuffer::seek(size_t offset_in_compressed_file, size_t offset_in_decompressed_block)
 {
     /// Nothing to do if we already at required position
-    if (file_pos == offset_in_compressed_file
+    if (!owned_cell && file_pos == offset_in_compressed_file
         && (offset() == offset_in_decompressed_block ||
             nextimpl_working_buffer_offset == offset_in_decompressed_block))
         return;
@@ -106,6 +106,8 @@ void CachedCompressedReadBuffer::seek(size_t offset_in_compressed_file, size_t o
         bytes += offset();
         /// No data, everything discarded
         pos = working_buffer.end();
+        owned_cell.reset();
+
         /// Remember required offset in decompressed block which will be set in
         /// the next ReadBuffer::next() call
         nextimpl_working_buffer_offset = offset_in_decompressed_block;

src/Compression/CompressedReadBufferFromFile.h

@@ -49,7 +49,7 @@ public:
     explicit CompressedReadBufferFromFile(std::unique_ptr<ReadBufferFromFileBase> buf, bool allow_different_codecs_ = false);
 
     /// Seek is lazy in some sense. We move position in compressed file_in to offset_in_compressed_file, but don't
-    /// read data into working_buffer and don't shit our position to offset_in_decompressed_block. Instead
+    /// read data into working_buffer and don't shift our position to offset_in_decompressed_block. Instead
     /// we store this offset inside nextimpl_working_buffer_offset.
     void seek(size_t offset_in_compressed_file, size_t offset_in_decompressed_block);
 

src/Compression/CompressionCodecEncrypted.cpp

@@ -7,11 +7,9 @@
 #include <Compression/CompressionCodecEncrypted.h>
 #include <Poco/Logger.h>
 #include <base/logger_useful.h>
-#include <Common/ErrorCodes.h>
 
 // This depends on BoringSSL-specific API, notably <openssl/aead.h>.
 #if USE_SSL && USE_INTERNAL_SSL_LIBRARY
-#include <Parsers/ASTLiteral.h>
 #include <openssl/digest.h>
 #include <openssl/err.h>
 #include <boost/algorithm/hex.hpp>

src/Compression/CompressionCodecGorilla.cpp

@@ -7,7 +7,6 @@
 #include <Compression/CompressionFactory.h>
 #include <base/unaligned.h>
 #include <Parsers/IAST_fwd.h>
-#include <Parsers/ASTIdentifier.h>
 #include <IO/WriteHelpers.h>
 #include <IO/ReadBufferFromMemory.h>
 #include <IO/BitHelpers.h>

src/Compression/CompressionCodecNone.cpp

@@ -1,7 +1,6 @@
 #include <Compression/CompressionCodecNone.h>
 #include <Compression/CompressionInfo.h>
 #include <Compression/CompressionFactory.h>
-#include <Parsers/ASTIdentifier.h>
 
 
 namespace DB

src/Compression/CompressionCodecT64.cpp

@@ -5,7 +5,6 @@
 #include <base/unaligned.h>
 #include <Parsers/IAST.h>
 #include <Parsers/ASTLiteral.h>
-#include <Parsers/ASTIdentifier.h>
 #include <Parsers/ASTFunction.h>
 #include <IO/WriteHelpers.h>
 #include <Core/Types.h>

src/Core/Settings.h

@@ -514,6 +514,7 @@ class IColumn;
     M(Int64, read_priority, 0, "Priority to read data from local filesystem. Only supported for 'pread_threadpool' method.", 0) \
     M(UInt64, merge_tree_min_rows_for_concurrent_read_for_remote_filesystem, (20 * 8192), "If at least as many lines are read from one file, the reading can be parallelized, when reading from remote filesystem.", 0) \
     M(UInt64, merge_tree_min_bytes_for_concurrent_read_for_remote_filesystem, (24 * 10 * 1024 * 1024), "If at least as many bytes are read from one file, the reading can be parallelized, when reading from remote filesystem.", 0) \
+    M(UInt64, remote_read_min_bytes_for_seek, DBMS_DEFAULT_BUFFER_SIZE, "Min bytes required for remote read (url, s3) to do seek, instead for read with ignore.", 0) \
     \
     M(UInt64, async_insert_threads, 16, "Maximum number of threads to actually parse and insert data in background. Zero means asynchronous mode is disabled", 0) \
     M(Bool, async_insert, false, "If true, data from INSERT query is stored in queue and later flushed to table in background. Makes sense only for inserts via HTTP protocol. If wait_for_async_insert is false, INSERT query is processed almost instantly, otherwise client will wait until data will be flushed to table", 0) \
@@ -582,6 +583,7 @@ class IColumn;
     M(Bool, input_format_arrow_import_nested, false, "Allow to insert array of structs into Nested table in Arrow input format.", 0) \
     M(Bool, input_format_orc_import_nested, false, "Allow to insert array of structs into Nested table in ORC input format.", 0) \
     M(Bool, input_format_parquet_import_nested, false, "Allow to insert array of structs into Nested table in Parquet input format.", 0) \
+    M(Bool, input_format_allow_seeks, true, "Allow seeks while reading in ORC/Parquet/Arrow input formats", 0) \
     \
     M(DateTimeInputFormat, date_time_input_format, FormatSettings::DateTimeInputFormat::Basic, "Method to read DateTime from text input formats. Possible values: 'basic' and 'best_effort'.", 0) \
     M(DateTimeOutputFormat, date_time_output_format, FormatSettings::DateTimeOutputFormat::Simple, "Method to write DateTime to text output. Possible values: 'simple', 'iso', 'unix_timestamp'.", 0) \

src/DataTypes/DataTypeAggregateFunction.cpp

@@ -15,8 +15,8 @@
 
 #include <AggregateFunctions/AggregateFunctionFactory.h>
 #include <Parsers/ASTFunction.h>
+#include <Parsers/ASTIdentifier_fwd.h>
 #include <Parsers/ASTLiteral.h>
-#include <Parsers/ASTIdentifier.h>
 
 
 namespace DB

src/DataTypes/DataTypeCustomSimpleAggregateFunction.cpp

@@ -8,8 +8,9 @@
 
 #include <AggregateFunctions/AggregateFunctionFactory.h>
 #include <Parsers/ASTFunction.h>
-#include <Parsers/ASTLiteral.h>
 #include <Parsers/ASTIdentifier.h>
+#include <Parsers/ASTLiteral.h>
+#include <Parsers/ASTSelectWithUnionQuery.h>
 
 #include <boost/algorithm/string/join.hpp>
 

src/DataTypes/DataTypeLowCardinality.cpp

@@ -6,7 +6,7 @@
 #include <Common/assert_cast.h>
 #include <Common/typeid_cast.h>
 #include <Core/Field.h>
-#include <base/Typelists.h>
+#include <base/TypeLists.h>
 #include <DataTypes/DataTypeDate.h>
 #include <DataTypes/DataTypeDateTime.h>
 #include <DataTypes/DataTypeFactory.h>
@@ -53,7 +53,7 @@ namespace
         {
         }
 
-        template <class T>
+        template <typename T>
         void operator()(Id<T>)
         {
             if (typeid_cast<const DataTypeNumber<T> *>(&keys_type))
@@ -89,7 +89,7 @@ MutableColumnUniquePtr DataTypeLowCardinality::createColumnUniqueImpl(const IDat
     else if (which.isInt() || which.isUInt() || which.isFloat())
     {
         MutableColumnUniquePtr column;
-        TLUtils::forEach(TLIntegral{}, CreateColumnVector(column, *type, creator));
+        TypeListUtils::forEach(TypeListIntAndFloat{}, CreateColumnVector(column, *type, creator));
 
         if (!column)
             throw Exception("Unexpected numeric type: " + type->getName(), ErrorCodes::LOGICAL_ERROR);

src/DataTypes/Serializations/ISerialization.cpp

@@ -15,6 +15,7 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int MULTIPLE_STREAMS_REQUIRED;
+    extern const int UNEXPECTED_DATA_AFTER_PARSED_VALUE;
 }
 
 String ISerialization::Substream::toString() const
@@ -256,5 +257,17 @@ ISerialization::SubstreamData ISerialization::createFromPath(const SubstreamPath
     return res;
 }
 
+void ISerialization::throwUnexpectedDataAfterParsedValue(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, const String & type_name) const
+{
+    WriteBufferFromOwnString ostr;
+    serializeText(column, column.size() - 1, ostr, settings);
+    throw Exception(
+        ErrorCodes::UNEXPECTED_DATA_AFTER_PARSED_VALUE,
+        "Unexpected data '{}' after parsed {} value '{}'",
+        std::string(istr.position(), std::min(size_t(10), istr.available())),
+        type_name,
+        ostr.str());
+}
+
 }
 

src/DataTypes/Serializations/ISerialization.h

@@ -304,6 +304,9 @@ public:
 
     static bool hasSubcolumnForPath(const SubstreamPath & path, size_t prefix_len);
     static SubstreamData createFromPath(const SubstreamPath & path, size_t prefix_len);
+
+protected:
+    [[noreturn]] void throwUnexpectedDataAfterParsedValue(IColumn & column, ReadBuffer & istr, const FormatSettings &, const String & type_name) const;
 };
 
 using SerializationPtr = std::shared_ptr<const ISerialization>;

src/DataTypes/Serializations/SerializationArray.cpp

@@ -450,13 +450,16 @@ void SerializationArray::serializeText(const IColumn & column, size_t row_num, W
 }
 
 
-void SerializationArray::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
+void SerializationArray::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, bool whole) const
 {
     deserializeTextImpl(column, istr,
         [&](IColumn & nested_column)
         {
             nested->deserializeTextQuoted(nested_column, istr, settings);
         }, false);
+
+    if (whole && !istr.eof())
+        throwUnexpectedDataAfterParsedValue(column, istr, settings, "Array");
 }
 
 void SerializationArray::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const

src/DataTypes/Serializations/SerializationArray.h

@@ -19,7 +19,7 @@ public:
     void deserializeBinary(IColumn & column, ReadBuffer & istr) const override;
 
     void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
-    void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
+    void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &, bool whole) const override;
 
     void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
     void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;

src/DataTypes/Serializations/SerializationCustomSimpleText.cpp

@@ -21,7 +21,7 @@ String serializeToString(const SerializationCustomSimpleText & domain, const ICo
 void deserializeFromString(const SerializationCustomSimpleText & domain, IColumn & column, const String & s, const FormatSettings & settings)
 {
     ReadBufferFromString istr(s);
-    domain.deserializeText(column, istr, settings);
+    domain.deserializeText(column, istr, settings, true);
 }
 
 }
@@ -37,7 +37,7 @@ SerializationCustomSimpleText::SerializationCustomSimpleText(const Serialization
 void SerializationCustomSimpleText::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
     String str;
-    readString(str, istr);
+    readStringUntilEOF(str, istr);
     deserializeFromString(*this, column, str, settings);
 }
 

src/DataTypes/Serializations/SerializationCustomSimpleText.h

@@ -19,9 +19,11 @@ public:
 
     // Methods that subclasses must override in order to get full serialization/deserialization support.
     virtual void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override = 0;
-    virtual void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const = 0;
+    /// whole = true means that buffer contains only one value, so we should read until EOF.
+    /// It's needed to check if there is garbage after parsed field.
+    virtual void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &, bool whole) const = 0;
 
-    /** Text deserialization without quoting or escaping.
+    /** Text deserialization in case when buffer contains only one value, without any escaping and delimiters.
       */
     void deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
 

src/DataTypes/Serializations/SerializationDate.cpp

@@ -5,7 +5,6 @@
 
 #include <Columns/ColumnsNumber.h>
 #include <Formats/ProtobufReader.h>
-#include <Formats/ProtobufWriter.h>
 
 #include <Common/assert_cast.h>
 
@@ -20,6 +19,8 @@ void SerializationDate::serializeText(const IColumn & column, size_t row_num, Wr
 void SerializationDate::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
     deserializeTextEscaped(column, istr, settings);
+    if (!istr.eof())
+        throwUnexpectedDataAfterParsedValue(column, istr, settings, "Date");
 }
 
 void SerializationDate::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const

src/DataTypes/Serializations/SerializationDate32.cpp

@@ -17,6 +17,8 @@ void SerializationDate32::serializeText(const IColumn & column, size_t row_num,
 void SerializationDate32::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
     deserializeTextEscaped(column, istr, settings);
+    if (!istr.eof())
+        throwUnexpectedDataAfterParsedValue(column, istr, settings, "Date32");
 }
 
 void SerializationDate32::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const

src/DataTypes/Serializations/SerializationDateTime.cpp

@@ -62,6 +62,8 @@ void SerializationDateTime::serializeTextEscaped(const IColumn & column, size_t
 void SerializationDateTime::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
     deserializeTextEscaped(column, istr, settings);
+    if (!istr.eof())
+        throwUnexpectedDataAfterParsedValue(column, istr, settings, "DateTime");
 }
 
 void SerializationDateTime::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const

src/DataTypes/Serializations/SerializationDateTime64.cpp

@@ -2,12 +2,9 @@
 
 #include <Columns/ColumnVector.h>
 #include <Common/assert_cast.h>
-#include <Common/typeid_cast.h>
 #include <base/DateLUT.h>
 #include <Formats/FormatSettings.h>
 #include <Formats/ProtobufReader.h>
-#include <Formats/ProtobufWriter.h>
-#include <IO/Operators.h>
 #include <IO/ReadHelpers.h>
 #include <IO/WriteBufferFromString.h>
 #include <IO/WriteHelpers.h>
@@ -40,16 +37,21 @@ void SerializationDateTime64::serializeText(const IColumn & column, size_t row_n
     }
 }
 
-void SerializationDateTime64::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
+void SerializationDateTime64::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, bool whole) const
 {
     DateTime64 result = 0;
     readDateTime64Text(result, scale, istr, time_zone);
     assert_cast<ColumnType &>(column).getData().push_back(result);
+
+    if (whole && !istr.eof())
+        throwUnexpectedDataAfterParsedValue(column, istr, settings, "DateTime64");
 }
 
 void SerializationDateTime64::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
     deserializeTextEscaped(column, istr, settings);
+    if (!istr.eof())
+        throwUnexpectedDataAfterParsedValue(column, istr, settings, "DateTime64");
 }
 
 void SerializationDateTime64::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const

src/DataTypes/Serializations/SerializationDateTime64.h

@@ -14,7 +14,7 @@ public:
     SerializationDateTime64(UInt32 scale_, const TimezoneMixin & time_zone_);
 
     void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
-    void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
+    void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, bool whole) const override;
     void deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override;
     void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
     void deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;

src/DataTypes/Serializations/SerializationDecimal.cpp

@@ -51,11 +51,14 @@ void SerializationDecimal<T>::serializeText(const IColumn & column, size_t row_n
 }
 
 template <typename T>
-void SerializationDecimal<T>::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
+void SerializationDecimal<T>::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, bool whole) const
 {
     T x;
     readText(x, istr);
     assert_cast<ColumnType &>(column).getData().push_back(x);
+
+    if (whole && !istr.eof())
+        ISerialization::throwUnexpectedDataAfterParsedValue(column, istr, settings, "Decimal");
 }
 
 template <typename T>

src/DataTypes/Serializations/SerializationDecimal.h

@@ -15,7 +15,7 @@ public:
         : SerializationDecimalBase<T>(precision_, scale_) {}
 
     void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
-    void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
+    void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &, bool whole) const override;
     void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
 
     void readText(T & x, ReadBuffer & istr, bool csv = false) const { readText(x, istr, this->precision, this->scale, csv); }

src/DataTypes/Serializations/SerializationEnum.cpp

@@ -53,11 +53,15 @@ template <typename Type>
 void SerializationEnum<Type>::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
     if (settings.tsv.input_format_enum_as_number)
+    {
         assert_cast<ColumnType &>(column).getData().push_back(readValue(istr));
+        if (!istr.eof())
+            ISerialization::throwUnexpectedDataAfterParsedValue(column, istr, settings, "Enum");
+    }
     else
     {
         std::string field_name;
-        readString(field_name, istr);
+        readStringUntilEOF(field_name, istr);
         assert_cast<ColumnType &>(column).getData().push_back(this->getValue(StringRef(field_name), true));
     }
 }

src/DataTypes/Serializations/SerializationIP.cpp

@@ -36,7 +36,7 @@ void SerializationIPv4::serializeText(const IColumn & column, size_t row_num, Wr
     ostr.write(buffer, strlen(buffer));
 }
 
-void SerializationIPv4::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
+void SerializationIPv4::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, bool whole) const
 {
     ColumnUInt32 * col = typeid_cast<ColumnUInt32 *>(&column);
     if (!col)
@@ -53,6 +53,9 @@ void SerializationIPv4::deserializeText(IColumn & column, ReadBuffer & istr, con
     }
 
     col->insert(ipv4_value);
+
+    if (whole && !istr.eof())
+        throwUnexpectedDataAfterParsedValue(column, istr, settings, "IPv4");
 }
 
 SerializationIPv6::SerializationIPv6(const SerializationPtr & nested_)
@@ -74,7 +77,7 @@ void SerializationIPv6::serializeText(const IColumn & column, size_t row_num, Wr
     ostr.write(buffer, strlen(buffer));
 }
 
-void SerializationIPv6::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
+void SerializationIPv6::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, bool whole) const
 {
     ColumnFixedString * col = typeid_cast<ColumnFixedString *>(&column);
     if (!col)
@@ -92,6 +95,9 @@ void SerializationIPv6::deserializeText(IColumn & column, ReadBuffer & istr, con
     }
 
     col->insertString(ipv6_value);
+
+    if (whole && !istr.eof())
+        throwUnexpectedDataAfterParsedValue(column, istr, settings, "IPv6");
 }
 
 }

src/DataTypes/Serializations/SerializationIP.h

@@ -11,7 +11,7 @@ public:
     SerializationIPv4(const SerializationPtr & nested_);
 
     void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
-    void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
+    void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, bool whole) const override;
 };
 
 class SerializationIPv6 : public SerializationCustomSimpleText
@@ -20,7 +20,7 @@ public:
     SerializationIPv6(const SerializationPtr & nested_);
 
     void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
-    void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
+    void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, bool whole) const override;
 };
 
 }

src/DataTypes/Serializations/SerializationMap.cpp

@@ -1,23 +1,16 @@
-#include <base/map.h>
-
 #include <DataTypes/Serializations/SerializationMap.h>
-#include <DataTypes/Serializations/SerializationArray.h>
-#include <DataTypes/Serializations/SerializationTuple.h>
 #include <DataTypes/DataTypeMap.h>
 
 #include <Common/StringUtils/StringUtils.h>
 #include <Columns/ColumnMap.h>
-#include <Columns/ColumnArray.h>
 #include <Core/Field.h>
 #include <Formats/FormatSettings.h>
-#include <Common/typeid_cast.h>
 #include <Common/assert_cast.h>
 #include <Common/quoteString.h>
 #include <IO/WriteHelpers.h>
 #include <IO/ReadHelpers.h>
 #include <IO/WriteBufferFromString.h>
 #include <IO/ReadBufferFromString.h>
-#include <IO/Operators.h>
 
 
 namespace DB
@@ -176,13 +169,16 @@ void SerializationMap::serializeText(const IColumn & column, size_t row_num, Wri
     serializeTextImpl(column, row_num, ostr, writer, writer);
 }
 
-void SerializationMap::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
+void SerializationMap::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, bool whole) const
 {
     deserializeTextImpl(column, istr,
         [&settings](ReadBuffer & buf, const SerializationPtr & subcolumn_serialization, IColumn & subcolumn)
         {
             subcolumn_serialization->deserializeTextQuoted(subcolumn, buf, settings);
         });
+
+    if (whole && !istr.eof())
+        throwUnexpectedDataAfterParsedValue(column, istr, settings, "Map");
 }
 
 void SerializationMap::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
@@ -248,7 +244,7 @@ void SerializationMap::deserializeTextCSV(IColumn & column, ReadBuffer & istr, c
     String s;
     readCSV(s, istr, settings.csv);
     ReadBufferFromString rb(s);
-    deserializeText(column, rb, settings);
+    deserializeText(column, rb, settings, true);
 }
 
 void SerializationMap::enumerateStreams(

src/DataTypes/Serializations/SerializationMap.h

@@ -23,7 +23,7 @@ public:
     void serializeBinary(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
     void deserializeBinary(IColumn & column, ReadBuffer & istr) const override;
     void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
-    void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
+    void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &, bool whole) const override;
     void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
     void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
     void serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;

src/DataTypes/Serializations/SerializationNothing.h

@@ -24,7 +24,7 @@ public:
     void serializeBinary(const IColumn &, size_t, WriteBuffer &) const override             { throwNoSerialization(); }
     void deserializeBinary(IColumn &, ReadBuffer &) const override                          { throwNoSerialization(); }
     void serializeText(const IColumn &, size_t, WriteBuffer &, const FormatSettings &) const override { throwNoSerialization(); }
-    void deserializeText(IColumn &, ReadBuffer &, const FormatSettings &) const override    { throwNoSerialization(); }
+    void deserializeText(IColumn &, ReadBuffer &, const FormatSettings &, bool) const override    { throwNoSerialization(); }
 
     /// These methods read and write zero bytes just to allow to figure out size of column.
     void serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const override;

src/DataTypes/Serializations/SerializationNumber.cpp

@@ -21,7 +21,7 @@ void SerializationNumber<T>::serializeText(const IColumn & column, size_t row_nu
 }
 
 template <typename T>
-void SerializationNumber<T>::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
+void SerializationNumber<T>::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, bool whole) const
 {
     T x;
 
@@ -31,6 +31,9 @@ void SerializationNumber<T>::deserializeText(IColumn & column, ReadBuffer & istr
         readText(x, istr);
 
     assert_cast<ColumnVector<T> &>(column).getData().push_back(x);
+
+    if (whole && !istr.eof())
+        throwUnexpectedDataAfterParsedValue(column, istr, settings, "Number");
 }
 
 template <typename T>

src/DataTypes/Serializations/SerializationNumber.h

@@ -16,7 +16,7 @@ public:
     using ColumnType = ColumnVector<T>;
 
     void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
-    void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override;
+    void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, bool whole) const override;
     void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override;
     void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
     void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override;

src/DataTypes/Serializations/SerializationTuple.cpp

@@ -1,4 +1,3 @@
-#include <base/map.h>
 #include <base/range.h>
 #include <DataTypes/Serializations/SerializationTuple.h>
 #include <DataTypes/DataTypeTuple.h>
@@ -8,7 +7,6 @@
 #include <IO/WriteHelpers.h>
 #include <IO/ReadHelpers.h>
 #include <IO/WriteBufferFromString.h>
-#include <IO/Operators.h>
 
 
 namespace DB
@@ -121,7 +119,7 @@ void SerializationTuple::serializeText(const IColumn & column, size_t row_num, W
     writeChar(')', ostr);
 }
 
-void SerializationTuple::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
+void SerializationTuple::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, bool whole) const
 {
     const size_t size = elems.size();
     assertChar('(', istr);
@@ -149,6 +147,9 @@ void SerializationTuple::deserializeText(IColumn & column, ReadBuffer & istr, co
     }
     skipWhitespaceIfAny(istr);
     assertChar(')', istr);
+
+    if (whole && !istr.eof())
+        throwUnexpectedDataAfterParsedValue(column, istr, settings, "Tuple");
 }
 
 void SerializationTuple::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const

src/DataTypes/Serializations/SerializationTuple.h

@@ -20,7 +20,7 @@ public:
     void serializeBinary(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
     void deserializeBinary(IColumn & column, ReadBuffer & istr) const override;
     void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
-    void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
+    void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &, bool whole) const override;
     void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
     void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
     void serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;

src/DataTypes/Serializations/SerializationUUID.cpp

@@ -15,16 +15,19 @@ void SerializationUUID::serializeText(const IColumn & column, size_t row_num, Wr
     writeText(assert_cast<const ColumnUUID &>(column).getData()[row_num], ostr);
 }
 
-void SerializationUUID::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
+void SerializationUUID::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, bool whole) const
 {
     UUID x;
     readText(x, istr);
     assert_cast<ColumnUUID &>(column).getData().push_back(x);
+
+    if (whole && !istr.eof())
+        throwUnexpectedDataAfterParsedValue(column, istr, settings, "UUID");
 }
 
 void SerializationUUID::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
-    deserializeText(column, istr, settings);
+    deserializeText(column, istr, settings, false);
 }
 
 void SerializationUUID::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const

src/DataTypes/Serializations/SerializationUUID.h

@@ -9,7 +9,7 @@ class SerializationUUID : public SimpleTextSerialization
 {
 public:
     void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
-    void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
+    void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, bool whole) const override;
     void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
     void deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
     void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;

src/DataTypes/Serializations/SimpleTextSerialization.h

@@ -33,30 +33,32 @@ protected:
 
     void deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override
     {
-        deserializeText(column, istr, settings);
+        deserializeText(column, istr, settings, true);
     }
 
     void deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override
     {
-        deserializeText(column, istr, settings);
+        deserializeText(column, istr, settings, false);
     }
 
     void deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override
     {
-        deserializeText(column, istr, settings);
+        deserializeText(column, istr, settings, false);
     }
 
     void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override
     {
-        deserializeText(column, istr, settings);
+        deserializeText(column, istr, settings, false);
     }
 
     void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override
     {
-        deserializeText(column, istr, settings);
+        deserializeText(column, istr, settings, false);
     }
 
-    virtual void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const = 0;
+    /// whole = true means that buffer contains only one value, so we should read until EOF.
+    /// It's needed to check if there is garbage after parsed field.
+    virtual void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &, bool whole) const = 0;
 };
 
 }

src/DataTypes/convertMySQLDataType.cpp

@@ -5,7 +5,6 @@
 #include <Core/MultiEnum.h>
 #include <Core/SettingsEnums.h>
 #include <Parsers/ASTFunction.h>
-#include <Parsers/ASTIdentifier.h>
 #include <Parsers/IAST.h>
 #include "DataTypeDate.h"
 #include "DataTypeDateTime.h"

src/Databases/DDLDependencyVisitor.cpp

@@ -1,10 +1,11 @@
 #include <Databases/DDLDependencyVisitor.h>
-#include <Parsers/ASTFunction.h>
-#include <Parsers/ASTCreateQuery.h>
-#include <Parsers/ASTLiteral.h>
-#include <Parsers/ASTIdentifier.h>
 #include <Dictionaries/getDictionaryConfigurationFromAST.h>
 #include <Interpreters/Context.h>
+#include <Parsers/ASTCreateQuery.h>
+#include <Parsers/ASTFunction.h>
+#include <Parsers/ASTIdentifier.h>
+#include <Parsers/ASTLiteral.h>
+#include <Parsers/ASTSelectWithUnionQuery.h>
 #include <Poco/String.h>
 
 namespace DB

src/Databases/DatabaseAtomic.cpp

@@ -229,15 +229,8 @@ void DatabaseAtomic::renameTable(ContextPtr local_context, const String & table_
 
     StoragePtr table = getTableUnlocked(table_name, db_lock);
 
-    if (table->isDictionary() && !dictionary)
+    if (dictionary && !table->isDictionary())
-    {
+        throw Exception(ErrorCodes::INCORRECT_QUERY, "Use RENAME/EXCHANGE TABLE (instead of RENAME/EXCHANGE DICTIONARY) for tables");
-        if (exchange)
-            throw Exception(ErrorCodes::INCORRECT_QUERY,
-                "Use EXCHANGE DICTIONARIES for dictionaries and EXCHANGE TABLES for tables.");
-        else
-            throw Exception(ErrorCodes::INCORRECT_QUERY,
-                "Use RENAME DICTIONARY for dictionaries and RENAME TABLE for tables.");
-    }
 
     table->checkTableCanBeRenamed();
     assert_can_move_mat_view(table);
@@ -245,6 +238,8 @@ void DatabaseAtomic::renameTable(ContextPtr local_context, const String & table_
     if (exchange)
     {
         other_table = other_db.getTableUnlocked(to_table_name, other_db_lock);
+        if (dictionary && !other_table->isDictionary())
+            throw Exception(ErrorCodes::INCORRECT_QUERY, "Use RENAME/EXCHANGE TABLE (instead of RENAME/EXCHANGE DICTIONARY) for tables");
         other_table->checkTableCanBeRenamed();
         assert_can_move_mat_view(other_table);
     }

src/Databases/DatabaseFactory.cpp

@@ -11,9 +11,7 @@
 #include <Interpreters/evaluateConstantExpression.h>
 #include <Parsers/ASTCreateQuery.h>
 #include <Parsers/ASTFunction.h>
-#include <Parsers/ASTIdentifier.h>
 #include <Parsers/ASTLiteral.h>
-#include <Parsers/formatAST.h>
 #include <Parsers/queryToString.h>
 #include <Storages/ExternalDataSourceConfiguration.h>
 #include <Common/Macros.h>

src/Databases/DatabaseMemory.cpp

@@ -5,6 +5,7 @@
 #include <Databases/DDLDependencyVisitor.h>
 #include <Interpreters/Context.h>
 #include <Parsers/ASTCreateQuery.h>
+#include <Parsers/ASTFunction.h>
 #include <Storages/IStorage.h>
 #include <filesystem>
 

src/Databases/DatabaseOnDisk.cpp

@@ -7,6 +7,7 @@
 #include <Interpreters/Context.h>
 #include <Interpreters/InterpreterCreateQuery.h>
 #include <Parsers/ASTCreateQuery.h>
+#include <Parsers/ASTFunction.h>
 #include <Parsers/ParserCreateQuery.h>
 #include <Parsers/formatAST.h>
 #include <Parsers/parseQuery.h>
@@ -40,6 +41,7 @@ namespace ErrorCodes
     extern const int TABLE_ALREADY_EXISTS;
     extern const int EMPTY_LIST_OF_COLUMNS_PASSED;
     extern const int DATABASE_NOT_EMPTY;
+    extern const int INCORRECT_QUERY;
 }
 
 
@@ -349,8 +351,6 @@ void DatabaseOnDisk::renameTable(
 {
     if (exchange)
         throw Exception("Tables can be exchanged only in Atomic databases", ErrorCodes::NOT_IMPLEMENTED);
-    if (dictionary)
-        throw Exception("Dictionaries can be renamed only in Atomic databases", ErrorCodes::NOT_IMPLEMENTED);
 
     bool from_ordinary_to_atomic = false;
     bool from_atomic_to_ordinary = false;
@@ -372,7 +372,11 @@ void DatabaseOnDisk::renameTable(
     ASTPtr attach_query;
     /// DatabaseLazy::detachTable may return nullptr even if table exists, so we need tryGetTable for this case.
     StoragePtr table = tryGetTable(table_name, local_context);
+    if (dictionary && table && !table->isDictionary())
+        throw Exception("Use RENAME/EXCHANGE TABLE (instead of RENAME/EXCHANGE DICTIONARY) for tables", ErrorCodes::INCORRECT_QUERY);
+
     detachTable(local_context, table_name);
+
     UUID prev_uuid = UUIDHelpers::Nil;
     try
     {

src/Databases/DatabaseReplicated.cpp

@@ -18,6 +18,7 @@
 #include <base/getFQDNOrHostName.h>
 #include <Parsers/ASTAlterQuery.h>
 #include <Parsers/ASTDropQuery.h>
+#include <Parsers/ASTFunction.h>
 #include <Parsers/ParserCreateQuery.h>
 #include <Parsers/parseQuery.h>
 #include <Interpreters/InterpreterCreateQuery.h>