Merge remote-tracking branch 'upstream/master' into fix25

CMakeLists.txt

@@ -1,15 +1,23 @@
-project(ClickHouse)
-cmake_minimum_required(VERSION 3.3)
-
 foreach(policy
         CMP0023
+        CMP0048 # CMake 3.0
         CMP0074 # CMake 3.12
+        CMP0077
+        CMP0079
     )
     if(POLICY ${policy})
         cmake_policy(SET ${policy} NEW)
     endif()
 endforeach()
 
+project(ClickHouse)
+cmake_minimum_required(VERSION 3.3)
+
+# Ignore export() since we don't use it,
+# but it gets broken with a global targets via link_libraries()
+macro (export)
+endmacro ()
+
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/")
 set(CMAKE_EXPORT_COMPILE_COMMANDS 1) # Write compile_commands.json
 set(CMAKE_LINK_DEPENDS_NO_SHARED 1) # Do not relink all depended targets on .so
@@ -128,12 +136,6 @@ if (CMAKE_SYSTEM_PROCESSOR MATCHES "amd64|x86_64")
     endif ()
 endif ()
 
-if (GLIBC_COMPATIBILITY)
-    set (USE_INTERNAL_MEMCPY ON)
-else ()
-    message (WARNING "Option GLIBC_COMPATIBILITY must be turned on for production builds.")
-endif ()
-
 string(REGEX MATCH "-?[0-9]+(.[0-9]+)?$" COMPILER_POSTFIX ${CMAKE_CXX_COMPILER})
 
 find_program (LLD_PATH NAMES "lld${COMPILER_POSTFIX}" "lld")
@@ -172,20 +174,15 @@ if (ARCH_NATIVE)
     set (COMPILER_FLAGS "${COMPILER_FLAGS} -march=native")
 endif ()
 
-# Special options for better optimized code with clang
-#if (COMPILER_CLANG)
-#    set (CMAKE_CXX_FLAGS_RELWITHDEBINFO  "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -Wno-unused-command-line-argument -mllvm -inline-threshold=10000")
-#endif ()
-
 if (CMAKE_VERSION VERSION_LESS "3.8.0")
     if (NOT MSVC)
-        set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++1z")
+        set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17")
     endif ()
 else ()
     set (CMAKE_CXX_STANDARD 17)
     set (CMAKE_CXX_EXTENSIONS 0) # https://cmake.org/cmake/help/latest/prop_tgt/CXX_EXTENSIONS.html#prop_tgt:CXX_EXTENSIONS
     set (CMAKE_CXX_STANDARD_REQUIRED ON)
-    set (CXX_FLAGS_INTERNAL_COMPILER "-std=c++1z")
+    set (CXX_FLAGS_INTERNAL_COMPILER "-std=c++17")
 endif ()
 
 if (COMPILER_GCC OR COMPILER_CLANG)
@@ -207,17 +204,13 @@ endif()
 
 set (CMAKE_BUILD_COLOR_MAKEFILE          ON)
 set (CMAKE_CXX_FLAGS                     "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS} ${PLATFORM_EXTRA_CXX_FLAG} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${CXX_WARNING_FLAGS}")
-#set (CMAKE_CXX_FLAGS_RELEASE             "${CMAKE_CXX_FLAGS_RELEASE} ${CMAKE_CXX_FLAGS_ADD}")
 set (CMAKE_CXX_FLAGS_RELWITHDEBINFO      "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3 ${CMAKE_CXX_FLAGS_ADD}")
 set (CMAKE_CXX_FLAGS_DEBUG               "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_CXX_FLAGS_ADD}")
 
 set (CMAKE_C_FLAGS                       "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${CMAKE_C_FLAGS_ADD}")
-#set (CMAKE_C_FLAGS_RELEASE               "${CMAKE_C_FLAGS_RELEASE} ${CMAKE_C_FLAGS_ADD}")
 set (CMAKE_C_FLAGS_RELWITHDEBINFO        "${CMAKE_C_FLAGS_RELWITHDEBINFO} -O3 ${CMAKE_C_FLAGS_ADD}")
 set (CMAKE_C_FLAGS_DEBUG                 "${CMAKE_C_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_C_FLAGS_ADD}")
 
-# Uses MAKE_STATIC_LIBRARIES
-
 
 option (UNBUNDLED "Try find all libraries in system. We recommend to avoid this mode for production builds, because we cannot guarantee exact versions and variants of libraries your system has installed. This mode exists for enthusiastic developers who search for trouble. Also it is useful for maintainers of OS packages." OFF)
 if (UNBUNDLED)
@@ -225,149 +218,28 @@ if (UNBUNDLED)
 else ()
     set(NOT_UNBUNDLED 1)
 endif ()
+
 # Using system libs can cause lot of warnings in includes.
 if (UNBUNDLED OR NOT (OS_LINUX OR APPLE) OR ARCH_32)
     option (NO_WERROR "Disable -Werror compiler option" ON)
 endif ()
 
-
-set(THREADS_PREFER_PTHREAD_FLAG ON)
-find_package (Threads)
-
-include (cmake/find_cxx.cmake)
-
-include (cmake/test_compiler.cmake)
-
-if (OS_LINUX AND COMPILER_CLANG AND USE_STATIC_LIBRARIES)
-    option (USE_LIBCXX "Use libc++ and libc++abi instead of libstdc++ (only make sense on Linux)" ${HAVE_LIBCXX})
-
-    if (USE_LIBCXX)
-        set (CMAKE_CXX_FLAGS_DEBUG       "${CMAKE_CXX_FLAGS_DEBUG} -D_LIBCPP_DEBUG=0") # More checks in debug build.
-    endif ()
-endif ()
-
-if (USE_LIBCXX)
-    set (STATIC_STDLIB_FLAGS "")
-else ()
-    set (STATIC_STDLIB_FLAGS "-static-libgcc -static-libstdc++")
-endif ()
-
-if (MAKE_STATIC_LIBRARIES AND NOT APPLE AND NOT (COMPILER_CLANG AND OS_FREEBSD))
-    set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${STATIC_STDLIB_FLAGS}")
-
-    # Along with executables, we also build example of shared library for "library dictionary source"; and it also should be self-contained.
-    set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${STATIC_STDLIB_FLAGS}")
-endif ()
-
-if (USE_STATIC_LIBRARIES AND HAVE_NO_PIE)
-    set (CMAKE_CXX_FLAGS                 "${CMAKE_CXX_FLAGS} ${FLAG_NO_PIE}")
-    set (CMAKE_C_FLAGS                   "${CMAKE_C_FLAGS} ${FLAG_NO_PIE}")
-endif ()
-
 # Make this extra-checks for correct library dependencies.
 if (NOT SANITIZE)
     set (CMAKE_EXE_LINKER_FLAGS    "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-undefined")
     set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-undefined")
 endif ()
 
-include (cmake/find_unwind.cmake)
+include(cmake/dbms_glob_sources.cmake)
+include(cmake/default_libs.cmake)
 
-if (USE_INTERNAL_UNWIND_LIBRARY)
+######################################
-    option (USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING "Use internal unwind library for exception handling" ${USE_STATIC_LIBRARIES})
+### Add targets below this comment ###
-endif ()
+######################################
-
-
-# Set standard, system and compiler libraries explicitly.
-# This is intended for more control of what we are linking.
 
 string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC)
 set (CMAKE_POSTFIX_VARIABLE "CMAKE_${CMAKE_BUILD_TYPE_UC}_POSTFIX")
 
-set (DEFAULT_LIBS "")
-if (OS_LINUX AND NOT UNBUNDLED AND (GLIBC_COMPATIBILITY OR USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING OR USE_LIBCXX))
-    # Note: this probably has no effect, but I'm not an expert in CMake.
-    set (CMAKE_C_IMPLICIT_LINK_LIBRARIES "")
-    set (CMAKE_CXX_IMPLICIT_LINK_LIBRARIES "")
-
-    # Disable default linked libraries.
-    set (DEFAULT_LIBS "-nodefaultlibs")
-
-    # We need builtins from Clang's RT even without libcxx - for ubsan+int128. See https://bugs.llvm.org/show_bug.cgi?id=16404
-    set (BUILTINS_LIB_PATH "")
-    if (COMPILER_CLANG)
-        execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-${CMAKE_SYSTEM_PROCESSOR}.a OUTPUT_VARIABLE BUILTINS_LIB_PATH OUTPUT_STRIP_TRAILING_WHITESPACE)
-    else ()
-        set (BUILTINS_LIB_PATH "-lgcc")
-    endif ()
-
-    string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC)
-    set (CMAKE_POSTFIX_VARIABLE "CMAKE_${CMAKE_BUILD_TYPE_UC}_POSTFIX")
-
-    # Add C++ libraries.
-    #
-    # This consist of:
-    # - C++ standard library (like implementation of std::string);
-    # - C++ ABI implementation (functions for exceptions like __cxa_throw, RTTI, etc);
-    # - functions for internal implementation of exception handling (stack unwinding based on DWARF info; TODO replace with bundled libunwind);
-    # - compiler builtins (example: functions for implementation of __int128 operations);
-    #
-    # There are two variants of C++ library: libc++ (from LLVM compiler infrastructure) and libstdc++ (from GCC).
-
-    if (USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING)
-        if (USE_STATIC_LIBRARIES)
-            set (EXCEPTION_HANDLING_LIBRARY "${ClickHouse_BINARY_DIR}/contrib/libunwind-cmake/libunwind_static${${CMAKE_POSTFIX_VARIABLE}}.a")
-        else ()
-            set (EXCEPTION_HANDLING_LIBRARY "${ClickHouse_BINARY_DIR}/contrib/libunwind-cmake/libunwind_shared${${CMAKE_POSTFIX_VARIABLE}}.so")
-        endif ()
-    else ()
-        set (EXCEPTION_HANDLING_LIBRARY "-lgcc_eh")
-    endif ()
-
-    message (STATUS "Using exception handling library: ${EXCEPTION_HANDLING_LIBRARY}")
-
-    if (USE_LIBCXX)
-        if (USE_INTERNAL_LIBCXX_LIBRARY)
-            set (LIBCXX_LIBS "${ClickHouse_BINARY_DIR}/contrib/libcxx-cmake/libcxx_static${${CMAKE_POSTFIX_VARIABLE}}.a ${ClickHouse_BINARY_DIR}/contrib/libcxxabi-cmake/libcxxabi_static${${CMAKE_POSTFIX_VARIABLE}}.a")
-        else ()
-            set (LIBCXX_LIBS "-lc++ -lc++abi -lc++fs")
-        endif ()
-
-        set (DEFAULT_LIBS "${DEFAULT_LIBS} -Wl,-Bstatic ${LIBCXX_LIBS} ${EXCEPTION_HANDLING_LIBRARY} ${BUILTINS_LIB_PATH} -Wl,-Bdynamic")
-    else ()
-        set (DEFAULT_LIBS "${DEFAULT_LIBS} -Wl,-Bstatic -lstdc++ -lstdc++fs ${EXCEPTION_HANDLING_LIBRARY} ${COVERAGE_OPTION} ${BUILTINS_LIB_PATH} -Wl,-Bdynamic")
-    endif ()
-
-    # Linking with GLIBC prevents portability of binaries to older systems.
-    # We overcome this behaviour by statically linking with our own implementation of all new symbols (that don't exist in older Libc or have infamous "symbol versioning").
-    # The order of linking is important: 'glibc-compatibility' must be before libc but after all other libraries.
-    if (GLIBC_COMPATIBILITY)
-        message (STATUS "Some symbols from glibc will be replaced for compatibility")
-
-        string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC)
-        set (CMAKE_POSTFIX_VARIABLE "CMAKE_${CMAKE_BUILD_TYPE_UC}_POSTFIX")
-
-        # FIXME: glibc-compatibility may be non-static in some builds!
-        set (DEFAULT_LIBS "${DEFAULT_LIBS} ${ClickHouse_BINARY_DIR}/libs/libglibc-compatibility/libglibc-compatibility${${CMAKE_POSTFIX_VARIABLE}}.a")
-    endif ()
-
-    # Add Libc. GLIBC is actually a collection of interdependent libraries.
-    set (DEFAULT_LIBS "${DEFAULT_LIBS} -lrt -ldl -lpthread -lm -lc")
-
-    # Note: we'd rather use Musl libc library, but it's little bit more difficult to use.
-
-    message(STATUS "Default libraries: ${DEFAULT_LIBS}")
-endif ()
-
-if (NOT GLIBC_COMPATIBILITY)
-    set (M_LIBRARY m)
-endif ()
-
-if (DEFAULT_LIBS)
-    # Add default libs to all targets as the last dependency.
-    set(CMAKE_CXX_STANDARD_LIBRARIES ${DEFAULT_LIBS})
-    set(CMAKE_C_STANDARD_LIBRARIES ${DEFAULT_LIBS})
-endif ()
-
 if (NOT MAKE_STATIC_LIBRARIES)
     set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 endif ()
@@ -420,20 +292,12 @@ if (UNBUNDLED)
 else ()
     set(NOT_UNBUNDLED 1)
 endif ()
+
 # Using system libs can cause lot of warnings in includes.
 if (UNBUNDLED OR NOT (OS_LINUX OR APPLE) OR ARCH_32)
     option (NO_WERROR "Disable -Werror compiler option" ON)
 endif ()
 
-if (USE_LIBCXX)
-    set (HAVE_LIBCXX 1)
-    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
-endif()
-
-if (USE_LIBCXX AND USE_INTERNAL_LIBCXX_LIBRARY)
-    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -nostdinc++ -isystem ${LIBCXX_INCLUDE_DIR} -isystem ${LIBCXXABI_INCLUDE_DIR}")
-endif ()
-
 message (STATUS "Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_LIBRARY_ARCHITECTURE} ; USE_STATIC_LIBRARIES=${USE_STATIC_LIBRARIES} MAKE_STATIC_LIBRARIES=${MAKE_STATIC_LIBRARIES} SPLIT_SHARED=${SPLIT_SHARED_LIBRARIES} UNBUNDLED=${UNBUNDLED} CCACHE=${CCACHE_FOUND} ${CCACHE_VERSION}")
 
 include(GNUInstallDirs)
@@ -499,79 +363,11 @@ include (libs/libmysqlxx/cmake/find_mysqlclient.cmake)
 
 include (cmake/print_flags.cmake)
 
+install (EXPORT global DESTINATION cmake)
+
 add_subdirectory (contrib EXCLUDE_FROM_ALL)
 add_subdirectory (libs)
 add_subdirectory (utils)
 add_subdirectory (dbms)
 
 include (cmake/print_include_directories.cmake)
-
-if (GLIBC_COMPATIBILITY OR USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING)
-    # FIXME: actually glibc-compatibility should always be built first,
-    #        because it's unconditionally linked via $DEFAULT_LIBS,
-    #        and these looks like the first places that get linked.
-    function (add_default_dependencies target_name)
-        if (TARGET ${target_name})
-            if (GLIBC_COMPATIBILITY)
-                add_dependencies(${target_name} glibc-compatibility)
-            endif ()
-
-            if (USE_LIBCXX AND USE_INTERNAL_LIBCXX_LIBRARY)
-                add_dependencies(${target_name} cxx_static cxxabi_static)
-            endif ()
-
-            if (USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING)
-                add_dependencies(${target_name} unwind_static)
-            endif ()
-        endif ()
-    endfunction ()
-
-    add_default_dependencies(ltdl)
-    add_default_dependencies(zlibstatic)
-    add_default_dependencies(jemalloc)
-    add_default_dependencies(memcpy)
-    add_default_dependencies(Foundation)
-    add_default_dependencies(common)
-    add_default_dependencies(gtest)
-    add_default_dependencies(lz4)
-    add_default_dependencies(zstd)
-    add_default_dependencies(snappy)
-    add_default_dependencies(arrow)
-    add_default_dependencies(protoc)
-    add_default_dependencies(thrift_static)
-    add_default_dependencies(cityhash)
-    add_default_dependencies(farmhash)
-    add_default_dependencies(murmurhash)
-    add_default_dependencies(metrohash)
-    add_default_dependencies(metrohash128)
-    add_default_dependencies(consistent-hashing)
-    add_default_dependencies(double-conversion)
-    add_default_dependencies(cctz)
-    add_default_dependencies(kj)
-    add_default_dependencies(simdjson)
-    add_default_dependencies(apple_rt)
-    add_default_dependencies(h3)
-    add_default_dependencies(re2)
-    add_default_dependencies(re2_st)
-    add_default_dependencies(hs_compile_shared)
-    add_default_dependencies(hs_exec_shared)
-    add_default_dependencies(hs_shared)
-    add_default_dependencies(widechar_width)
-    add_default_dependencies(string_utils)
-    add_default_dependencies(consistent-hashing-sumbur)
-    add_default_dependencies(boost_program_options_internal)
-    add_default_dependencies(boost_system_internal)
-    add_default_dependencies(boost_regex_internal)
-    add_default_dependencies(roaring)
-    add_default_dependencies(btrie)
-    add_default_dependencies(cpuid)
-    add_default_dependencies(mysqlclient)
-    add_default_dependencies(zlib)
-    add_default_dependencies(thrift)
-    add_default_dependencies(brotli)
-    add_default_dependencies(libprotobuf)
-    add_default_dependencies(base64)
-    add_default_dependencies(readpassphrase)
-    add_default_dependencies(unwind_static)
-    add_default_dependencies(fastops)
-endif ()

cmake/default_libs.cmake

@@ -0,0 +1,48 @@
+# Set standard, system and compiler libraries explicitly.
+# This is intended for more control of what we are linking.
+
+set (DEFAULT_LIBS "-nodefaultlibs")
+
+if (OS_LINUX)
+    # We need builtins from Clang's RT even without libcxx - for ubsan+int128.
+    # See https://bugs.llvm.org/show_bug.cgi?id=16404
+    if (COMPILER_CLANG)
+        execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-${CMAKE_SYSTEM_PROCESSOR}.a OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE)
+    else ()
+        set (BUILTINS_LIBRARY "-lgcc")
+    endif ()
+
+    set (DEFAULT_LIBS "${DEFAULT_LIBS} ${BUILTINS_LIBRARY} ${COVERAGE_OPTION} -lc -lm -lrt -lpthread -ldl")
+
+    message(STATUS "Default libraries: ${DEFAULT_LIBS}")
+endif ()
+
+set(CMAKE_CXX_STANDARD_LIBRARIES ${DEFAULT_LIBS})
+set(CMAKE_C_STANDARD_LIBRARIES ${DEFAULT_LIBS})
+
+# Global libraries
+
+add_library(global-libs INTERFACE)
+
+# Unfortunately '-pthread' doesn't work with '-nodefaultlibs'.
+# Just make sure we have pthreads at all.
+set(THREADS_PREFER_PTHREAD_FLAG ON)
+find_package(Threads REQUIRED)
+
+add_subdirectory(libs/libglibc-compatibility)
+include (cmake/find_unwind.cmake)
+include (cmake/find_cxx.cmake)
+
+add_library(global-group INTERFACE)
+target_link_libraries(global-group INTERFACE
+    -Wl,--start-group
+    $<TARGET_PROPERTY:global-libs,INTERFACE_LINK_LIBRARIES>
+    -Wl,--end-group
+)
+
+link_libraries(global-group)
+
+install(
+    TARGETS global-group global-libs
+    EXPORT global
+)

cmake/find_capnp.cmake

@@ -1,50 +1,20 @@
-option (ENABLE_CAPNP "Enable Cap'n Proto" ON)
+option (USE_CAPNP "Enable Cap'n Proto" ON)
-
-if (ENABLE_CAPNP)
-    # cmake 3.5.1 bug:
-    # capnproto uses this cmake feature:
-    # target_compile_features(kj PUBLIC cxx_constexpr)
-    # old cmake adds -std=gnu++11 to end of all compile commands (even if -std=gnu++17 already present in compile string)
-    # cmake 3.9.1 (ubuntu artful) have no this bug (c++17 support added to cmake 3.8.2)
-    if (CMAKE_VERSION VERSION_LESS "3.8.0")
-       set (USE_INTERNAL_CAPNP_LIBRARY_DEFAULT 0)
-       set (MISSING_INTERNAL_CAPNP_LIBRARY 1)
-    else ()
-       set (USE_INTERNAL_CAPNP_LIBRARY_DEFAULT ${NOT_UNBUNDLED})
-    endif ()
-
-    option (USE_INTERNAL_CAPNP_LIBRARY "Set to FALSE to use system capnproto library instead of bundled" ${USE_INTERNAL_CAPNP_LIBRARY_DEFAULT})
-
-    if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/capnproto/c++/CMakeLists.txt")
-       if (USE_INTERNAL_CAPNP_LIBRARY)
-           message (WARNING "submodule contrib/capnproto is missing. to fix try run: \n git submodule update --init --recursive")
-       endif ()
-       set (USE_INTERNAL_CAPNP_LIBRARY 0)
-       set (MISSING_INTERNAL_CAPNP_LIBRARY 1)
-    endif ()
-
-    if (NOT USE_INTERNAL_CAPNP_LIBRARY)
-        set (CAPNP_PATHS "/usr/local/opt/capnp/lib")
-        set (CAPNP_INCLUDE_PATHS "/usr/local/opt/capnp/include")
-        find_library (CAPNP capnp PATHS ${CAPNP_PATHS})
-        find_library (CAPNPC capnpc PATHS ${CAPNP_PATHS})
-        find_library (KJ kj PATHS ${CAPNP_PATHS})
-        set (CAPNP_LIBRARY ${CAPNPC} ${CAPNP} ${KJ})
-        find_path (CAPNP_INCLUDE_DIR NAMES capnp/schema-parser.h PATHS ${CAPNP_INCLUDE_PATHS})
-    endif ()
-
-    if (CAPNP_INCLUDE_DIR AND CAPNP_LIBRARY)
-        set(USE_CAPNP 1)
-    elseif (NOT MISSING_INTERNAL_CAPNP_LIBRARY)
-        set (USE_INTERNAL_CAPNP_LIBRARY 1)
-        set (CAPNP_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/capnproto/c++/src")
-        set (CAPNP_LIBRARY capnpc)
-        set (USE_CAPNP 1)
-    endif ()
-endif ()
 
 if (USE_CAPNP)
-    message (STATUS "Using capnp=${USE_CAPNP}: ${CAPNP_INCLUDE_DIR} : ${CAPNP_LIBRARY}")
+    option (USE_INTERNAL_CAPNP_LIBRARY "Set to FALSE to use system capnproto library instead of bundled" ${NOT_UNBUNDLED})
-else ()
+
-    message (STATUS "Build without capnp (support for Cap'n Proto format will be disabled)")
+    # FIXME: refactor to use `add_library(… IMPORTED)` if possible.
+    if (NOT USE_INTERNAL_CAPNP_LIBRARY)
+        find_library (KJ kj)
+        find_library (CAPNP capnp)
+        find_library (CAPNPC capnpc)
+
+        set (CAPNP_LIBRARIES ${CAPNPC} ${CAPNP} ${KJ})
+    else ()
+        add_subdirectory(contrib/capnproto-cmake)
+
+        set (CAPNP_LIBRARIES capnpc)
+    endif ()
+
+    message (STATUS "Using capnp: ${CAPNP_LIBRARIES}")
 endif ()

cmake/find_cxx.cmake

@@ -1,26 +1,49 @@
-if (NOT APPLE)
+if (OS_LINUX AND COMPILER_CLANG)
+    option (USE_LIBCXX "Use libc++ and libc++abi instead of libstdc++" ${HAVE_LIBCXX})
     option (USE_INTERNAL_LIBCXX_LIBRARY "Set to FALSE to use system libcxx and libcxxabi libraries instead of bundled" ${NOT_UNBUNDLED})
+endif()
+
+if (USE_LIBCXX)
+    set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_LIBCPP_DEBUG=0") # More checks in debug build.
 endif ()
 
+# FIXME: make better check for submodule presence
 if (USE_INTERNAL_LIBCXX_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libcxx/include/vector")
     message (WARNING "submodule contrib/libcxx is missing. to fix try run: \n git submodule update --init --recursive")
     set (USE_INTERNAL_LIBCXX_LIBRARY 0)
 endif ()
 
+# FIXME: make better check for submodule presence
 if (USE_INTERNAL_LIBCXX_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libcxxabi/src")
     message (WARNING "submodule contrib/libcxxabi is missing. to fix try run: \n git submodule update --init --recursive")
-    set (USE_INTERNAL_LIBCXXABI_LIBRARY 0)
+    set (USE_INTERNAL_LIBCXX_LIBRARY 0)
 endif ()
 
-if (NOT USE_INTERNAL_LIBCXX_LIBRARY)
+if (USE_LIBCXX)
+    if (NOT USE_INTERNAL_LIBCXX_LIBRARY)
         find_library (LIBCXX_LIBRARY c++)
+        find_library (LIBCXXFS_LIBRARY c++fs)
         find_library (LIBCXXABI_LIBRARY c++abi)
-else ()
-    set (LIBCXX_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libcxx/include)
-    set (LIBCXXABI_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libcxxabi/include)
-    set (LIBCXX_LIBRARY cxx_static)
-    set (LIBCXXABI_LIBRARY cxxabi_static)
-endif ()
 
-message (STATUS "Using libcxx: ${LIBCXX_LIBRARY}")
+        target_link_libraries(global-libs INTERFACE ${EXCEPTION_HANDLING_LIBRARY})
-message (STATUS "Using libcxxabi: ${LIBCXXABI_LIBRARY}")
+    else ()
+        set (LIBCXX_LIBRARY cxx)
+        set (LIBCXXABI_LIBRARY cxxabi)
+        add_subdirectory(contrib/libcxxabi-cmake)
+        add_subdirectory(contrib/libcxx-cmake)
+
+        # Exception handling library is embedded into libcxxabi.
+    endif ()
+
+    target_link_libraries(global-libs INTERFACE ${LIBCXX_LIBRARY} ${LIBCXXABI_LIBRARY} ${LIBCXXFS_LIBRARY})
+
+    set (HAVE_LIBCXX 1)
+    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
+
+    message (STATUS "Using libcxx: ${LIBCXX_LIBRARY}")
+    message (STATUS "Using libcxxfs: ${LIBCXXFS_LIBRARY}")
+    message (STATUS "Using libcxxabi: ${LIBCXXABI_LIBRARY}")
+else ()
+    target_link_libraries(global-libs INTERFACE -l:libstdc++.a -l:libstdc++fs.a) # Always link these libraries as static
+    target_link_libraries(global-libs INTERFACE ${EXCEPTION_HANDLING_LIBRARY})
+endif ()

cmake/find_unwind.cmake

@@ -1,59 +1,17 @@
-include (CMakePushCheckState)
+option (USE_UNWIND "Enable libunwind (better stacktraces)" ON)
-cmake_push_check_state ()
 
-option (ENABLE_UNWIND "Enable libunwind (better stacktraces)" ON)
+if (NOT CMAKE_SYSTEM MATCHES "Linux" OR ARCH_ARM OR ARCH_32)
+    set (USE_UNWIND OFF)
+endif ()
 
-if (ENABLE_UNWIND)
+if (USE_UNWIND)
+    add_subdirectory(contrib/libunwind-cmake)
+    set (UNWIND_LIBRARIES unwind)
+    set (EXCEPTION_HANDLING_LIBRARY ${UNWIND_LIBRARIES})
 
-if (CMAKE_SYSTEM MATCHES "Linux" AND NOT ARCH_ARM AND NOT ARCH_32)
+    message (STATUS "Using libunwind: ${UNWIND_LIBRARIES}")
-    option (USE_INTERNAL_UNWIND_LIBRARY "Set to FALSE to use system unwind library instead of bundled" ${NOT_UNBUNDLED})
 else ()
-    option (USE_INTERNAL_UNWIND_LIBRARY "Set to FALSE to use system unwind library instead of bundled" OFF)
+    set (EXCEPTION_HANDLING_LIBRARY gcc_eh)
 endif ()
 
-if (NOT USE_INTERNAL_UNWIND_LIBRARY)
+message (STATUS "Using exception handler: ${EXCEPTION_HANDLING_LIBRARY}")
-    find_library (UNWIND_LIBRARY unwind)
-    find_path (UNWIND_INCLUDE_DIR NAMES unwind.h PATHS ${UNWIND_INCLUDE_PATHS})
-
-    include (CheckCXXSourceCompiles)
-    set(CMAKE_REQUIRED_INCLUDES ${UNWIND_INCLUDE_DIR})
-    set(CMAKE_REQUIRED_LIBRARIES ${UNWIND_LIBRARY})
-    check_cxx_source_compiles("
-    #include <ucontext.h>
-    #define UNW_LOCAL_ONLY
-    #include <libunwind.h>
-    int main () {
-       ucontext_t context;
-       unw_cursor_t cursor;
-       unw_init_local(&cursor, &context);
-       return 0;
-    }
-    " HAVE_UNW_INIT_LOCAL)
-    if (NOT HAVE_UNW_INIT_LOCAL)
-       set(UNWIND_LIBRARY "")
-       set(UNWIND_INCLUDE_DIR "")
-    endif ()
-
-endif ()
-
-if (UNWIND_LIBRARY AND UNWIND_INCLUDE_DIR)
-    set (USE_UNWIND 1)
-elseif (CMAKE_SYSTEM MATCHES "Linux" AND NOT ARCH_ARM AND NOT ARCH_32 AND NOT UNBUNDLED)
-    set (USE_INTERNAL_UNWIND_LIBRARY 1)
-
-    set (PACKAGE_VERSION "9.0.0svn" CACHE STRING "")
-
-    set (UNWIND_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libunwind/include")
-
-    set (LIBUNWIND_ENABLE_SHARED OFF CACHE BOOL "")
-    set (LIBUNWIND_ENABLE_STATIC ON CACHE BOOL "")
-    set (UNWIND_LIBRARY unwind_static)
-
-    set (USE_UNWIND 1)
-endif ()
-
-endif ()
-
-message (STATUS "Using unwind=${USE_UNWIND}: ${UNWIND_INCLUDE_DIR} : ${UNWIND_LIBRARY}")
-
-cmake_pop_check_state ()

cmake/test_compiler.cmake

@@ -1,47 +0,0 @@
-include (CheckCXXSourceCompiles)
-include (CMakePushCheckState)
-
-set(THREADS_PREFER_PTHREAD_FLAG ON)
-find_package(Threads)
-
-cmake_push_check_state ()
-
-if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
-# clang4 : -no-pie cause error
-# clang6 : -no-pie cause warning
-
-    if (MAKE_STATIC_LIBRARIES)
-        set (TEST_FLAG "-Wl,-Bstatic -stdlib=libc++ -lc++ -lc++abi -Wl,-Bdynamic")
-    else ()
-        set (TEST_FLAG "-stdlib=libc++ -lc++ -lc++abi")
-    endif ()
-
-    set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG}")
-    set (CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} Threads::Threads)
-
-    check_cxx_source_compiles("
-        #include <iostream>
-        int main() {
-            std::cerr << std::endl;
-            return 0;
-        }
-        " HAVE_LIBCXX)
-
-else ()
-
-    set (TEST_FLAG "-no-pie")
-    set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG}")
-
-    check_cxx_source_compiles("
-        int main() {
-            return 0;
-        }
-        " HAVE_NO_PIE)
-
-    if (HAVE_NO_PIE)
-        set (FLAG_NO_PIE ${TEST_FLAG})
-    endif ()
-
-endif ()
-
-cmake_pop_check_state ()

contrib/CMakeLists.txt

@@ -23,16 +23,6 @@ if (USE_INTERNAL_ORC_LIBRARY)
     add_subdirectory(orc)
 endif()
 
-if (USE_INTERNAL_UNWIND_LIBRARY)
-    add_subdirectory (libunwind-cmake)
-endif ()
-
-if (USE_LIBCXX AND USE_INTERNAL_LIBCXX_LIBRARY)
-    add_subdirectory(libcxx-cmake)
-    add_subdirectory(libcxxabi-cmake)
-endif()
-
-
 if (USE_INTERNAL_BOOST_LIBRARY)
     add_subdirectory (boost-cmake)
 endif ()
@@ -172,15 +162,6 @@ if (ENABLE_ODBC AND USE_INTERNAL_ODBC_LIBRARY)
     add_library(ODBC::ODBC ALIAS ${ODBC_LIBRARIES})
 endif ()
 
-if (ENABLE_CAPNP AND USE_INTERNAL_CAPNP_LIBRARY)
-    set (BUILD_TESTING 0 CACHE INTERNAL "")
-    set (_save ${CMAKE_CXX_EXTENSIONS})
-    set (CMAKE_CXX_EXTENSIONS)
-    add_subdirectory (capnproto/c++)
-    set (CMAKE_CXX_EXTENSIONS ${_save})
-    target_include_directories(${CAPNP_LIBRARY} PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/capnproto/c++/src>)
-endif ()
-
 if (USE_INTERNAL_PARQUET_LIBRARY)
 if (USE_INTERNAL_PARQUET_LIBRARY_NATIVE_CMAKE)
     # We dont use arrow's cmakefiles because they uses too many depends and download some libs in compile time

contrib/arrow-cmake/CMakeLists.txt

@@ -44,7 +44,6 @@ set( thriftcpp_threads_SOURCES
 add_library(${THRIFT_LIBRARY} ${thriftcpp_SOURCES} ${thriftcpp_threads_SOURCES})
 set_target_properties(${THRIFT_LIBRARY} PROPERTIES CXX_STANDARD 14) # REMOVE after https://github.com/apache/thrift/pull/1641
 target_include_directories(${THRIFT_LIBRARY} SYSTEM PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/thrift/lib/cpp/src PRIVATE ${Boost_INCLUDE_DIRS})
-target_link_libraries(${THRIFT_LIBRARY} PRIVATE Threads::Threads)
 
 
 # === orc
@@ -219,7 +218,7 @@ endif()
 add_library(${ARROW_LIBRARY} ${ARROW_SRCS})
 add_dependencies(${ARROW_LIBRARY} protoc)
 target_include_directories(${ARROW_LIBRARY} SYSTEM PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/cpp/src ${Boost_INCLUDE_DIRS})
-target_link_libraries(${ARROW_LIBRARY} PRIVATE ${DOUBLE_CONVERSION_LIBRARIES} ${Protobuf_LIBRARY} Threads::Threads)
+target_link_libraries(${ARROW_LIBRARY} PRIVATE ${DOUBLE_CONVERSION_LIBRARIES} ${Protobuf_LIBRARY})
 if (ARROW_WITH_LZ4)
     target_link_libraries(${ARROW_LIBRARY} PRIVATE ${LZ4_LIBRARY})
 endif()

contrib/capnproto-cmake/CMakeLists.txt

@@ -0,0 +1,68 @@
+set (CAPNPROTO_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/capnproto/c++/src)
+
+set (KJ_SRCS
+    ${CAPNPROTO_SOURCE_DIR}/kj/array.c++
+    ${CAPNPROTO_SOURCE_DIR}/kj/common.c++
+    ${CAPNPROTO_SOURCE_DIR}/kj/debug.c++
+    ${CAPNPROTO_SOURCE_DIR}/kj/exception.c++
+    ${CAPNPROTO_SOURCE_DIR}/kj/io.c++
+    ${CAPNPROTO_SOURCE_DIR}/kj/memory.c++
+    ${CAPNPROTO_SOURCE_DIR}/kj/mutex.c++
+    ${CAPNPROTO_SOURCE_DIR}/kj/string.c++
+    ${CAPNPROTO_SOURCE_DIR}/kj/hash.c++
+    ${CAPNPROTO_SOURCE_DIR}/kj/table.c++
+    ${CAPNPROTO_SOURCE_DIR}/kj/thread.c++
+    ${CAPNPROTO_SOURCE_DIR}/kj/main.c++
+    ${CAPNPROTO_SOURCE_DIR}/kj/arena.c++
+    ${CAPNPROTO_SOURCE_DIR}/kj/test-helpers.c++
+    ${CAPNPROTO_SOURCE_DIR}/kj/units.c++
+    ${CAPNPROTO_SOURCE_DIR}/kj/encoding.c++
+
+    ${CAPNPROTO_SOURCE_DIR}/kj/refcount.c++
+    ${CAPNPROTO_SOURCE_DIR}/kj/string-tree.c++
+    ${CAPNPROTO_SOURCE_DIR}/kj/time.c++
+    ${CAPNPROTO_SOURCE_DIR}/kj/filesystem.c++
+    ${CAPNPROTO_SOURCE_DIR}/kj/filesystem-disk-unix.c++
+    ${CAPNPROTO_SOURCE_DIR}/kj/filesystem-disk-win32.c++
+    ${CAPNPROTO_SOURCE_DIR}/kj/parse/char.c++
+)
+
+add_library(kj ${KJ_SRCS})
+target_include_directories(kj INTERFACE ${CAPNPROTO_SOURCE_DIR})
+
+set (CAPNP_SRCS
+    ${CAPNPROTO_SOURCE_DIR}/capnp/c++.capnp.c++
+    ${CAPNPROTO_SOURCE_DIR}/capnp/blob.c++
+    ${CAPNPROTO_SOURCE_DIR}/capnp/arena.c++
+    ${CAPNPROTO_SOURCE_DIR}/capnp/layout.c++
+    ${CAPNPROTO_SOURCE_DIR}/capnp/list.c++
+    ${CAPNPROTO_SOURCE_DIR}/capnp/any.c++
+    ${CAPNPROTO_SOURCE_DIR}/capnp/message.c++
+    ${CAPNPROTO_SOURCE_DIR}/capnp/schema.capnp.c++
+    ${CAPNPROTO_SOURCE_DIR}/capnp/serialize.c++
+    ${CAPNPROTO_SOURCE_DIR}/capnp/serialize-packed.c++
+
+    ${CAPNPROTO_SOURCE_DIR}/capnp/schema.c++
+    ${CAPNPROTO_SOURCE_DIR}/capnp/schema-loader.c++
+    ${CAPNPROTO_SOURCE_DIR}/capnp/dynamic.c++
+    ${CAPNPROTO_SOURCE_DIR}/capnp/stringify.c++
+)
+
+add_library(capnp ${CAPNP_SRCS})
+target_link_libraries(capnp PUBLIC kj)
+
+set (CAPNPC_SRCS
+    ${CAPNPROTO_SOURCE_DIR}/capnp/compiler/type-id.c++
+    ${CAPNPROTO_SOURCE_DIR}/capnp/compiler/error-reporter.c++
+    ${CAPNPROTO_SOURCE_DIR}/capnp/compiler/lexer.capnp.c++
+    ${CAPNPROTO_SOURCE_DIR}/capnp/compiler/lexer.c++
+    ${CAPNPROTO_SOURCE_DIR}/capnp/compiler/grammar.capnp.c++
+    ${CAPNPROTO_SOURCE_DIR}/capnp/compiler/parser.c++
+    ${CAPNPROTO_SOURCE_DIR}/capnp/compiler/node-translator.c++
+    ${CAPNPROTO_SOURCE_DIR}/capnp/compiler/compiler.c++
+    ${CAPNPROTO_SOURCE_DIR}/capnp/schema-parser.c++
+    ${CAPNPROTO_SOURCE_DIR}/capnp/serialize-text.c++
+)
+
+add_library(capnpc ${CAPNPC_SRCS})
+target_link_libraries(capnpc PUBLIC capnp)

contrib/jemalloc-cmake/CMakeLists.txt

@@ -59,7 +59,6 @@ if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")
 
     if (USE_UNWIND)
         target_compile_definitions (jemalloc PRIVATE -DJEMALLOC_PROF_LIBUNWIND=1)
-        target_include_directories (jemalloc BEFORE PRIVATE ${UNWIND_INCLUDE_DIR})
+        target_link_libraries (jemalloc PRIVATE ${UNWIND_LIBRARIES})
-        target_link_libraries (jemalloc PRIVATE ${UNWIND_LIBRARY})
     endif ()
 endif ()

contrib/libcxx-cmake/CMakeLists.txt

@@ -1,5 +1,4 @@
 set(LIBCXX_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libcxx)
-#set(LIBCXX_BINARY_DIR ${ClickHouse_BINARY_DIR}/contrib/libcxx)
 
 set(SRCS
 ${LIBCXX_SOURCE_DIR}/src/optional.cpp
@@ -16,10 +15,6 @@ ${LIBCXX_SOURCE_DIR}/src/condition_variable.cpp
 ${LIBCXX_SOURCE_DIR}/src/hash.cpp
 ${LIBCXX_SOURCE_DIR}/src/string.cpp
 ${LIBCXX_SOURCE_DIR}/src/debug.cpp
-#${LIBCXX_SOURCE_DIR}/src/support/win32/support.cpp
-#${LIBCXX_SOURCE_DIR}/src/support/win32/locale_win32.cpp
-#${LIBCXX_SOURCE_DIR}/src/support/win32/thread_win32.cpp
-#${LIBCXX_SOURCE_DIR}/src/support/solaris/xlocale.cpp
 ${LIBCXX_SOURCE_DIR}/src/stdexcept.cpp
 ${LIBCXX_SOURCE_DIR}/src/utility.cpp
 ${LIBCXX_SOURCE_DIR}/src/any.cpp
@@ -43,9 +38,16 @@ ${LIBCXX_SOURCE_DIR}/src/system_error.cpp
 ${LIBCXX_SOURCE_DIR}/src/random.cpp
 )
 
-add_library(cxx_static ${SRCS})
+add_library(cxx ${SRCS})
 
-target_include_directories(cxx_static PUBLIC ${LIBCXX_SOURCE_DIR}/include)
+target_include_directories(cxx SYSTEM BEFORE PUBLIC $<BUILD_INTERFACE:${LIBCXX_SOURCE_DIR}/include>)
-target_compile_definitions(cxx_static PRIVATE -D_LIBCPP_BUILDING_LIBRARY -DLIBCXX_BUILDING_LIBCXXABI)
+target_compile_definitions(cxx PRIVATE -D_LIBCPP_BUILDING_LIBRARY -DLIBCXX_BUILDING_LIBCXXABI)
-target_compile_options(cxx_static PRIVATE -nostdinc++)
+target_compile_options(cxx PRIVATE -nostdinc++)
+target_link_libraries(cxx PUBLIC cxxabi)
 
+install(
+    TARGETS cxx
+    EXPORT global
+    ARCHIVE DESTINATION lib
+    RUNTIME DESTINATION lib
+)

contrib/libcxxabi-cmake/CMakeLists.txt

@@ -1,13 +1,10 @@
 set(LIBCXXABI_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libcxxabi)
-set(LIBCXX_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libcxx)
-#set(LIBCXXABI_BINARY_DIR ${ClickHouse_BINARY_DIR}/contrib/libcxxabi)
 
 set(SRCS
 ${LIBCXXABI_SOURCE_DIR}/src/stdlib_stdexcept.cpp
 ${LIBCXXABI_SOURCE_DIR}/src/cxa_virtual.cpp
 ${LIBCXXABI_SOURCE_DIR}/src/cxa_thread_atexit.cpp
 ${LIBCXXABI_SOURCE_DIR}/src/fallback_malloc.cpp
-#${LIBCXXABI_SOURCE_DIR}/src/cxa_noexception.cpp
 ${LIBCXXABI_SOURCE_DIR}/src/cxa_guard.cpp
 ${LIBCXXABI_SOURCE_DIR}/src/cxa_default_handlers.cpp
 ${LIBCXXABI_SOURCE_DIR}/src/cxa_personality.cpp
@@ -25,10 +22,24 @@ ${LIBCXXABI_SOURCE_DIR}/src/cxa_vector.cpp
 ${LIBCXXABI_SOURCE_DIR}/src/stdlib_new_delete.cpp
 )
 
-add_library(cxxabi_static ${SRCS})
+add_library(cxxabi ${SRCS})
 
-target_include_directories(cxxabi_static PUBLIC ${LIBCXXABI_SOURCE_DIR}/include ${LIBCXX_SOURCE_DIR}/include)
+target_include_directories(cxxabi SYSTEM BEFORE
-target_compile_definitions(cxxabi_static PRIVATE -D_LIBCPP_BUILDING_LIBRARY)
+    PUBLIC $<BUILD_INTERFACE:${LIBCXXABI_SOURCE_DIR}/include>
-target_compile_options(cxxabi_static PRIVATE -nostdinc++ -fno-sanitize=undefined) # If we don't disable UBSan, infinite recursion happens in dynamic_cast.
+    PRIVATE $<BUILD_INTERFACE:${LIBCXXABI_SOURCE_DIR}/../libcxx/include>
+)
+target_compile_definitions(cxxabi PRIVATE -D_LIBCPP_BUILDING_LIBRARY)
+target_compile_options(cxxabi PRIVATE -nostdinc++ -fno-sanitize=undefined) # If we don't disable UBSan, infinite recursion happens in dynamic_cast.
 
+if (USE_UNWIND)
+    target_link_libraries(cxxabi PRIVATE ${UNWIND_LIBRARIES})
+else ()
+    target_link_libraries(cxxabi PRIVATE gcc_eh)
+endif ()
 
+install(
+    TARGETS cxxabi
+    EXPORT global
+    ARCHIVE DESTINATION lib
+    RUNTIME DESTINATION lib
+)

contrib/librdkafka-cmake/CMakeLists.txt

@@ -65,7 +65,7 @@ add_library(rdkafka ${SRCS})
 target_include_directories(rdkafka SYSTEM PUBLIC include)
 target_include_directories(rdkafka SYSTEM PUBLIC ${RDKAFKA_SOURCE_DIR})         # Because weird logic with "include_next" is used.
 target_include_directories(rdkafka SYSTEM PRIVATE ${ZSTD_INCLUDE_DIR}/common)   # Because wrong path to "zstd_errors.h" is used.
-target_link_libraries(rdkafka PRIVATE ${ZLIB_LIBRARIES} ${ZSTD_LIBRARY} ${LZ4_LIBRARY} ${LIBGSASL_LIBRARY} Threads::Threads)
+target_link_libraries(rdkafka PRIVATE ${ZLIB_LIBRARIES} ${ZSTD_LIBRARY} ${LZ4_LIBRARY} ${LIBGSASL_LIBRARY})
 if(OPENSSL_SSL_LIBRARY AND OPENSSL_CRYPTO_LIBRARY)
     target_link_libraries(rdkafka PRIVATE ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY})
 endif()

contrib/libunwind-cmake/CMakeLists.txt

@@ -24,9 +24,15 @@ set(LIBUNWIND_SOURCES
     ${LIBUNWIND_C_SOURCES}
     ${LIBUNWIND_ASM_SOURCES})
 
-add_library(unwind_static ${LIBUNWIND_SOURCES})
+add_library(unwind ${LIBUNWIND_SOURCES})
 
-target_include_directories(unwind_static SYSTEM BEFORE PUBLIC ${LIBUNWIND_SOURCE_DIR}/include)
+target_include_directories(unwind SYSTEM BEFORE PUBLIC $<BUILD_INTERFACE:${LIBUNWIND_SOURCE_DIR}/include>)
-target_compile_definitions(unwind_static PRIVATE -D_LIBUNWIND_NO_HEAP=1 -D_DEBUG -D_LIBUNWIND_IS_NATIVE_ONLY)
+target_compile_definitions(unwind PRIVATE -D_LIBUNWIND_NO_HEAP=1 -D_DEBUG -D_LIBUNWIND_IS_NATIVE_ONLY)
-target_compile_options(unwind_static PRIVATE -fno-exceptions -funwind-tables -fno-sanitize=all -nostdinc++ -fno-rtti)
+target_compile_options(unwind PRIVATE -fno-exceptions -funwind-tables -fno-sanitize=all -nostdinc++ -fno-rtti)
-target_link_libraries(unwind_static PRIVATE Threads::Threads ${CMAKE_DL_LIBS})
+
+install(
+    TARGETS unwind
+    EXPORT global
+    LIBRARY DESTINATION lib
+    ARCHIVE DESTINATION lib
+)

contrib/mariadb-connector-c-cmake/CMakeLists.txt

@@ -62,11 +62,6 @@ endif()
 
 add_library(mysqlclient ${SRCS})
 
-target_link_libraries(mysqlclient PRIVATE ${CMAKE_DL_LIBS} Threads::Threads)
-if(M_LIBRARY)
-    target_link_libraries(mysqlclient PRIVATE ${M_LIBRARY})
-endif()
-
 if(OPENSSL_LIBRARIES)
     target_link_libraries(mysqlclient PRIVATE ${OPENSSL_LIBRARIES})
     target_compile_definitions(mysqlclient PRIVATE -D HAVE_OPENSSL -D HAVE_TLS)

dbms/CMakeLists.txt

@@ -97,8 +97,6 @@ add_subdirectory (src)
 set(dbms_headers)
 set(dbms_sources)
 
-include(../cmake/dbms_glob_sources.cmake)
-
 add_headers_and_sources(clickhouse_common_io src/Common)
 add_headers_and_sources(clickhouse_common_io src/Common/HashTable)
 add_headers_and_sources(clickhouse_common_io src/IO)
@@ -163,9 +161,7 @@ if (OS_FREEBSD)
 endif ()
 
 if (USE_UNWIND)
-    if (NOT USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING)
+    target_link_libraries (clickhouse_common_io PRIVATE ${UNWIND_LIBRARIES})
-        target_link_libraries (clickhouse_common_io PRIVATE ${UNWIND_LIBRARY})
-    endif ()
 endif ()
 
 add_subdirectory(src/Common/ZooKeeper)
@@ -241,15 +237,10 @@ target_link_libraries(clickhouse_common_io
     ${EXECINFO_LIBRARIES}
         PUBLIC
     ${Boost_SYSTEM_LIBRARY}
+    ${Boost_PROGRAM_OPTIONS_LIBRARY}
         PRIVATE
     apple_rt
         PUBLIC
-    Threads::Threads
-        PRIVATE
-    ${CMAKE_DL_LIBS}
-        PRIVATE
-    rt
-        PUBLIC
     roaring
 )
 
@@ -297,7 +288,6 @@ target_link_libraries (dbms
     ${Boost_FILESYSTEM_LIBRARY}
         PUBLIC
     ${Boost_SYSTEM_LIBRARY}
-    Threads::Threads
 )
 
 target_include_directories(dbms PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/src/Core/include)
@@ -364,10 +354,7 @@ if (USE_ICU)
 endif ()
 
 if (USE_CAPNP)
-    target_link_libraries (dbms PRIVATE ${CAPNP_LIBRARY})
+    target_link_libraries (dbms PRIVATE ${CAPNP_LIBRARIES})
-    if (NOT USE_INTERNAL_CAPNP_LIBRARY)
-        target_include_directories (dbms SYSTEM BEFORE PRIVATE ${CAPNP_INCLUDE_DIR})
-    endif ()
 endif ()
 
 if (USE_PARQUET)
@@ -380,7 +367,6 @@ endif ()
 if(OPENSSL_CRYPTO_LIBRARY)
     target_link_libraries(dbms PRIVATE ${OPENSSL_CRYPTO_LIBRARY})
 endif ()
-target_link_libraries(dbms PRIVATE Threads::Threads)
 
 target_include_directories (dbms SYSTEM BEFORE PRIVATE ${DIVIDE_INCLUDE_DIR})
 target_include_directories (dbms SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})

dbms/programs/benchmark/Benchmark.cpp

@@ -32,6 +32,7 @@
 #include <Client/Connection.h>
 #include <Common/InterruptListener.h>
 #include <Common/Config/configReadClient.h>
+#include <Common/TerminalSize.h>
 #include <Common/StudentTTest.h>
 
 
@@ -504,7 +505,7 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv)
     {
         using boost::program_options::value;
 
-        boost::program_options::options_description desc("Allowed options");
+        boost::program_options::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth());
         desc.add_options()
             ("help",                                                            "produce help message")
             ("concurrency,c", value<unsigned>()->default_value(1),              "number of parallel queries")

dbms/programs/client/Client.cpp

@@ -67,6 +67,7 @@
 #include <Common/Config/configReadClient.h>
 #include <Storages/ColumnsDescription.h>
 #include <common/argsToConfig.h>
+#include <Common/TerminalSize.h>
 
 #if USE_READLINE
 #include "Suggest.h"
@@ -130,7 +131,7 @@ private:
     bool print_time_to_stderr = false;   /// Output execution time to stderr in batch mode.
     bool stdin_is_not_tty = false;       /// stdin is not a terminal.
 
-    winsize terminal_size {};            /// Terminal size is needed to render progress bar.
+    uint16_t terminal_width = 0;         /// Terminal width is needed to render progress bar.
 
     std::unique_ptr<Connection> connection;    /// Connection to DB.
     String query_id;                     /// Current query_id.
@@ -671,7 +672,7 @@ private:
         String text;
 
         if (config().has("query"))
-            text = config().getString("query");
+            text = config().getRawString("query");  /// Poco configuration should not process substitutions in form of ${...} inside query.
         else
         {
             /// If 'query' parameter is not set, read a query from stdin.
@@ -1465,7 +1466,7 @@ private:
 
                 if (show_progress_bar)
                 {
-                    ssize_t width_of_progress_bar = static_cast<ssize_t>(terminal_size.ws_col) - written_progress_chars - strlen(" 99%");
+                    ssize_t width_of_progress_bar = static_cast<ssize_t>(terminal_width) - written_progress_chars - strlen(" 99%");
                     if (width_of_progress_bar > 0)
                     {
                         std::string bar = UnicodeBar::render(UnicodeBar::getWidth(progress.read_rows, 0, total_rows_corrected, width_of_progress_bar));
@@ -1642,22 +1643,13 @@ public:
 
         stdin_is_not_tty = !isatty(STDIN_FILENO);
 
+        if (!stdin_is_not_tty)
+            terminal_width = getTerminalWidth();
+
         namespace po = boost::program_options;
 
-        unsigned line_length = po::options_description::m_default_line_length;
-        unsigned min_description_length = line_length / 2;
-        if (!stdin_is_not_tty)
-        {
-            if (ioctl(STDIN_FILENO, TIOCGWINSZ, &terminal_size))
-                throwFromErrno("Cannot obtain terminal window size (ioctl TIOCGWINSZ)", ErrorCodes::SYSTEM_ERROR);
-            line_length = std::max(
-                static_cast<unsigned>(strlen("--http_native_compression_disable_checksumming_on_decompress ")),
-                static_cast<unsigned>(terminal_size.ws_col));
-            min_description_length = std::min(min_description_length, line_length - 2);
-        }
-
         /// Main commandline options related to client functionality and all parameters from Settings.
-        po::options_description main_description("Main options", line_length, min_description_length);
+        po::options_description main_description = createOptionsDescription("Main options", terminal_width);
         main_description.add_options()
             ("help", "produce help message")
             ("config-file,C", po::value<std::string>(), "config-file path")
@@ -1672,7 +1664,7 @@ public:
               * the "\n" is used to distinguish this case because there is hardly a chance an user would use "\n"
               * as the password.
               */
-            ("password", po::value<std::string>()->implicit_value("\n"), "password")
+            ("password", po::value<std::string>()->implicit_value("\n", ""), "password")
             ("ask-password", "ask-password")
             ("query_id", po::value<std::string>(), "query_id")
             ("query,q", po::value<std::string>(), "query")
@@ -1703,7 +1695,7 @@ public:
         context.getSettingsRef().addProgramOptions(main_description);
 
         /// Commandline options related to external tables.
-        po::options_description external_description("External tables options");
+        po::options_description external_description = createOptionsDescription("External tables options", terminal_width);
         external_description.add_options()
             ("file", po::value<std::string>(), "data file or - for stdin")
             ("name", po::value<std::string>()->default_value("_data"), "name of the table")

dbms/programs/compressor/Compressor.cpp

@@ -12,8 +12,9 @@
 #include <IO/copyData.h>
 #include <Parsers/parseQuery.h>
 #include <Parsers/ExpressionElementParsers.h>
-
 #include <Compression/CompressionFactory.h>
+#include <Common/TerminalSize.h>
+
 
 namespace DB
 {
@@ -59,7 +60,7 @@ void checkAndWriteHeader(DB::ReadBuffer & in, DB::WriteBuffer & out)
 
 int mainEntryClickHouseCompressor(int argc, char ** argv)
 {
-    boost::program_options::options_description desc("Allowed options");
+    boost::program_options::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth());
     desc.add_options()
         ("help,h", "produce help message")
         ("decompress,d", "decompress")

dbms/programs/format/Format.cpp

@@ -6,13 +6,13 @@
 #include <Parsers/ParserQuery.h>
 #include <Parsers/parseQuery.h>
 #include <Parsers/formatAST.h>
-
+#include <Common/TerminalSize.h>
 
 int mainEntryClickHouseFormat(int argc, char ** argv)
 {
     using namespace DB;
 
-    boost::program_options::options_description desc("Allowed options");
+    boost::program_options::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth());
     desc.add_options()
         ("help,h", "produce help message")
         ("hilite", "add syntax highlight with ANSI terminal escape sequences")

dbms/programs/local/LocalServer.cpp

@@ -35,6 +35,7 @@
 #include <boost/program_options/options_description.hpp>
 #include <boost/program_options.hpp>
 #include <common/argsToConfig.h>
+#include <Common/TerminalSize.h>
 
 
 namespace DB
@@ -267,7 +268,7 @@ void LocalServer::attachSystemTables()
 void LocalServer::processQueries()
 {
     String initial_create_query = getInitialCreateTableQuery();
-    String queries_str = initial_create_query + config().getString("query");
+    String queries_str = initial_create_query + config().getRawString("query");
 
     std::vector<String> queries;
     auto parse_res = splitMultipartQuery(queries_str, queries);
@@ -409,17 +410,7 @@ void LocalServer::init(int argc, char ** argv)
     /// Don't parse options with Poco library, we prefer neat boost::program_options
     stopOptionsProcessing();
 
-    unsigned line_length = po::options_description::m_default_line_length;
+    po::options_description description = createOptionsDescription("Main options", getTerminalWidth());
-    unsigned min_description_length = line_length / 2;
-    if (isatty(STDIN_FILENO))
-    {
-        winsize terminal_size{};
-        ioctl(0, TIOCGWINSZ, &terminal_size);
-        line_length = std::max(3U, static_cast<unsigned>(terminal_size.ws_col));
-        min_description_length = std::min(min_description_length, line_length - 2);
-    }
-
-    po::options_description description("Main options", line_length, min_description_length);
     description.add_options()
         ("help", "produce help message")
         ("config-file,c", po::value<std::string>(), "config-file path")

dbms/programs/obfuscator/Obfuscator.cpp

@@ -37,6 +37,7 @@
 #include <boost/program_options.hpp>
 #include <boost/algorithm/string.hpp>
 #include <boost/container/flat_map.hpp>
+#include <Common/TerminalSize.h>
 
 
 static const char * documantation = R"(
@@ -949,7 +950,7 @@ try
     using namespace DB;
     namespace po = boost::program_options;
 
-    po::options_description description("Options");
+    po::options_description description = createOptionsDescription("Options", getTerminalWidth());
     description.add_options()
         ("help", "produce help message")
         ("structure,S", po::value<std::string>(), "structure of the initial table (list of column and type names)")

dbms/programs/performance-test/PerformanceTestSuite.cpp

@@ -28,6 +28,7 @@
 #include <Core/Settings.h>
 #include <Common/Exception.h>
 #include <Common/InterruptListener.h>
+#include <Common/TerminalSize.h>
 
 #include "TestStopConditions.h"
 #include "TestStats.h"
@@ -324,7 +325,7 @@ try
     using po::value;
     using Strings = DB::Strings;
 
-    po::options_description desc("Allowed options");
+    po::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth());
     desc.add_options()
         ("help", "produce help message")
         ("lite", "use lite version of output")

dbms/programs/server/Server.cpp

@@ -520,7 +520,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
 
     /// Init trace collector only after trace_log system table was created
     /// Disable it if we collect test coverage information, because it will work extremely slow.
-#if USE_INTERNAL_UNWIND_LIBRARY && !WITH_COVERAGE
+#if USE_UNWIND && !WITH_COVERAGE
     /// QueryProfiler cannot work reliably with any other libunwind or without PHDR cache.
     if (hasPHDRCache())
         global_context->initializeTraceCollector();

dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h

@@ -90,7 +90,7 @@ public:
         auto & set = this->data(place).value;
         size_t size = set.size();
         writeVarUInt(size, buf);
-        for (auto & elem : set)
+        for (const auto & elem : set)
             writeIntBinary(elem, buf);
     }
 

dbms/src/Client/ConnectionPool.h

@@ -74,7 +74,7 @@ public:
     {
         Entry entry;
         if (settings)
-            entry = Base::get(settings->queue_max_wait_ms.totalMilliseconds());
+            entry = Base::get(settings->connection_pool_max_wait_ms.totalMilliseconds());
         else
             entry = Base::get(-1);
 

dbms/src/Columns/ColumnLowCardinality.cpp

@@ -34,7 +34,7 @@ namespace
         auto & data = res_col->getData();
 
         data.resize(hash_map.size());
-        for (auto val : hash_map)
+        for (const auto & val : hash_map)
             data[val.getSecond()] = val.getFirst();
 
         for (auto & ind : index)

dbms/src/Columns/ColumnTuple.cpp

@@ -81,6 +81,16 @@ MutableColumnPtr ColumnTuple::cloneEmpty() const
     return ColumnTuple::create(std::move(new_columns));
 }
 
+MutableColumnPtr ColumnTuple::cloneResized(size_t new_size) const
+{
+    const size_t tuple_size = columns.size();
+    MutableColumns new_columns(tuple_size);
+    for (size_t i = 0; i < tuple_size; ++i)
+        new_columns[i] = columns[i]->cloneResized(new_size);
+
+    return ColumnTuple::create(std::move(new_columns));
+}
+
 Field ColumnTuple::operator[](size_t n) const
 {
     return Tuple{ext::map<TupleBackend>(columns, [n] (const auto & column) { return (*column)[n]; })};

dbms/src/Columns/ColumnTuple.h

@@ -42,6 +42,7 @@ public:
     const char * getFamilyName() const override { return "Tuple"; }
 
     MutableColumnPtr cloneEmpty() const override;
+    MutableColumnPtr cloneResized(size_t size) const override;
 
     size_t size() const override
     {

dbms/src/Common/Config/CMakeLists.txt

@@ -4,5 +4,5 @@ add_headers_and_sources(clickhouse_common_config .)
 
 add_library(clickhouse_common_config ${clickhouse_common_config_headers} ${clickhouse_common_config_sources})
 
-target_link_libraries(clickhouse_common_config PUBLIC common PRIVATE clickhouse_common_zookeeper string_utils PUBLIC ${Poco_XML_LIBRARY} ${Poco_Util_LIBRARY} Threads::Threads)
+target_link_libraries(clickhouse_common_config PUBLIC common PRIVATE clickhouse_common_zookeeper string_utils PUBLIC ${Poco_XML_LIBRARY} ${Poco_Util_LIBRARY})
 target_include_directories(clickhouse_common_config PUBLIC ${DBMS_INCLUDE_DIR})

dbms/src/Common/HashTable/FixedHashMap.h

@@ -11,8 +11,8 @@ struct FixedHashMapCell
     using State = TState;
 
     using value_type = PairNoInit<Key, Mapped>;
-    bool full;
     Mapped mapped;
+    bool full;
 
     FixedHashMapCell() {}
     FixedHashMapCell(const Key &, const State &) : full(true) {}

dbms/src/Common/HashTable/HashMap.h

@@ -128,14 +128,12 @@ struct HashMapCellWithSavedHash : public HashMapCell<Key, TMapped, Hash, TState>
 };
 
 
-template
+template <
-<
     typename Key,
     typename Cell,
     typename Hash = DefaultHash<Key>,
     typename Grower = HashTableGrower<>,
-    typename Allocator = HashTableAllocator
+    typename Allocator = HashTableAllocator>
->
 class HashMapTable : public HashTable<Key, Cell, Hash, Grower, Allocator>
 {
 public:
@@ -173,23 +171,19 @@ public:
 };
 
 
-template
+template <
-<
     typename Key,
     typename Mapped,
     typename Hash = DefaultHash<Key>,
     typename Grower = HashTableGrower<>,
-    typename Allocator = HashTableAllocator
+    typename Allocator = HashTableAllocator>
->
 using HashMap = HashMapTable<Key, HashMapCell<Key, Mapped, Hash>, Hash, Grower, Allocator>;
 
 
-template
+template <
-<
     typename Key,
     typename Mapped,
     typename Hash = DefaultHash<Key>,
     typename Grower = HashTableGrower<>,
-    typename Allocator = HashTableAllocator
+    typename Allocator = HashTableAllocator>
->
 using HashMapWithSavedHash = HashMapTable<Key, HashMapCellWithSavedHash<Key, Mapped, Hash>, Hash, Grower, Allocator>;

dbms/src/Common/HashTable/HashTable.h

@@ -95,7 +95,6 @@ struct HashTableCell
 
     /// Create a cell with the given key / key and value.
     HashTableCell(const Key & key_, const State &) : key(key_) {}
-/// HashTableCell(const value_type & value_, const State & state) : key(value_) {}
 
     /// Get what the value_type of the container will be.
     value_type & getValueMutable() { return key; }

dbms/src/Common/QueryProfiler.cpp

@@ -100,7 +100,7 @@ QueryProfilerBase<ProfilerImpl>::QueryProfilerBase(const Int32 thread_id, const
     : log(&Logger::get("QueryProfiler"))
     , pause_signal(pause_signal_)
 {
-#if USE_INTERNAL_UNWIND_LIBRARY
+#if USE_UNWIND
     /// Sanity check.
     if (!hasPHDRCache())
         throw Exception("QueryProfiler cannot be used without PHDR cache, that is not available for TSan build", ErrorCodes::NOT_IMPLEMENTED);
@@ -173,7 +173,7 @@ QueryProfilerBase<ProfilerImpl>::~QueryProfilerBase()
 template <typename ProfilerImpl>
 void QueryProfilerBase<ProfilerImpl>::tryCleanup()
 {
-#if USE_INTERNAL_UNWIND_LIBRARY
+#if USE_UNWIND
     if (timer_id != nullptr && timer_delete(timer_id))
         LOG_ERROR(log, "Failed to delete query profiler timer " + errnoToString(ErrorCodes::CANNOT_DELETE_TIMER));
 

dbms/src/Common/QueryProfiler.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <Core/Types.h>
+#include <Common/config.h>
 #include <common/config_common.h>
 #include <signal.h>
 #include <time.h>
@@ -43,7 +44,7 @@ private:
 
     Poco::Logger * log;
 
-#if USE_INTERNAL_UNWIND_LIBRARY
+#if USE_UNWIND
     /// Timer id from timer_create(2)
     timer_t timer_id = nullptr;
 #endif

dbms/src/Common/StackTrace.cpp

@@ -1,15 +1,20 @@
-#include <common/SimpleCache.h>
-#include <common/demangle.h>
-#include <Common/config.h>
 #include <Common/StackTrace.h>
-#include <Common/SymbolIndex.h>
+
 #include <Common/Dwarf.h>
 #include <Common/Elf.h>
-#include <sstream>
+#include <Common/SymbolIndex.h>
-#include <filesystem>
+#include <Common/config.h>
-#include <unordered_map>
+#include <common/SimpleCache.h>
-#include <cstring>
+#include <common/demangle.h>
 
+#include <cstring>
+#include <filesystem>
+#include <sstream>
+#include <unordered_map>
+
+#if USE_UNWIND
+#   include <libunwind.h>
+#endif
 
 std::string signalToErrorMessage(int sig, const siginfo_t & info, const ucontext_t & context)
 {
@@ -215,12 +220,6 @@ StackTrace::StackTrace(NoCapture)
 {
 }
 
-
-#if USE_UNWIND
-extern "C" int unw_backtrace(void **, int);
-#endif
-
-
 void StackTrace::tryCapture()
 {
     size = 0;

dbms/src/Common/TerminalSize.cpp

@@ -0,0 +1,37 @@
+#include <unistd.h>
+#include <sys/ioctl.h>
+#include <Common/Exception.h>
+#include <Common/TerminalSize.h>
+#include <boost/program_options.hpp>
+
+
+namespace DB::ErrorCodes
+{
+    extern const int SYSTEM_ERROR;
+}
+
+uint16_t getTerminalWidth()
+{
+    if (isatty(STDIN_FILENO))
+    {
+        winsize terminal_size {};
+
+        if (ioctl(STDIN_FILENO, TIOCGWINSZ, &terminal_size))
+            DB::throwFromErrno("Cannot obtain terminal window size (ioctl TIOCGWINSZ)", DB::ErrorCodes::SYSTEM_ERROR);
+
+        return terminal_size.ws_col;
+    }
+    return 0;
+}
+
+po::options_description createOptionsDescription(const std::string & caption, uint16_t terminal_width)
+{
+    unsigned line_length = po::options_description::m_default_line_length;
+    unsigned min_description_length = line_length / 2;
+    std::string longest_option_desc = "--http_native_compression_disable_checksumming_on_decompress";
+
+    line_length = std::max(static_cast<uint16_t>(longest_option_desc.size()), terminal_width);
+    min_description_length = std::min(min_description_length, line_length - 2);
+
+    return po::options_description(caption, line_length, min_description_length);
+}

dbms/src/Common/TerminalSize.h

@@ -0,0 +1,16 @@
+#pragma once
+
+#include <string>
+#include <boost/program_options.hpp>
+
+
+namespace po = boost::program_options;
+
+
+uint16_t getTerminalWidth();
+
+/** Creates po::options_description with name and an appropriate size for option displaying
+ *  when program is called with option --help
+ * */
+po::options_description createOptionsDescription(const std::string &caption, unsigned short terminal_width);
+

dbms/src/Common/ZooKeeper/CMakeLists.txt

@@ -4,7 +4,7 @@ add_headers_and_sources(clickhouse_common_zookeeper .)
 
 add_library(clickhouse_common_zookeeper ${clickhouse_common_zookeeper_headers} ${clickhouse_common_zookeeper_sources})
 
-target_link_libraries (clickhouse_common_zookeeper PUBLIC clickhouse_common_io common PRIVATE string_utils PUBLIC ${Poco_Util_LIBRARY} Threads::Threads)
+target_link_libraries (clickhouse_common_zookeeper PUBLIC clickhouse_common_io common PRIVATE string_utils PUBLIC ${Poco_Util_LIBRARY})
 target_include_directories(clickhouse_common_zookeeper PUBLIC ${DBMS_INCLUDE_DIR})
 
 if (ENABLE_TESTS)

dbms/src/Core/Settings.h

@@ -61,7 +61,10 @@ struct Settings : public SettingsCollection<Settings>
     M(SettingSeconds, receive_timeout, DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC, "") \
     M(SettingSeconds, send_timeout, DBMS_DEFAULT_SEND_TIMEOUT_SEC, "") \
     M(SettingSeconds, tcp_keep_alive_timeout, 0, "") \
-    M(SettingMilliseconds, queue_max_wait_ms, 5000, "The wait time in the request queue, if the number of concurrent requests exceeds the maximum.") \
+    M(SettingMilliseconds, queue_max_wait_ms, 0, "The wait time in the request queue, if the number of concurrent requests exceeds the maximum.") \
+    M(SettingMilliseconds, connection_pool_max_wait_ms, 0, "The wait time when connection pool is full.") \
+    M(SettingMilliseconds, replace_running_query_max_wait_ms, 5000, "The wait time for running query with the same query_id to finish when setting 'replace_running_query' is active.") \
+    M(SettingMilliseconds, kafka_max_wait_ms, 5000, "The wait time for reading from Kafka before retry.") \
     M(SettingUInt64, poll_interval, DBMS_DEFAULT_POLL_INTERVAL, "Block at the query wait loop on the server for the specified number of seconds.") \
     M(SettingUInt64, idle_connection_timeout, 3600, "Close idle TCP connections after specified number of seconds.") \
     M(SettingUInt64, distributed_connections_pool_size, DBMS_DEFAULT_DISTRIBUTED_CONNECTIONS_POOL_SIZE, "Maximum number of connections with one remote server in the pool.") \
@@ -302,7 +305,7 @@ struct Settings : public SettingsCollection<Settings>
     M(SettingChar, format_csv_delimiter, ',', "The character to be considered as a delimiter in CSV data. If setting with a string, a string has to have a length of 1.") \
     M(SettingBool, format_csv_allow_single_quotes, 1, "If it is set to true, allow strings in single quotes.") \
     M(SettingBool, format_csv_allow_double_quotes, 1, "If it is set to true, allow strings in double quotes.") \
-    M(SettingBool, input_format_csv_unquoted_null_literal_as_null, false, "Consider unquoted NULL literal as \N") \
+    M(SettingBool, input_format_csv_unquoted_null_literal_as_null, false, "Consider unquoted NULL literal as \\N") \
     \
     M(SettingDateTimeInputFormat, date_time_input_format, FormatSettings::DateTimeInputFormat::Basic, "Method to read DateTime from text input formats. Possible values: 'basic' and 'best_effort'.") \
     M(SettingBool, log_profile_events, true, "Log query performance statistics into the query_log and query_thread_log.") \

dbms/src/Core/SettingsCommon.h

@@ -695,7 +695,7 @@ public:
 
 #define IMPLEMENT_SETTINGS_COLLECTION_ADD_MUTABLE_MEMBER_INFO_HELPER_(TYPE, NAME, DEFAULT, DESCRIPTION) \
     add({[](const Derived & d) { return d.NAME.changed; },          \
-         StringRef(#NAME, strlen(#NAME)), StringRef(#DESCRIPTION, strlen(#DESCRIPTION)), true, \
+         StringRef(#NAME, strlen(#NAME)), StringRef(DESCRIPTION, strlen(DESCRIPTION)), true, \
          &Functions::NAME##_getString, &Functions::NAME##_getField, \
          &Functions::NAME##_setString, &Functions::NAME##_setField, \
          &Functions::NAME##_serialize, &Functions::NAME##_deserialize, \
@@ -703,7 +703,7 @@ public:
 
 #define IMPLEMENT_SETTINGS_COLLECTION_ADD_IMMUTABLE_MEMBER_INFO_HELPER_(TYPE, NAME, DEFAULT, DESCRIPTION) \
     add({[](const Derived & d) { return d.NAME.changed; },              \
-        StringRef(#NAME, strlen(#NAME)), StringRef(#DESCRIPTION, strlen(#DESCRIPTION)), false, \
+        StringRef(#NAME, strlen(#NAME)), StringRef(DESCRIPTION, strlen(DESCRIPTION)), false, \
         &Functions::NAME##_getString, &Functions::NAME##_getField, \
         &Functions::NAME##_setString, &Functions::NAME##_setField, \
         &Functions::NAME##_serialize, &Functions::NAME##_deserialize, \

dbms/src/Databases/DatabaseDictionary.cpp

@@ -115,26 +115,6 @@ void DatabaseDictionary::removeTable(
     throw Exception("DatabaseDictionary: removeTable() is not supported", ErrorCodes::NOT_IMPLEMENTED);
 }
 
-void DatabaseDictionary::renameTable(
-    const Context &,
-    const String &,
-    IDatabase &,
-    const String &)
-{
-    throw Exception("DatabaseDictionary: renameTable() is not supported", ErrorCodes::NOT_IMPLEMENTED);
-}
-
-void DatabaseDictionary::alterTable(
-    const Context &,
-    const String &,
-    const ColumnsDescription &,
-    const IndicesDescription &,
-    const ConstraintsDescription &,
-    const ASTModifier &)
-{
-    throw Exception("DatabaseDictionary: alterTable() is not supported", ErrorCodes::NOT_IMPLEMENTED);
-}
-
 time_t DatabaseDictionary::getTableMetadataModificationTime(
     const Context &,
     const String &)

dbms/src/Databases/DatabaseDictionary.h

@@ -60,20 +60,6 @@ public:
     void attachTable(const String & table_name, const StoragePtr & table) override;
     StoragePtr detachTable(const String & table_name) override;
 
-    void renameTable(
-        const Context & context,
-        const String & table_name,
-        IDatabase & to_database,
-        const String & to_table_name) override;
-
-    void alterTable(
-        const Context & context,
-        const String & name,
-        const ColumnsDescription & columns,
-        const IndicesDescription & indices,
-        const ConstraintsDescription & constraints,
-        const ASTModifier & engine_modifier) override;
-
     time_t getTableMetadataModificationTime(
         const Context & context,
         const String & table_name) override;

dbms/src/Databases/DatabaseFactory.cpp

@@ -5,6 +5,7 @@
 #include <Parsers/ASTLiteral.h>
 #include <Parsers/formatAST.h>
 #include <Parsers/ASTCreateQuery.h>
+#include <Parsers/ASTFunction.h>
 #include <Common/parseAddress.h>
 #include "config_core.h"
 #if USE_MYSQL

dbms/src/Databases/DatabaseMemory.cpp

@@ -39,26 +39,6 @@ void DatabaseMemory::removeTable(
     detachTable(table_name);
 }
 
-void DatabaseMemory::renameTable(
-    const Context &,
-    const String &,
-    IDatabase &,
-    const String &)
-{
-    throw Exception("DatabaseMemory: renameTable() is not supported", ErrorCodes::NOT_IMPLEMENTED);
-}
-
-void DatabaseMemory::alterTable(
-    const Context &,
-    const String &,
-    const ColumnsDescription &,
-    const IndicesDescription &,
-    const ConstraintsDescription &,
-    const ASTModifier &)
-{
-    throw Exception("DatabaseMemory: alterTable() is not supported", ErrorCodes::NOT_IMPLEMENTED);
-}
-
 time_t DatabaseMemory::getTableMetadataModificationTime(
     const Context &,
     const String &)

dbms/src/Databases/DatabaseMemory.h

@@ -37,20 +37,6 @@ public:
         const Context & context,
         const String & table_name) override;
 
-    void renameTable(
-        const Context & context,
-        const String & table_name,
-        IDatabase & to_database,
-        const String & to_table_name) override;
-
-    void alterTable(
-        const Context & context,
-        const String & name,
-        const ColumnsDescription & columns,
-        const IndicesDescription & indices,
-        const ConstraintsDescription & constraints,
-        const ASTModifier & engine_modifier) override;
-
     time_t getTableMetadataModificationTime(
         const Context & context,
         const String & table_name) override;

dbms/src/Databases/DatabaseMySQL.h

@@ -5,6 +5,8 @@
 
 #include <mysqlxx/Pool.h>
 #include <Databases/DatabasesCommon.h>
+#include <Interpreters/Context.h>
+
 
 namespace DB
 {
@@ -61,21 +63,11 @@ public:
         throw Exception("MySQL database engine does not support attach table.", ErrorCodes::NOT_IMPLEMENTED);
     }
 
-    void renameTable(const Context &, const String &, IDatabase &, const String &) override
-    {
-        throw Exception("MySQL database engine does not support rename table.", ErrorCodes::NOT_IMPLEMENTED);
-    }
-
     void createTable(const Context &, const String &, const StoragePtr &, const ASTPtr &) override
     {
         throw Exception("MySQL database engine does not support create table.", ErrorCodes::NOT_IMPLEMENTED);
     }
 
-    void alterTable(const Context &, const String &, const ColumnsDescription &, const IndicesDescription &, const ConstraintsDescription &, const ASTModifier &) override
-    {
-        throw Exception("MySQL database engine does not support alter table.", ErrorCodes::NOT_IMPLEMENTED);
-    }
-
 private:
     struct MySQLStorageInfo
     {

dbms/src/Databases/DatabaseOrdinary.cpp

@@ -135,7 +135,25 @@ void DatabaseOrdinary::loadTables(
         if (endsWith(dir_it.name(), ".sql.bak"))
             continue;
 
-        /// There are files .sql.tmp - delete.
+        // There are files that we tried to delete previously
+        static const char * tmp_drop_ext = ".sql.tmp_drop";
+        if (endsWith(dir_it.name(), tmp_drop_ext))
+        {
+            const std::string table_name = dir_it.name().substr(0, dir_it.name().size() - strlen(tmp_drop_ext));
+            if (Poco::File(data_path + '/' + table_name).exists())
+            {
+                Poco::File(dir_it->path()).renameTo(table_name + ".sql");
+                LOG_WARNING(log, "Table " << backQuote(table_name) << " was not dropped previously");
+            }
+            else
+            {
+                LOG_INFO(log, "Removing file " << dir_it->path());
+                Poco::File(dir_it->path()).remove();
+            }
+            continue;
+        }
+
+        /// There are files .sql.tmp - delete
         if (endsWith(dir_it.name(), ".sql.tmp"))
         {
             LOG_INFO(log, "Removing file " << dir_it->path());
@@ -302,6 +320,15 @@ void DatabaseOrdinary::removeTable(
     }
     catch (...)
     {
+        try
+        {
+            Poco::File(table_metadata_path + ".tmp_drop").remove();
+            return;
+        }
+        catch (...)
+        {
+            LOG_WARNING(log, getCurrentExceptionMessage(__PRETTY_FUNCTION__));
+        }
         attachTable(table_name, res);
         throw;
     }
@@ -355,7 +382,8 @@ void DatabaseOrdinary::renameTable(
     const Context & context,
     const String & table_name,
     IDatabase & to_database,
-    const String & to_table_name)
+    const String & to_table_name,
+    TableStructureWriteLockHolder & lock)
 {
     DatabaseOrdinary * to_database_concrete = typeid_cast<DatabaseOrdinary *>(&to_database);
 
@@ -372,7 +400,7 @@ void DatabaseOrdinary::renameTable(
     {
         table->rename(context.getPath() + "/data/" + escapeForFileName(to_database_concrete->name) + "/",
             to_database_concrete->name,
-            to_table_name);
+            to_table_name, lock);
     }
     catch (const Exception &)
     {

dbms/src/Databases/DatabaseOrdinary.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <Databases/DatabasesCommon.h>
+#include <Common/ThreadPool.h>
 
 
 namespace DB
@@ -35,7 +36,8 @@ public:
         const Context & context,
         const String & table_name,
         IDatabase & to_database,
-        const String & to_table_name) override;
+        const String & to_table_name,
+        TableStructureWriteLockHolder &) override;
 
     void alterTable(
         const Context & context,

dbms/src/Databases/DatabasesCommon.h

@@ -4,6 +4,7 @@
 #include <Parsers/IAST.h>
 #include <Storages/IStorage_fwd.h>
 #include <Databases/IDatabase.h>
+#include <mutex>
 
 
 /// General functionality for several different database engines.

dbms/src/Databases/IDatabase.h

@@ -1,16 +1,9 @@
 #pragma once
 
-#include <Core/NamesAndTypes.h>
 #include <Core/Types.h>
-#include <Interpreters/Context.h>
 #include <Parsers/IAST_fwd.h>
-#include <Storages/ColumnsDescription.h>
-#include <Storages/IndicesDescription.h>
-#include <Storages/ConstraintsDescription.h>
 #include <Storages/IStorage_fwd.h>
-#include <Poco/File.h>
+#include <Common/Exception.h>
-#include <Common/ThreadPool.h>
-#include <Common/escapeForFileName.h>
 
 #include <ctime>
 #include <functional>
@@ -21,8 +14,16 @@ namespace DB
 {
 
 class Context;
-
 struct Settings;
+struct ConstraintsDescription;
+class ColumnsDescription;
+struct IndicesDescription;
+struct TableStructureWriteLockHolder;
+
+namespace ErrorCodes
+{
+    extern const int NOT_IMPLEMENTED;
+}
 
 
 /** Allows to iterate over tables.
@@ -102,22 +103,29 @@ public:
 
     /// Rename the table and possibly move the table to another database.
     virtual void renameTable(
-        const Context & context,
+        const Context & /*context*/,
-        const String & name,
+        const String & /*name*/,
-        IDatabase & to_database,
+        IDatabase & /*to_database*/,
-        const String & to_name) = 0;
+        const String & /*to_name*/,
+        TableStructureWriteLockHolder &)
+    {
+        throw Exception(getEngineName() + ": renameTable() is not supported", ErrorCodes::NOT_IMPLEMENTED);
+    }
 
     using ASTModifier = std::function<void(IAST &)>;
 
     /// Change the table structure in metadata.
     /// You must call under the TableStructureLock of the corresponding table . If engine_modifier is empty, then engine does not change.
     virtual void alterTable(
-        const Context & context,
+        const Context & /*context*/,
-        const String & name,
+        const String & /*name*/,
-        const ColumnsDescription & columns,
+        const ColumnsDescription & /*columns*/,
-        const IndicesDescription & indices,
+        const IndicesDescription & /*indices*/,
-        const ConstraintsDescription & constraints,
+        const ConstraintsDescription & /*constraints*/,
-        const ASTModifier & engine_modifier) = 0;
+        const ASTModifier & /*engine_modifier*/)
+    {
+        throw Exception(getEngineName() + ": renameTable() is not supported", ErrorCodes::NOT_IMPLEMENTED);
+    }
 
     /// Returns time of table's metadata change, 0 if there is no corresponding metadata file.
     virtual time_t getTableMetadataModificationTime(

dbms/src/Dictionaries/CMakeLists.txt

@@ -15,7 +15,7 @@ list(REMOVE_ITEM clickhouse_dictionaries_sources DictionaryFactory.cpp Dictionar
 list(REMOVE_ITEM clickhouse_dictionaries_headers DictionaryFactory.h DictionarySourceFactory.h DictionaryStructure.h)
 
 add_library(clickhouse_dictionaries ${clickhouse_dictionaries_sources})
-target_link_libraries(clickhouse_dictionaries PRIVATE dbms clickhouse_common_io ${BTRIE_LIBRARIES} PUBLIC Threads::Threads)
+target_link_libraries(clickhouse_dictionaries PRIVATE dbms clickhouse_common_io ${BTRIE_LIBRARIES})
 
 if(Poco_SQL_FOUND AND NOT USE_INTERNAL_POCO_LIBRARY)
     target_include_directories(clickhouse_dictionaries SYSTEM PRIVATE ${Poco_SQL_INCLUDE_DIR})

dbms/src/Interpreters/Aggregator.h

@@ -196,8 +196,6 @@ struct AggregationMethodString
     using Data = TData;
     using Key = typename Data::key_type;
     using Mapped = typename Data::mapped_type;
-    using iterator = typename Data::iterator;
-    using const_iterator = typename Data::const_iterator;
 
     Data data;
 
@@ -224,8 +222,6 @@ struct AggregationMethodFixedString
     using Data = TData;
     using Key = typename Data::key_type;
     using Mapped = typename Data::mapped_type;
-    using iterator = typename Data::iterator;
-    using const_iterator = typename Data::const_iterator;
 
     Data data;
 
@@ -254,8 +250,6 @@ struct AggregationMethodSingleLowCardinalityColumn : public SingleColumnMethod
     using Data = typename Base::Data;
     using Key = typename Base::Key;
     using Mapped = typename Base::Mapped;
-    using iterator = typename Base::iterator;
-    using const_iterator = typename Base::const_iterator;
 
     using Base::data;
 
@@ -365,8 +359,6 @@ struct AggregationMethodSerialized
     using Data = TData;
     using Key = typename Data::key_type;
     using Mapped = typename Data::mapped_type;
-    using iterator = typename Data::iterator;
-    using const_iterator = typename Data::const_iterator;
 
     Data data;
 
@@ -460,8 +452,8 @@ struct AggregatedDataVariants : private boost::noncopyable
     std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithKeys256TwoLevel, true>>     nullable_keys256_two_level;
 
     /// Support for low cardinality.
-    std::unique_ptr<AggregationMethodSingleLowCardinalityColumn<AggregationMethodOneNumber<UInt8, AggregatedDataWithNullableUInt8Key>>> low_cardinality_key8;
+    std::unique_ptr<AggregationMethodSingleLowCardinalityColumn<AggregationMethodOneNumber<UInt8, AggregatedDataWithNullableUInt8Key, false>>> low_cardinality_key8;
-    std::unique_ptr<AggregationMethodSingleLowCardinalityColumn<AggregationMethodOneNumber<UInt16, AggregatedDataWithNullableUInt16Key>>> low_cardinality_key16;
+    std::unique_ptr<AggregationMethodSingleLowCardinalityColumn<AggregationMethodOneNumber<UInt16, AggregatedDataWithNullableUInt16Key, false>>> low_cardinality_key16;
     std::unique_ptr<AggregationMethodSingleLowCardinalityColumn<AggregationMethodOneNumber<UInt32, AggregatedDataWithNullableUInt64Key>>> low_cardinality_key32;
     std::unique_ptr<AggregationMethodSingleLowCardinalityColumn<AggregationMethodOneNumber<UInt64, AggregatedDataWithNullableUInt64Key>>> low_cardinality_key64;
     std::unique_ptr<AggregationMethodSingleLowCardinalityColumn<AggregationMethodString<AggregatedDataWithNullableStringKey>>> low_cardinality_key_string;

dbms/src/Interpreters/ClusterProxy/executeQuery.cpp

@@ -18,6 +18,8 @@ Context removeUserRestrictionsFromSettings(const Context & context, const Settin
 {
     Settings new_settings = settings;
     new_settings.queue_max_wait_ms = Cluster::saturate(new_settings.queue_max_wait_ms, settings.max_execution_time);
+    new_settings.connection_pool_max_wait_ms = Cluster::saturate(new_settings.connection_pool_max_wait_ms, settings.max_execution_time);
+    new_settings.replace_running_query_max_wait_ms = Cluster::saturate(new_settings.replace_running_query_max_wait_ms, settings.max_execution_time);
 
     /// Does not matter on remote servers, because queries are sent under different user.
     new_settings.max_concurrent_queries_for_user = 0;

dbms/src/Interpreters/ExpressionActions.cpp

@@ -291,8 +291,8 @@ void ExpressionAction::prepare(Block & sample_block, const Settings & settings,
 
                 bool make_nullable = is_null_used_as_default && right_or_full_join;
 
-                if (make_nullable && !col.type->isNullable())
+                if (make_nullable && col.type->canBeInsideNullable())
-                    col.type = std::make_shared<DataTypeNullable>(col.type);
+                    col.type = makeNullable(col.type);
             }
 
             for (const auto & col : columns_added_by_join)
@@ -316,8 +316,8 @@ void ExpressionAction::prepare(Block & sample_block, const Settings & settings,
                     }
                 }
 
-                if (make_nullable && !res_type->isNullable())
+                if (make_nullable && res_type->canBeInsideNullable())
-                    res_type = std::make_shared<DataTypeNullable>(res_type);
+                    res_type = makeNullable(res_type);
 
                 sample_block.insert(ColumnWithTypeAndName(nullptr, res_type, col.name));
             }

dbms/src/Interpreters/InterpreterDropQuery.cpp

@@ -80,7 +80,7 @@ BlockIO InterpreterDropQuery::executeToTable(String & database_name_, String & t
             /// If table was already dropped by anyone, an exception will be thrown
             auto table_lock = database_and_table.second->lockExclusively(context.getCurrentQueryId());
             /// Drop table data, don't touch metadata
-            database_and_table.second->truncate(query_ptr, context);
+            database_and_table.second->truncate(query_ptr, context, table_lock);
         }
         else if (kind == ASTDropQuery::Kind::Drop)
         {
@@ -90,11 +90,32 @@ BlockIO InterpreterDropQuery::executeToTable(String & database_name_, String & t
             /// If table was already dropped by anyone, an exception will be thrown
 
             auto table_lock = database_and_table.second->lockExclusively(context.getCurrentQueryId());
-            /// Delete table metadata and table itself from memory
 
-            database_and_table.first->removeTable(context, database_and_table.second->getTableName());
+            const std::string metadata_file_without_extension =
+                database_and_table.first->getMetadataPath()
+                + escapeForFileName(database_and_table.second->getTableName());
+
+            const auto prev_metadata_name = metadata_file_without_extension + ".sql";
+            const auto drop_metadata_name = metadata_file_without_extension + ".sql.tmp_drop";
+
+            /// Try to rename metadata file and delete the data
+            try
+            {
+                /// There some kind of tables that have no metadata - ignore renaming
+                if (Poco::File(prev_metadata_name).exists())
+                    Poco::File(prev_metadata_name).renameTo(drop_metadata_name);
                 /// Delete table data
-            database_and_table.second->drop();
+                database_and_table.second->drop(table_lock);
+            }
+            catch (...)
+            {
+                if (Poco::File(drop_metadata_name).exists())
+                    Poco::File(drop_metadata_name).renameTo(prev_metadata_name);
+                throw;
+            }
+
+            /// Delete table metadata and table itself from memory
+            database_and_table.first->removeTable(context, database_and_table.second->getTableName());
             database_and_table.second->is_dropped = true;
 
             String database_data_path = database_and_table.first->getDataPath();
@@ -128,7 +149,7 @@ BlockIO InterpreterDropQuery::executeToTemporaryTable(String & table_name, ASTDr
                 /// If table was already dropped by anyone, an exception will be thrown
                 auto table_lock = table->lockExclusively(context.getCurrentQueryId());
                 /// Drop table data, don't touch metadata
-                table->truncate(query_ptr, context);
+                table->truncate(query_ptr, context, table_lock);
             }
             else if (kind == ASTDropQuery::Kind::Drop)
             {
@@ -137,7 +158,7 @@ BlockIO InterpreterDropQuery::executeToTemporaryTable(String & table_name, ASTDr
                 /// If table was already dropped by anyone, an exception will be thrown
                 auto table_lock = table->lockExclusively(context.getCurrentQueryId());
                 /// Delete table data
-                table->drop();
+                table->drop(table_lock);
                 table->is_dropped = true;
             }
         }

dbms/src/Interpreters/InterpreterRenameQuery.cpp

@@ -26,6 +26,8 @@ struct RenameDescription
         to_table_name(elem.to.table)
     {}
 
+    TableStructureWriteLockHolder from_table_lock;
+
     String from_database_name;
     String from_table_name;
 
@@ -75,7 +77,7 @@ BlockIO InterpreterRenameQuery::execute()
         }
     };
 
-    std::set<UniqueTableName> unique_tables_from;
+    std::map<UniqueTableName, TableStructureWriteLockHolder> tables_from_locks;
 
     /// Don't allow to drop tables (that we are renaming); don't allow to create tables in places where tables will be renamed.
     std::map<UniqueTableName, std::unique_ptr<DDLGuard>> table_guards;
@@ -87,7 +89,11 @@ BlockIO InterpreterRenameQuery::execute()
         UniqueTableName from(descriptions.back().from_database_name, descriptions.back().from_table_name);
         UniqueTableName to(descriptions.back().to_database_name, descriptions.back().to_table_name);
 
-        unique_tables_from.emplace(from);
+        if (!tables_from_locks.count(from))
+            if (auto table = context.tryGetTable(from.database_name, from.table_name))
+                tables_from_locks.emplace(from, table->lockExclusively(context.getCurrentQueryId()));
+
+        descriptions.back().from_table_lock = tables_from_locks[from];
 
         if (!table_guards.count(from))
             table_guards.emplace(from, context.getDDLGuard(from.database_name, from.table_name));
@@ -96,13 +102,6 @@ BlockIO InterpreterRenameQuery::execute()
             table_guards.emplace(to, context.getDDLGuard(to.database_name, to.table_name));
     }
 
-    std::vector<TableStructureWriteLockHolder> locks;
-    locks.reserve(unique_tables_from.size());
-
-    for (const auto & names : unique_tables_from)
-        if (auto table = context.tryGetTable(names.database_name, names.table_name))
-            locks.emplace_back(table->lockExclusively(context.getCurrentQueryId()));
-
     /** All tables are locked. If there are more than one rename in chain,
       *  we need to hold global lock while doing all renames. Order matters to avoid deadlocks.
       * It provides atomicity of all RENAME chain as a whole, from the point of view of DBMS client,
@@ -114,12 +113,12 @@ BlockIO InterpreterRenameQuery::execute()
     if (descriptions.size() > 1)
         lock = context.getLock();
 
-    for (const auto & elem : descriptions)
+    for (auto & elem : descriptions)
     {
         context.assertTableDoesntExist(elem.to_database_name, elem.to_table_name);
 
         context.getDatabase(elem.from_database_name)->renameTable(
-            context, elem.from_table_name, *context.getDatabase(elem.to_database_name), elem.to_table_name);
+            context, elem.from_table_name, *context.getDatabase(elem.to_database_name), elem.to_table_name, elem.from_table_lock);
     }
 
     return {};

dbms/src/Interpreters/Join.cpp

@@ -50,7 +50,7 @@ static std::unordered_map<String, DataTypePtr> requiredRightKeys(const Names & k
 
 static void convertColumnToNullable(ColumnWithTypeAndName & column)
 {
-    if (column.type->isNullable())
+    if (column.type->isNullable() || !column.type->canBeInsideNullable())
         return;
 
     column.type = makeNullable(column.type);
@@ -71,7 +71,7 @@ static ColumnWithTypeAndName correctNullability(ColumnWithTypeAndName && column,
     if (nullable)
     {
         convertColumnToNullable(column);
-        if (negative_null_map.size())
+        if (column.type->isNullable() && negative_null_map.size())
         {
             MutableColumnPtr mutable_column = (*std::move(column.column)).mutate();
             assert_cast<ColumnNullable &>(*mutable_column).applyNegatedNullMap(negative_null_map);

dbms/src/Interpreters/ProcessList.cpp

@@ -87,10 +87,10 @@ ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * as
     {
         std::unique_lock lock(mutex);
 
-        const auto max_wait_ms = settings.queue_max_wait_ms.totalMilliseconds();
+        const auto queue_max_wait_ms = settings.queue_max_wait_ms.totalMilliseconds();
         if (!is_unlimited_query && max_size && processes.size() >= max_size)
         {
-            if (!max_wait_ms || !have_space.wait_for(lock, std::chrono::milliseconds(max_wait_ms), [&]{ return processes.size() < max_size; }))
+            if (!queue_max_wait_ms || !have_space.wait_for(lock, std::chrono::milliseconds(queue_max_wait_ms), [&]{ return processes.size() < max_size; }))
                 throw Exception("Too many simultaneous queries. Maximum: " + toString(max_size), ErrorCodes::TOO_MANY_SIMULTANEOUS_QUERIES);
         }
 
@@ -127,7 +127,9 @@ ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * as
                     /// Ask queries to cancel. They will check this flag.
                     running_query->second->is_killed.store(true, std::memory_order_relaxed);
 
-                    if (!max_wait_ms || !have_space.wait_for(lock, std::chrono::milliseconds(max_wait_ms), [&]
+                    const auto replace_running_query_max_wait_ms = settings.replace_running_query_max_wait_ms.totalMilliseconds();
+                    if (!replace_running_query_max_wait_ms || !have_space.wait_for(lock, std::chrono::milliseconds(replace_running_query_max_wait_ms),
+                        [&]
                         {
                             running_query = user_process_list->second.queries.find(client_info.current_query_id);
                             if (running_query == user_process_list->second.queries.end())
@@ -135,11 +137,13 @@ ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * as
                             running_query->second->is_killed.store(true, std::memory_order_relaxed);
                             return false;
                         }))
+                    {
                         throw Exception("Query with id = " + client_info.current_query_id + " is already running and can't be stopped",
                             ErrorCodes::QUERY_WITH_SAME_ID_IS_ALREADY_RUNNING);
                     }
                  }
             }
+        }
 
         /// Check other users running query with our query_id
         for (const auto & user_process_list : user_to_queries)

dbms/src/Interpreters/tests/CMakeLists.txt

@@ -57,5 +57,5 @@ target_link_libraries (users PRIVATE dbms clickhouse_common_config stdc++fs)
 
 if (OS_LINUX)
     add_executable (internal_iotop internal_iotop.cpp)
-    target_link_libraries (internal_iotop PRIVATE dbms Threads::Threads)
+    target_link_libraries (internal_iotop PRIVATE dbms)
 endif ()

dbms/src/Storages/IStorage.cpp

@@ -22,6 +22,7 @@ namespace ErrorCodes
     extern const int TYPE_MISMATCH;
     extern const int SETTINGS_ARE_NOT_SUPPORTED;
     extern const int UNKNOWN_SETTING;
+    extern const int TABLE_IS_DROPPED;
 }
 
 IStorage::IStorage(ColumnsDescription virtuals_) : virtuals(std::move(virtuals_))

dbms/src/Storages/IStorage.h

@@ -9,11 +9,13 @@
 #include <Storages/SelectQueryInfo.h>
 #include <Storages/TableStructureLockHolder.h>
 #include <Storages/CheckResults.h>
+#include <Storages/ColumnsDescription.h>
+#include <Storages/IndicesDescription.h>
+#include <Storages/ConstraintsDescription.h>
 #include <Common/ActionLock.h>
 #include <Common/Exception.h>
 #include <Common/RWLock.h>
 #include <Common/SettingsChanges.h>
-#include <Storages/ConstraintsDescription.h>
 
 #include <optional>
 #include <shared_mutex>
@@ -24,7 +26,6 @@ namespace DB
 
 namespace ErrorCodes
 {
-    extern const int TABLE_IS_DROPPED;
     extern const int NOT_IMPLEMENTED;
 }
 
@@ -261,12 +262,12 @@ public:
       * The table is not usable during and after call to this method.
       * If you do not need any action other than deleting the directory with data, you can leave this method blank.
       */
-    virtual void drop() {}
+    virtual void drop(TableStructureWriteLockHolder &) {}
 
     /** Clear the table data and leave it empty.
       * Must be called under lockForAlter.
       */
-    virtual void truncate(const ASTPtr & /*query*/, const Context & /* context */)
+    virtual void truncate(const ASTPtr & /*query*/, const Context & /* context */, TableStructureWriteLockHolder &)
     {
         throw Exception("Truncate is not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED);
     }
@@ -276,7 +277,8 @@ public:
       * In this function, you need to rename the directory with the data, if any.
       * Called when the table structure is locked for write.
       */
-    virtual void rename(const String & /*new_path_to_db*/, const String & /*new_database_name*/, const String & /*new_table_name*/)
+    virtual void rename(const String & /*new_path_to_db*/, const String & /*new_database_name*/, const String & /*new_table_name*/,
+                        TableStructureWriteLockHolder &)
     {
         throw Exception("Method rename is not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED);
     }

dbms/src/Storages/Kafka/KafkaBlockInputStream.cpp

@@ -43,7 +43,7 @@ Block KafkaBlockInputStream::getHeader() const
 
 void KafkaBlockInputStream::readPrefixImpl()
 {
-    auto timeout = std::chrono::milliseconds(context.getSettingsRef().queue_max_wait_ms.totalMilliseconds());
+    auto timeout = std::chrono::milliseconds(context.getSettingsRef().kafka_max_wait_ms.totalMilliseconds());
     buffer = storage.popReadBuffer(timeout);
     claimed = !!buffer;
 

dbms/src/Storages/Kafka/KafkaBlockInputStream.h

@@ -4,6 +4,8 @@
 #include <Interpreters/Context.h>
 
 #include <Storages/Kafka/StorageKafka.h>
+#include <Storages/Kafka/ReadBufferFromKafkaConsumer.h>
+
 
 namespace DB
 {

dbms/src/Storages/Kafka/StorageKafka.cpp

@@ -189,7 +189,7 @@ void StorageKafka::shutdown()
 }
 
 
-void StorageKafka::rename(const String & /* new_path_to_db */, const String & new_database_name, const String & new_table_name)
+void StorageKafka::rename(const String & /* new_path_to_db */, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &)
 {
     table_name = new_table_name;
     database_name = new_database_name;

dbms/src/Storages/Kafka/StorageKafka.h

@@ -1,8 +1,6 @@
 #pragma once
 
 #include <Core/BackgroundSchedulePool.h>
-#include <Core/NamesAndTypes.h>
-#include <DataStreams/IBlockOutputStream.h>
 #include <Storages/IStorage.h>
 #include <Storages/Kafka/ReadBufferFromKafkaConsumer.h>
 #include <Storages/Kafka/WriteBufferToKafkaProducer.h>
@@ -11,6 +9,8 @@
 #include <ext/shared_ptr_helper.h>
 
 #include <mutex>
+#include <atomic>
+
 
 namespace DB
 {
@@ -40,10 +40,9 @@ public:
 
     BlockOutputStreamPtr write(
         const ASTPtr & query,
-        const Context & context
+        const Context & context) override;
-    ) override;
 
-    void rename(const String & /* new_path_to_db */, const String & new_database_name, const String & new_table_name) override;
+    void rename(const String & /* new_path_to_db */, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override;
 
     void updateDependencies() override;
 

dbms/src/Storages/LiveView/LiveViewBlockInputStream.h

@@ -31,7 +31,12 @@ public:
         const bool has_limit_, const UInt64 limit_,
         const UInt64 heartbeat_interval_sec_,
         const UInt64 temporary_live_view_timeout_sec_)
-        : storage(std::move(storage_)), blocks_ptr(std::move(blocks_ptr_)), blocks_metadata_ptr(std::move(blocks_metadata_ptr_)), active_ptr(std::move(active_ptr_)), has_limit(has_limit_), limit(limit_), heartbeat_interval_usec(heartbeat_interval_sec_ * 1000000), temporary_live_view_timeout_sec(temporary_live_view_timeout_sec_)
+        : storage(std::move(storage_)), blocks_ptr(std::move(blocks_ptr_)),
+          blocks_metadata_ptr(std::move(blocks_metadata_ptr_)),
+          active_ptr(std::move(active_ptr_)),
+          has_limit(has_limit_), limit(limit_),
+          heartbeat_interval_usec(heartbeat_interval_sec_ * 1000000),
+          temporary_live_view_timeout_sec(temporary_live_view_timeout_sec_)
     {
         /// grab active pointer
         active = active_ptr.lock();

dbms/src/Storages/LiveView/LiveViewEventsBlockInputStream.h

@@ -51,7 +51,12 @@ public:
         const bool has_limit_, const UInt64 limit_,
         const UInt64 heartbeat_interval_sec_,
         const UInt64 temporary_live_view_timeout_sec_)
-        : storage(std::move(storage_)), blocks_ptr(std::move(blocks_ptr_)), blocks_metadata_ptr(std::move(blocks_metadata_ptr_)), active_ptr(std::move(active_ptr_)), has_limit(has_limit_), limit(limit_), heartbeat_interval_usec(heartbeat_interval_sec_ * 1000000), temporary_live_view_timeout_sec(temporary_live_view_timeout_sec_)
+        : storage(std::move(storage_)), blocks_ptr(std::move(blocks_ptr_)),
+          blocks_metadata_ptr(std::move(blocks_metadata_ptr_)),
+          active_ptr(std::move(active_ptr_)), has_limit(has_limit_),
+          limit(limit_),
+          heartbeat_interval_usec(heartbeat_interval_sec_ * 1000000),
+          temporary_live_view_timeout_sec(temporary_live_view_timeout_sec_)
     {
         /// grab active pointer
         active = active_ptr.lock();

dbms/src/Storages/LiveView/StorageLiveView.cpp

@@ -365,25 +365,25 @@ void StorageLiveView::checkTableCanBeDropped() const
     }
 }
 
-void StorageLiveView::noUsersThread(const UInt64 & timeout)
+void StorageLiveView::noUsersThread(std::shared_ptr<StorageLiveView> storage, const UInt64 & timeout)
 {
-    if (shutdown_called)
-        return;
-
     bool drop_table = false;
 
+    if (storage->shutdown_called)
+        return;
+
     {
         while (1)
         {
-            std::unique_lock lock(no_users_thread_mutex);
+            std::unique_lock lock(storage->no_users_thread_mutex);
-            if (!no_users_thread_condition.wait_for(lock, std::chrono::seconds(timeout), [&] { return no_users_thread_wakeup; }))
+            if (!storage->no_users_thread_condition.wait_for(lock, std::chrono::seconds(timeout), [&] { return storage->no_users_thread_wakeup; }))
             {
-                no_users_thread_wakeup = false;
+                storage->no_users_thread_wakeup = false;
-                if (shutdown_called)
+                if (storage->shutdown_called)
                     return;
-                if (hasUsers())
+                if (storage->hasUsers())
                     return;
-                if (!global_context.getDependencies(database_name, table_name).empty())
+                if (!storage->global_context.getDependencies(storage->database_name, storage->table_name).empty())
                     continue;
                 drop_table = true;
             }
@@ -393,17 +393,17 @@ void StorageLiveView::noUsersThread(const UInt64 & timeout)
 
     if (drop_table)
     {
-        if (global_context.tryGetTable(database_name, table_name))
+        if (storage->global_context.tryGetTable(storage->database_name, storage->table_name))
         {
             try
             {
                 /// We create and execute `drop` query for this table
                 auto drop_query = std::make_shared<ASTDropQuery>();
-                drop_query->database = database_name;
+                drop_query->database = storage->database_name;
-                drop_query->table = table_name;
+                drop_query->table = storage->table_name;
                 drop_query->kind = ASTDropQuery::Kind::Drop;
                 ASTPtr ast_drop_query = drop_query;
-                InterpreterDropQuery drop_interpreter(ast_drop_query, global_context);
+                InterpreterDropQuery drop_interpreter(ast_drop_query, storage->global_context);
                 drop_interpreter.execute();
             }
             catch (...)
@@ -419,9 +419,6 @@ void StorageLiveView::startNoUsersThread(const UInt64 & timeout)
     if (!start_no_users_thread_called.compare_exchange_strong(expected, true))
         return;
 
-    if (is_dropped)
-        return;
-
     if (is_temporary)
     {
         if (no_users_thread.joinable())
@@ -438,8 +435,10 @@ void StorageLiveView::startNoUsersThread(const UInt64 & timeout)
             no_users_thread_wakeup = false;
         }
         if (!is_dropped)
-            no_users_thread = std::thread(&StorageLiveView::noUsersThread, this, timeout);
+            no_users_thread = std::thread(&StorageLiveView::noUsersThread,
+                std::static_pointer_cast<StorageLiveView>(shared_from_this()), timeout);
     }
+
     start_no_users_thread_called = false;
 }
 
@@ -455,23 +454,23 @@ void StorageLiveView::shutdown()
         return;
 
     if (no_users_thread.joinable())
+    {
         {
             std::lock_guard lock(no_users_thread_mutex);
             no_users_thread_wakeup = true;
             no_users_thread_condition.notify_one();
-        /// Must detach the no users thread
+        }
-        /// as we can't join it as it will result
-        /// in a deadlock
-        no_users_thread.detach();   /// TODO Not viable at all.
     }
 }
 
 StorageLiveView::~StorageLiveView()
 {
     shutdown();
+    if (no_users_thread.joinable())
+        no_users_thread.detach();
 }
 
-void StorageLiveView::drop()
+void StorageLiveView::drop(TableStructureWriteLockHolder &)
 {
     global_context.removeDependency(
         DatabaseAndTableName(select_database_name, select_table_name),
@@ -534,7 +533,10 @@ BlockInputStreams StorageLiveView::watch(
 
     if (query.is_watch_events)
     {
-        auto reader = std::make_shared<LiveViewEventsBlockInputStream>(std::static_pointer_cast<StorageLiveView>(shared_from_this()), blocks_ptr, blocks_metadata_ptr, active_ptr, has_limit, limit, context.getSettingsRef().live_view_heartbeat_interval.totalSeconds(),
+        auto reader = std::make_shared<LiveViewEventsBlockInputStream>(
+            std::static_pointer_cast<StorageLiveView>(shared_from_this()),
+            blocks_ptr, blocks_metadata_ptr, active_ptr, has_limit, limit,
+            context.getSettingsRef().live_view_heartbeat_interval.totalSeconds(),
             context.getSettingsRef().temporary_live_view_timeout.totalSeconds());
 
         if (no_users_thread.joinable())
@@ -559,7 +561,10 @@ BlockInputStreams StorageLiveView::watch(
     }
     else
     {
-        auto reader = std::make_shared<LiveViewBlockInputStream>(std::static_pointer_cast<StorageLiveView>(shared_from_this()), blocks_ptr, blocks_metadata_ptr, active_ptr, has_limit, limit, context.getSettingsRef().live_view_heartbeat_interval.totalSeconds(),
+        auto reader = std::make_shared<LiveViewBlockInputStream>(
+            std::static_pointer_cast<StorageLiveView>(shared_from_this()),
+            blocks_ptr, blocks_metadata_ptr, active_ptr, has_limit, limit,
+            context.getSettingsRef().live_view_heartbeat_interval.totalSeconds(),
             context.getSettingsRef().temporary_live_view_timeout.totalSeconds());
 
         if (no_users_thread.joinable())

dbms/src/Storages/LiveView/StorageLiveView.h

@@ -71,11 +71,10 @@ public:
     {
         return active_ptr.use_count() > 1;
     }
-    /// Background thread for temporary tables
+    /// No users thread mutex, predicate and wake up condition
-    /// which drops this table if there are no users
     void startNoUsersThread(const UInt64 & timeout);
     std::mutex no_users_thread_mutex;
-    bool no_users_thread_wakeup{false};
+    bool no_users_thread_wakeup = false;
     std::condition_variable no_users_thread_condition;
     /// Get blocks hash
     /// must be called with mutex locked
@@ -105,7 +104,7 @@ public:
     }
 
     void checkTableCanBeDropped() const override;
-    void drop() override;
+    void drop(TableStructureWriteLockHolder &) override;
     void startup() override;
     void shutdown() override;
 
@@ -149,7 +148,7 @@ private:
     String database_name;
     ASTPtr inner_query;
     Context & global_context;
-    bool is_temporary {false};
+    bool is_temporary = false;
     mutable Block sample_block;
 
     /// Mutex for the blocks and ready condition
@@ -166,10 +165,12 @@ private:
     std::shared_ptr<BlocksMetadataPtr> blocks_metadata_ptr;
     BlocksPtrs mergeable_blocks;
 
-    void noUsersThread(const UInt64 & timeout);
+    /// Background thread for temporary tables
+    /// which drops this table if there are no users
+    static void noUsersThread(std::shared_ptr<StorageLiveView> storage, const UInt64 & timeout);
     std::thread no_users_thread;
-    std::atomic<bool> shutdown_called{false};
+    std::atomic<bool> shutdown_called = false;
-    std::atomic<bool> start_no_users_thread_called{false};
+    std::atomic<bool> start_no_users_thread_called = false;
     UInt64 temporary_live_view_timeout;
 
     StorageLiveView(

dbms/src/Storages/MergeTree/MergeTreeData.cpp

@@ -1690,7 +1690,7 @@ void MergeTreeData::removeEmptyColumnsFromPart(MergeTreeData::MutableDataPartPtr
     empty_columns.clear();
 }
 
-void MergeTreeData::freezeAll(const String & with_name, const Context & context)
+void MergeTreeData::freezeAll(const String & with_name, const Context & context, TableStructureReadLockHolder &)
 {
     freezePartitionsByMatcher([] (const DataPartPtr &){ return true; }, with_name, context);
 }
@@ -2550,7 +2550,7 @@ void MergeTreeData::removePartContributionToColumnSizes(const DataPartPtr & part
 }
 
 
-void MergeTreeData::freezePartition(const ASTPtr & partition_ast, const String & with_name, const Context & context)
+void MergeTreeData::freezePartition(const ASTPtr & partition_ast, const String & with_name, const Context & context, TableStructureReadLockHolder &)
 {
     std::optional<String> prefix;
     String partition_id;

dbms/src/Storages/MergeTree/MergeTreeData.h

@@ -549,7 +549,7 @@ public:
     void removeEmptyColumnsFromPart(MergeTreeData::MutableDataPartPtr & data_part);
 
     /// Freezes all parts.
-    void freezeAll(const String & with_name, const Context & context);
+    void freezeAll(const String & with_name, const Context & context, TableStructureReadLockHolder & table_lock_holder);
 
     /// Should be called if part data is suspected to be corrupted.
     void reportBrokenPart(const String & name) const
@@ -577,7 +577,7 @@ public:
       * Backup is created in directory clickhouse_dir/shadow/i/, where i - incremental number,
       *  or if 'with_name' is specified - backup is created in directory with specified name.
       */
-    void freezePartition(const ASTPtr & partition, const String & with_name, const Context & context);
+    void freezePartition(const ASTPtr & partition, const String & with_name, const Context & context, TableStructureReadLockHolder & table_lock_holder);
 
     size_t getColumnCompressedSize(const std::string & name) const
     {

dbms/src/Storages/StorageBuffer.h

@@ -73,7 +73,11 @@ public:
     void shutdown() override;
     bool optimize(const ASTPtr & query, const ASTPtr & partition, bool final, bool deduplicate, const Context & context) override;
 
-    void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name) override { table_name = new_table_name; database_name = new_database_name; }
+    void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override
+    {
+        table_name = new_table_name;
+        database_name = new_database_name;
+    }
 
     bool supportsSampling() const override { return true; }
     bool supportsPrewhere() const override

dbms/src/Storages/StorageDictionary.h

@@ -34,7 +34,6 @@ public:
         size_t max_block_size = DEFAULT_BLOCK_SIZE,
         unsigned threads = 1) override;
 
-    void drop() override {}
     static NamesAndTypesList getNamesAndTypes(const DictionaryStructure & dictionary_structure);
 
     template <typename ForwardIterator>

dbms/src/Storages/StorageDistributed.cpp

@@ -415,7 +415,7 @@ void StorageDistributed::shutdown()
 }
 
 
-void StorageDistributed::truncate(const ASTPtr &, const Context &)
+void StorageDistributed::truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &)
 {
     std::lock_guard lock(cluster_nodes_mutex);
 

dbms/src/Storages/StorageDistributed.h

@@ -77,12 +77,17 @@ public:
 
     BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override;
 
-    void drop() override {}
+    void drop(TableStructureWriteLockHolder &) override {}
 
     /// Removes temporary data in local filesystem.
-    void truncate(const ASTPtr &, const Context &) override;
+    void truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) override;
+
+    void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override
+    {
+        table_name = new_table_name;
+        database_name = new_database_name;
+    }
 
-    void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name) override { table_name = new_table_name; database_name = new_database_name; }
     /// in the sub-tables, you need to manually add and delete columns
     /// the structure of the sub-table is not checked
     void alter(

dbms/src/Storages/StorageFile.cpp

@@ -264,13 +264,7 @@ BlockOutputStreamPtr StorageFile::write(
 }
 
 
-void StorageFile::drop()
+void StorageFile::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &)
-{
-    /// Extra actions are not required.
-}
-
-
-void StorageFile::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name)
 {
     if (!is_db_table)
         throw Exception("Can't rename table '" + table_name + "' binded to user-defined file (or FD)", ErrorCodes::DATABASE_ACCESS_DENIED);

dbms/src/Storages/StorageFile.h

@@ -38,9 +38,7 @@ public:
         const ASTPtr & query,
         const Context & context) override;
 
-    void drop() override;
+    void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override;
-
-    void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override;
 
     String getDataPath() const override { return path; }
 

dbms/src/Storages/StorageHDFS.cpp

@@ -148,7 +148,7 @@ BlockInputStreams StorageHDFS::read(
         max_block_size)};
 }
 
-void StorageHDFS::rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name)
+void StorageHDFS::rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &)
 {
     table_name = new_table_name;
     database_name = new_database_name;

dbms/src/Storages/StorageHDFS.h

@@ -30,7 +30,7 @@ public:
 
     BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override;
 
-    void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override;
+    void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override;
 
 protected:
     StorageHDFS(const String & uri_,

dbms/src/Storages/StorageJoin.cpp

@@ -55,7 +55,7 @@ StorageJoin::StorageJoin(
 }
 
 
-void StorageJoin::truncate(const ASTPtr &, const Context &)
+void StorageJoin::truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &)
 {
     Poco::File(path).remove(true);
     Poco::File(path).createDirectories();

dbms/src/Storages/StorageJoin.h

@@ -26,7 +26,7 @@ class StorageJoin : public ext::shared_ptr_helper<StorageJoin>, public StorageSe
 public:
     String getName() const override { return "Join"; }
 
-    void truncate(const ASTPtr &, const Context &) override;
+    void truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) override;
 
     /// Access the innards.
     JoinPtr & getJoin() { return join; }

dbms/src/Storages/StorageLog.cpp

@@ -512,7 +512,7 @@ void StorageLog::loadMarks()
 }
 
 
-void StorageLog::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name)
+void StorageLog::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &)
 {
     std::unique_lock<std::shared_mutex> lock(rwlock);
 
@@ -530,7 +530,7 @@ void StorageLog::rename(const String & new_path_to_db, const String & new_databa
     marks_file = Poco::File(path + escapeForFileName(table_name) + '/' + DBMS_STORAGE_LOG_MARKS_FILE_NAME);
 }
 
-void StorageLog::truncate(const ASTPtr &, const Context &)
+void StorageLog::truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &)
 {
     std::shared_lock<std::shared_mutex> lock(rwlock);
 

dbms/src/Storages/StorageLog.h

@@ -38,11 +38,11 @@ public:
 
     BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override;
 
-    void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override;
+    void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override;
 
     CheckResults checkData(const ASTPtr & /* query */, const Context & /* context */) override;
 
-    void truncate(const ASTPtr &, const Context &) override;
+    void truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) override;
 
     std::string full_path() const { return path + escapeForFileName(table_name) + '/';}
 

dbms/src/Storages/StorageMaterializedView.cpp

@@ -232,7 +232,7 @@ static void executeDropQuery(ASTDropQuery::Kind kind, Context & global_context,
 }
 
 
-void StorageMaterializedView::drop()
+void StorageMaterializedView::drop(TableStructureWriteLockHolder &)
 {
     global_context.removeDependency(
         DatabaseAndTableName(select_database_name, select_table_name),
@@ -242,7 +242,7 @@ void StorageMaterializedView::drop()
         executeDropQuery(ASTDropQuery::Kind::Drop, global_context, target_database_name, target_table_name);
 }
 
-void StorageMaterializedView::truncate(const ASTPtr &, const Context &)
+void StorageMaterializedView::truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &)
 {
     if (has_inner_table)
         executeDropQuery(ASTDropQuery::Kind::Truncate, global_context, target_database_name, target_table_name);
@@ -299,7 +299,8 @@ static void executeRenameQuery(Context & global_context, const String & database
 }
 
 
-void StorageMaterializedView::rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name)
+void StorageMaterializedView::rename(
+    const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &)
 {
     if (has_inner_table && tryGetTargetTable())
     {

dbms/src/Storages/StorageMaterializedView.h

@@ -33,9 +33,9 @@ public:
 
     BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override;
 
-    void drop() override;
+    void drop(TableStructureWriteLockHolder &) override;
 
-    void truncate(const ASTPtr &, const Context &) override;
+    void truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) override;
 
     bool optimize(const ASTPtr & query, const ASTPtr & partition, bool final, bool deduplicate, const Context & context) override;
 
@@ -43,7 +43,7 @@ public:
 
     void mutate(const MutationCommands & commands, const Context & context) override;
 
-    void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override;
+    void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override;
 
     void shutdown() override;
 

dbms/src/Storages/StorageMemory.cpp

@@ -123,13 +123,13 @@ BlockOutputStreamPtr StorageMemory::write(
 }
 
 
-void StorageMemory::drop()
+void StorageMemory::drop(TableStructureWriteLockHolder &)
 {
     std::lock_guard lock(mutex);
     data.clear();
 }
 
-void StorageMemory::truncate(const ASTPtr &, const Context &)
+void StorageMemory::truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &)
 {
     std::lock_guard lock(mutex);
     data.clear();

dbms/src/Storages/StorageMemory.h

@@ -40,11 +40,15 @@ public:
 
     BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override;
 
-    void drop() override;
+    void drop(TableStructureWriteLockHolder &) override;
 
-    void truncate(const ASTPtr &, const Context &) override;
+    void truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) override;
 
-    void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name) override { table_name = new_table_name; database_name = new_database_name; }
+    void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override
+    {
+        table_name = new_table_name;
+        database_name = new_database_name;
+    }
 
 private:
     String database_name;

dbms/src/Storages/StorageMerge.h

@@ -42,8 +42,11 @@ public:
         size_t max_block_size,
         unsigned num_streams) override;
 
-    void drop() override {}
+    void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override
-    void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name) override { table_name = new_table_name; database_name = new_database_name; }
+    {
+        table_name = new_table_name;
+        database_name = new_database_name;
+    }
 
     /// you need to add and remove columns in the sub-tables manually
     /// the structure of sub-tables is not checked

dbms/src/Storages/StorageMergeTree.cpp

@@ -157,13 +157,13 @@ void StorageMergeTree::checkPartitionCanBeDropped(const ASTPtr & partition)
     global_context.checkPartitionCanBeDropped(database_name, table_name, partition_size);
 }
 
-void StorageMergeTree::drop()
+void StorageMergeTree::drop(TableStructureWriteLockHolder &)
 {
     shutdown();
     dropAllData();
 }
 
-void StorageMergeTree::truncate(const ASTPtr &, const Context &)
+void StorageMergeTree::truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &)
 {
     {
         /// Asks to complete merges and does not allow them to start.
@@ -181,7 +181,7 @@ void StorageMergeTree::truncate(const ASTPtr &, const Context &)
     clearOldPartsFromFilesystem();
 }
 
-void StorageMergeTree::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name)
+void StorageMergeTree::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &)
 {
     std::string new_full_path = new_path_to_db + escapeForFileName(new_table_name) + '/';
 
@@ -995,7 +995,7 @@ void StorageMergeTree::alterPartition(const ASTPtr & query, const PartitionComma
             case PartitionCommand::FREEZE_PARTITION:
             {
                 auto lock = lockStructureForShare(false, context.getCurrentQueryId());
-                freezePartition(command.partition, command.with_name, context);
+                freezePartition(command.partition, command.with_name, context, lock);
             }
             break;
 
@@ -1020,7 +1020,7 @@ void StorageMergeTree::alterPartition(const ASTPtr & query, const PartitionComma
             case PartitionCommand::FREEZE_ALL_PARTITIONS:
             {
                 auto lock = lockStructureForShare(false, context.getCurrentQueryId());
-                freezeAll(command.with_name, context);
+                freezeAll(command.with_name, context, lock);
             }
             break;
 

dbms/src/Storages/StorageMergeTree.h

@@ -55,10 +55,10 @@ public:
     std::vector<MergeTreeMutationStatus> getMutationsStatus() const override;
     CancellationCode killMutation(const String & mutation_id) override;
 
-    void drop() override;
+    void drop(TableStructureWriteLockHolder &) override;
-    void truncate(const ASTPtr &, const Context &) override;
+    void truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) override;
 
-    void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override;
+    void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override;
 
     void alter(const AlterCommands & params, const Context & context, TableStructureWriteLockHolder & table_lock_holder) override;
 

dbms/src/Storages/StorageNull.h

@@ -38,7 +38,7 @@ public:
         return std::make_shared<NullBlockOutputStream>(getSampleBlock());
     }
 
-    void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name) override
+    void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override
     {
         table_name = new_table_name;
         database_name = new_database_name;

dbms/src/Storages/StorageReplicatedMergeTree.cpp

@@ -3132,7 +3132,6 @@ bool StorageReplicatedMergeTree::optimize(const ASTPtr & query, const ASTPtr & p
     if (query_context.getSettingsRef().replication_alter_partitions_sync != 0)
     {
         /// NOTE Table lock must not be held while waiting. Some combination of R-W-R locks from different threads will yield to deadlock.
-        /// TODO Check all other "wait" places.
         for (auto & merge_entry : merge_entries)
             waitForAllReplicasToProcessLogEntry(merge_entry);
     }
@@ -3484,7 +3483,7 @@ void StorageReplicatedMergeTree::alterPartition(const ASTPtr & query, const Part
             case PartitionCommand::FREEZE_PARTITION:
             {
                 auto lock = lockStructureForShare(false, query_context.getCurrentQueryId());
-                freezePartition(command.partition, command.with_name, query_context);
+                freezePartition(command.partition, command.with_name, query_context, lock);
             }
             break;
 
@@ -3509,7 +3508,7 @@ void StorageReplicatedMergeTree::alterPartition(const ASTPtr & query, const Part
             case PartitionCommand::FREEZE_ALL_PARTITIONS:
             {
                 auto lock = lockStructureForShare(false, query_context.getCurrentQueryId());
-                freezeAll(command.with_name, query_context);
+                freezeAll(command.with_name, query_context, lock);
             }
             break;
         }
@@ -3633,8 +3632,10 @@ void StorageReplicatedMergeTree::dropPartition(const ASTPtr & query, const ASTPt
 }
 
 
-void StorageReplicatedMergeTree::truncate(const ASTPtr & query, const Context & query_context)
+void StorageReplicatedMergeTree::truncate(const ASTPtr & query, const Context & query_context, TableStructureWriteLockHolder & table_lock)
 {
+    table_lock.release();   /// Truncate is done asynchronously.
+
     assertNotReadonly();
 
     zkutil::ZooKeeperPtr zookeeper = getZooKeeper();
@@ -3701,7 +3702,7 @@ void StorageReplicatedMergeTree::checkPartitionCanBeDropped(const ASTPtr & parti
 }
 
 
-void StorageReplicatedMergeTree::drop()
+void StorageReplicatedMergeTree::drop(TableStructureWriteLockHolder &)
 {
     {
         auto zookeeper = tryGetZooKeeper();
@@ -3731,7 +3732,8 @@ void StorageReplicatedMergeTree::drop()
 }
 
 
-void StorageReplicatedMergeTree::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name)
+void StorageReplicatedMergeTree::rename(
+    const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &)
 {
     std::string new_full_path = new_path_to_db + escapeForFileName(new_table_name) + '/';
 

dbms/src/Storages/StorageReplicatedMergeTree.h

@@ -109,11 +109,11 @@ public:
 
     /** Removes a replica from ZooKeeper. If there are no other replicas, it deletes the entire table from ZooKeeper.
       */
-    void drop() override;
+    void drop(TableStructureWriteLockHolder &) override;
 
-    void truncate(const ASTPtr &, const Context &) override;
+    void truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) override;
 
-    void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override;
+    void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override;
 
     bool supportsIndexForIn() const override { return true; }
 

dbms/src/Storages/StorageSet.cpp

@@ -126,7 +126,7 @@ void StorageSet::insertBlock(const Block & block) { set->insertFromBlock(block);
 size_t StorageSet::getSize() const { return set->getTotalRowCount(); }
 
 
-void StorageSet::truncate(const ASTPtr &, const Context &)
+void StorageSet::truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &)
 {
     Poco::File(path).remove(true);
     Poco::File(path).createDirectories();
@@ -193,7 +193,8 @@ void StorageSetOrJoinBase::restoreFromFile(const String & file_path)
 }
 
 
-void StorageSetOrJoinBase::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name)
+void StorageSetOrJoinBase::rename(
+    const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &)
 {
     /// Rename directory with data.
     String new_path = new_path_to_db + escapeForFileName(new_table_name);

dbms/src/Storages/StorageSet.h

@@ -22,7 +22,7 @@ public:
     String getTableName() const override { return table_name; }
     String getDatabaseName() const override { return database_name; }
 
-    void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override;
+    void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override;
 
     BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override;
 
@@ -69,7 +69,7 @@ public:
     /// Access the insides.
     SetPtr & getSet() { return set; }
 
-    void truncate(const ASTPtr &, const Context &) override;
+    void truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) override;
 
 private:
     SetPtr set;

dbms/src/Storages/StorageStripeLog.cpp

@@ -223,7 +223,7 @@ StorageStripeLog::StorageStripeLog(
 }
 
 
-void StorageStripeLog::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name)
+void StorageStripeLog::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &)
 {
     std::unique_lock<std::shared_mutex> lock(rwlock);
 
@@ -294,7 +294,7 @@ CheckResults StorageStripeLog::checkData(const ASTPtr & /* query */, const Conte
     return file_checker.check();
 }
 
-void StorageStripeLog::truncate(const ASTPtr &, const Context &)
+void StorageStripeLog::truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &)
 {
     if (table_name.empty())
         throw Exception("Logical error: table name is empty", ErrorCodes::LOGICAL_ERROR);

dbms/src/Storages/StorageStripeLog.h

@@ -40,7 +40,7 @@ public:
 
     BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override;
 
-    void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override;
+    void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override;
 
     CheckResults checkData(const ASTPtr & /* query */, const Context & /* context */) override;
 
@@ -55,7 +55,7 @@ public:
 
     String getDataPath() const override { return full_path(); }
 
-    void truncate(const ASTPtr &, const Context &) override;
+    void truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) override;
 
 private:
     String path;

dbms/src/Storages/StorageTinyLog.cpp

@@ -378,7 +378,7 @@ void StorageTinyLog::addFiles(const String & column_name, const IDataType & type
 }
 
 
-void StorageTinyLog::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name)
+void StorageTinyLog::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &)
 {
     std::unique_lock<std::shared_mutex> lock(rwlock);
 
@@ -424,7 +424,7 @@ CheckResults StorageTinyLog::checkData(const ASTPtr & /* query */, const Context
     return file_checker.check();
 }
 
-void StorageTinyLog::truncate(const ASTPtr &, const Context &)
+void StorageTinyLog::truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &)
 {
     if (table_name.empty())
         throw Exception("Logical error: table name is empty", ErrorCodes::LOGICAL_ERROR);

dbms/src/Storages/StorageTinyLog.h

@@ -39,7 +39,7 @@ public:
 
     BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override;
 
-    void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override;
+    void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override;
 
     CheckResults checkData(const ASTPtr & /* query */, const Context & /* context */) override;
 
@@ -54,7 +54,7 @@ public:
 
     String getDataPath() const override { return full_path(); }
 
-    void truncate(const ASTPtr &, const Context &) override;
+    void truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) override;
 
 private:
     String path;