thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-12-17 20:02:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master' into fix25

Browse Source
This commit is contained in:
proller 2019-08-29 13:27:16 +03:00
commit 286f36477a
192 changed files with 2320 additions and 837 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

258
CMakeLists.txt
View File

@ -1,14 +1,22 @@
foreach(policy
CMP0023
CMP0048 # CMake 3.0
CMP0074 # CMake 3.12
CMP0077
CMP0079
)
if(POLICY ${policy})
cmake_policy(SET ${policy} NEW)
endif()
endforeach()
project(ClickHouse)
cmake_minimum_required(VERSION 3.3)
foreach(policy
CMP0023
CMP0074 # CMake 3.12
)
if(POLICY ${policy})
cmake_policy(SET ${policy} NEW)
endif()
endforeach()
# Ignore export() since we don't use it,
# but it gets broken with a global targets via link_libraries()
macro (export)
endmacro ()
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/")
set(CMAKE_EXPORT_COMPILE_COMMANDS 1) # Write compile_commands.json
@ -128,12 +136,6 @@ if (CMAKE_SYSTEM_PROCESSOR MATCHES "amd64|x86_64")
endif ()
endif ()
if (GLIBC_COMPATIBILITY)
set (USE_INTERNAL_MEMCPY ON)
else ()
message (WARNING "Option GLIBC_COMPATIBILITY must be turned on for production builds.")
endif ()
string(REGEX MATCH "-?[0-9]+(.[0-9]+)?$" COMPILER_POSTFIX ${CMAKE_CXX_COMPILER})
find_program (LLD_PATH NAMES "lld${COMPILER_POSTFIX}" "lld")
@ -172,20 +174,15 @@ if (ARCH_NATIVE)
set (COMPILER_FLAGS "${COMPILER_FLAGS} -march=native")
endif ()
# Special options for better optimized code with clang
#if (COMPILER_CLANG)
# set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -Wno-unused-command-line-argument -mllvm -inline-threshold=10000")
#endif ()
if (CMAKE_VERSION VERSION_LESS "3.8.0")
if (NOT MSVC)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++1z")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17")
endif ()
else ()
set (CMAKE_CXX_STANDARD 17)
set (CMAKE_CXX_EXTENSIONS 0) # https://cmake.org/cmake/help/latest/prop_tgt/CXX_EXTENSIONS.html#prop_tgt:CXX_EXTENSIONS
set (CMAKE_CXX_STANDARD_REQUIRED ON)
set (CXX_FLAGS_INTERNAL_COMPILER "-std=c++1z")
set (CXX_FLAGS_INTERNAL_COMPILER "-std=c++17")
endif ()
if (COMPILER_GCC OR COMPILER_CLANG)
@ -207,17 +204,13 @@ endif()
set (CMAKE_BUILD_COLOR_MAKEFILE ON)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS} ${PLATFORM_EXTRA_CXX_FLAG} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${CXX_WARNING_FLAGS}")
#set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${CMAKE_CXX_FLAGS_ADD}")
set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3 ${CMAKE_CXX_FLAGS_ADD}")
set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_CXX_FLAGS_ADD}")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${CMAKE_C_FLAGS_ADD}")
#set (CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${CMAKE_C_FLAGS_ADD}")
set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -O3 ${CMAKE_C_FLAGS_ADD}")
set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_C_FLAGS_ADD}")
# Uses MAKE_STATIC_LIBRARIES
option (UNBUNDLED "Try find all libraries in system. We recommend to avoid this mode for production builds, because we cannot guarantee exact versions and variants of libraries your system has installed. This mode exists for enthusiastic developers who search for trouble. Also it is useful for maintainers of OS packages." OFF)
if (UNBUNDLED)
@ -225,149 +218,28 @@ if (UNBUNDLED)
else ()
set(NOT_UNBUNDLED 1)
endif ()
# Using system libs can cause lot of warnings in includes.
if (UNBUNDLED OR NOT (OS_LINUX OR APPLE) OR ARCH_32)
option (NO_WERROR "Disable -Werror compiler option" ON)
endif ()
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package (Threads)
include (cmake/find_cxx.cmake)
include (cmake/test_compiler.cmake)
if (OS_LINUX AND COMPILER_CLANG AND USE_STATIC_LIBRARIES)
option (USE_LIBCXX "Use libc++ and libc++abi instead of libstdc++ (only make sense on Linux)" ${HAVE_LIBCXX})
if (USE_LIBCXX)
set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_LIBCPP_DEBUG=0") # More checks in debug build.
endif ()
endif ()
if (USE_LIBCXX)
set (STATIC_STDLIB_FLAGS "")
else ()
set (STATIC_STDLIB_FLAGS "-static-libgcc -static-libstdc++")
endif ()
if (MAKE_STATIC_LIBRARIES AND NOT APPLE AND NOT (COMPILER_CLANG AND OS_FREEBSD))
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${STATIC_STDLIB_FLAGS}")
# Along with executables, we also build example of shared library for "library dictionary source"; and it also should be self-contained.
set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${STATIC_STDLIB_FLAGS}")
endif ()
if (USE_STATIC_LIBRARIES AND HAVE_NO_PIE)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${FLAG_NO_PIE}")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${FLAG_NO_PIE}")
endif ()
# Make this extra-checks for correct library dependencies.
if (NOT SANITIZE)
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-undefined")
set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-undefined")
endif ()
include (cmake/find_unwind.cmake)
include(cmake/dbms_glob_sources.cmake)
include(cmake/default_libs.cmake)
if (USE_INTERNAL_UNWIND_LIBRARY)
option (USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING "Use internal unwind library for exception handling" ${USE_STATIC_LIBRARIES})
endif ()
# Set standard, system and compiler libraries explicitly.
# This is intended for more control of what we are linking.
######################################
### Add targets below this comment ###
######################################
string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC)
set (CMAKE_POSTFIX_VARIABLE "CMAKE_${CMAKE_BUILD_TYPE_UC}_POSTFIX")
set (DEFAULT_LIBS "")
if (OS_LINUX AND NOT UNBUNDLED AND (GLIBC_COMPATIBILITY OR USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING OR USE_LIBCXX))
# Note: this probably has no effect, but I'm not an expert in CMake.
set (CMAKE_C_IMPLICIT_LINK_LIBRARIES "")
set (CMAKE_CXX_IMPLICIT_LINK_LIBRARIES "")
# Disable default linked libraries.
set (DEFAULT_LIBS "-nodefaultlibs")
# We need builtins from Clang's RT even without libcxx - for ubsan+int128. See https://bugs.llvm.org/show_bug.cgi?id=16404
set (BUILTINS_LIB_PATH "")
if (COMPILER_CLANG)
execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-${CMAKE_SYSTEM_PROCESSOR}.a OUTPUT_VARIABLE BUILTINS_LIB_PATH OUTPUT_STRIP_TRAILING_WHITESPACE)
else ()
set (BUILTINS_LIB_PATH "-lgcc")
endif ()
string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC)
set (CMAKE_POSTFIX_VARIABLE "CMAKE_${CMAKE_BUILD_TYPE_UC}_POSTFIX")
# Add C++ libraries.
#
# This consist of:
# - C++ standard library (like implementation of std::string);
# - C++ ABI implementation (functions for exceptions like __cxa_throw, RTTI, etc);
# - functions for internal implementation of exception handling (stack unwinding based on DWARF info; TODO replace with bundled libunwind);
# - compiler builtins (example: functions for implementation of __int128 operations);
#
# There are two variants of C++ library: libc++ (from LLVM compiler infrastructure) and libstdc++ (from GCC).
if (USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING)
if (USE_STATIC_LIBRARIES)
set (EXCEPTION_HANDLING_LIBRARY "${ClickHouse_BINARY_DIR}/contrib/libunwind-cmake/libunwind_static${${CMAKE_POSTFIX_VARIABLE}}.a")
else ()
set (EXCEPTION_HANDLING_LIBRARY "${ClickHouse_BINARY_DIR}/contrib/libunwind-cmake/libunwind_shared${${CMAKE_POSTFIX_VARIABLE}}.so")
endif ()
else ()
set (EXCEPTION_HANDLING_LIBRARY "-lgcc_eh")
endif ()
message (STATUS "Using exception handling library: ${EXCEPTION_HANDLING_LIBRARY}")
if (USE_LIBCXX)
if (USE_INTERNAL_LIBCXX_LIBRARY)
set (LIBCXX_LIBS "${ClickHouse_BINARY_DIR}/contrib/libcxx-cmake/libcxx_static${${CMAKE_POSTFIX_VARIABLE}}.a ${ClickHouse_BINARY_DIR}/contrib/libcxxabi-cmake/libcxxabi_static${${CMAKE_POSTFIX_VARIABLE}}.a")
else ()
set (LIBCXX_LIBS "-lc++ -lc++abi -lc++fs")
endif ()
set (DEFAULT_LIBS "${DEFAULT_LIBS} -Wl,-Bstatic ${LIBCXX_LIBS} ${EXCEPTION_HANDLING_LIBRARY} ${BUILTINS_LIB_PATH} -Wl,-Bdynamic")
else ()
set (DEFAULT_LIBS "${DEFAULT_LIBS} -Wl,-Bstatic -lstdc++ -lstdc++fs ${EXCEPTION_HANDLING_LIBRARY} ${COVERAGE_OPTION} ${BUILTINS_LIB_PATH} -Wl,-Bdynamic")
endif ()
# Linking with GLIBC prevents portability of binaries to older systems.
# We overcome this behaviour by statically linking with our own implementation of all new symbols (that don't exist in older Libc or have infamous "symbol versioning").
# The order of linking is important: 'glibc-compatibility' must be before libc but after all other libraries.
if (GLIBC_COMPATIBILITY)
message (STATUS "Some symbols from glibc will be replaced for compatibility")
string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC)
set (CMAKE_POSTFIX_VARIABLE "CMAKE_${CMAKE_BUILD_TYPE_UC}_POSTFIX")
# FIXME: glibc-compatibility may be non-static in some builds!
set (DEFAULT_LIBS "${DEFAULT_LIBS} ${ClickHouse_BINARY_DIR}/libs/libglibc-compatibility/libglibc-compatibility${${CMAKE_POSTFIX_VARIABLE}}.a")
endif ()
# Add Libc. GLIBC is actually a collection of interdependent libraries.
set (DEFAULT_LIBS "${DEFAULT_LIBS} -lrt -ldl -lpthread -lm -lc")
# Note: we'd rather use Musl libc library, but it's little bit more difficult to use.
message(STATUS "Default libraries: ${DEFAULT_LIBS}")
endif ()
if (NOT GLIBC_COMPATIBILITY)
set (M_LIBRARY m)
endif ()
if (DEFAULT_LIBS)
# Add default libs to all targets as the last dependency.
set(CMAKE_CXX_STANDARD_LIBRARIES ${DEFAULT_LIBS})
set(CMAKE_C_STANDARD_LIBRARIES ${DEFAULT_LIBS})
endif ()
if (NOT MAKE_STATIC_LIBRARIES)
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
endif ()
@ -420,20 +292,12 @@ if (UNBUNDLED)
else ()
set(NOT_UNBUNDLED 1)
endif ()
# Using system libs can cause lot of warnings in includes.
if (UNBUNDLED OR NOT (OS_LINUX OR APPLE) OR ARCH_32)
option (NO_WERROR "Disable -Werror compiler option" ON)
endif ()
if (USE_LIBCXX)
set (HAVE_LIBCXX 1)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
endif()
if (USE_LIBCXX AND USE_INTERNAL_LIBCXX_LIBRARY)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -nostdinc++ -isystem ${LIBCXX_INCLUDE_DIR} -isystem ${LIBCXXABI_INCLUDE_DIR}")
endif ()
message (STATUS "Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_LIBRARY_ARCHITECTURE} ; USE_STATIC_LIBRARIES=${USE_STATIC_LIBRARIES} MAKE_STATIC_LIBRARIES=${MAKE_STATIC_LIBRARIES} SPLIT_SHARED=${SPLIT_SHARED_LIBRARIES} UNBUNDLED=${UNBUNDLED} CCACHE=${CCACHE_FOUND} ${CCACHE_VERSION}")
include(GNUInstallDirs)
@ -499,79 +363,11 @@ include (libs/libmysqlxx/cmake/find_mysqlclient.cmake)
include (cmake/print_flags.cmake)
install (EXPORT global DESTINATION cmake)
add_subdirectory (contrib EXCLUDE_FROM_ALL)
add_subdirectory (libs)
add_subdirectory (utils)
add_subdirectory (dbms)
include (cmake/print_include_directories.cmake)
if (GLIBC_COMPATIBILITY OR USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING)
# FIXME: actually glibc-compatibility should always be built first,
# because it's unconditionally linked via $DEFAULT_LIBS,
# and these looks like the first places that get linked.
function (add_default_dependencies target_name)
if (TARGET ${target_name})
if (GLIBC_COMPATIBILITY)
add_dependencies(${target_name} glibc-compatibility)
endif ()
if (USE_LIBCXX AND USE_INTERNAL_LIBCXX_LIBRARY)
add_dependencies(${target_name} cxx_static cxxabi_static)
endif ()
if (USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING)
add_dependencies(${target_name} unwind_static)
endif ()
endif ()
endfunction ()
add_default_dependencies(ltdl)
add_default_dependencies(zlibstatic)
add_default_dependencies(jemalloc)
add_default_dependencies(memcpy)
add_default_dependencies(Foundation)
add_default_dependencies(common)
add_default_dependencies(gtest)
add_default_dependencies(lz4)
add_default_dependencies(zstd)
add_default_dependencies(snappy)
add_default_dependencies(arrow)
add_default_dependencies(protoc)
add_default_dependencies(thrift_static)
add_default_dependencies(cityhash)
add_default_dependencies(farmhash)
add_default_dependencies(murmurhash)
add_default_dependencies(metrohash)
add_default_dependencies(metrohash128)
add_default_dependencies(consistent-hashing)
add_default_dependencies(double-conversion)
add_default_dependencies(cctz)
add_default_dependencies(kj)
add_default_dependencies(simdjson)
add_default_dependencies(apple_rt)
add_default_dependencies(h3)
add_default_dependencies(re2)
add_default_dependencies(re2_st)
add_default_dependencies(hs_compile_shared)
add_default_dependencies(hs_exec_shared)
add_default_dependencies(hs_shared)
add_default_dependencies(widechar_width)
add_default_dependencies(string_utils)
add_default_dependencies(consistent-hashing-sumbur)
add_default_dependencies(boost_program_options_internal)
add_default_dependencies(boost_system_internal)
add_default_dependencies(boost_regex_internal)
add_default_dependencies(roaring)
add_default_dependencies(btrie)
add_default_dependencies(cpuid)
add_default_dependencies(mysqlclient)
add_default_dependencies(zlib)
add_default_dependencies(thrift)
add_default_dependencies(brotli)
add_default_dependencies(libprotobuf)
add_default_dependencies(base64)
add_default_dependencies(readpassphrase)
add_default_dependencies(unwind_static)
add_default_dependencies(fastops)
endif ()

48
cmake/default_libs.cmake Normal file
View File

@ -0,0 +1,48 @@
# Set standard, system and compiler libraries explicitly.
# This is intended for more control of what we are linking.
set (DEFAULT_LIBS "-nodefaultlibs")
if (OS_LINUX)
# We need builtins from Clang's RT even without libcxx - for ubsan+int128.
# See https://bugs.llvm.org/show_bug.cgi?id=16404
if (COMPILER_CLANG)
execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-${CMAKE_SYSTEM_PROCESSOR}.a OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE)
else ()
set (BUILTINS_LIBRARY "-lgcc")
endif ()
set (DEFAULT_LIBS "${DEFAULT_LIBS} ${BUILTINS_LIBRARY} ${COVERAGE_OPTION} -lc -lm -lrt -lpthread -ldl")
message(STATUS "Default libraries: ${DEFAULT_LIBS}")
endif ()
set(CMAKE_CXX_STANDARD_LIBRARIES ${DEFAULT_LIBS})
set(CMAKE_C_STANDARD_LIBRARIES ${DEFAULT_LIBS})
# Global libraries
add_library(global-libs INTERFACE)
# Unfortunately '-pthread' doesn't work with '-nodefaultlibs'.
# Just make sure we have pthreads at all.
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
add_subdirectory(libs/libglibc-compatibility)
include (cmake/find_unwind.cmake)
include (cmake/find_cxx.cmake)
add_library(global-group INTERFACE)
target_link_libraries(global-group INTERFACE
-Wl,--start-group
$<TARGET_PROPERTY:global-libs,INTERFACE_LINK_LIBRARIES>
-Wl,--end-group
)
link_libraries(global-group)
install(
TARGETS global-group global-libs
EXPORT global
)

64
cmake/find_capnp.cmake
View File

@ -1,50 +1,20 @@
option (ENABLE_CAPNP "Enable Cap'n Proto" ON)
if (ENABLE_CAPNP)
# cmake 3.5.1 bug:
# capnproto uses this cmake feature:
# target_compile_features(kj PUBLIC cxx_constexpr)
# old cmake adds -std=gnu++11 to end of all compile commands (even if -std=gnu++17 already present in compile string)
# cmake 3.9.1 (ubuntu artful) have no this bug (c++17 support added to cmake 3.8.2)
if (CMAKE_VERSION VERSION_LESS "3.8.0")
set (USE_INTERNAL_CAPNP_LIBRARY_DEFAULT 0)
set (MISSING_INTERNAL_CAPNP_LIBRARY 1)
else ()
set (USE_INTERNAL_CAPNP_LIBRARY_DEFAULT ${NOT_UNBUNDLED})
endif ()
option (USE_INTERNAL_CAPNP_LIBRARY "Set to FALSE to use system capnproto library instead of bundled" ${USE_INTERNAL_CAPNP_LIBRARY_DEFAULT})
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/capnproto/c++/CMakeLists.txt")
if (USE_INTERNAL_CAPNP_LIBRARY)
message (WARNING "submodule contrib/capnproto is missing. to fix try run: \n git submodule update --init --recursive")
endif ()
set (USE_INTERNAL_CAPNP_LIBRARY 0)
set (MISSING_INTERNAL_CAPNP_LIBRARY 1)
endif ()
if (NOT USE_INTERNAL_CAPNP_LIBRARY)
set (CAPNP_PATHS "/usr/local/opt/capnp/lib")
set (CAPNP_INCLUDE_PATHS "/usr/local/opt/capnp/include")
find_library (CAPNP capnp PATHS ${CAPNP_PATHS})
find_library (CAPNPC capnpc PATHS ${CAPNP_PATHS})
find_library (KJ kj PATHS ${CAPNP_PATHS})
set (CAPNP_LIBRARY ${CAPNPC} ${CAPNP} ${KJ})
find_path (CAPNP_INCLUDE_DIR NAMES capnp/schema-parser.h PATHS ${CAPNP_INCLUDE_PATHS})
endif ()
if (CAPNP_INCLUDE_DIR AND CAPNP_LIBRARY)
set(USE_CAPNP 1)
elseif (NOT MISSING_INTERNAL_CAPNP_LIBRARY)
set (USE_INTERNAL_CAPNP_LIBRARY 1)
set (CAPNP_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/capnproto/c++/src")
set (CAPNP_LIBRARY capnpc)
set (USE_CAPNP 1)
endif ()
endif ()
option (USE_CAPNP "Enable Cap'n Proto" ON)
if (USE_CAPNP)
message (STATUS "Using capnp=${USE_CAPNP}: ${CAPNP_INCLUDE_DIR} : ${CAPNP_LIBRARY}")
else ()
message (STATUS "Build without capnp (support for Cap'n Proto format will be disabled)")
option (USE_INTERNAL_CAPNP_LIBRARY "Set to FALSE to use system capnproto library instead of bundled" ${NOT_UNBUNDLED})
# FIXME: refactor to use `add_library(… IMPORTED)` if possible.
if (NOT USE_INTERNAL_CAPNP_LIBRARY)
find_library (KJ kj)
find_library (CAPNP capnp)
find_library (CAPNPC capnpc)
set (CAPNP_LIBRARIES ${CAPNPC} ${CAPNP} ${KJ})
else ()
add_subdirectory(contrib/capnproto-cmake)
set (CAPNP_LIBRARIES capnpc)
endif ()
message (STATUS "Using capnp: ${CAPNP_LIBRARIES}")
endif ()

47
cmake/find_cxx.cmake
View File

@ -1,26 +1,49 @@
if (NOT APPLE)
if (OS_LINUX AND COMPILER_CLANG)
option (USE_LIBCXX "Use libc++ and libc++abi instead of libstdc++" ${HAVE_LIBCXX})
option (USE_INTERNAL_LIBCXX_LIBRARY "Set to FALSE to use system libcxx and libcxxabi libraries instead of bundled" ${NOT_UNBUNDLED})
endif()
if (USE_LIBCXX)
set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_LIBCPP_DEBUG=0") # More checks in debug build.
endif ()
# FIXME: make better check for submodule presence
if (USE_INTERNAL_LIBCXX_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libcxx/include/vector")
message (WARNING "submodule contrib/libcxx is missing. to fix try run: \n git submodule update --init --recursive")
set (USE_INTERNAL_LIBCXX_LIBRARY 0)
endif ()
# FIXME: make better check for submodule presence
if (USE_INTERNAL_LIBCXX_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libcxxabi/src")
message (WARNING "submodule contrib/libcxxabi is missing. to fix try run: \n git submodule update --init --recursive")
set (USE_INTERNAL_LIBCXXABI_LIBRARY 0)
set (USE_INTERNAL_LIBCXX_LIBRARY 0)
endif ()
if (NOT USE_INTERNAL_LIBCXX_LIBRARY)
find_library (LIBCXX_LIBRARY c++)
find_library (LIBCXXABI_LIBRARY c++abi)
if (USE_LIBCXX)
if (NOT USE_INTERNAL_LIBCXX_LIBRARY)
find_library (LIBCXX_LIBRARY c++)
find_library (LIBCXXFS_LIBRARY c++fs)
find_library (LIBCXXABI_LIBRARY c++abi)
target_link_libraries(global-libs INTERFACE ${EXCEPTION_HANDLING_LIBRARY})
else ()
set (LIBCXX_LIBRARY cxx)
set (LIBCXXABI_LIBRARY cxxabi)
add_subdirectory(contrib/libcxxabi-cmake)
add_subdirectory(contrib/libcxx-cmake)
# Exception handling library is embedded into libcxxabi.
endif ()
target_link_libraries(global-libs INTERFACE ${LIBCXX_LIBRARY} ${LIBCXXABI_LIBRARY} ${LIBCXXFS_LIBRARY})
set (HAVE_LIBCXX 1)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
message (STATUS "Using libcxx: ${LIBCXX_LIBRARY}")
message (STATUS "Using libcxxfs: ${LIBCXXFS_LIBRARY}")
message (STATUS "Using libcxxabi: ${LIBCXXABI_LIBRARY}")
else ()
set (LIBCXX_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libcxx/include)
set (LIBCXXABI_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libcxxabi/include)
set (LIBCXX_LIBRARY cxx_static)
set (LIBCXXABI_LIBRARY cxxabi_static)
target_link_libraries(global-libs INTERFACE -l:libstdc++.a -l:libstdc++fs.a) # Always link these libraries as static
target_link_libraries(global-libs INTERFACE ${EXCEPTION_HANDLING_LIBRARY})
endif ()
message (STATUS "Using libcxx: ${LIBCXX_LIBRARY}")
message (STATUS "Using libcxxabi: ${LIBCXXABI_LIBRARY}")

64
cmake/find_unwind.cmake
View File

@ -1,59 +1,17 @@
include (CMakePushCheckState)
cmake_push_check_state ()
option (USE_UNWIND "Enable libunwind (better stacktraces)" ON)
option (ENABLE_UNWIND "Enable libunwind (better stacktraces)" ON)
if (NOT CMAKE_SYSTEM MATCHES "Linux" OR ARCH_ARM OR ARCH_32)
set (USE_UNWIND OFF)
endif ()
if (ENABLE_UNWIND)
if (USE_UNWIND)
add_subdirectory(contrib/libunwind-cmake)
set (UNWIND_LIBRARIES unwind)
set (EXCEPTION_HANDLING_LIBRARY ${UNWIND_LIBRARIES})
if (CMAKE_SYSTEM MATCHES "Linux" AND NOT ARCH_ARM AND NOT ARCH_32)
option (USE_INTERNAL_UNWIND_LIBRARY "Set to FALSE to use system unwind library instead of bundled" ${NOT_UNBUNDLED})
message (STATUS "Using libunwind: ${UNWIND_LIBRARIES}")
else ()
option (USE_INTERNAL_UNWIND_LIBRARY "Set to FALSE to use system unwind library instead of bundled" OFF)
set (EXCEPTION_HANDLING_LIBRARY gcc_eh)
endif ()
if (NOT USE_INTERNAL_UNWIND_LIBRARY)
find_library (UNWIND_LIBRARY unwind)
find_path (UNWIND_INCLUDE_DIR NAMES unwind.h PATHS ${UNWIND_INCLUDE_PATHS})
include (CheckCXXSourceCompiles)
set(CMAKE_REQUIRED_INCLUDES ${UNWIND_INCLUDE_DIR})
set(CMAKE_REQUIRED_LIBRARIES ${UNWIND_LIBRARY})
check_cxx_source_compiles("
#include <ucontext.h>
#define UNW_LOCAL_ONLY
#include <libunwind.h>
int main () {
ucontext_t context;
unw_cursor_t cursor;
unw_init_local(&cursor, &context);
return 0;
}
" HAVE_UNW_INIT_LOCAL)
if (NOT HAVE_UNW_INIT_LOCAL)
set(UNWIND_LIBRARY "")
set(UNWIND_INCLUDE_DIR "")
endif ()
endif ()
if (UNWIND_LIBRARY AND UNWIND_INCLUDE_DIR)
set (USE_UNWIND 1)
elseif (CMAKE_SYSTEM MATCHES "Linux" AND NOT ARCH_ARM AND NOT ARCH_32 AND NOT UNBUNDLED)
set (USE_INTERNAL_UNWIND_LIBRARY 1)
set (PACKAGE_VERSION "9.0.0svn" CACHE STRING "")
set (UNWIND_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libunwind/include")
set (LIBUNWIND_ENABLE_SHARED OFF CACHE BOOL "")
set (LIBUNWIND_ENABLE_STATIC ON CACHE BOOL "")
set (UNWIND_LIBRARY unwind_static)
set (USE_UNWIND 1)
endif ()
endif ()
message (STATUS "Using unwind=${USE_UNWIND}: ${UNWIND_INCLUDE_DIR} : ${UNWIND_LIBRARY}")
cmake_pop_check_state ()
message (STATUS "Using exception handler: ${EXCEPTION_HANDLING_LIBRARY}")

47
cmake/test_compiler.cmake
View File

@ -1,47 +0,0 @@
include (CheckCXXSourceCompiles)
include (CMakePushCheckState)
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads)
cmake_push_check_state ()
if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
# clang4 : -no-pie cause error
# clang6 : -no-pie cause warning
if (MAKE_STATIC_LIBRARIES)
set (TEST_FLAG "-Wl,-Bstatic -stdlib=libc++ -lc++ -lc++abi -Wl,-Bdynamic")
else ()
set (TEST_FLAG "-stdlib=libc++ -lc++ -lc++abi")
endif ()
set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG}")
set (CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} Threads::Threads)
check_cxx_source_compiles("
#include <iostream>
int main() {
std::cerr << std::endl;
return 0;
}
" HAVE_LIBCXX)
else ()
set (TEST_FLAG "-no-pie")
set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG}")
check_cxx_source_compiles("
int main() {
return 0;
}
" HAVE_NO_PIE)
if (HAVE_NO_PIE)
set (FLAG_NO_PIE ${TEST_FLAG})
endif ()
endif ()
cmake_pop_check_state ()

19
contrib/CMakeLists.txt vendored
View File

@ -23,16 +23,6 @@ if (USE_INTERNAL_ORC_LIBRARY)
add_subdirectory(orc)
endif()
if (USE_INTERNAL_UNWIND_LIBRARY)
add_subdirectory (libunwind-cmake)
endif ()
if (USE_LIBCXX AND USE_INTERNAL_LIBCXX_LIBRARY)
add_subdirectory(libcxx-cmake)
add_subdirectory(libcxxabi-cmake)
endif()
if (USE_INTERNAL_BOOST_LIBRARY)
add_subdirectory (boost-cmake)
endif ()
@ -172,15 +162,6 @@ if (ENABLE_ODBC AND USE_INTERNAL_ODBC_LIBRARY)
add_library(ODBC::ODBC ALIAS ${ODBC_LIBRARIES})
endif ()
if (ENABLE_CAPNP AND USE_INTERNAL_CAPNP_LIBRARY)
set (BUILD_TESTING 0 CACHE INTERNAL "")
set (_save ${CMAKE_CXX_EXTENSIONS})
set (CMAKE_CXX_EXTENSIONS)
add_subdirectory (capnproto/c++)
set (CMAKE_CXX_EXTENSIONS ${_save})
target_include_directories(${CAPNP_LIBRARY} PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/capnproto/c++/src>)
endif ()
if (USE_INTERNAL_PARQUET_LIBRARY)
if (USE_INTERNAL_PARQUET_LIBRARY_NATIVE_CMAKE)
# We dont use arrow's cmakefiles because they uses too many depends and download some libs in compile time

3
contrib/arrow-cmake/CMakeLists.txt
View File

@ -44,7 +44,6 @@ set( thriftcpp_threads_SOURCES
add_library(${THRIFT_LIBRARY} ${thriftcpp_SOURCES} ${thriftcpp_threads_SOURCES})
set_target_properties(${THRIFT_LIBRARY} PROPERTIES CXX_STANDARD 14) # REMOVE after https://github.com/apache/thrift/pull/1641
target_include_directories(${THRIFT_LIBRARY} SYSTEM PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/thrift/lib/cpp/src PRIVATE ${Boost_INCLUDE_DIRS})
target_link_libraries(${THRIFT_LIBRARY} PRIVATE Threads::Threads)
# === orc
@ -219,7 +218,7 @@ endif()
add_library(${ARROW_LIBRARY} ${ARROW_SRCS})
add_dependencies(${ARROW_LIBRARY} protoc)
target_include_directories(${ARROW_LIBRARY} SYSTEM PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/cpp/src ${Boost_INCLUDE_DIRS})
target_link_libraries(${ARROW_LIBRARY} PRIVATE ${DOUBLE_CONVERSION_LIBRARIES} ${Protobuf_LIBRARY} Threads::Threads)
target_link_libraries(${ARROW_LIBRARY} PRIVATE ${DOUBLE_CONVERSION_LIBRARIES} ${Protobuf_LIBRARY})
if (ARROW_WITH_LZ4)
target_link_libraries(${ARROW_LIBRARY} PRIVATE ${LZ4_LIBRARY})
endif()

68
contrib/capnproto-cmake/CMakeLists.txt Normal file
View File

@ -0,0 +1,68 @@
set (CAPNPROTO_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/capnproto/c++/src)
set (KJ_SRCS
${CAPNPROTO_SOURCE_DIR}/kj/array.c++
${CAPNPROTO_SOURCE_DIR}/kj/common.c++
${CAPNPROTO_SOURCE_DIR}/kj/debug.c++
${CAPNPROTO_SOURCE_DIR}/kj/exception.c++
${CAPNPROTO_SOURCE_DIR}/kj/io.c++
${CAPNPROTO_SOURCE_DIR}/kj/memory.c++
${CAPNPROTO_SOURCE_DIR}/kj/mutex.c++
${CAPNPROTO_SOURCE_DIR}/kj/string.c++
${CAPNPROTO_SOURCE_DIR}/kj/hash.c++
${CAPNPROTO_SOURCE_DIR}/kj/table.c++
${CAPNPROTO_SOURCE_DIR}/kj/thread.c++
${CAPNPROTO_SOURCE_DIR}/kj/main.c++
${CAPNPROTO_SOURCE_DIR}/kj/arena.c++
${CAPNPROTO_SOURCE_DIR}/kj/test-helpers.c++
${CAPNPROTO_SOURCE_DIR}/kj/units.c++
${CAPNPROTO_SOURCE_DIR}/kj/encoding.c++
${CAPNPROTO_SOURCE_DIR}/kj/refcount.c++
${CAPNPROTO_SOURCE_DIR}/kj/string-tree.c++
${CAPNPROTO_SOURCE_DIR}/kj/time.c++
${CAPNPROTO_SOURCE_DIR}/kj/filesystem.c++
${CAPNPROTO_SOURCE_DIR}/kj/filesystem-disk-unix.c++
${CAPNPROTO_SOURCE_DIR}/kj/filesystem-disk-win32.c++
${CAPNPROTO_SOURCE_DIR}/kj/parse/char.c++
)
add_library(kj ${KJ_SRCS})
target_include_directories(kj INTERFACE ${CAPNPROTO_SOURCE_DIR})
set (CAPNP_SRCS
${CAPNPROTO_SOURCE_DIR}/capnp/c++.capnp.c++
${CAPNPROTO_SOURCE_DIR}/capnp/blob.c++
${CAPNPROTO_SOURCE_DIR}/capnp/arena.c++
${CAPNPROTO_SOURCE_DIR}/capnp/layout.c++
${CAPNPROTO_SOURCE_DIR}/capnp/list.c++
${CAPNPROTO_SOURCE_DIR}/capnp/any.c++
${CAPNPROTO_SOURCE_DIR}/capnp/message.c++
${CAPNPROTO_SOURCE_DIR}/capnp/schema.capnp.c++
${CAPNPROTO_SOURCE_DIR}/capnp/serialize.c++
${CAPNPROTO_SOURCE_DIR}/capnp/serialize-packed.c++
${CAPNPROTO_SOURCE_DIR}/capnp/schema.c++
${CAPNPROTO_SOURCE_DIR}/capnp/schema-loader.c++
${CAPNPROTO_SOURCE_DIR}/capnp/dynamic.c++
${CAPNPROTO_SOURCE_DIR}/capnp/stringify.c++
)
add_library(capnp ${CAPNP_SRCS})
target_link_libraries(capnp PUBLIC kj)
set (CAPNPC_SRCS
${CAPNPROTO_SOURCE_DIR}/capnp/compiler/type-id.c++
${CAPNPROTO_SOURCE_DIR}/capnp/compiler/error-reporter.c++
${CAPNPROTO_SOURCE_DIR}/capnp/compiler/lexer.capnp.c++
${CAPNPROTO_SOURCE_DIR}/capnp/compiler/lexer.c++
${CAPNPROTO_SOURCE_DIR}/capnp/compiler/grammar.capnp.c++
${CAPNPROTO_SOURCE_DIR}/capnp/compiler/parser.c++
${CAPNPROTO_SOURCE_DIR}/capnp/compiler/node-translator.c++
${CAPNPROTO_SOURCE_DIR}/capnp/compiler/compiler.c++
${CAPNPROTO_SOURCE_DIR}/capnp/schema-parser.c++
${CAPNPROTO_SOURCE_DIR}/capnp/serialize-text.c++
)
add_library(capnpc ${CAPNPC_SRCS})
target_link_libraries(capnpc PUBLIC capnp)

3
contrib/jemalloc-cmake/CMakeLists.txt
View File

@ -59,7 +59,6 @@ if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")
if (USE_UNWIND)
target_compile_definitions (jemalloc PRIVATE -DJEMALLOC_PROF_LIBUNWIND=1)
target_include_directories (jemalloc BEFORE PRIVATE ${UNWIND_INCLUDE_DIR})
target_link_libraries (jemalloc PRIVATE ${UNWIND_LIBRARY})
target_link_libraries (jemalloc PRIVATE ${UNWIND_LIBRARIES})
endif ()
endif ()

20
contrib/libcxx-cmake/CMakeLists.txt
View File

@ -1,5 +1,4 @@
set(LIBCXX_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libcxx)
#set(LIBCXX_BINARY_DIR ${ClickHouse_BINARY_DIR}/contrib/libcxx)
set(SRCS
${LIBCXX_SOURCE_DIR}/src/optional.cpp
@ -16,10 +15,6 @@ ${LIBCXX_SOURCE_DIR}/src/condition_variable.cpp
${LIBCXX_SOURCE_DIR}/src/hash.cpp
${LIBCXX_SOURCE_DIR}/src/string.cpp
${LIBCXX_SOURCE_DIR}/src/debug.cpp
#${LIBCXX_SOURCE_DIR}/src/support/win32/support.cpp
#${LIBCXX_SOURCE_DIR}/src/support/win32/locale_win32.cpp
#${LIBCXX_SOURCE_DIR}/src/support/win32/thread_win32.cpp
#${LIBCXX_SOURCE_DIR}/src/support/solaris/xlocale.cpp
${LIBCXX_SOURCE_DIR}/src/stdexcept.cpp
${LIBCXX_SOURCE_DIR}/src/utility.cpp
${LIBCXX_SOURCE_DIR}/src/any.cpp
@ -43,9 +38,16 @@ ${LIBCXX_SOURCE_DIR}/src/system_error.cpp
${LIBCXX_SOURCE_DIR}/src/random.cpp
)
add_library(cxx_static ${SRCS})
add_library(cxx ${SRCS})
target_include_directories(cxx_static PUBLIC ${LIBCXX_SOURCE_DIR}/include)
target_compile_definitions(cxx_static PRIVATE -D_LIBCPP_BUILDING_LIBRARY -DLIBCXX_BUILDING_LIBCXXABI)
target_compile_options(cxx_static PRIVATE -nostdinc++)
target_include_directories(cxx SYSTEM BEFORE PUBLIC $<BUILD_INTERFACE:${LIBCXX_SOURCE_DIR}/include>)
target_compile_definitions(cxx PRIVATE -D_LIBCPP_BUILDING_LIBRARY -DLIBCXX_BUILDING_LIBCXXABI)
target_compile_options(cxx PRIVATE -nostdinc++)
target_link_libraries(cxx PUBLIC cxxabi)
install(
TARGETS cxx
EXPORT global
ARCHIVE DESTINATION lib
RUNTIME DESTINATION lib
)

25
contrib/libcxxabi-cmake/CMakeLists.txt
View File

@ -1,13 +1,10 @@
set(LIBCXXABI_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libcxxabi)
set(LIBCXX_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libcxx)
#set(LIBCXXABI_BINARY_DIR ${ClickHouse_BINARY_DIR}/contrib/libcxxabi)
set(SRCS
${LIBCXXABI_SOURCE_DIR}/src/stdlib_stdexcept.cpp
${LIBCXXABI_SOURCE_DIR}/src/cxa_virtual.cpp
${LIBCXXABI_SOURCE_DIR}/src/cxa_thread_atexit.cpp
${LIBCXXABI_SOURCE_DIR}/src/fallback_malloc.cpp
#${LIBCXXABI_SOURCE_DIR}/src/cxa_noexception.cpp
${LIBCXXABI_SOURCE_DIR}/src/cxa_guard.cpp
${LIBCXXABI_SOURCE_DIR}/src/cxa_default_handlers.cpp
${LIBCXXABI_SOURCE_DIR}/src/cxa_personality.cpp
@ -25,10 +22,24 @@ ${LIBCXXABI_SOURCE_DIR}/src/cxa_vector.cpp
${LIBCXXABI_SOURCE_DIR}/src/stdlib_new_delete.cpp
)
add_library(cxxabi_static ${SRCS})
add_library(cxxabi ${SRCS})
target_include_directories(cxxabi_static PUBLIC ${LIBCXXABI_SOURCE_DIR}/include ${LIBCXX_SOURCE_DIR}/include)
target_compile_definitions(cxxabi_static PRIVATE -D_LIBCPP_BUILDING_LIBRARY)
target_compile_options(cxxabi_static PRIVATE -nostdinc++ -fno-sanitize=undefined) # If we don't disable UBSan, infinite recursion happens in dynamic_cast.
target_include_directories(cxxabi SYSTEM BEFORE
PUBLIC $<BUILD_INTERFACE:${LIBCXXABI_SOURCE_DIR}/include>
PRIVATE $<BUILD_INTERFACE:${LIBCXXABI_SOURCE_DIR}/../libcxx/include>
)
target_compile_definitions(cxxabi PRIVATE -D_LIBCPP_BUILDING_LIBRARY)
target_compile_options(cxxabi PRIVATE -nostdinc++ -fno-sanitize=undefined) # If we don't disable UBSan, infinite recursion happens in dynamic_cast.
if (USE_UNWIND)
target_link_libraries(cxxabi PRIVATE ${UNWIND_LIBRARIES})
else ()
target_link_libraries(cxxabi PRIVATE gcc_eh)
endif ()
install(
TARGETS cxxabi
EXPORT global
ARCHIVE DESTINATION lib
RUNTIME DESTINATION lib
)

2
contrib/librdkafka-cmake/CMakeLists.txt
View File

@ -65,7 +65,7 @@ add_library(rdkafka ${SRCS})
target_include_directories(rdkafka SYSTEM PUBLIC include)
target_include_directories(rdkafka SYSTEM PUBLIC ${RDKAFKA_SOURCE_DIR}) # Because weird logic with "include_next" is used.
target_include_directories(rdkafka SYSTEM PRIVATE ${ZSTD_INCLUDE_DIR}/common) # Because wrong path to "zstd_errors.h" is used.
target_link_libraries(rdkafka PRIVATE ${ZLIB_LIBRARIES} ${ZSTD_LIBRARY} ${LZ4_LIBRARY} ${LIBGSASL_LIBRARY} Threads::Threads)
target_link_libraries(rdkafka PRIVATE ${ZLIB_LIBRARIES} ${ZSTD_LIBRARY} ${LZ4_LIBRARY} ${LIBGSASL_LIBRARY})
if(OPENSSL_SSL_LIBRARY AND OPENSSL_CRYPTO_LIBRARY)
target_link_libraries(rdkafka PRIVATE ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY})
endif()

16
contrib/libunwind-cmake/CMakeLists.txt
View File

@ -24,9 +24,15 @@ set(LIBUNWIND_SOURCES
${LIBUNWIND_C_SOURCES}
${LIBUNWIND_ASM_SOURCES})
add_library(unwind_static ${LIBUNWIND_SOURCES})
add_library(unwind ${LIBUNWIND_SOURCES})
target_include_directories(unwind_static SYSTEM BEFORE PUBLIC ${LIBUNWIND_SOURCE_DIR}/include)
target_compile_definitions(unwind_static PRIVATE -D_LIBUNWIND_NO_HEAP=1 -D_DEBUG -D_LIBUNWIND_IS_NATIVE_ONLY)
target_compile_options(unwind_static PRIVATE -fno-exceptions -funwind-tables -fno-sanitize=all -nostdinc++ -fno-rtti)
target_link_libraries(unwind_static PRIVATE Threads::Threads ${CMAKE_DL_LIBS})
target_include_directories(unwind SYSTEM BEFORE PUBLIC $<BUILD_INTERFACE:${LIBUNWIND_SOURCE_DIR}/include>)
target_compile_definitions(unwind PRIVATE -D_LIBUNWIND_NO_HEAP=1 -D_DEBUG -D_LIBUNWIND_IS_NATIVE_ONLY)
target_compile_options(unwind PRIVATE -fno-exceptions -funwind-tables -fno-sanitize=all -nostdinc++ -fno-rtti)
install(
TARGETS unwind
EXPORT global
LIBRARY DESTINATION lib
ARCHIVE DESTINATION lib
)

5
contrib/mariadb-connector-c-cmake/CMakeLists.txt
View File

@ -62,11 +62,6 @@ endif()
add_library(mysqlclient ${SRCS})
target_link_libraries(mysqlclient PRIVATE ${CMAKE_DL_LIBS} Threads::Threads)
if(M_LIBRARY)
target_link_libraries(mysqlclient PRIVATE ${M_LIBRARY})
endif()
if(OPENSSL_LIBRARIES)
target_link_libraries(mysqlclient PRIVATE ${OPENSSL_LIBRARIES})
target_compile_definitions(mysqlclient PRIVATE -D HAVE_OPENSSL -D HAVE_TLS)

20
dbms/CMakeLists.txt
View File

@ -97,8 +97,6 @@ add_subdirectory (src)
set(dbms_headers)
set(dbms_sources)
include(../cmake/dbms_glob_sources.cmake)
add_headers_and_sources(clickhouse_common_io src/Common)
add_headers_and_sources(clickhouse_common_io src/Common/HashTable)
add_headers_and_sources(clickhouse_common_io src/IO)
@ -163,9 +161,7 @@ if (OS_FREEBSD)
endif ()
if (USE_UNWIND)
if (NOT USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING)
target_link_libraries (clickhouse_common_io PRIVATE ${UNWIND_LIBRARY})
endif ()
target_link_libraries (clickhouse_common_io PRIVATE ${UNWIND_LIBRARIES})
endif ()
add_subdirectory(src/Common/ZooKeeper)
@ -241,15 +237,10 @@ target_link_libraries(clickhouse_common_io
${EXECINFO_LIBRARIES}
PUBLIC
${Boost_SYSTEM_LIBRARY}
${Boost_PROGRAM_OPTIONS_LIBRARY}
PRIVATE
apple_rt
PUBLIC
Threads::Threads
PRIVATE
${CMAKE_DL_LIBS}
PRIVATE
rt
PUBLIC
roaring
)
@ -297,7 +288,6 @@ target_link_libraries (dbms
${Boost_FILESYSTEM_LIBRARY}
PUBLIC
${Boost_SYSTEM_LIBRARY}
Threads::Threads
)
target_include_directories(dbms PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/src/Core/include)
@ -364,10 +354,7 @@ if (USE_ICU)
endif ()
if (USE_CAPNP)
target_link_libraries (dbms PRIVATE ${CAPNP_LIBRARY})
if (NOT USE_INTERNAL_CAPNP_LIBRARY)
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${CAPNP_INCLUDE_DIR})
endif ()
target_link_libraries (dbms PRIVATE ${CAPNP_LIBRARIES})
endif ()
if (USE_PARQUET)
@ -380,7 +367,6 @@ endif ()
if(OPENSSL_CRYPTO_LIBRARY)
target_link_libraries(dbms PRIVATE ${OPENSSL_CRYPTO_LIBRARY})
endif ()
target_link_libraries(dbms PRIVATE Threads::Threads)
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${DIVIDE_INCLUDE_DIR})
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})

3
dbms/programs/benchmark/Benchmark.cpp
View File

@ -32,6 +32,7 @@
#include <Client/Connection.h>
#include <Common/InterruptListener.h>
#include <Common/Config/configReadClient.h>
#include <Common/TerminalSize.h>
#include <Common/StudentTTest.h>
@ -504,7 +505,7 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv)
{
using boost::program_options::value;
boost::program_options::options_description desc("Allowed options");
boost::program_options::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth());
desc.add_options()
("help", "produce help message")
("concurrency,c", value<unsigned>()->default_value(1), "number of parallel queries")

28
dbms/programs/client/Client.cpp
View File

@ -67,6 +67,7 @@
#include <Common/Config/configReadClient.h>
#include <Storages/ColumnsDescription.h>
#include <common/argsToConfig.h>
#include <Common/TerminalSize.h>
#if USE_READLINE
#include "Suggest.h"
@ -130,7 +131,7 @@ private:
bool print_time_to_stderr = false; /// Output execution time to stderr in batch mode.
bool stdin_is_not_tty = false; /// stdin is not a terminal.
winsize terminal_size {}; /// Terminal size is needed to render progress bar.
uint16_t terminal_width = 0; /// Terminal width is needed to render progress bar.
std::unique_ptr<Connection> connection; /// Connection to DB.
String query_id; /// Current query_id.
@ -671,7 +672,7 @@ private:
String text;
if (config().has("query"))
text = config().getString("query");
text = config().getRawString("query"); /// Poco configuration should not process substitutions in form of ${...} inside query.
else
{
/// If 'query' parameter is not set, read a query from stdin.
@ -1465,7 +1466,7 @@ private:
if (show_progress_bar)
{
ssize_t width_of_progress_bar = static_cast<ssize_t>(terminal_size.ws_col) - written_progress_chars - strlen(" 99%");
ssize_t width_of_progress_bar = static_cast<ssize_t>(terminal_width) - written_progress_chars - strlen(" 99%");
if (width_of_progress_bar > 0)
{
std::string bar = UnicodeBar::render(UnicodeBar::getWidth(progress.read_rows, 0, total_rows_corrected, width_of_progress_bar));
@ -1642,22 +1643,13 @@ public:
stdin_is_not_tty = !isatty(STDIN_FILENO);
if (!stdin_is_not_tty)
terminal_width = getTerminalWidth();
namespace po = boost::program_options;
unsigned line_length = po::options_description::m_default_line_length;
unsigned min_description_length = line_length / 2;
if (!stdin_is_not_tty)
{
if (ioctl(STDIN_FILENO, TIOCGWINSZ, &terminal_size))
throwFromErrno("Cannot obtain terminal window size (ioctl TIOCGWINSZ)", ErrorCodes::SYSTEM_ERROR);
line_length = std::max(
static_cast<unsigned>(strlen("--http_native_compression_disable_checksumming_on_decompress ")),
static_cast<unsigned>(terminal_size.ws_col));
min_description_length = std::min(min_description_length, line_length - 2);
}
/// Main commandline options related to client functionality and all parameters from Settings.
po::options_description main_description("Main options", line_length, min_description_length);
po::options_description main_description = createOptionsDescription("Main options", terminal_width);
main_description.add_options()
("help", "produce help message")
("config-file,C", po::value<std::string>(), "config-file path")
@ -1672,7 +1664,7 @@ public:
* the "\n" is used to distinguish this case because there is hardly a chance an user would use "\n"
* as the password.
*/
("password", po::value<std::string>()->implicit_value("\n"), "password")
("password", po::value<std::string>()->implicit_value("\n", ""), "password")
("ask-password", "ask-password")
("query_id", po::value<std::string>(), "query_id")
("query,q", po::value<std::string>(), "query")
@ -1703,7 +1695,7 @@ public:
context.getSettingsRef().addProgramOptions(main_description);
/// Commandline options related to external tables.
po::options_description external_description("External tables options");
po::options_description external_description = createOptionsDescription("External tables options", terminal_width);
external_description.add_options()
("file", po::value<std::string>(), "data file or - for stdin")
("name", po::value<std::string>()->default_value("_data"), "name of the table")

5
dbms/programs/compressor/Compressor.cpp
View File

@ -12,8 +12,9 @@
#include <IO/copyData.h>
#include <Parsers/parseQuery.h>
#include <Parsers/ExpressionElementParsers.h>
#include <Compression/CompressionFactory.h>
#include <Common/TerminalSize.h>
namespace DB
{
@ -59,7 +60,7 @@ void checkAndWriteHeader(DB::ReadBuffer & in, DB::WriteBuffer & out)
int mainEntryClickHouseCompressor(int argc, char ** argv)
{
boost::program_options::options_description desc("Allowed options");
boost::program_options::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth());
desc.add_options()
("help,h", "produce help message")
("decompress,d", "decompress")

4
dbms/programs/format/Format.cpp
View File

@ -6,13 +6,13 @@
#include <Parsers/ParserQuery.h>
#include <Parsers/parseQuery.h>
#include <Parsers/formatAST.h>
#include <Common/TerminalSize.h>
int mainEntryClickHouseFormat(int argc, char ** argv)
{
using namespace DB;
boost::program_options::options_description desc("Allowed options");
boost::program_options::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth());
desc.add_options()
("help,h", "produce help message")
("hilite", "add syntax highlight with ANSI terminal escape sequences")

15
dbms/programs/local/LocalServer.cpp
View File

@ -35,6 +35,7 @@
#include <boost/program_options/options_description.hpp>
#include <boost/program_options.hpp>
#include <common/argsToConfig.h>
#include <Common/TerminalSize.h>
namespace DB
@ -267,7 +268,7 @@ void LocalServer::attachSystemTables()
void LocalServer::processQueries()
{
String initial_create_query = getInitialCreateTableQuery();
String queries_str = initial_create_query + config().getString("query");
String queries_str = initial_create_query + config().getRawString("query");
std::vector<String> queries;
auto parse_res = splitMultipartQuery(queries_str, queries);
@ -409,17 +410,7 @@ void LocalServer::init(int argc, char ** argv)
/// Don't parse options with Poco library, we prefer neat boost::program_options
stopOptionsProcessing();
unsigned line_length = po::options_description::m_default_line_length;
unsigned min_description_length = line_length / 2;
if (isatty(STDIN_FILENO))
{
winsize terminal_size{};
ioctl(0, TIOCGWINSZ, &terminal_size);
line_length = std::max(3U, static_cast<unsigned>(terminal_size.ws_col));
min_description_length = std::min(min_description_length, line_length - 2);
}
po::options_description description("Main options", line_length, min_description_length);
po::options_description description = createOptionsDescription("Main options", getTerminalWidth());
description.add_options()
("help", "produce help message")
("config-file,c", po::value<std::string>(), "config-file path")

3
dbms/programs/obfuscator/Obfuscator.cpp
View File

@ -37,6 +37,7 @@
#include <boost/program_options.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/container/flat_map.hpp>
#include <Common/TerminalSize.h>
static const char * documantation = R"(
@ -949,7 +950,7 @@ try
using namespace DB;
namespace po = boost::program_options;
po::options_description description("Options");
po::options_description description = createOptionsDescription("Options", getTerminalWidth());
description.add_options()
("help", "produce help message")
("structure,S", po::value<std::string>(), "structure of the initial table (list of column and type names)")

3
dbms/programs/performance-test/PerformanceTestSuite.cpp
View File

@ -28,6 +28,7 @@
#include <Core/Settings.h>
#include <Common/Exception.h>
#include <Common/InterruptListener.h>
#include <Common/TerminalSize.h>
#include "TestStopConditions.h"
#include "TestStats.h"
@ -324,7 +325,7 @@ try
using po::value;
using Strings = DB::Strings;
po::options_description desc("Allowed options");
po::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth());
desc.add_options()
("help", "produce help message")
("lite", "use lite version of output")

2
dbms/programs/server/Server.cpp
View File

@ -520,7 +520,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
/// Init trace collector only after trace_log system table was created
/// Disable it if we collect test coverage information, because it will work extremely slow.
#if USE_INTERNAL_UNWIND_LIBRARY && !WITH_COVERAGE
#if USE_UNWIND && !WITH_COVERAGE
/// QueryProfiler cannot work reliably with any other libunwind or without PHDR cache.
if (hasPHDRCache())
global_context->initializeTraceCollector();

2
dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h
View File

@ -90,7 +90,7 @@ public:
auto & set = this->data(place).value;
size_t size = set.size();
writeVarUInt(size, buf);
for (auto & elem : set)
for (const auto & elem : set)
writeIntBinary(elem, buf);
}

2
dbms/src/Client/ConnectionPool.h
View File

@ -74,7 +74,7 @@ public:
{
Entry entry;
if (settings)
entry = Base::get(settings->queue_max_wait_ms.totalMilliseconds());
entry = Base::get(settings->connection_pool_max_wait_ms.totalMilliseconds());
else
entry = Base::get(-1);

2
dbms/src/Columns/ColumnLowCardinality.cpp
View File

@ -34,7 +34,7 @@ namespace
auto & data = res_col->getData();
data.resize(hash_map.size());
for (auto val : hash_map)
for (const auto & val : hash_map)
data[val.getSecond()] = val.getFirst();
for (auto & ind : index)

10
dbms/src/Columns/ColumnTuple.cpp
View File

@ -81,6 +81,16 @@ MutableColumnPtr ColumnTuple::cloneEmpty() const
return ColumnTuple::create(std::move(new_columns));
}
MutableColumnPtr ColumnTuple::cloneResized(size_t new_size) const
{
const size_t tuple_size = columns.size();
MutableColumns new_columns(tuple_size);
for (size_t i = 0; i < tuple_size; ++i)
new_columns[i] = columns[i]->cloneResized(new_size);
return ColumnTuple::create(std::move(new_columns));
}
Field ColumnTuple::operator[](size_t n) const
{
return Tuple{ext::map<TupleBackend>(columns, [n] (const auto & column) { return (*column)[n]; })};

1
dbms/src/Columns/ColumnTuple.h
View File

@ -42,6 +42,7 @@ public:
const char * getFamilyName() const override { return "Tuple"; }
MutableColumnPtr cloneEmpty() const override;
MutableColumnPtr cloneResized(size_t size) const override;
size_t size() const override
{

2
dbms/src/Common/Config/CMakeLists.txt
View File

@ -4,5 +4,5 @@ add_headers_and_sources(clickhouse_common_config .)
add_library(clickhouse_common_config ${clickhouse_common_config_headers} ${clickhouse_common_config_sources})
target_link_libraries(clickhouse_common_config PUBLIC common PRIVATE clickhouse_common_zookeeper string_utils PUBLIC ${Poco_XML_LIBRARY} ${Poco_Util_LIBRARY} Threads::Threads)
target_link_libraries(clickhouse_common_config PUBLIC common PRIVATE clickhouse_common_zookeeper string_utils PUBLIC ${Poco_XML_LIBRARY} ${Poco_Util_LIBRARY})
target_include_directories(clickhouse_common_config PUBLIC ${DBMS_INCLUDE_DIR})

2
dbms/src/Common/HashTable/FixedHashMap.h
View File

@ -11,8 +11,8 @@ struct FixedHashMapCell
using State = TState;
using value_type = PairNoInit<Key, Mapped>;
bool full;
Mapped mapped;
bool full;
FixedHashMapCell() {}
FixedHashMapCell(const Key &, const State &) : full(true) {}

18
dbms/src/Common/HashTable/HashMap.h
View File

@ -128,14 +128,12 @@ struct HashMapCellWithSavedHash : public HashMapCell<Key, TMapped, Hash, TState>
};
template
<
template <
typename Key,
typename Cell,
typename Hash = DefaultHash<Key>,
typename Grower = HashTableGrower<>,
typename Allocator = HashTableAllocator
>
typename Allocator = HashTableAllocator>
class HashMapTable : public HashTable<Key, Cell, Hash, Grower, Allocator>
{
public:
@ -173,23 +171,19 @@ public:
};
template
<
template <
typename Key,
typename Mapped,
typename Hash = DefaultHash<Key>,
typename Grower = HashTableGrower<>,
typename Allocator = HashTableAllocator
>
typename Allocator = HashTableAllocator>
using HashMap = HashMapTable<Key, HashMapCell<Key, Mapped, Hash>, Hash, Grower, Allocator>;
template
<
template <
typename Key,
typename Mapped,
typename Hash = DefaultHash<Key>,
typename Grower = HashTableGrower<>,
typename Allocator = HashTableAllocator
>
typename Allocator = HashTableAllocator>
using HashMapWithSavedHash = HashMapTable<Key, HashMapCellWithSavedHash<Key, Mapped, Hash>, Hash, Grower, Allocator>;

1
dbms/src/Common/HashTable/HashTable.h
View File

@ -95,7 +95,6 @@ struct HashTableCell
/// Create a cell with the given key / key and value.
HashTableCell(const Key & key_, const State &) : key(key_) {}
/// HashTableCell(const value_type & value_, const State & state) : key(value_) {}
/// Get what the value_type of the container will be.
value_type & getValueMutable() { return key; }

4
dbms/src/Common/QueryProfiler.cpp
View File

@ -100,7 +100,7 @@ QueryProfilerBase<ProfilerImpl>::QueryProfilerBase(const Int32 thread_id, const
: log(&Logger::get("QueryProfiler"))
, pause_signal(pause_signal_)
{
#if USE_INTERNAL_UNWIND_LIBRARY
#if USE_UNWIND
/// Sanity check.
if (!hasPHDRCache())
throw Exception("QueryProfiler cannot be used without PHDR cache, that is not available for TSan build", ErrorCodes::NOT_IMPLEMENTED);
@ -173,7 +173,7 @@ QueryProfilerBase<ProfilerImpl>::~QueryProfilerBase()
template <typename ProfilerImpl>
void QueryProfilerBase<ProfilerImpl>::tryCleanup()
{
#if USE_INTERNAL_UNWIND_LIBRARY
#if USE_UNWIND
if (timer_id != nullptr && timer_delete(timer_id))
LOG_ERROR(log, "Failed to delete query profiler timer " + errnoToString(ErrorCodes::CANNOT_DELETE_TIMER));

3
dbms/src/Common/QueryProfiler.h
View File

@ -1,6 +1,7 @@
#pragma once
#include <Core/Types.h>
#include <Common/config.h>
#include <common/config_common.h>
#include <signal.h>
#include <time.h>
@ -43,7 +44,7 @@ private:
Poco::Logger * log;
#if USE_INTERNAL_UNWIND_LIBRARY
#if USE_UNWIND
/// Timer id from timer_create(2)
timer_t timer_id = nullptr;
#endif

27
dbms/src/Common/StackTrace.cpp
View File

@ -1,15 +1,20 @@
#include <common/SimpleCache.h>
#include <common/demangle.h>
#include <Common/config.h>
#include <Common/StackTrace.h>
#include <Common/SymbolIndex.h>
#include <Common/Dwarf.h>
#include <Common/Elf.h>
#include <sstream>
#include <filesystem>
#include <unordered_map>
#include <cstring>
#include <Common/SymbolIndex.h>
#include <Common/config.h>
#include <common/SimpleCache.h>
#include <common/demangle.h>
#include <cstring>
#include <filesystem>
#include <sstream>
#include <unordered_map>
#if USE_UNWIND
# include <libunwind.h>
#endif
std::string signalToErrorMessage(int sig, const siginfo_t & info, const ucontext_t & context)
{
@ -215,12 +220,6 @@ StackTrace::StackTrace(NoCapture)
{
}
#if USE_UNWIND
extern "C" int unw_backtrace(void **, int);
#endif
void StackTrace::tryCapture()
{
size = 0;

37
dbms/src/Common/TerminalSize.cpp Normal file
View File

@ -0,0 +1,37 @@
#include <unistd.h>
#include <sys/ioctl.h>
#include <Common/Exception.h>
#include <Common/TerminalSize.h>
#include <boost/program_options.hpp>
namespace DB::ErrorCodes
{
extern const int SYSTEM_ERROR;
}
uint16_t getTerminalWidth()
{
if (isatty(STDIN_FILENO))
{
winsize terminal_size {};
if (ioctl(STDIN_FILENO, TIOCGWINSZ, &terminal_size))
DB::throwFromErrno("Cannot obtain terminal window size (ioctl TIOCGWINSZ)", DB::ErrorCodes::SYSTEM_ERROR);
return terminal_size.ws_col;
}
return 0;
}
po::options_description createOptionsDescription(const std::string & caption, uint16_t terminal_width)
{
unsigned line_length = po::options_description::m_default_line_length;
unsigned min_description_length = line_length / 2;
std::string longest_option_desc = "--http_native_compression_disable_checksumming_on_decompress";
line_length = std::max(static_cast<uint16_t>(longest_option_desc.size()), terminal_width);
min_description_length = std::min(min_description_length, line_length - 2);
return po::options_description(caption, line_length, min_description_length);
}

16
dbms/src/Common/TerminalSize.h Normal file
View File

@ -0,0 +1,16 @@
#pragma once
#include <string>
#include <boost/program_options.hpp>
namespace po = boost::program_options;
uint16_t getTerminalWidth();
/** Creates po::options_description with name and an appropriate size for option displaying
* when program is called with option --help
* */
po::options_description createOptionsDescription(const std::string &caption, unsigned short terminal_width);

2
dbms/src/Common/ZooKeeper/CMakeLists.txt
View File

@ -4,7 +4,7 @@ add_headers_and_sources(clickhouse_common_zookeeper .)
add_library(clickhouse_common_zookeeper ${clickhouse_common_zookeeper_headers} ${clickhouse_common_zookeeper_sources})
target_link_libraries (clickhouse_common_zookeeper PUBLIC clickhouse_common_io common PRIVATE string_utils PUBLIC ${Poco_Util_LIBRARY} Threads::Threads)
target_link_libraries (clickhouse_common_zookeeper PUBLIC clickhouse_common_io common PRIVATE string_utils PUBLIC ${Poco_Util_LIBRARY})
target_include_directories(clickhouse_common_zookeeper PUBLIC ${DBMS_INCLUDE_DIR})
if (ENABLE_TESTS)

7
dbms/src/Core/Settings.h
View File

@ -61,7 +61,10 @@ struct Settings : public SettingsCollection<Settings>
M(SettingSeconds, receive_timeout, DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC, "") \
M(SettingSeconds, send_timeout, DBMS_DEFAULT_SEND_TIMEOUT_SEC, "") \
M(SettingSeconds, tcp_keep_alive_timeout, 0, "") \
M(SettingMilliseconds, queue_max_wait_ms, 5000, "The wait time in the request queue, if the number of concurrent requests exceeds the maximum.") \
M(SettingMilliseconds, queue_max_wait_ms, 0, "The wait time in the request queue, if the number of concurrent requests exceeds the maximum.") \
M(SettingMilliseconds, connection_pool_max_wait_ms, 0, "The wait time when connection pool is full.") \
M(SettingMilliseconds, replace_running_query_max_wait_ms, 5000, "The wait time for running query with the same query_id to finish when setting 'replace_running_query' is active.") \
M(SettingMilliseconds, kafka_max_wait_ms, 5000, "The wait time for reading from Kafka before retry.") \
M(SettingUInt64, poll_interval, DBMS_DEFAULT_POLL_INTERVAL, "Block at the query wait loop on the server for the specified number of seconds.") \
M(SettingUInt64, idle_connection_timeout, 3600, "Close idle TCP connections after specified number of seconds.") \
M(SettingUInt64, distributed_connections_pool_size, DBMS_DEFAULT_DISTRIBUTED_CONNECTIONS_POOL_SIZE, "Maximum number of connections with one remote server in the pool.") \
@ -302,7 +305,7 @@ struct Settings : public SettingsCollection<Settings>
M(SettingChar, format_csv_delimiter, ',', "The character to be considered as a delimiter in CSV data. If setting with a string, a string has to have a length of 1.") \
M(SettingBool, format_csv_allow_single_quotes, 1, "If it is set to true, allow strings in single quotes.") \
M(SettingBool, format_csv_allow_double_quotes, 1, "If it is set to true, allow strings in double quotes.") \
M(SettingBool, input_format_csv_unquoted_null_literal_as_null, false, "Consider unquoted NULL literal as \N") \
M(SettingBool, input_format_csv_unquoted_null_literal_as_null, false, "Consider unquoted NULL literal as \\N") \
\
M(SettingDateTimeInputFormat, date_time_input_format, FormatSettings::DateTimeInputFormat::Basic, "Method to read DateTime from text input formats. Possible values: 'basic' and 'best_effort'.") \
M(SettingBool, log_profile_events, true, "Log query performance statistics into the query_log and query_thread_log.") \

4
dbms/src/Core/SettingsCommon.h
View File

@ -695,7 +695,7 @@ public:
#define IMPLEMENT_SETTINGS_COLLECTION_ADD_MUTABLE_MEMBER_INFO_HELPER_(TYPE, NAME, DEFAULT, DESCRIPTION) \
add({[](const Derived & d) { return d.NAME.changed; }, \
StringRef(#NAME, strlen(#NAME)), StringRef(#DESCRIPTION, strlen(#DESCRIPTION)), true, \
StringRef(#NAME, strlen(#NAME)), StringRef(DESCRIPTION, strlen(DESCRIPTION)), true, \
&Functions::NAME##_getString, &Functions::NAME##_getField, \
&Functions::NAME##_setString, &Functions::NAME##_setField, \
&Functions::NAME##_serialize, &Functions::NAME##_deserialize, \
@ -703,7 +703,7 @@ public:
#define IMPLEMENT_SETTINGS_COLLECTION_ADD_IMMUTABLE_MEMBER_INFO_HELPER_(TYPE, NAME, DEFAULT, DESCRIPTION) \
add({[](const Derived & d) { return d.NAME.changed; }, \
StringRef(#NAME, strlen(#NAME)), StringRef(#DESCRIPTION, strlen(#DESCRIPTION)), false, \
StringRef(#NAME, strlen(#NAME)), StringRef(DESCRIPTION, strlen(DESCRIPTION)), false, \
&Functions::NAME##_getString, &Functions::NAME##_getField, \
&Functions::NAME##_setString, &Functions::NAME##_setField, \
&Functions::NAME##_serialize, &Functions::NAME##_deserialize, \

20
dbms/src/Databases/DatabaseDictionary.cpp
View File

@ -115,26 +115,6 @@ void DatabaseDictionary::removeTable(
throw Exception("DatabaseDictionary: removeTable() is not supported", ErrorCodes::NOT_IMPLEMENTED);
}
void DatabaseDictionary::renameTable(
const Context &,
const String &,
IDatabase &,
const String &)
{
throw Exception("DatabaseDictionary: renameTable() is not supported", ErrorCodes::NOT_IMPLEMENTED);
}
void DatabaseDictionary::alterTable(
const Context &,
const String &,
const ColumnsDescription &,
const IndicesDescription &,
const ConstraintsDescription &,
const ASTModifier &)
{
throw Exception("DatabaseDictionary: alterTable() is not supported", ErrorCodes::NOT_IMPLEMENTED);
}
time_t DatabaseDictionary::getTableMetadataModificationTime(
const Context &,
const String &)

14
dbms/src/Databases/DatabaseDictionary.h
View File

@ -60,20 +60,6 @@ public:
void attachTable(const String & table_name, const StoragePtr & table) override;
StoragePtr detachTable(const String & table_name) override;
void renameTable(
const Context & context,
const String & table_name,
IDatabase & to_database,
const String & to_table_name) override;
void alterTable(
const Context & context,
const String & name,
const ColumnsDescription & columns,
const IndicesDescription & indices,
const ConstraintsDescription & constraints,
const ASTModifier & engine_modifier) override;
time_t getTableMetadataModificationTime(
const Context & context,
const String & table_name) override;

1
dbms/src/Databases/DatabaseFactory.cpp
View File

@ -5,6 +5,7 @@
#include <Parsers/ASTLiteral.h>
#include <Parsers/formatAST.h>
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/ASTFunction.h>
#include <Common/parseAddress.h>
#include "config_core.h"
#if USE_MYSQL

20
dbms/src/Databases/DatabaseMemory.cpp
View File

@ -39,26 +39,6 @@ void DatabaseMemory::removeTable(
detachTable(table_name);
}
void DatabaseMemory::renameTable(
const Context &,
const String &,
IDatabase &,
const String &)
{
throw Exception("DatabaseMemory: renameTable() is not supported", ErrorCodes::NOT_IMPLEMENTED);
}
void DatabaseMemory::alterTable(
const Context &,
const String &,
const ColumnsDescription &,
const IndicesDescription &,
const ConstraintsDescription &,
const ASTModifier &)
{
throw Exception("DatabaseMemory: alterTable() is not supported", ErrorCodes::NOT_IMPLEMENTED);
}
time_t DatabaseMemory::getTableMetadataModificationTime(
const Context &,
const String &)

14
dbms/src/Databases/DatabaseMemory.h
View File

@ -37,20 +37,6 @@ public:
const Context & context,
const String & table_name) override;
void renameTable(
const Context & context,
const String & table_name,
IDatabase & to_database,
const String & to_table_name) override;
void alterTable(
const Context & context,
const String & name,
const ColumnsDescription & columns,
const IndicesDescription & indices,
const ConstraintsDescription & constraints,
const ASTModifier & engine_modifier) override;
time_t getTableMetadataModificationTime(
const Context & context,
const String & table_name) override;

12
dbms/src/Databases/DatabaseMySQL.h
View File

@ -5,6 +5,8 @@
#include <mysqlxx/Pool.h>
#include <Databases/DatabasesCommon.h>
#include <Interpreters/Context.h>
namespace DB
{
@ -61,21 +63,11 @@ public:
throw Exception("MySQL database engine does not support attach table.", ErrorCodes::NOT_IMPLEMENTED);
}
void renameTable(const Context &, const String &, IDatabase &, const String &) override
{
throw Exception("MySQL database engine does not support rename table.", ErrorCodes::NOT_IMPLEMENTED);
}
void createTable(const Context &, const String &, const StoragePtr &, const ASTPtr &) override
{
throw Exception("MySQL database engine does not support create table.", ErrorCodes::NOT_IMPLEMENTED);
}
void alterTable(const Context &, const String &, const ColumnsDescription &, const IndicesDescription &, const ConstraintsDescription &, const ASTModifier &) override
{
throw Exception("MySQL database engine does not support alter table.", ErrorCodes::NOT_IMPLEMENTED);
}
private:
struct MySQLStorageInfo
{

34
dbms/src/Databases/DatabaseOrdinary.cpp
View File

@ -135,7 +135,25 @@ void DatabaseOrdinary::loadTables(
if (endsWith(dir_it.name(), ".sql.bak"))
continue;
/// There are files .sql.tmp - delete.
// There are files that we tried to delete previously
static const char * tmp_drop_ext = ".sql.tmp_drop";
if (endsWith(dir_it.name(), tmp_drop_ext))
{
const std::string table_name = dir_it.name().substr(0, dir_it.name().size() - strlen(tmp_drop_ext));
if (Poco::File(data_path + '/' + table_name).exists())
{
Poco::File(dir_it->path()).renameTo(table_name + ".sql");
LOG_WARNING(log, "Table " << backQuote(table_name) << " was not dropped previously");
}
else
{
LOG_INFO(log, "Removing file " << dir_it->path());
Poco::File(dir_it->path()).remove();
}
continue;
}
/// There are files .sql.tmp - delete
if (endsWith(dir_it.name(), ".sql.tmp"))
{
LOG_INFO(log, "Removing file " << dir_it->path());
@ -302,6 +320,15 @@ void DatabaseOrdinary::removeTable(
}
catch (...)
{
try
{
Poco::File(table_metadata_path + ".tmp_drop").remove();
return;
}
catch (...)
{
LOG_WARNING(log, getCurrentExceptionMessage(__PRETTY_FUNCTION__));
}
attachTable(table_name, res);
throw;
}
@ -355,7 +382,8 @@ void DatabaseOrdinary::renameTable(
const Context & context,
const String & table_name,
IDatabase & to_database,
const String & to_table_name)
const String & to_table_name,
TableStructureWriteLockHolder & lock)
{
DatabaseOrdinary * to_database_concrete = typeid_cast<DatabaseOrdinary *>(&to_database);
@ -372,7 +400,7 @@ void DatabaseOrdinary::renameTable(
{
table->rename(context.getPath() + "/data/" + escapeForFileName(to_database_concrete->name) + "/",
to_database_concrete->name,
to_table_name);
to_table_name, lock);
}
catch (const Exception &)
{

4
dbms/src/Databases/DatabaseOrdinary.h
View File

@ -1,6 +1,7 @@
#pragma once
#include <Databases/DatabasesCommon.h>
#include <Common/ThreadPool.h>
namespace DB
@ -35,7 +36,8 @@ public:
const Context & context,
const String & table_name,
IDatabase & to_database,
const String & to_table_name) override;
const String & to_table_name,
TableStructureWriteLockHolder &) override;
void alterTable(
const Context & context,

1
dbms/src/Databases/DatabasesCommon.h
View File

@ -4,6 +4,7 @@
#include <Parsers/IAST.h>
#include <Storages/IStorage_fwd.h>
#include <Databases/IDatabase.h>
#include <mutex>
/// General functionality for several different database engines.

46
dbms/src/Databases/IDatabase.h
View File

@ -1,16 +1,9 @@
#pragma once
#include <Core/NamesAndTypes.h>
#include <Core/Types.h>
#include <Interpreters/Context.h>
#include <Parsers/IAST_fwd.h>
#include <Storages/ColumnsDescription.h>
#include <Storages/IndicesDescription.h>
#include <Storages/ConstraintsDescription.h>
#include <Storages/IStorage_fwd.h>
#include <Poco/File.h>
#include <Common/ThreadPool.h>
#include <Common/escapeForFileName.h>
#include <Common/Exception.h>
#include <ctime>
#include <functional>
@ -21,8 +14,16 @@ namespace DB
{
class Context;
struct Settings;
struct ConstraintsDescription;
class ColumnsDescription;
struct IndicesDescription;
struct TableStructureWriteLockHolder;
namespace ErrorCodes
{
extern const int NOT_IMPLEMENTED;
}
/** Allows to iterate over tables.
@ -102,22 +103,29 @@ public:
/// Rename the table and possibly move the table to another database.
virtual void renameTable(
const Context & context,
const String & name,
IDatabase & to_database,
const String & to_name) = 0;
const Context & /*context*/,
const String & /*name*/,
IDatabase & /*to_database*/,
const String & /*to_name*/,
TableStructureWriteLockHolder &)
{
throw Exception(getEngineName() + ": renameTable() is not supported", ErrorCodes::NOT_IMPLEMENTED);
}
using ASTModifier = std::function<void(IAST &)>;
/// Change the table structure in metadata.
/// You must call under the TableStructureLock of the corresponding table . If engine_modifier is empty, then engine does not change.
virtual void alterTable(
const Context & context,
const String & name,
const ColumnsDescription & columns,
const IndicesDescription & indices,
const ConstraintsDescription & constraints,
const ASTModifier & engine_modifier) = 0;
const Context & /*context*/,
const String & /*name*/,
const ColumnsDescription & /*columns*/,
const IndicesDescription & /*indices*/,
const ConstraintsDescription & /*constraints*/,
const ASTModifier & /*engine_modifier*/)
{
throw Exception(getEngineName() + ": renameTable() is not supported", ErrorCodes::NOT_IMPLEMENTED);
}
/// Returns time of table's metadata change, 0 if there is no corresponding metadata file.
virtual time_t getTableMetadataModificationTime(

2
dbms/src/Dictionaries/CMakeLists.txt
View File

@ -15,7 +15,7 @@ list(REMOVE_ITEM clickhouse_dictionaries_sources DictionaryFactory.cpp Dictionar
list(REMOVE_ITEM clickhouse_dictionaries_headers DictionaryFactory.h DictionarySourceFactory.h DictionaryStructure.h)
add_library(clickhouse_dictionaries ${clickhouse_dictionaries_sources})
target_link_libraries(clickhouse_dictionaries PRIVATE dbms clickhouse_common_io ${BTRIE_LIBRARIES} PUBLIC Threads::Threads)
target_link_libraries(clickhouse_dictionaries PRIVATE dbms clickhouse_common_io ${BTRIE_LIBRARIES})
if(Poco_SQL_FOUND AND NOT USE_INTERNAL_POCO_LIBRARY)
target_include_directories(clickhouse_dictionaries SYSTEM PRIVATE ${Poco_SQL_INCLUDE_DIR})

12
dbms/src/Interpreters/Aggregator.h
View File

@ -196,8 +196,6 @@ struct AggregationMethodString
using Data = TData;
using Key = typename Data::key_type;
using Mapped = typename Data::mapped_type;
using iterator = typename Data::iterator;
using const_iterator = typename Data::const_iterator;
Data data;
@ -224,8 +222,6 @@ struct AggregationMethodFixedString
using Data = TData;
using Key = typename Data::key_type;
using Mapped = typename Data::mapped_type;
using iterator = typename Data::iterator;
using const_iterator = typename Data::const_iterator;
Data data;
@ -254,8 +250,6 @@ struct AggregationMethodSingleLowCardinalityColumn : public SingleColumnMethod
using Data = typename Base::Data;
using Key = typename Base::Key;
using Mapped = typename Base::Mapped;
using iterator = typename Base::iterator;
using const_iterator = typename Base::const_iterator;
using Base::data;
@ -365,8 +359,6 @@ struct AggregationMethodSerialized
using Data = TData;
using Key = typename Data::key_type;
using Mapped = typename Data::mapped_type;
using iterator = typename Data::iterator;
using const_iterator = typename Data::const_iterator;
Data data;
@ -460,8 +452,8 @@ struct AggregatedDataVariants : private boost::noncopyable
std::unique_ptr<AggregationMethodKeysFixed<AggregatedDataWithKeys256TwoLevel, true>> nullable_keys256_two_level;
/// Support for low cardinality.
std::unique_ptr<AggregationMethodSingleLowCardinalityColumn<AggregationMethodOneNumber<UInt8, AggregatedDataWithNullableUInt8Key>>> low_cardinality_key8;
std::unique_ptr<AggregationMethodSingleLowCardinalityColumn<AggregationMethodOneNumber<UInt16, AggregatedDataWithNullableUInt16Key>>> low_cardinality_key16;
std::unique_ptr<AggregationMethodSingleLowCardinalityColumn<AggregationMethodOneNumber<UInt8, AggregatedDataWithNullableUInt8Key, false>>> low_cardinality_key8;
std::unique_ptr<AggregationMethodSingleLowCardinalityColumn<AggregationMethodOneNumber<UInt16, AggregatedDataWithNullableUInt16Key, false>>> low_cardinality_key16;
std::unique_ptr<AggregationMethodSingleLowCardinalityColumn<AggregationMethodOneNumber<UInt32, AggregatedDataWithNullableUInt64Key>>> low_cardinality_key32;
std::unique_ptr<AggregationMethodSingleLowCardinalityColumn<AggregationMethodOneNumber<UInt64, AggregatedDataWithNullableUInt64Key>>> low_cardinality_key64;
std::unique_ptr<AggregationMethodSingleLowCardinalityColumn<AggregationMethodString<AggregatedDataWithNullableStringKey>>> low_cardinality_key_string;

2
dbms/src/Interpreters/ClusterProxy/executeQuery.cpp
View File

@ -18,6 +18,8 @@ Context removeUserRestrictionsFromSettings(const Context & context, const Settin
{
Settings new_settings = settings;
new_settings.queue_max_wait_ms = Cluster::saturate(new_settings.queue_max_wait_ms, settings.max_execution_time);
new_settings.connection_pool_max_wait_ms = Cluster::saturate(new_settings.connection_pool_max_wait_ms, settings.max_execution_time);
new_settings.replace_running_query_max_wait_ms = Cluster::saturate(new_settings.replace_running_query_max_wait_ms, settings.max_execution_time);
/// Does not matter on remote servers, because queries are sent under different user.
new_settings.max_concurrent_queries_for_user = 0;

8
dbms/src/Interpreters/ExpressionActions.cpp
View File

@ -291,8 +291,8 @@ void ExpressionAction::prepare(Block & sample_block, const Settings & settings,
bool make_nullable = is_null_used_as_default && right_or_full_join;
if (make_nullable && !col.type->isNullable())
col.type = std::make_shared<DataTypeNullable>(col.type);
if (make_nullable && col.type->canBeInsideNullable())
col.type = makeNullable(col.type);
}
for (const auto & col : columns_added_by_join)
@ -316,8 +316,8 @@ void ExpressionAction::prepare(Block & sample_block, const Settings & settings,
}
}
if (make_nullable && !res_type->isNullable())
res_type = std::make_shared<DataTypeNullable>(res_type);
if (make_nullable && res_type->canBeInsideNullable())
res_type = makeNullable(res_type);
sample_block.insert(ColumnWithTypeAndName(nullptr, res_type, col.name));
}

33
dbms/src/Interpreters/InterpreterDropQuery.cpp
View File

@ -80,7 +80,7 @@ BlockIO InterpreterDropQuery::executeToTable(String & database_name_, String & t
/// If table was already dropped by anyone, an exception will be thrown
auto table_lock = database_and_table.second->lockExclusively(context.getCurrentQueryId());
/// Drop table data, don't touch metadata
database_and_table.second->truncate(query_ptr, context);
database_and_table.second->truncate(query_ptr, context, table_lock);
}
else if (kind == ASTDropQuery::Kind::Drop)
{
@ -90,11 +90,32 @@ BlockIO InterpreterDropQuery::executeToTable(String & database_name_, String & t
/// If table was already dropped by anyone, an exception will be thrown
auto table_lock = database_and_table.second->lockExclusively(context.getCurrentQueryId());
/// Delete table metadata and table itself from memory
const std::string metadata_file_without_extension =
database_and_table.first->getMetadataPath()
+ escapeForFileName(database_and_table.second->getTableName());
const auto prev_metadata_name = metadata_file_without_extension + ".sql";
const auto drop_metadata_name = metadata_file_without_extension + ".sql.tmp_drop";
/// Try to rename metadata file and delete the data
try
{
/// There some kind of tables that have no metadata - ignore renaming
if (Poco::File(prev_metadata_name).exists())
Poco::File(prev_metadata_name).renameTo(drop_metadata_name);
/// Delete table data
database_and_table.second->drop(table_lock);
}
catch (...)
{
if (Poco::File(drop_metadata_name).exists())
Poco::File(drop_metadata_name).renameTo(prev_metadata_name);
throw;
}
/// Delete table metadata and table itself from memory
database_and_table.first->removeTable(context, database_and_table.second->getTableName());
/// Delete table data
database_and_table.second->drop();
database_and_table.second->is_dropped = true;
String database_data_path = database_and_table.first->getDataPath();
@ -128,7 +149,7 @@ BlockIO InterpreterDropQuery::executeToTemporaryTable(String & table_name, ASTDr
/// If table was already dropped by anyone, an exception will be thrown
auto table_lock = table->lockExclusively(context.getCurrentQueryId());
/// Drop table data, don't touch metadata
table->truncate(query_ptr, context);
table->truncate(query_ptr, context, table_lock);
}
else if (kind == ASTDropQuery::Kind::Drop)
{
@ -137,7 +158,7 @@ BlockIO InterpreterDropQuery::executeToTemporaryTable(String & table_name, ASTDr
/// If table was already dropped by anyone, an exception will be thrown
auto table_lock = table->lockExclusively(context.getCurrentQueryId());
/// Delete table data
table->drop();
table->drop(table_lock);
table->is_dropped = true;
}
}

21
dbms/src/Interpreters/InterpreterRenameQuery.cpp
View File

@ -26,6 +26,8 @@ struct RenameDescription
to_table_name(elem.to.table)
{}
TableStructureWriteLockHolder from_table_lock;
String from_database_name;
String from_table_name;
@ -75,7 +77,7 @@ BlockIO InterpreterRenameQuery::execute()
}
};
std::set<UniqueTableName> unique_tables_from;
std::map<UniqueTableName, TableStructureWriteLockHolder> tables_from_locks;
/// Don't allow to drop tables (that we are renaming); don't allow to create tables in places where tables will be renamed.
std::map<UniqueTableName, std::unique_ptr<DDLGuard>> table_guards;
@ -87,7 +89,11 @@ BlockIO InterpreterRenameQuery::execute()
UniqueTableName from(descriptions.back().from_database_name, descriptions.back().from_table_name);
UniqueTableName to(descriptions.back().to_database_name, descriptions.back().to_table_name);
unique_tables_from.emplace(from);
if (!tables_from_locks.count(from))
if (auto table = context.tryGetTable(from.database_name, from.table_name))
tables_from_locks.emplace(from, table->lockExclusively(context.getCurrentQueryId()));
descriptions.back().from_table_lock = tables_from_locks[from];
if (!table_guards.count(from))
table_guards.emplace(from, context.getDDLGuard(from.database_name, from.table_name));
@ -96,13 +102,6 @@ BlockIO InterpreterRenameQuery::execute()
table_guards.emplace(to, context.getDDLGuard(to.database_name, to.table_name));
}
std::vector<TableStructureWriteLockHolder> locks;
locks.reserve(unique_tables_from.size());
for (const auto & names : unique_tables_from)
if (auto table = context.tryGetTable(names.database_name, names.table_name))
locks.emplace_back(table->lockExclusively(context.getCurrentQueryId()));
/** All tables are locked. If there are more than one rename in chain,
* we need to hold global lock while doing all renames. Order matters to avoid deadlocks.
* It provides atomicity of all RENAME chain as a whole, from the point of view of DBMS client,
@ -114,12 +113,12 @@ BlockIO InterpreterRenameQuery::execute()
if (descriptions.size() > 1)
lock = context.getLock();
for (const auto & elem : descriptions)
for (auto & elem : descriptions)
{
context.assertTableDoesntExist(elem.to_database_name, elem.to_table_name);
context.getDatabase(elem.from_database_name)->renameTable(
context, elem.from_table_name, *context.getDatabase(elem.to_database_name), elem.to_table_name);
context, elem.from_table_name, *context.getDatabase(elem.to_database_name), elem.to_table_name, elem.from_table_lock);
}
return {};

4
dbms/src/Interpreters/Join.cpp
View File

@ -50,7 +50,7 @@ static std::unordered_map<String, DataTypePtr> requiredRightKeys(const Names & k
static void convertColumnToNullable(ColumnWithTypeAndName & column)
{
if (column.type->isNullable())
if (column.type->isNullable() || !column.type->canBeInsideNullable())
return;
column.type = makeNullable(column.type);
@ -71,7 +71,7 @@ static ColumnWithTypeAndName correctNullability(ColumnWithTypeAndName && column,
if (nullable)
{
convertColumnToNullable(column);
if (negative_null_map.size())
if (column.type->isNullable() && negative_null_map.size())
{
MutableColumnPtr mutable_column = (*std::move(column.column)).mutate();
assert_cast<ColumnNullable &>(*mutable_column).applyNegatedNullMap(negative_null_map);

10
dbms/src/Interpreters/ProcessList.cpp
View File

@ -87,10 +87,10 @@ ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * as
{
std::unique_lock lock(mutex);
const auto max_wait_ms = settings.queue_max_wait_ms.totalMilliseconds();
const auto queue_max_wait_ms = settings.queue_max_wait_ms.totalMilliseconds();
if (!is_unlimited_query && max_size && processes.size() >= max_size)
{
if (!max_wait_ms || !have_space.wait_for(lock, std::chrono::milliseconds(max_wait_ms), [&]{ return processes.size() < max_size; }))
if (!queue_max_wait_ms || !have_space.wait_for(lock, std::chrono::milliseconds(queue_max_wait_ms), [&]{ return processes.size() < max_size; }))
throw Exception("Too many simultaneous queries. Maximum: " + toString(max_size), ErrorCodes::TOO_MANY_SIMULTANEOUS_QUERIES);
}
@ -127,7 +127,9 @@ ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * as
/// Ask queries to cancel. They will check this flag.
running_query->second->is_killed.store(true, std::memory_order_relaxed);
if (!max_wait_ms || !have_space.wait_for(lock, std::chrono::milliseconds(max_wait_ms), [&]
const auto replace_running_query_max_wait_ms = settings.replace_running_query_max_wait_ms.totalMilliseconds();
if (!replace_running_query_max_wait_ms || !have_space.wait_for(lock, std::chrono::milliseconds(replace_running_query_max_wait_ms),
[&]
{
running_query = user_process_list->second.queries.find(client_info.current_query_id);
if (running_query == user_process_list->second.queries.end())
@ -135,8 +137,10 @@ ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * as
running_query->second->is_killed.store(true, std::memory_order_relaxed);
return false;
}))
{
throw Exception("Query with id = " + client_info.current_query_id + " is already running and can't be stopped",
ErrorCodes::QUERY_WITH_SAME_ID_IS_ALREADY_RUNNING);
}
}
}
}

2
dbms/src/Interpreters/tests/CMakeLists.txt
View File

@ -57,5 +57,5 @@ target_link_libraries (users PRIVATE dbms clickhouse_common_config stdc++fs)
if (OS_LINUX)
add_executable (internal_iotop internal_iotop.cpp)
target_link_libraries (internal_iotop PRIVATE dbms Threads::Threads)
target_link_libraries (internal_iotop PRIVATE dbms)
endif ()

1
dbms/src/Storages/IStorage.cpp
View File

@ -22,6 +22,7 @@ namespace ErrorCodes
extern const int TYPE_MISMATCH;
extern const int SETTINGS_ARE_NOT_SUPPORTED;
extern const int UNKNOWN_SETTING;
extern const int TABLE_IS_DROPPED;
}
IStorage::IStorage(ColumnsDescription virtuals_) : virtuals(std::move(virtuals_))

12
dbms/src/Storages/IStorage.h
View File

@ -9,11 +9,13 @@
#include <Storages/SelectQueryInfo.h>
#include <Storages/TableStructureLockHolder.h>
#include <Storages/CheckResults.h>
#include <Storages/ColumnsDescription.h>
#include <Storages/IndicesDescription.h>
#include <Storages/ConstraintsDescription.h>
#include <Common/ActionLock.h>
#include <Common/Exception.h>
#include <Common/RWLock.h>
#include <Common/SettingsChanges.h>
#include <Storages/ConstraintsDescription.h>
#include <optional>
#include <shared_mutex>
@ -24,7 +26,6 @@ namespace DB
namespace ErrorCodes
{
extern const int TABLE_IS_DROPPED;
extern const int NOT_IMPLEMENTED;
}
@ -261,12 +262,12 @@ public:
* The table is not usable during and after call to this method.
* If you do not need any action other than deleting the directory with data, you can leave this method blank.
*/
virtual void drop() {}
virtual void drop(TableStructureWriteLockHolder &) {}
/** Clear the table data and leave it empty.
* Must be called under lockForAlter.
*/
virtual void truncate(const ASTPtr & /*query*/, const Context & /* context */)
virtual void truncate(const ASTPtr & /*query*/, const Context & /* context */, TableStructureWriteLockHolder &)
{
throw Exception("Truncate is not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED);
}
@ -276,7 +277,8 @@ public:
* In this function, you need to rename the directory with the data, if any.
* Called when the table structure is locked for write.
*/
virtual void rename(const String & /*new_path_to_db*/, const String & /*new_database_name*/, const String & /*new_table_name*/)
virtual void rename(const String & /*new_path_to_db*/, const String & /*new_database_name*/, const String & /*new_table_name*/,
TableStructureWriteLockHolder &)
{
throw Exception("Method rename is not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED);
}

2
dbms/src/Storages/Kafka/KafkaBlockInputStream.cpp
View File

@ -43,7 +43,7 @@ Block KafkaBlockInputStream::getHeader() const
void KafkaBlockInputStream::readPrefixImpl()
{
auto timeout = std::chrono::milliseconds(context.getSettingsRef().queue_max_wait_ms.totalMilliseconds());
auto timeout = std::chrono::milliseconds(context.getSettingsRef().kafka_max_wait_ms.totalMilliseconds());
buffer = storage.popReadBuffer(timeout);
claimed = !!buffer;

2
dbms/src/Storages/Kafka/KafkaBlockInputStream.h
View File

@ -4,6 +4,8 @@
#include <Interpreters/Context.h>
#include <Storages/Kafka/StorageKafka.h>
#include <Storages/Kafka/ReadBufferFromKafkaConsumer.h>
namespace DB
{

2
dbms/src/Storages/Kafka/StorageKafka.cpp
View File

@ -189,7 +189,7 @@ void StorageKafka::shutdown()
}
void StorageKafka::rename(const String & /* new_path_to_db */, const String & new_database_name, const String & new_table_name)
void StorageKafka::rename(const String & /* new_path_to_db */, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &)
{
table_name = new_table_name;
database_name = new_database_name;

9
dbms/src/Storages/Kafka/StorageKafka.h
View File

@ -1,8 +1,6 @@
#pragma once
#include <Core/BackgroundSchedulePool.h>
#include <Core/NamesAndTypes.h>
#include <DataStreams/IBlockOutputStream.h>
#include <Storages/IStorage.h>
#include <Storages/Kafka/ReadBufferFromKafkaConsumer.h>
#include <Storages/Kafka/WriteBufferToKafkaProducer.h>
@ -11,6 +9,8 @@
#include <ext/shared_ptr_helper.h>
#include <mutex>
#include <atomic>
namespace DB
{
@ -40,10 +40,9 @@ public:
BlockOutputStreamPtr write(
const ASTPtr & query,
const Context & context
) override;
const Context & context) override;
void rename(const String & /* new_path_to_db */, const String & new_database_name, const String & new_table_name) override;
void rename(const String & /* new_path_to_db */, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override;
void updateDependencies() override;

7
dbms/src/Storages/LiveView/LiveViewBlockInputStream.h
View File

@ -31,7 +31,12 @@ public:
const bool has_limit_, const UInt64 limit_,
const UInt64 heartbeat_interval_sec_,
const UInt64 temporary_live_view_timeout_sec_)
: storage(std::move(storage_)), blocks_ptr(std::move(blocks_ptr_)), blocks_metadata_ptr(std::move(blocks_metadata_ptr_)), active_ptr(std::move(active_ptr_)), has_limit(has_limit_), limit(limit_), heartbeat_interval_usec(heartbeat_interval_sec_ * 1000000), temporary_live_view_timeout_sec(temporary_live_view_timeout_sec_)
: storage(std::move(storage_)), blocks_ptr(std::move(blocks_ptr_)),
blocks_metadata_ptr(std::move(blocks_metadata_ptr_)),
active_ptr(std::move(active_ptr_)),
has_limit(has_limit_), limit(limit_),
heartbeat_interval_usec(heartbeat_interval_sec_ * 1000000),
temporary_live_view_timeout_sec(temporary_live_view_timeout_sec_)
{
/// grab active pointer
active = active_ptr.lock();

7
dbms/src/Storages/LiveView/LiveViewEventsBlockInputStream.h
View File

@ -51,7 +51,12 @@ public:
const bool has_limit_, const UInt64 limit_,
const UInt64 heartbeat_interval_sec_,
const UInt64 temporary_live_view_timeout_sec_)
: storage(std::move(storage_)), blocks_ptr(std::move(blocks_ptr_)), blocks_metadata_ptr(std::move(blocks_metadata_ptr_)), active_ptr(std::move(active_ptr_)), has_limit(has_limit_), limit(limit_), heartbeat_interval_usec(heartbeat_interval_sec_ * 1000000), temporary_live_view_timeout_sec(temporary_live_view_timeout_sec_)
: storage(std::move(storage_)), blocks_ptr(std::move(blocks_ptr_)),
blocks_metadata_ptr(std::move(blocks_metadata_ptr_)),
active_ptr(std::move(active_ptr_)), has_limit(has_limit_),
limit(limit_),
heartbeat_interval_usec(heartbeat_interval_sec_ * 1000000),
temporary_live_view_timeout_sec(temporary_live_view_timeout_sec_)
{
/// grab active pointer
active = active_ptr.lock();

65
dbms/src/Storages/LiveView/StorageLiveView.cpp
View File

@ -365,25 +365,25 @@ void StorageLiveView::checkTableCanBeDropped() const
}
}
void StorageLiveView::noUsersThread(const UInt64 & timeout)
void StorageLiveView::noUsersThread(std::shared_ptr<StorageLiveView> storage, const UInt64 & timeout)
{
if (shutdown_called)
return;
bool drop_table = false;
if (storage->shutdown_called)
return;
{
while (1)
{
std::unique_lock lock(no_users_thread_mutex);
if (!no_users_thread_condition.wait_for(lock, std::chrono::seconds(timeout), [&] { return no_users_thread_wakeup; }))
std::unique_lock lock(storage->no_users_thread_mutex);
if (!storage->no_users_thread_condition.wait_for(lock, std::chrono::seconds(timeout), [&] { return storage->no_users_thread_wakeup; }))
{
no_users_thread_wakeup = false;
if (shutdown_called)
storage->no_users_thread_wakeup = false;
if (storage->shutdown_called)
return;
if (hasUsers())
if (storage->hasUsers())
return;
if (!global_context.getDependencies(database_name, table_name).empty())
if (!storage->global_context.getDependencies(storage->database_name, storage->table_name).empty())
continue;
drop_table = true;
}
@ -393,17 +393,17 @@ void StorageLiveView::noUsersThread(const UInt64 & timeout)
if (drop_table)
{
if (global_context.tryGetTable(database_name, table_name))
if (storage->global_context.tryGetTable(storage->database_name, storage->table_name))
{
try
{
/// We create and execute `drop` query for this table
auto drop_query = std::make_shared<ASTDropQuery>();
drop_query->database = database_name;
drop_query->table = table_name;
drop_query->database = storage->database_name;
drop_query->table = storage->table_name;
drop_query->kind = ASTDropQuery::Kind::Drop;
ASTPtr ast_drop_query = drop_query;
InterpreterDropQuery drop_interpreter(ast_drop_query, global_context);
InterpreterDropQuery drop_interpreter(ast_drop_query, storage->global_context);
drop_interpreter.execute();
}
catch (...)
@ -419,9 +419,6 @@ void StorageLiveView::startNoUsersThread(const UInt64 & timeout)
if (!start_no_users_thread_called.compare_exchange_strong(expected, true))
return;
if (is_dropped)
return;
if (is_temporary)
{
if (no_users_thread.joinable())
@ -438,8 +435,10 @@ void StorageLiveView::startNoUsersThread(const UInt64 & timeout)
no_users_thread_wakeup = false;
}
if (!is_dropped)
no_users_thread = std::thread(&StorageLiveView::noUsersThread, this, timeout);
no_users_thread = std::thread(&StorageLiveView::noUsersThread,
std::static_pointer_cast<StorageLiveView>(shared_from_this()), timeout);
}
start_no_users_thread_called = false;
}
@ -456,22 +455,22 @@ void StorageLiveView::shutdown()
if (no_users_thread.joinable())
{
std::lock_guard lock(no_users_thread_mutex);
no_users_thread_wakeup = true;
no_users_thread_condition.notify_one();
/// Must detach the no users thread
/// as we can't join it as it will result
/// in a deadlock
no_users_thread.detach(); /// TODO Not viable at all.
{
std::lock_guard lock(no_users_thread_mutex);
no_users_thread_wakeup = true;
no_users_thread_condition.notify_one();
}
}
}
StorageLiveView::~StorageLiveView()
{
shutdown();
if (no_users_thread.joinable())
no_users_thread.detach();
}
void StorageLiveView::drop()
void StorageLiveView::drop(TableStructureWriteLockHolder &)
{
global_context.removeDependency(
DatabaseAndTableName(select_database_name, select_table_name),
@ -534,8 +533,11 @@ BlockInputStreams StorageLiveView::watch(
if (query.is_watch_events)
{
auto reader = std::make_shared<LiveViewEventsBlockInputStream>(std::static_pointer_cast<StorageLiveView>(shared_from_this()), blocks_ptr, blocks_metadata_ptr, active_ptr, has_limit, limit, context.getSettingsRef().live_view_heartbeat_interval.totalSeconds(),
context.getSettingsRef().temporary_live_view_timeout.totalSeconds());
auto reader = std::make_shared<LiveViewEventsBlockInputStream>(
std::static_pointer_cast<StorageLiveView>(shared_from_this()),
blocks_ptr, blocks_metadata_ptr, active_ptr, has_limit, limit,
context.getSettingsRef().live_view_heartbeat_interval.totalSeconds(),
context.getSettingsRef().temporary_live_view_timeout.totalSeconds());
if (no_users_thread.joinable())
{
@ -559,8 +561,11 @@ BlockInputStreams StorageLiveView::watch(
}
else
{
auto reader = std::make_shared<LiveViewBlockInputStream>(std::static_pointer_cast<StorageLiveView>(shared_from_this()), blocks_ptr, blocks_metadata_ptr, active_ptr, has_limit, limit, context.getSettingsRef().live_view_heartbeat_interval.totalSeconds(),
context.getSettingsRef().temporary_live_view_timeout.totalSeconds());
auto reader = std::make_shared<LiveViewBlockInputStream>(
std::static_pointer_cast<StorageLiveView>(shared_from_this()),
blocks_ptr, blocks_metadata_ptr, active_ptr, has_limit, limit,
context.getSettingsRef().live_view_heartbeat_interval.totalSeconds(),
context.getSettingsRef().temporary_live_view_timeout.totalSeconds());
if (no_users_thread.joinable())
{

17
dbms/src/Storages/LiveView/StorageLiveView.h
View File

@ -71,11 +71,10 @@ public:
{
return active_ptr.use_count() > 1;
}
/// Background thread for temporary tables
/// which drops this table if there are no users
/// No users thread mutex, predicate and wake up condition
void startNoUsersThread(const UInt64 & timeout);
std::mutex no_users_thread_mutex;
bool no_users_thread_wakeup{false};
bool no_users_thread_wakeup = false;
std::condition_variable no_users_thread_condition;
/// Get blocks hash
/// must be called with mutex locked
@ -105,7 +104,7 @@ public:
}
void checkTableCanBeDropped() const override;
void drop() override;
void drop(TableStructureWriteLockHolder &) override;
void startup() override;
void shutdown() override;
@ -149,7 +148,7 @@ private:
String database_name;
ASTPtr inner_query;
Context & global_context;
bool is_temporary {false};
bool is_temporary = false;
mutable Block sample_block;
/// Mutex for the blocks and ready condition
@ -166,10 +165,12 @@ private:
std::shared_ptr<BlocksMetadataPtr> blocks_metadata_ptr;
BlocksPtrs mergeable_blocks;
void noUsersThread(const UInt64 & timeout);
/// Background thread for temporary tables
/// which drops this table if there are no users
static void noUsersThread(std::shared_ptr<StorageLiveView> storage, const UInt64 & timeout);
std::thread no_users_thread;
std::atomic<bool> shutdown_called{false};
std::atomic<bool> start_no_users_thread_called{false};
std::atomic<bool> shutdown_called = false;
std::atomic<bool> start_no_users_thread_called = false;
UInt64 temporary_live_view_timeout;
StorageLiveView(

4
dbms/src/Storages/MergeTree/MergeTreeData.cpp
View File

@ -1690,7 +1690,7 @@ void MergeTreeData::removeEmptyColumnsFromPart(MergeTreeData::MutableDataPartPtr
empty_columns.clear();
}
void MergeTreeData::freezeAll(const String & with_name, const Context & context)
void MergeTreeData::freezeAll(const String & with_name, const Context & context, TableStructureReadLockHolder &)
{
freezePartitionsByMatcher([] (const DataPartPtr &){ return true; }, with_name, context);
}
@ -2550,7 +2550,7 @@ void MergeTreeData::removePartContributionToColumnSizes(const DataPartPtr & part
}
void MergeTreeData::freezePartition(const ASTPtr & partition_ast, const String & with_name, const Context & context)
void MergeTreeData::freezePartition(const ASTPtr & partition_ast, const String & with_name, const Context & context, TableStructureReadLockHolder &)
{
std::optional<String> prefix;
String partition_id;

4
dbms/src/Storages/MergeTree/MergeTreeData.h
View File

@ -549,7 +549,7 @@ public:
void removeEmptyColumnsFromPart(MergeTreeData::MutableDataPartPtr & data_part);
/// Freezes all parts.
void freezeAll(const String & with_name, const Context & context);
void freezeAll(const String & with_name, const Context & context, TableStructureReadLockHolder & table_lock_holder);
/// Should be called if part data is suspected to be corrupted.
void reportBrokenPart(const String & name) const
@ -577,7 +577,7 @@ public:
* Backup is created in directory clickhouse_dir/shadow/i/, where i - incremental number,
* or if 'with_name' is specified - backup is created in directory with specified name.
*/
void freezePartition(const ASTPtr & partition, const String & with_name, const Context & context);
void freezePartition(const ASTPtr & partition, const String & with_name, const Context & context, TableStructureReadLockHolder & table_lock_holder);
size_t getColumnCompressedSize(const std::string & name) const
{

6
dbms/src/Storages/StorageBuffer.h
View File

@ -73,7 +73,11 @@ public:
void shutdown() override;
bool optimize(const ASTPtr & query, const ASTPtr & partition, bool final, bool deduplicate, const Context & context) override;
void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name) override { table_name = new_table_name; database_name = new_database_name; }
void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override
{
table_name = new_table_name;
database_name = new_database_name;
}
bool supportsSampling() const override { return true; }
bool supportsPrewhere() const override

1
dbms/src/Storages/StorageDictionary.h
View File

@ -34,7 +34,6 @@ public:
size_t max_block_size = DEFAULT_BLOCK_SIZE,
unsigned threads = 1) override;
void drop() override {}
static NamesAndTypesList getNamesAndTypes(const DictionaryStructure & dictionary_structure);
template <typename ForwardIterator>

2
dbms/src/Storages/StorageDistributed.cpp
View File

@ -415,7 +415,7 @@ void StorageDistributed::shutdown()
}
void StorageDistributed::truncate(const ASTPtr &, const Context &)
void StorageDistributed::truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &)
{
std::lock_guard lock(cluster_nodes_mutex);

11
dbms/src/Storages/StorageDistributed.h
View File

@ -77,12 +77,17 @@ public:
BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override;
void drop() override {}
void drop(TableStructureWriteLockHolder &) override {}
/// Removes temporary data in local filesystem.
void truncate(const ASTPtr &, const Context &) override;
void truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) override;
void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override
{
table_name = new_table_name;
database_name = new_database_name;
}
void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name) override { table_name = new_table_name; database_name = new_database_name; }
/// in the sub-tables, you need to manually add and delete columns
/// the structure of the sub-table is not checked
void alter(

8
dbms/src/Storages/StorageFile.cpp
View File

@ -264,13 +264,7 @@ BlockOutputStreamPtr StorageFile::write(
}
void StorageFile::drop()
{
/// Extra actions are not required.
}
void StorageFile::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name)
void StorageFile::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &)
{
if (!is_db_table)
throw Exception("Can't rename table '" + table_name + "' binded to user-defined file (or FD)", ErrorCodes::DATABASE_ACCESS_DENIED);

4
dbms/src/Storages/StorageFile.h
View File

@ -38,9 +38,7 @@ public:
const ASTPtr & query,
const Context & context) override;
void drop() override;
void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override;
void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override;
String getDataPath() const override { return path; }

2
dbms/src/Storages/StorageHDFS.cpp
View File

@ -148,7 +148,7 @@ BlockInputStreams StorageHDFS::read(
max_block_size)};
}
void StorageHDFS::rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name)
void StorageHDFS::rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &)
{
table_name = new_table_name;
database_name = new_database_name;

2
dbms/src/Storages/StorageHDFS.h
View File

@ -30,7 +30,7 @@ public:
BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override;
void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override;
void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override;
protected:
StorageHDFS(const String & uri_,

2
dbms/src/Storages/StorageJoin.cpp
View File

@ -55,7 +55,7 @@ StorageJoin::StorageJoin(
}
void StorageJoin::truncate(const ASTPtr &, const Context &)
void StorageJoin::truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &)
{
Poco::File(path).remove(true);
Poco::File(path).createDirectories();

2
dbms/src/Storages/StorageJoin.h
View File

@ -26,7 +26,7 @@ class StorageJoin : public ext::shared_ptr_helper<StorageJoin>, public StorageSe
public:
String getName() const override { return "Join"; }
void truncate(const ASTPtr &, const Context &) override;
void truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) override;
/// Access the innards.
JoinPtr & getJoin() { return join; }

4
dbms/src/Storages/StorageLog.cpp
View File

@ -512,7 +512,7 @@ void StorageLog::loadMarks()
}
void StorageLog::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name)
void StorageLog::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &)
{
std::unique_lock<std::shared_mutex> lock(rwlock);
@ -530,7 +530,7 @@ void StorageLog::rename(const String & new_path_to_db, const String & new_databa
marks_file = Poco::File(path + escapeForFileName(table_name) + '/' + DBMS_STORAGE_LOG_MARKS_FILE_NAME);
}
void StorageLog::truncate(const ASTPtr &, const Context &)
void StorageLog::truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &)
{
std::shared_lock<std::shared_mutex> lock(rwlock);

4
dbms/src/Storages/StorageLog.h
View File

@ -38,11 +38,11 @@ public:
BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override;
void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override;
void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override;
CheckResults checkData(const ASTPtr & /* query */, const Context & /* context */) override;
void truncate(const ASTPtr &, const Context &) override;
void truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) override;
std::string full_path() const { return path + escapeForFileName(table_name) + '/';}

7
dbms/src/Storages/StorageMaterializedView.cpp
View File

@ -232,7 +232,7 @@ static void executeDropQuery(ASTDropQuery::Kind kind, Context & global_context,
}
void StorageMaterializedView::drop()
void StorageMaterializedView::drop(TableStructureWriteLockHolder &)
{
global_context.removeDependency(
DatabaseAndTableName(select_database_name, select_table_name),
@ -242,7 +242,7 @@ void StorageMaterializedView::drop()
executeDropQuery(ASTDropQuery::Kind::Drop, global_context, target_database_name, target_table_name);
}
void StorageMaterializedView::truncate(const ASTPtr &, const Context &)
void StorageMaterializedView::truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &)
{
if (has_inner_table)
executeDropQuery(ASTDropQuery::Kind::Truncate, global_context, target_database_name, target_table_name);
@ -299,7 +299,8 @@ static void executeRenameQuery(Context & global_context, const String & database
}
void StorageMaterializedView::rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name)
void StorageMaterializedView::rename(
const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &)
{
if (has_inner_table && tryGetTargetTable())
{

6
dbms/src/Storages/StorageMaterializedView.h
View File

@ -33,9 +33,9 @@ public:
BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override;
void drop() override;
void drop(TableStructureWriteLockHolder &) override;
void truncate(const ASTPtr &, const Context &) override;
void truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) override;
bool optimize(const ASTPtr & query, const ASTPtr & partition, bool final, bool deduplicate, const Context & context) override;
@ -43,7 +43,7 @@ public:
void mutate(const MutationCommands & commands, const Context & context) override;
void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override;
void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override;
void shutdown() override;

4
dbms/src/Storages/StorageMemory.cpp
View File

@ -123,13 +123,13 @@ BlockOutputStreamPtr StorageMemory::write(
}
void StorageMemory::drop()
void StorageMemory::drop(TableStructureWriteLockHolder &)
{
std::lock_guard lock(mutex);
data.clear();
}
void StorageMemory::truncate(const ASTPtr &, const Context &)
void StorageMemory::truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &)
{
std::lock_guard lock(mutex);
data.clear();

10
dbms/src/Storages/StorageMemory.h
View File

@ -40,11 +40,15 @@ public:
BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override;
void drop() override;
void drop(TableStructureWriteLockHolder &) override;
void truncate(const ASTPtr &, const Context &) override;
void truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) override;
void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name) override { table_name = new_table_name; database_name = new_database_name; }
void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override
{
table_name = new_table_name;
database_name = new_database_name;
}
private:
String database_name;

7
dbms/src/Storages/StorageMerge.h
View File

@ -42,8 +42,11 @@ public:
size_t max_block_size,
unsigned num_streams) override;
void drop() override {}
void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name) override { table_name = new_table_name; database_name = new_database_name; }
void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override
{
table_name = new_table_name;
database_name = new_database_name;
}
/// you need to add and remove columns in the sub-tables manually
/// the structure of sub-tables is not checked

10
dbms/src/Storages/StorageMergeTree.cpp
View File

@ -157,13 +157,13 @@ void StorageMergeTree::checkPartitionCanBeDropped(const ASTPtr & partition)
global_context.checkPartitionCanBeDropped(database_name, table_name, partition_size);
}
void StorageMergeTree::drop()
void StorageMergeTree::drop(TableStructureWriteLockHolder &)
{
shutdown();
dropAllData();
}
void StorageMergeTree::truncate(const ASTPtr &, const Context &)
void StorageMergeTree::truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &)
{
{
/// Asks to complete merges and does not allow them to start.
@ -181,7 +181,7 @@ void StorageMergeTree::truncate(const ASTPtr &, const Context &)
clearOldPartsFromFilesystem();
}
void StorageMergeTree::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name)
void StorageMergeTree::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &)
{
std::string new_full_path = new_path_to_db + escapeForFileName(new_table_name) + '/';
@ -995,7 +995,7 @@ void StorageMergeTree::alterPartition(const ASTPtr & query, const PartitionComma
case PartitionCommand::FREEZE_PARTITION:
{
auto lock = lockStructureForShare(false, context.getCurrentQueryId());
freezePartition(command.partition, command.with_name, context);
freezePartition(command.partition, command.with_name, context, lock);
}
break;
@ -1020,7 +1020,7 @@ void StorageMergeTree::alterPartition(const ASTPtr & query, const PartitionComma
case PartitionCommand::FREEZE_ALL_PARTITIONS:
{
auto lock = lockStructureForShare(false, context.getCurrentQueryId());
freezeAll(command.with_name, context);
freezeAll(command.with_name, context, lock);
}
break;

6
dbms/src/Storages/StorageMergeTree.h
View File

@ -55,10 +55,10 @@ public:
std::vector<MergeTreeMutationStatus> getMutationsStatus() const override;
CancellationCode killMutation(const String & mutation_id) override;
void drop() override;
void truncate(const ASTPtr &, const Context &) override;
void drop(TableStructureWriteLockHolder &) override;
void truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) override;
void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override;
void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override;
void alter(const AlterCommands & params, const Context & context, TableStructureWriteLockHolder & table_lock_holder) override;

2
dbms/src/Storages/StorageNull.h
View File

@ -38,7 +38,7 @@ public:
return std::make_shared<NullBlockOutputStream>(getSampleBlock());
}
void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name) override
void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override
{
table_name = new_table_name;
database_name = new_database_name;

14
dbms/src/Storages/StorageReplicatedMergeTree.cpp
View File

@ -3132,7 +3132,6 @@ bool StorageReplicatedMergeTree::optimize(const ASTPtr & query, const ASTPtr & p
if (query_context.getSettingsRef().replication_alter_partitions_sync != 0)
{
/// NOTE Table lock must not be held while waiting. Some combination of R-W-R locks from different threads will yield to deadlock.
/// TODO Check all other "wait" places.
for (auto & merge_entry : merge_entries)
waitForAllReplicasToProcessLogEntry(merge_entry);
}
@ -3484,7 +3483,7 @@ void StorageReplicatedMergeTree::alterPartition(const ASTPtr & query, const Part
case PartitionCommand::FREEZE_PARTITION:
{
auto lock = lockStructureForShare(false, query_context.getCurrentQueryId());
freezePartition(command.partition, command.with_name, query_context);
freezePartition(command.partition, command.with_name, query_context, lock);
}
break;
@ -3509,7 +3508,7 @@ void StorageReplicatedMergeTree::alterPartition(const ASTPtr & query, const Part
case PartitionCommand::FREEZE_ALL_PARTITIONS:
{
auto lock = lockStructureForShare(false, query_context.getCurrentQueryId());
freezeAll(command.with_name, query_context);
freezeAll(command.with_name, query_context, lock);
}
break;
}
@ -3633,8 +3632,10 @@ void StorageReplicatedMergeTree::dropPartition(const ASTPtr & query, const ASTPt
}
void StorageReplicatedMergeTree::truncate(const ASTPtr & query, const Context & query_context)
void StorageReplicatedMergeTree::truncate(const ASTPtr & query, const Context & query_context, TableStructureWriteLockHolder & table_lock)
{
table_lock.release(); /// Truncate is done asynchronously.
assertNotReadonly();
zkutil::ZooKeeperPtr zookeeper = getZooKeeper();
@ -3701,7 +3702,7 @@ void StorageReplicatedMergeTree::checkPartitionCanBeDropped(const ASTPtr & parti
}
void StorageReplicatedMergeTree::drop()
void StorageReplicatedMergeTree::drop(TableStructureWriteLockHolder &)
{
{
auto zookeeper = tryGetZooKeeper();
@ -3731,7 +3732,8 @@ void StorageReplicatedMergeTree::drop()
}
void StorageReplicatedMergeTree::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name)
void StorageReplicatedMergeTree::rename(
const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &)
{
std::string new_full_path = new_path_to_db + escapeForFileName(new_table_name) + '/';

6
dbms/src/Storages/StorageReplicatedMergeTree.h
View File

@ -109,11 +109,11 @@ public:
/** Removes a replica from ZooKeeper. If there are no other replicas, it deletes the entire table from ZooKeeper.
*/
void drop() override;
void drop(TableStructureWriteLockHolder &) override;
void truncate(const ASTPtr &, const Context &) override;
void truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) override;
void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override;
void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override;
bool supportsIndexForIn() const override { return true; }

5
dbms/src/Storages/StorageSet.cpp
View File

@ -126,7 +126,7 @@ void StorageSet::insertBlock(const Block & block) { set->insertFromBlock(block);
size_t StorageSet::getSize() const { return set->getTotalRowCount(); }
void StorageSet::truncate(const ASTPtr &, const Context &)
void StorageSet::truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &)
{
Poco::File(path).remove(true);
Poco::File(path).createDirectories();
@ -193,7 +193,8 @@ void StorageSetOrJoinBase::restoreFromFile(const String & file_path)
}
void StorageSetOrJoinBase::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name)
void StorageSetOrJoinBase::rename(
const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &)
{
/// Rename directory with data.
String new_path = new_path_to_db + escapeForFileName(new_table_name);

4
dbms/src/Storages/StorageSet.h
View File

@ -22,7 +22,7 @@ public:
String getTableName() const override { return table_name; }
String getDatabaseName() const override { return database_name; }
void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override;
void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override;
BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override;
@ -69,7 +69,7 @@ public:
/// Access the insides.
SetPtr & getSet() { return set; }
void truncate(const ASTPtr &, const Context &) override;
void truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) override;
private:
SetPtr set;

4
dbms/src/Storages/StorageStripeLog.cpp
View File

@ -223,7 +223,7 @@ StorageStripeLog::StorageStripeLog(
}
void StorageStripeLog::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name)
void StorageStripeLog::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &)
{
std::unique_lock<std::shared_mutex> lock(rwlock);
@ -294,7 +294,7 @@ CheckResults StorageStripeLog::checkData(const ASTPtr & /* query */, const Conte
return file_checker.check();
}
void StorageStripeLog::truncate(const ASTPtr &, const Context &)
void StorageStripeLog::truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &)
{
if (table_name.empty())
throw Exception("Logical error: table name is empty", ErrorCodes::LOGICAL_ERROR);

4
dbms/src/Storages/StorageStripeLog.h
View File

@ -40,7 +40,7 @@ public:
BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override;
void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override;
void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override;
CheckResults checkData(const ASTPtr & /* query */, const Context & /* context */) override;
@ -55,7 +55,7 @@ public:
String getDataPath() const override { return full_path(); }
void truncate(const ASTPtr &, const Context &) override;
void truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) override;
private:
String path;

4
dbms/src/Storages/StorageTinyLog.cpp
View File

@ -378,7 +378,7 @@ void StorageTinyLog::addFiles(const String & column_name, const IDataType & type
}
void StorageTinyLog::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name)
void StorageTinyLog::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &)
{
std::unique_lock<std::shared_mutex> lock(rwlock);
@ -424,7 +424,7 @@ CheckResults StorageTinyLog::checkData(const ASTPtr & /* query */, const Context
return file_checker.check();
}
void StorageTinyLog::truncate(const ASTPtr &, const Context &)
void StorageTinyLog::truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &)
{
if (table_name.empty())
throw Exception("Logical error: table name is empty", ErrorCodes::LOGICAL_ERROR);

4
dbms/src/Storages/StorageTinyLog.h
View File

@ -39,7 +39,7 @@ public:
BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override;
void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override;
void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override;
CheckResults checkData(const ASTPtr & /* query */, const Context & /* context */) override;
@ -54,7 +54,7 @@ public:
String getDataPath() const override { return full_path(); }
void truncate(const ASTPtr &, const Context &) override;
void truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) override;
private:
String path;

Some files were not shown because too many files have changed in this diff Show More