Merge branch 'master' into merging_values_with_expressions

.github/ISSUE_TEMPLATE/bug-report-or-unexpected-behaviour.md

@@ -17,7 +17,7 @@ A clear and concise description of what works not as it is supposed to.
 * Which interface to use, if matters
 * Non-default settings, if any
 * `CREATE TABLE` statements for all tables involved
-* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/yandex/ClickHouse/blob/master/dbms/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
+* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/dbms/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
 * Queries to run that lead to unexpected result
 
 **Expected behavior**

.github/ISSUE_TEMPLATE/performance-issue.md

@@ -17,7 +17,7 @@ What exactly works slower than expected?
 * Which interface to use, if matters
 * Non-default settings, if any
 * `CREATE TABLE` statements for all tables involved
-* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/yandex/ClickHouse/blob/master/dbms/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
+* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/dbms/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
 * Queries to run that lead to slow performance
 
 **Expected performance**

CMakeLists.txt

@@ -1,4 +1,3 @@
-project(ClickHouse)
 cmake_minimum_required(VERSION 3.3)
 
 foreach(policy
@@ -13,6 +12,9 @@ foreach(policy
     endif()
 endforeach()
 
+project(ClickHouse)
+include (cmake/target.cmake)
+
 # Ignore export() since we don't use it,
 # but it gets broken with a global targets via link_libraries()
 macro (export)
@@ -41,28 +43,12 @@ else()
     message(STATUS "IPO/LTO not enabled.")
 endif()
 
-if (COMPILER_GCC)
-    # Require minimum version of gcc
-    set (GCC_MINIMUM_VERSION 8)
-    if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${GCC_MINIMUM_VERSION} AND NOT CMAKE_VERSION VERSION_LESS 2.8.9)
-        message (FATAL_ERROR "GCC version must be at least ${GCC_MINIMUM_VERSION}. For example, if GCC ${GCC_MINIMUM_VERSION} is available under gcc-${GCC_MINIMUM_VERSION}, g++-${GCC_MINIMUM_VERSION} names, do the following: export CC=gcc-${GCC_MINIMUM_VERSION} CXX=g++-${GCC_MINIMUM_VERSION}; rm -rf CMakeCache.txt CMakeFiles; and re run cmake or ./release.")
-    endif ()
-elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
-    # Require minimum version of clang
-    set (CLANG_MINIMUM_VERSION 7)
-    if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${CLANG_MINIMUM_VERSION})
-        message (FATAL_ERROR "Clang version must be at least ${CLANG_MINIMUM_VERSION}.")
-    endif ()
-else ()
-    message (WARNING "You are using an unsupported compiler. Compilation has only been tested with Clang 6+ and GCC 7+.")
-endif ()
-
 # Check that submodules are present only if source was downloaded with git
 if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git" AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/boost/boost")
     message (FATAL_ERROR "Submodules are not initialized. Run\n\tgit submodule update --init --recursive")
 endif ()
 
-include (cmake/find_ccache.cmake)
+include (cmake/find/ccache.cmake)
 
 if (NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "None")
     message (STATUS "CMAKE_BUILD_TYPE is not set, set to default = RELWITHDEBINFO")
@@ -91,6 +77,14 @@ if (USE_STATIC_LIBRARIES)
     list(REVERSE CMAKE_FIND_LIBRARY_SUFFIXES)
 endif ()
 
+option (ENABLE_FUZZING "Enables fuzzing instrumentation" OFF)
+
+if (ENABLE_FUZZING)
+    message (STATUS "Fuzzing instrumentation enabled")
+    set (WITH_COVERAGE ON)
+    set (SANITIZE "libfuzzer")
+endif()
+
 include (cmake/sanitize.cmake)
 
 
@@ -136,23 +130,6 @@ if (CMAKE_SYSTEM_PROCESSOR MATCHES "amd64|x86_64")
     endif ()
 endif ()
 
-string(REGEX MATCH "-?[0-9]+(.[0-9]+)?$" COMPILER_POSTFIX ${CMAKE_CXX_COMPILER})
-
-find_program (LLD_PATH NAMES "lld${COMPILER_POSTFIX}" "lld")
-find_program (GOLD_PATH NAMES "gold")
-
-if (COMPILER_CLANG AND LLD_PATH AND NOT LINKER_NAME)
-    set (LINKER_NAME "lld")
-elseif (GOLD_PATH)
-    set (LINKER_NAME "gold")
-endif ()
-
-if (LINKER_NAME)
-    message(STATUS "Using linker: ${LINKER_NAME} (selected from: LLD_PATH=${LLD_PATH}; GOLD_PATH=${GOLD_PATH}; COMPILER_POSTFIX=${COMPILER_POSTFIX})")
-    set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}")
-    set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}")
-endif ()
-
 # Make sure the final executable has symbols exported
 set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -rdynamic")
 
@@ -187,7 +164,7 @@ else()
 endif()
 
 if(NOT DISABLE_CPU_OPTIMIZE)
-    include(cmake/test_cpu.cmake)
+    include(cmake/cpu_features.cmake)
 endif()
 
 if(NOT COMPILER_CLANG) # clang: error: the clang compiler does not support '-march=native'
@@ -198,16 +175,9 @@ if (ARCH_NATIVE)
     set (COMPILER_FLAGS "${COMPILER_FLAGS} -march=native")
 endif ()
 
-if (CMAKE_VERSION VERSION_LESS "3.8.0")
+set (CMAKE_CXX_STANDARD 17)
-    if (NOT MSVC)
+set (CMAKE_CXX_EXTENSIONS 0) # https://cmake.org/cmake/help/latest/prop_tgt/CXX_EXTENSIONS.html#prop_tgt:CXX_EXTENSIONS
-        set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17")
+set (CMAKE_CXX_STANDARD_REQUIRED ON)
-    endif ()
-else ()
-    set (CMAKE_CXX_STANDARD 17)
-    set (CMAKE_CXX_EXTENSIONS 0) # https://cmake.org/cmake/help/latest/prop_tgt/CXX_EXTENSIONS.html#prop_tgt:CXX_EXTENSIONS
-    set (CMAKE_CXX_STANDARD_REQUIRED ON)
-    set (CXX_FLAGS_INTERNAL_COMPILER "-std=c++17")
-endif ()
 
 if (COMPILER_GCC OR COMPILER_CLANG)
     # Enable C++14 sized global deallocation functions. It should be enabled by setting -std=c++14 but I'm not sure.
@@ -249,13 +219,19 @@ if (UNBUNDLED OR NOT (OS_LINUX OR APPLE) OR ARCH_32)
 endif ()
 
 # Make this extra-checks for correct library dependencies.
-if (NOT SANITIZE)
+if (OS_LINUX AND NOT SANITIZE)
     set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-undefined")
     set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-undefined")
 endif ()
 
 include(cmake/dbms_glob_sources.cmake)
-include(cmake/default_libs.cmake)
+
+if (OS_LINUX)
+    include(cmake/linux/default_libs.cmake)
+elseif (OS_DARWIN)
+    include(cmake/darwin/sdk.cmake)
+    include(cmake/darwin/default_libs.cmake)
+endif ()
 
 ######################################
 ### Add targets below this comment ###
@@ -313,49 +289,49 @@ endif ()
 message (STATUS "Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_LIBRARY_ARCHITECTURE} ; USE_STATIC_LIBRARIES=${USE_STATIC_LIBRARIES} MAKE_STATIC_LIBRARIES=${MAKE_STATIC_LIBRARIES} SPLIT_SHARED=${SPLIT_SHARED_LIBRARIES} UNBUNDLED=${UNBUNDLED} CCACHE=${CCACHE_FOUND} ${CCACHE_VERSION}")
 
 include(GNUInstallDirs)
-include (cmake/find_contrib_lib.cmake)
+include (cmake/contrib_finder.cmake)
 include (cmake/lib_name.cmake)
 
 find_contrib_lib(double-conversion) # Must be before parquet
-include (cmake/find_ssl.cmake)
+include (cmake/find/ssl.cmake)
-include (cmake/find_icu.cmake)
+include (cmake/find/icu.cmake)
-include (cmake/find_boost.cmake)
+include (cmake/find/boost.cmake)
-include (cmake/find_zlib.cmake)
+include (cmake/find/zlib.cmake)
-include (cmake/find_zstd.cmake)
+include (cmake/find/zstd.cmake)
-include (cmake/find_ltdl.cmake) # for odbc
+include (cmake/find/ltdl.cmake) # for odbc
-include (cmake/find_termcap.cmake)
+include (cmake/find/termcap.cmake)
-include (cmake/find_odbc.cmake)
+include (cmake/find/odbc.cmake)
 # openssl, zlib, odbc before poco
-include (cmake/find_poco.cmake)
+include (cmake/find/poco.cmake)
-include (cmake/find_lz4.cmake)
+include (cmake/find/lz4.cmake)
-include (cmake/find_xxhash.cmake)
+include (cmake/find/xxhash.cmake)
-include (cmake/find_sparsehash.cmake)
+include (cmake/find/sparsehash.cmake)
-include (cmake/find_rt.cmake)
+include (cmake/find/rt.cmake)
-include (cmake/find_execinfo.cmake)
+include (cmake/find/execinfo.cmake)
-include (cmake/find_readline_edit.cmake)
+include (cmake/find/readline_edit.cmake)
-include (cmake/find_re2.cmake)
+include (cmake/find/re2.cmake)
-include (cmake/find_libgsasl.cmake)
+include (cmake/find/libgsasl.cmake)
-include (cmake/find_rdkafka.cmake)
+include (cmake/find/rdkafka.cmake)
-include (cmake/find_capnp.cmake)
+include (cmake/find/capnp.cmake)
-include (cmake/find_llvm.cmake)
+include (cmake/find/llvm.cmake)
-include (cmake/find_h3.cmake)
+include (cmake/find/h3.cmake)
-include (cmake/find_cpuid.cmake) # Freebsd, bundled
+include (cmake/find/cpuid.cmake) # Freebsd, bundled
 if (NOT USE_CPUID)
-    include (cmake/find_cpuinfo.cmake) # Debian
+    include (cmake/find/cpuinfo.cmake) # Debian
 endif()
-include (cmake/find_libxml2.cmake)
+include (cmake/find/libxml2.cmake)
-include (cmake/find_brotli.cmake)
+include (cmake/find/brotli.cmake)
-include (cmake/find_protobuf.cmake)
+include (cmake/find/protobuf.cmake)
-include (cmake/find_pdqsort.cmake)
+include (cmake/find/pdqsort.cmake)
-include (cmake/find_hdfs3.cmake) # uses protobuf
+include (cmake/find/hdfs3.cmake) # uses protobuf
-include (cmake/find_consistent-hashing.cmake)
+include (cmake/find/consistent-hashing.cmake)
-include (cmake/find_base64.cmake)
+include (cmake/find/base64.cmake)
-include (cmake/find_parquet.cmake)
+include (cmake/find/parquet.cmake)
-include (cmake/find_hyperscan.cmake)
+include (cmake/find/hyperscan.cmake)
-include (cmake/find_simdjson.cmake)
+include (cmake/find/simdjson.cmake)
-include (cmake/find_rapidjson.cmake)
+include (cmake/find/rapidjson.cmake)
-include (cmake/find_fastops.cmake)
+include (cmake/find/fastops.cmake)
-#include (cmake/find_orc.cmake)
+include (cmake/find/orc.cmake)
 
 find_contrib_lib(cityhash)
 find_contrib_lib(farmhash)
@@ -363,7 +339,7 @@ find_contrib_lib(metrohash)
 find_contrib_lib(btrie)
 
 if (ENABLE_TESTS)
-    include (cmake/find_gtest.cmake)
+    include (cmake/find/gtest.cmake)
 endif ()
 
 # Need to process before "contrib" dir:
@@ -372,7 +348,7 @@ include (libs/libcommon/cmake/find_jemalloc.cmake)
 include (libs/libcommon/cmake/find_cctz.cmake)
 include (libs/libmysqlxx/cmake/find_mysqlclient.cmake)
 
-# When testing for memory leaks with Valgrind, dont link tcmalloc or jemalloc.
+# When testing for memory leaks with Valgrind, don't link tcmalloc or jemalloc.
 
 if (USE_JEMALLOC)
     message (STATUS "Link jemalloc: ${JEMALLOC_LIBRARIES}")
@@ -391,7 +367,7 @@ elseif (USE_TCMALLOC)
     endif ()
 elseif (SANITIZE)
     message (STATUS "Will use ${SANITIZE} sanitizer.")
-else ()
+elseif (OS_LINUX)
     message (WARNING "Non default allocator is disabled. This is not recommended for production Linux builds.")
 endif ()
 
@@ -400,6 +376,22 @@ include (cmake/print_flags.cmake)
 install (EXPORT global DESTINATION cmake)
 
 add_subdirectory (contrib EXCLUDE_FROM_ALL)
+
+macro (add_executable target)
+    # invoke built-in add_executable
+    _add_executable (${ARGV})
+    get_target_property (type ${target} TYPE)
+    if (${type} STREQUAL EXECUTABLE)
+        file (RELATIVE_PATH dir ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR})
+        if (${dir} MATCHES "^dbms")
+            # Only interpose operator::new/delete for dbms executables (MemoryTracker stuff)
+            target_link_libraries (${target} PRIVATE clickhouse_new_delete ${MALLOC_LIBRARIES})
+        else ()
+            target_link_libraries (${target} PRIVATE ${MALLOC_LIBRARIES})
+        endif ()
+    endif()
+endmacro()
+
 add_subdirectory (libs)
 add_subdirectory (utils)
 add_subdirectory (dbms)

README.md

@@ -1,4 +1,4 @@
-[![ClickHouse — open source distributed column-oriented DBMS](https://github.com/yandex/ClickHouse/raw/master/website/images/logo-400x240.png)](https://clickhouse.yandex)
+[![ClickHouse — open source distributed column-oriented DBMS](https://github.com/ClickHouse/ClickHouse/raw/master/website/images/logo-400x240.png)](https://clickhouse.yandex)
 
 ClickHouse is an open-source column-oriented database management system that allows generating analytical data reports in real time.
 
@@ -14,11 +14,11 @@ ClickHouse is an open-source column-oriented database management system that all
 
 ## Upcoming Events
 * [ClickHouse Meetup in Paris](https://www.eventbrite.com/e/clickhouse-paris-meetup-2019-registration-68493270215) on October 3.
-* [ClickHouse Meetup in San Francisco](https://www.meetup.com/San-Francisco-Bay-Area-ClickHouse-Meetup/events/264242199/) on October 9.
+* [ClickHouse Meetup in San Francisco](https://www.meetup.com/San-Francisco-Bay-Area-ClickHouse-Meetup/events/264242199/) on October 9.
-* [ClickHouse Meetup in Hong Kong](https://www.meetup.com/Hong-Kong-Machine-Learning-Meetup/events/263580542/) on October 17.
+* [ClickHouse Meetup in Hong Kong](https://www.meetup.com/Hong-Kong-Machine-Learning-Meetup/events/263580542/) on October 17.
 * [ClickHouse Meetup in Shenzhen](https://www.huodongxing.com/event/3483759917300) on October 20.
 * [ClickHouse Meetup in Shanghai](https://www.huodongxing.com/event/4483760336000) on October 27.
 * [ClickHouse Meetup in Tokyo](https://clickhouse.connpass.com/event/147001/) on November 14.
 * [ClickHouse Meetup in Istanbul](https://www.eventbrite.com/e/clickhouse-meetup-istanbul-create-blazing-fast-experiences-w-clickhouse-tickets-73101120419) on November 19.
 * [ClickHouse Meetup in Ankara](https://www.eventbrite.com/e/clickhouse-meetup-ankara-create-blazing-fast-experiences-w-clickhouse-tickets-73100530655) on November 21.
-
+* [ClickHouse Meetup in Singapore](https://www.meetup.com/Singapore-Clickhouse-Meetup-Group/events/265085331/) on November 23.

SECURITY.md

@@ -14,4 +14,4 @@ currently being supported with security updates:
 ## Reporting a Vulnerability
 
 To report a potential vulnerability in ClickHouse please use the security advisory feature of GitHub:
-https://github.com/yandex/ClickHouse/security/advisories
+https://github.com/ClickHouse/ClickHouse/security/advisories

cmake/arch.cmake

@@ -12,19 +12,6 @@ if ((ARCH_ARM AND NOT ARCH_AARCH64) OR ARCH_I386)
     message (FATAL_ERROR "32bit platforms are not supported")
 endif ()
 
-if (CMAKE_SYSTEM MATCHES "Linux")
-    set (OS_LINUX 1)
-endif ()
-if (CMAKE_SYSTEM MATCHES "FreeBSD")
-    set (OS_FREEBSD 1)
-endif ()
-
-if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-    set (COMPILER_GCC 1)
-elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
-    set (COMPILER_CLANG 1)
-endif ()
-
 if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(ppc64le.*|PPC64LE.*)")
     set (ARCH_PPC64LE 1)
     if (COMPILER_CLANG OR (COMPILER_GCC AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8))

cmake/darwin/default_libs.cmake

@@ -0,0 +1,37 @@
+set (DEFAULT_LIBS "-nodefaultlibs")
+
+if (NOT COMPILER_CLANG)
+    message (FATAL_ERROR "Darwin build is supported only for Clang")
+endif ()
+
+set (DEFAULT_LIBS "${DEFAULT_LIBS} ${COVERAGE_OPTION} -lc -lm -lpthread -ldl")
+
+message(STATUS "Default libraries: ${DEFAULT_LIBS}")
+
+set(CMAKE_CXX_STANDARD_LIBRARIES ${DEFAULT_LIBS})
+set(CMAKE_C_STANDARD_LIBRARIES ${DEFAULT_LIBS})
+
+# Global libraries
+
+add_library(global-libs INTERFACE)
+
+# Unfortunately '-pthread' doesn't work with '-nodefaultlibs'.
+# Just make sure we have pthreads at all.
+set(THREADS_PREFER_PTHREAD_FLAG ON)
+find_package(Threads REQUIRED)
+
+include (cmake/find/cxx.cmake)
+
+add_library(global-group INTERFACE)
+
+target_link_libraries(global-group INTERFACE
+    $<TARGET_PROPERTY:global-libs,INTERFACE_LINK_LIBRARIES>
+)
+
+link_libraries(global-group)
+
+# FIXME: remove when all contribs will get custom cmake lists
+install(
+    TARGETS global-group global-libs
+    EXPORT global
+)

cmake/darwin/sdk.cmake

@@ -0,0 +1,11 @@
+option (SDK_PATH "Path to the SDK to build with" "")
+
+if (NOT EXISTS "${SDK_PATH}/SDKSettings.plist")
+    message (FATAL_ERROR "Wrong SDK path provided: ${SDK_PATH}")
+endif ()
+
+set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -isysroot ${SDK_PATH} -mmacosx-version-min=10.14")
+set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -isysroot ${SDK_PATH} -mmacosx-version-min=10.14")
+
+set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -isysroot ${SDK_PATH} -mmacosx-version-min=10.14")
+set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -isysroot ${SDK_PATH} -mmacosx-version-min=10.14")

cmake/find/cxx.cmake

@@ -1,25 +1,11 @@
-if (OS_LINUX AND COMPILER_CLANG)
+if (COMPILER_CLANG)
     option (USE_LIBCXX "Use libc++ and libc++abi instead of libstdc++" ON)
     option (USE_INTERNAL_LIBCXX_LIBRARY "Set to FALSE to use system libcxx and libcxxabi libraries instead of bundled" ${NOT_UNBUNDLED})
 endif()
 
 if (USE_LIBCXX)
     set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_LIBCPP_DEBUG=0") # More checks in debug build.
-endif ()
 
-# FIXME: make better check for submodule presence
-if (USE_INTERNAL_LIBCXX_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libcxx/include/vector")
-    message (WARNING "submodule contrib/libcxx is missing. to fix try run: \n git submodule update --init --recursive")
-    set (USE_INTERNAL_LIBCXX_LIBRARY 0)
-endif ()
-
-# FIXME: make better check for submodule presence
-if (USE_INTERNAL_LIBCXX_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libcxxabi/src")
-    message (WARNING "submodule contrib/libcxxabi is missing. to fix try run: \n git submodule update --init --recursive")
-    set (USE_INTERNAL_LIBCXX_LIBRARY 0)
-endif ()
-
-if (USE_LIBCXX)
     if (NOT USE_INTERNAL_LIBCXX_LIBRARY)
         find_library (LIBCXX_LIBRARY c++)
         find_library (LIBCXXFS_LIBRARY c++fs)

cmake/find/parquet.cmake

@@ -21,7 +21,7 @@ endif()
 
 if(ARROW_INCLUDE_DIR AND PARQUET_INCLUDE_DIR)
 elseif(NOT MISSING_INTERNAL_PARQUET_LIBRARY AND NOT OS_FREEBSD)
-    include(cmake/find_snappy.cmake)
+    include(cmake/find/snappy.cmake)
     set(CAN_USE_INTERNAL_PARQUET_LIBRARY 1)
     include(CheckCXXSourceCompiles)
     if(NOT USE_INTERNAL_DOUBLE_CONVERSION_LIBRARY)

cmake/find/poco.cmake

@@ -18,6 +18,9 @@ if (NOT DEFINED ENABLE_POCO_MONGODB OR ENABLE_POCO_MONGODB)
 else ()
     set(ENABLE_POCO_MONGODB 0 CACHE BOOL "")
 endif ()
+if (NOT DEFINED ENABLE_POCO_REDIS OR ENABLE_POCO_REDIS)
+    list (APPEND POCO_COMPONENTS Redis)
+endif ()
 # TODO: after new poco release with SQL library rename ENABLE_POCO_ODBC -> ENABLE_POCO_SQLODBC
 if (NOT DEFINED ENABLE_POCO_ODBC OR ENABLE_POCO_ODBC)
     list (APPEND POCO_COMPONENTS DataODBC)
@@ -35,7 +38,6 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY)
     set (ENABLE_ZIP 0 CACHE BOOL "")
     set (ENABLE_PAGECOMPILER 0 CACHE BOOL "")
     set (ENABLE_PAGECOMPILER_FILE2PAGE 0 CACHE BOOL "")
-    set (ENABLE_REDIS 0 CACHE BOOL "")
     set (ENABLE_DATA_SQLITE 0 CACHE BOOL "")
     set (ENABLE_DATA_MYSQL 0 CACHE BOOL "")
     set (ENABLE_DATA_POSTGRESQL 0 CACHE BOOL "")
@@ -46,7 +48,6 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY)
     set (POCO_ENABLE_ZIP 0 CACHE BOOL "")
     set (POCO_ENABLE_PAGECOMPILER 0 CACHE BOOL "")
     set (POCO_ENABLE_PAGECOMPILER_FILE2PAGE 0 CACHE BOOL "")
-    set (POCO_ENABLE_REDIS 0 CACHE BOOL "")
     set (POCO_ENABLE_SQL_SQLITE 0 CACHE BOOL "")
     set (POCO_ENABLE_SQL_MYSQL 0 CACHE BOOL "")
     set (POCO_ENABLE_SQL_POSTGRESQL 0 CACHE BOOL "")
@@ -69,6 +70,11 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY)
         set (Poco_MongoDB_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/poco/MongoDB/include/")
     endif ()
 
+    if (NOT DEFINED ENABLE_POCO_REDIS OR ENABLE_POCO_REDIS)
+        set (Poco_Redis_LIBRARY PocoRedis)
+        set (Poco_Redis_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/poco/Redis/include/")
+    endif ()
+
     if (EXISTS "${ClickHouse_SOURCE_DIR}/contrib/poco/SQL/ODBC/include/")
         set (Poco_SQL_FOUND 1)
         set (Poco_SQL_LIBRARY PocoSQL)
@@ -122,6 +128,9 @@ endif ()
 if (Poco_MongoDB_LIBRARY)
     set (USE_POCO_MONGODB 1)
 endif ()
+if (Poco_Redis_LIBRARY)
+    set (USE_POCO_REDIS 1)
+endif ()
 if (Poco_DataODBC_LIBRARY AND ODBC_FOUND)
     set (USE_POCO_DATAODBC 1)
 endif ()
@@ -129,7 +138,7 @@ if (Poco_SQLODBC_LIBRARY AND ODBC_FOUND)
     set (USE_POCO_SQLODBC 1)
 endif ()
 
-message(STATUS "Using Poco: ${Poco_INCLUDE_DIRS} : ${Poco_Foundation_LIBRARY},${Poco_Util_LIBRARY},${Poco_Net_LIBRARY},${Poco_NetSSL_LIBRARY},${Poco_Crypto_LIBRARY},${Poco_XML_LIBRARY},${Poco_Data_LIBRARY},${Poco_DataODBC_LIBRARY},${Poco_SQL_LIBRARY},${Poco_SQLODBC_LIBRARY},${Poco_MongoDB_LIBRARY}; MongoDB=${USE_POCO_MONGODB}, DataODBC=${USE_POCO_DATAODBC}, NetSSL=${USE_POCO_NETSSL}")
+message(STATUS "Using Poco: ${Poco_INCLUDE_DIRS} : ${Poco_Foundation_LIBRARY},${Poco_Util_LIBRARY},${Poco_Net_LIBRARY},${Poco_NetSSL_LIBRARY},${Poco_Crypto_LIBRARY},${Poco_XML_LIBRARY},${Poco_Data_LIBRARY},${Poco_DataODBC_LIBRARY},${Poco_SQL_LIBRARY},${Poco_SQLODBC_LIBRARY},${Poco_MongoDB_LIBRARY},${Poco_Redis_LIBRARY}; MongoDB=${USE_POCO_MONGODB}, Redis=${USE_POCO_REDIS}, DataODBC=${USE_POCO_DATAODBC}, NetSSL=${USE_POCO_NETSSL}")
 
 # How to make sutable poco:
 # use branch:

cmake/find/snappy.cmake

@@ -0,0 +1,13 @@
+option(USE_SNAPPY "Enable support of snappy library" ON)
+
+if (USE_SNAPPY)
+    option (USE_INTERNAL_SNAPPY_LIBRARY "Set to FALSE to use system snappy library instead of bundled" ${NOT_UNBUNDLED})
+
+    if(NOT USE_INTERNAL_SNAPPY_LIBRARY)
+        find_library(SNAPPY_LIBRARY snappy)
+    else ()
+        set(SNAPPY_LIBRARY snappy)
+    endif()
+
+    message (STATUS "Using snappy: ${SNAPPY_LIBRARY}")
+endif ()

cmake/find_snappy.cmake

@@ -1,27 +0,0 @@
-option(USE_INTERNAL_SNAPPY_LIBRARY "Set to FALSE to use system snappy library instead of bundled" ${NOT_UNBUNDLED})
-
-if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/snappy/snappy.h")
-   if(USE_INTERNAL_SNAPPY_LIBRARY)
-       message(WARNING "submodule contrib/snappy is missing. to fix try run: \n git submodule update --init --recursive")
-       set(USE_INTERNAL_SNAPPY_LIBRARY 0)
-    endif()
-    set(MISSING_INTERNAL_SNAPPY_LIBRARY 1)
-endif()
-
-if(NOT USE_INTERNAL_SNAPPY_LIBRARY)
-    find_library(SNAPPY_LIBRARY snappy)
-    find_path(SNAPPY_INCLUDE_DIR NAMES snappy.h PATHS ${SNAPPY_INCLUDE_PATHS})
-endif()
-
-if(SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
-elseif(NOT MISSING_INTERNAL_SNAPPY_LIBRARY)
-    set(SNAPPY_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/snappy)
-    set(USE_INTERNAL_SNAPPY_LIBRARY 1)
-    set(SNAPPY_LIBRARY snappy)
-endif()
-
-if(SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
-    set(USE_SNAPPY 1)
-endif()
-
-message(STATUS "Using snappy=${USE_SNAPPY}: ${SNAPPY_INCLUDE_DIR} : ${SNAPPY_LIBRARY}")

cmake/linux/default_libs.cmake

@@ -3,20 +3,18 @@
 
 set (DEFAULT_LIBS "-nodefaultlibs")
 
-if (OS_LINUX)
+# We need builtins from Clang's RT even without libcxx - for ubsan+int128.
-    # We need builtins from Clang's RT even without libcxx - for ubsan+int128.
+# See https://bugs.llvm.org/show_bug.cgi?id=16404
-    # See https://bugs.llvm.org/show_bug.cgi?id=16404
+if (COMPILER_CLANG)
-    if (COMPILER_CLANG)
+    execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-${CMAKE_SYSTEM_PROCESSOR}.a OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE)
-        execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-${CMAKE_SYSTEM_PROCESSOR}.a OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE)
+else ()
-    else ()
+    set (BUILTINS_LIBRARY "-lgcc")
-        set (BUILTINS_LIBRARY "-lgcc")
-    endif ()
-
-    set (DEFAULT_LIBS "${DEFAULT_LIBS} ${BUILTINS_LIBRARY} ${COVERAGE_OPTION} -lc -lm -lrt -lpthread -ldl")
-
-    message(STATUS "Default libraries: ${DEFAULT_LIBS}")
 endif ()
 
+set (DEFAULT_LIBS "${DEFAULT_LIBS} ${BUILTINS_LIBRARY} ${COVERAGE_OPTION} -lc -lm -lrt -lpthread -ldl")
+
+message(STATUS "Default libraries: ${DEFAULT_LIBS}")
+
 set(CMAKE_CXX_STANDARD_LIBRARIES ${DEFAULT_LIBS})
 set(CMAKE_C_STANDARD_LIBRARIES ${DEFAULT_LIBS})
 
@@ -30,8 +28,8 @@ set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
 
 add_subdirectory(libs/libglibc-compatibility)
-include (cmake/find_unwind.cmake)
+include (cmake/find/unwind.cmake)
-include (cmake/find_cxx.cmake)
+include (cmake/find/cxx.cmake)
 
 add_library(global-group INTERFACE)
 target_link_libraries(global-group INTERFACE
@@ -42,6 +40,7 @@ target_link_libraries(global-group INTERFACE
 
 link_libraries(global-group)
 
+# FIXME: remove when all contribs will get custom cmake lists
 install(
     TARGETS global-group global-libs
     EXPORT global

cmake/sanitize.cmake

@@ -14,8 +14,20 @@ if (SANITIZE)
         endif ()
 
     elseif (SANITIZE STREQUAL "memory")
-        set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=memory -fsanitize-memory-track-origins")
+        # MemorySanitizer flags are set according to the official documentation:
-        set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=memory -fsanitize-memory-track-origins")
+        # https://clang.llvm.org/docs/MemorySanitizer.html#usage
+        #
+        # For now, it compiles with `cmake -DSANITIZE=memory -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_CXX_FLAGS_ADD="-O1" -DCMAKE_C_FLAGS_ADD="-O1"`
+        # Compiling with -DCMAKE_BUILD_TYPE=Debug leads to ld.lld failures because
+        # of large files (was not tested with ld.gold). This is why we compile with
+        # RelWithDebInfo, and downgrade optimizations to -O1 but not to -Og, to
+        # keep the binary size down.
+        # TODO: try compiling with -Og and with ld.gold.
+        set (MSAN_FLAGS "-fsanitize=memory -fsanitize-memory-track-origins -fno-optimize-sibling-calls")
+
+        set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${MSAN_FLAGS}")
+        set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} ${MSAN_FLAGS}")
+
         if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
             set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=memory")
         endif()
@@ -23,6 +35,28 @@ if (SANITIZE)
             set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libmsan")
         endif ()
 
+        # Temporarily disable many external libraries that don't work under
+        # MemorySanitizer yet.
+        set (ENABLE_HDFS 0 CACHE BOOL "")
+        set (ENABLE_CAPNP 0 CACHE BOOL "")
+        set (ENABLE_RDKAFKA 0 CACHE BOOL "")
+        set (ENABLE_ICU 0 CACHE BOOL "")
+        set (ENABLE_POCO_MONGODB 0 CACHE BOOL "")
+        set (ENABLE_POCO_NETSSL 0 CACHE BOOL "")
+        set (ENABLE_POCO_ODBC 0 CACHE BOOL "")
+        set (ENABLE_ODBC 0 CACHE BOOL "")
+        set (ENABLE_MYSQL 0 CACHE BOOL "")
+        set (ENABLE_EMBEDDED_COMPILER 0 CACHE BOOL "")
+        set (USE_INTERNAL_CAPNP_LIBRARY 0 CACHE BOOL "")
+        set (USE_SIMDJSON 0 CACHE BOOL "")
+        set (ENABLE_READLINE 0 CACHE BOOL "")
+        set (ENABLE_ORC 0 CACHE BOOL "")
+        set (ENABLE_PARQUET 0 CACHE BOOL "")
+        set (USE_CAPNP 0 CACHE BOOL "")
+        set (USE_INTERNAL_ORC_LIBRARY 0 CACHE BOOL "")
+        set (USE_ORC 0 CACHE BOOL "")
+        set (ENABLE_SSL 0 CACHE BOOL "")
+
     elseif (SANITIZE STREQUAL "thread")
         set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=thread")
         set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=thread")
@@ -34,14 +68,27 @@ if (SANITIZE)
         endif ()
 
     elseif (SANITIZE STREQUAL "undefined")
-        set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=undefined -fno-sanitize-recover=all")
+        set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=undefined -fno-sanitize-recover=all -fno-sanitize=float-divide-by-zero")
-        set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=undefined -fno-sanitize-recover=all")
+        set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=undefined -fno-sanitize-recover=all -fno-sanitize=float-divide-by-zero")
         if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
             set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined")
         endif()
         if (MAKE_STATIC_LIBRARIES AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
             set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libubsan")
         endif ()
+
+    elseif (SANITIZE STREQUAL "libfuzzer")
+        # NOTE: Eldar Zaitov decided to name it "libfuzzer" instead of "fuzzer" to keep in mind another possible fuzzer backends.
+        # NOTE: no-link means that all the targets are built with instrumentation for fuzzer, but only some of them (tests) have entry point for fuzzer and it's not checked.
+        set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=fuzzer-no-link,address,undefined -fsanitize-address-use-after-scope")
+        set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=fuzzer-no-link,address,undefined -fsanitize-address-use-after-scope")
+        if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+            set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=fuzzer-no-link,address,undefined -fsanitize-address-use-after-scope")
+        endif()
+        if (MAKE_STATIC_LIBRARIES AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+            set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libasan -static-libubsan")
+        endif ()
+        set (LIBFUZZER_CMAKE_CXX_FLAGS "-fsanitize=fuzzer,address,undefined -fsanitize-address-use-after-scope")
     else ()
         message (FATAL_ERROR "Unknown sanitizer type: ${SANITIZE}")
     endif ()

cmake/target.cmake

@@ -0,0 +1,80 @@
+if (CMAKE_SYSTEM_NAME MATCHES "Linux")
+    set (OS_LINUX 1)
+    add_definitions(-D OS_LINUX)
+elseif (CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
+    set (OS_FREEBSD 1)
+    add_definitions(-D OS_FREEBSD)
+elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin")
+    set (OS_DARWIN 1)
+    add_definitions(-D OS_DARWIN)
+endif ()
+
+if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+    set (COMPILER_GCC 1)
+elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
+    set (COMPILER_CLANG 1)
+endif ()
+
+if (COMPILER_GCC)
+    # Require minimum version of gcc
+    set (GCC_MINIMUM_VERSION 8)
+    if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${GCC_MINIMUM_VERSION} AND NOT CMAKE_VERSION VERSION_LESS 2.8.9)
+        message (FATAL_ERROR "GCC version must be at least ${GCC_MINIMUM_VERSION}. For example, if GCC ${GCC_MINIMUM_VERSION} is available under gcc-${GCC_MINIMUM_VERSION}, g++-${GCC_MINIMUM_VERSION} names, do the following: export CC=gcc-${GCC_MINIMUM_VERSION} CXX=g++-${GCC_MINIMUM_VERSION}; rm -rf CMakeCache.txt CMakeFiles; and re run cmake or ./release.")
+    endif ()
+elseif (COMPILER_CLANG)
+    # Require minimum version of clang
+    set (CLANG_MINIMUM_VERSION 7)
+    if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${CLANG_MINIMUM_VERSION})
+        message (FATAL_ERROR "Clang version must be at least ${CLANG_MINIMUM_VERSION}.")
+    endif ()
+else ()
+    message (WARNING "You are using an unsupported compiler. Compilation has only been tested with Clang 6+ and GCC 7+.")
+endif ()
+
+string(REGEX MATCH "-?[0-9]+(.[0-9]+)?$" COMPILER_POSTFIX ${CMAKE_CXX_COMPILER})
+
+if (OS_LINUX)
+    find_program (LLD_PATH NAMES "lld${COMPILER_POSTFIX}" "lld")
+    find_program (GOLD_PATH NAMES "ld.gold" "gold")
+endif()
+
+option (LINKER_NAME "Linker name or full path")
+if (NOT LINKER_NAME)
+    if (COMPILER_CLANG AND LLD_PATH)
+        set (LINKER_NAME "lld")
+    elseif (GOLD_PATH)
+        set (LINKER_NAME "gold")
+    endif ()
+endif ()
+
+if (LINKER_NAME)
+    message(STATUS "Using linker: ${LINKER_NAME} (selected from: LLD_PATH=${LLD_PATH}; GOLD_PATH=${GOLD_PATH}; COMPILER_POSTFIX=${COMPILER_POSTFIX})")
+    set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}")
+    set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}")
+endif ()
+
+if (CMAKE_CROSSCOMPILING)
+    if (CMAKE_SYSTEM_NAME MATCHES "Darwin")
+        set (CMAKE_SYSTEM_PROCESSOR x86_64)
+        set (CMAKE_C_COMPILER_TARGET x86_64-apple-darwin)
+        set (CMAKE_CXX_COMPILER_TARGET x86_64-apple-darwin)
+
+        set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
+        set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
+
+        set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
+        set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
+
+        # FIXME: broken dependencies
+        set (USE_SNAPPY OFF)
+        set (ENABLE_SSL OFF)
+        set (ENABLE_PROTOBUF OFF)
+        set (ENABLE_PARQUET OFF)
+        set (ENABLE_READLINE OFF)
+        set (ENABLE_ICU OFF)
+        set (ENABLE_FASTOPS OFF)
+    endif ()
+
+    # Don't know why but CXX_STANDARD doesn't work for cross-compilation
+    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17")
+endif ()

contrib/CMakeLists.txt

@@ -65,7 +65,7 @@ if (USE_INTERNAL_ZLIB_LIBRARY)
     endif ()
 
     add_subdirectory (${INTERNAL_ZLIB_NAME})
-    # todo: make pull to Dead2/zlib-ng and remove:
+    # TODO: make pull to Dead2/zlib-ng and remove:
     # We should use same defines when including zlib.h as used when zlib compiled
     target_compile_definitions (zlib PUBLIC ZLIB_COMPAT WITH_GZFILEOP)
     target_compile_definitions (zlibstatic PUBLIC ZLIB_COMPAT WITH_GZFILEOP)
@@ -125,11 +125,18 @@ if (USE_INTERNAL_SSL_LIBRARY)
 endif ()
 
 if (ENABLE_MYSQL AND USE_INTERNAL_MYSQL_LIBRARY)
-    add_subdirectory (mariadb-connector-c-cmake)
+    set(CLIENT_PLUGIN_CACHING_SHA2_PASSWORD STATIC)
-    target_include_directories(mysqlclient BEFORE PRIVATE ${ZLIB_INCLUDE_DIR})
+    set(CLIENT_PLUGIN_SHA256_PASSWORD STATIC)
-    if(OPENSSL_INCLUDE_DIR)
+    set(CLIENT_PLUGIN_REMOTE_IO OFF)
-        target_include_directories(mysqlclient BEFORE PRIVATE ${OPENSSL_INCLUDE_DIR})
+    set(CLIENT_PLUGIN_DIALOG OFF)
+    set(CLIENT_PLUGIN_AUTH_GSSAPI_CLIENT OFF)
+    set(CLIENT_PLUGIN_CLIENT_ED25519 OFF)
+    set(CLIENT_PLUGIN_MYSQL_CLEAR_PASSWORD OFF)
+    set(SKIP_TESTS 1)
+    if (GLIBC_COMPATIBILITY)
+        set(LIBM glibc-compatibility)
     endif()
+    add_subdirectory (mariadb-connector-c)
 endif ()
 
 if (USE_INTERNAL_RDKAFKA_LIBRARY)
@@ -193,13 +200,15 @@ if (USE_INTERNAL_PARQUET_LIBRARY_NATIVE_CMAKE)
     endif()
 
 else()
-
     if(USE_INTERNAL_SNAPPY_LIBRARY)
         set(SNAPPY_BUILD_TESTS 0 CACHE INTERNAL "")
         if (NOT MAKE_STATIC_LIBRARIES)
             set(BUILD_SHARED_LIBS 1) # TODO: set at root dir
         endif()
+
         add_subdirectory(snappy)
+
+        set (SNAPPY_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/snappy")
         if(SANITIZE STREQUAL "undefined")
             target_compile_options(${SNAPPY_LIBRARY} PRIVATE -fno-sanitize=undefined)
         endif()

contrib/libcxx-cmake/CMakeLists.txt

@@ -42,7 +42,12 @@ add_library(cxx ${SRCS})
 
 target_include_directories(cxx SYSTEM BEFORE PUBLIC $<BUILD_INTERFACE:${LIBCXX_SOURCE_DIR}/include>)
 target_compile_definitions(cxx PRIVATE -D_LIBCPP_BUILDING_LIBRARY -DLIBCXX_BUILDING_LIBCXXABI)
+
 target_compile_options(cxx PUBLIC -nostdinc++ -Wno-reserved-id-macro)
+if (OS_DARWIN)
+    target_compile_options(cxx PUBLIC -Wno-ctad-maybe-unsupported)
+endif ()
+
 target_link_libraries(cxx PUBLIC cxxabi)
 
 install(

contrib/mariadb-connector-c

@@ -1 +1 @@
-Subproject commit c6503d3acc85ca1a7f5e7e38b605d7c9410aac1e
+Subproject commit 18016300b00825a3fcbc6fb2aa37ac3e51416f71

contrib/mariadb-connector-c-cmake/CMakeLists.txt

@@ -1,74 +0,0 @@
-set(MARIADB_CLIENT_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/mariadb-connector-c)
-set(MARIADB_CLIENT_BINARY_DIR ${ClickHouse_BINARY_DIR}/contrib/mariadb-connector-c)
-
-set(SRCS
-#${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/bmove_upp.c
-${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/get_password.c
-${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_alloc.c
-${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_array.c
-${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_charset.c
-${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_compress.c
-${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_context.c
-${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_default.c
-${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_dtoa.c
-${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_errmsg.c
-${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_hash.c
-${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_init.c
-${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_io.c
-${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_list.c
-${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_ll2str.c
-${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_loaddata.c
-${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_net.c
-${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_password.c
-${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_pvio.c
-${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/mariadb_async.c
-${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/mariadb_charset.c
-#${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/mariadb_dyncol.c
-${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/mariadb_lib.c
-${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/mariadb_stmt.c
-${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_sha1.c
-${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_stmt_codec.c
-${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_string.c
-${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_time.c
-${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_tls.c
-${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/secure/openssl_crypt.c
-#${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/secure/gnutls.c
-#${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/secure/ma_schannel.c
-#${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/secure/schannel.c
-#${MARIADB_CLIENT_SOURCE_DIR}/plugins/auth/auth_gssapi_client.c
-#${MARIADB_CLIENT_SOURCE_DIR}/plugins/auth/dialog.c
-#${MARIADB_CLIENT_SOURCE_DIR}/plugins/auth/gssapi_client.c
-#${MARIADB_CLIENT_SOURCE_DIR}/plugins/auth/gssapi_errmsg.c
-${MARIADB_CLIENT_SOURCE_DIR}/plugins/auth/mariadb_cleartext.c
-${MARIADB_CLIENT_SOURCE_DIR}/plugins/auth/my_auth.c
-${MARIADB_CLIENT_SOURCE_DIR}/plugins/auth/old_password.c
-${MARIADB_CLIENT_SOURCE_DIR}/plugins/auth/sha256_pw.c
-${MARIADB_CLIENT_SOURCE_DIR}/plugins/auth/caching_sha2_pw.c
-#${MARIADB_CLIENT_SOURCE_DIR}/plugins/auth/sspi_client.c
-#${MARIADB_CLIENT_SOURCE_DIR}/plugins/auth/sspi_errmsg.c
-${MARIADB_CLIENT_SOURCE_DIR}/plugins/connection/aurora.c
-${MARIADB_CLIENT_SOURCE_DIR}/plugins/connection/replication.c
-#${MARIADB_CLIENT_SOURCE_DIR}/plugins/io/remote_io.c
-#${MARIADB_CLIENT_SOURCE_DIR}/plugins/pvio/pvio_npipe.c
-#${MARIADB_CLIENT_SOURCE_DIR}/plugins/pvio/pvio_shmem.c
-${MARIADB_CLIENT_SOURCE_DIR}/plugins/pvio/pvio_socket.c
-#${MARIADB_CLIENT_SOURCE_DIR}/plugins/trace/trace_example.c
-${CMAKE_CURRENT_SOURCE_DIR}/linux_x86_64/libmariadb/ma_client_plugin.c
-)
-
-if(OPENSSL_LIBRARIES)
-    list(APPEND SRCS ${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/secure/openssl.c)
-endif()
-
-add_library(mysqlclient ${SRCS})
-
-if(OPENSSL_LIBRARIES)
-    target_link_libraries(mysqlclient PRIVATE ${OPENSSL_LIBRARIES})
-    target_compile_definitions(mysqlclient PRIVATE -D HAVE_OPENSSL -D HAVE_TLS)
-endif()
-
-target_include_directories(mysqlclient PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/linux_x86_64/include)
-target_include_directories(mysqlclient PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/common/include)
-target_include_directories(mysqlclient PUBLIC ${MARIADB_CLIENT_SOURCE_DIR}/include)
-
-target_compile_definitions(mysqlclient PRIVATE -D THREAD)

contrib/mariadb-connector-c-cmake/common/include/mysql/mysql.h

@@ -1 +0,0 @@
-#include <mysql.h>

contrib/mariadb-connector-c-cmake/common/include/mysql/mysqld_error.h

@@ -1 +0,0 @@
-#include <mysqld_error.h>

contrib/mariadb-connector-c-cmake/linux_x86_64/include/config.h

@@ -1,269 +0,0 @@
-
-/*
- * Include file constants (processed in LibmysqlIncludeFiles.txt 1
- */
-#define HAVE_ALLOCA_H 1
-/* #undef HAVE_BIGENDIAN */
-#define HAVE_SETLOCALE 1
-#define HAVE_NL_LANGINFO 1
-#define HAVE_ARPA_INET_H 1
-#define HAVE_CRYPT_H 1
-#define HAVE_DIRENT_H 1
-#define HAVE_DLFCN_H 1
-#define HAVE_EXECINFO_H 1
-#define HAVE_FCNTL_H 1
-#define HAVE_FENV_H 1
-#define HAVE_FLOAT_H 1
-/* #undef HAVE_FPU_CONTROL_H */
-#define HAVE_GRP_H 1
-/* #undef HAVE_IEEEFP_H */
-#define HAVE_LIMITS_H 1
-#define HAVE_MALLOC_H 1
-#define HAVE_MEMORY_H 1
-#define HAVE_NETINET_IN_H 1
-#define HAVE_PATHS_H 1
-#define HAVE_PWD_H 1
-#define HAVE_SCHED_H 1
-/* #undef HAVE_SELECT_H */
-#define HAVE_STDDEF_H 1
-#define HAVE_STDINT_H 1
-#define HAVE_STDLIB_H 1
-#define HAVE_STRING_H 1
-#define HAVE_STRINGS_H 1
-/* #undef HAVE_SYNCH_H */
-/* #undef HAVE_SYS_FPU_H */
-#define HAVE_SYS_IOCTL_H 1
-#define HAVE_SYS_IPC_H 1
-#define HAVE_SYS_MMAN_H 1
-#define HAVE_SYS_PRCTL_H 1
-#define HAVE_SYS_SELECT_H 1
-#define HAVE_SYS_SHM_H 1
-#define HAVE_SYS_SOCKET_H 1
-#define HAVE_SYS_STAT_H 1
-/* #undef HAVE_SYS_STREAM_H */
-#define HAVE_SYS_TIMEB_H 1
-#define HAVE_SYS_TYPES_H 1
-#define HAVE_SYS_UN_H 1
-/* #undef HAVE_SYSENT_H */
-#define HAVE_TERMIO_H 1
-#define HAVE_TERMIOS_H 1
-#define HAVE_UNISTD_H 1
-#define HAVE_UTIME_H 1
-#define HAVE_UCONTEXT_H 1
-
-/*
- * function definitions - processed in LibmysqlFunctions.txt 
- */
-#define HAVE_ACCESS 1
-/* #undef HAVE_AIOWAIT */
-#define HAVE_ALARM 1
-/* #undef HAVE_ALLOCA */
-#define HAVE_BCMP 1
-/* #undef HAVE_BFILL */
-/* #undef HAVE_BMOVE */
-#define HAVE_BZERO 1
-#define HAVE_CLOCK_GETTIME 1
-/* #undef HAVE_COMPRESS */
-/* #undef HAVE_CRYPT */
-#define HAVE_DLERROR 1
-#define HAVE_DLOPEN 1
-#define HAVE_FCHMOD 1
-#define HAVE_FCNTL 1
-/* #undef HAVE_FCONVERT */
-#define HAVE_FDATASYNC 1
-#define HAVE_FESETROUND 1
-#define HAVE_FINITE 1
-#define HAVE_FSEEKO 1
-#define HAVE_FSYNC 1
-#define HAVE_GETADDRINFO 1
-#define HAVE_GETCWD 1
-#define HAVE_GETHOSTBYADDR_R 1
-#define HAVE_GETHOSTBYNAME_R 1
-/* #undef HAVE_GETHRTIME */
-#define HAVE_GETNAMEINFO 1
-#define HAVE_GETPAGESIZE 1
-#define HAVE_GETPASS 1
-/* #undef HAVE_GETPASSPHRASE */
-#define HAVE_GETPWNAM 1
-#define HAVE_GETPWUID 1
-#define HAVE_GETRLIMIT 1
-#define HAVE_GETRUSAGE 1
-#define HAVE_GETWD 1
-#define HAVE_GMTIME_R 1
-#define HAVE_INITGROUPS 1
-#define HAVE_LDIV 1
-#define HAVE_LOCALTIME_R 1
-#define HAVE_LOG2 1
-#define HAVE_LONGJMP 1
-#define HAVE_LSTAT 1
-#define HAVE_MADVISE 1
-#define HAVE_MALLINFO 1
-#define HAVE_MEMALIGN 1
-#define HAVE_MEMCPY 1
-#define HAVE_MEMMOVE 1
-#define HAVE_MKSTEMP 1
-#define HAVE_MLOCK 1
-#define HAVE_MLOCKALL 1
-#define HAVE_MMAP 1
-#define HAVE_MMAP64 1
-#define HAVE_PERROR 1
-#define HAVE_POLL 1
-#define HAVE_PREAD 1
-/* #undef HAVE_PTHREAD_ATTR_CREATE */
-#define HAVE_PTHREAD_ATTR_GETSTACKSIZE 1
-/* #undef HAVE_PTHREAD_ATTR_SETPRIO */
-#define HAVE_PTHREAD_ATTR_SETSCHEDPARAM 1
-#define HAVE_PTHREAD_ATTR_SETSCOPE 1
-#define HAVE_PTHREAD_ATTR_SETSTACKSIZE 1
-/* #undef HAVE_PTHREAD_CONDATTR_CREATE */
-/* #undef HAVE_PTHREAD_INIT */
-#define HAVE_PTHREAD_KEY_DELETE 1
-#define HAVE_PTHREAD_KILL 1
-#define HAVE_PTHREAD_RWLOCK_RDLOCK 1
-/* #undef HAVE_PTHREAD_SETPRIO_NP */
-#define HAVE_PTHREAD_SETSCHEDPARAM 1
-#define HAVE_PTHREAD_SIGMASK 1
-/* #undef HAVE_PTHREAD_THREADMASK */
-/* #undef HAVE_PTHREAD_YIELD_NP */
-#define HAVE_READDIR_R 1
-#define HAVE_READLINK 1
-#define HAVE_REALPATH 1
-#define HAVE_RENAME 1
-#define HAVE_SCHED_YIELD 1
-#define HAVE_SELECT 1
-/* #undef HAVE_SETFD */
-/* #undef HAVE_SETFILEPOINTER */
-#define HAVE_SIGNAL 1
-#define HAVE_SIGACTION 1
-/* #undef HAVE_SIGTHREADMASK */
-#define HAVE_SIGWAIT 1
-#define HAVE_SLEEP 1
-#define HAVE_SNPRINTF 1
-/* #undef HAVE_SQLITE */
-#define HAVE_STPCPY 1
-#define HAVE_STRERROR 1
-/* #undef HAVE_STRLCPY */
-#define HAVE_STRNLEN 1
-#define HAVE_STRPBRK 1
-#define HAVE_STRSEP 1
-#define HAVE_STRSTR 1
-#define HAVE_STRTOK_R 1
-#define HAVE_STRTOL 1
-#define HAVE_STRTOLL 1
-#define HAVE_STRTOUL 1
-#define HAVE_STRTOULL 1
-/* #undef HAVE_TELL */
-/* #undef HAVE_THR_SETCONCURRENCY */
-/* #undef HAVE_THR_YIELD */
-#define HAVE_VASPRINTF 1
-#define HAVE_VSNPRINTF 1
-
-/*
- * types and sizes
- */
-/* Types we may use */
-#define SIZEOF_CHAR 1
-#if defined(SIZEOF_CHAR)
-# define HAVE_CHAR 1
-#endif
-
-#define SIZEOF_CHARP 8
-#if defined(SIZEOF_CHARP)
-# define HAVE_CHARP 1
-#endif
-
-#define SIZEOF_SHORT 2
-#if defined(SIZEOF_SHORT)
-# define HAVE_SHORT 1
-#endif
-
-#define SIZEOF_INT 4
-#if defined(SIZEOF_INT)
-# define HAVE_INT 1
-#endif
-
-#define SIZEOF_LONG 8
-#if defined(SIZEOF_LONG)
-# define HAVE_LONG 1
-#endif
-
-#define SIZEOF_LONG_LONG 8
-#if defined(SIZEOF_LONG_LONG)
-# define HAVE_LONG_LONG 1
-#endif
-
-
-#define SIZEOF_SIGSET_T 128
-#if defined(SIZEOF_SIGSET_T)
-# define HAVE_SIGSET_T 1
-#endif
-
-#define SIZEOF_SIZE_T 8
-#if defined(SIZEOF_SIZE_T)
-# define HAVE_SIZE_T 1
-#endif
-
-/* #undef SIZEOF_UCHAR */
-#if defined(SIZEOF_UCHAR)
-# define HAVE_UCHAR 1
-#endif
-
-#define SIZEOF_UINT 4
-#if defined(SIZEOF_UINT)
-# define HAVE_UINT 1
-#endif
-
-#define SIZEOF_ULONG 8
-#if defined(SIZEOF_ULONG)
-# define HAVE_ULONG 1
-#endif
-
-/* #undef SIZEOF_INT8 */
-#if defined(SIZEOF_INT8)
-# define HAVE_INT8 1
-#endif
-/* #undef SIZEOF_UINT8 */
-#if defined(SIZEOF_UINT8)
-# define HAVE_UINT8 1
-#endif
-
-/* #undef SIZEOF_INT16 */
-#if defined(SIZEOF_INT16)
-# define HAVE_INT16 1
-#endif
-/* #undef SIZEOF_UINT16 */
-#if defined(SIZEOF_UINT16)
-# define HAVE_UINT16 1
-#endif
-
-/* #undef SIZEOF_INT32 */
-#if defined(SIZEOF_INT32)
-# define HAVE_INT32 1
-#endif
-/* #undef SIZEOF_UINT32 */
-#if defined(SIZEOF_UINT32)
-# define HAVE_UINT32 1
-#endif
-/* #undef SIZEOF_U_INT32_T */
-#if defined(SIZEOF_U_INT32_T)
-# define HAVE_U_INT32_T 1
-#endif
-
-/* #undef SIZEOF_INT64 */
-#if defined(SIZEOF_INT64)
-# define HAVE_INT64 1
-#endif
-/* #undef SIZEOF_UINT64 */
-#if defined(SIZEOF_UINT64)
-# define HAVE_UINT64 1
-#endif
-
-/* #undef SIZEOF_SOCKLEN_T */
-#if defined(SIZEOF_SOCKLEN_T)
-# define HAVE_SOCKLEN_T 1
-#endif
-
-#define SOCKET_SIZE_TYPE socklen_t
-
-#define MARIADB_DEFAULT_CHARSET "latin1"
-

contrib/mariadb-connector-c-cmake/linux_x86_64/include/ma_config.h

@@ -1,269 +0,0 @@
-
-/*
- * Include file constants (processed in LibmysqlIncludeFiles.txt 1
- */
-#define HAVE_ALLOCA_H 1
-/* #undef HAVE_BIGENDIAN */
-#define HAVE_SETLOCALE 1
-#define HAVE_NL_LANGINFO 1
-#define HAVE_ARPA_INET_H 1
-#define HAVE_CRYPT_H 1
-#define HAVE_DIRENT_H 1
-#define HAVE_DLFCN_H 1
-#define HAVE_EXECINFO_H 1
-#define HAVE_FCNTL_H 1
-#define HAVE_FENV_H 1
-#define HAVE_FLOAT_H 1
-/* #undef HAVE_FPU_CONTROL_H */
-#define HAVE_GRP_H 1
-/* #undef HAVE_IEEEFP_H */
-#define HAVE_LIMITS_H 1
-#define HAVE_MALLOC_H 1
-#define HAVE_MEMORY_H 1
-#define HAVE_NETINET_IN_H 1
-#define HAVE_PATHS_H 1
-#define HAVE_PWD_H 1
-#define HAVE_SCHED_H 1
-/* #undef HAVE_SELECT_H */
-#define HAVE_STDDEF_H 1
-#define HAVE_STDINT_H 1
-#define HAVE_STDLIB_H 1
-#define HAVE_STRING_H 1
-#define HAVE_STRINGS_H 1
-/* #undef HAVE_SYNCH_H */
-/* #undef HAVE_SYS_FPU_H */
-#define HAVE_SYS_IOCTL_H 1
-#define HAVE_SYS_IPC_H 1
-#define HAVE_SYS_MMAN_H 1
-#define HAVE_SYS_PRCTL_H 1
-#define HAVE_SYS_SELECT_H 1
-#define HAVE_SYS_SHM_H 1
-#define HAVE_SYS_SOCKET_H 1
-#define HAVE_SYS_STAT_H 1
-/* #undef HAVE_SYS_STREAM_H */
-#define HAVE_SYS_TIMEB_H 1
-#define HAVE_SYS_TYPES_H 1
-#define HAVE_SYS_UN_H 1
-/* #undef HAVE_SYSENT_H */
-#define HAVE_TERMIO_H 1
-#define HAVE_TERMIOS_H 1
-#define HAVE_UNISTD_H 1
-#define HAVE_UTIME_H 1
-#define HAVE_UCONTEXT_H 1
-
-/*
- * function definitions - processed in LibmysqlFunctions.txt 
- */
-#define HAVE_ACCESS 1
-/* #undef HAVE_AIOWAIT */
-#define HAVE_ALARM 1
-/* #undef HAVE_ALLOCA */
-#define HAVE_BCMP 1
-/* #undef HAVE_BFILL */
-/* #undef HAVE_BMOVE */
-#define HAVE_BZERO 1
-#define HAVE_CLOCK_GETTIME 1
-/* #undef HAVE_COMPRESS */
-/* #undef HAVE_CRYPT */
-#define HAVE_DLERROR 1
-#define HAVE_DLOPEN 1
-#define HAVE_FCHMOD 1
-#define HAVE_FCNTL 1
-/* #undef HAVE_FCONVERT */
-#define HAVE_FDATASYNC 1
-#define HAVE_FESETROUND 1
-#define HAVE_FINITE 1
-#define HAVE_FSEEKO 1
-#define HAVE_FSYNC 1
-#define HAVE_GETADDRINFO 1
-#define HAVE_GETCWD 1
-#define HAVE_GETHOSTBYADDR_R 1
-#define HAVE_GETHOSTBYNAME_R 1
-/* #undef HAVE_GETHRTIME */
-#define HAVE_GETNAMEINFO 1
-#define HAVE_GETPAGESIZE 1
-#define HAVE_GETPASS 1
-/* #undef HAVE_GETPASSPHRASE */
-#define HAVE_GETPWNAM 1
-#define HAVE_GETPWUID 1
-#define HAVE_GETRLIMIT 1
-#define HAVE_GETRUSAGE 1
-#define HAVE_GETWD 1
-#define HAVE_GMTIME_R 1
-#define HAVE_INITGROUPS 1
-#define HAVE_LDIV 1
-#define HAVE_LOCALTIME_R 1
-#define HAVE_LOG2 1
-#define HAVE_LONGJMP 1
-#define HAVE_LSTAT 1
-#define HAVE_MADVISE 1
-#define HAVE_MALLINFO 1
-#define HAVE_MEMALIGN 1
-#define HAVE_MEMCPY 1
-#define HAVE_MEMMOVE 1
-#define HAVE_MKSTEMP 1
-#define HAVE_MLOCK 1
-#define HAVE_MLOCKALL 1
-#define HAVE_MMAP 1
-#define HAVE_MMAP64 1
-#define HAVE_PERROR 1
-#define HAVE_POLL 1
-#define HAVE_PREAD 1
-/* #undef HAVE_PTHREAD_ATTR_CREATE */
-#define HAVE_PTHREAD_ATTR_GETSTACKSIZE 1
-/* #undef HAVE_PTHREAD_ATTR_SETPRIO */
-#define HAVE_PTHREAD_ATTR_SETSCHEDPARAM 1
-#define HAVE_PTHREAD_ATTR_SETSCOPE 1
-#define HAVE_PTHREAD_ATTR_SETSTACKSIZE 1
-/* #undef HAVE_PTHREAD_CONDATTR_CREATE */
-/* #undef HAVE_PTHREAD_INIT */
-#define HAVE_PTHREAD_KEY_DELETE 1
-#define HAVE_PTHREAD_KILL 1
-#define HAVE_PTHREAD_RWLOCK_RDLOCK 1
-/* #undef HAVE_PTHREAD_SETPRIO_NP */
-#define HAVE_PTHREAD_SETSCHEDPARAM 1
-#define HAVE_PTHREAD_SIGMASK 1
-/* #undef HAVE_PTHREAD_THREADMASK */
-/* #undef HAVE_PTHREAD_YIELD_NP */
-#define HAVE_READDIR_R 1
-#define HAVE_READLINK 1
-#define HAVE_REALPATH 1
-#define HAVE_RENAME 1
-#define HAVE_SCHED_YIELD 1
-#define HAVE_SELECT 1
-/* #undef HAVE_SETFD */
-/* #undef HAVE_SETFILEPOINTER */
-#define HAVE_SIGNAL 1
-#define HAVE_SIGACTION 1
-/* #undef HAVE_SIGTHREADMASK */
-#define HAVE_SIGWAIT 1
-#define HAVE_SLEEP 1
-#define HAVE_SNPRINTF 1
-/* #undef HAVE_SQLITE */
-#define HAVE_STPCPY 1
-#define HAVE_STRERROR 1
-/* #undef HAVE_STRLCPY */
-#define HAVE_STRNLEN 1
-#define HAVE_STRPBRK 1
-#define HAVE_STRSEP 1
-#define HAVE_STRSTR 1
-#define HAVE_STRTOK_R 1
-#define HAVE_STRTOL 1
-#define HAVE_STRTOLL 1
-#define HAVE_STRTOUL 1
-#define HAVE_STRTOULL 1
-/* #undef HAVE_TELL */
-/* #undef HAVE_THR_SETCONCURRENCY */
-/* #undef HAVE_THR_YIELD */
-#define HAVE_VASPRINTF 1
-#define HAVE_VSNPRINTF 1
-
-/*
- * types and sizes
- */
-/* Types we may use */
-#define SIZEOF_CHAR 1
-#if defined(SIZEOF_CHAR)
-# define HAVE_CHAR 1
-#endif
-
-#define SIZEOF_CHARP 8
-#if defined(SIZEOF_CHARP)
-# define HAVE_CHARP 1
-#endif
-
-#define SIZEOF_SHORT 2
-#if defined(SIZEOF_SHORT)
-# define HAVE_SHORT 1
-#endif
-
-#define SIZEOF_INT 4
-#if defined(SIZEOF_INT)
-# define HAVE_INT 1
-#endif
-
-#define SIZEOF_LONG 8
-#if defined(SIZEOF_LONG)
-# define HAVE_LONG 1
-#endif
-
-#define SIZEOF_LONG_LONG 8
-#if defined(SIZEOF_LONG_LONG)
-# define HAVE_LONG_LONG 1
-#endif
-
-
-#define SIZEOF_SIGSET_T 128
-#if defined(SIZEOF_SIGSET_T)
-# define HAVE_SIGSET_T 1
-#endif
-
-#define SIZEOF_SIZE_T 8
-#if defined(SIZEOF_SIZE_T)
-# define HAVE_SIZE_T 1
-#endif
-
-/* #undef SIZEOF_UCHAR */
-#if defined(SIZEOF_UCHAR)
-# define HAVE_UCHAR 1
-#endif
-
-#define SIZEOF_UINT 4
-#if defined(SIZEOF_UINT)
-# define HAVE_UINT 1
-#endif
-
-#define SIZEOF_ULONG 8
-#if defined(SIZEOF_ULONG)
-# define HAVE_ULONG 1
-#endif
-
-/* #undef SIZEOF_INT8 */
-#if defined(SIZEOF_INT8)
-# define HAVE_INT8 1
-#endif
-/* #undef SIZEOF_UINT8 */
-#if defined(SIZEOF_UINT8)
-# define HAVE_UINT8 1
-#endif
-
-/* #undef SIZEOF_INT16 */
-#if defined(SIZEOF_INT16)
-# define HAVE_INT16 1
-#endif
-/* #undef SIZEOF_UINT16 */
-#if defined(SIZEOF_UINT16)
-# define HAVE_UINT16 1
-#endif
-
-/* #undef SIZEOF_INT32 */
-#if defined(SIZEOF_INT32)
-# define HAVE_INT32 1
-#endif
-/* #undef SIZEOF_UINT32 */
-#if defined(SIZEOF_UINT32)
-# define HAVE_UINT32 1
-#endif
-/* #undef SIZEOF_U_INT32_T */
-#if defined(SIZEOF_U_INT32_T)
-# define HAVE_U_INT32_T 1
-#endif
-
-/* #undef SIZEOF_INT64 */
-#if defined(SIZEOF_INT64)
-# define HAVE_INT64 1
-#endif
-/* #undef SIZEOF_UINT64 */
-#if defined(SIZEOF_UINT64)
-# define HAVE_UINT64 1
-#endif
-
-/* #undef SIZEOF_SOCKLEN_T */
-#if defined(SIZEOF_SOCKLEN_T)
-# define HAVE_SOCKLEN_T 1
-#endif
-
-#define SOCKET_SIZE_TYPE socklen_t
-
-#define MARIADB_DEFAULT_CHARSET "latin1"
-

contrib/mariadb-connector-c-cmake/linux_x86_64/include/mariadb_version.h

@@ -1,36 +0,0 @@
-/* Copyright Abandoned 1996, 1999, 2001 MySQL AB
-   This file is public domain and comes with NO WARRANTY of any kind */
-
-/* Version numbers for protocol & mysqld */
-
-#ifndef _mariadb_version_h_
-#define _mariadb_version_h_
-
-#ifdef _CUSTOMCONFIG_
-#include <custom_conf.h>
-#else
-#define PROTOCOL_VERSION		10
-#define MARIADB_CLIENT_VERSION_STR	"10.3.6"
-#define MARIADB_BASE_VERSION		"mariadb-10.3"
-#define MARIADB_VERSION_ID		100306
-#define MYSQL_VERSION_ID		100306
-#define MARIADB_PORT	        	3306
-#define MARIADB_UNIX_ADDR               "/var/run/mysqld/mysqld.sock"
-#define MYSQL_CONFIG_NAME		"my"
-
-#define MARIADB_PACKAGE_VERSION "3.0.6"
-#define MARIADB_PACKAGE_VERSION_ID 30006
-#define MARIADB_SYSTEM_TYPE "Linux"
-#define MARIADB_MACHINE_TYPE "x86_64"
-#define MARIADB_PLUGINDIR "lib/mariadb/plugin"
-
-/* mysqld compile time options */
-#ifndef MYSQL_CHARSET
-#define MYSQL_CHARSET			""
-#endif
-#endif
-
-/* Source information */
-#define CC_SOURCE_REVISION "a0fd36cc5a5313414a5a2ebe9322577a29b4782a"
-
-#endif /* _mariadb_version_h_ */

contrib/mariadb-connector-c-cmake/linux_x86_64/libmariadb/ma_client_plugin.c

@@ -1,502 +0,0 @@
-/* Copyright (C) 2010 - 2012 Sergei Golubchik and Monty Program Ab
-                 2015-2016 MariaDB Corporation AB
-
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public
-   License as published by the Free Software Foundation; either
-   version 2 of the License, or (at your option) any later version.
-   
-   This library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-   
-   You should have received a copy of the GNU Library General Public
-   License along with this library; if not see <http://www.gnu.org/licenses>
-   or write to the Free Software Foundation, Inc., 
-   51 Franklin St., Fifth Floor, Boston, MA 02110, USA */
-
-/**
-  @file
-  
-  Support code for the client side (libmariadb) plugins
-
-  Client plugins are somewhat different from server plugins, they are simpler.
-
-  They do not need to be installed or in any way explicitly loaded on the
-  client, they are loaded automatically on demand.
-  One client plugin per shared object, soname *must* match the plugin name.
-
-  There is no reference counting and no unloading either.
-*/
-
-#if _MSC_VER
-/* Silence warnings about variable 'unused' being used. */
-#define FORCE_INIT_OF_VARS 1
-#endif
-
-#include <ma_global.h>
-#include <ma_sys.h>
-#include <ma_common.h> 
-#include <ma_string.h>
-#include <ma_pthread.h>
-
-#include "errmsg.h"
-#include <mysql/client_plugin.h>
-
-struct st_client_plugin_int {
-  struct st_client_plugin_int *next;
-  void   *dlhandle;
-  struct st_mysql_client_plugin *plugin;
-};
-
-static my_bool initialized= 0;
-static MA_MEM_ROOT mem_root;
-
-static uint valid_plugins[][2]= {
-  {MYSQL_CLIENT_AUTHENTICATION_PLUGIN, MYSQL_CLIENT_AUTHENTICATION_PLUGIN_INTERFACE_VERSION},
-  {MARIADB_CLIENT_PVIO_PLUGIN, MARIADB_CLIENT_PVIO_PLUGIN_INTERFACE_VERSION},
-  {MARIADB_CLIENT_TRACE_PLUGIN, MARIADB_CLIENT_TRACE_PLUGIN_INTERFACE_VERSION},
-  {MARIADB_CLIENT_CONNECTION_PLUGIN, MARIADB_CLIENT_CONNECTION_PLUGIN_INTERFACE_VERSION},
-  {0, 0}
-};
-
-/*
-  Loaded plugins are stored in a linked list.
-  The list is append-only, the elements are added to the head (like in a stack).
-  The elements are added under a mutex, but the list can be read and traversed
-  without any mutex because once an element is added to the list, it stays
-  there. The main purpose of a mutex is to prevent two threads from
-  loading the same plugin twice in parallel.
-*/
-
-
-struct st_client_plugin_int *plugin_list[MYSQL_CLIENT_MAX_PLUGINS + MARIADB_CLIENT_MAX_PLUGINS];
-#ifdef THREAD
-static pthread_mutex_t LOCK_load_client_plugin;
-#endif
-
-extern struct st_mysql_client_plugin mysql_native_password_client_plugin;
-extern struct st_mysql_client_plugin mysql_old_password_client_plugin;
-extern struct st_mysql_client_plugin pvio_socket_client_plugin;
-extern struct st_mysql_client_plugin sha256_password_client_plugin;
-extern struct st_mysql_client_plugin caching_sha2_password_client_plugin;
-
-
-struct st_mysql_client_plugin *mysql_client_builtins[]=
-{
-  (struct st_mysql_client_plugin *)&mysql_native_password_client_plugin,
-  (struct st_mysql_client_plugin *)&mysql_old_password_client_plugin,
-  (struct st_mysql_client_plugin *)&pvio_socket_client_plugin,
-  (struct st_mysql_client_plugin *)&sha256_password_client_plugin,
-  (struct st_mysql_client_plugin *)&caching_sha2_password_client_plugin,
-  0
-};
-
-
-static int is_not_initialized(MYSQL *mysql, const char *name)
-{
-  if (initialized)
-    return 0;
-
-  my_set_error(mysql, CR_AUTH_PLUGIN_CANNOT_LOAD,
-               SQLSTATE_UNKNOWN, ER(CR_AUTH_PLUGIN_CANNOT_LOAD),
-               name, "not initialized");
-  return 1;
-}
-
-static int get_plugin_nr(uint type)
-{
-  uint i= 0;
-  for(; valid_plugins[i][1]; i++)
-    if (valid_plugins[i][0] == type)
-      return i;
-  return -1;
-}
-
-static const char *check_plugin_version(struct st_mysql_client_plugin *plugin, unsigned int version)
-{
-  if (plugin->interface_version < version ||
-      (plugin->interface_version >> 8) > (version >> 8))
-    return "Incompatible client plugin interface";
-  return 0;
-}
-
-/**
-  finds a plugin in the list
-
-  @param name   plugin name to search for
-  @param type   plugin type
-
-  @note this does NOT necessarily need a mutex, take care!
-  
-  @retval a pointer to a found plugin or 0
-*/
-static struct st_mysql_client_plugin *find_plugin(const char *name, int type)
-{
-  struct st_client_plugin_int *p;
-  int plugin_nr= get_plugin_nr(type);
-
-  DBUG_ASSERT(initialized);
-  if (plugin_nr == -1)
-    return 0;
-
-  if (!name)
-    return plugin_list[plugin_nr]->plugin;
-
-  for (p= plugin_list[plugin_nr]; p; p= p->next)
-  {
-    if (strcmp(p->plugin->name, name) == 0)
-      return p->plugin;
-  }
-  return NULL;
-}
-
-
-/**
-  verifies the plugin and adds it to the list
-
-  @param mysql          MYSQL structure (for error reporting)
-  @param plugin         plugin to install
-  @param dlhandle       a handle to the shared object (returned by dlopen)
-                        or 0 if the plugin was not dynamically loaded
-  @param argc           number of arguments in the 'va_list args'
-  @param args           arguments passed to the plugin initialization function
-
-  @retval a pointer to an installed plugin or 0
-*/
-
-static struct st_mysql_client_plugin *
-add_plugin(MYSQL *mysql, struct st_mysql_client_plugin *plugin, void *dlhandle,
-           int argc, va_list args)
-{
-  const char *errmsg;
-  struct st_client_plugin_int plugin_int, *p;
-  char errbuf[1024];
-  int plugin_nr;
-
-  DBUG_ASSERT(initialized);
-
-  plugin_int.plugin= plugin;
-  plugin_int.dlhandle= dlhandle;
-
-  if ((plugin_nr= get_plugin_nr(plugin->type)) == -1)
-  {
-    errmsg= "Unknown client plugin type";
-    goto err1;
-  }
-  if ((errmsg= check_plugin_version(plugin, valid_plugins[plugin_nr][1])))
-    goto err1;
-
-  /* Call the plugin initialization function, if any */
-  if (plugin->init && plugin->init(errbuf, sizeof(errbuf), argc, args))
-  {
-    errmsg= errbuf;
-    goto err1;
-  }
-
-  p= (struct st_client_plugin_int *)
-    ma_memdup_root(&mem_root, (char *)&plugin_int, sizeof(plugin_int));
-
-  if (!p)
-  {
-    errmsg= "Out of memory";
-    goto err2;
-  }
-
-#ifdef THREAD
-  safe_mutex_assert_owner(&LOCK_load_client_plugin);
-#endif
-
-  p->next= plugin_list[plugin_nr];
-  plugin_list[plugin_nr]= p;
-
-  return plugin;
-
-err2:
-  if (plugin->deinit)
-    plugin->deinit();
-err1:
-  my_set_error(mysql, CR_AUTH_PLUGIN_CANNOT_LOAD, SQLSTATE_UNKNOWN,
-               ER(CR_AUTH_PLUGIN_CANNOT_LOAD), plugin->name, errmsg);
-  if (dlhandle)
-    (void)dlclose(dlhandle);
-  return NULL;
-}
-
-
-/**
-  Loads plugins which are specified in the environment variable
-  LIBMYSQL_PLUGINS.
-  
-  Multiple plugins must be separated by semicolon. This function doesn't
-  return or log an error.
-
-  The function is be called by mysql_client_plugin_init
-
-  @todo
-  Support extended syntax, passing parameters to plugins, for example
-  LIBMYSQL_PLUGINS="plugin1(param1,param2);plugin2;..."
-  or
-  LIBMYSQL_PLUGINS="plugin1=int:param1,str:param2;plugin2;..."
-*/
-
-static void load_env_plugins(MYSQL *mysql)
-{
-  char *plugs, *free_env, *s= getenv("LIBMYSQL_PLUGINS");
-
-  if (ma_check_env_str(s))
-    return;
-
-  free_env= strdup(s);
-  plugs= s= free_env;
-
-  do {
-    if ((s= strchr(plugs, ';')))
-      *s= '\0';
-    mysql_load_plugin(mysql, plugs, -1, 0);
-    plugs= s + 1;
-  } while (s);
-
-  free(free_env);
-}
-
-/********** extern functions to be used by libmariadb *********************/
-
-/**
-  Initializes the client plugin layer.
-
-  This function must be called before any other client plugin function.
-
-  @retval 0    successful
-  @retval != 0 error occurred
-*/
-
-int mysql_client_plugin_init()
-{
-  MYSQL mysql;
-  struct st_mysql_client_plugin **builtin;
-  va_list unused;
-  LINT_INIT_STRUCT(unused);
-
-  if (initialized)
-    return 0;
-
-  memset(&mysql, 0, sizeof(mysql)); /* dummy mysql for set_mysql_extended_error */
-
-  pthread_mutex_init(&LOCK_load_client_plugin, MY_MUTEX_INIT_SLOW);
-  ma_init_alloc_root(&mem_root, 128, 128);
-
-  memset(&plugin_list, 0, sizeof(plugin_list));
-
-  initialized= 1;
-
-  pthread_mutex_lock(&LOCK_load_client_plugin);
-  for (builtin= mysql_client_builtins; *builtin; builtin++)
-    add_plugin(&mysql, *builtin, 0, 0, unused);
-
-  pthread_mutex_unlock(&LOCK_load_client_plugin);
-
-  load_env_plugins(&mysql);
-
-  return 0;
-}
-
-
-/**
-  Deinitializes the client plugin layer.
-
-  Unloades all client plugins and frees any associated resources.
-*/
-
-void mysql_client_plugin_deinit()
-{
-  int i;
-  struct st_client_plugin_int *p;
-
-  if (!initialized)
-    return;
-
-  for (i=0; i < MYSQL_CLIENT_MAX_PLUGINS; i++)
-    for (p= plugin_list[i]; p; p= p->next)
-    {
-      if (p->plugin->deinit)
-        p->plugin->deinit();
-      if (p->dlhandle)
-        (void)dlclose(p->dlhandle);
-    }
-
-  memset(&plugin_list, 0, sizeof(plugin_list));
-  initialized= 0;
-  ma_free_root(&mem_root, MYF(0));
-  pthread_mutex_destroy(&LOCK_load_client_plugin);
-}
-
-/************* public facing functions, for client consumption *********/
-
-/* see <mysql/client_plugin.h> for a full description */
-struct st_mysql_client_plugin * STDCALL
-mysql_client_register_plugin(MYSQL *mysql,
-                             struct st_mysql_client_plugin *plugin)
-{
-  va_list unused;
-  LINT_INIT_STRUCT(unused);
-
-  if (is_not_initialized(mysql, plugin->name))
-    return NULL;
-
-  pthread_mutex_lock(&LOCK_load_client_plugin);
-
-  /* make sure the plugin wasn't loaded meanwhile */
-  if (find_plugin(plugin->name, plugin->type))
-  {
-    my_set_error(mysql, CR_AUTH_PLUGIN_CANNOT_LOAD,
-                 SQLSTATE_UNKNOWN, ER(CR_AUTH_PLUGIN_CANNOT_LOAD),
-                 plugin->name, "it is already loaded");
-    plugin= NULL;
-  }
-  else
-    plugin= add_plugin(mysql, plugin, 0, 0, unused);
-
-  pthread_mutex_unlock(&LOCK_load_client_plugin);
-  return plugin;
-}
-
-
-/* see <mysql/client_plugin.h> for a full description */
-struct st_mysql_client_plugin * STDCALL
-mysql_load_plugin_v(MYSQL *mysql, const char *name, int type,
-                    int argc, va_list args)
-{
-  const char *errmsg;
-#ifdef _WIN32
-  char errbuf[1024];
-#endif
-  char dlpath[FN_REFLEN+1];
-  void *sym, *dlhandle = NULL;
-  struct st_mysql_client_plugin *plugin;
-  char *env_plugin_dir= getenv("MARIADB_PLUGIN_DIR");
-
-  CLEAR_CLIENT_ERROR(mysql);
-  if (is_not_initialized(mysql, name))
-    return NULL;
-
-  pthread_mutex_lock(&LOCK_load_client_plugin);
-
-  /* make sure the plugin wasn't loaded meanwhile */
-  if (type >= 0 && find_plugin(name, type))
-  {
-    errmsg= "it is already loaded";
-    goto err;
-  }
-
-  /* Compile dll path */
-  snprintf(dlpath, sizeof(dlpath) - 1, "%s/%s%s",
-           mysql->options.extension && mysql->options.extension->plugin_dir ?
-           mysql->options.extension->plugin_dir : (env_plugin_dir) ? env_plugin_dir :
-           MARIADB_PLUGINDIR, name, SO_EXT);
-
-  /* Open new dll handle */
-  if (!(dlhandle= dlopen((const char *)dlpath, RTLD_NOW)))
-  {
-#ifdef _WIN32
-   char winmsg[255];
-   size_t len;
-   winmsg[0] = 0;
-   FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM,
-                 NULL,
-                 GetLastError(),
-                 MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
-                 winmsg, 255, NULL);
-   len= strlen(winmsg);
-   while (len > 0 && (winmsg[len - 1] == '\n' || winmsg[len - 1] == '\r'))
-     len--;
-   if (len)
-     winmsg[len] = 0;
-   snprintf(errbuf, sizeof(errbuf), "%s Library path is '%s'", winmsg, dlpath);
-   errmsg= errbuf;
-#else
-    errmsg= dlerror();
-#endif
-    goto err;
-  }
-
-
-  if (!(sym= dlsym(dlhandle, plugin_declarations_sym)))
-  {
-    errmsg= "not a plugin";
-    (void)dlclose(dlhandle);
-    goto err;
-  }
-
-  plugin= (struct st_mysql_client_plugin*)sym;
-
-  if (type >=0 && type != plugin->type)
-  {
-    errmsg= "type mismatch";
-    goto err;
-  }
-
-  if (strcmp(name, plugin->name))
-  {
-    errmsg= "name mismatch";
-    goto err;
-  }
-
-  if (type < 0 && find_plugin(name, plugin->type))
-  {
-    errmsg= "it is already loaded";
-    goto err;
-  }
-
-  plugin= add_plugin(mysql, plugin, dlhandle, argc, args);
-
-  pthread_mutex_unlock(&LOCK_load_client_plugin);
-
-  return plugin;
-
-err:
-  if (dlhandle)
-    dlclose(dlhandle);
-  pthread_mutex_unlock(&LOCK_load_client_plugin);
-  my_set_error(mysql, CR_AUTH_PLUGIN_CANNOT_LOAD, SQLSTATE_UNKNOWN,
-               ER(CR_AUTH_PLUGIN_CANNOT_LOAD), name, errmsg);
-  return NULL;
-}
-
-
-/* see <mysql/client_plugin.h> for a full description */
-struct st_mysql_client_plugin * STDCALL
-mysql_load_plugin(MYSQL *mysql, const char *name, int type, int argc, ...)
-{
-  struct st_mysql_client_plugin *p;
-  va_list args;
-  va_start(args, argc);
-  p= mysql_load_plugin_v(mysql, name, type, argc, args);
-  va_end(args);
-  return p;
-}
-
-/* see <mysql/client_plugin.h> for a full description */
-struct st_mysql_client_plugin * STDCALL
-mysql_client_find_plugin(MYSQL *mysql, const char *name, int type)
-{
-  struct st_mysql_client_plugin *p;
-  int plugin_nr= get_plugin_nr(type);
-
-  if (is_not_initialized(mysql, name))
-    return NULL;
-
-  if (plugin_nr == -1)
-  {
-    my_set_error(mysql, CR_AUTH_PLUGIN_CANNOT_LOAD, SQLSTATE_UNKNOWN,
-                 ER(CR_AUTH_PLUGIN_CANNOT_LOAD), name, "invalid type");
-  }
-
-  if ((p= find_plugin(name, type)))
-    return p;
-
-  /* not found, load it */
-  return mysql_load_plugin(mysql, name, type, 0);
-}
-

contrib/zlib-ng

@@ -1 +1 @@
-Subproject commit cb43e7fa08ec29fd76d84e3bb35258a0f0bf3df3
+Subproject commit cff0f500d9399d7cd3b9461a693d211e4b86fcc9

dbms/CMakeLists.txt

@@ -100,6 +100,7 @@ set(dbms_sources)
 add_headers_and_sources(clickhouse_common_io src/Common)
 add_headers_and_sources(clickhouse_common_io src/Common/HashTable)
 add_headers_and_sources(clickhouse_common_io src/IO)
+list (REMOVE_ITEM clickhouse_common_io_sources src/Common/new_delete.cpp)
 
 if(USE_RDKAFKA)
     add_headers_and_sources(dbms src/Storages/Kafka)
@@ -132,8 +133,16 @@ list (APPEND dbms_headers src/TableFunctions/ITableFunction.h   src/TableFunctio
 list (APPEND dbms_sources src/Dictionaries/DictionaryFactory.cpp src/Dictionaries/DictionarySourceFactory.cpp src/Dictionaries/DictionaryStructure.cpp)
 list (APPEND dbms_headers src/Dictionaries/DictionaryFactory.h   src/Dictionaries/DictionarySourceFactory.h   src/Dictionaries/DictionaryStructure.h)
 
+if (NOT ENABLE_SSL)
+    list (REMOVE_ITEM clickhouse_common_io_sources src/Common/OpenSSLHelpers.cpp)
+    list (REMOVE_ITEM clickhouse_common_io_headers src/Common/OpenSSLHelpers.h)
+endif ()
+
 add_library(clickhouse_common_io ${clickhouse_common_io_headers} ${clickhouse_common_io_sources})
 
+add_library (clickhouse_new_delete STATIC src/Common/new_delete.cpp)
+target_link_libraries (clickhouse_new_delete PRIVATE clickhouse_common_io)
+
 if (OS_FREEBSD)
     target_compile_definitions (clickhouse_common_io PUBLIC CLOCK_MONOTONIC_COARSE=CLOCK_MONOTONIC_FAST)
 endif ()
@@ -156,7 +165,7 @@ macro(add_object_library name common_path)
 endmacro()
 
 add_object_library(clickhouse_core src/Core)
-add_object_library(clickhouse_compression src/Compression/)
+add_object_library(clickhouse_compression src/Compression)
 add_object_library(clickhouse_datastreams src/DataStreams)
 add_object_library(clickhouse_datatypes src/DataTypes)
 add_object_library(clickhouse_databases src/Databases)
@@ -414,17 +423,7 @@ endif()
 
 if (USE_JEMALLOC)
     dbms_target_include_directories (SYSTEM BEFORE PRIVATE ${JEMALLOC_INCLUDE_DIR}) # used in Interpreters/AsynchronousMetrics.cpp
-    target_include_directories (clickhouse_common_io SYSTEM BEFORE PRIVATE ${JEMALLOC_INCLUDE_DIR}) # new_delete.cpp
+    target_include_directories (clickhouse_new_delete SYSTEM BEFORE PRIVATE ${JEMALLOC_INCLUDE_DIR})
-    # common/memory.h
-    if (MAKE_STATIC_LIBRARIES OR NOT SPLIT_SHARED_LIBRARIES)
-        # skip if we have bundled build, since jemalloc is static in this case
-    elseif (${JEMALLOC_LIBRARIES} MATCHES "${CMAKE_STATIC_LIBRARY_SUFFIX}$")
-        # if the library is static we do not need to link with it,
-        # since in this case it will be in libs/libcommon,
-        # and we do not want to link with jemalloc multiple times.
-    else()
-        target_link_libraries(clickhouse_common_io PRIVATE ${JEMALLOC_LIBRARIES})
-    endif()
 endif ()
 
 dbms_target_include_directories (PUBLIC ${DBMS_INCLUDE_DIR} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/src/Formats/include)

dbms/cmake/version.cmake

@@ -1,11 +1,11 @@
 # This strings autochanged from release_lib.sh:
-set(VERSION_REVISION 54426)
+set(VERSION_REVISION 54427)
 set(VERSION_MAJOR 19)
-set(VERSION_MINOR 15)
+set(VERSION_MINOR 16)
 set(VERSION_PATCH 1)
-set(VERSION_GITHASH 6f1a8c37abe6ee4e7ee74c0b5cb9c05a87417b61)
+set(VERSION_GITHASH 38f65a6a2120d2e76bcf71131068f41195149dfc)
-set(VERSION_DESCRIBE v19.15.1.1-prestable)
+set(VERSION_DESCRIBE v19.16.1.1-prestable)
-set(VERSION_STRING 19.15.1.1)
+set(VERSION_STRING 19.16.1.1)
 # end of autochange
 
 set(VERSION_EXTRA "" CACHE STRING "")

dbms/programs/CMakeLists.txt

@@ -24,9 +24,9 @@ configure_file (config_tools.h.in ${CMAKE_CURRENT_BINARY_DIR}/config_tools.h)
 
 macro(clickhouse_target_link_split_lib target name)
     if(NOT CLICKHOUSE_ONE_SHARED)
-        target_link_libraries(${target} PRIVATE clickhouse-${name}-lib ${MALLOC_LIBRARIES})
+        target_link_libraries(${target} PRIVATE clickhouse-${name}-lib)
     else()
-        target_link_libraries(${target} PRIVATE clickhouse-lib ${MALLOC_LIBRARIES})
+        target_link_libraries(${target} PRIVATE clickhouse-lib)
     endif()
 endmacro()
 
@@ -111,7 +111,7 @@ if (CLICKHOUSE_SPLIT_BINARY)
     install(PROGRAMS clickhouse-split-helper DESTINATION ${CMAKE_INSTALL_BINDIR} RENAME clickhouse COMPONENT clickhouse)
 else ()
     add_executable (clickhouse main.cpp)
-    target_link_libraries (clickhouse PRIVATE clickhouse_common_io string_utils ${MALLOC_LIBRARIES})
+    target_link_libraries (clickhouse PRIVATE clickhouse_common_io string_utils)
     target_include_directories (clickhouse BEFORE PRIVATE ${COMMON_INCLUDE_DIR})
     target_include_directories (clickhouse PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
 

dbms/programs/client/readpassphrase/readpassphrase.c

@@ -46,7 +46,7 @@
 #  define _POSIX_VDISABLE       VDISABLE
 #endif
 
-static volatile sig_atomic_t signo[_NSIG];
+static volatile sig_atomic_t signo[NSIG];
 
 static void handler(int);
 
@@ -67,7 +67,7 @@ readpassphrase(const char *prompt, char *buf, size_t bufsiz, int flags)
 	}
 
 restart:
-	for (i = 0; i < _NSIG; i++)
+	for (i = 0; i < NSIG; i++)
 		signo[i] = 0;
 	nr = -1;
 	save_errno = 0;
@@ -173,7 +173,7 @@ restart:
 	 * If we were interrupted by a signal, resend it to ourselves
 	 * now that we have restored the signal handlers.
 	 */
-	for (i = 0; i < _NSIG; i++) {
+	for (i = 0; i < NSIG; i++) {
 		if (signo[i]) {
 			kill(getpid(), i);
 			switch (i) {

dbms/programs/obfuscator/Obfuscator.cpp

@@ -670,13 +670,13 @@ public:
 
         while (pos < end)
         {
-            Table::iterator it = table.end();
+            Table::LookupResult it;
 
             size_t context_size = params.order;
             while (true)
             {
                 it = table.find(hashContext(code_points.data() + code_points.size() - context_size, code_points.data() + code_points.size()));
-                if (table.end() != it && it->getSecond().total + it->getSecond().count_end != 0)
+                if (it && lookupResultGetMapped(it)->total + lookupResultGetMapped(it)->count_end != 0)
                     break;
 
                 if (context_size == 0)
@@ -684,7 +684,7 @@ public:
                 --context_size;
             }
 
-            if (table.end() == it)
+            if (!it)
                 throw Exception("Logical error in markov model", ErrorCodes::LOGICAL_ERROR);
 
             size_t offset_from_begin_of_string = pos - data;
@@ -710,7 +710,7 @@ public:
             if (num_bytes_after_desired_size > 0)
                 end_probability_multiplier = std::pow(1.25, num_bytes_after_desired_size);
 
-            CodePoint code = it->getSecond().sample(determinator, end_probability_multiplier);
+            CodePoint code = lookupResultGetMapped(it)->sample(determinator, end_probability_multiplier);
 
             if (code == END)
                 break;

dbms/programs/odbc-bridge/CMakeLists.txt

@@ -33,9 +33,11 @@ endif ()
 
 clickhouse_program_add_library(odbc-bridge)
 
-# clickhouse-odbc-bridge is always a separate binary.
+if (OS_LINUX)
-# Reason: it must not export symbols from SSL, mariadb-client, etc. to not break ABI compatibility with ODBC drivers.
+    # clickhouse-odbc-bridge is always a separate binary.
-set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-export-dynamic")
+    # Reason: it must not export symbols from SSL, mariadb-client, etc. to not break ABI compatibility with ODBC drivers.
+    set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-export-dynamic")
+endif ()
 
 add_executable(clickhouse-odbc-bridge odbc-bridge.cpp)
 set_target_properties(clickhouse-odbc-bridge PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..)

dbms/programs/server/MetricsTransmitter.cpp

@@ -21,6 +21,7 @@ MetricsTransmitter::MetricsTransmitter(
 {
     interval_seconds = config.getInt(config_name + ".interval", 60);
     send_events = config.getBool(config_name + ".events", true);
+    send_events_cumulative = config.getBool(config_name + ".events_cumulative", false);
     send_metrics = config.getBool(config_name + ".metrics", true);
     send_asynchronous_metrics = config.getBool(config_name + ".asynchronous_metrics", true);
 }
@@ -95,6 +96,16 @@ void MetricsTransmitter::transmit(std::vector<ProfileEvents::Count> & prev_count
         }
     }
 
+    if (send_events_cumulative)
+    {
+        for (size_t i = 0, end = ProfileEvents::end(); i < end; ++i)
+        {
+            const auto counter = ProfileEvents::global_counters[i].load(std::memory_order_relaxed);
+            std::string key{ProfileEvents::getName(static_cast<ProfileEvents::Event>(i))};
+            key_vals.emplace_back(profile_events_cumulative_path_prefix + key, counter);
+        }
+    }
+
     if (send_metrics)
     {
         for (size_t i = 0, end = CurrentMetrics::end(); i < end; ++i)

dbms/programs/server/MetricsTransmitter.h

@@ -24,7 +24,8 @@ class AsynchronousMetrics;
 
 
 /** Automatically sends
-  * - difference of ProfileEvents;
+  * - delta values of ProfileEvents;
+  * - cumulative values of ProfileEvents;
   * - values of CurrentMetrics;
   * - values of AsynchronousMetrics;
   *  to Graphite at beginning of every minute.
@@ -44,6 +45,7 @@ private:
     std::string config_name;
     UInt32 interval_seconds;
     bool send_events;
+    bool send_events_cumulative;
     bool send_metrics;
     bool send_asynchronous_metrics;
 
@@ -53,6 +55,7 @@ private:
     ThreadFromGlobalPool thread{&MetricsTransmitter::run, this};
 
     static inline constexpr auto profile_events_path_prefix = "ClickHouse.ProfileEvents.";
+    static inline constexpr auto profile_events_cumulative_path_prefix = "ClickHouse.ProfileEventsCumulative.";
     static inline constexpr auto current_metrics_path_prefix = "ClickHouse.Metrics.";
     static inline constexpr auto asynchronous_metrics_path_prefix = "ClickHouse.AsynchronousMetrics.";
 };

dbms/programs/server/Server.cpp

@@ -36,7 +36,7 @@
 #include <IO/UseSSL.h>
 #include <Interpreters/AsynchronousMetrics.h>
 #include <Interpreters/DDLWorker.h>
-#include <Interpreters/ExternalDictionaries.h>
+#include <Interpreters/ExternalDictionariesLoader.h>
 #include <Interpreters/ProcessList.h>
 #include <Interpreters/loadMetadata.h>
 #include <Interpreters/DNSCacheUpdater.h>
@@ -54,16 +54,16 @@
 #include <Common/StatusFile.h>
 #include "TCPHandlerFactory.h"
 #include "Common/config_version.h"
-#include "MySQLHandlerFactory.h"
 #include <Common/SensitiveDataMasker.h>
 
 
-#if defined(__linux__)
+#if defined(OS_LINUX)
 #include <Common/hasLinuxCapability.h>
 #include <sys/mman.h>
 #endif
 
 #if USE_POCO_NETSSL
+#include "MySQLHandlerFactory.h"
 #include <Poco/Net/Context.h>
 #include <Poco/Net/SecureServerSocket.h>
 #endif
@@ -918,7 +918,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
             if (!config().getBool("dictionaries_lazy_load", true))
             {
                 global_context->tryCreateEmbeddedDictionaries();
-                global_context->getExternalDictionaries().enableAlwaysLoadEverything(true);
+                global_context->getExternalDictionariesLoader().enableAlwaysLoadEverything(true);
             }
         }
         catch (...)

dbms/programs/server/config.xml

@@ -258,6 +258,7 @@
 
         <metrics>true</metrics>
         <events>true</events>
+        <events_cumulative>false</events_cumulative>
         <asynchronous_metrics>true</asynchronous_metrics>
     </graphite>
     <graphite>
@@ -269,6 +270,7 @@
 
         <metrics>true</metrics>
         <events>true</events>
+        <events_cumulative>false</events_cumulative>
         <asynchronous_metrics>false</asynchronous_metrics>
     </graphite>
     -->

dbms/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h

@@ -63,7 +63,12 @@ public:
             roaring_bitmap_add(rb, value);
     }
 
-    UInt64 size() const { return isSmall() ? small.size() : roaring_bitmap_get_cardinality(rb); }
+    UInt64 size() const
+    {
+        return isSmall()
+            ? small.size()
+            : roaring_bitmap_get_cardinality(rb);
+    }
 
     void merge(const RoaringBitmapWithSmallSet & r1)
     {
@@ -91,7 +96,7 @@ public:
             std::string s;
             readStringBinary(s,in);
             rb = roaring_bitmap_portable_deserialize(s.c_str());
-            for (const auto & x : small) //merge from small
+            for (const auto & x : small) // merge from small
                 roaring_bitmap_add(rb, x.getValue());
         }
         else
@@ -245,13 +250,13 @@ public:
         {
             for (const auto & x : small)
                 if (r1.small.find(x.getValue()) != r1.small.end())
-                    retSize++;
+                    ++retSize;
         }
         else if (isSmall() && r1.isLarge())
         {
             for (const auto & x : small)
                 if (roaring_bitmap_contains(r1.rb, x.getValue()))
-                    retSize++;
+                    ++retSize;
         }
         else
         {
@@ -391,8 +396,7 @@ public:
      */
     UInt8 rb_contains(const UInt32 x) const
     {
-        return isSmall() ? small.find(x) != small.end() :
+        return isSmall() ? small.find(x) != small.end() : roaring_bitmap_contains(rb, x);
-            roaring_bitmap_contains(rb, x);
     }
 
     /**
@@ -460,21 +464,20 @@ public:
     /**
      * Return new set with specified range (not include the range_end)
      */
-    UInt64 rb_range(UInt32 range_start, UInt32 range_end, RoaringBitmapWithSmallSet& r1) const
+    UInt64 rb_range(UInt32 range_start, UInt32 range_end, RoaringBitmapWithSmallSet & r1) const
     {
         UInt64 count = 0;
         if (range_start >= range_end)
             return count;
         if (isSmall())
         {
-            std::vector<T> ans;
             for (const auto & x : small)
             {
                 T val = x.getValue();
-                if ((UInt32)val >= range_start && (UInt32)val < range_end)
+                if (UInt32(val) >= range_start && UInt32(val) < range_end)
                 {
                     r1.add(val);
-                    count++;
+                    ++count;
                 }
             }
         }
@@ -483,13 +486,50 @@ public:
             roaring_uint32_iterator_t iterator;
             roaring_init_iterator(rb, &iterator);
             roaring_move_uint32_iterator_equalorlarger(&iterator, range_start);
-            while (iterator.has_value)
+            while (iterator.has_value && UInt32(iterator.current_value) < range_end)
             {
-                if ((UInt32)iterator.current_value >= range_end)
-                    break;
                 r1.add(iterator.current_value);
                 roaring_advance_uint32_iterator(&iterator);
-                count++;
+                ++count;
+            }
+        }
+        return count;
+    }
+
+    /**
+     * Return new set of the smallest `limit` values in set which is no less than `range_start`.
+     */
+    UInt64 rb_limit(UInt32 range_start, UInt32 limit, RoaringBitmapWithSmallSet & r1) const
+    {
+        UInt64 count = 0;
+        if (isSmall())
+        {
+            std::vector<T> ans;
+            for (const auto & x : small)
+            {
+                T val = x.getValue();
+                if (UInt32(val) >= range_start)
+                {
+                    ans.push_back(val);
+                }
+            }
+            sort(ans.begin(), ans.end());
+            if (limit > ans.size())
+                limit = ans.size();
+            for (size_t i = 0; i < limit; ++i)
+                r1.add(ans[i]);
+            count = UInt64(limit);
+        }
+        else
+        {
+            roaring_uint32_iterator_t iterator;
+            roaring_init_iterator(rb, &iterator);
+            roaring_move_uint32_iterator_equalorlarger(&iterator, range_start);
+            while (UInt32(count) < limit && iterator.has_value)
+            {
+                r1.add(iterator.current_value);
+                roaring_advance_uint32_iterator(&iterator);
+                ++count;
             }
         }
         return count;
@@ -552,8 +592,8 @@ private:
         readBinary(val, dbBuf);
         container = containerptr_roaring_bitmap_add(r, val, &typecode, &containerindex);
         prev = val;
-        i++;
+        ++i;
-        for (; i < n_args; i++)
+        for (; i < n_args; ++i)
         {
             readBinary(val, dbBuf);
             if (((prev ^ val) >> 16) == 0)

dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h

@@ -218,7 +218,7 @@ public:
             return;
 
         bool inserted;
-        State::Set::iterator it;
+        State::Set::LookupResult it;
         auto key_holder = getKeyHolder(*columns[0], row_num, *arena);
         set.emplace(key_holder, it, inserted);
     }
@@ -229,7 +229,7 @@ public:
         auto & rhs_set = this->data(rhs).value;
 
         bool inserted;
-        State::Set::iterator it;
+        State::Set::LookupResult it;
         for (auto & rhs_elem : rhs_set)
         {
             if (limit_num_elems && cur_set.size() >= max_elems)

dbms/src/Client/tests/CMakeLists.txt

@@ -1,2 +1,2 @@
 add_executable(test-connect test_connect.cpp)
-target_link_libraries (test-connect dbms)
+target_link_libraries (test-connect PRIVATE dbms)

dbms/src/Columns/ReverseIndex.h

@@ -151,6 +151,7 @@ namespace
     public:
         using Base::Base;
         using iterator = typename Base::iterator;
+        using LookupResult = typename Base::LookupResult;
         State & getState() { return *this; }
 
 
@@ -168,7 +169,7 @@ namespace
         }
 
         template <typename ObjectToCompareWith>
-        void ALWAYS_INLINE reverseIndexEmplaceNonZero(const Key & key, iterator & it,
+        void ALWAYS_INLINE reverseIndexEmplaceNonZero(const Key & key, LookupResult & it,
             bool & inserted, size_t hash_value, const ObjectToCompareWith & object)
         {
             size_t place_value = reverseIndexFindCell(object, hash_value,
@@ -184,10 +185,14 @@ namespace
         void ALWAYS_INLINE reverseIndexEmplace(Key key, iterator & it, bool & inserted,
             size_t hash_value, const ObjectToCompareWith& object)
         {
-            if (!this->emplaceIfZero(key, it, inserted, hash_value))
+            LookupResult impl_it = nullptr;
+
+            if (!this->emplaceIfZero(key, impl_it, inserted, hash_value))
             {
-                reverseIndexEmplaceNonZero(key, it, inserted, hash_value, object);
+                reverseIndexEmplaceNonZero(key, impl_it, inserted, hash_value, object);
             }
+            assert(impl_it != nullptr);
+            it = iterator(this, impl_it);
         }
 
         template <typename ObjectToCompareWith>

dbms/src/Common/Allocator.h

@@ -179,12 +179,11 @@ protected:
     // MAP_POPULATE to mmap(). This takes some time, but should be faster
     // overall than having a hot loop interrupted by page faults.
     // It is only supported on Linux.
-#if defined(__linux__)
     static constexpr int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS
-            | (mmap_populate ? MAP_POPULATE : 0);
+#if defined(OS_LINUX)
-#else
+        | (mmap_populate ? MAP_POPULATE : 0)
-    static constexpr int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS;
 #endif
+        ;
 
 private:
     void * allocNoTrack(size_t size, size_t alignment)

dbms/src/Common/ColumnsHashing.h

@@ -349,7 +349,7 @@ struct HashMethodSingleLowCardinalityColumn : public SingleColumnMethod
         auto key_holder = getKeyHolder(row_, pool);
 
         bool inserted = false;
-        typename Data::iterator it;
+        typename Data::LookupResult it;
         if (saved_hash)
             data.emplace(key_holder, it, inserted, saved_hash[row]);
         else
@@ -359,12 +359,13 @@ struct HashMethodSingleLowCardinalityColumn : public SingleColumnMethod
 
         if constexpr (has_mapped)
         {
+            auto & mapped = *lookupResultGetMapped(it);
             if (inserted)
             {
-                new (&it->getSecond()) Mapped();
+                new (&mapped) Mapped();
             }
-            mapped_cache[row] = it->getSecond();
+            mapped_cache[row] = mapped;
-            return EmplaceResult(it->getSecond(), mapped_cache[row], inserted);
+            return EmplaceResult(mapped, mapped_cache[row], inserted);
         }
         else
             return EmplaceResult(inserted);

dbms/src/Common/ColumnsHashingImpl.h

@@ -168,34 +168,41 @@ protected:
             }
         }
 
-        typename Data::iterator it;
+        typename Data::LookupResult it;
         bool inserted = false;
         data.emplace(key_holder, it, inserted);
 
         [[maybe_unused]] Mapped * cached = nullptr;
         if constexpr (has_mapped)
-            cached = &it->getSecond();
+            cached = lookupResultGetMapped(it);
 
         if (inserted)
         {
             if constexpr (has_mapped)
             {
-                new(&it->getSecond()) Mapped();
+                new(lookupResultGetMapped(it)) Mapped();
             }
         }
 
         if constexpr (consecutive_keys_optimization)
         {
-            cache.value = it->getValue();
             cache.found = true;
             cache.empty = false;
 
             if constexpr (has_mapped)
+            {
+                cache.value.first = *lookupResultGetKey(it);
+                cache.value.second = *lookupResultGetMapped(it);
                 cached = &cache.value.second;
+            }
+            else
+            {
+                cache.value = *lookupResultGetKey(it);
+            }
         }
 
         if constexpr (has_mapped)
-            return EmplaceResult(it->getSecond(), *cached, inserted);
+            return EmplaceResult(*lookupResultGetMapped(it), *cached, inserted);
         else
             return EmplaceResult(inserted);
     }
@@ -215,28 +222,30 @@ protected:
         }
 
         auto it = data.find(key);
-        bool found = it != data.end();
 
         if constexpr (consecutive_keys_optimization)
         {
-            cache.found = found;
+            cache.found = it != nullptr;
             cache.empty = false;
 
-            if (found)
+            if constexpr (has_mapped)
-                cache.value = it->getValue();
+            {
+                cache.value.first = key;
+                if (it)
+                {
+                    cache.value.second = *lookupResultGetMapped(it);
+                }
+            }
             else
             {
-                if constexpr (has_mapped)
+                cache.value = key;
-                    cache.value.first = key;
-                else
-                    cache.value = key;
             }
         }
 
         if constexpr (has_mapped)
-            return FindResult(found ? &it->getSecond() : nullptr, found);
+            return FindResult(it ? lookupResultGetMapped(it) : nullptr, it != nullptr);
         else
-            return FindResult(found);
+            return FindResult(it != nullptr);
     }
 };
 

dbms/src/Common/DiskSpaceMonitor.cpp

@@ -1,10 +1,12 @@
 #include <Common/DiskSpaceMonitor.h>
+#include <Common/escapeForFileName.h>
+#include <IO/WriteHelpers.h>
 
 #include <set>
 
-#include <Common/escapeForFileName.h>
 #include <Poco/File.h>
 
+
 namespace DB
 {
 
@@ -45,7 +47,7 @@ std::filesystem::path getMountPoint(std::filesystem::path absolute_path)
     return absolute_path;
 }
 
-    /// Returns name of filesystem mounted to mount_point
+/// Returns name of filesystem mounted to mount_point
 #if !defined(__linux__)
 [[noreturn]]
 #endif
@@ -65,7 +67,7 @@ std::string getFilesystemName([[maybe_unused]] const std::string & mount_point)
         throw DB::Exception("Cannot find name of filesystem by mount point " + mount_point, ErrorCodes::SYSTEM_ERROR);
     return fs_info.mnt_fsname;
 #else
-    throw DB::Exception("Supported on linux only", ErrorCodes::NOT_IMPLEMENTED);
+    throw DB::Exception("The function getFilesystemName is supported on Linux only", ErrorCodes::NOT_IMPLEMENTED);
 #endif
 }
 
@@ -82,7 +84,7 @@ bool Disk::tryReserve(UInt64 bytes) const
     std::lock_guard lock(mutex);
     if (bytes == 0)
     {
-        LOG_DEBUG(&Logger::get("DiskSpaceMonitor"), "Reserving 0 bytes on disk " << name);
+        LOG_DEBUG(&Logger::get("DiskSpaceMonitor"), "Reserving 0 bytes on disk " << backQuote(name));
         ++reservation_count;
         return true;
     }
@@ -93,7 +95,8 @@ bool Disk::tryReserve(UInt64 bytes) const
     {
         LOG_DEBUG(
             &Logger::get("DiskSpaceMonitor"),
-            "Reserving " << bytes << " bytes on disk " << name << " having unreserved " << unreserved_space << " bytes.");
+            "Reserving " << formatReadableSizeWithBinarySuffix(bytes) << " on disk " << backQuote(name)
+                << ", having unreserved " << formatReadableSizeWithBinarySuffix(unreserved_space) << ".");
         ++reservation_count;
         reserved_bytes += bytes;
         return true;
@@ -283,14 +286,14 @@ Volume::Volume(
         max_data_part_size = static_cast<decltype(max_data_part_size)>(sum_size * ratio / disks.size());
         for (size_t i = 0; i < disks.size(); ++i)
             if (sizes[i] < max_data_part_size)
-                LOG_WARNING(logger, "Disk " << disks[i]->getName() << " on volume " << config_prefix <<
+                LOG_WARNING(logger, "Disk " << backQuote(disks[i]->getName()) << " on volume " << backQuote(config_prefix) <<
-                                    " have not enough space (" << sizes[i] <<
+                                    " have not enough space (" << formatReadableSizeWithBinarySuffix(sizes[i]) <<
                                     ") for containing part the size of max_data_part_size (" <<
-                                    max_data_part_size << ")");
+                                    formatReadableSizeWithBinarySuffix(max_data_part_size) << ")");
     }
     constexpr UInt64 MIN_PART_SIZE = 8u * 1024u * 1024u;
     if (max_data_part_size < MIN_PART_SIZE)
-        LOG_WARNING(logger, "Volume '" << name << "' max_data_part_size is too low ("
+        LOG_WARNING(logger, "Volume " << backQuote(name) << " max_data_part_size is too low ("
             << formatReadableSizeWithBinarySuffix(max_data_part_size) << " < "
             << formatReadableSizeWithBinarySuffix(MIN_PART_SIZE) << ")");
 }
@@ -505,7 +508,7 @@ StoragePolicySelector::StoragePolicySelector(
                 ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG);
 
         policies.emplace(name, std::make_shared<StoragePolicy>(name, config, config_prefix + "." + name, disks));
-        LOG_INFO(&Logger::get("StoragePolicySelector"), "Storage policy " << name << " loaded");
+        LOG_INFO(&Logger::get("StoragePolicySelector"), "Storage policy " << backQuote(name) << " loaded");
     }
 
     constexpr auto default_storage_policy_name = "default";

dbms/src/Common/ErrorCodes.cpp

@@ -457,6 +457,8 @@ namespace ErrorCodes
     extern const int UNKNOWN_PROTOCOL = 480;
     extern const int PATH_ACCESS_DENIED = 481;
     extern const int DICTIONARY_ACCESS_DENIED = 482;
+    extern const int TOO_MANY_REDIRECTS = 483;
+    extern const int INTERNAL_REDIS_ERROR = 484;
 
     extern const int KEEPER_EXCEPTION = 999;
     extern const int POCO_EXCEPTION = 1000;

dbms/src/Common/HashTable/ClearableHashMap.h

@@ -14,6 +14,11 @@ struct ClearableHashMapCell : public ClearableHashTableCell<Key, HashMapCell<Key
         : Base::BaseCell(value_, state), Base::version(state.version) {}
 };
 
+template<typename Key, typename Mapped, typename Hash>
+ALWAYS_INLINE inline auto lookupResultGetKey(ClearableHashMapCell<Key, Mapped, Hash> * cell) { return &cell->getFirst(); }
+
+template<typename Key, typename Mapped, typename Hash>
+ALWAYS_INLINE inline auto lookupResultGetMapped(ClearableHashMapCell<Key, Mapped, Hash> * cell) { return &cell->getSecond(); }
 
 template
 <
@@ -32,14 +37,14 @@ public:
 
     mapped_type & operator[](Key x)
     {
-        typename ClearableHashMap::iterator it;
+        typename ClearableHashMap::LookupResult it;
         bool inserted;
         this->emplace(x, it, inserted);
 
         if (inserted)
-            new(&it->getSecond()) mapped_type();
+            new(lookupResultGetMapped(it)) mapped_type();
 
-        return it->getSecond();
+        return *lookupResultGetMapped(it);
     }
 
     void clear()

dbms/src/Common/HashTable/ClearableHashSet.h

@@ -48,6 +48,11 @@ struct ClearableHashTableCell : public BaseCell
     ClearableHashTableCell(const Key & key_, const State & state) : BaseCell(key_, state), version(state.version) {}
 };
 
+template<typename Key, typename BaseCell>
+ALWAYS_INLINE inline auto lookupResultGetKey(ClearableHashTableCell<Key, BaseCell> * cell) { return &cell->key; }
+
+template<typename Key, typename BaseCell>
+ALWAYS_INLINE inline void * lookupResultGetMapped(ClearableHashTableCell<Key, BaseCell> *) { return nullptr; }
 
 template
 <
@@ -62,6 +67,9 @@ public:
     using key_type = Key;
     using value_type = typename ClearableHashSet::cell_type::value_type;
 
+    using Base = HashTable<Key, ClearableHashTableCell<Key, HashTableCell<Key, Hash, ClearableHashSetState>>, Hash, Grower, Allocator>;
+    using typename Base::LookupResult;
+
     void clear()
     {
         ++this->version;

dbms/src/Common/HashTable/FixedClearableHashMap.h

@@ -23,7 +23,6 @@ struct FixedClearableHashMapCell
     bool isZero(const State & state) const { return version != state.version; }
     void setZero() { version = 0; }
     static constexpr bool need_zero_value_storage = false;
-    void setMapped(const value_type & value) { mapped = value.getSecond(); }
 
     struct CellExt
     {

dbms/src/Common/HashTable/FixedClearableHashSet.h

@@ -10,6 +10,7 @@ struct FixedClearableHashTableCell
     using State = ClearableHashSetState;
 
     using value_type = Key;
+    using mapped_type = void;
     UInt32 version;
 
     FixedClearableHashTableCell() {}
@@ -18,7 +19,6 @@ struct FixedClearableHashTableCell
     bool isZero(const State & state) const { return version != state.version; }
     void setZero() { version = 0; }
     static constexpr bool need_zero_value_storage = false;
-    void setMapped(const value_type & /*value*/) {}
 
     struct CellExt
     {
@@ -33,8 +33,10 @@ template <typename Key, typename Allocator = HashTableAllocator>
 class FixedClearableHashSet : public FixedHashTable<Key, FixedClearableHashTableCell<Key>, Allocator>
 {
 public:
+    using Base = FixedHashTable<Key, FixedClearableHashTableCell<Key>, Allocator>;
     using key_type = Key;
     using value_type = typename FixedClearableHashSet::cell_type::value_type;
+    using LookupResult = typename Base::LookupResult;
 
     void clear()
     {

dbms/src/Common/HashTable/FixedHashMap.h

@@ -11,6 +11,8 @@ struct FixedHashMapCell
     using State = TState;
 
     using value_type = PairNoInit<Key, Mapped>;
+    using mapped_type = TMapped;
+
     Mapped mapped;
     bool full;
 
@@ -23,7 +25,6 @@ struct FixedHashMapCell
     bool isZero(const State &) const { return !full; }
     void setZero() { full = false; }
     static constexpr bool need_zero_value_storage = false;
-    void setMapped(const value_type & value) { mapped = value.getSecond(); }
 
     /// Similar to FixedHashSetCell except that we need to contain a pointer to the Mapped field.
     ///  Note that we have to assemble a continuous layout for the value_type on each call of getValue().
@@ -46,6 +47,13 @@ struct FixedHashMapCell
     };
 };
 
+template<typename Key, typename Mapped, typename State>
+ALWAYS_INLINE inline void * lookupResultGetKey(FixedHashMapCell<Key, Mapped, State> *)
+{ return nullptr; }
+
+template<typename Key, typename Mapped, typename State>
+ALWAYS_INLINE inline auto lookupResultGetMapped(FixedHashMapCell<Key, Mapped, State> * cell)
+{ return &cell->getSecond(); }
 
 template <typename Key, typename Mapped, typename Allocator = HashTableAllocator>
 class FixedHashMap : public FixedHashTable<Key, FixedHashMapCell<Key, Mapped>, Allocator>
@@ -54,21 +62,23 @@ public:
     using Base = FixedHashTable<Key, FixedHashMapCell<Key, Mapped>, Allocator>;
     using Self = FixedHashMap;
     using key_type = Key;
-    using mapped_type = Mapped;
     using Cell = typename Base::cell_type;
     using value_type = typename Cell::value_type;
+    using mapped_type = typename Cell::Mapped;
 
     using Base::Base;
 
+    using LookupResult = typename Base::LookupResult;
+
     template <typename Func>
     void ALWAYS_INLINE mergeToViaEmplace(Self & that, Func && func)
     {
         for (auto it = this->begin(), end = this->end(); it != end; ++it)
         {
-            decltype(it) res_it;
+            typename Self::LookupResult res_it;
             bool inserted;
             that.emplace(it->getFirst(), res_it, inserted, it.getHash());
-            func(res_it->getSecond(), it->getSecond(), inserted);
+            func(*lookupResultGetMapped(res_it), it->getSecond(), inserted);
         }
     }
 
@@ -77,11 +87,11 @@ public:
     {
         for (auto it = this->begin(), end = this->end(); it != end; ++it)
         {
-            decltype(it) res_it = that.find(it->getFirst(), it.getHash());
+            auto res_it = that.find(it->getFirst(), it.getHash());
-            if (res_it == that.end())
+            if (!res_it)
                 func(it->getSecond(), it->getSecond(), false);
             else
-                func(res_it->getSecond(), it->getSecond(), true);
+                func(*lookupResultGetMapped(res_it), it->getSecond(), true);
         }
     }
 
@@ -101,12 +111,12 @@ public:
 
     mapped_type & ALWAYS_INLINE operator[](Key x)
     {
-        typename Base::iterator it;
+        typename Base::LookupResult it;
         bool inserted;
         this->emplace(x, it, inserted);
         if (inserted)
-            new (&it->getSecond()) mapped_type();
+            new (it) mapped_type();
 
-        return it->getSecond();
+        return it;
     }
 };

dbms/src/Common/HashTable/FixedHashTable.h

@@ -8,6 +8,7 @@ struct FixedHashTableCell
     using State = TState;
 
     using value_type = Key;
+    using mapped_type = void;
     bool full;
 
     FixedHashTableCell() {}
@@ -16,7 +17,6 @@ struct FixedHashTableCell
     bool isZero(const State &) const { return !full; }
     void setZero() { full = false; }
     static constexpr bool need_zero_value_storage = false;
-    void setMapped(const value_type & /*value*/) {}
 
     /// This Cell is only stored inside an iterator. It's used to accomodate the fact
     ///  that the iterator based API always provide a reference to a continuous memory
@@ -141,6 +141,11 @@ protected:
 public:
     using key_type = Key;
     using value_type = typename Cell::value_type;
+    using mapped_type = typename Cell::mapped_type;
+
+    using LookupResult = Cell *;
+    using ConstLookupResult = const Cell *;
+
 
     size_t hash(const Key & x) const { return x; }
 
@@ -263,9 +268,9 @@ public:
 
 public:
     /// The last parameter is unused but exists for compatibility with HashTable interface.
-    void ALWAYS_INLINE emplace(Key x, iterator & it, bool & inserted, size_t /* hash */ = 0)
+    void ALWAYS_INLINE emplace(Key x, LookupResult & it, bool & inserted, size_t /* hash */ = 0)
     {
-        it = iterator(this, &buf[x]);
+        it = &buf[x];
 
         if (!buf[x].isZero(*this))
         {
@@ -278,34 +283,34 @@ public:
         ++m_size;
     }
 
-    std::pair<iterator, bool> ALWAYS_INLINE insert(const value_type & x)
+    std::pair<LookupResult, bool> ALWAYS_INLINE insert(const value_type & x)
     {
-        std::pair<iterator, bool> res;
+        std::pair<LookupResult, bool> res;
         emplace(Cell::getKey(x), res.first, res.second);
         if (res.second)
-            res.first.ptr->setMapped(x);
+            insertSetMapped(lookupResultGetMapped(res.first), x);
 
         return res;
     }
 
-    iterator ALWAYS_INLINE find(Key x)
+    LookupResult ALWAYS_INLINE find(Key x)
     {
-        return !buf[x].isZero(*this) ? iterator(this, &buf[x]) : end();
+        return !buf[x].isZero(*this) ? &buf[x] : nullptr;
     }
 
-    const_iterator ALWAYS_INLINE find(Key x) const
+    ConstLookupResult ALWAYS_INLINE find(Key x) const
     {
-        return !buf[x].isZero(*this) ? const_iterator(this, &buf[x]) : end();
+        return const_cast<std::decay_t<decltype(*this)> *>(this)->find(x);
     }
 
-    iterator ALWAYS_INLINE find(Key, size_t hash_value)
+    LookupResult ALWAYS_INLINE find(Key, size_t hash_value)
     {
-        return !buf[hash_value].isZero(*this) ? iterator(this, &buf[hash_value]) : end();
+        return !buf[hash_value].isZero(*this) ? &buf[hash_value] : nullptr;
     }
 
-    const_iterator ALWAYS_INLINE find(Key, size_t hash_value) const
+    ConstLookupResult ALWAYS_INLINE find(Key key, size_t hash_value) const
     {
-        return !buf[hash_value].isZero(*this) ? const_iterator(this, &buf[hash_value]) : end();
+        return const_cast<std::decay_t<decltype(*this)> *>(this)->find(key, hash_value);
     }
 
     bool ALWAYS_INLINE has(Key x) const { return !buf[x].isZero(*this); }

dbms/src/Common/HashTable/HashMap.h

@@ -43,6 +43,9 @@ struct HashMapCell
     using State = TState;
 
     using value_type = PairNoInit<Key, Mapped>;
+    using mapped_type = Mapped;
+    using key_type = Key;
+
     value_type value;
 
     HashMapCell() {}
@@ -107,6 +110,14 @@ struct HashMapCell
     }
 };
 
+template<typename Key, typename Mapped, typename Hash, typename State>
+ALWAYS_INLINE inline auto lookupResultGetKey(HashMapCell<Key, Mapped, Hash, State> * cell)
+{ return &cell->getFirst(); }
+
+template<typename Key, typename Mapped, typename Hash, typename State>
+ALWAYS_INLINE inline auto lookupResultGetMapped(HashMapCell<Key, Mapped, Hash, State> * cell)
+{ return &cell->getSecond(); }
+
 
 template <typename Key, typename TMapped, typename Hash, typename TState = HashTableNoState>
 struct HashMapCellWithSavedHash : public HashMapCell<Key, TMapped, Hash, TState>
@@ -125,6 +136,14 @@ struct HashMapCellWithSavedHash : public HashMapCell<Key, TMapped, Hash, TState>
     size_t getHash(const Hash & /*hash_function*/) const { return saved_hash; }
 };
 
+template<typename Key, typename Mapped, typename Hash, typename State>
+ALWAYS_INLINE inline auto lookupResultGetKey(HashMapCellWithSavedHash<Key, Mapped, Hash, State> * cell)
+{ return &cell->getFirst(); }
+
+template<typename Key, typename Mapped, typename Hash, typename State>
+ALWAYS_INLINE inline auto lookupResultGetMapped(HashMapCellWithSavedHash<Key, Mapped, Hash, State> * cell)
+{ return &cell->getSecond(); }
+
 
 template <
     typename Key,
@@ -136,9 +155,13 @@ class HashMapTable : public HashTable<Key, Cell, Hash, Grower, Allocator>
 {
 public:
     using Self = HashMapTable;
+    using Base = HashTable<Key, Cell, Hash, Grower, Allocator>;
+
     using key_type = Key;
-    using mapped_type = typename Cell::Mapped;
     using value_type = typename Cell::value_type;
+    using mapped_type = typename Cell::Mapped;
+
+    using LookupResult = typename Base::LookupResult;
 
     using HashTable<Key, Cell, Hash, Grower, Allocator>::HashTable;
 
@@ -153,10 +176,10 @@ public:
     {
         for (auto it = this->begin(), end = this->end(); it != end; ++it)
         {
-            decltype(it) res_it;
+            typename Self::LookupResult res_it;
             bool inserted;
             that.emplace(it->getFirst(), res_it, inserted, it.getHash());
-            func(res_it->getSecond(), it->getSecond(), inserted);
+            func(*lookupResultGetMapped(res_it), it->getSecond(), inserted);
         }
     }
 
@@ -170,11 +193,11 @@ public:
     {
         for (auto it = this->begin(), end = this->end(); it != end; ++it)
         {
-            decltype(it) res_it = that.find(it->getFirst(), it.getHash());
+            auto res_it = that.find(it->getFirst(), it.getHash());
-            if (res_it == that.end())
+            if (!res_it)
                 func(it->getSecond(), it->getSecond(), false);
             else
-                func(res_it->getSecond(), it->getSecond(), true);
+                func(*lookupResultGetMapped(res_it), it->getSecond(), true);
         }
     }
 
@@ -196,7 +219,7 @@ public:
 
     mapped_type & ALWAYS_INLINE operator[](Key x)
     {
-        typename HashMapTable::iterator it;
+        typename HashMapTable::LookupResult it;
         bool inserted;
         this->emplace(x, it, inserted);
 
@@ -215,9 +238,9 @@ public:
           *  the compiler can not guess about this, and generates the `load`, `increment`, `store` code.
           */
         if (inserted)
-            new(&it->getSecond()) mapped_type();
+            new(lookupResultGetMapped(it)) mapped_type();
 
-        return it->getSecond();
+        return *lookupResultGetMapped(it);
     }
 };
 

dbms/src/Common/HashTable/HashSet.h

@@ -30,6 +30,9 @@ public:
     using Self = HashSetTable;
     using Cell = TCell;
 
+    using Base = HashTable<Key, TCell, Hash, Grower, Allocator>;
+    using typename Base::LookupResult;
+
     void merge(const Self & rhs)
     {
         if (!this->hasZero() && rhs.hasZero())
@@ -81,6 +84,13 @@ struct HashSetCellWithSavedHash : public HashTableCell<Key, Hash, TState>
     size_t getHash(const Hash & /*hash_function*/) const { return saved_hash; }
 };
 
+template<typename Key, typename Hash, typename State>
+ALWAYS_INLINE inline auto lookupResultGetKey(HashSetCellWithSavedHash<Key, Hash, State> * cell)
+{ return &cell->key; }
+
+template<typename Key, typename Hash, typename State>
+ALWAYS_INLINE inline void * lookupResultGetMapped(HashSetCellWithSavedHash<Key, Hash, State> *)
+{ return nullptr; }
 
 template
 <

dbms/src/Common/HashTable/HashTable.h

@@ -77,6 +77,67 @@ void set(T & x) { x = 0; }
 
 }
 
+/**
+  * lookupResultGetKey/Mapped -- functions to get key/"mapped" values from the
+  * LookupResult returned by find() and emplace() methods of HashTable.
+  * Must not be called for a null LookupResult.
+  *
+  * We don't use iterators for lookup result to avoid creating temporary
+  * objects. Instead, LookupResult is a pointer of some kind. There are global
+  * functions lookupResultGetKey/Mapped, overloaded for this pointer type, that
+  * return pointers to key/"mapped" values. They are implemented as global
+  * functions and not as methods, because they have to be overloaded for POD
+  * types, e.g. in StringHashTable where different components have different
+  * Cell format.
+  *
+  * Different hash table implementations support this interface to a varying
+  * degree:
+  *
+  * 1) Hash tables that store neither the key in its original form, nor a
+  *    "mapped" value: FixedHashTable or StringHashTable.
+  *    Neither GetKey nor GetMapped are supported, the only valid operation is
+  *    checking LookupResult for null.
+  *
+  * 2) Hash maps that do not store the key, e.g. FixedHashMap or StringHashMap.
+  *    Only GetMapped is supported.
+  *
+  * 3) Hash tables that store the key and do not have a "mapped" value, e.g. the
+  *    normal HashTable.
+  *    GetKey returns the key, and GetMapped returns a zero void pointer. This
+  *    simplifies generic code that works with mapped values: it can overload
+  *    on the return type of GetMapped(), and doesn't need other parameters. One
+  *    example is insertSetMapped() function.
+  *
+  * 4) Hash tables that store both the key and the "mapped" value, e.g. HashMap.
+  *    Both GetKey and GetMapped are supported.
+  *
+  * The implementation side goes as follows:
+  * for (1), LookupResult = void *, no getters;
+  * for (2), LookupResult = Mapped *, GetMapped is a default implementation that
+  * takes any pointer-like object;
+  * for (3) and (4), LookupResult = Cell *, and both getters are implemented.
+  * They have to be specialized for each particular Cell class to supersede the
+  * default verision that takes a generic pointer-like object.
+  */
+
+/**
+  * The default implementation of GetMapped that is used for the above case (2).
+  */
+template<typename PointerLike>
+ALWAYS_INLINE inline auto lookupResultGetMapped(PointerLike && ptr) { return &*ptr; }
+
+/**
+  * Generic const wrapper for lookupResultGetMapped, that calls a non-const
+  * version. Should be safe, given that these functions only do pointer
+  * arithmetics.
+  */
+template<typename T>
+ALWAYS_INLINE inline auto lookupResultGetMapped(const T * obj)
+{
+    auto mapped_ptr = lookupResultGetMapped(const_cast<T *>(obj));
+    const auto const_mapped_ptr = mapped_ptr;
+    return const_mapped_ptr;
+}
 
 /** Compile-time interface for cell of the hash table.
   * Different cell types are used to implement different hash tables.
@@ -89,7 +150,10 @@ struct HashTableCell
 {
     using State = TState;
 
+    using key_type = Key;
     using value_type = Key;
+    using mapped_type = void;
+
     Key key;
 
     HashTableCell() {}
@@ -143,6 +207,24 @@ struct HashTableCell
     void readText(DB::ReadBuffer & rb)    { DB::readDoubleQuoted(key, rb); }
 };
 
+template<typename Key, typename Hash, typename State>
+ALWAYS_INLINE inline auto lookupResultGetKey(HashTableCell<Key, Hash, State> * cell)
+{ return &cell->key; }
+
+template<typename Key, typename Hash, typename State>
+ALWAYS_INLINE inline void * lookupResultGetMapped(HashTableCell<Key, Hash, State> *)
+{ return nullptr; }
+
+/**
+  * A helper function for HashTable::insert() to set the "mapped" value.
+  * Overloaded on the mapped type, does nothing if it's void.
+  */
+template <typename ValueType>
+void insertSetMapped(void * /* dest */, const ValueType & /* src */) {}
+
+template <typename MappedType, typename ValueType>
+void insertSetMapped(MappedType * dest, const ValueType & src) { *dest = src.second; }
+
 
 /** Determines the size of the hash table, and when and how much it should be resized.
   */
@@ -476,6 +558,23 @@ protected:
         {
             return container->grower.place((ptr - container->buf) - container->grower.place(getHash()));
         }
+
+        /**
+          * A hack for HashedDictionary.
+          *
+          * The problem: std-like find() returns an iterator, which has to be
+          * compared to end(). On the other hand, HashMap::find() returns
+          * LookupResult, which is compared to nullptr. HashedDictionary has to
+          * support both hash maps with the same code, hence the need for this
+          * hack.
+          *
+          * The proper way would be to remove iterator interface from our
+          * HashMap completely, change all its users to the existing internal
+          * iteration interface, and redefine end() to return LookupResult for
+          * compatibility with std find(). Unfortunately, now is not the time to
+          * do this.
+          */
+        operator Cell * () const { return nullptr; }
     };
 
 
@@ -483,6 +582,10 @@ public:
     using key_type = Key;
     using value_type = typename Cell::value_type;
 
+    // Use lookupResultGetMapped/Key to work with these values.
+    using LookupResult = Cell *;
+    using ConstLookupResult = const Cell *;
+
     size_t hash(const Key & x) const { return Hash::operator()(x); }
 
 
@@ -642,7 +745,7 @@ protected:
     /// If the key is zero, insert it into a special place and return true.
     /// We don't have to persist a zero key, because it's not actually inserted.
     /// That's why we just take a Key by value, an not a key holder.
-    bool ALWAYS_INLINE emplaceIfZero(Key x, iterator & it, bool & inserted, size_t hash_value)
+    bool ALWAYS_INLINE emplaceIfZero(Key x, LookupResult & it, bool & inserted, size_t hash_value)
     {
         /// If it is claimed that the zero key can not be inserted into the table.
         if (!Cell::need_zero_value_storage)
@@ -650,12 +753,13 @@ protected:
 
         if (Cell::isZero(x, *this))
         {
-            it = iteratorToZero();
+            it = this->zeroValue();
+
             if (!this->hasZero())
             {
                 ++m_size;
                 this->setHasZero();
-                it.ptr->setHash(hash_value);
+                this->zeroValue()->setHash(hash_value);
                 inserted = true;
             }
             else
@@ -669,9 +773,9 @@ protected:
 
     template <typename KeyHolder>
     void ALWAYS_INLINE emplaceNonZeroImpl(size_t place_value, KeyHolder && key_holder,
-                                          iterator & it, bool & inserted, size_t hash_value)
+                                          LookupResult & it, bool & inserted, size_t hash_value)
     {
-        it = iterator(this, &buf[place_value]);
+        it = &buf[place_value];
 
         if (!buf[place_value].isZero(*this))
         {
@@ -705,13 +809,16 @@ protected:
                 throw;
             }
 
-            it = find(keyHolderGetKey(key_holder), hash_value);
+            // The hash table was rehashed, so we have to re-find the key.
+            size_t new_place = findCell(key, hash_value, grower.place(hash_value));
+            assert(!buf[new_place].isZero(*this));
+            it = &buf[new_place];
         }
     }
 
     /// Only for non-zero keys. Find the right place, insert the key there, if it does not already exist. Set iterator to the cell in output parameter.
     template <typename KeyHolder>
-    void ALWAYS_INLINE emplaceNonZero(KeyHolder && key_holder, iterator & it,
+    void ALWAYS_INLINE emplaceNonZero(KeyHolder && key_holder, LookupResult & it,
                                       bool & inserted, size_t hash_value)
     {
         const auto & key = keyHolderGetKey(key_holder);
@@ -722,9 +829,9 @@ protected:
 
 public:
     /// Insert a value. In the case of any more complex values, it is better to use the `emplace` function.
-    std::pair<iterator, bool> ALWAYS_INLINE insert(const value_type & x)
+    std::pair<LookupResult, bool> ALWAYS_INLINE insert(const value_type & x)
     {
-        std::pair<iterator, bool> res;
+        std::pair<LookupResult, bool> res;
 
         size_t hash_value = hash(Cell::getKey(x));
         if (!emplaceIfZero(Cell::getKey(x), res.first, res.second, hash_value))
@@ -733,7 +840,7 @@ public:
         }
 
         if (res.second)
-            res.first.ptr->setMapped(x);
+            insertSetMapped(lookupResultGetMapped(res.first), x);
 
         return res;
     }
@@ -746,9 +853,10 @@ public:
     }
 
 
-    /** Insert the key,
+    /** Insert the key.
-      * return an iterator to a position that can be used for `placement new` of value,
+      * Return values:
-      * as well as the flag - whether a new key was inserted.
+      * 'it' -- a LookupResult pointing to the corresponding key/mapped pair.
+      * 'inserted' -- whether a new key was inserted.
       *
       * You have to make `placement new` of value if you inserted a new key,
       * since when destroying a hash table, it will call the destructor!
@@ -762,14 +870,14 @@ public:
       *     new(&it->second) Mapped(value);
       */
     template <typename KeyHolder>
-    void ALWAYS_INLINE emplace(KeyHolder && key_holder, iterator & it, bool & inserted)
+    void ALWAYS_INLINE emplace(KeyHolder && key_holder, LookupResult & it, bool & inserted)
     {
         const auto & key = keyHolderGetKey(key_holder);
         emplace(key_holder, it, inserted, hash(key));
     }
 
     template <typename KeyHolder>
-    void ALWAYS_INLINE emplace(KeyHolder && key_holder, iterator & it,
+    void ALWAYS_INLINE emplace(KeyHolder && key_holder, LookupResult & it,
                                   bool & inserted, size_t hash_value)
     {
         const auto & key = keyHolderGetKey(key_holder);
@@ -789,48 +897,30 @@ public:
             resize();
     }
 
-    iterator ALWAYS_INLINE find(Key x)
+    LookupResult ALWAYS_INLINE find(Key x)
     {
         if (Cell::isZero(x, *this))
-            return this->hasZero() ? iteratorToZero() : end();
+            return this->hasZero() ? this->zeroValue() : nullptr;
 
         size_t hash_value = hash(x);
         size_t place_value = findCell(x, hash_value, grower.place(hash_value));
-        return !buf[place_value].isZero(*this) ? iterator(this, &buf[place_value]) : end();
+        return !buf[place_value].isZero(*this) ? &buf[place_value] : nullptr;
     }
 
+    ConstLookupResult ALWAYS_INLINE find(Key x) const
+    {
+        return const_cast<std::decay_t<decltype(*this)> *>(this)->find(x);
+    }
 
-    const_iterator ALWAYS_INLINE find(Key x) const
+    LookupResult ALWAYS_INLINE find(Key x, size_t hash_value)
     {
         if (Cell::isZero(x, *this))
-            return this->hasZero() ? iteratorToZero() : end();
+            return this->hasZero() ? this->zeroValue() : nullptr;
-
-        size_t hash_value = hash(x);
-        size_t place_value = findCell(x, hash_value, grower.place(hash_value));
-        return !buf[place_value].isZero(*this) ? const_iterator(this, &buf[place_value]) : end();
-    }
-
-
-    iterator ALWAYS_INLINE find(Key x, size_t hash_value)
-    {
-        if (Cell::isZero(x, *this))
-            return this->hasZero() ? iteratorToZero() : end();
 
         size_t place_value = findCell(x, hash_value, grower.place(hash_value));
-        return !buf[place_value].isZero(*this) ? iterator(this, &buf[place_value]) : end();
+        return !buf[place_value].isZero(*this) ? &buf[place_value] : nullptr;
     }
 
-
-    const_iterator ALWAYS_INLINE find(Key x, size_t hash_value) const
-    {
-        if (Cell::isZero(x, *this))
-            return this->hasZero() ? iteratorToZero() : end();
-
-        size_t place_value = findCell(x, hash_value, grower.place(hash_value));
-        return !buf[place_value].isZero(*this) ? const_iterator(this, &buf[place_value]) : end();
-    }
-
-
     bool ALWAYS_INLINE has(Key x) const
     {
         if (Cell::isZero(x, *this))

dbms/src/Common/HashTable/TwoLevelHashMap.h

@@ -20,6 +20,9 @@ public:
     using mapped_type = typename Cell::Mapped;
     using value_type = typename Cell::value_type;
 
+    using Impl = ImplTable<Key, Cell, Hash, Grower, Allocator>;
+    using LookupResult = typename Impl::LookupResult;
+
     using TwoLevelHashTable<Key, Cell, Hash, Grower, Allocator, ImplTable<Key, Cell, Hash, Grower, Allocator>>::TwoLevelHashTable;
 
     template <typename Func>
@@ -31,14 +34,14 @@ public:
 
     mapped_type & ALWAYS_INLINE operator[](Key x)
     {
-        typename TwoLevelHashMapTable::iterator it;
+        typename TwoLevelHashMapTable::LookupResult it;
         bool inserted;
         this->emplace(x, it, inserted);
 
         if (inserted)
-            new(&it->getSecond()) mapped_type();
+            new(lookupResultGetMapped(it)) mapped_type();
 
-        return it->getSecond();
+        return *lookupResultGetMapped(it);
     }
 };
 

dbms/src/Common/HashTable/TwoLevelHashTable.h

@@ -84,6 +84,9 @@ public:
     using key_type = typename Impl::key_type;
     using value_type = typename Impl::value_type;
 
+    using LookupResult = typename Impl::LookupResult;
+    using ConstLookupResult = typename Impl::ConstLookupResult;
+
     Impl impls[NUM_BUCKETS];
 
 
@@ -206,15 +209,15 @@ public:
 
 
     /// Insert a value. In the case of any more complex values, it is better to use the `emplace` function.
-    std::pair<iterator, bool> ALWAYS_INLINE insert(const value_type & x)
+    std::pair<LookupResult, bool> ALWAYS_INLINE insert(const value_type & x)
     {
         size_t hash_value = hash(Cell::getKey(x));
 
-        std::pair<iterator, bool> res;
+        std::pair<LookupResult, bool> res;
         emplace(Cell::getKey(x), res.first, res.second, hash_value);
 
         if (res.second)
-            res.first.getPtr()->setMapped(x);
+            insertSetMapped(lookupResultGetMapped(res.first), x);
 
         return res;
     }
@@ -236,7 +239,7 @@ public:
       *     new(&it->second) Mapped(value);
       */
     template <typename KeyHolder>
-    void ALWAYS_INLINE emplace(KeyHolder && key_holder, iterator & it, bool & inserted)
+    void ALWAYS_INLINE emplace(KeyHolder && key_holder, LookupResult & it, bool & inserted)
     {
         size_t hash_value = hash(keyHolderGetKey(key_holder));
         emplace(key_holder, it, inserted, hash_value);
@@ -245,40 +248,27 @@ public:
 
     /// Same, but with a precalculated values of hash function.
     template <typename KeyHolder>
-    void ALWAYS_INLINE emplace(KeyHolder && key_holder, iterator & it,
+    void ALWAYS_INLINE emplace(KeyHolder && key_holder, LookupResult & it,
                                   bool & inserted, size_t hash_value)
     {
         size_t buck = getBucketFromHash(hash_value);
-        typename Impl::iterator impl_it;
+        impls[buck].emplace(key_holder, it, inserted, hash_value);
-        impls[buck].emplace(key_holder, impl_it, inserted, hash_value);
-        it = iterator(this, buck, impl_it);
     }
 
-
+    LookupResult ALWAYS_INLINE find(Key x, size_t hash_value)
-    iterator ALWAYS_INLINE find(Key x, size_t hash_value)
     {
         size_t buck = getBucketFromHash(hash_value);
-
+        return impls[buck].find(x, hash_value);
-        typename Impl::iterator found = impls[buck].find(x, hash_value);
-        return found != impls[buck].end()
-            ? iterator(this, buck, found)
-            : end();
     }
 
-
+    ConstLookupResult ALWAYS_INLINE find(Key x, size_t hash_value) const
-    const_iterator ALWAYS_INLINE find(Key x, size_t hash_value) const
     {
-        size_t buck = getBucketFromHash(hash_value);
+        return const_cast<std::decay_t<decltype(*this)> *>(this)->find(x, hash_value);
-
-        typename Impl::const_iterator found = impls[buck].find(x, hash_value);
-        return found != impls[buck].end()
-            ? const_iterator(this, buck, found)
-            : end();
     }
 
+    LookupResult ALWAYS_INLINE find(Key x) { return find(x, hash(x)); }
 
-    iterator ALWAYS_INLINE find(Key x) { return find(x, hash(x)); }
+    ConstLookupResult ALWAYS_INLINE find(Key x) const { return find(x, hash(x)); }
-    const_iterator ALWAYS_INLINE find(Key x) const { return find(x, hash(x)); }
 
 
     void write(DB::WriteBuffer & wb) const

dbms/src/Common/PoolWithFailoverBase.h

@@ -199,7 +199,7 @@ PoolWithFailoverBase<TNestedPool>::getMany(
         for (const ShuffledPool & pool: shuffled_pools)
         {
             auto & pool_state = shared_pool_states[pool.index];
-            pool_state.error_count = std::min(max_error_cap, pool_state.error_count + pool.error_count);
+            pool_state.error_count = std::min<UInt64>(max_error_cap, pool_state.error_count + pool.error_count);
         }
     });
 

dbms/src/Common/ProfileEvents.cpp

@@ -114,6 +114,7 @@
     M(SelectedRanges, "Number of (non-adjacent) ranges in all data parts selected to read from a MergeTree table.") \
     M(SelectedMarks, "Number of marks (index granules) selected to read from a MergeTree table.") \
     \
+    M(Merge, "Number of launched background merges.") \
     M(MergedRows, "Rows read for background merges. This is the number of rows before merge.") \
     M(MergedUncompressedBytes, "Uncompressed bytes (for columns as they stored in memory) that was read for background merges. This is the number before merge.") \
     M(MergesTimeMilliseconds, "Total time spent for background merges.")\