Merge branch 'ClickHouse:master' into master

.github/workflows/main.yml

@@ -250,7 +250,7 @@ jobs:
           REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
           CACHES_PATH: ${{runner.temp}}/../ccaches
           CHECK_NAME: 'ClickHouse build check (actions)'
-          BUILD_NUMBER: 9
+          BUILD_NUMBER: 8
         run: |
           sudo rm -fr $TEMP_PATH
           mkdir -p $TEMP_PATH
@@ -1426,7 +1426,7 @@ jobs:
         env:
           TEMP_PATH: ${{runner.temp}}/unit_tests_ubsan
           REPORTS_PATH: ${{runner.temp}}/reports_dir
-          CHECK_NAME: 'Unit tests (msan, actions)'
+          CHECK_NAME: 'Unit tests (ubsan, actions)'
           REPO_COPY: ${{runner.temp}}/unit_tests_ubsan/ClickHouse
         run: |
           sudo rm -fr $TEMP_PATH

.gitmodules

@@ -171,12 +171,6 @@
 [submodule "contrib/sentry-native"]
 	path = contrib/sentry-native
 	url = https://github.com/ClickHouse-Extras/sentry-native.git
-[submodule "contrib/gcem"]
-	path = contrib/gcem
-	url = https://github.com/kthohr/gcem.git
-[submodule "contrib/stats"]
-	path = contrib/stats
-	url = https://github.com/kthohr/stats.git
 [submodule "contrib/krb5"]
 	path = contrib/krb5
 	url = https://github.com/ClickHouse-Extras/krb5

CMakeLists.txt

@@ -201,7 +201,7 @@ endif ()
 option(ENABLE_TESTS "Provide unit_test_dbms target with Google.Test unit tests" ON)
 option(ENABLE_EXAMPLES "Build all example programs in 'examples' subdirectories" OFF)
 
-if (OS_LINUX AND (ARCH_AMD64 OR ARCH_AARCH64) AND NOT UNBUNDLED AND MAKE_STATIC_LIBRARIES AND NOT SPLIT_SHARED_LIBRARIES AND NOT USE_MUSL)
+if (OS_LINUX AND (ARCH_AMD64 OR ARCH_AARCH64) AND MAKE_STATIC_LIBRARIES AND NOT SPLIT_SHARED_LIBRARIES AND NOT USE_MUSL)
     # Only for Linux, x86_64 or aarch64.
     option(GLIBC_COMPATIBILITY "Enable compatibility with older glibc libraries." ON)
 elseif(GLIBC_COMPATIBILITY)
@@ -435,20 +435,7 @@ endif ()
 # Turns on all external libs like s3, kafka, ODBC, ...
 option(ENABLE_LIBRARIES "Enable all external libraries by default" ON)
 
-# We recommend avoiding this mode for production builds because we can't guarantee
+if (NOT (OS_LINUX OR OS_DARWIN))
-# all needed libraries exist in your system.
-# This mode exists for enthusiastic developers who are searching for trouble.
-# The whole idea of using unknown version of libraries from the OS distribution is deeply flawed.
-# Useful for maintainers of OS packages.
-option (UNBUNDLED "Use system libraries instead of ones in contrib/" OFF)
-
-if (UNBUNDLED)
-    set(NOT_UNBUNDLED OFF)
-else ()
-    set(NOT_UNBUNDLED ON)
-endif ()
-
-if (UNBUNDLED OR NOT (OS_LINUX OR OS_DARWIN))
     # Using system libs can cause a lot of warnings in includes (on macro expansion).
     option(WERROR "Enable -Werror compiler option" OFF)
 else ()
@@ -527,7 +514,6 @@ message (STATUS
     USE_STATIC_LIBRARIES=${USE_STATIC_LIBRARIES}
     MAKE_STATIC_LIBRARIES=${MAKE_STATIC_LIBRARIES}
     SPLIT_SHARED=${SPLIT_SHARED_LIBRARIES}
-    UNBUNDLED=${UNBUNDLED}
     CCACHE=${CCACHE_FOUND} ${CCACHE_VERSION}")
 
 include (GNUInstallDirs)
@@ -590,7 +576,6 @@ include (cmake/find/avro.cmake)
 include (cmake/find/msgpack.cmake)
 include (cmake/find/cassandra.cmake)
 include (cmake/find/sentry.cmake)
-include (cmake/find/stats.cmake)
 include (cmake/find/datasketches.cmake)
 include (cmake/find/libprotobuf-mutator.cmake)
 

base/daemon/GraphiteWriter.cpp

@@ -37,11 +37,3 @@ GraphiteWriter::GraphiteWriter(const std::string & config_name, const std::strin
         root_path += sub_path;
     }
 }
-
-
-std::string GraphiteWriter::getPerServerPath(const std::string & server_name, const std::string & root_path)
-{
-    std::string path = root_path + "." + server_name;
-    std::replace(path.begin() + root_path.size() + 1, path.end(), '.', '_');
-    return path;
-}

base/daemon/GraphiteWriter.h

@@ -8,10 +8,10 @@
 #include <base/logger_useful.h>
 
 
-/// пишет в Graphite данные в формате
+/// Writes to Graphite in the following format
 /// path value timestamp\n
-/// path может иметь любую вложенность. Директории разделяются с помощью "."
+/// path can be arbitrary nested. Elements are separated by '.'
-/// у нас принят следующий формат path - root_path.server_name.sub_path.key
+/// Example: root_path.server_name.sub_path.key
 class GraphiteWriter
 {
 public:
@@ -32,8 +32,6 @@ public:
         writeImpl(key_val_vec, timestamp, custom_root_path);
     }
 
-    /// возвращает путь root_path.server_name
-    static std::string getPerServerPath(const std::string & server_name, const std::string & root_path = "one_min");
 private:
     template <typename T>
     void writeImpl(const T & data, time_t timestamp, const std::string & custom_root_path)

base/mysqlxx/CMakeLists.txt

@@ -13,10 +13,7 @@ add_library (mysqlxx
 
 target_include_directories (mysqlxx PUBLIC ..)
 
-if (USE_INTERNAL_MYSQL_LIBRARY)
+if (NOT USE_INTERNAL_MYSQL_LIBRARY)
-    target_include_directories (mysqlxx PUBLIC "${ClickHouse_SOURCE_DIR}/contrib/mariadb-connector-c/include")
-    target_include_directories (mysqlxx PUBLIC "${ClickHouse_BINARY_DIR}/contrib/mariadb-connector-c/include")
-else ()
     set(PLATFORM_LIBRARIES ${CMAKE_DL_LIBS})
 
     if (USE_MYSQL)

cmake/contrib_finder.cmake

@@ -4,7 +4,7 @@ macro(find_contrib_lib LIB_NAME)
     string(TOUPPER ${LIB_NAME} LIB_NAME_UC)
     string(REPLACE "-" "_" LIB_NAME_UC ${LIB_NAME_UC})
 
-    option (USE_INTERNAL_${LIB_NAME_UC}_LIBRARY "Use bundled library ${LIB_NAME} instead of system" ${NOT_UNBUNDLED})
+    option (USE_INTERNAL_${LIB_NAME_UC}_LIBRARY "Use bundled library ${LIB_NAME} instead of system" ON)
 
     if (NOT USE_INTERNAL_${LIB_NAME_UC}_LIBRARY)
         find_package ("${LIB_NAME}")

cmake/cpu_features.cmake

@@ -28,6 +28,9 @@ option (ARCH_NATIVE "Add -march=native compiler flag. This makes your binaries n
 if (ARCH_NATIVE)
     set (COMPILER_FLAGS "${COMPILER_FLAGS} -march=native")
 
+elseif (ARCH_AARCH64)
+    set (COMPILER_FLAGS "${COMPILER_FLAGS} -march=armv8-a+crc")
+
 else ()
     set (TEST_FLAG "-mssse3")
     set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
@@ -43,7 +46,6 @@ else ()
         set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}")
     endif ()
 
-
     set (TEST_FLAG "-msse4.1")
     set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
     check_cxx_source_compiles("

cmake/find/amqpcpp.cmake

@@ -9,7 +9,7 @@ if (NOT ENABLE_AMQPCPP)
     return()
 endif()
 
-if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP/CMakeLists.txt")
+if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP/src")
     message (WARNING "submodule contrib/AMQP-CPP is missing. to fix try run: \n git submodule update --init")
     message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal AMQP-CPP library")
     set (USE_AMQPCPP 0)

cmake/find/avro.cmake

@@ -8,10 +8,9 @@ if (NOT ENABLE_AVRO)
     return()
 endif()
 
-option (USE_INTERNAL_AVRO_LIBRARY
+option (USE_INTERNAL_AVRO_LIBRARY "Set to FALSE to use system avro library instead of bundled" ON)
-        "Set to FALSE to use system avro library instead of bundled" ON) # TODO: provide unbundled support
 
-if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/avro/lang/c++/CMakeLists.txt")
+if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/avro/lang")
     if (USE_INTERNAL_AVRO_LIBRARY)
         message(WARNING "submodule contrib/avro is missing. to fix try run: \n git submodule update --init")
         message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot find internal avro")

cmake/find/brotli.cmake

@@ -7,12 +7,7 @@ if (NOT ENABLE_BROTLI)
     return()
 endif()
 
-if (UNBUNDLED)
-    # Many system ship only dynamic brotly libraries, so we back off to bundled by default
-    option (USE_INTERNAL_BROTLI_LIBRARY "Set to FALSE to use system libbrotli library instead of bundled" ${USE_STATIC_LIBRARIES})
-else()
 option (USE_INTERNAL_BROTLI_LIBRARY "Set to FALSE to use system libbrotli library instead of bundled" ON)
-endif()
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/brotli/c/include/brotli/decode.h")
     if (USE_INTERNAL_BROTLI_LIBRARY)

cmake/find/capnp.cmake

@@ -7,9 +7,9 @@ if (NOT ENABLE_CAPNP)
     return()
 endif()
 
-option (USE_INTERNAL_CAPNP_LIBRARY "Set to FALSE to use system capnproto library instead of bundled" ${NOT_UNBUNDLED})
+option (USE_INTERNAL_CAPNP_LIBRARY "Set to FALSE to use system capnproto library instead of bundled" ON)
 
-if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/capnproto/CMakeLists.txt")
+if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/capnproto/c++")
     if(USE_INTERNAL_CAPNP_LIBRARY)
         message(WARNING "submodule contrib/capnproto is missing. to fix try run: \n git submodule update --init")
         message(${RECONFIGURE_MESSAGE_LEVEL} "cannot find internal capnproto")

cmake/find/curl.cmake

@@ -7,7 +7,7 @@ if (NOT ENABLE_CURL)
     return()
 endif()
 
-option (USE_INTERNAL_CURL "Use internal curl library" ${NOT_UNBUNDLED})
+option (USE_INTERNAL_CURL "Use internal curl library" ON)
 
 if (NOT USE_INTERNAL_CURL)
     find_package (CURL)
@@ -22,8 +22,6 @@ if (NOT CURL_FOUND)
 
     # find_package(CURL) compatibility for the following packages that uses
     # find_package(CURL)/include(FindCURL):
-    # - mariadb-connector-c
-    # - aws-s3-cmake
     # - sentry-native
     set (CURL_FOUND ON CACHE BOOL "")
     set (CURL_ROOT_DIR ${CURL_LIBRARY_DIR} CACHE PATH "")

cmake/find/cxx.cmake

@@ -1,4 +1,4 @@
-option (USE_LIBCXX "Use libc++ and libc++abi instead of libstdc++" ${NOT_UNBUNDLED})
+option (USE_LIBCXX "Use libc++ and libc++abi instead of libstdc++" ON)
 
 if (NOT USE_LIBCXX)
     if (USE_INTERNAL_LIBCXX_LIBRARY)
@@ -10,12 +10,12 @@ if (NOT USE_LIBCXX)
     return()
 endif()
 
-set(USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT ${NOT_UNBUNDLED})
+set(USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT ON)
 
 option (USE_INTERNAL_LIBCXX_LIBRARY "Disable to use system libcxx and libcxxabi libraries instead of bundled"
     ${USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT})
 
-if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libcxx/CMakeLists.txt")
+if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libcxx/src")
     if (USE_INTERNAL_LIBCXX_LIBRARY)
         message(WARNING "submodule contrib/libcxx is missing. to fix try run: \n git submodule update --init")
         message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal libcxx")

cmake/find/datasketches.cmake

@@ -2,7 +2,7 @@ option (ENABLE_DATASKETCHES "Enable DataSketches" ${ENABLE_LIBRARIES})
 
 if (ENABLE_DATASKETCHES)
 
-option (USE_INTERNAL_DATASKETCHES_LIBRARY "Set to FALSE to use system DataSketches library instead of bundled" ${NOT_UNBUNDLED})
+option (USE_INTERNAL_DATASKETCHES_LIBRARY "Set to FALSE to use system DataSketches library instead of bundled" ON)
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/datasketches-cpp/theta/CMakeLists.txt")
     if (USE_INTERNAL_DATASKETCHES_LIBRARY)

cmake/find/grpc.cmake

@@ -22,7 +22,7 @@ endif()
 # You can set USE_INTERNAL_GRPC_LIBRARY to OFF to force using the external gRPC framework, which should be installed in the system in this case.
 # The external gRPC framework can be installed in the system by running
 # sudo apt-get install libgrpc++-dev protobuf-compiler-grpc
-option(USE_INTERNAL_GRPC_LIBRARY "Set to FALSE to use system gRPC library instead of bundled. (Experimental. Set to OFF on your own risk)" ${NOT_UNBUNDLED})
+option(USE_INTERNAL_GRPC_LIBRARY "Set to FALSE to use system gRPC library instead of bundled. (Experimental. Set to OFF on your own risk)" ON)
 
 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/grpc/CMakeLists.txt")
   if(USE_INTERNAL_GRPC_LIBRARY)

cmake/find/gtest.cmake

@@ -1,6 +1,6 @@
 # included only if ENABLE_TESTS=1
 
-option (USE_INTERNAL_GTEST_LIBRARY "Set to FALSE to use system Google Test instead of bundled" ${NOT_UNBUNDLED})
+option (USE_INTERNAL_GTEST_LIBRARY "Set to FALSE to use system Google Test instead of bundled" ON)
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/googletest/googletest/CMakeLists.txt")
    if (USE_INTERNAL_GTEST_LIBRARY)

cmake/find/icu.cmake

@@ -12,7 +12,7 @@ if (NOT ENABLE_ICU)
     return()
 endif()
 
-option (USE_INTERNAL_ICU_LIBRARY "Set to FALSE to use system ICU library instead of bundled" ${NOT_UNBUNDLED})
+option (USE_INTERNAL_ICU_LIBRARY "Set to FALSE to use system ICU library instead of bundled" ON)
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/icu/icu4c/LICENSE")
     if (USE_INTERNAL_ICU_LIBRARY)

cmake/find/ldap.cmake

@@ -1,7 +1,3 @@
-if (UNBUNDLED AND USE_STATIC_LIBRARIES)
-    set (ENABLE_LDAP OFF CACHE INTERNAL "")
-endif()
-
 option (ENABLE_LDAP "Enable LDAP" ${ENABLE_LIBRARIES})
 
 if (NOT ENABLE_LDAP)
@@ -11,7 +7,7 @@ if (NOT ENABLE_LDAP)
     return()
 endif()
 
-option (USE_INTERNAL_LDAP_LIBRARY "Set to FALSE to use system *LDAP library instead of bundled" ${NOT_UNBUNDLED})
+option (USE_INTERNAL_LDAP_LIBRARY "Set to FALSE to use system *LDAP library instead of bundled" ON)
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/openldap/README")
     if (USE_INTERNAL_LDAP_LIBRARY)

cmake/find/libgsasl.cmake

@@ -7,12 +7,7 @@ if (NOT ENABLE_GSASL_LIBRARY)
     return()
 endif()
 
-if (UNBUNDLED)
-    # when USE_STATIC_LIBRARIES we usually need to pick up hell a lot of dependencies for libgsasl
-    option (USE_INTERNAL_LIBGSASL_LIBRARY "Set to FALSE to use system libgsasl library instead of bundled" ${USE_STATIC_LIBRARIES})
-else()
 option (USE_INTERNAL_LIBGSASL_LIBRARY "Set to FALSE to use system libgsasl library instead of bundled" ON)
-endif()
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libgsasl/src/gsasl.h")
     if (USE_INTERNAL_LIBGSASL_LIBRARY)
@@ -35,7 +30,7 @@ if (LIBGSASL_LIBRARY AND LIBGSASL_INCLUDE_DIR)
 elseif (NOT MISSING_INTERNAL_LIBGSASL_LIBRARY)
     set (LIBGSASL_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libgsasl/src" "${ClickHouse_SOURCE_DIR}/contrib/libgsasl/linux_x86_64/include")
     set (USE_INTERNAL_LIBGSASL_LIBRARY 1)
-    set (LIBGSASL_LIBRARY libgsasl)
+    set (LIBGSASL_LIBRARY gsasl)
 endif ()
 
 if(LIBGSASL_LIBRARY AND LIBGSASL_INCLUDE_DIR)

cmake/find/libpqxx.cmake

@@ -4,7 +4,7 @@ if (NOT ENABLE_LIBPQXX)
     return()
 endif()
 
-if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libpqxx/CMakeLists.txt")
+if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libpqxx/src")
     message (WARNING "submodule contrib/libpqxx is missing. to fix try run: \n git submodule update --init")
     message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal libpqxx library")
     set (USE_LIBPQXX 0)

cmake/find/libxml2.cmake

@@ -1,4 +1,4 @@
-option (USE_INTERNAL_LIBXML2_LIBRARY "Set to FALSE to use system libxml2 library instead of bundled" ${NOT_UNBUNDLED})
+option (USE_INTERNAL_LIBXML2_LIBRARY "Set to FALSE to use system libxml2 library instead of bundled" ON)
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libxml2/libxml.h")
     if (USE_INTERNAL_LIBXML2_LIBRARY)

cmake/find/msgpack.cmake

@@ -7,7 +7,7 @@ if(NOT ENABLE_MSGPACK)
     return()
 endif()
 
-option (USE_INTERNAL_MSGPACK_LIBRARY "Set to FALSE to use system msgpack library instead of bundled" ${NOT_UNBUNDLED})
+option (USE_INTERNAL_MSGPACK_LIBRARY "Set to FALSE to use system msgpack library instead of bundled" ON)
 
 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/msgpack-c/include/msgpack.hpp")
     if(USE_INTERNAL_MSGPACK_LIBRARY)

cmake/find/mysqlclient.cmake

@@ -12,7 +12,7 @@ if(NOT ENABLE_MYSQL)
     return()
 endif()
 
-option(USE_INTERNAL_MYSQL_LIBRARY "Set to FALSE to use system mysqlclient library instead of bundled" ${NOT_UNBUNDLED})
+option(USE_INTERNAL_MYSQL_LIBRARY "Set to FALSE to use system mysqlclient library instead of bundled" ON)
 
 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/mariadb-connector-c/README")
     if(USE_INTERNAL_MYSQL_LIBRARY)

cmake/find/nanodbc.cmake

@@ -6,7 +6,7 @@ if (NOT USE_INTERNAL_NANODBC_LIBRARY)
     message (FATAL_ERROR "Only the bundled nanodbc library can be used")
 endif ()
 
-if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/nanodbc/CMakeLists.txt")
+if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/nanodbc/nanodbc")
     message (FATAL_ERROR "submodule contrib/nanodbc is missing. to fix try run: \n git submodule update --init")
 endif()
 

cmake/find/nlp.cmake

@@ -13,7 +13,7 @@ if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libstemmer_c/Makefile")
     return()
 endif ()
 
-if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/wordnet-blast/CMakeLists.txt")
+if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/wordnet-blast/wnb")
     message (WARNING "submodule contrib/wordnet-blast is missing. to fix try run: \n git submodule update --init")
     message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal wordnet-blast library, NLP functions will be disabled")
     set (USE_NLP 0)

cmake/find/nuraft.cmake

@@ -4,7 +4,7 @@ if (NOT ENABLE_NURAFT)
     return()
 endif()
 
-if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/NuRaft/CMakeLists.txt")
+if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/NuRaft/src")
     message (WARNING "submodule contrib/NuRaft is missing. to fix try run: \n git submodule update --init")
     message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal NuRaft library")
     set (USE_NURAFT 0)

cmake/find/odbc.cmake

@@ -19,7 +19,7 @@ if (NOT ENABLE_ODBC)
     return()
 endif()
 
-option (USE_INTERNAL_ODBC_LIBRARY "Use internal ODBC library" ${NOT_UNBUNDLED})
+option (USE_INTERNAL_ODBC_LIBRARY "Use internal ODBC library" ON)
 
 if (NOT USE_INTERNAL_ODBC_LIBRARY)
     find_library (LIBRARY_ODBC NAMES unixodbc odbc)

cmake/find/parquet.cmake

@@ -1,5 +1,5 @@
 if (Protobuf_PROTOC_EXECUTABLE)
-    option (ENABLE_PARQUET "Enable parquet" ${ENABLE_LIBRARIES})
+    option (ENABLE_PARQUET "Enable parquet" ON)
 elseif(ENABLE_PARQUET OR USE_INTERNAL_PARQUET_LIBRARY)
     message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use parquet without protoc executable")
 endif()
@@ -13,7 +13,7 @@ if (NOT ENABLE_PARQUET)
 endif()
 
 if (NOT OS_FREEBSD) # Freebsd: ../contrib/arrow/cpp/src/arrow/util/bit-util.h:27:10: fatal error: endian.h: No such file or directory
-    option(USE_INTERNAL_PARQUET_LIBRARY "Set to FALSE to use system parquet library instead of bundled" ${NOT_UNBUNDLED})
+    option(USE_INTERNAL_PARQUET_LIBRARY "Set to FALSE to use system parquet library instead of bundled" ON)
 elseif(USE_INTERNAL_PARQUET_LIBRARY)
     message (${RECONFIGURE_MESSAGE_LEVEL} "Using internal parquet is not supported on freebsd")
 endif()

cmake/find/protobuf.cmake

@@ -11,7 +11,7 @@ endif()
 # You can set USE_INTERNAL_PROTOBUF_LIBRARY to OFF to force using the external protobuf library, which should be installed in the system in this case.
 # The external protobuf library can be installed in the system by running
 # sudo apt-get install libprotobuf-dev protobuf-compiler libprotoc-dev
-option(USE_INTERNAL_PROTOBUF_LIBRARY "Set to FALSE to use system protobuf instead of bundled. (Experimental. Set to OFF on your own risk)" ${NOT_UNBUNDLED})
+option(USE_INTERNAL_PROTOBUF_LIBRARY "Set to FALSE to use system protobuf instead of bundled. (Experimental. Set to OFF on your own risk)" ON)
 
 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/protobuf/cmake/CMakeLists.txt")
   if(USE_INTERNAL_PROTOBUF_LIBRARY)

cmake/find/rapidjson.cmake

@@ -6,7 +6,7 @@ if(NOT ENABLE_RAPIDJSON)
     return()
 endif()
 
-option(USE_INTERNAL_RAPIDJSON_LIBRARY "Set to FALSE to use system rapidjson library instead of bundled" ${NOT_UNBUNDLED})
+option(USE_INTERNAL_RAPIDJSON_LIBRARY "Set to FALSE to use system rapidjson library instead of bundled" ON)
 
 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/rapidjson/include/rapidjson/rapidjson.h")
     if(USE_INTERNAL_RAPIDJSON_LIBRARY)

cmake/find/rdkafka.cmake

@@ -7,9 +7,9 @@ if (NOT ENABLE_RDKAFKA)
     return()
 endif()
 
-option (USE_INTERNAL_RDKAFKA_LIBRARY "Set to FALSE to use system librdkafka instead of the bundled" ${NOT_UNBUNDLED})
+option (USE_INTERNAL_RDKAFKA_LIBRARY "Set to FALSE to use system librdkafka instead of the bundled" ON)
 
-if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cppkafka/CMakeLists.txt")
+if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cppkafka/src")
     if(USE_INTERNAL_RDKAFKA_LIBRARY)
         message (WARNING "submodule contrib/cppkafka is missing. to fix try run: \n git submodule update --init")
         message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal cppkafka")
@@ -18,7 +18,7 @@ if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cppkafka/CMakeLists.txt")
     set (MISSING_INTERNAL_CPPKAFKA_LIBRARY 1)
 endif ()
 
-if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/librdkafka/CMakeLists.txt")
+if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/librdkafka/src")
     if(USE_INTERNAL_RDKAFKA_LIBRARY OR MISSING_INTERNAL_CPPKAFKA_LIBRARY)
         message (WARNING "submodule contrib/librdkafka is missing. to fix try run: \n git submodule update --init")
         message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal rdkafka")
@@ -40,7 +40,7 @@ if (NOT USE_INTERNAL_RDKAFKA_LIBRARY)
            message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system sasl2 library needed for static librdkafka")
        endif()
     endif ()
-    set (CPPKAFKA_LIBRARY cppkafka) # TODO: try to use unbundled version.
+    set (CPPKAFKA_LIBRARY cppkafka)
 endif ()
 
 if (RDKAFKA_LIB AND RDKAFKA_INCLUDE_DIR)

cmake/find/re2.cmake

@@ -1,6 +1,6 @@
-option (USE_INTERNAL_RE2_LIBRARY "Set to FALSE to use system re2 library instead of bundled [slower]" ${NOT_UNBUNDLED})
+option (USE_INTERNAL_RE2_LIBRARY "Set to FALSE to use system re2 library instead of bundled [slower]" ON)
 
-if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/re2/CMakeLists.txt")
+if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/re2/re2")
     if(USE_INTERNAL_RE2_LIBRARY)
         message(WARNING "submodule contrib/re2 is missing. to fix try run: \n git submodule update --init")
         message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal re2 library")

cmake/find/rocksdb.cmake

@@ -11,9 +11,9 @@ if (NOT ENABLE_ROCKSDB)
     return()
 endif()
 
-option(USE_INTERNAL_ROCKSDB_LIBRARY "Set to FALSE to use system ROCKSDB library instead of bundled" ${NOT_UNBUNDLED})
+option(USE_INTERNAL_ROCKSDB_LIBRARY "Set to FALSE to use system ROCKSDB library instead of bundled" ON)
 
-if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/rocksdb/CMakeLists.txt")
+if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/rocksdb/include")
     if (USE_INTERNAL_ROCKSDB_LIBRARY)
         message (WARNING "submodule contrib is missing. to fix try run: \n git submodule update --init")
         message(${RECONFIGURE_MESSAGE_LEVEL} "cannot find internal rocksdb")

cmake/find/sentry.cmake

@@ -9,7 +9,7 @@ if (NOT EXISTS "${SENTRY_INCLUDE_DIR}/sentry.h")
     return()
 endif ()
 
-if (NOT OS_FREEBSD AND NOT SPLIT_SHARED_LIBRARIES AND NOT_UNBUNDLED AND NOT (OS_DARWIN AND COMPILER_CLANG))
+if (NOT OS_FREEBSD AND NOT SPLIT_SHARED_LIBRARIES AND NOT (OS_DARWIN AND COMPILER_CLANG))
     option (USE_SENTRY "Use Sentry" ${ENABLE_LIBRARIES})
     set (SENTRY_TRANSPORT "curl" CACHE STRING "")
     set (SENTRY_BACKEND "none" CACHE STRING "")
@@ -18,8 +18,6 @@ if (NOT OS_FREEBSD AND NOT SPLIT_SHARED_LIBRARIES AND NOT_UNBUNDLED AND NOT (OS_
     set (SENTRY_PIC OFF CACHE BOOL "")
     set (BUILD_SHARED_LIBS OFF)
     message (STATUS "Using sentry=${USE_SENTRY}: ${SENTRY_LIBRARY}")
-
-    include_directories("${SENTRY_INCLUDE_DIR}")
 elseif (USE_SENTRY)
     message (${RECONFIGURE_MESSAGE_LEVEL} "Sentry is not supported in current configuration")
 endif ()

cmake/find/snappy.cmake

@@ -1,4 +1,4 @@
-option(USE_SNAPPY "Enable snappy library" ${ENABLE_LIBRARIES})
+option(USE_SNAPPY "Enable snappy library" ON)
 
 if(NOT USE_SNAPPY)
     if (USE_INTERNAL_SNAPPY_LIBRARY)
@@ -7,7 +7,7 @@ if(NOT USE_SNAPPY)
     return()
 endif()
 
-option (USE_INTERNAL_SNAPPY_LIBRARY "Set to FALSE to use system snappy library instead of bundled" ${NOT_UNBUNDLED})
+option (USE_INTERNAL_SNAPPY_LIBRARY "Set to FALSE to use system snappy library instead of bundled" ON)
 
 if(NOT USE_INTERNAL_SNAPPY_LIBRARY)
     find_library(SNAPPY_LIBRARY snappy)

cmake/find/ssl.cmake

@@ -9,7 +9,7 @@ if(NOT ENABLE_SSL)
     return()
 endif()
 
-option(USE_INTERNAL_SSL_LIBRARY "Set to FALSE to use system *ssl library instead of bundled" ${NOT_UNBUNDLED})
+option(USE_INTERNAL_SSL_LIBRARY "Set to FALSE to use system *ssl library instead of bundled" ON)
 
 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/boringssl/README.md")
     if(USE_INTERNAL_SSL_LIBRARY)

cmake/find/stats.cmake

@@ -1,24 +0,0 @@
-option(ENABLE_STATS "Enable StatsLib library" ${ENABLE_LIBRARIES})
-
-if (ENABLE_STATS)
-    if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/stats")
-        message (WARNING "submodule contrib/stats is missing. to fix try run: \n git submodule update --init")
-        set (ENABLE_STATS 0)
-        set (USE_STATS 0)
-    elseif (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/gcem")
-        message (WARNING "submodule contrib/gcem is missing. to fix try run: \n git submodule update --init")
-        set (ENABLE_STATS 0)
-        set (USE_STATS 0)
-    else()
-        set(STATS_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/stats/include)
-        set(GCEM_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/gcem/include)
-        set (USE_STATS 1)
-    endif()
-
-    if (NOT USE_STATS)
-        message (${RECONFIGURE_MESSAGE_LEVEL} "Can't enable stats library")
-    endif()
-endif()
-
-message (STATUS "Using stats=${USE_STATS} : ${STATS_INCLUDE_DIR}")
-message (STATUS "Using gcem=${USE_STATS}: ${GCEM_INCLUDE_DIR}")

cmake/find/xz.cmake

@@ -1,4 +1,4 @@
-option (USE_INTERNAL_XZ_LIBRARY "Set to OFF to use system xz (lzma) library instead of bundled" ${NOT_UNBUNDLED})
+option (USE_INTERNAL_XZ_LIBRARY "Set to OFF to use system xz (lzma) library instead of bundled" ON)
 
 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/xz/src/liblzma/api/lzma.h")
     if(USE_INTERNAL_XZ_LIBRARY)

cmake/find/zlib.cmake

@@ -1,4 +1,4 @@
-option (USE_INTERNAL_ZLIB_LIBRARY "Set to FALSE to use system zlib library instead of bundled" ${NOT_UNBUNDLED})
+option (USE_INTERNAL_ZLIB_LIBRARY "Set to FALSE to use system zlib library instead of bundled" ON)
 
 if (NOT MSVC)
     set (INTERNAL_ZLIB_NAME "zlib-ng" CACHE INTERNAL "")
@@ -29,9 +29,6 @@ if (NOT USE_INTERNAL_ZLIB_LIBRARY)
 endif ()
 
 if (NOT ZLIB_FOUND AND NOT MISSING_INTERNAL_ZLIB_LIBRARY)
-    # https://github.com/zlib-ng/zlib-ng/pull/733
-    # This is disabed by default
-    add_compile_definitions(Z_TLS=__thread)
     set (USE_INTERNAL_ZLIB_LIBRARY 1)
     set (ZLIB_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/${INTERNAL_ZLIB_NAME}" "${ClickHouse_BINARY_DIR}/contrib/${INTERNAL_ZLIB_NAME}" CACHE INTERNAL "") # generated zconf.h
     set (ZLIB_INCLUDE_DIRS ${ZLIB_INCLUDE_DIR}) # for poco

cmake/find/zstd.cmake

@@ -1,4 +1,4 @@
-option (USE_INTERNAL_ZSTD_LIBRARY "Set to FALSE to use system zstd library instead of bundled" ${NOT_UNBUNDLED})
+option (USE_INTERNAL_ZSTD_LIBRARY "Set to FALSE to use system zstd library instead of bundled" ON)
 
 if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/zstd/lib/zstd.h")
     if(USE_INTERNAL_ZSTD_LIBRARY)

cmake/linux/default_libs.cmake

@@ -28,7 +28,7 @@ set(CMAKE_C_STANDARD_LIBRARIES ${DEFAULT_LIBS})
 # glibc-compatibility library relies to constant version of libc headers
 # (because minor changes in function attributes between different glibc versions will introduce incompatibilities)
 # This is for x86_64. For other architectures we have separate toolchains.
-if (ARCH_AMD64 AND NOT_UNBUNDLED AND NOT CMAKE_CROSSCOMPILING)
+if (ARCH_AMD64 AND NOT CMAKE_CROSSCOMPILING)
     set(CMAKE_C_STANDARD_INCLUDE_DIRECTORIES ${ClickHouse_SOURCE_DIR}/contrib/libc-headers/x86_64-linux-gnu ${ClickHouse_SOURCE_DIR}/contrib/libc-headers)
     set(CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES ${ClickHouse_SOURCE_DIR}/contrib/libc-headers/x86_64-linux-gnu ${ClickHouse_SOURCE_DIR}/contrib/libc-headers)
 endif ()

cmake/warnings.cmake

@@ -7,9 +7,7 @@
 # - sometimes warnings from 3rd party libraries may come from macro substitutions in our code
 #   and we have to wrap them with #pragma GCC/clang diagnostic ignored
 
-if (NOT MSVC)
 set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wextra")
-endif ()
 
 # Add some warnings that are not available even with -Wall -Wextra -Wpedantic.
 # Intended for exploration of new compiler warnings that may be found useful.

contrib/CMakeLists.txt

@@ -51,7 +51,7 @@ if (USE_YAML_CPP)
 endif()
 
 if (USE_INTERNAL_XZ_LIBRARY)
-    add_subdirectory (xz)
+    add_subdirectory (xz-cmake)
 endif()
 
 add_subdirectory (poco-cmake)
@@ -64,9 +64,7 @@ if (USE_INTERNAL_ZSTD_LIBRARY)
 endif ()
 
 if (USE_INTERNAL_RE2_LIBRARY)
-    set(RE2_BUILD_TESTING 0 CACHE INTERNAL "")
+    add_subdirectory (re2-cmake)
-    add_subdirectory (re2)
-    add_subdirectory (re2_st)
 endif ()
 
 if (USE_INTERNAL_DOUBLE_CONVERSION_LIBRARY)
@@ -82,23 +80,10 @@ if (USE_INTERNAL_FARMHASH_LIBRARY)
 endif ()
 
 if (USE_INTERNAL_ZLIB_LIBRARY)
-    set (ZLIB_ENABLE_TESTS 0 CACHE INTERNAL "")
+    if (INTERNAL_ZLIB_NAME STREQUAL "zlib-ng")
-    set (SKIP_INSTALL_ALL 1 CACHE INTERNAL "")
+        add_subdirectory (zlib-ng-cmake)
-    set (ZLIB_COMPAT 1 CACHE INTERNAL "") # also enables WITH_GZFILEOP
+    else ()
-    set (WITH_NATIVE_INSTRUCTIONS ${ARCH_NATIVE} CACHE INTERNAL "")
-    if (OS_FREEBSD OR ARCH_I386)
-        set (WITH_OPTIM 0 CACHE INTERNAL "") # Bug in assembler
-    endif ()
-    if (ARCH_AARCH64)
-        set(WITH_NEON 1 CACHE INTERNAL "")
-        set(WITH_ACLE 1 CACHE INTERNAL "")
-    endif ()
-
         add_subdirectory (${INTERNAL_ZLIB_NAME})
-    # We should use same defines when including zlib.h as used when zlib compiled
-    target_compile_definitions (zlib PUBLIC ZLIB_COMPAT WITH_GZFILEOP)
-    if (ARCH_AMD64 OR ARCH_AARCH64)
-        target_compile_definitions (zlib PUBLIC X86_64 UNALIGNED_OK)
     endif ()
 endif ()
 
@@ -117,28 +102,8 @@ if (USE_INTERNAL_LDAP_LIBRARY)
     add_subdirectory (openldap-cmake)
 endif ()
 
-function(mysql_support)
+if (USE_INTERNAL_MYSQL_LIBRARY)
-    set(CLIENT_PLUGIN_CACHING_SHA2_PASSWORD STATIC)
+    add_subdirectory (mariadb-connector-c-cmake)
-    set(CLIENT_PLUGIN_SHA256_PASSWORD STATIC)
-    set(CLIENT_PLUGIN_REMOTE_IO OFF)
-    set(CLIENT_PLUGIN_DIALOG OFF)
-    set(CLIENT_PLUGIN_AUTH_GSSAPI_CLIENT OFF)
-    set(CLIENT_PLUGIN_CLIENT_ED25519 OFF)
-    set(CLIENT_PLUGIN_MYSQL_CLEAR_PASSWORD OFF)
-    set(SKIP_TESTS 1)
-    if (GLIBC_COMPATIBILITY)
-        set(LIBM glibc-compatibility)
-    endif()
-    if (USE_INTERNAL_ZLIB_LIBRARY)
-        set(ZLIB_FOUND ON)
-        set(ZLIB_LIBRARY ${ZLIB_LIBRARIES})
-        set(WITH_EXTERNAL_ZLIB ON)
-    endif()
-    set(WITH_CURL OFF)
-    add_subdirectory (mariadb-connector-c)
-endfunction()
-if (ENABLE_MYSQL AND USE_INTERNAL_MYSQL_LIBRARY)
-    mysql_support()
 endif ()
 
 if (USE_INTERNAL_RDKAFKA_LIBRARY)
@@ -194,11 +159,7 @@ if (USE_INTERNAL_AVRO_LIBRARY)
 endif()
 
 if(USE_INTERNAL_GTEST_LIBRARY)
-    set(GOOGLETEST_VERSION 1.10.0) # master
+    add_subdirectory(googletest-cmake)
-    # Google Test from sources
-    add_subdirectory(${ClickHouse_SOURCE_DIR}/contrib/googletest/googletest ${CMAKE_CURRENT_BINARY_DIR}/googletest)
-    # avoid problems with <regexp.h>
-    target_compile_definitions (gtest INTERFACE GTEST_HAS_POSIX_RE=0)
 elseif(GTEST_SRC_DIR)
     add_subdirectory(${GTEST_SRC_DIR}/googletest ${CMAKE_CURRENT_BINARY_DIR}/googletest)
     target_compile_definitions(gtest INTERFACE GTEST_HAS_POSIX_RE=0)
@@ -229,7 +190,7 @@ if (USE_EMBEDDED_COMPILER)
 endif ()
 
 if (USE_INTERNAL_LIBGSASL_LIBRARY)
-    add_subdirectory(libgsasl)
+    add_subdirectory(libgsasl-cmake)
 endif()
 
 if (USE_INTERNAL_LIBXML2_LIBRARY)
@@ -281,14 +242,7 @@ if (USE_AMQPCPP)
     add_subdirectory (amqpcpp-cmake)
 endif()
 if (USE_CASSANDRA)
-    # Need to use C++17 since the compilation is not possible with C++20 currently.
+    add_subdirectory (cassandra-cmake)
-    set (CMAKE_CXX_STANDARD_bak ${CMAKE_CXX_STANDARD})
-    set (CMAKE_CXX_STANDARD 17)
-
-    add_subdirectory (cassandra)
-
-    set (CMAKE_CXX_STANDARD ${CMAKE_CXX_STANDARD_bak})
-    unset (CMAKE_CXX_STANDARD_bak)
 endif()
 
 # Should go before:
@@ -296,16 +250,11 @@ endif()
 add_subdirectory (curl-cmake)
 
 if (USE_SENTRY)
-    add_subdirectory (sentry-native)
+    add_subdirectory (sentry-native-cmake)
 endif()
 
 add_subdirectory (fmtlib-cmake)
 
-if (USE_STATS)
-    add_subdirectory (stats-cmake)
-    add_subdirectory (gcem)
-endif()
-
 if (USE_KRB5)
     add_subdirectory (krb5-cmake)
     if (USE_CYRUS_SASL)
@@ -326,7 +275,7 @@ if (USE_NURAFT)
   add_subdirectory(nuraft-cmake)
 endif()
 
-add_subdirectory(fast_float)
+add_subdirectory(fast_float-cmake)
 
 if (USE_NLP)
     add_subdirectory(libstemmer-c-cmake)

contrib/arrow-cmake/CMakeLists.txt

@@ -417,7 +417,49 @@ set(PARQUET_SRCS
 #list(TRANSFORM PARQUET_SRCS PREPEND "${LIBRARY_DIR}/") # cmake 3.12
 add_library(${PARQUET_LIBRARY} ${PARQUET_SRCS})
 target_include_directories(${PARQUET_LIBRARY} SYSTEM PUBLIC "${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src" "${CMAKE_CURRENT_SOURCE_DIR}/cpp/src" PRIVATE ${OPENSSL_INCLUDE_DIR})
-include("${ClickHouse_SOURCE_DIR}/contrib/thrift/build/cmake/ConfigureChecks.cmake") # makes config.h
+
+set (HAVE_ARPA_INET_H 1)
+set (HAVE_FCNTL_H 1)
+set (HAVE_GETOPT_H 1)
+set (HAVE_INTTYPES_H 1)
+set (HAVE_NETDB_H 1)
+set (HAVE_NETINET_IN_H 1)
+set (HAVE_SIGNAL_H 1)
+set (HAVE_STDINT_H 1)
+set (HAVE_UNISTD_H 1)
+set (HAVE_PTHREAD_H 1)
+set (HAVE_SYS_IOCTL_H 1)
+set (HAVE_SYS_PARAM_H 1)
+set (HAVE_SYS_RESOURCE_H 1)
+set (HAVE_SYS_SOCKET_H 1)
+set (HAVE_SYS_STAT_H 1)
+set (HAVE_SYS_TIME_H 1)
+set (HAVE_SYS_UN_H 1)
+set (HAVE_POLL_H 1)
+set (HAVE_SYS_POLL_H 1)
+set (HAVE_SYS_SELECT_H 1)
+set (HAVE_SCHED_H 1)
+set (HAVE_STRING_H 1)
+set (HAVE_STRINGS_H 1)
+set (HAVE_GETHOSTBYNAME 1)
+set (HAVE_STRERROR_R 1)
+set (HAVE_SCHED_GET_PRIORITY_MAX 1)
+set (HAVE_SCHED_GET_PRIORITY_MIN 1)
+
+if (OS_LINUX)
+    set (STRERROR_R_CHAR_P 1)
+endif ()
+
+#set(PACKAGE ${PACKAGE_NAME})
+#set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}")
+#set(VERSION ${thrift_VERSION})
+
+# generate a config.h file
+configure_file("${CMAKE_CURRENT_SOURCE_DIR}/build/cmake/config.h.in" "${CMAKE_CURRENT_BINARY_DIR}/thrift/config.h")
+
+include_directories("${CMAKE_CURRENT_BINARY_DIR}")
+
+
 target_link_libraries(${PARQUET_LIBRARY} PUBLIC ${ARROW_LIBRARY} PRIVATE ${THRIFT_LIBRARY} boost::headers_only boost::regex ${OPENSSL_LIBRARIES})
 
 if (SANITIZE STREQUAL "undefined")

contrib/boost-cmake/CMakeLists.txt

@@ -1,4 +1,4 @@
-option (USE_INTERNAL_BOOST_LIBRARY "Use internal Boost library" ${NOT_UNBUNDLED})
+option (USE_INTERNAL_BOOST_LIBRARY "Use internal Boost library" ON)
 
 if (NOT USE_INTERNAL_BOOST_LIBRARY)
     # 1.70 like in contrib/boost

contrib/cassandra-cmake/CMakeLists.txt

@@ -0,0 +1,127 @@
+# Need to use C++17 since the compilation is not possible with C++20 currently.
+set (CMAKE_CXX_STANDARD 17)
+
+set(CASS_ROOT_DIR ${CMAKE_SOURCE_DIR}/contrib/cassandra)
+set(CASS_SRC_DIR "${CASS_ROOT_DIR}/src")
+set(CASS_INCLUDE_DIR "${CASS_ROOT_DIR}/include")
+
+# Ensure functions/modules are available
+list(APPEND CMAKE_MODULE_PATH ${CASS_ROOT_DIR}/cmake)
+
+set(CASS_BUILD_SHARED 1)
+set(CASS_BUILD_STATIC 1)
+set(CASS_USE_KERBEROS 0)
+set(CASS_USE_LIBSSH2 0)
+set(CASS_USE_OPENSSL 1)
+set(CASS_USE_STD_ATOMIC 1)
+set(CASS_USE_ZLIB 1)
+
+
+file(GLOB SOURCES ${CASS_SRC_DIR}/*.cpp)
+
+if(APPLE)
+  list(REMOVE_ITEM SOURCES ${CASS_SRC_DIR}/get_time-unix.cpp ${CASS_SRC_DIR}/get_time-win.cpp)
+elseif(UNIX)
+  list(REMOVE_ITEM SOURCES ${CASS_SRC_DIR}/get_time-mac.cpp ${CASS_SRC_DIR}/get_time-win.cpp)
+elseif(WIN32)
+  list(REMOVE_ITEM SOURCES ${CASS_SRC_DIR}/get_time-mac.cpp ${CASS_SRC_DIR}/get_time-unix.cpp)
+endif()
+
+if(CASS_USE_OPENSSL)
+  list(APPEND INCLUDE_DIRS ${CASS_SRC_DIR}/ssl)
+  list(APPEND SOURCES ${CASS_SRC_DIR}/ssl/ssl_openssl_impl.cpp ${CASS_SRC_DIR}/ssl/ring_buffer_bio.cpp)
+else()
+  list(APPEND SOURCES ${CASS_SRC_DIR}/ssl/ssl_no_impl.cpp)
+endif()
+
+if(CASS_USE_KERBEROS)
+  list(APPEND INCLUDE_DIRS ${CASS_SRC_DIR}/gssapi)
+  list(APPEND SOURCES ${CASS_SRC_DIR}/gssapi/dse_auth_gssapi.cpp ${CASS_SRC_DIR}/gssapi/dse_auth_gssapi.hpp)
+endif()
+
+list(APPEND SOURCES ${CASS_SRC_DIR}/atomic/atomic_std.hpp)
+
+
+add_library(curl_hostcheck OBJECT ${CASS_SRC_DIR}/third_party/curl/hostcheck.cpp)
+add_library(hdr_histogram OBJECT ${CASS_SRC_DIR}/third_party/hdr_histogram/hdr_histogram.cpp)
+add_library(http-parser OBJECT ${CASS_SRC_DIR}/third_party/http-parser/http_parser.c)
+add_library(minizip OBJECT
+    ${CASS_SRC_DIR}/third_party/minizip/ioapi.c
+    ${CASS_SRC_DIR}/third_party/minizip/zip.c
+    ${CASS_SRC_DIR}/third_party/minizip/unzip.c)
+
+target_link_libraries(minizip zlib)
+target_compile_definitions(minizip PRIVATE "-Dz_crc_t=unsigned long")
+
+list(APPEND INCLUDE_DIRS
+  ${CASS_SRC_DIR}/third_party/curl
+  ${CASS_SRC_DIR}/third_party/hdr_histogram
+  ${CASS_SRC_DIR}/third_party/http-parser
+  ${CASS_SRC_DIR}/third_party/minizip
+  ${CASS_SRC_DIR}/third_party/mt19937_64
+  ${CASS_SRC_DIR}/third_party/rapidjson/rapidjson
+  ${CASS_SRC_DIR}/third_party/sparsehash/src)
+
+list(APPEND INCLUDE_DIRS ${CASS_INCLUDE_DIR} ${CASS_SRC_DIR})
+
+set(HASH_FUN_H "functional")
+set(HASH_NAME hash)
+set(HASH_NAMESPACE "std")
+set(HAVE_INTTYPES_H 1)
+set(HAVE_STDINT_H 1)
+set(HAVE_SYS_TYPES_H 1)
+set(HAVE_MEMCPY 1)
+set(HAVE_LONG_LONG 1)
+set(HAVE_UINT16_T 1)
+
+configure_file("${CASS_SRC_DIR}/third_party/sparsehash/config.h.cmake" "${CMAKE_CURRENT_BINARY_DIR}/sparsehash/internal/sparseconfig.h")
+
+
+
+# Determine random availability
+if (OS_LINUX)
+  #set (HAVE_GETRANDOM 1) - not on every Linux kernel
+elseif (OS_FREEBSD OR OS_DARWIN)
+  set (HAVE_ARC4RANDOM 1)
+endif ()
+
+# Determine if sigpipe is available
+if (OS_LINUX)
+    set (HAVE_SIGTIMEDWAIT 1)
+else (OS_FREEBSD OR OS_DARWIN)
+    set (HAVE_NOSIGPIPE 1)
+endif()
+
+set (HAVE_BUILTIN_BSWAP32 1)
+set (HAVE_BUILTIN_BSWAP64 1)
+
+set(HAVE_BOOST_ATOMIC 0)
+set(HAVE_STD_ATOMIC 1)
+
+set(HAVE_KERBEROS ${CASS_USE_KERBEROS})
+set(HAVE_OPENSSL ${CASS_USE_OPENSSL})
+set(HAVE_ZLIB ${CASS_USE_ZLIB})
+
+# Generate the driver_config.hpp file
+configure_file(
+  ${CASS_ROOT_DIR}/driver_config.hpp.in
+  ${CMAKE_CURRENT_BINARY_DIR}/driver_config.hpp)
+
+
+add_library(cassandra
+    ${SOURCES}
+    $<TARGET_OBJECTS:curl_hostcheck>
+    $<TARGET_OBJECTS:hdr_histogram>
+    $<TARGET_OBJECTS:http-parser>
+    $<TARGET_OBJECTS:minizip>)
+
+target_link_libraries(cassandra zlib)
+add_library(cassandra_static ALIAS cassandra)
+target_include_directories(cassandra PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${INCLUDE_DIRS})
+target_compile_definitions(cassandra PRIVATE CASS_BUILDING)
+
+target_link_libraries(cassandra uv)
+
+if(CASS_USE_OPENSSL)
+    target_link_libraries(cassandra ssl)
+endif()

contrib/cctz-cmake/CMakeLists.txt

@@ -1,4 +1,4 @@
-option (USE_INTERNAL_CCTZ_LIBRARY "Use internal cctz library" ${NOT_UNBUNDLED})
+option (USE_INTERNAL_CCTZ_LIBRARY "Use internal cctz library" ON)
 
 if (NOT USE_INTERNAL_CCTZ_LIBRARY)
     find_library (LIBRARY_CCTZ cctz)

contrib/fast_float-cmake/CMakeLists.txt

@@ -0,0 +1,2 @@
+add_library(fast_float INTERFACE)
+target_include_directories(fast_float INTERFACE "${ClickHouse_SOURCE_DIR}/contrib/fast_float/include/")

contrib/gcem

@@ -1 +0,0 @@
-Subproject commit 8d4f1b5d76ea8f6ff12f3f4f34cda45424556b00

contrib/googletest-cmake/CMakeLists.txt

@@ -0,0 +1,11 @@
+set (SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/googletest/googletest")
+
+add_library(gtest "${SRC_DIR}/src/gtest-all.cc")
+set_target_properties(gtest PROPERTIES VERSION "1.0.0")
+target_compile_definitions (gtest INTERFACE GTEST_HAS_POSIX_RE=0)
+target_include_directories(gtest SYSTEM PUBLIC "${SRC_DIR}/include")
+target_include_directories(gtest PRIVATE "${SRC_DIR}")
+
+add_library(gtest_main "${SRC_DIR}/src/gtest_main.cc")
+set_target_properties(gtest_main PROPERTIES VERSION "1.0.0")
+target_link_libraries(gtest_main PUBLIC gtest)

contrib/hyperscan-cmake/CMakeLists.txt

@@ -17,7 +17,7 @@ if (NOT ENABLE_HYPERSCAN)
     return()
 endif()
 
-option (USE_INTERNAL_HYPERSCAN_LIBRARY "Use internal hyperscan library" ${NOT_UNBUNDLED})
+option (USE_INTERNAL_HYPERSCAN_LIBRARY "Use internal hyperscan library" ON)
 
 if (NOT USE_INTERNAL_HYPERSCAN_LIBRARY)
     find_library (LIBRARY_HYPERSCAN hs)

contrib/libgsasl-cmake/CMakeLists.txt

@@ -0,0 +1,107 @@
+set (SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/libgsasl")
+
+set(SRCS
+    ${SRC_DIR}/gl/gc-gnulib.c
+    ${SRC_DIR}/gl/printf-parse.c
+    ${SRC_DIR}/gl/c-ctype.c
+    ${SRC_DIR}/gl/float.c
+    ${SRC_DIR}/gl/printf-args.c
+    ${SRC_DIR}/gl/hmac-sha1.c
+    ${SRC_DIR}/gl/itold.c
+    ${SRC_DIR}/gl/hmac-md5.c
+    ${SRC_DIR}/gl/gc-pbkdf2-sha1.c
+    ${SRC_DIR}/gl/md5.c
+    ${SRC_DIR}/gl/base64.c
+    ${SRC_DIR}/gl/memxor.c
+    ${SRC_DIR}/gl/sha1.c
+    ${SRC_DIR}/openid20/client.c
+    ${SRC_DIR}/openid20/mechinfo.c
+    ${SRC_DIR}/openid20/server.c
+    ${SRC_DIR}/anonymous/client.c
+    ${SRC_DIR}/anonymous/mechinfo.c
+    ${SRC_DIR}/anonymous/server.c
+    ${SRC_DIR}/saml20/client.c
+    ${SRC_DIR}/saml20/mechinfo.c
+    ${SRC_DIR}/saml20/server.c
+    ${SRC_DIR}/scram/parser.c
+    ${SRC_DIR}/scram/printer.c
+    ${SRC_DIR}/scram/tokens.c
+    ${SRC_DIR}/scram/client.c
+    ${SRC_DIR}/scram/mechinfo.c
+    ${SRC_DIR}/scram/server.c
+    ${SRC_DIR}/scram/validate.c
+    ${SRC_DIR}/src/free.c
+    ${SRC_DIR}/src/supportp.c
+    ${SRC_DIR}/src/init.c
+    ${SRC_DIR}/src/mechtools.c
+    ${SRC_DIR}/src/error.c
+    ${SRC_DIR}/src/property.c
+    ${SRC_DIR}/src/done.c
+    ${SRC_DIR}/src/callback.c
+    ${SRC_DIR}/src/xstart.c
+    ${SRC_DIR}/src/xfinish.c
+    ${SRC_DIR}/src/version.c
+    ${SRC_DIR}/src/xstep.c
+    ${SRC_DIR}/src/mechname.c
+    ${SRC_DIR}/src/xcode.c
+    ${SRC_DIR}/src/crypto.c
+    ${SRC_DIR}/src/doxygen.c
+    ${SRC_DIR}/src/suggest.c
+    ${SRC_DIR}/src/saslprep.c
+    ${SRC_DIR}/src/listmech.c
+    ${SRC_DIR}/src/register.c
+    ${SRC_DIR}/src/base64.c
+    ${SRC_DIR}/src/md5pwd.c
+    ${SRC_DIR}/external/client.c
+    ${SRC_DIR}/external/mechinfo.c
+    ${SRC_DIR}/external/server.c
+    ${SRC_DIR}/securid/client.c
+    ${SRC_DIR}/securid/mechinfo.c
+    ${SRC_DIR}/securid/server.c
+    ${SRC_DIR}/plain/client.c
+    ${SRC_DIR}/plain/mechinfo.c
+    ${SRC_DIR}/plain/server.c
+    ${SRC_DIR}/cram-md5/client.c
+    ${SRC_DIR}/cram-md5/challenge.c
+    ${SRC_DIR}/cram-md5/mechinfo.c
+    ${SRC_DIR}/cram-md5/server.c
+    ${SRC_DIR}/cram-md5/digest.c
+    ${SRC_DIR}/digest-md5/client.c
+    ${SRC_DIR}/digest-md5/digesthmac.c
+    ${SRC_DIR}/digest-md5/free.c
+    ${SRC_DIR}/digest-md5/getsubopt.c
+    ${SRC_DIR}/digest-md5/mechinfo.c
+    ${SRC_DIR}/digest-md5/nonascii.c
+    ${SRC_DIR}/digest-md5/parser.c
+    ${SRC_DIR}/digest-md5/printer.c
+    ${SRC_DIR}/digest-md5/qop.c
+    ${SRC_DIR}/digest-md5/server.c
+    ${SRC_DIR}/digest-md5/session.c
+    ${SRC_DIR}/digest-md5/test-parser.c
+    ${SRC_DIR}/digest-md5/validate.c
+    ${SRC_DIR}/login/client.c
+    ${SRC_DIR}/login/mechinfo.c
+    ${SRC_DIR}/login/server.c
+)
+
+if (USE_KRB5)
+    set(SRCS ${SRCS}
+        ${SRC_DIR}/gssapi/client.c
+        ${SRC_DIR}/gssapi/mechinfo.c
+        ${SRC_DIR}/gssapi/server.c)
+endif()
+
+add_library(gsasl ${SRCS})
+
+target_include_directories(gsasl PUBLIC ${SRC_DIR})
+target_include_directories(gsasl PUBLIC ${SRC_DIR}/gl)
+target_include_directories(gsasl PUBLIC ${SRC_DIR}/src)
+target_include_directories(gsasl PUBLIC ${SRC_DIR}/digest-md5)
+target_include_directories(gsasl PRIVATE "${ClickHouse_SOURCE_DIR}/contrib/libgsasl-cmake/linux_x86_64/include")
+
+target_compile_definitions (gsasl PRIVATE HAVE_CONFIG_H=1)
+
+if (USE_KRB5)
+    target_link_libraries(gsasl PUBLIC ${KRB5_LIBRARY})
+    target_compile_definitions (gsasl PRIVATE HAVE_GSSAPI_H=1 USE_GSSAPI=1)
+endif()

contrib/libhdfs3-cmake/CMakeLists.txt

@@ -1,23 +1,4 @@
-if (ENABLE_PROTOBUF AND NOT USE_INTERNAL_PROTOBUF_LIBRARY)
+if (${ENABLE_KRB5})
-    option(PROTOBUF_OLD_ABI_COMPAT "Set to ON for compatiability with external protobuf which was compiled old C++ ABI" OFF)
-endif()
-
-if (PROTOBUF_OLD_ABI_COMPAT)
-    if (NOT ENABLE_PROTOBUF OR USE_INTERNAL_PROTOBUF_LIBRARY)
-        message (${RECONFIGURE_MESSAGE_LEVEL} "PROTOBUF_OLD_ABI_COMPAT option is ignored")
-    endif()
-endif()
-
-if (NOT USE_INTERNAL_PROTOBUF_LIBRARY AND PROTOBUF_OLD_ABI_COMPAT)
-    # compatiable with protobuf which was compiled old C++ ABI
-    set(CMAKE_CXX_FLAGS "-D_GLIBCXX_USE_CXX11_ABI=0")
-    set(CMAKE_C_FLAGS "")
-    if (NOT (CMAKE_VERSION VERSION_LESS "3.8.0"))
-        unset(CMAKE_CXX_STANDARD)
-    endif ()
-endif()
-
-if (${ENABLE_LIBRARIES} AND ${ENABLE_KRB5})
     SET(WITH_KERBEROS 1)
 else()
     SET(WITH_KERBEROS 0)
@@ -46,9 +27,7 @@ set(PROTO_FILES
     "${HDFS3_SOURCE_DIR}/proto/datatransfer.proto"
     )
 
-if(USE_PROTOBUF)
 PROTOBUF_GENERATE_CPP(PROTO_SOURCES PROTO_HEADERS ${PROTO_FILES})
-endif()
 
 configure_file("${HDFS3_SOURCE_DIR}/platform.h.in" "${CMAKE_CURRENT_BINARY_DIR}/platform.h")
 
@@ -108,95 +87,14 @@ set(SRCS
     "${HDFS3_SOURCE_DIR}/common/Hash.cpp"
     "${HDFS3_SOURCE_DIR}/common/SWCrc32c.cpp"
     "${HDFS3_SOURCE_DIR}/common/Thread.cpp"
-
+    ${PROTO_SOURCES}
-    "${HDFS3_SOURCE_DIR}/network/TcpSocket.h"
-    "${HDFS3_SOURCE_DIR}/network/BufferedSocketReader.h"
-    "${HDFS3_SOURCE_DIR}/network/Socket.h"
-    "${HDFS3_SOURCE_DIR}/network/DomainSocket.h"
-    "${HDFS3_SOURCE_DIR}/network/Syscall.h"
-    "${HDFS3_SOURCE_DIR}/client/InputStreamImpl.h"
-    "${HDFS3_SOURCE_DIR}/client/FileSystem.h"
-    "${HDFS3_SOURCE_DIR}/client/ReadShortCircuitInfo.h"
-    "${HDFS3_SOURCE_DIR}/client/InputStreamInter.h"
-    "${HDFS3_SOURCE_DIR}/client/FileSystemImpl.h"
-    "${HDFS3_SOURCE_DIR}/client/PacketPool.h"
-    "${HDFS3_SOURCE_DIR}/client/Pipeline.h"
-    "${HDFS3_SOURCE_DIR}/client/OutputStreamInter.h"
-    "${HDFS3_SOURCE_DIR}/client/RemoteBlockReader.h"
-    "${HDFS3_SOURCE_DIR}/client/Token.h"
-    "${HDFS3_SOURCE_DIR}/client/KerberosName.h"
-    "${HDFS3_SOURCE_DIR}/client/DirectoryIterator.h"
-    "${HDFS3_SOURCE_DIR}/client/hdfs.h"
-    "${HDFS3_SOURCE_DIR}/client/FileSystemStats.h"
-    "${HDFS3_SOURCE_DIR}/client/FileSystemKey.h"
-    "${HDFS3_SOURCE_DIR}/client/DataTransferProtocolSender.h"
-    "${HDFS3_SOURCE_DIR}/client/Packet.h"
-    "${HDFS3_SOURCE_DIR}/client/PacketHeader.h"
-    "${HDFS3_SOURCE_DIR}/client/FileSystemInter.h"
-    "${HDFS3_SOURCE_DIR}/client/LocalBlockReader.h"
-    "${HDFS3_SOURCE_DIR}/client/TokenInternal.h"
-    "${HDFS3_SOURCE_DIR}/client/InputStream.h"
-    "${HDFS3_SOURCE_DIR}/client/PipelineAck.h"
-    "${HDFS3_SOURCE_DIR}/client/BlockReader.h"
-    "${HDFS3_SOURCE_DIR}/client/Permission.h"
-    "${HDFS3_SOURCE_DIR}/client/OutputStreamImpl.h"
-    "${HDFS3_SOURCE_DIR}/client/LeaseRenewer.h"
-    "${HDFS3_SOURCE_DIR}/client/UserInfo.h"
-    "${HDFS3_SOURCE_DIR}/client/PeerCache.h"
-    "${HDFS3_SOURCE_DIR}/client/OutputStream.h"
-    "${HDFS3_SOURCE_DIR}/client/FileStatus.h"
-    "${HDFS3_SOURCE_DIR}/client/DataTransferProtocol.h"
-    "${HDFS3_SOURCE_DIR}/client/BlockLocation.h"
-    "${HDFS3_SOURCE_DIR}/rpc/RpcConfig.h"
-    "${HDFS3_SOURCE_DIR}/rpc/SaslClient.h"
-    "${HDFS3_SOURCE_DIR}/rpc/RpcAuth.h"
-    "${HDFS3_SOURCE_DIR}/rpc/RpcClient.h"
-    "${HDFS3_SOURCE_DIR}/rpc/RpcCall.h"
-    "${HDFS3_SOURCE_DIR}/rpc/RpcContentWrapper.h"
-    "${HDFS3_SOURCE_DIR}/rpc/RpcProtocolInfo.h"
-    "${HDFS3_SOURCE_DIR}/rpc/RpcRemoteCall.h"
-    "${HDFS3_SOURCE_DIR}/rpc/RpcServerInfo.h"
-    "${HDFS3_SOURCE_DIR}/rpc/RpcChannel.h"
-    "${HDFS3_SOURCE_DIR}/rpc/RpcChannelKey.h"
-    "${HDFS3_SOURCE_DIR}/server/BlockLocalPathInfo.h"
-    "${HDFS3_SOURCE_DIR}/server/LocatedBlocks.h"
-    "${HDFS3_SOURCE_DIR}/server/DatanodeInfo.h"
-    "${HDFS3_SOURCE_DIR}/server/RpcHelper.h"
-    "${HDFS3_SOURCE_DIR}/server/ExtendedBlock.h"
-    "${HDFS3_SOURCE_DIR}/server/NamenodeInfo.h"
-    "${HDFS3_SOURCE_DIR}/server/NamenodeImpl.h"
-    "${HDFS3_SOURCE_DIR}/server/LocatedBlock.h"
-    "${HDFS3_SOURCE_DIR}/server/NamenodeProxy.h"
-    "${HDFS3_SOURCE_DIR}/server/Datanode.h"
-    "${HDFS3_SOURCE_DIR}/server/Namenode.h"
-    "${HDFS3_SOURCE_DIR}/common/XmlConfig.h"
-    "${HDFS3_SOURCE_DIR}/common/Logger.h"
-    "${HDFS3_SOURCE_DIR}/common/WriteBuffer.h"
-    "${HDFS3_SOURCE_DIR}/common/HWCrc32c.h"
-    "${HDFS3_SOURCE_DIR}/common/Checksum.h"
-    "${HDFS3_SOURCE_DIR}/common/SessionConfig.h"
-    "${HDFS3_SOURCE_DIR}/common/Unordered.h"
-    "${HDFS3_SOURCE_DIR}/common/BigEndian.h"
-    "${HDFS3_SOURCE_DIR}/common/Thread.h"
-    "${HDFS3_SOURCE_DIR}/common/StackPrinter.h"
-    "${HDFS3_SOURCE_DIR}/common/Exception.h"
-    "${HDFS3_SOURCE_DIR}/common/WritableUtils.h"
-    "${HDFS3_SOURCE_DIR}/common/StringUtil.h"
-    "${HDFS3_SOURCE_DIR}/common/LruMap.h"
-    "${HDFS3_SOURCE_DIR}/common/Function.h"
-    "${HDFS3_SOURCE_DIR}/common/DateTime.h"
-    "${HDFS3_SOURCE_DIR}/common/Hash.h"
-    "${HDFS3_SOURCE_DIR}/common/SWCrc32c.h"
-    "${HDFS3_SOURCE_DIR}/common/ExceptionInternal.h"
-    "${HDFS3_SOURCE_DIR}/common/Memory.h"
-    "${HDFS3_SOURCE_DIR}/common/FileWrapper.h"
 )
 
 # old kernels (< 3.17) doesn't have SYS_getrandom. Always use POSIX implementation to have better compatibility
 set_source_files_properties("${HDFS3_SOURCE_DIR}/rpc/RpcClient.cpp" PROPERTIES COMPILE_FLAGS "-DBOOST_UUID_RANDOM_PROVIDER_FORCE_POSIX=1")
 
 # target
-add_library(hdfs3 ${SRCS} ${PROTO_SOURCES} ${PROTO_HEADERS})
+add_library(hdfs3 ${SRCS})
 
 if(USE_INTERNAL_PROTOBUF_LIBRARY)
     add_dependencies(hdfs3 protoc)
@@ -218,6 +116,7 @@ target_link_libraries(hdfs3 PRIVATE ${LIBXML2_LIBRARIES})
 # inherit from parent cmake
 target_include_directories(hdfs3 PRIVATE ${Protobuf_INCLUDE_DIR})
 target_link_libraries(hdfs3 PRIVATE ${Protobuf_LIBRARY} boost::headers_only)
+
 if(OPENSSL_INCLUDE_DIR AND OPENSSL_LIBRARIES)
     target_include_directories(hdfs3 PRIVATE ${OPENSSL_INCLUDE_DIR})
     target_link_libraries(hdfs3 PRIVATE ${OPENSSL_LIBRARIES})

contrib/librdkafka-cmake/config.h.in

@@ -66,7 +66,7 @@
 #cmakedefine WITH_SASL_OAUTHBEARER 1
 #cmakedefine WITH_SASL_CYRUS 1
 // crc32chw
-#if !defined(__PPC__) && !defined(__riscv) && (!defined(__aarch64__) || defined(__ARM_FEATURE_CRC32)) && !(defined(__aarch64__) && defined(__APPLE__))
+#if !defined(__PPC__) && !defined(__riscv) && !defined(__aarch64__)
 #define WITH_CRC32C_HW 1
 #endif
 // regex

contrib/lz4-cmake/CMakeLists.txt

@@ -1,4 +1,4 @@
-option (USE_INTERNAL_LZ4_LIBRARY "Use internal lz4 library" ${NOT_UNBUNDLED})
+option (USE_INTERNAL_LZ4_LIBRARY "Use internal lz4 library" ON)
 
 if (NOT USE_INTERNAL_LZ4_LIBRARY)
     find_library (LIBRARY_LZ4 lz4)

contrib/mariadb-connector-c-cmake/CMakeLists.txt

@@ -0,0 +1,243 @@
+if (GLIBC_COMPATIBILITY)
+    set(LIBM glibc-compatibility)
+endif()
+
+# This is the LGPL libmariadb project.
+
+set(CC_SOURCE_DIR ${CMAKE_SOURCE_DIR}/contrib/mariadb-connector-c)
+set(CC_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
+
+set(WITH_SSL ON)
+
+set(MARIADB_CONNECTOR_C_COPYRIGHT "2013-2017 MariaDB Corporation Ab")
+
+set(PROTOCOL_VERSION 10) # we adapted new password option from PHP's mysqlnd !
+
+# if C/C is build as subproject inside MariaDB server tree we will
+# use the version defined by server
+if(MAJOR_VERSION)
+  set(MARIADB_CLIENT_VERSION_MAJOR ${MAJOR_VERSION})
+  set(MARIADB_CLIENT_VERSION_MINOR ${MINOR_VERSION})
+  set(MARIADB_CLIENT_VERSION_PATCH ${PATCH_VERSION})
+  set(MARIADB_CLIENT_VERSION_EXTRA ${EXTRA_VERSION})
+else()
+  set(MARIADB_CLIENT_VERSION_MAJOR "10")
+  set(MARIADB_CLIENT_VERSION_MINOR "4")
+  set(MARIADB_CLIENT_VERSION_PATCH "3")
+  set(MARIADB_CLIENT_VERSION_EXTRA "")
+endif()
+set(MARIADB_CLIENT_VERSION "${MARIADB_CLIENT_VERSION_MAJOR}.${MARIADB_CLIENT_VERSION_MINOR}.${MARIADB_CLIENT_VERSION_PATCH}${MARIADB_CLIENT_VERSION_EXTRA}")
+set(MARIADB_BASE_VERSION "mariadb-${MARIADB_CLIENT_VERSION_MAJOR}.${MARIADB_CLIENT_VERSION_MINOR}")
+MATH(EXPR MARIADB_VERSION_ID "${MARIADB_CLIENT_VERSION_MAJOR} * 10000 +
+                              ${MARIADB_CLIENT_VERSION_MINOR} * 100   +
+                              ${MARIADB_CLIENT_VERSION_PATCH}")
+
+IF (NOT MARIADB_PORT)
+  set(MARIADB_PORT 3306)
+ENDIF ()
+if(NOT MARIADB_UNIX_ADDR)
+  set(MARIADB_UNIX_ADDR "/tmp/mysql.sock")
+endif()
+
+set(HAVE_ALLOCA_H 1)
+set(HAVE_ARPA_INET_H 1)
+set(HAVE_DLFCN_H 1)
+set(HAVE_FCNTL_H 1)
+set(HAVE_FLOAT_H 1)
+set(HAVE_LIMITS_H 1)
+set(HAVE_PWD_H 1)
+set(HAVE_SCHED_H 1)
+set(HAVE_SELECT_H 0)
+set(INCLUDE_SIGNAL 1)
+set(HAVE_SIGNAL 1)
+set(HAVE_STDDEF_H 1)
+set(HAVE_STDINT_H 1)
+set(HAVE_STDLIB_H 1)
+set(HAVE_STRING_H 1)
+set(HAVE_STRINGS_H 1)
+set(HAVE_SYS_IOCTL_H 1)
+set(HAVE_SYS_SELECT_H 1)
+set(HAVE_SYS_SOCKET_H 1)
+set(HAVE_SYS_TYPES_H 1)
+set(HAVE_SYS_UN_H 1)
+set(HAVE_UNISTD_H 1)
+set(HAVE_UTIME_H 1)
+set(HAVE_UCONTEXT_H 1)
+set(HAVE_ALLOCA 1)
+set(HAVE_DLERROR 0)
+set(HAVE_DLOPEN 0)
+set(HAVE_FCNTL 1)
+set(HAVE_MEMCPY 1)
+set(HAVE_NL_LANGINFO 0)
+set(HAVE_SETLOCALE 0)
+set(HAVE_POLL 1)
+
+set(SIZEOF_CHARP 8)
+set(SIZEOF_INT 4)
+set(SIZEOF_LONG 8)
+set(SIZEOF_LONG_LONG 8)
+set(SIZEOF_SIZE_T 8)
+set(SOCKET_SIZE_TYPE socklen_t)
+
+
+set(SYSTEM_LIBS ${SYSTEM_LIBS} zlib)
+
+if(CMAKE_HAVE_PTHREAD_H)
+  set(CMAKE_REQUIRED_INCLUDES pthread.h)
+endif()
+
+add_definitions(-DMARIADB_SYSTEM_TYPE="${CMAKE_SYSTEM_NAME}")
+add_definitions(-DMARIADB_MACHINE_TYPE="${CMAKE_SYSTEM_PROCESSOR}")
+
+set(HAVE_THREADS 1)
+set(DEFAULT_CHARSET "utf8mb4")
+
+add_definitions(-DHAVE_OPENSSL -DHAVE_TLS)
+set(SSL_LIBRARIES ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY})
+include_directories(BEFORE ${OPENSSL_INCLUDE_DIR})
+set(TLS_LIBRARY_VERSION "OpenSSL ${OPENSSL_VERSION}")
+
+set(ENABLED_LOCAL_INFILE OFF)
+
+CONFIGURE_FILE(${CC_SOURCE_DIR}/include/ma_config.h.in
+               ${CC_BINARY_DIR}/include-private/ma_config.h)
+CONFIGURE_FILE(${CC_SOURCE_DIR}/include/ma_config.h.in
+               ${CC_BINARY_DIR}/include-private/config.h)
+CONFIGURE_FILE(${CC_SOURCE_DIR}/include/mariadb_version.h.in
+               ${CC_BINARY_DIR}/include-public/mariadb_version.h)
+
+if(WITH_SSL)
+  set(SYSTEM_LIBS ${SYSTEM_LIBS} ${SSL_LIBRARIES})
+endif()
+
+
+function(REGISTER_PLUGIN)
+
+  SET(one_value_keywords TARGET TYPE)
+  SET(multi_value_keywords SOURCES)
+
+  cmake_parse_arguments(CC_PLUGIN
+                        "${options}"
+                        "${one_value_keywords}"
+                        "${multi_value_keywords}"
+                        ${ARGN})
+
+  # overwrite default if it was specified with cmake option
+  string(TOUPPER ${CC_PLUGIN_TARGET} cc_plugin)
+  if(NOT "${CLIENT_PLUGIN_${cc_plugin}}" STREQUAL "")
+    SET(CC_PLUGIN_DEFAULT ${CLIENT_PLUGIN_${cc_plugin}})
+  endif()
+
+  # use uppercase
+  string(TOUPPER ${CC_PLUGIN_TARGET} target_name)
+  string(TOUPPER "${CC_PLUGIN_CONFIGURATIONS}" CC_PLUGIN_CONFIGURATIONS)
+
+  if(NOT ${PLUGIN_${target_name}} STREQUAL "")
+    string(TOUPPER ${PLUGIN_${target_name}} PLUGIN_${target_name})
+    set(CC_PLUGIN_DEFAULT ${PLUGIN_${target_name}})
+  endif()
+
+  set(PLUGINS_STATIC ${PLUGINS_STATIC} ${CC_PLUGIN_TARGET} PARENT_SCOPE)
+  set(LIBMARIADB_PLUGIN_CFLAGS ${LIBMARIADB_PLUGIN_CFLAGS} ${CC_PLUGIN_COMPILE_OPTIONS} PARENT_SCOPE)
+  set(LIBMARIADB_PLUGIN_INCLUDES ${LIBMARIADB_PLUGIN_INCLUDES} ${CC_PLUGIN_INCLUDES} PARENT_SCOPE)
+  set(LIBMARIADB_PLUGIN_SOURCES ${LIBMARIADB_PLUGIN_SOURCES} ${CC_PLUGIN_SOURCES} PARENT_SCOPE)
+  set(LIBMARIADB_PLUGIN_LIBS ${LIBMARIADB_PLUGIN_LIBS} ${CC_PLUGIN_LIBRARIES} PARENT_SCOPE)
+endfunction()
+
+SET(PLUGIN_EXTRA_FILES ${CC_SOURCE_DIR}/libmariadb/ma_errmsg.c)
+
+#native password
+REGISTER_PLUGIN(TARGET pvio_socket
+                TYPE MARIADB_CLIENT_PLUGIN_PVIO
+                SOURCES "${CC_SOURCE_DIR}/plugins/pvio/pvio_socket.c")
+
+# SHA256 caching plugin for MySQL 8.0 connection
+REGISTER_PLUGIN(TARGET caching_sha2_password
+                TYPE MARIADB_CLIENT_PLUGIN_AUTH
+                SOURCES "${CC_SOURCE_DIR}/plugins/auth/caching_sha2_pw.c")
+
+REGISTER_PLUGIN(TARGET sha256_password
+                TYPE MARIADB_CLIENT_PLUGIN_AUTH
+                SOURCES "${CC_SOURCE_DIR}/plugins/auth/sha256_pw.c")
+
+#native password
+REGISTER_PLUGIN(TARGET mysql_native_password
+                TYPE MARIADB_CLIENT_PLUGIN_AUTH
+                SOURCES "${CC_SOURCE_DIR}/plugins/auth/my_auth.c")
+
+REGISTER_PLUGIN(TARGET aurora
+                TYPE MARIADB_CLIENT_PLUGIN_CONNECTION
+                SOURCES "${CC_SOURCE_DIR}/plugins/connection/aurora.c")
+
+
+add_definitions(-D HAVE_COMPRESS)
+add_definitions(-D LIBMARIADB)
+add_definitions(-D THREAD)
+
+# handle static plugins
+set(LIBMARIADB_SOURCES ${LIBMARIADB_PLUGIN_SOURCES})
+set(SYSTEM_LIBS ${SYSTEM_LIBS} ${LIBMARIADB_PLUGIN_LIBS})
+add_definitions(${LIBMARIADB_PLUGIN_DEFS})
+FOREACH(plugin ${PLUGINS_STATIC})
+  set(EXTERNAL_PLUGINS "${EXTERNAL_PLUGINS} extern struct st_mysql_client_plugin ${plugin}_client_plugin;\n")
+  set(BUILTIN_PLUGINS "${BUILTIN_PLUGINS}   (struct st_mysql_client_plugin *)&${plugin}_client_plugin,\n")
+ENDFOREACH()
+CONFIGURE_FILE(${CC_SOURCE_DIR}/libmariadb/ma_client_plugin.c.in
+               ${CC_BINARY_DIR}/libmariadb/ma_client_plugin.c)
+
+set(LIBMARIADB_SOURCES ${LIBMARIADB_SOURCES}
+${CC_SOURCE_DIR}/plugins/auth/my_auth.c
+${CC_SOURCE_DIR}/libmariadb/ma_array.c
+${CC_SOURCE_DIR}/libmariadb/ma_charset.c
+${CC_SOURCE_DIR}/libmariadb/ma_hash.c
+${CC_SOURCE_DIR}/libmariadb/ma_net.c
+${CC_SOURCE_DIR}/libmariadb/mariadb_charset.c
+${CC_SOURCE_DIR}/libmariadb/ma_time.c
+${CC_SOURCE_DIR}/libmariadb/ma_default.c
+${CC_SOURCE_DIR}/libmariadb/ma_errmsg.c
+${CC_SOURCE_DIR}/libmariadb/mariadb_lib.c
+${CC_SOURCE_DIR}/libmariadb/ma_list.c
+${CC_SOURCE_DIR}/libmariadb/ma_pvio.c
+${CC_SOURCE_DIR}/libmariadb/ma_tls.c
+${CC_SOURCE_DIR}/libmariadb/ma_alloc.c
+${CC_SOURCE_DIR}/libmariadb/ma_compress.c
+${CC_SOURCE_DIR}/libmariadb/ma_init.c
+${CC_SOURCE_DIR}/libmariadb/ma_password.c
+${CC_SOURCE_DIR}/libmariadb/ma_ll2str.c
+${CC_SOURCE_DIR}/libmariadb/ma_sha1.c
+${CC_SOURCE_DIR}/libmariadb/mariadb_stmt.c
+${CC_SOURCE_DIR}/libmariadb/ma_loaddata.c
+${CC_SOURCE_DIR}/libmariadb/ma_stmt_codec.c
+${CC_SOURCE_DIR}/libmariadb/ma_string.c
+${CC_SOURCE_DIR}/libmariadb/ma_dtoa.c
+${CC_SOURCE_DIR}/libmariadb/mariadb_rpl.c
+${CC_SOURCE_DIR}/libmariadb/ma_io.c
+${CC_SOURCE_DIR}/libmariadb/secure/openssl.c
+${CC_SOURCE_DIR}/libmariadb/secure/openssl_crypt.c
+${CC_BINARY_DIR}/libmariadb/ma_client_plugin.c
+)
+
+if(ICONV_INCLUDE_DIR)
+   include_directories(BEFORE ${ICONV_INCLUDE_DIR})
+endif()
+add_definitions(-DLIBICONV_PLUG)
+
+if(ZLIB_FOUND AND WITH_EXTERNAL_ZLIB)
+  include_directories(${ZLIB_INCLUDE_DIR})
+endif()
+
+if(WITH_DYNCOL)
+  set(LIBMARIADB_SOURCES ${LIBMARIADB_SOURCES} ${CC_SOURCE_DIR}/libmariadb/mariadb_dyncol.c)
+endif()
+
+set(LIBMARIADB_SOURCES ${LIBMARIADB_SOURCES} ${CC_SOURCE_DIR}/libmariadb/mariadb_async.c ${CC_SOURCE_DIR}/libmariadb/ma_context.c)
+
+
+add_library(mariadbclient STATIC ${LIBMARIADB_SOURCES})
+target_link_libraries(mariadbclient ${SYSTEM_LIBS})
+
+target_include_directories(mariadbclient
+    PRIVATE ${CC_BINARY_DIR}/include-private
+    PUBLIC ${CC_BINARY_DIR}/include-public ${CC_SOURCE_DIR}/include ${CC_SOURCE_DIR}/libmariadb)
+
+set_target_properties(mariadbclient PROPERTIES IMPORTED_INTERFACE_LINK_LIBRARIES "${SYSTEM_LIBS}")

contrib/protobuf-cmake/CMakeLists.txt

@@ -1,21 +1,220 @@
-set(protobuf_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/protobuf")
+set(protobuf_source_dir "${ClickHouse_SOURCE_DIR}/contrib/protobuf")
-set(protobuf_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/protobuf")
+set(protobuf_binary_dir "${ClickHouse_BINARY_DIR}/contrib/protobuf")
 
-set(protobuf_WITH_ZLIB 0 CACHE INTERNAL "" FORCE) # actually will use zlib, but skip find
-set(protobuf_BUILD_TESTS OFF CACHE INTERNAL "" FORCE)
 
-if (MAKE_STATIC_LIBRARIES)
+add_definitions(-DGOOGLE_PROTOBUF_CMAKE_BUILD)
-    set(protobuf_BUILD_SHARED_LIBS OFF CACHE INTERNAL "" FORCE)
+
-else ()
+add_definitions(-DHAVE_PTHREAD)
-    set(protobuf_BUILD_SHARED_LIBS ON CACHE INTERNAL "" FORCE)
+add_definitions(-DHAVE_ZLIB)
+
+include_directories(
+  ${ZLIB_INCLUDE_DIRECTORIES}
+  ${protobuf_binary_dir}
+  ${protobuf_source_dir}/src)
+
+set(libprotobuf_lite_files
+  ${protobuf_source_dir}/src/google/protobuf/any_lite.cc
+  ${protobuf_source_dir}/src/google/protobuf/arena.cc
+  ${protobuf_source_dir}/src/google/protobuf/arenastring.cc
+  ${protobuf_source_dir}/src/google/protobuf/extension_set.cc
+  ${protobuf_source_dir}/src/google/protobuf/field_access_listener.cc
+  ${protobuf_source_dir}/src/google/protobuf/generated_enum_util.cc
+  ${protobuf_source_dir}/src/google/protobuf/generated_message_table_driven_lite.cc
+  ${protobuf_source_dir}/src/google/protobuf/generated_message_util.cc
+  ${protobuf_source_dir}/src/google/protobuf/implicit_weak_message.cc
+  ${protobuf_source_dir}/src/google/protobuf/io/coded_stream.cc
+  ${protobuf_source_dir}/src/google/protobuf/io/io_win32.cc
+  ${protobuf_source_dir}/src/google/protobuf/io/strtod.cc
+  ${protobuf_source_dir}/src/google/protobuf/io/zero_copy_stream.cc
+  ${protobuf_source_dir}/src/google/protobuf/io/zero_copy_stream_impl.cc
+  ${protobuf_source_dir}/src/google/protobuf/io/zero_copy_stream_impl_lite.cc
+  ${protobuf_source_dir}/src/google/protobuf/map.cc
+  ${protobuf_source_dir}/src/google/protobuf/message_lite.cc
+  ${protobuf_source_dir}/src/google/protobuf/parse_context.cc
+  ${protobuf_source_dir}/src/google/protobuf/repeated_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/stubs/bytestream.cc
+  ${protobuf_source_dir}/src/google/protobuf/stubs/common.cc
+  ${protobuf_source_dir}/src/google/protobuf/stubs/int128.cc
+  ${protobuf_source_dir}/src/google/protobuf/stubs/status.cc
+  ${protobuf_source_dir}/src/google/protobuf/stubs/statusor.cc
+  ${protobuf_source_dir}/src/google/protobuf/stubs/stringpiece.cc
+  ${protobuf_source_dir}/src/google/protobuf/stubs/stringprintf.cc
+  ${protobuf_source_dir}/src/google/protobuf/stubs/structurally_valid.cc
+  ${protobuf_source_dir}/src/google/protobuf/stubs/strutil.cc
+  ${protobuf_source_dir}/src/google/protobuf/stubs/time.cc
+  ${protobuf_source_dir}/src/google/protobuf/wire_format_lite.cc
+)
+
+add_library(libprotobuf-lite ${libprotobuf_lite_files})
+target_link_libraries(libprotobuf-lite pthread)
+if(${CMAKE_SYSTEM_NAME} STREQUAL "Android")
+    target_link_libraries(libprotobuf-lite log)
 endif()
+target_include_directories(libprotobuf-lite SYSTEM PUBLIC ${protobuf_source_dir}/src)
+add_library(protobuf::libprotobuf-lite ALIAS libprotobuf-lite)
 
-if (CMAKE_CROSSCOMPILING)
+
-    # Will build 'protoc' for host arch instead of cross-compiling
+set(libprotobuf_files
-    set(protobuf_BUILD_PROTOC_BINARIES OFF CACHE INTERNAL "" FORCE)
+  ${protobuf_source_dir}/src/google/protobuf/any.cc
+  ${protobuf_source_dir}/src/google/protobuf/any.pb.cc
+  ${protobuf_source_dir}/src/google/protobuf/api.pb.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/importer.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/parser.cc
+  ${protobuf_source_dir}/src/google/protobuf/descriptor.cc
+  ${protobuf_source_dir}/src/google/protobuf/descriptor.pb.cc
+  ${protobuf_source_dir}/src/google/protobuf/descriptor_database.cc
+  ${protobuf_source_dir}/src/google/protobuf/duration.pb.cc
+  ${protobuf_source_dir}/src/google/protobuf/dynamic_message.cc
+  ${protobuf_source_dir}/src/google/protobuf/empty.pb.cc
+  ${protobuf_source_dir}/src/google/protobuf/extension_set_heavy.cc
+  ${protobuf_source_dir}/src/google/protobuf/field_mask.pb.cc
+  ${protobuf_source_dir}/src/google/protobuf/generated_message_reflection.cc
+  ${protobuf_source_dir}/src/google/protobuf/generated_message_table_driven.cc
+  ${protobuf_source_dir}/src/google/protobuf/io/gzip_stream.cc
+  ${protobuf_source_dir}/src/google/protobuf/io/printer.cc
+  ${protobuf_source_dir}/src/google/protobuf/io/tokenizer.cc
+  ${protobuf_source_dir}/src/google/protobuf/map_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/message.cc
+  ${protobuf_source_dir}/src/google/protobuf/reflection_ops.cc
+  ${protobuf_source_dir}/src/google/protobuf/service.cc
+  ${protobuf_source_dir}/src/google/protobuf/source_context.pb.cc
+  ${protobuf_source_dir}/src/google/protobuf/struct.pb.cc
+  ${protobuf_source_dir}/src/google/protobuf/stubs/substitute.cc
+  ${protobuf_source_dir}/src/google/protobuf/text_format.cc
+  ${protobuf_source_dir}/src/google/protobuf/timestamp.pb.cc
+  ${protobuf_source_dir}/src/google/protobuf/type.pb.cc
+  ${protobuf_source_dir}/src/google/protobuf/unknown_field_set.cc
+  ${protobuf_source_dir}/src/google/protobuf/util/delimited_message_util.cc
+  ${protobuf_source_dir}/src/google/protobuf/util/field_comparator.cc
+  ${protobuf_source_dir}/src/google/protobuf/util/field_mask_util.cc
+  ${protobuf_source_dir}/src/google/protobuf/util/internal/datapiece.cc
+  ${protobuf_source_dir}/src/google/protobuf/util/internal/default_value_objectwriter.cc
+  ${protobuf_source_dir}/src/google/protobuf/util/internal/error_listener.cc
+  ${protobuf_source_dir}/src/google/protobuf/util/internal/field_mask_utility.cc
+  ${protobuf_source_dir}/src/google/protobuf/util/internal/json_escaping.cc
+  ${protobuf_source_dir}/src/google/protobuf/util/internal/json_objectwriter.cc
+  ${protobuf_source_dir}/src/google/protobuf/util/internal/json_stream_parser.cc
+  ${protobuf_source_dir}/src/google/protobuf/util/internal/object_writer.cc
+  ${protobuf_source_dir}/src/google/protobuf/util/internal/proto_writer.cc
+  ${protobuf_source_dir}/src/google/protobuf/util/internal/protostream_objectsource.cc
+  ${protobuf_source_dir}/src/google/protobuf/util/internal/protostream_objectwriter.cc
+  ${protobuf_source_dir}/src/google/protobuf/util/internal/type_info.cc
+  ${protobuf_source_dir}/src/google/protobuf/util/internal/type_info_test_helper.cc
+  ${protobuf_source_dir}/src/google/protobuf/util/internal/utility.cc
+  ${protobuf_source_dir}/src/google/protobuf/util/json_util.cc
+  ${protobuf_source_dir}/src/google/protobuf/util/message_differencer.cc
+  ${protobuf_source_dir}/src/google/protobuf/util/time_util.cc
+  ${protobuf_source_dir}/src/google/protobuf/util/type_resolver_util.cc
+  ${protobuf_source_dir}/src/google/protobuf/wire_format.cc
+  ${protobuf_source_dir}/src/google/protobuf/wrappers.pb.cc
+)
+
+add_library(libprotobuf ${libprotobuf_lite_files} ${libprotobuf_files})
+target_link_libraries(libprotobuf pthread)
+target_link_libraries(libprotobuf ${ZLIB_LIBRARIES})
+if(${CMAKE_SYSTEM_NAME} STREQUAL "Android")
+    target_link_libraries(libprotobuf log)
 endif()
+target_include_directories(libprotobuf SYSTEM PUBLIC ${protobuf_source_dir}/src)
+add_library(protobuf::libprotobuf ALIAS libprotobuf)
 
-add_subdirectory("${protobuf_SOURCE_DIR}/cmake" "${protobuf_BINARY_DIR}")
+
+set(libprotoc_files
+  ${protobuf_source_dir}/src/google/protobuf/compiler/code_generator.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/command_line_interface.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/cpp_enum.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/cpp_enum_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/cpp_extension.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/cpp_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/cpp_file.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/cpp_generator.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/cpp_helpers.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/cpp_map_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/cpp_message.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/cpp_message_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/cpp_padding_optimizer.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/cpp_parse_function_generator.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/cpp_primitive_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/cpp_service.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/cpp_string_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/csharp/csharp_doc_comment.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/csharp/csharp_enum.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/csharp/csharp_enum_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/csharp/csharp_field_base.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/csharp/csharp_generator.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/csharp/csharp_helpers.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/csharp/csharp_map_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/csharp/csharp_message.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/csharp/csharp_message_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/csharp/csharp_primitive_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/csharp/csharp_reflection_class.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/csharp/csharp_repeated_enum_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/csharp/csharp_repeated_message_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/csharp/csharp_repeated_primitive_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/csharp/csharp_source_generator_base.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/csharp/csharp_wrapper_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_context.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_doc_comment.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_enum.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_enum_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_enum_field_lite.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_enum_lite.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_extension.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_extension_lite.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_file.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_generator.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_generator_factory.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_helpers.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_kotlin_generator.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_map_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_map_field_lite.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_message.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_message_builder.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_message_builder_lite.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_message_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_message_field_lite.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_message_lite.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_name_resolver.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_primitive_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_primitive_field_lite.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_service.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_shared_code_generator.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_string_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_string_field_lite.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/js/js_generator.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/js/well_known_types_embed.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_enum.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_enum_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_extension.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_file.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_generator.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_helpers.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_map_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_message.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_message_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_oneof.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_primitive_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/php/php_generator.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/plugin.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/plugin.pb.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/python/python_generator.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/ruby/ruby_generator.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/subprocess.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/zip_writer.cc
+)
+
+add_library(libprotoc ${libprotoc_files})
+target_link_libraries(libprotoc libprotobuf)
+add_library(protobuf::libprotoc ALIAS libprotoc)
+
+set(protoc_files ${protobuf_source_dir}/src/google/protobuf/compiler/main.cc)
+
+if (NOT CMAKE_CROSSCOMPILING)
+    add_executable(protoc ${protoc_files})
+    target_link_libraries(protoc libprotoc libprotobuf pthread)
+    add_executable(protobuf::protoc ALIAS protoc)
 
     if (ENABLE_FUZZING)
         # `protoc` will be built with sanitizer and it could fail during ClickHouse build
@@ -26,14 +225,9 @@ if (ENABLE_FUZZING)
         # export MSAN_OPTIONS=exit_code=0
         target_compile_options(protoc PRIVATE "-fsanitize-recover=all")
     endif()
-
+else ()
-# We don't want to stop compilation on warnings in protobuf's headers.
-# The following line overrides the value assigned by the command target_include_directories() in libprotobuf.cmake
-set_property(TARGET libprotobuf PROPERTY INTERFACE_SYSTEM_INCLUDE_DIRECTORIES "${protobuf_SOURCE_DIR}/src")
-
-if (CMAKE_CROSSCOMPILING)
     # Build 'protoc' for host arch
-    set (PROTOC_BUILD_DIR "${protobuf_BINARY_DIR}/build")
+    set (PROTOC_BUILD_DIR "${protobuf_binary_dir}/build")
 
     if (NOT EXISTS "${PROTOC_BUILD_DIR}/protoc")
 
@@ -53,7 +247,7 @@ if (CMAKE_CROSSCOMPILING)
                 "-Dprotobuf_BUILD_CONFORMANCE=0"
                 "-Dprotobuf_BUILD_EXAMPLES=0"
                 "-Dprotobuf_BUILD_PROTOC_BINARIES=1"
-                "${protobuf_SOURCE_DIR}/cmake"
+                "${protobuf_source_dir}/cmake"
             WORKING_DIRECTORY "${PROTOC_BUILD_DIR}"
             COMMAND_ECHO STDOUT)
 
@@ -78,7 +272,7 @@ if (CMAKE_CROSSCOMPILING)
 #             -Dprotobuf_BUILD_CONFORMANCE=0
 #             -Dprotobuf_BUILD_EXAMPLES=0
 #             -Dprotobuf_BUILD_PROTOC_BINARIES=1
-#             "${protobuf_SOURCE_DIR}/cmake"
+#             "${protobuf_source_dir}/cmake"
 #
 #         DEPENDS "${PROTOC_BUILD_DIR}"
 #         WORKING_DIRECTORY "${PROTOC_BUILD_DIR}"
@@ -97,5 +291,4 @@ if (CMAKE_CROSSCOMPILING)
     add_executable(protoc IMPORTED GLOBAL)
     set_target_properties (protoc PROPERTIES IMPORTED_LOCATION "${PROTOC_BUILD_DIR}/protoc")
     add_dependencies(protoc "${PROTOC_BUILD_DIR}/protoc")
-
 endif ()

contrib/re2-cmake/CMakeLists.txt

@@ -1,24 +1,54 @@
+# Copyright 2015 The RE2 Authors.  All Rights Reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+# This file was edited for ClickHouse
+
+set(SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/re2")
+
+set(RE2_SOURCES
+    ${SRC_DIR}/re2/bitstate.cc
+    ${SRC_DIR}/re2/compile.cc
+    ${SRC_DIR}/re2/dfa.cc
+    ${SRC_DIR}/re2/filtered_re2.cc
+    ${SRC_DIR}/re2/mimics_pcre.cc
+    ${SRC_DIR}/re2/nfa.cc
+    ${SRC_DIR}/re2/onepass.cc
+    ${SRC_DIR}/re2/parse.cc
+    ${SRC_DIR}/re2/perl_groups.cc
+    ${SRC_DIR}/re2/prefilter.cc
+    ${SRC_DIR}/re2/prefilter_tree.cc
+    ${SRC_DIR}/re2/prog.cc
+    ${SRC_DIR}/re2/re2.cc
+    ${SRC_DIR}/re2/regexp.cc
+    ${SRC_DIR}/re2/set.cc
+    ${SRC_DIR}/re2/simplify.cc
+    ${SRC_DIR}/re2/stringpiece.cc
+    ${SRC_DIR}/re2/tostring.cc
+    ${SRC_DIR}/re2/unicode_casefold.cc
+    ${SRC_DIR}/re2/unicode_groups.cc
+    ${SRC_DIR}/util/rune.cc
+    ${SRC_DIR}/util/strutil.cc
+)
+
+add_library(re2 ${RE2_SOURCES})
+target_include_directories(re2 PUBLIC "${SRC_DIR}")
+
+
 # Building re2 which is thread-safe and re2_st which is not.
 # re2 changes its state during matching of regular expression, e.g. creates temporary DFA.
 # It uses RWLock to process the same regular expression object from different threads.
 # In order to avoid redundant locks in some cases, we use not thread-safe version of the library (re2_st).
 
-set (RE2_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/re2/)
+add_library(re2_st ${RE2_SOURCES})
-
-get_target_property (RE2_SOURCES_ re2 SOURCES)
-foreach (src ${RE2_SOURCES_})
-    list(APPEND RE2_ST_SOURCES ${RE2_SOURCE_DIR}/${src})
-endforeach ()
-
-add_library(re2_st ${RE2_ST_SOURCES})
 target_compile_definitions (re2_st PRIVATE NDEBUG NO_THREADS re2=re2_st)
 target_include_directories (re2_st PRIVATE .)
-target_include_directories (re2_st SYSTEM PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${RE2_SOURCE_DIR})
+target_include_directories (re2_st SYSTEM PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${SRC_DIR})
 
 file (MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/re2_st)
 foreach (FILENAME filtered_re2.h re2.h set.h stringpiece.h)
     add_custom_command (OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/re2_st/${FILENAME}"
-        COMMAND ${CMAKE_COMMAND} -DSOURCE_FILENAME="${RE2_SOURCE_DIR}/re2/${FILENAME}"
+        COMMAND ${CMAKE_COMMAND} -DSOURCE_FILENAME="${SRC_DIR}/re2/${FILENAME}"
             -DTARGET_FILENAME="${CMAKE_CURRENT_BINARY_DIR}/re2_st/${FILENAME}"
             -P "${CMAKE_CURRENT_SOURCE_DIR}/re2_transform.cmake"
         COMMENT "Creating ${FILENAME} for re2_st library.")
@@ -29,7 +59,7 @@ endforeach ()
 file (MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/util)
 foreach (FILENAME mutex.h)
     add_custom_command (OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/util/${FILENAME}"
-        COMMAND ${CMAKE_COMMAND} -DSOURCE_FILENAME="${RE2_SOURCE_DIR}/util/${FILENAME}"
+        COMMAND ${CMAKE_COMMAND} -DSOURCE_FILENAME="${SRC_DIR}/util/${FILENAME}"
             -DTARGET_FILENAME="${CMAKE_CURRENT_BINARY_DIR}/util/${FILENAME}"
             -P "${CMAKE_CURRENT_SOURCE_DIR}/re2_transform.cmake"
         COMMENT "Creating ${FILENAME} for re2_st library.")

contrib/rocksdb-cmake/CMakeLists.txt

@@ -190,7 +190,7 @@ if(HAVE_PTHREAD_MUTEX_ADAPTIVE_NP)
 endif()
 
 include(CheckCXXSymbolExists)
-if(CMAKE_SYSTEM_NAME MATCHES "^FreeBSD")
+if (OS_FREEBSD)
   check_cxx_symbol_exists(malloc_usable_size "${ROCKSDB_SOURCE_DIR}/malloc_np.h" HAVE_MALLOC_USABLE_SIZE)
 else()
   check_cxx_symbol_exists(malloc_usable_size "${ROCKSDB_SOURCE_DIR}/malloc.h" HAVE_MALLOC_USABLE_SIZE)
@@ -199,20 +199,14 @@ if(HAVE_MALLOC_USABLE_SIZE)
   add_definitions(-DROCKSDB_MALLOC_USABLE_SIZE)
 endif()
 
-check_cxx_symbol_exists(sched_getcpu sched.h HAVE_SCHED_GETCPU)
+if (OS_LINUX)
-if(HAVE_SCHED_GETCPU)
   add_definitions(-DROCKSDB_SCHED_GETCPU_PRESENT)
+  add_definitions(-DROCKSDB_AUXV_SYSAUXV_PRESENT)
+  add_definitions(-DROCKSDB_AUXV_GETAUXVAL_PRESENT)
+elseif (OS_FREEBSD)
+  add_definitions(-DROCKSDB_AUXV_SYSAUXV_PRESENT)
 endif()
 
-check_cxx_symbol_exists(getauxval auvx.h HAVE_AUXV_GETAUXVAL)
-if(HAVE_AUXV_GETAUXVAL)
-  add_definitions(-DROCKSDB_AUXV_GETAUXVAL_PRESENT)
-endif()
-
-check_cxx_symbol_exists(elf_aux_info sys/auxv.h HAVE_ELF_AUX_INFO)
-if(HAVE_ELF_AUX_INFO)
-  add_definitions(-DROCKSDB_AUXV_GETAUXVAL_PRESENT)
-endif()
 
 include_directories(${ROCKSDB_SOURCE_DIR})
 include_directories("${ROCKSDB_SOURCE_DIR}/include")

contrib/sentry-native-cmake/CMakeLists.txt

@@ -0,0 +1,47 @@
+set (SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/sentry-native")
+
+set (SRCS
+    ${SRC_DIR}/vendor/mpack.c
+    ${SRC_DIR}/src/sentry_alloc.c
+    ${SRC_DIR}/src/sentry_backend.c
+    ${SRC_DIR}/src/sentry_core.c
+    ${SRC_DIR}/src/sentry_database.c
+    ${SRC_DIR}/src/sentry_envelope.c
+    ${SRC_DIR}/src/sentry_json.c
+    ${SRC_DIR}/src/sentry_logger.c
+    ${SRC_DIR}/src/sentry_options.c
+    ${SRC_DIR}/src/sentry_random.c
+    ${SRC_DIR}/src/sentry_ratelimiter.c
+    ${SRC_DIR}/src/sentry_scope.c
+    ${SRC_DIR}/src/sentry_session.c
+    ${SRC_DIR}/src/sentry_slice.c
+    ${SRC_DIR}/src/sentry_string.c
+    ${SRC_DIR}/src/sentry_sync.c
+    ${SRC_DIR}/src/sentry_transport.c
+    ${SRC_DIR}/src/sentry_utils.c
+    ${SRC_DIR}/src/sentry_uuid.c
+    ${SRC_DIR}/src/sentry_value.c
+    ${SRC_DIR}/src/path/sentry_path.c
+    ${SRC_DIR}/src/transports/sentry_disk_transport.c
+    ${SRC_DIR}/src/transports/sentry_function_transport.c
+    ${SRC_DIR}/src/unwinder/sentry_unwinder.c
+    ${SRC_DIR}/src/sentry_unix_pageallocator.c
+    ${SRC_DIR}/src/path/sentry_path_unix.c
+    ${SRC_DIR}/src/symbolizer/sentry_symbolizer_unix.c
+    ${SRC_DIR}/src/modulefinder/sentry_modulefinder_linux.c
+    ${SRC_DIR}/src/transports/sentry_transport_curl.c
+    ${SRC_DIR}/src/backends/sentry_backend_none.c
+)
+
+add_library(sentry ${SRCS})
+add_library(sentry::sentry ALIAS sentry)
+
+if(BUILD_SHARED_LIBS)
+    target_compile_definitions(sentry PRIVATE SENTRY_BUILD_SHARED)
+else()
+    target_compile_definitions(sentry PUBLIC SENTRY_BUILD_STATIC)
+endif()
+
+target_link_libraries(sentry PRIVATE curl pthread)
+target_include_directories(sentry PUBLIC "${SRC_DIR}/include" PRIVATE "${SRC_DIR}/src")
+target_compile_definitions(sentry PRIVATE SENTRY_WITH_INPROC_BACKEND SIZEOF_LONG=8)

contrib/snappy-cmake/CMakeLists.txt

@@ -2,15 +2,12 @@ set (SOURCE_DIR "${CMAKE_SOURCE_DIR}/contrib/snappy")
 
 set (SNAPPY_IS_BIG_ENDIAN 0)
 
-include(CheckIncludeFile)
+set (HAVE_BYTESWAP_H 1)
-check_include_file("byteswap.h" HAVE_BYTESWAP_H)
+set (HAVE_SYS_MMAN_H 1)
-check_include_file("sys/endian.h" HAVE_SYS_ENDIAN_H)
+set (HAVE_SYS_RESOURCE_H 1)
-check_include_file("sys/mman.h" HAVE_SYS_MMAN_H)
+set (HAVE_SYS_TIME_H 1)
-check_include_file("sys/resource.h" HAVE_SYS_RESOURCE_H)
+set (HAVE_SYS_UIO_H 1)
-check_include_file("sys/time.h" HAVE_SYS_TIME_H)
+set (HAVE_UNISTD_H 1)
-check_include_file("sys/uio.h" HAVE_SYS_UIO_H)
-check_include_file("unistd.h" HAVE_UNISTD_H)
-check_include_file("windows.h" HAVE_WINDOWS_H)
 
 set (HAVE_BUILTIN_EXPECT 1)
 set (HAVE_BUILTIN_CTZ 1)

contrib/stats

@@ -1 +0,0 @@
-Subproject commit b6dd459c10a88c7ea04693c007e9e35820c5d9ad

contrib/xz-cmake/CMakeLists.txt

@@ -0,0 +1,263 @@
+set (SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/xz")
+
+# Author: Lasse Collin
+#
+# This file has been put into the public domain.
+# You can do whatever you want with this file.
+#
+# The file was edited for ClickHouse
+
+# Get the package version from version.h into XZ_VERSION variable.
+file(READ ${SRC_DIR}/src/liblzma/api/lzma/version.h XZ_VERSION)
+string(REGEX REPLACE
+"^.*\n\
+#define LZMA_VERSION_MAJOR ([0-9]+)\n\
+#define LZMA_VERSION_MINOR ([0-9]+)\n\
+#define LZMA_VERSION_PATCH ([0-9]+)\n\
+.*$"
+       "\\1.\\2.\\3" XZ_VERSION "${XZ_VERSION}")
+
+# Definitions common to all targets:
+add_compile_definitions(
+    # Package info:
+    PACKAGE_NAME="XZ Utils"
+    PACKAGE_BUGREPORT="lasse.collin@tukaani.org"
+    PACKAGE_URL="https://tukaani.org/xz/"
+
+    # Features:
+    HAVE_CHECK_CRC32
+    HAVE_CHECK_CRC64
+    HAVE_CHECK_SHA256
+    HAVE_DECODERS
+    HAVE_DECODER_ARM
+    HAVE_DECODER_ARMTHUMB
+    HAVE_DECODER_DELTA
+    HAVE_DECODER_IA64
+    HAVE_DECODER_LZMA1
+    HAVE_DECODER_LZMA2
+    HAVE_DECODER_POWERPC
+    HAVE_DECODER_SPARC
+    HAVE_DECODER_X86
+    HAVE_ENCODERS
+    HAVE_ENCODER_ARM
+    HAVE_ENCODER_ARMTHUMB
+    HAVE_ENCODER_DELTA
+    HAVE_ENCODER_IA64
+    HAVE_ENCODER_LZMA1
+    HAVE_ENCODER_LZMA2
+    HAVE_ENCODER_POWERPC
+    HAVE_ENCODER_SPARC
+    HAVE_ENCODER_X86
+    HAVE_MF_BT2
+    HAVE_MF_BT3
+    HAVE_MF_BT4
+    HAVE_MF_HC3
+    HAVE_MF_HC4
+
+    # Standard headers and types are available:
+    HAVE_STDBOOL_H
+    HAVE__BOOL
+    HAVE_STDINT_H
+    HAVE_INTTYPES_H
+
+    HAVE___BUILTIN_BSWAPXX
+    HAVE___BUILTIN_ASSUME_ALIGNED
+
+    _GNU_SOURCE
+    __EXTENSIONS__
+    _POSIX_PTHREAD_SEMANTICS
+    _TANDEM_SOURCE
+    _ALL_SOURCE
+
+    HAVE_CLOCK_GETTIME=1
+    HAVE_DECL_CLOCK_MONOTONIC=1
+
+    HAVE_PTHREAD_CONDATTR_SETCLOCK
+    MYTHREAD_POSIX
+)
+
+if (OS_LINUX)
+    add_compile_definitions(
+        TUKLIB_CPUCORES_SCHED_GETAFFINITY
+        TUKLIB_PHYSMEM_SYSCONF)
+elseif (OS_FREEBSD)
+    add_compile_definitions(
+        TUKLIB_CPUCORES_CPUSET
+        TUKLIB_PHYSMEM_SYSCTL)
+elseif (OS_DARWIN)
+    add_compile_definitions(
+        TUKLIB_CPUCORES_SYSCTL
+        TUKLIB_PHYSMEM_SYSCTL)
+endif ()
+
+if (ARCH_AMD64 OR ARCH_AARCH64)
+    add_compile_definitions(TUKLIB_FAST_UNALIGNED_ACCESS=1)
+endif ()
+
+find_package(Threads REQUIRED)
+
+
+add_library(liblzma
+    ${SRC_DIR}/src/common/mythread.h
+    ${SRC_DIR}/src/common/sysdefs.h
+    ${SRC_DIR}/src/common/tuklib_common.h
+    ${SRC_DIR}/src/common/tuklib_config.h
+    ${SRC_DIR}/src/common/tuklib_cpucores.c
+    ${SRC_DIR}/src/common/tuklib_cpucores.h
+    ${SRC_DIR}/src/common/tuklib_integer.h
+    ${SRC_DIR}/src/common/tuklib_physmem.c
+    ${SRC_DIR}/src/common/tuklib_physmem.h
+    ${SRC_DIR}/src/liblzma/api/lzma.h
+    ${SRC_DIR}/src/liblzma/api/lzma/base.h
+    ${SRC_DIR}/src/liblzma/api/lzma/bcj.h
+    ${SRC_DIR}/src/liblzma/api/lzma/block.h
+    ${SRC_DIR}/src/liblzma/api/lzma/check.h
+    ${SRC_DIR}/src/liblzma/api/lzma/container.h
+    ${SRC_DIR}/src/liblzma/api/lzma/delta.h
+    ${SRC_DIR}/src/liblzma/api/lzma/filter.h
+    ${SRC_DIR}/src/liblzma/api/lzma/hardware.h
+    ${SRC_DIR}/src/liblzma/api/lzma/index.h
+    ${SRC_DIR}/src/liblzma/api/lzma/index_hash.h
+    ${SRC_DIR}/src/liblzma/api/lzma/lzma12.h
+    ${SRC_DIR}/src/liblzma/api/lzma/stream_flags.h
+    ${SRC_DIR}/src/liblzma/api/lzma/version.h
+    ${SRC_DIR}/src/liblzma/api/lzma/vli.h
+    ${SRC_DIR}/src/liblzma/check/check.c
+    ${SRC_DIR}/src/liblzma/check/check.h
+    ${SRC_DIR}/src/liblzma/check/crc32_fast.c
+    ${SRC_DIR}/src/liblzma/check/crc32_table.c
+    ${SRC_DIR}/src/liblzma/check/crc32_table_be.h
+    ${SRC_DIR}/src/liblzma/check/crc32_table_le.h
+    ${SRC_DIR}/src/liblzma/check/crc64_fast.c
+    ${SRC_DIR}/src/liblzma/check/crc64_table.c
+    ${SRC_DIR}/src/liblzma/check/crc64_table_be.h
+    ${SRC_DIR}/src/liblzma/check/crc64_table_le.h
+    ${SRC_DIR}/src/liblzma/check/crc_macros.h
+    ${SRC_DIR}/src/liblzma/check/sha256.c
+    ${SRC_DIR}/src/liblzma/common/alone_decoder.c
+    ${SRC_DIR}/src/liblzma/common/alone_decoder.h
+    ${SRC_DIR}/src/liblzma/common/alone_encoder.c
+    ${SRC_DIR}/src/liblzma/common/auto_decoder.c
+    ${SRC_DIR}/src/liblzma/common/block_buffer_decoder.c
+    ${SRC_DIR}/src/liblzma/common/block_buffer_encoder.c
+    ${SRC_DIR}/src/liblzma/common/block_buffer_encoder.h
+    ${SRC_DIR}/src/liblzma/common/block_decoder.c
+    ${SRC_DIR}/src/liblzma/common/block_decoder.h
+    ${SRC_DIR}/src/liblzma/common/block_encoder.c
+    ${SRC_DIR}/src/liblzma/common/block_encoder.h
+    ${SRC_DIR}/src/liblzma/common/block_header_decoder.c
+    ${SRC_DIR}/src/liblzma/common/block_header_encoder.c
+    ${SRC_DIR}/src/liblzma/common/block_util.c
+    ${SRC_DIR}/src/liblzma/common/common.c
+    ${SRC_DIR}/src/liblzma/common/common.h
+    ${SRC_DIR}/src/liblzma/common/easy_buffer_encoder.c
+    ${SRC_DIR}/src/liblzma/common/easy_decoder_memusage.c
+    ${SRC_DIR}/src/liblzma/common/easy_encoder.c
+    ${SRC_DIR}/src/liblzma/common/easy_encoder_memusage.c
+    ${SRC_DIR}/src/liblzma/common/easy_preset.c
+    ${SRC_DIR}/src/liblzma/common/easy_preset.h
+    ${SRC_DIR}/src/liblzma/common/file_info.c
+    ${SRC_DIR}/src/liblzma/common/filter_buffer_decoder.c
+    ${SRC_DIR}/src/liblzma/common/filter_buffer_encoder.c
+    ${SRC_DIR}/src/liblzma/common/filter_common.c
+    ${SRC_DIR}/src/liblzma/common/filter_common.h
+    ${SRC_DIR}/src/liblzma/common/filter_decoder.c
+    ${SRC_DIR}/src/liblzma/common/filter_decoder.h
+    ${SRC_DIR}/src/liblzma/common/filter_encoder.c
+    ${SRC_DIR}/src/liblzma/common/filter_encoder.h
+    ${SRC_DIR}/src/liblzma/common/filter_flags_decoder.c
+    ${SRC_DIR}/src/liblzma/common/filter_flags_encoder.c
+    ${SRC_DIR}/src/liblzma/common/hardware_cputhreads.c
+    ${SRC_DIR}/src/liblzma/common/hardware_physmem.c
+    ${SRC_DIR}/src/liblzma/common/index.c
+    ${SRC_DIR}/src/liblzma/common/index.h
+    ${SRC_DIR}/src/liblzma/common/index_decoder.c
+    ${SRC_DIR}/src/liblzma/common/index_decoder.h
+    ${SRC_DIR}/src/liblzma/common/index_encoder.c
+    ${SRC_DIR}/src/liblzma/common/index_encoder.h
+    ${SRC_DIR}/src/liblzma/common/index_hash.c
+    ${SRC_DIR}/src/liblzma/common/memcmplen.h
+    ${SRC_DIR}/src/liblzma/common/outqueue.c
+    ${SRC_DIR}/src/liblzma/common/outqueue.h
+    ${SRC_DIR}/src/liblzma/common/stream_buffer_decoder.c
+    ${SRC_DIR}/src/liblzma/common/stream_buffer_encoder.c
+    ${SRC_DIR}/src/liblzma/common/stream_decoder.c
+    ${SRC_DIR}/src/liblzma/common/stream_decoder.h
+    ${SRC_DIR}/src/liblzma/common/stream_encoder.c
+    ${SRC_DIR}/src/liblzma/common/stream_encoder_mt.c
+    ${SRC_DIR}/src/liblzma/common/stream_flags_common.c
+    ${SRC_DIR}/src/liblzma/common/stream_flags_common.h
+    ${SRC_DIR}/src/liblzma/common/stream_flags_decoder.c
+    ${SRC_DIR}/src/liblzma/common/stream_flags_encoder.c
+    ${SRC_DIR}/src/liblzma/common/vli_decoder.c
+    ${SRC_DIR}/src/liblzma/common/vli_encoder.c
+    ${SRC_DIR}/src/liblzma/common/vli_size.c
+    ${SRC_DIR}/src/liblzma/delta/delta_common.c
+    ${SRC_DIR}/src/liblzma/delta/delta_common.h
+    ${SRC_DIR}/src/liblzma/delta/delta_decoder.c
+    ${SRC_DIR}/src/liblzma/delta/delta_decoder.h
+    ${SRC_DIR}/src/liblzma/delta/delta_encoder.c
+    ${SRC_DIR}/src/liblzma/delta/delta_encoder.h
+    ${SRC_DIR}/src/liblzma/delta/delta_private.h
+    ${SRC_DIR}/src/liblzma/lz/lz_decoder.c
+    ${SRC_DIR}/src/liblzma/lz/lz_decoder.h
+    ${SRC_DIR}/src/liblzma/lz/lz_encoder.c
+    ${SRC_DIR}/src/liblzma/lz/lz_encoder.h
+    ${SRC_DIR}/src/liblzma/lz/lz_encoder_hash.h
+    ${SRC_DIR}/src/liblzma/lz/lz_encoder_hash_table.h
+    ${SRC_DIR}/src/liblzma/lz/lz_encoder_mf.c
+    ${SRC_DIR}/src/liblzma/lzma/fastpos.h
+    ${SRC_DIR}/src/liblzma/lzma/fastpos_table.c
+    ${SRC_DIR}/src/liblzma/lzma/lzma2_decoder.c
+    ${SRC_DIR}/src/liblzma/lzma/lzma2_decoder.h
+    ${SRC_DIR}/src/liblzma/lzma/lzma2_encoder.c
+    ${SRC_DIR}/src/liblzma/lzma/lzma2_encoder.h
+    ${SRC_DIR}/src/liblzma/lzma/lzma_common.h
+    ${SRC_DIR}/src/liblzma/lzma/lzma_decoder.c
+    ${SRC_DIR}/src/liblzma/lzma/lzma_decoder.h
+    ${SRC_DIR}/src/liblzma/lzma/lzma_encoder.c
+    ${SRC_DIR}/src/liblzma/lzma/lzma_encoder.h
+    ${SRC_DIR}/src/liblzma/lzma/lzma_encoder_optimum_fast.c
+    ${SRC_DIR}/src/liblzma/lzma/lzma_encoder_optimum_normal.c
+    ${SRC_DIR}/src/liblzma/lzma/lzma_encoder_presets.c
+    ${SRC_DIR}/src/liblzma/lzma/lzma_encoder_private.h
+    ${SRC_DIR}/src/liblzma/rangecoder/price.h
+    ${SRC_DIR}/src/liblzma/rangecoder/price_table.c
+    ${SRC_DIR}/src/liblzma/rangecoder/range_common.h
+    ${SRC_DIR}/src/liblzma/rangecoder/range_decoder.h
+    ${SRC_DIR}/src/liblzma/rangecoder/range_encoder.h
+    ${SRC_DIR}/src/liblzma/simple/arm.c
+    ${SRC_DIR}/src/liblzma/simple/armthumb.c
+    ${SRC_DIR}/src/liblzma/simple/ia64.c
+    ${SRC_DIR}/src/liblzma/simple/powerpc.c
+    ${SRC_DIR}/src/liblzma/simple/simple_coder.c
+    ${SRC_DIR}/src/liblzma/simple/simple_coder.h
+    ${SRC_DIR}/src/liblzma/simple/simple_decoder.c
+    ${SRC_DIR}/src/liblzma/simple/simple_decoder.h
+    ${SRC_DIR}/src/liblzma/simple/simple_encoder.c
+    ${SRC_DIR}/src/liblzma/simple/simple_encoder.h
+    ${SRC_DIR}/src/liblzma/simple/simple_private.h
+    ${SRC_DIR}/src/liblzma/simple/sparc.c
+    ${SRC_DIR}/src/liblzma/simple/x86.c
+)
+
+target_include_directories(liblzma PRIVATE
+    ${SRC_DIR}/src/liblzma/api
+    ${SRC_DIR}/src/liblzma/common
+    ${SRC_DIR}/src/liblzma/check
+    ${SRC_DIR}/src/liblzma/lz
+    ${SRC_DIR}/src/liblzma/rangecoder
+    ${SRC_DIR}/src/liblzma/lzma
+    ${SRC_DIR}/src/liblzma/delta
+    ${SRC_DIR}/src/liblzma/simple
+    ${SRC_DIR}/src/common
+)
+
+target_link_libraries(liblzma Threads::Threads)
+
+# Put the tuklib functions under the lzma_ namespace.
+target_compile_definitions(liblzma PRIVATE TUKLIB_SYMBOL_PREFIX=lzma_)
+
+if (ENABLE_SSE2)
+    target_compile_definitions(liblzma PRIVATE HAVE_IMMINTRIN_H HAVE__MM_MOVEMASK_EPI8)
+endif()

contrib/zlib-ng-cmake/CMakeLists.txt

@@ -0,0 +1,161 @@
+set (SOURCE_DIR ${CMAKE_SOURCE_DIR}/contrib/zlib-ng)
+
+add_definitions(-DZLIB_COMPAT)
+add_definitions(-DWITH_GZFILEOP)
+add_definitions(-DUNALIGNED_OK)
+add_definitions(-DUNALIGNED64_OK)
+
+set (HAVE_UNISTD_H 1)
+add_definitions(-D_LARGEFILE64_SOURCE=1 -D__USE_LARGEFILE64)
+
+add_definitions(-DHAVE_VISIBILITY_HIDDEN)
+add_definitions(-DHAVE_VISIBILITY_INTERNAL)
+add_definitions(-DHAVE_BUILTIN_CTZ)
+add_definitions(-DHAVE_BUILTIN_CTZLL)
+
+set(ZLIB_ARCH_SRCS)
+set(ZLIB_ARCH_HDRS)
+
+set(ARCHDIR "arch/generic")
+
+if(ARCH_AARCH64)
+    set(ARCHDIR "${SOURCE_DIR}/arch/arm")
+
+    add_definitions(-DARM_FEATURES)
+    add_definitions(-DARM_AUXV_HAS_CRC32 -DARM_ASM_HWCAP)
+    add_definitions(-DARM_AUXV_HAS_NEON)
+    add_definitions(-DARM_ACLE_CRC_HASH)
+    add_definitions(-DARM_NEON_ADLER32 -DARM_NEON_CHUNKSET -DARM_NEON_SLIDEHASH)
+
+    list(APPEND ZLIB_ARCH_HDRS ${ARCHDIR}/arm.h)
+    list(APPEND ZLIB_ARCH_SRCS ${ARCHDIR}/armfeature.c)
+    set(ACLE_SRCS ${ARCHDIR}/crc32_acle.c ${ARCHDIR}/insert_string_acle.c)
+    list(APPEND ZLIB_ARCH_SRCS ${ACLE_SRCS})
+    set(NEON_SRCS ${ARCHDIR}/adler32_neon.c ${ARCHDIR}/chunkset_neon.c ${ARCHDIR}/slide_neon.c)
+    list(APPEND ZLIB_ARCH_SRCS ${NEON_SRCS})
+
+elseif(ARCH_PPC64LE)
+    set(ARCHDIR "${SOURCE_DIR}/arch/power")
+
+    add_definitions(-DPOWER8)
+    add_definitions(-DPOWER_FEATURES)
+    add_definitions(-DPOWER8_VSX_ADLER32)
+    add_definitions(-DPOWER8_VSX_SLIDEHASH)
+
+    list(APPEND ZLIB_ARCH_HDRS ${ARCHDIR}/power.h)
+    list(APPEND ZLIB_ARCH_SRCS ${ARCHDIR}/power.c)
+    set(POWER8_SRCS ${ARCHDIR}/adler32_power8.c ${ARCHDIR}/slide_hash_power8.c)
+    list(APPEND ZLIB_ARCH_SRCS ${POWER8_SRCS})
+
+elseif(ARCH_AMD64)
+    set(ARCHDIR "${SOURCE_DIR}/arch/x86")
+
+    add_definitions(-DX86_FEATURES)
+    list(APPEND ZLIB_ARCH_HDRS ${ARCHDIR}/x86.h)
+    list(APPEND ZLIB_ARCH_SRCS ${ARCHDIR}/x86.c)
+    if(ENABLE_AVX2)
+        add_definitions(-DX86_AVX2 -DX86_AVX2_ADLER32 -DX86_AVX_CHUNKSET)
+        set(AVX2_SRCS ${ARCHDIR}/slide_avx.c)
+        list(APPEND AVX2_SRCS ${ARCHDIR}/chunkset_avx.c)
+        list(APPEND AVX2_SRCS ${ARCHDIR}/compare258_avx.c)
+        list(APPEND AVX2_SRCS ${ARCHDIR}/adler32_avx.c)
+        list(APPEND ZLIB_ARCH_SRCS ${AVX2_SRCS})
+    endif()
+    if(ENABLE_SSE42)
+        add_definitions(-DX86_SSE42_CRC_HASH)
+        set(SSE42_SRCS ${ARCHDIR}/insert_string_sse.c)
+        list(APPEND ZLIB_ARCH_SRCS ${SSE42_SRCS})
+        add_definitions(-DX86_SSE42_CRC_INTRIN)
+        add_definitions(-DX86_SSE42_CMP_STR)
+        set(SSE42_SRCS ${ARCHDIR}/compare258_sse.c)
+        list(APPEND ZLIB_ARCH_SRCS ${SSE42_SRCS})
+    endif()
+    if(ENABLE_SSSE3)
+        add_definitions(-DX86_SSSE3 -DX86_SSSE3_ADLER32)
+        set(SSSE3_SRCS ${ARCHDIR}/adler32_ssse3.c)
+        list(APPEND ZLIB_ARCH_SRCS ${SSSE3_SRCS})
+    endif()
+    if(ENABLE_PCLMULQDQ)
+        add_definitions(-DX86_PCLMULQDQ_CRC)
+        set(PCLMULQDQ_SRCS ${ARCHDIR}/crc_folding.c)
+        list(APPEND ZLIB_ARCH_SRCS ${PCLMULQDQ_SRCS})
+    endif()
+
+    add_definitions(-DX86_SSE2 -DX86_SSE2_CHUNKSET -DX86_SSE2_SLIDEHASH)
+    set(SSE2_SRCS ${ARCHDIR}/chunkset_sse.c ${ARCHDIR}/slide_sse.c)
+    list(APPEND ZLIB_ARCH_SRCS ${SSE2_SRCS})
+    add_definitions(-DX86_NOCHECK_SSE2)
+endif ()
+
+macro(generate_cmakein input output)
+    file(REMOVE ${output})
+    file(STRINGS ${input} _lines)
+    foreach(_line IN LISTS _lines)
+        string(REGEX REPLACE "#ifdef HAVE_UNISTD_H.*" "@ZCONF_UNISTD_LINE@" _line "${_line}")
+        string(REGEX REPLACE "#ifdef NEED_PTRDIFF_T.*" "@ZCONF_PTRDIFF_LINE@" _line "${_line}")
+        if(NEED_PTRDIFF_T)
+            string(REGEX REPLACE "typedef PTRDIFF_TYPE" "typedef @PTRDIFF_TYPE@" _line "${_line}")
+        endif()
+        file(APPEND ${output} "${_line}\n")
+    endforeach()
+endmacro(generate_cmakein)
+
+generate_cmakein(${SOURCE_DIR}/zconf.h.in ${CMAKE_CURRENT_BINARY_DIR}/zconf.h.cmakein)
+
+set(ZLIB_SRCS
+    ${SOURCE_DIR}/adler32.c
+    ${SOURCE_DIR}/chunkset.c
+    ${SOURCE_DIR}/compare258.c
+    ${SOURCE_DIR}/compress.c
+    ${SOURCE_DIR}/crc32.c
+    ${SOURCE_DIR}/crc32_comb.c
+    ${SOURCE_DIR}/deflate.c
+    ${SOURCE_DIR}/deflate_fast.c
+    ${SOURCE_DIR}/deflate_medium.c
+    ${SOURCE_DIR}/deflate_quick.c
+    ${SOURCE_DIR}/deflate_slow.c
+    ${SOURCE_DIR}/functable.c
+    ${SOURCE_DIR}/infback.c
+    ${SOURCE_DIR}/inffast.c
+    ${SOURCE_DIR}/inflate.c
+    ${SOURCE_DIR}/inftrees.c
+    ${SOURCE_DIR}/insert_string.c
+    ${SOURCE_DIR}/trees.c
+    ${SOURCE_DIR}/uncompr.c
+    ${SOURCE_DIR}/zutil.c
+    ${SOURCE_DIR}/gzlib.c
+    ${SOURCE_DIR}/gzread.c
+    ${SOURCE_DIR}/gzwrite.c
+)
+
+set(ZLIB_ALL_SRCS ${ZLIB_SRCS} ${ZLIB_ARCH_SRCS})
+
+add_library(zlib ${ZLIB_ALL_SRCS})
+add_library(zlibstatic ALIAS zlib)
+
+# https://github.com/zlib-ng/zlib-ng/pull/733
+# This is disabed by default
+add_compile_definitions(Z_TLS=__thread)
+
+if(HAVE_UNISTD_H)
+  SET(ZCONF_UNISTD_LINE "#if 1    /* was set to #if 1 by configure/cmake/etc */")
+else()
+  SET(ZCONF_UNISTD_LINE "#if 0    /* was set to #if 0 by configure/cmake/etc */")
+endif()
+if(NEED_PTRDIFF_T)
+    SET(ZCONF_PTRDIFF_LINE "#if 1    /* was set to #if 1 by configure/cmake/etc */")
+else()
+    SET(ZCONF_PTRDIFF_LINE "#ifdef NEED_PTRDIFF_T    /* may be set to #if 1 by configure/cmake/etc */")
+endif()
+
+set(ZLIB_PC ${CMAKE_CURRENT_BINARY_DIR}/zlib.pc)
+configure_file(${SOURCE_DIR}/zlib.pc.cmakein ${ZLIB_PC} @ONLY)
+configure_file(${CMAKE_CURRENT_BINARY_DIR}/zconf.h.cmakein ${CMAKE_CURRENT_BINARY_DIR}/zconf.h @ONLY)
+
+# We should use same defines when including zlib.h as used when zlib compiled
+target_compile_definitions (zlib PUBLIC ZLIB_COMPAT WITH_GZFILEOP)
+if (ARCH_AMD64 OR ARCH_AARCH64)
+    target_compile_definitions (zlib PUBLIC X86_64 UNALIGNED_OK)
+endif ()
+
+target_include_directories(zlib PUBLIC ${SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})

docker/images.json

@@ -1,9 +1,7 @@
 {
     "docker/packager/deb": {
         "name": "clickhouse/deb-builder",
-        "dependent": [
+        "dependent": []
-            "docker/packager/unbundled"
-        ]
     },
     "docker/packager/binary": {
         "name": "clickhouse/binary-builder",
@@ -134,23 +132,11 @@
          "name": "clickhouse/test-base",
          "dependent": [
             "docker/test/stateless",
-            "docker/test/stateless_unbundled",
             "docker/test/integration/base",
             "docker/test/fuzzer",
             "docker/test/keeper-jepsen"
          ]
     },
-    "docker/packager/unbundled": {
-         "name": "clickhouse/unbundled-builder",
-         "dependent": [
-            "docker/test/stateless_unbundled"
-         ]
-    },
-    "docker/test/stateless_unbundled": {
-         "name": "clickhouse/stateless-unbundled-test",
-         "dependent": [
-         ]
-    },
     "docker/test/integration/kerberized_hadoop": {
         "name": "clickhouse/kerberized-hadoop",
         "dependent": []

docker/packager/deb/Dockerfile

@@ -33,7 +33,6 @@ RUN curl -O https://clickhouse-datasets.s3.yandex.net/utils/1/dpkg-deb \
     && chmod +x dpkg-deb \
     && cp dpkg-deb /usr/bin
 
-# Libraries from OS are only needed to test the "unbundled" build (this is not used in production).
 RUN apt-get update \
     && apt-get install \
         alien \

docker/packager/packager

@@ -11,7 +11,6 @@ SCRIPT_PATH = os.path.realpath(__file__)
 IMAGE_MAP = {
     "deb": "clickhouse/deb-builder",
     "binary": "clickhouse/binary-builder",
-    "unbundled": "clickhouse/unbundled-builder"
 }
 
 def check_image_exists_locally(image_name):
@@ -55,7 +54,7 @@ def run_docker_image_with_env(image_name, output, env_variables, ch_root, ccache
 
     subprocess.check_call(cmd, shell=True)
 
-def parse_env_variables(build_type, compiler, sanitizer, package_type, image_type, cache, distcc_hosts, unbundled, split_binary, clang_tidy, version, author, official, alien_pkgs, with_coverage, with_binaries):
+def parse_env_variables(build_type, compiler, sanitizer, package_type, image_type, cache, distcc_hosts, split_binary, clang_tidy, version, author, official, alien_pkgs, with_coverage, with_binaries):
     CLANG_PREFIX = "clang"
     DARWIN_SUFFIX = "-darwin"
     DARWIN_ARM_SUFFIX = "-darwin-aarch64"
@@ -107,7 +106,7 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ
 
     cxx = cc.replace('gcc', 'g++').replace('clang', 'clang++')
 
-    if image_type == "deb" or image_type == "unbundled":
+    if image_type == "deb":
         result.append("DEB_CC={}".format(cc))
         result.append("DEB_CXX={}".format(cxx))
         # For building fuzzers
@@ -159,15 +158,6 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ
         cmake_flags.append('-DENABLE_TESTS=1')
         cmake_flags.append('-DUSE_GTEST=1')
 
-    # "Unbundled" build is not suitable for any production usage.
-    # But it is occasionally used by some developers.
-    # The whole idea of using unknown version of libraries from the OS distribution is deeply flawed.
-    # We wish these developers good luck.
-    if unbundled:
-        # We also disable all CPU features except basic x86_64.
-        # It is only slightly related to "unbundled" build, but it is a good place to test if code compiles without these instruction sets.
-        cmake_flags.append('-DUNBUNDLED=1 -DUSE_INTERNAL_RDKAFKA_LIBRARY=1 -DENABLE_ARROW=0 -DENABLE_AVRO=0 -DENABLE_ORC=0 -DENABLE_PARQUET=0 -DENABLE_SSSE3=0 -DENABLE_SSE41=0 -DENABLE_SSE42=0 -DENABLE_PCLMULQDQ=0 -DENABLE_POPCNT=0 -DENABLE_AVX=0 -DENABLE_AVX2=0')
-
     if split_binary:
         cmake_flags.append('-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1')
         # We can't always build utils because it requires too much space, but
@@ -213,7 +203,6 @@ if __name__ == "__main__":
                                                "clang-13", "clang-13-darwin", "clang-13-darwin-aarch64", "clang-13-aarch64", "clang-13-ppc64le",
                                                "clang-11-freebsd", "clang-12-freebsd", "clang-13-freebsd", "gcc-11"), default="clang-13")
     parser.add_argument("--sanitizer", choices=("address", "thread", "memory", "undefined", ""), default="")
-    parser.add_argument("--unbundled", action="store_true")
     parser.add_argument("--split-binary", action="store_true")
     parser.add_argument("--clang-tidy", action="store_true")
     parser.add_argument("--cache", choices=("", "ccache", "distcc"), default="")
@@ -232,7 +221,7 @@ if __name__ == "__main__":
     if not os.path.isabs(args.output_dir):
         args.output_dir = os.path.abspath(os.path.join(os.getcwd(), args.output_dir))
 
-    image_type = 'binary' if args.package_type == 'performance' else 'unbundled' if args.unbundled else args.package_type
+    image_type = 'binary' if args.package_type == 'performance' else args.package_type
     image_name = IMAGE_MAP[image_type]
 
     if not os.path.isabs(args.clickhouse_repo_path):
@@ -256,7 +245,7 @@ if __name__ == "__main__":
             build_image(image_with_version, dockerfile)
     env_prepared = parse_env_variables(
         args.build_type, args.compiler, args.sanitizer, args.package_type, image_type,
-        args.cache, args.distcc_hosts, args.unbundled, args.split_binary, args.clang_tidy,
+        args.cache, args.distcc_hosts, args.split_binary, args.clang_tidy,
         args.version, args.author, args.official, args.alien_pkgs, args.with_coverage, args.with_binaries)
 
     run_docker_image_with_env(image_name, args.output_dir, env_prepared, ch_root, args.ccache_dir, args.docker_image_version)

docker/packager/unbundled/Dockerfile

@@ -1,69 +0,0 @@
-# docker build -t clickhouse/unbundled-builder .
-FROM clickhouse/deb-builder
-
-RUN export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \
-    && wget -nv -O /tmp/arrow-keyring.deb "https://apache.jfrog.io/artifactory/arrow/ubuntu/apache-arrow-apt-source-latest-${CODENAME}.deb" \
-    && dpkg -i /tmp/arrow-keyring.deb
-
-RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
-
-# Libraries from OS are only needed to test the "unbundled" build (that is not used in production).
-RUN apt-get update \
-    && apt-get install \
-        libicu-dev \
-        gperf \
-        perl \
-        pkg-config \
-        devscripts \
-        libc++-dev \
-        libc++abi-dev \
-        libboost-all-dev \
-        zlib1g-dev \
-        liblz4-dev \
-        libdouble-conversion-dev \
-        libxml2-dev \
-        librdkafka-dev \
-        libgoogle-perftools-dev \
-        libzstd-dev \
-        libltdl-dev \
-        libre2-dev \
-        libjemalloc-dev \
-        libmsgpack-dev \
-        libcurl4-openssl-dev \
-        unixodbc-dev \
-        odbcinst \
-        tzdata \
-        alien \
-        libcapnp-dev \
-        cmake \
-        gdb  \
-        pigz \
-        moreutils \
-        libcctz-dev \
-        libldap2-dev \
-        libsasl2-dev \
-        libgsasl7-dev \
-        heimdal-multidev \
-        libhyperscan-dev \
-        libbrotli-dev \
-        protobuf-compiler \
-        libprotoc-dev \
-        libgrpc++-dev \
-        protobuf-compiler-grpc \
-        libc-ares-dev \
-        rapidjson-dev \
-        libsnappy-dev \
-        libparquet-dev \
-        libthrift-dev \
-        libutf8proc-dev \
-        libbz2-dev \
-        libavro-dev \
-        libfarmhash-dev \
-        librocksdb-dev \
-        libgflags-dev \
-        libmysqlclient-dev \
-        --yes --no-install-recommends
-
-COPY build.sh /
-
-CMD ["/bin/bash", "/build.sh"]

docker/packager/unbundled/build.sh

@@ -1,15 +0,0 @@
-#!/usr/bin/env bash
-
-set -x -e
-
-ccache --show-stats ||:
-ccache --zero-stats ||:
-read -ra ALIEN_PKGS <<< "${ALIEN_PKGS:-}"
-build/release "${ALIEN_PKGS[@]}" | ts '%Y-%m-%d %H:%M:%S'
-mv /*.deb /output
-mv -- *.changes /output
-mv -- *.buildinfo /output
-mv /*.rpm /output ||: # if exists
-mv /*.tgz /output ||: # if exists
-
-ccache --show-stats ||:

docker/test/stateless_unbundled/Dockerfile

@@ -1,82 +0,0 @@
-# docker build -t clickhouse/stateless-unbundled-test .
-FROM clickhouse/test-base
-
-ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.4.20200302/clickhouse-odbc-1.1.4-Linux.tar.gz"
-
-RUN apt-get --allow-unauthenticated update -y \
-    && env DEBIAN_FRONTEND=noninteractive \
-        apt-get --allow-unauthenticated install --yes --no-install-recommends \
-            alien \
-            brotli \
-            zstd \
-            cmake \
-            devscripts \
-            expect \
-            gdb  \
-            gperf \
-            heimdal-multidev \
-            libboost-filesystem-dev \
-            libboost-iostreams-dev \
-            libboost-program-options-dev \
-            libboost-regex-dev \
-            libboost-system-dev \
-            libboost-thread-dev \
-            libc++-dev \
-            libc++abi-dev \
-            libcapnp-dev \
-            libcctz-dev \
-            libcurl4-openssl-dev \
-            libdouble-conversion-dev \
-            libgoogle-perftools-dev \
-            libhyperscan-dev \
-            libicu-dev \
-            libjemalloc-dev \
-            libldap2-dev \
-            libltdl-dev \
-            liblz4-dev \
-            libmsgpack-dev \
-            libpoco-dev \
-            libpoconetssl62 \
-            librdkafka-dev \
-            libre2-dev \
-            libreadline-dev \
-            libsasl2-dev \
-            libzstd-dev \
-            librocksdb-dev \
-            libgflags-dev \
-            lsof \
-            moreutils \
-            ncdu \
-            netcat-openbsd \
-            odbcinst \
-            openssl \
-            perl \
-            pigz \
-            pkg-config \
-            python3 \
-            python3-lxml \
-            python3-requests \
-            python3-termcolor \
-            python3-pip \
-            qemu-user-static \
-            sudo \
-            telnet \
-            tree \
-            tzdata \
-            unixodbc \
-            unixodbc-dev \
-            wget \
-            zlib1g-dev
-
-RUN pip3 install numpy scipy pandas
-
-RUN mkdir -p /tmp/clickhouse-odbc-tmp \
-   && wget -nv -O - ${odbc_driver_url} | tar --strip-components=1 -xz -C /tmp/clickhouse-odbc-tmp \
-   && cp /tmp/clickhouse-odbc-tmp/lib64/*.so /usr/local/lib/ \
-   && odbcinst -i -d -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbcinst.ini.sample \
-   && odbcinst -i -s -l -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbc.ini.sample \
-   && rm -rf /tmp/clickhouse-odbc-tmp
-
-COPY run.sh /
-CMD ["/bin/bash", "/run.sh"]
-

docker/test/stateless_unbundled/clickhouse-statelest-test-runner.Dockerfile

@@ -1,15 +0,0 @@
-# Since right now we can't set volumes to the docker during build, we split building container in stages:
-# 1. build base container
-# 2. run base conatiner with mounted volumes
-# 3. commit container as image
-FROM ubuntu:18.10 as clickhouse-test-runner-base
-
-# A volume where directory with clickhouse packages to be mounted,
-# for later installing.
-VOLUME /packages
-
-CMD apt-get update ;\
-	DEBIAN_FRONTEND=noninteractive \
-	apt install -y /packages/clickhouse-common-static_*.deb \
-		/packages/clickhouse-client_*.deb \
-		/packages/clickhouse-test_*.deb

docker/test/stateless_unbundled/run.sh

@@ -1,16 +0,0 @@
-#!/bin/bash
-
-set -e -x
-
-dpkg -i package_folder/clickhouse-common-static_*.deb
-dpkg -i package_folder/clickhouse-common-static-dbg_*.deb
-dpkg -i package_folder/clickhouse-server_*.deb
-dpkg -i package_folder/clickhouse-client_*.deb
-dpkg -i package_folder/clickhouse-test_*.deb
-
-# install test configs
-/usr/share/clickhouse-test/config/install.sh
-
-service clickhouse-server start && sleep 5
-
-clickhouse-test --testname --shard --zookeeper "$ADDITIONAL_OPTIONS" "$SKIP_TESTS_OPTION" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt

docs/en/development/continuous-integration.md

@@ -126,7 +126,6 @@ Builds ClickHouse in various configurations for use in further steps. You have t
 - **Compiler**: `gcc-9` or `clang-10` (or `clang-10-xx` for other architectures e.g. `clang-10-freebsd`).
 - **Build type**: `Debug` or `RelWithDebInfo` (cmake).
 - **Sanitizer**: `none` (without sanitizers), `address` (ASan), `memory` (MSan), `undefined` (UBSan), or `thread` (TSan).
-- **Bundled**: `bundled` build uses libraries from `contrib` folder, and `unbundled` build uses system libraries.
 - **Splitted** `splitted` is a [split build](../development/build.md#split-build)
 - **Status**: `success` or `fail`
 - **Build log**: link to the building and files copying log, useful when build failed.

docs/zh/development/continuous-integration.md

@@ -86,7 +86,6 @@ git push
 - **Compiler**: `gcc-9` 或 `clang-10` (或其他架构的`clang-10-xx`, 比如`clang-10-freebsd`).
 - **Build type**: `Debug` or `RelWithDebInfo` (cmake).
 - **Sanitizer**: `none` (without sanitizers), `address` (ASan), `memory` (MSan), `undefined` (UBSan), or `thread` (TSan).
-- **Bundled**: `bundled` 构建使用来自 `contrib` 库, 而 `unbundled` 构建使用系统库.
 - **Splitted**: `splitted` is a [split build](https://clickhouse.com/docs/en/development/build/#split-build)
 - **Status**: `成功` 或 `失败`
 - **Build log**: 链接到构建和文件复制日志, 当构建失败时很有用.

src/AggregateFunctions/AggregateFunctionFactory.cpp

@@ -178,6 +178,12 @@ AggregateFunctionPtr AggregateFunctionFactory::getImpl(
         /// uniqCombinedIfMergeIf is useful in cases when the underlying
         /// storage stores AggregateFunction(uniqCombinedIf) and in SELECT you
         /// need to filter aggregation result based on another column.
+
+#if defined(UNBUNDLED)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wstringop-overread"
+#endif
+
         if (!combinator->supportsNesting() && nested_name.ends_with(combinator_name))
         {
             throw Exception(ErrorCodes::ILLEGAL_AGGREGATION,
@@ -185,6 +191,10 @@ AggregateFunctionPtr AggregateFunctionFactory::getImpl(
                 combinator_name);
         }
 
+#if defined(UNBUNDLED)
+#pragma GCC diagnostic pop
+#endif
+
         DataTypes nested_types = combinator->transformArguments(argument_types);
         Array nested_parameters = combinator->transformParameters(parameters);
 

src/CMakeLists.txt

@@ -366,13 +366,10 @@ dbms_target_link_libraries (
         clickhouse_common_io
 )
 
-if (NOT_UNBUNDLED)
 dbms_target_link_libraries (
     PUBLIC
         boost::circular_buffer
-            boost::heap
+        boost::heap)
-    )
-endif()
 
 target_include_directories(clickhouse_common_io PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/Core/include") # uses some includes from core
 dbms_target_include_directories(PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/Core/include")

src/Common/HashTable/Hash.h

@@ -46,7 +46,6 @@ inline DB::UInt64 intHash64(DB::UInt64 x)
 
 #if defined(__aarch64__) && defined(__ARM_FEATURE_CRC32)
 #include <arm_acle.h>
-#include <arm_neon.h>
 #endif
 
 inline DB::UInt64 intHashCRC32(DB::UInt64 x)

src/Common/Macros.cpp

@@ -106,6 +106,16 @@ String Macros::expand(const String & s,
             res += toString(info.table_id.uuid);
             info.expanded_uuid = true;
         }
+        else if (info.shard && macro_name == "shard")
+        {
+            res += *info.shard;
+            info.expanded_uuid = true;
+        }
+        else if (info.replica && macro_name == "replica")
+        {
+            res += *info.replica;
+            info.expanded_uuid = true;
+        }
         else if (info.ignore_unknown || info.expand_special_macros_only)
         {
             if (info.expand_special_macros_only)

src/Common/Macros.h

@@ -34,6 +34,8 @@ public:
         StorageID table_id = StorageID::createEmpty();
         bool ignore_unknown = false;
         bool expand_special_macros_only = false;
+        std::optional<String> shard;
+        std::optional<String> replica;
 
         /// Information about macro expansion
         size_t level = 0;

src/Common/config.h.in

@@ -15,7 +15,6 @@
 #cmakedefine01 USE_CASSANDRA
 #cmakedefine01 USE_SENTRY
 #cmakedefine01 USE_GRPC
-#cmakedefine01 USE_STATS
 #cmakedefine01 USE_DATASKETCHES
 #cmakedefine01 USE_YAML_CPP
 #cmakedefine01 CLICKHOUSE_SPLIT_BINARY

src/Databases/DatabaseFactory.cpp

@@ -8,6 +8,7 @@
 #include <Databases/DatabaseOrdinary.h>
 #include <Databases/DatabaseReplicated.h>
 #include <Interpreters/Context.h>
+#include <Interpreters/evaluateConstantExpression.h>
 #include <Parsers/ASTCreateQuery.h>
 #include <Parsers/ASTFunction.h>
 #include <Parsers/ASTIdentifier.h>
@@ -30,7 +31,6 @@
 
 #if USE_MYSQL || USE_LIBPQXX
 #include <Common/parseRemoteDescription.h>
-#include <Interpreters/evaluateConstantExpression.h>
 #include <Common/parseAddress.h>
 #endif
 
@@ -258,7 +258,9 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String
         if (!engine->arguments || engine->arguments->children.size() != 3)
             throw Exception("Replicated database requires 3 arguments: zookeeper path, shard name and replica name", ErrorCodes::BAD_ARGUMENTS);
 
-        const auto & arguments = engine->arguments->children;
+        auto & arguments = engine->arguments->children;
+        for (auto & engine_arg : arguments)
+            engine_arg = evaluateConstantExpressionOrIdentifierAsLiteral(engine_arg, context);
 
         String zookeeper_path = safeGetLiteralValue<String>(arguments[0], "Replicated");
         String shard_name = safeGetLiteralValue<String>(arguments[1], "Replicated");

src/Databases/DatabaseReplicated.h

@@ -50,6 +50,8 @@ public:
 
     void stopReplication() override;
 
+    String getShardName() const { return shard_name; }
+    String getReplicaName() const { return replica_name; }
     String getFullReplicaName() const;
     static std::pair<String, String> parseFullReplicaName(const String & name);
 

src/Databases/DatabaseReplicatedHelpers.cpp

@@ -0,0 +1,17 @@
+#include <Databases/DatabaseReplicatedHelpers.h>
+#include <Databases/DatabaseReplicated.h>
+
+namespace DB
+{
+
+String getReplicatedDatabaseShardName(const DatabasePtr & database)
+{
+    return assert_cast<const DatabaseReplicated *>(database.get())->getShardName();
+}
+
+String getReplicatedDatabaseReplicaName(const DatabasePtr & database)
+{
+    return assert_cast<const DatabaseReplicated *>(database.get())->getReplicaName();
+}
+
+}

src/Databases/DatabaseReplicatedHelpers.h

@@ -0,0 +1,14 @@
+#pragma once
+#include <Core/Types.h>
+#include <memory>
+
+namespace DB
+{
+
+class IDatabase;
+using DatabasePtr = std::shared_ptr<IDatabase>;
+
+String getReplicatedDatabaseShardName(const DatabasePtr & database);
+String getReplicatedDatabaseReplicaName(const DatabasePtr & database);
+
+}

src/Databases/TablesLoader.cpp

@@ -36,8 +36,19 @@ void mergeDependenciesGraphs(DependenciesInfos & main_dependencies_info, const D
             if (maybe_existing_info.dependencies.empty())
                 maybe_existing_info.dependencies = dependencies;
             else if (maybe_existing_info.dependencies != dependencies)
-                throw Exception(ErrorCodes::LOGICAL_ERROR, "Have different dependencies for {}: {} and {}, it's a bug",
+            {
-                                table, fmt::join(maybe_existing_info.dependencies, ", "), fmt::join(dependencies, ", "));
+                /// Can happen on DatabaseReplicated recovery
+                LOG_WARNING(&Poco::Logger::get("TablesLoader"), "Replacing outdated dependencies ({}) of {} with: {}",
+                            fmt::join(maybe_existing_info.dependencies, ", "),
+                            table,
+                            fmt::join(dependencies, ", "));
+                for (const auto & old_dependency : maybe_existing_info.dependencies)
+                {
+                    [[maybe_unused]] bool removed = main_dependencies_info[old_dependency].dependent_database_objects.erase(table);
+                    assert(removed);
+                }
+                maybe_existing_info.dependencies = dependencies;
+            }
         }
     }
 }

src/Functions/CMakeLists.txt

@@ -118,10 +118,6 @@ target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_array)
 
 add_subdirectory(JSONPath)
 
-if (USE_STATS)
-    target_link_libraries(clickhouse_functions PRIVATE stats)
-endif()
-
 # Signed integer overflow on user-provided data inside boost::geometry - ignore.
 set_source_files_properties("pointInPolygon.cpp" PROPERTIES COMPILE_FLAGS -fno-sanitize=signed-integer-overflow)
 

src/Storages/MergeTree/registerStorageMergeTree.cpp

@@ -22,6 +22,7 @@
 #include <Interpreters/Context.h>
 #include <Interpreters/evaluateConstantExpression.h>
 
+#include <Databases/DatabaseReplicatedHelpers.h>
 
 namespace DB
 {
@@ -541,6 +542,12 @@ static StoragePtr create(const StorageFactory::Arguments & args)
         /// to make possible copying metadata files between replicas.
         Macros::MacroExpansionInfo info;
         info.table_id = args.table_id;
+        if (is_replicated_database)
+        {
+            auto database = DatabaseCatalog::instance().getDatabase(args.table_id.database_name);
+            info.shard = getReplicatedDatabaseShardName(database);
+            info.replica = getReplicatedDatabaseReplicaName(database);
+        }
         if (!allow_uuid_macro)
             info.table_id.uuid = UUIDHelpers::Nil;
         zookeeper_path = args.getContext()->getMacros()->expand(zookeeper_path, info);

src/Storages/System/StorageSystemBuildOptions.generated.cpp.in

@@ -24,7 +24,6 @@ const char * auto_config_build[]
     "BUILD_INCLUDE_DIRECTORIES", "@BUILD_INCLUDE_DIRECTORIES@",
     "STATIC", "@USE_STATIC_LIBRARIES@",
     "SPLIT_BINARY", "@CLICKHOUSE_SPLIT_BINARY@",
-    "UNBUNDLED", "@UNBUNDLED@",
     "USE_EMBEDDED_COMPILER", "@USE_EMBEDDED_COMPILER@",
     "USE_GLIBC_COMPATIBILITY", "@GLIBC_COMPATIBILITY@",
     "USE_JEMALLOC", "@ENABLE_JEMALLOC@",

tests/ci/build_check.py

@@ -50,8 +50,6 @@ def get_packager_cmd(build_config, packager_path, output_path, build_version, im
         cmd += ' --build-type={}'.format(build_config['build_type'])
     if build_config['sanitizer']:
         cmd += ' --sanitizer={}'.format(build_config['sanitizer'])
-    if build_config['bundled'] == 'unbundled':
-        cmd += ' --unbundled'
     if build_config['splitted'] == 'splitted':
         cmd += ' --split-binary'
     if build_config['tidy'] == 'enable':
@@ -73,9 +71,7 @@ def get_packager_cmd(build_config, packager_path, output_path, build_version, im
     return cmd
 
 def get_image_name(build_config):
-    if build_config['bundled'] != 'bundled':
+    if build_config['package_type'] != 'deb':
-        return 'clickhouse/unbundled-builder'
-    elif build_config['package_type'] != 'deb':
         return 'clickhouse/binary-builder'
     else:
         return 'clickhouse/deb-builder'

tests/ci/ci_config.json

@@ -81,16 +81,6 @@
             "tidy": "disable",
             "with_coverage": false
         },
-        {
-            "compiler": "gcc-11",
-            "build-type": "",
-            "sanitizer": "",
-            "package-type": "deb",
-            "bundled": "unbundled",
-            "splitted": "unsplitted",
-            "tidy": "disable",
-            "with_coverage": false
-        },
         {
             "compiler": "clang-13",
             "build-type": "",
@@ -343,30 +333,6 @@
                 "with_coverage": false
             }
         },
-        "Functional stateless tests (pytest)": {
-            "required_build_properties": {
-                "compiler": "clang-13",
-                "package_type": "deb",
-                "build_type": "relwithdebuginfo",
-                "sanitizer": "none",
-                "bundled": "bundled",
-                "splitted": "unsplitted",
-                "clang-tidy": "disable",
-                "with_coverage": false
-            }
-        },
-        "Functional stateless tests (unbundled)": {
-            "required_build_properties": {
-                "compiler": "gcc-11",
-                "package_type": "deb",
-                "build_type": "relwithdebuginfo",
-                "sanitizer": "none",
-                "bundled": "unbundled",
-                "splitted": "unsplitted",
-                "clang-tidy": "disable",
-                "with_coverage": false
-            }
-        },
         "Functional stateless tests (release, wide parts enabled)": {
             "required_build_properties": {
                 "compiler": "clang-13",
@@ -624,7 +590,7 @@
                 "compiler": "clang-13",
                 "package_type": "binary",
                 "build_type": "relwithdebuginfo",
-                "sanitizer": "thread",
+                "sanitizer": "undefined",
                 "bundled": "bundled",
                 "splitted": "unsplitted",
                 "clang-tidy": "disable",

tests/ci/ci_config.py

@@ -83,16 +83,6 @@ CI_CONFIG = {
             "tidy": "disable",
             "with_coverage": False
         },
-        {
-            "compiler": "gcc-11",
-            "build_type": "",
-            "sanitizer": "",
-            "package_type": "deb",
-            "bundled": "unbundled",
-            "splitted": "unsplitted",
-            "tidy": "disable",
-            "with_coverage": False
-        },
         {
             "compiler": "clang-13",
             "build_type": "",
@@ -345,18 +335,6 @@ CI_CONFIG = {
                 "with_coverage": False
             }
         },
-        "Stateless tests (unbundled, actions)": {
-            "required_build_properties": {
-                "compiler": "gcc-11",
-                "package_type": "deb",
-                "build_type": "relwithdebuginfo",
-                "sanitizer": "none",
-                "bundled": "unbundled",
-                "splitted": "unsplitted",
-                "clang_tidy": "disable",
-                "with_coverage": False
-            }
-        },
         "Stateless tests (release, wide parts enabled, actions)": {
             "required_build_properties": {
                 "compiler": "clang-13",
@@ -614,7 +592,7 @@ CI_CONFIG = {
                 "compiler": "clang-13",
                 "package_type": "deb",
                 "build_type": "relwithdebuginfo",
-                "sanitizer": "thread",
+                "sanitizer": "undefined",
                 "bundled": "bundled",
                 "splitted": "unsplitted",
                 "clang_tidy": "disable",

tests/clickhouse-test

@@ -996,7 +996,6 @@ class BuildFlags():
     UNDEFINED = 'ubsan'
     MEMORY = 'msan'
     DEBUG = 'debug'
-    UNBUNDLED = 'unbundled'
     RELEASE = 'release'
     ORDINARY_DATABASE = 'ordinary-database'
     POLYMORPHIC_PARTS = 'polymorphic-parts'
@@ -1021,10 +1020,6 @@ def collect_build_flags(args):
     elif b'RelWithDebInfo' in value or b'Release' in value:
         result.append(BuildFlags.RELEASE)
 
-    value = clickhouse_execute(args, "SELECT value FROM system.build_options WHERE name = 'UNBUNDLED'")
-    if value in (b'ON', b'1'):
-        result.append(BuildFlags.UNBUNDLED)
-
     value = clickhouse_execute(args, "SELECT value FROM system.settings WHERE name = 'default_database_engine'")
     if value == b'Ordinary':
         result.append(BuildFlags.ORDINARY_DATABASE)

tests/integration/test_replicated_database/test.py

@@ -16,7 +16,7 @@ main_node = cluster.add_instance('main_node', main_configs=['configs/config.xml'
 dummy_node = cluster.add_instance('dummy_node', main_configs=['configs/config.xml'], user_configs=['configs/settings.xml'], with_zookeeper=True, stay_alive=True, macros={"shard": 1, "replica": 2})
 competing_node = cluster.add_instance('competing_node', main_configs=['configs/config.xml'], user_configs=['configs/settings.xml'], with_zookeeper=True, macros={"shard": 1, "replica": 3})
 snapshotting_node = cluster.add_instance('snapshotting_node', main_configs=['configs/config.xml'], user_configs=['configs/settings.xml'], with_zookeeper=True, macros={"shard": 2, "replica": 1})
-snapshot_recovering_node = cluster.add_instance('snapshot_recovering_node', main_configs=['configs/config.xml'], user_configs=['configs/settings.xml'], with_zookeeper=True, macros={"shard": 2, "replica": 2})
+snapshot_recovering_node = cluster.add_instance('snapshot_recovering_node', main_configs=['configs/config.xml'], user_configs=['configs/settings.xml'], with_zookeeper=True)
 
 all_nodes = [main_node, dummy_node, competing_node, snapshotting_node, snapshot_recovering_node]
 
@@ -37,7 +37,7 @@ def started_cluster():
         cluster.shutdown()
 
 def test_create_replicated_table(started_cluster):
-    main_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');")
+    main_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica' || '1');")
     dummy_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica2');")
     assert "Explicit zookeeper_path and replica_name are specified" in \
            main_node.query_and_get_error("CREATE TABLE testdb.replicated_table (d Date, k UInt64, i32 Int32) "

tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh

@@ -1,5 +1,4 @@
 #!/usr/bin/env bash
-# Tags: no-unbundled
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

tests/queries/0_stateless/00429_long_http_bufferization.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long, no-unbundled, no-parallel
+# Tags: long, no-parallel
 
 set -e
 

tests/queries/0_stateless/00429_point_in_ellipses.sql

@@ -1,4 +1,3 @@
--- Tags: no-unbundled
 
 SELECT pointInEllipses(33.3, 55.3, 33.4, 55.1, 1.0, 1.0) AS distance;
 SELECT pointInEllipses(33.3 + v, 55.3 + v, 33.4, 55.1, 1.0, 1.0) AS distance from

tests/queries/0_stateless/00505_secure.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: no-unbundled, no-fasttest
+# Tags: no-fasttest
 
 # set -x