Build fixes (#5278)

CMakeLists.txt

@@ -310,6 +310,7 @@ include (cmake/find_rt.cmake)
 include (cmake/find_execinfo.cmake)
 include (cmake/find_readline_edit.cmake)
 include (cmake/find_re2.cmake)
+include (cmake/find_libgsasl.cmake)
 include (cmake/find_rdkafka.cmake)
 include (cmake/find_capnp.cmake)
 include (cmake/find_llvm.cmake)
@@ -317,7 +318,6 @@ include (cmake/find_cpuid.cmake) # Freebsd, bundled
 if (NOT USE_CPUID)
     include (cmake/find_cpuinfo.cmake) # Debian
 endif()
-include (cmake/find_libgsasl.cmake)
 include (cmake/find_libxml2.cmake)
 include (cmake/find_brotli.cmake)
 include (cmake/find_protobuf.cmake)

cmake/find_boost.cmake

@@ -1,9 +1,12 @@
 option (USE_INTERNAL_BOOST_LIBRARY "Set to FALSE to use system boost library instead of bundled" ${NOT_UNBUNDLED})
 
 # Test random file existing in all package variants
-if (USE_INTERNAL_BOOST_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/boost/libs/system/src/error_code.cpp")
-   message (WARNING "submodules in contrib/boost is missing. to fix try run: \n git submodule update --init --recursive")
+if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/boost/libs/system/src/error_code.cpp")
+    if(USE_INTERNAL_BOOST_LIBRARY)
+        message(WARNING "submodules in contrib/boost is missing. to fix try run: \n git submodule update --init --recursive")
+    endif()
     set (USE_INTERNAL_BOOST_LIBRARY 0)
+    set (MISSING_INTERNAL_BOOST_LIBRARY 1)
 endif ()
 
 if (NOT USE_INTERNAL_BOOST_LIBRARY)
@@ -21,10 +24,9 @@ if (NOT USE_INTERNAL_BOOST_LIBRARY)
         set (Boost_INCLUDE_DIRS "")
         set (Boost_SYSTEM_LIBRARY "")
     endif ()
-
 endif ()
 
-if (NOT Boost_SYSTEM_LIBRARY)
+if (NOT Boost_SYSTEM_LIBRARY AND NOT MISSING_INTERNAL_BOOST_LIBRARY)
     set (USE_INTERNAL_BOOST_LIBRARY 1)
     set (Boost_SYSTEM_LIBRARY boost_system_internal)
     set (Boost_PROGRAM_OPTIONS_LIBRARY boost_program_options_internal)
@@ -44,7 +46,6 @@ if (NOT Boost_SYSTEM_LIBRARY)
 
     # For packaged version:
     list (APPEND Boost_INCLUDE_DIRS "${ClickHouse_SOURCE_DIR}/contrib/boost")
-
 endif ()
 
 message (STATUS "Using Boost: ${Boost_INCLUDE_DIRS} : ${Boost_PROGRAM_OPTIONS_LIBRARY},${Boost_SYSTEM_LIBRARY},${Boost_FILESYSTEM_LIBRARY},${Boost_REGEX_LIBRARY}")

cmake/find_libgsasl.cmake

@@ -22,4 +22,8 @@ elseif (NOT MISSING_INTERNAL_LIBGSASL_LIBRARY AND NOT APPLE AND NOT ARCH_32)
     set (LIBGSASL_LIBRARY libgsasl)
 endif ()
 
-message (STATUS "Using libgsasl: ${LIBGSASL_INCLUDE_DIR} : ${LIBGSASL_LIBRARY}")
+if(LIBGSASL_LIBRARY AND LIBGSASL_INCLUDE_DIR)
+    set (USE_LIBGSASL 1)
+endif()
+
+message (STATUS "Using libgsasl=${USE_LIBGSASL}: ${LIBGSASL_INCLUDE_DIR} : ${LIBGSASL_LIBRARY}")

cmake/find_rdkafka.cmake

@@ -10,7 +10,7 @@ endif ()
 
 if (ENABLE_RDKAFKA)
 
-if (OS_LINUX AND NOT ARCH_ARM)
+if (OS_LINUX AND NOT ARCH_ARM AND USE_LIBGSASL)
     option (USE_INTERNAL_RDKAFKA_LIBRARY "Set to FALSE to use system librdkafka instead of the bundled" ${NOT_UNBUNDLED})
 endif ()
 

cmake/find_re2.cmake

@@ -1,5 +1,13 @@
 option (USE_INTERNAL_RE2_LIBRARY "Set to FALSE to use system re2 library instead of bundled [slower]" ${NOT_UNBUNDLED})
 
+if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/re2/CMakeLists.txt")
+    if(USE_INTERNAL_RE2_LIBRARY)
+        message(WARNING "submodule contrib/re2 is missing. to fix try run: \n git submodule update --init --recursive")
+    endif()
+    set(USE_INTERNAL_RE2_LIBRARY 0)
+    set(MISSING_INTERNAL_RE2_LIBRARY 1)
+endif()
+
 if (NOT USE_INTERNAL_RE2_LIBRARY)
     find_library (RE2_LIBRARY re2)
     find_path (RE2_INCLUDE_DIR NAMES re2/re2.h PATHS ${RE2_INCLUDE_PATHS})

cmake/find_simdjson.cmake

@@ -9,6 +9,6 @@ if (NOT HAVE_AVX2)
 endif ()
 
 option (USE_SIMDJSON "Use simdjson" ON)
-
-set (SIMDJSON_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/simdjson/include")
 set (SIMDJSON_LIBRARY "simdjson")
+
+message(STATUS "Using simdjson=${USE_SIMDJSON}: ${SIMDJSON_LIBRARY}")

cmake/find_zlib.cmake

@@ -2,20 +2,28 @@ if (NOT OS_FREEBSD AND NOT ARCH_32)
     option (USE_INTERNAL_ZLIB_LIBRARY "Set to FALSE to use system zlib library instead of bundled" ${NOT_UNBUNDLED})
 endif ()
 
+if (NOT MSVC)
+    set (INTERNAL_ZLIB_NAME "zlib-ng" CACHE INTERNAL "")
+else ()
+    set (INTERNAL_ZLIB_NAME "zlib" CACHE INTERNAL "")
+    if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/${INTERNAL_ZLIB_NAME}")
+        message (WARNING "Will use standard zlib, please clone manually:\n git clone https://github.com/madler/zlib.git ${ClickHouse_SOURCE_DIR}/contrib/${INTERNAL_ZLIB_NAME}")
+    endif ()
+endif ()
+
+if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/${INTERNAL_ZLIB_NAME}/zlib.h")
+    if(USE_INTERNAL_ZLIB_LIBRARY)
+        message(WARNING "submodule contrib/${INTERNAL_ZLIB_NAME} is missing. to fix try run: \n git submodule update --init --recursive")
+    endif()
+    set(USE_INTERNAL_ZLIB_LIBRARY 0)
+    set(MISSING_INTERNAL_ZLIB_LIBRARY 1)
+endif()
+
 if (NOT USE_INTERNAL_ZLIB_LIBRARY)
     find_package (ZLIB)
 endif ()
 
-if (NOT ZLIB_FOUND)
-    if (NOT MSVC)
-        set (INTERNAL_ZLIB_NAME "zlib-ng" CACHE INTERNAL "")
-    else ()
-        set (INTERNAL_ZLIB_NAME "zlib" CACHE INTERNAL "")
-        if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/${INTERNAL_ZLIB_NAME}")
-            message (WARNING "Will use standard zlib, please clone manually:\n git clone https://github.com/madler/zlib.git ${ClickHouse_SOURCE_DIR}/contrib/${INTERNAL_ZLIB_NAME}")
-        endif ()
-    endif ()
-
+if (NOT ZLIB_FOUND AND NOT MISSING_INTERNAL_ZLIB_LIBRARY)
     set (USE_INTERNAL_ZLIB_LIBRARY 1)
     set (ZLIB_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/${INTERNAL_ZLIB_NAME}" "${ClickHouse_BINARY_DIR}/contrib/${INTERNAL_ZLIB_NAME}" CACHE INTERNAL "") # generated zconf.h
     set (ZLIB_INCLUDE_DIRS ${ZLIB_INCLUDE_DIR}) # for poco

cmake/find_zstd.cmake

@@ -1,9 +1,12 @@
 option (USE_INTERNAL_ZSTD_LIBRARY "Set to FALSE to use system zstd library instead of bundled" ${NOT_UNBUNDLED})
 
-if (USE_INTERNAL_ZSTD_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/zstd/lib/zstd.h")
-   message (WARNING "submodule contrib/zstd is missing. to fix try run: \n git submodule update --init --recursive")
-   set (USE_INTERNAL_ZSTD_LIBRARY 0)
-endif ()
+if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/zstd/lib/zstd.h")
+    if(USE_INTERNAL_ZSTD_LIBRARY)
+        message(WARNING "submodule contrib/zstd is missing. to fix try run: \n git submodule update --init --recursive")
+    endif()
+    set(USE_INTERNAL_ZSTD_LIBRARY 0)
+    set(MISSING_INTERNAL_ZSTD_LIBRARY 1)
+endif()
 
 if (NOT USE_INTERNAL_ZSTD_LIBRARY)
     find_library (ZSTD_LIBRARY zstd)
@@ -11,7 +14,7 @@ if (NOT USE_INTERNAL_ZSTD_LIBRARY)
 endif ()
 
 if (ZSTD_LIBRARY AND ZSTD_INCLUDE_DIR)
-else ()
+elseif (NOT MISSING_INTERNAL_ZSTD_LIBRARY)
     set (USE_INTERNAL_ZSTD_LIBRARY 1)
     set (ZSTD_LIBRARY zstd)
     set (ZSTD_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/zstd/lib)

contrib/librdkafka-cmake/CMakeLists.txt

@@ -59,7 +59,7 @@ add_library(rdkafka ${SRCS})
 target_include_directories(rdkafka SYSTEM PUBLIC include)
 target_include_directories(rdkafka SYSTEM PUBLIC ${RDKAFKA_SOURCE_DIR})         # Because weird logic with "include_next" is used.
 target_include_directories(rdkafka SYSTEM PRIVATE ${ZSTD_INCLUDE_DIR}/common)   # Because wrong path to "zstd_errors.h" is used.
-target_link_libraries(rdkafka PUBLIC ${ZLIB_LIBRARIES} ${ZSTD_LIBRARY} ${LZ4_LIBRARY} libgsasl)
+target_link_libraries(rdkafka PUBLIC ${ZLIB_LIBRARIES} ${ZSTD_LIBRARY} ${LZ4_LIBRARY} ${LIBGSASL_LIBRARY})
 if(OPENSSL_SSL_LIBRARY AND OPENSSL_CRYPTO_LIBRARY)
     target_link_libraries(rdkafka PUBLIC ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY})
 endif()

contrib/simdjson-cmake/CMakeLists.txt

@@ -1,15 +1,7 @@
 if (NOT HAVE_AVX2)
     message (FATAL_ERROR "No AVX2 support")
 endif ()
-
-if(MAKE_STATIC_LIBRARIES)
-    set(SIMDJSON_LIB_TYPE STATIC)
-    MESSAGE(STATUS "Building static library ${SIMDJSON_LIBRARY}")
-else()
-    set(SIMDJSON_LIB_TYPE SHARED)
-    MESSAGE(STATUS "Building dynamic library ${SIMDJSON_LIBRARY}")
-endif()
-
+set(SIMDJSON_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/simdjson/include")
 set(SIMDJSON_SRC_DIR "${SIMDJSON_INCLUDE_DIR}/../src")
 set(SIMDJSON_SRC
     ${SIMDJSON_SRC_DIR}/jsonioutil.cpp
@@ -21,6 +13,6 @@ set(SIMDJSON_SRC
     ${SIMDJSON_SRC_DIR}/parsedjsoniterator.cpp
 )
 
-add_library(${SIMDJSON_LIBRARY} ${SIMDJSON_LIB_TYPE} ${SIMDJSON_SRC})
-target_include_directories(${SIMDJSON_LIBRARY} PRIVATE "${SIMDJSON_INCLUDE_DIR}")
+add_library(${SIMDJSON_LIBRARY} ${SIMDJSON_SRC})
+target_include_directories(${SIMDJSON_LIBRARY} PUBLIC "${SIMDJSON_INCLUDE_DIR}")
 target_compile_options(${SIMDJSON_LIBRARY} PRIVATE -mavx2 -mbmi -mbmi2 -mpclmul)

dbms/CMakeLists.txt

@@ -189,8 +189,17 @@ target_link_libraries (clickhouse_common_io
     ${Poco_Net_LIBRARY}
     ${Poco_Util_LIBRARY}
     ${Poco_Foundation_LIBRARY}
-    ${RE2_LIBRARY}
-    ${RE2_ST_LIBRARY}
+)
+
+if(RE2_LIBRARY)
+    target_link_libraries(clickhouse_common_io PUBLIC ${RE2_LIBRARY})
+endif()
+if(RE2_ST_LIBRARY)
+    target_link_libraries(clickhouse_common_io PUBLIC ${RE2_ST_LIBRARY})
+endif()
+
+target_link_libraries(clickhouse_common_io
+        PUBLIC
     ${CITYHASH_LIBRARIES}
         PRIVATE
     ${ZLIB_LIBRARIES}
@@ -208,7 +217,9 @@ target_link_libraries (clickhouse_common_io
 )
 
 
-target_include_directories(clickhouse_common_io SYSTEM BEFORE PUBLIC ${RE2_INCLUDE_DIR})
+if(RE2_INCLUDE_DIR)
+    target_include_directories(clickhouse_common_io SYSTEM BEFORE PUBLIC ${RE2_INCLUDE_DIR})
+endif()
 
 if (USE_LFALLOC)
     target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${LFALLOC_INCLUDE_DIR})

dbms/src/Compression/CMakeLists.txt

@@ -1,14 +1,18 @@
 include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
 add_headers_and_sources(clickhouse_compression .)
 add_library(clickhouse_compression ${clickhouse_compression_headers} ${clickhouse_compression_sources})
-target_link_libraries(clickhouse_compression PRIVATE clickhouse_parsers clickhouse_common_io ${ZSTD_LIBRARY} ${LZ4_LIBRARY} ${CITYHASH_LIBRARIES})
+target_link_libraries(clickhouse_compression PRIVATE clickhouse_parsers clickhouse_common_io ${LZ4_LIBRARY} ${CITYHASH_LIBRARIES})
+if(ZSTD_LIBRARY)
+    target_link_libraries(clickhouse_compression PRIVATE ${ZSTD_LIBRARY})
+endif()
+
 target_include_directories(clickhouse_compression PUBLIC ${DBMS_INCLUDE_DIR})
 target_include_directories(clickhouse_compression SYSTEM PUBLIC ${PCG_RANDOM_INCLUDE_DIR})
 
 if (NOT USE_INTERNAL_LZ4_LIBRARY)
     target_include_directories(clickhouse_compression SYSTEM BEFORE PRIVATE ${LZ4_INCLUDE_DIR})
 endif ()
-if (NOT USE_INTERNAL_ZSTD_LIBRARY)
+if (NOT USE_INTERNAL_ZSTD_LIBRARY AND ZSTD_INCLUDE_DIR)
     target_include_directories(clickhouse_compression SYSTEM BEFORE PRIVATE ${ZSTD_INCLUDE_DIR})
 endif ()
 

dbms/src/Functions/CMakeLists.txt

@@ -60,17 +60,16 @@ if(USE_BASE64)
     target_include_directories(clickhouse_functions SYSTEM PRIVATE ${BASE64_INCLUDE_DIR})
 endif()
 
-if (USE_XXHASH)
+if(USE_XXHASH)
     target_link_libraries(clickhouse_functions PRIVATE ${XXHASH_LIBRARY})
     target_include_directories(clickhouse_functions SYSTEM PRIVATE ${XXHASH_INCLUDE_DIR})
 endif()
 
-if (USE_HYPERSCAN)
-    target_link_libraries (clickhouse_functions PRIVATE ${HYPERSCAN_LIBRARY})
-    target_include_directories (clickhouse_functions SYSTEM PRIVATE ${HYPERSCAN_INCLUDE_DIR})
-endif ()
+if(USE_HYPERSCAN)
+    target_link_libraries(clickhouse_functions PRIVATE ${HYPERSCAN_LIBRARY})
+    target_include_directories(clickhouse_functions SYSTEM PRIVATE ${HYPERSCAN_INCLUDE_DIR})
+endif()
 
-if (USE_SIMDJSON)
+if(USE_SIMDJSON)
     target_link_libraries(clickhouse_functions PRIVATE ${SIMDJSON_LIBRARY})
-    target_include_directories(clickhouse_functions PRIVATE ${SIMDJSON_INCLUDE_DIR})
-endif ()
+endif()

dbms/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in

@@ -37,11 +37,14 @@ const char * auto_config_build[]
     "USE_GLIBC_COMPATIBILITY", "@GLIBC_COMPATIBILITY@",
     "USE_JEMALLOC", "@USE_JEMALLOC@",
     "USE_TCMALLOC", "@USE_TCMALLOC@",
+    "USE_LFALLOC", "@USE_LFALLOC@",
+    "USE_LFALLOC_RANDOM_HINT", "@USE_LFALLOC_RANDOM_HINT@",
     "USE_UNWIND", "@USE_UNWIND@",
     "USE_ICU", "@USE_ICU@",
     "USE_MYSQL", "@USE_MYSQL@",
     "USE_RE2_ST", "@USE_RE2_ST@",
     "USE_VECTORCLASS", "@USE_VECTORCLASS@",
+    "USE_LIBGSASL", "@USE_LIBGSASL@",
     "USE_RDKAFKA", "@USE_RDKAFKA@",
     "USE_CAPNP", "@USE_CAPNP@",
     "USE_POCO_SQLODBC", "@USE_POCO_SQLODBC@",
@@ -57,8 +60,7 @@ const char * auto_config_build[]
     "USE_BROTLI", "@USE_BROTLI@",
     "USE_SSL", "@USE_SSL@",
     "USE_HYPERSCAN", "@USE_HYPERSCAN@",
-    "USE_LFALLOC", "@USE_LFALLOC@",
-    "USE_LFALLOC_RANDOM_HINT", "@USE_LFALLOC_RANDOM_HINT@",
+    "USE_SIMDJSON", "@USE_SIMDJSON@",
 
     nullptr, nullptr
 };

debian/pbuilder-hooks/A00ccache

@@ -14,4 +14,5 @@ fi
 
 df -h
 ccache --show-stats
-ccache -M ${CCACHE_SIZE:=32G}
+ccache --zero-stats
+ccache --max-size=${CCACHE_SIZE:=32G}

docker/packager/binary/build.sh

@@ -4,10 +4,11 @@ set -x -e
 
 mkdir -p build/build_docker
 cd build/build_docker
-ccache -s ||:
+ccache --show-stats ||:
+ccache --zero-stats ||:
 rm -f CMakeCache.txt
 cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DSANITIZE=$SANITIZER $CMAKE_FLAGS
 ninja
-ccache -s ||:
+ccache --show-stats ||:
 mv ./dbms/programs/clickhouse* /output
 mv ./dbms/unit_tests_dbms /output

docker/packager/deb/build.sh

@@ -2,9 +2,10 @@
 
 set -x -e
 
-ccache -s ||:
+ccache --show-stats ||:
+ccache --zero-stats ||:
 build/release --no-pbuilder
 mv /*.deb /output
 mv *.changes /output
 mv *.buildinfo /output
-ccache -s ||:
+ccache --show-stats ||:

libs/libcommon/CMakeLists.txt

@@ -87,7 +87,10 @@ endif ()
 
 find_package (Threads)
 
-target_include_directories (common BEFORE PRIVATE ${CCTZ_INCLUDE_DIR})
+if(CCTZ_INCLUDE_DIR)
+    target_include_directories(common BEFORE PRIVATE ${CCTZ_INCLUDE_DIR})
+endif()
+
 target_include_directories (common PUBLIC ${COMMON_INCLUDE_DIR})
 
 if (NOT USE_INTERNAL_BOOST_LIBRARY)
@@ -98,12 +101,15 @@ if(NOT USE_INTERNAL_POCO_LIBRARY)
     target_include_directories (common SYSTEM BEFORE PUBLIC ${Poco_Foundation_INCLUDE_DIR})
 endif()
 
+if(CCTZ_LIBRARY)
+    target_link_libraries(common PRIVATE ${CCTZ_LIBRARY})
+endif()
+
 target_link_libraries (common
         PUBLIC
     ${Poco_Foundation_LIBRARY}
     ${CITYHASH_LIBRARIES}
         PRIVATE
-    ${CCTZ_LIBRARY}
     ${Boost_FILESYSTEM_LIBRARY}
         PUBLIC
     ${Boost_SYSTEM_LIBRARY}

libs/libcommon/cmake/find_cctz.cmake

@@ -1,10 +1,12 @@
 option (USE_INTERNAL_CCTZ_LIBRARY "Set to FALSE to use system cctz library instead of bundled" ${NOT_UNBUNDLED})
 
-if (USE_INTERNAL_CCTZ_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cctz/include/cctz/time_zone.h")
-   message (WARNING "submodule contrib/cctz is missing. to fix try run: \n git submodule update --init --recursive")
-   set (MISSING_INTERNAL_CCTZ_LIBRARY 1)
-   set (USE_INTERNAL_CCTZ_LIBRARY 0)
-endif ()
+if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cctz/include/cctz/time_zone.h")
+    if(USE_INTERNAL_CCTZ_LIBRARY)
+       message(WARNING "submodule contrib/cctz is missing. to fix try run: \n git submodule update --init --recursive")
+    endif()
+    set(USE_INTERNAL_CCTZ_LIBRARY 0)
+    set(MISSING_INTERNAL_CCTZ_LIBRARY 1)
+endif()
 
 if (NOT USE_INTERNAL_CCTZ_LIBRARY)
     find_library (CCTZ_LIBRARY cctz)

utils/compressor/CMakeLists.txt

@@ -1,7 +1,10 @@
 find_package (Threads)
 
 add_executable (zstd_test zstd_test.cpp)
-target_link_libraries (zstd_test PRIVATE ${ZSTD_LIBRARY} common Threads::Threads)
+if(ZSTD_LIBRARY)
+    target_link_libraries(zstd_test PRIVATE ${ZSTD_LIBRARY})
+endif()
+target_link_libraries (zstd_test PRIVATE common Threads::Threads)
 
 add_executable (mutator mutator.cpp)
 target_link_libraries(mutator PRIVATE clickhouse_common_io)
@@ -9,6 +12,6 @@ target_link_libraries(mutator PRIVATE clickhouse_common_io)
 add_executable (decompress_perf decompress_perf.cpp)
 target_link_libraries(decompress_perf PRIVATE clickhouse_common_io clickhouse_compression ${LZ4_LIBRARY})
 
-if (NOT USE_INTERNAL_ZSTD_LIBRARY)
+if (NOT USE_INTERNAL_ZSTD_LIBRARY AND ZSTD_INCLUDE_DIR)
     target_include_directories (zstd_test BEFORE PRIVATE ${ZSTD_INCLUDE_DIR})
 endif ()