From 61b781971cb9e97879ac4584e5d82a9302908bdf Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 08:21:56 +0300 Subject: [PATCH] Remove unbundled librdkafka/cppkafka support --- CMakeLists.txt | 1 - cmake/find/rdkafka.cmake | 68 ------------------------- contrib/CMakeLists.txt | 13 +---- contrib/cppkafka-cmake/CMakeLists.txt | 16 ++++-- contrib/librdkafka-cmake/CMakeLists.txt | 52 ++++++++++--------- src/CMakeLists.txt | 9 ++-- src/configure_config.cmake | 3 ++ 7 files changed, 48 insertions(+), 114 deletions(-) delete mode 100644 cmake/find/rdkafka.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 3c424b07584..88365c70d5f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -501,7 +501,6 @@ include (cmake/find/re2.cmake) include (cmake/find/krb5.cmake) include (cmake/find/libgsasl.cmake) include (cmake/find/cyrus-sasl.cmake) -include (cmake/find/rdkafka.cmake) include (cmake/find/libuv.cmake) # for amqpcpp and cassandra include (cmake/find/amqpcpp.cmake) include (cmake/find/capnp.cmake) diff --git a/cmake/find/rdkafka.cmake b/cmake/find/rdkafka.cmake deleted file mode 100644 index cad267bacff..00000000000 --- a/cmake/find/rdkafka.cmake +++ /dev/null @@ -1,68 +0,0 @@ -option (ENABLE_RDKAFKA "Enable kafka" ${ENABLE_LIBRARIES}) - -if (NOT ENABLE_RDKAFKA) - if (USE_INTERNAL_RDKAFKA_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal librdkafka with ENABLE_RDKAFKA=OFF") - endif() - return() -endif() - -option (USE_INTERNAL_RDKAFKA_LIBRARY "Set to FALSE to use system librdkafka instead of the bundled" ON) - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cppkafka/src") - if(USE_INTERNAL_RDKAFKA_LIBRARY) - message (WARNING "submodule contrib/cppkafka is missing. to fix try run: \n git submodule update --init") - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal cppkafka") - set (USE_INTERNAL_RDKAFKA_LIBRARY 0) - endif() - set (MISSING_INTERNAL_CPPKAFKA_LIBRARY 1) -endif () - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/librdkafka/src") - if(USE_INTERNAL_RDKAFKA_LIBRARY OR MISSING_INTERNAL_CPPKAFKA_LIBRARY) - message (WARNING "submodule contrib/librdkafka is missing. to fix try run: \n git submodule update --init") - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal rdkafka") - set (USE_INTERNAL_RDKAFKA_LIBRARY 0) - endif() - set (MISSING_INTERNAL_RDKAFKA_LIBRARY 1) -endif () - -if (NOT USE_INTERNAL_RDKAFKA_LIBRARY) - find_library (RDKAFKA_LIB rdkafka) - find_path (RDKAFKA_INCLUDE_DIR NAMES librdkafka/rdkafka.h PATHS ${RDKAFKA_INCLUDE_PATHS}) - if (NOT RDKAFKA_LIB OR NOT RDKAFKA_INCLUDE_DIR) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system librdkafka") - endif() - - if (USE_STATIC_LIBRARIES AND NOT OS_FREEBSD) - find_library (SASL2_LIBRARY sasl2) - if (NOT SASL2_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system sasl2 library needed for static librdkafka") - endif() - endif () - set (CPPKAFKA_LIBRARY cppkafka) -endif () - -if (RDKAFKA_LIB AND RDKAFKA_INCLUDE_DIR) - set (USE_RDKAFKA 1) - add_library (rdkafka_imp UNKNOWN IMPORTED) - set_target_properties (rdkafka_imp PROPERTIES IMPORTED_LOCATION ${RDKAFKA_LIB}) - set_target_properties (rdkafka_imp PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${RDKAFKA_INCLUDE_DIR}) - - set (RDKAFKA_LIBRARY rdkafka_imp ${OPENSSL_LIBRARIES}) - set (CPPKAFKA_LIBRARY cppkafka) - if (SASL2_LIBRARY) - list (APPEND RDKAFKA_LIBRARY ${SASL2_LIBRARY}) - endif () - if (LZ4_LIBRARY) - list (APPEND RDKAFKA_LIBRARY ${LZ4_LIBRARY}) - endif () -elseif (NOT MISSING_INTERNAL_RDKAFKA_LIBRARY AND NOT MISSING_INTERNAL_CPPKAFKA_LIBRARY) - set (USE_INTERNAL_RDKAFKA_LIBRARY 1) - set (RDKAFKA_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/librdkafka/src") - set (RDKAFKA_LIBRARY rdkafka) - set (CPPKAFKA_LIBRARY cppkafka) - set (USE_RDKAFKA 1) -endif () - -message (STATUS "Using librdkafka=${USE_RDKAFKA}: ${RDKAFKA_INCLUDE_DIR} : ${RDKAFKA_LIBRARY} ${CPPKAFKA_LIBRARY}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index a7c06f31717..8b8f74bb698 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -66,6 +66,8 @@ add_subdirectory (thrift-cmake) add_subdirectory (arrow-cmake) # requires: snappy, thrift add_subdirectory (avro-cmake) # requires: snappy add_subdirectory (hive-metastore-cmake) # requires: thrift/avro/arrow +add_subdirectory (librdkafka-cmake) +add_subdirectory (cppkafka-cmake) # TODO: refactor the contrib libraries below this comment. @@ -104,17 +106,6 @@ if (USE_INTERNAL_MYSQL_LIBRARY) add_subdirectory (mariadb-connector-c-cmake) endif () -if (USE_INTERNAL_RDKAFKA_LIBRARY) - add_subdirectory (librdkafka-cmake) - if(OPENSSL_INCLUDE_DIR) - target_include_directories(rdkafka BEFORE PRIVATE ${OPENSSL_INCLUDE_DIR}) - endif() -endif () - -if (USE_RDKAFKA) - add_subdirectory (cppkafka-cmake) -endif() - if (ENABLE_ICU AND USE_INTERNAL_ICU_LIBRARY) add_subdirectory (icu-cmake) endif () diff --git a/contrib/cppkafka-cmake/CMakeLists.txt b/contrib/cppkafka-cmake/CMakeLists.txt index 0bc33ada529..87bf2356a80 100644 --- a/contrib/cppkafka-cmake/CMakeLists.txt +++ b/contrib/cppkafka-cmake/CMakeLists.txt @@ -1,3 +1,8 @@ +if (NOT ENABLE_KAFKA) + message(STATUS "Not using librdkafka (skip cppkafka)") + return() +endif() + set(LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/cppkafka") set(SRCS @@ -22,12 +27,13 @@ set(SRCS "${LIBRARY_DIR}/src/topic.cpp" ) -add_library(cppkafka ${SRCS}) +add_library(_cppkafka ${SRCS}) +add_library(ch_contrib::cppkafka ALIAS _cppkafka) -target_link_libraries(cppkafka +target_link_libraries(_cppkafka PRIVATE - ${RDKAFKA_LIBRARY} + ch_contrib::rdkafka boost::headers_only ) -target_include_directories(cppkafka PRIVATE "${LIBRARY_DIR}/include/cppkafka") -target_include_directories(cppkafka SYSTEM BEFORE PUBLIC "${LIBRARY_DIR}/include") +target_include_directories(_cppkafka PRIVATE "${LIBRARY_DIR}/include/cppkafka") +target_include_directories(_cppkafka SYSTEM BEFORE PUBLIC "${LIBRARY_DIR}/include") diff --git a/contrib/librdkafka-cmake/CMakeLists.txt b/contrib/librdkafka-cmake/CMakeLists.txt index 9fb69a49e87..0505d06d2fc 100644 --- a/contrib/librdkafka-cmake/CMakeLists.txt +++ b/contrib/librdkafka-cmake/CMakeLists.txt @@ -1,3 +1,10 @@ +option (ENABLE_KAFKA "Enable kafka" ${ENABLE_LIBRARIES}) + +if (NOT ENABLE_KAFKA) + message(STATUS "Not using librdkafka") + return() +endif() + set(RDKAFKA_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/librdkafka/src") set(SRCS @@ -85,19 +92,13 @@ if(${ENABLE_CYRUS_SASL}) set(WITH_SASL_CYRUS 1) endif() -if(OPENSSL_FOUND) - message (STATUS "librdkafka with SSL support") - set(WITH_SSL 1) - - if(${ENABLE_CYRUS_SASL}) - set(WITH_SASL_SCRAM 1) - set(WITH_SASL_OAUTHBEARER 1) - endif() -endif() - -if(WITH_SSL) - list(APPEND SRCS "${RDKAFKA_SOURCE_DIR}/rdkafka_ssl.c") +message (STATUS "librdkafka with SSL support") +set(WITH_SSL 1) +if(${ENABLE_CYRUS_SASL}) + set(WITH_SASL_SCRAM 1) + set(WITH_SASL_OAUTHBEARER 1) endif() +list(APPEND SRCS "${RDKAFKA_SOURCE_DIR}/rdkafka_ssl.c") if(WITH_SASL_CYRUS) list(APPEND SRCS "${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_cyrus.c") # needed to support Kerberos, requires cyrus-sasl @@ -111,18 +112,23 @@ if(WITH_SASL_OAUTHBEARER) list(APPEND SRCS "${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_oauthbearer.c") endif() -add_library(rdkafka ${SRCS}) -target_compile_options(rdkafka PRIVATE -fno-sanitize=undefined) -# target_include_directories(rdkafka SYSTEM PUBLIC include) -target_include_directories(rdkafka SYSTEM PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include") # for "librdkafka/rdkafka.h" -target_include_directories(rdkafka SYSTEM PUBLIC ${RDKAFKA_SOURCE_DIR}) # Because weird logic with "include_next" is used. -target_include_directories(rdkafka SYSTEM PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/auxdir") # for "../config.h" -target_link_libraries(rdkafka PRIVATE lz4 ch_contrib::zlib ch_contrib::zstd) -if(OPENSSL_SSL_LIBRARY AND OPENSSL_CRYPTO_LIBRARY) - target_link_libraries(rdkafka PRIVATE ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY}) -endif() +add_library(_rdkafka ${SRCS}) +add_library(ch_contrib::rdkafka ALIAS _rdkafka) + +target_compile_options(_rdkafka PRIVATE -fno-sanitize=undefined) +# target_include_directories(_rdkafka SYSTEM PUBLIC include) +target_include_directories(_rdkafka SYSTEM PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include") # for "librdkafka/rdkafka.h" +target_include_directories(_rdkafka SYSTEM PUBLIC ${RDKAFKA_SOURCE_DIR}) # Because weird logic with "include_next" is used. +target_include_directories(_rdkafka SYSTEM PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/auxdir") # for "../config.h" +target_link_libraries(_rdkafka + PRIVATE + lz4 + ch_contrib::zlib + ch_contrib::zstd + OpenSSL::Crypto OpenSSL::SSL +) if(${ENABLE_CYRUS_SASL}) - target_link_libraries(rdkafka PRIVATE ${CYRUS_SASL_LIBRARY}) + target_link_libraries(_rdkafka PRIVATE ${CYRUS_SASL_LIBRARY}) endif() file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/auxdir") diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4a10f90f942..97bcedf9ff6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -87,7 +87,7 @@ if (USE_SQLITE) add_headers_and_sources(dbms Databases/SQLite) endif() -if(USE_RDKAFKA) +if (TARGET ch_contrib::rdkafka) add_headers_and_sources(dbms Storages/Kafka) endif() @@ -339,11 +339,8 @@ target_link_libraries(clickhouse_common_io # That way we we can redirect malloc/free functions avoiding circular dependencies dbms_target_link_libraries(PUBLIC roaring) -if (USE_RDKAFKA) - dbms_target_link_libraries(PRIVATE ${CPPKAFKA_LIBRARY} ${RDKAFKA_LIBRARY}) - if(NOT USE_INTERNAL_RDKAFKA_LIBRARY) - dbms_target_include_directories(SYSTEM BEFORE PRIVATE ${RDKAFKA_INCLUDE_DIR}) - endif() +if (TARGET ch_contrib::rdkafka) + dbms_target_link_libraries(PRIVATE ch_contrib::rdkafka ch_contrib::cppkafka) endif() if (USE_CYRUS_SASL) diff --git a/src/configure_config.cmake b/src/configure_config.cmake index 1bac4e07fb1..2a084d328a2 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -10,3 +10,6 @@ endif() if (TARGET ch_contrib::hivemetastore) set(USE_HIVE 1) endif() +if (TARGET ch_contrib::rdkafka) + set(USE_RDKAFKA 1) +endif()