Remove unbundled librdkafka/cppkafka support

2024-11-23 16:12:01 +00:00 · 2022-01-17 08:21:56 +03:00 · 2022-01-17 08:21:56 +03:00 · 61b781971c
commit 61b781971c
parent 313716be94
7 changed files with 48 additions and 114 deletions
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -501,7 +501,6 @@ include (cmake/find/re2.cmake)
 include (cmake/find/krb5.cmake)
 include (cmake/find/libgsasl.cmake)
 include (cmake/find/cyrus-sasl.cmake)
-include (cmake/find/rdkafka.cmake)
 include (cmake/find/libuv.cmake) # for amqpcpp and cassandra
 include (cmake/find/amqpcpp.cmake)
 include (cmake/find/capnp.cmake)
--- a/cmake/find/rdkafka.cmake
+++ b/cmake/find/rdkafka.cmake
@ -1,68 +0,0 @@
-option (ENABLE_RDKAFKA "Enable kafka" ${ENABLE_LIBRARIES})
-
-if (NOT ENABLE_RDKAFKA)
-    if (USE_INTERNAL_RDKAFKA_LIBRARY)
-        message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal librdkafka with ENABLE_RDKAFKA=OFF")
-    endif()
-    return()
-endif()
-
-option (USE_INTERNAL_RDKAFKA_LIBRARY "Set to FALSE to use system librdkafka instead of the bundled" ON)
-
-if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cppkafka/src")
-    if(USE_INTERNAL_RDKAFKA_LIBRARY)
-        message (WARNING "submodule contrib/cppkafka is missing. to fix try run: \n git submodule update --init")
-        message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal cppkafka")
-        set (USE_INTERNAL_RDKAFKA_LIBRARY 0)
-    endif()
-    set (MISSING_INTERNAL_CPPKAFKA_LIBRARY 1)
-endif ()
-
-if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/librdkafka/src")
-    if(USE_INTERNAL_RDKAFKA_LIBRARY OR MISSING_INTERNAL_CPPKAFKA_LIBRARY)
-        message (WARNING "submodule contrib/librdkafka is missing. to fix try run: \n git submodule update --init")
-        message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal rdkafka")
-        set (USE_INTERNAL_RDKAFKA_LIBRARY 0)
-    endif()
-    set (MISSING_INTERNAL_RDKAFKA_LIBRARY 1)
-endif ()
-
-if (NOT USE_INTERNAL_RDKAFKA_LIBRARY)
-    find_library (RDKAFKA_LIB rdkafka)
-    find_path (RDKAFKA_INCLUDE_DIR NAMES librdkafka/rdkafka.h PATHS ${RDKAFKA_INCLUDE_PATHS})
-    if (NOT RDKAFKA_LIB OR NOT RDKAFKA_INCLUDE_DIR)
-        message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system librdkafka")
-    endif()
-
-    if (USE_STATIC_LIBRARIES AND NOT OS_FREEBSD)
-       find_library (SASL2_LIBRARY sasl2)
-       if (NOT SASL2_LIBRARY)
-           message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system sasl2 library needed for static librdkafka")
-       endif()
-    endif ()
-    set (CPPKAFKA_LIBRARY cppkafka)
-endif ()
-
-if (RDKAFKA_LIB AND RDKAFKA_INCLUDE_DIR)
-    set (USE_RDKAFKA 1)
-    add_library (rdkafka_imp UNKNOWN IMPORTED)
-    set_target_properties (rdkafka_imp PROPERTIES IMPORTED_LOCATION ${RDKAFKA_LIB})
-    set_target_properties (rdkafka_imp PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${RDKAFKA_INCLUDE_DIR})
-
-    set (RDKAFKA_LIBRARY rdkafka_imp ${OPENSSL_LIBRARIES})
-    set (CPPKAFKA_LIBRARY cppkafka)
-    if (SASL2_LIBRARY)
-       list (APPEND RDKAFKA_LIBRARY ${SASL2_LIBRARY})
-    endif ()
-    if (LZ4_LIBRARY)
-       list (APPEND RDKAFKA_LIBRARY ${LZ4_LIBRARY})
-    endif ()
-elseif (NOT MISSING_INTERNAL_RDKAFKA_LIBRARY AND NOT MISSING_INTERNAL_CPPKAFKA_LIBRARY)
-    set (USE_INTERNAL_RDKAFKA_LIBRARY 1)
-    set (RDKAFKA_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/librdkafka/src")
-    set (RDKAFKA_LIBRARY rdkafka)
-    set (CPPKAFKA_LIBRARY cppkafka)
-    set (USE_RDKAFKA 1)
-endif ()
-
-message (STATUS "Using librdkafka=${USE_RDKAFKA}: ${RDKAFKA_INCLUDE_DIR} : ${RDKAFKA_LIBRARY} ${CPPKAFKA_LIBRARY}")
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@ -66,6 +66,8 @@ add_subdirectory (thrift-cmake)
 add_subdirectory (arrow-cmake) # requires: snappy, thrift
 add_subdirectory (avro-cmake) # requires: snappy
 add_subdirectory (hive-metastore-cmake) # requires: thrift/avro/arrow
+add_subdirectory (librdkafka-cmake)
+add_subdirectory (cppkafka-cmake)

 # TODO: refactor the contrib libraries below this comment.

@ -104,17 +106,6 @@ if (USE_INTERNAL_MYSQL_LIBRARY)
    add_subdirectory (mariadb-connector-c-cmake)
 endif ()

-if (USE_INTERNAL_RDKAFKA_LIBRARY)
-    add_subdirectory (librdkafka-cmake)
-    if(OPENSSL_INCLUDE_DIR)
-        target_include_directories(rdkafka BEFORE PRIVATE ${OPENSSL_INCLUDE_DIR})
-    endif()
-endif ()
-
-if (USE_RDKAFKA)
-    add_subdirectory (cppkafka-cmake)
-endif()
-
 if (ENABLE_ICU AND USE_INTERNAL_ICU_LIBRARY)
    add_subdirectory (icu-cmake)
 endif ()
--- a/contrib/cppkafka-cmake/CMakeLists.txt
+++ b/contrib/cppkafka-cmake/CMakeLists.txt
@ -1,3 +1,8 @@
+if (NOT ENABLE_KAFKA)
+    message(STATUS "Not using librdkafka (skip cppkafka)")
+    return()
+endif()
+
 set(LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/cppkafka")

 set(SRCS
@ -22,12 +27,13 @@ set(SRCS
    "${LIBRARY_DIR}/src/topic.cpp"
 )

-add_library(cppkafka ${SRCS})
+add_library(_cppkafka ${SRCS})
+add_library(ch_contrib::cppkafka ALIAS _cppkafka)

-target_link_libraries(cppkafka
+target_link_libraries(_cppkafka
    PRIVATE
-        ${RDKAFKA_LIBRARY}
+        ch_contrib::rdkafka
        boost::headers_only
 )
-target_include_directories(cppkafka PRIVATE "${LIBRARY_DIR}/include/cppkafka")
-target_include_directories(cppkafka SYSTEM BEFORE PUBLIC "${LIBRARY_DIR}/include")
+target_include_directories(_cppkafka PRIVATE "${LIBRARY_DIR}/include/cppkafka")
+target_include_directories(_cppkafka SYSTEM BEFORE PUBLIC "${LIBRARY_DIR}/include")
--- a/contrib/librdkafka-cmake/CMakeLists.txt
+++ b/contrib/librdkafka-cmake/CMakeLists.txt
@ -1,3 +1,10 @@
+option (ENABLE_KAFKA "Enable kafka" ${ENABLE_LIBRARIES})
+
+if (NOT ENABLE_KAFKA)
+    message(STATUS "Not using librdkafka")
+    return()
+endif()
+
 set(RDKAFKA_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/librdkafka/src")

 set(SRCS
@ -85,19 +92,13 @@ if(${ENABLE_CYRUS_SASL})
    set(WITH_SASL_CYRUS 1)
 endif()

-if(OPENSSL_FOUND)
-    message (STATUS "librdkafka with SSL support")
-    set(WITH_SSL 1)
-
-    if(${ENABLE_CYRUS_SASL})
-       set(WITH_SASL_SCRAM 1)
-       set(WITH_SASL_OAUTHBEARER 1)
-    endif()
-endif()
-
-if(WITH_SSL)
-  list(APPEND SRCS "${RDKAFKA_SOURCE_DIR}/rdkafka_ssl.c")
+message (STATUS "librdkafka with SSL support")
+set(WITH_SSL 1)
+if(${ENABLE_CYRUS_SASL})
+   set(WITH_SASL_SCRAM 1)
+   set(WITH_SASL_OAUTHBEARER 1)
 endif()
+list(APPEND SRCS "${RDKAFKA_SOURCE_DIR}/rdkafka_ssl.c")

 if(WITH_SASL_CYRUS)
  list(APPEND SRCS "${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_cyrus.c") # needed to support Kerberos, requires cyrus-sasl
@ -111,18 +112,23 @@ if(WITH_SASL_OAUTHBEARER)
  list(APPEND SRCS "${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_oauthbearer.c")
 endif()

-add_library(rdkafka ${SRCS})
-target_compile_options(rdkafka PRIVATE -fno-sanitize=undefined)
-# target_include_directories(rdkafka SYSTEM PUBLIC include)
-target_include_directories(rdkafka SYSTEM PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include") # for "librdkafka/rdkafka.h"
-target_include_directories(rdkafka SYSTEM PUBLIC ${RDKAFKA_SOURCE_DIR})                 # Because weird logic with "include_next" is used.
-target_include_directories(rdkafka SYSTEM PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/auxdir")  # for "../config.h"
-target_link_libraries(rdkafka PRIVATE lz4 ch_contrib::zlib ch_contrib::zstd)
-if(OPENSSL_SSL_LIBRARY AND OPENSSL_CRYPTO_LIBRARY)
-    target_link_libraries(rdkafka PRIVATE ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY})
-endif()
+add_library(_rdkafka ${SRCS})
+add_library(ch_contrib::rdkafka ALIAS _rdkafka)
+
+target_compile_options(_rdkafka PRIVATE -fno-sanitize=undefined)
+# target_include_directories(_rdkafka SYSTEM PUBLIC include)
+target_include_directories(_rdkafka SYSTEM PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include") # for "librdkafka/rdkafka.h"
+target_include_directories(_rdkafka SYSTEM PUBLIC ${RDKAFKA_SOURCE_DIR})                 # Because weird logic with "include_next" is used.
+target_include_directories(_rdkafka SYSTEM PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/auxdir")  # for "../config.h"
+target_link_libraries(_rdkafka
+  PRIVATE
+    lz4
+    ch_contrib::zlib
+    ch_contrib::zstd
+    OpenSSL::Crypto OpenSSL::SSL
+)
 if(${ENABLE_CYRUS_SASL})
-    target_link_libraries(rdkafka PRIVATE ${CYRUS_SASL_LIBRARY})
+    target_link_libraries(_rdkafka PRIVATE ${CYRUS_SASL_LIBRARY})
 endif()

 file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/auxdir")
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@ -87,7 +87,7 @@ if (USE_SQLITE)
    add_headers_and_sources(dbms Databases/SQLite)
 endif()

-if(USE_RDKAFKA)
+if (TARGET ch_contrib::rdkafka)
    add_headers_and_sources(dbms Storages/Kafka)
 endif()

@ -339,11 +339,8 @@ target_link_libraries(clickhouse_common_io
 # That way we we can redirect malloc/free functions avoiding circular dependencies
 dbms_target_link_libraries(PUBLIC roaring)

-if (USE_RDKAFKA)
-    dbms_target_link_libraries(PRIVATE ${CPPKAFKA_LIBRARY} ${RDKAFKA_LIBRARY})
-    if(NOT USE_INTERNAL_RDKAFKA_LIBRARY)
-        dbms_target_include_directories(SYSTEM BEFORE PRIVATE ${RDKAFKA_INCLUDE_DIR})
-    endif()
+if (TARGET ch_contrib::rdkafka)
+    dbms_target_link_libraries(PRIVATE ch_contrib::rdkafka ch_contrib::cppkafka)
 endif()

 if (USE_CYRUS_SASL)
--- a/src/configure_config.cmake
+++ b/src/configure_config.cmake
@ -10,3 +10,6 @@ endif()
 if (TARGET ch_contrib::hivemetastore)
    set(USE_HIVE 1)
 endif()
+if (TARGET ch_contrib::rdkafka)
+    set(USE_RDKAFKA 1)
+endif()