diff --git a/contrib/librdkafka b/contrib/librdkafka index 39d4ed49ccf..806991401e3 160000 --- a/contrib/librdkafka +++ b/contrib/librdkafka @@ -1 +1 @@ -Subproject commit 39d4ed49ccf3406e2bf825d5d7b0903b5a290782 +Subproject commit 806991401e3622823665a6e771c5a618229263b5 diff --git a/contrib/librdkafka-cmake/CMakeLists.txt b/contrib/librdkafka-cmake/CMakeLists.txt index d84abd06dec..037e6807ead 100644 --- a/contrib/librdkafka-cmake/CMakeLists.txt +++ b/contrib/librdkafka-cmake/CMakeLists.txt @@ -8,26 +8,29 @@ endif() set(RDKAFKA_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/librdkafka/src") set(SRCS +# "${RDKAFKA_SOURCE_DIR}/cJSON.c" include for future versions "${RDKAFKA_SOURCE_DIR}/crc32c.c" # "${RDKAFKA_SOURCE_DIR}/lz4.c" # "${RDKAFKA_SOURCE_DIR}/lz4frame.c" # "${RDKAFKA_SOURCE_DIR}/lz4hc.c" "${RDKAFKA_SOURCE_DIR}/rdaddr.c" "${RDKAFKA_SOURCE_DIR}/rdavl.c" + # "${RDKAFKA_SOURCE_DIR}/rdbase64.c" include for future (2.x.x) versions "${RDKAFKA_SOURCE_DIR}/rdbuf.c" "${RDKAFKA_SOURCE_DIR}/rdcrc32.c" "${RDKAFKA_SOURCE_DIR}/rddl.c" "${RDKAFKA_SOURCE_DIR}/rdfnv1a.c" "${RDKAFKA_SOURCE_DIR}/rdgz.c" "${RDKAFKA_SOURCE_DIR}/rdhdrhistogram.c" - "${RDKAFKA_SOURCE_DIR}/rdkafka_admin.c" # looks optional +# "${RDKAFKA_SOURCE_DIR}/rdhttp.c" include for future versions + "${RDKAFKA_SOURCE_DIR}/rdkafka.c" + "${RDKAFKA_SOURCE_DIR}/rdkafka_admin.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_assignment.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_assignor.c" - "${RDKAFKA_SOURCE_DIR}/rdkafka_aux.c" # looks optional + "${RDKAFKA_SOURCE_DIR}/rdkafka_aux.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_background.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_broker.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_buf.c" - "${RDKAFKA_SOURCE_DIR}/rdkafka.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_cert.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_cgrp.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_conf.c" @@ -35,6 +38,7 @@ set(SRCS "${RDKAFKA_SOURCE_DIR}/rdkafka_error.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_event.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_feature.c" + # "${RDKAFKA_SOURCE_DIR}/rdkafka_fetcher.c" include for future (2.x.x) versions "${RDKAFKA_SOURCE_DIR}/rdkafka_header.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_idempotence.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_interceptor.c" @@ -57,14 +61,18 @@ set(SRCS "${RDKAFKA_SOURCE_DIR}/rdkafka_request.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_roundrobin_assignor.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_sasl.c" -# "${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_cyrus.c" # optionally included below -# "${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_oauthbearer.c" # optionally included below +# "${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_cyrus.c" # optionally included below +# "${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_oauthbearer.c" # optionally included below +# "${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_oauthbearer_oidc.c" # optionally included below "${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_plain.c" -# "${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_scram.c" # optionally included below +# "${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_scram.c" # optionally included below # "${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_win32.c" -# "${RDKAFKA_SOURCE_DIR}/rdkafka_ssl.c" # optionally included below +# "${RDKAFKA_SOURCE_DIR}/rdkafka_ssl.c" # optionally included below "${RDKAFKA_SOURCE_DIR}/rdkafka_sticky_assignor.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_subscription.c" + # "${RDKAFKA_SOURCE_DIR}/rdkafka_telemetry.c" include for future (2.x.x) versions + # "${RDKAFKA_SOURCE_DIR}/rdkafka_telemetry_decode.c" include for future (2.x.x) versions + # "${RDKAFKA_SOURCE_DIR}/rdkafka_telemetry_encode.c" include for future (2.x.x) versions "${RDKAFKA_SOURCE_DIR}/rdkafka_timer.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_topic.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_transport.c" @@ -81,10 +89,16 @@ set(SRCS "${RDKAFKA_SOURCE_DIR}/rdunittest.c" "${RDKAFKA_SOURCE_DIR}/rdvarint.c" "${RDKAFKA_SOURCE_DIR}/rdxxhash.c" - # "${RDKAFKA_SOURCE_DIR}/regexp.c" +# "${RDKAFKA_SOURCE_DIR}/regexp.c" "${RDKAFKA_SOURCE_DIR}/snappy.c" "${RDKAFKA_SOURCE_DIR}/tinycthread.c" "${RDKAFKA_SOURCE_DIR}/tinycthread_extra.c" + # "${RDKAFKA_SOURCE_DIR}/nanopb/pb_common.c" include for future versions + # "${RDKAFKA_SOURCE_DIR}/nanopb/pb_decode.c" include for future versions + # "${RDKAFKA_SOURCE_DIR}/nanopb/pb_encode.c" include for future versions + # "${RDKAFKA_SOURCE_DIR}/opentelemetry/common.pb.c" include for future versions + # "${RDKAFKA_SOURCE_DIR}/opentelemetry/metrics.pb.c" include for future versions + # "${RDKAFKA_SOURCE_DIR}/opentelemetry/resource.pb.c" include for future versions ) if(TARGET ch_contrib::sasl2) @@ -92,6 +106,11 @@ if(TARGET ch_contrib::sasl2) set(WITH_SASL_CYRUS 1) endif() +if(TARGET ch_contrib::curl) + message (STATUS "librdkafka with curl") + set(WITH_CURL 1) +endif() + message (STATUS "librdkafka with SSL support") set(WITH_SSL 1) if(WITH_SASL_CYRUS) @@ -110,6 +129,7 @@ endif() if(WITH_SASL_OAUTHBEARER) list(APPEND SRCS "${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_oauthbearer.c") +# list(APPEND SRCS "${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_oauthbearer_oidc.c") include for future versions endif() add_library(_rdkafka ${SRCS}) @@ -128,7 +148,11 @@ target_link_libraries(_rdkafka OpenSSL::Crypto OpenSSL::SSL ) if(WITH_SASL_CYRUS) - target_link_libraries(_rdkafka PRIVATE ch_contrib::sasl2) + target_link_libraries(_rdkafka PRIVATE ch_contrib::sasl2) +endif() + +if(WITH_CURL) + target_link_libraries(_rdkafka PRIVATE ch_contrib::curl) endif() file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/auxdir") diff --git a/contrib/librdkafka-cmake/config.h.in b/contrib/librdkafka-cmake/config.h.in index f6ec3bc0e79..a33f64b727a 100644 --- a/contrib/librdkafka-cmake/config.h.in +++ b/contrib/librdkafka-cmake/config.h.in @@ -1,33 +1,66 @@ -// Originally generated by ./configure +// Automatically generated by ./configure +/// NOTE: Version 2.5.3 was used to generate and manually modified after that. Therefore this should be fine +/// until we upgrade to something newer than that. +/// Commented out the followings: +/// - #define ARCH "x86_64": we build on multiple archs +/// - ENABLE_XXX: the commented out ones are only used in librdkafka's configure and CMake, but not in source +/// NOTE: ENABLE_DEVEL and ENABLE_REFCNT_DEBUG is used in the source at the time of writing +/// +/// Commented out the following to disable them: +/// - ENABLE_C11THREADS: to maintain compatibility with old libc, maybe not necessary anymore +/// - RDKAFKA_VERSION_STR, MKL_APP_VERSION: nothing depends on them currently, so make sure nothing starts +// to depend on them as it is easy to forget to update it in the future +/// - WITH_GCC +/// - WITH_GXX +/// - WITH_INSTALL +/// - HAS_GNU_AR +/// - HAVE_PIC +/// - WITH_GNULD +/// - WITH_C11THREADS +/// - HAVE_PYTHON +/// +/// Modified the follwoings: +/// - WITH_{CURL,SASL_CYRUS,SASL_OAUTHBEARER,SASL_SCRAM}: made them CMake dependant +/// - BUILT_WITH: tried to make some sense of it, don't spend to much time on it +/// +/// Added: +/// - special handling of __APPLE__ + #ifndef _CONFIG_H_ #define _CONFIG_H_ -#define BUILT_WITH "GCC GXX PKGCONFIG OSXLD LIBDL PLUGINS ZLIB SSL SASL_CYRUS ZSTD HDRHISTOGRAM LZ4_EXT SNAPPY SOCKEM SASL_SCRAM CRC32C_HW" - -#define CPU "generic" -#define WITHOUT_OPTIMIZATION 0 -#define ENABLE_DEVEL 0 -#define ENABLE_VALGRIND 0 -#define ENABLE_REFCNT_DEBUG 0 -#define ENABLE_SHAREDPTR_DEBUG 0 -#define ENABLE_LZ4_EXT 1 -#define ENABLE_SSL 1 -#define ENABLE_SASL 1 -#define MKL_APP_NAME "librdkafka" -#define MKL_APP_DESC_ONELINE "The Apache Kafka C/C++ library" // distro #define SOLIB_EXT ".so" +// #define ARCH "x86_64" +#define CPU "generic" +#define WITHOUT_OPTIMIZATION 0 +#define WITH_STRIP 0 +// #define ENABLE_ZLIB "try" +// #define ENABLE_ZSTD "try" +// #define ENABLE_SSL "try" +// #define ENABLE_GSSAPI "try" +// #define ENABLE_CURL "try" +#define ENABLE_DEVEL 0 +// #define ENABLE_VALGRIND 0 +#define ENABLE_REFCNT_DEBUG 0 +// #define ENABLE_LZ4_EXT 1 +// #define ENABLE_LZ4_EXT 1 +// #define ENABLE_REGEX_EXT 1 +// #define ENABLE_C11THREADS "try" +// #define ENABLE_SYSLOG 0 +#define MKL_APP_NAME "librdkafka" +#define MKL_APP_DESC_ONELINE "The Apache Kafka C/C++ library" // gcc -//#define WITH_GCC 1 +// #define WITH_GCC 1 // gxx -//#define WITH_GXX 1 -// pkgconfig -//#define WITH_PKGCONFIG 1 +// #define WITH_GXX 1 // install -//#define WITH_INSTALL 1 +// #define WITH_INSTALL 1 +// gnuar +// #define HAS_GNU_AR 1 // PIC -//#define HAVE_PIC 1 +// #define HAVE_PIC 1 // gnulib -//#define WITH_GNULD 1 +// #define WITH_GNULD 1 // __atomic_32 #define HAVE_ATOMICS_32 1 // __atomic_32 @@ -43,9 +76,11 @@ // atomic_64 #define ATOMIC_OP(OP1,OP2,PTR,VAL) __atomic_ ## OP1 ## _ ## OP2(PTR, VAL, __ATOMIC_SEQ_CST) // parseversion -#define RDKAFKA_VERSION_STR "1.6.0" +// #define RDKAFKA_VERSION_STR "1.7.0" // parseversion -#define MKL_APP_VERSION "1.6.0" +// #define MKL_APP_VERSION "1.7.0" +// c11threads +// #define WITH_C11THREADS 1 // libdl #define WITH_LIBDL 1 // WITH_PLUGINS @@ -54,32 +89,41 @@ #define WITH_ZLIB 1 // zstd #define WITH_ZSTD 1 +// libssl +#cmakedefine WITH_SSL 1 +// libcrypto +#define OPENSSL_SUPPRESS_DEPRECATED "OPENSSL_SUPPRESS_DEPRECATED" +// WITH_HDRHISTOGRAM +#define WITH_HDRHISTOGRAM 1 +// syslog +#define WITH_SYSLOG 0 // WITH_SNAPPY #define WITH_SNAPPY 1 // WITH_SOCKEM #define WITH_SOCKEM 1 -// libssl -#cmakedefine WITH_SSL 1 // WITH_SASL_SCRAM #cmakedefine WITH_SASL_SCRAM 1 // WITH_SASL_OAUTHBEARER #cmakedefine WITH_SASL_OAUTHBEARER 1 +// WITH_SASL_CYRUS #cmakedefine WITH_SASL_CYRUS 1 +// WITH_CURL +#cmakedefine WITH_CURL 1 // crc32chw #if !defined(__PPC__) && !defined(__riscv) && !defined(__aarch64__) && !defined(__s390x__) && !defined(__loongarch64) #define WITH_CRC32C_HW 1 #endif // regex #define HAVE_REGEX 1 +// rand_r +#define HAVE_RAND_R 1 // strndup #define HAVE_STRNDUP 1 // strerror_r #define HAVE_STRERROR_R 1 -// rand_r -#define HAVE_RAND_R 1 - +// strcasestr +#define HAVE_STRCASESTR 1 #ifdef __APPLE__ -// pthread_setname_np #define HAVE_PTHREAD_SETNAME_DARWIN 1 #if (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ <= 101400) #define _TTHREAD_EMULATE_TIMESPEC_GET_ @@ -90,8 +134,10 @@ // pthread_setname_gnu #define HAVE_PTHREAD_SETNAME_GNU 1 #endif -// python -//#define HAVE_PYTHON 1 -// disable C11 threads for compatibility with old libc -//#define WITH_C11THREADS 1 +// python3 +// #define HAVE_PYTHON 1 +// getrusage +#define HAVE_GETRUSAGE 1 +// BUILT_WITH +#define BUILT_WITH "GCC GXX INSTALL GNULD LDS LIBDL PLUGINS ZLIB SSL SASL_CYRUS ZSTD HDRHISTOGRAM LZ4_EXT SNAPPY SOCKEM SASL_SCRAM SASL_OAUTHBEARER CRC32C_HW" #endif /* _CONFIG_H_ */ diff --git a/tests/integration/test_storage_kafka/configs/kafka.xml b/tests/integration/test_storage_kafka/configs/kafka.xml index a846fdbb295..5c25f995e6b 100644 --- a/tests/integration/test_storage_kafka/configs/kafka.xml +++ b/tests/integration/test_storage_kafka/configs/kafka.xml @@ -12,6 +12,11 @@ 600 + + + + + 10000 consumer_hang @@ -28,12 +33,12 @@ consumer_hang - + 6000 separate_settings - + 6001