diff --git a/contrib/librdkafka b/contrib/librdkafka
index 39d4ed49ccf..806991401e3 160000
--- a/contrib/librdkafka
+++ b/contrib/librdkafka
@@ -1 +1 @@
-Subproject commit 39d4ed49ccf3406e2bf825d5d7b0903b5a290782
+Subproject commit 806991401e3622823665a6e771c5a618229263b5
diff --git a/contrib/librdkafka-cmake/CMakeLists.txt b/contrib/librdkafka-cmake/CMakeLists.txt
index d84abd06dec..037e6807ead 100644
--- a/contrib/librdkafka-cmake/CMakeLists.txt
+++ b/contrib/librdkafka-cmake/CMakeLists.txt
@@ -8,26 +8,29 @@ endif()
set(RDKAFKA_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/librdkafka/src")
set(SRCS
+# "${RDKAFKA_SOURCE_DIR}/cJSON.c" include for future versions
"${RDKAFKA_SOURCE_DIR}/crc32c.c"
# "${RDKAFKA_SOURCE_DIR}/lz4.c"
# "${RDKAFKA_SOURCE_DIR}/lz4frame.c"
# "${RDKAFKA_SOURCE_DIR}/lz4hc.c"
"${RDKAFKA_SOURCE_DIR}/rdaddr.c"
"${RDKAFKA_SOURCE_DIR}/rdavl.c"
+ # "${RDKAFKA_SOURCE_DIR}/rdbase64.c" include for future (2.x.x) versions
"${RDKAFKA_SOURCE_DIR}/rdbuf.c"
"${RDKAFKA_SOURCE_DIR}/rdcrc32.c"
"${RDKAFKA_SOURCE_DIR}/rddl.c"
"${RDKAFKA_SOURCE_DIR}/rdfnv1a.c"
"${RDKAFKA_SOURCE_DIR}/rdgz.c"
"${RDKAFKA_SOURCE_DIR}/rdhdrhistogram.c"
- "${RDKAFKA_SOURCE_DIR}/rdkafka_admin.c" # looks optional
+# "${RDKAFKA_SOURCE_DIR}/rdhttp.c" include for future versions
+ "${RDKAFKA_SOURCE_DIR}/rdkafka.c"
+ "${RDKAFKA_SOURCE_DIR}/rdkafka_admin.c"
"${RDKAFKA_SOURCE_DIR}/rdkafka_assignment.c"
"${RDKAFKA_SOURCE_DIR}/rdkafka_assignor.c"
- "${RDKAFKA_SOURCE_DIR}/rdkafka_aux.c" # looks optional
+ "${RDKAFKA_SOURCE_DIR}/rdkafka_aux.c"
"${RDKAFKA_SOURCE_DIR}/rdkafka_background.c"
"${RDKAFKA_SOURCE_DIR}/rdkafka_broker.c"
"${RDKAFKA_SOURCE_DIR}/rdkafka_buf.c"
- "${RDKAFKA_SOURCE_DIR}/rdkafka.c"
"${RDKAFKA_SOURCE_DIR}/rdkafka_cert.c"
"${RDKAFKA_SOURCE_DIR}/rdkafka_cgrp.c"
"${RDKAFKA_SOURCE_DIR}/rdkafka_conf.c"
@@ -35,6 +38,7 @@ set(SRCS
"${RDKAFKA_SOURCE_DIR}/rdkafka_error.c"
"${RDKAFKA_SOURCE_DIR}/rdkafka_event.c"
"${RDKAFKA_SOURCE_DIR}/rdkafka_feature.c"
+ # "${RDKAFKA_SOURCE_DIR}/rdkafka_fetcher.c" include for future (2.x.x) versions
"${RDKAFKA_SOURCE_DIR}/rdkafka_header.c"
"${RDKAFKA_SOURCE_DIR}/rdkafka_idempotence.c"
"${RDKAFKA_SOURCE_DIR}/rdkafka_interceptor.c"
@@ -57,14 +61,18 @@ set(SRCS
"${RDKAFKA_SOURCE_DIR}/rdkafka_request.c"
"${RDKAFKA_SOURCE_DIR}/rdkafka_roundrobin_assignor.c"
"${RDKAFKA_SOURCE_DIR}/rdkafka_sasl.c"
-# "${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_cyrus.c" # optionally included below
-# "${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_oauthbearer.c" # optionally included below
+# "${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_cyrus.c" # optionally included below
+# "${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_oauthbearer.c" # optionally included below
+# "${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_oauthbearer_oidc.c" # optionally included below
"${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_plain.c"
-# "${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_scram.c" # optionally included below
+# "${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_scram.c" # optionally included below
# "${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_win32.c"
-# "${RDKAFKA_SOURCE_DIR}/rdkafka_ssl.c" # optionally included below
+# "${RDKAFKA_SOURCE_DIR}/rdkafka_ssl.c" # optionally included below
"${RDKAFKA_SOURCE_DIR}/rdkafka_sticky_assignor.c"
"${RDKAFKA_SOURCE_DIR}/rdkafka_subscription.c"
+ # "${RDKAFKA_SOURCE_DIR}/rdkafka_telemetry.c" include for future (2.x.x) versions
+ # "${RDKAFKA_SOURCE_DIR}/rdkafka_telemetry_decode.c" include for future (2.x.x) versions
+ # "${RDKAFKA_SOURCE_DIR}/rdkafka_telemetry_encode.c" include for future (2.x.x) versions
"${RDKAFKA_SOURCE_DIR}/rdkafka_timer.c"
"${RDKAFKA_SOURCE_DIR}/rdkafka_topic.c"
"${RDKAFKA_SOURCE_DIR}/rdkafka_transport.c"
@@ -81,10 +89,16 @@ set(SRCS
"${RDKAFKA_SOURCE_DIR}/rdunittest.c"
"${RDKAFKA_SOURCE_DIR}/rdvarint.c"
"${RDKAFKA_SOURCE_DIR}/rdxxhash.c"
- # "${RDKAFKA_SOURCE_DIR}/regexp.c"
+# "${RDKAFKA_SOURCE_DIR}/regexp.c"
"${RDKAFKA_SOURCE_DIR}/snappy.c"
"${RDKAFKA_SOURCE_DIR}/tinycthread.c"
"${RDKAFKA_SOURCE_DIR}/tinycthread_extra.c"
+ # "${RDKAFKA_SOURCE_DIR}/nanopb/pb_common.c" include for future versions
+ # "${RDKAFKA_SOURCE_DIR}/nanopb/pb_decode.c" include for future versions
+ # "${RDKAFKA_SOURCE_DIR}/nanopb/pb_encode.c" include for future versions
+ # "${RDKAFKA_SOURCE_DIR}/opentelemetry/common.pb.c" include for future versions
+ # "${RDKAFKA_SOURCE_DIR}/opentelemetry/metrics.pb.c" include for future versions
+ # "${RDKAFKA_SOURCE_DIR}/opentelemetry/resource.pb.c" include for future versions
)
if(TARGET ch_contrib::sasl2)
@@ -92,6 +106,11 @@ if(TARGET ch_contrib::sasl2)
set(WITH_SASL_CYRUS 1)
endif()
+if(TARGET ch_contrib::curl)
+ message (STATUS "librdkafka with curl")
+ set(WITH_CURL 1)
+endif()
+
message (STATUS "librdkafka with SSL support")
set(WITH_SSL 1)
if(WITH_SASL_CYRUS)
@@ -110,6 +129,7 @@ endif()
if(WITH_SASL_OAUTHBEARER)
list(APPEND SRCS "${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_oauthbearer.c")
+# list(APPEND SRCS "${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_oauthbearer_oidc.c") include for future versions
endif()
add_library(_rdkafka ${SRCS})
@@ -128,7 +148,11 @@ target_link_libraries(_rdkafka
OpenSSL::Crypto OpenSSL::SSL
)
if(WITH_SASL_CYRUS)
- target_link_libraries(_rdkafka PRIVATE ch_contrib::sasl2)
+ target_link_libraries(_rdkafka PRIVATE ch_contrib::sasl2)
+endif()
+
+if(WITH_CURL)
+ target_link_libraries(_rdkafka PRIVATE ch_contrib::curl)
endif()
file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/auxdir")
diff --git a/contrib/librdkafka-cmake/config.h.in b/contrib/librdkafka-cmake/config.h.in
index f6ec3bc0e79..a33f64b727a 100644
--- a/contrib/librdkafka-cmake/config.h.in
+++ b/contrib/librdkafka-cmake/config.h.in
@@ -1,33 +1,66 @@
-// Originally generated by ./configure
+// Automatically generated by ./configure
+/// NOTE: Version 2.5.3 was used to generate and manually modified after that. Therefore this should be fine
+/// until we upgrade to something newer than that.
+/// Commented out the followings:
+/// - #define ARCH "x86_64": we build on multiple archs
+/// - ENABLE_XXX: the commented out ones are only used in librdkafka's configure and CMake, but not in source
+/// NOTE: ENABLE_DEVEL and ENABLE_REFCNT_DEBUG is used in the source at the time of writing
+///
+/// Commented out the following to disable them:
+/// - ENABLE_C11THREADS: to maintain compatibility with old libc, maybe not necessary anymore
+/// - RDKAFKA_VERSION_STR, MKL_APP_VERSION: nothing depends on them currently, so make sure nothing starts
+// to depend on them as it is easy to forget to update it in the future
+/// - WITH_GCC
+/// - WITH_GXX
+/// - WITH_INSTALL
+/// - HAS_GNU_AR
+/// - HAVE_PIC
+/// - WITH_GNULD
+/// - WITH_C11THREADS
+/// - HAVE_PYTHON
+///
+/// Modified the follwoings:
+/// - WITH_{CURL,SASL_CYRUS,SASL_OAUTHBEARER,SASL_SCRAM}: made them CMake dependant
+/// - BUILT_WITH: tried to make some sense of it, don't spend to much time on it
+///
+/// Added:
+/// - special handling of __APPLE__
+
#ifndef _CONFIG_H_
#define _CONFIG_H_
-#define BUILT_WITH "GCC GXX PKGCONFIG OSXLD LIBDL PLUGINS ZLIB SSL SASL_CYRUS ZSTD HDRHISTOGRAM LZ4_EXT SNAPPY SOCKEM SASL_SCRAM CRC32C_HW"
-
-#define CPU "generic"
-#define WITHOUT_OPTIMIZATION 0
-#define ENABLE_DEVEL 0
-#define ENABLE_VALGRIND 0
-#define ENABLE_REFCNT_DEBUG 0
-#define ENABLE_SHAREDPTR_DEBUG 0
-#define ENABLE_LZ4_EXT 1
-#define ENABLE_SSL 1
-#define ENABLE_SASL 1
-#define MKL_APP_NAME "librdkafka"
-#define MKL_APP_DESC_ONELINE "The Apache Kafka C/C++ library"
// distro
#define SOLIB_EXT ".so"
+// #define ARCH "x86_64"
+#define CPU "generic"
+#define WITHOUT_OPTIMIZATION 0
+#define WITH_STRIP 0
+// #define ENABLE_ZLIB "try"
+// #define ENABLE_ZSTD "try"
+// #define ENABLE_SSL "try"
+// #define ENABLE_GSSAPI "try"
+// #define ENABLE_CURL "try"
+#define ENABLE_DEVEL 0
+// #define ENABLE_VALGRIND 0
+#define ENABLE_REFCNT_DEBUG 0
+// #define ENABLE_LZ4_EXT 1
+// #define ENABLE_LZ4_EXT 1
+// #define ENABLE_REGEX_EXT 1
+// #define ENABLE_C11THREADS "try"
+// #define ENABLE_SYSLOG 0
+#define MKL_APP_NAME "librdkafka"
+#define MKL_APP_DESC_ONELINE "The Apache Kafka C/C++ library"
// gcc
-//#define WITH_GCC 1
+// #define WITH_GCC 1
// gxx
-//#define WITH_GXX 1
-// pkgconfig
-//#define WITH_PKGCONFIG 1
+// #define WITH_GXX 1
// install
-//#define WITH_INSTALL 1
+// #define WITH_INSTALL 1
+// gnuar
+// #define HAS_GNU_AR 1
// PIC
-//#define HAVE_PIC 1
+// #define HAVE_PIC 1
// gnulib
-//#define WITH_GNULD 1
+// #define WITH_GNULD 1
// __atomic_32
#define HAVE_ATOMICS_32 1
// __atomic_32
@@ -43,9 +76,11 @@
// atomic_64
#define ATOMIC_OP(OP1,OP2,PTR,VAL) __atomic_ ## OP1 ## _ ## OP2(PTR, VAL, __ATOMIC_SEQ_CST)
// parseversion
-#define RDKAFKA_VERSION_STR "1.6.0"
+// #define RDKAFKA_VERSION_STR "1.7.0"
// parseversion
-#define MKL_APP_VERSION "1.6.0"
+// #define MKL_APP_VERSION "1.7.0"
+// c11threads
+// #define WITH_C11THREADS 1
// libdl
#define WITH_LIBDL 1
// WITH_PLUGINS
@@ -54,32 +89,41 @@
#define WITH_ZLIB 1
// zstd
#define WITH_ZSTD 1
+// libssl
+#cmakedefine WITH_SSL 1
+// libcrypto
+#define OPENSSL_SUPPRESS_DEPRECATED "OPENSSL_SUPPRESS_DEPRECATED"
+// WITH_HDRHISTOGRAM
+#define WITH_HDRHISTOGRAM 1
+// syslog
+#define WITH_SYSLOG 0
// WITH_SNAPPY
#define WITH_SNAPPY 1
// WITH_SOCKEM
#define WITH_SOCKEM 1
-// libssl
-#cmakedefine WITH_SSL 1
// WITH_SASL_SCRAM
#cmakedefine WITH_SASL_SCRAM 1
// WITH_SASL_OAUTHBEARER
#cmakedefine WITH_SASL_OAUTHBEARER 1
+// WITH_SASL_CYRUS
#cmakedefine WITH_SASL_CYRUS 1
+// WITH_CURL
+#cmakedefine WITH_CURL 1
// crc32chw
#if !defined(__PPC__) && !defined(__riscv) && !defined(__aarch64__) && !defined(__s390x__) && !defined(__loongarch64)
#define WITH_CRC32C_HW 1
#endif
// regex
#define HAVE_REGEX 1
+// rand_r
+#define HAVE_RAND_R 1
// strndup
#define HAVE_STRNDUP 1
// strerror_r
#define HAVE_STRERROR_R 1
-// rand_r
-#define HAVE_RAND_R 1
-
+// strcasestr
+#define HAVE_STRCASESTR 1
#ifdef __APPLE__
-// pthread_setname_np
#define HAVE_PTHREAD_SETNAME_DARWIN 1
#if (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ <= 101400)
#define _TTHREAD_EMULATE_TIMESPEC_GET_
@@ -90,8 +134,10 @@
// pthread_setname_gnu
#define HAVE_PTHREAD_SETNAME_GNU 1
#endif
-// python
-//#define HAVE_PYTHON 1
-// disable C11 threads for compatibility with old libc
-//#define WITH_C11THREADS 1
+// python3
+// #define HAVE_PYTHON 1
+// getrusage
+#define HAVE_GETRUSAGE 1
+// BUILT_WITH
+#define BUILT_WITH "GCC GXX INSTALL GNULD LDS LIBDL PLUGINS ZLIB SSL SASL_CYRUS ZSTD HDRHISTOGRAM LZ4_EXT SNAPPY SOCKEM SASL_SCRAM SASL_OAUTHBEARER CRC32C_HW"
#endif /* _CONFIG_H_ */
diff --git a/tests/integration/test_storage_kafka/configs/kafka.xml b/tests/integration/test_storage_kafka/configs/kafka.xml
index a846fdbb295..5c25f995e6b 100644
--- a/tests/integration/test_storage_kafka/configs/kafka.xml
+++ b/tests/integration/test_storage_kafka/configs/kafka.xml
@@ -12,6 +12,11 @@
600
+
+
+
+
+ 10000
consumer_hang
@@ -28,12 +33,12 @@
consumer_hang
-
+
6000
separate_settings
-
+
6001