From a5cbeda1e36d2976ce5da074b9ce8393eb0565a3 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Thu, 2 Jul 2020 19:13:42 +0300 Subject: [PATCH 1/6] fix segfault with -StateResample combinators --- src/Interpreters/Aggregator.cpp | 10 ++++++++-- .../0_stateless/01356_state_resample.reference | 8 ++++++++ tests/queries/0_stateless/01356_state_resample.sql | 14 ++++++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/01356_state_resample.reference create mode 100644 tests/queries/0_stateless/01356_state_resample.sql diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 5f7414b774b..1b8439fc704 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include @@ -1180,10 +1181,15 @@ Block Aggregator::prepareBlockAndFill( if (aggregate_functions[i]->isState()) { /// The ColumnAggregateFunction column captures the shared ownership of the arena with aggregate function states. - ColumnAggregateFunction & column_aggregate_func = assert_cast(*final_aggregate_columns[i]); + ColumnAggregateFunction * column_aggregate_func = nullptr; + /// Aggregate state can be wrapped into array if aggregate function ends with -Resample combinator. + if (auto * column_array = typeid_cast(final_aggregate_columns[i].get())) + column_aggregate_func = &assert_cast(column_array->getData()); + else + column_aggregate_func = &assert_cast(*final_aggregate_columns[i]); for (auto & pool : data_variants.aggregates_pools) - column_aggregate_func.addArena(pool); + column_aggregate_func->addArena(pool); } } } diff --git a/tests/queries/0_stateless/01356_state_resample.reference b/tests/queries/0_stateless/01356_state_resample.reference new file mode 100644 index 00000000000..40c606b4a68 --- /dev/null +++ b/tests/queries/0_stateless/01356_state_resample.reference @@ -0,0 +1,8 @@ +[900,910,920,930,940,950,960,970,980,990,1000,1010,1020,1030,1040,1050,1060,1070,1080,1090] +[900,910,920,930,940,950,960,970,980,990,1000,1010,1020,1030,1040,1050,1060,1070,1080,1090] +[360,243,306,372,252,315,384,261,324,396,270,333,408,279,342,420,288,351,432,297] +[300,364,246,309,376,255,318,388,264,327,400,273,336,412,282,345,424,291,354,436] +[240,303,368,249,312,380,258,321,392,267,330,404,276,339,416,285,348,428,294,357] +[[0,20,40],[1,21,41],[2,22,42],[3,23,43],[4,24,44],[5,25,45],[6,26,46],[7,27,47],[8,28,48],[9,29,49],[10,30],[11,31],[12,32],[13,33],[14,34],[15,35],[16,36],[17,37],[18,38],[19,39]] +[[0,20,40],[1,21,41],[2,22,42],[3,23,43],[4,24,44],[5,25,45],[6,26,46],[7,27,47],[8,28,48],[9,29,49],[10,30],[11,31],[12,32],[13,33],[14,34],[15,35],[16,36],[17,37],[18,38],[19,39]] +[1800,1820,1840,1860,1880,1900,1920,1940,1960,1980,2000,2020,2040,2060,2080,2100,2120,2140,2160,2180] diff --git a/tests/queries/0_stateless/01356_state_resample.sql b/tests/queries/0_stateless/01356_state_resample.sql new file mode 100644 index 00000000000..6be28e19d87 --- /dev/null +++ b/tests/queries/0_stateless/01356_state_resample.sql @@ -0,0 +1,14 @@ +select sumResample(0, 20, 1)(number, number % 20) from numbers(200); +select arrayMap(x -> finalizeAggregation(x), state) from (select sumStateResample(0, 20, 1)(number, number % 20) as state from numbers(200)); +select arrayMap(x -> finalizeAggregation(x), state) from +( + select sumStateResample(0,20,1)(number, number%20) as state from numbers(200) group by number % 3 +); + +select groupArrayResample(0, 20, 1)(number, number % 20) from numbers(50); +select arrayMap(x -> finalizeAggregation(x), state) from (select groupArrayStateResample(0, 20, 1)(number, number % 20) state from numbers(50)); + +select arrayMap(x -> finalizeAggregation(x), state) from +( + select sumStateResample(0, 20, 1)(number, number % 20) as state from remote('127.0.0.{1,2}', numbers(200)) +); From 0700a705bc186105d1d459ab80e4c2a1a368317b Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 29 Jun 2020 23:07:45 +0300 Subject: [PATCH 2/6] Rewrite curl dependency in a more ch compatible way - add support of unbundled curl - add CURL::libcurl libraries - avoid explicit linkage of daemon with curl (added with sentry) - set CACHE variables for non-direct users: - mariadb-connector-c - aws-s3-cmake - sentry-native Cc: @alexey-milovidov Cc: @alesapin (requires docker image update) Cc: @abyss7 Refs: #11300 Refs: #8011 Refs: #8905 v2: replace cmake/find/curl.cmake with proper contrib/curl-cmake (as pointed by @abyss7, cmake/find/*.cmake is deprecated) --- base/daemon/CMakeLists.txt | 1 - cmake/find/sentry.cmake | 2 - contrib/CMakeLists.txt | 23 +- contrib/curl-cmake/CMakeLists.txt | 329 ++++++++++++---------- docker/test/integration/runner/Dockerfile | 1 + utils/build/build_debian_unbundled.sh | 2 +- 6 files changed, 190 insertions(+), 168 deletions(-) diff --git a/base/daemon/CMakeLists.txt b/base/daemon/CMakeLists.txt index 04d2f059b39..26d59a57e7f 100644 --- a/base/daemon/CMakeLists.txt +++ b/base/daemon/CMakeLists.txt @@ -8,6 +8,5 @@ target_include_directories (daemon PUBLIC ..) target_link_libraries (daemon PUBLIC loggers PRIVATE clickhouse_common_io clickhouse_common_config common ${EXECINFO_LIBRARIES}) if (USE_SENTRY) - target_link_libraries (daemon PRIVATE curl) target_link_libraries (daemon PRIVATE ${SENTRY_LIBRARY}) endif () diff --git a/cmake/find/sentry.cmake b/cmake/find/sentry.cmake index eadf071141e..f202c9100a8 100644 --- a/cmake/find/sentry.cmake +++ b/cmake/find/sentry.cmake @@ -7,8 +7,6 @@ endif () if (NOT OS_FREEBSD AND NOT SPLIT_SHARED_LIBRARIES AND NOT_UNBUNDLED AND NOT (OS_DARWIN AND COMPILER_CLANG)) option (USE_SENTRY "Use Sentry" ON) - set (CURL_LIBRARY ${ClickHouse_SOURCE_DIR}/contrib/curl/lib) - set (CURL_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/curl/include) set (SENTRY_TRANSPORT "curl" CACHE STRING "") set (SENTRY_BACKEND "none" CACHE STRING "") set (SENTRY_EXPORT_SYMBOLS OFF CACHE BOOL "") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index f2222797bff..ba0bad90c5e 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -106,6 +106,12 @@ if (ENABLE_LDAP AND USE_INTERNAL_LDAP_LIBRARY) add_subdirectory (openldap-cmake) endif () +# Should go before: +# - mariadb-connector-c +# - aws-s3-cmake +# - sentry-native +add_subdirectory (curl-cmake) + function(mysql_support) set(CLIENT_PLUGIN_CACHING_SHA2_PASSWORD STATIC) set(CLIENT_PLUGIN_SHA256_PASSWORD STATIC) @@ -263,23 +269,6 @@ if (USE_INTERNAL_GRPC_LIBRARY) add_subdirectory(grpc-cmake) endif () -if (USE_INTERNAL_AWS_S3_LIBRARY OR USE_SENTRY) - set (save_CMAKE_C_FLAGS ${CMAKE_C_FLAGS}) - set (save_CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES}) - set (save_CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES}) - set (save_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS}) - set (save_CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH}) - add_subdirectory(curl-cmake) - set (CMAKE_C_FLAGS ${save_CMAKE_C_FLAGS}) - set (CMAKE_REQUIRED_LIBRARIES ${save_CMAKE_REQUIRED_LIBRARIES}) - set (CMAKE_CMAKE_REQUIRED_INCLUDES ${save_CMAKE_REQUIRED_INCLUDES}) - set (CMAKE_REQUIRED_FLAGS ${save_CMAKE_REQUIRED_FLAGS}) - set (CMAKE_CMAKE_MODULE_PATH ${save_CMAKE_MODULE_PATH}) - - # The library is large - avoid bloat. - target_compile_options (curl PRIVATE -g0) -endif () - if (USE_INTERNAL_AWS_S3_LIBRARY) add_subdirectory(aws-s3-cmake) diff --git a/contrib/curl-cmake/CMakeLists.txt b/contrib/curl-cmake/CMakeLists.txt index d0f6a7773b0..9edec1633c0 100644 --- a/contrib/curl-cmake/CMakeLists.txt +++ b/contrib/curl-cmake/CMakeLists.txt @@ -1,152 +1,187 @@ -set (CURL_DIR ${ClickHouse_SOURCE_DIR}/contrib/curl) -set (CURL_LIBRARY ${ClickHouse_SOURCE_DIR}/contrib/curl/lib) -set (CURL_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/curl/include) +option (ENABLE_CURL "Enable curl" ${ENABLE_LIBRARIES}) -set (SRCS - ${CURL_DIR}/lib/file.c - ${CURL_DIR}/lib/timeval.c - ${CURL_DIR}/lib/base64.c - ${CURL_DIR}/lib/hostip.c - ${CURL_DIR}/lib/progress.c - ${CURL_DIR}/lib/formdata.c - ${CURL_DIR}/lib/cookie.c - ${CURL_DIR}/lib/http.c - ${CURL_DIR}/lib/sendf.c - ${CURL_DIR}/lib/url.c - ${CURL_DIR}/lib/dict.c - ${CURL_DIR}/lib/if2ip.c - ${CURL_DIR}/lib/speedcheck.c - ${CURL_DIR}/lib/ldap.c - ${CURL_DIR}/lib/version.c - ${CURL_DIR}/lib/getenv.c - ${CURL_DIR}/lib/escape.c - ${CURL_DIR}/lib/mprintf.c - ${CURL_DIR}/lib/telnet.c - ${CURL_DIR}/lib/netrc.c - ${CURL_DIR}/lib/getinfo.c - ${CURL_DIR}/lib/transfer.c - ${CURL_DIR}/lib/strcase.c - ${CURL_DIR}/lib/easy.c - ${CURL_DIR}/lib/security.c - ${CURL_DIR}/lib/curl_fnmatch.c - ${CURL_DIR}/lib/fileinfo.c - ${CURL_DIR}/lib/wildcard.c - ${CURL_DIR}/lib/krb5.c - ${CURL_DIR}/lib/memdebug.c - ${CURL_DIR}/lib/http_chunks.c - ${CURL_DIR}/lib/strtok.c - ${CURL_DIR}/lib/connect.c - ${CURL_DIR}/lib/llist.c - ${CURL_DIR}/lib/hash.c - ${CURL_DIR}/lib/multi.c - ${CURL_DIR}/lib/content_encoding.c - ${CURL_DIR}/lib/share.c - ${CURL_DIR}/lib/http_digest.c - ${CURL_DIR}/lib/md4.c - ${CURL_DIR}/lib/md5.c - ${CURL_DIR}/lib/http_negotiate.c - ${CURL_DIR}/lib/inet_pton.c - ${CURL_DIR}/lib/strtoofft.c - ${CURL_DIR}/lib/strerror.c - ${CURL_DIR}/lib/amigaos.c - ${CURL_DIR}/lib/hostasyn.c - ${CURL_DIR}/lib/hostip4.c - ${CURL_DIR}/lib/hostip6.c - ${CURL_DIR}/lib/hostsyn.c - ${CURL_DIR}/lib/inet_ntop.c - ${CURL_DIR}/lib/parsedate.c - ${CURL_DIR}/lib/select.c - ${CURL_DIR}/lib/splay.c - ${CURL_DIR}/lib/strdup.c - ${CURL_DIR}/lib/socks.c - ${CURL_DIR}/lib/curl_addrinfo.c - ${CURL_DIR}/lib/socks_gssapi.c - ${CURL_DIR}/lib/socks_sspi.c - ${CURL_DIR}/lib/curl_sspi.c - ${CURL_DIR}/lib/slist.c - ${CURL_DIR}/lib/nonblock.c - ${CURL_DIR}/lib/curl_memrchr.c - ${CURL_DIR}/lib/imap.c - ${CURL_DIR}/lib/pop3.c - ${CURL_DIR}/lib/smtp.c - ${CURL_DIR}/lib/pingpong.c - ${CURL_DIR}/lib/rtsp.c - ${CURL_DIR}/lib/curl_threads.c - ${CURL_DIR}/lib/warnless.c - ${CURL_DIR}/lib/hmac.c - ${CURL_DIR}/lib/curl_rtmp.c - ${CURL_DIR}/lib/openldap.c - ${CURL_DIR}/lib/curl_gethostname.c - ${CURL_DIR}/lib/gopher.c - ${CURL_DIR}/lib/idn_win32.c - ${CURL_DIR}/lib/http_proxy.c - ${CURL_DIR}/lib/non-ascii.c - ${CURL_DIR}/lib/asyn-thread.c - ${CURL_DIR}/lib/curl_gssapi.c - ${CURL_DIR}/lib/http_ntlm.c - ${CURL_DIR}/lib/curl_ntlm_wb.c - ${CURL_DIR}/lib/curl_ntlm_core.c - ${CURL_DIR}/lib/curl_sasl.c - ${CURL_DIR}/lib/rand.c - ${CURL_DIR}/lib/curl_multibyte.c - ${CURL_DIR}/lib/hostcheck.c - ${CURL_DIR}/lib/conncache.c - ${CURL_DIR}/lib/dotdot.c - ${CURL_DIR}/lib/x509asn1.c - ${CURL_DIR}/lib/http2.c - ${CURL_DIR}/lib/smb.c - ${CURL_DIR}/lib/curl_endian.c - ${CURL_DIR}/lib/curl_des.c - ${CURL_DIR}/lib/system_win32.c - ${CURL_DIR}/lib/mime.c - ${CURL_DIR}/lib/sha256.c - ${CURL_DIR}/lib/setopt.c - ${CURL_DIR}/lib/curl_path.c - ${CURL_DIR}/lib/curl_ctype.c - ${CURL_DIR}/lib/curl_range.c - ${CURL_DIR}/lib/psl.c - ${CURL_DIR}/lib/doh.c - ${CURL_DIR}/lib/urlapi.c - ${CURL_DIR}/lib/curl_get_line.c - ${CURL_DIR}/lib/altsvc.c - ${CURL_DIR}/lib/socketpair.c - ${CURL_DIR}/lib/vauth/vauth.c - ${CURL_DIR}/lib/vauth/cleartext.c - ${CURL_DIR}/lib/vauth/cram.c - ${CURL_DIR}/lib/vauth/digest.c - ${CURL_DIR}/lib/vauth/digest_sspi.c - ${CURL_DIR}/lib/vauth/krb5_gssapi.c - ${CURL_DIR}/lib/vauth/krb5_sspi.c - ${CURL_DIR}/lib/vauth/ntlm.c - ${CURL_DIR}/lib/vauth/ntlm_sspi.c - ${CURL_DIR}/lib/vauth/oauth2.c - ${CURL_DIR}/lib/vauth/spnego_gssapi.c - ${CURL_DIR}/lib/vauth/spnego_sspi.c - ${CURL_DIR}/lib/vtls/openssl.c - ${CURL_DIR}/lib/vtls/gtls.c - ${CURL_DIR}/lib/vtls/vtls.c - ${CURL_DIR}/lib/vtls/nss.c - ${CURL_DIR}/lib/vtls/polarssl.c - ${CURL_DIR}/lib/vtls/polarssl_threadlock.c - ${CURL_DIR}/lib/vtls/wolfssl.c - ${CURL_DIR}/lib/vtls/schannel.c - ${CURL_DIR}/lib/vtls/schannel_verify.c - ${CURL_DIR}/lib/vtls/sectransp.c - ${CURL_DIR}/lib/vtls/gskit.c - ${CURL_DIR}/lib/vtls/mbedtls.c - ${CURL_DIR}/lib/vtls/mesalink.c - ${CURL_DIR}/lib/vtls/bearssl.c - ${CURL_DIR}/lib/vquic/ngtcp2.c - ${CURL_DIR}/lib/vquic/quiche.c - ${CURL_DIR}/lib/vssh/libssh2.c - ${CURL_DIR}/lib/vssh/libssh.c -) +if (ENABLE_CURL) + option (USE_INTERNAL_CURL "Use internal curl library" ${NOT_UNBUNDLED}) -add_library (curl ${SRCS}) + if (USE_INTERNAL_CURL) + set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/curl") -target_compile_definitions(curl PRIVATE HAVE_CONFIG_H BUILDING_LIBCURL CURL_HIDDEN_SYMBOLS libcurl_EXPORTS) -target_include_directories(curl PUBLIC ${CURL_DIR}/include ${CURL_DIR}/lib .) + set (SRCS + ${LIBRARY_DIR}/lib/file.c + ${LIBRARY_DIR}/lib/timeval.c + ${LIBRARY_DIR}/lib/base64.c + ${LIBRARY_DIR}/lib/hostip.c + ${LIBRARY_DIR}/lib/progress.c + ${LIBRARY_DIR}/lib/formdata.c + ${LIBRARY_DIR}/lib/cookie.c + ${LIBRARY_DIR}/lib/http.c + ${LIBRARY_DIR}/lib/sendf.c + ${LIBRARY_DIR}/lib/url.c + ${LIBRARY_DIR}/lib/dict.c + ${LIBRARY_DIR}/lib/if2ip.c + ${LIBRARY_DIR}/lib/speedcheck.c + ${LIBRARY_DIR}/lib/ldap.c + ${LIBRARY_DIR}/lib/version.c + ${LIBRARY_DIR}/lib/getenv.c + ${LIBRARY_DIR}/lib/escape.c + ${LIBRARY_DIR}/lib/mprintf.c + ${LIBRARY_DIR}/lib/telnet.c + ${LIBRARY_DIR}/lib/netrc.c + ${LIBRARY_DIR}/lib/getinfo.c + ${LIBRARY_DIR}/lib/transfer.c + ${LIBRARY_DIR}/lib/strcase.c + ${LIBRARY_DIR}/lib/easy.c + ${LIBRARY_DIR}/lib/security.c + ${LIBRARY_DIR}/lib/curl_fnmatch.c + ${LIBRARY_DIR}/lib/fileinfo.c + ${LIBRARY_DIR}/lib/wildcard.c + ${LIBRARY_DIR}/lib/krb5.c + ${LIBRARY_DIR}/lib/memdebug.c + ${LIBRARY_DIR}/lib/http_chunks.c + ${LIBRARY_DIR}/lib/strtok.c + ${LIBRARY_DIR}/lib/connect.c + ${LIBRARY_DIR}/lib/llist.c + ${LIBRARY_DIR}/lib/hash.c + ${LIBRARY_DIR}/lib/multi.c + ${LIBRARY_DIR}/lib/content_encoding.c + ${LIBRARY_DIR}/lib/share.c + ${LIBRARY_DIR}/lib/http_digest.c + ${LIBRARY_DIR}/lib/md4.c + ${LIBRARY_DIR}/lib/md5.c + ${LIBRARY_DIR}/lib/http_negotiate.c + ${LIBRARY_DIR}/lib/inet_pton.c + ${LIBRARY_DIR}/lib/strtoofft.c + ${LIBRARY_DIR}/lib/strerror.c + ${LIBRARY_DIR}/lib/amigaos.c + ${LIBRARY_DIR}/lib/hostasyn.c + ${LIBRARY_DIR}/lib/hostip4.c + ${LIBRARY_DIR}/lib/hostip6.c + ${LIBRARY_DIR}/lib/hostsyn.c + ${LIBRARY_DIR}/lib/inet_ntop.c + ${LIBRARY_DIR}/lib/parsedate.c + ${LIBRARY_DIR}/lib/select.c + ${LIBRARY_DIR}/lib/splay.c + ${LIBRARY_DIR}/lib/strdup.c + ${LIBRARY_DIR}/lib/socks.c + ${LIBRARY_DIR}/lib/curl_addrinfo.c + ${LIBRARY_DIR}/lib/socks_gssapi.c + ${LIBRARY_DIR}/lib/socks_sspi.c + ${LIBRARY_DIR}/lib/curl_sspi.c + ${LIBRARY_DIR}/lib/slist.c + ${LIBRARY_DIR}/lib/nonblock.c + ${LIBRARY_DIR}/lib/curl_memrchr.c + ${LIBRARY_DIR}/lib/imap.c + ${LIBRARY_DIR}/lib/pop3.c + ${LIBRARY_DIR}/lib/smtp.c + ${LIBRARY_DIR}/lib/pingpong.c + ${LIBRARY_DIR}/lib/rtsp.c + ${LIBRARY_DIR}/lib/curl_threads.c + ${LIBRARY_DIR}/lib/warnless.c + ${LIBRARY_DIR}/lib/hmac.c + ${LIBRARY_DIR}/lib/curl_rtmp.c + ${LIBRARY_DIR}/lib/openldap.c + ${LIBRARY_DIR}/lib/curl_gethostname.c + ${LIBRARY_DIR}/lib/gopher.c + ${LIBRARY_DIR}/lib/idn_win32.c + ${LIBRARY_DIR}/lib/http_proxy.c + ${LIBRARY_DIR}/lib/non-ascii.c + ${LIBRARY_DIR}/lib/asyn-thread.c + ${LIBRARY_DIR}/lib/curl_gssapi.c + ${LIBRARY_DIR}/lib/http_ntlm.c + ${LIBRARY_DIR}/lib/curl_ntlm_wb.c + ${LIBRARY_DIR}/lib/curl_ntlm_core.c + ${LIBRARY_DIR}/lib/curl_sasl.c + ${LIBRARY_DIR}/lib/rand.c + ${LIBRARY_DIR}/lib/curl_multibyte.c + ${LIBRARY_DIR}/lib/hostcheck.c + ${LIBRARY_DIR}/lib/conncache.c + ${LIBRARY_DIR}/lib/dotdot.c + ${LIBRARY_DIR}/lib/x509asn1.c + ${LIBRARY_DIR}/lib/http2.c + ${LIBRARY_DIR}/lib/smb.c + ${LIBRARY_DIR}/lib/curl_endian.c + ${LIBRARY_DIR}/lib/curl_des.c + ${LIBRARY_DIR}/lib/system_win32.c + ${LIBRARY_DIR}/lib/mime.c + ${LIBRARY_DIR}/lib/sha256.c + ${LIBRARY_DIR}/lib/setopt.c + ${LIBRARY_DIR}/lib/curl_path.c + ${LIBRARY_DIR}/lib/curl_ctype.c + ${LIBRARY_DIR}/lib/curl_range.c + ${LIBRARY_DIR}/lib/psl.c + ${LIBRARY_DIR}/lib/doh.c + ${LIBRARY_DIR}/lib/urlapi.c + ${LIBRARY_DIR}/lib/curl_get_line.c + ${LIBRARY_DIR}/lib/altsvc.c + ${LIBRARY_DIR}/lib/socketpair.c + ${LIBRARY_DIR}/lib/vauth/vauth.c + ${LIBRARY_DIR}/lib/vauth/cleartext.c + ${LIBRARY_DIR}/lib/vauth/cram.c + ${LIBRARY_DIR}/lib/vauth/digest.c + ${LIBRARY_DIR}/lib/vauth/digest_sspi.c + ${LIBRARY_DIR}/lib/vauth/krb5_gssapi.c + ${LIBRARY_DIR}/lib/vauth/krb5_sspi.c + ${LIBRARY_DIR}/lib/vauth/ntlm.c + ${LIBRARY_DIR}/lib/vauth/ntlm_sspi.c + ${LIBRARY_DIR}/lib/vauth/oauth2.c + ${LIBRARY_DIR}/lib/vauth/spnego_gssapi.c + ${LIBRARY_DIR}/lib/vauth/spnego_sspi.c + ${LIBRARY_DIR}/lib/vtls/openssl.c + ${LIBRARY_DIR}/lib/vtls/gtls.c + ${LIBRARY_DIR}/lib/vtls/vtls.c + ${LIBRARY_DIR}/lib/vtls/nss.c + ${LIBRARY_DIR}/lib/vtls/polarssl.c + ${LIBRARY_DIR}/lib/vtls/polarssl_threadlock.c + ${LIBRARY_DIR}/lib/vtls/wolfssl.c + ${LIBRARY_DIR}/lib/vtls/schannel.c + ${LIBRARY_DIR}/lib/vtls/schannel_verify.c + ${LIBRARY_DIR}/lib/vtls/sectransp.c + ${LIBRARY_DIR}/lib/vtls/gskit.c + ${LIBRARY_DIR}/lib/vtls/mbedtls.c + ${LIBRARY_DIR}/lib/vtls/mesalink.c + ${LIBRARY_DIR}/lib/vtls/bearssl.c + ${LIBRARY_DIR}/lib/vquic/ngtcp2.c + ${LIBRARY_DIR}/lib/vquic/quiche.c + ${LIBRARY_DIR}/lib/vssh/libssh2.c + ${LIBRARY_DIR}/lib/vssh/libssh.c + ) -target_compile_definitions(curl PRIVATE OS="${CMAKE_SYSTEM_NAME}") + add_library (curl ${SRCS}) -target_link_libraries(curl PRIVATE ssl) + target_compile_definitions (curl PRIVATE + HAVE_CONFIG_H + BUILDING_LIBCURL + CURL_HIDDEN_SYMBOLS + libcurl_EXPORTS + OS="${CMAKE_SYSTEM_NAME}" + ) + target_include_directories (curl PUBLIC + ${LIBRARY_DIR}/include + ${LIBRARY_DIR}/lib + . # curl_config.h + ) + + target_link_libraries (curl PRIVATE ssl) + + # The library is large - avoid bloat (XXX: is it?) + target_compile_options (curl PRIVATE -g0) + + # find_package(CURL) compatibility for the following packages that uses + # find_package(CURL)/include(FindCURL): + # - mariadb-connector-c + # - aws-s3-cmake + # - sentry-native + set (CURL_FOUND ON CACHE BOOL "") + set (CURL_ROOT_DIR ${LIBRARY_DIR} CACHE PATH "") + set (CURL_INCLUDE_DIR ${LIBRARY_DIR}/include CACHE PATH "") + set (CURL_INCLUDE_DIRS ${LIBRARY_DIR}/include CACHE PATH "") + set (CURL_LIBRARY curl CACHE STRING "") + set (CURL_LIBRARIES ${CURL_LIBRARY} CACHE STRING "") + set (CURL_VERSION_STRING 7.67.0 CACHE STRING "") + add_library (CURL::libcurl ALIAS ${CURL_LIBRARY}) + else () + find_package (CURL) + endif () +endif () + +message (STATUS "Using curl: ${CURL_INCLUDE_DIRS} : ${CURL_LIBRARIES}") diff --git a/docker/test/integration/runner/Dockerfile b/docker/test/integration/runner/Dockerfile index 9c1fe66cf7b..423ecb06122 100644 --- a/docker/test/integration/runner/Dockerfile +++ b/docker/test/integration/runner/Dockerfile @@ -26,6 +26,7 @@ RUN apt-get update \ liblua5.1-dev \ luajit \ libssl-dev \ + libcurl4-openssl-dev \ gdb \ && rm -rf \ /var/lib/apt/lists/* \ diff --git a/utils/build/build_debian_unbundled.sh b/utils/build/build_debian_unbundled.sh index f5f59ce7a13..5b2129fc5bf 100755 --- a/utils/build/build_debian_unbundled.sh +++ b/utils/build/build_debian_unbundled.sh @@ -22,5 +22,5 @@ env TEST_RUN=1 \ `# Use all possible contrib libs from system` \ `# psmisc - killall` \ `# gdb - symbol test in pbuilder` \ - EXTRAPACKAGES="psmisc libboost-program-options-dev libboost-system-dev libboost-filesystem-dev libboost-thread-dev libboost-regex-dev libboost-iostreams-dev zlib1g-dev liblz4-dev libdouble-conversion-dev libsparsehash-dev librdkafka-dev libpoco-dev unixodbc-dev libsparsehash-dev libgoogle-perftools-dev libzstd-dev libre2-dev libunwind-dev googletest libcctz-dev libcapnp-dev libjemalloc-dev libssl-dev libunwind-dev libgsasl7-dev libxml2-dev libbrotli-dev libhyperscan-dev rapidjson-dev $EXTRAPACKAGES" \ + EXTRAPACKAGES="psmisc libboost-program-options-dev libboost-system-dev libboost-filesystem-dev libboost-thread-dev libboost-regex-dev libboost-iostreams-dev zlib1g-dev liblz4-dev libdouble-conversion-dev libsparsehash-dev librdkafka-dev libpoco-dev unixodbc-dev libsparsehash-dev libgoogle-perftools-dev libzstd-dev libre2-dev libunwind-dev googletest libcctz-dev libcapnp-dev libjemalloc-dev libssl-dev libcurl4-openssl-dev libunwind-dev libgsasl7-dev libxml2-dev libbrotli-dev libhyperscan-dev rapidjson-dev $EXTRAPACKAGES" \ pdebuild --configfile $ROOT_DIR/debian/.pbuilderrc $PDEBUILD_OPT From e6aacaac5c69bf682e24c2521033f6f97bc3fbec Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 4 Jul 2020 16:40:32 +0300 Subject: [PATCH 3/6] Do not enable sentry if ENABLE_LIBRARIES is not set --- cmake/find/sentry.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/find/sentry.cmake b/cmake/find/sentry.cmake index f202c9100a8..84425220f12 100644 --- a/cmake/find/sentry.cmake +++ b/cmake/find/sentry.cmake @@ -6,7 +6,7 @@ if (NOT EXISTS "${SENTRY_INCLUDE_DIR}/sentry.h") endif () if (NOT OS_FREEBSD AND NOT SPLIT_SHARED_LIBRARIES AND NOT_UNBUNDLED AND NOT (OS_DARWIN AND COMPILER_CLANG)) - option (USE_SENTRY "Use Sentry" ON) + option (USE_SENTRY "Use Sentry" ${ENABLE_LIBRARIES}) set (SENTRY_TRANSPORT "curl" CACHE STRING "") set (SENTRY_BACKEND "none" CACHE STRING "") set (SENTRY_EXPORT_SYMBOLS OFF CACHE BOOL "") From 9258368de28d12348dc732adea392357aec1dff3 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 4 Jul 2020 16:48:25 +0300 Subject: [PATCH 4/6] Fail if curl library was enabled and was not found in case of unbundled build --- contrib/curl-cmake/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/curl-cmake/CMakeLists.txt b/contrib/curl-cmake/CMakeLists.txt index 9edec1633c0..3c3226cae9e 100644 --- a/contrib/curl-cmake/CMakeLists.txt +++ b/contrib/curl-cmake/CMakeLists.txt @@ -180,7 +180,7 @@ if (ENABLE_CURL) set (CURL_VERSION_STRING 7.67.0 CACHE STRING "") add_library (CURL::libcurl ALIAS ${CURL_LIBRARY}) else () - find_package (CURL) + find_package (CURL REQUIRED) endif () endif () From f91e34b2e80001f0c3b2eec55d65ee8c5aa3431f Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 5 Jul 2020 00:56:48 +0300 Subject: [PATCH 5/6] Include libcurl4-openssl-dev into yandex/clickhouse-deb-builder --- docker/packager/deb/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/packager/deb/Dockerfile b/docker/packager/deb/Dockerfile index 4f1be197668..48e90d16f5d 100644 --- a/docker/packager/deb/Dockerfile +++ b/docker/packager/deb/Dockerfile @@ -68,6 +68,7 @@ RUN apt-get --allow-unauthenticated update -y \ libre2-dev \ libjemalloc-dev \ libmsgpack-dev \ + libcurl4-openssl-dev \ opencl-headers \ ocl-icd-libopencl1 \ intel-opencl-icd \ From e2e92afd981b0c3af79fd7d9f70e653dd86f2065 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Mon, 6 Jul 2020 02:17:46 +0300 Subject: [PATCH 6/6] fix segfault with -StateResample combinators --- src/Interpreters/Aggregator.cpp | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 1b8439fc704..39fcb382e57 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -1181,15 +1181,17 @@ Block Aggregator::prepareBlockAndFill( if (aggregate_functions[i]->isState()) { /// The ColumnAggregateFunction column captures the shared ownership of the arena with aggregate function states. - ColumnAggregateFunction * column_aggregate_func = nullptr; - /// Aggregate state can be wrapped into array if aggregate function ends with -Resample combinator. - if (auto * column_array = typeid_cast(final_aggregate_columns[i].get())) - column_aggregate_func = &assert_cast(column_array->getData()); - else - column_aggregate_func = &assert_cast(*final_aggregate_columns[i]); + if (auto * column_aggregate_func = typeid_cast(final_aggregate_columns[i].get())) + for (auto & pool : data_variants.aggregates_pools) + column_aggregate_func->addArena(pool); - for (auto & pool : data_variants.aggregates_pools) - column_aggregate_func->addArena(pool); + /// Aggregate state can be wrapped into array if aggregate function ends with -Resample combinator. + final_aggregate_columns[i]->forEachSubcolumn([&data_variants](auto & subcolumn) + { + if (auto * column_aggregate_func = typeid_cast(subcolumn.get())) + for (auto & pool : data_variants.aggregates_pools) + column_aggregate_func->addArena(pool); + }); } } }