thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-10-03 23:20:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:ClickHouse/ClickHouse

Browse Source
This commit is contained in:
Sergei Shtykov 2019-12-17 11:18:52 +03:00
commit c14eeba002
626 changed files with 19900 additions and 5408 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
.gitmodules vendored
View File

@ -29,9 +29,6 @@
[submodule "contrib/re2"]
path = contrib/re2
url = https://github.com/google/re2.git
[submodule "contrib/ssl"]
path = contrib/ssl
url = https://github.com/ClickHouse-Extras/ssl.git
[submodule "contrib/llvm"]
path = contrib/llvm
url = https://github.com/ClickHouse-Extras/llvm
@ -107,3 +104,27 @@
[submodule "contrib/sparsehash-c11"]
path = contrib/sparsehash-c11
url = https://github.com/sparsehash/sparsehash-c11.git
[submodule "contrib/aws"]
path = contrib/aws
url = https://github.com/aws/aws-sdk-cpp.git
[submodule "aws-c-event-stream"]
path = contrib/aws-c-event-stream
url = https://github.com/awslabs/aws-c-event-stream.git
[submodule "aws-c-common"]
path = contrib/aws-c-common
url = https://github.com/awslabs/aws-c-common.git
[submodule "aws-checksums"]
path = contrib/aws-checksums
url = https://github.com/awslabs/aws-checksums.git
[submodule "contrib/curl"]
path = contrib/curl
url = https://github.com/curl/curl.git
[submodule "contrib/openssl"]
path = contrib/openssl
url = https://github.com/ClickHouse-Extras/openssl.git
[submodule "contrib/icudata"]
path = contrib/icudata
url = https://github.com/ClickHouse-Extras/icudata.git
[submodule "contrib/icu"]
path = contrib/icu
url = https://github.com/unicode-org/icu.git

2
CHANGELOG.md
View File

@ -128,6 +128,8 @@ Yu](https://github.com/yuzhichang))
* Introduce CustomSeparated data format that supports custom escaping and
delimiter rules. [#7118](https://github.com/ClickHouse/ClickHouse/pull/7118)
([tavplubix](https://github.com/tavplubix))
* Support Redis as source of external dictionary. [#4361](https://github.com/ClickHouse/ClickHouse/pull/4361) [#6962](https://github.com/ClickHouse/ClickHouse/pull/6962) ([comunodi](https://github.com/comunodi), [Anton
Popov](https://github.com/CurtizJ))
### Bug Fix
* Fix wrong query result if it has `WHERE IN (SELECT ...)` section and `optimize_read_in_order` is

5
CMakeLists.txt
View File

@ -198,11 +198,11 @@ if(WITH_COVERAGE AND COMPILER_GCC)
endif()
set (CMAKE_BUILD_COLOR_MAKEFILE ON)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS} ${PLATFORM_EXTRA_CXX_FLAG} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${CXX_WARNING_FLAGS}")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS} ${PLATFORM_EXTRA_CXX_FLAG} ${COMMON_WARNING_FLAGS} ${CXX_WARNING_FLAGS}")
set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3 ${CMAKE_CXX_FLAGS_ADD}")
set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_CXX_FLAGS_ADD}")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${CMAKE_C_FLAGS_ADD}")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} ${COMMON_WARNING_FLAGS} ${CMAKE_C_FLAGS_ADD}")
set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -O3 ${CMAKE_C_FLAGS_ADD}")
set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_C_FLAGS_ADD}")
@ -325,6 +325,7 @@ include (cmake/find/brotli.cmake)
include (cmake/find/protobuf.cmake)
include (cmake/find/pdqsort.cmake)
include (cmake/find/hdfs3.cmake) # uses protobuf
include (cmake/find/s3.cmake)
include (cmake/find/consistent-hashing.cmake)
include (cmake/find/base64.cmake)
include (cmake/find/parquet.cmake)

4
README.md
View File

@ -11,7 +11,3 @@ ClickHouse is an open-source column-oriented database management system that all
* [Blog](https://clickhouse.yandex/blog/en/) contains various ClickHouse-related articles, as well as announces and reports about events.
* [Contacts](https://clickhouse.yandex/#contacts) can help to get your questions answered if there are any.
* You can also [fill this form](https://forms.yandex.com/surveys/meet-yandex-clickhouse-team/) to meet Yandex ClickHouse team in person.
## Upcoming Events
* [ClickHouse Meetup in Moscow](https://yandex.ru/promo/clickhouse/moscow-december-2019) on December 11.

24
cmake/find/icu.cmake
View File

@ -1,6 +1,18 @@
option(ENABLE_ICU "Enable ICU" ${ENABLE_LIBRARIES})
if(ENABLE_ICU)
if (ENABLE_ICU)
option (USE_INTERNAL_ICU_LIBRARY "Set to FALSE to use system ICU library instead of bundled" ${NOT_UNBUNDLED})
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/icu/icu4c/LICENSE")
if (USE_INTERNAL_ICU_LIBRARY)
message (WARNING "submodule contrib/icu is missing. to fix try run: \n git submodule update --init --recursive")
set (USE_INTERNAL_ICU_LIBRARY 0)
endif ()
set (MISSING_INTERNAL_ICU_LIBRARY 1)
endif ()
if(NOT USE_INTERNAL_ICU_LIBRARY)
if (APPLE)
set(ICU_ROOT "/usr/local/opt/icu4c" CACHE STRING "")
endif()
@ -11,6 +23,16 @@ if(ENABLE_ICU)
endif()
endif()
if (ICU_LIBRARY AND ICU_INCLUDE_DIR)
set (USE_ICU 1)
elseif (NOT MISSING_INTERNAL_ICU_LIBRARY)
set (USE_INTERNAL_ICU_LIBRARY 1)
set (ICU_LIBRARIES icui18n icuuc icudata)
set (USE_ICU 1)
endif ()
endif()
if(USE_ICU)
message(STATUS "Using icu=${USE_ICU}: ${ICU_INCLUDE_DIR} : ${ICU_LIBRARIES}")
else()

26
cmake/find/s3.cmake Normal file
View File

@ -0,0 +1,26 @@
if(NOT OS_FREEBSD AND NOT APPLE)
option(ENABLE_S3 "Enable S3" ${ENABLE_LIBRARIES})
endif()
if(ENABLE_S3)
option(USE_INTERNAL_AWS_S3_LIBRARY "Set to FALSE to use system S3 instead of bundled" ${NOT_UNBUNDLED})
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/aws/aws-cpp-sdk-s3")
message (WARNING "submodule contrib/aws is missing. to fix try run: \n git submodule update --init --recursive")
set (MISSING_AWS_S3 1)
endif ()
if (USE_INTERNAL_AWS_S3_LIBRARY AND NOT MISSING_AWS_S3)
set(AWS_S3_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws/aws-cpp-sdk-s3/include")
set(AWS_S3_CORE_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws/aws-cpp-sdk-core/include")
set(AWS_S3_LIBRARY aws_s3)
set(USE_INTERNAL_AWS_S3_LIBRARY 1)
set(USE_AWS_S3 1)
else()
set(USE_INTERNAL_AWS_S3_LIBRARY 0)
set(USE_AWS_S3 0)
endif ()
endif()
message (STATUS "Using aws_s3=${USE_AWS_S3}: ${AWS_S3_INCLUDE_DIR} : ${AWS_S3_LIBRARY}")

22
cmake/find/ssl.cmake
View File

@ -6,9 +6,9 @@ if(NOT ARCH_32)
option(USE_INTERNAL_SSL_LIBRARY "Set to FALSE to use system *ssl library instead of bundled" ${NOT_UNBUNDLED})
endif()
if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/ssl/CMakeLists.txt")
if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/openssl/README")
if(USE_INTERNAL_SSL_LIBRARY)
message(WARNING "submodule contrib/ssl is missing. to fix try run: \n git submodule update --init --recursive")
message(WARNING "submodule contrib/openssl is missing. to fix try run: \n git submodule update --init --recursive")
endif()
set(USE_INTERNAL_SSL_LIBRARY 0)
set(MISSING_INTERNAL_SSL_LIBRARY 1)
@ -42,17 +42,17 @@ endif ()
if (NOT OPENSSL_FOUND AND NOT MISSING_INTERNAL_SSL_LIBRARY)
set (USE_INTERNAL_SSL_LIBRARY 1)
set (OPENSSL_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/ssl")
set (OPENSSL_INCLUDE_DIR "${OPENSSL_ROOT_DIR}/include")
if (NOT USE_STATIC_LIBRARIES AND TARGET crypto-shared AND TARGET ssl-shared)
set (OPENSSL_CRYPTO_LIBRARY crypto-shared)
set (OPENSSL_SSL_LIBRARY ssl-shared)
else ()
set (OPENSSL_CRYPTO_LIBRARY crypto)
set (OPENSSL_SSL_LIBRARY ssl)
set (OPENSSL_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/openssl")
if (ARCH_AMD64)
set (OPENSSL_INCLUDE_DIR "${OPENSSL_ROOT_DIR}/include" "${ClickHouse_SOURCE_DIR}/contrib/openssl-cmake/linux_x86_64/include")
elseif (ARCH_AARCH64)
set (OPENSSL_INCLUDE_DIR "${OPENSSL_ROOT_DIR}/include" "${ClickHouse_SOURCE_DIR}/contrib/openssl-cmake/linux_aarch64/include")
endif ()
set (OPENSSL_LIBRARIES ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY})
set (OPENSSL_CRYPTO_LIBRARY crypto)
set (OPENSSL_SSL_LIBRARY ssl)
set (OPENSSL_FOUND 1)
set (OPENSSL_LIBRARIES ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY})
endif ()
if(OPENSSL_FOUND)

7
cmake/find/termcap.cmake
View File

@ -1,5 +1,8 @@
find_library (TERMCAP_LIBRARY termcap)
find_library (TERMCAP_LIBRARY tinfo)
if (NOT TERMCAP_LIBRARY)
find_library (TERMCAP_LIBRARY tinfo)
find_library (TERMCAP_LIBRARY ncurses)
endif()
if (NOT TERMCAP_LIBRARY)
find_library (TERMCAP_LIBRARY termcap)
endif()
message (STATUS "Using termcap: ${TERMCAP_LIBRARY}")

3
cmake/sanitize.cmake
View File

@ -23,7 +23,7 @@ if (SANITIZE)
# RelWithDebInfo, and downgrade optimizations to -O1 but not to -Og, to
# keep the binary size down.
# TODO: try compiling with -Og and with ld.gold.
set (MSAN_FLAGS "-fsanitize=memory -fsanitize-memory-track-origins -fno-optimize-sibling-calls")
set (MSAN_FLAGS "-fsanitize=memory -fsanitize-memory-track-origins -fno-optimize-sibling-calls -fsanitize-blacklist=${CMAKE_SOURCE_DIR}/dbms/tests/msan_suppressions.txt")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${MSAN_FLAGS}")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} ${MSAN_FLAGS}")
@ -40,7 +40,6 @@ if (SANITIZE)
set (ENABLE_HDFS 0 CACHE BOOL "")
set (ENABLE_CAPNP 0 CACHE BOOL "")
set (ENABLE_RDKAFKA 0 CACHE BOOL "")
set (ENABLE_ICU 0 CACHE BOOL "")
set (ENABLE_POCO_MONGODB 0 CACHE BOOL "")
set (ENABLE_POCO_NETSSL 0 CACHE BOOL "")
set (ENABLE_POCO_ODBC 0 CACHE BOOL "")

2
cmake/toolchain/darwin-x86_64/README.txt
View File

@ -1,2 +1,2 @@
wget https://github.com/phracker/MacOSX-SDKs/releases/download/10.14-beta4/MacOSX10.14.sdk.tar.xz
tar --strip-components=1 xJf MacOSX10.14.sdk.tar.xz
tar xJf MacOSX10.14.sdk.tar.xz --strip-components=1

4
cmake/toolchain/linux-aarch64/README.txt
View File

@ -1,2 +1,2 @@
wget https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel/gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz?revision=2e88a73f-d233-4f96-b1f4-d8b36e9bb0b9&la=en -O gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz
tar --strip-components=1 xJf gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz
wget 'https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel/gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz?revision=2e88a73f-d233-4f96-b1f4-d8b36e9bb0b9&la=en' -O gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz
tar xJf gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz --strip-components=1

30
cmake/tools.cmake
View File

@ -24,14 +24,34 @@ STRING(REGEX MATCHALL "[0-9]+" COMPILER_VERSION_LIST ${CMAKE_CXX_COMPILER_VERSIO
LIST(GET COMPILER_VERSION_LIST 0 COMPILER_VERSION_MAJOR)
option (LINKER_NAME "Linker name or full path")
find_program (LLD_PATH NAMES "ld.lld" "lld" "lld-${COMPILER_VERSION_MAJOR}")
find_program (GOLD_PATH NAMES "ld.gold" "gold")
if (COMPILER_GCC)
find_program (LLD_PATH NAMES "ld.lld")
find_program (GOLD_PATH NAMES "ld.gold")
else ()
find_program (LLD_PATH NAMES "ld.lld-${COMPILER_VERSION_MAJOR}" "lld-${COMPILER_VERSION_MAJOR}" "ld.lld" "lld")
find_program (GOLD_PATH NAMES "ld.gold" "gold")
endif ()
# We prefer LLD linker over Gold or BFD.
if (NOT LINKER_NAME)
if (LLD_PATH)
set (LINKER_NAME "lld")
elseif (GOLD_PATH)
set (LINKER_NAME "gold")
if (COMPILER_GCC)
# GCC driver requires one of supported linker names like "lld".
set (LINKER_NAME "lld")
else ()
# Clang driver simply allows full linker path.
set (LINKER_NAME ${LLD_PATH})
endif ()
endif ()
endif ()
if (NOT LINKER_NAME)
if (GOLD_PATH)
if (COMPILER_GCC)
set (LINKER_NAME "gold")
else ()
set (LINKER_NAME ${GOLD_PATH})
endif ()
endif ()
endif ()

34
contrib/CMakeLists.txt vendored
View File

@ -52,6 +52,7 @@ if (USE_INTERNAL_BTRIE_LIBRARY)
endif ()
if (USE_INTERNAL_ZLIB_LIBRARY)
unset (BUILD_SHARED_LIBS CACHE)
set (ZLIB_ENABLE_TESTS 0 CACHE INTERNAL "")
set (SKIP_INSTALL_ALL 1 CACHE INTERNAL "")
set (ZLIB_COMPAT 1 CACHE INTERNAL "") # also enables WITH_GZFILEOP
@ -106,20 +107,10 @@ if (USE_INTERNAL_H3_LIBRARY)
endif ()
if (USE_INTERNAL_SSL_LIBRARY)
if (NOT MAKE_STATIC_LIBRARIES)
set (BUILD_SHARED 1)
endif ()
add_subdirectory (openssl-cmake)
# By default, ${CMAKE_INSTALL_PREFIX}/etc/ssl is selected - that is not what we need.
# We need to use system wide ssl directory.
set (OPENSSLDIR "/etc/ssl")
set (LIBRESSL_SKIP_INSTALL 1 CACHE INTERNAL "")
add_subdirectory (ssl)
target_include_directories(${OPENSSL_CRYPTO_LIBRARY} SYSTEM PUBLIC ${OPENSSL_INCLUDE_DIR})
target_include_directories(${OPENSSL_SSL_LIBRARY} SYSTEM PUBLIC ${OPENSSL_INCLUDE_DIR})
# This is for Poco library
set (POCO_SKIP_OPENSSL_FIND 1)
add_library(OpenSSL::Crypto ALIAS ${OPENSSL_CRYPTO_LIBRARY})
add_library(OpenSSL::SSL ALIAS ${OPENSSL_SSL_LIBRARY})
endif ()
@ -164,6 +155,10 @@ if (ENABLE_ODBC AND USE_INTERNAL_ODBC_LIBRARY)
add_library(ODBC::ODBC ALIAS ${ODBC_LIBRARIES})
endif ()
if (ENABLE_ICU AND USE_INTERNAL_ICU_LIBRARY)
add_subdirectory (icu-cmake)
endif ()
if (USE_INTERNAL_PARQUET_LIBRARY)
if (USE_INTERNAL_PARQUET_LIBRARY_NATIVE_CMAKE)
# We dont use arrow's cmakefiles because they uses too many depends and download some libs in compile time
@ -312,6 +307,21 @@ if (USE_INTERNAL_HDFS3_LIBRARY)
add_subdirectory(libhdfs3-cmake)
endif ()
if (USE_INTERNAL_AWS_S3_LIBRARY)
set (save_CMAKE_C_FLAGS ${CMAKE_C_FLAGS})
set (save_CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES})
set (save_CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES})
set (save_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS})
set (save_CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH})
add_subdirectory(curl-cmake)
set (CMAKE_C_FLAGS ${save_CMAKE_C_FLAGS})
set (CMAKE_REQUIRED_LIBRARIES ${save_CMAKE_REQUIRED_LIBRARIES})
set (CMAKE_CMAKE_REQUIRED_INCLUDES ${save_CMAKE_REQUIRED_INCLUDES})
set (CMAKE_REQUIRED_FLAGS ${save_CMAKE_REQUIRED_FLAGS})
set (CMAKE_CMAKE_MODULE_PATH ${save_CMAKE_MODULE_PATH})
add_subdirectory(aws-s3-cmake)
endif ()
if (USE_BASE64)
add_subdirectory (base64-cmake)
endif()

1
contrib/aws vendored Submodule

@ -0,0 +1 @@
Subproject commit 45dd8552d3c492defca79d2720bcc809e35654da

1
contrib/aws-c-common vendored Submodule

@ -0,0 +1 @@
Subproject commit 736a82d1697c108b04a277e66438a7f4e19b6857

1
contrib/aws-c-event-stream vendored Submodule

@ -0,0 +1 @@
Subproject commit 3bc33662f9ccff4f4cbcf9509cc78c26e022fde0

1
contrib/aws-checksums vendored Submodule

@ -0,0 +1 @@
Subproject commit 519d6d9093819b6cf89ffff589a27ef8f83d0f65

107
contrib/aws-s3-cmake/CMakeLists.txt Normal file
View File

@ -0,0 +1,107 @@
SET(AWS_S3_LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/aws/aws-cpp-sdk-s3)
SET(AWS_CORE_LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/aws/aws-cpp-sdk-core)
SET(AWS_CHECKSUMS_LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/aws-checksums)
SET(AWS_COMMON_LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/aws-c-common)
SET(AWS_EVENT_STREAM_LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/aws-c-event-stream)
OPTION(USE_AWS_MEMORY_MANAGEMENT "Aws memory management" OFF)
configure_file("${AWS_CORE_LIBRARY_DIR}/include/aws/core/SDKConfig.h.in"
"${CMAKE_CURRENT_BINARY_DIR}/include/aws/core/SDKConfig.h" @ONLY)
configure_file("${AWS_COMMON_LIBRARY_DIR}/include/aws/common/config.h.in"
"${CMAKE_CURRENT_BINARY_DIR}/include/aws/common/config.h" @ONLY)
file(GLOB AWS_CORE_SOURCES
"${AWS_CORE_LIBRARY_DIR}/source/*.cpp"
"${AWS_CORE_LIBRARY_DIR}/source/auth/*.cpp"
"${AWS_CORE_LIBRARY_DIR}/source/client/*.cpp"
"${AWS_CORE_LIBRARY_DIR}/source/http/*.cpp"
"${AWS_CORE_LIBRARY_DIR}/source/http/standard/*.cpp"
"${AWS_CORE_LIBRARY_DIR}/source/http/curl/*.cpp"
"${AWS_CORE_LIBRARY_DIR}/source/config/*.cpp"
"${AWS_CORE_LIBRARY_DIR}/source/external/cjson/*.cpp"
"${AWS_CORE_LIBRARY_DIR}/source/external/tinyxml2/*.cpp"
"${AWS_CORE_LIBRARY_DIR}/source/internal/*.cpp"
"${AWS_CORE_LIBRARY_DIR}/source/monitoring/*.cpp"
"${AWS_CORE_LIBRARY_DIR}/source/net/*.cpp"
"${AWS_CORE_LIBRARY_DIR}/source/linux-shared/*.cpp"
"${AWS_CORE_LIBRARY_DIR}/source/platform/linux-shared/*.cpp"
"${AWS_CORE_LIBRARY_DIR}/source/utils/*.cpp"
"${AWS_CORE_LIBRARY_DIR}/source/utils/base64/*.cpp"
"${AWS_CORE_LIBRARY_DIR}/source/utils/event/*.cpp"
"${AWS_CORE_LIBRARY_DIR}/source/utils/crypto/*.cpp"
"${AWS_CORE_LIBRARY_DIR}/source/utils/crypto/openssl/*.cpp"
"${AWS_CORE_LIBRARY_DIR}/source/utils/crypto/factory/*.cpp"
"${AWS_CORE_LIBRARY_DIR}/source/utils/json/*.cpp"
"${AWS_CORE_LIBRARY_DIR}/source/utils/logging/*.cpp"
"${AWS_CORE_LIBRARY_DIR}/source/utils/memory/*.cpp"
"${AWS_CORE_LIBRARY_DIR}/source/utils/memory/stl/*.cpp"
"${AWS_CORE_LIBRARY_DIR}/source/utils/stream/*.cpp"
"${AWS_CORE_LIBRARY_DIR}/source/utils/threading/*.cpp"
"${AWS_CORE_LIBRARY_DIR}/source/utils/xml/*.cpp"
)
file(GLOB AWS_S3_SOURCES
"${AWS_S3_LIBRARY_DIR}/source/*.cpp"
)
file(GLOB AWS_S3_MODEL_SOURCES
"${AWS_S3_LIBRARY_DIR}/source/model/*.cpp"
)
file(GLOB AWS_EVENT_STREAM_SOURCES
"${AWS_EVENT_STREAM_LIBRARY_DIR}/source/*.c"
)
file(GLOB AWS_COMMON_SOURCES
"${AWS_COMMON_LIBRARY_DIR}/source/*.c"
"${AWS_COMMON_LIBRARY_DIR}/source/posix/*.c"
)
file(GLOB AWS_CHECKSUMS_SOURCES
"${AWS_CHECKSUMS_LIBRARY_DIR}/source/*.c"
"${AWS_CHECKSUMS_LIBRARY_DIR}/source/intel/*.c"
"${AWS_CHECKSUMS_LIBRARY_DIR}/source/arm/*.c"
)
file(GLOB S3_UNIFIED_SRC
${AWS_EVENT_STREAM_SOURCES}
${AWS_COMMON_SOURCES}
${AWS_S3_SOURCES}
${AWS_S3_MODEL_SOURCES}
${AWS_CORE_SOURCES}
)
set(S3_INCLUDES
"${CMAKE_CURRENT_SOURCE_DIR}/include/"
"${AWS_COMMON_LIBRARY_DIR}/include/"
"${AWS_EVENT_STREAM_LIBRARY_DIR}/include/"
"${AWS_S3_LIBRARY_DIR}/include/"
"${AWS_CORE_LIBRARY_DIR}/include/"
"${CMAKE_CURRENT_BINARY_DIR}/include/"
)
add_library(aws_s3_checksums ${AWS_CHECKSUMS_SOURCES})
target_include_directories(aws_s3_checksums PUBLIC "${AWS_CHECKSUMS_LIBRARY_DIR}/include/")
if(CMAKE_BUILD_TYPE STREQUAL "" OR CMAKE_BUILD_TYPE STREQUAL "Debug")
target_compile_definitions(aws_s3_checksums PRIVATE "-DDEBUG_BUILD")
endif()
set_target_properties(aws_s3_checksums PROPERTIES COMPILE_OPTIONS -fPIC)
set_target_properties(aws_s3_checksums PROPERTIES LINKER_LANGUAGE C)
set_property(TARGET aws_s3_checksums PROPERTY C_STANDARD 99)
add_library(aws_s3 ${S3_UNIFIED_SRC})
target_compile_definitions(aws_s3 PUBLIC -DENABLE_CURL_CLIENT)
target_compile_definitions(aws_s3 PUBLIC "AWS_SDK_VERSION_MAJOR=1")
target_compile_definitions(aws_s3 PUBLIC "AWS_SDK_VERSION_MINOR=7")
target_compile_definitions(aws_s3 PUBLIC "AWS_SDK_VERSION_PATCH=231")
target_include_directories(aws_s3 PUBLIC ${S3_INCLUDES} "${CMAKE_BINARY_DIR}/install")
if (OPENSSL_FOUND)
target_compile_definitions(aws_s3 PUBLIC -DENABLE_OPENSSL_ENCRYPTION)
target_link_libraries(aws_s3 PRIVATE ${OPENSSL_LIBRARIES})
endif()
target_link_libraries(aws_s3 PRIVATE aws_s3_checksums libcurl)

14
contrib/capnproto-cmake/CMakeLists.txt
View File

@ -28,8 +28,7 @@ set (KJ_SRCS
)
add_library(kj ${KJ_SRCS})
target_include_directories(kj PUBLIC ${CAPNPROTO_SOURCE_DIR})
target_compile_options(kj PUBLIC -Wno-non-virtual-dtor)
target_include_directories(kj SYSTEM PUBLIC ${CAPNPROTO_SOURCE_DIR})
set (CAPNP_SRCS
${CAPNPROTO_SOURCE_DIR}/capnp/c++.capnp.c++
@ -52,6 +51,17 @@ set (CAPNP_SRCS
add_library(capnp ${CAPNP_SRCS})
target_link_libraries(capnp PUBLIC kj)
# The library has substandard code
if (COMPILER_GCC)
set (SUPPRESS_WARNINGS -Wno-non-virtual-dtor -Wno-sign-compare -Wno-strict-aliasing -Wno-maybe-uninitialized
-Wno-deprecated-declarations -Wno-class-memaccess)
elseif (COMPILER_CLANG)
set (SUPPRESS_WARNINGS -Wno-non-virtual-dtor -Wno-sign-compare -Wno-strict-aliasing -Wno-deprecated-declarations)
endif ()
target_compile_options(kj PRIVATE ${SUPPRESS_WARNINGS})
target_compile_options(capnp PRIVATE ${SUPPRESS_WARNINGS})
set (CAPNPC_SRCS
${CAPNPROTO_SOURCE_DIR}/capnp/compiler/type-id.c++
${CAPNPROTO_SOURCE_DIR}/capnp/compiler/error-reporter.c++

1
contrib/curl vendored Submodule

@ -0,0 +1 @@
Subproject commit 3b8bbbbd1609c638a3d3d0acb148a33dedb67be3

61
contrib/curl-cmake/CMake/CurlSymbolHiding.cmake Normal file
View File

@ -0,0 +1,61 @@
include(CheckCSourceCompiles)
option(CURL_HIDDEN_SYMBOLS "Set to ON to hide libcurl internal symbols (=hide all symbols that aren't officially external)." ON)
mark_as_advanced(CURL_HIDDEN_SYMBOLS)
if(CURL_HIDDEN_SYMBOLS)
set(SUPPORTS_SYMBOL_HIDING FALSE)
if(CMAKE_C_COMPILER_ID MATCHES "Clang")
set(SUPPORTS_SYMBOL_HIDING TRUE)
set(_SYMBOL_EXTERN "__attribute__ ((__visibility__ (\"default\")))")
set(_CFLAG_SYMBOLS_HIDE "-fvisibility=hidden")
elseif(CMAKE_COMPILER_IS_GNUCC)
if(NOT CMAKE_VERSION VERSION_LESS 2.8.10)
set(GCC_VERSION ${CMAKE_C_COMPILER_VERSION})
else()
execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion
OUTPUT_VARIABLE GCC_VERSION)
endif()
if(NOT GCC_VERSION VERSION_LESS 3.4)
# note: this is considered buggy prior to 4.0 but the autotools don't care, so let's ignore that fact
set(SUPPORTS_SYMBOL_HIDING TRUE)
set(_SYMBOL_EXTERN "__attribute__ ((__visibility__ (\"default\")))")
set(_CFLAG_SYMBOLS_HIDE "-fvisibility=hidden")
endif()
elseif(CMAKE_C_COMPILER_ID MATCHES "SunPro" AND NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 8.0)
set(SUPPORTS_SYMBOL_HIDING TRUE)
set(_SYMBOL_EXTERN "__global")
set(_CFLAG_SYMBOLS_HIDE "-xldscope=hidden")
elseif(CMAKE_C_COMPILER_ID MATCHES "Intel" AND NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 9.0)
# note: this should probably just check for version 9.1.045 but I'm not 100% sure
# so let's do it the same way autotools do.
set(SUPPORTS_SYMBOL_HIDING TRUE)
set(_SYMBOL_EXTERN "__attribute__ ((__visibility__ (\"default\")))")
set(_CFLAG_SYMBOLS_HIDE "-fvisibility=hidden")
check_c_source_compiles("#include <stdio.h>
int main (void) { printf(\"icc fvisibility bug test\"); return 0; }" _no_bug)
if(NOT _no_bug)
set(SUPPORTS_SYMBOL_HIDING FALSE)
set(_SYMBOL_EXTERN "")
set(_CFLAG_SYMBOLS_HIDE "")
endif()
elseif(MSVC)
set(SUPPORTS_SYMBOL_HIDING TRUE)
endif()
set(HIDES_CURL_PRIVATE_SYMBOLS ${SUPPORTS_SYMBOL_HIDING})
elseif(MSVC)
if(NOT CMAKE_VERSION VERSION_LESS 3.7)
set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS TRUE) #present since 3.4.3 but broken
set(HIDES_CURL_PRIVATE_SYMBOLS FALSE)
else()
message(WARNING "Hiding private symbols regardless CURL_HIDDEN_SYMBOLS being disabled.")
set(HIDES_CURL_PRIVATE_SYMBOLS TRUE)
endif()
else()
set(HIDES_CURL_PRIVATE_SYMBOLS FALSE)
endif()
set(CURL_CFLAG_SYMBOLS_HIDE ${_CFLAG_SYMBOLS_HIDE})
set(CURL_EXTERN_SYMBOL ${_SYMBOL_EXTERN})

617
contrib/curl-cmake/CMake/CurlTests.c Normal file
View File

@ -0,0 +1,617 @@
/***************************************************************************
* _ _ ____ _
* Project ___| | | | _ \| |
* / __| | | | |_) | |
* | (__| |_| | _ <| |___
* \___|\___/|_| \_\_____|
*
* Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
*
* This software is licensed as described in the file COPYING, which
* you should have received as part of this distribution. The terms
* are also available at https://curl.haxx.se/docs/copyright.html.
*
* You may opt to use, copy, modify, merge, publish, distribute and/or sell
* copies of the Software, and permit persons to whom the Software is
* furnished to do so, under the terms of the COPYING file.
*
* This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
* KIND, either express or implied.
*
***************************************************************************/
#ifdef TIME_WITH_SYS_TIME
/* Time with sys/time test */
#include <sys/types.h>
#include <sys/time.h>
#include <time.h>
int
main ()
{
if ((struct tm *) 0)
return 0;
;
return 0;
}
#endif
#ifdef HAVE_FCNTL_O_NONBLOCK
/* headers for FCNTL_O_NONBLOCK test */
#include <sys/types.h>
#include <unistd.h>
#include <fcntl.h>
/* */
#if defined(sun) || defined(__sun__) || \
defined(__SUNPRO_C) || defined(__SUNPRO_CC)
# if defined(__SVR4) || defined(__srv4__)
# define PLATFORM_SOLARIS
# else
# define PLATFORM_SUNOS4
# endif
#endif
#if (defined(_AIX) || defined(__xlC__)) && !defined(_AIX41)
# define PLATFORM_AIX_V3
#endif
/* */
#if defined(PLATFORM_SUNOS4) || defined(PLATFORM_AIX_V3) || defined(__BEOS__)
#error "O_NONBLOCK does not work on this platform"
#endif
int
main ()
{
/* O_NONBLOCK source test */
int flags = 0;
if(0 != fcntl(0, F_SETFL, flags | O_NONBLOCK))
return 1;
return 0;
}
#endif
/* tests for gethostbyaddr_r or gethostbyname_r */
#if defined(HAVE_GETHOSTBYADDR_R_5_REENTRANT) || \
defined(HAVE_GETHOSTBYADDR_R_7_REENTRANT) || \
defined(HAVE_GETHOSTBYADDR_R_8_REENTRANT) || \
defined(HAVE_GETHOSTBYNAME_R_3_REENTRANT) || \
defined(HAVE_GETHOSTBYNAME_R_5_REENTRANT) || \
defined(HAVE_GETHOSTBYNAME_R_6_REENTRANT)
# define _REENTRANT
/* no idea whether _REENTRANT is always set, just invent a new flag */
# define TEST_GETHOSTBYFOO_REENTRANT
#endif
#if defined(HAVE_GETHOSTBYADDR_R_5) || \
defined(HAVE_GETHOSTBYADDR_R_7) || \
defined(HAVE_GETHOSTBYADDR_R_8) || \
defined(HAVE_GETHOSTBYNAME_R_3) || \
defined(HAVE_GETHOSTBYNAME_R_5) || \
defined(HAVE_GETHOSTBYNAME_R_6) || \
defined(TEST_GETHOSTBYFOO_REENTRANT)
#include <sys/types.h>
#include <netdb.h>
int main(void)
{
char *address = "example.com";
int length = 0;
int type = 0;
struct hostent h;
int rc = 0;
#if defined(HAVE_GETHOSTBYADDR_R_5) || \
defined(HAVE_GETHOSTBYADDR_R_5_REENTRANT) || \
\
defined(HAVE_GETHOSTBYNAME_R_3) || \
defined(HAVE_GETHOSTBYNAME_R_3_REENTRANT)
struct hostent_data hdata;
#elif defined(HAVE_GETHOSTBYADDR_R_7) || \
defined(HAVE_GETHOSTBYADDR_R_7_REENTRANT) || \
defined(HAVE_GETHOSTBYADDR_R_8) || \
defined(HAVE_GETHOSTBYADDR_R_8_REENTRANT) || \
\
defined(HAVE_GETHOSTBYNAME_R_5) || \
defined(HAVE_GETHOSTBYNAME_R_5_REENTRANT) || \
defined(HAVE_GETHOSTBYNAME_R_6) || \
defined(HAVE_GETHOSTBYNAME_R_6_REENTRANT)
char buffer[8192];
int h_errnop;
struct hostent *hp;
#endif
#ifndef gethostbyaddr_r
(void)gethostbyaddr_r;
#endif
#if defined(HAVE_GETHOSTBYADDR_R_5) || \
defined(HAVE_GETHOSTBYADDR_R_5_REENTRANT)
rc = gethostbyaddr_r(address, length, type, &h, &hdata);
(void)rc;
#elif defined(HAVE_GETHOSTBYADDR_R_7) || \
defined(HAVE_GETHOSTBYADDR_R_7_REENTRANT)
hp = gethostbyaddr_r(address, length, type, &h, buffer, 8192, &h_errnop);
(void)hp;
#elif defined(HAVE_GETHOSTBYADDR_R_8) || \
defined(HAVE_GETHOSTBYADDR_R_8_REENTRANT)
rc = gethostbyaddr_r(address, length, type, &h, buffer, 8192, &hp, &h_errnop);
(void)rc;
#endif
#if defined(HAVE_GETHOSTBYNAME_R_3) || \
defined(HAVE_GETHOSTBYNAME_R_3_REENTRANT)
rc = gethostbyname_r(address, &h, &hdata);
#elif defined(HAVE_GETHOSTBYNAME_R_5) || \
defined(HAVE_GETHOSTBYNAME_R_5_REENTRANT)
rc = gethostbyname_r(address, &h, buffer, 8192, &h_errnop);
(void)hp; /* not used for test */
#elif defined(HAVE_GETHOSTBYNAME_R_6) || \
defined(HAVE_GETHOSTBYNAME_R_6_REENTRANT)
rc = gethostbyname_r(address, &h, buffer, 8192, &hp, &h_errnop);
#endif
(void)length;
(void)type;
(void)rc;
return 0;
}
#endif
#ifdef HAVE_SOCKLEN_T
#ifdef _WIN32
#include <ws2tcpip.h>
#else
#include <sys/types.h>
#include <sys/socket.h>
#endif
int
main ()
{
if ((socklen_t *) 0)
return 0;
if (sizeof (socklen_t))
return 0;
;
return 0;
}
#endif
#ifdef HAVE_IN_ADDR_T
#include <sys/types.h>
#include <sys/socket.h>
#include <arpa/inet.h>
int
main ()
{
if ((in_addr_t *) 0)
return 0;
if (sizeof (in_addr_t))
return 0;
;
return 0;
}
#endif
#ifdef HAVE_BOOL_T
#ifdef HAVE_SYS_TYPES_H
#include <sys/types.h>
#endif
#ifdef HAVE_STDBOOL_H
#include <stdbool.h>
#endif
int
main ()
{
if (sizeof (bool *) )
return 0;
;
return 0;
}
#endif
#ifdef STDC_HEADERS
#include <stdlib.h>
#include <stdarg.h>
#include <string.h>
#include <float.h>
int main() { return 0; }
#endif
#ifdef RETSIGTYPE_TEST
#include <sys/types.h>
#include <signal.h>
#ifdef signal
# undef signal
#endif
#ifdef __cplusplus
extern "C" void (*signal (int, void (*)(int)))(int);
#else
void (*signal ()) ();
#endif
int
main ()
{
return 0;
}
#endif
#ifdef HAVE_INET_NTOA_R_DECL
#include <arpa/inet.h>
typedef void (*func_type)();
int main()
{
#ifndef inet_ntoa_r
func_type func;
func = (func_type)inet_ntoa_r;
(void)func;
#endif
return 0;
}
#endif
#ifdef HAVE_INET_NTOA_R_DECL_REENTRANT
#define _REENTRANT
#include <arpa/inet.h>
typedef void (*func_type)();
int main()
{
#ifndef inet_ntoa_r
func_type func;
func = (func_type)&inet_ntoa_r;
(void)func;
#endif
return 0;
}
#endif
#ifdef HAVE_GETADDRINFO
#include <netdb.h>
#include <sys/types.h>
#include <sys/socket.h>
int main(void) {
struct addrinfo hints, *ai;
int error;
memset(&hints, 0, sizeof(hints));
hints.ai_family = AF_UNSPEC;
hints.ai_socktype = SOCK_STREAM;
#ifndef getaddrinfo
(void)getaddrinfo;
#endif
error = getaddrinfo("127.0.0.1", "8080", &hints, &ai);
if (error) {
return 1;
}
return 0;
}
#endif
#ifdef HAVE_FILE_OFFSET_BITS
#ifdef _FILE_OFFSET_BITS
#undef _FILE_OFFSET_BITS
#endif
#define _FILE_OFFSET_BITS 64
#include <sys/types.h>
/* Check that off_t can represent 2**63 - 1 correctly.
We can't simply define LARGE_OFF_T to be 9223372036854775807,
since some C++ compilers masquerading as C compilers
incorrectly reject 9223372036854775807. */
#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
&& LARGE_OFF_T % 2147483647 == 1)
? 1 : -1];
int main () { ; return 0; }
#endif
#ifdef HAVE_IOCTLSOCKET
/* includes start */
#ifdef HAVE_WINDOWS_H
# ifndef WIN32_LEAN_AND_MEAN
# define WIN32_LEAN_AND_MEAN
# endif
# include <windows.h>
# ifdef HAVE_WINSOCK2_H
# include <winsock2.h>
# else
# ifdef HAVE_WINSOCK_H
# include <winsock.h>
# endif
# endif
#endif
int
main ()
{
/* ioctlsocket source code */
int socket;
unsigned long flags = ioctlsocket(socket, FIONBIO, &flags);
;
return 0;
}
#endif
#ifdef HAVE_IOCTLSOCKET_CAMEL
/* includes start */
#ifdef HAVE_WINDOWS_H
# ifndef WIN32_LEAN_AND_MEAN
# define WIN32_LEAN_AND_MEAN
# endif
# include <windows.h>
# ifdef HAVE_WINSOCK2_H
# include <winsock2.h>
# else
# ifdef HAVE_WINSOCK_H
# include <winsock.h>
# endif
# endif
#endif
int
main ()
{
/* IoctlSocket source code */
if(0 != IoctlSocket(0, 0, 0))
return 1;
;
return 0;
}
#endif
#ifdef HAVE_IOCTLSOCKET_CAMEL_FIONBIO
/* includes start */
#ifdef HAVE_WINDOWS_H
# ifndef WIN32_LEAN_AND_MEAN
# define WIN32_LEAN_AND_MEAN
# endif
# include <windows.h>
# ifdef HAVE_WINSOCK2_H
# include <winsock2.h>
# else
# ifdef HAVE_WINSOCK_H
# include <winsock.h>
# endif
# endif
#endif
int
main ()
{
/* IoctlSocket source code */
long flags = 0;
if(0 != ioctlsocket(0, FIONBIO, &flags))
return 1;
;
return 0;
}
#endif
#ifdef HAVE_IOCTLSOCKET_FIONBIO
/* includes start */
#ifdef HAVE_WINDOWS_H
# ifndef WIN32_LEAN_AND_MEAN
# define WIN32_LEAN_AND_MEAN
# endif
# include <windows.h>
# ifdef HAVE_WINSOCK2_H
# include <winsock2.h>
# else
# ifdef HAVE_WINSOCK_H
# include <winsock.h>
# endif
# endif
#endif
int
main ()
{
int flags = 0;
if(0 != ioctlsocket(0, FIONBIO, &flags))
return 1;
;
return 0;
}
#endif
#ifdef HAVE_IOCTL_FIONBIO
/* headers for FIONBIO test */
/* includes start */
#ifdef HAVE_SYS_TYPES_H
# include <sys/types.h>
#endif
#ifdef HAVE_UNISTD_H
# include <unistd.h>
#endif
#ifdef HAVE_SYS_SOCKET_H
# include <sys/socket.h>
#endif
#ifdef HAVE_SYS_IOCTL_H
# include <sys/ioctl.h>
#endif
#ifdef HAVE_STROPTS_H
# include <stropts.h>
#endif
int
main ()
{
int flags = 0;
if(0 != ioctl(0, FIONBIO, &flags))
return 1;
;
return 0;
}
#endif
#ifdef HAVE_IOCTL_SIOCGIFADDR
/* headers for FIONBIO test */
/* includes start */
#ifdef HAVE_SYS_TYPES_H
# include <sys/types.h>
#endif
#ifdef HAVE_UNISTD_H
# include <unistd.h>
#endif
#ifdef HAVE_SYS_SOCKET_H
# include <sys/socket.h>
#endif
#ifdef HAVE_SYS_IOCTL_H
# include <sys/ioctl.h>
#endif
#ifdef HAVE_STROPTS_H
# include <stropts.h>
#endif
#include <net/if.h>
int
main ()
{
struct ifreq ifr;
if(0 != ioctl(0, SIOCGIFADDR, &ifr))
return 1;
;
return 0;
}
#endif
#ifdef HAVE_SETSOCKOPT_SO_NONBLOCK
/* includes start */
#ifdef HAVE_WINDOWS_H
# ifndef WIN32_LEAN_AND_MEAN
# define WIN32_LEAN_AND_MEAN
# endif
# include <windows.h>
# ifdef HAVE_WINSOCK2_H
# include <winsock2.h>
# else
# ifdef HAVE_WINSOCK_H
# include <winsock.h>
# endif
# endif
#endif
/* includes start */
#ifdef HAVE_SYS_TYPES_H
# include <sys/types.h>
#endif
#ifdef HAVE_SYS_SOCKET_H
# include <sys/socket.h>
#endif
/* includes end */
int
main ()
{
if(0 != setsockopt(0, SOL_SOCKET, SO_NONBLOCK, 0, 0))
return 1;
;
return 0;
}
#endif
#ifdef HAVE_GLIBC_STRERROR_R
#include <string.h>
#include <errno.h>
void check(char c) {}
int
main () {
char buffer[1024];
/* This will not compile if strerror_r does not return a char* */
check(strerror_r(EACCES, buffer, sizeof(buffer))[0]);
return 0;
}
#endif
#ifdef HAVE_POSIX_STRERROR_R
#include <string.h>
#include <errno.h>
/* float, because a pointer can't be implicitly cast to float */
void check(float f) {}
int
main () {
char buffer[1024];
/* This will not compile if strerror_r does not return an int */
check(strerror_r(EACCES, buffer, sizeof(buffer)));
return 0;
}
#endif
#ifdef HAVE_FSETXATTR_6
#include <sys/xattr.h> /* header from libc, not from libattr */
int
main() {
fsetxattr(0, 0, 0, 0, 0, 0);
return 0;
}
#endif
#ifdef HAVE_FSETXATTR_5
#include <sys/xattr.h> /* header from libc, not from libattr */
int
main() {
fsetxattr(0, 0, 0, 0, 0);
return 0;
}
#endif
#ifdef HAVE_CLOCK_GETTIME_MONOTONIC
#include <time.h>
int
main() {
struct timespec ts = {0, 0};
clock_gettime(CLOCK_MONOTONIC, &ts);
return 0;
}
#endif
#ifdef HAVE_BUILTIN_AVAILABLE
int
main() {
if(__builtin_available(macOS 10.12, *)) {}
return 0;
}
#endif
#ifdef HAVE_VARIADIC_MACROS_C99
#define c99_vmacro3(first, ...) fun3(first, __VA_ARGS__)
#define c99_vmacro2(first, ...) fun2(first, __VA_ARGS__)
int fun3(int arg1, int arg2, int arg3);
int fun2(int arg1, int arg2);
int fun3(int arg1, int arg2, int arg3) {
return arg1 + arg2 + arg3;
}
int fun2(int arg1, int arg2) {
return arg1 + arg2;
}
int
main() {
int res3 = c99_vmacro3(1, 2, 3);
int res2 = c99_vmacro2(1, 2);
(void)res3;
(void)res2;
return 0;
}
#endif
#ifdef HAVE_VARIADIC_MACROS_GCC
#define gcc_vmacro3(first, args...) fun3(first, args)
#define gcc_vmacro2(first, args...) fun2(first, args)
int fun3(int arg1, int arg2, int arg3);
int fun2(int arg1, int arg2);
int fun3(int arg1, int arg2, int arg3) {
return arg1 + arg2 + arg3;
}
int fun2(int arg1, int arg2) {
return arg1 + arg2;
}
int
main() {
int res3 = gcc_vmacro3(1, 2, 3);
int res2 = gcc_vmacro2(1, 2);
(void)res3;
(void)res2;
return 0;
}
#endif

84
contrib/curl-cmake/CMake/Macros.cmake Normal file
View File

@ -0,0 +1,84 @@
#File defines convenience macros for available feature testing
# This macro checks if the symbol exists in the library and if it
# does, it prepends library to the list. It is intended to be called
# multiple times with a sequence of possibly dependent libraries in
# order of least-to-most-dependent. Some libraries depend on others
# to link correctly.
macro(check_library_exists_concat LIBRARY SYMBOL VARIABLE)
check_library_exists("${LIBRARY};${CURL_LIBS}" ${SYMBOL} "${CMAKE_LIBRARY_PATH}"
${VARIABLE})
if(${VARIABLE})
set(CURL_LIBS ${LIBRARY} ${CURL_LIBS})
endif()
endmacro()
# Check if header file exists and add it to the list.
# This macro is intended to be called multiple times with a sequence of
# possibly dependent header files. Some headers depend on others to be
# compiled correctly.
macro(check_include_file_concat FILE VARIABLE)
check_include_files("${CURL_INCLUDES};${FILE}" ${VARIABLE})
if(${VARIABLE})
set(CURL_INCLUDES ${CURL_INCLUDES} ${FILE})
set(CURL_TEST_DEFINES "${CURL_TEST_DEFINES} -D${VARIABLE}")
endif()
endmacro()
# For other curl specific tests, use this macro.
macro(curl_internal_test CURL_TEST)
if(NOT DEFINED "${CURL_TEST}")
set(MACRO_CHECK_FUNCTION_DEFINITIONS
"-D${CURL_TEST} ${CURL_TEST_DEFINES} ${CMAKE_REQUIRED_FLAGS}")
if(CMAKE_REQUIRED_LIBRARIES)
set(CURL_TEST_ADD_LIBRARIES
"-DLINK_LIBRARIES:STRING=${CMAKE_REQUIRED_LIBRARIES}")
endif()
try_compile(${CURL_TEST}
${CMAKE_BINARY_DIR}
${CMAKE_CURRENT_SOURCE_DIR}/CMake/CurlTests.c
CMAKE_FLAGS -DCOMPILE_DEFINITIONS:STRING=${MACRO_CHECK_FUNCTION_DEFINITIONS}
"${CURL_TEST_ADD_LIBRARIES}"
OUTPUT_VARIABLE OUTPUT)
if(${CURL_TEST})
set(${CURL_TEST} 1 CACHE INTERNAL "Curl test ${FUNCTION}")
file(APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeOutput.log
"Performing Curl Test ${CURL_TEST} passed with the following output:\n"
"${OUTPUT}\n")
else()
set(${CURL_TEST} "" CACHE INTERNAL "Curl test ${FUNCTION}")
file(APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeError.log
"Performing Curl Test ${CURL_TEST} failed with the following output:\n"
"${OUTPUT}\n")
endif()
endif()
endmacro()
macro(curl_nroff_check)
find_program(NROFF NAMES gnroff nroff)
if(NROFF)
# Need a way to write to stdin, this will do
file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/nroff-input.txt" "test")
# Tests for a valid nroff option to generate a manpage
foreach(_MANOPT "-man" "-mandoc")
execute_process(COMMAND "${NROFF}" ${_MANOPT}
OUTPUT_VARIABLE NROFF_MANOPT_OUTPUT
INPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/nroff-input.txt"
ERROR_QUIET)
# Save the option if it was valid
if(NROFF_MANOPT_OUTPUT)
set(NROFF_MANOPT ${_MANOPT})
set(NROFF_USEFUL ON)
break()
endif()
endforeach()
# No need for the temporary file
file(REMOVE "${CMAKE_CURRENT_BINARY_DIR}/nroff-input.txt")
if(NOT NROFF_USEFUL)
message(WARNING "Found no *nroff option to get plaintext from man pages")
endif()
else()
message(WARNING "Found no *nroff program")
endif()
endmacro()

260
contrib/curl-cmake/CMake/OtherTests.cmake Normal file
View File

@ -0,0 +1,260 @@
include(CheckCSourceCompiles)
# The begin of the sources (macros and includes)
set(_source_epilogue "#undef inline")
macro(add_header_include check header)
if(${check})
set(_source_epilogue "${_source_epilogue}\n#include <${header}>")
endif()
endmacro()
set(signature_call_conv)
if(HAVE_WINDOWS_H)
add_header_include(HAVE_WINSOCK2_H "winsock2.h")
add_header_include(HAVE_WINDOWS_H "windows.h")
add_header_include(HAVE_WINSOCK_H "winsock.h")
set(_source_epilogue
"${_source_epilogue}\n#ifndef WIN32_LEAN_AND_MEAN\n#define WIN32_LEAN_AND_MEAN\n#endif")
set(signature_call_conv "PASCAL")
if(HAVE_LIBWS2_32)
set(CMAKE_REQUIRED_LIBRARIES ws2_32)
endif()
else()
add_header_include(HAVE_SYS_TYPES_H "sys/types.h")
add_header_include(HAVE_SYS_SOCKET_H "sys/socket.h")
endif()
set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
check_c_source_compiles("${_source_epilogue}
int main(void) {
recv(0, 0, 0, 0);
return 0;
}" curl_cv_recv)
if(curl_cv_recv)
if(NOT DEFINED curl_cv_func_recv_args OR "${curl_cv_func_recv_args}" STREQUAL "unknown")
foreach(recv_retv "int" "ssize_t" )
foreach(recv_arg1 "SOCKET" "int" )
foreach(recv_arg2 "char *" "void *" )
foreach(recv_arg3 "int" "size_t" "socklen_t" "unsigned int")
foreach(recv_arg4 "int" "unsigned int")
if(NOT curl_cv_func_recv_done)
unset(curl_cv_func_recv_test CACHE)
check_c_source_compiles("
${_source_epilogue}
extern ${recv_retv} ${signature_call_conv}
recv(${recv_arg1}, ${recv_arg2}, ${recv_arg3}, ${recv_arg4});
int main(void) {
${recv_arg1} s=0;
${recv_arg2} buf=0;
${recv_arg3} len=0;
${recv_arg4} flags=0;
${recv_retv} res = recv(s, buf, len, flags);
(void) res;
return 0;
}"
curl_cv_func_recv_test)
if(curl_cv_func_recv_test)
set(curl_cv_func_recv_args
"${recv_arg1},${recv_arg2},${recv_arg3},${recv_arg4},${recv_retv}")
set(RECV_TYPE_ARG1 "${recv_arg1}")
set(RECV_TYPE_ARG2 "${recv_arg2}")
set(RECV_TYPE_ARG3 "${recv_arg3}")
set(RECV_TYPE_ARG4 "${recv_arg4}")
set(RECV_TYPE_RETV "${recv_retv}")
set(HAVE_RECV 1)
set(curl_cv_func_recv_done 1)
endif()
endif()
endforeach()
endforeach()
endforeach()
endforeach()
endforeach()
else()
string(REGEX REPLACE "^([^,]*),[^,]*,[^,]*,[^,]*,[^,]*$" "\\1" RECV_TYPE_ARG1 "${curl_cv_func_recv_args}")
string(REGEX REPLACE "^[^,]*,([^,]*),[^,]*,[^,]*,[^,]*$" "\\1" RECV_TYPE_ARG2 "${curl_cv_func_recv_args}")
string(REGEX REPLACE "^[^,]*,[^,]*,([^,]*),[^,]*,[^,]*$" "\\1" RECV_TYPE_ARG3 "${curl_cv_func_recv_args}")
string(REGEX REPLACE "^[^,]*,[^,]*,[^,]*,([^,]*),[^,]*$" "\\1" RECV_TYPE_ARG4 "${curl_cv_func_recv_args}")
string(REGEX REPLACE "^[^,]*,[^,]*,[^,]*,[^,]*,([^,]*)$" "\\1" RECV_TYPE_RETV "${curl_cv_func_recv_args}")
endif()
if("${curl_cv_func_recv_args}" STREQUAL "unknown")
message(FATAL_ERROR "Cannot find proper types to use for recv args")
endif()
else()
message(FATAL_ERROR "Unable to link function recv")
endif()
set(curl_cv_func_recv_args "${curl_cv_func_recv_args}" CACHE INTERNAL "Arguments for recv")
set(HAVE_RECV 1)
check_c_source_compiles("${_source_epilogue}
int main(void) {
send(0, 0, 0, 0);
return 0;
}" curl_cv_send)
if(curl_cv_send)
if(NOT DEFINED curl_cv_func_send_args OR "${curl_cv_func_send_args}" STREQUAL "unknown")
foreach(send_retv "int" "ssize_t" )
foreach(send_arg1 "SOCKET" "int" "ssize_t" )
foreach(send_arg2 "const char *" "const void *" "void *" "char *")
foreach(send_arg3 "int" "size_t" "socklen_t" "unsigned int")
foreach(send_arg4 "int" "unsigned int")
if(NOT curl_cv_func_send_done)
unset(curl_cv_func_send_test CACHE)
check_c_source_compiles("
${_source_epilogue}
extern ${send_retv} ${signature_call_conv}
send(${send_arg1}, ${send_arg2}, ${send_arg3}, ${send_arg4});
int main(void) {
${send_arg1} s=0;
${send_arg2} buf=0;
${send_arg3} len=0;
${send_arg4} flags=0;
${send_retv} res = send(s, buf, len, flags);
(void) res;
return 0;
}"
curl_cv_func_send_test)
if(curl_cv_func_send_test)
string(REGEX REPLACE "(const) .*" "\\1" send_qual_arg2 "${send_arg2}")
string(REGEX REPLACE "const (.*)" "\\1" send_arg2 "${send_arg2}")
set(curl_cv_func_send_args
"${send_arg1},${send_arg2},${send_arg3},${send_arg4},${send_retv},${send_qual_arg2}")
set(SEND_TYPE_ARG1 "${send_arg1}")
set(SEND_TYPE_ARG2 "${send_arg2}")
set(SEND_TYPE_ARG3 "${send_arg3}")
set(SEND_TYPE_ARG4 "${send_arg4}")
set(SEND_TYPE_RETV "${send_retv}")
set(HAVE_SEND 1)
set(curl_cv_func_send_done 1)
endif()
endif()
endforeach()
endforeach()
endforeach()
endforeach()
endforeach()
else()
string(REGEX REPLACE "^([^,]*),[^,]*,[^,]*,[^,]*,[^,]*,[^,]*$" "\\1" SEND_TYPE_ARG1 "${curl_cv_func_send_args}")
string(REGEX REPLACE "^[^,]*,([^,]*),[^,]*,[^,]*,[^,]*,[^,]*$" "\\1" SEND_TYPE_ARG2 "${curl_cv_func_send_args}")
string(REGEX REPLACE "^[^,]*,[^,]*,([^,]*),[^,]*,[^,]*,[^,]*$" "\\1" SEND_TYPE_ARG3 "${curl_cv_func_send_args}")
string(REGEX REPLACE "^[^,]*,[^,]*,[^,]*,([^,]*),[^,]*,[^,]*$" "\\1" SEND_TYPE_ARG4 "${curl_cv_func_send_args}")
string(REGEX REPLACE "^[^,]*,[^,]*,[^,]*,[^,]*,([^,]*),[^,]*$" "\\1" SEND_TYPE_RETV "${curl_cv_func_send_args}")
string(REGEX REPLACE "^[^,]*,[^,]*,[^,]*,[^,]*,[^,]*,([^,]*)$" "\\1" SEND_QUAL_ARG2 "${curl_cv_func_send_args}")
endif()
if("${curl_cv_func_send_args}" STREQUAL "unknown")
message(FATAL_ERROR "Cannot find proper types to use for send args")
endif()
set(SEND_QUAL_ARG2 "const")
else()
message(FATAL_ERROR "Unable to link function send")
endif()
set(curl_cv_func_send_args "${curl_cv_func_send_args}" CACHE INTERNAL "Arguments for send")
set(HAVE_SEND 1)
check_c_source_compiles("${_source_epilogue}
int main(void) {
int flag = MSG_NOSIGNAL;
(void)flag;
return 0;
}" HAVE_MSG_NOSIGNAL)
if(NOT HAVE_WINDOWS_H)
add_header_include(HAVE_SYS_TIME_H "sys/time.h")
add_header_include(TIME_WITH_SYS_TIME "time.h")
add_header_include(HAVE_TIME_H "time.h")
endif()
check_c_source_compiles("${_source_epilogue}
int main(void) {
struct timeval ts;
ts.tv_sec = 0;
ts.tv_usec = 0;
(void)ts;
return 0;
}" HAVE_STRUCT_TIMEVAL)
set(HAVE_SIG_ATOMIC_T 1)
set(CMAKE_REQUIRED_FLAGS)
if(HAVE_SIGNAL_H)
set(CMAKE_REQUIRED_FLAGS "-DHAVE_SIGNAL_H")
set(CMAKE_EXTRA_INCLUDE_FILES "signal.h")
endif()
check_type_size("sig_atomic_t" SIZEOF_SIG_ATOMIC_T)
if(HAVE_SIZEOF_SIG_ATOMIC_T)
check_c_source_compiles("
#ifdef HAVE_SIGNAL_H
# include <signal.h>
#endif
int main(void) {
static volatile sig_atomic_t dummy = 0;
(void)dummy;
return 0;
}" HAVE_SIG_ATOMIC_T_NOT_VOLATILE)
if(NOT HAVE_SIG_ATOMIC_T_NOT_VOLATILE)
set(HAVE_SIG_ATOMIC_T_VOLATILE 1)
endif()
endif()
if(HAVE_WINDOWS_H)
set(CMAKE_EXTRA_INCLUDE_FILES winsock2.h)
else()
set(CMAKE_EXTRA_INCLUDE_FILES)
if(HAVE_SYS_SOCKET_H)
set(CMAKE_EXTRA_INCLUDE_FILES sys/socket.h)
endif()
endif()
check_type_size("struct sockaddr_storage" SIZEOF_STRUCT_SOCKADDR_STORAGE)
if(HAVE_SIZEOF_STRUCT_SOCKADDR_STORAGE)
set(HAVE_STRUCT_SOCKADDR_STORAGE 1)
endif()
unset(CMAKE_TRY_COMPILE_TARGET_TYPE)
if(NOT DEFINED CMAKE_TOOLCHAIN_FILE)
# if not cross-compilation...
include(CheckCSourceRuns)
set(CMAKE_REQUIRED_FLAGS "")
if(HAVE_SYS_POLL_H)
set(CMAKE_REQUIRED_FLAGS "-DHAVE_SYS_POLL_H")
elseif(HAVE_POLL_H)
set(CMAKE_REQUIRED_FLAGS "-DHAVE_POLL_H")
endif()
check_c_source_runs("
#include <stdlib.h>
#include <sys/time.h>
#ifdef HAVE_SYS_POLL_H
# include <sys/poll.h>
#elif HAVE_POLL_H
# include <poll.h>
#endif
int main(void)
{
if(0 != poll(0, 0, 10)) {
return 1; /* fail */
}
else {
/* detect the 10.12 poll() breakage */
struct timeval before, after;
int rc;
size_t us;
gettimeofday(&before, NULL);
rc = poll(NULL, 0, 500);
gettimeofday(&after, NULL);
us = (after.tv_sec - before.tv_sec) * 1000000 +
(after.tv_usec - before.tv_usec);
if(us < 400000) {
return 1;
}
}
return 0;
}" HAVE_POLL_FINE)
endif()

28
contrib/curl-cmake/CMake/README.md Normal file
View File

@ -0,0 +1,28 @@
#***************************************************************************
# _ _ ____ _
# Project ___| | | | _ \| |
# / __| | | | |_) | |
# | (__| |_| | _ <| |___
# \___|\___/|_| \_\_____|
#
# Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at https://curl.haxx.se/docs/copyright.html.
#
# You may opt to use, copy, modify, merge, publish, distribute and/or sell
# copies of the Software, and permit persons to whom the Software is
# furnished to do so, under the terms of the COPYING file.
#
# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
# KIND, either express or implied.
#
###########################################################################
CMake files under this directory were reused from project curl.
Here are links to original source files:
https://github.com/curl/curl/blob/master/CMake/CurlSymbolHiding.cmake
https://github.com/curl/curl/blob/master/CMake/CurlTests,c
https://github.com/curl/curl/blob/master/CMake/Macros.cmake
https://github.com/curl/curl/blob/master/CMake/OtherTests.cmake

1509
contrib/curl-cmake/CMakeLists.patch Normal file
View File

File diff suppressed because it is too large Load Diff

634
contrib/curl-cmake/CMakeLists.txt Normal file
View File

@ -0,0 +1,634 @@
#***************************************************************************
# _ _ ____ _
# Project ___| | | | _ \| |
# / __| | | | |_) | |
# | (__| |_| | _ <| |___
# \___|\___/|_| \_\_____|
#
# Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at https://curl.haxx.se/docs/copyright.html.
#
# You may opt to use, copy, modify, merge, publish, distribute and/or sell
# copies of the Software, and permit persons to whom the Software is
# furnished to do so, under the terms of the COPYING file.
#
# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
# KIND, either express or implied.
#
###########################################################################
# NOTE:
# This file is shrinked and reworked version of original curl CMakeLists.txt
# Original file link https://github.com/curl/curl/blob/3b8bbbbd1609c638a3d3d0acb148a33dedb67be3/CMakeLists.txt
# If you need to update curl building you can find patch file in this directory
# and apply it to fresh original CMakeLists.txt file.
cmake_minimum_required(VERSION 3.0 FATAL_ERROR)
SET(CURL_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/curl)
SET(CURL_LIBRARY_DIR ${CURL_SOURCE_DIR}/lib)
set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake;${CMAKE_MODULE_PATH}")
# Disable status messages when perform checks
set(CMAKE_REQUIRED_QUIET TRUE)
include(Macros)
include(CMakeDependentOption)
include(CheckCCompilerFlag)
file(READ ${CURL_SOURCE_DIR}/include/curl/curlver.h CURL_VERSION_H_CONTENTS)
string(REGEX MATCH "#define LIBCURL_VERSION \"[^\"]*"
CURL_VERSION ${CURL_VERSION_H_CONTENTS})
string(REGEX REPLACE "[^\"]+\"" "" CURL_VERSION ${CURL_VERSION})
string(REGEX MATCH "#define LIBCURL_VERSION_NUM 0x[0-9a-fA-F]+"
CURL_VERSION_NUM ${CURL_VERSION_H_CONTENTS})
string(REGEX REPLACE "[^0]+0x" "" CURL_VERSION_NUM ${CURL_VERSION_NUM})
message(STATUS "Use curl version=[${CURL_VERSION}]")
set(OPERATING_SYSTEM "${CMAKE_SYSTEM_NAME}")
set(OS "\"${CMAKE_SYSTEM_NAME}\"")
option(PICKY_COMPILER "Enable picky compiler options" ON)
option(ENABLE_THREADED_RESOLVER "Set to ON to enable threaded DNS lookup" ON)
if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_CLANG)
if(PICKY_COMPILER)
foreach(_CCOPT -pedantic -Wall -W -Wpointer-arith -Wwrite-strings -Wunused -Wshadow -Winline -Wnested-externs -Wmissing-declarations -Wmissing-prototypes -Wno-long-long -Wfloat-equal -Wno-multichar -Wsign-compare -Wundef -Wno-format-nonliteral -Wendif-labels -Wstrict-prototypes -Wdeclaration-after-statement -Wstrict-aliasing=3 -Wcast-align -Wtype-limits -Wold-style-declaration -Wmissing-parameter-type -Wempty-body -Wclobbered -Wignored-qualifiers -Wconversion -Wno-sign-conversion -Wvla -Wdouble-promotion -Wno-system-headers -Wno-pedantic-ms-format)
# surprisingly, CHECK_C_COMPILER_FLAG needs a new variable to store each new
# test result in.
check_c_compiler_flag(${_CCOPT} OPT${_CCOPT})
if(OPT${_CCOPT})
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${_CCOPT}")
endif()
endforeach()
endif()
endif()
# For debug libs and exes, add "-d" postfix
if(NOT DEFINED CMAKE_DEBUG_POSTFIX)
set(CMAKE_DEBUG_POSTFIX "-d")
endif()
# initialize CURL_LIBS
set(CURL_LIBS "")
include(CurlSymbolHiding)
# Http only
set(CURL_DISABLE_FTP ON)
set(CURL_DISABLE_LDAP ON)
set(CURL_DISABLE_LDAPS ON)
set(CURL_DISABLE_TELNET ON)
set(CURL_DISABLE_DICT ON)
set(CURL_DISABLE_FILE ON)
set(CURL_DISABLE_TFTP ON)
set(CURL_DISABLE_RTSP ON)
set(CURL_DISABLE_POP3 ON)
set(CURL_DISABLE_IMAP ON)
set(CURL_DISABLE_SMTP ON)
set(CURL_DISABLE_GOPHER ON)
option(CURL_DISABLE_COOKIES "to disable cookies support" OFF)
mark_as_advanced(CURL_DISABLE_COOKIES)
option(CURL_DISABLE_CRYPTO_AUTH "to disable cryptographic authentication" OFF)
mark_as_advanced(CURL_DISABLE_CRYPTO_AUTH)
option(CURL_DISABLE_VERBOSE_STRINGS "to disable verbose strings" OFF)
mark_as_advanced(CURL_DISABLE_VERBOSE_STRINGS)
option(ENABLE_IPV6 "Define if you want to enable IPv6 support" ON)
mark_as_advanced(ENABLE_IPV6)
if(ENABLE_IPV6 AND NOT WIN32)
include(CheckStructHasMember)
check_struct_has_member("struct sockaddr_in6" sin6_addr "netinet/in.h"
HAVE_SOCKADDR_IN6_SIN6_ADDR)
check_struct_has_member("struct sockaddr_in6" sin6_scope_id "netinet/in.h"
HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID)
if(NOT HAVE_SOCKADDR_IN6_SIN6_ADDR)
message(WARNING "struct sockaddr_in6 not available, disabling IPv6 support")
# Force the feature off as this name is used as guard macro...
set(ENABLE_IPV6 OFF
CACHE BOOL "Define if you want to enable IPv6 support" FORCE)
endif()
endif()
curl_nroff_check()
# We need ansi c-flags, especially on HP
set(CMAKE_C_FLAGS "${CMAKE_ANSI_CFLAGS} ${CMAKE_C_FLAGS}")
set(CMAKE_REQUIRED_FLAGS ${CMAKE_ANSI_CFLAGS})
# Include all the necessary files for macros
include(CheckFunctionExists)
include(CheckIncludeFile)
include(CheckIncludeFiles)
include(CheckLibraryExists)
include(CheckSymbolExists)
include(CheckTypeSize)
include(CheckCSourceCompiles)
if(ENABLE_THREADED_RESOLVER)
find_package(Threads REQUIRED)
if(WIN32)
set(USE_THREADS_WIN32 ON)
else()
set(USE_THREADS_POSIX ${CMAKE_USE_PTHREADS_INIT})
set(HAVE_PTHREAD_H ${CMAKE_USE_PTHREADS_INIT})
endif()
set(CURL_LIBS ${CURL_LIBS} ${CMAKE_THREAD_LIBS_INIT})
endif()
# Check for all needed libraries
check_library_exists_concat("${CMAKE_DL_LIBS}" dlopen HAVE_LIBDL)
check_library_exists_concat("socket" connect HAVE_LIBSOCKET)
check_library_exists("c" gethostbyname "" NOT_NEED_LIBNSL)
check_function_exists(gethostname HAVE_GETHOSTNAME)
# From cmake/find/ssl.cmake
if (OPENSSL_FOUND)
set(SSL_ENABLED ON)
set(USE_OPENSSL ON)
list(APPEND CURL_LIBS ${OPENSSL_LIBRARIES})
check_include_file("openssl/crypto.h" HAVE_OPENSSL_CRYPTO_H)
check_include_file("openssl/err.h" HAVE_OPENSSL_ERR_H)
check_include_file("openssl/pem.h" HAVE_OPENSSL_PEM_H)
check_include_file("openssl/rsa.h" HAVE_OPENSSL_RSA_H)
check_include_file("openssl/ssl.h" HAVE_OPENSSL_SSL_H)
check_include_file("openssl/x509.h" HAVE_OPENSSL_X509_H)
check_include_file("openssl/rand.h" HAVE_OPENSSL_RAND_H)
check_symbol_exists(RAND_status "${CURL_INCLUDES}" HAVE_RAND_STATUS)
check_symbol_exists(RAND_screen "${CURL_INCLUDES}" HAVE_RAND_SCREEN)
check_symbol_exists(RAND_egd "${CURL_INCLUDES}" HAVE_RAND_EGD)
endif()
# Check for idn
check_library_exists_concat("idn2" idn2_lookup_ul HAVE_LIBIDN2)
# Check for symbol dlopen (same as HAVE_LIBDL)
check_library_exists("${CURL_LIBS}" dlopen "" HAVE_DLOPEN)
# From /cmake/find/zlib.cmake
if (ZLIB_FOUND)
set(HAVE_ZLIB_H ON)
set(HAVE_LIBZ ON)
set(USE_ZLIB ON)
list(APPEND CURL_LIBS ${ZLIB_LIBRARIES})
endif()
option(ENABLE_UNIX_SOCKETS "Define if you want Unix domain sockets support" ON)
if(ENABLE_UNIX_SOCKETS)
include(CheckStructHasMember)
check_struct_has_member("struct sockaddr_un" sun_path "sys/un.h" USE_UNIX_SOCKETS)
else()
unset(USE_UNIX_SOCKETS CACHE)
endif()
# CA handling
# Explicitly set to most common case
if (OPENSSL_FOUND)
set(CURL_CA_BUNDLE "/etc/ssl/certs/ca-certificates.crt")
set(CURL_CA_BUNDLE_SET TRUE CACHE BOOL "Path to the CA bundle has been set")
set(CURL_CA_PATH "/etc/ssl/certs")
set(CURL_CA_PATH_SET TRUE CACHE BOOL "Path to the CA bundle has been set")
endif()
check_include_file_concat("stdio.h" HAVE_STDIO_H)
check_include_file_concat("inttypes.h" HAVE_INTTYPES_H)
check_include_file_concat("sys/filio.h" HAVE_SYS_FILIO_H)
check_include_file_concat("sys/ioctl.h" HAVE_SYS_IOCTL_H)
check_include_file_concat("sys/param.h" HAVE_SYS_PARAM_H)
check_include_file_concat("sys/poll.h" HAVE_SYS_POLL_H)
check_include_file_concat("sys/resource.h" HAVE_SYS_RESOURCE_H)
check_include_file_concat("sys/select.h" HAVE_SYS_SELECT_H)
check_include_file_concat("sys/socket.h" HAVE_SYS_SOCKET_H)
check_include_file_concat("sys/sockio.h" HAVE_SYS_SOCKIO_H)
check_include_file_concat("sys/stat.h" HAVE_SYS_STAT_H)
check_include_file_concat("sys/time.h" HAVE_SYS_TIME_H)
check_include_file_concat("sys/types.h" HAVE_SYS_TYPES_H)
check_include_file_concat("sys/uio.h" HAVE_SYS_UIO_H)
check_include_file_concat("sys/un.h" HAVE_SYS_UN_H)
check_include_file_concat("sys/utime.h" HAVE_SYS_UTIME_H)
check_include_file_concat("sys/xattr.h" HAVE_SYS_XATTR_H)
check_include_file_concat("alloca.h" HAVE_ALLOCA_H)
check_include_file_concat("arpa/inet.h" HAVE_ARPA_INET_H)
check_include_file_concat("arpa/tftp.h" HAVE_ARPA_TFTP_H)
check_include_file_concat("assert.h" HAVE_ASSERT_H)
check_include_file_concat("crypto.h" HAVE_CRYPTO_H)
check_include_file_concat("des.h" HAVE_DES_H)
check_include_file_concat("err.h" HAVE_ERR_H)
check_include_file_concat("errno.h" HAVE_ERRNO_H)
check_include_file_concat("fcntl.h" HAVE_FCNTL_H)
check_include_file_concat("idn2.h" HAVE_IDN2_H)
check_include_file_concat("ifaddrs.h" HAVE_IFADDRS_H)
check_include_file_concat("io.h" HAVE_IO_H)
check_include_file_concat("krb.h" HAVE_KRB_H)
check_include_file_concat("libgen.h" HAVE_LIBGEN_H)
check_include_file_concat("locale.h" HAVE_LOCALE_H)
check_include_file_concat("net/if.h" HAVE_NET_IF_H)
check_include_file_concat("netdb.h" HAVE_NETDB_H)
check_include_file_concat("netinet/in.h" HAVE_NETINET_IN_H)
check_include_file_concat("netinet/tcp.h" HAVE_NETINET_TCP_H)
check_include_file_concat("pem.h" HAVE_PEM_H)
check_include_file_concat("poll.h" HAVE_POLL_H)
check_include_file_concat("pwd.h" HAVE_PWD_H)
check_include_file_concat("rsa.h" HAVE_RSA_H)
check_include_file_concat("setjmp.h" HAVE_SETJMP_H)
check_include_file_concat("sgtty.h" HAVE_SGTTY_H)
check_include_file_concat("signal.h" HAVE_SIGNAL_H)
check_include_file_concat("ssl.h" HAVE_SSL_H)
check_include_file_concat("stdbool.h" HAVE_STDBOOL_H)
check_include_file_concat("stdint.h" HAVE_STDINT_H)
check_include_file_concat("stdio.h" HAVE_STDIO_H)
check_include_file_concat("stdlib.h" HAVE_STDLIB_H)
check_include_file_concat("string.h" HAVE_STRING_H)
check_include_file_concat("strings.h" HAVE_STRINGS_H)
check_include_file_concat("stropts.h" HAVE_STROPTS_H)
check_include_file_concat("termio.h" HAVE_TERMIO_H)
check_include_file_concat("termios.h" HAVE_TERMIOS_H)
check_include_file_concat("time.h" HAVE_TIME_H)
check_include_file_concat("unistd.h" HAVE_UNISTD_H)
check_include_file_concat("utime.h" HAVE_UTIME_H)
check_include_file_concat("x509.h" HAVE_X509_H)
check_include_file_concat("process.h" HAVE_PROCESS_H)
check_include_file_concat("stddef.h" HAVE_STDDEF_H)
check_include_file_concat("dlfcn.h" HAVE_DLFCN_H)
check_include_file_concat("malloc.h" HAVE_MALLOC_H)
check_include_file_concat("memory.h" HAVE_MEMORY_H)
check_include_file_concat("netinet/if_ether.h" HAVE_NETINET_IF_ETHER_H)
check_include_file_concat("stdint.h" HAVE_STDINT_H)
check_include_file_concat("sockio.h" HAVE_SOCKIO_H)
check_include_file_concat("sys/utsname.h" HAVE_SYS_UTSNAME_H)
check_type_size(size_t SIZEOF_SIZE_T)
check_type_size(ssize_t SIZEOF_SSIZE_T)
check_type_size("long long" SIZEOF_LONG_LONG)
check_type_size("long" SIZEOF_LONG)
check_type_size("short" SIZEOF_SHORT)
check_type_size("int" SIZEOF_INT)
check_type_size("__int64" SIZEOF___INT64)
check_type_size("long double" SIZEOF_LONG_DOUBLE)
check_type_size("time_t" SIZEOF_TIME_T)
if(NOT HAVE_SIZEOF_SSIZE_T)
if(SIZEOF_LONG EQUAL SIZEOF_SIZE_T)
set(ssize_t long)
endif()
if(NOT ssize_t AND SIZEOF___INT64 EQUAL SIZEOF_SIZE_T)
set(ssize_t __int64)
endif()
endif()
# off_t is sized later, after the HAVE_FILE_OFFSET_BITS test
if(HAVE_SIZEOF_LONG_LONG)
set(HAVE_LONGLONG 1)
set(HAVE_LL 1)
endif()
find_file(RANDOM_FILE urandom /dev)
mark_as_advanced(RANDOM_FILE)
# Check for some functions that are used
if(HAVE_LIBWS2_32)
set(CMAKE_REQUIRED_LIBRARIES ws2_32)
elseif(HAVE_LIBSOCKET)
set(CMAKE_REQUIRED_LIBRARIES socket)
endif()
check_symbol_exists(basename "${CURL_INCLUDES}" HAVE_BASENAME)
check_symbol_exists(socket "${CURL_INCLUDES}" HAVE_SOCKET)
check_symbol_exists(select "${CURL_INCLUDES}" HAVE_SELECT)
check_symbol_exists(poll "${CURL_INCLUDES}" HAVE_POLL)
check_symbol_exists(strdup "${CURL_INCLUDES}" HAVE_STRDUP)
check_symbol_exists(strstr "${CURL_INCLUDES}" HAVE_STRSTR)
check_symbol_exists(strtok_r "${CURL_INCLUDES}" HAVE_STRTOK_R)
check_symbol_exists(strftime "${CURL_INCLUDES}" HAVE_STRFTIME)
check_symbol_exists(uname "${CURL_INCLUDES}" HAVE_UNAME)
check_symbol_exists(strcasecmp "${CURL_INCLUDES}" HAVE_STRCASECMP)
check_symbol_exists(stricmp "${CURL_INCLUDES}" HAVE_STRICMP)
check_symbol_exists(strcmpi "${CURL_INCLUDES}" HAVE_STRCMPI)
check_symbol_exists(strncmpi "${CURL_INCLUDES}" HAVE_STRNCMPI)
check_symbol_exists(alarm "${CURL_INCLUDES}" HAVE_ALARM)
if(NOT HAVE_STRNCMPI)
set(HAVE_STRCMPI)
endif()
check_symbol_exists(gethostbyaddr "${CURL_INCLUDES}" HAVE_GETHOSTBYADDR)
check_symbol_exists(gethostbyaddr_r "${CURL_INCLUDES}" HAVE_GETHOSTBYADDR_R)
check_symbol_exists(gettimeofday "${CURL_INCLUDES}" HAVE_GETTIMEOFDAY)
check_symbol_exists(inet_addr "${CURL_INCLUDES}" HAVE_INET_ADDR)
check_symbol_exists(inet_ntoa "${CURL_INCLUDES}" HAVE_INET_NTOA)
check_symbol_exists(inet_ntoa_r "${CURL_INCLUDES}" HAVE_INET_NTOA_R)
check_symbol_exists(tcsetattr "${CURL_INCLUDES}" HAVE_TCSETATTR)
check_symbol_exists(tcgetattr "${CURL_INCLUDES}" HAVE_TCGETATTR)
check_symbol_exists(perror "${CURL_INCLUDES}" HAVE_PERROR)
check_symbol_exists(closesocket "${CURL_INCLUDES}" HAVE_CLOSESOCKET)
check_symbol_exists(setvbuf "${CURL_INCLUDES}" HAVE_SETVBUF)
check_symbol_exists(sigsetjmp "${CURL_INCLUDES}" HAVE_SIGSETJMP)
check_symbol_exists(getpass_r "${CURL_INCLUDES}" HAVE_GETPASS_R)
check_symbol_exists(strlcat "${CURL_INCLUDES}" HAVE_STRLCAT)
check_symbol_exists(getpwuid "${CURL_INCLUDES}" HAVE_GETPWUID)
check_symbol_exists(getpwuid_r "${CURL_INCLUDES}" HAVE_GETPWUID_R)
check_symbol_exists(geteuid "${CURL_INCLUDES}" HAVE_GETEUID)
check_symbol_exists(usleep "${CURL_INCLUDES}" HAVE_USLEEP)
check_symbol_exists(utime "${CURL_INCLUDES}" HAVE_UTIME)
check_symbol_exists(gmtime_r "${CURL_INCLUDES}" HAVE_GMTIME_R)
check_symbol_exists(localtime_r "${CURL_INCLUDES}" HAVE_LOCALTIME_R)
check_symbol_exists(gethostbyname "${CURL_INCLUDES}" HAVE_GETHOSTBYNAME)
check_symbol_exists(gethostbyname_r "${CURL_INCLUDES}" HAVE_GETHOSTBYNAME_R)
check_symbol_exists(signal "${CURL_INCLUDES}" HAVE_SIGNAL_FUNC)
check_symbol_exists(SIGALRM "${CURL_INCLUDES}" HAVE_SIGNAL_MACRO)
if(HAVE_SIGNAL_FUNC AND HAVE_SIGNAL_MACRO)
set(HAVE_SIGNAL 1)
endif()
check_symbol_exists(uname "${CURL_INCLUDES}" HAVE_UNAME)
check_symbol_exists(strtoll "${CURL_INCLUDES}" HAVE_STRTOLL)
check_symbol_exists(_strtoi64 "${CURL_INCLUDES}" HAVE__STRTOI64)
check_symbol_exists(strerror_r "${CURL_INCLUDES}" HAVE_STRERROR_R)
check_symbol_exists(siginterrupt "${CURL_INCLUDES}" HAVE_SIGINTERRUPT)
check_symbol_exists(perror "${CURL_INCLUDES}" HAVE_PERROR)
check_symbol_exists(fork "${CURL_INCLUDES}" HAVE_FORK)
check_symbol_exists(getaddrinfo "${CURL_INCLUDES}" HAVE_GETADDRINFO)
check_symbol_exists(freeaddrinfo "${CURL_INCLUDES}" HAVE_FREEADDRINFO)
check_symbol_exists(freeifaddrs "${CURL_INCLUDES}" HAVE_FREEIFADDRS)
check_symbol_exists(pipe "${CURL_INCLUDES}" HAVE_PIPE)
check_symbol_exists(ftruncate "${CURL_INCLUDES}" HAVE_FTRUNCATE)
check_symbol_exists(getprotobyname "${CURL_INCLUDES}" HAVE_GETPROTOBYNAME)
check_symbol_exists(getpeername "${CURL_INCLUDES}" HAVE_GETPEERNAME)
check_symbol_exists(getsockname "${CURL_INCLUDES}" HAVE_GETSOCKNAME)
check_symbol_exists(if_nametoindex "${CURL_INCLUDES}" HAVE_IF_NAMETOINDEX)
check_symbol_exists(getrlimit "${CURL_INCLUDES}" HAVE_GETRLIMIT)
check_symbol_exists(setlocale "${CURL_INCLUDES}" HAVE_SETLOCALE)
check_symbol_exists(setmode "${CURL_INCLUDES}" HAVE_SETMODE)
check_symbol_exists(setrlimit "${CURL_INCLUDES}" HAVE_SETRLIMIT)
check_symbol_exists(fcntl "${CURL_INCLUDES}" HAVE_FCNTL)
check_symbol_exists(ioctl "${CURL_INCLUDES}" HAVE_IOCTL)
check_symbol_exists(setsockopt "${CURL_INCLUDES}" HAVE_SETSOCKOPT)
check_function_exists(mach_absolute_time HAVE_MACH_ABSOLUTE_TIME)
check_symbol_exists(fsetxattr "${CURL_INCLUDES}" HAVE_FSETXATTR)
if(HAVE_FSETXATTR)
foreach(CURL_TEST HAVE_FSETXATTR_5 HAVE_FSETXATTR_6)
curl_internal_test(${CURL_TEST})
endforeach()
endif()
# sigaction and sigsetjmp are special. Use special mechanism for
# detecting those, but only if previous attempt failed.
if(HAVE_SIGNAL_H)
check_symbol_exists(sigaction "signal.h" HAVE_SIGACTION)
endif()
if(NOT HAVE_SIGSETJMP)
if(HAVE_SETJMP_H)
check_symbol_exists(sigsetjmp "setjmp.h" HAVE_MACRO_SIGSETJMP)
if(HAVE_MACRO_SIGSETJMP)
set(HAVE_SIGSETJMP 1)
endif()
endif()
endif()
# If there is no stricmp(), do not allow LDAP to parse URLs
if(NOT HAVE_STRICMP)
set(HAVE_LDAP_URL_PARSE 1)
endif()
# Do curl specific tests
foreach(CURL_TEST
HAVE_FCNTL_O_NONBLOCK
HAVE_IOCTLSOCKET
HAVE_IOCTLSOCKET_CAMEL
HAVE_IOCTLSOCKET_CAMEL_FIONBIO
HAVE_IOCTLSOCKET_FIONBIO
HAVE_IOCTL_FIONBIO
HAVE_IOCTL_SIOCGIFADDR
HAVE_SETSOCKOPT_SO_NONBLOCK
HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID
TIME_WITH_SYS_TIME
HAVE_O_NONBLOCK
HAVE_GETHOSTBYADDR_R_5
HAVE_GETHOSTBYADDR_R_7
HAVE_GETHOSTBYADDR_R_8
HAVE_GETHOSTBYADDR_R_5_REENTRANT
HAVE_GETHOSTBYADDR_R_7_REENTRANT
HAVE_GETHOSTBYADDR_R_8_REENTRANT
HAVE_GETHOSTBYNAME_R_3
HAVE_GETHOSTBYNAME_R_5
HAVE_GETHOSTBYNAME_R_6
HAVE_GETHOSTBYNAME_R_3_REENTRANT
HAVE_GETHOSTBYNAME_R_5_REENTRANT
HAVE_GETHOSTBYNAME_R_6_REENTRANT
HAVE_IN_ADDR_T
HAVE_BOOL_T
STDC_HEADERS
RETSIGTYPE_TEST
HAVE_INET_NTOA_R_DECL
HAVE_INET_NTOA_R_DECL_REENTRANT
HAVE_GETADDRINFO
HAVE_FILE_OFFSET_BITS
HAVE_VARIADIC_MACROS_C99
HAVE_VARIADIC_MACROS_GCC
)
curl_internal_test(${CURL_TEST})
endforeach()
if(HAVE_FILE_OFFSET_BITS)
set(_FILE_OFFSET_BITS 64)
set(CMAKE_REQUIRED_FLAGS "-D_FILE_OFFSET_BITS=64")
endif()
check_type_size("off_t" SIZEOF_OFF_T)
# include this header to get the type
set(CMAKE_REQUIRED_INCLUDES "${CURL_SOURCE_DIR}/include")
set(CMAKE_EXTRA_INCLUDE_FILES "curl/system.h")
check_type_size("curl_off_t" SIZEOF_CURL_OFF_T)
set(CMAKE_EXTRA_INCLUDE_FILES "")
foreach(CURL_TEST
HAVE_GLIBC_STRERROR_R
HAVE_POSIX_STRERROR_R
)
curl_internal_test(${CURL_TEST})
endforeach()
# Check for reentrant
foreach(CURL_TEST
HAVE_GETHOSTBYADDR_R_5
HAVE_GETHOSTBYADDR_R_7
HAVE_GETHOSTBYADDR_R_8
HAVE_GETHOSTBYNAME_R_3
HAVE_GETHOSTBYNAME_R_5
HAVE_GETHOSTBYNAME_R_6
HAVE_INET_NTOA_R_DECL_REENTRANT)
if(NOT ${CURL_TEST})
if(${CURL_TEST}_REENTRANT)
set(NEED_REENTRANT 1)
endif()
endif()
endforeach()
if(NEED_REENTRANT)
foreach(CURL_TEST
HAVE_GETHOSTBYADDR_R_5
HAVE_GETHOSTBYADDR_R_7
HAVE_GETHOSTBYADDR_R_8
HAVE_GETHOSTBYNAME_R_3
HAVE_GETHOSTBYNAME_R_5
HAVE_GETHOSTBYNAME_R_6)
set(${CURL_TEST} 0)
if(${CURL_TEST}_REENTRANT)
set(${CURL_TEST} 1)
endif()
endforeach()
endif()
if(HAVE_INET_NTOA_R_DECL_REENTRANT)
set(HAVE_INET_NTOA_R_DECL 1)
set(NEED_REENTRANT 1)
endif()
# Check clock_gettime(CLOCK_MONOTONIC, x) support
curl_internal_test(HAVE_CLOCK_GETTIME_MONOTONIC)
# Check compiler support of __builtin_available()
curl_internal_test(HAVE_BUILTIN_AVAILABLE)
# Some other minor tests
if(NOT HAVE_IN_ADDR_T)
set(in_addr_t "unsigned long")
endif()
# Check for nonblocking
set(HAVE_DISABLED_NONBLOCKING 1)
if(HAVE_FIONBIO OR
HAVE_IOCTLSOCKET OR
HAVE_IOCTLSOCKET_CASE OR
HAVE_O_NONBLOCK)
set(HAVE_DISABLED_NONBLOCKING)
endif()
set(CURL_PULL_SYS_TYPES_H ${HAVE_SYS_TYPES_H})
set(CURL_PULL_SYS_SOCKET_H ${HAVE_SYS_SOCKET_H})
set(CURL_PULL_SYS_POLL_H ${HAVE_SYS_POLL_H})
set(CURL_PULL_STDINT_H ${HAVE_STDINT_H})
set(CURL_PULL_INTTYPES_H ${HAVE_INTTYPES_H})
include(CMake/OtherTests.cmake)
SET(LIB_VAUTH_CFILES
"${CURL_LIBRARY_DIR}/vauth/vauth.c" "${CURL_LIBRARY_DIR}/vauth/cleartext.c" "${CURL_LIBRARY_DIR}/vauth/cram.c"
"${CURL_LIBRARY_DIR}/vauth/digest.c" "${CURL_LIBRARY_DIR}/vauth/digest_sspi.c" "${CURL_LIBRARY_DIR}/vauth/krb5_gssapi.c"
"${CURL_LIBRARY_DIR}/vauth/krb5_sspi.c" "${CURL_LIBRARY_DIR}/vauth/ntlm.c" "${CURL_LIBRARY_DIR}/vauth/ntlm_sspi.c" "${CURL_LIBRARY_DIR}/vauth/oauth2.c"
"${CURL_LIBRARY_DIR}/vauth/spnego_gssapi.c" "${CURL_LIBRARY_DIR}/vauth/spnego_sspi.c")
SET(LIB_VAUTH_HFILES "${CURL_LIBRARY_DIR}/vauth/vauth.h" "${CURL_LIBRARY_DIR}/vauth/digest.h" "${CURL_LIBRARY_DIR}/vauth/ntlm.h")
SET(LIB_VTLS_CFILES "${CURL_LIBRARY_DIR}/vtls/openssl.c" "${CURL_LIBRARY_DIR}/vtls/gtls.c" "${CURL_LIBRARY_DIR}/vtls/vtls.c" "${CURL_LIBRARY_DIR}/vtls/nss.c"
"${CURL_LIBRARY_DIR}/vtls/polarssl.c" "${CURL_LIBRARY_DIR}/vtls/polarssl_threadlock.c"
"${CURL_LIBRARY_DIR}/vtls/wolfssl.c" "${CURL_LIBRARY_DIR}/vtls/schannel.c" "${CURL_LIBRARY_DIR}/vtls/schannel_verify.c"
"${CURL_LIBRARY_DIR}/vtls/sectransp.c" "${CURL_LIBRARY_DIR}/vtls/gskit.c" "${CURL_LIBRARY_DIR}/vtls/mbedtls.c" "${CURL_LIBRARY_DIR}/vtls/mesalink.c"
"${CURL_LIBRARY_DIR}/vtls/bearssl.c")
SET(LIB_VTLS_HFILES "${CURL_LIBRARY_DIR}/vtls/openssl.h" "${CURL_LIBRARY_DIR}/vtls/vtls.h" "${CURL_LIBRARY_DIR}/vtls/gtls.h"
"${CURL_LIBRARY_DIR}/vtls/nssg.h" "${CURL_LIBRARY_DIR}/vtls/polarssl.h" "${CURL_LIBRARY_DIR}/vtls/polarssl_threadlock.h"
"${CURL_LIBRARY_DIR}/vtls/wolfssl.h" "${CURL_LIBRARY_DIR}/vtls/schannel.h" "${CURL_LIBRARY_DIR}/vtls/sectransp.h" "${CURL_LIBRARY_DIR}/vtls/gskit.h"
"${CURL_LIBRARY_DIR}/vtls/mbedtls.h" "${CURL_LIBRARY_DIR}/vtls/mesalink.h" "${CURL_LIBRARY_DIR}/vtls/bearssl.h")
SET(LIB_VQUIC_CFILES "${CURL_LIBRARY_DIR}/vquic/ngtcp2.c" "${CURL_LIBRARY_DIR}/vquic/quiche.c")
SET(LIB_VQUIC_HFILES "${CURL_LIBRARY_DIR}/vquic/ngtcp2.h" "${CURL_LIBRARY_DIR}/vquic/quiche.h")
SET(LIB_VSSH_CFILES "${CURL_LIBRARY_DIR}/vssh/libssh2.c" "${CURL_LIBRARY_DIR}/vssh/libssh.c")
SET(LIB_VSSH_HFILES "${CURL_LIBRARY_DIR}/vssh/ssh.h")
SET(LIB_CFILES "${CURL_LIBRARY_DIR}/file.c"
"${CURL_LIBRARY_DIR}/timeval.c" "${CURL_LIBRARY_DIR}/base64.c" "${CURL_LIBRARY_DIR}/hostip.c" "${CURL_LIBRARY_DIR}/progress.c" "${CURL_LIBRARY_DIR}/formdata.c"
"${CURL_LIBRARY_DIR}/cookie.c" "${CURL_LIBRARY_DIR}/http.c" "${CURL_LIBRARY_DIR}/sendf.c" "${CURL_LIBRARY_DIR}/url.c" "${CURL_LIBRARY_DIR}/dict.c" "${CURL_LIBRARY_DIR}/if2ip.c" "${CURL_LIBRARY_DIR}/speedcheck.c"
"${CURL_LIBRARY_DIR}/ldap.c" "${CURL_LIBRARY_DIR}/version.c" "${CURL_LIBRARY_DIR}/getenv.c" "${CURL_LIBRARY_DIR}/escape.c" "${CURL_LIBRARY_DIR}/mprintf.c" "${CURL_LIBRARY_DIR}/telnet.c" "${CURL_LIBRARY_DIR}/netrc.c"
"${CURL_LIBRARY_DIR}/getinfo.c" "${CURL_LIBRARY_DIR}/transfer.c" "${CURL_LIBRARY_DIR}/strcase.c" "${CURL_LIBRARY_DIR}/easy.c" "${CURL_LIBRARY_DIR}/security.c" "${CURL_LIBRARY_DIR}/curl_fnmatch.c"
"${CURL_LIBRARY_DIR}/fileinfo.c" "${CURL_LIBRARY_DIR}/wildcard.c" "${CURL_LIBRARY_DIR}/krb5.c" "${CURL_LIBRARY_DIR}/memdebug.c" "${CURL_LIBRARY_DIR}/http_chunks.c"
"${CURL_LIBRARY_DIR}/strtok.c" "${CURL_LIBRARY_DIR}/connect.c" "${CURL_LIBRARY_DIR}/llist.c" "${CURL_LIBRARY_DIR}/hash.c" "${CURL_LIBRARY_DIR}/multi.c" "${CURL_LIBRARY_DIR}/content_encoding.c" "${CURL_LIBRARY_DIR}/share.c"
"${CURL_LIBRARY_DIR}/http_digest.c" "${CURL_LIBRARY_DIR}/md4.c" "${CURL_LIBRARY_DIR}/md5.c" "${CURL_LIBRARY_DIR}/http_negotiate.c" "${CURL_LIBRARY_DIR}/inet_pton.c" "${CURL_LIBRARY_DIR}/strtoofft.c"
"${CURL_LIBRARY_DIR}/strerror.c" "${CURL_LIBRARY_DIR}/amigaos.c" "${CURL_LIBRARY_DIR}/hostasyn.c" "${CURL_LIBRARY_DIR}/hostip4.c" "${CURL_LIBRARY_DIR}/hostip6.c" "${CURL_LIBRARY_DIR}/hostsyn.c"
"${CURL_LIBRARY_DIR}/inet_ntop.c" "${CURL_LIBRARY_DIR}/parsedate.c" "${CURL_LIBRARY_DIR}/select.c" "${CURL_LIBRARY_DIR}/splay.c" "${CURL_LIBRARY_DIR}/strdup.c" "${CURL_LIBRARY_DIR}/socks.c"
"${CURL_LIBRARY_DIR}/curl_addrinfo.c" "${CURL_LIBRARY_DIR}/socks_gssapi.c" "${CURL_LIBRARY_DIR}/socks_sspi.c"
"${CURL_LIBRARY_DIR}/curl_sspi.c" "${CURL_LIBRARY_DIR}/slist.c" "${CURL_LIBRARY_DIR}/nonblock.c" "${CURL_LIBRARY_DIR}/curl_memrchr.c" "${CURL_LIBRARY_DIR}/imap.c" "${CURL_LIBRARY_DIR}/pop3.c" "${CURL_LIBRARY_DIR}/smtp.c"
"${CURL_LIBRARY_DIR}/pingpong.c" "${CURL_LIBRARY_DIR}/rtsp.c" "${CURL_LIBRARY_DIR}/curl_threads.c" "${CURL_LIBRARY_DIR}/warnless.c" "${CURL_LIBRARY_DIR}/hmac.c" "${CURL_LIBRARY_DIR}/curl_rtmp.c"
"${CURL_LIBRARY_DIR}/openldap.c" "${CURL_LIBRARY_DIR}/curl_gethostname.c" "${CURL_LIBRARY_DIR}/gopher.c" "${CURL_LIBRARY_DIR}/idn_win32.c"
"${CURL_LIBRARY_DIR}/http_proxy.c" "${CURL_LIBRARY_DIR}/non-ascii.c" "${CURL_LIBRARY_DIR}/asyn-ares.c" "${CURL_LIBRARY_DIR}/asyn-thread.c" "${CURL_LIBRARY_DIR}/curl_gssapi.c"
"${CURL_LIBRARY_DIR}/http_ntlm.c" "${CURL_LIBRARY_DIR}/curl_ntlm_wb.c" "${CURL_LIBRARY_DIR}/curl_ntlm_core.c" "${CURL_LIBRARY_DIR}/curl_sasl.c" "${CURL_LIBRARY_DIR}/rand.c"
"${CURL_LIBRARY_DIR}/curl_multibyte.c" "${CURL_LIBRARY_DIR}/hostcheck.c" "${CURL_LIBRARY_DIR}/conncache.c" "${CURL_LIBRARY_DIR}/dotdot.c"
"${CURL_LIBRARY_DIR}/x509asn1.c" "${CURL_LIBRARY_DIR}/http2.c" "${CURL_LIBRARY_DIR}/smb.c" "${CURL_LIBRARY_DIR}/curl_endian.c" "${CURL_LIBRARY_DIR}/curl_des.c" "${CURL_LIBRARY_DIR}/system_win32.c"
"${CURL_LIBRARY_DIR}/mime.c" "${CURL_LIBRARY_DIR}/sha256.c" "${CURL_LIBRARY_DIR}/setopt.c" "${CURL_LIBRARY_DIR}/curl_path.c" "${CURL_LIBRARY_DIR}/curl_ctype.c" "${CURL_LIBRARY_DIR}/curl_range.c" "${CURL_LIBRARY_DIR}/psl.c"
"${CURL_LIBRARY_DIR}/doh.c" "${CURL_LIBRARY_DIR}/urlapi.c" "${CURL_LIBRARY_DIR}/curl_get_line.c" "${CURL_LIBRARY_DIR}/altsvc.c" "${CURL_LIBRARY_DIR}/socketpair.c")
SET(LIB_HFILES "${CURL_LIBRARY_DIR}/arpa_telnet.h" "${CURL_LIBRARY_DIR}/netrc.h" "${CURL_LIBRARY_DIR}/file.h" "${CURL_LIBRARY_DIR}/timeval.h" "${CURL_LIBRARY_DIR}/hostip.h" "${CURL_LIBRARY_DIR}/progress.h"
"${CURL_LIBRARY_DIR}/formdata.h" "${CURL_LIBRARY_DIR}/cookie.h" "${CURL_LIBRARY_DIR}/http.h" "${CURL_LIBRARY_DIR}/sendf.h" "${CURL_LIBRARY_DIR}/url.h" "${CURL_LIBRARY_DIR}/dict.h" "${CURL_LIBRARY_DIR}/if2ip.h"
"${CURL_LIBRARY_DIR}/speedcheck.h" "${CURL_LIBRARY_DIR}/urldata.h" "${CURL_LIBRARY_DIR}/curl_ldap.h" "${CURL_LIBRARY_DIR}/escape.h" "${CURL_LIBRARY_DIR}/telnet.h" "${CURL_LIBRARY_DIR}/getinfo.h"
"${CURL_LIBRARY_DIR}/strcase.h" "${CURL_LIBRARY_DIR}/curl_sec.h" "${CURL_LIBRARY_DIR}/memdebug.h" "${CURL_LIBRARY_DIR}/http_chunks.h" "${CURL_LIBRARY_DIR}/curl_fnmatch.h"
"${CURL_LIBRARY_DIR}/wildcard.h" "${CURL_LIBRARY_DIR}/fileinfo.h" "${CURL_LIBRARY_DIR}/strtok.h" "${CURL_LIBRARY_DIR}/connect.h" "${CURL_LIBRARY_DIR}/llist.h"
"${CURL_LIBRARY_DIR}/hash.h" "${CURL_LIBRARY_DIR}/content_encoding.h" "${CURL_LIBRARY_DIR}/share.h" "${CURL_LIBRARY_DIR}/curl_md4.h" "${CURL_LIBRARY_DIR}/curl_md5.h" "${CURL_LIBRARY_DIR}/http_digest.h"
"${CURL_LIBRARY_DIR}/http_negotiate.h" "${CURL_LIBRARY_DIR}/inet_pton.h" "${CURL_LIBRARY_DIR}/amigaos.h" "${CURL_LIBRARY_DIR}/strtoofft.h" "${CURL_LIBRARY_DIR}/strerror.h"
"${CURL_LIBRARY_DIR}/inet_ntop.h" "${CURL_LIBRARY_DIR}/curlx.h" "${CURL_LIBRARY_DIR}/curl_memory.h" "${CURL_LIBRARY_DIR}/curl_setup.h" "${CURL_LIBRARY_DIR}/transfer.h" "${CURL_LIBRARY_DIR}/select.h"
"${CURL_LIBRARY_DIR}/easyif.h" "${CURL_LIBRARY_DIR}/multiif.h" "${CURL_LIBRARY_DIR}/parsedate.h" "${CURL_LIBRARY_DIR}/sockaddr.h" "${CURL_LIBRARY_DIR}/splay.h" "${CURL_LIBRARY_DIR}/strdup.h"
"${CURL_LIBRARY_DIR}/socks.h" "${CURL_LIBRARY_DIR}/curl_base64.h" "${CURL_LIBRARY_DIR}/curl_addrinfo.h" "${CURL_LIBRARY_DIR}/curl_sspi.h"
"${CURL_LIBRARY_DIR}/slist.h" "${CURL_LIBRARY_DIR}/nonblock.h" "${CURL_LIBRARY_DIR}/curl_memrchr.h" "${CURL_LIBRARY_DIR}/imap.h" "${CURL_LIBRARY_DIR}/pop3.h" "${CURL_LIBRARY_DIR}/smtp.h" "${CURL_LIBRARY_DIR}/pingpong.h"
"${CURL_LIBRARY_DIR}/rtsp.h" "${CURL_LIBRARY_DIR}/curl_threads.h" "${CURL_LIBRARY_DIR}/warnless.h" "${CURL_LIBRARY_DIR}/curl_hmac.h" "${CURL_LIBRARY_DIR}/curl_rtmp.h"
"${CURL_LIBRARY_DIR}/curl_gethostname.h" "${CURL_LIBRARY_DIR}/gopher.h" "${CURL_LIBRARY_DIR}/http_proxy.h" "${CURL_LIBRARY_DIR}/non-ascii.h" "${CURL_LIBRARY_DIR}/asyn.h"
"${CURL_LIBRARY_DIR}/http_ntlm.h" "${CURL_LIBRARY_DIR}/curl_gssapi.h" "${CURL_LIBRARY_DIR}/curl_ntlm_wb.h" "${CURL_LIBRARY_DIR}/curl_ntlm_core.h"
"${CURL_LIBRARY_DIR}/curl_sasl.h" "${CURL_LIBRARY_DIR}/curl_multibyte.h" "${CURL_LIBRARY_DIR}/hostcheck.h" "${CURL_LIBRARY_DIR}/conncache.h"
"${CURL_LIBRARY_DIR}/multihandle.h" "${CURL_LIBRARY_DIR}/setup-vms.h" "${CURL_LIBRARY_DIR}/dotdot.h"
"${CURL_LIBRARY_DIR}/x509asn1.h" "${CURL_LIBRARY_DIR}/http2.h" "${CURL_LIBRARY_DIR}/sigpipe.h" "${CURL_LIBRARY_DIR}/smb.h" "${CURL_LIBRARY_DIR}/curl_endian.h" "${CURL_LIBRARY_DIR}/curl_des.h"
"${CURL_LIBRARY_DIR}/curl_printf.h" "${CURL_LIBRARY_DIR}/system_win32.h" "${CURL_LIBRARY_DIR}/rand.h" "${CURL_LIBRARY_DIR}/mime.h" "${CURL_LIBRARY_DIR}/curl_sha256.h" "${CURL_LIBRARY_DIR}/setopt.h"
"${CURL_LIBRARY_DIR}/curl_path.h" "${CURL_LIBRARY_DIR}/curl_ctype.h" "${CURL_LIBRARY_DIR}/curl_range.h" "${CURL_LIBRARY_DIR}/psl.h" "${CURL_LIBRARY_DIR}/doh.h" "${CURL_LIBRARY_DIR}/urlapi-int.h"
"${CURL_LIBRARY_DIR}/curl_get_line.h" "${CURL_LIBRARY_DIR}/altsvc.h" "${CURL_LIBRARY_DIR}/quic.h" "${CURL_LIBRARY_DIR}/socketpair.h")
SET(LIB_RCFILES "${CURL_LIBRARY_DIR}/libcurl.rc")
SET(CSOURCES ${LIB_CFILES} ${LIB_VAUTH_CFILES} ${LIB_VTLS_CFILES}
${LIB_VQUIC_CFILES} ${LIB_VSSH_CFILES})
SET(HHEADERS ${LIB_HFILES} ${LIB_VAUTH_HFILES} ${LIB_VTLS_HFILES}
${LIB_VQUIC_HFILES} ${LIB_VSSH_HFILES})
configure_file(${CURL_SOURCE_DIR}/lib/curl_config.h.cmake
${CMAKE_CURRENT_BINARY_DIR}/curl/curl_config.h)
list(APPEND HHEADERS
${CMAKE_CURRENT_BINARY_DIR}/curl/curl_config.h
)
add_library(libcurl ${HHEADERS} ${CSOURCES})
if(NOT BUILD_SHARED_LIBS)
set_target_properties(libcurl PROPERTIES INTERFACE_COMPILE_DEFINITIONS CURL_STATICLIB)
endif()
if(HIDES_CURL_PRIVATE_SYMBOLS)
set_property(TARGET libcurl APPEND PROPERTY COMPILE_DEFINITIONS "CURL_HIDDEN_SYMBOLS")
set_property(TARGET libcurl APPEND PROPERTY COMPILE_FLAGS ${CURL_CFLAG_SYMBOLS_HIDE})
endif()
if(OPENSSL_FOUND)
target_include_directories(libcurl PUBLIC ${OPENSSL_INCLUDE_DIR})
message("-- Including openssl ${OPENSSL_INCLUDE_DIR} to curl")
endif()
if(ZLIB_FOUND)
target_include_directories(libcurl PUBLIC ${ZLIB_INCLUDE_DIRS}})
message("-- Including zlib ${ZLIB_INCLUDE_DIRS} to curl")
endif()
target_compile_definitions(libcurl PUBLIC -DHAVE_CONFIG_H)
target_compile_definitions(libcurl PUBLIC -DBUILDING_LIBCURL)
target_include_directories(libcurl PUBLIC "${CURL_SOURCE_DIR}/include" "${CURL_LIBRARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/curl")
target_link_libraries(libcurl ${CURL_LIBS})

1
contrib/icu vendored Submodule

@ -0,0 +1 @@
Subproject commit faa2f9f9e1fe74c5ed00eba371d2830134cdbea1

459
contrib/icu-cmake/CMakeLists.txt Normal file
View File

@ -0,0 +1,459 @@
set(ICU_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/icu/icu4c/source)
set(ICUDATA_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/icudata/)
# These lists of sources were generated from build log of the original ICU build system (configure + make).
set(ICUUC_SOURCES
${ICU_SOURCE_DIR}/common/errorcode.cpp
${ICU_SOURCE_DIR}/common/putil.cpp
${ICU_SOURCE_DIR}/common/umath.cpp
${ICU_SOURCE_DIR}/common/utypes.cpp
${ICU_SOURCE_DIR}/common/uinvchar.cpp
${ICU_SOURCE_DIR}/common/umutex.cpp
${ICU_SOURCE_DIR}/common/ucln_cmn.cpp
${ICU_SOURCE_DIR}/common/uinit.cpp
${ICU_SOURCE_DIR}/common/uobject.cpp
${ICU_SOURCE_DIR}/common/cmemory.cpp
${ICU_SOURCE_DIR}/common/charstr.cpp
${ICU_SOURCE_DIR}/common/cstr.cpp
${ICU_SOURCE_DIR}/common/udata.cpp
${ICU_SOURCE_DIR}/common/ucmndata.cpp
${ICU_SOURCE_DIR}/common/udatamem.cpp
${ICU_SOURCE_DIR}/common/umapfile.cpp
${ICU_SOURCE_DIR}/common/udataswp.cpp
${ICU_SOURCE_DIR}/common/utrie_swap.cpp
${ICU_SOURCE_DIR}/common/ucol_swp.cpp
${ICU_SOURCE_DIR}/common/utrace.cpp
${ICU_SOURCE_DIR}/common/uhash.cpp
${ICU_SOURCE_DIR}/common/uhash_us.cpp
${ICU_SOURCE_DIR}/common/uenum.cpp
${ICU_SOURCE_DIR}/common/ustrenum.cpp
${ICU_SOURCE_DIR}/common/uvector.cpp
${ICU_SOURCE_DIR}/common/ustack.cpp
${ICU_SOURCE_DIR}/common/uvectr32.cpp
${ICU_SOURCE_DIR}/common/uvectr64.cpp
${ICU_SOURCE_DIR}/common/ucnv.cpp
${ICU_SOURCE_DIR}/common/ucnv_bld.cpp
${ICU_SOURCE_DIR}/common/ucnv_cnv.cpp
${ICU_SOURCE_DIR}/common/ucnv_io.cpp
${ICU_SOURCE_DIR}/common/ucnv_cb.cpp
${ICU_SOURCE_DIR}/common/ucnv_err.cpp
${ICU_SOURCE_DIR}/common/ucnvlat1.cpp
${ICU_SOURCE_DIR}/common/ucnv_u7.cpp
${ICU_SOURCE_DIR}/common/ucnv_u8.cpp
${ICU_SOURCE_DIR}/common/ucnv_u16.cpp
${ICU_SOURCE_DIR}/common/ucnv_u32.cpp
${ICU_SOURCE_DIR}/common/ucnvscsu.cpp
${ICU_SOURCE_DIR}/common/ucnvbocu.cpp
${ICU_SOURCE_DIR}/common/ucnv_ext.cpp
${ICU_SOURCE_DIR}/common/ucnvmbcs.cpp
${ICU_SOURCE_DIR}/common/ucnv2022.cpp
${ICU_SOURCE_DIR}/common/ucnvhz.cpp
${ICU_SOURCE_DIR}/common/ucnv_lmb.cpp
${ICU_SOURCE_DIR}/common/ucnvisci.cpp
${ICU_SOURCE_DIR}/common/ucnvdisp.cpp
${ICU_SOURCE_DIR}/common/ucnv_set.cpp
${ICU_SOURCE_DIR}/common/ucnv_ct.cpp
${ICU_SOURCE_DIR}/common/resource.cpp
${ICU_SOURCE_DIR}/common/uresbund.cpp
${ICU_SOURCE_DIR}/common/ures_cnv.cpp
${ICU_SOURCE_DIR}/common/uresdata.cpp
${ICU_SOURCE_DIR}/common/resbund.cpp
${ICU_SOURCE_DIR}/common/resbund_cnv.cpp
${ICU_SOURCE_DIR}/common/ucurr.cpp
${ICU_SOURCE_DIR}/common/localebuilder.cpp
${ICU_SOURCE_DIR}/common/localeprioritylist.cpp
${ICU_SOURCE_DIR}/common/messagepattern.cpp
${ICU_SOURCE_DIR}/common/ucat.cpp
${ICU_SOURCE_DIR}/common/locmap.cpp
${ICU_SOURCE_DIR}/common/uloc.cpp
${ICU_SOURCE_DIR}/common/locid.cpp
${ICU_SOURCE_DIR}/common/locutil.cpp
${ICU_SOURCE_DIR}/common/locavailable.cpp
${ICU_SOURCE_DIR}/common/locdispnames.cpp
${ICU_SOURCE_DIR}/common/locdspnm.cpp
${ICU_SOURCE_DIR}/common/loclikely.cpp
${ICU_SOURCE_DIR}/common/locresdata.cpp
${ICU_SOURCE_DIR}/common/lsr.cpp
${ICU_SOURCE_DIR}/common/loclikelysubtags.cpp
${ICU_SOURCE_DIR}/common/locdistance.cpp
${ICU_SOURCE_DIR}/common/localematcher.cpp
${ICU_SOURCE_DIR}/common/bytestream.cpp
${ICU_SOURCE_DIR}/common/stringpiece.cpp
${ICU_SOURCE_DIR}/common/bytesinkutil.cpp
${ICU_SOURCE_DIR}/common/stringtriebuilder.cpp
${ICU_SOURCE_DIR}/common/bytestriebuilder.cpp
${ICU_SOURCE_DIR}/common/bytestrie.cpp
${ICU_SOURCE_DIR}/common/bytestrieiterator.cpp
${ICU_SOURCE_DIR}/common/ucharstrie.cpp
${ICU_SOURCE_DIR}/common/ucharstriebuilder.cpp
${ICU_SOURCE_DIR}/common/ucharstrieiterator.cpp
${ICU_SOURCE_DIR}/common/dictionarydata.cpp
${ICU_SOURCE_DIR}/common/edits.cpp
${ICU_SOURCE_DIR}/common/appendable.cpp
${ICU_SOURCE_DIR}/common/ustr_cnv.cpp
${ICU_SOURCE_DIR}/common/unistr_cnv.cpp
${ICU_SOURCE_DIR}/common/unistr.cpp
${ICU_SOURCE_DIR}/common/unistr_case.cpp
${ICU_SOURCE_DIR}/common/unistr_props.cpp
${ICU_SOURCE_DIR}/common/utf_impl.cpp
${ICU_SOURCE_DIR}/common/ustring.cpp
${ICU_SOURCE_DIR}/common/ustrcase.cpp
${ICU_SOURCE_DIR}/common/ucasemap.cpp
${ICU_SOURCE_DIR}/common/ucasemap_titlecase_brkiter.cpp
${ICU_SOURCE_DIR}/common/cstring.cpp
${ICU_SOURCE_DIR}/common/ustrfmt.cpp
${ICU_SOURCE_DIR}/common/ustrtrns.cpp
${ICU_SOURCE_DIR}/common/ustr_wcs.cpp
${ICU_SOURCE_DIR}/common/utext.cpp
${ICU_SOURCE_DIR}/common/unistr_case_locale.cpp
${ICU_SOURCE_DIR}/common/ustrcase_locale.cpp
${ICU_SOURCE_DIR}/common/unistr_titlecase_brkiter.cpp
${ICU_SOURCE_DIR}/common/ustr_titlecase_brkiter.cpp
${ICU_SOURCE_DIR}/common/normalizer2impl.cpp
${ICU_SOURCE_DIR}/common/normalizer2.cpp
${ICU_SOURCE_DIR}/common/filterednormalizer2.cpp
${ICU_SOURCE_DIR}/common/normlzr.cpp
${ICU_SOURCE_DIR}/common/unorm.cpp
${ICU_SOURCE_DIR}/common/unormcmp.cpp
${ICU_SOURCE_DIR}/common/loadednormalizer2impl.cpp
${ICU_SOURCE_DIR}/common/chariter.cpp
${ICU_SOURCE_DIR}/common/schriter.cpp
${ICU_SOURCE_DIR}/common/uchriter.cpp
${ICU_SOURCE_DIR}/common/uiter.cpp
${ICU_SOURCE_DIR}/common/patternprops.cpp
${ICU_SOURCE_DIR}/common/uchar.cpp
${ICU_SOURCE_DIR}/common/uprops.cpp
${ICU_SOURCE_DIR}/common/ucase.cpp
${ICU_SOURCE_DIR}/common/propname.cpp
${ICU_SOURCE_DIR}/common/ubidi_props.cpp
${ICU_SOURCE_DIR}/common/characterproperties.cpp
${ICU_SOURCE_DIR}/common/ubidi.cpp
${ICU_SOURCE_DIR}/common/ubidiwrt.cpp
${ICU_SOURCE_DIR}/common/ubidiln.cpp
${ICU_SOURCE_DIR}/common/ushape.cpp
${ICU_SOURCE_DIR}/common/uscript.cpp
${ICU_SOURCE_DIR}/common/uscript_props.cpp
${ICU_SOURCE_DIR}/common/usc_impl.cpp
${ICU_SOURCE_DIR}/common/unames.cpp
${ICU_SOURCE_DIR}/common/utrie.cpp
${ICU_SOURCE_DIR}/common/utrie2.cpp
${ICU_SOURCE_DIR}/common/utrie2_builder.cpp
${ICU_SOURCE_DIR}/common/ucptrie.cpp
${ICU_SOURCE_DIR}/common/umutablecptrie.cpp
${ICU_SOURCE_DIR}/common/bmpset.cpp
${ICU_SOURCE_DIR}/common/unisetspan.cpp
${ICU_SOURCE_DIR}/common/uset_props.cpp
${ICU_SOURCE_DIR}/common/uniset_props.cpp
${ICU_SOURCE_DIR}/common/uniset_closure.cpp
${ICU_SOURCE_DIR}/common/uset.cpp
${ICU_SOURCE_DIR}/common/uniset.cpp
${ICU_SOURCE_DIR}/common/usetiter.cpp
${ICU_SOURCE_DIR}/common/ruleiter.cpp
${ICU_SOURCE_DIR}/common/caniter.cpp
${ICU_SOURCE_DIR}/common/unifilt.cpp
${ICU_SOURCE_DIR}/common/unifunct.cpp
${ICU_SOURCE_DIR}/common/uarrsort.cpp
${ICU_SOURCE_DIR}/common/brkiter.cpp
${ICU_SOURCE_DIR}/common/ubrk.cpp
${ICU_SOURCE_DIR}/common/brkeng.cpp
${ICU_SOURCE_DIR}/common/dictbe.cpp
${ICU_SOURCE_DIR}/common/filteredbrk.cpp
${ICU_SOURCE_DIR}/common/rbbi.cpp
${ICU_SOURCE_DIR}/common/rbbidata.cpp
${ICU_SOURCE_DIR}/common/rbbinode.cpp
${ICU_SOURCE_DIR}/common/rbbirb.cpp
${ICU_SOURCE_DIR}/common/rbbiscan.cpp
${ICU_SOURCE_DIR}/common/rbbisetb.cpp
${ICU_SOURCE_DIR}/common/rbbistbl.cpp
${ICU_SOURCE_DIR}/common/rbbitblb.cpp
${ICU_SOURCE_DIR}/common/rbbi_cache.cpp
${ICU_SOURCE_DIR}/common/serv.cpp
${ICU_SOURCE_DIR}/common/servnotf.cpp
${ICU_SOURCE_DIR}/common/servls.cpp
${ICU_SOURCE_DIR}/common/servlk.cpp
${ICU_SOURCE_DIR}/common/servlkf.cpp
${ICU_SOURCE_DIR}/common/servrbf.cpp
${ICU_SOURCE_DIR}/common/servslkf.cpp
${ICU_SOURCE_DIR}/common/uidna.cpp
${ICU_SOURCE_DIR}/common/usprep.cpp
${ICU_SOURCE_DIR}/common/uts46.cpp
${ICU_SOURCE_DIR}/common/punycode.cpp
${ICU_SOURCE_DIR}/common/util.cpp
${ICU_SOURCE_DIR}/common/util_props.cpp
${ICU_SOURCE_DIR}/common/parsepos.cpp
${ICU_SOURCE_DIR}/common/locbased.cpp
${ICU_SOURCE_DIR}/common/cwchar.cpp
${ICU_SOURCE_DIR}/common/wintz.cpp
${ICU_SOURCE_DIR}/common/dtintrv.cpp
${ICU_SOURCE_DIR}/common/ucnvsel.cpp
${ICU_SOURCE_DIR}/common/propsvec.cpp
${ICU_SOURCE_DIR}/common/ulist.cpp
${ICU_SOURCE_DIR}/common/uloc_tag.cpp
${ICU_SOURCE_DIR}/common/icudataver.cpp
${ICU_SOURCE_DIR}/common/icuplug.cpp
${ICU_SOURCE_DIR}/common/sharedobject.cpp
${ICU_SOURCE_DIR}/common/simpleformatter.cpp
${ICU_SOURCE_DIR}/common/unifiedcache.cpp
${ICU_SOURCE_DIR}/common/uloc_keytype.cpp
${ICU_SOURCE_DIR}/common/ubiditransform.cpp
${ICU_SOURCE_DIR}/common/pluralmap.cpp
${ICU_SOURCE_DIR}/common/static_unicode_sets.cpp
${ICU_SOURCE_DIR}/common/restrace.cpp)
set(ICUI18N_SOURCES
${ICU_SOURCE_DIR}/i18n/ucln_in.cpp
${ICU_SOURCE_DIR}/i18n/fmtable.cpp
${ICU_SOURCE_DIR}/i18n/format.cpp
${ICU_SOURCE_DIR}/i18n/msgfmt.cpp
${ICU_SOURCE_DIR}/i18n/umsg.cpp
${ICU_SOURCE_DIR}/i18n/numfmt.cpp
${ICU_SOURCE_DIR}/i18n/unum.cpp
${ICU_SOURCE_DIR}/i18n/decimfmt.cpp
${ICU_SOURCE_DIR}/i18n/dcfmtsym.cpp
${ICU_SOURCE_DIR}/i18n/fmtable_cnv.cpp
${ICU_SOURCE_DIR}/i18n/choicfmt.cpp
${ICU_SOURCE_DIR}/i18n/datefmt.cpp
${ICU_SOURCE_DIR}/i18n/smpdtfmt.cpp
${ICU_SOURCE_DIR}/i18n/reldtfmt.cpp
${ICU_SOURCE_DIR}/i18n/dtfmtsym.cpp
${ICU_SOURCE_DIR}/i18n/udat.cpp
${ICU_SOURCE_DIR}/i18n/dtptngen.cpp
${ICU_SOURCE_DIR}/i18n/udatpg.cpp
${ICU_SOURCE_DIR}/i18n/nfrs.cpp
${ICU_SOURCE_DIR}/i18n/nfrule.cpp
${ICU_SOURCE_DIR}/i18n/nfsubs.cpp
${ICU_SOURCE_DIR}/i18n/rbnf.cpp
${ICU_SOURCE_DIR}/i18n/numsys.cpp
${ICU_SOURCE_DIR}/i18n/unumsys.cpp
${ICU_SOURCE_DIR}/i18n/ucsdet.cpp
${ICU_SOURCE_DIR}/i18n/ucal.cpp
${ICU_SOURCE_DIR}/i18n/calendar.cpp
${ICU_SOURCE_DIR}/i18n/gregocal.cpp
${ICU_SOURCE_DIR}/i18n/timezone.cpp
${ICU_SOURCE_DIR}/i18n/simpletz.cpp
${ICU_SOURCE_DIR}/i18n/olsontz.cpp
${ICU_SOURCE_DIR}/i18n/astro.cpp
${ICU_SOURCE_DIR}/i18n/taiwncal.cpp
${ICU_SOURCE_DIR}/i18n/buddhcal.cpp
${ICU_SOURCE_DIR}/i18n/persncal.cpp
${ICU_SOURCE_DIR}/i18n/islamcal.cpp
${ICU_SOURCE_DIR}/i18n/japancal.cpp
${ICU_SOURCE_DIR}/i18n/gregoimp.cpp
${ICU_SOURCE_DIR}/i18n/hebrwcal.cpp
${ICU_SOURCE_DIR}/i18n/indiancal.cpp
${ICU_SOURCE_DIR}/i18n/chnsecal.cpp
${ICU_SOURCE_DIR}/i18n/cecal.cpp
${ICU_SOURCE_DIR}/i18n/coptccal.cpp
${ICU_SOURCE_DIR}/i18n/dangical.cpp
${ICU_SOURCE_DIR}/i18n/ethpccal.cpp
${ICU_SOURCE_DIR}/i18n/coleitr.cpp
${ICU_SOURCE_DIR}/i18n/coll.cpp
${ICU_SOURCE_DIR}/i18n/sortkey.cpp
${ICU_SOURCE_DIR}/i18n/bocsu.cpp
${ICU_SOURCE_DIR}/i18n/ucoleitr.cpp
${ICU_SOURCE_DIR}/i18n/ucol.cpp
${ICU_SOURCE_DIR}/i18n/ucol_res.cpp
${ICU_SOURCE_DIR}/i18n/ucol_sit.cpp
${ICU_SOURCE_DIR}/i18n/collation.cpp
${ICU_SOURCE_DIR}/i18n/collationsettings.cpp
${ICU_SOURCE_DIR}/i18n/collationdata.cpp
${ICU_SOURCE_DIR}/i18n/collationtailoring.cpp
${ICU_SOURCE_DIR}/i18n/collationdatareader.cpp
${ICU_SOURCE_DIR}/i18n/collationdatawriter.cpp
${ICU_SOURCE_DIR}/i18n/collationfcd.cpp
${ICU_SOURCE_DIR}/i18n/collationiterator.cpp
${ICU_SOURCE_DIR}/i18n/utf16collationiterator.cpp
${ICU_SOURCE_DIR}/i18n/utf8collationiterator.cpp
${ICU_SOURCE_DIR}/i18n/uitercollationiterator.cpp
${ICU_SOURCE_DIR}/i18n/collationsets.cpp
${ICU_SOURCE_DIR}/i18n/collationcompare.cpp
${ICU_SOURCE_DIR}/i18n/collationfastlatin.cpp
${ICU_SOURCE_DIR}/i18n/collationkeys.cpp
${ICU_SOURCE_DIR}/i18n/rulebasedcollator.cpp
${ICU_SOURCE_DIR}/i18n/collationroot.cpp
${ICU_SOURCE_DIR}/i18n/collationrootelements.cpp
${ICU_SOURCE_DIR}/i18n/collationdatabuilder.cpp
${ICU_SOURCE_DIR}/i18n/collationweights.cpp
${ICU_SOURCE_DIR}/i18n/collationruleparser.cpp
${ICU_SOURCE_DIR}/i18n/collationbuilder.cpp
${ICU_SOURCE_DIR}/i18n/collationfastlatinbuilder.cpp
${ICU_SOURCE_DIR}/i18n/listformatter.cpp
${ICU_SOURCE_DIR}/i18n/ulistformatter.cpp
${ICU_SOURCE_DIR}/i18n/strmatch.cpp
${ICU_SOURCE_DIR}/i18n/usearch.cpp
${ICU_SOURCE_DIR}/i18n/search.cpp
${ICU_SOURCE_DIR}/i18n/stsearch.cpp
${ICU_SOURCE_DIR}/i18n/translit.cpp
${ICU_SOURCE_DIR}/i18n/utrans.cpp
${ICU_SOURCE_DIR}/i18n/esctrn.cpp
${ICU_SOURCE_DIR}/i18n/unesctrn.cpp
${ICU_SOURCE_DIR}/i18n/funcrepl.cpp
${ICU_SOURCE_DIR}/i18n/strrepl.cpp
${ICU_SOURCE_DIR}/i18n/tridpars.cpp
${ICU_SOURCE_DIR}/i18n/cpdtrans.cpp
${ICU_SOURCE_DIR}/i18n/rbt.cpp
${ICU_SOURCE_DIR}/i18n/rbt_data.cpp
${ICU_SOURCE_DIR}/i18n/rbt_pars.cpp
${ICU_SOURCE_DIR}/i18n/rbt_rule.cpp
${ICU_SOURCE_DIR}/i18n/rbt_set.cpp
${ICU_SOURCE_DIR}/i18n/nultrans.cpp
${ICU_SOURCE_DIR}/i18n/remtrans.cpp
${ICU_SOURCE_DIR}/i18n/casetrn.cpp
${ICU_SOURCE_DIR}/i18n/titletrn.cpp
${ICU_SOURCE_DIR}/i18n/tolowtrn.cpp
${ICU_SOURCE_DIR}/i18n/toupptrn.cpp
${ICU_SOURCE_DIR}/i18n/anytrans.cpp
${ICU_SOURCE_DIR}/i18n/name2uni.cpp
${ICU_SOURCE_DIR}/i18n/uni2name.cpp
${ICU_SOURCE_DIR}/i18n/nortrans.cpp
${ICU_SOURCE_DIR}/i18n/quant.cpp
${ICU_SOURCE_DIR}/i18n/transreg.cpp
${ICU_SOURCE_DIR}/i18n/brktrans.cpp
${ICU_SOURCE_DIR}/i18n/regexcmp.cpp
${ICU_SOURCE_DIR}/i18n/rematch.cpp
${ICU_SOURCE_DIR}/i18n/repattrn.cpp
${ICU_SOURCE_DIR}/i18n/regexst.cpp
${ICU_SOURCE_DIR}/i18n/regextxt.cpp
${ICU_SOURCE_DIR}/i18n/regeximp.cpp
${ICU_SOURCE_DIR}/i18n/uregex.cpp
${ICU_SOURCE_DIR}/i18n/uregexc.cpp
${ICU_SOURCE_DIR}/i18n/ulocdata.cpp
${ICU_SOURCE_DIR}/i18n/measfmt.cpp
${ICU_SOURCE_DIR}/i18n/currfmt.cpp
${ICU_SOURCE_DIR}/i18n/curramt.cpp
${ICU_SOURCE_DIR}/i18n/currunit.cpp
${ICU_SOURCE_DIR}/i18n/measure.cpp
${ICU_SOURCE_DIR}/i18n/utmscale.cpp
${ICU_SOURCE_DIR}/i18n/csdetect.cpp
${ICU_SOURCE_DIR}/i18n/csmatch.cpp
${ICU_SOURCE_DIR}/i18n/csr2022.cpp
${ICU_SOURCE_DIR}/i18n/csrecog.cpp
${ICU_SOURCE_DIR}/i18n/csrmbcs.cpp
${ICU_SOURCE_DIR}/i18n/csrsbcs.cpp
${ICU_SOURCE_DIR}/i18n/csrucode.cpp
${ICU_SOURCE_DIR}/i18n/csrutf8.cpp
${ICU_SOURCE_DIR}/i18n/inputext.cpp
${ICU_SOURCE_DIR}/i18n/wintzimpl.cpp
${ICU_SOURCE_DIR}/i18n/windtfmt.cpp
${ICU_SOURCE_DIR}/i18n/winnmfmt.cpp
${ICU_SOURCE_DIR}/i18n/basictz.cpp
${ICU_SOURCE_DIR}/i18n/dtrule.cpp
${ICU_SOURCE_DIR}/i18n/rbtz.cpp
${ICU_SOURCE_DIR}/i18n/tzrule.cpp
${ICU_SOURCE_DIR}/i18n/tztrans.cpp
${ICU_SOURCE_DIR}/i18n/vtzone.cpp
${ICU_SOURCE_DIR}/i18n/zonemeta.cpp
${ICU_SOURCE_DIR}/i18n/standardplural.cpp
${ICU_SOURCE_DIR}/i18n/upluralrules.cpp
${ICU_SOURCE_DIR}/i18n/plurrule.cpp
${ICU_SOURCE_DIR}/i18n/plurfmt.cpp
${ICU_SOURCE_DIR}/i18n/selfmt.cpp
${ICU_SOURCE_DIR}/i18n/dtitvfmt.cpp
${ICU_SOURCE_DIR}/i18n/dtitvinf.cpp
${ICU_SOURCE_DIR}/i18n/udateintervalformat.cpp
${ICU_SOURCE_DIR}/i18n/tmunit.cpp
${ICU_SOURCE_DIR}/i18n/tmutamt.cpp
${ICU_SOURCE_DIR}/i18n/tmutfmt.cpp
${ICU_SOURCE_DIR}/i18n/currpinf.cpp
${ICU_SOURCE_DIR}/i18n/uspoof.cpp
${ICU_SOURCE_DIR}/i18n/uspoof_impl.cpp
${ICU_SOURCE_DIR}/i18n/uspoof_build.cpp
${ICU_SOURCE_DIR}/i18n/uspoof_conf.cpp
${ICU_SOURCE_DIR}/i18n/smpdtfst.cpp
${ICU_SOURCE_DIR}/i18n/ztrans.cpp
${ICU_SOURCE_DIR}/i18n/zrule.cpp
${ICU_SOURCE_DIR}/i18n/vzone.cpp
${ICU_SOURCE_DIR}/i18n/fphdlimp.cpp
${ICU_SOURCE_DIR}/i18n/fpositer.cpp
${ICU_SOURCE_DIR}/i18n/ufieldpositer.cpp
${ICU_SOURCE_DIR}/i18n/decNumber.cpp
${ICU_SOURCE_DIR}/i18n/decContext.cpp
${ICU_SOURCE_DIR}/i18n/alphaindex.cpp
${ICU_SOURCE_DIR}/i18n/tznames.cpp
${ICU_SOURCE_DIR}/i18n/tznames_impl.cpp
${ICU_SOURCE_DIR}/i18n/tzgnames.cpp
${ICU_SOURCE_DIR}/i18n/tzfmt.cpp
${ICU_SOURCE_DIR}/i18n/compactdecimalformat.cpp
${ICU_SOURCE_DIR}/i18n/gender.cpp
${ICU_SOURCE_DIR}/i18n/region.cpp
${ICU_SOURCE_DIR}/i18n/scriptset.cpp
${ICU_SOURCE_DIR}/i18n/uregion.cpp
${ICU_SOURCE_DIR}/i18n/reldatefmt.cpp
${ICU_SOURCE_DIR}/i18n/quantityformatter.cpp
${ICU_SOURCE_DIR}/i18n/measunit.cpp
${ICU_SOURCE_DIR}/i18n/sharedbreakiterator.cpp
${ICU_SOURCE_DIR}/i18n/scientificnumberformatter.cpp
${ICU_SOURCE_DIR}/i18n/dayperiodrules.cpp
${ICU_SOURCE_DIR}/i18n/nounit.cpp
${ICU_SOURCE_DIR}/i18n/number_affixutils.cpp
${ICU_SOURCE_DIR}/i18n/number_compact.cpp
${ICU_SOURCE_DIR}/i18n/number_decimalquantity.cpp
${ICU_SOURCE_DIR}/i18n/number_decimfmtprops.cpp
${ICU_SOURCE_DIR}/i18n/number_fluent.cpp
${ICU_SOURCE_DIR}/i18n/number_formatimpl.cpp
${ICU_SOURCE_DIR}/i18n/number_grouping.cpp
${ICU_SOURCE_DIR}/i18n/number_integerwidth.cpp
${ICU_SOURCE_DIR}/i18n/number_longnames.cpp
${ICU_SOURCE_DIR}/i18n/number_modifiers.cpp
${ICU_SOURCE_DIR}/i18n/number_notation.cpp
${ICU_SOURCE_DIR}/i18n/number_output.cpp
${ICU_SOURCE_DIR}/i18n/number_padding.cpp
${ICU_SOURCE_DIR}/i18n/number_patternmodifier.cpp
${ICU_SOURCE_DIR}/i18n/number_patternstring.cpp
${ICU_SOURCE_DIR}/i18n/number_rounding.cpp
${ICU_SOURCE_DIR}/i18n/number_scientific.cpp
${ICU_SOURCE_DIR}/i18n/number_utils.cpp
${ICU_SOURCE_DIR}/i18n/number_asformat.cpp
${ICU_SOURCE_DIR}/i18n/number_mapper.cpp
${ICU_SOURCE_DIR}/i18n/number_multiplier.cpp
${ICU_SOURCE_DIR}/i18n/number_currencysymbols.cpp
${ICU_SOURCE_DIR}/i18n/number_skeletons.cpp
${ICU_SOURCE_DIR}/i18n/number_capi.cpp
${ICU_SOURCE_DIR}/i18n/double-conversion-string-to-double.cpp
${ICU_SOURCE_DIR}/i18n/double-conversion-double-to-string.cpp
${ICU_SOURCE_DIR}/i18n/double-conversion-bignum-dtoa.cpp
${ICU_SOURCE_DIR}/i18n/double-conversion-bignum.cpp
${ICU_SOURCE_DIR}/i18n/double-conversion-cached-powers.cpp
${ICU_SOURCE_DIR}/i18n/double-conversion-fast-dtoa.cpp
${ICU_SOURCE_DIR}/i18n/double-conversion-strtod.cpp
${ICU_SOURCE_DIR}/i18n/string_segment.cpp
${ICU_SOURCE_DIR}/i18n/numparse_parsednumber.cpp
${ICU_SOURCE_DIR}/i18n/numparse_impl.cpp
${ICU_SOURCE_DIR}/i18n/numparse_symbols.cpp
${ICU_SOURCE_DIR}/i18n/numparse_decimal.cpp
${ICU_SOURCE_DIR}/i18n/numparse_scientific.cpp
${ICU_SOURCE_DIR}/i18n/numparse_currency.cpp
${ICU_SOURCE_DIR}/i18n/numparse_affixes.cpp
${ICU_SOURCE_DIR}/i18n/numparse_compositions.cpp
${ICU_SOURCE_DIR}/i18n/numparse_validators.cpp
${ICU_SOURCE_DIR}/i18n/numrange_fluent.cpp
${ICU_SOURCE_DIR}/i18n/numrange_impl.cpp
${ICU_SOURCE_DIR}/i18n/erarules.cpp
${ICU_SOURCE_DIR}/i18n/formattedvalue.cpp
${ICU_SOURCE_DIR}/i18n/formattedval_iterimpl.cpp
${ICU_SOURCE_DIR}/i18n/formattedval_sbimpl.cpp
${ICU_SOURCE_DIR}/i18n/formatted_string_builder.cpp)
enable_language(ASM)
set(ICUDATA_SOURCES ${ICUDATA_SOURCE_DIR}/icudt66l_dat.S)
# Note that we don't like any kind of binary plugins (because of runtime dependencies, vulnerabilities, ABI incompatibilities).
add_definitions(-D_REENTRANT -DU_HAVE_ELF_H=1 -DU_HAVE_STRTOD_L=1 -DU_HAVE_XLOCALE_H=0 -DDEFAULT_ICU_PLUGINS="/dev/null")
add_library(icuuc ${ICUUC_SOURCES})
add_library(icui18n ${ICUI18N_SOURCES})
add_library(icudata ${ICUDATA_SOURCES})
target_link_libraries(icuuc icudata)
target_link_libraries(icui18n icuuc)
target_include_directories(icuuc SYSTEM PUBLIC ${ICU_SOURCE_DIR}/common/)
target_include_directories(icui18n SYSTEM PUBLIC ${ICU_SOURCE_DIR}/i18n/)
target_compile_definitions(icuuc PRIVATE -DU_COMMON_IMPLEMENTATION)
target_compile_definitions(icui18n PRIVATE -DU_I18N_IMPLEMENTATION)
if (COMPILER_CLANG)
target_compile_options(icudata PRIVATE -Wno-unused-command-line-argument)
endif ()

1
contrib/icudata vendored Submodule

@ -0,0 +1 @@
Subproject commit f020820388e3faafb44cc643574a2d563dfde572

2
contrib/lz4 vendored

@ -1 +1 @@
Subproject commit 7a4e3b1fac5cd9d4ec7c8d0091329ba107ec2131
Subproject commit 3d67671559be723b0912bbee2fcd2eb14783a721

1
contrib/openssl vendored Submodule

@ -0,0 +1 @@
Subproject commit c74e7895eb0d219d4007775eec134dd2bcd9d1ae

1019
contrib/openssl-cmake/CMakeLists.txt Normal file
View File

File diff suppressed because it is too large Load Diff

28
contrib/openssl-cmake/linux_aarch64/include/crypto/bn_conf.h Normal file
View File

@ -0,0 +1,28 @@
/* WARNING: do not edit! */
/* Generated by Makefile from /home/milovidov/work/ClickHouse/contrib/openssl/include/crypto/bn_conf.h.in */
/*
* Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
*
* Licensed under the Apache License 2.0 (the "License"). You may not use
* this file except in compliance with the License. You can obtain a copy
* in the file LICENSE in the source distribution or at
* https://www.openssl.org/source/license.html
*/
#ifndef OSSL_CRYPTO_BN_CONF_H
# define OSSL_CRYPTO_BN_CONF_H
/*
* The contents of this file are not used in the UEFI build, as
* both 32-bit and 64-bit builds are supported from a single run
* of the Configure script.
*/
/* Should we define BN_DIV2W here? */
/* Only one for the following should be defined */
#define SIXTY_FOUR_BIT_LONG
#undef SIXTY_FOUR_BIT
#undef THIRTY_TWO_BIT
#endif

17
contrib/openssl-cmake/linux_aarch64/include/crypto/dso_conf.h Normal file
View File

@ -0,0 +1,17 @@
/* WARNING: do not edit! */
/* Generated by Makefile from /home/milovidov/work/ClickHouse/contrib/openssl/include/crypto/dso_conf.h.in */
/*
* Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved.
*
* Licensed under the Apache License 2.0 (the "License"). You may not use
* this file except in compliance with the License. You can obtain a copy
* in the file LICENSE in the source distribution or at
* https://www.openssl.org/source/license.html
*/
#ifndef OSSL_CRYPTO_DSO_CONF_H
# define OSSL_CRYPTO_DSO_CONF_H
# define DSO_DLFCN
# define HAVE_DLFCN_H
# define DSO_EXTENSION ".so"
#endif

132
contrib/openssl-cmake/linux_aarch64/include/openssl/configuration.h Normal file
View File

@ -0,0 +1,132 @@
/*
* WARNING: do not edit!
* Generated by Makefile from include/openssl/configuration.h.in
*
* Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved.
*
* Licensed under the Apache License 2.0 (the "License"). You may not use
* this file except in compliance with the License. You can obtain a copy
* in the file LICENSE in the source distribution or at
* https://www.openssl.org/source/license.html
*/
#ifndef OPENSSL_CONFIGURATION_H
# define OPENSSL_CONFIGURATION_H
# ifdef __cplusplus
extern "C" {
# endif
# ifdef OPENSSL_ALGORITHM_DEFINES
# error OPENSSL_ALGORITHM_DEFINES no longer supported
# endif
/*
* OpenSSL was configured with the following options:
*/
# define OPENSSL_CONFIGURED_API 30000
/// This fragment was edited to avoid dependency on "getrandom" function that is not available on old libc and old Linux kernels.
/// The DEVRANDOM method is also good.
//# ifndef OPENSSL_RAND_SEED_OS
//# define OPENSSL_RAND_SEED_OS
//# endif
#define OPENSSL_RAND_SEED_DEVRANDOM
# ifndef OPENSSL_THREADS
# define OPENSSL_THREADS
# endif
# ifndef OPENSSL_NO_ASAN
# define OPENSSL_NO_ASAN
# endif
# ifndef OPENSSL_NO_CRYPTO_MDEBUG
# define OPENSSL_NO_CRYPTO_MDEBUG
# endif
# ifndef OPENSSL_NO_CRYPTO_MDEBUG_BACKTRACE
# define OPENSSL_NO_CRYPTO_MDEBUG_BACKTRACE
# endif
# ifndef OPENSSL_NO_DEVCRYPTOENG
# define OPENSSL_NO_DEVCRYPTOENG
# endif
# ifndef OPENSSL_NO_EC_NISTP_64_GCC_128
# define OPENSSL_NO_EC_NISTP_64_GCC_128
# endif
# ifndef OPENSSL_NO_EGD
# define OPENSSL_NO_EGD
# endif
# ifndef OPENSSL_NO_EXTERNAL_TESTS
# define OPENSSL_NO_EXTERNAL_TESTS
# endif
# ifndef OPENSSL_NO_FUZZ_AFL
# define OPENSSL_NO_FUZZ_AFL
# endif
# ifndef OPENSSL_NO_FUZZ_LIBFUZZER
# define OPENSSL_NO_FUZZ_LIBFUZZER
# endif
# ifndef OPENSSL_NO_KTLS
# define OPENSSL_NO_KTLS
# endif
# ifndef OPENSSL_NO_MD2
# define OPENSSL_NO_MD2
# endif
# ifndef OPENSSL_NO_MSAN
# define OPENSSL_NO_MSAN
# endif
# ifndef OPENSSL_NO_RC5
# define OPENSSL_NO_RC5
# endif
# ifndef OPENSSL_NO_SCTP
# define OPENSSL_NO_SCTP
# endif
# ifndef OPENSSL_NO_SSL_TRACE
# define OPENSSL_NO_SSL_TRACE
# endif
# ifndef OPENSSL_NO_SSL3
# define OPENSSL_NO_SSL3
# endif
# ifndef OPENSSL_NO_SSL3_METHOD
# define OPENSSL_NO_SSL3_METHOD
# endif
# ifndef OPENSSL_NO_TRACE
# define OPENSSL_NO_TRACE
# endif
# ifndef OPENSSL_NO_UBSAN
# define OPENSSL_NO_UBSAN
# endif
# ifndef OPENSSL_NO_UNIT_TEST
# define OPENSSL_NO_UNIT_TEST
# endif
# ifndef OPENSSL_NO_UPLINK
# define OPENSSL_NO_UPLINK
# endif
# ifndef OPENSSL_NO_WEAK_SSL_CIPHERS
# define OPENSSL_NO_WEAK_SSL_CIPHERS
# endif
# ifndef OPENSSL_NO_STATIC_ENGINE
# define OPENSSL_NO_STATIC_ENGINE
# endif
/* Generate 80386 code? */
# undef I386_ONLY
/*
* The following are cipher-specific, but are part of the public API.
*/
# if !defined(OPENSSL_SYS_UEFI)
# undef BN_LLONG
/* Only one for the following should be defined */
# define SIXTY_FOUR_BIT_LONG
# undef SIXTY_FOUR_BIT
# undef THIRTY_TWO_BIT
# endif
# define RC4_INT unsigned char
# ifdef __cplusplus
}
# endif
#endif /* OPENSSL_CONFIGURATION_H */

114
contrib/openssl-cmake/linux_aarch64/include/openssl/opensslv.h Normal file
View File

@ -0,0 +1,114 @@
/*
* WARNING: do not edit!
* Generated by Makefile from /home/milovidov/work/ClickHouse/contrib/openssl/include/openssl/opensslv.h.in
*
* Copyright 1999-2019 The OpenSSL Project Authors. All Rights Reserved.
*
* Licensed under the Apache License 2.0 (the "License"). You may not use
* this file except in compliance with the License. You can obtain a copy
* in the file LICENSE in the source distribution or at
* https://www.openssl.org/source/license.html
*/
#ifndef OPENSSL_OPENSSLV_H
# define OPENSSL_OPENSSLV_H
# pragma once
# ifdef __cplusplus
extern "C" {
# endif
/*
* SECTION 1: VERSION DATA. These will change for each release
*/
/*
* Base version macros
*
* These macros express version number MAJOR.MINOR.PATCH exactly
*/
# define OPENSSL_VERSION_MAJOR 3
# define OPENSSL_VERSION_MINOR 0
# define OPENSSL_VERSION_PATCH 0
/*
* Additional version information
*
* These are also part of the new version scheme, but aren't part
* of the version number itself.
*/
/* Could be: #define OPENSSL_VERSION_PRE_RELEASE "-alpha.1" */
# define OPENSSL_VERSION_PRE_RELEASE "-dev"
/* Could be: #define OPENSSL_VERSION_BUILD_METADATA "+fips" */
/* Could be: #define OPENSSL_VERSION_BUILD_METADATA "+vendor.1" */
# define OPENSSL_VERSION_BUILD_METADATA ""
/*
* Note: The OpenSSL Project will never define OPENSSL_VERSION_BUILD_METADATA
* to be anything but the empty string. Its use is entirely reserved for
* others
*/
/*
* Shared library version
*
* This is strictly to express ABI version, which may or may not
* be related to the API version expressed with the macros above.
* This is defined in free form.
*/
# define OPENSSL_SHLIB_VERSION 3
/*
* SECTION 2: USEFUL MACROS
*/
/* For checking general API compatibility when preprocessing */
# define OPENSSL_VERSION_PREREQ(maj,min) \
((OPENSSL_VERSION_MAJOR << 16) + OPENSSL_VERSION_MINOR >= ((maj) << 16) + (min))
/*
* Macros to get the version in easily digested string form, both the short
* "MAJOR.MINOR.PATCH" variant (where MAJOR, MINOR and PATCH are replaced
* with the values from the corresponding OPENSSL_VERSION_ macros) and the
* longer variant with OPENSSL_VERSION_PRE_RELEASE_STR and
* OPENSSL_VERSION_BUILD_METADATA_STR appended.
*/
# define OPENSSL_VERSION_STR "3.0.0"
# define OPENSSL_FULL_VERSION_STR "3.0.0-dev"
/*
* SECTION 3: ADDITIONAL METADATA
*
* These strings are defined separately to allow them to be parsable.
*/
# define OPENSSL_RELEASE_DATE "xx XXX xxxx"
/*
* SECTION 4: BACKWARD COMPATIBILITY
*/
# define OPENSSL_VERSION_TEXT "OpenSSL 3.0.0-dev xx XXX xxxx"
/* Synthesize OPENSSL_VERSION_NUMBER with the layout 0xMNN00PPSL */
# ifdef OPENSSL_VERSION_PRE_RELEASE
# define _OPENSSL_VERSION_PRE_RELEASE 0x0
# else
# define _OPENSSL_VERSION_PRE_RELEASE 0xf
# endif
# define OPENSSL_VERSION_NUMBER \
( (OPENSSL_VERSION_MAJOR<<28) \
|(OPENSSL_VERSION_MINOR<<20) \
|(OPENSSL_VERSION_PATCH<<4) \
|_OPENSSL_VERSION_PRE_RELEASE )
# ifdef __cplusplus
}
# endif
# include <openssl/macros.h>
# ifndef OPENSSL_NO_DEPRECATED_3_0
# define HEADER_OPENSSLV_H
# endif
#endif /* OPENSSL_OPENSSLV_H */

23
contrib/openssl-cmake/linux_aarch64/include_private/buildinf.h Normal file
View File

@ -0,0 +1,23 @@
/*
* WARNING: do not edit!
* Generated by util/mkbuildinf.pl
*
* Copyright 2014-2017 The OpenSSL Project Authors. All Rights Reserved.
*
* Licensed under the Apache License 2.0 (the "License"). You may not use
* this file except in compliance with the License. You can obtain a copy
* in the file LICENSE in the source distribution or at
* https://www.openssl.org/source/license.html
*/
#define PLATFORM "platform: linux-aarch64"
#define DATE "built on: Sun Dec 15 01:54:14 2019 UTC"
/*
* Generate compiler_flags as an array of individual characters. This is a
* workaround for the situation where CFLAGS gets too long for a C90 string
* literal
*/
/// Edited because I don't care.
static const char compiler_flags[] = "";

28
contrib/openssl-cmake/linux_x86_64/include/crypto/bn_conf.h Normal file
View File

@ -0,0 +1,28 @@
/* WARNING: do not edit! */
/* Generated by Makefile from /home/milovidov/work/ClickHouse/contrib/openssl/include/crypto/bn_conf.h.in */
/*
* Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
*
* Licensed under the Apache License 2.0 (the "License"). You may not use
* this file except in compliance with the License. You can obtain a copy
* in the file LICENSE in the source distribution or at
* https://www.openssl.org/source/license.html
*/
#ifndef OSSL_CRYPTO_BN_CONF_H
# define OSSL_CRYPTO_BN_CONF_H
/*
* The contents of this file are not used in the UEFI build, as
* both 32-bit and 64-bit builds are supported from a single run
* of the Configure script.
*/
/* Should we define BN_DIV2W here? */
/* Only one for the following should be defined */
#define SIXTY_FOUR_BIT_LONG
#undef SIXTY_FOUR_BIT
#undef THIRTY_TWO_BIT
#endif

17
contrib/openssl-cmake/linux_x86_64/include/crypto/dso_conf.h Normal file
View File

@ -0,0 +1,17 @@
/* WARNING: do not edit! */
/* Generated by Makefile from /home/milovidov/work/ClickHouse/contrib/openssl/include/crypto/dso_conf.h.in */
/*
* Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved.
*
* Licensed under the Apache License 2.0 (the "License"). You may not use
* this file except in compliance with the License. You can obtain a copy
* in the file LICENSE in the source distribution or at
* https://www.openssl.org/source/license.html
*/
#ifndef OSSL_CRYPTO_DSO_CONF_H
# define OSSL_CRYPTO_DSO_CONF_H
# define DSO_DLFCN
# define HAVE_DLFCN_H
# define DSO_EXTENSION ".so"
#endif

132
contrib/openssl-cmake/linux_x86_64/include/openssl/configuration.h Normal file
View File

@ -0,0 +1,132 @@
/*
* WARNING: do not edit!
* Generated by Makefile from /home/milovidov/work/ClickHouse/contrib/openssl/include/openssl/configuration.h.in
*
* Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved.
*
* Licensed under the Apache License 2.0 (the "License"). You may not use
* this file except in compliance with the License. You can obtain a copy
* in the file LICENSE in the source distribution or at
* https://www.openssl.org/source/license.html
*/
#ifndef OPENSSL_CONFIGURATION_H
# define OPENSSL_CONFIGURATION_H
# ifdef __cplusplus
extern "C" {
# endif
# ifdef OPENSSL_ALGORITHM_DEFINES
# error OPENSSL_ALGORITHM_DEFINES no longer supported
# endif
/*
* OpenSSL was configured with the following options:
*/
# define OPENSSL_CONFIGURED_API 30000
/// This fragment was edited to avoid dependency on "getrandom" function that is not available on old libc and old Linux kernels.
/// The DEVRANDOM method is also good.
//# ifndef OPENSSL_RAND_SEED_OS
//# define OPENSSL_RAND_SEED_OS
//# endif
#define OPENSSL_RAND_SEED_DEVRANDOM
# ifndef OPENSSL_THREADS
# define OPENSSL_THREADS
# endif
# ifndef OPENSSL_NO_ASAN
# define OPENSSL_NO_ASAN
# endif
# ifndef OPENSSL_NO_CRYPTO_MDEBUG
# define OPENSSL_NO_CRYPTO_MDEBUG
# endif
# ifndef OPENSSL_NO_CRYPTO_MDEBUG_BACKTRACE
# define OPENSSL_NO_CRYPTO_MDEBUG_BACKTRACE
# endif
# ifndef OPENSSL_NO_DEVCRYPTOENG
# define OPENSSL_NO_DEVCRYPTOENG
# endif
# ifndef OPENSSL_NO_EC_NISTP_64_GCC_128
# define OPENSSL_NO_EC_NISTP_64_GCC_128
# endif
# ifndef OPENSSL_NO_EGD
# define OPENSSL_NO_EGD
# endif
# ifndef OPENSSL_NO_EXTERNAL_TESTS
# define OPENSSL_NO_EXTERNAL_TESTS
# endif
# ifndef OPENSSL_NO_FUZZ_AFL
# define OPENSSL_NO_FUZZ_AFL
# endif
# ifndef OPENSSL_NO_FUZZ_LIBFUZZER
# define OPENSSL_NO_FUZZ_LIBFUZZER
# endif
# ifndef OPENSSL_NO_KTLS
# define OPENSSL_NO_KTLS
# endif
# ifndef OPENSSL_NO_MD2
# define OPENSSL_NO_MD2
# endif
# ifndef OPENSSL_NO_MSAN
# define OPENSSL_NO_MSAN
# endif
# ifndef OPENSSL_NO_RC5
# define OPENSSL_NO_RC5
# endif
# ifndef OPENSSL_NO_SCTP
# define OPENSSL_NO_SCTP
# endif
# ifndef OPENSSL_NO_SSL_TRACE
# define OPENSSL_NO_SSL_TRACE
# endif
# ifndef OPENSSL_NO_SSL3
# define OPENSSL_NO_SSL3
# endif
# ifndef OPENSSL_NO_SSL3_METHOD
# define OPENSSL_NO_SSL3_METHOD
# endif
# ifndef OPENSSL_NO_TRACE
# define OPENSSL_NO_TRACE
# endif
# ifndef OPENSSL_NO_UBSAN
# define OPENSSL_NO_UBSAN
# endif
# ifndef OPENSSL_NO_UNIT_TEST
# define OPENSSL_NO_UNIT_TEST
# endif
# ifndef OPENSSL_NO_UPLINK
# define OPENSSL_NO_UPLINK
# endif
# ifndef OPENSSL_NO_WEAK_SSL_CIPHERS
# define OPENSSL_NO_WEAK_SSL_CIPHERS
# endif
# ifndef OPENSSL_NO_STATIC_ENGINE
# define OPENSSL_NO_STATIC_ENGINE
# endif
/* Generate 80386 code? */
# undef I386_ONLY
/*
* The following are cipher-specific, but are part of the public API.
*/
# if !defined(OPENSSL_SYS_UEFI)
# undef BN_LLONG
/* Only one for the following should be defined */
# define SIXTY_FOUR_BIT_LONG
# undef SIXTY_FOUR_BIT
# undef THIRTY_TWO_BIT
# endif
# define RC4_INT unsigned int
# ifdef __cplusplus
}
# endif
#endif /* OPENSSL_CONFIGURATION_H */

114
contrib/openssl-cmake/linux_x86_64/include/openssl/opensslv.h Normal file
View File

@ -0,0 +1,114 @@
/*
* WARNING: do not edit!
* Generated by Makefile from /home/milovidov/work/ClickHouse/contrib/openssl/include/openssl/opensslv.h.in
*
* Copyright 1999-2019 The OpenSSL Project Authors. All Rights Reserved.
*
* Licensed under the Apache License 2.0 (the "License"). You may not use
* this file except in compliance with the License. You can obtain a copy
* in the file LICENSE in the source distribution or at
* https://www.openssl.org/source/license.html
*/
#ifndef OPENSSL_OPENSSLV_H
# define OPENSSL_OPENSSLV_H
# pragma once
# ifdef __cplusplus
extern "C" {
# endif
/*
* SECTION 1: VERSION DATA. These will change for each release
*/
/*
* Base version macros
*
* These macros express version number MAJOR.MINOR.PATCH exactly
*/
# define OPENSSL_VERSION_MAJOR 3
# define OPENSSL_VERSION_MINOR 0
# define OPENSSL_VERSION_PATCH 0
/*
* Additional version information
*
* These are also part of the new version scheme, but aren't part
* of the version number itself.
*/
/* Could be: #define OPENSSL_VERSION_PRE_RELEASE "-alpha.1" */
# define OPENSSL_VERSION_PRE_RELEASE "-dev"
/* Could be: #define OPENSSL_VERSION_BUILD_METADATA "+fips" */
/* Could be: #define OPENSSL_VERSION_BUILD_METADATA "+vendor.1" */
# define OPENSSL_VERSION_BUILD_METADATA ""
/*
* Note: The OpenSSL Project will never define OPENSSL_VERSION_BUILD_METADATA
* to be anything but the empty string. Its use is entirely reserved for
* others
*/
/*
* Shared library version
*
* This is strictly to express ABI version, which may or may not
* be related to the API version expressed with the macros above.
* This is defined in free form.
*/
# define OPENSSL_SHLIB_VERSION 3
/*
* SECTION 2: USEFUL MACROS
*/
/* For checking general API compatibility when preprocessing */
# define OPENSSL_VERSION_PREREQ(maj,min) \
((OPENSSL_VERSION_MAJOR << 16) + OPENSSL_VERSION_MINOR >= ((maj) << 16) + (min))
/*
* Macros to get the version in easily digested string form, both the short
* "MAJOR.MINOR.PATCH" variant (where MAJOR, MINOR and PATCH are replaced
* with the values from the corresponding OPENSSL_VERSION_ macros) and the
* longer variant with OPENSSL_VERSION_PRE_RELEASE_STR and
* OPENSSL_VERSION_BUILD_METADATA_STR appended.
*/
# define OPENSSL_VERSION_STR "3.0.0"
# define OPENSSL_FULL_VERSION_STR "3.0.0-dev"
/*
* SECTION 3: ADDITIONAL METADATA
*
* These strings are defined separately to allow them to be parsable.
*/
# define OPENSSL_RELEASE_DATE "xx XXX xxxx"
/*
* SECTION 4: BACKWARD COMPATIBILITY
*/
# define OPENSSL_VERSION_TEXT "OpenSSL 3.0.0-dev xx XXX xxxx"
/* Synthesize OPENSSL_VERSION_NUMBER with the layout 0xMNN00PPSL */
# ifdef OPENSSL_VERSION_PRE_RELEASE
# define _OPENSSL_VERSION_PRE_RELEASE 0x0
# else
# define _OPENSSL_VERSION_PRE_RELEASE 0xf
# endif
# define OPENSSL_VERSION_NUMBER \
( (OPENSSL_VERSION_MAJOR<<28) \
|(OPENSSL_VERSION_MINOR<<20) \
|(OPENSSL_VERSION_PATCH<<4) \
|_OPENSSL_VERSION_PRE_RELEASE )
# ifdef __cplusplus
}
# endif
# include <openssl/macros.h>
# ifndef OPENSSL_NO_DEPRECATED_3_0
# define HEADER_OPENSSLV_H
# endif
#endif /* OPENSSL_OPENSSLV_H */

23
contrib/openssl-cmake/linux_x86_64/include_private/buildinf.h Normal file
View File

@ -0,0 +1,23 @@
/*
* WARNING: do not edit!
* Generated by util/mkbuildinf.pl
*
* Copyright 2014-2017 The OpenSSL Project Authors. All Rights Reserved.
*
* Licensed under the Apache License 2.0 (the "License"). You may not use
* this file except in compliance with the License. You can obtain a copy
* in the file LICENSE in the source distribution or at
* https://www.openssl.org/source/license.html
*/
#define PLATFORM "platform: linux-x86_64"
#define DATE "built on: Sat Dec 14 00:28:08 2019 UTC"
/*
* Generate compiler_flags as an array of individual characters. This is a
* workaround for the situation where CFLAGS gets too long for a C90 string
* literal
*/
/// Edited because I don't care.
static const char compiler_flags[] = "";

2
contrib/poco vendored

@ -1 +1 @@
Subproject commit 2b273bfe9db89429b2040c024484dee0197e48c7
Subproject commit d478f62bd93c9cd14eb343756ef73a4ae622ddf5

1
contrib/ssl vendored

@ -1 +0,0 @@
Subproject commit ba8de796195ff9d8bb0249ce289b83226b848b77

2
contrib/zlib-ng vendored

@ -1 +1 @@
Subproject commit cff0f500d9399d7cd3b9461a693d211e4b86fcc9
Subproject commit 5673222fbd37ea89afb2ea73096f9bf5ec68ea31

9
dbms/CMakeLists.txt
View File

@ -111,7 +111,7 @@ list (APPEND clickhouse_common_io_sources ${CONFIG_BUILD})
list (APPEND clickhouse_common_io_headers ${CONFIG_VERSION} ${CONFIG_COMMON})
list (APPEND dbms_sources src/Functions/IFunction.cpp src/Functions/FunctionFactory.cpp src/Functions/FunctionHelpers.cpp)
list (APPEND dbms_headers src/Functions/IFunction.h src/Functions/FunctionFactory.h src/Functions/FunctionHelpers.h)
list (APPEND dbms_headers src/Functions/IFunctionImpl.h src/Functions/FunctionFactory.h src/Functions/FunctionHelpers.h)
list (APPEND dbms_sources
src/AggregateFunctions/AggregateFunctionFactory.cpp
@ -171,6 +171,7 @@ add_object_library(clickhouse_compression src/Compression)
add_object_library(clickhouse_datastreams src/DataStreams)
add_object_library(clickhouse_datatypes src/DataTypes)
add_object_library(clickhouse_databases src/Databases)
add_object_library(clickhouse_disks src/Disks)
add_object_library(clickhouse_interpreters src/Interpreters)
add_object_library(clickhouse_interpreters_clusterproxy src/Interpreters/ClusterProxy)
add_object_library(clickhouse_columns src/Columns)
@ -424,6 +425,12 @@ if (USE_HDFS)
target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${HDFS3_INCLUDE_DIR})
endif()
if (USE_AWS_S3)
target_link_libraries (clickhouse_common_io PUBLIC ${AWS_S3_LIBRARY})
target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${AWS_S3_CORE_INCLUDE_DIR})
target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${AWS_S3_INCLUDE_DIR})
endif()
if (USE_BROTLI)
target_link_libraries (clickhouse_common_io PRIVATE ${BROTLI_LIBRARY})
target_include_directories (clickhouse_common_io SYSTEM BEFORE PRIVATE ${BROTLI_INCLUDE_DIR})

67
dbms/programs/copier/ClusterCopier.cpp
View File

@ -65,6 +65,7 @@
#include <Storages/registerStorages.h>
#include <Storages/StorageDistributed.h>
#include <Dictionaries/registerDictionaries.h>
#include <Disks/registerDisks.h>
#include <Databases/DatabaseMemory.h>
#include <Common/StatusFile.h>
@ -1281,10 +1282,66 @@ protected:
return res;
}
/** Allows to compare two incremental counters of type UInt32 in presence of possible overflow.
* We assume that we compare values that are not too far away.
* For example, when we increment 0xFFFFFFFF, we get 0. So, 0xFFFFFFFF is less than 0.
*/
class WrappingUInt32
{
public:
UInt32 value;
WrappingUInt32(UInt32 _value)
: value(_value)
{}
bool operator<(const WrappingUInt32 & other) const
{
return value != other.value && *this <= other;
}
bool operator<=(const WrappingUInt32 & other) const
{
const UInt32 HALF = 1 << 31;
return (value <= other.value && other.value - value < HALF)
|| (value > other.value && value - other.value > HALF);
}
bool operator==(const WrappingUInt32 & other) const
{
return value == other.value;
}
};
/** Conforming Zxid definition.
* cf. https://github.com/apache/zookeeper/blob/631d1b284f0edb1c4f6b0fb221bf2428aec71aaa/zookeeper-docs/src/main/resources/markdown/zookeeperInternals.md#guarantees-properties-and-definitions
*/
class Zxid
{
public:
WrappingUInt32 epoch;
WrappingUInt32 counter;
Zxid(UInt64 _zxid)
: epoch(_zxid >> 32)
, counter(_zxid)
{}
bool operator<=(const Zxid & other) const
{
return (epoch < other.epoch)
|| (epoch == other.epoch && counter <= other.counter);
}
bool operator==(const Zxid & other) const
{
return epoch == other.epoch && counter == other.counter;
}
};
class LogicalClock
{
public:
std::optional<UInt64> zxid;
std::optional<Zxid> zxid;
LogicalClock() = default;
@ -1300,11 +1357,8 @@ protected:
// happens-before relation with a reasonable time bound
bool happensBefore(const LogicalClock & other) const
{
const UInt64 HALF = 1ull << 63;
return
!zxid ||
(other.zxid && *zxid <= *other.zxid && *other.zxid - *zxid < HALF) ||
(other.zxid && *zxid >= *other.zxid && *zxid - *other.zxid > HALF);
return !zxid
|| (other.zxid && *zxid <= *other.zxid);
}
bool operator<=(const LogicalClock & other) const
@ -2410,6 +2464,7 @@ void ClusterCopierApp::mainImpl()
registerTableFunctions();
registerStorages();
registerDictionaries();
registerDisks();
static const std::string default_database = "_local";
context->addDatabase(default_database, std::make_shared<DatabaseMemory>(default_database));

4
dbms/programs/local/LocalServer.cpp
View File

@ -32,6 +32,7 @@
#include <TableFunctions/registerTableFunctions.h>
#include <Storages/registerStorages.h>
#include <Dictionaries/registerDictionaries.h>
#include <Disks/registerDisks.h>
#include <boost/program_options/options_description.hpp>
#include <boost/program_options.hpp>
#include <common/argsToConfig.h>
@ -152,6 +153,7 @@ try
registerTableFunctions();
registerStorages();
registerDictionaries();
registerDisks();
/// Maybe useless
if (config().has("macros"))
@ -441,7 +443,7 @@ void LocalServer::init(int argc, char ** argv)
exit(0);
}
if (options.count("help"))
if (options.empty() || options.count("help"))
{
std::cout << getHelpHeader() << "\n";
std::cout << description << "\n";

12
dbms/programs/main.cpp
View File

@ -14,10 +14,6 @@
#include "config_core.h"
#endif
#if USE_TCMALLOC
#include <gperftools/malloc_extension.h>
#endif
#include <Common/StringUtils/StringUtils.h>
#include <common/phdr_cache.h>
@ -153,14 +149,6 @@ int main(int argc_, char ** argv_)
/// will work only after additional call of this function.
updatePHDRCache();
#if USE_TCMALLOC
/** Without this option, tcmalloc returns memory to OS too frequently for medium-sized memory allocations
* (like IO buffers, column vectors, hash tables, etc.),
* that lead to page faults and significantly hurts performance.
*/
MallocExtension::instance()->SetNumericProperty("tcmalloc.aggressive_memory_decommit", false);
#endif
std::vector<char *> argv(argv_, argv_ + argc_);
/// Print a basic help if nothing was matched

3
dbms/programs/server/CMakeLists.txt
View File

@ -1,9 +1,12 @@
set(CLICKHOUSE_SERVER_SOURCES
${CMAKE_CURRENT_SOURCE_DIR}/HTTPHandler.cpp
${CMAKE_CURRENT_SOURCE_DIR}/HTTPHandlerFactory.cpp
${CMAKE_CURRENT_SOURCE_DIR}/InterserverIOHTTPHandler.cpp
${CMAKE_CURRENT_SOURCE_DIR}/MetricsTransmitter.cpp
${CMAKE_CURRENT_SOURCE_DIR}/NotFoundHandler.cpp
${CMAKE_CURRENT_SOURCE_DIR}/PingRequestHandler.cpp
${CMAKE_CURRENT_SOURCE_DIR}/PrometheusMetricsWriter.cpp
${CMAKE_CURRENT_SOURCE_DIR}/PrometheusRequestHandler.cpp
${CMAKE_CURRENT_SOURCE_DIR}/ReplicasStatusHandler.cpp
${CMAKE_CURRENT_SOURCE_DIR}/RootRequestHandler.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Server.cpp

43
dbms/programs/server/HTTPHandlerFactory.cpp Normal file
View File

@ -0,0 +1,43 @@
#include "HTTPHandlerFactory.h"
namespace DB
{
HTTPRequestHandlerFactoryMain::HTTPRequestHandlerFactoryMain(IServer & server_, const std::string & name_)
: server(server_), log(&Logger::get(name_)), name(name_)
{
}
Poco::Net::HTTPRequestHandler * HTTPRequestHandlerFactoryMain::createRequestHandler(
const Poco::Net::HTTPServerRequest & request) // override
{
LOG_TRACE(log, "HTTP Request for " << name << ". "
<< "Method: "
<< request.getMethod()
<< ", Address: "
<< request.clientAddress().toString()
<< ", User-Agent: "
<< (request.has("User-Agent") ? request.get("User-Agent") : "none")
<< (request.hasContentLength() ? (", Length: " + std::to_string(request.getContentLength())) : (""))
<< ", Content Type: " << request.getContentType()
<< ", Transfer Encoding: " << request.getTransferEncoding());
for (auto & handlerFactory: child_handler_factories)
{
auto handler = handlerFactory->createRequestHandler(request);
if (handler != nullptr)
return handler;
}
if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET
|| request.getMethod() == Poco::Net::HTTPRequest::HTTP_HEAD
|| request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST)
{
return new NotFoundHandler;
}
return nullptr;
}
}

120
dbms/programs/server/HTTPHandlerFactory.h
View File

@ -9,6 +9,7 @@
#include "InterserverIOHTTPHandler.h"
#include "NotFoundHandler.h"
#include "PingRequestHandler.h"
#include "PrometheusRequestHandler.h"
#include "ReplicasStatusHandler.h"
#include "RootRequestHandler.h"
@ -16,60 +17,111 @@
namespace DB
{
template <typename HandlerType>
class HTTPRequestHandlerFactory : public Poco::Net::HTTPRequestHandlerFactory
/// Handle request using child handlers
class HTTPRequestHandlerFactoryMain : public Poco::Net::HTTPRequestHandlerFactory
{
private:
using TThis = HTTPRequestHandlerFactoryMain;
IServer & server;
Logger * log;
std::string name;
std::vector<std::unique_ptr<Poco::Net::HTTPRequestHandlerFactory>> child_handler_factories;
public:
HTTPRequestHandlerFactory(IServer & server_, const std::string & name_) : server(server_), log(&Logger::get(name_)), name(name_)
HTTPRequestHandlerFactoryMain(IServer & server_, const std::string & name_);
Poco::Net::HTTPRequestHandler * createRequestHandler(const Poco::Net::HTTPServerRequest & request) override;
template <typename T, typename... TArgs>
TThis * addHandler(TArgs &&... args)
{
child_handler_factories.emplace_back(std::make_unique<T>(server, std::forward<TArgs>(args)...));
return this;
}
};
/// Handle POST or GET with params
template <typename HandleType>
class HTTPQueryRequestHandlerFactory : public Poco::Net::HTTPRequestHandlerFactory
{
private:
IServer & server;
public:
HTTPQueryRequestHandlerFactory(IServer & server_) : server(server_) {}
Poco::Net::HTTPRequestHandler * createRequestHandler(const Poco::Net::HTTPServerRequest & request) override
{
LOG_TRACE(log, "HTTP Request for " << name << ". "
<< "Method: "
<< request.getMethod()
<< ", Address: "
<< request.clientAddress().toString()
<< ", User-Agent: "
<< (request.has("User-Agent") ? request.get("User-Agent") : "none")
<< (request.hasContentLength() ? (", Length: " + std::to_string(request.getContentLength())) : (""))
<< ", Content Type: " << request.getContentType()
<< ", Transfer Encoding: " << request.getTransferEncoding());
if (request.getURI().find('?') != std::string::npos || request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST)
return new HandleType(server);
return nullptr;
}
};
const auto & uri = request.getURI();
if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET || request.getMethod() == Poco::Net::HTTPRequest::HTTP_HEAD)
{
if (uri == "/")
return new RootRequestHandler(server);
if (uri == "/ping")
return new PingRequestHandler(server);
else if (startsWith(uri, "/replicas_status"))
return new ReplicasStatusHandler(server.context());
}
/// Handle GET or HEAD endpoint on specified path
template <typename TGetEndpoint>
class HTTPGetRequestHandlerFactory : public Poco::Net::HTTPRequestHandlerFactory
{
private:
IServer & server;
public:
HTTPGetRequestHandlerFactory(IServer & server_) : server(server_) {}
if (uri.find('?') != std::string::npos || request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST)
{
return new HandlerType(server);
}
Poco::Net::HTTPRequestHandler * createRequestHandler(const Poco::Net::HTTPServerRequest & request) override
{
auto & method = request.getMethod();
if (!(method == Poco::Net::HTTPRequest::HTTP_GET || method == Poco::Net::HTTPRequest::HTTP_HEAD))
return nullptr;
if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET || request.getMethod() == Poco::Net::HTTPRequest::HTTP_HEAD
|| request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST)
{
return new NotFoundHandler;
}
auto & uri = request.getURI();
bool uri_match = TGetEndpoint::strict_path ? uri == TGetEndpoint::path : startsWith(uri, TGetEndpoint::path);
if (uri_match)
return new typename TGetEndpoint::HandleType(server);
return nullptr;
}
};
using HTTPHandlerFactory = HTTPRequestHandlerFactory<HTTPHandler>;
using InterserverIOHTTPHandlerFactory = HTTPRequestHandlerFactory<InterserverIOHTTPHandler>;
struct RootEndpoint
{
static constexpr auto path = "/";
static constexpr auto strict_path = true;
using HandleType = RootRequestHandler;
};
struct PingEndpoint
{
static constexpr auto path = "/ping";
static constexpr auto strict_path = true;
using HandleType = PingRequestHandler;
};
struct ReplicasStatusEndpoint
{
static constexpr auto path = "/replicas_status";
static constexpr auto strict_path = false;
using HandleType = ReplicasStatusHandler;
};
using HTTPRootRequestHandlerFactory = HTTPGetRequestHandlerFactory<RootEndpoint>;
using HTTPPingRequestHandlerFactory = HTTPGetRequestHandlerFactory<PingEndpoint>;
using HTTPReplicasStatusRequestHandlerFactory = HTTPGetRequestHandlerFactory<ReplicasStatusEndpoint>;
template <typename HandleType>
HTTPRequestHandlerFactoryMain * createDefaultHandlerFatory(IServer & server, const std::string & name)
{
auto handlerFactory = new HTTPRequestHandlerFactoryMain(server, name);
handlerFactory->addHandler<HTTPRootRequestHandlerFactory>()
->addHandler<HTTPPingRequestHandlerFactory>()
->addHandler<HTTPReplicasStatusRequestHandlerFactory>()
->addHandler<HTTPQueryRequestHandlerFactory<HandleType>>();
return handlerFactory;
}
}

3
dbms/programs/server/MySQLHandler.cpp
View File

@ -221,7 +221,8 @@ void MySQLHandler::authenticate(const String & user_name, const String & auth_pl
{
// For compatibility with JavaScript MySQL client, Native41 authentication plugin is used when possible (if password is specified using double SHA1). Otherwise SHA256 plugin is used.
auto user = connection_context.getUser(user_name);
if (user->authentication.getType() != DB::Authentication::DOUBLE_SHA1_PASSWORD)
const DB::Authentication::Type user_auth_type = user->authentication.getType();
if (user_auth_type != DB::Authentication::DOUBLE_SHA1_PASSWORD && user_auth_type != DB::Authentication::PLAINTEXT_PASSWORD && user_auth_type != DB::Authentication::NO_PASSWORD)
{
authPluginSSL();
}

90
dbms/programs/server/PrometheusMetricsWriter.cpp Normal file
View File

@ -0,0 +1,90 @@
#include "PrometheusMetricsWriter.h"
#include <IO/WriteHelpers.h>
namespace
{
template <typename T>
void writeOutLine(DB::WriteBuffer & wb, T && val)
{
DB::writeText(std::forward<T>(val), wb);
DB::writeChar('\n', wb);
}
template <typename T, typename... TArgs>
void writeOutLine(DB::WriteBuffer & wb, T && val, TArgs &&... args)
{
DB::writeText(std::forward<T>(val), wb);
DB::writeChar(' ', wb);
writeOutLine(wb, std::forward<TArgs>(args)...);
}
}
namespace DB
{
PrometheusMetricsWriter::PrometheusMetricsWriter(
const Poco::Util::AbstractConfiguration & config, const std::string & config_name,
const AsynchronousMetrics & async_metrics_)
: async_metrics(async_metrics_)
, send_events(config.getBool(config_name + ".events", true))
, send_metrics(config.getBool(config_name + ".metrics", true))
, send_asynchronous_metrics(config.getBool(config_name + ".asynchronous_metrics", true))
{
}
void PrometheusMetricsWriter::write(WriteBuffer & wb) const
{
if (send_events)
{
for (size_t i = 0, end = ProfileEvents::end(); i < end; ++i)
{
const auto counter = ProfileEvents::global_counters[i].load(std::memory_order_relaxed);
std::string metric_name{ProfileEvents::getName(static_cast<ProfileEvents::Event>(i))};
std::string metric_doc{ProfileEvents::getDocumentation(static_cast<ProfileEvents::Event>(i))};
std::string key{profile_events_prefix + metric_name};
writeOutLine(wb, "# HELP", key, metric_doc);
writeOutLine(wb, "# TYPE", key, "counter");
writeOutLine(wb, key, counter);
}
}
if (send_metrics)
{
for (size_t i = 0, end = CurrentMetrics::end(); i < end; ++i)
{
const auto value = CurrentMetrics::values[i].load(std::memory_order_relaxed);
std::string metric_name{CurrentMetrics::getName(static_cast<CurrentMetrics::Metric>(i))};
std::string metric_doc{CurrentMetrics::getDocumentation(static_cast<CurrentMetrics::Metric>(i))};
std::string key{current_metrics_prefix + metric_name};
writeOutLine(wb, "# HELP", key, metric_doc);
writeOutLine(wb, "# TYPE", key, "gauge");
writeOutLine(wb, key, value);
}
}
if (send_asynchronous_metrics)
{
auto async_metrics_values = async_metrics.getValues();
for (const auto & name_value : async_metrics_values)
{
std::string key{asynchronous_metrics_prefix + name_value.first};
auto value = name_value.second;
// TODO: add HELP section? asynchronous_metrics contains only key and value
writeOutLine(wb, "# TYPE", key, "gauge");
writeOutLine(wb, key, value);
}
}
}
}

36
dbms/programs/server/PrometheusMetricsWriter.h Normal file
View File

@ -0,0 +1,36 @@
#pragma once
#include <string>
#include <Interpreters/AsynchronousMetrics.h>
#include <IO/WriteBuffer.h>
#include <Poco/Util/AbstractConfiguration.h>
namespace DB
{
/// Write metrics in Prometheus format
class PrometheusMetricsWriter
{
public:
PrometheusMetricsWriter(
const Poco::Util::AbstractConfiguration & config, const std::string & config_name,
const AsynchronousMetrics & async_metrics_);
void write(WriteBuffer & wb) const;
private:
const AsynchronousMetrics & async_metrics;
const bool send_events;
const bool send_metrics;
const bool send_asynchronous_metrics;
static inline constexpr auto profile_events_prefix = "ClickHouseProfileEvents";
static inline constexpr auto current_metrics_prefix = "ClickHouseMetrics";
static inline constexpr auto asynchronous_metrics_prefix = "ClickHouseAsyncMetrics";
};
}

42
dbms/programs/server/PrometheusRequestHandler.cpp Normal file
View File

@ -0,0 +1,42 @@
#include "PrometheusRequestHandler.h"
#include <IO/HTTPCommon.h>
#include <Common/Exception.h>
#include <Poco/Net/HTTPServerRequest.h>
#include <Poco/Net/HTTPServerResponse.h>
#include <Common/ProfileEvents.h>
#include <Common/CurrentMetrics.h>
#include <IO/WriteBufferFromHTTPServerResponse.h>
namespace DB
{
void PrometheusRequestHandler::handleRequest(
Poco::Net::HTTPServerRequest & request,
Poco::Net::HTTPServerResponse & response)
{
try
{
const auto & config = server.config();
unsigned keep_alive_timeout = config.getUInt("keep_alive_timeout", 10);
setResponseDefaultHeaders(response, keep_alive_timeout);
response.setContentType("text/plain; version=0.0.4; charset=UTF-8");
auto wb = WriteBufferFromHTTPServerResponse(request, response, keep_alive_timeout);
metrics_writer.write(wb);
wb.finalize();
}
catch (...)
{
tryLogCurrentException("PrometheusRequestHandler");
}
}
}

61
dbms/programs/server/PrometheusRequestHandler.h Normal file
View File

@ -0,0 +1,61 @@
#pragma once
#include "IServer.h"
#include "PrometheusMetricsWriter.h"
#include <Poco/Net/HTTPServerRequest.h>
#include <Poco/Net/HTTPServerResponse.h>
#include <Poco/Net/HTTPRequestHandler.h>
#include <Poco/Net/HTTPRequestHandlerFactory.h>
namespace DB
{
class PrometheusRequestHandler : public Poco::Net::HTTPRequestHandler
{
private:
IServer & server;
const PrometheusMetricsWriter & metrics_writer;
public:
explicit PrometheusRequestHandler(IServer & server_, PrometheusMetricsWriter & metrics_writer_)
: server(server_)
, metrics_writer(metrics_writer_)
{
}
void handleRequest(
Poco::Net::HTTPServerRequest & request,
Poco::Net::HTTPServerResponse & response) override;
};
template <typename HandlerType>
class PrometeusRequestHandlerFactory : public Poco::Net::HTTPRequestHandlerFactory
{
private:
IServer & server;
std::string endpoint_path;
PrometheusMetricsWriter metrics_writer;
public:
PrometeusRequestHandlerFactory(IServer & server_, const AsynchronousMetrics & async_metrics_)
: server(server_)
, endpoint_path(server_.config().getString("prometheus.endpoint", "/metrics"))
, metrics_writer(server_.config(), "prometheus", async_metrics_)
{
}
Poco::Net::HTTPRequestHandler * createRequestHandler(const Poco::Net::HTTPServerRequest & request) override
{
if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET
&& request.getURI() == endpoint_path)
return new HandlerType(server, metrics_writer);
return nullptr;
}
};
using PrometeusHandlerFactory = PrometeusRequestHandlerFactory<PrometheusRequestHandler>;
}

4
dbms/programs/server/ReplicasStatusHandler.cpp
View File

@ -15,8 +15,8 @@ namespace DB
{
ReplicasStatusHandler::ReplicasStatusHandler(Context & context_)
: context(context_)
ReplicasStatusHandler::ReplicasStatusHandler(IServer & server)
: context(server.context())
{
}

4
dbms/programs/server/ReplicasStatusHandler.h
View File

@ -1,5 +1,7 @@
#pragma once
#include "IServer.h"
#include <Poco/Net/HTTPRequestHandler.h>
@ -15,7 +17,7 @@ private:
Context & context;
public:
explicit ReplicasStatusHandler(Context & context_);
explicit ReplicasStatusHandler(IServer & server);
void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override;
};

48
dbms/programs/server/Server.cpp
View File

@ -50,6 +50,7 @@
#include <TableFunctions/registerTableFunctions.h>
#include <Storages/registerStorages.h>
#include <Dictionaries/registerDictionaries.h>
#include <Disks/registerDisks.h>
#include <Common/Config/ConfigReloader.h>
#include "HTTPHandlerFactory.h"
#include "MetricsTransmitter.h"
@ -187,6 +188,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
registerTableFunctions();
registerStorages();
registerDictionaries();
registerDisks();
CurrentMetrics::set(CurrentMetrics::Revision, ClickHouseRevision::get());
CurrentMetrics::set(CurrentMetrics::VersionInteger, ClickHouseRevision::getVersionInteger());
@ -670,6 +672,10 @@ int Server::main(const std::vector<std::string> & /*args*/)
return address;
};
/// This object will periodically calculate some metrics.
AsynchronousMetrics async_metrics(*global_context);
attachSystemTablesAsync(*global_context->getDatabase("system"), async_metrics);
for (const auto & listen_host : listen_hosts)
{
auto create_server = [&](const char * port_name, auto && func)
@ -709,13 +715,17 @@ int Server::main(const std::vector<std::string> & /*args*/)
auto address = socket_bind_listen(socket, listen_host, port);
socket.setReceiveTimeout(settings.http_receive_timeout);
socket.setSendTimeout(settings.http_send_timeout);
auto handler_factory = createDefaultHandlerFatory<HTTPHandler>(*this, "HTTPHandler-factory");
if (config().has("prometheus") && config().getInt("prometheus.port", 0) == 0)
handler_factory->addHandler<PrometeusHandlerFactory>(async_metrics);
servers.emplace_back(std::make_unique<Poco::Net::HTTPServer>(
new HTTPHandlerFactory(*this, "HTTPHandler-factory"),
handler_factory,
server_pool,
socket,
http_params));
LOG_INFO(log, "Listening http://" + address.toString());
LOG_INFO(log, "Listening for http://" + address.toString());
});
/// HTTPS
@ -727,12 +737,12 @@ int Server::main(const std::vector<std::string> & /*args*/)
socket.setReceiveTimeout(settings.http_receive_timeout);
socket.setSendTimeout(settings.http_send_timeout);
servers.emplace_back(std::make_unique<Poco::Net::HTTPServer>(
new HTTPHandlerFactory(*this, "HTTPSHandler-factory"),
createDefaultHandlerFatory<HTTPHandler>(*this, "HTTPSHandler-factory"),
server_pool,
socket,
http_params));
LOG_INFO(log, "Listening https://" + address.toString());
LOG_INFO(log, "Listening for https://" + address.toString());
#else
UNUSED(port);
throw Exception{"HTTPS protocol is disabled because Poco library was built without NetSSL support.",
@ -785,12 +795,12 @@ int Server::main(const std::vector<std::string> & /*args*/)
socket.setReceiveTimeout(settings.http_receive_timeout);
socket.setSendTimeout(settings.http_send_timeout);
servers.emplace_back(std::make_unique<Poco::Net::HTTPServer>(
new InterserverIOHTTPHandlerFactory(*this, "InterserverIOHTTPHandler-factory"),
createDefaultHandlerFatory<InterserverIOHTTPHandler>(*this, "InterserverIOHTTPHandler-factory"),
server_pool,
socket,
http_params));
LOG_INFO(log, "Listening for replica communication (interserver) http://" + address.toString());
LOG_INFO(log, "Listening for replica communication (interserver): http://" + address.toString());
});
create_server("interserver_https_port", [&](UInt16 port)
@ -801,12 +811,12 @@ int Server::main(const std::vector<std::string> & /*args*/)
socket.setReceiveTimeout(settings.http_receive_timeout);
socket.setSendTimeout(settings.http_send_timeout);
servers.emplace_back(std::make_unique<Poco::Net::HTTPServer>(
new InterserverIOHTTPHandlerFactory(*this, "InterserverIOHTTPHandler-factory"),
createDefaultHandlerFatory<InterserverIOHTTPHandler>(*this, "InterserverIOHTTPHandler-factory"),
server_pool,
socket,
http_params));
LOG_INFO(log, "Listening for secure replica communication (interserver) https://" + address.toString());
LOG_INFO(log, "Listening for secure replica communication (interserver): https://" + address.toString());
#else
UNUSED(port);
throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support.",
@ -828,6 +838,24 @@ int Server::main(const std::vector<std::string> & /*args*/)
LOG_INFO(log, "Listening for MySQL compatibility protocol: " + address.toString());
});
/// Prometheus (if defined and not setup yet with http_port)
create_server("prometheus.port", [&](UInt16 port)
{
Poco::Net::ServerSocket socket;
auto address = socket_bind_listen(socket, listen_host, port);
socket.setReceiveTimeout(settings.http_receive_timeout);
socket.setSendTimeout(settings.http_send_timeout);
auto handler_factory = new HTTPRequestHandlerFactoryMain(*this, "PrometheusHandler-factory");
handler_factory->addHandler<PrometeusHandlerFactory>(async_metrics);
servers.emplace_back(std::make_unique<Poco::Net::HTTPServer>(
handler_factory,
server_pool,
socket,
http_params));
LOG_INFO(log, "Listening for Prometheus: http://" + address.toString());
});
}
if (servers.empty())
@ -930,10 +958,6 @@ int Server::main(const std::vector<std::string> & /*args*/)
throw;
}
/// This object will periodically calculate some metrics.
AsynchronousMetrics async_metrics(*global_context);
attachSystemTablesAsync(*global_context->getDatabase("system"), async_metrics);
std::vector<std::unique_ptr<MetricsTransmitter>> metrics_transmitters;
for (const auto & graphite_key : DB::getMultipleKeysFromConfig(config(), "", "graphite"))
{

112
dbms/programs/server/TCPHandler.cpp
View File

@ -484,80 +484,63 @@ void TCPHandler::processOrdinaryQuery()
/// Pull query execution result, if exists, and send it to network.
if (state.io.in)
{
/// Send header-block, to allow client to prepare output format for data to send.
{
Block header = state.io.in->getHeader();
if (header)
sendData(header);
}
/// This allows the client to prepare output format
if (Block header = state.io.in->getHeader())
sendData(header);
/// Use of async mode here enables reporting progress and monitoring client cancelling the query
AsynchronousBlockInputStream async_in(state.io.in);
async_in.readPrefix();
async_in.readPrefix();
while (true)
{
Block block;
while (true)
if (isQueryCancelled())
{
if (isQueryCancelled())
{
/// A packet was received requesting to stop execution of the request.
async_in.cancel(false);
break;
}
else
{
if (after_send_progress.elapsed() / 1000 >= query_context->getSettingsRef().interactive_delay)
{
/// Some time passed and there is a progress.
after_send_progress.restart();
sendProgress();
}
sendLogs();
if (async_in.poll(query_context->getSettingsRef().interactive_delay / 1000))
{
/// There is the following result block.
block = async_in.read();
break;
}
}
}
/** If data has run out, we will send the profiling data and total values to
* the last zero block to be able to use
* this information in the suffix output of stream.
* If the request was interrupted, then `sendTotals` and other methods could not be called,
* because we have not read all the data yet,
* and there could be ongoing calculations in other threads at the same time.
*/
if (!block && !isQueryCancelled())
{
/// Wait till inner thread finish to avoid possible race with getTotals.
async_in.waitInnerThread();
sendTotals(state.io.in->getTotals());
sendExtremes(state.io.in->getExtremes());
sendProfileInfo(state.io.in->getProfileInfo());
sendProgress();
sendLogs();
}
if (!block || !state.io.null_format)
sendData(block);
if (!block)
async_in.cancel(false);
break;
}
if (after_send_progress.elapsed() / 1000 >= query_context->getSettingsRef().interactive_delay)
{
/// Some time passed and there is a progress.
after_send_progress.restart();
sendProgress();
}
sendLogs();
if (async_in.poll(query_context->getSettingsRef().interactive_delay / 1000))
{
const auto block = async_in.read();
if (!block)
break;
if (!state.io.null_format)
sendData(block);
}
}
async_in.readSuffix();
/** When the data has run out, we send the profiling data and totals up to the terminating empty block,
* so that this information can be used in the suffix output of stream.
* If the request has been interrupted, then sendTotals and other methods should not be called,
* because we have not read all the data.
*/
if (!isQueryCancelled())
{
sendTotals(state.io.in->getTotals());
sendExtremes(state.io.in->getExtremes());
sendProfileInfo(state.io.in->getProfileInfo());
sendProgress();
}
async_in.readSuffix();
sendData({});
}
state.io.onFinish();
}
void TCPHandler::processOrdinaryQueryWithProcessors(size_t num_threads)
{
auto & pipeline = state.io.pipeline;
@ -936,10 +919,15 @@ void TCPHandler::receiveQuery()
}
/// Per query settings.
Settings & settings = query_context->getSettingsRef();
Settings custom_settings{};
auto settings_format = (client_revision >= DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS) ? SettingsBinaryFormat::STRINGS
: SettingsBinaryFormat::OLD;
settings.deserialize(*in, settings_format);
custom_settings.deserialize(*in, settings_format);
auto settings_changes = custom_settings.changes();
query_context->checkSettingsConstraints(settings_changes);
query_context->applySettingsChanges(settings_changes);
Settings & settings = query_context->getSettingsRef();
/// Sync timeouts on client and server during current query to avoid dangling queries on server
/// NOTE: We use settings.send_timeout for the receive timeout and vice versa (change arguments ordering in TimeoutSetter),

18
dbms/programs/server/config.xml
View File

@ -307,6 +307,24 @@
</graphite>
-->
<!-- Serve endpoint fot Prometheus monitoring. -->
<!--
endpoint - mertics path (relative to root, statring with "/")
port - port to setup server. If not defined or 0 than http_port used
metrics - send data from table system.metrics
events - send data from table system.events
asynchronous_metrics - send data from table system.asynchronous_metrics
-->
<!--
<prometheus>
<endpoint>/metrics</endpoint>
<port>8001</port>
<metrics>true</metrics>
<events>true</events>
<asynchronous_metrics>true</asynchronous_metrics>
</prometheus>
-->
<!-- Query log. Used only for queries with setting log_queries = 1. -->
<query_log>

38
dbms/src/Access/Authentication.cpp
View File

@ -160,6 +160,35 @@ void Authentication::setPasswordHashBinary(const Digest & hash)
}
Digest Authentication::getPasswordDoubleSHA1() const
{
switch (type)
{
case NO_PASSWORD:
{
Poco::SHA1Engine engine;
return engine.digest();
}
case PLAINTEXT_PASSWORD:
{
Poco::SHA1Engine engine;
engine.update(getPassword());
const Digest & first_sha1 = engine.digest();
engine.update(first_sha1.data(), first_sha1.size());
return engine.digest();
}
case SHA256_PASSWORD:
throw Exception("Cannot get password double SHA1 for user with 'SHA256_PASSWORD' authentication.", ErrorCodes::BAD_ARGUMENTS);
case DOUBLE_SHA1_PASSWORD:
return password_hash;
}
throw Exception("Unknown authentication type: " + std::to_string(static_cast<int>(type)), ErrorCodes::LOGICAL_ERROR);
}
bool Authentication::isCorrectPassword(const String & password_) const
{
switch (type)
@ -168,7 +197,14 @@ bool Authentication::isCorrectPassword(const String & password_) const
return true;
case PLAINTEXT_PASSWORD:
return password_ == StringRef{reinterpret_cast<const char *>(password_hash.data()), password_hash.size()};
{
if (password_ == StringRef{reinterpret_cast<const char *>(password_hash.data()), password_hash.size()})
return true;
// For compatibility with MySQL clients which support only native authentication plugin, SHA1 can be passed instead of password.
auto password_sha1 = encodeSHA1(password_hash);
return password_ == StringRef{reinterpret_cast<const char *>(password_sha1.data()), password_sha1.size()};
}
case SHA256_PASSWORD:
return encodeSHA256(password_) == password_hash;

4
dbms/src/Access/Authentication.h
View File

@ -49,6 +49,10 @@ public:
void setPasswordHashBinary(const Digest & hash);
const Digest & getPasswordHashBinary() const { return password_hash; }
/// Returns SHA1(SHA1(password)) used by MySQL compatibility server for authentication.
/// Allowed to use for Type::NO_PASSWORD, Type::PLAINTEXT_PASSWORD, Type::DOUBLE_SHA1_PASSWORD.
Digest getPasswordDoubleSHA1() const;
/// Checks if the provided password is correct. Returns false if not.
bool isCorrectPassword(const String & password) const;

53
dbms/src/AggregateFunctions/AggregateFunctionCategoricalInformationValue.cpp Normal file
View File

@ -0,0 +1,53 @@
#include <AggregateFunctions/AggregateFunctionCategoricalInformationValue.h>
#include <AggregateFunctions/AggregateFunctionFactory.h>
#include <AggregateFunctions/FactoryHelpers.h>
#include <AggregateFunctions/Helpers.h>
namespace DB
{
namespace ErrorCodes
{
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
}
namespace
{
AggregateFunctionPtr createAggregateFunctionCategoricalIV(
const std::string & name,
const DataTypes & arguments,
const Array & params
)
{
assertNoParameters(name, params);
if (arguments.size() < 2)
throw Exception(
"Aggregate function " + name + " requires two or more arguments",
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
for (auto & argument : arguments)
{
if (!WhichDataType(argument).isUInt8())
throw Exception(
"All the arguments of aggregate function " + name + " should be UInt8",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
}
return std::make_shared<AggregateFunctionCategoricalIV<>>(arguments, params);
}
}
void registerAggregateFunctionCategoricalIV(
AggregateFunctionFactory & factory
)
{
factory.registerFunction("categoricalInformationValue", createAggregateFunctionCategoricalIV);
}
}

156
dbms/src/AggregateFunctions/AggregateFunctionCategoricalInformationValue.h Normal file
View File

@ -0,0 +1,156 @@
#pragma once
#include <AggregateFunctions/IAggregateFunction.h>
#include <Columns/ColumnArray.h>
#include <Columns/ColumnsNumber.h>
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypesNumber.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>
#include <ext/range.h>
namespace DB
{
template <typename T = UInt64>
class AggregateFunctionCategoricalIV final : public IAggregateFunctionHelper<AggregateFunctionCategoricalIV<T>>
{
private:
size_t category_count;
public:
AggregateFunctionCategoricalIV(const DataTypes & arguments_, const Array & params_) :
IAggregateFunctionHelper<AggregateFunctionCategoricalIV<T>> {arguments_, params_},
category_count {arguments_.size() - 1}
{
// notice: argument types has been checked before
}
String getName() const override
{
return "categoricalInformationValue";
}
const char * getHeaderFilePath() const override
{
return __FILE__;
}
void create(AggregateDataPtr place) const override
{
memset(place, 0, sizeOfData());
}
void destroy(AggregateDataPtr) const noexcept override
{
// nothing
}
bool hasTrivialDestructor() const override
{
return true;
}
size_t sizeOfData() const override
{
return sizeof(T) * (category_count + 1) * 2;
}
size_t alignOfData() const override
{
return alignof(T);
}
void add(
AggregateDataPtr place,
const IColumn ** columns,
size_t row_num,
Arena *
) const override
{
auto y_col = static_cast<const ColumnUInt8 *>(columns[category_count]);
bool y = y_col->getData()[row_num];
for (size_t i : ext::range(0, category_count))
{
auto x_col = static_cast<const ColumnUInt8 *>(columns[i]);
bool x = x_col->getData()[row_num];
if (x)
reinterpret_cast<T *>(place)[i * 2 + size_t(y)] += 1;
}
reinterpret_cast<T *>(place)[category_count * 2 + size_t(y)] += 1;
}
void merge(
AggregateDataPtr place,
ConstAggregateDataPtr rhs,
Arena *
) const override
{
for (size_t i : ext::range(0, category_count + 1))
{
reinterpret_cast<T *>(place)[i * 2] += reinterpret_cast<const T *>(rhs)[i * 2];
reinterpret_cast<T *>(place)[i * 2 + 1] += reinterpret_cast<const T *>(rhs)[i * 2 + 1];
}
}
void serialize(
ConstAggregateDataPtr place,
WriteBuffer & buf
) const override
{
buf.write(place, sizeOfData());
}
void deserialize(
AggregateDataPtr place,
ReadBuffer & buf,
Arena *
) const override
{
buf.read(place, sizeOfData());
}
DataTypePtr getReturnType() const override
{
return std::make_shared<DataTypeArray>(
std::make_shared<DataTypeNumber<Float64>>()
);
}
void insertResultInto(
ConstAggregateDataPtr place,
IColumn & to
) const override
{
auto & col = static_cast<ColumnArray &>(to);
auto & data_col = static_cast<ColumnFloat64 &>(col.getData());
auto & offset_col = static_cast<ColumnArray::ColumnOffsets &>(
col.getOffsetsColumn()
);
data_col.reserve(data_col.size() + category_count);
T sum_no = reinterpret_cast<const T *>(place)[category_count * 2];
T sum_yes = reinterpret_cast<const T *>(place)[category_count * 2 + 1];
Float64 rev_no = 1. / sum_no;
Float64 rev_yes = 1. / sum_yes;
for (size_t i : ext::range(0, category_count))
{
T no = reinterpret_cast<const T *>(place)[i * 2];
T yes = reinterpret_cast<const T *>(place)[i * 2 + 1];
data_col.insertValue((no * rev_no - yes * rev_yes) * (log(no * rev_no) - log(yes * rev_yes)));
}
offset_col.insertValue(data_col.size());
}
};
}

14
dbms/src/AggregateFunctions/AggregateFunctionMinMaxAny.h
View File

@ -4,6 +4,7 @@
#include <IO/ReadHelpers.h>
#include <Columns/ColumnVector.h>
#include <Columns/ColumnDecimal.h>
#include <Columns/ColumnString.h>
#include <DataTypes/IDataType.h>
#include <common/StringRef.h>
@ -26,6 +27,7 @@ struct SingleValueDataFixed
{
private:
using Self = SingleValueDataFixed;
using ColVecType = std::conditional_t<IsDecimalNumber<T>, ColumnDecimal<T>, ColumnVector<T>>;
bool has_value = false; /// We need to remember if at least one value has been passed. This is necessary for AggregateFunctionIf.
T value;
@ -39,9 +41,9 @@ public:
void insertResultInto(IColumn & to) const
{
if (has())
assert_cast<ColumnVector<T> &>(to).getData().push_back(value);
assert_cast<ColVecType &>(to).getData().push_back(value);
else
assert_cast<ColumnVector<T> &>(to).insertDefault();
assert_cast<ColVecType &>(to).insertDefault();
}
void write(WriteBuffer & buf, const IDataType & /*data_type*/) const
@ -62,7 +64,7 @@ public:
void change(const IColumn & column, size_t row_num, Arena *)
{
has_value = true;
value = assert_cast<const ColumnVector<T> &>(column).getData()[row_num];
value = assert_cast<const ColVecType &>(column).getData()[row_num];
}
/// Assuming to.has()
@ -113,7 +115,7 @@ public:
bool changeIfLess(const IColumn & column, size_t row_num, Arena * arena)
{
if (!has() || assert_cast<const ColumnVector<T> &>(column).getData()[row_num] < value)
if (!has() || assert_cast<const ColVecType &>(column).getData()[row_num] < value)
{
change(column, row_num, arena);
return true;
@ -135,7 +137,7 @@ public:
bool changeIfGreater(const IColumn & column, size_t row_num, Arena * arena)
{
if (!has() || assert_cast<const ColumnVector<T> &>(column).getData()[row_num] > value)
if (!has() || assert_cast<const ColVecType &>(column).getData()[row_num] > value)
{
change(column, row_num, arena);
return true;
@ -162,7 +164,7 @@ public:
bool isEqualTo(const IColumn & column, size_t row_num) const
{
return has() && assert_cast<const ColumnVector<T> &>(column).getData()[row_num] == value;
return has() && assert_cast<const ColVecType &>(column).getData()[row_num] == value;
}
};

24
dbms/src/AggregateFunctions/HelpersMinMaxAny.h
View File

@ -32,6 +32,14 @@ static IAggregateFunction * createAggregateFunctionSingleValue(const String & na
return new AggregateFunctionTemplate<Data<SingleValueDataFixed<DataTypeDate::FieldType>>, false>(argument_type);
if (which.idx == TypeIndex::DateTime)
return new AggregateFunctionTemplate<Data<SingleValueDataFixed<DataTypeDateTime::FieldType>>, false>(argument_type);
if (which.idx == TypeIndex::DateTime64)
return new AggregateFunctionTemplate<Data<SingleValueDataFixed<DateTime64>>, false>(argument_type);
if (which.idx == TypeIndex::Decimal32)
return new AggregateFunctionTemplate<Data<SingleValueDataFixed<Decimal32>>, false>(argument_type);
if (which.idx == TypeIndex::Decimal64)
return new AggregateFunctionTemplate<Data<SingleValueDataFixed<Decimal64>>, false>(argument_type);
if (which.idx == TypeIndex::Decimal128)
return new AggregateFunctionTemplate<Data<SingleValueDataFixed<Decimal128>>, false>(argument_type);
if (which.idx == TypeIndex::String)
return new AggregateFunctionTemplate<Data<SingleValueDataString>, true>(argument_type);
@ -54,6 +62,14 @@ static IAggregateFunction * createAggregateFunctionArgMinMaxSecond(const DataTyp
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<DataTypeDate::FieldType>>>, false>(res_type, val_type);
if (which.idx == TypeIndex::DateTime)
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<DataTypeDateTime::FieldType>>>, false>(res_type, val_type);
if (which.idx == TypeIndex::DateTime64)
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<DateTime64>>>, false>(res_type, val_type);
if (which.idx == TypeIndex::Decimal32)
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<Decimal32>>>, false>(res_type, val_type);
if (which.idx == TypeIndex::Decimal64)
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<Decimal64>>>, false>(res_type, val_type);
if (which.idx == TypeIndex::Decimal128)
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<Decimal128>>>, false>(res_type, val_type);
if (which.idx == TypeIndex::String)
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataString>>, true>(res_type, val_type);
@ -80,6 +96,14 @@ static IAggregateFunction * createAggregateFunctionArgMinMax(const String & name
return createAggregateFunctionArgMinMaxSecond<MinMaxData, SingleValueDataFixed<DataTypeDate::FieldType>>(res_type, val_type);
if (which.idx == TypeIndex::DateTime)
return createAggregateFunctionArgMinMaxSecond<MinMaxData, SingleValueDataFixed<DataTypeDateTime::FieldType>>(res_type, val_type);
if (which.idx == TypeIndex::DateTime64)
return createAggregateFunctionArgMinMaxSecond<MinMaxData, SingleValueDataFixed<DateTime64>>(res_type, val_type);
if (which.idx == TypeIndex::Decimal32)
return createAggregateFunctionArgMinMaxSecond<MinMaxData, SingleValueDataFixed<Decimal32>>(res_type, val_type);
if (which.idx == TypeIndex::Decimal64)
return createAggregateFunctionArgMinMaxSecond<MinMaxData, SingleValueDataFixed<Decimal64>>(res_type, val_type);
if (which.idx == TypeIndex::Decimal128)
return createAggregateFunctionArgMinMaxSecond<MinMaxData, SingleValueDataFixed<Decimal128>>(res_type, val_type);
if (which.idx == TypeIndex::String)
return createAggregateFunctionArgMinMaxSecond<MinMaxData, SingleValueDataString>(res_type, val_type);

2
dbms/src/AggregateFunctions/registerAggregateFunctions.cpp
View File

@ -35,6 +35,7 @@ void registerAggregateFunctionMLMethod(AggregateFunctionFactory &);
void registerAggregateFunctionEntropy(AggregateFunctionFactory &);
void registerAggregateFunctionSimpleLinearRegression(AggregateFunctionFactory &);
void registerAggregateFunctionMoving(AggregateFunctionFactory &);
void registerAggregateFunctionCategoricalIV(AggregateFunctionFactory &);
void registerAggregateFunctionCombinatorIf(AggregateFunctionCombinatorFactory &);
void registerAggregateFunctionCombinatorArray(AggregateFunctionCombinatorFactory &);
@ -78,6 +79,7 @@ void registerAggregateFunctions()
registerAggregateFunctionEntropy(factory);
registerAggregateFunctionSimpleLinearRegression(factory);
registerAggregateFunctionMoving(factory);
registerAggregateFunctionCategoricalIV(factory);
}
{

1
dbms/src/CMakeLists.txt
View File

@ -5,6 +5,7 @@ add_subdirectory (Core)
add_subdirectory (DataStreams)
add_subdirectory (DataTypes)
add_subdirectory (Dictionaries)
add_subdirectory (Disks)
add_subdirectory (Storages)
add_subdirectory (Parsers)
add_subdirectory (IO)

1
dbms/src/Columns/ColumnDecimal.h
View File

@ -66,6 +66,7 @@ private:
friend class COWHelper<ColumnVectorHelper, Self>;
public:
using ValueType = T;
using Container = DecimalPaddedPODArray<T>;
private:

2
dbms/src/Columns/ColumnVector.cpp
View File

@ -204,7 +204,7 @@ MutableColumnPtr ColumnVector<T>::cloneResized(size_t size) const
memcpy(new_col.data.data(), data.data(), count * sizeof(data[0]));
if (size > count)
memset(static_cast<void *>(&new_col.data[count]), static_cast<int>(value_type()), (size - count) * sizeof(value_type));
memset(static_cast<void *>(&new_col.data[count]), static_cast<int>(ValueType()), (size - count) * sizeof(ValueType));
}
return res;

6
dbms/src/Columns/ColumnVector.h
View File

@ -104,13 +104,13 @@ private:
struct greater;
public:
using value_type = T;
using Container = PaddedPODArray<value_type>;
using ValueType = T;
using Container = PaddedPODArray<ValueType>;
private:
ColumnVector() {}
ColumnVector(const size_t n) : data(n) {}
ColumnVector(const size_t n, const value_type x) : data(n, x) {}
ColumnVector(const size_t n, const ValueType x) : data(n, x) {}
ColumnVector(const ColumnVector & src) : data(src.data.begin(), src.data.end()) {}
/// Sugar constructor.

4
dbms/src/Columns/ReverseIndex.h
View File

@ -116,7 +116,7 @@ namespace
return (*state.saved_hash_column)[index];
else
{
using ValueType = typename ColumnType::value_type;
using ValueType = typename ColumnType::ValueType;
ValueType value = unalignedLoad<ValueType>(state.index_column->getDataAt(index).data);
return DefaultHash<ValueType>()(value);
}
@ -367,7 +367,7 @@ private:
{
if constexpr (is_numeric_column)
{
using ValueType = typename ColumnType::value_type;
using ValueType = typename ColumnType::ValueType;
ValueType value = unalignedLoad<ValueType>(ref.data);
return DefaultHash<ValueType>()(value);
}

2
dbms/src/Common/CurrentMetrics.cpp
View File

@ -10,6 +10,7 @@
M(ReplicatedSend, "Number of data parts being sent to replicas") \
M(ReplicatedChecks, "Number of data parts checking for consistency") \
M(BackgroundPoolTask, "Number of active tasks in BackgroundProcessingPool (merges, mutations, fetches, or replication queue bookkeeping)") \
M(BackgroundMovePoolTask, "Number of active tasks in BackgroundProcessingPool for moves") \
M(BackgroundSchedulePoolTask, "Number of active tasks in BackgroundSchedulePool. This pool is used for periodic ReplicatedMergeTree tasks, like cleaning old data parts, altering data parts, replica re-initialization, etc.") \
M(DiskSpaceReservedForMerge, "Disk space reserved for currently running background merges. It is slightly more than the total size of currently merging parts.") \
M(DistributedSend, "Number of connections to remote servers sending data that was INSERTed into Distributed tables. Both synchronous and asynchronous mode.") \
@ -28,6 +29,7 @@
M(LeaderReplica, "Number of Replicated tables that are leaders. Leader replica is responsible for assigning merges, cleaning old blocks for deduplications and a few more bookkeeping tasks. There may be no more than one leader across all replicas at one moment of time. If there is no leader it will be elected soon or it indicate an issue.") \
M(MemoryTracking, "Total amount of memory (bytes) allocated in currently executing queries. Note that some memory allocations may not be accounted.") \
M(MemoryTrackingInBackgroundProcessingPool, "Total amount of memory (bytes) allocated in background processing pool (that is dedicated for backround merges, mutations and fetches). Note that this value may include a drift when the memory was allocated in a context of background processing pool and freed in other context or vice-versa. This happens naturally due to caches for tables indexes and doesn't indicate memory leaks.") \
M(MemoryTrackingInBackgroundMoveProcessingPool, "Total amount of memory (bytes) allocated in background processing pool (that is dedicated for backround moves). Note that this value may include a drift when the memory was allocated in a context of background processing pool and freed in other context or vice-versa. This happens naturally due to caches for tables indexes and doesn't indicate memory leaks.") \
M(MemoryTrackingInBackgroundSchedulePool, "Total amount of memory (bytes) allocated in background schedule pool (that is dedicated for bookkeeping tasks of Replicated tables).") \
M(MemoryTrackingForMerges, "Total amount of memory (bytes) allocated for background merges. Included in MemoryTrackingInBackgroundProcessingPool. Note that this value may include a drift when the memory was allocated in a context of background processing pool and freed in other context or vice-versa. This happens naturally due to caches for tables indexes and doesn't indicate memory leaks.") \
M(LeaderElection, "Number of Replicas participating in leader election. Equals to total number of replicas in usual cases.") \

360
dbms/src/Common/DiskSpaceMonitor.h
View File

@ -1,360 +0,0 @@
#pragma once
#include <mutex>
#include <sys/statvfs.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#if defined(__linux__)
#include <cstdio>
#include <mntent.h>
#endif
#include <memory>
#include <filesystem>
#include <boost/noncopyable.hpp>
#include <Poco/Util/AbstractConfiguration.h>
#include <common/logger_useful.h>
#include <Common/Exception.h>
#include <IO/WriteHelpers.h>
#include <Common/formatReadable.h>
#include <Common/CurrentMetrics.h>
namespace CurrentMetrics
{
extern const Metric DiskSpaceReservedForMerge;
}
namespace DB
{
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
extern const int CANNOT_STATVFS;
extern const int NOT_ENOUGH_SPACE;
extern const int NOT_IMPLEMENTED;
extern const int SYSTEM_ERROR;
extern const int UNKNOWN_ELEMENT_IN_CONFIG;
extern const int EXCESSIVE_ELEMENT_IN_CONFIG;
extern const int UNKNOWN_POLICY;
extern const int UNKNOWN_DISK;
}
namespace DiskSpace
{
class Reservation;
using ReservationPtr = std::unique_ptr<Reservation>;
/// Returns mount point of filesystem where absoulte_path (must exist) is located
std::filesystem::path getMountPoint(std::filesystem::path absolute_path);
/// Returns name of filesystem mounted to mount_point
#if !defined(__linux__)
[[noreturn]]
#endif
std::string getFilesystemName([[maybe_unused]] const std::string & mount_point);
inline struct statvfs getStatVFS(const std::string & path)
{
struct statvfs fs;
if (statvfs(path.c_str(), &fs) != 0)
throwFromErrnoWithPath(
"Could not calculate available disk space (statvfs)", path, ErrorCodes::CANNOT_STATVFS);
return fs;
}
/**
* Provide interface for reservation
*/
class Space : public std::enable_shared_from_this<Space>
{
public:
virtual ReservationPtr reserve(UInt64 bytes) const = 0;
virtual const String & getName() const = 0;
virtual ~Space() = default;
};
using SpacePtr = std::shared_ptr<const Space>;
/** Disk - Smallest space unit.
* path - Path to space. Ends with /
* name - Unique key using for disk space reservation.
*/
class Disk : public Space
{
public:
friend class Reservation;
/// Snapshot of disk space state (free and total space)
class Stat
{
struct statvfs fs{};
UInt64 keep_free_space_bytes;
public:
explicit Stat(const Disk & disk)
{
if (statvfs(disk.path.c_str(), &fs) != 0)
throwFromErrno("Could not calculate available disk space (statvfs)", ErrorCodes::CANNOT_STATVFS);
keep_free_space_bytes = disk.keep_free_space_bytes;
}
/// Total space on disk using information from statvfs
UInt64 getTotalSpace() const;
/// Available space on disk using information from statvfs
UInt64 getAvailableSpace() const;
};
Disk(const String & name_, const String & path_, UInt64 keep_free_space_bytes_)
: name(name_)
, path(path_)
, keep_free_space_bytes(keep_free_space_bytes_)
{
if (path.back() != '/')
throw Exception("Disk path must ends with '/', but '" + path + "' doesn't.", ErrorCodes::LOGICAL_ERROR);
}
/// Reserves bytes on disk, if not possible returns nullptr.
ReservationPtr reserve(UInt64 bytes) const override;
/// Disk name from configuration;
const String & getName() const override { return name; }
/// Path on fs to disk
const String & getPath() const { return path; }
/// Path to clickhouse data directory on this disk
String getClickHouseDataPath() const { return path + "data/"; }
/// Amount of bytes which should be kept free on this disk
UInt64 getKeepingFreeSpace() const { return keep_free_space_bytes; }
/// Snapshot of disk space state (free and total space)
Stat getSpaceInformation() const { return Stat(*this); }
/// Total available space on disk
UInt64 getTotalSpace() const { return getSpaceInformation().getTotalSpace(); }
/// Space currently available on disk, take information from statvfs call
UInt64 getAvailableSpace() const { return getSpaceInformation().getAvailableSpace(); }
/// Currently available (prev method) minus already reserved space
UInt64 getUnreservedSpace() const;
private:
const String name;
const String path;
const UInt64 keep_free_space_bytes;
/// Used for reservation counters modification
static std::mutex mutex;
mutable UInt64 reserved_bytes = 0;
mutable UInt64 reservation_count = 0;
private:
/// Reserves bytes on disk, if not possible returns false
bool tryReserve(UInt64 bytes) const;
};
/// It is not possible to change disk runtime.
using DiskPtr = std::shared_ptr<const Disk>;
using Disks = std::vector<DiskPtr>;
/** Information about reserved size on concrete disk.
* Unreserve on destroy. Doesn't reserve bytes in constructor.
*/
class Reservation final : private boost::noncopyable
{
public:
Reservation(UInt64 size_, DiskPtr disk_ptr_)
: size(size_)
, metric_increment(CurrentMetrics::DiskSpaceReservedForMerge, size)
, disk_ptr(disk_ptr_)
{
}
/// Unreserves reserved space and decrement reservations count on disk
~Reservation();
/// Changes amount of reserved space. When new_size is greater than before,
/// availability of free space is not checked.
void update(UInt64 new_size);
/// Get reservation size
UInt64 getSize() const { return size; }
/// Get disk where reservation take place
const DiskPtr & getDisk() const { return disk_ptr; }
private:
UInt64 size;
CurrentMetrics::Increment metric_increment;
DiskPtr disk_ptr;
};
/// Parse .xml configuration and store information about disks
/// Mostly used for introspection.
class DiskSelector
{
public:
DiskSelector(const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix, const String & default_path);
/// Get disk by name
const DiskPtr & operator[](const String & name) const;
/// Get all disks name
const auto & getDisksMap() const { return disks; }
private:
std::map<String, DiskPtr> disks;
};
/**
* Disks group by some (user) criteria. For example,
* - Volume("slow_disks", [d1, d2], 100)
* - Volume("fast_disks", [d3, d4], 200)
* Cannot store parts larger than max_data_part_size.
*/
class Volume : public Space
{
friend class StoragePolicy;
public:
Volume(String name_, std::vector<DiskPtr> disks_, UInt64 max_data_part_size_)
: max_data_part_size(max_data_part_size_)
, disks(std::move(disks_))
, name(std::move(name_))
{
}
Volume(String name_, const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix, const DiskSelector & disk_selector);
/// Uses Round-robin to choose disk for reservation.
/// Returns valid reservation or nullptr if there is no space left on any disk.
ReservationPtr reserve(UInt64 bytes) const override;
/// Return biggest unreserved space across all disks
UInt64 getMaxUnreservedFreeSpace() const;
/// Volume name from config
const String & getName() const override { return name; }
/// Max size of reservation
UInt64 max_data_part_size = 0;
/// Disks in volume
Disks disks;
private:
mutable std::atomic<size_t> last_used = 0;
const String name;
};
using VolumePtr = std::shared_ptr<const Volume>;
using Volumes = std::vector<VolumePtr>;
/**
* Contains all information about volumes configuration for Storage.
* Can determine appropriate Volume and Disk for each reservation.
*/
class StoragePolicy : public Space
{
public:
StoragePolicy(String name_, const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix, const DiskSelector & disks);
StoragePolicy(String name_, Volumes volumes_, double move_factor_);
/// Returns disks ordered by volumes priority
Disks getDisks() const;
/// Returns any disk
/// Used when it's not important, for example for
/// mutations files
DiskPtr getAnyDisk() const;
DiskPtr getDiskByName(const String & disk_name) const;
/// Get free space from most free disk
UInt64 getMaxUnreservedFreeSpace() const;
const String & getName() const override { return name; }
/// Returns valid reservation or null
ReservationPtr reserve(UInt64 bytes) const override;
/// Reserve space on any volume with index > min_volume_index
ReservationPtr reserve(UInt64 bytes, size_t min_volume_index) const;
/// Find volume index, which contains disk
size_t getVolumeIndexByDisk(const DiskPtr & disk_ptr) const;
/// Reserves 0 bytes on disk with max available space
/// Do not use this function when it is possible to predict size.
ReservationPtr makeEmptyReservationOnLargestDisk() const;
const Volumes & getVolumes() const { return volumes; }
/// Returns number [0., 1.] -- fraction of free space on disk
/// which should be kept with help of background moves
double getMoveFactor() const { return move_factor; }
/// Get volume by index from storage_policy
VolumePtr getVolume(size_t i) const { return (i < volumes_names.size() ? volumes[i] : VolumePtr()); }
VolumePtr getVolumeByName(const String & volume_name) const
{
auto it = volumes_names.find(volume_name);
if (it == volumes_names.end())
return {};
return getVolume(it->second);
}
private:
Volumes volumes;
const String name;
std::map<String, size_t> volumes_names;
/// move_factor from interval [0., 1.]
/// We move something if disk from this policy
/// filled more than total_size * move_factor
double move_factor = 0.1; /// by default move factor is 10%
};
using StoragePolicyPtr = std::shared_ptr<const StoragePolicy>;
/// Parse .xml configuration and store information about policies
/// Mostly used for introspection.
class StoragePolicySelector
{
public:
StoragePolicySelector(const Poco::Util::AbstractConfiguration & config,
const String & config_prefix, const DiskSelector & disks);
/// Policy by name
const StoragePolicyPtr & operator[](const String & name) const;
/// All policies
const std::map<String, StoragePolicyPtr> & getPoliciesMap() const { return policies; }
private:
std::map<String, StoragePolicyPtr> policies;
};
}
}

12
dbms/src/Common/Dwarf.cpp
View File

@ -424,7 +424,7 @@ Dwarf::AttributeValue Dwarf::readAttributeValue(std::string_view & sp, uint64_t
switch (form)
{
case DW_FORM_addr:
return read<uintptr_t>(sp);
return uint64_t(read<uintptr_t>(sp));
case DW_FORM_block1:
return readBytes(sp, read<uint8_t>(sp));
case DW_FORM_block2:
@ -436,23 +436,23 @@ Dwarf::AttributeValue Dwarf::readAttributeValue(std::string_view & sp, uint64_t
return readBytes(sp, readULEB(sp));
case DW_FORM_data1: [[fallthrough]];
case DW_FORM_ref1:
return read<uint8_t>(sp);
return uint64_t(read<uint8_t>(sp));
case DW_FORM_data2: [[fallthrough]];
case DW_FORM_ref2:
return read<uint16_t>(sp);
return uint64_t(read<uint16_t>(sp));
case DW_FORM_data4: [[fallthrough]];
case DW_FORM_ref4:
return read<uint32_t>(sp);
return uint64_t(read<uint32_t>(sp));
case DW_FORM_data8: [[fallthrough]];
case DW_FORM_ref8:
return read<uint64_t>(sp);
case DW_FORM_sdata:
return readSLEB(sp);
return uint64_t(readSLEB(sp));
case DW_FORM_udata: [[fallthrough]];
case DW_FORM_ref_udata:
return readULEB(sp);
case DW_FORM_flag:
return read<uint8_t>(sp);
return uint64_t(read<uint8_t>(sp));
case DW_FORM_flag_present:
return 1;
case DW_FORM_sec_offset: [[fallthrough]];

3
dbms/src/Common/ErrorCodes.cpp
View File

@ -473,6 +473,9 @@ namespace ErrorCodes
extern const int QUOTA_REQUIRES_CLIENT_KEY = 496;
extern const int NOT_ENOUGH_PRIVILEGES = 497;
extern const int LIMIT_BY_WITH_TIES_IS_NOT_SUPPORTED = 498;
extern const int S3_ERROR = 499;
extern const int CANNOT_CREATE_DICTIONARY_FROM_METADATA = 500;
extern const int CANNOT_CREATE_DATABASE = 501;
extern const int KEEPER_EXCEPTION = 999;
extern const int POCO_EXCEPTION = 1000;

8
dbms/src/Common/Exception.cpp
View File

@ -10,7 +10,7 @@
#include <common/demangle.h>
#include <Common/config_version.h>
#include <Common/formatReadable.h>
#include <Common/DiskSpaceMonitor.h>
#include <Common/filesystemHelpers.h>
#include <filesystem>
namespace DB
@ -84,16 +84,16 @@ void getNoSpaceLeftInfoMessage(std::filesystem::path path, std::string & msg)
while (!std::filesystem::exists(path) && path.has_relative_path())
path = path.parent_path();
auto fs = DiskSpace::getStatVFS(path);
auto fs = getStatVFS(path);
msg += "\nTotal space: " + formatReadableSizeWithBinarySuffix(fs.f_blocks * fs.f_bsize)
+ "\nAvailable space: " + formatReadableSizeWithBinarySuffix(fs.f_bavail * fs.f_bsize)
+ "\nTotal inodes: " + formatReadableQuantity(fs.f_files)
+ "\nAvailable inodes: " + formatReadableQuantity(fs.f_favail);
auto mount_point = DiskSpace::getMountPoint(path).string();
auto mount_point = getMountPoint(path).string();
msg += "\nMount point: " + mount_point;
#if defined(__linux__)
msg += "\nFilesystem: " + DiskSpace::getFilesystemName(mount_point);
msg += "\nFilesystem: " + getFilesystemName(mount_point);
#endif
}

108
dbms/src/Common/FieldVisitors.h
View File

@ -34,97 +34,23 @@ struct StaticVisitor
/// F is template parameter, to allow universal reference for field, that is useful for const and non-const values.
template <typename Visitor, typename F>
typename std::decay_t<Visitor>::ResultType applyVisitor(Visitor && visitor, F && field)
auto applyVisitor(Visitor && visitor, F && field)
{
switch (field.getType())
{
case Field::Types::Null: return visitor(field.template get<Null>());
case Field::Types::UInt64: return visitor(field.template get<UInt64>());
case Field::Types::UInt128: return visitor(field.template get<UInt128>());
case Field::Types::Int64: return visitor(field.template get<Int64>());
case Field::Types::Float64: return visitor(field.template get<Float64>());
case Field::Types::String: return visitor(field.template get<String>());
case Field::Types::Array: return visitor(field.template get<Array>());
case Field::Types::Tuple: return visitor(field.template get<Tuple>());
case Field::Types::Decimal32: return visitor(field.template get<DecimalField<Decimal32>>());
case Field::Types::Decimal64: return visitor(field.template get<DecimalField<Decimal64>>());
case Field::Types::Decimal128: return visitor(field.template get<DecimalField<Decimal128>>());
case Field::Types::AggregateFunctionState: return visitor(field.template get<AggregateFunctionStateData>());
default:
throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD);
}
}
template <typename Visitor, typename F1, typename F2>
static typename std::decay_t<Visitor>::ResultType applyBinaryVisitorImpl(Visitor && visitor, F1 && field1, F2 && field2)
{
switch (field2.getType())
{
case Field::Types::Null: return visitor(field1, field2.template get<Null>());
case Field::Types::UInt64: return visitor(field1, field2.template get<UInt64>());
case Field::Types::UInt128: return visitor(field1, field2.template get<UInt128>());
case Field::Types::Int64: return visitor(field1, field2.template get<Int64>());
case Field::Types::Float64: return visitor(field1, field2.template get<Float64>());
case Field::Types::String: return visitor(field1, field2.template get<String>());
case Field::Types::Array: return visitor(field1, field2.template get<Array>());
case Field::Types::Tuple: return visitor(field1, field2.template get<Tuple>());
case Field::Types::Decimal32: return visitor(field1, field2.template get<DecimalField<Decimal32>>());
case Field::Types::Decimal64: return visitor(field1, field2.template get<DecimalField<Decimal64>>());
case Field::Types::Decimal128: return visitor(field1, field2.template get<DecimalField<Decimal128>>());
case Field::Types::AggregateFunctionState: return visitor(field1, field2.template get<AggregateFunctionStateData>());
default:
throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD);
}
return Field::dispatch(visitor, field);
}
template <typename Visitor, typename F1, typename F2>
typename std::decay_t<Visitor>::ResultType applyVisitor(Visitor && visitor, F1 && field1, F2 && field2)
auto applyVisitor(Visitor && visitor, F1 && field1, F2 && field2)
{
switch (field1.getType())
{
case Field::Types::Null:
return applyBinaryVisitorImpl(
std::forward<Visitor>(visitor), field1.template get<Null>(), std::forward<F2>(field2));
case Field::Types::UInt64:
return applyBinaryVisitorImpl(
std::forward<Visitor>(visitor), field1.template get<UInt64>(), std::forward<F2>(field2));
case Field::Types::UInt128:
return applyBinaryVisitorImpl(
std::forward<Visitor>(visitor), field1.template get<UInt128>(), std::forward<F2>(field2));
case Field::Types::Int64:
return applyBinaryVisitorImpl(
std::forward<Visitor>(visitor), field1.template get<Int64>(), std::forward<F2>(field2));
case Field::Types::Float64:
return applyBinaryVisitorImpl(
std::forward<Visitor>(visitor), field1.template get<Float64>(), std::forward<F2>(field2));
case Field::Types::String:
return applyBinaryVisitorImpl(
std::forward<Visitor>(visitor), field1.template get<String>(), std::forward<F2>(field2));
case Field::Types::Array:
return applyBinaryVisitorImpl(
std::forward<Visitor>(visitor), field1.template get<Array>(), std::forward<F2>(field2));
case Field::Types::Tuple:
return applyBinaryVisitorImpl(
std::forward<Visitor>(visitor), field1.template get<Tuple>(), std::forward<F2>(field2));
case Field::Types::Decimal32:
return applyBinaryVisitorImpl(
std::forward<Visitor>(visitor), field1.template get<DecimalField<Decimal32>>(), std::forward<F2>(field2));
case Field::Types::Decimal64:
return applyBinaryVisitorImpl(
std::forward<Visitor>(visitor), field1.template get<DecimalField<Decimal64>>(), std::forward<F2>(field2));
case Field::Types::Decimal128:
return applyBinaryVisitorImpl(
std::forward<Visitor>(visitor), field1.template get<DecimalField<Decimal128>>(), std::forward<F2>(field2));
case Field::Types::AggregateFunctionState:
return applyBinaryVisitorImpl(
std::forward<Visitor>(visitor), field1.template get<AggregateFunctionStateData>(), std::forward<F2>(field2));
default:
throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD);
}
return Field::dispatch([&](auto & field1_value)
{
return Field::dispatch([&](auto & field2_value)
{
return visitor(field1_value, field2_value);
},
field2);
},
field1);
}
@ -473,8 +399,14 @@ private:
public:
explicit FieldVisitorSum(const Field & rhs_) : rhs(rhs_) {}
bool operator() (UInt64 & x) const { x += get<UInt64>(rhs); return x != 0; }
bool operator() (Int64 & x) const { x += get<Int64>(rhs); return x != 0; }
// We can add all ints as unsigned regardless of their actual signedness.
bool operator() (Int64 & x) const { return this->operator()(reinterpret_cast<UInt64 &>(x)); }
bool operator() (UInt64 & x) const
{
x += rhs.reinterpret<UInt64>();
return x != 0;
}
bool operator() (Float64 & x) const { x += get<Float64>(rhs); return x != 0; }
bool operator() (Null &) const { throw Exception("Cannot sum Nulls", ErrorCodes::LOGICAL_ERROR); }

1
dbms/src/Common/config.h.in
View File

@ -6,6 +6,7 @@
#cmakedefine01 USE_SSL
#cmakedefine01 USE_POCO_NETSSL
#cmakedefine01 USE_HDFS
#cmakedefine01 USE_AWS_S3
#cmakedefine01 USE_CPUID
#cmakedefine01 USE_CPUINFO
#cmakedefine01 USE_BROTLI

69
dbms/src/Common/filesystemHelpers.cpp
View File

@ -1,10 +1,22 @@
#include <Common/filesystemHelpers.h>
#include "filesystemHelpers.h"
#include <sys/stat.h>
#if defined(__linux__)
# include <cstdio>
# include <mntent.h>
#endif
#include <Poco/File.h>
#include <Poco/Path.h>
#include <Poco/Version.h>
namespace DB
{
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
extern const int SYSTEM_ERROR;
extern const int NOT_IMPLEMENTED;
}
bool enoughSpaceInDirectory(const std::string & path [[maybe_unused]], size_t data_size [[maybe_unused]])
{
@ -24,4 +36,59 @@ std::unique_ptr<TemporaryFile> createTemporaryFile(const std::string & path)
return std::make_unique<TemporaryFile>(path);
}
std::filesystem::path getMountPoint(std::filesystem::path absolute_path)
{
if (absolute_path.is_relative())
throw Exception("Path is relative. It's a bug.", ErrorCodes::LOGICAL_ERROR);
absolute_path = std::filesystem::canonical(absolute_path);
const auto get_device_id = [](const std::filesystem::path & p)
{
struct stat st;
if (stat(p.c_str(), &st))
throwFromErrnoWithPath("Cannot stat " + p.string(), p.string(), ErrorCodes::SYSTEM_ERROR);
return st.st_dev;
};
/// If /some/path/to/dir/ and /some/path/to/ have different device id,
/// then device which contains /some/path/to/dir/filename is mounted to /some/path/to/dir/
auto device_id = get_device_id(absolute_path);
while (absolute_path.has_relative_path())
{
auto parent = absolute_path.parent_path();
auto parent_device_id = get_device_id(parent);
if (device_id != parent_device_id)
return absolute_path;
absolute_path = parent;
device_id = parent_device_id;
}
return absolute_path;
}
/// Returns name of filesystem mounted to mount_point
#if !defined(__linux__)
[[noreturn]]
#endif
String getFilesystemName([[maybe_unused]] const String & mount_point)
{
#if defined(__linux__)
auto mounted_filesystems = setmntent("/etc/mtab", "r");
if (!mounted_filesystems)
throw DB::Exception("Cannot open /etc/mtab to get name of filesystem", ErrorCodes::SYSTEM_ERROR);
mntent fs_info;
constexpr size_t buf_size = 4096; /// The same as buffer used for getmntent in glibc. It can happen that it's not enough
char buf[buf_size];
while (getmntent_r(mounted_filesystems, &fs_info, buf, buf_size) && fs_info.mnt_dir != mount_point)
;
endmntent(mounted_filesystems);
if (fs_info.mnt_dir != mount_point)
throw DB::Exception("Cannot find name of filesystem by mount point " + mount_point, ErrorCodes::SYSTEM_ERROR);
return fs_info.mnt_fsname;
#else
throw DB::Exception("The function getFilesystemName is supported on Linux only", ErrorCodes::NOT_IMPLEMENTED);
#endif
}
}

30
dbms/src/Common/filesystemHelpers.h
View File

@ -1,16 +1,42 @@
#pragma once
#include <string>
#include <memory>
#include <Core/Types.h>
#include <Common/Exception.h>
#include <filesystem>
#include <memory>
#include <string>
#include <sys/statvfs.h>
#include <Poco/TemporaryFile.h>
namespace DB
{
namespace ErrorCodes
{
extern const int CANNOT_STATVFS;
}
using TemporaryFile = Poco::TemporaryFile;
bool enoughSpaceInDirectory(const std::string & path, size_t data_size);
std::unique_ptr<TemporaryFile> createTemporaryFile(const std::string & path);
/// Returns mount point of filesystem where absoulte_path (must exist) is located
std::filesystem::path getMountPoint(std::filesystem::path absolute_path);
/// Returns name of filesystem mounted to mount_point
#if !defined(__linux__)
[[noreturn]]
#endif
String getFilesystemName([[maybe_unused]] const String & mount_point);
inline struct statvfs getStatVFS(const String & path)
{
struct statvfs fs;
if (statvfs(path.c_str(), &fs) != 0)
throwFromErrnoWithPath("Could not calculate available disk space (statvfs)", path, ErrorCodes::CANNOT_STATVFS);
return fs;
}
}

9
dbms/src/Common/quoteString.cpp
View File

@ -14,6 +14,15 @@ String quoteString(const StringRef & x)
}
String doubleQuoteString(const StringRef & x)
{
String res(x.size, '\0');
WriteBufferFromString wb(res);
writeDoubleQuotedString(x, wb);
return res;
}
String backQuote(const StringRef & x)
{
String res(x.size, '\0');

3
dbms/src/Common/quoteString.h
View File

@ -9,6 +9,9 @@ namespace DB
/// Quote the string.
String quoteString(const StringRef & x);
/// Double quote the string.
String doubleQuoteString(const StringRef & x);
/// Quote the identifier with backquotes.
String backQuote(const StringRef & x);

9
dbms/src/Core/DecimalComparison.h
View File

@ -4,6 +4,7 @@
#include <Core/Block.h>
#include <Core/AccurateComparison.h>
#include <Core/callOnTypeIndex.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypesDecimal.h>
#include <Columns/ColumnVector.h>
#include <Columns/ColumnsNumber.h>
@ -22,12 +23,12 @@ namespace ErrorCodes
///
inline bool allowDecimalComparison(const DataTypePtr & left_type, const DataTypePtr & right_type)
{
if (isDecimal(left_type))
if (isColumnedAsDecimal(left_type))
{
if (isDecimal(right_type) || isNotDecimalButComparableToDecimal(right_type))
if (isColumnedAsDecimal(right_type) || isNotDecimalButComparableToDecimal(right_type))
return true;
}
else if (isNotDecimalButComparableToDecimal(left_type) && isDecimal(right_type))
else if (isNotDecimalButComparableToDecimal(left_type) && isColumnedAsDecimal(right_type))
return true;
return false;
}
@ -82,7 +83,7 @@ public:
static bool compare(A a, B b, UInt32 scale_a, UInt32 scale_b)
{
static const UInt32 max_scale = maxDecimalPrecision<Decimal128>();
static const UInt32 max_scale = DecimalUtils::maxPrecision<Decimal128>();
if (scale_a > max_scale || scale_b > max_scale)
throw Exception("Bad scale of decimal field", ErrorCodes::DECIMAL_OVERFLOW);

142
dbms/src/Core/DecimalFunctions.h Normal file
View File

@ -0,0 +1,142 @@
#pragma once
// Moved Decimal-related functions out from Core/Types.h to reduce compilation time.
#include <Core/Types.h>
#include <Common/intExp.h>
#include <limits>
class DateLUTImpl;
namespace DB
{
namespace DecimalUtils
{
static constexpr size_t minPrecision() { return 1; }
template <typename T> static constexpr size_t maxPrecision() { return 0; }
template <> constexpr size_t maxPrecision<Decimal32>() { return 9; }
template <> constexpr size_t maxPrecision<Decimal64>() { return 18; }
template <> constexpr size_t maxPrecision<Decimal128>() { return 38; }
template <typename T> T scaleMultiplier(UInt32 scale);
template <> inline Int32 scaleMultiplier<Int32>(UInt32 scale) { return common::exp10_i32(scale); }
template <> inline Int64 scaleMultiplier<Int64>(UInt32 scale) { return common::exp10_i64(scale); }
template <> inline Int128 scaleMultiplier<Int128>(UInt32 scale) { return common::exp10_i128(scale); }
/** Components of DecimalX value:
* whole - represents whole part of decimal, can be negatve or positive.
* fractional - for fractional part of decimal, always positive.
*/
template <typename T>
struct DecimalComponents
{
T whole;
T fractional;
};
/** Make a decimal value from whole and fractional components with given scale multiplier.
* where scale_multiplier = scaleMultiplier<T>(scale)
* this is to reduce number of calls to scaleMultiplier when scale is known.
*
* Sign of `whole` controls sign of result: negative whole => negative result, positive whole => positive result.
* Sign of `fractional` is expected to be positive, otherwise result is undefined.
* If `scale` is to big (scale > maxPrecision<DecimalType::NativeType>), result is undefined.
*/
template <typename DecimalType>
DecimalType decimalFromComponentsWithMultiplier(const typename DecimalType::NativeType & whole,
const typename DecimalType::NativeType & fractional,
typename DecimalType::NativeType scale_multiplier)
{
using T = typename DecimalType::NativeType;
const auto fractional_sign = whole < 0 ? -1 : 1;
const T value = whole * scale_multiplier + fractional_sign * (fractional % scale_multiplier);
return DecimalType(value);
}
/** Make a decimal value from whole and fractional components with given scale.
*
* @see `decimalFromComponentsWithMultiplier` for details.
*/
template <typename DecimalType>
DecimalType decimalFromComponents(const typename DecimalType::NativeType & whole, const typename DecimalType::NativeType & fractional, UInt32 scale)
{
using T = typename DecimalType::NativeType;
return decimalFromComponentsWithMultiplier<DecimalType>(whole, fractional, scaleMultiplier<T>(scale));
}
/** Make a decimal value from whole and fractional components with given scale.
* @see `decimalFromComponentsWithMultiplier` for details.
*/
template <typename DecimalType>
DecimalType decimalFromComponents(const DecimalComponents<typename DecimalType::NativeType> & components, UInt32 scale)
{
return decimalFromComponents<DecimalType>(components.whole, components.fractional, scale);
}
/** Split decimal into whole and fractional parts with given scale_multiplier.
* This is an optimization to reduce number of calls to scaleMultiplier on known scale.
*/
template <typename DecimalType>
DecimalComponents<typename DecimalType::NativeType> splitWithScaleMultiplier(const DecimalType & decimal, typename DecimalType::NativeType scale_multiplier)
{
using T = typename DecimalType::NativeType;
const auto whole = decimal.value / scale_multiplier;
auto fractional = decimal.value % scale_multiplier;
if (fractional < T(0))
fractional *= T(-1);
return {whole, fractional};
}
/// Split decimal into components: whole and fractional part, @see `DecimalComponents` for details.
template <typename DecimalType>
DecimalComponents<typename DecimalType::NativeType> split(const DecimalType & decimal, UInt32 scale)
{
if (scale == 0)
{
return {decimal.value, 0};
}
return splitWithScaleMultiplier(decimal, scaleMultiplier<typename DecimalType::NativeType>(scale));
}
/** Get whole part from decimal.
*
* Sign of result follows sign of `decimal` value.
* If scale is to big, result is undefined.
*/
template <typename DecimalType>
typename DecimalType::NativeType getWholePart(const DecimalType & decimal, size_t scale)
{
if (scale == 0)
return decimal.value;
return decimal.value / scaleMultiplier<typename DecimalType::NativeType>(scale);
}
/** Get fractional part from decimal
*
* Result is always positive.
* If scale is to big, result is undefined.
*/
template <typename DecimalType>
typename DecimalType::NativeType getFractionalPart(const DecimalType & decimal, size_t scale)
{
using T = typename DecimalType::NativeType;
if (scale == 0)
return 0;
T result = decimal.value;
if (result < T(0))
result *= T(-1);
return result % scaleMultiplier<T>(scale);
}
}
}

2
dbms/src/Core/Field.cpp
View File

@ -295,7 +295,7 @@ namespace DB
void writeFieldText(const Field & x, WriteBuffer & buf)
{
DB::String res = applyVisitor(DB::FieldVisitorToString(), x);
DB::String res = Field::dispatch(DB::FieldVisitorToString(), x);
buf.write(res.data(), res.size());
}

218
dbms/src/Core/Field.h
View File

@ -27,7 +27,7 @@ namespace ErrorCodes
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
}
template <typename T>
template <typename T, typename SFINAE = void>
struct NearestFieldTypeImpl;
template <typename T>
@ -151,6 +151,54 @@ private:
UInt32 scale;
};
/// char may be signed or unsigned, and behave identically to signed char or unsigned char,
/// but they are always three different types.
/// signedness of char is different in Linux on x86 and Linux on ARM.
template <> struct NearestFieldTypeImpl<char> { using Type = std::conditional_t<is_signed_v<char>, Int64, UInt64>; };
template <> struct NearestFieldTypeImpl<signed char> { using Type = Int64; };
template <> struct NearestFieldTypeImpl<unsigned char> { using Type = UInt64; };
template <> struct NearestFieldTypeImpl<UInt16> { using Type = UInt64; };
template <> struct NearestFieldTypeImpl<UInt32> { using Type = UInt64; };
template <> struct NearestFieldTypeImpl<DayNum> { using Type = UInt64; };
template <> struct NearestFieldTypeImpl<UInt128> { using Type = UInt128; };
template <> struct NearestFieldTypeImpl<UUID> { using Type = UInt128; };
template <> struct NearestFieldTypeImpl<Int16> { using Type = Int64; };
template <> struct NearestFieldTypeImpl<Int32> { using Type = Int64; };
/// long and long long are always different types that may behave identically or not.
/// This is different on Linux and Mac.
template <> struct NearestFieldTypeImpl<long> { using Type = Int64; };
template <> struct NearestFieldTypeImpl<long long> { using Type = Int64; };
template <> struct NearestFieldTypeImpl<unsigned long> { using Type = UInt64; };
template <> struct NearestFieldTypeImpl<unsigned long long> { using Type = UInt64; };
template <> struct NearestFieldTypeImpl<Int128> { using Type = Int128; };
template <> struct NearestFieldTypeImpl<Decimal32> { using Type = DecimalField<Decimal32>; };
template <> struct NearestFieldTypeImpl<Decimal64> { using Type = DecimalField<Decimal64>; };
template <> struct NearestFieldTypeImpl<Decimal128> { using Type = DecimalField<Decimal128>; };
template <> struct NearestFieldTypeImpl<DecimalField<Decimal32>> { using Type = DecimalField<Decimal32>; };
template <> struct NearestFieldTypeImpl<DecimalField<Decimal64>> { using Type = DecimalField<Decimal64>; };
template <> struct NearestFieldTypeImpl<DecimalField<Decimal128>> { using Type = DecimalField<Decimal128>; };
template <> struct NearestFieldTypeImpl<Float32> { using Type = Float64; };
template <> struct NearestFieldTypeImpl<Float64> { using Type = Float64; };
template <> struct NearestFieldTypeImpl<const char *> { using Type = String; };
template <> struct NearestFieldTypeImpl<String> { using Type = String; };
template <> struct NearestFieldTypeImpl<Array> { using Type = Array; };
template <> struct NearestFieldTypeImpl<Tuple> { using Type = Tuple; };
template <> struct NearestFieldTypeImpl<bool> { using Type = UInt64; };
template <> struct NearestFieldTypeImpl<Null> { using Type = Null; };
template <> struct NearestFieldTypeImpl<AggregateFunctionStateData> { using Type = AggregateFunctionStateData; };
// For enum types, use the field type that corresponds to their underlying type.
template <typename T>
struct NearestFieldTypeImpl<T, std::enable_if_t<std::is_enum_v<T>>>
{
using Type = NearestFieldType<std::underlying_type_t<T>>;
};
/** 32 is enough. Round number is used for alignment and for better arithmetic inside std::vector.
* NOTE: Actually, sizeof(std::string) is 32 when using libc++, so Field is 40 bytes.
*/
@ -314,18 +362,24 @@ public:
bool isNull() const { return which == Types::Null; }
template <typename T> T & get()
template <typename T>
T & get();
template <typename T>
const T & get() const
{
using TWithoutRef = std::remove_reference_t<T>;
TWithoutRef * MAY_ALIAS ptr = reinterpret_cast<TWithoutRef*>(&storage);
return *ptr;
auto mutable_this = const_cast<std::decay_t<decltype(*this)> *>(this);
return mutable_this->get<T>();
}
template <typename T> const T & get() const
template <typename T>
T & reinterpret();
template <typename T>
const T & reinterpret() const
{
using TWithoutRef = std::remove_reference_t<T>;
const TWithoutRef * MAY_ALIAS ptr = reinterpret_cast<const TWithoutRef*>(&storage);
return *ptr;
auto mutable_this = const_cast<std::decay_t<decltype(*this)> *>(this);
return mutable_this->reinterpret<T>();
}
template <typename T> bool tryGet(T & result)
@ -427,6 +481,8 @@ public:
return rhs <= *this;
}
// More like bitwise equality as opposed to semantic equality:
// Null equals Null and NaN equals NaN.
bool operator== (const Field & rhs) const
{
if (which != rhs.which)
@ -435,9 +491,13 @@ public:
switch (which)
{
case Types::Null: return true;
case Types::UInt64:
case Types::Int64:
case Types::Float64: return get<UInt64>() == rhs.get<UInt64>();
case Types::UInt64: return get<UInt64>() == rhs.get<UInt64>();
case Types::Int64: return get<Int64>() == rhs.get<Int64>();
case Types::Float64:
{
// Compare as UInt64 so that NaNs compare as equal.
return reinterpret<UInt64>() == rhs.reinterpret<UInt64>();
}
case Types::String: return get<String>() == rhs.get<String>();
case Types::Array: return get<Array>() == rhs.get<Array>();
case Types::Tuple: return get<Tuple>() == rhs.get<Tuple>();
@ -457,6 +517,50 @@ public:
return !(*this == rhs);
}
/// Field is template parameter, to allow universal reference for field,
/// that is useful for const and non-const .
template <typename F, typename FieldRef>
static auto dispatch(F && f, FieldRef && field)
{
switch (field.which)
{
case Types::Null: return f(field.template get<Null>());
// gcc 8.2.1
#if !__clang__
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#endif
case Types::UInt64: return f(field.template get<UInt64>());
case Types::UInt128: return f(field.template get<UInt128>());
case Types::Int64: return f(field.template get<Int64>());
case Types::Float64: return f(field.template get<Float64>());
case Types::String: return f(field.template get<String>());
case Types::Array: return f(field.template get<Array>());
case Types::Tuple: return f(field.template get<Tuple>());
#if !__clang__
#pragma GCC diagnostic pop
#endif
case Types::Decimal32: return f(field.template get<DecimalField<Decimal32>>());
case Types::Decimal64: return f(field.template get<DecimalField<Decimal64>>());
case Types::Decimal128: return f(field.template get<DecimalField<Decimal128>>());
case Types::AggregateFunctionState: return f(field.template get<AggregateFunctionStateData>());
case Types::Int128:
// TODO: investigate where we need Int128 Fields. There are no
// field visitors that support them, and they only arise indirectly
// in some functions that use Decimal columns: they get the
// underlying Field value with get<Int128>(). Probably should be
// switched to DecimalField, but this is a whole endeavor in itself.
throw Exception("Unexpected Int128 in Field::dispatch()", ErrorCodes::LOGICAL_ERROR);
}
// GCC 9 complains that control reaches the end, despite that we handle
// all the cases above (maybe because of throw?). Return something to
// silence it.
Null null{};
return f(null);
}
private:
std::aligned_union_t<DBMS_MIN_FIELD_SIZE - sizeof(Types::Which),
Null, UInt64, UInt128, Int64, Int128, Float64, String, Array, Tuple,
@ -493,37 +597,6 @@ private:
}
template <typename F, typename Field> /// Field template parameter may be const or non-const Field.
static void dispatch(F && f, Field & field)
{
switch (field.which)
{
case Types::Null: f(field.template get<Null>()); return;
// gcc 7.3.0
#if !__clang__
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#endif
case Types::UInt64: f(field.template get<UInt64>()); return;
case Types::UInt128: f(field.template get<UInt128>()); return;
case Types::Int64: f(field.template get<Int64>()); return;
case Types::Int128: f(field.template get<Int128>()); return;
case Types::Float64: f(field.template get<Float64>()); return;
#if !__clang__
#pragma GCC diagnostic pop
#endif
case Types::String: f(field.template get<String>()); return;
case Types::Array: f(field.template get<Array>()); return;
case Types::Tuple: f(field.template get<Tuple>()); return;
case Types::Decimal32: f(field.template get<DecimalField<Decimal32>>()); return;
case Types::Decimal64: f(field.template get<DecimalField<Decimal64>>()); return;
case Types::Decimal128: f(field.template get<DecimalField<Decimal128>>()); return;
case Types::AggregateFunctionState: f(field.template get<AggregateFunctionStateData>()); return;
}
}
void create(const Field & x)
{
dispatch([this] (auto & value) { createConcrete(value); }, x);
@ -621,6 +694,22 @@ template <> struct Field::EnumToType<Field::Types::Decimal64> { using Type = Dec
template <> struct Field::EnumToType<Field::Types::Decimal128> { using Type = DecimalField<Decimal128>; };
template <> struct Field::EnumToType<Field::Types::AggregateFunctionState> { using Type = DecimalField<AggregateFunctionStateData>; };
template <typename T>
T & Field::get()
{
using ValueType = std::decay_t<T>;
//assert(TypeToEnum<NearestFieldType<ValueType>>::value == which);
ValueType * MAY_ALIAS ptr = reinterpret_cast<ValueType *>(&storage);
return *ptr;
}
template <typename T>
T & Field::reinterpret()
{
using ValueType = std::decay_t<T>;
ValueType * MAY_ALIAS ptr = reinterpret_cast<ValueType *>(&storage);
return *ptr;
}
template <typename T>
T get(const Field & field)
@ -651,49 +740,6 @@ template <> struct TypeName<Array> { static std::string get() { return "Array";
template <> struct TypeName<Tuple> { static std::string get() { return "Tuple"; } };
template <> struct TypeName<AggregateFunctionStateData> { static std::string get() { return "AggregateFunctionState"; } };
/// char may be signed or unsigned, and behave identically to signed char or unsigned char,
/// but they are always three different types.
/// signedness of char is different in Linux on x86 and Linux on ARM.
template <> struct NearestFieldTypeImpl<char> { using Type = std::conditional_t<is_signed_v<char>, Int64, UInt64>; };
template <> struct NearestFieldTypeImpl<signed char> { using Type = Int64; };
template <> struct NearestFieldTypeImpl<unsigned char> { using Type = UInt64; };
template <> struct NearestFieldTypeImpl<UInt16> { using Type = UInt64; };
template <> struct NearestFieldTypeImpl<UInt32> { using Type = UInt64; };
template <> struct NearestFieldTypeImpl<DayNum> { using Type = UInt64; };
template <> struct NearestFieldTypeImpl<UInt128> { using Type = UInt128; };
template <> struct NearestFieldTypeImpl<UUID> { using Type = UInt128; };
template <> struct NearestFieldTypeImpl<Int16> { using Type = Int64; };
template <> struct NearestFieldTypeImpl<Int32> { using Type = Int64; };
/// long and long long are always different types that may behave identically or not.
/// This is different on Linux and Mac.
template <> struct NearestFieldTypeImpl<long> { using Type = Int64; };
template <> struct NearestFieldTypeImpl<long long> { using Type = Int64; };
template <> struct NearestFieldTypeImpl<unsigned long> { using Type = UInt64; };
template <> struct NearestFieldTypeImpl<unsigned long long> { using Type = UInt64; };
template <> struct NearestFieldTypeImpl<Int128> { using Type = Int128; };
template <> struct NearestFieldTypeImpl<Decimal32> { using Type = DecimalField<Decimal32>; };
template <> struct NearestFieldTypeImpl<Decimal64> { using Type = DecimalField<Decimal64>; };
template <> struct NearestFieldTypeImpl<Decimal128> { using Type = DecimalField<Decimal128>; };
template <> struct NearestFieldTypeImpl<DecimalField<Decimal32>> { using Type = DecimalField<Decimal32>; };
template <> struct NearestFieldTypeImpl<DecimalField<Decimal64>> { using Type = DecimalField<Decimal64>; };
template <> struct NearestFieldTypeImpl<DecimalField<Decimal128>> { using Type = DecimalField<Decimal128>; };
template <> struct NearestFieldTypeImpl<Float32> { using Type = Float64; };
template <> struct NearestFieldTypeImpl<Float64> { using Type = Float64; };
template <> struct NearestFieldTypeImpl<const char *> { using Type = String; };
template <> struct NearestFieldTypeImpl<String> { using Type = String; };
template <> struct NearestFieldTypeImpl<Array> { using Type = Array; };
template <> struct NearestFieldTypeImpl<Tuple> { using Type = Tuple; };
template <> struct NearestFieldTypeImpl<bool> { using Type = UInt64; };
template <> struct NearestFieldTypeImpl<Null> { using Type = Null; };
template <> struct NearestFieldTypeImpl<AggregateFunctionStateData> { using Type = AggregateFunctionStateData; };
template <typename T>
decltype(auto) castToNearestFieldType(T && x)
{

5
dbms/src/Core/MySQLProtocol.h
View File

@ -953,10 +953,7 @@ public:
auto user = context.getUser(user_name);
if (user->authentication.getType() != DB::Authentication::DOUBLE_SHA1_PASSWORD)
throw Exception("Cannot use " + getName() + " auth plugin for user " + user_name + " since its password isn't specified using double SHA1.", ErrorCodes::UNKNOWN_EXCEPTION);
Poco::SHA1Engine::Digest double_sha1_value = user->authentication.getPasswordHashBinary();
Poco::SHA1Engine::Digest double_sha1_value = user->authentication.getPasswordDoubleSHA1();
assert(double_sha1_value.size() == Poco::SHA1Engine::DIGEST_SIZE);
Poco::SHA1Engine engine;

2
dbms/src/Core/Settings.h
View File

@ -188,7 +188,7 @@ struct Settings : public SettingsCollection<Settings>
M(SettingBool, input_format_null_as_default, false, "For text input formats initialize null fields with default values if data type of this field is not nullable", 0) \
\
M(SettingBool, input_format_values_interpret_expressions, true, "For Values format: if field could not be parsed by streaming parser, run SQL parser and try to interpret it as SQL expression.", 0) \
M(SettingBool, input_format_values_deduce_templates_of_expressions, false, "For Values format: if field could not be parsed by streaming parser, run SQL parser, deduce template of the SQL expression, try to parse all rows using template and then interpret expression for all rows.", 0) \
M(SettingBool, input_format_values_deduce_templates_of_expressions, true, "For Values format: if field could not be parsed by streaming parser, run SQL parser, deduce template of the SQL expression, try to parse all rows using template and then interpret expression for all rows.", 0) \
M(SettingBool, input_format_values_accurate_types_of_literals, true, "For Values format: when parsing and interpreting expressions using template, check actual type of literal to avoid possible overflow and precision issues.", 0) \
\
M(SettingBool, output_format_json_quote_64bit_integers, true, "Controls quoting of 64-bit integers in JSON output format.", 0) \

8
dbms/src/Core/Types.h
View File

@ -31,6 +31,8 @@ enum class TypeIndex
Float64,
Date,
DateTime,
DateTime32 = DateTime,
DateTime64,
String,
FixedString,
Enum8,
@ -151,10 +153,15 @@ struct Decimal
T value;
};
using Decimal32 = Decimal<Int32>;
using Decimal64 = Decimal<Int64>;
using Decimal128 = Decimal<Int128>;
// TODO (nemkov): consider making a strong typedef
//using DateTime32 = time_t;
using DateTime64 = Decimal64;
template <> struct TypeName<Decimal32> { static const char * get() { return "Decimal32"; } };
template <> struct TypeName<Decimal64> { static const char * get() { return "Decimal64"; } };
template <> struct TypeName<Decimal128> { static const char * get() { return "Decimal128"; } };
@ -196,6 +203,7 @@ inline const char * getTypeName(TypeIndex idx)
case TypeIndex::Float64: return TypeName<Float64>::get();
case TypeIndex::Date: return "Date";
case TypeIndex::DateTime: return "DateTime";
case TypeIndex::DateTime64: return "DateTime64";
case TypeIndex::String: return TypeName<String>::get();
case TypeIndex::FixedString: return "FixedString";
case TypeIndex::Enum8: return "Enum8";

6
dbms/src/Core/callOnTypeIndex.h
View File

@ -71,6 +71,7 @@ bool callOnBasicType(TypeIndex number, F && f)
{
case TypeIndex::Date: return f(TypePair<T, UInt16>());
case TypeIndex::DateTime: return f(TypePair<T, UInt32>());
case TypeIndex::DateTime64: return f(TypePair<T, DateTime64>());
default:
break;
}
@ -135,6 +136,7 @@ inline bool callOnBasicTypes(TypeIndex type_num1, TypeIndex type_num2, F && f)
{
case TypeIndex::Date: return callOnBasicType<UInt16, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
case TypeIndex::DateTime: return callOnBasicType<UInt32, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
case TypeIndex::DateTime64: return callOnBasicType<Decimal64, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
default:
break;
}
@ -145,10 +147,11 @@ inline bool callOnBasicTypes(TypeIndex type_num1, TypeIndex type_num2, F && f)
class DataTypeDate;
class DataTypeDateTime;
class DataTypeString;
class DataTypeFixedString;
class DataTypeUUID;
class DataTypeDateTime;
class DataTypeDateTime64;
template <typename T> class DataTypeEnum;
template <typename T> class DataTypeNumber;
template <typename T> class DataTypeDecimal;
@ -178,6 +181,7 @@ bool callOnIndexAndDataType(TypeIndex number, F && f)
case TypeIndex::Date: return f(TypePair<DataTypeDate, T>());
case TypeIndex::DateTime: return f(TypePair<DataTypeDateTime, T>());
case TypeIndex::DateTime64: return f(TypePair<DataTypeDateTime64, T>());
case TypeIndex::String: return f(TypePair<DataTypeString, T>());
case TypeIndex::FixedString: return f(TypePair<DataTypeFixedString, T>());

2
dbms/src/Core/iostream_debug_helpers.cpp
View File

@ -58,7 +58,7 @@ std::ostream & operator<<(std::ostream & stream, const TableStructureReadLock &)
return stream;
}
std::ostream & operator<<(std::ostream & stream, const IFunctionBuilder & what)
std::ostream & operator<<(std::ostream & stream, const IFunctionOverloadResolver & what)
{
stream << "IFunction(name = " << what.getName() << ", variadic = " << what.isVariadic() << ", args = " << what.getNumberOfArguments()
<< ")";

171
dbms/src/Core/tests/gtest_DecimalFunctions.cpp Normal file
View File

@ -0,0 +1,171 @@
#include <gtest/gtest.h>
#include <Core/DecimalFunctions.h>
namespace
{
using namespace DB;
struct DecimalUtilsSplitAndCombineTestParam
{
const char * description;
Decimal64 decimal_value;
UInt8 scale;
DecimalUtils::DecimalComponents<typename Decimal64::NativeType> components;
};
std::ostream & operator << (std::ostream & ostr, const DecimalUtilsSplitAndCombineTestParam & param)
{
return ostr << param.description;
}
class DecimalUtilsSplitAndCombineTest : public ::testing::TestWithParam<DecimalUtilsSplitAndCombineTestParam>
{};
template <typename DecimalType>
void testSplit(const DecimalUtilsSplitAndCombineTestParam & param)
{
const DecimalType decimal_value = param.decimal_value;
const auto & actual_components = DecimalUtils::split(decimal_value, param.scale);
EXPECT_EQ(param.components.whole, actual_components.whole);
EXPECT_EQ(param.components.fractional, actual_components.fractional);
}
template <typename DecimalType>
void testDecimalFromComponents(const DecimalUtilsSplitAndCombineTestParam & param)
{
EXPECT_EQ(param.decimal_value,
DecimalUtils::decimalFromComponents<DecimalType>(param.components.whole, param.components.fractional, param.scale));
}
template <typename DecimalType>
void testGetWhole(const DecimalUtilsSplitAndCombineTestParam & param)
{
EXPECT_EQ(param.components.whole,
DecimalUtils::getWholePart(DecimalType{param.decimal_value}, param.scale));
}
template <typename DecimalType>
void testGetFractional(const DecimalUtilsSplitAndCombineTestParam & param)
{
EXPECT_EQ(param.components.fractional,
DecimalUtils::getFractionalPart(DecimalType{param.decimal_value}, param.scale));
}
// unfortunatelly typed parametrized tests () are not supported in this version of gtest, so I have to emulate by hand.
TEST_P(DecimalUtilsSplitAndCombineTest, split_Decimal32)
{
testSplit<Decimal32>(GetParam());
}
TEST_P(DecimalUtilsSplitAndCombineTest, split_Decimal64)
{
testSplit<Decimal64>(GetParam());
}
TEST_P(DecimalUtilsSplitAndCombineTest, split_Decimal128)
{
testSplit<Decimal128>(GetParam());
}
TEST_P(DecimalUtilsSplitAndCombineTest, combine_Decimal32)
{
testDecimalFromComponents<Decimal32>(GetParam());
}
TEST_P(DecimalUtilsSplitAndCombineTest, combine_Decimal64)
{
testDecimalFromComponents<Decimal64>(GetParam());
}
TEST_P(DecimalUtilsSplitAndCombineTest, combine_Decimal128)
{
testDecimalFromComponents<Decimal64>(GetParam());
}
TEST_P(DecimalUtilsSplitAndCombineTest, getWholePart_Decimal32)
{
testGetWhole<Decimal32>(GetParam());
}
TEST_P(DecimalUtilsSplitAndCombineTest, getWholePart_Decimal64)
{
testGetWhole<Decimal64>(GetParam());
}
TEST_P(DecimalUtilsSplitAndCombineTest, getWholePart_Decimal128)
{
testGetWhole<Decimal128>(GetParam());
}
TEST_P(DecimalUtilsSplitAndCombineTest, getFractionalPart_Decimal32)
{
testGetFractional<Decimal32>(GetParam());
}
TEST_P(DecimalUtilsSplitAndCombineTest, getFractionalPart_Decimal64)
{
testGetFractional<Decimal64>(GetParam());
}
TEST_P(DecimalUtilsSplitAndCombineTest, getFractionalPart_Decimal128)
{
testGetFractional<Decimal128>(GetParam());
}
}
// Intentionally small values that fit into 32-bit in order to cover Decimal32, Decimal64 and Decimal128 with single set of data.
INSTANTIATE_TEST_CASE_P(Basic,
DecimalUtilsSplitAndCombineTest,
::testing::ValuesIn(std::initializer_list<DecimalUtilsSplitAndCombineTestParam>{
{
"Positive value with non-zero scale, whole, and fractional parts.",
1234567'89,
2,
{
1234567,
89
}
},
{
"When scale is 0, fractional part is 0.",
1234567'89,
0,
{
123456789,
0
}
},
{
"When scale is not 0 and fractional part is 0.",
1234567'00,
2,
{
1234567,
0
}
},
{
"When scale is not 0 and whole part is 0.",
123,
3,
{
0,
123
}
},
{
"For negative Decimal value whole part is negative, fractional is non-negative.",
-1234567'89,
2,
{
-1234567,
89
}
}
}
),);

1
dbms/src/DataStreams/AddingDefaultsBlockInputStream.cpp
View File

@ -14,6 +14,7 @@
#include <DataTypes/DataTypesDecimal.h>
#include <DataTypes/DataTypeDate.h>
#include <DataTypes/DataTypeDateTime.h>
#include <DataTypes/DataTypeDateTime64.h>
#include <DataTypes/DataTypeEnum.h>
#include <DataTypes/DataTypeUUID.h>
#include <DataTypes/DataTypeString.h>

3
dbms/src/DataStreams/AggregatingSortedBlockInputStream.cpp
View File

@ -213,7 +213,8 @@ void AggregatingSortedBlockInputStream::merge(MutableColumns & merged_columns, s
}
/// Write the simple aggregation result for the previous group.
insertSimpleAggregationResult(merged_columns);
if (merged_rows > 0)
insertSimpleAggregationResult(merged_columns);
finished = true;
}

39
dbms/src/DataStreams/TTLBlockInputStream.cpp
View File

@ -85,6 +85,8 @@ Block TTLBlockInputStream::readImpl()
removeValuesWithExpiredColumnTTL(block);
updateMovesTTL(block);
return block;
}
@ -145,7 +147,8 @@ void TTLBlockInputStream::removeValuesWithExpiredColumnTTL(Block & block)
defaults_expression->execute(block_with_defaults);
}
for (const auto & [name, ttl_entry] : storage.ttl_entries_by_name)
std::vector<String> columns_to_remove;
for (const auto & [name, ttl_entry] : storage.column_ttl_entries_by_name)
{
const auto & old_ttl_info = old_ttl_infos.columns_ttl[name];
auto & new_ttl_info = new_ttl_infos.columns_ttl[name];
@ -159,7 +162,10 @@ void TTLBlockInputStream::removeValuesWithExpiredColumnTTL(Block & block)
continue;
if (!block.has(ttl_entry.result_column))
{
columns_to_remove.push_back(ttl_entry.result_column);
ttl_entry.expression->execute(block);
}
ColumnPtr default_column = nullptr;
if (block_with_defaults.has(name))
@ -192,9 +198,34 @@ void TTLBlockInputStream::removeValuesWithExpiredColumnTTL(Block & block)
column_with_type.column = std::move(result_column);
}
for (const auto & elem : storage.ttl_entries_by_name)
if (block.has(elem.second.result_column))
block.erase(elem.second.result_column);
for (const String & column : columns_to_remove)
block.erase(column);
}
void TTLBlockInputStream::updateMovesTTL(Block & block)
{
std::vector<String> columns_to_remove;
for (const auto & ttl_entry : storage.move_ttl_entries)
{
auto & new_ttl_info = new_ttl_infos.moves_ttl[ttl_entry.result_column];
if (!block.has(ttl_entry.result_column))
{
columns_to_remove.push_back(ttl_entry.result_column);
ttl_entry.expression->execute(block);
}
const IColumn * ttl_column = block.getByName(ttl_entry.result_column).column.get();
for (size_t i = 0; i < block.rows(); ++i)
{
UInt32 cur_ttl = getTimestampByIndex(ttl_column, i);
new_ttl_info.update(cur_ttl);
}
}
for (const String & column : columns_to_remove)
block.erase(column);
}
UInt32 TTLBlockInputStream::getTimestampByIndex(const IColumn * column, size_t ind)

3
dbms/src/DataStreams/TTLBlockInputStream.h
View File

@ -58,6 +58,9 @@ private:
/// Removes rows with expired table ttl and computes new ttl_infos for part
void removeRowsWithExpiredTableTTL(Block & block);
/// Updates TTL for moves
void updateMovesTTL(Block & block);
UInt32 getTimestampByIndex(const IColumn * column, size_t ind);
bool isTTLExpired(time_t ttl);
};

Some files were not shown because too many files have changed in this diff Show More